From e239a77223382d1290b2fd7c301a4b304da3eff3 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Thu, 9 May 2019 19:42:39 +0300 Subject: [PATCH 01/48] Fixing build for publish (not yet) --- buildSrc/build.gradle.kts | 2 +- .../src/main/kotlin/npm-artifactory.gradle.kts | 5 ----- buildSrc/src/main/kotlin/npm-bintray.gradle.kts | 13 ++++++++++--- .../dataforge/workspace/SimpleWorkspaceTest.kt | 17 ++++++++++++++--- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 1ebbdf4d..31818916 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -12,7 +12,7 @@ val kotlinVersion = "1.3.31" // Add plugins used in buildSrc as dependencies, also we should specify version only here dependencies { implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion") - implementation("org.jfrog.buildinfo:build-info-extractor-gradle:4.9.5") + implementation("org.jfrog.buildinfo:build-info-extractor-gradle:4.9.6") implementation("com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.4") implementation("org.jetbrains.dokka:dokka-gradle-plugin:0.9.18") implementation("com.moowork.gradle:gradle-node-plugin:1.3.1") diff --git a/buildSrc/src/main/kotlin/npm-artifactory.gradle.kts b/buildSrc/src/main/kotlin/npm-artifactory.gradle.kts index d792dffb..27e5ebce 100644 --- a/buildSrc/src/main/kotlin/npm-artifactory.gradle.kts +++ b/buildSrc/src/main/kotlin/npm-artifactory.gradle.kts @@ -21,11 +21,6 @@ artifactory { defaults(delegateClosureOf{ invokeMethod("publications", arrayOf("jvm", "js", "kotlinMultiplatform", "metadata")) - //TODO: This property is not available for ArtifactoryTask - //setProperty("publishBuildInfo", false) - setProperty("publishArtifacts", true) - setProperty("publishPom", true) - setProperty("publishIvy", false) }) }) resolve(delegateClosureOf { diff --git a/buildSrc/src/main/kotlin/npm-bintray.gradle.kts b/buildSrc/src/main/kotlin/npm-bintray.gradle.kts index b152d163..a8987279 100644 --- a/buildSrc/src/main/kotlin/npm-bintray.gradle.kts +++ b/buildSrc/src/main/kotlin/npm-bintray.gradle.kts @@ -67,9 +67,9 @@ bintray { // this is a problem of this plugin pkg(delegateClosureOf { userOrg = "mipt-npm" - repo = "scientifik" - name = "scientifik.kmath" - issueTrackerUrl = "https://github.com/mipt-npm/kmath/issues" + repo = "dataforge" + name = project.name + issueTrackerUrl = "https://github.com/altavir/dataforge-core/issues" setLicenses("Apache-2.0") vcsUrl = vcs version(delegateClosureOf { @@ -94,4 +94,11 @@ bintray { } } + + val publications = project.publishing.publications.filter { !it.name.contains("-test") }.map { + println("Uploading artifact '$it.groupId:$it.artifactId:$it.version' from publication '$it.name'") + it.name.toString() + }.toTypedArray() + + setPublications(*publications) } diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index 52ba85b5..82bb2eb7 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -2,6 +2,8 @@ package hep.dataforge.workspace import hep.dataforge.data.first import hep.dataforge.data.get +import hep.dataforge.meta.boolean +import hep.dataforge.meta.get import org.junit.Test import kotlin.test.assertEquals @@ -19,6 +21,9 @@ class SimpleWorkspaceTest { allData() } pipe { data -> + if (meta["testFlag"].boolean == true) { + println("flag") + } context.logger.info { "Starting square on $data" } data * data } @@ -54,11 +59,11 @@ class SimpleWorkspaceTest { } } - task("delta"){ - model{ + task("delta") { + model { dependsOn("average") } - join {data-> + join { data -> data["even"]!! - data["odd"]!! } } @@ -70,4 +75,10 @@ class SimpleWorkspaceTest { val res = node.first() assertEquals(328350, res.get()) } + + @Test + fun testMetaPropagation() { + val node = workspace.run("sum"){"testFlag" to true} + val res = node.first().get() + } } \ No newline at end of file From 7daeedb9ff046b9093b982248e401257a0f1574e Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Thu, 9 May 2019 22:05:23 +0300 Subject: [PATCH 02/48] Fixed build for publish --- build.gradle.kts | 3 +- .../main/kotlin/npm-artifactory.gradle.kts | 33 ----- .../src/main/kotlin/npm-bintray.gradle.kts | 104 -------------- .../src/main/kotlin/npm-publish.gradle.kts | 134 ++++++++++++++++++ 4 files changed, 135 insertions(+), 139 deletions(-) delete mode 100644 buildSrc/src/main/kotlin/npm-artifactory.gradle.kts delete mode 100644 buildSrc/src/main/kotlin/npm-bintray.gradle.kts create mode 100644 buildSrc/src/main/kotlin/npm-publish.gradle.kts diff --git a/build.gradle.kts b/build.gradle.kts index 8494144e..4fc08ab3 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -12,7 +12,6 @@ allprojects { subprojects { if (name.startsWith("dataforge")) { - apply(plugin = "npm-bintray") - apply(plugin = "npm-artifactory") + apply(plugin = "npm-publish") } } \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/npm-artifactory.gradle.kts b/buildSrc/src/main/kotlin/npm-artifactory.gradle.kts deleted file mode 100644 index 27e5ebce..00000000 --- a/buildSrc/src/main/kotlin/npm-artifactory.gradle.kts +++ /dev/null @@ -1,33 +0,0 @@ -import groovy.lang.GroovyObject -import org.jfrog.gradle.plugin.artifactory.dsl.PublisherConfig -import org.jfrog.gradle.plugin.artifactory.dsl.ResolverConfig - -plugins { - id("com.jfrog.artifactory") -} - -artifactory { - val artifactoryUser: String? by project - val artifactoryPassword: String? by project - val artifactoryContextUrl = "http://npm.mipt.ru:8081/artifactory" - - setContextUrl(artifactoryContextUrl)//The base Artifactory URL if not overridden by the publisher/resolver - publish(delegateClosureOf { - repository(delegateClosureOf { - setProperty("repoKey", "gradle-dev-local") - setProperty("username", artifactoryUser) - setProperty("password", artifactoryPassword) - }) - - defaults(delegateClosureOf{ - invokeMethod("publications", arrayOf("jvm", "js", "kotlinMultiplatform", "metadata")) - }) - }) - resolve(delegateClosureOf { - repository(delegateClosureOf { - setProperty("repoKey", "gradle-dev") - setProperty("username", artifactoryUser) - setProperty("password", artifactoryPassword) - }) - }) -} diff --git a/buildSrc/src/main/kotlin/npm-bintray.gradle.kts b/buildSrc/src/main/kotlin/npm-bintray.gradle.kts deleted file mode 100644 index a8987279..00000000 --- a/buildSrc/src/main/kotlin/npm-bintray.gradle.kts +++ /dev/null @@ -1,104 +0,0 @@ -@file:Suppress("UnstableApiUsage") - -import com.jfrog.bintray.gradle.BintrayExtension.PackageConfig -import com.jfrog.bintray.gradle.BintrayExtension.VersionConfig - -// Old bintray.gradle script converted to real Gradle plugin (precompiled script plugin) -// It now has own dependencies and support type safe accessors -// Syntax is pretty close to what we had in Groovy -// (excluding Property.set and bintray dynamic configs) - -plugins { - id("com.jfrog.bintray") - `maven-publish` -} - -val vcs = "https://github.com/mipt-npm/kmath" - -// Configure publishing -publishing { - repositories { - maven("https://bintray.com/mipt-npm/scientifik") - } - - // Process each publication we have in this project - publications.filterIsInstance().forEach { publication -> - - // use type safe pom config GSL insterad of old dynamic - publication.pom { - name.set(project.name) - description.set(project.description) - url.set(vcs) - - licenses { - license { - name.set("The Apache Software License, Version 2.0") - url.set("http://www.apache.org/licenses/LICENSE-2.0.txt") - distribution.set("repo") - } - } - developers { - developer { - id.set("MIPT-NPM") - name.set("MIPT nuclear physics methods laboratory") - organization.set("MIPT") - organizationUrl.set("http://npm.mipt.ru") - } - - } - scm { - url.set(vcs) - } - } - - } -} - -bintray { - // delegates for runtime properties - val bintrayUser: String? by project - val bintrayApiKey: String? by project - user = bintrayUser ?: System.getenv("BINTRAY_USER") - key = bintrayApiKey ?: System.getenv("BINTRAY_API_KEY") - publish = true - override = true // for multi-platform Kotlin/Native publishing - - // We have to use delegateClosureOf because bintray supports only dynamic groovy syntax - // this is a problem of this plugin - pkg(delegateClosureOf { - userOrg = "mipt-npm" - repo = "dataforge" - name = project.name - issueTrackerUrl = "https://github.com/altavir/dataforge-core/issues" - setLicenses("Apache-2.0") - vcsUrl = vcs - version(delegateClosureOf { - name = project.version.toString() - vcsTag = project.version.toString() - released = java.util.Date().toString() - }) - }) - - tasks { - bintrayUpload { - dependsOn(publishToMavenLocal) - doFirst { - setPublications(project.publishing.publications - .filterIsInstance() - .filter { !it.name.contains("-test") && it.name != "kotlinMultiplatform" } - .map { - println("""Uploading artifact "${it.groupId}:${it.artifactId}:${it.version}" from publication "${it.name}""") - it.name //https://github.com/bintray/gradle-bintray-plugin/issues/256 - }) - } - } - - } - - val publications = project.publishing.publications.filter { !it.name.contains("-test") }.map { - println("Uploading artifact '$it.groupId:$it.artifactId:$it.version' from publication '$it.name'") - it.name.toString() - }.toTypedArray() - - setPublications(*publications) -} diff --git a/buildSrc/src/main/kotlin/npm-publish.gradle.kts b/buildSrc/src/main/kotlin/npm-publish.gradle.kts new file mode 100644 index 00000000..316a1c65 --- /dev/null +++ b/buildSrc/src/main/kotlin/npm-publish.gradle.kts @@ -0,0 +1,134 @@ +@file:Suppress("UnstableApiUsage") + +import com.jfrog.bintray.gradle.tasks.BintrayUploadTask +import groovy.lang.GroovyObject +import org.gradle.api.publish.maven.internal.artifact.FileBasedMavenArtifact +import org.jfrog.gradle.plugin.artifactory.dsl.PublisherConfig +import org.jfrog.gradle.plugin.artifactory.dsl.ResolverConfig + +// Old bintray.gradle script converted to real Gradle plugin (precompiled script plugin) +// It now has own dependencies and support type safe accessors +// Syntax is pretty close to what we had in Groovy +// (excluding Property.set and bintray dynamic configs) + +plugins { + `maven-publish` + id("com.jfrog.bintray") + id("com.jfrog.artifactory") +} + +val vcs = "https://github.com/altavir/dataforge-core" + +// Configure publishing +publishing { + repositories { + maven("https://bintray.com/mipt-npm/dataforge") + } + + // Process each publication we have in this project + publications.filterIsInstance().forEach { publication -> + + // use type safe pom config GSL insterad of old dynamic + publication.pom { + name.set(project.name) + description.set(project.description) + url.set(vcs) + + licenses { + license { + name.set("The Apache Software License, Version 2.0") + url.set("http://www.apache.org/licenses/LICENSE-2.0.txt") + distribution.set("repo") + } + } + developers { + developer { + id.set("MIPT-NPM") + name.set("MIPT nuclear physics methods laboratory") + organization.set("MIPT") + organizationUrl.set("http://npm.mipt.ru") + } + + } + scm { + url.set(vcs) + } + } + } + +} + +bintray { + user = findProperty("bintrayUser") as? String ?: System.getenv("BINTRAY_USER") + key = findProperty("bintrayApiKey") as? String? ?: System.getenv("BINTRAY_API_KEY") + publish = true + override = true // for multi-platform Kotlin/Native publishing + + // We have to use delegateClosureOf because bintray supports only dynamic groovy syntax + // this is a problem of this plugin + pkg.apply { + userOrg = "mipt-npm" + repo = "dataforge" + name = project.name + issueTrackerUrl = "https://github.com/altavir/dataforge-core/issues" + setLicenses("Apache-2.0") + vcsUrl = vcs + version.apply { + name = project.version.toString() + vcsTag = project.version.toString() + released = java.util.Date().toString() + } + } + + afterEvaluate { + setPublications(*publishing.publications.names.toTypedArray()) + } + + tasks { + bintrayUpload { + dependsOn(publishToMavenLocal) + } + } +} + +//workaround for bintray +tasks.withType { + doFirst { + publishing.publications + .filterIsInstance() + .forEach { publication -> + val moduleFile = buildDir.resolve("publications/${publication.name}/module.json") + if (moduleFile.exists()) { + publication.artifact(object : FileBasedMavenArtifact(moduleFile) { + override fun getDefaultExtension() = "module" + }) + } + } + } +} + +artifactory { + val artifactoryUser: String? by project + val artifactoryPassword: String? by project + val artifactoryContextUrl = "http://npm.mipt.ru:8081/artifactory" + + setContextUrl(artifactoryContextUrl)//The base Artifactory URL if not overridden by the publisher/resolver + publish(delegateClosureOf { + repository(delegateClosureOf { + setProperty("repoKey", "gradle-dev-local") + setProperty("username", artifactoryUser) + setProperty("password", artifactoryPassword) + }) + + defaults(delegateClosureOf { + invokeMethod("publications", arrayOf("jvm", "js", "kotlinMultiplatform", "metadata")) + }) + }) + resolve(delegateClosureOf { + repository(delegateClosureOf { + setProperty("repoKey", "gradle-dev") + setProperty("username", artifactoryUser) + setProperty("password", artifactoryPassword) + }) + }) +} From 1344471f40eefd5dc93292e524ba4e9d613b4159 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 10 May 2019 09:51:31 +0300 Subject: [PATCH 03/48] build fix --- build.gradle.kts | 1 + .../src/main/kotlin/dokka-publish.gradle.kts | 85 +++++++++++-------- .../main/kotlin/npm-multiplatform.gradle.kts | 7 +- .../src/main/kotlin/npm-publish.gradle.kts | 8 +- 4 files changed, 60 insertions(+), 41 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 4fc08ab3..e2482330 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -11,6 +11,7 @@ allprojects { } subprojects { + apply(plugin = "dokka-publish") if (name.startsWith("dataforge")) { apply(plugin = "npm-publish") } diff --git a/buildSrc/src/main/kotlin/dokka-publish.gradle.kts b/buildSrc/src/main/kotlin/dokka-publish.gradle.kts index 318e08ae..b816a5c4 100644 --- a/buildSrc/src/main/kotlin/dokka-publish.gradle.kts +++ b/buildSrc/src/main/kotlin/dokka-publish.gradle.kts @@ -1,58 +1,73 @@ import org.jetbrains.dokka.gradle.DokkaTask +import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension +import org.jetbrains.kotlin.gradle.plugin.KotlinPlatformJvmPlugin +import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinMultiplatformPlugin plugins { - kotlin("multiplatform") id("org.jetbrains.dokka") `maven-publish` } -kotlin { +plugins.withType(KotlinMultiplatformPlugin::class){ + configure{ + val dokka by tasks.getting(DokkaTask::class) { + outputFormat = "html" + outputDirectory = "$buildDir/javadoc" + jdkVersion = 8 + kotlinTasks { + // dokka fails to retrieve sources from MPP-tasks so we only define the jvm task + listOf(tasks.getByPath("compileKotlinJvm")) + } + sourceRoot { + // assuming only single source dir + path = sourceSets["commonMain"].kotlin.srcDirs.first().toString() + platforms = listOf("Common") + } + // although the JVM sources are now taken from the task, + // we still define the jvm source root to get the JVM marker in the generated html + sourceRoot { + // assuming only single source dir + path = sourceSets["jvmMain"].kotlin.srcDirs.first().toString() + platforms = listOf("JVM") + } + } + + val javadocJar by tasks.registering(Jar::class) { + group = JavaBasePlugin.DOCUMENTATION_GROUP + dependsOn(dokka) + archiveClassifier.set("javadoc") + from("$buildDir/javadoc") + } + + configure{ + + targets.all { + val publication = publications.findByName(name) as MavenPublication + + // Patch publications with fake javadoc + publication.artifact(javadocJar.get()) + } + } + } +} + +plugins.withType(KotlinPlatformJvmPlugin::class){ val dokka by tasks.getting(DokkaTask::class) { outputFormat = "html" outputDirectory = "$buildDir/javadoc" jdkVersion = 8 - - kotlinTasks { - // dokka fails to retrieve sources from MPP-tasks so we only define the jvm task - listOf(tasks.getByPath("compileKotlinJvm")) - } - sourceRoot { - // assuming only single source dir - path = sourceSets["commonMain"].kotlin.srcDirs.first().toString() - platforms = listOf("Common") - } - // although the JVM sources are now taken from the task, - // we still define the jvm source root to get the JVM marker in the generated html - sourceRoot { - // assuming only single source dir - path = sourceSets["jvmMain"].kotlin.srcDirs.first().toString() - platforms = listOf("JVM") - } } val javadocJar by tasks.registering(Jar::class) { + group = JavaBasePlugin.DOCUMENTATION_GROUP dependsOn(dokka) archiveClassifier.set("javadoc") from("$buildDir/javadoc") } - publishing { - - // publications.filterIsInstance().forEach { publication -> -// if (publication.name == "kotlinMultiplatform") { -// // for our root metadata publication, set artifactId with a package and project name -// publication.artifactId = project.name -// } else { -// // for targets, set artifactId with a package, project name and target name (e.g. iosX64) -// publication.artifactId = "${project.name}-${publication.name}" -// } -// } - - targets.all { - val publication = publications.findByName(name) as MavenPublication - - // Patch publications with fake javadoc + configure{ + publications.filterIsInstance().forEach {publication -> publication.artifact(javadocJar.get()) } } diff --git a/buildSrc/src/main/kotlin/npm-multiplatform.gradle.kts b/buildSrc/src/main/kotlin/npm-multiplatform.gradle.kts index 671986b8..dd048a39 100644 --- a/buildSrc/src/main/kotlin/npm-multiplatform.gradle.kts +++ b/buildSrc/src/main/kotlin/npm-multiplatform.gradle.kts @@ -1,4 +1,7 @@ -import org.gradle.kotlin.dsl.* +import org.gradle.kotlin.dsl.`maven-publish` +import org.gradle.kotlin.dsl.apply +import org.gradle.kotlin.dsl.dependencies +import org.gradle.kotlin.dsl.kotlin plugins { kotlin("multiplatform") @@ -74,8 +77,6 @@ kotlin { } } - apply(plugin = "dokka-publish") - // Apply JS test configuration val runJsTests by ext(false) diff --git a/buildSrc/src/main/kotlin/npm-publish.gradle.kts b/buildSrc/src/main/kotlin/npm-publish.gradle.kts index 316a1c65..851348ba 100644 --- a/buildSrc/src/main/kotlin/npm-publish.gradle.kts +++ b/buildSrc/src/main/kotlin/npm-publish.gradle.kts @@ -18,17 +18,18 @@ plugins { } val vcs = "https://github.com/altavir/dataforge-core" +val bintrayRepo = "https://bintray.com/mipt-npm/dataforge" // Configure publishing publishing { repositories { - maven("https://bintray.com/mipt-npm/dataforge") + maven(bintrayRepo) } // Process each publication we have in this project publications.filterIsInstance().forEach { publication -> - // use type safe pom config GSL insterad of old dynamic + // use type safe pom config GSL instead of old dynamic publication.pom { name.set(project.name) description.set(project.description) @@ -70,7 +71,7 @@ bintray { userOrg = "mipt-npm" repo = "dataforge" name = project.name - issueTrackerUrl = "https://github.com/altavir/dataforge-core/issues" + issueTrackerUrl = "$vcs/issues" setLicenses("Apache-2.0") vcsUrl = vcs version.apply { @@ -80,6 +81,7 @@ bintray { } } + //workaround bintray bug afterEvaluate { setPublications(*publishing.publications.names.toTypedArray()) } From fdd5b1137079078af671f7c8a8d9ebc1a257ad57 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 10 May 2019 19:53:54 +0300 Subject: [PATCH 04/48] Configurable updater fix --- .../src/main/kotlin/dokka-publish.gradle.kts | 47 ++++++++++--------- .../kotlin/hep/dataforge/meta/Config.kt | 2 +- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/buildSrc/src/main/kotlin/dokka-publish.gradle.kts b/buildSrc/src/main/kotlin/dokka-publish.gradle.kts index b816a5c4..b7b48fb6 100644 --- a/buildSrc/src/main/kotlin/dokka-publish.gradle.kts +++ b/buildSrc/src/main/kotlin/dokka-publish.gradle.kts @@ -1,15 +1,15 @@ import org.jetbrains.dokka.gradle.DokkaTask +import org.jetbrains.kotlin.gradle.dsl.KotlinJvmProjectExtension import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension -import org.jetbrains.kotlin.gradle.plugin.KotlinPlatformJvmPlugin -import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinMultiplatformPlugin plugins { id("org.jetbrains.dokka") `maven-publish` } -plugins.withType(KotlinMultiplatformPlugin::class){ - configure{ +afterEvaluate { + + extensions.findByType()?.apply{ val dokka by tasks.getting(DokkaTask::class) { outputFormat = "html" outputDirectory = "$buildDir/javadoc" @@ -33,42 +33,43 @@ plugins.withType(KotlinMultiplatformPlugin::class){ } } - val javadocJar by tasks.registering(Jar::class) { + val kdocJar by tasks.registering(Jar::class) { group = JavaBasePlugin.DOCUMENTATION_GROUP dependsOn(dokka) archiveClassifier.set("javadoc") from("$buildDir/javadoc") } - configure{ + configure { targets.all { val publication = publications.findByName(name) as MavenPublication // Patch publications with fake javadoc - publication.artifact(javadocJar.get()) + publication.artifact(kdocJar.get()) } } } -} -plugins.withType(KotlinPlatformJvmPlugin::class){ - val dokka by tasks.getting(DokkaTask::class) { - outputFormat = "html" - outputDirectory = "$buildDir/javadoc" - jdkVersion = 8 - } - val javadocJar by tasks.registering(Jar::class) { - group = JavaBasePlugin.DOCUMENTATION_GROUP - dependsOn(dokka) - archiveClassifier.set("javadoc") - from("$buildDir/javadoc") - } + extensions.findByType()?.apply{ + val dokka by tasks.getting(DokkaTask::class) { + outputFormat = "html" + outputDirectory = "$buildDir/javadoc" + jdkVersion = 8 + } - configure{ - publications.filterIsInstance().forEach {publication -> - publication.artifact(javadocJar.get()) + val kdocJar by tasks.registering(Jar::class) { + group = JavaBasePlugin.DOCUMENTATION_GROUP + dependsOn(dokka) + archiveClassifier.set("javadoc") + from("$buildDir/javadoc") + } + + configure { + publications.filterIsInstance().forEach { publication -> + publication.artifact(kdocJar.get()) + } } } } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt index 1f9a3522..00fcec23 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt @@ -41,6 +41,6 @@ interface Configurable { fun T.configure(meta: Meta): T = this.apply { config.update(meta) } -fun T.configure(action: Config.() -> Unit): T = this.apply { config.apply(action) } +fun T.configure(action: MetaBuilder.() -> Unit): T = configure(buildMeta(action)) open class SimpleConfigurable(override val config: Config) : Configurable \ No newline at end of file From f1692297b8c08f4df77e5b3ef89a1cfa1ca2338f Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 18 May 2019 11:12:41 +0300 Subject: [PATCH 05/48] Minor delegates optimization --- .../hep/dataforge/meta/ConfigDelegates.kt | 32 +++++++++++------ .../hep/dataforge/meta/ExtraMetaDelegates.kt | 36 ------------------- .../kotlin/hep/dataforge/meta/Specific.kt | 13 ++++--- 3 files changed, 29 insertions(+), 52 deletions(-) delete mode 100644 dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ExtraMetaDelegates.kt diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ConfigDelegates.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ConfigDelegates.kt index e848ec31..39a5f569 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ConfigDelegates.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ConfigDelegates.kt @@ -10,25 +10,19 @@ import kotlin.jvm.JvmName /** * A property delegate that uses custom key */ -fun Configurable.value(default: Any = Null, key: String? = null) = +fun Configurable.value(default: Any = Null, key: String? = null): MutableValueDelegate = MutableValueDelegate(config, key, Value.of(default)) -fun Configurable.value(default: T? = null, key: String? = null, transform: (Value?) -> T) = +fun Configurable.value(default: T? = null, key: String? = null, transform: (Value?) -> T): ReadWriteDelegateWrapper = MutableValueDelegate(config, key, Value.of(default)).transform(reader = transform) -fun Configurable.stringList(key: String? = null) = - value(key) { it?.list?.map { value -> value.string } ?: emptyList() } - -fun Configurable.numberList(key: String? = null) = - value(key) { it?.list?.map { value -> value.number } ?: emptyList() } - -fun Configurable.string(default: String? = null, key: String? = null) = +fun Configurable.string(default: String? = null, key: String? = null): MutableStringDelegate = MutableStringDelegate(config, key, default) -fun Configurable.boolean(default: Boolean? = null, key: String? = null) = +fun Configurable.boolean(default: Boolean? = null, key: String? = null): MutableBooleanDelegate = MutableBooleanDelegate(config, key, default) -fun Configurable.number(default: Number? = null, key: String? = null) = +fun Configurable.number(default: Number? = null, key: String? = null): MutableNumberDelegate = MutableNumberDelegate(config, key, default) /* Number delegates*/ @@ -111,3 +105,19 @@ fun Configurable.spec(spec: Specification, key: String? = null fun Configurable.spec(builder: (Config) -> T, key: String? = null) = MutableMorphDelegate(config, key) { specification(builder).wrap(it) } + + +/* + * Extra delegates for special cases + */ + +fun Configurable.stringList(key: String? = null): ReadWriteDelegateWrapper> = + value(emptyList(), key) { it?.list?.map { value -> value.string } ?: emptyList() } + +fun Configurable.numberList(key: String? = null): ReadWriteDelegateWrapper> = + value(emptyList(), key) { it?.list?.map { value -> value.number } ?: emptyList() } + +fun Metoid.child(key: String? = null, converter: (Meta) -> T) = ChildDelegate(meta, key, converter) + +fun Configurable.child(key: String? = null, converter: (Meta) -> T) = + MutableMorphDelegate(config, key, converter) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ExtraMetaDelegates.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ExtraMetaDelegates.kt deleted file mode 100644 index 6e24c6d0..00000000 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ExtraMetaDelegates.kt +++ /dev/null @@ -1,36 +0,0 @@ -package hep.dataforge.meta - -import kotlin.properties.ReadWriteProperty -import kotlin.reflect.KProperty - -/* - * Extra delegates for special cases - */ - -/** - * A delegate for a string list - */ -class StringListConfigDelegate( - val config: Config, - private val key: String? = null, - private val default: List = emptyList() -) : - ReadWriteProperty> { - override fun getValue(thisRef: Any?, property: KProperty<*>): List { - return config[key ?: property.name]?.value?.list?.map { it.string } ?: default - } - - override fun setValue(thisRef: Any?, property: KProperty<*>, value: List) { - val name = key ?: property.name - config[name] = value - } -} - -fun Configurable.stringList(vararg default: String = emptyArray(), key: String? = null) = - StringListConfigDelegate(config, key, default.toList()) - - -fun Metoid.child(key: String? = null, converter: (Meta) -> T) = ChildDelegate(meta, key, converter) - -fun Configurable.child(key: String? = null, converter: (Meta) -> T) = - MutableMorphDelegate(config, key, converter) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt index 10210fb4..a89a44a1 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specific.kt @@ -1,11 +1,13 @@ package hep.dataforge.meta /** - * Marker interface for specifications + * Marker interface for classes with specifications */ -interface Specific : Configurable { - operator fun get(name: String): MetaItem? = config[name] -} +interface Specific : Configurable + +//TODO separate mutable config from immutable meta to allow free wrapping of meta + +operator fun Specific.get(name: String): MetaItem<*>? = config[name] /** * Allows to apply custom configuration in a type safe way to simple untyped configuration. @@ -29,6 +31,7 @@ interface Specification { */ fun wrap(config: Config): T + //TODO replace by free wrapper fun wrap(meta: Meta): T = wrap(meta.toConfig()) } @@ -59,4 +62,4 @@ fun > S.createStyle(action: C.() -> Unit): Me fun Specific.spec( spec: Specification, key: String? = null -) = MutableMorphDelegate(config, key) { spec.wrap(it) } \ No newline at end of file +): MutableMorphDelegate = MutableMorphDelegate(config, key) { spec.wrap(it) } \ No newline at end of file From 2a395f80644bc4ada7471f4320536227ff8555b2 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 19 May 2019 10:05:21 +0300 Subject: [PATCH 06/48] Test for plugin task loading --- .../kotlin/hep/dataforge/context/Context.kt | 4 +- .../kotlin/hep/dataforge/provider/Provider.kt | 5 +- .../dataforge/descriptors/NodeDescriptor.kt | 4 +- .../dataforge/workspace/SimpleWorkspace.kt | 4 +- .../dataforge/workspace/WorkspaceBuilder.kt | 2 +- .../hep/dataforge/workspace/TaskBuilder.kt | 2 +- .../workspace/SimpleWorkspaceTest.kt | 47 ++++++++++++++++++- 7 files changed, 58 insertions(+), 10 deletions(-) diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt index cf6bb662..8d355173 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt @@ -116,11 +116,11 @@ open class Context(final override val name: String, val parent: Context? = Globa } } +fun Context.content(target: String): Map = content(target) + /** * A sequences of all objects provided by plugins with given target and type */ -fun Context.content(target: String): Map = content(target) - @JvmName("typedContent") inline fun Context.content(target: String): Map = plugins.flatMap { plugin -> diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Provider.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Provider.kt index 657f272d..35f1aca0 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Provider.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Provider.kt @@ -85,11 +85,12 @@ inline fun Provider.provide(target: String, name: Name): T? { inline fun Provider.provide(target: String, name: String): T? = provide(target, name.toName()) + +fun Provider.top(target: String): Map = top(target) + /** * A top level content with names */ -fun Provider.top(target: String): Map = top(target) - @JvmName("typedTop") inline fun Provider.top(target: String): Map { return listNames(target).associate { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt index 902c81a3..238cc865 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt @@ -104,7 +104,7 @@ class NodeDescriptor(override val config: Config) : Specific { */ val nodes: Map get() = config.getAll("node".toName()).entries.associate { (name, node) -> - name to NodeDescriptor.wrap(node.node ?: error("Node descriptor must be a node")) + name to wrap(node.node ?: error("Node descriptor must be a node")) } @@ -114,7 +114,7 @@ class NodeDescriptor(override val config: Config) : Specific { } fun node(name: String, block: NodeDescriptor.() -> Unit) { - node(name, NodeDescriptor.build { this.name = name }.apply(block)) + node(name, build { this.name = name }.apply(block)) } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt index c7a54f3d..d707545d 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt @@ -2,6 +2,7 @@ package hep.dataforge.workspace import hep.dataforge.context.Context import hep.dataforge.context.Global +import hep.dataforge.context.content import hep.dataforge.data.DataNode import hep.dataforge.meta.Meta import hep.dataforge.names.Name @@ -18,8 +19,9 @@ class SimpleWorkspace( override val targets: Map, tasks: Collection> ) : Workspace { + override val tasks: Map> by lazy { - context.top>(Task.TYPE) + tasks.associate { it.name.toName() to it } + context.content>(Task.TYPE) + tasks.associate { it.name.toName() to it } } companion object { diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt index f62753a2..b8df2fef 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt @@ -26,7 +26,7 @@ interface WorkspaceBuilder { /** * Set the context for future workspcace */ -fun WorkspaceBuilder.context(name: String, block: ContextBuilder.() -> Unit = {}) { +fun WorkspaceBuilder.context(name: String = "WORKSPACE", block: ContextBuilder.() -> Unit = {}) { context = ContextBuilder(name, parentContext).apply(block).build() } diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index 761dc8f3..d2e1b864 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -207,7 +207,7 @@ class TaskBuilder(val name: String) { } } -fun task(name: String, builder: TaskBuilder.() -> Unit): GenericTask { +fun Workspace.Companion.task(name: String, builder: TaskBuilder.() -> Unit): GenericTask { return TaskBuilder(name).apply(builder).build() } diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index 82bb2eb7..ff57c53b 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -1,16 +1,52 @@ package hep.dataforge.workspace +import hep.dataforge.context.AbstractPlugin +import hep.dataforge.context.PluginTag import hep.dataforge.data.first import hep.dataforge.data.get import hep.dataforge.meta.boolean import hep.dataforge.meta.get +import hep.dataforge.names.Name +import hep.dataforge.names.toName import org.junit.Test import kotlin.test.assertEquals +import kotlin.test.assertTrue class SimpleWorkspaceTest { + val testPlugin = object : AbstractPlugin() { + override val tag: PluginTag = PluginTag("test") + + val contextTask = Workspace.task("test") { + pipe { + context.logger.info { "Test: $it" } + } + } + + override fun provideTop(target: String, name: Name): Any? { + return if (target == Task.TYPE && name == "test".toName()) { + contextTask + } else { + null + } + } + + override fun listNames(target: String): Sequence { + return if(target== Task.TYPE){ + sequenceOf(contextTask.name.toName()) + } else{ + emptySequence() + } + } + + } + val workspace = SimpleWorkspace.build { + context{ + plugin(testPlugin) + } + repeat(100) { static("myData[$it]", it) } @@ -67,6 +103,8 @@ class SimpleWorkspaceTest { data["even"]!! - data["odd"]!! } } + + target("empty") {} } @Test @@ -78,7 +116,14 @@ class SimpleWorkspaceTest { @Test fun testMetaPropagation() { - val node = workspace.run("sum"){"testFlag" to true} + val node = workspace.run("sum") { "testFlag" to true } val res = node.first().get() } + + @Test + fun testPluginTask() { + val tasks = workspace.tasks + assertTrue { tasks["test.test"] != null } + //val node = workspace.run("test.test", "empty") + } } \ No newline at end of file From 343ba84118504ebfd137322c546c2f673322fbf2 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 19 May 2019 10:14:49 +0300 Subject: [PATCH 07/48] Introduced a WorkspacePlugin to access tasks from plugin easily --- .../dataforge/workspace/WorkspacePlugin.kt | 28 +++++++++++++++++++ .../workspace/SimpleWorkspaceTest.kt | 25 ++--------------- 2 files changed, 31 insertions(+), 22 deletions(-) create mode 100644 dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt new file mode 100644 index 00000000..7f808166 --- /dev/null +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt @@ -0,0 +1,28 @@ +package hep.dataforge.workspace + +import hep.dataforge.context.AbstractPlugin +import hep.dataforge.names.Name +import hep.dataforge.names.toName + +/** + * An abstract plugin with some additional boilerplate to effectively work with workspace context + */ +abstract class WorkspacePlugin : AbstractPlugin() { + abstract val tasks: Collection> + + override fun provideTop(target: String, name: Name): Any? { + return if (target == Task.TYPE) { + tasks.find { it.name == name.toString() } + } else { + super.provideTop(target, name) + } + } + + override fun listNames(target: String): Sequence { + return if (target == Task.TYPE) { + tasks.asSequence().map { it.name.toName() } + } else { + return super.listNames(target) + } + } +} \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index ff57c53b..962dab53 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -1,20 +1,17 @@ package hep.dataforge.workspace -import hep.dataforge.context.AbstractPlugin import hep.dataforge.context.PluginTag import hep.dataforge.data.first import hep.dataforge.data.get import hep.dataforge.meta.boolean import hep.dataforge.meta.get -import hep.dataforge.names.Name -import hep.dataforge.names.toName import org.junit.Test import kotlin.test.assertEquals import kotlin.test.assertTrue class SimpleWorkspaceTest { - val testPlugin = object : AbstractPlugin() { + val testPlugin = object : WorkspacePlugin() { override val tag: PluginTag = PluginTag("test") val contextTask = Workspace.task("test") { @@ -22,28 +19,12 @@ class SimpleWorkspaceTest { context.logger.info { "Test: $it" } } } - - override fun provideTop(target: String, name: Name): Any? { - return if (target == Task.TYPE && name == "test".toName()) { - contextTask - } else { - null - } - } - - override fun listNames(target: String): Sequence { - return if(target== Task.TYPE){ - sequenceOf(contextTask.name.toName()) - } else{ - emptySequence() - } - } - + override val tasks: Collection> = listOf(contextTask) } val workspace = SimpleWorkspace.build { - context{ + context { plugin(testPlugin) } From 54c7b55bc4010b98553b3d924d33b1dfdd7b3771 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 19 May 2019 21:42:56 +0300 Subject: [PATCH 08/48] Working on meta transformations --- build.gradle.kts | 2 +- .../kotlin/hep/dataforge/meta/Meta.kt | 25 +++++-- .../kotlin/hep/dataforge/meta/MetaBuilder.kt | 10 ++- .../hep/dataforge/meta/MetaTransformation.kt | 72 +++++++++++++++++++ .../kotlin/hep/dataforge/meta/MutableMeta.kt | 20 ++++-- 5 files changed, 115 insertions(+), 14 deletions(-) create mode 100644 dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt diff --git a/build.gradle.kts b/build.gradle.kts index e2482330..35e241e4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,4 +1,4 @@ -val dataforgeVersion by extra("0.1.2") +val dataforgeVersion by extra("0.1.3-dev-1") allprojects { repositories { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index e62c3621..329989de 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -94,16 +94,31 @@ fun Meta.getAll(name: String): Map> = getAll(name.toN * Get a sequence of [Name]-[Value] pairs */ fun Meta.values(): Sequence> { - return items.asSequence().flatMap { entry -> - val item = entry.value + return items.asSequence().flatMap { (key, item) -> when (item) { - is ValueItem -> sequenceOf(entry.key.asName() to item.value) - is NodeItem -> item.node.values().map { pair -> (entry.key.asName() + pair.first) to pair.second } + is ValueItem -> sequenceOf(key.asName() to item.value) + is NodeItem -> item.node.values().map { pair -> (key.asName() + pair.first) to pair.second } } } } -operator fun Meta.iterator(): Iterator> = values().iterator() +/** + * Get a sequence of all [Name]-[MetaItem] pairs for all items including nodes + */ +fun Meta.sequence(): Sequence>> { + return sequence { + items.forEach { (key, item) -> + yield(key.asName() to item) + if(item is NodeItem<*>) { + yieldAll(item.node.sequence().map { (innerKey, innerItem)-> + (key + innerKey) to innerItem + }) + } + } + } +} + +operator fun Meta.iterator(): Iterator>> = sequence().iterator() /** * A meta node that ensures that all of its descendants has at least the same type diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt index fe6829d4..7debc39b 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt @@ -46,4 +46,12 @@ fun Meta.builder(): MetaBuilder { } } -fun buildMeta(builder: MetaBuilder.() -> Unit): MetaBuilder = MetaBuilder().apply(builder) \ No newline at end of file +/** + * Build a [MetaBuilder] using given transformation + */ +fun buildMeta(builder: MetaBuilder.() -> Unit): MetaBuilder = MetaBuilder().apply(builder) + +/** + * Build meta using given source meta as a base + */ +fun buildMeta(source: Meta, builder: MetaBuilder.() -> Unit): MetaBuilder = source.builder().apply(builder) \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt new file mode 100644 index 00000000..0b8321d9 --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt @@ -0,0 +1,72 @@ +package hep.dataforge.meta + +import hep.dataforge.names.Name + +/** + * A transformation for meta item or a group of items + */ +interface TransformationRule { + + /** + * Check if this transformation + */ + fun matches(name: Name, item: MetaItem<*>?): Boolean + + /** + * Select all items to be transformed. Item could be a value as well as node + * + * @return a sequence of item paths to be transformed + */ + fun selectItems(meta: Meta): Sequence = + meta.sequence().filter { matches(it.first, it.second) }.map { it.first } + + /** + * Apply transformation for a single item (Node or Value) and return resulting tree with absolute path + */ + fun > transformItem(name: Name, item: MetaItem<*>?, target: M): Unit +} + +data class SelfTransformationRule(val name: Name) : TransformationRule { + override fun matches(name: Name, item: MetaItem<*>?): Boolean { + return name == name + } + + override fun selectItems(meta: Meta): Sequence = sequenceOf(name) + + override fun > transformItem(name: Name, item: MetaItem<*>?, target: M) { + if (name == this.name) target[name] = item + } +} + +data class SingleItemTransformationRule( + val from: Name, + val to: Name, + val transform: MutableMetaNode<*>.(MetaItem<*>?) -> Unit +) : TransformationRule { + override fun matches(name: Name, item: MetaItem<*>?): Boolean { + return name == from + } + + override fun selectItems(meta: Meta): Sequence = sequenceOf(from) + + override fun > transformItem(name: Name, item: MetaItem<*>?, target: M) { + if (name == this.from) { + target.transform(item) + } + } +} + +class MetaTransformation { + private val transformations = HashSet() + + /** + * Produce new meta using only those items that match transformation rules + */ + fun produce(source: Meta): Meta = buildMeta { + transformations.forEach { rule -> + rule.selectItems(source).forEach { name -> + rule.transformItem(name, source[name], this) + } + } + } +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt index 8182495b..2b3e9c7a 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt @@ -96,11 +96,20 @@ fun > MutableMeta.remove(name: Name) = set(name, null) fun > MutableMeta.remove(name: String) = remove(name.toName()) fun > MutableMeta.setValue(name: Name, value: Value) = set(name, MetaItem.ValueItem(value)) -fun > MutableMeta.setItem(name: String, item: MetaItem) = set(name.toName(), item) +//fun > MutableMeta.setItem(name: String, item: MetaItem) = set(name.toName(), item) fun > MutableMeta.setValue(name: String, value: Value) = set(name.toName(), MetaItem.ValueItem(value)) -fun > MutableMeta.setItem(token: NameToken, item: MetaItem?) = set(token.asName(), item) +//fun > MutableMeta.setItem(token: NameToken, item: MetaItem?) = set(token.asName(), item) + +fun > MutableMetaNode.setItem(name: Name, item: MetaItem<*>) { + when (item) { + is MetaItem.ValueItem<*> -> setValue(name, item.value) + is MetaItem.NodeItem<*> -> setNode(name, item.node) + } +} + +fun > MutableMetaNode.setItem(name: String, item: MetaItem<*>) = setItem(name.toName(), item) fun > MutableMetaNode.setNode(name: Name, node: Meta) = set(name, MetaItem.NodeItem(wrap(name, node))) @@ -110,13 +119,10 @@ fun > MutableMetaNode.setNode(name: String, node: Meta /** * Universal set method */ -operator fun > M.set(name: Name, value: Any?) { +operator fun > MutableMetaNode.set(name: Name, value: Any?) { when (value) { null -> remove(name) - is MetaItem<*> -> when (value) { - is MetaItem.ValueItem<*> -> setValue(name, value.value) - is MetaItem.NodeItem<*> -> setNode(name, value.node) - } + is MetaItem<*> -> setItem(name, value) is Meta -> setNode(name, value) is Specific -> setNode(name, value.config) else -> setValue(name, Value.of(value)) From 7c38cadddd53e0c64498e5149892949574049820 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Thu, 23 May 2019 22:16:34 +0300 Subject: [PATCH 09/48] Envelope IO --- .../kotlin/hep/dataforge/io/Binary.kt | 60 ++++++++++++++ .../hep/dataforge/io/BinaryMetaFormat.kt | 20 ++--- .../kotlin/hep/dataforge/io/Envelope.kt | 9 +-- .../kotlin/hep/dataforge/io/IOFormat.kt | 16 ++-- .../kotlin/hep/dataforge/io/JsonMetaFormat.kt | 18 ++--- .../kotlin/hep/dataforge/io/MetaFormat.kt | 19 ++--- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 80 +++++++++++++++++++ .../kotlin/hep/dataforge/io/FileBinary.kt | 16 ++++ .../kotlin/hep/dataforge/meta/MutableMeta.kt | 5 +- 9 files changed, 191 insertions(+), 52 deletions(-) create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt create mode 100644 dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt new file mode 100644 index 00000000..a0374968 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt @@ -0,0 +1,60 @@ +package hep.dataforge.io + +import kotlinx.io.core.ByteReadPacket +import kotlinx.io.core.Input +import kotlinx.io.core.readBytes + +/** + * A source of binary data + */ +interface Binary { + /** + * The size of binary in bytes + */ + val size: ULong + + /** + * Read continuous [Input] from this binary stating from the beginning. + * The input is automatically closed on scope close. + * Some implementation may forbid this to be called twice. In this case second call will throw an exception. + */ + fun read(block: Input.() -> R): R +} + +/** + * A [Binary] with addition random access functionality. It by default allows multiple [read] operations. + */ +interface RandomAccessBinary : Binary { + /** + * Read at most [size] of bytes starting at [from] offset from the beginning of the binary. + * This method could be called multiple times simultaneously. + */ + fun read(from: UInt, size: UInt = UInt.MAX_VALUE, block: Input.() -> R): R + + override fun read(block: Input.() -> R): R = read(0.toUInt(), UInt.MAX_VALUE, block) +} + +fun Binary.readAll(): ByteReadPacket = read { + ByteReadPacket(this.readBytes()) +} + +fun RandomAccessBinary.readPacket(from: UInt, size: UInt): ByteReadPacket = read(from, size) { + ByteReadPacket(this.readBytes()) +} + +object EmptyBinary : RandomAccessBinary { + + override val size: ULong = 0.toULong() + + override fun read(from: UInt, size: UInt, block: Input.() -> R): R { + error("The binary is empty") + } +} + +class ArrayBinary(val array: ByteArray) : RandomAccessBinary { + override val size: ULong = array.size.toULong() + + override fun read(from: UInt, size: UInt, block: Input.() -> R): R { + return ByteReadPacket(array, from.toInt(), size.toInt()).block() + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt index 0e73934a..79145796 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt @@ -8,12 +8,11 @@ import kotlinx.io.core.readText import kotlinx.io.core.writeText object BinaryMetaFormat : MetaFormat { - override fun write(obj: Meta, out: Output) { - out.writeMeta(obj) - } + override val name: String = "bin" + override val key: Short = 0x4249//BI - override fun read(input: Input): Meta { - return (input.readMetaItem() as MetaItem.NodeItem).node + override fun Input.readObject(): Meta { + return (readMetaItem() as MetaItem.NodeItem).node } private fun Output.writeChar(char: Char) = writeByte(char.toByte()) @@ -70,7 +69,7 @@ object BinaryMetaFormat : MetaFormat { } } - private fun Output.writeMeta(meta: Meta) { + override fun Output.writeObject(meta: Meta) { writeChar('M') writeInt(meta.items.size) meta.items.forEach { (key, item) -> @@ -80,7 +79,7 @@ object BinaryMetaFormat : MetaFormat { writeValue(item.value) } is MetaItem.NodeItem -> { - writeMeta(item.node) + writeObject(item.node) } } } @@ -122,11 +121,4 @@ object BinaryMetaFormat : MetaFormat { else -> error("Unknown serialization key character: $keyChar") } } -} - -class BinaryMetaFormatFactory : MetaFormatFactory { - override val name: String = "bin" - override val key: Short = 0x4249//BI - - override fun build(): MetaFormat = BinaryMetaFormat } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt index 1a9e58d7..fc3ae90d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt @@ -7,7 +7,7 @@ import kotlinx.io.core.Input interface Envelope { val meta: Meta - val data: Input? + val data: Binary? companion object { @@ -23,11 +23,7 @@ interface Envelope { } } -class SimpleEnvelope(override val meta: Meta, val dataProvider: () -> Input?) : Envelope{ - override val data: Input? - get() = dataProvider() - -} +class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope /** * The purpose of the envelope @@ -50,3 +46,4 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str */ val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string +typealias EnvelopeFormat = IOFormat \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index ce58c05d..9055b249 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -1,10 +1,14 @@ package hep.dataforge.io -import kotlinx.io.core.Input -import kotlinx.io.core.Output - +import kotlinx.io.core.* +/** + * And interface for serialization facilities + */ interface IOFormat { - fun write(obj: T, out: Output) - fun read(input: Input): T -} \ No newline at end of file + fun Output.writeObject(obj: T) + fun Input.readObject(): T +} + +fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeObject(obj) } +fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeObject(obj) }.readBytes() \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index 9d7b5739..13cbc717 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -14,13 +14,16 @@ import kotlinx.serialization.json.* object JsonMetaFormat : MetaFormat { - override fun write(obj: Meta, out: Output) { + override val name: String = "json" + override val key: Short = 0x4a53//"JS" + + override fun Output.writeObject(obj: Meta) { val str = obj.toJson().toString() - out.writeText(str) + writeText(str) } - override fun read(input: Input): Meta { - val str = input.readText() + override fun Input.readObject(): Meta { + val str = readText() val json = Json.plain.parseJson(str) if (json is JsonObject) { @@ -97,11 +100,4 @@ class JsonMeta(val json: JsonObject) : Meta { json.forEach { (key, value) -> map[key] = value } map.mapKeys { it.key.toName().first()!! } } -} - -class JsonMetaFormatFactory : MetaFormatFactory { - override val name: String = "json" - override val key: Short = 0x4a53//"JS" - - override fun build() = JsonMetaFormat } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index 61969536..52f01525 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -1,33 +1,26 @@ package hep.dataforge.io import hep.dataforge.meta.Meta -import kotlinx.io.core.BytePacketBuilder import kotlinx.io.core.ByteReadPacket +import kotlinx.io.core.buildPacket import kotlinx.io.core.toByteArray /** * A format for meta serialization */ -interface MetaFormat: IOFormat - -/** - * ServiceLoader compatible factory - */ -interface MetaFormatFactory { +interface MetaFormat : IOFormat { val name: String val key: Short - - fun build(): MetaFormat } fun Meta.asString(format: MetaFormat = JsonMetaFormat): String { - val builder = BytePacketBuilder() - format.write(this, builder) - return builder.build().readText() + return buildPacket { + format.run { writeObject(this@asString) } + }.readText() } fun MetaFormat.parse(str: String): Meta { - return read(ByteReadPacket(str.toByteArray())) + return ByteReadPacket(str.toByteArray()).readObject() } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt new file mode 100644 index 00000000..40363eed --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -0,0 +1,80 @@ +package hep.dataforge.io + +import kotlinx.io.core.* + +class TaggedEnvelopeFormat( + val metaFormats: Collection, + val outputMetaFormat: MetaFormat = metaFormats.first() +) : EnvelopeFormat { + + override fun Output.writeObject(obj: Envelope) { + write(obj, this, outputMetaFormat) + } + + /** + * Read an envelope from input into memory + * + * @param input an input to read from + * @param metaFormats a collection of meta formats to resolve + */ + override fun Input.readObject(): Envelope = read(this, metaFormats) + + + private data class Tag( + val metaFormatKey: Short, + val metaSize: UInt, + val dataSize: ULong + ) + + companion object { + private const val VERSION = "DF03" + private const val START_SEQUENCE = "#~" + private const val END_SEQUENCE = "~#\r\n" + + private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) { + writeText(START_SEQUENCE) + writeText(VERSION) + writeShort(metaFormatKey) + writeUInt(metaSize) + writeULong(dataSize) + writeText(END_SEQUENCE) + } + + private fun Input.readTag(): Tag { + val start = readTextExactBytes(2) + if (start != START_SEQUENCE) error("The input is not an envelope") + val version = readTextExactBytes(4) + if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version") + val metaFormatKey = readShort() + val metaLength = readUInt() + val dataLength = readULong() + return Tag(metaFormatKey, metaLength, dataLength) + } + + fun read(input: Input, metaFormats: Collection): Envelope { + val tag = input.readTag() + + val metaFormat = metaFormats.find { it.key == tag.metaFormatKey } + ?: error("Meta format with key ${tag.metaFormatKey} not found") + + val metaPacket = ByteReadPacket(input.readBytes(tag.metaSize.toInt())) + val meta = metaFormat.run { metaPacket.readObject() } + + val dataBytes = input.readBytes(tag.dataSize.toInt()) + + return SimpleEnvelope(meta, ArrayBinary(dataBytes)) + } + + fun write(obj: Envelope, out: Output, metaFormat: MetaFormat) { + val metaBytes = metaFormat.writeBytes(obj.meta) + val tag = Tag(metaFormat.key, metaBytes.size.toUInt(), obj.data?.size ?: 0.toULong()) + out.writePacket(tag.toBytes()) + out.writeFully(metaBytes) + obj.data?.read { + while (!endOfInput){ + out.writeByte(readByte()) + } + } + } + } +} \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt new file mode 100644 index 00000000..3154a2cf --- /dev/null +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt @@ -0,0 +1,16 @@ +package hep.dataforge.io + +import kotlinx.io.core.ByteReadPacket +import kotlinx.io.core.Input +import java.nio.channels.FileChannel +import java.nio.file.Path +import java.nio.file.StandardOpenOption + +class FileBinary(val path: Path, private val offset: Int = 0) : RandomAccessBinary { + override fun read(from: Long, size: Long, block: Input.() -> R): R { + FileChannel.open(path, StandardOpenOption.READ).use { + val buffer = it.map(FileChannel.MapMode.READ_ONLY, from + offset, size) + return ByteReadPacket(buffer).block() + } + } +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt index 2b3e9c7a..000bff7a 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt @@ -95,12 +95,13 @@ abstract class MutableMetaNode> : AbstractMetaNode(), fun > MutableMeta.remove(name: Name) = set(name, null) fun > MutableMeta.remove(name: String) = remove(name.toName()) -fun > MutableMeta.setValue(name: Name, value: Value) = set(name, MetaItem.ValueItem(value)) -//fun > MutableMeta.setItem(name: String, item: MetaItem) = set(name.toName(), item) +fun > MutableMeta.setValue(name: Name, value: Value) = + set(name, MetaItem.ValueItem(value)) fun > MutableMeta.setValue(name: String, value: Value) = set(name.toName(), MetaItem.ValueItem(value)) //fun > MutableMeta.setItem(token: NameToken, item: MetaItem?) = set(token.asName(), item) +//fun > MutableMeta.setItem(name: String, item: MetaItem) = set(name.toName(), item) fun > MutableMetaNode.setItem(name: Name, item: MetaItem<*>) { when (item) { From 13c5a579edca7f23a2cb26c7422adff586a2416c Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 26 May 2019 22:05:20 +0300 Subject: [PATCH 10/48] Envelope IO --- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 6 +---- .../hep/dataforge/meta/MetaTransformation.kt | 25 ++++++++++++++++++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index 40363eed..ccb6e382 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -70,11 +70,7 @@ class TaggedEnvelopeFormat( val tag = Tag(metaFormat.key, metaBytes.size.toUInt(), obj.data?.size ?: 0.toULong()) out.writePacket(tag.toBytes()) out.writeFully(metaBytes) - obj.data?.read { - while (!endOfInput){ - out.writeByte(readByte()) - } - } + obj.data?.read { copyTo(out) } } } } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt index 0b8321d9..2e00bd11 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt @@ -26,6 +26,9 @@ interface TransformationRule { fun > transformItem(name: Name, item: MetaItem<*>?, target: M): Unit } +/** + * A transformation which transforms an element with given [name] to itself. + */ data class SelfTransformationRule(val name: Name) : TransformationRule { override fun matches(name: Name, item: MetaItem<*>?): Boolean { return name == name @@ -38,6 +41,9 @@ data class SelfTransformationRule(val name: Name) : TransformationRule { } } +/** + * A transformation which transforms element with specific name + */ data class SingleItemTransformationRule( val from: Name, val to: Name, @@ -59,14 +65,31 @@ data class SingleItemTransformationRule( class MetaTransformation { private val transformations = HashSet() + /** * Produce new meta using only those items that match transformation rules */ - fun produce(source: Meta): Meta = buildMeta { + fun transform(source: Meta): Meta = buildMeta { transformations.forEach { rule -> rule.selectItems(source).forEach { name -> rule.transformItem(name, source[name], this) } } } + + /** + * Transform a meta, replacing all elements found in rules with transformed entries + */ + fun apply(source: Meta): Meta = buildMeta(source) { + transformations.forEach { rule -> + rule.selectItems(source).forEach { name -> + remove(name) + rule.transformItem(name, source[name], this) + } + } + } + + companion object{ + + } } \ No newline at end of file From 65a5379f4a3d78da076eb8f61374bf05d8e29389 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 28 May 2019 17:59:55 +0300 Subject: [PATCH 11/48] build fix --- .../kotlin/hep/dataforge/descriptors/Described.kt | 3 +++ .../hep/dataforge/descriptors/NodeDescriptor.kt | 12 +++++++----- .../hep/dataforge/descriptors/ValueDescriptor.kt | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/Described.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/Described.kt index a0780a3e..0ae7a13c 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/Described.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/Described.kt @@ -1,5 +1,8 @@ package hep.dataforge.descriptors +/** + * An object which provides its descriptor + */ interface Described { val descriptor: NodeDescriptor } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt index 238cc865..df748acf 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt @@ -83,12 +83,12 @@ class NodeDescriptor(override val config: Config) : Specific { * The list of value descriptors */ val values: Map - get() = config.getAll("value".toName()).entries.associate { (name, node) -> + get() = config.getAll(VALUE_KEY.toName()).entries.associate { (name, node) -> name to ValueDescriptor.wrap(node.node ?: error("Value descriptor must be a node")) } fun value(name: String, descriptor: ValueDescriptor) { - val token = NameToken("value", name) + val token = NameToken(VALUE_KEY, name) config[token] = descriptor.config } @@ -103,13 +103,13 @@ class NodeDescriptor(override val config: Config) : Specific { * The map of children node descriptors */ val nodes: Map - get() = config.getAll("node".toName()).entries.associate { (name, node) -> + get() = config.getAll(NODE_KEY.toName()).entries.associate { (name, node) -> name to wrap(node.node ?: error("Node descriptor must be a node")) } fun node(name: String, descriptor: NodeDescriptor) { - val token = NameToken("node", name) + val token = NameToken(NODE_KEY, name) config[token] = descriptor.config } @@ -122,7 +122,9 @@ class NodeDescriptor(override val config: Config) : Specific { companion object : Specification { - override fun wrap(config: Config): NodeDescriptor = NodeDescriptor(config) + const val NODE_KEY = "node" + const val VALUE_KEY = "value" + override fun wrap(config: Config): NodeDescriptor = NodeDescriptor(config) } } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ValueDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ValueDescriptor.kt index e3acd07b..cc3352c0 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ValueDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ValueDescriptor.kt @@ -126,7 +126,7 @@ class ValueDescriptor(override val config: Config) : Specific { override fun wrap(config: Config): ValueDescriptor = ValueDescriptor(config) inline fun > enum(name: String) = - ValueDescriptor.build { + build { this.name = name type(ValueType.STRING) this.allowedValues = enumValues().map { Value.of(it.name) } From cffb02d483af8ab67e6a564256fe792eb33fa964 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 28 May 2019 18:56:22 +0300 Subject: [PATCH 12/48] fixed FileBinary --- .../src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt index 3154a2cf..db74e8e0 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt @@ -3,13 +3,19 @@ package hep.dataforge.io import kotlinx.io.core.ByteReadPacket import kotlinx.io.core.Input import java.nio.channels.FileChannel +import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption class FileBinary(val path: Path, private val offset: Int = 0) : RandomAccessBinary { - override fun read(from: Long, size: Long, block: Input.() -> R): R { + + override val size: ULong + get() = (Files.size(path) - offset).toULong() + + + override fun read(from: UInt, size: UInt, block: Input.() -> R): R { FileChannel.open(path, StandardOpenOption.READ).use { - val buffer = it.map(FileChannel.MapMode.READ_ONLY, from + offset, size) + val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from.toLong() + offset), size.toLong()) return ByteReadPacket(buffer).block() } } From 59c46344fa898ef7f85134924301a9c760e027d2 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 28 May 2019 19:24:23 +0300 Subject: [PATCH 13/48] Simplified mutable meta hierarchy --- .../kotlin/hep/dataforge/meta/Config.kt | 2 +- .../kotlin/hep/dataforge/meta/Delegates.kt | 6 +-- .../kotlin/hep/dataforge/meta/MetaBuilder.kt | 2 +- .../hep/dataforge/meta/MetaTransformation.kt | 8 ++-- .../kotlin/hep/dataforge/meta/MutableMeta.kt | 47 ++++++++++--------- .../kotlin/hep/dataforge/meta/Styled.kt | 10 +++- 6 files changed, 42 insertions(+), 33 deletions(-) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt index 00fcec23..f6370942 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt @@ -9,7 +9,7 @@ import hep.dataforge.names.asName /** * Mutable meta representing object state */ -open class Config : MutableMetaNode() { +open class Config : AbstractMutableMeta() { /** * Attach configuration node instead of creating one diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Delegates.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Delegates.kt index 377ec994..bcd60cc8 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Delegates.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Delegates.kt @@ -327,7 +327,7 @@ class MutableSafeEnumvDelegate, E : Enum>( //Child node delegate -class MutableNodeDelegate>( +class MutableNodeDelegate>( val meta: M, private val key: String? = null ) : ReadWriteProperty { @@ -340,7 +340,7 @@ class MutableNodeDelegate>( } } -class MutableMorphDelegate, T : Configurable>( +class MutableMorphDelegate, T : Configurable>( val meta: M, private val key: String? = null, private val converter: (Meta) -> T @@ -390,7 +390,7 @@ fun > M.boolean(default: Boolean? = null, key: String? = null fun > M.number(default: Number? = null, key: String? = null) = MutableNumberDelegate(this, key, default) -fun > M.node(key: String? = null) = MutableNodeDelegate(this, key) +fun > M.node(key: String? = null) = MutableNodeDelegate(this, key) @JvmName("safeString") fun > M.string(default: String, key: String? = null) = diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt index 7debc39b..c8911c60 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt @@ -7,7 +7,7 @@ import hep.dataforge.values.Value /** * DSL builder for meta. Is not intended to store mutable state */ -class MetaBuilder : MutableMetaNode() { +class MetaBuilder : AbstractMutableMeta() { override fun wrap(name: Name, meta: Meta): MetaBuilder = meta.builder() override fun empty(): MetaBuilder = MetaBuilder() diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt index 2e00bd11..22a294b1 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt @@ -23,7 +23,7 @@ interface TransformationRule { /** * Apply transformation for a single item (Node or Value) and return resulting tree with absolute path */ - fun > transformItem(name: Name, item: MetaItem<*>?, target: M): Unit + fun > transformItem(name: Name, item: MetaItem<*>?, target: M): Unit } /** @@ -36,7 +36,7 @@ data class SelfTransformationRule(val name: Name) : TransformationRule { override fun selectItems(meta: Meta): Sequence = sequenceOf(name) - override fun > transformItem(name: Name, item: MetaItem<*>?, target: M) { + override fun > transformItem(name: Name, item: MetaItem<*>?, target: M) { if (name == this.name) target[name] = item } } @@ -47,7 +47,7 @@ data class SelfTransformationRule(val name: Name) : TransformationRule { data class SingleItemTransformationRule( val from: Name, val to: Name, - val transform: MutableMetaNode<*>.(MetaItem<*>?) -> Unit + val transform: MutableMeta<*>.(MetaItem<*>?) -> Unit ) : TransformationRule { override fun matches(name: Name, item: MetaItem<*>?): Boolean { return name == from @@ -55,7 +55,7 @@ data class SingleItemTransformationRule( override fun selectItems(meta: Meta): Sequence = sequenceOf(from) - override fun > transformItem(name: Name, item: MetaItem<*>?, target: M) { + override fun > transformItem(name: Name, item: MetaItem<*>?, target: M) { if (name == this.from) { target.transform(item) } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt index 000bff7a..83edc56b 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt @@ -10,6 +10,13 @@ internal data class MetaListener( interface MutableMeta> : MetaNode { + /** + * Transform given meta to node type of this meta tree + * @param name the name of the node where meta should be attached. Needed for correct assignment validators and styles + * @param meta the node itself + */ + fun wrap(name: Name, meta: Meta): M + override val items: Map> operator fun set(name: Name, item: MetaItem?) fun onChange(owner: Any? = null, action: (Name, MetaItem<*>?, MetaItem<*>?) -> Unit) @@ -21,7 +28,7 @@ interface MutableMeta> : MetaNode { * * Changes in Meta are not thread safe. */ -abstract class MutableMetaNode> : AbstractMetaNode(), MutableMeta { +abstract class AbstractMutableMeta> : AbstractMetaNode(), MutableMeta { private val listeners = HashSet() /** @@ -62,13 +69,6 @@ abstract class MutableMetaNode> : AbstractMetaNode(), itemChanged(key.asName(), oldItem, newItem) } - /** - * Transform given meta to node type of this meta tree - * @param name the name of the node where meta should be attached. Needed for correct assignment validators and styles - * @param meta the node itself - */ - internal abstract fun wrap(name: Name, meta: Meta): M - /** * Create empty node */ @@ -97,30 +97,31 @@ fun > MutableMeta.remove(name: String) = remove(name.toNam fun > MutableMeta.setValue(name: Name, value: Value) = set(name, MetaItem.ValueItem(value)) + fun > MutableMeta.setValue(name: String, value: Value) = set(name.toName(), MetaItem.ValueItem(value)) //fun > MutableMeta.setItem(token: NameToken, item: MetaItem?) = set(token.asName(), item) //fun > MutableMeta.setItem(name: String, item: MetaItem) = set(name.toName(), item) -fun > MutableMetaNode.setItem(name: Name, item: MetaItem<*>) { +fun > MutableMeta.setItem(name: Name, item: MetaItem<*>) { when (item) { is MetaItem.ValueItem<*> -> setValue(name, item.value) is MetaItem.NodeItem<*> -> setNode(name, item.node) } } -fun > MutableMetaNode.setItem(name: String, item: MetaItem<*>) = setItem(name.toName(), item) +fun > MutableMeta.setItem(name: String, item: MetaItem<*>) = setItem(name.toName(), item) -fun > MutableMetaNode.setNode(name: Name, node: Meta) = +fun > MutableMeta.setNode(name: Name, node: Meta) = set(name, MetaItem.NodeItem(wrap(name, node))) -fun > MutableMetaNode.setNode(name: String, node: Meta) = setNode(name.toName(), node) +fun > MutableMeta.setNode(name: String, node: Meta) = setNode(name.toName(), node) /** * Universal set method */ -operator fun > MutableMetaNode.set(name: Name, value: Any?) { +operator fun > MutableMeta.set(name: Name, value: Any?) { when (value) { null -> remove(name) is MetaItem<*> -> setItem(name, value) @@ -130,9 +131,9 @@ operator fun > MutableMetaNode.set(name: Name, value: } } -operator fun > M.set(name: NameToken, value: Any?) = set(name.asName(), value) +operator fun > M.set(name: NameToken, value: Any?) = set(name.asName(), value) -operator fun > M.set(key: String, value: Any?) = set(key.toName(), value) +operator fun > M.set(key: String, value: Any?) = set(key.toName(), value) /** * Update existing mutable node with another node. The rules are following: @@ -140,7 +141,7 @@ operator fun > M.set(key: String, value: Any?) = set(key. * * node updates node and replaces anything but node * * node list updates node list if number of nodes in the list is the same and replaces anything otherwise */ -fun > M.update(meta: Meta) { +fun > M.update(meta: Meta) { meta.items.forEach { entry -> val value = entry.value when (value) { @@ -153,7 +154,7 @@ fun > M.update(meta: Meta) { /* Same name siblings generation */ -fun > M.setIndexed( +fun > M.setIndexedItems( name: Name, items: Iterable>, indexFactory: MetaItem.(index: Int) -> String = { it.toString() } @@ -167,21 +168,21 @@ fun > M.setIndexed( } } -fun > M.setIndexed( +fun > M.setIndexed( name: Name, metas: Iterable, indexFactory: MetaItem.(index: Int) -> String = { it.toString() } ) { - setIndexed(name, metas.map { MetaItem.NodeItem(wrap(name, it)) }, indexFactory) + setIndexedItems(name, metas.map { MetaItem.NodeItem(wrap(name, it)) }, indexFactory) } -operator fun > M.set(name: Name, metas: Iterable) = setIndexed(name, metas) -operator fun > M.set(name: String, metas: Iterable) = setIndexed(name.toName(), metas) +operator fun > M.set(name: Name, metas: Iterable) = setIndexed(name, metas) +operator fun > M.set(name: String, metas: Iterable) = setIndexed(name.toName(), metas) /** * Append the node with a same-name-sibling, automatically generating numerical index */ -fun > M.append(name: Name, value: Any?) { +fun > M.append(name: Name, value: Any?) { require(!name.isEmpty()) { "Name could not be empty for append operation" } val newIndex = name.last()!!.index if (newIndex.isNotEmpty()) { @@ -192,4 +193,4 @@ fun > M.append(name: Name, value: Any?) { } } -fun > M.append(name: String, value: Any?) = append(name.toName(), value) \ No newline at end of file +fun > M.append(name: String, value: Any?) = append(name.toName(), value) \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt index dcc36a5a..136c5617 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt @@ -11,7 +11,15 @@ import kotlin.reflect.KProperty * @param base - unchangeable base * @param style - the style */ -class Styled(val base: Meta, val style: Config = Config().empty()) : MutableMeta { +class Styled(val base: Meta, val style: Config = Config().empty()) : AbstractMutableMeta() { + override fun wrap(name: Name, meta: Meta): Styled { + TODO("not implemented") //To change body of created functions use File | Settings | File Templates. + } + + override fun empty(): Styled { + TODO("not implemented") //To change body of created functions use File | Settings | File Templates. + } + override val items: Map> get() = (base.items.keys + style.items.keys).associate { key -> val value = base.items[key] From f0413464a320d3f2089d3cc30a711aed15249d95 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 1 Jun 2019 08:14:37 +0300 Subject: [PATCH 14/48] Name comparison --- .../hep/dataforge/descriptors/Described.kt | 23 ++++++- .../dataforge/descriptors/NodeDescriptor.kt | 2 + .../hep/dataforge/meta/MetaTransformation.kt | 67 ++++++++++++++++--- .../kotlin/hep/dataforge/names/Name.kt | 12 +++- .../kotlin/hep/dataforge/names/NameTest.kt | 13 +++- 5 files changed, 102 insertions(+), 15 deletions(-) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/Described.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/Described.kt index 0ae7a13c..f355c828 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/Described.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/Described.kt @@ -1,8 +1,29 @@ package hep.dataforge.descriptors +import hep.dataforge.descriptors.Described.Companion.DESCRIPTOR_NODE +import hep.dataforge.meta.Meta +import hep.dataforge.meta.get +import hep.dataforge.meta.node + /** * An object which provides its descriptor */ interface Described { val descriptor: NodeDescriptor -} \ No newline at end of file + + companion object { + const val DESCRIPTOR_NODE = "@descriptor" + } +} + +/** + * If meta node supplies explicit descriptor, return it, otherwise try to use descriptor node from meta itself + */ +val Meta.descriptor: NodeDescriptor? + get() { + return if (this is Described) { + descriptor + } else { + get(DESCRIPTOR_NODE).node?.let { NodeDescriptor.wrap(it) } + } + } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt index df748acf..a9f73af5 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt @@ -126,5 +126,7 @@ class NodeDescriptor(override val config: Config) : Specific { const val VALUE_KEY = "value" override fun wrap(config: Config): NodeDescriptor = NodeDescriptor(config) + + //TODO infer descriptor from spec } } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt index 22a294b1..ab1a95fa 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt @@ -1,6 +1,7 @@ package hep.dataforge.meta import hep.dataforge.names.Name +import hep.dataforge.names.startsWith /** * A transformation for meta item or a group of items @@ -27,17 +28,18 @@ interface TransformationRule { } /** - * A transformation which transforms an element with given [name] to itself. + * A transformation which keeps all elements, matching [selector] unchanged. */ -data class SelfTransformationRule(val name: Name) : TransformationRule { +data class KeepTransformationRule(val selector: (Name) -> Boolean) : TransformationRule { override fun matches(name: Name, item: MetaItem<*>?): Boolean { - return name == name + return selector(name) } - override fun selectItems(meta: Meta): Sequence = sequenceOf(name) + override fun selectItems(meta: Meta): Sequence = + meta.sequence().map { it.first }.filter(selector) override fun > transformItem(name: Name, item: MetaItem<*>?, target: M) { - if (name == this.name) target[name] = item + if (selector(name)) target[name] = item } } @@ -46,8 +48,7 @@ data class SelfTransformationRule(val name: Name) : TransformationRule { */ data class SingleItemTransformationRule( val from: Name, - val to: Name, - val transform: MutableMeta<*>.(MetaItem<*>?) -> Unit + val transform: MutableMeta<*>.(Name, MetaItem<*>?) -> Unit ) : TransformationRule { override fun matches(name: Name, item: MetaItem<*>?): Boolean { return name == from @@ -57,14 +58,29 @@ data class SingleItemTransformationRule( override fun > transformItem(name: Name, item: MetaItem<*>?, target: M) { if (name == this.from) { - target.transform(item) + target.transform(name, item) } } } -class MetaTransformation { - private val transformations = HashSet() +data class RegexpItemTransformationRule( + val from: Regex, + val transform: MutableMeta<*>.(MatchResult, MetaItem<*>?) -> Unit +) : TransformationRule { + override fun matches(name: Name, item: MetaItem<*>?): Boolean { + return from.matches(name.toString()) + } + override fun > transformItem(name: Name, item: MetaItem<*>?, target: M) { + val match = from.matchEntire(name.toString()) + if (match != null) { + target.transform(match, item) + } + } + +} + +inline class MetaTransformation(val transformations: Collection) { /** * Produce new meta using only those items that match transformation rules @@ -89,7 +105,36 @@ class MetaTransformation { } } - companion object{ + /** + * Listens for changes in the source node and translates them into second node if transformation set contains a corresponding rule. + */ + fun > bind(source: MutableMeta<*>, target: M) { + source.onChange(target) { name, oldItem, newItem -> + transformations.forEach { t -> + if (t.matches(name, newItem)) { + t.transformItem(name, newItem, target) + } + } + } + } + + companion object { } +} + +class MetaTransformationBuilder { + val transformations = HashSet() + + fun keep(selector: (Name) -> Boolean) { + transformations.add(KeepTransformationRule(selector)) + } + + fun keep(name: Name) { + keep{it == name} + } + + fun keepNode(name: Name){ + keep{it.startsWith(name)} + } } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt index d8513e9c..b0aa8a7d 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt @@ -122,4 +122,14 @@ fun Name.withIndex(index: String): Name { } operator fun Map.get(name: String) = get(name.toName()) -operator fun MutableMap.set(name: String, value: T) = set(name.toName(), value) \ No newline at end of file +operator fun MutableMap.set(name: String, value: T) = set(name.toName(), value) + +/* Name comparison operations */ + +fun Name.startsWith(token: NameToken): Boolean = first() == token + +fun Name.endsWith(token: NameToken): Boolean = last() == token + +fun Name.startsWith(name: Name): Boolean = tokens.subList(0, name.length) == name.tokens + +fun Name.endsWith(name: Name): Boolean = tokens.subList(length - name.length, length) == name.tokens \ No newline at end of file diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameTest.kt index 1302777e..a3116040 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameTest.kt @@ -1,7 +1,6 @@ package hep.dataforge.names -import kotlin.test.Test -import kotlin.test.assertEquals +import kotlin.test.* class NameTest { @Test @@ -16,4 +15,14 @@ class NameTest { val name2 = "token1".toName() + "token2[2].token3" assertEquals(name1, name2) } + + @Test + fun comparisonTest(){ + val name1 = "token1.token2.token3".toName() + val name2 = "token1.token2".toName() + val name3 = "token3".toName() + assertTrue { name1.startsWith(name2) } + assertTrue { name1.endsWith(name3) } + assertFalse { name1.startsWith(name3) } + } } \ No newline at end of file From d08e1ee57da713b9816d01433a4321bb4ea5e588 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 1 Jun 2019 16:04:45 +0300 Subject: [PATCH 15/48] Optimized type projections in meta. Set operations are now non-generic --- .../hep/dataforge/io/BinaryMetaFormat.kt | 3 +- .../kotlin/hep/dataforge/io/JsonMetaFormat.kt | 10 +-- .../kotlin/hep/dataforge/meta/Config.kt | 7 +- .../kotlin/hep/dataforge/meta/Meta.kt | 50 +++++------ .../kotlin/hep/dataforge/meta/MetaBuilder.kt | 5 +- .../hep/dataforge/meta/MetaTransformation.kt | 20 ++++- .../kotlin/hep/dataforge/meta/MutableMeta.kt | 85 ++++++++++--------- .../kotlin/hep/dataforge/meta/Styled.kt | 15 ++-- .../kotlin/hep/dataforge/names/Name.kt | 5 ++ .../kotlin/hep/dataforge/meta/DynamicMeta.kt | 11 +-- .../workspace/SimpleWorkspaceTest.kt | 4 +- 11 files changed, 117 insertions(+), 98 deletions(-) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt index 79145796..843e5acb 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt @@ -90,6 +90,7 @@ object BinaryMetaFormat : MetaFormat { return readText(max = length) } + @Suppress("UNCHECKED_CAST") private fun Input.readMetaItem(): MetaItem { val keyChar = readByte().toChar() return when (keyChar) { @@ -119,6 +120,6 @@ object BinaryMetaFormat : MetaFormat { MetaItem.NodeItem(meta) } else -> error("Unknown serialization key character: $keyChar") - } + } as MetaItem } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index 13cbc717..7815b69c 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -49,8 +49,7 @@ fun Value.toJson(): JsonElement { fun Meta.toJson(): JsonObject { val map = this.items.mapValues { entry -> - val value = entry.value - when (value) { + when (val value = entry.value) { is MetaItem.ValueItem -> value.value.toJson() is MetaItem.NodeItem -> value.node.toJson() } @@ -70,8 +69,9 @@ class JsonMeta(val json: JsonObject) : Meta { } } + @Suppress("UNCHECKED_CAST") private operator fun MutableMap>.set(key: String, value: JsonElement) = when (value) { - is JsonPrimitive -> this[key] = MetaItem.ValueItem(value.toValue()) + is JsonPrimitive -> this[key] = MetaItem.ValueItem(value.toValue()) as MetaItem is JsonObject -> this[key] = MetaItem.NodeItem(value.toMeta()) is JsonArray -> { when { @@ -82,12 +82,12 @@ class JsonMeta(val json: JsonObject) : Meta { (it as JsonPrimitive).toValue() } ) - this[key] = MetaItem.ValueItem(listValue) + this[key] = MetaItem.ValueItem(listValue) as MetaItem } else -> value.forEachIndexed { index, jsonElement -> when (jsonElement) { is JsonObject -> this["$key[$index]"] = MetaItem.NodeItem(JsonMeta(jsonElement)) - is JsonPrimitive -> this["$key[$index]"] = MetaItem.ValueItem(jsonElement.toValue()) + is JsonPrimitive -> this["$key[$index]"] = MetaItem.ValueItem(jsonElement.toValue()) as MetaItem is JsonArray -> TODO("Nested arrays not supported") } } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt index f6370942..3ea2a39e 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt @@ -1,6 +1,5 @@ package hep.dataforge.meta -import hep.dataforge.names.Name import hep.dataforge.names.NameToken import hep.dataforge.names.asName @@ -9,12 +8,12 @@ import hep.dataforge.names.asName /** * Mutable meta representing object state */ -open class Config : AbstractMutableMeta() { +class Config : AbstractMutableMeta() { /** * Attach configuration node instead of creating one */ - override fun wrap(name: Name, meta: Meta): Config = meta.toConfig() + override fun wrapNode(meta: Meta): Config = meta.toConfig() override fun empty(): Config = Config() @@ -29,7 +28,7 @@ fun Meta.toConfig(): Config = this as? Config ?: Config().also { builder -> this.items.mapValues { entry -> val item = entry.value builder[entry.key.asName()] = when (item) { - is MetaItem.ValueItem -> MetaItem.ValueItem(item.value) + is MetaItem.ValueItem -> item.value is MetaItem.NodeItem -> MetaItem.NodeItem(item.node.toConfig()) } } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 329989de..4fb554ca 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -14,8 +14,8 @@ import hep.dataforge.values.boolean * * a [ValueItem] (leaf) * * a [NodeItem] (node) */ -sealed class MetaItem { - data class ValueItem(val value: Value) : MetaItem() +sealed class MetaItem { + data class ValueItem(val value: Value) : MetaItem() data class NodeItem(val node: M) : MetaItem() } @@ -35,7 +35,7 @@ interface MetaRepr { * * Same name siblings are supported via elements with the same [Name] but different queries */ interface Meta : MetaRepr { - val items: Map> + val items: Map> override fun toMeta(): Meta = this @@ -50,12 +50,7 @@ interface Meta : MetaRepr { /* Get operations*/ -/** - * Fast [String]-based accessor for item map - */ -operator fun Map.get(body: String, query: String = ""): T? = get(NameToken(body, query)) - -operator fun Meta?.get(name: Name): MetaItem? { +operator fun Meta?.get(name: Name): MetaItem<*>? { if (this == null) return null return name.first()?.let { token -> val tail = name.cutFirst() @@ -66,13 +61,13 @@ operator fun Meta?.get(name: Name): MetaItem? { } } -operator fun Meta?.get(token: NameToken): MetaItem? = this?.items?.get(token) -operator fun Meta?.get(key: String): MetaItem? = get(key.toName()) +operator fun Meta?.get(token: NameToken): MetaItem<*>? = this?.items?.get(token) +operator fun Meta?.get(key: String): MetaItem<*>? = get(key.toName()) /** * Get all items matching given name. */ -fun Meta.getAll(name: Name): Map> { +fun Meta.getAll(name: Name): Map> { val root = when (name.length) { 0 -> error("Can't use empty name for that") 1 -> this @@ -88,7 +83,7 @@ fun Meta.getAll(name: Name): Map> { ?: emptyMap() } -fun Meta.getAll(name: String): Map> = getAll(name.toName()) +fun Meta.getAll(name: String): Map> = getAll(name.toName()) /** * Get a sequence of [Name]-[Value] pairs @@ -109,8 +104,8 @@ fun Meta.sequence(): Sequence>> { return sequence { items.forEach { (key, item) -> yield(key.asName() to item) - if(item is NodeItem<*>) { - yieldAll(item.node.sequence().map { (innerKey, innerItem)-> + if (item is NodeItem<*>) { + yieldAll(item.node.sequence().map { (innerKey, innerItem) -> (key + innerKey) to innerItem }) } @@ -130,7 +125,7 @@ interface MetaNode> : Meta { /** * Get all items matching given name. */ -fun > MetaNode.getAll(name: Name): Map> { +fun > M.getAll(name: Name): Map> { val root: MetaNode? = when (name.length) { 0 -> error("Can't use empty name for that") 1 -> this @@ -158,7 +153,11 @@ operator fun > MetaNode.get(name: Name): MetaItem? { } } -operator fun > MetaNode?.get(key: String): MetaItem? = this?.let { get(key.toName()) } +operator fun > MetaNode?.get(key: String): MetaItem? = if (this == null) { + null +} else { + this[key.toName()] +} /** * Equals and hash code implementation for meta node @@ -189,13 +188,14 @@ class SealedMeta internal constructor(override val items: Map entry.value.seal() }) +@Suppress("UNCHECKED_CAST") fun MetaItem<*>.seal(): MetaItem = when (this) { - is MetaItem.ValueItem -> MetaItem.ValueItem(value) - is MetaItem.NodeItem -> MetaItem.NodeItem(node.seal()) + is ValueItem -> this as MetaItem + is NodeItem -> NodeItem(node.seal()) } object EmptyMeta : Meta { - override val items: Map> = emptyMap() + override val items: Map> = emptyMap() } /** @@ -203,8 +203,8 @@ object EmptyMeta : Meta { */ val MetaItem<*>?.value - get() = (this as? MetaItem.ValueItem)?.value - ?: (this?.node?.get(VALUE_KEY) as? MetaItem.ValueItem)?.value + get() = (this as? ValueItem)?.value + ?: (this?.node?.get(VALUE_KEY) as? ValueItem)?.value val MetaItem<*>?.string get() = value?.string val MetaItem<*>?.boolean get() = value?.boolean @@ -226,8 +226,8 @@ val MetaItem<*>?.stringList get() = value?.list?.map { it.string } ?: emptyList( val MetaItem?.node: M? get() = when (this) { null -> null - is MetaItem.ValueItem -> error("Trying to interpret value meta item as node item") - is MetaItem.NodeItem -> node + is ValueItem -> error("Trying to interpret value meta item as node item") + is NodeItem -> node } /** @@ -239,4 +239,4 @@ interface Metoid { fun Value.toMeta() = buildMeta { Meta.VALUE_KEY to this } -fun Meta.isEmpty() = this === EmptyMeta || this.items.isEmpty() \ No newline at end of file +fun Meta.isEmpty() = this === EmptyMeta || this.items.isEmpty() diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt index c8911c60..0b5c83d6 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt @@ -1,6 +1,5 @@ package hep.dataforge.meta -import hep.dataforge.names.Name import hep.dataforge.names.asName import hep.dataforge.values.Value @@ -8,7 +7,7 @@ import hep.dataforge.values.Value * DSL builder for meta. Is not intended to store mutable state */ class MetaBuilder : AbstractMutableMeta() { - override fun wrap(name: Name, meta: Meta): MetaBuilder = meta.builder() + override fun wrapNode(meta: Meta): MetaBuilder = meta.builder() override fun empty(): MetaBuilder = MetaBuilder() infix fun String.to(value: Any) { @@ -39,7 +38,7 @@ fun Meta.builder(): MetaBuilder { items.mapValues { entry -> val item = entry.value builder[entry.key.asName()] = when (item) { - is MetaItem.ValueItem -> MetaItem.ValueItem(item.value) + is MetaItem.ValueItem -> item.value is MetaItem.NodeItem -> MetaItem.NodeItem(item.node.builder()) } } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt index ab1a95fa..11ef9cdb 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt @@ -80,6 +80,9 @@ data class RegexpItemTransformationRule( } +/** + * A set of [TransformationRule] to either transform static meta or create dynamically updated [MutableMeta] + */ inline class MetaTransformation(val transformations: Collection) { /** @@ -109,7 +112,7 @@ inline class MetaTransformation(val transformations: Collection> bind(source: MutableMeta<*>, target: M) { - source.onChange(target) { name, oldItem, newItem -> + source.onChange(target) { name, _, newItem -> transformations.forEach { t -> if (t.matches(name, newItem)) { t.transformItem(name, newItem, target) @@ -123,18 +126,29 @@ inline class MetaTransformation(val transformations: Collection() + /** + * Keep all items with name satisfying the criteria + */ fun keep(selector: (Name) -> Boolean) { transformations.add(KeepTransformationRule(selector)) } + /** + * Keep specific item (including its descendants) + */ fun keep(name: Name) { keep{it == name} } - fun keepNode(name: Name){ - keep{it.startsWith(name)} + fun move(from: Name, to: Name){ + transformations.add( + SingleItemTransformationRule(from){ _, item -> setItem(to, item)} + ) } } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt index 83edc56b..aebb18d6 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt @@ -10,15 +10,8 @@ internal data class MetaListener( interface MutableMeta> : MetaNode { - /** - * Transform given meta to node type of this meta tree - * @param name the name of the node where meta should be attached. Needed for correct assignment validators and styles - * @param meta the node itself - */ - fun wrap(name: Name, meta: Meta): M - override val items: Map> - operator fun set(name: Name, item: MetaItem?) + operator fun set(name: Name, item: MetaItem<*>?) fun onChange(owner: Any? = null, action: (Name, MetaItem<*>?, MetaItem<*>?) -> Unit) fun removeListener(owner: Any? = null) } @@ -69,59 +62,70 @@ abstract class AbstractMutableMeta> : AbstractMetaNode(), itemChanged(key.asName(), oldItem, newItem) } + @Suppress("UNCHECKED_CAST") + protected fun wrapItem(item: MetaItem<*>?): MetaItem? = when (item) { + null -> null + is MetaItem.ValueItem -> item as MetaItem + is MetaItem.NodeItem<*> -> MetaItem.NodeItem(wrapNode(item.node)) + } + + /** + * Transform given meta to node type of this meta tree + */ + protected abstract fun wrapNode(meta: Meta): M + /** * Create empty node */ internal abstract fun empty(): M - override operator fun set(name: Name, item: MetaItem?) { + override operator fun set(name: Name, item: MetaItem<*>?) { when (name.length) { 0 -> error("Can't setValue meta item for empty name") 1 -> { val token = name.first()!! - replaceItem(token, get(name), item) + replaceItem(token, get(name), wrapItem(item)) } else -> { val token = name.first()!! //get existing or create new node. Query is ignored for new node - val child = this.items[token]?.node - ?: empty().also { this[token.body.toName()] = MetaItem.NodeItem(it) } - child[name.cutFirst()] = item + if(items[token] == null){ + replaceItem(token,null, MetaItem.NodeItem(empty())) + } + items[token]?.node!![name.cutFirst()] = item } } } } -fun > MutableMeta.remove(name: Name) = set(name, null) -fun > MutableMeta.remove(name: String) = remove(name.toName()) +fun MutableMeta<*>.remove(name: Name) = set(name, null) +fun MutableMeta<*>.remove(name: String) = remove(name.toName()) -fun > MutableMeta.setValue(name: Name, value: Value) = +fun MutableMeta<*>.setValue(name: Name, value: Value) = set(name, MetaItem.ValueItem(value)) -fun > MutableMeta.setValue(name: String, value: Value) = +fun MutableMeta<*>.setValue(name: String, value: Value) = set(name.toName(), MetaItem.ValueItem(value)) -//fun > MutableMeta.setItem(token: NameToken, item: MetaItem?) = set(token.asName(), item) -//fun > MutableMeta.setItem(name: String, item: MetaItem) = set(name.toName(), item) - -fun > MutableMeta.setItem(name: Name, item: MetaItem<*>) { +fun MutableMeta<*>.setItem(name: Name, item: MetaItem<*>?) { when (item) { - is MetaItem.ValueItem<*> -> setValue(name, item.value) + null -> remove(name) + is MetaItem.ValueItem -> setValue(name, item.value) is MetaItem.NodeItem<*> -> setNode(name, item.node) } } -fun > MutableMeta.setItem(name: String, item: MetaItem<*>) = setItem(name.toName(), item) +fun MutableMeta<*>.setItem(name: String, item: MetaItem<*>?) = setItem(name.toName(), item) -fun > MutableMeta.setNode(name: Name, node: Meta) = - set(name, MetaItem.NodeItem(wrap(name, node))) +fun MutableMeta<*>.setNode(name: Name, node: Meta) = + set(name, MetaItem.NodeItem(node)) -fun > MutableMeta.setNode(name: String, node: Meta) = setNode(name.toName(), node) +fun MutableMeta<*>.setNode(name: String, node: Meta) = setNode(name.toName(), node) /** * Universal set method */ -operator fun > MutableMeta.set(name: Name, value: Any?) { +operator fun MutableMeta<*>.set(name: Name, value: Any?) { when (value) { null -> remove(name) is MetaItem<*> -> setItem(name, value) @@ -131,9 +135,9 @@ operator fun > MutableMeta.set(name: Name, value: Any?) { } } -operator fun > M.set(name: NameToken, value: Any?) = set(name.asName(), value) +operator fun MutableMeta<*>.set(name: NameToken, value: Any?) = set(name.asName(), value) -operator fun > M.set(key: String, value: Any?) = set(key.toName(), value) +operator fun MutableMeta<*>.set(key: String, value: Any?) = set(key.toName(), value) /** * Update existing mutable node with another node. The rules are following: @@ -143,8 +147,7 @@ operator fun > M.set(key: String, value: Any?) = set(key.toNa */ fun > M.update(meta: Meta) { meta.items.forEach { entry -> - val value = entry.value - when (value) { + when (val value = entry.value) { is MetaItem.ValueItem -> setValue(entry.key.asName(), value.value) is MetaItem.NodeItem -> (this[entry.key.asName()] as? MetaItem.NodeItem)?.node?.update(value.node) ?: run { setNode(entry.key.asName(), value.node) } @@ -154,10 +157,10 @@ fun > M.update(meta: Meta) { /* Same name siblings generation */ -fun > M.setIndexedItems( +fun MutableMeta<*>.setIndexedItems( name: Name, - items: Iterable>, - indexFactory: MetaItem.(index: Int) -> String = { it.toString() } + items: Iterable>, + indexFactory: MetaItem<*>.(index: Int) -> String = { it.toString() } ) { val tokens = name.tokens.toMutableList() val last = tokens.last() @@ -168,21 +171,21 @@ fun > M.setIndexedItems( } } -fun > M.setIndexed( +fun MutableMeta<*>.setIndexed( name: Name, metas: Iterable, - indexFactory: MetaItem.(index: Int) -> String = { it.toString() } + indexFactory: MetaItem<*>.(index: Int) -> String = { it.toString() } ) { - setIndexedItems(name, metas.map { MetaItem.NodeItem(wrap(name, it)) }, indexFactory) + setIndexedItems(name, metas.map { MetaItem.NodeItem(it) }, indexFactory) } -operator fun > M.set(name: Name, metas: Iterable) = setIndexed(name, metas) -operator fun > M.set(name: String, metas: Iterable) = setIndexed(name.toName(), metas) +operator fun MutableMeta<*>.set(name: Name, metas: Iterable): Unit = setIndexed(name, metas) +operator fun MutableMeta<*>.set(name: String, metas: Iterable): Unit = setIndexed(name.toName(), metas) /** * Append the node with a same-name-sibling, automatically generating numerical index */ -fun > M.append(name: Name, value: Any?) { +fun MutableMeta<*>.append(name: Name, value: Any?) { require(!name.isEmpty()) { "Name could not be empty for append operation" } val newIndex = name.last()!!.index if (newIndex.isNotEmpty()) { @@ -193,4 +196,4 @@ fun > M.append(name: Name, value: Any?) { } } -fun > M.append(name: String, value: Any?) = append(name.toName(), value) \ No newline at end of file +fun MutableMeta<*>.append(name: String, value: Any?) = append(name.toName(), value) \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt index 136c5617..39b47227 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt @@ -12,14 +12,11 @@ import kotlin.reflect.KProperty * @param style - the style */ class Styled(val base: Meta, val style: Config = Config().empty()) : AbstractMutableMeta() { - override fun wrap(name: Name, meta: Meta): Styled { - TODO("not implemented") //To change body of created functions use File | Settings | File Templates. - } + override fun wrapNode(meta: Meta): Styled = Styled(meta) - override fun empty(): Styled { - TODO("not implemented") //To change body of created functions use File | Settings | File Templates. - } + override fun empty(): Styled = Styled(EmptyMeta) + @Suppress("UNCHECKED_CAST") override val items: Map> get() = (base.items.keys + style.items.keys).associate { key -> val value = base.items[key] @@ -27,10 +24,10 @@ class Styled(val base: Meta, val style: Config = Config().empty()) : AbstractMut val item: MetaItem = when (value) { null -> when (styleValue) { null -> error("Should be unreachable") - is MetaItem.ValueItem -> MetaItem.ValueItem(styleValue.value) is MetaItem.NodeItem -> MetaItem.NodeItem(Styled(style.empty(), styleValue.node)) + else -> styleValue.value as MetaItem } - is MetaItem.ValueItem -> MetaItem.ValueItem(value.value) + is MetaItem.ValueItem -> value as MetaItem is MetaItem.NodeItem -> MetaItem.NodeItem( Styled(value.node, styleValue?.node ?: Config.empty()) ) @@ -38,7 +35,7 @@ class Styled(val base: Meta, val style: Config = Config().empty()) : AbstractMut key to item } - override fun set(name: Name, item: MetaItem?) { + override fun set(name: Name, item: MetaItem<*>?) { if (item == null) { style.remove(name) } else { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt index b0aa8a7d..504c07d2 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt @@ -121,6 +121,11 @@ fun Name.withIndex(index: String): Name { return Name(tokens) } +/** + * Fast [String]-based accessor for item map + */ +operator fun Map.get(body: String, query: String = ""): T? = get(NameToken(body, query)) + operator fun Map.get(name: String) = get(name.toName()) operator fun MutableMap.set(name: String, value: T) = set(name.toName(), value) diff --git a/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt b/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt index 4719831b..12b8db9e 100644 --- a/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt +++ b/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt @@ -35,12 +35,13 @@ class DynamicMeta(val obj: dynamic) : Meta { private fun isArray(@Suppress("UNUSED_PARAMETER") obj: dynamic): Boolean = js("Array.isArray(obj)") as Boolean + @Suppress("UNCHECKED_CAST") private fun asItem(obj: dynamic): MetaItem? { - if (obj == null) return MetaItem.ValueItem(Null) - return when (jsTypeOf(obj)) { - "boolean" -> MetaItem.ValueItem(Value.of(obj as Boolean)) - "number" -> MetaItem.ValueItem(Value.of(obj as Number)) - "string" -> MetaItem.ValueItem(Value.of(obj as String)) + if (obj == null) return MetaItem.ValueItem(Null) as MetaItem + return when (jsTypeOf(obj as? Any)) { + "boolean" -> MetaItem.ValueItem(Value.of(obj as Boolean)) as MetaItem + "number" -> MetaItem.ValueItem(Value.of(obj as Number)) as MetaItem + "string" -> MetaItem.ValueItem(Value.of(obj as String)) as MetaItem "object" -> MetaItem.NodeItem(DynamicMeta(obj)) else -> null } diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index 962dab53..d47f972f 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -61,13 +61,13 @@ class SimpleWorkspaceTest { allData() } joinByGroup { context -> - group("even", filter = { name, data -> name.toString().toInt() % 2 == 0 }) { + group("even", filter = { name, _ -> name.toString().toInt() % 2 == 0 }) { result { data -> context.logger.info { "Starting even" } data.values.average() } } - group("odd", filter = { name, data -> name.toString().toInt() % 2 == 1 }) { + group("odd", filter = { name, _ -> name.toString().toInt() % 2 == 1 }) { result { data -> context.logger.info { "Starting odd" } data.values.average() From 846ad4582f903241631b0575ac1d3627c4206b2d Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 1 Jun 2019 20:21:19 +0300 Subject: [PATCH 16/48] MetaTransformation update --- .../hep/dataforge/meta/MetaTransformation.kt | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt index 11ef9cdb..3d0e50e4 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt @@ -1,7 +1,6 @@ package hep.dataforge.meta import hep.dataforge.names.Name -import hep.dataforge.names.startsWith /** * A transformation for meta item or a group of items @@ -63,9 +62,9 @@ data class SingleItemTransformationRule( } } -data class RegexpItemTransformationRule( +data class RegexItemTransformationRule( val from: Regex, - val transform: MutableMeta<*>.(MatchResult, MetaItem<*>?) -> Unit + val transform: MutableMeta<*>.(name: Name, MatchResult, MetaItem<*>?) -> Unit ) : TransformationRule { override fun matches(name: Name, item: MetaItem<*>?): Boolean { return from.matches(name.toString()) @@ -74,7 +73,7 @@ data class RegexpItemTransformationRule( override fun > transformItem(name: Name, item: MetaItem<*>?, target: M) { val match = from.matchEntire(name.toString()) if (match != null) { - target.transform(match, item) + target.transform(name, match, item) } } @@ -122,7 +121,8 @@ inline class MetaTransformation(val transformations: Collection Unit): MetaTransformation = + MetaTransformationBuilder().apply(block).build() } } @@ -143,12 +143,28 @@ class MetaTransformationBuilder { * Keep specific item (including its descendants) */ fun keep(name: Name) { - keep{it == name} + keep { it == name } } - fun move(from: Name, to: Name){ + /** + * Keep nodes by regex + */ + fun keep(regex: String) { + transformations.add(RegexItemTransformationRule(regex.toRegex()) { name, _, metaItem -> + setItem(name, metaItem) + }) + } + + /** + * Move an item from [from] to [to], optionally applying [operation] it defined + */ + fun move(from: Name, to: Name, operation: (MetaItem<*>?) -> Any? = { it }) { transformations.add( - SingleItemTransformationRule(from){ _, item -> setItem(to, item)} + SingleItemTransformationRule(from) { _, item -> + set(to, operation(item)) + } ) } + + fun build() = MetaTransformation(transformations) } \ No newline at end of file From 4e6cf3e7859702f950dfd930f6af5b0b6e655250 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 1 Jun 2019 20:30:47 +0300 Subject: [PATCH 17/48] bump version to 0.1.3-dev-2 --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 35e241e4..9d89d17b 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,4 +1,4 @@ -val dataforgeVersion by extra("0.1.3-dev-1") +val dataforgeVersion by extra("0.1.3-dev-2") allprojects { repositories { From 07ff982d0881e000606d26b7a75d55c0b2abf348 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 9 Jun 2019 18:48:26 +0300 Subject: [PATCH 18/48] Added common ancestor to descriptors --- .../hep/dataforge/descriptors/ItemDescriptor.kt | 9 +++++++++ .../hep/dataforge/descriptors/NodeDescriptor.kt | 14 ++++++++------ .../hep/dataforge/descriptors/ValueDescriptor.kt | 12 ++++++------ .../kotlin/hep/dataforge/meta/Laminate.kt | 2 +- .../commonMain/kotlin/hep/dataforge/meta/Meta.kt | 9 ++++++++- 5 files changed, 32 insertions(+), 14 deletions(-) create mode 100644 dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt new file mode 100644 index 00000000..1616fff4 --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt @@ -0,0 +1,9 @@ +package hep.dataforge.descriptors + +interface ItemDescriptor { + val name: String + val multiple: Boolean + val required: Boolean + val info: String? + val tags: List +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt index a9f73af5..0c001889 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt @@ -31,14 +31,14 @@ import hep.dataforge.names.toName * * @author Alexander Nozik */ -class NodeDescriptor(override val config: Config) : Specific { +class NodeDescriptor(override val config: Config) : ItemDescriptor, Specific { /** * The name of this node * * @return */ - var name: String by string { error("Anonymous descriptors are not allowed") } + override var name: String by string { error("Anonymous descriptors are not allowed") } /** @@ -54,28 +54,28 @@ class NodeDescriptor(override val config: Config) : Specific { * * @return */ - var multiple: Boolean by boolean(false) + override var multiple: Boolean by boolean(false) /** * True if the node is required * * @return */ - var required: Boolean by boolean { default == null } + override var required: Boolean by boolean { default == null } /** * The node description * * @return */ - var info: String? by string() + override var info: String? by string() /** * A list of tags for this node. Tags used to customize node usage * * @return */ - var tags: List by value{ value -> + override var tags: List by value { value -> value?.list?.map { it.string } ?: emptyList() } @@ -117,6 +117,8 @@ class NodeDescriptor(override val config: Config) : Specific { node(name, build { this.name = name }.apply(block)) } + val items: Map get() = nodes + values + //override val descriptor: NodeDescriptor = empty("descriptor") diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ValueDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ValueDescriptor.kt index cc3352c0..cfaaf5f9 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ValueDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ValueDescriptor.kt @@ -29,7 +29,7 @@ import hep.dataforge.values.ValueType * * @author Alexander Nozik */ -class ValueDescriptor(override val config: Config) : Specific { +class ValueDescriptor(override val config: Config) : ItemDescriptor, Specific { /** * The default for this value. Null if there is no default. @@ -47,28 +47,28 @@ class ValueDescriptor(override val config: Config) : Specific { * * @return */ - var multiple: Boolean by boolean(false) + override var multiple: Boolean by boolean(false) /** * True if the value is required * * @return */ - var required: Boolean by boolean { default == null } + override var required: Boolean by boolean { default == null } /** * Value name * * @return */ - var name: String by string { error("Anonymous descriptors are not allowed") } + override var name: String by string { error("Anonymous descriptors are not allowed") } /** * The value info * * @return */ - var info: String? by string() + override var info: String? by string() /** * A list of allowed ValueTypes. Empty if any value type allowed @@ -83,7 +83,7 @@ class ValueDescriptor(override val config: Config) : Specific { this.type = listOf(*t) } - var tags: List by value { value -> + override var tags: List by value { value -> value?.list?.map { it.string } ?: emptyList() } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt index b16248ac..28240908 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt @@ -19,7 +19,7 @@ class Laminate(layers: List) : Meta { constructor(vararg layers: Meta) : this(layers.asList()) - override val items: Map> + override val items: Map> get() = layers.map { it.items.keys }.flatten().associateWith { key -> layers.asSequence().map { it.items[key] }.filterNotNull().let(replaceRule) } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 4fb554ca..81b31175 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -143,7 +143,8 @@ fun > M.getAll(name: Name): Map> { fun > M.getAll(name: String): Map> = getAll(name.toName()) -operator fun > MetaNode.get(name: Name): MetaItem? { +operator fun > MetaNode?.get(name: Name): MetaItem? { + if (this == null) return null return name.first()?.let { token -> val tail = name.cutFirst() when (tail.length) { @@ -159,6 +160,12 @@ operator fun > MetaNode?.get(key: String): MetaItem? = if this[key.toName()] } +operator fun > MetaNode?.get(key: NameToken): MetaItem? = if (this == null) { + null +} else { + this[key.asName()] +} + /** * Equals and hash code implementation for meta node */ From 4bb95ca79336d05fc13ba23602d578500d3dea8f Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 10 Jun 2019 20:07:40 +0300 Subject: [PATCH 19/48] Joined descriptors --- build.gradle.kts | 2 +- .../dataforge/descriptors/ItemDescriptor.kt | 280 +++++++++++++++++- .../dataforge/descriptors/NodeDescriptor.kt | 134 --------- .../dataforge/descriptors/ValueDescriptor.kt | 193 ------------ 4 files changed, 274 insertions(+), 335 deletions(-) delete mode 100644 dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt delete mode 100644 dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ValueDescriptor.kt diff --git a/build.gradle.kts b/build.gradle.kts index 9d89d17b..b24c5a70 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,4 +1,4 @@ -val dataforgeVersion by extra("0.1.3-dev-2") +val dataforgeVersion by extra("0.1.3-dev-3") allprojects { repositories { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt index 1616fff4..09e4d11e 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ItemDescriptor.kt @@ -1,9 +1,275 @@ package hep.dataforge.descriptors -interface ItemDescriptor { - val name: String - val multiple: Boolean - val required: Boolean - val info: String? - val tags: List -} \ No newline at end of file +import hep.dataforge.meta.* +import hep.dataforge.names.NameToken +import hep.dataforge.names.toName +import hep.dataforge.values.False +import hep.dataforge.values.True +import hep.dataforge.values.Value +import hep.dataforge.values.ValueType + +sealed class ItemDescriptor(override val config: Config) : Specific { + + /** + * The name of this item + * + * @return + */ + var name: String by string { error("Anonymous descriptors are not allowed") } + + /** + * True if same name siblings with this name are allowed + * + * @return + */ + var multiple: Boolean by boolean(false) + + /** + * The item description + * + * @return + */ + var info: String? by string() + + /** + * A list of tags for this item. Tags used to customize item usage + * + * @return + */ + var tags: List by value { value -> + value?.list?.map { it.string } ?: emptyList() + } + + /** + * True if the item is required + * + * @return + */ + abstract var required: Boolean +} + +/** + * Descriptor for meta node. Could contain additional information for viewing + * and editing. + * + * @author Alexander Nozik + */ +class NodeDescriptor(config: Config) : ItemDescriptor(config){ + + /** + * True if the node is required + * + * @return + */ + override var required: Boolean by boolean { default == null } + + /** + * The default for this node. Null if there is no default. + * + * @return + */ + var default: Meta? by node() + + /** + * The list of value descriptors + */ + val values: Map + get() = config.getAll(VALUE_KEY.toName()).entries.associate { (name, node) -> + name to ValueDescriptor.wrap(node.node ?: error("Value descriptor must be a node")) + } + + fun value(name: String, descriptor: ValueDescriptor) { + if(items.keys.contains(name)) error("The key $name already exists in descriptor") + val token = NameToken(VALUE_KEY, name) + config[token] = descriptor.config + } + + /** + * Add a value descriptor using block for + */ + fun value(name: String, block: ValueDescriptor.() -> Unit) { + value(name, ValueDescriptor.build { this.name = name }.apply(block)) + } + + /** + * The map of children node descriptors + */ + val nodes: Map + get() = config.getAll(NODE_KEY.toName()).entries.associate { (name, node) -> + name to wrap(node.node ?: error("Node descriptor must be a node")) + } + + + fun node(name: String, descriptor: NodeDescriptor) { + if(items.keys.contains(name)) error("The key $name already exists in descriptor") + val token = NameToken(NODE_KEY, name) + config[token] = descriptor.config + } + + fun node(name: String, block: NodeDescriptor.() -> Unit) { + node(name, build { this.name = name }.apply(block)) + } + + val items: Map get() = nodes + values + + + //override val descriptor: NodeDescriptor = empty("descriptor") + + companion object : Specification { + +// const val ITEM_KEY = "item" + const val NODE_KEY = "node" + const val VALUE_KEY = "value" + + override fun wrap(config: Config): NodeDescriptor = NodeDescriptor(config) + + //TODO infer descriptor from spec + } +} + + +/** + * A descriptor for meta value + * + * Descriptor can have non-atomic path. It is resolved when descriptor is added to the node + * + * @author Alexander Nozik + */ +class ValueDescriptor(config: Config) : ItemDescriptor(config){ + + + /** + * True if the value is required + * + * @return + */ + override var required: Boolean by boolean { default == null } + + /** + * The default for this value. Null if there is no default. + * + * @return + */ + var default: Value? by value() + + fun default(v: Any) { + this.default = Value.of(v) + } + + /** + * A list of allowed ValueTypes. Empty if any value type allowed + * + * @return + */ + var type: List by value { + it?.list?.map { v -> ValueType.valueOf(v.string) } ?: emptyList() + } + + fun type(vararg t: ValueType) { + this.type = listOf(*t) + } + + /** + * Check if given value is allowed for here. The type should be allowed and + * if it is value should be within allowed values + * + * @param value + * @return + */ + fun isAllowedValue(value: Value): Boolean { + return (type.isEmpty() || type.contains(ValueType.STRING) || type.contains(value.type)) && (allowedValues.isEmpty() || allowedValues.contains( + value + )) + } + + /** + * A list of allowed values with descriptions. If empty than any value is + * allowed. + * + * @return + */ + var allowedValues: List by value { + it?.list ?: if (type.size == 1 && type[0] === ValueType.BOOLEAN) { + listOf(True, False) + } else { + emptyList() + } + } + + /** + * Allow given list of value and forbid others + */ + fun allow(vararg v: Any) { + this.allowedValues = v.map { Value.of(it) } + } + + companion object : Specification { + + override fun wrap(config: Config): ValueDescriptor = ValueDescriptor(config) + + inline fun > enum(name: String) = + build { + this.name = name + type(ValueType.STRING) + this.allowedValues = enumValues().map { Value.of(it.name) } + } + +// /** +// * Build a value descriptor from annotation +// */ +// fun build(def: ValueDef): ValueDescriptor { +// val builder = MetaBuilder("value") +// .setValue("name", def.key) +// +// if (def.type.isNotEmpty()) { +// builder.setValue("type", def.type) +// } +// +// if (def.multiple) { +// builder.setValue("multiple", def.multiple) +// } +// +// if (!def.info.isEmpty()) { +// builder.setValue("info", def.info) +// } +// +// if (def.allowed.isNotEmpty()) { +// builder.setValue("allowedValues", def.allowed) +// } else if (def.enumeration != Any::class) { +// if (def.enumeration.java.isEnum) { +// val values = def.enumeration.java.enumConstants +// builder.setValue("allowedValues", values.map { it.toString() }) +// } else { +// throw RuntimeException("Only enumeration classes are allowed in 'enumeration' annotation property") +// } +// } +// +// if (def.def.isNotEmpty()) { +// builder.setValue("default", def.def) +// } else if (!def.required) { +// builder.setValue("required", def.required) +// } +// +// if (def.tags.isNotEmpty()) { +// builder.setValue("tags", def.tags) +// } +// return ValueDescriptor(builder) +// } +// +// /** +// * Build empty value descriptor +// */ +// fun empty(valueName: String): ValueDescriptor { +// val builder = MetaBuilder("value") +// .setValue("name", valueName) +// return ValueDescriptor(builder) +// } +// +// /** +// * Merge two separate value descriptors +// */ +// fun merge(primary: ValueDescriptor, secondary: ValueDescriptor): ValueDescriptor { +// return ValueDescriptor(Laminate(primary.meta, secondary.meta)) +// } + } +} diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt deleted file mode 100644 index 0c001889..00000000 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/NodeDescriptor.kt +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright 2018 Alexander Nozik. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package hep.dataforge.descriptors - -import hep.dataforge.meta.* -import hep.dataforge.names.NameToken -import hep.dataforge.names.toName - -/** - * Descriptor for meta node. Could contain additional information for viewing - * and editing. - * - * @author Alexander Nozik - */ -class NodeDescriptor(override val config: Config) : ItemDescriptor, Specific { - - /** - * The name of this node - * - * @return - */ - override var name: String by string { error("Anonymous descriptors are not allowed") } - - - /** - * The default for this node. Null if there is no default. - * - * @return - */ - var default: Meta? by node() - - /** - * True if multiple children with this nodes name are allowed. Anonymous - * nodes are always single - * - * @return - */ - override var multiple: Boolean by boolean(false) - - /** - * True if the node is required - * - * @return - */ - override var required: Boolean by boolean { default == null } - - /** - * The node description - * - * @return - */ - override var info: String? by string() - - /** - * A list of tags for this node. Tags used to customize node usage - * - * @return - */ - override var tags: List by value { value -> - value?.list?.map { it.string } ?: emptyList() - } - - /** - * The list of value descriptors - */ - val values: Map - get() = config.getAll(VALUE_KEY.toName()).entries.associate { (name, node) -> - name to ValueDescriptor.wrap(node.node ?: error("Value descriptor must be a node")) - } - - fun value(name: String, descriptor: ValueDescriptor) { - val token = NameToken(VALUE_KEY, name) - config[token] = descriptor.config - } - - /** - * Add a value descriptor using block for - */ - fun value(name: String, block: ValueDescriptor.() -> Unit) { - value(name, ValueDescriptor.build { this.name = name }.apply(block)) - } - - /** - * The map of children node descriptors - */ - val nodes: Map - get() = config.getAll(NODE_KEY.toName()).entries.associate { (name, node) -> - name to wrap(node.node ?: error("Node descriptor must be a node")) - } - - - fun node(name: String, descriptor: NodeDescriptor) { - val token = NameToken(NODE_KEY, name) - config[token] = descriptor.config - } - - fun node(name: String, block: NodeDescriptor.() -> Unit) { - node(name, build { this.name = name }.apply(block)) - } - - val items: Map get() = nodes + values - - - //override val descriptor: NodeDescriptor = empty("descriptor") - - companion object : Specification { - - const val NODE_KEY = "node" - const val VALUE_KEY = "value" - - override fun wrap(config: Config): NodeDescriptor = NodeDescriptor(config) - - //TODO infer descriptor from spec - } -} diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ValueDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ValueDescriptor.kt deleted file mode 100644 index cfaaf5f9..00000000 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/descriptors/ValueDescriptor.kt +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Copyright 2018 Alexander Nozik. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package hep.dataforge.descriptors - -import hep.dataforge.meta.* -import hep.dataforge.values.False -import hep.dataforge.values.True -import hep.dataforge.values.Value -import hep.dataforge.values.ValueType - -/** - * A descriptor for meta value - * - * Descriptor can have non-atomic path. It is resolved when descriptor is added to the node - * - * @author Alexander Nozik - */ -class ValueDescriptor(override val config: Config) : ItemDescriptor, Specific { - - /** - * The default for this value. Null if there is no default. - * - * @return - */ - var default: Value? by value() - - fun default(v: Any) { - this.default = Value.of(v) - } - - /** - * True if multiple values with this name are allowed. - * - * @return - */ - override var multiple: Boolean by boolean(false) - - /** - * True if the value is required - * - * @return - */ - override var required: Boolean by boolean { default == null } - - /** - * Value name - * - * @return - */ - override var name: String by string { error("Anonymous descriptors are not allowed") } - - /** - * The value info - * - * @return - */ - override var info: String? by string() - - /** - * A list of allowed ValueTypes. Empty if any value type allowed - * - * @return - */ - var type: List by value { - it?.list?.map { v -> ValueType.valueOf(v.string) } ?: emptyList() - } - - fun type(vararg t: ValueType) { - this.type = listOf(*t) - } - - override var tags: List by value { value -> - value?.list?.map { it.string } ?: emptyList() - } - - /** - * Check if given value is allowed for here. The type should be allowed and - * if it is value should be within allowed values - * - * @param value - * @return - */ - fun isAllowedValue(value: Value): Boolean { - return (type.isEmpty() || type.contains(ValueType.STRING) || type.contains(value.type)) && (allowedValues.isEmpty() || allowedValues.contains( - value - )) - } - - /** - * A list of allowed values with descriptions. If empty than any value is - * allowed. - * - * @return - */ - var allowedValues: List by value { - it?.list ?: if (type.size == 1 && type[0] === ValueType.BOOLEAN) { - listOf(True, False) - } else { - emptyList() - } - } - - /** - * Allow given list of value and forbid others - */ - fun allow(vararg v: Any) { - this.allowedValues = v.map { Value.of(it) } - } - - companion object : Specification { - - override fun wrap(config: Config): ValueDescriptor = ValueDescriptor(config) - - inline fun > enum(name: String) = - build { - this.name = name - type(ValueType.STRING) - this.allowedValues = enumValues().map { Value.of(it.name) } - } - -// /** -// * Build a value descriptor from annotation -// */ -// fun build(def: ValueDef): ValueDescriptor { -// val builder = MetaBuilder("value") -// .setValue("name", def.key) -// -// if (def.type.isNotEmpty()) { -// builder.setValue("type", def.type) -// } -// -// if (def.multiple) { -// builder.setValue("multiple", def.multiple) -// } -// -// if (!def.info.isEmpty()) { -// builder.setValue("info", def.info) -// } -// -// if (def.allowed.isNotEmpty()) { -// builder.setValue("allowedValues", def.allowed) -// } else if (def.enumeration != Any::class) { -// if (def.enumeration.java.isEnum) { -// val values = def.enumeration.java.enumConstants -// builder.setValue("allowedValues", values.map { it.toString() }) -// } else { -// throw RuntimeException("Only enumeration classes are allowed in 'enumeration' annotation property") -// } -// } -// -// if (def.def.isNotEmpty()) { -// builder.setValue("default", def.def) -// } else if (!def.required) { -// builder.setValue("required", def.required) -// } -// -// if (def.tags.isNotEmpty()) { -// builder.setValue("tags", def.tags) -// } -// return ValueDescriptor(builder) -// } -// -// /** -// * Build empty value descriptor -// */ -// fun empty(valueName: String): ValueDescriptor { -// val builder = MetaBuilder("value") -// .setValue("name", valueName) -// return ValueDescriptor(builder) -// } -// -// /** -// * Merge two separate value descriptors -// */ -// fun merge(primary: ValueDescriptor, secondary: ValueDescriptor): ValueDescriptor { -// return ValueDescriptor(Laminate(primary.meta, secondary.meta)) -// } - } -} From c6c4509d6c1b7fb6a5705cfd17dc90cd70d91554 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 12 Jun 2019 19:45:57 +0300 Subject: [PATCH 20/48] Fixed bug with meta value removal --- build.gradle.kts | 2 +- .../kotlin/hep/dataforge/meta/MutableMeta.kt | 15 ++++++++----- .../kotlin/hep/dataforge/values/Value.kt | 4 ++-- .../hep/dataforge/meta/MutableMetaTest.kt | 22 +++++++++++++++++++ 4 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MutableMetaTest.kt diff --git a/build.gradle.kts b/build.gradle.kts index b24c5a70..2dcf5b24 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,4 +1,4 @@ -val dataforgeVersion by extra("0.1.3-dev-3") +val dataforgeVersion by extra("0.1.3-dev-4") allprojects { repositories { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt index aebb18d6..2be5c1a0 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt @@ -50,7 +50,9 @@ abstract class AbstractMutableMeta> : AbstractMetaNode(), protected open fun replaceItem(key: NameToken, oldItem: MetaItem?, newItem: MetaItem?) { if (newItem == null) { _items.remove(key) - oldItem?.node?.removeListener(this) + if(oldItem!= null && oldItem is MetaItem.NodeItem) { + oldItem.node.removeListener(this) + } } else { _items[key] = newItem if (newItem is MetaItem.NodeItem) { @@ -65,8 +67,8 @@ abstract class AbstractMutableMeta> : AbstractMetaNode(), @Suppress("UNCHECKED_CAST") protected fun wrapItem(item: MetaItem<*>?): MetaItem? = when (item) { null -> null - is MetaItem.ValueItem -> item as MetaItem - is MetaItem.NodeItem<*> -> MetaItem.NodeItem(wrapNode(item.node)) + is MetaItem.ValueItem -> item + is MetaItem.NodeItem -> MetaItem.NodeItem(wrapNode(item.node)) } /** @@ -98,8 +100,11 @@ abstract class AbstractMutableMeta> : AbstractMetaNode(), } } -fun MutableMeta<*>.remove(name: Name) = set(name, null) -fun MutableMeta<*>.remove(name: String) = remove(name.toName()) + +@Suppress("NOTHING_TO_INLINE") +inline fun MutableMeta<*>.remove(name: Name) = set(name, null) +@Suppress("NOTHING_TO_INLINE") +inline fun MutableMeta<*>.remove(name: String) = remove(name.toName()) fun MutableMeta<*>.setValue(name: Name, value: Value) = set(name, MetaItem.ValueItem(value)) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt index 20f86779..5753512b 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt @@ -97,7 +97,7 @@ object True : Value { override val value: Any? get() = true override val type: ValueType get() = ValueType.BOOLEAN override val number: Number get() = 1.0 - override val string: String get() = "+" + override val string: String get() = "true" override fun toString(): String = value.toString() } @@ -109,7 +109,7 @@ object False : Value { override val value: Any? get() = false override val type: ValueType get() = ValueType.BOOLEAN override val number: Number get() = -1.0 - override val string: String get() = "-" + override val string: String get() = "false" } val Value.boolean get() = this == True || this.list.firstOrNull() == True || (type == ValueType.STRING && string.toBoolean()) diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MutableMetaTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MutableMetaTest.kt new file mode 100644 index 00000000..5ab75fd4 --- /dev/null +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MutableMetaTest.kt @@ -0,0 +1,22 @@ +package hep.dataforge.meta + +import kotlin.test.Test +import kotlin.test.assertEquals + +class MutableMetaTest{ + @Test + fun testRemove(){ + val meta = buildMeta { + "aNode" to { + "innerNode" to { + "innerValue" to true + } + "b" to 22 + "c" to "StringValue" + } + }.toConfig() + + meta.remove("aNode.c") + assertEquals(meta["aNode.c"], null) + } +} \ No newline at end of file From 4ae1f71a05e881e5c0282fdf02c4c9b020f23574 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 12 Jun 2019 21:12:54 +0300 Subject: [PATCH 21/48] Fixed value delegate nullability bug. --- build.gradle.kts | 2 +- .../hep/dataforge/meta/ConfigDelegates.kt | 2 +- .../dataforge/descriptors/DescriptorTest.kt | 31 +++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 dataforge-meta/src/commonTest/kotlin/hep/dataforge/descriptors/DescriptorTest.kt diff --git a/build.gradle.kts b/build.gradle.kts index 2dcf5b24..c05ec51c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,4 +1,4 @@ -val dataforgeVersion by extra("0.1.3-dev-4") +val dataforgeVersion by extra("0.1.3-dev-5") allprojects { repositories { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ConfigDelegates.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ConfigDelegates.kt index 39a5f569..71094888 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ConfigDelegates.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ConfigDelegates.kt @@ -14,7 +14,7 @@ fun Configurable.value(default: Any = Null, key: String? = null): MutableValueDe MutableValueDelegate(config, key, Value.of(default)) fun Configurable.value(default: T? = null, key: String? = null, transform: (Value?) -> T): ReadWriteDelegateWrapper = - MutableValueDelegate(config, key, Value.of(default)).transform(reader = transform) + MutableValueDelegate(config, key, default?.let { Value.of(it)}).transform(reader = transform) fun Configurable.string(default: String? = null, key: String? = null): MutableStringDelegate = MutableStringDelegate(config, key, default) diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/descriptors/DescriptorTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/descriptors/DescriptorTest.kt new file mode 100644 index 00000000..79800ec5 --- /dev/null +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/descriptors/DescriptorTest.kt @@ -0,0 +1,31 @@ +package hep.dataforge.descriptors + +import hep.dataforge.values.ValueType +import kotlin.test.Test +import kotlin.test.assertEquals + +class DescriptorTest { + + val descriptor = NodeDescriptor.build { + node("aNode") { + info = "A root demo node" + value("b") { + info = "b number value" + type(ValueType.NUMBER) + } + node("otherNode") { + value("otherValue") { + type(ValueType.BOOLEAN) + default(false) + info = "default value" + } + } + } + } + + @Test + fun testAllowedValues() { + val allowed = descriptor.nodes["aNode"]?.values?.get("b")?.allowedValues + assertEquals(allowed, emptyList()) + } +} \ No newline at end of file From 535e877eb0c1fecc8541610668ff98cac888de9e Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Thu, 13 Jun 2019 09:52:24 +0300 Subject: [PATCH 22/48] Fixed mpp deploy to artifactory --- buildSrc/src/main/kotlin/npm-publish.gradle.kts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/buildSrc/src/main/kotlin/npm-publish.gradle.kts b/buildSrc/src/main/kotlin/npm-publish.gradle.kts index 851348ba..054bf034 100644 --- a/buildSrc/src/main/kotlin/npm-publish.gradle.kts +++ b/buildSrc/src/main/kotlin/npm-publish.gradle.kts @@ -5,6 +5,7 @@ import groovy.lang.GroovyObject import org.gradle.api.publish.maven.internal.artifact.FileBasedMavenArtifact import org.jfrog.gradle.plugin.artifactory.dsl.PublisherConfig import org.jfrog.gradle.plugin.artifactory.dsl.ResolverConfig +import org.jfrog.gradle.plugin.artifactory.task.ArtifactoryTask // Old bintray.gradle script converted to real Gradle plugin (precompiled script plugin) // It now has own dependencies and support type safe accessors @@ -134,3 +135,19 @@ artifactory { }) }) } + +// Fixed module artifact uploading to artifactory +tasks.withType { + doFirst { + publishing.publications + .filterIsInstance() + .forEach { publication -> + val moduleFile = buildDir.resolve("publications/${publication.name}/module.json") + if (moduleFile.exists()) { + publication.artifact(object : FileBasedMavenArtifact(moduleFile) { + override fun getDefaultExtension() = "module" + }) + } + } + } +} From b275288c55fa760e20ba29201cf4be0144c561bf Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 22 Jun 2019 14:29:37 +0300 Subject: [PATCH 23/48] Moved to 1.3.40 and plugin-based build. Fix fo Styled. --- build.gradle.kts | 5 +- buildSrc/build.gradle.kts | 15 +- .../src/main/kotlin/ScientifikMPPlugin.kt | 76 +++++++ .../main/kotlin/ScientifikPublishPlugin.kt | 210 ++++++++++++++++++ buildSrc/src/main/kotlin/Versions.kt | 8 +- .../src/main/kotlin/dokka-publish.gradle.kts | 75 ------- buildSrc/src/main/kotlin/js-test.gradle.kts | 44 ---- .../main/kotlin/npm-multiplatform.gradle.kts | 87 -------- .../src/main/kotlin/npm-publish.gradle.kts | 153 ------------- dataforge-context/build.gradle.kts | 2 +- dataforge-data/build.gradle.kts | 2 +- dataforge-io/build.gradle.kts | 2 +- dataforge-meta/build.gradle.kts | 2 +- .../kotlin/hep/dataforge/meta/Styled.kt | 3 +- .../build.gradle.kts | 4 +- .../hep/dataforge/output/html/HtmlOutput.kt | 0 dataforge-output/build.gradle.kts | 4 +- dataforge-scripting/build.gradle.kts | 2 +- dataforge-workspace/build.gradle.kts | 2 +- settings.gradle.kts | 2 +- 20 files changed, 316 insertions(+), 382 deletions(-) create mode 100644 buildSrc/src/main/kotlin/ScientifikMPPlugin.kt create mode 100644 buildSrc/src/main/kotlin/ScientifikPublishPlugin.kt delete mode 100644 buildSrc/src/main/kotlin/dokka-publish.gradle.kts delete mode 100644 buildSrc/src/main/kotlin/js-test.gradle.kts delete mode 100644 buildSrc/src/main/kotlin/npm-multiplatform.gradle.kts delete mode 100644 buildSrc/src/main/kotlin/npm-publish.gradle.kts rename {dataforge-output/dataforge-output-html => dataforge-output-html}/build.gradle.kts (93%) rename {dataforge-output/dataforge-output-html => dataforge-output-html}/src/jvmMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt (100%) diff --git a/build.gradle.kts b/build.gradle.kts index c05ec51c..55a823d2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,4 +1,4 @@ -val dataforgeVersion by extra("0.1.3-dev-5") +val dataforgeVersion by extra("0.1.3-dev-6") allprojects { repositories { @@ -11,8 +11,7 @@ allprojects { } subprojects { - apply(plugin = "dokka-publish") if (name.startsWith("dataforge")) { - apply(plugin = "npm-publish") + apply(plugin = "scientifik.publish") } } \ No newline at end of file diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 31818916..d0036af4 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -7,7 +7,7 @@ repositories { jcenter() } -val kotlinVersion = "1.3.31" +val kotlinVersion = "1.3.40" // Add plugins used in buildSrc as dependencies, also we should specify version only here dependencies { @@ -18,3 +18,16 @@ dependencies { implementation("com.moowork.gradle:gradle-node-plugin:1.3.1") implementation("org.openjfx:javafx-plugin:0.0.7") } + +gradlePlugin{ + plugins { + create("scientifik-publish") { + id = "scientifik.publish" + implementationClass = "ScientifikPublishPlugin" + } + create("scientifik-mpp"){ + id = "scientifik.mpp" + implementationClass = "ScientifikMPPlugin" + } + } +} diff --git a/buildSrc/src/main/kotlin/ScientifikMPPlugin.kt b/buildSrc/src/main/kotlin/ScientifikMPPlugin.kt new file mode 100644 index 00000000..4cc366c7 --- /dev/null +++ b/buildSrc/src/main/kotlin/ScientifikMPPlugin.kt @@ -0,0 +1,76 @@ +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.kotlin.dsl.configure +import org.gradle.kotlin.dsl.getValue +import org.gradle.kotlin.dsl.getting +import org.gradle.kotlin.dsl.invoke +import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension + +open class ScientifikMPPlugin : Plugin { + override fun apply(project: Project) { + project.plugins.apply("org.jetbrains.kotlin.multiplatform") + + project.configure { + jvm { + compilations.all { + kotlinOptions { + jvmTarget = "1.8" + } + } + } + + js { + compilations.all { + kotlinOptions { + sourceMap = true + sourceMapEmbedSources = "always" + moduleKind = "commonjs" + } + } + } + + sourceSets.invoke { + val commonMain by getting { + dependencies { + api(kotlin("stdlib")) + } + } + val commonTest by getting { + dependencies { + implementation(kotlin("test-common")) + implementation(kotlin("test-annotations-common")) + } + } + val jvmMain by getting { + dependencies { + api(kotlin("stdlib-jdk8")) + } + } + val jvmTest by getting { + dependencies { + implementation(kotlin("test")) + implementation(kotlin("test-junit")) + } + } + val jsMain by getting { + dependencies { + api(kotlin("stdlib-js")) + } + } + val jsTest by getting { + dependencies { + implementation(kotlin("test-js")) + } + } + } + + targets.all { + sourceSets.all { + languageSettings.progressiveMode = true + languageSettings.enableLanguageFeature("InlineClasses") + } + } + } + + } +} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/ScientifikPublishPlugin.kt b/buildSrc/src/main/kotlin/ScientifikPublishPlugin.kt new file mode 100644 index 00000000..fe0631bf --- /dev/null +++ b/buildSrc/src/main/kotlin/ScientifikPublishPlugin.kt @@ -0,0 +1,210 @@ +import com.jfrog.bintray.gradle.BintrayExtension +import groovy.lang.GroovyObject +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.plugins.JavaBasePlugin +import org.gradle.api.publish.PublishingExtension +import org.gradle.api.publish.maven.MavenPublication +import org.gradle.api.publish.maven.internal.artifact.FileBasedMavenArtifact +import org.gradle.api.tasks.bundling.Jar +import org.gradle.kotlin.dsl.* +import org.jetbrains.dokka.gradle.DokkaTask +import org.jetbrains.kotlin.gradle.dsl.KotlinJvmProjectExtension +import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension +import org.jfrog.gradle.plugin.artifactory.dsl.ArtifactoryPluginConvention +import org.jfrog.gradle.plugin.artifactory.dsl.PublisherConfig +import org.jfrog.gradle.plugin.artifactory.dsl.ResolverConfig + + +open class ScientifikExtension { + var vcs = "https://github.com/altavir/dataforge-core" + var bintrayRepo = "dataforge" + var dokka = true +} + +open class ScientifikPublishPlugin : Plugin { + + override fun apply(project: Project) { + + project.plugins.apply("maven-publish") + val extension = project.extensions.create("scientifik") + + + + project.configure { + repositories { + maven("https://bintray.com/mipt-npm/${extension.bintrayRepo}") + } + + // Process each publication we have in this project + publications.filterIsInstance().forEach { publication -> + + @Suppress("UnstableApiUsage") + publication.pom { + name.set(project.name) + description.set(project.description) + url.set(extension.vcs) + + licenses { + license { + name.set("The Apache Software License, Version 2.0") + url.set("http://www.apache.org/licenses/LICENSE-2.0.txt") + distribution.set("repo") + } + } + developers { + developer { + id.set("MIPT-NPM") + name.set("MIPT nuclear physics methods laboratory") + organization.set("MIPT") + organizationUrl.set("http://npm.mipt.ru") + } + + } + scm { + url.set(extension.vcs) + } + } + + val moduleFile = project.buildDir.resolve("publications/${publication.name}/module.json") + if (moduleFile.exists()) { + publication.artifact(object : FileBasedMavenArtifact(moduleFile) { + override fun getDefaultExtension() = "module" + }) + } + } + } + + if(extension.dokka){ + project.plugins.apply("org.jetbrains.dokka") + + project.afterEvaluate { + extensions.findByType()?.apply{ + val dokka by tasks.getting(DokkaTask::class) { + outputFormat = "html" + outputDirectory = "$buildDir/javadoc" + jdkVersion = 8 + + kotlinTasks { + // dokka fails to retrieve sources from MPP-tasks so we only define the jvm task + listOf(tasks.getByPath("compileKotlinJvm")) + } + sourceRoot { + // assuming only single source dir + path = sourceSets["commonMain"].kotlin.srcDirs.first().toString() + platforms = listOf("Common") + } + // although the JVM sources are now taken from the task, + // we still define the jvm source root to get the JVM marker in the generated html + sourceRoot { + // assuming only single source dir + path = sourceSets["jvmMain"].kotlin.srcDirs.first().toString() + platforms = listOf("JVM") + } + } + + val kdocJar by tasks.registering(Jar::class) { + group = JavaBasePlugin.DOCUMENTATION_GROUP + dependsOn(dokka) + archiveClassifier.set("javadoc") + from("$buildDir/javadoc") + } + + configure { + + targets.all { + val publication = publications.findByName(name) as MavenPublication + + // Patch publications with fake javadoc + publication.artifact(kdocJar.get()) + } + } + } + + + extensions.findByType()?.apply{ + val dokka by tasks.getting(DokkaTask::class) { + outputFormat = "html" + outputDirectory = "$buildDir/javadoc" + jdkVersion = 8 + } + + val kdocJar by tasks.registering(Jar::class) { + group = JavaBasePlugin.DOCUMENTATION_GROUP + dependsOn(dokka) + archiveClassifier.set("javadoc") + from("$buildDir/javadoc") + } + + configure { + publications.filterIsInstance().forEach { publication -> + publication.artifact(kdocJar.get()) + } + } + } + } + } + + project.plugins.apply("com.jfrog.bintray") + + project.configure { + user = project.findProperty("bintrayUser") as? String ?: System.getenv("BINTRAY_USER") + key = project.findProperty("bintrayApiKey") as? String? ?: System.getenv("BINTRAY_API_KEY") + publish = true + override = true // for multi-platform Kotlin/Native publishing + + // We have to use delegateClosureOf because bintray supports only dynamic groovy syntax + // this is a problem of this plugin + pkg.apply { + userOrg = "mipt-npm" + repo = extension.bintrayRepo + name = project.name + issueTrackerUrl = "${extension.vcs}/issues" + setLicenses("Apache-2.0") + vcsUrl = extension.vcs + version.apply { + name = project.version.toString() + vcsTag = project.version.toString() + released = java.util.Date().toString() + } + } + + //workaround bintray bug + project.afterEvaluate { + setPublications(*project.extensions.findByType()!!.publications.names.toTypedArray()) + } + +// project.tasks.figetByPath("bintrayUpload") { +// dependsOn(publishToMavenLocal) +// } + } + + project.plugins.apply("com.jfrog.artifactory") + + project.configure { + val artifactoryUser: String? by project + val artifactoryPassword: String? by project + val artifactoryContextUrl = "http://npm.mipt.ru:8081/artifactory" + + setContextUrl(artifactoryContextUrl)//The base Artifactory URL if not overridden by the publisher/resolver + publish(delegateClosureOf { + repository(delegateClosureOf { + setProperty("repoKey", "gradle-dev-local") + setProperty("username", artifactoryUser) + setProperty("password", artifactoryPassword) + }) + + defaults(delegateClosureOf { + invokeMethod("publications", arrayOf("jvm", "js", "kotlinMultiplatform", "metadata")) + }) + }) + resolve(delegateClosureOf { + repository(delegateClosureOf { + setProperty("repoKey", "gradle-dev") + setProperty("username", artifactoryUser) + setProperty("password", artifactoryPassword) + }) + }) + } + } +} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index 883af120..c3fc6d6d 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -2,8 +2,8 @@ // define version in buildSrc and have autocompletion and compile-time check // Also dependencies itself can be moved here object Versions { - val ioVersion = "0.1.8" - val coroutinesVersion = "1.2.1" - val atomicfuVersion = "0.12.6" - val serializationVersion = "0.11.0" + val ioVersion = "0.1.10" + val coroutinesVersion = "1.2.2" + val atomicfuVersion = "0.12.9" + val serializationVersion = "0.11.1" } diff --git a/buildSrc/src/main/kotlin/dokka-publish.gradle.kts b/buildSrc/src/main/kotlin/dokka-publish.gradle.kts deleted file mode 100644 index b7b48fb6..00000000 --- a/buildSrc/src/main/kotlin/dokka-publish.gradle.kts +++ /dev/null @@ -1,75 +0,0 @@ -import org.jetbrains.dokka.gradle.DokkaTask -import org.jetbrains.kotlin.gradle.dsl.KotlinJvmProjectExtension -import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension - -plugins { - id("org.jetbrains.dokka") - `maven-publish` -} - -afterEvaluate { - - extensions.findByType()?.apply{ - val dokka by tasks.getting(DokkaTask::class) { - outputFormat = "html" - outputDirectory = "$buildDir/javadoc" - jdkVersion = 8 - - kotlinTasks { - // dokka fails to retrieve sources from MPP-tasks so we only define the jvm task - listOf(tasks.getByPath("compileKotlinJvm")) - } - sourceRoot { - // assuming only single source dir - path = sourceSets["commonMain"].kotlin.srcDirs.first().toString() - platforms = listOf("Common") - } - // although the JVM sources are now taken from the task, - // we still define the jvm source root to get the JVM marker in the generated html - sourceRoot { - // assuming only single source dir - path = sourceSets["jvmMain"].kotlin.srcDirs.first().toString() - platforms = listOf("JVM") - } - } - - val kdocJar by tasks.registering(Jar::class) { - group = JavaBasePlugin.DOCUMENTATION_GROUP - dependsOn(dokka) - archiveClassifier.set("javadoc") - from("$buildDir/javadoc") - } - - configure { - - targets.all { - val publication = publications.findByName(name) as MavenPublication - - // Patch publications with fake javadoc - publication.artifact(kdocJar.get()) - } - } - } - - - extensions.findByType()?.apply{ - val dokka by tasks.getting(DokkaTask::class) { - outputFormat = "html" - outputDirectory = "$buildDir/javadoc" - jdkVersion = 8 - } - - val kdocJar by tasks.registering(Jar::class) { - group = JavaBasePlugin.DOCUMENTATION_GROUP - dependsOn(dokka) - archiveClassifier.set("javadoc") - from("$buildDir/javadoc") - } - - configure { - publications.filterIsInstance().forEach { publication -> - publication.artifact(kdocJar.get()) - } - } - } -} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/js-test.gradle.kts b/buildSrc/src/main/kotlin/js-test.gradle.kts deleted file mode 100644 index 61759a28..00000000 --- a/buildSrc/src/main/kotlin/js-test.gradle.kts +++ /dev/null @@ -1,44 +0,0 @@ -import com.moowork.gradle.node.npm.NpmTask -import com.moowork.gradle.node.task.NodeTask -import org.gradle.kotlin.dsl.* -import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile - -plugins { - id("com.moowork.node") - kotlin("multiplatform") -} - -node { - nodeModulesDir = file("$buildDir/node_modules") -} - -val compileKotlinJs by tasks.getting(Kotlin2JsCompile::class) -val compileTestKotlinJs by tasks.getting(Kotlin2JsCompile::class) - -val populateNodeModules by tasks.registering(Copy::class) { - dependsOn(compileKotlinJs) - from(compileKotlinJs.destinationDir) - - kotlin.js().compilations["test"].runtimeDependencyFiles.forEach { - if (it.exists() && !it.isDirectory) { - from(zipTree(it.absolutePath).matching { include("*.js") }) - } - } - - into("$buildDir/node_modules") -} - -val installMocha by tasks.registering(NpmTask::class) { - setWorkingDir(buildDir) - setArgs(listOf("install", "mocha")) -} - -val runMocha by tasks.registering(NodeTask::class) { - dependsOn(compileTestKotlinJs, populateNodeModules, installMocha) - setScript(file("$buildDir/node_modules/mocha/bin/mocha")) - setArgs(listOf(compileTestKotlinJs.outputFile)) -} - -tasks["jsTest"].dependsOn(runMocha) - - diff --git a/buildSrc/src/main/kotlin/npm-multiplatform.gradle.kts b/buildSrc/src/main/kotlin/npm-multiplatform.gradle.kts deleted file mode 100644 index dd048a39..00000000 --- a/buildSrc/src/main/kotlin/npm-multiplatform.gradle.kts +++ /dev/null @@ -1,87 +0,0 @@ -import org.gradle.kotlin.dsl.`maven-publish` -import org.gradle.kotlin.dsl.apply -import org.gradle.kotlin.dsl.dependencies -import org.gradle.kotlin.dsl.kotlin - -plugins { - kotlin("multiplatform") - `maven-publish` -} - - -kotlin { - jvm { - compilations.all { - kotlinOptions { - jvmTarget = "1.8" - } - } - } - - js { - compilations.all { - kotlinOptions { - metaInfo = true - sourceMap = true - sourceMapEmbedSources = "always" - moduleKind = "commonjs" - } - } - - compilations.named("main") { - kotlinOptions { - main = "call" - } - } - } - - sourceSets { - val commonMain by getting { - dependencies { - api(kotlin("stdlib")) - } - } - val commonTest by getting { - dependencies { - implementation(kotlin("test-common")) - implementation(kotlin("test-annotations-common")) - } - } - val jvmMain by getting { - dependencies { - api(kotlin("stdlib-jdk8")) - } - } - val jvmTest by getting { - dependencies { - implementation(kotlin("test")) - implementation(kotlin("test-junit")) - } - } - val jsMain by getting { - dependencies { - api(kotlin("stdlib-js")) - } - } - val jsTest by getting { - dependencies { - implementation(kotlin("test-js")) - } - } - } - - targets.all { - sourceSets.all { - languageSettings.progressiveMode = true - languageSettings.enableLanguageFeature("InlineClasses") - } - } - - // Apply JS test configuration - val runJsTests by ext(false) - - if (runJsTests) { - apply(plugin = "js-test") - } - -} diff --git a/buildSrc/src/main/kotlin/npm-publish.gradle.kts b/buildSrc/src/main/kotlin/npm-publish.gradle.kts deleted file mode 100644 index 054bf034..00000000 --- a/buildSrc/src/main/kotlin/npm-publish.gradle.kts +++ /dev/null @@ -1,153 +0,0 @@ -@file:Suppress("UnstableApiUsage") - -import com.jfrog.bintray.gradle.tasks.BintrayUploadTask -import groovy.lang.GroovyObject -import org.gradle.api.publish.maven.internal.artifact.FileBasedMavenArtifact -import org.jfrog.gradle.plugin.artifactory.dsl.PublisherConfig -import org.jfrog.gradle.plugin.artifactory.dsl.ResolverConfig -import org.jfrog.gradle.plugin.artifactory.task.ArtifactoryTask - -// Old bintray.gradle script converted to real Gradle plugin (precompiled script plugin) -// It now has own dependencies and support type safe accessors -// Syntax is pretty close to what we had in Groovy -// (excluding Property.set and bintray dynamic configs) - -plugins { - `maven-publish` - id("com.jfrog.bintray") - id("com.jfrog.artifactory") -} - -val vcs = "https://github.com/altavir/dataforge-core" -val bintrayRepo = "https://bintray.com/mipt-npm/dataforge" - -// Configure publishing -publishing { - repositories { - maven(bintrayRepo) - } - - // Process each publication we have in this project - publications.filterIsInstance().forEach { publication -> - - // use type safe pom config GSL instead of old dynamic - publication.pom { - name.set(project.name) - description.set(project.description) - url.set(vcs) - - licenses { - license { - name.set("The Apache Software License, Version 2.0") - url.set("http://www.apache.org/licenses/LICENSE-2.0.txt") - distribution.set("repo") - } - } - developers { - developer { - id.set("MIPT-NPM") - name.set("MIPT nuclear physics methods laboratory") - organization.set("MIPT") - organizationUrl.set("http://npm.mipt.ru") - } - - } - scm { - url.set(vcs) - } - } - } - -} - -bintray { - user = findProperty("bintrayUser") as? String ?: System.getenv("BINTRAY_USER") - key = findProperty("bintrayApiKey") as? String? ?: System.getenv("BINTRAY_API_KEY") - publish = true - override = true // for multi-platform Kotlin/Native publishing - - // We have to use delegateClosureOf because bintray supports only dynamic groovy syntax - // this is a problem of this plugin - pkg.apply { - userOrg = "mipt-npm" - repo = "dataforge" - name = project.name - issueTrackerUrl = "$vcs/issues" - setLicenses("Apache-2.0") - vcsUrl = vcs - version.apply { - name = project.version.toString() - vcsTag = project.version.toString() - released = java.util.Date().toString() - } - } - - //workaround bintray bug - afterEvaluate { - setPublications(*publishing.publications.names.toTypedArray()) - } - - tasks { - bintrayUpload { - dependsOn(publishToMavenLocal) - } - } -} - -//workaround for bintray -tasks.withType { - doFirst { - publishing.publications - .filterIsInstance() - .forEach { publication -> - val moduleFile = buildDir.resolve("publications/${publication.name}/module.json") - if (moduleFile.exists()) { - publication.artifact(object : FileBasedMavenArtifact(moduleFile) { - override fun getDefaultExtension() = "module" - }) - } - } - } -} - -artifactory { - val artifactoryUser: String? by project - val artifactoryPassword: String? by project - val artifactoryContextUrl = "http://npm.mipt.ru:8081/artifactory" - - setContextUrl(artifactoryContextUrl)//The base Artifactory URL if not overridden by the publisher/resolver - publish(delegateClosureOf { - repository(delegateClosureOf { - setProperty("repoKey", "gradle-dev-local") - setProperty("username", artifactoryUser) - setProperty("password", artifactoryPassword) - }) - - defaults(delegateClosureOf { - invokeMethod("publications", arrayOf("jvm", "js", "kotlinMultiplatform", "metadata")) - }) - }) - resolve(delegateClosureOf { - repository(delegateClosureOf { - setProperty("repoKey", "gradle-dev") - setProperty("username", artifactoryUser) - setProperty("password", artifactoryPassword) - }) - }) -} - -// Fixed module artifact uploading to artifactory -tasks.withType { - doFirst { - publishing.publications - .filterIsInstance() - .forEach { publication -> - val moduleFile = buildDir.resolve("publications/${publication.name}/module.json") - if (moduleFile.exists()) { - publication.artifact(object : FileBasedMavenArtifact(moduleFile) { - override fun getDefaultExtension() = "module" - }) - } - } - } -} diff --git a/dataforge-context/build.gradle.kts b/dataforge-context/build.gradle.kts index f454e2ef..75c74910 100644 --- a/dataforge-context/build.gradle.kts +++ b/dataforge-context/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - `npm-multiplatform` + id("scientifik.mpp") } description = "Context and provider definitions" diff --git a/dataforge-data/build.gradle.kts b/dataforge-data/build.gradle.kts index 7ebb46ce..7527bf6f 100644 --- a/dataforge-data/build.gradle.kts +++ b/dataforge-data/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - `npm-multiplatform` + id("scientifik.mpp") } val coroutinesVersion: String = Versions.coroutinesVersion diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index 67ffe124..4d7772a9 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - `npm-multiplatform` + id("scientifik.mpp") } description = "IO for meta" diff --git a/dataforge-meta/build.gradle.kts b/dataforge-meta/build.gradle.kts index 17aaebf8..40ddbb17 100644 --- a/dataforge-meta/build.gradle.kts +++ b/dataforge-meta/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - `npm-multiplatform` + id("scientifik.mpp") } description = "Meta definition and basic operations on meta" diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt index 39b47227..bea41c5d 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt @@ -16,7 +16,6 @@ class Styled(val base: Meta, val style: Config = Config().empty()) : AbstractMut override fun empty(): Styled = Styled(EmptyMeta) - @Suppress("UNCHECKED_CAST") override val items: Map> get() = (base.items.keys + style.items.keys).associate { key -> val value = base.items[key] @@ -25,7 +24,7 @@ class Styled(val base: Meta, val style: Config = Config().empty()) : AbstractMut null -> when (styleValue) { null -> error("Should be unreachable") is MetaItem.NodeItem -> MetaItem.NodeItem(Styled(style.empty(), styleValue.node)) - else -> styleValue.value as MetaItem + is MetaItem.ValueItem -> styleValue } is MetaItem.ValueItem -> value as MetaItem is MetaItem.NodeItem -> MetaItem.NodeItem( diff --git a/dataforge-output/dataforge-output-html/build.gradle.kts b/dataforge-output-html/build.gradle.kts similarity index 93% rename from dataforge-output/dataforge-output-html/build.gradle.kts rename to dataforge-output-html/build.gradle.kts index d1693bab..98b9d0bd 100644 --- a/dataforge-output/dataforge-output-html/build.gradle.kts +++ b/dataforge-output-html/build.gradle.kts @@ -1,12 +1,10 @@ plugins { - `npm-multiplatform` + id("scientifik.mpp") } val htmlVersion by rootProject.extra("0.6.12") kotlin { - jvm() - js() sourceSets { val commonMain by getting { dependencies { diff --git a/dataforge-output/dataforge-output-html/src/jvmMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt b/dataforge-output-html/src/jvmMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt similarity index 100% rename from dataforge-output/dataforge-output-html/src/jvmMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt rename to dataforge-output-html/src/jvmMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt diff --git a/dataforge-output/build.gradle.kts b/dataforge-output/build.gradle.kts index 36811267..6c0a1e53 100644 --- a/dataforge-output/build.gradle.kts +++ b/dataforge-output/build.gradle.kts @@ -1,10 +1,8 @@ plugins { - `npm-multiplatform` + id("scientifik.mpp") } kotlin { - jvm() - js() sourceSets { val commonMain by getting{ dependencies { diff --git a/dataforge-scripting/build.gradle.kts b/dataforge-scripting/build.gradle.kts index eb8f7742..757f0c33 100644 --- a/dataforge-scripting/build.gradle.kts +++ b/dataforge-scripting/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - `npm-multiplatform` + id("scientifik.mpp") } kotlin { diff --git a/dataforge-workspace/build.gradle.kts b/dataforge-workspace/build.gradle.kts index e6aa9dd0..ff74d591 100644 --- a/dataforge-workspace/build.gradle.kts +++ b/dataforge-workspace/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - `npm-multiplatform` + id("scientifik.mpp") } kotlin { diff --git a/settings.gradle.kts b/settings.gradle.kts index 03b9bff9..05b41d36 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -25,7 +25,7 @@ include( ":dataforge-context", ":dataforge-data", ":dataforge-output", - ":dataforge-output:dataforge-output-html", + ":dataforge-output-html", ":dataforge-workspace", ":dataforge-scripting" ) \ No newline at end of file From 7f95dcf02cbb27bd9e26e9675bdcc8d6e315ee3b Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 22 Jun 2019 15:33:33 +0300 Subject: [PATCH 24/48] Build fix for module deploy --- .../src/main/kotlin/ScientifikMPPlugin.kt | 23 +++++++++++++++++++ .../main/kotlin/ScientifikPublishPlugin.kt | 15 ++++-------- dataforge-context/build.gradle.kts | 3 --- .../kotlin/hep/dataforge/io/Binary.kt | 1 + .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 1 + dataforge-meta/build.gradle.kts | 4 ---- .../kotlin/hep/dataforge/meta/Meta.kt | 2 +- .../kotlin/hep/dataforge/meta/Styled.kt | 2 +- .../kotlin/hep/dataforge/meta/DynamicMeta.kt | 8 +++---- 9 files changed, 36 insertions(+), 23 deletions(-) diff --git a/buildSrc/src/main/kotlin/ScientifikMPPlugin.kt b/buildSrc/src/main/kotlin/ScientifikMPPlugin.kt index 4cc366c7..0aef37be 100644 --- a/buildSrc/src/main/kotlin/ScientifikMPPlugin.kt +++ b/buildSrc/src/main/kotlin/ScientifikMPPlugin.kt @@ -1,10 +1,15 @@ +import com.jfrog.bintray.gradle.tasks.BintrayUploadTask import org.gradle.api.Plugin import org.gradle.api.Project +import org.gradle.api.publish.PublishingExtension +import org.gradle.api.publish.maven.MavenPublication +import org.gradle.api.publish.maven.internal.artifact.FileBasedMavenArtifact import org.gradle.kotlin.dsl.configure import org.gradle.kotlin.dsl.getValue import org.gradle.kotlin.dsl.getting import org.gradle.kotlin.dsl.invoke import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension +import org.jfrog.gradle.plugin.artifactory.task.ArtifactoryTask open class ScientifikMPPlugin : Plugin { override fun apply(project: Project) { @@ -72,5 +77,23 @@ open class ScientifikMPPlugin : Plugin { } } + + project.tasks.filter { it is ArtifactoryTask || it is BintrayUploadTask }.forEach { + it.doFirst { + project.configure { + publications + .filterIsInstance() + .forEach { publication -> + val moduleFile = project.buildDir.resolve("publications/${publication.name}/module.json") + if (moduleFile.exists()) { + publication.artifact(object : FileBasedMavenArtifact(moduleFile) { + override fun getDefaultExtension() = "module" + }) + } + } + } + } + } + } } \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/ScientifikPublishPlugin.kt b/buildSrc/src/main/kotlin/ScientifikPublishPlugin.kt index fe0631bf..81e56ad6 100644 --- a/buildSrc/src/main/kotlin/ScientifikPublishPlugin.kt +++ b/buildSrc/src/main/kotlin/ScientifikPublishPlugin.kt @@ -1,4 +1,5 @@ import com.jfrog.bintray.gradle.BintrayExtension +import com.jfrog.bintray.gradle.tasks.BintrayUploadTask import groovy.lang.GroovyObject import org.gradle.api.Plugin import org.gradle.api.Project @@ -14,6 +15,7 @@ import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension import org.jfrog.gradle.plugin.artifactory.dsl.ArtifactoryPluginConvention import org.jfrog.gradle.plugin.artifactory.dsl.PublisherConfig import org.jfrog.gradle.plugin.artifactory.dsl.ResolverConfig +import org.jfrog.gradle.plugin.artifactory.task.ArtifactoryTask open class ScientifikExtension { @@ -65,21 +67,14 @@ open class ScientifikPublishPlugin : Plugin { url.set(extension.vcs) } } - - val moduleFile = project.buildDir.resolve("publications/${publication.name}/module.json") - if (moduleFile.exists()) { - publication.artifact(object : FileBasedMavenArtifact(moduleFile) { - override fun getDefaultExtension() = "module" - }) - } } } - if(extension.dokka){ + if (extension.dokka) { project.plugins.apply("org.jetbrains.dokka") project.afterEvaluate { - extensions.findByType()?.apply{ + extensions.findByType()?.apply { val dokka by tasks.getting(DokkaTask::class) { outputFormat = "html" outputDirectory = "$buildDir/javadoc" @@ -122,7 +117,7 @@ open class ScientifikPublishPlugin : Plugin { } - extensions.findByType()?.apply{ + extensions.findByType()?.apply { val dokka by tasks.getting(DokkaTask::class) { outputFormat = "html" outputDirectory = "$buildDir/javadoc" diff --git a/dataforge-context/build.gradle.kts b/dataforge-context/build.gradle.kts index 75c74910..a4f8faba 100644 --- a/dataforge-context/build.gradle.kts +++ b/dataforge-context/build.gradle.kts @@ -7,9 +7,6 @@ description = "Context and provider definitions" val coroutinesVersion: String = Versions.coroutinesVersion kotlin { - jvm() - js() - sourceSets { val commonMain by getting { dependencies { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt index a0374968..75ef0422 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt @@ -7,6 +7,7 @@ import kotlinx.io.core.readBytes /** * A source of binary data */ + interface Binary { /** * The size of binary in bytes diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index ccb6e382..a993620e 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -2,6 +2,7 @@ package hep.dataforge.io import kotlinx.io.core.* +@ExperimentalUnsignedTypes class TaggedEnvelopeFormat( val metaFormats: Collection, val outputMetaFormat: MetaFormat = metaFormats.first() diff --git a/dataforge-meta/build.gradle.kts b/dataforge-meta/build.gradle.kts index 40ddbb17..859c2ed6 100644 --- a/dataforge-meta/build.gradle.kts +++ b/dataforge-meta/build.gradle.kts @@ -4,7 +4,3 @@ plugins { description = "Meta definition and basic operations on meta" -kotlin { - jvm() - js() -} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 81b31175..153beab1 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -197,7 +197,7 @@ fun Meta.seal(): SealedMeta = this as? SealedMeta ?: SealedMeta(items.mapValues @Suppress("UNCHECKED_CAST") fun MetaItem<*>.seal(): MetaItem = when (this) { - is ValueItem -> this as MetaItem + is ValueItem -> this is NodeItem -> NodeItem(node.seal()) } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt index bea41c5d..c34d933f 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt @@ -26,7 +26,7 @@ class Styled(val base: Meta, val style: Config = Config().empty()) : AbstractMut is MetaItem.NodeItem -> MetaItem.NodeItem(Styled(style.empty(), styleValue.node)) is MetaItem.ValueItem -> styleValue } - is MetaItem.ValueItem -> value as MetaItem + is MetaItem.ValueItem -> value is MetaItem.NodeItem -> MetaItem.NodeItem( Styled(value.node, styleValue?.node ?: Config.empty()) ) diff --git a/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt b/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt index 12b8db9e..57375125 100644 --- a/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt +++ b/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt @@ -37,11 +37,11 @@ class DynamicMeta(val obj: dynamic) : Meta { @Suppress("UNCHECKED_CAST") private fun asItem(obj: dynamic): MetaItem? { - if (obj == null) return MetaItem.ValueItem(Null) as MetaItem + if (obj == null) return MetaItem.ValueItem(Null) return when (jsTypeOf(obj as? Any)) { - "boolean" -> MetaItem.ValueItem(Value.of(obj as Boolean)) as MetaItem - "number" -> MetaItem.ValueItem(Value.of(obj as Number)) as MetaItem - "string" -> MetaItem.ValueItem(Value.of(obj as String)) as MetaItem + "boolean" -> MetaItem.ValueItem(Value.of(obj as Boolean)) + "number" -> MetaItem.ValueItem(Value.of(obj as Number)) + "string" -> MetaItem.ValueItem(Value.of(obj as String)) "object" -> MetaItem.NodeItem(DynamicMeta(obj)) else -> null } From 3bf49d3fae56de3011566138d721e6df6563a81e Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 28 Jun 2019 10:19:58 +0300 Subject: [PATCH 25/48] Isolating build logic --- build.gradle.kts | 5 +- buildSrc/src/main/kotlin/Scientifik.kt | 9 ++++ .../src/main/kotlin/ScientifikMPPlugin.kt | 32 +++---------- .../main/kotlin/ScientifikPublishPlugin.kt | 46 ++++++++++++++++--- buildSrc/src/main/kotlin/Versions.kt | 9 ---- dataforge-context/build.gradle.kts | 2 +- dataforge-data/build.gradle.kts | 2 +- dataforge-io/build.gradle.kts | 4 +- .../hep/dataforge/io/BinaryMetaFormat.kt | 5 +- dataforge-meta/build.gradle.kts | 3 ++ ...{ConfigDelegates.kt => configDelegates.kt} | 21 +++++++-- .../meta/{Delegates.kt => metaDelegates.kt} | 11 +++-- .../kotlin/hep/dataforge/values/Value.kt | 18 +------- .../hep/dataforge/values/exoticValues.kt | 46 +++++++++++++++++++ 14 files changed, 140 insertions(+), 73 deletions(-) create mode 100644 buildSrc/src/main/kotlin/Scientifik.kt delete mode 100644 buildSrc/src/main/kotlin/Versions.kt rename dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/{ConfigDelegates.kt => configDelegates.kt} (86%) rename dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/{Delegates.kt => metaDelegates.kt} (98%) create mode 100644 dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt diff --git a/build.gradle.kts b/build.gradle.kts index 55a823d2..5723c33f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,4 +1,7 @@ -val dataforgeVersion by extra("0.1.3-dev-6") +val dataforgeVersion by extra("0.1.3-dev-7") + +val bintrayRepo by extra("dataforge") +val vcs by extra("https://github.com/mipt-npm/dataforge-core") allprojects { repositories { diff --git a/buildSrc/src/main/kotlin/Scientifik.kt b/buildSrc/src/main/kotlin/Scientifik.kt new file mode 100644 index 00000000..617b2e67 --- /dev/null +++ b/buildSrc/src/main/kotlin/Scientifik.kt @@ -0,0 +1,9 @@ +/** + * Build constants + */ +object Scientifik { + val ioVersion = "0.1.10" + val coroutinesVersion = "1.2.2" + val atomicfuVersion = "0.12.9" + val serializationVersion = "0.11.1" +} diff --git a/buildSrc/src/main/kotlin/ScientifikMPPlugin.kt b/buildSrc/src/main/kotlin/ScientifikMPPlugin.kt index 0aef37be..e34c675b 100644 --- a/buildSrc/src/main/kotlin/ScientifikMPPlugin.kt +++ b/buildSrc/src/main/kotlin/ScientifikMPPlugin.kt @@ -1,15 +1,10 @@ -import com.jfrog.bintray.gradle.tasks.BintrayUploadTask import org.gradle.api.Plugin import org.gradle.api.Project -import org.gradle.api.publish.PublishingExtension -import org.gradle.api.publish.maven.MavenPublication -import org.gradle.api.publish.maven.internal.artifact.FileBasedMavenArtifact import org.gradle.kotlin.dsl.configure import org.gradle.kotlin.dsl.getValue import org.gradle.kotlin.dsl.getting import org.gradle.kotlin.dsl.invoke import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension -import org.jfrog.gradle.plugin.artifactory.task.ArtifactoryTask open class ScientifikMPPlugin : Plugin { override fun apply(project: Project) { @@ -29,7 +24,7 @@ open class ScientifikMPPlugin : Plugin { kotlinOptions { sourceMap = true sourceMapEmbedSources = "always" - moduleKind = "commonjs" + moduleKind = "umd" } } } @@ -71,26 +66,11 @@ open class ScientifikMPPlugin : Plugin { targets.all { sourceSets.all { - languageSettings.progressiveMode = true - languageSettings.enableLanguageFeature("InlineClasses") - } - } - } - - - project.tasks.filter { it is ArtifactoryTask || it is BintrayUploadTask }.forEach { - it.doFirst { - project.configure { - publications - .filterIsInstance() - .forEach { publication -> - val moduleFile = project.buildDir.resolve("publications/${publication.name}/module.json") - if (moduleFile.exists()) { - publication.artifact(object : FileBasedMavenArtifact(moduleFile) { - override fun getDefaultExtension() = "module" - }) - } - } + languageSettings.apply{ + progressiveMode = true + enableLanguageFeature("InlineClasses") + useExperimentalAnnotation("ExperimentalUnsignedType") + } } } } diff --git a/buildSrc/src/main/kotlin/ScientifikPublishPlugin.kt b/buildSrc/src/main/kotlin/ScientifikPublishPlugin.kt index 81e56ad6..c94751a9 100644 --- a/buildSrc/src/main/kotlin/ScientifikPublishPlugin.kt +++ b/buildSrc/src/main/kotlin/ScientifikPublishPlugin.kt @@ -19,11 +19,22 @@ import org.jfrog.gradle.plugin.artifactory.task.ArtifactoryTask open class ScientifikExtension { - var vcs = "https://github.com/altavir/dataforge-core" - var bintrayRepo = "dataforge" - var dokka = true + var vcs: String? = null + var bintrayRepo: String? = null + var kdoc: Boolean = true } +// recursively search up the project chain for configuration +private val Project.bintrayRepo: String? + get() = extensions.findByType()?.bintrayRepo + ?: parent?.bintrayRepo + ?: (findProperty("bintrayRepo") as? String) + +private val Project.vcs: String? + get() = extensions.findByType()?.vcs + ?: parent?.vcs + ?: (findProperty("vcs") as? String) + open class ScientifikPublishPlugin : Plugin { override fun apply(project: Project) { @@ -31,11 +42,16 @@ open class ScientifikPublishPlugin : Plugin { project.plugins.apply("maven-publish") val extension = project.extensions.create("scientifik") + val bintrayRepo = project.bintrayRepo + val vcs = project.vcs + if (bintrayRepo == null || vcs == null) { + project.logger.warn("[${project.name}] Missing deployment configuration. Skipping publish.") + } project.configure { repositories { - maven("https://bintray.com/mipt-npm/${extension.bintrayRepo}") + maven("https://bintray.com/mipt-npm/$bintrayRepo") } // Process each publication we have in this project @@ -45,7 +61,7 @@ open class ScientifikPublishPlugin : Plugin { publication.pom { name.set(project.name) description.set(project.description) - url.set(extension.vcs) + url.set(vcs) licenses { license { @@ -70,7 +86,7 @@ open class ScientifikPublishPlugin : Plugin { } } - if (extension.dokka) { + if (extension.kdoc) { project.plugins.apply("org.jetbrains.dokka") project.afterEvaluate { @@ -96,6 +112,7 @@ open class ScientifikPublishPlugin : Plugin { path = sourceSets["jvmMain"].kotlin.srcDirs.first().toString() platforms = listOf("JVM") } + } val kdocJar by tasks.registering(Jar::class) { @@ -113,6 +130,21 @@ open class ScientifikPublishPlugin : Plugin { // Patch publications with fake javadoc publication.artifact(kdocJar.get()) } + + tasks.filter { it is ArtifactoryTask || it is BintrayUploadTask }.forEach { + it.doFirst { + publications.filterIsInstance() + .forEach { publication -> + val moduleFile = + buildDir.resolve("publications/${publication.name}/module.json") + if (moduleFile.exists()) { + publication.artifact(object : FileBasedMavenArtifact(moduleFile) { + override fun getDefaultExtension() = "module" + }) + } + } + } + } } } @@ -152,7 +184,7 @@ open class ScientifikPublishPlugin : Plugin { // this is a problem of this plugin pkg.apply { userOrg = "mipt-npm" - repo = extension.bintrayRepo + repo = bintrayRepo name = project.name issueTrackerUrl = "${extension.vcs}/issues" setLicenses("Apache-2.0") diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt deleted file mode 100644 index c3fc6d6d..00000000 --- a/buildSrc/src/main/kotlin/Versions.kt +++ /dev/null @@ -1,9 +0,0 @@ -// Instead of defining runtime properties and use them dynamically -// define version in buildSrc and have autocompletion and compile-time check -// Also dependencies itself can be moved here -object Versions { - val ioVersion = "0.1.10" - val coroutinesVersion = "1.2.2" - val atomicfuVersion = "0.12.9" - val serializationVersion = "0.11.1" -} diff --git a/dataforge-context/build.gradle.kts b/dataforge-context/build.gradle.kts index a4f8faba..f550b58a 100644 --- a/dataforge-context/build.gradle.kts +++ b/dataforge-context/build.gradle.kts @@ -4,7 +4,7 @@ plugins { description = "Context and provider definitions" -val coroutinesVersion: String = Versions.coroutinesVersion +val coroutinesVersion: String = Scientifik.coroutinesVersion kotlin { sourceSets { diff --git a/dataforge-data/build.gradle.kts b/dataforge-data/build.gradle.kts index 7527bf6f..00052375 100644 --- a/dataforge-data/build.gradle.kts +++ b/dataforge-data/build.gradle.kts @@ -2,7 +2,7 @@ plugins { id("scientifik.mpp") } -val coroutinesVersion: String = Versions.coroutinesVersion +val coroutinesVersion: String = Scientifik.coroutinesVersion kotlin { jvm() diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index 4d7772a9..6ed06857 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -5,8 +5,8 @@ plugins { description = "IO for meta" -val ioVersion: String = Versions.ioVersion -val serializationVersion: String = Versions.serializationVersion +val ioVersion: String = Scientifik.ioVersion +val serializationVersion: String = Scientifik.serializationVersion kotlin { jvm() diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt index 843e5acb..941b4abc 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt @@ -92,8 +92,7 @@ object BinaryMetaFormat : MetaFormat { @Suppress("UNCHECKED_CAST") private fun Input.readMetaItem(): MetaItem { - val keyChar = readByte().toChar() - return when (keyChar) { + return when (val keyChar = readByte().toChar()) { 'S' -> MetaItem.ValueItem(StringValue(readString())) 'N' -> MetaItem.ValueItem(Null) '+' -> MetaItem.ValueItem(True) @@ -120,6 +119,6 @@ object BinaryMetaFormat : MetaFormat { MetaItem.NodeItem(meta) } else -> error("Unknown serialization key character: $keyChar") - } as MetaItem + } } } \ No newline at end of file diff --git a/dataforge-meta/build.gradle.kts b/dataforge-meta/build.gradle.kts index 859c2ed6..3fd7b1e0 100644 --- a/dataforge-meta/build.gradle.kts +++ b/dataforge-meta/build.gradle.kts @@ -4,3 +4,6 @@ plugins { description = "Meta definition and basic operations on meta" +scientifik{ + +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ConfigDelegates.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt similarity index 86% rename from dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ConfigDelegates.kt rename to dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt index 71094888..6871b6e3 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ConfigDelegates.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/configDelegates.kt @@ -1,5 +1,6 @@ package hep.dataforge.meta +import hep.dataforge.values.DoubleArrayValue import hep.dataforge.values.Null import hep.dataforge.values.Value import kotlin.jvm.JvmName @@ -13,8 +14,13 @@ import kotlin.jvm.JvmName fun Configurable.value(default: Any = Null, key: String? = null): MutableValueDelegate = MutableValueDelegate(config, key, Value.of(default)) -fun Configurable.value(default: T? = null, key: String? = null, transform: (Value?) -> T): ReadWriteDelegateWrapper = - MutableValueDelegate(config, key, default?.let { Value.of(it)}).transform(reader = transform) +fun Configurable.value( + default: T? = null, + key: String? = null, + writer: (T) -> Value = { Value.of(it) }, + reader: (Value?) -> T +): ReadWriteDelegateWrapper = + MutableValueDelegate(config, key, default?.let { Value.of(it) }).transform(reader = reader, writer = writer) fun Configurable.string(default: String? = null, key: String? = null): MutableStringDelegate = MutableStringDelegate(config, key, default) @@ -106,7 +112,6 @@ fun Configurable.spec(spec: Specification, key: String? = null fun Configurable.spec(builder: (Config) -> T, key: String? = null) = MutableMorphDelegate(config, key) { specification(builder).wrap(it) } - /* * Extra delegates for special cases */ @@ -117,7 +122,15 @@ fun Configurable.stringList(key: String? = null): ReadWriteDelegateWrapper> = value(emptyList(), key) { it?.list?.map { value -> value.number } ?: emptyList() } -fun Metoid.child(key: String? = null, converter: (Meta) -> T) = ChildDelegate(meta, key, converter) +/** + * A special delegate for double arrays + */ +fun Configurable.doubleArray(key: String? = null): ReadWriteDelegateWrapper = + value(doubleArrayOf(), key) { + (it as? DoubleArrayValue)?.value + ?: it?.list?.map { value -> value.number.toDouble() }?.toDoubleArray() + ?: doubleArrayOf() + } fun Configurable.child(key: String? = null, converter: (Meta) -> T) = MutableMorphDelegate(config, key, converter) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Delegates.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt similarity index 98% rename from dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Delegates.kt rename to dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt index bcd60cc8..9d606b6f 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Delegates.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaDelegates.kt @@ -113,8 +113,11 @@ class SafeEnumDelegate>( //Child node delegate -class ChildDelegate(val meta: Meta, private val key: String? = null, private val converter: (Meta) -> T) : - ReadOnlyProperty { +class ChildDelegate( + val meta: Meta, + private val key: String? = null, + private val converter: (Meta) -> T +) : ReadOnlyProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): T? { return meta[key ?: property.name]?.node?.let { converter(it) } } @@ -164,6 +167,8 @@ inline fun > Meta.enum(default: E, key: String? = null) = SafeEnumDelegate(this, key, default) { enumValueOf(it) } +fun Metoid.child(key: String? = null, converter: (Meta) -> T) = ChildDelegate(meta, key, converter) + /* Read-write delegates */ class MutableValueDelegate>( @@ -418,4 +423,4 @@ fun > M.number(key: String? = null, default: () -> Number) = inline fun , reified E : Enum> M.enum(default: E, key: String? = null) = - MutableSafeEnumvDelegate(this, key, default) { enumValueOf(it) } + MutableSafeEnumvDelegate(this, key, default) { enumValueOf(it) } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt index 5753512b..ac651c20 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt @@ -110,6 +110,8 @@ object False : Value { override val type: ValueType get() = ValueType.BOOLEAN override val number: Number get() = -1.0 override val string: String get() = "false" + + override fun toString(): String = True.value.toString() } val Value.boolean get() = this == True || this.list.firstOrNull() == True || (type == ValueType.STRING && string.toBoolean()) @@ -206,9 +208,6 @@ fun String.asValue(): Value = StringValue(this) fun Iterable.asValue(): Value = ListValue(this.toList()) -//TODO maybe optimized storage performance -fun DoubleArray.asValue(): Value = ListValue(map{NumberValue(it)}) - fun IntArray.asValue(): Value = ListValue(map{NumberValue(it)}) fun LongArray.asValue(): Value = ListValue(map{NumberValue(it)}) @@ -253,17 +252,4 @@ fun String.parseValue(): Value { //Give up and return a StringValue return StringValue(this) -} - -class LazyParsedValue(override val string: String) : Value { - private val parsedValue by lazy { string.parseValue() } - - override val value: Any? - get() = parsedValue.value - override val type: ValueType - get() = parsedValue.type - override val number: Number - get() = parsedValue.number - - override fun toString(): String = value.toString() } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt new file mode 100644 index 00000000..9f13d73a --- /dev/null +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt @@ -0,0 +1,46 @@ +package hep.dataforge.values + + +/** + * A value built from string which content and type are parsed on-demand + */ +class LazyParsedValue(override val string: String) : Value { + private val parsedValue by lazy { string.parseValue() } + + override val value: Any? get() = parsedValue.value + override val type: ValueType get() = parsedValue.type + override val number: Number get() = parsedValue.number + + override fun toString(): String = string +} + +fun String.lazyParseValue(): LazyParsedValue = LazyParsedValue(this) + +/** + * A performance optimized version of list value for doubles + */ +class DoubleArrayValue(override val value: DoubleArray) : Value { + override val type: ValueType get() = ValueType.NUMBER + override val number: Double get() = value.first() + override val string: String get() = value.first().toString() + override val list: List get() = value.map { NumberValue(it) } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is Value) return false + + return if (other is DoubleArrayValue) { + value.contentEquals(other.value) + } else { + list == other.list + } + } + + override fun hashCode(): Int { + return value.contentHashCode() + } + + override fun toString(): String = value.toString() +} + +fun DoubleArray.asValue(): DoubleArrayValue = DoubleArrayValue(this) From b13d980c0214704497e777e2e76757d0f50d36fe Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 28 Jun 2019 16:28:54 +0300 Subject: [PATCH 26/48] migrating to common toolset --- README.md | 96 +++++++ build.gradle.kts | 5 + buildSrc/build.gradle.kts | 33 --- buildSrc/settings.gradle.kts | 0 buildSrc/src/main/kotlin/Scientifik.kt | 9 - .../src/main/kotlin/ScientifikMPPlugin.kt | 79 ------ .../main/kotlin/ScientifikPublishPlugin.kt | 237 ------------------ dataforge-meta/build.gradle.kts | 6 +- settings.gradle.kts | 2 + 9 files changed, 104 insertions(+), 363 deletions(-) delete mode 100644 buildSrc/build.gradle.kts delete mode 100644 buildSrc/settings.gradle.kts delete mode 100644 buildSrc/src/main/kotlin/Scientifik.kt delete mode 100644 buildSrc/src/main/kotlin/ScientifikMPPlugin.kt delete mode 100644 buildSrc/src/main/kotlin/ScientifikPublishPlugin.kt diff --git a/README.md b/README.md index e69de29b..79fa4cf1 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,96 @@ + + + +# Questions and Answers # + +In this section we will try to cover DataForge main ideas in the form of questions and answers. + +## General ## + +**Q:** I have a lot of data to analyze. The analysis process is complicated, requires a lot of stages and data flow is not always obvious. To top it the data size is huge, so I don't want to perform operation I don't need (calculate something I won't need or calculate something twice). And yes, I need it to be performed in parallel and probably on remote computer. By the way, I am sick and tired of scripts that modify other scripts that control scripts. Could you help me? + +**A:** Yes, that is the precisely the problem DataForge was made to solve. It allows to perform some automated data manipulations with automatic optimization and parallelization. The important thing that data processing recipes are made in the declarative way, so it is quite easy to perform computations on a remote station. Also DataForge guarantees reproducibility of analysis results. +
+ +**Q:** How does it work? + +**A:** At the core of DataForge lies the idea of **metadata processor**. It utilizes the statement that in order to analyze something you need data itself and some additional information about what does that data represent and what does user want as a result. This additional information is called metadata and could be organized in a regular structure (a tree of values not unlike XML or JSON). The important thing is that this distinction leaves no place for user instructions (or scripts). Indeed, the idea of DataForge logic is that one do not need imperative commands. The framework configures itself according to input meta-data and decides what operations should be performed in the most efficient way. +
+ +**Q:** But where does it take algorithms to use? + +**A:** Of course algorithms must be written somewhere. No magic here. The logic is written in specialized modules. Some modules are provided out of the box at the system core, some need to be developed for specific problem. +
+ +**Q:** So I still need to write the code? What is the difference then? + +**A:** Yes, someone still need to write the code. But not necessary you. Simple operations could be performed using provided core logic. Also your group can have one programmer writing the logic and all other using it without any real programming expertise. Also the framework organized in a such way that one writes some additional logic, he do not need to thing about complicated thing like parallel computing, resource handling, logging, caching etc. Most of the things are done by the DataForge. +
+ +## Platform ## + +**Q:** Which platform does DataForge use? Which operation system is it working on? + +**A:** The DataForge is mostly written in Java and utilizes JVM as a platform. It works on any system that supports JVM (meaning almost any modern system excluding some mobile platforms). +
+ + **Q:** But Java... it is slow! + + **A:** [It is not](https://stackoverflow.com/questions/2163411/is-java-really-slow/2163570#2163570). It lacks some hardware specific optimizations and requires some additional time to start (due to JIT nature), but otherwise it is at least as fast as other languages traditionally used in science. More importantly, the memory safety, tooling support and vast ecosystem makes it №1 candidate for data analysis framework. + +
+ + **Q:** Can I use my C++/Fortran/Python code in DataForge? + + **A:** Yes, as long as the code could be called from Java. Most of common languages have a bridge for Java access. There are completely no problems with compiled C/Fortran libraries. Python code could be called via one of existing python-java interfaces. It is also planned to implement remote method invocation for common languages, so your Python, or, say, Julia, code could run in its native environment. The metadata processor paradigm makes it much easier to do so. + +
+ +## Features ## + +**Q:** What other features does DataForge provide? + +**A:** Alongside metadata processing (and a lot of tools for metadata manipulation and layering), DataForge has two additional important concepts: + +* **Modularisation**. Contrary to lot other frameworks, DataForge is intrinsically modular. The mandatory part is a rather tiny core module. Everything else could be customized. + +* **Context encapsulation**. Every DataForge task is executed in some context. The context isolates environment for the task and also works as dependency injection base and specifies interaction of the task with the external world. + + +
+ +**Q:** OK, but now I want to work directly with my measuring devices. How can I do that? + +**A:** The [dataforge-control](${site.url}/docs.html#control) module provides interfaces to interact with the hardware. Out of the box it supports safe communication with TCP/IP or COM/tty based devices. Specific device declaration could be done via additional modules. It is also possible to maintain data storage with [datforge-storage](${site.url}/docs.htm#storage) module. + +
+ +**Q:** Declarations and metadata are good, but I want my scripts back! + +**A:** We can do that. [GRIND](${site.url}/docs.html#grind) provides a shell-like environment called GrindShell. It allows to run imperative scripts with full access to all of the DataForge functionality. Grind scripts are basically context-encapsulated. Also there are convenient feature wrappers called helpers that could be loaded into the shell when new features modules are added. + +
+ +## Misc ## + +**Q:** So everything looks great, can I replace my ROOT / other data analysis framework with DataForge? + +**A:** One must note, that DataForge is made for analysis, not for visualisation. The visualisation and user interaction capabilities of DataForge are rather limited compared to frameworks like ROOT, JAS3 or DataMelt. The idea is to provide reliable API and core functionality. In fact JAS3 and DataMelt could be used as a frontend for DataForge mechanics. It is planned to add an interface to ROOT via JFreeHep AIDA. + +
+ +**Q:** How does DataForge compare to cluster computation frameworks like Hadoop or Spark? + +**A:** Again, it is not the purpose of DataForge to replace cluster software. DataForge has some internal parallelism mechanics and implementations, but they are most certainly worse then specially developed programs. Still, DataForge is not fixed on one single implementation. Your favourite parallel processing tool could be still used as a back-end for the DataForge. With full benefit of configuration tools, integrations and no performance overhead. + +
+ +**Q:** Is it possible to use DataForge in notebook mode? + +**A:** Yes, it is. DataForge can be used as is from [beaker/beakerx](http://beakernotebook.com/) groovy kernel with minor additional adjustments. It is planned to provide separate DataForge kernel to `beakerx` which will automatically call a specific GRIND shell. + +
+ +**Q:** Can I use DataForge on a mobile platform? + +**A:** DataForge is modular. Core and the most of api are pretty compact, so it could be used in Android applications. Some modules are designed for PC and could not be used on other platforms. IPhone does not support Java and therefore could use only client-side DataForge applications. diff --git a/build.gradle.kts b/build.gradle.kts index 5723c33f..3b494f3d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,3 +1,8 @@ +plugins { + id("scientifik.mpp") apply false + id("scientifik.publish") apply false +} + val dataforgeVersion by extra("0.1.3-dev-7") val bintrayRepo by extra("dataforge") diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts deleted file mode 100644 index d0036af4..00000000 --- a/buildSrc/build.gradle.kts +++ /dev/null @@ -1,33 +0,0 @@ -plugins { - `kotlin-dsl` -} - -repositories { - gradlePluginPortal() - jcenter() -} - -val kotlinVersion = "1.3.40" - -// Add plugins used in buildSrc as dependencies, also we should specify version only here -dependencies { - implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion") - implementation("org.jfrog.buildinfo:build-info-extractor-gradle:4.9.6") - implementation("com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.4") - implementation("org.jetbrains.dokka:dokka-gradle-plugin:0.9.18") - implementation("com.moowork.gradle:gradle-node-plugin:1.3.1") - implementation("org.openjfx:javafx-plugin:0.0.7") -} - -gradlePlugin{ - plugins { - create("scientifik-publish") { - id = "scientifik.publish" - implementationClass = "ScientifikPublishPlugin" - } - create("scientifik-mpp"){ - id = "scientifik.mpp" - implementationClass = "ScientifikMPPlugin" - } - } -} diff --git a/buildSrc/settings.gradle.kts b/buildSrc/settings.gradle.kts deleted file mode 100644 index e69de29b..00000000 diff --git a/buildSrc/src/main/kotlin/Scientifik.kt b/buildSrc/src/main/kotlin/Scientifik.kt deleted file mode 100644 index 617b2e67..00000000 --- a/buildSrc/src/main/kotlin/Scientifik.kt +++ /dev/null @@ -1,9 +0,0 @@ -/** - * Build constants - */ -object Scientifik { - val ioVersion = "0.1.10" - val coroutinesVersion = "1.2.2" - val atomicfuVersion = "0.12.9" - val serializationVersion = "0.11.1" -} diff --git a/buildSrc/src/main/kotlin/ScientifikMPPlugin.kt b/buildSrc/src/main/kotlin/ScientifikMPPlugin.kt deleted file mode 100644 index e34c675b..00000000 --- a/buildSrc/src/main/kotlin/ScientifikMPPlugin.kt +++ /dev/null @@ -1,79 +0,0 @@ -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.kotlin.dsl.configure -import org.gradle.kotlin.dsl.getValue -import org.gradle.kotlin.dsl.getting -import org.gradle.kotlin.dsl.invoke -import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension - -open class ScientifikMPPlugin : Plugin { - override fun apply(project: Project) { - project.plugins.apply("org.jetbrains.kotlin.multiplatform") - - project.configure { - jvm { - compilations.all { - kotlinOptions { - jvmTarget = "1.8" - } - } - } - - js { - compilations.all { - kotlinOptions { - sourceMap = true - sourceMapEmbedSources = "always" - moduleKind = "umd" - } - } - } - - sourceSets.invoke { - val commonMain by getting { - dependencies { - api(kotlin("stdlib")) - } - } - val commonTest by getting { - dependencies { - implementation(kotlin("test-common")) - implementation(kotlin("test-annotations-common")) - } - } - val jvmMain by getting { - dependencies { - api(kotlin("stdlib-jdk8")) - } - } - val jvmTest by getting { - dependencies { - implementation(kotlin("test")) - implementation(kotlin("test-junit")) - } - } - val jsMain by getting { - dependencies { - api(kotlin("stdlib-js")) - } - } - val jsTest by getting { - dependencies { - implementation(kotlin("test-js")) - } - } - } - - targets.all { - sourceSets.all { - languageSettings.apply{ - progressiveMode = true - enableLanguageFeature("InlineClasses") - useExperimentalAnnotation("ExperimentalUnsignedType") - } - } - } - } - - } -} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/ScientifikPublishPlugin.kt b/buildSrc/src/main/kotlin/ScientifikPublishPlugin.kt deleted file mode 100644 index c94751a9..00000000 --- a/buildSrc/src/main/kotlin/ScientifikPublishPlugin.kt +++ /dev/null @@ -1,237 +0,0 @@ -import com.jfrog.bintray.gradle.BintrayExtension -import com.jfrog.bintray.gradle.tasks.BintrayUploadTask -import groovy.lang.GroovyObject -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.plugins.JavaBasePlugin -import org.gradle.api.publish.PublishingExtension -import org.gradle.api.publish.maven.MavenPublication -import org.gradle.api.publish.maven.internal.artifact.FileBasedMavenArtifact -import org.gradle.api.tasks.bundling.Jar -import org.gradle.kotlin.dsl.* -import org.jetbrains.dokka.gradle.DokkaTask -import org.jetbrains.kotlin.gradle.dsl.KotlinJvmProjectExtension -import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension -import org.jfrog.gradle.plugin.artifactory.dsl.ArtifactoryPluginConvention -import org.jfrog.gradle.plugin.artifactory.dsl.PublisherConfig -import org.jfrog.gradle.plugin.artifactory.dsl.ResolverConfig -import org.jfrog.gradle.plugin.artifactory.task.ArtifactoryTask - - -open class ScientifikExtension { - var vcs: String? = null - var bintrayRepo: String? = null - var kdoc: Boolean = true -} - -// recursively search up the project chain for configuration -private val Project.bintrayRepo: String? - get() = extensions.findByType()?.bintrayRepo - ?: parent?.bintrayRepo - ?: (findProperty("bintrayRepo") as? String) - -private val Project.vcs: String? - get() = extensions.findByType()?.vcs - ?: parent?.vcs - ?: (findProperty("vcs") as? String) - -open class ScientifikPublishPlugin : Plugin { - - override fun apply(project: Project) { - - project.plugins.apply("maven-publish") - val extension = project.extensions.create("scientifik") - - val bintrayRepo = project.bintrayRepo - val vcs = project.vcs - - if (bintrayRepo == null || vcs == null) { - project.logger.warn("[${project.name}] Missing deployment configuration. Skipping publish.") - } - - project.configure { - repositories { - maven("https://bintray.com/mipt-npm/$bintrayRepo") - } - - // Process each publication we have in this project - publications.filterIsInstance().forEach { publication -> - - @Suppress("UnstableApiUsage") - publication.pom { - name.set(project.name) - description.set(project.description) - url.set(vcs) - - licenses { - license { - name.set("The Apache Software License, Version 2.0") - url.set("http://www.apache.org/licenses/LICENSE-2.0.txt") - distribution.set("repo") - } - } - developers { - developer { - id.set("MIPT-NPM") - name.set("MIPT nuclear physics methods laboratory") - organization.set("MIPT") - organizationUrl.set("http://npm.mipt.ru") - } - - } - scm { - url.set(extension.vcs) - } - } - } - } - - if (extension.kdoc) { - project.plugins.apply("org.jetbrains.dokka") - - project.afterEvaluate { - extensions.findByType()?.apply { - val dokka by tasks.getting(DokkaTask::class) { - outputFormat = "html" - outputDirectory = "$buildDir/javadoc" - jdkVersion = 8 - - kotlinTasks { - // dokka fails to retrieve sources from MPP-tasks so we only define the jvm task - listOf(tasks.getByPath("compileKotlinJvm")) - } - sourceRoot { - // assuming only single source dir - path = sourceSets["commonMain"].kotlin.srcDirs.first().toString() - platforms = listOf("Common") - } - // although the JVM sources are now taken from the task, - // we still define the jvm source root to get the JVM marker in the generated html - sourceRoot { - // assuming only single source dir - path = sourceSets["jvmMain"].kotlin.srcDirs.first().toString() - platforms = listOf("JVM") - } - - } - - val kdocJar by tasks.registering(Jar::class) { - group = JavaBasePlugin.DOCUMENTATION_GROUP - dependsOn(dokka) - archiveClassifier.set("javadoc") - from("$buildDir/javadoc") - } - - configure { - - targets.all { - val publication = publications.findByName(name) as MavenPublication - - // Patch publications with fake javadoc - publication.artifact(kdocJar.get()) - } - - tasks.filter { it is ArtifactoryTask || it is BintrayUploadTask }.forEach { - it.doFirst { - publications.filterIsInstance() - .forEach { publication -> - val moduleFile = - buildDir.resolve("publications/${publication.name}/module.json") - if (moduleFile.exists()) { - publication.artifact(object : FileBasedMavenArtifact(moduleFile) { - override fun getDefaultExtension() = "module" - }) - } - } - } - } - } - } - - - extensions.findByType()?.apply { - val dokka by tasks.getting(DokkaTask::class) { - outputFormat = "html" - outputDirectory = "$buildDir/javadoc" - jdkVersion = 8 - } - - val kdocJar by tasks.registering(Jar::class) { - group = JavaBasePlugin.DOCUMENTATION_GROUP - dependsOn(dokka) - archiveClassifier.set("javadoc") - from("$buildDir/javadoc") - } - - configure { - publications.filterIsInstance().forEach { publication -> - publication.artifact(kdocJar.get()) - } - } - } - } - } - - project.plugins.apply("com.jfrog.bintray") - - project.configure { - user = project.findProperty("bintrayUser") as? String ?: System.getenv("BINTRAY_USER") - key = project.findProperty("bintrayApiKey") as? String? ?: System.getenv("BINTRAY_API_KEY") - publish = true - override = true // for multi-platform Kotlin/Native publishing - - // We have to use delegateClosureOf because bintray supports only dynamic groovy syntax - // this is a problem of this plugin - pkg.apply { - userOrg = "mipt-npm" - repo = bintrayRepo - name = project.name - issueTrackerUrl = "${extension.vcs}/issues" - setLicenses("Apache-2.0") - vcsUrl = extension.vcs - version.apply { - name = project.version.toString() - vcsTag = project.version.toString() - released = java.util.Date().toString() - } - } - - //workaround bintray bug - project.afterEvaluate { - setPublications(*project.extensions.findByType()!!.publications.names.toTypedArray()) - } - -// project.tasks.figetByPath("bintrayUpload") { -// dependsOn(publishToMavenLocal) -// } - } - - project.plugins.apply("com.jfrog.artifactory") - - project.configure { - val artifactoryUser: String? by project - val artifactoryPassword: String? by project - val artifactoryContextUrl = "http://npm.mipt.ru:8081/artifactory" - - setContextUrl(artifactoryContextUrl)//The base Artifactory URL if not overridden by the publisher/resolver - publish(delegateClosureOf { - repository(delegateClosureOf { - setProperty("repoKey", "gradle-dev-local") - setProperty("username", artifactoryUser) - setProperty("password", artifactoryPassword) - }) - - defaults(delegateClosureOf { - invokeMethod("publications", arrayOf("jvm", "js", "kotlinMultiplatform", "metadata")) - }) - }) - resolve(delegateClosureOf { - repository(delegateClosureOf { - setProperty("repoKey", "gradle-dev") - setProperty("username", artifactoryUser) - setProperty("password", artifactoryPassword) - }) - }) - } - } -} \ No newline at end of file diff --git a/dataforge-meta/build.gradle.kts b/dataforge-meta/build.gradle.kts index 3fd7b1e0..6f2a5160 100644 --- a/dataforge-meta/build.gradle.kts +++ b/dataforge-meta/build.gradle.kts @@ -2,8 +2,4 @@ plugins { id("scientifik.mpp") } -description = "Meta definition and basic operations on meta" - -scientifik{ - -} \ No newline at end of file +description = "Meta definition and basic operations on meta" \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 05b41d36..73726216 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -3,6 +3,7 @@ pluginManagement { jcenter() gradlePluginPortal() maven("https://dl.bintray.com/kotlin/kotlin-eap") + maven("https://dl.bintray.com/mipt-npm/scientifik") } resolutionStrategy { eachPlugin { @@ -11,6 +12,7 @@ pluginManagement { "kotlin-multiplatform" -> useModule("org.jetbrains.kotlin:kotlin-gradle-plugin:${requested.version}") "kotlin2js" -> useModule("org.jetbrains.kotlin:kotlin-gradle-plugin:${requested.version}") "org.jetbrains.kotlin.frontend" -> useModule("org.jetbrains.kotlin:kotlin-frontend-plugin:0.0.45") + "scientifik.mpp", "scientifik.publish" -> useModule("scientifik:gradle-tools:0.1.0") } } } From 1722a7dd82c5916d2d48d32a1547347ff8392408 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 30 Jun 2019 10:34:15 +0300 Subject: [PATCH 27/48] Slightly modified plugin lookup --- build.gradle.kts | 2 +- .../hep/dataforge/context/PluginManager.kt | 22 +++++++++++-------- .../hep/dataforge/output/OutputManager.kt | 7 ++++-- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 3b494f3d..4fda7bec 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,7 +3,7 @@ plugins { id("scientifik.publish") apply false } -val dataforgeVersion by extra("0.1.3-dev-7") +val dataforgeVersion by extra("0.1.3-dev-8") val bintrayRepo by extra("dataforge") val vcs by extra("https://github.com/mipt-npm/dataforge-core") diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt index bcce8eb6..7c6175b3 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt @@ -51,7 +51,10 @@ class PluginManager(override val context: Context) : ContextAware, Iterable get(type: KClass, recursive: Boolean = true): T? = - get(recursive) { type.isInstance(it) } as T? + operator fun get(type: KClass, tag: PluginTag? = null, recursive: Boolean = true): T? = + get(recursive) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) } as T? - inline fun get(recursive: Boolean = true): T? = get(T::class, recursive) + inline fun get(tag: PluginTag? = null, recursive: Boolean = true): T? = + get(T::class, tag, recursive) /** @@ -75,7 +79,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable load(plugin: T): T { if (context.isActive) error("Can't load plugin into active context") - if (get(plugin::class, false) != null) { + if (get(plugin::class, recursive = false) != null) { throw RuntimeException("Plugin of type ${plugin::class} already exists in ${context.name}") } else { loadDependencies(plugin) @@ -115,13 +119,13 @@ class PluginManager(override val context: Context) : ContextAware, Iterable load(PluginRepository.fetch(tag,meta)) + loaded == null -> load(PluginRepository.fetch(tag, meta)) loaded.meta == meta -> loaded // if meta is the same, return existing plugin else -> throw RuntimeException("Can't load plugin with tag $tag. Plugin with this tag and different configuration already exists in context.") } } - fun load(factory: PluginFactory<*>, meta: Meta = EmptyMeta): Plugin{ + fun load(factory: PluginFactory<*>, meta: Meta = EmptyMeta): Plugin { val loaded = get(factory.tag, false) return when { loaded == null -> load(factory(meta)) @@ -135,7 +139,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable load(type: KClass, meta: Meta = EmptyMeta): T { - val loaded = get(type, false) + val loaded = get(type, recursive = false) return when { loaded == null -> { val plugin = PluginRepository.list().first { it.type == type }.invoke(meta) @@ -165,7 +169,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable getOrLoad(noinline metaBuilder: MetaBuilder.() -> Unit = {}): T { - return get(true) ?: load(metaBuilder) + return get(recursive = true) ?: load(metaBuilder) } } \ No newline at end of file diff --git a/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt b/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt index 47d9f59c..448b3adb 100644 --- a/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt +++ b/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt @@ -1,6 +1,9 @@ package hep.dataforge.output -import hep.dataforge.context.* +import hep.dataforge.context.AbstractPlugin +import hep.dataforge.context.Context +import hep.dataforge.context.PluginFactory +import hep.dataforge.context.PluginTag import hep.dataforge.context.PluginTag.Companion.DATAFORGE_GROUP import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta @@ -13,7 +16,7 @@ import kotlin.reflect.KClass /** * A manager for outputs */ -interface OutputManager : Plugin { +interface OutputManager { /** * Get an output specialized for given type, name and stage. From 167dc752ea481a6ed47a988fc0019d4c1efc92aa Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 30 Jun 2019 14:13:45 +0300 Subject: [PATCH 28/48] Few more updates for context builder --- .../commonMain/kotlin/hep/dataforge/context/Context.kt | 10 +++++++--- .../kotlin/hep/dataforge/context/ContextBuilder.kt | 8 ++++++-- .../kotlin/hep/dataforge/context/PluginManager.kt | 3 ++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt index 8d355173..1309eea8 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt @@ -148,14 +148,18 @@ object Global : Context("GLOBAL", null) { private val contextRegistry = HashMap() /** - * Get previously builder context o builder a new one + * Get previously built context * * @param name * @return */ - fun getContext(name: String): Context { - return contextRegistry.getOrPut(name) { Context(name) } + fun getContext(name: String): Context? { + return contextRegistry[name] } + + fun context(name: String, parent: Context = this, block: ContextBuilder.() -> Unit): Context = + ContextBuilder(name, parent).apply(block).build() + } diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt index 203edd2c..92840862 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt @@ -18,11 +18,15 @@ class ContextBuilder(var name: String = "@anonimous", val parent: Context = Glob plugins.add(plugin) } - fun plugin(tag: PluginTag, action: MetaBuilder.() -> Unit) { + fun plugin(tag: PluginTag, action: MetaBuilder.() -> Unit = {}) { plugins.add(PluginRepository.fetch(tag, buildMeta(action))) } - fun plugin(name: String, group: String = "", version: String = "", action: MetaBuilder.() -> Unit) { + fun plugin(builder: PluginFactory<*>, action: MetaBuilder.() -> Unit = {}) { + plugins.add(builder.invoke(buildMeta(action))) + } + + fun plugin(name: String, group: String = "", version: String = "", action: MetaBuilder.() -> Unit = {}) { plugin(PluginTag(name, group, version), action) } diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt index 7c6175b3..96c64308 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt @@ -142,7 +142,8 @@ class PluginManager(override val context: Context) : ContextAware, Iterable { - val plugin = PluginRepository.list().first { it.type == type }.invoke(meta) + val plugin = PluginRepository.list().find { it.type == type }?.invoke(meta) + ?: error("Plugin factory for class $type not found") if (type.isInstance(plugin)) { @Suppress("UNCHECKED_CAST") load(plugin as T) From 5b053d60a2e0d272c47ff93874b8d15a63fc2071 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 2 Jul 2019 13:17:41 +0300 Subject: [PATCH 29/48] optimize plugin loading API --- .../hep/dataforge/context/PluginManager.kt | 81 +++++-------------- 1 file changed, 22 insertions(+), 59 deletions(-) diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt index 96c64308..dad483a8 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt @@ -71,7 +71,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable load(plugin: T): T { if (context.isActive) error("Can't load plugin into active context") - if (get(plugin::class, recursive = false) != null) { - throw RuntimeException("Plugin of type ${plugin::class} already exists in ${context.name}") + if (get(plugin::class, plugin.tag, recursive = false) != null) { + error("Plugin of type ${plugin::class} already exists in ${context.name}") } else { - loadDependencies(plugin) + for (tag in plugin.dependsOn()) { + fetch(tag, true) + } logger.info { "Loading plugin ${plugin.name} into ${context.name}" } plugin.attach(context) @@ -91,11 +93,14 @@ class PluginManager(override val context: Context) : ContextAware, Iterable load(factory: PluginFactory, meta: Meta = EmptyMeta): T = + load(factory(meta)) + + fun load(factory: PluginFactory, metaBuilder: MetaBuilder.() -> Unit): T = + load(factory, buildMeta(metaBuilder)) /** * Remove a plugin from [PluginManager] @@ -111,22 +116,11 @@ class PluginManager(override val context: Context) : ContextAware, Iterable load(PluginRepository.fetch(tag, meta)) - loaded.meta == meta -> loaded // if meta is the same, return existing plugin - else -> throw RuntimeException("Can't load plugin with tag $tag. Plugin with this tag and different configuration already exists in context.") - } - } - - fun load(factory: PluginFactory<*>, meta: Meta = EmptyMeta): Plugin { - val loaded = get(factory.tag, false) + fun fetch(factory: PluginFactory, recursive: Boolean = true, meta: Meta = EmptyMeta): T { + val loaded = get(factory.type, factory.tag, recursive) return when { loaded == null -> load(factory(meta)) loaded.meta == meta -> loaded // if meta is the same, return existing plugin @@ -134,43 +128,12 @@ class PluginManager(override val context: Context) : ContextAware, Iterable load(type: KClass, meta: Meta = EmptyMeta): T { - val loaded = get(type, recursive = false) - return when { - loaded == null -> { - val plugin = PluginRepository.list().find { it.type == type }?.invoke(meta) - ?: error("Plugin factory for class $type not found") - if (type.isInstance(plugin)) { - @Suppress("UNCHECKED_CAST") - load(plugin as T) - } else { - error("Corrupt type information in plugin repository") - } - } - loaded.meta == meta -> loaded // if meta is the same, return existing plugin - else -> throw RuntimeException("Can't load plugin with type $type. Plugin with this type and different configuration already exists in context.") - } - } - - inline fun load(noinline metaBuilder: MetaBuilder.() -> Unit = {}): T { - return load(T::class, buildMeta(metaBuilder)) - } - - fun load(name: String, meta: Meta = EmptyMeta): Plugin { - return load(PluginTag.fromString(name), meta) - } + fun fetch( + factory: PluginFactory, + recursive: Boolean = true, + metaBuilder: MetaBuilder.() -> Unit + ): T = fetch(factory, recursive, buildMeta(metaBuilder)) override fun iterator(): Iterator = plugins.iterator() - /** - * Get a plugin if it exists or load it with given meta if it is not. - */ - inline fun getOrLoad(noinline metaBuilder: MetaBuilder.() -> Unit = {}): T { - return get(recursive = true) ?: load(metaBuilder) - } - } \ No newline at end of file From da4a9ebe9d3b0440979b9c0a56111c5209ea1d23 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 5 Jul 2019 18:49:36 +0300 Subject: [PATCH 30/48] Simplified Provider API --- build.gradle.kts | 2 +- .../hep/dataforge/context/AbstractPlugin.kt | 4 +-- .../kotlin/hep/dataforge/context/Context.kt | 18 +++------- .../kotlin/hep/dataforge/provider/Provider.kt | 33 ++++++------------- .../hep/dataforge/context/ContextTest.kt | 15 +++------ .../kotlin/hep/dataforge/provider/Types.kt | 4 +-- .../dataforge/workspace/SimpleWorkspace.kt | 1 - .../hep/dataforge/workspace/Workspace.kt | 32 ++++-------------- .../dataforge/workspace/WorkspacePlugin.kt | 17 +++------- 9 files changed, 32 insertions(+), 94 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 4fda7bec..00c7dfea 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,7 +3,7 @@ plugins { id("scientifik.publish") apply false } -val dataforgeVersion by extra("0.1.3-dev-8") +val dataforgeVersion by extra("0.1.3-dev-9") val bintrayRepo by extra("dataforge") val vcs by extra("https://github.com/mipt-npm/dataforge-core") diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt index 9a091ad1..25f10232 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt @@ -18,7 +18,5 @@ abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin { this._context = null } - override fun provideTop(target: String, name: Name): Any? = null - - override fun listNames(target: String): Sequence = emptySequence() + override fun provideTop(target: String): Map = emptyMap() } \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt index 1309eea8..9db32a01 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt @@ -59,19 +59,11 @@ open class Context(final override val name: String, val parent: Context? = Globa override val defaultTarget: String get() = Plugin.PLUGIN_TARGET - override fun provideTop(target: String, name: Name): Any? { - return when (target) { - Plugin.PLUGIN_TARGET -> plugins[PluginTag.fromString(name.toString())] - Value.TYPE -> properties[name]?.value - else -> null - } - } - - override fun listNames(target: String): Sequence { - return when (target) { - Plugin.PLUGIN_TARGET -> plugins.asSequence().map { it.name.toName() } - Value.TYPE -> properties.values().map { it.first } - else -> emptySequence() + override fun provideTop(target: String): Map { + return when(target){ + Value.TYPE -> properties.sequence().toMap() + Plugin.PLUGIN_TARGET -> plugins.sequence(true).associateBy { it.name.toName() } + else-> emptyMap() } } diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Provider.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Provider.kt index 35f1aca0..b1d769e2 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Provider.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/provider/Provider.kt @@ -17,7 +17,6 @@ package hep.dataforge.provider import hep.dataforge.names.Name import hep.dataforge.names.toName -import kotlin.jvm.JvmName /** * A marker utility interface for providers. @@ -42,29 +41,21 @@ interface Provider { /** - * Provide a top level element for this [Provider] or return null if element is not present + * A map of direct children for specific target */ - fun provideTop(target: String, name: Name): Any? - - /** - * [Sequence] of available names with given target. Only top level names are listed, no chain path. - * - * @param target - * @return - */ - fun listNames(target: String): Sequence + fun provideTop(target: String): Map } fun Provider.provide(path: Path, targetOverride: String? = null): Any? { if (path.length == 0) throw IllegalArgumentException("Can't provide by empty path") val first = path.first() - val top = provideTop(targetOverride ?: first.target ?: defaultTarget, first.name) + val target = targetOverride ?: first.target ?: defaultTarget + val res = provideTop(target)[first.name] ?: return null return when (path.length) { - 1 -> top + 1 -> res else -> { - when (top) { - null -> null - is Provider -> top.provide(path.tail!!, targetOverride = defaultChainTarget) + when (res) { + is Provider -> res.provide(path.tail!!, targetOverride = defaultChainTarget) else -> throw IllegalStateException("Chain path not supported: child is not a provider") } } @@ -85,16 +76,12 @@ inline fun Provider.provide(target: String, name: Name): T? { inline fun Provider.provide(target: String, name: String): T? = provide(target, name.toName()) - -fun Provider.top(target: String): Map = top(target) - /** - * A top level content with names + * Typed top level content */ -@JvmName("typedTop") inline fun Provider.top(target: String): Map { - return listNames(target).associate { - it to (provideTop(target, it) as? T ?: error("The element $it is declared but not provided")) + return provideTop(target).mapValues { + it.value as? T ?: error("The type of element $it is ${it::class} but ${T::class} is expected") } } diff --git a/dataforge-context/src/commonTest/kotlin/hep/dataforge/context/ContextTest.kt b/dataforge-context/src/commonTest/kotlin/hep/dataforge/context/ContextTest.kt index 1d37c69e..c77439d6 100644 --- a/dataforge-context/src/commonTest/kotlin/hep/dataforge/context/ContextTest.kt +++ b/dataforge-context/src/commonTest/kotlin/hep/dataforge/context/ContextTest.kt @@ -12,17 +12,10 @@ class ContextTest { class DummyPlugin : AbstractPlugin() { override val tag get() = PluginTag("test") - override fun provideTop(target: String, name: Name): Any? { - return when (target) { - "test" -> return name - else -> super.provideTop(target, name) - } - } - - override fun listNames(target: String): Sequence { - return when (target) { - "test" -> sequenceOf("a", "b", "c.d").map { it.toName() } - else -> super.listNames(target) + override fun provideTop(target: String): Map { + return when(target){ + "test" -> listOf("a", "b", "c.d").associate { it.toName() to it.toName() } + else -> emptyMap() } } } diff --git a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/Types.kt b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/Types.kt index d6ed723d..dfe81ce0 100644 --- a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/Types.kt +++ b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/Types.kt @@ -34,9 +34,7 @@ inline fun Provider.provideByType(name: Name): T? { inline fun Provider.top(): Map { val target = Types[T::class] - return listNames(target).associate { name -> - name to (provideByType(name) ?: error("The element $name is declared but not provided")) - } + return top(target) } /** diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt index d707545d..68438440 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt @@ -7,7 +7,6 @@ import hep.dataforge.data.DataNode import hep.dataforge.meta.Meta import hep.dataforge.names.Name import hep.dataforge.names.toName -import hep.dataforge.provider.top /** diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt index 1945c74e..58fe081e 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt @@ -29,27 +29,16 @@ interface Workspace : ContextAware, Provider { */ val tasks: Map> - override fun provideTop(target: String, name: Name): Any? { + override fun provideTop(target: String): Map { return when (target) { - "target", Meta.TYPE -> targets[name.toString()] - Task.TYPE -> tasks[name] - Data.TYPE -> data[name] - DataNode.TYPE -> data.getNode(name) - else -> null + "target", Meta.TYPE -> targets.mapKeys { it.key.toName() } + Task.TYPE -> tasks + Data.TYPE -> data.data().toMap() + DataNode.TYPE -> data.nodes().toMap() + else -> emptyMap() } } - override fun listNames(target: String): Sequence { - return when (target) { - "target", Meta.TYPE -> targets.keys.asSequence().map { it.toName() } - Task.TYPE -> tasks.keys.asSequence().map { it } - Data.TYPE -> data.data().map { it.first } - DataNode.TYPE -> data.nodes().map { it.first } - else -> emptySequence() - } - } - - /** * Invoke a task in the workspace utilizing caching if possible */ @@ -64,15 +53,6 @@ interface Workspace : ContextAware, Provider { } } -// /** -// * Invoke a task in the workspace utilizing caching if possible -// */ -// operator fun Task.invoke(targetName: String): DataNode { -// val target = targets[targetName] ?: error("A target with name $targetName not found in ${this@Workspace}") -// context.logger.info { "Running ${this.name} on $target" } -// return invoke(target) -// } - companion object { const val TYPE = "workspace" } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt index 7f808166..2fefd4f8 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspacePlugin.kt @@ -10,19 +10,10 @@ import hep.dataforge.names.toName abstract class WorkspacePlugin : AbstractPlugin() { abstract val tasks: Collection> - override fun provideTop(target: String, name: Name): Any? { - return if (target == Task.TYPE) { - tasks.find { it.name == name.toString() } - } else { - super.provideTop(target, name) - } - } - - override fun listNames(target: String): Sequence { - return if (target == Task.TYPE) { - tasks.asSequence().map { it.name.toName() } - } else { - return super.listNames(target) + override fun provideTop(target: String): Map { + return when(target){ + Task.TYPE -> tasks.associateBy { it.name.toName() } + else -> emptyMap() } } } \ No newline at end of file From efddfa8e91a9cd2862edd17df7150e6e7d6d951a Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 27 Jul 2019 17:02:48 +0300 Subject: [PATCH 31/48] Meta and io fixes --- build.gradle.kts | 13 ++--- .../kotlin/hep/dataforge/context/Context.kt | 2 +- dataforge-io/build.gradle.kts | 47 ++++--------------- .../kotlin/hep/dataforge/io/FunctionServer.kt | 38 +++++++++++++++ .../kotlin/hep/dataforge/io/MetaFormat.kt | 15 ++++-- .../kotlin/hep/dataforge/io/MetaFormatTest.kt | 15 ++++-- .../kotlin/hep/dataforge/meta/Config.kt | 39 +++++++++++++++ .../kotlin/hep/dataforge/meta/Laminate.kt | 10 ++++ .../kotlin/hep/dataforge/meta/Meta.kt | 16 +++++-- .../hep/dataforge/meta/MetaTransformation.kt | 2 +- .../kotlin/hep/dataforge/meta/MutableMeta.kt | 36 ++------------ .../kotlin/hep/dataforge/meta/Styled.kt | 4 +- .../kotlin/hep/dataforge/values/Value.kt | 9 ++++ .../hep/dataforge/values/exoticValues.kt | 4 +- settings.gradle.kts | 8 ++-- 15 files changed, 159 insertions(+), 99 deletions(-) create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/FunctionServer.kt diff --git a/build.gradle.kts b/build.gradle.kts index 00c7dfea..9b042061 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,19 +1,14 @@ plugins { - id("scientifik.mpp") apply false - id("scientifik.publish") apply false + id("scientifik.mpp") version "0.1.4-dev" apply false + id("scientifik.publish") version "0.1.4-dev" apply false } -val dataforgeVersion by extra("0.1.3-dev-9") +val dataforgeVersion by extra("0.1.3-dev-10") val bintrayRepo by extra("dataforge") -val vcs by extra("https://github.com/mipt-npm/dataforge-core") +val githubProject by extra("dataforge-core") allprojects { - repositories { - jcenter() - maven("https://kotlin.bintray.com/kotlinx") - } - group = "hep.dataforge" version = dataforgeVersion } diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt index 9db32a01..21b8ebe6 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt @@ -149,7 +149,7 @@ object Global : Context("GLOBAL", null) { return contextRegistry[name] } - fun context(name: String, parent: Context = this, block: ContextBuilder.() -> Unit): Context = + fun context(name: String, parent: Context = this, block: ContextBuilder.() -> Unit = {}): Context = ContextBuilder(name, parent).apply(block).build() } diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index 6ed06857..b65783c6 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -2,56 +2,25 @@ plugins { id("scientifik.mpp") } -description = "IO for meta" +description = "IO module" +scientifik{ + serialization = true + io = true +} -val ioVersion: String = Scientifik.ioVersion -val serializationVersion: String = Scientifik.serializationVersion kotlin { - jvm() - js() sourceSets { val commonMain by getting{ dependencies { api(project(":dataforge-meta")) - //implementation 'org.jetbrains.kotlin:kotlin-reflect' - api("org.jetbrains.kotlinx:kotlinx-serialization-runtime-common:$serializationVersion") - api("org.jetbrains.kotlinx:kotlinx-io:$ioVersion") } } - val commonTest by getting { - dependencies { - implementation("org.jetbrains.kotlin:kotlin-test-common") - implementation("org.jetbrains.kotlin:kotlin-test-annotations-common") + val jsMain by getting{ + dependencies{ + api(npm("text-encoding")) } } - val jvmMain by getting { - dependencies { - api("org.jetbrains.kotlinx:kotlinx-serialization-runtime:$serializationVersion") - api("org.jetbrains.kotlinx:kotlinx-io-jvm:$ioVersion") - } - } - val jvmTest by getting { - dependencies { - implementation("org.jetbrains.kotlin:kotlin-test") - implementation("org.jetbrains.kotlin:kotlin-test-junit") - } - } - val jsMain by getting { - dependencies { - api("org.jetbrains.kotlinx:kotlinx-serialization-runtime-js:$serializationVersion") - api("org.jetbrains.kotlinx:kotlinx-io-js:$ioVersion") - } - } - val jsTest by getting { - dependencies { - implementation("org.jetbrains.kotlin:kotlin-test-js") - } - } -// iosMain { -// } -// iosTest { -// } } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/FunctionServer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/FunctionServer.kt new file mode 100644 index 00000000..2b254716 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/FunctionServer.kt @@ -0,0 +1,38 @@ +package hep.dataforge.io + +import kotlin.reflect.KClass + +/** + * A descriptor for specific type of functions + */ +interface FunctionSpec { + val inputType: KClass + val outputType: KClass +} + +/** + * A server that could produce asynchronous function values + */ +interface FunctionServer { + /** + * Call a function with given name and descriptor + */ + suspend fun > call(name: String, descriptor: D, arg: T): R + + /** + * Resolve a function descriptor for given types + */ + fun resolveType(inputType: KClass, outputType: KClass): FunctionSpec + + /** + * Get a generic suspended function with given name and descriptor + */ + operator fun > get(name: String, descriptor: D): (suspend (T) -> R) = + { call(name, descriptor, it) } +} + +suspend inline fun FunctionServer.call(name: String, arg: T): R = + call(name, resolveType(T::class, R::class), arg) + +inline operator fun FunctionServer.get(name: String): (suspend (T) -> R) = + get(name, resolveType(T::class, R::class)) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index 52f01525..4e1f6dec 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -13,14 +13,21 @@ interface MetaFormat : IOFormat { val key: Short } -fun Meta.asString(format: MetaFormat = JsonMetaFormat): String { - return buildPacket { - format.run { writeObject(this@asString) } - }.readText() +fun Meta.toString(format: MetaFormat = JsonMetaFormat): String = buildPacket { + format.run { writeObject(this@toString) } +}.readText() + +fun Meta.toBytes(format: MetaFormat = JsonMetaFormat): ByteReadPacket = buildPacket { + format.run { writeObject(this@toBytes) } } + fun MetaFormat.parse(str: String): Meta { return ByteReadPacket(str.toByteArray()).readObject() } +fun MetaFormat.fromBytes(packet: ByteReadPacket): Meta { + return packet.readObject() +} + diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt index e177e4d8..3788a0ef 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt @@ -1,6 +1,7 @@ package hep.dataforge.io import hep.dataforge.meta.buildMeta +import hep.dataforge.meta.get import kotlin.test.Test import kotlin.test.assertEquals @@ -12,10 +13,11 @@ class MetaFormatTest { "node" to { "b" to "DDD" "c" to 11.1 + "array" to doubleArrayOf(1.0, 2.0, 3.0) } } - val string = meta.asString(BinaryMetaFormat) - val result = BinaryMetaFormat.parse(string) + val bytes = meta.toBytes(BinaryMetaFormat) + val result = BinaryMetaFormat.fromBytes(bytes) assertEquals(meta, result) } @@ -26,11 +28,16 @@ class MetaFormatTest { "node" to { "b" to "DDD" "c" to 11.1 - "array" to doubleArrayOf(1.0,2.0,3.0) + "array" to doubleArrayOf(1.0, 2.0, 3.0) } } - val string = meta.asString(JsonMetaFormat) + val string = meta.toString(JsonMetaFormat) val result = JsonMetaFormat.parse(string) + + meta.items.keys.forEach { + if (meta[it] != result[it]) error("${meta[it]} != ${result[it]}") + } + assertEquals(meta, result) } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt index 3ea2a39e..07ce8f96 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt @@ -1,7 +1,9 @@ package hep.dataforge.meta +import hep.dataforge.names.Name import hep.dataforge.names.NameToken import hep.dataforge.names.asName +import hep.dataforge.names.plus //TODO add validator to configuration @@ -10,6 +12,43 @@ import hep.dataforge.names.asName */ class Config : AbstractMutableMeta() { + private val listeners = HashSet() + + private fun itemChanged(name: Name, oldItem: MetaItem<*>?, newItem: MetaItem<*>?) { + listeners.forEach { it.action(name, oldItem, newItem) } + } + + /** + * Add change listener to this meta. Owner is declared to be able to remove listeners later. Listener without owner could not be removed + */ + fun onChange(owner: Any?, action: (Name, MetaItem<*>?, MetaItem<*>?) -> Unit) { + listeners.add(MetaListener(owner, action)) + } + + /** + * Remove all listeners belonging to given owner + */ + fun removeListener(owner: Any?) { + listeners.removeAll { it.owner === owner } + } + + override fun replaceItem(key: NameToken, oldItem: MetaItem?, newItem: MetaItem?) { + if (newItem == null) { + _items.remove(key) + if(oldItem!= null && oldItem is MetaItem.NodeItem) { + oldItem.node.removeListener(this) + } + } else { + _items[key] = newItem + if (newItem is MetaItem.NodeItem) { + newItem.node.onChange(this) { name, oldChild, newChild -> + itemChanged(key + name, oldChild, newChild) + } + } + } + itemChanged(key.asName(), oldItem, newItem) + } + /** * Attach configuration node instead of creating one */ diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt index 28240908..d8be0806 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt @@ -79,4 +79,14 @@ class Laminate(layers: List) : Meta { } } +/** + * Create a new [Laminate] adding given layer to the top + */ +fun Laminate.withTop(meta: Meta): Laminate = Laminate(listOf(meta) + layers) + +/** + * Create a new [Laminate] adding given layer to the bottom + */ +fun Laminate.withBottom(meta: Meta): Laminate = Laminate(layers + meta) + //TODO add custom rules for Laminate merge diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 153beab1..9e71023c 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -15,8 +15,16 @@ import hep.dataforge.values.boolean * * a [NodeItem] (node) */ sealed class MetaItem { - data class ValueItem(val value: Value) : MetaItem() - data class NodeItem(val node: M) : MetaItem() + data class ValueItem(val value: Value) : MetaItem(){ + override fun toString(): String = value.string + } + data class NodeItem(val node: M) : MetaItem(){ + override fun toString(): String = node.toString() + + override fun equals(other: Any?): Boolean { + return this.node == (other as? NodeItem<*>).node + } + } } /** @@ -174,12 +182,14 @@ abstract class AbstractMetaNode> : MetaNode { if (this === other) return true if (other !is Meta) return false - return this.items == other.items + return this.items == other.items//this.items.keys == other.items.keys && items.keys.all { this[it] == other[it] } } override fun hashCode(): Int { return items.hashCode() } + + override fun toString(): String = items.toString() } /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt index 3d0e50e4..8deada19 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaTransformation.kt @@ -110,7 +110,7 @@ inline class MetaTransformation(val transformations: Collection> bind(source: MutableMeta<*>, target: M) { + fun > bind(source: Config, target: M) { source.onChange(target) { name, _, newItem -> transformations.forEach { t -> if (t.matches(name, newItem)) { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt index 2be5c1a0..b9ab6f62 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt @@ -12,8 +12,8 @@ internal data class MetaListener( interface MutableMeta> : MetaNode { override val items: Map> operator fun set(name: Name, item: MetaItem<*>?) - fun onChange(owner: Any? = null, action: (Name, MetaItem<*>?, MetaItem<*>?) -> Unit) - fun removeListener(owner: Any? = null) +// fun onChange(owner: Any? = null, action: (Name, MetaItem<*>?, MetaItem<*>?) -> Unit) +// fun removeListener(owner: Any? = null) } /** @@ -22,46 +22,20 @@ interface MutableMeta> : MetaNode { * Changes in Meta are not thread safe. */ abstract class AbstractMutableMeta> : AbstractMetaNode(), MutableMeta { - private val listeners = HashSet() - - /** - * Add change listener to this meta. Owner is declared to be able to remove listeners later. Listener without owner could not be removed - */ - override fun onChange(owner: Any?, action: (Name, MetaItem<*>?, MetaItem<*>?) -> Unit) { - listeners.add(MetaListener(owner, action)) - } - - /** - * Remove all listeners belonging to given owner - */ - override fun removeListener(owner: Any?) { - listeners.removeAll { it.owner === owner } - } - - private val _items: MutableMap> = HashMap() + protected val _items: MutableMap> = HashMap() override val items: Map> get() = _items - protected fun itemChanged(name: Name, oldItem: MetaItem<*>?, newItem: MetaItem<*>?) { - listeners.forEach { it.action(name, oldItem, newItem) } - } + //protected abstract fun itemChanged(name: Name, oldItem: MetaItem<*>?, newItem: MetaItem<*>?) protected open fun replaceItem(key: NameToken, oldItem: MetaItem?, newItem: MetaItem?) { if (newItem == null) { _items.remove(key) - if(oldItem!= null && oldItem is MetaItem.NodeItem) { - oldItem.node.removeListener(this) - } } else { _items[key] = newItem - if (newItem is MetaItem.NodeItem) { - newItem.node.onChange(this) { name, oldChild, newChild -> - itemChanged(key + name, oldChild, newChild) - } - } } - itemChanged(key.asName(), oldItem, newItem) + //itemChanged(key.asName(), oldItem, newItem) } @Suppress("UNCHECKED_CAST") diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt index c34d933f..55d652aa 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Styled.kt @@ -42,12 +42,12 @@ class Styled(val base: Meta, val style: Config = Config().empty()) : AbstractMut } } - override fun onChange(owner: Any?, action: (Name, before: MetaItem<*>?, after: MetaItem<*>?) -> Unit) { + fun onChange(owner: Any?, action: (Name, before: MetaItem<*>?, after: MetaItem<*>?) -> Unit) { //TODO test correct behavior style.onChange(owner) { name, before, after -> action(name, before ?: base[name], after ?: base[name]) } } - override fun removeListener(owner: Any?) { + fun removeListener(owner: Any?) { style.removeListener(owner) } } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt index ac651c20..140178c2 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt @@ -43,6 +43,8 @@ interface Value { val list: List get() = listOf(this) + override fun equals(other: Any?): Boolean + companion object { const val TYPE = "value" @@ -82,6 +84,8 @@ object Null : Value { override val string: String get() = "@null" override fun toString(): String = value.toString() + + override fun equals(other: Any?): Boolean = other === Null } /** @@ -100,6 +104,9 @@ object True : Value { override val string: String get() = "true" override fun toString(): String = value.toString() + + override fun equals(other: Any?): Boolean = other === True + } /** @@ -112,6 +119,8 @@ object False : Value { override val string: String get() = "false" override fun toString(): String = True.value.toString() + + override fun equals(other: Any?): Boolean = other === False } val Value.boolean get() = this == True || this.list.firstOrNull() == True || (type == ValueType.STRING && string.toBoolean()) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt index 9f13d73a..8ebc1c6b 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt @@ -12,9 +12,11 @@ class LazyParsedValue(override val string: String) : Value { override val number: Number get() = parsedValue.number override fun toString(): String = string + + override fun equals(other: Any?): Boolean = other is Value && this.parsedValue == other } -fun String.lazyParseValue(): LazyParsedValue = LazyParsedValue(this) +fun String.lazyParseValue(): LazyParsedValue = LazyParsedValue(this) /** * A performance optimized version of list value for doubles diff --git a/settings.gradle.kts b/settings.gradle.kts index 73726216..011123b9 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,18 +1,18 @@ pluginManagement { repositories { + mavenLocal() jcenter() gradlePluginPortal() maven("https://dl.bintray.com/kotlin/kotlin-eap") + maven("https://dl.bintray.com/kotlin/kotlinx") maven("https://dl.bintray.com/mipt-npm/scientifik") + maven("https://dl.bintray.com/mipt-npm/dev") } resolutionStrategy { eachPlugin { when (requested.id.id) { "kotlinx-atomicfu" -> useModule("org.jetbrains.kotlinx:atomicfu-gradle-plugin:${requested.version}") - "kotlin-multiplatform" -> useModule("org.jetbrains.kotlin:kotlin-gradle-plugin:${requested.version}") - "kotlin2js" -> useModule("org.jetbrains.kotlin:kotlin-gradle-plugin:${requested.version}") - "org.jetbrains.kotlin.frontend" -> useModule("org.jetbrains.kotlin:kotlin-frontend-plugin:0.0.45") - "scientifik.mpp", "scientifik.publish" -> useModule("scientifik:gradle-tools:0.1.0") + "scientifik.mpp", "scientifik.publish" -> useModule("scientifik:gradle-tools:${requested.version}") } } } From 9f6d53f214782597780dbdd91caee33cbf75c649 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 27 Jul 2019 17:45:14 +0300 Subject: [PATCH 32/48] Some additional fixes --- .../src/commonMain/kotlin/hep/dataforge/io/Binary.kt | 1 - .../commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt | 4 ++++ .../src/commonMain/kotlin/hep/dataforge/meta/Meta.kt | 6 +----- .../src/commonMain/kotlin/hep/dataforge/values/Value.kt | 2 +- .../kotlin/hep/dataforge/output/html/HtmlOutput.kt | 3 +-- .../kotlin/hep/dataforge/scripting/Placeholder.kt | 4 ++++ 6 files changed, 11 insertions(+), 9 deletions(-) rename dataforge-output-html/src/{jvmMain => commonMain}/kotlin/hep/dataforge/output/html/HtmlOutput.kt (93%) create mode 100644 dataforge-scripting/src/commonMain/kotlin/hep/dataforge/scripting/Placeholder.kt diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt index 75ef0422..a0374968 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt @@ -7,7 +7,6 @@ import kotlinx.io.core.readBytes /** * A source of binary data */ - interface Binary { /** * The size of binary in bytes diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt index 3788a0ef..ab4ab677 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt @@ -1,7 +1,9 @@ package hep.dataforge.io +import hep.dataforge.meta.Meta import hep.dataforge.meta.buildMeta import hep.dataforge.meta.get +import hep.dataforge.meta.seal import kotlin.test.Test import kotlin.test.assertEquals @@ -34,6 +36,8 @@ class MetaFormatTest { val string = meta.toString(JsonMetaFormat) val result = JsonMetaFormat.parse(string) + assertEquals(meta, meta.seal()) + meta.items.keys.forEach { if (meta[it] != result[it]) error("${meta[it]} != ${result[it]}") } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 9e71023c..1687b66a 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -16,14 +16,10 @@ import hep.dataforge.values.boolean */ sealed class MetaItem { data class ValueItem(val value: Value) : MetaItem(){ - override fun toString(): String = value.string + override fun toString(): String = value.toString() } data class NodeItem(val node: M) : MetaItem(){ override fun toString(): String = node.toString() - - override fun equals(other: Any?): Boolean { - return this.node == (other as? NodeItem<*>).node - } } } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt index 140178c2..7b42f639 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt @@ -188,7 +188,7 @@ class ListValue(override val list: List) : Value { override val number: Number get() = list.first().number override val string: String get() = list.first().string - override fun toString(): String = value.toString() + override fun toString(): String = list.joinToString (prefix = "[ ", postfix = " ]") override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/dataforge-output-html/src/jvmMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt b/dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt similarity index 93% rename from dataforge-output-html/src/jvmMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt rename to dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt index bfbd74eb..c7aea610 100644 --- a/dataforge-output-html/src/jvmMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt +++ b/dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt @@ -27,8 +27,7 @@ class HtmlOutput(override val context: Context, private val consumer: T } else { val value = cache[obj::class] if (value == null) { - val answer = context.top>().values - .filter { it.type.isInstance(obj) }.firstOrNull() + val answer = context.top>(HTML_CONVERTER_TYPE).values.firstOrNull { it.type.isInstance(obj) } if (answer != null) { cache[obj::class] = answer answer diff --git a/dataforge-scripting/src/commonMain/kotlin/hep/dataforge/scripting/Placeholder.kt b/dataforge-scripting/src/commonMain/kotlin/hep/dataforge/scripting/Placeholder.kt new file mode 100644 index 00000000..a09e35c1 --- /dev/null +++ b/dataforge-scripting/src/commonMain/kotlin/hep/dataforge/scripting/Placeholder.kt @@ -0,0 +1,4 @@ +package hep.dataforge.scripting + +internal object Placeholder { +} \ No newline at end of file From c82eda28d81c1a2e2cecde59d8098849b8b05bc9 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 31 Jul 2019 16:22:14 +0300 Subject: [PATCH 33/48] Minor updates for mutable meta and builder --- build.gradle.kts | 4 ++-- .../kotlin/hep/dataforge/meta/Config.kt | 5 +++++ .../kotlin/hep/dataforge/meta/MetaBuilder.kt | 20 +++++++++++++++++++ .../kotlin/hep/dataforge/meta/MutableMeta.kt | 6 ------ 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 9b042061..ecaee7a6 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,6 +1,6 @@ plugins { - id("scientifik.mpp") version "0.1.4-dev" apply false - id("scientifik.publish") version "0.1.4-dev" apply false + id("scientifik.mpp") version "0.1.4" apply false + id("scientifik.publish") version "0.1.4" apply false } val dataforgeVersion by extra("0.1.3-dev-10") diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt index 07ce8f96..f47d3bcd 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Config.kt @@ -7,6 +7,11 @@ import hep.dataforge.names.plus //TODO add validator to configuration +data class MetaListener( + val owner: Any? = null, + val action: (name: Name, oldItem: MetaItem<*>?, newItem: MetaItem<*>?) -> Unit +) + /** * Mutable meta representing object state */ diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt index 0b5c83d6..f79fed07 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt @@ -1,5 +1,6 @@ package hep.dataforge.meta +import hep.dataforge.names.Name import hep.dataforge.names.asName import hep.dataforge.values.Value @@ -28,6 +29,25 @@ class MetaBuilder : AbstractMutableMeta() { infix fun String.to(metaBuilder: MetaBuilder.() -> Unit) { this@MetaBuilder[this] = MetaBuilder().apply(metaBuilder) } + + infix fun Name.to(value: Any) { + if (value is Meta) { + this@MetaBuilder[this] = value + } + this@MetaBuilder[this] = Value.of(value) + } + + infix fun Name.to(meta: Meta) { + this@MetaBuilder[this] = meta + } + + infix fun Name.to(value: Iterable) { + this@MetaBuilder[this] = value.toList() + } + + infix fun Name.to(metaBuilder: MetaBuilder.() -> Unit) { + this@MetaBuilder[this] = MetaBuilder().apply(metaBuilder) + } } /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt index b9ab6f62..285edd89 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt @@ -3,12 +3,6 @@ package hep.dataforge.meta import hep.dataforge.names.* import hep.dataforge.values.Value -internal data class MetaListener( - val owner: Any? = null, - val action: (name: Name, oldItem: MetaItem<*>?, newItem: MetaItem<*>?) -> Unit -) - - interface MutableMeta> : MetaNode { override val items: Map> operator fun set(name: Name, item: MetaItem<*>?) From a729d27d1c939ebe13b98183dc7743973d88dbe1 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 2 Aug 2019 15:42:45 +0300 Subject: [PATCH 34/48] Meta serialization update --- .../hep/dataforge/io/BinaryMetaFormat.kt | 5 +- .../kotlin/hep/dataforge/io/JsonMetaFormat.kt | 98 +++++++++++++------ .../kotlin/hep/dataforge/io/MetaFormat.kt | 14 ++- .../kotlin/hep/dataforge/io/MetaSerializer.kt | 38 +++++++ .../kotlin/hep/dataforge/meta/Meta.kt | 3 + 5 files changed, 121 insertions(+), 37 deletions(-) create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt index 941b4abc..aa4be898 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt @@ -1,5 +1,6 @@ package hep.dataforge.io +import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.meta.* import hep.dataforge.values.* import kotlinx.io.core.Input @@ -11,7 +12,7 @@ object BinaryMetaFormat : MetaFormat { override val name: String = "bin" override val key: Short = 0x4249//BI - override fun Input.readObject(): Meta { + override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { return (readMetaItem() as MetaItem.NodeItem).node } @@ -69,7 +70,7 @@ object BinaryMetaFormat : MetaFormat { } } - override fun Output.writeObject(meta: Meta) { + override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) { writeChar('M') writeInt(meta.items.size) meta.items.forEach { (key, item) -> diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index 7815b69c..de2cf925 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -1,5 +1,8 @@ package hep.dataforge.io +import hep.dataforge.descriptors.ItemDescriptor +import hep.dataforge.descriptors.NodeDescriptor +import hep.dataforge.descriptors.ValueDescriptor import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaItem import hep.dataforge.names.NameToken @@ -10,6 +13,9 @@ import kotlinx.io.core.Output import kotlinx.io.core.readText import kotlinx.io.core.writeText import kotlinx.serialization.json.* +import kotlin.collections.component1 +import kotlin.collections.component2 +import kotlin.collections.set object JsonMetaFormat : MetaFormat { @@ -17,12 +23,12 @@ object JsonMetaFormat : MetaFormat { override val name: String = "json" override val key: Short = 0x4a53//"JS" - override fun Output.writeObject(obj: Meta) { - val str = obj.toJson().toString() + override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) { + val str = meta.toJson().toString() writeText(str) } - override fun Input.readObject(): Meta { + override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { val str = readText() val json = Json.plain.parseJson(str) @@ -34,8 +40,8 @@ object JsonMetaFormat : MetaFormat { } } -fun Value.toJson(): JsonElement { - return if(isList()){ +fun Value.toJson(descriptor: ValueDescriptor? = null): JsonElement { + return if (isList()) { JsonArray(list.map { it.toJson() }) } else { when (type) { @@ -47,22 +53,35 @@ fun Value.toJson(): JsonElement { } } -fun Meta.toJson(): JsonObject { - val map = this.items.mapValues { entry -> - when (val value = entry.value) { - is MetaItem.ValueItem -> value.value.toJson() - is MetaItem.NodeItem -> value.node.toJson() +//Use theese methods to customize JSON key mapping +private fun NameToken.toJsonKey(descriptor: ItemDescriptor?) = toString() +private fun NodeDescriptor?.getDescriptor(key: String) = this?.items?.get(key) + +fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject { + + //TODO search for same name siblings and arrange them into arrays + val map = this.items.entries.associate {(name,item)-> + val itemDescriptor = descriptor?.items?.get(name.body) + val key = name.toJsonKey(itemDescriptor) + val value = when (item) { + is MetaItem.ValueItem -> { + item.value.toJson(itemDescriptor as? ValueDescriptor) + } + is MetaItem.NodeItem -> { + item.node.toJson(itemDescriptor as? NodeDescriptor) + } } - }.mapKeys { it.key.toString() } + key to value + } return JsonObject(map) } -fun JsonObject.toMeta() = JsonMeta(this) +fun JsonObject.toMeta(descriptor: NodeDescriptor? = null) = JsonMeta(this, descriptor) -class JsonMeta(val json: JsonObject) : Meta { +class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : Meta { - private fun JsonPrimitive.toValue(): Value { + private fun JsonPrimitive.toValue(descriptor: ValueDescriptor?): Value { return when (this) { JsonNull -> Null else -> this.content.parseValue() // Optimize number and boolean parsing @@ -70,25 +89,40 @@ class JsonMeta(val json: JsonObject) : Meta { } @Suppress("UNCHECKED_CAST") - private operator fun MutableMap>.set(key: String, value: JsonElement) = when (value) { - is JsonPrimitive -> this[key] = MetaItem.ValueItem(value.toValue()) as MetaItem - is JsonObject -> this[key] = MetaItem.NodeItem(value.toMeta()) - is JsonArray -> { - when { - value.all { it is JsonPrimitive } -> { - val listValue = ListValue( - value.map { - //We already checked that all values are primitives - (it as JsonPrimitive).toValue() + private operator fun MutableMap>.set(key: String, value: JsonElement): Unit { + val itemDescriptor = descriptor.getDescriptor(key) + //use name from descriptor in case descriptor name differs from json key + val name = itemDescriptor?.name ?: key + return when (value) { + is JsonPrimitive -> { + this[name] = MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor)) as MetaItem + } + is JsonObject -> { + this[name] = MetaItem.NodeItem(value.toMeta(itemDescriptor as? NodeDescriptor)) + } + is JsonArray -> { + when { + value.all { it is JsonPrimitive } -> { + val listValue = ListValue( + value.map { + //We already checked that all values are primitives + (it as JsonPrimitive).toValue(itemDescriptor as? ValueDescriptor) + } + ) + this[name] = MetaItem.ValueItem(listValue) as MetaItem + } + else -> value.forEachIndexed { index, jsonElement -> + when (jsonElement) { + is JsonObject -> { + this["$name[$index]"] = + MetaItem.NodeItem(jsonElement.toMeta(itemDescriptor as? NodeDescriptor)) + } + is JsonPrimitive -> { + this["$name[$index]"] = + MetaItem.ValueItem(jsonElement.toValue(itemDescriptor as? ValueDescriptor)) + } + is JsonArray -> TODO("Nested arrays not supported") } - ) - this[key] = MetaItem.ValueItem(listValue) as MetaItem - } - else -> value.forEachIndexed { index, jsonElement -> - when (jsonElement) { - is JsonObject -> this["$key[$index]"] = MetaItem.NodeItem(JsonMeta(jsonElement)) - is JsonPrimitive -> this["$key[$index]"] = MetaItem.ValueItem(jsonElement.toValue()) as MetaItem - is JsonArray -> TODO("Nested arrays not supported") } } } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index 4e1f6dec..27bb921c 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -1,9 +1,8 @@ package hep.dataforge.io +import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.meta.Meta -import kotlinx.io.core.ByteReadPacket -import kotlinx.io.core.buildPacket -import kotlinx.io.core.toByteArray +import kotlinx.io.core.* /** * A format for meta serialization @@ -11,6 +10,15 @@ import kotlinx.io.core.toByteArray interface MetaFormat : IOFormat { val name: String val key: Short + + override fun Output.writeObject(obj: Meta) { + writeMeta(obj, null) + } + + override fun Input.readObject(): Meta = readMeta(null) + + fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) + fun Input.readMeta(descriptor: NodeDescriptor?): Meta } fun Meta.toString(format: MetaFormat = JsonMetaFormat): String = buildPacket { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt new file mode 100644 index 00000000..b935e0b2 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt @@ -0,0 +1,38 @@ +package hep.dataforge.io + +import hep.dataforge.meta.Config +import hep.dataforge.meta.Meta +import hep.dataforge.meta.toConfig +import kotlinx.serialization.Decoder +import kotlinx.serialization.Encoder +import kotlinx.serialization.KSerializer +import kotlinx.serialization.SerialDescriptor +import kotlinx.serialization.json.JsonObjectSerializer + +/** + * Serialized for meta + */ +object MetaSerializer : KSerializer { + override val descriptor: SerialDescriptor = JsonObjectSerializer.descriptor + + override fun deserialize(decoder: Decoder): Meta { + //currently just delegates serialization to json serializer + return JsonObjectSerializer.deserialize(decoder).toMeta() + } + + override fun serialize(encoder: Encoder, obj: Meta) { + JsonObjectSerializer.serialize(encoder, obj.toJson()) + } +} + +object ConfigSerializer: KSerializer{ + override val descriptor: SerialDescriptor = JsonObjectSerializer.descriptor + + override fun deserialize(decoder: Decoder): Config { + return JsonObjectSerializer.deserialize(decoder).toMeta().toConfig() + } + + override fun serialize(encoder: Encoder, obj: Config) { + JsonObjectSerializer.serialize(encoder, obj.toJson()) + } +} \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 1687b66a..e89eb950 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -39,6 +39,9 @@ interface MetaRepr { * * Same name siblings are supported via elements with the same [Name] but different queries */ interface Meta : MetaRepr { + /** + * Top level items of meta tree + */ val items: Map> override fun toMeta(): Meta = this From 44737faa2691b1970b241a68caf3e439f8ffed95 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 7 Aug 2019 11:38:25 +0300 Subject: [PATCH 35/48] A lot of minor fixes --- dataforge-context/build.gradle.kts | 6 +- .../hep/dataforge/context/AbstractPlugin.kt | 5 ++ .../kotlin/hep/dataforge/context/Context.kt | 12 ++-- .../kotlin/hep/dataforge/data/Goal.kt | 10 +-- .../hep/dataforge/data/{_Data.kt => data.kt} | 0 dataforge-io/build.gradle.kts | 2 +- .../kotlin/hep/dataforge/io/Envelope.kt | 18 ++++- .../kotlin/hep/dataforge/io/IOPlugin.kt | 39 ++++++++++ .../kotlin/hep/dataforge/io/MetaFormat.kt | 16 +++-- .../kotlin/hep/dataforge/io/MetaSerializer.kt | 25 +++++-- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 34 +++++---- .../kotlin/hep/dataforge/io/FileBinary.kt | 8 +-- .../kotlin/hep/dataforge/io/FileEnvelope.kt | 24 +++++++ .../kotlin/hep/dataforge/meta/Laminate.kt | 2 +- .../kotlin/hep/dataforge/meta/MetaBuilder.kt | 2 +- .../kotlin/hep/dataforge/names/Name.kt | 51 ++++++++++--- .../kotlin/hep/dataforge/names/NameTest.kt | 15 +++- dataforge-workspace/build.gradle.kts | 1 + .../hep/dataforge/workspace/fileData.kt | 72 +++++++++++++++++++ 19 files changed, 288 insertions(+), 54 deletions(-) rename dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/{_Data.kt => data.kt} (100%) create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt create mode 100644 dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt create mode 100644 dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt diff --git a/dataforge-context/build.gradle.kts b/dataforge-context/build.gradle.kts index f550b58a..896e7b89 100644 --- a/dataforge-context/build.gradle.kts +++ b/dataforge-context/build.gradle.kts @@ -12,20 +12,20 @@ kotlin { dependencies { api(project(":dataforge-meta")) api(kotlin("reflect")) - api("io.github.microutils:kotlin-logging-common:1.6.10") + api("io.github.microutils:kotlin-logging-common:1.7.2") api("org.jetbrains.kotlinx:kotlinx-coroutines-core-common:$coroutinesVersion") } } val jvmMain by getting { dependencies { - api("io.github.microutils:kotlin-logging:1.6.10") + api("io.github.microutils:kotlin-logging:1.7.2") api("ch.qos.logback:logback-classic:1.2.3") api("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion") } } val jsMain by getting { dependencies { - api("io.github.microutils:kotlin-logging-js:1.6.10") + api("io.github.microutils:kotlin-logging-js:1.7.2") api("org.jetbrains.kotlinx:kotlinx-coroutines-core-js:$coroutinesVersion") } } diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt index 25f10232..c9268790 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt @@ -3,6 +3,7 @@ package hep.dataforge.context import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.names.Name +import hep.dataforge.names.toName abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin { private var _context: Context? = null @@ -19,4 +20,8 @@ abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin { } override fun provideTop(target: String): Map = emptyMap() + + companion object{ + fun Collection.toMap(): Map = associate { it.name.toName() to it } + } } \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt index 21b8ebe6..80746456 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt @@ -26,8 +26,10 @@ import kotlin.jvm.JvmName * Since plugins could contain mutable state, context has two states: active and inactive. No changes are allowed to active context. * @author Alexander Nozik */ -open class Context(final override val name: String, val parent: Context? = Global) : Named, MetaRepr, Provider, - CoroutineScope { +open class Context( + final override val name: String, + val parent: Context? = Global +) : Named, MetaRepr, Provider, CoroutineScope { private val config = Config() @@ -60,10 +62,10 @@ open class Context(final override val name: String, val parent: Context? = Globa override val defaultTarget: String get() = Plugin.PLUGIN_TARGET override fun provideTop(target: String): Map { - return when(target){ + return when (target) { Value.TYPE -> properties.sequence().toMap() Plugin.PLUGIN_TARGET -> plugins.sequence(true).associateBy { it.name.toName() } - else-> emptyMap() + else -> emptyMap() } } @@ -111,7 +113,7 @@ open class Context(final override val name: String, val parent: Context? = Globa fun Context.content(target: String): Map = content(target) /** - * A sequences of all objects provided by plugins with given target and type + * A map of all objects provided by plugins with given target and type */ @JvmName("typedContent") inline fun Context.content(target: String): Map = diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt index 991ddbdb..128832c5 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt @@ -77,8 +77,8 @@ private class StaticGoalImpl(override val scope: CoroutineScope, deferred: Co * * **Important:** Unlike regular deferred, the [Goal] is started lazily, so the actual calculation is called only when result is requested. */ -fun CoroutineScope.createGoal( - dependencies: Collection>, +fun CoroutineScope.goal( + dependencies: Collection> = emptyList(), context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.() -> R ): Goal { @@ -102,7 +102,7 @@ fun CoroutineScope.createGoal( fun Goal.pipe( context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(T) -> R -): Goal = createGoal(listOf(this), context) { block(await()) } +): Goal = goal(listOf(this), context) { block(await()) } /** * Create a joining goal. @@ -112,7 +112,7 @@ fun Collection>.join( scope: CoroutineScope = first(), context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(Collection) -> R -): Goal = scope.createGoal(this, context) { +): Goal = scope.goal(this, context) { block(map { it.await() }) } @@ -126,6 +126,6 @@ fun Map>.join( scope: CoroutineScope = values.first(), context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(Map) -> R -): Goal = scope.createGoal(this.values, context) { +): Goal = scope.goal(this.values, context) { block(mapValues { it.value.await() }) } \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_Data.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/data.kt similarity index 100% rename from dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_Data.kt rename to dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/data.kt diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index b65783c6..3ed7ab62 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -14,7 +14,7 @@ kotlin { sourceSets { val commonMain by getting{ dependencies { - api(project(":dataforge-meta")) + api(project(":dataforge-context")) } } val jsMain by getting{ diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt index fc3ae90d..08fe44e5 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt @@ -1,9 +1,12 @@ package hep.dataforge.io +import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.string +import hep.dataforge.provider.Type import kotlinx.io.core.Input +import kotlinx.io.core.Output interface Envelope { val meta: Meta @@ -46,4 +49,17 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str */ val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string -typealias EnvelopeFormat = IOFormat \ No newline at end of file +data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?) + +@Type(ENVELOPE_FORMAT_TYPE) +interface EnvelopeFormat : IOFormat{ + fun readPartial(input: Input): PartialEnvelope + + fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat) + + override fun Output.writeObject(obj: Envelope) = writeEnvelope(obj, JsonMetaFormat) + + companion object { + const val ENVELOPE_FORMAT_TYPE = "envelopeFormat" + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt new file mode 100644 index 00000000..6416e5c4 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt @@ -0,0 +1,39 @@ +package hep.dataforge.io + +import hep.dataforge.context.AbstractPlugin +import hep.dataforge.context.PluginFactory +import hep.dataforge.context.PluginTag +import hep.dataforge.context.content +import hep.dataforge.meta.Meta +import hep.dataforge.names.Name +import hep.dataforge.names.asName +import kotlin.reflect.KClass + +class IOPlugin(meta: Meta) : AbstractPlugin(meta) { + override val tag: PluginTag get() = Companion.tag + + val metaFormats by lazy { + context.content(MetaFormat.META_FORMAT_TYPE).values + } + + fun metaFormat(key: Short): MetaFormat? = metaFormats.find { it.key == key } + fun metaFormat(name: String): MetaFormat? = metaFormats.find { it.name == name } + + override fun provideTop(target: String): Map { + return when (target) { + MetaFormat.META_FORMAT_TYPE -> internalMetaFormats + EnvelopeFormat.ENVELOPE_FORMAT_TYPE -> mapOf( + TaggedEnvelopeFormat.VERSION.asName() to TaggedEnvelopeFormat(metaFormats) + ) + else -> super.provideTop(target) + } + } + + companion object : PluginFactory { + private val internalMetaFormats = listOf(JsonMetaFormat, BinaryMetaFormat).toMap() + + override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP) + override val type: KClass = IOPlugin::class + override fun invoke(meta: Meta): IOPlugin = IOPlugin(meta) + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index 27bb921c..f658ca32 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -1,14 +1,18 @@ package hep.dataforge.io +import hep.dataforge.context.Named import hep.dataforge.descriptors.NodeDescriptor +import hep.dataforge.io.MetaFormat.Companion.META_FORMAT_TYPE import hep.dataforge.meta.Meta +import hep.dataforge.provider.Type import kotlinx.io.core.* /** * A format for meta serialization */ -interface MetaFormat : IOFormat { - val name: String +@Type(META_FORMAT_TYPE) +interface MetaFormat : IOFormat, Named { + override val name: String val key: Short override fun Output.writeObject(obj: Meta) { @@ -17,8 +21,12 @@ interface MetaFormat : IOFormat { override fun Input.readObject(): Meta = readMeta(null) - fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) - fun Input.readMeta(descriptor: NodeDescriptor?): Meta + fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor? = null) + fun Input.readMeta(descriptor: NodeDescriptor? = null): Meta + + companion object{ + const val META_FORMAT_TYPE = "metaFormat" + } } fun Meta.toString(format: MetaFormat = JsonMetaFormat): String = buildPacket { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt index b935e0b2..e1a754eb 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt @@ -3,15 +3,29 @@ package hep.dataforge.io import hep.dataforge.meta.Config import hep.dataforge.meta.Meta import hep.dataforge.meta.toConfig -import kotlinx.serialization.Decoder -import kotlinx.serialization.Encoder -import kotlinx.serialization.KSerializer -import kotlinx.serialization.SerialDescriptor +import hep.dataforge.names.Name +import hep.dataforge.names.toName +import kotlinx.serialization.* +import kotlinx.serialization.internal.StringDescriptor import kotlinx.serialization.json.JsonObjectSerializer +@Serializer(Name::class) +object NameSerializer : KSerializer { + override val descriptor: SerialDescriptor = StringDescriptor + + override fun deserialize(decoder: Decoder): Name { + return decoder.decodeString().toName() + } + + override fun serialize(encoder: Encoder, obj: Name) { + encoder.encodeString(obj.toString()) + } +} + /** * Serialized for meta */ +@Serializer(Meta::class) object MetaSerializer : KSerializer { override val descriptor: SerialDescriptor = JsonObjectSerializer.descriptor @@ -25,7 +39,8 @@ object MetaSerializer : KSerializer { } } -object ConfigSerializer: KSerializer{ +@Serializer(Config::class) +object ConfigSerializer : KSerializer { override val descriptor: SerialDescriptor = JsonObjectSerializer.descriptor override fun deserialize(decoder: Decoder): Config { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index a993620e..dbbe9c58 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -2,14 +2,10 @@ package hep.dataforge.io import kotlinx.io.core.* -@ExperimentalUnsignedTypes -class TaggedEnvelopeFormat( - val metaFormats: Collection, - val outputMetaFormat: MetaFormat = metaFormats.first() -) : EnvelopeFormat { +class TaggedEnvelopeFormat(val metaFormats: Collection) : EnvelopeFormat { - override fun Output.writeObject(obj: Envelope) { - write(obj, this, outputMetaFormat) + override fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat) { + write(this, envelope, format) } /** @@ -20,6 +16,7 @@ class TaggedEnvelopeFormat( */ override fun Input.readObject(): Envelope = read(this, metaFormats) + override fun readPartial(input: Input): PartialEnvelope = Companion.readPartial(input, metaFormats) private data class Tag( val metaFormatKey: Short, @@ -28,9 +25,10 @@ class TaggedEnvelopeFormat( ) companion object { - private const val VERSION = "DF03" + const val VERSION = "DF03" private const val START_SEQUENCE = "#~" private const val END_SEQUENCE = "~#\r\n" + private const val TAG_SIZE = 26u private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) { writeText(START_SEQUENCE) @@ -66,12 +64,24 @@ class TaggedEnvelopeFormat( return SimpleEnvelope(meta, ArrayBinary(dataBytes)) } - fun write(obj: Envelope, out: Output, metaFormat: MetaFormat) { - val metaBytes = metaFormat.writeBytes(obj.meta) - val tag = Tag(metaFormat.key, metaBytes.size.toUInt(), obj.data?.size ?: 0.toULong()) + fun readPartial(input: Input, metaFormats: Collection): PartialEnvelope { + val tag = input.readTag() + + val metaFormat = metaFormats.find { it.key == tag.metaFormatKey } + ?: error("Meta format with key ${tag.metaFormatKey} not found") + + val metaPacket = ByteReadPacket(input.readBytes(tag.metaSize.toInt())) + val meta = metaFormat.run { metaPacket.readObject() } + + return PartialEnvelope(meta, TAG_SIZE + tag.metaSize, tag.dataSize) + } + + fun write(out: Output, envelope: Envelope, metaFormat: MetaFormat) { + val metaBytes = metaFormat.writeBytes(envelope.meta) + val tag = Tag(metaFormat.key, metaBytes.size.toUInt(), envelope.data?.size ?: 0.toULong()) out.writePacket(tag.toBytes()) out.writeFully(metaBytes) - obj.data?.read { copyTo(out) } + envelope.data?.read { copyTo(out) } } } } \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt index db74e8e0..e68d002d 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt @@ -7,15 +7,13 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption -class FileBinary(val path: Path, private val offset: Int = 0) : RandomAccessBinary { - - override val size: ULong - get() = (Files.size(path) - offset).toULong() +class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = null) : RandomAccessBinary { + override val size: ULong = size ?: (Files.size(path).toULong() - offset).toULong() override fun read(from: UInt, size: UInt, block: Input.() -> R): R { FileChannel.open(path, StandardOpenOption.READ).use { - val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from.toLong() + offset), size.toLong()) + val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from + offset).toLong(), size.toLong()) return ByteReadPacket(buffer).block() } } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt new file mode 100644 index 00000000..de926db8 --- /dev/null +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt @@ -0,0 +1,24 @@ +package hep.dataforge.io + +import hep.dataforge.meta.Meta +import kotlinx.io.nio.asInput +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption + +class FileEnvelope internal constructor(val path: Path, val format: EnvelopeFormat) : Envelope { + //TODO do not like this constructor. Hope to replace it later + + private val partialEnvelope: PartialEnvelope + + init { + val input = Files.newByteChannel(path, StandardOpenOption.READ).asInput() + partialEnvelope = format.readPartial(input) + } + + override val meta: Meta get() = partialEnvelope.meta + + override val data: Binary? = FileBinary(path, partialEnvelope.dataOffset, partialEnvelope.dataSize) +} + +fun Path.readEnvelope(format: EnvelopeFormat) = FileEnvelope(this,format) \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt index d8be0806..6f7c41c8 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt @@ -17,7 +17,7 @@ class Laminate(layers: List) : Meta { } } - constructor(vararg layers: Meta) : this(layers.asList()) + constructor(vararg layers: Meta?) : this(layers.filterNotNull()) override val items: Map> get() = layers.map { it.items.keys }.flatten().associateWith { key -> diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt index f79fed07..322b660a 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt @@ -8,7 +8,7 @@ import hep.dataforge.values.Value * DSL builder for meta. Is not intended to store mutable state */ class MetaBuilder : AbstractMutableMeta() { - override fun wrapNode(meta: Meta): MetaBuilder = meta.builder() + override fun wrapNode(meta: Meta): MetaBuilder = if (meta is MetaBuilder) meta else meta.builder() override fun empty(): MetaBuilder = MetaBuilder() infix fun String.to(value: Any) { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt index 504c07d2..40502c8c 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt @@ -51,32 +51,55 @@ data class NameToken(val body: String, val index: String = "") { if (body.isEmpty()) error("Syntax error: Name token body is empty") } + private fun String.escape() = + replace("\\", "\\\\") + .replace(".", "\\.") + .replace("[", "\\[") + .replace("]", "\\]") + override fun toString(): String = if (hasIndex()) { - "$body[$index]" + "${body.escape()}[$index]" } else { - body + body.escape() } fun hasIndex() = index.isNotEmpty() } +/** + * Convert a [String] to name parsing it and extracting name tokens and index syntax. + * This operation is rather heavy so it should be used with care in high performance code. + */ fun String.toName(): Name { if (isBlank()) return EmptyName val tokens = sequence { var bodyBuilder = StringBuilder() var queryBuilder = StringBuilder() var bracketCount: Int = 0 + var escape: Boolean = false fun queryOn() = bracketCount > 0 - asSequence().forEach { - if (queryOn()) { - when (it) { - '[' -> bracketCount++ - ']' -> bracketCount-- + for (it in this@toName) { + when { + escape -> { + if (queryOn()) { + queryBuilder.append(it) + } else { + bodyBuilder.append(it) + } + escape = false } - if (queryOn()) queryBuilder.append(it) - } else { - when (it) { + it == '\\' -> { + escape = true + } + queryOn() -> { + when (it) { + '[' -> bracketCount++ + ']' -> bracketCount-- + } + if (queryOn()) queryBuilder.append(it) + } + else -> when (it) { '.' -> { yield(NameToken(bodyBuilder.toString(), queryBuilder.toString())) bodyBuilder = StringBuilder() @@ -96,6 +119,14 @@ fun String.toName(): Name { return Name(tokens.toList()) } +/** + * Convert the [String] to a [Name] by simply wrapping it in a single name token without parsing. + * The input string could contain dots and braces, but they are just escaped, not parsed. + */ +fun String.asName(): Name { + return NameToken(this).asName() +} + operator fun NameToken.plus(other: Name): Name = Name(listOf(this) + other.tokens) operator fun Name.plus(other: Name): Name = Name(this.tokens + other.tokens) diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameTest.kt index a3116040..bcd75154 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameTest.kt @@ -1,6 +1,9 @@ package hep.dataforge.names -import kotlin.test.* +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue class NameTest { @Test @@ -25,4 +28,14 @@ class NameTest { assertTrue { name1.endsWith(name3) } assertFalse { name1.startsWith(name3) } } + + @Test + fun escapeTest(){ + val escapedName = "token\\.one.token2".toName() + val unescapedName = "token\\.one.token2".asName() + + assertEquals(2, escapedName.length) + assertEquals(1, unescapedName.length) + assertEquals(escapedName, escapedName.toString().toName()) + } } \ No newline at end of file diff --git a/dataforge-workspace/build.gradle.kts b/dataforge-workspace/build.gradle.kts index ff74d591..c576ef8e 100644 --- a/dataforge-workspace/build.gradle.kts +++ b/dataforge-workspace/build.gradle.kts @@ -10,6 +10,7 @@ kotlin { dependencies { api(project(":dataforge-context")) api(project(":dataforge-data")) + api(project(":dataforge-output")) } } } diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt new file mode 100644 index 00000000..62fc97bd --- /dev/null +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -0,0 +1,72 @@ +package hep.dataforge.workspace + +import hep.dataforge.context.Context +import hep.dataforge.data.Data +import hep.dataforge.data.goal +import hep.dataforge.descriptors.NodeDescriptor +import hep.dataforge.io.IOFormat +import hep.dataforge.io.JsonMetaFormat +import hep.dataforge.io.MetaFormat +import hep.dataforge.meta.EmptyMeta +import hep.dataforge.meta.Meta +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.withContext +import kotlinx.io.nio.asInput +import kotlinx.io.nio.asOutput +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption +import kotlin.reflect.KClass + +/** + * Read meta from file in a given [format] + */ +suspend fun Path.readMeta(format: MetaFormat, descriptor: NodeDescriptor? = null): Meta { + return withContext(Dispatchers.IO) { + format.run { + Files.newByteChannel(this@readMeta, StandardOpenOption.READ) + .asInput() + .readMeta(descriptor) + } + } +} + +/** + * Write meta to file in a given [format] + */ +suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) { + withContext(Dispatchers.IO) { + format.run { + Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW) + .asOutput() + .writeMeta(this@write, descriptor) + } + } +} + +suspend fun Context.readData( + type: KClass, + path: Path, + format: IOFormat, + metaFile: Path = path.resolveSibling("${path.fileName}.meta"), + metaFileFormat: MetaFormat = JsonMetaFormat +): Data { + return coroutineScope { + val externalMeta = if (Files.exists(metaFile)) { + metaFile.readMeta(metaFileFormat) + } else { + null + } + val goal = goal { + withContext(Dispatchers.IO) { + format.run { + Files.newByteChannel(path, StandardOpenOption.READ) + .asInput() + .readObject() + } + } + } + Data.of(type, goal, externalMeta ?: EmptyMeta) + } +} \ No newline at end of file From 1f0a317cd875a6bfaebe1c671d4f8c78fd70f9f6 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 7 Aug 2019 19:28:44 +0300 Subject: [PATCH 36/48] Complete refactoring of goals and DataNodes --- dataforge-data/build.gradle.kts | 2 +- .../hep/dataforge/data/CoroutineMonitor.kt | 48 +++++ .../kotlin/hep/dataforge/data/Data.kt | 25 +-- .../kotlin/hep/dataforge/data/DataFilter.kt | 4 +- .../kotlin/hep/dataforge/data/DataNode.kt | 180 ++++++++++-------- .../kotlin/hep/dataforge/data/Goal.kt | 131 ------------- .../kotlin/hep/dataforge/data/GroupBuilder.kt | 2 +- .../kotlin/hep/dataforge/data/JoinAction.kt | 9 +- .../kotlin/hep/dataforge/data/PipeAction.kt | 4 +- .../kotlin/hep/dataforge/data/SplitAction.kt | 4 +- .../kotlin/hep/dataforge/data/tasks.kt | 60 ++++++ .../kotlin/hep/dataforge/data/CastDataNode.kt | 34 ++-- .../hep/dataforge/data/{data.kt => _data.kt} | 2 +- .../hep/dataforge/workspace/TaskModel.kt | 2 +- .../hep/dataforge/workspace/Workspace.kt | 4 +- .../hep/dataforge/workspace/fileData.kt | 4 +- 16 files changed, 253 insertions(+), 262 deletions(-) create mode 100644 dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CoroutineMonitor.kt delete mode 100644 dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt create mode 100644 dataforge-data/src/commonMain/kotlin/hep/dataforge/data/tasks.kt rename dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/{data.kt => _data.kt} (65%) diff --git a/dataforge-data/build.gradle.kts b/dataforge-data/build.gradle.kts index 00052375..1faff685 100644 --- a/dataforge-data/build.gradle.kts +++ b/dataforge-data/build.gradle.kts @@ -11,7 +11,6 @@ kotlin { val commonMain by getting{ dependencies { api(project(":dataforge-meta")) - api(kotlin("reflect")) api("org.jetbrains.kotlinx:kotlinx-coroutines-core-common:$coroutinesVersion") } } @@ -19,6 +18,7 @@ kotlin { val jvmMain by getting{ dependencies { api("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion") + api(kotlin("reflect")) } } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CoroutineMonitor.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CoroutineMonitor.kt new file mode 100644 index 00000000..8cbd6192 --- /dev/null +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/CoroutineMonitor.kt @@ -0,0 +1,48 @@ +package hep.dataforge.data + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlin.coroutines.CoroutineContext + +/** + * A monitor of goal state that could be accessed only form inside the goal + */ +class CoroutineMonitor : CoroutineContext.Element { + override val key: CoroutineContext.Key<*> get() = CoroutineMonitor + + var totalWork: Double = 1.0 + var workDone: Double = 0.0 + var status: String = "" + + /** + * Mark the goal as started + */ + fun start() { + + } + + /** + * Mark the goal as completed + */ + fun finish() { + workDone = totalWork + } + + companion object : CoroutineContext.Key +} + +class Dependencies(val values: Collection) : CoroutineContext.Element { + override val key: CoroutineContext.Key<*> get() = Dependencies + + companion object : CoroutineContext.Key +} + +val CoroutineContext.monitor: CoroutineMonitor? get() = this[CoroutineMonitor] +val CoroutineScope.monitor: CoroutineMonitor? get() = coroutineContext.monitor + +val Job.dependencies: Collection get() = this[Dependencies]?.values ?: emptyList() + +val Job.totalWork: Double get() = dependencies.sumByDouble { totalWork } + (monitor?.totalWork ?: 0.0) +val Job.workDone: Double get() = dependencies.sumByDouble { workDone } + (monitor?.workDone ?: 0.0) +val Job.status: String get() = monitor?.status ?: "" +val Job.progress: Double get() = workDone / totalWork \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt index 20957824..6bbd06f8 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt @@ -2,7 +2,8 @@ package hep.dataforge.data import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaRepr -import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Deferred import kotlin.reflect.KClass /** @@ -21,25 +22,25 @@ interface Data : MetaRepr { /** * Lazy data value */ - val goal: Goal + val task: Deferred override fun toMeta(): Meta = meta companion object { const val TYPE = "data" - fun of(type: KClass, goal: Goal, meta: Meta): Data = DataImpl(type, goal, meta) + fun of(type: KClass, goal: Deferred, meta: Meta): Data = DataImpl(type, goal, meta) - inline fun of(goal: Goal, meta: Meta): Data = of(T::class, goal, meta) + inline fun of(goal: Deferred, meta: Meta): Data = of(T::class, goal, meta) - fun of(name: String, type: KClass, goal: Goal, meta: Meta): Data = + fun of(name: String, type: KClass, goal: Deferred, meta: Meta): Data = NamedData(name, of(type, goal, meta)) - inline fun of(name: String, goal: Goal, meta: Meta): Data = + inline fun of(name: String, goal: Deferred, meta: Meta): Data = of(name, T::class, goal, meta) - fun static(scope: CoroutineScope, value: T, meta: Meta): Data = - DataImpl(value::class, Goal.static(scope, value), meta) + fun static(value: T, meta: Meta): Data = + DataImpl(value::class, CompletableDeferred(value), meta) } } @@ -47,21 +48,21 @@ interface Data : MetaRepr { * Upcast a [Data] to a supertype */ inline fun Data.cast(): Data { - return Data.of(R::class, goal, meta) + return Data.of(R::class, task, meta) } fun Data.cast(type: KClass): Data { - return Data.of(type, goal, meta) + return Data.of(type, task, meta) } -suspend fun Data.await(): T = goal.await() +suspend fun Data.await(): T = task.await() /** * Generic Data implementation */ private class DataImpl( override val type: KClass, - override val goal: Goal, + override val task: Deferred, override val meta: Meta ) : Data diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt index 6c920f57..a23b550d 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt @@ -20,10 +20,10 @@ class DataFilter(override val config: Config) : Specific { * Apply meta-based filter to given data node */ fun DataNode.filter(filter: DataFilter): DataNode { - val sourceNode = filter.from?.let { getNode(it.toName()) } ?: this@filter + val sourceNode = filter.from?.let { get(it.toName()).node } ?: this@filter val regex = filter.pattern.toRegex() val targetNode = DataTreeBuilder(type).apply { - sourceNode.data().forEach { (name, data) -> + sourceNode.dataSequence().forEach { (name, data) -> if (name.toString().matches(regex)) { this[name] = data } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index 02fc6a9e..08c5af0c 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -1,8 +1,26 @@ package hep.dataforge.data import hep.dataforge.names.* +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.launch +import kotlin.collections.component1 +import kotlin.collections.component2 +import kotlin.collections.set import kotlin.reflect.KClass +sealed class DataItem { + abstract val type: KClass + + class Node(val value: DataNode) : DataItem() { + override val type: KClass get() = value.type + } + + class Leaf(val value: Data) : DataItem() { + override val type: KClass get() = value.type + } +} + /** * A tree-like data structure grouped into the node. All data inside the node must inherit its type */ @@ -13,93 +31,89 @@ interface DataNode { */ val type: KClass - /** - * Get the specific data if it exists - */ - operator fun get(name: Name): Data? - - /** - * Get a subnode with given name if it exists. - */ - fun getNode(name: Name): DataNode? - - /** - * Walk the tree upside down and provide all data nodes with full names - */ - fun data(): Sequence>> - - /** - * A sequence of all nodes in the tree walking upside down, excluding self - */ - fun nodes(): Sequence>> - - operator fun iterator(): Iterator>> = data().iterator() + val items: Map> companion object { const val TYPE = "dataNode" fun build(type: KClass, block: DataTreeBuilder.() -> Unit) = - DataTreeBuilder(type).apply(block).build() + DataTreeBuilder(type).apply(block).build() fun builder(type: KClass) = DataTreeBuilder(type) } } -internal sealed class DataTreeItem { - class Node(val tree: DataTree) : DataTreeItem() - class Value(val value: Data) : DataTreeItem() +val DataItem?.node: DataNode? get() = (this as? DataItem.Node)?.value +val DataItem?.data: Data? get() = (this as? DataItem.Leaf)?.value + +/** + * Start computation for all goals in data node + */ +fun DataNode<*>.startAll(): Unit = items.values.forEach { + when (it) { + is DataItem.Node<*> -> it.value.startAll() + is DataItem.Leaf<*> -> it.value.task.start() + } } +fun DataNode<*>.joinAll(scope: CoroutineScope): Job = scope.launch { + startAll() + items.forEach { + when (val value = it.value) { + is DataItem.Node -> value.value.joinAll(this).join() + is DataItem.Leaf -> value.value.task.await() + } + } +} + +operator fun DataNode.get(name: Name): DataItem? = when (name.length) { + 0 -> error("Empty name") + 1 -> (items[name.first()] as? DataItem.Leaf) + else -> get(name.first()!!.asName()).node?.get(name.cutFirst()) +} + +/** + * Sequence of all children including nodes + */ +fun DataNode.asSequence(): Sequence>> = sequence { + items.forEach { (head, item) -> + yield(head.asName() to item) + if (item is DataItem.Node) { + val subSequence = item.value.asSequence() + .map { (name, data) -> (head.asName() + name) to data } + yieldAll(subSequence) + } + } +} + +/** + * Sequence of data entries + */ +fun DataNode.dataSequence(): Sequence>> = sequence { + items.forEach { (head, item) -> + when (item) { + is DataItem.Leaf -> yield(head.asName() to item.value) + is DataItem.Node -> { + val subSequence = item.value.dataSequence() + .map { (name, data) -> (head.asName() + name) to data } + yieldAll(subSequence) + } + } + } +} + +operator fun DataNode.iterator(): Iterator>> = asSequence().iterator() + class DataTree internal constructor( override val type: KClass, - private val items: Map> + override val items: Map> ) : DataNode { //TODO add node-level meta? - - override fun get(name: Name): Data? = when (name.length) { - 0 -> error("Empty name") - 1 -> (items[name.first()] as? DataTreeItem.Value)?.value - else -> getNode(name.first()!!.asName())?.get(name.cutFirst()) - } - - override fun getNode(name: Name): DataTree? = when (name.length) { - 0 -> this - 1 -> (items[name.first()] as? DataTreeItem.Node)?.tree - else -> getNode(name.first()!!.asName())?.getNode(name.cutFirst()) - } - - override fun data(): Sequence>> { - return sequence { - items.forEach { (head, tree) -> - when (tree) { - is DataTreeItem.Value -> yield(head.asName() to tree.value) - is DataTreeItem.Node -> { - val subSequence = - tree.tree.data().map { (name, data) -> (head.asName() + name) to data } - yieldAll(subSequence) - } - } - } - } - } - - override fun nodes(): Sequence>> { - return sequence { - items.forEach { (head, tree) -> - if (tree is DataTreeItem.Node) { - yield(head.asName() to tree.tree) - val subSequence = - tree.tree.nodes().map { (name, node) -> (head.asName() + name) to node } - yieldAll(subSequence) - } - } - } - } } private sealed class DataTreeBuilderItem { class Node(val tree: DataTreeBuilder) : DataTreeBuilderItem() - class Value(val value: Data) : DataTreeBuilderItem() + class Leaf(val value: Data) : DataTreeBuilderItem() } /** @@ -115,7 +129,7 @@ class DataTreeBuilder(private val type: KClass) { operator fun set(token: NameToken, data: Data) { if (map.containsKey(token)) error("Tree entry with name $token is not empty") - map[token] = DataTreeBuilderItem.Value(data) + map[token] = DataTreeBuilderItem.Leaf(data) } private fun buildNode(token: NameToken): DataTreeBuilder { @@ -152,6 +166,11 @@ class DataTreeBuilder(private val type: KClass) { operator fun set(name: Name, node: DataNode) = set(name, node.builder()) + operator fun set(name: Name, item: DataItem) = when (item) { + is DataItem.Node -> set(name, item.value.builder()) + is DataItem.Leaf -> set(name, item.value) + } + /** * Append data to node */ @@ -162,14 +181,16 @@ class DataTreeBuilder(private val type: KClass) { */ infix fun String.to(node: DataNode) = set(toName(), node) + infix fun String.to(item: DataItem) = set(toName(), item) + /** * Build and append node */ infix fun String.to(block: DataTreeBuilder.() -> Unit) = set(toName(), DataTreeBuilder(type).apply(block)) - fun update(node: DataNode){ - node.data().forEach { + fun update(node: DataNode) { + node.dataSequence().forEach { //TODO check if the place is occupied this[it.first] = it.second } @@ -178,8 +199,8 @@ class DataTreeBuilder(private val type: KClass) { fun build(): DataTree { val resMap = map.mapValues { (_, value) -> when (value) { - is DataTreeBuilderItem.Value -> DataTreeItem.Value(value.value) - is DataTreeBuilderItem.Node -> DataTreeItem.Node(value.tree.build()) + is DataTreeBuilderItem.Leaf -> DataItem.Leaf(value.value) + is DataTreeBuilderItem.Node -> DataItem.Node(value.tree.build()) } } return DataTree(type, resMap) @@ -190,27 +211,20 @@ class DataTreeBuilder(private val type: KClass) { * Generate a mutable builder from this node. Node content is not changed */ fun DataNode.builder(): DataTreeBuilder = DataTreeBuilder(type).apply { - data().forEach { (name, data) -> this[name] = data } + dataSequence().forEach { (name, data) -> this[name] = data } } -/** - * Start computation for all goals in data node - */ -fun DataNode<*>.startAll() = data().forEach { (_, data) -> data.goal.start() } - fun DataNode.filter(predicate: (Name, Data) -> Boolean): DataNode = DataNode.build(type) { - data().forEach { (name, data) -> + dataSequence().forEach { (name, data) -> if (predicate(name, data)) { this[name] = data } } } -fun DataNode.first(): Data = data().first().second +fun DataNode.first(): Data? = dataSequence().first().second /** * Check that node is compatible with given type meaning that each element could be cast to the type */ -expect fun DataNode<*>.checkType(type: KClass<*>) - -//fun DataNode.filterIsInstance(type: KClass): DataNode = filter{_,data -> type.} \ No newline at end of file +expect fun DataNode<*>.checkType(type: KClass<*>) \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt deleted file mode 100644 index 128832c5..00000000 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt +++ /dev/null @@ -1,131 +0,0 @@ -package hep.dataforge.data - -import kotlinx.coroutines.* -import kotlin.coroutines.CoroutineContext -import kotlin.coroutines.EmptyCoroutineContext - -/** - * A special deferred with explicit dependencies and some additional information like progress and unique id - */ -interface Goal : Deferred, CoroutineScope { - val scope: CoroutineScope - override val coroutineContext get() = scope.coroutineContext - - val dependencies: Collection> - - val totalWork: Double get() = dependencies.sumByDouble { totalWork } + (monitor?.totalWork ?: 0.0) - val workDone: Double get() = dependencies.sumByDouble { workDone } + (monitor?.workDone ?: 0.0) - val status: String get() = monitor?.status ?: "" - val progress: Double get() = workDone / totalWork - - companion object { - /** - * Create goal wrapping static value. This goal is always completed - */ - fun static(scope: CoroutineScope, value: T): Goal = - StaticGoalImpl(scope, CompletableDeferred(value)) - } -} - -/** - * A monitor of goal state that could be accessed only form inside the goal - */ -class GoalMonitor : CoroutineContext.Element { - override val key: CoroutineContext.Key<*> get() = GoalMonitor - - var totalWork: Double = 1.0 - var workDone: Double = 0.0 - var status: String = "" - - /** - * Mark the goal as started - */ - fun start() { - - } - - /** - * Mark the goal as completed - */ - fun finish() { - workDone = totalWork - } - - companion object : CoroutineContext.Key -} - -val CoroutineScope.monitor: GoalMonitor? get() = coroutineContext[GoalMonitor] - -private class GoalImpl( - override val scope: CoroutineScope, - override val dependencies: Collection>, - deferred: Deferred -) : Goal, Deferred by deferred - -private class StaticGoalImpl(override val scope: CoroutineScope, deferred: CompletableDeferred) : Goal, - Deferred by deferred { - override val dependencies: Collection> get() = emptyList() - override val status: String get() = "" - override val totalWork: Double get() = 0.0 - override val workDone: Double get() = 0.0 -} - - -/** - * Create a new [Goal] with given [dependencies] and execution [block]. The block takes monitor as parameter. - * The goal block runs in a supervised scope, meaning that when it fails, it won't affect external scope. - * - * **Important:** Unlike regular deferred, the [Goal] is started lazily, so the actual calculation is called only when result is requested. - */ -fun CoroutineScope.goal( - dependencies: Collection> = emptyList(), - context: CoroutineContext = EmptyCoroutineContext, - block: suspend CoroutineScope.() -> R -): Goal { - val deferred = async(context + GoalMonitor(), start = CoroutineStart.LAZY) { - dependencies.forEach { it.start() } - monitor?.start() - //Running in supervisor scope in order to allow manual error handling - return@async supervisorScope { - block().also { - monitor?.finish() - } - } - } - - return GoalImpl(this, dependencies, deferred) -} - -/** - * Create a one-to-one goal based on existing goal - */ -fun Goal.pipe( - context: CoroutineContext = EmptyCoroutineContext, - block: suspend CoroutineScope.(T) -> R -): Goal = goal(listOf(this), context) { block(await()) } - -/** - * Create a joining goal. - * @param scope the scope for resulting goal. By default use first goal in list - */ -fun Collection>.join( - scope: CoroutineScope = first(), - context: CoroutineContext = EmptyCoroutineContext, - block: suspend CoroutineScope.(Collection) -> R -): Goal = scope.goal(this, context) { - block(map { it.await() }) -} - -/** - * A joining goal for a map - * @param K type of the map key - * @param T type of the input goal - * @param R type of the result goal - */ -fun Map>.join( - scope: CoroutineScope = values.first(), - context: CoroutineContext = EmptyCoroutineContext, - block: suspend CoroutineScope.(Map) -> R -): Goal = scope.goal(this.values, context) { - block(mapValues { it.value.await() }) -} \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupBuilder.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupBuilder.kt index 0820c162..409be5bf 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupBuilder.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupBuilder.kt @@ -44,7 +44,7 @@ object GroupBuilder { override fun invoke(node: DataNode): Map> { val map = HashMap>() - node.data().forEach { (name, data) -> + node.dataSequence().forEach { (name, data) -> val tagValue = data.meta[key]?.string ?: defaultTagValue map.getOrPut(tagValue) { DataNode.builder(node.type) }[name] = data } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt index 2f5979fe..0a78fa7e 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt @@ -6,6 +6,7 @@ import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.builder import hep.dataforge.names.Name import hep.dataforge.names.toName +import kotlinx.coroutines.Deferred import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext import kotlin.reflect.KClass @@ -89,15 +90,13 @@ class JoinAction( val laminate = Laminate(group.meta, meta) - val goalMap: Map> = group.node - .data() - .associate { it.first to it.second.goal } + val goalMap: Map> = group.node.dataSequence().associate { it.first to it.second.task } val groupName: String = group.name; val env = ActionEnv(groupName.toName(), laminate.builder()) - val goal = goalMap.join(context = context) { group.result.invoke(env, it) } + val goal = goalMap.join(context) { group.result.invoke(env, it) } val res = Data.of(outputType, goal, env.meta) @@ -108,4 +107,4 @@ class JoinAction( } } -operator fun Map.get(name:String) = get(name.toName()) +operator fun Map.get(name: String) = get(name.toName()) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt index f106df40..17f22911 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt @@ -35,7 +35,7 @@ class PipeAction( node.checkType(inputType) return DataNode.build(outputType) { - node.data().forEach { (name, data) -> + node.dataSequence().forEach { (name, data) -> //merging data meta with action meta (data meta is primary) val oldMeta = meta.builder().apply { update(data.meta) } // creating environment from old meta and name @@ -47,7 +47,7 @@ class PipeAction( //getting new meta val newMeta = builder.meta.seal() //creating a goal with custom context if provided - val goal = data.goal.pipe(context) { builder.result(env, it) } + val goal = data.task.pipe(context) { builder.result(env, it) } //setting the data node this[newName] = Data.of(outputType, goal, newMeta) } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt index 3aa08990..5606bae5 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt @@ -45,7 +45,7 @@ class SplitAction( node.checkType(inputType) return DataNode.build(outputType) { - node.data().forEach { (name, data) -> + node.dataSequence().forEach { (name, data) -> val laminate = Laminate(data.meta, meta) @@ -58,7 +58,7 @@ class SplitAction( rule(env) - val goal = data.goal.pipe(context = context) { env.result(it) } + val goal = data.task.pipe(context) { env.result(it) } val res = Data.of(outputType, goal, env.meta) set(env.name, res) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/tasks.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/tasks.kt new file mode 100644 index 00000000..c46a02bc --- /dev/null +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/tasks.kt @@ -0,0 +1,60 @@ +package hep.dataforge.data + +import kotlinx.coroutines.* +import kotlin.coroutines.CoroutineContext +import kotlin.coroutines.EmptyCoroutineContext + +/** + * Create a new [Deferred] with given [dependencies] and execution [block]. The block takes monitor as parameter. + * + * **Important:** Unlike regular deferred, the [Deferred] is started lazily, so the actual calculation is called only when result is requested. + */ +fun CoroutineScope.task( + context: CoroutineContext, + dependencies: Collection = emptyList(), + block: suspend CoroutineScope.() -> T +): Deferred = async(context + CoroutineMonitor() + Dependencies(dependencies), start = CoroutineStart.LAZY) { + dependencies.forEach { job -> + job.start() + job.invokeOnCompletion { error -> + if (error != null) cancel(CancellationException("Dependency $job failed with error: ${error.message}")) + } + } + return@async block() +} + +/** + * Create a one-to-one goal based on existing goal + */ +fun Deferred.pipe( + context: CoroutineContext = EmptyCoroutineContext, + block: suspend CoroutineScope.(T) -> R +): Deferred = CoroutineScope(this + context).task(context, listOf(this)) { + block(await()) +} + +/** + * Create a joining goal. + * @param scope the scope for resulting goal. By default use first goal in list + */ +fun Collection>.join( + scope: CoroutineScope, + context: CoroutineContext = EmptyCoroutineContext, + block: suspend CoroutineScope.(Collection) -> R +): Deferred = scope.task(context, this) { + block(map { it.await() }) +} + +/** + * A joining goal for a map + * @param K type of the map key + * @param T type of the input goal + * @param R type of the result goal + */ +fun Map>.join( + context: CoroutineContext = EmptyCoroutineContext, + block: suspend CoroutineScope.(Map) -> R +): Deferred = CoroutineScope(values.first() + context).task(context, this.values) { + block(mapValues { it.value.await() }) +} + diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt index 9ff277cd..512af4a1 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt @@ -1,13 +1,14 @@ package hep.dataforge.data -import hep.dataforge.names.Name +import hep.dataforge.names.NameToken +import kotlinx.coroutines.Deferred import kotlin.reflect.KClass import kotlin.reflect.full.isSubclassOf fun Data.safeCast(type: KClass): Data? { return if (type.isSubclassOf(type)) { @Suppress("UNCHECKED_CAST") - Data.of(type, goal as Goal, meta) + Data.of(type, task as Deferred, meta) } else { null } @@ -17,7 +18,7 @@ fun Data.safeCast(type: KClass): Data? { * Filter a node by data and node type. Resulting node and its subnodes is guaranteed to have border type [type], * but could contain empty nodes */ -fun DataNode.cast(type: KClass): DataNode { +fun DataNode.cast(type: KClass): DataNode { return if (this is CastDataNode) { origin.cast(type) } else { @@ -28,19 +29,18 @@ fun DataNode.cast(type: KClass): DataNode { inline fun DataNode.cast(): DataNode = cast(R::class) class CastDataNode(val origin: DataNode, override val type: KClass) : DataNode { - - override fun get(name: Name): Data? = - origin[name]?.safeCast(type) - - override fun getNode(name: Name): DataNode? { - return origin.getNode(name)?.cast(type) + override val items: Map> by lazy { + origin.items.mapNotNull { (key, item) -> + when (item) { + is DataItem.Leaf -> { + (item.value.safeCast(type))?.let { + key to DataItem.Leaf(it) + } + } + is DataItem.Node -> { + key to DataItem.Node(item.value.cast(type)) + } + } + }.associate { it } } - - override fun data(): Sequence>> = - origin.data().mapNotNull { pair -> - pair.second.safeCast(type)?.let { pair.first to it } - } - - override fun nodes(): Sequence>> = - origin.nodes().map { it.first to it.second.cast(type) } } \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/data.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_data.kt similarity index 65% rename from dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/data.kt rename to dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_data.kt index 00c9e656..df6cc33e 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/data.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_data.kt @@ -5,4 +5,4 @@ import kotlinx.coroutines.runBlocking /** * Block the thread and get data content */ -fun Data.get(): T = runBlocking { await() } \ No newline at end of file +fun Data.get(): T = runBlocking { task.await() } \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index eaaac235..534f88f0 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -51,7 +51,7 @@ data class TaskModel( */ fun TaskModel.buildInput(workspace: Workspace): DataTree { return DataTreeBuilder(Any::class).apply { - dependencies.asSequence().flatMap { it.apply(workspace).data() }.forEach { (name, data) -> + dependencies.asSequence().flatMap { it.apply(workspace).data }.forEach { (name, data) -> //TODO add concise error on replacement this[name] = data } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt index 58fe081e..169e1cc0 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt @@ -33,8 +33,8 @@ interface Workspace : ContextAware, Provider { return when (target) { "target", Meta.TYPE -> targets.mapKeys { it.key.toName() } Task.TYPE -> tasks - Data.TYPE -> data.data().toMap() - DataNode.TYPE -> data.nodes().toMap() + Data.TYPE -> data.data.toMap() + DataNode.TYPE -> data.nodes.toMap() else -> emptyMap() } } diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index 62fc97bd..fa8f6ddf 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -2,7 +2,7 @@ package hep.dataforge.workspace import hep.dataforge.context.Context import hep.dataforge.data.Data -import hep.dataforge.data.goal +import hep.dataforge.data.task import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.io.IOFormat import hep.dataforge.io.JsonMetaFormat @@ -58,7 +58,7 @@ suspend fun Context.readData( } else { null } - val goal = goal { + val goal = task { withContext(Dispatchers.IO) { format.run { Files.newByteChannel(path, StandardOpenOption.READ) From 72954a837004e8b56708a1d3d390834fb0403e1d Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 7 Aug 2019 20:41:27 +0300 Subject: [PATCH 37/48] Meta equality for all --- .../kotlin/hep/dataforge/io/JsonMetaFormat.kt | 3 +- .../kotlin/hep/dataforge/io/MetaFormatTest.kt | 2 +- .../kotlin/hep/dataforge/meta/Laminate.kt | 2 +- .../kotlin/hep/dataforge/meta/Meta.kt | 34 ++++++++++++------- .../kotlin/hep/dataforge/values/Value.kt | 7 ++-- .../hep/dataforge/values/exoticValues.kt | 9 +++-- .../kotlin/hep/dataforge/meta/DynamicMeta.kt | 2 +- 7 files changed, 36 insertions(+), 23 deletions(-) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index de2cf925..5dabf499 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -4,6 +4,7 @@ import hep.dataforge.descriptors.ItemDescriptor import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.descriptors.ValueDescriptor import hep.dataforge.meta.Meta +import hep.dataforge.meta.MetaBase import hep.dataforge.meta.MetaItem import hep.dataforge.names.NameToken import hep.dataforge.names.toName @@ -79,7 +80,7 @@ fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject { fun JsonObject.toMeta(descriptor: NodeDescriptor? = null) = JsonMeta(this, descriptor) -class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : Meta { +class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : MetaBase() { private fun JsonPrimitive.toValue(descriptor: ValueDescriptor?): Value { return when (this) { diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt index ab4ab677..8348f7b0 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt @@ -42,7 +42,7 @@ class MetaFormatTest { if (meta[it] != result[it]) error("${meta[it]} != ${result[it]}") } - assertEquals(meta, result) + assertEquals(meta, result) } } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt index 6f7c41c8..b76fd46f 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt @@ -7,7 +7,7 @@ import hep.dataforge.names.NameToken * * */ -class Laminate(layers: List) : Meta { +class Laminate(layers: List) : MetaBase() { val layers: List = layers.flatMap { if (it is Laminate) { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index e89eb950..1f918da3 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -15,10 +15,11 @@ import hep.dataforge.values.boolean * * a [NodeItem] (node) */ sealed class MetaItem { - data class ValueItem(val value: Value) : MetaItem(){ + data class ValueItem(val value: Value) : MetaItem() { override fun toString(): String = value.toString() } - data class NodeItem(val node: M) : MetaItem(){ + + data class NodeItem(val node: M) : MetaItem() { override fun toString(): String = node.toString() } } @@ -46,6 +47,12 @@ interface Meta : MetaRepr { override fun toMeta(): Meta = this + override fun equals(other: Any?): Boolean + + override fun hashCode(): Int + + override fun toString(): String + companion object { const val TYPE = "meta" /** @@ -174,23 +181,26 @@ operator fun > MetaNode?.get(key: NameToken): MetaItem? = } /** - * Equals and hash code implementation for meta node + * Equals, hashcode and to string for any meta */ -abstract class AbstractMetaNode> : MetaNode { - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (other !is Meta) return false +abstract class MetaBase: Meta{ - return this.items == other.items//this.items.keys == other.items.keys && items.keys.all { this[it] == other[it] } + override fun equals(other: Any?): Boolean = if(other is Meta) { + items == other.items + } else { + false } - override fun hashCode(): Int { - return items.hashCode() - } + override fun hashCode(): Int = items.hashCode() override fun toString(): String = items.toString() } +/** + * Equals and hash code implementation for meta node + */ +abstract class AbstractMetaNode> : MetaNode, MetaBase() + /** * The meta implementation which is guaranteed to be immutable. * @@ -210,7 +220,7 @@ fun MetaItem<*>.seal(): MetaItem = when (this) { is NodeItem -> NodeItem(node.seal()) } -object EmptyMeta : Meta { +object EmptyMeta : MetaBase() { override val items: Map> = emptyMap() } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt index 7b42f639..5da65a10 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt @@ -159,7 +159,7 @@ class StringValue(override val string: String) : Value { override fun hashCode(): Int = string.hashCode() - override fun toString(): String = value.toString() + override fun toString(): String = "\"${value.toString()}\"" } class EnumValue>(override val value: E) : Value { @@ -188,11 +188,14 @@ class ListValue(override val list: List) : Value { override val number: Number get() = list.first().number override val string: String get() = list.first().string - override fun toString(): String = list.joinToString (prefix = "[ ", postfix = " ]") + override fun toString(): String = list.joinToString (prefix = "[", postfix = "]") override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is Value) return false + if( other is DoubleArrayValue){ + + } return list == other.list } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt index 8ebc1c6b..f6122dbb 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/exoticValues.kt @@ -31,10 +31,9 @@ class DoubleArrayValue(override val value: DoubleArray) : Value { if (this === other) return true if (other !is Value) return false - return if (other is DoubleArrayValue) { - value.contentEquals(other.value) - } else { - list == other.list + return when (other) { + is DoubleArrayValue -> value.contentEquals(other.value) + else -> list == other.list } } @@ -42,7 +41,7 @@ class DoubleArrayValue(override val value: DoubleArray) : Value { return value.contentHashCode() } - override fun toString(): String = value.toString() + override fun toString(): String = list.joinToString (prefix = "[", postfix = "]") } fun DoubleArray.asValue(): DoubleArrayValue = DoubleArrayValue(this) diff --git a/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt b/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt index 57375125..694e2441 100644 --- a/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt +++ b/dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt @@ -29,7 +29,7 @@ fun Meta.toDynamic(): dynamic { return res } -class DynamicMeta(val obj: dynamic) : Meta { +class DynamicMeta(val obj: dynamic) : MetaBase() { private fun keys() = js("Object.keys(this.obj)") as Array private fun isArray(@Suppress("UNUSED_PARAMETER") obj: dynamic): Boolean = From 65633bbd0d915d104fe2afeed1c8539dbd556c7a Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 7 Aug 2019 21:14:20 +0300 Subject: [PATCH 38/48] Build passes, but a lot of tests fails. Need more work. --- .../hep/dataforge/data/{tasks.kt => goals.kt} | 16 +++++++++------- .../commonMain/kotlin/hep/dataforge/meta/Meta.kt | 7 ++++++- .../kotlin/hep/dataforge/workspace/TaskModel.kt | 3 ++- .../kotlin/hep/dataforge/workspace/Workspace.kt | 5 +++-- .../hep/dataforge/workspace/WorkspaceBuilder.kt | 14 ++++++-------- .../hep/dataforge/workspace/TaskBuilder.kt | 2 +- .../kotlin/hep/dataforge/workspace/fileData.kt | 4 ++-- .../dataforge/workspace/SimpleWorkspaceTest.kt | 4 ++-- 8 files changed, 31 insertions(+), 24 deletions(-) rename dataforge-data/src/commonMain/kotlin/hep/dataforge/data/{tasks.kt => goals.kt} (79%) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/tasks.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/goals.kt similarity index 79% rename from dataforge-data/src/commonMain/kotlin/hep/dataforge/data/tasks.kt rename to dataforge-data/src/commonMain/kotlin/hep/dataforge/data/goals.kt index c46a02bc..f4f200f0 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/tasks.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/goals.kt @@ -9,11 +9,14 @@ import kotlin.coroutines.EmptyCoroutineContext * * **Important:** Unlike regular deferred, the [Deferred] is started lazily, so the actual calculation is called only when result is requested. */ -fun CoroutineScope.task( - context: CoroutineContext, +fun goal( + context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection = emptyList(), block: suspend CoroutineScope.() -> T -): Deferred = async(context + CoroutineMonitor() + Dependencies(dependencies), start = CoroutineStart.LAZY) { +): Deferred = CoroutineScope(context).async( + CoroutineMonitor() + Dependencies(dependencies), + start = CoroutineStart.LAZY +) { dependencies.forEach { job -> job.start() job.invokeOnCompletion { error -> @@ -29,7 +32,7 @@ fun CoroutineScope.task( fun Deferred.pipe( context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(T) -> R -): Deferred = CoroutineScope(this + context).task(context, listOf(this)) { +): Deferred = goal(this + context,listOf(this)) { block(await()) } @@ -38,10 +41,9 @@ fun Deferred.pipe( * @param scope the scope for resulting goal. By default use first goal in list */ fun Collection>.join( - scope: CoroutineScope, context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(Collection) -> R -): Deferred = scope.task(context, this) { +): Deferred = goal(context, this) { block(map { it.await() }) } @@ -54,7 +56,7 @@ fun Collection>.join( fun Map>.join( context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(Map) -> R -): Deferred = CoroutineScope(values.first() + context).task(context, this.values) { +): Deferred = goal(context, this.values) { block(mapValues { it.value.await() }) } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 1f918da3..9e0dedf9 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -186,7 +186,12 @@ operator fun > MetaNode?.get(key: NameToken): MetaItem? = abstract class MetaBase: Meta{ override fun equals(other: Any?): Boolean = if(other is Meta) { - items == other.items + this.items == other.items +// val items = items +// val otherItems = other.items +// (items.keys == otherItems.keys) && items.keys.all { +// items[it] == otherItems[it] +// } } else { false } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index 534f88f0..b6c4b2b6 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -8,6 +8,7 @@ package hep.dataforge.workspace import hep.dataforge.data.DataFilter import hep.dataforge.data.DataTree import hep.dataforge.data.DataTreeBuilder +import hep.dataforge.data.dataSequence import hep.dataforge.meta.* import hep.dataforge.names.EmptyName import hep.dataforge.names.Name @@ -51,7 +52,7 @@ data class TaskModel( */ fun TaskModel.buildInput(workspace: Workspace): DataTree { return DataTreeBuilder(Any::class).apply { - dependencies.asSequence().flatMap { it.apply(workspace).data }.forEach { (name, data) -> + dependencies.asSequence().flatMap { it.apply(workspace).dataSequence() }.forEach { (name, data) -> //TODO add concise error on replacement this[name] = data } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt index 169e1cc0..f5f0f3a6 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/Workspace.kt @@ -3,6 +3,7 @@ package hep.dataforge.workspace import hep.dataforge.context.ContextAware import hep.dataforge.data.Data import hep.dataforge.data.DataNode +import hep.dataforge.data.dataSequence import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.buildMeta @@ -33,8 +34,8 @@ interface Workspace : ContextAware, Provider { return when (target) { "target", Meta.TYPE -> targets.mapKeys { it.key.toName() } Task.TYPE -> tasks - Data.TYPE -> data.data.toMap() - DataNode.TYPE -> data.nodes.toMap() + Data.TYPE -> data.dataSequence().toMap() + //DataNode.TYPE -> data.nodes.toMap() else -> emptyMap() } } diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt index b8df2fef..999ee50c 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt @@ -8,8 +8,6 @@ import hep.dataforge.data.DataTreeBuilder import hep.dataforge.meta.* import hep.dataforge.names.Name import hep.dataforge.names.toName -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.GlobalScope @TaskBuildScope interface WorkspaceBuilder { @@ -36,14 +34,14 @@ fun WorkspaceBuilder.data(name: Name, data: Data) { fun WorkspaceBuilder.data(name: String, data: Data) = data(name.toName(), data) -fun WorkspaceBuilder.static(name: Name, data: Any, scope: CoroutineScope = GlobalScope, meta: Meta = EmptyMeta) = - data(name, Data.static(scope, data, meta)) +fun WorkspaceBuilder.static(name: Name, data: Any, meta: Meta = EmptyMeta) = + data(name, Data.static(data, meta)) -fun WorkspaceBuilder.static(name: Name, data: Any, scope: CoroutineScope = GlobalScope, block: MetaBuilder.() -> Unit = {}) = - data(name, Data.static(scope, data, buildMeta(block))) +fun WorkspaceBuilder.static(name: Name, data: Any, block: MetaBuilder.() -> Unit = {}) = + data(name, Data.static(data, buildMeta(block))) -fun WorkspaceBuilder.static(name: String, data: Any, scope: CoroutineScope = GlobalScope, block: MetaBuilder.() -> Unit = {}) = - data(name, Data.static(scope, data, buildMeta(block))) +fun WorkspaceBuilder.static(name: String, data: Any, block: MetaBuilder.() -> Unit = {}) = + data(name, Data.static(data, buildMeta(block))) fun WorkspaceBuilder.data(name: Name, node: DataNode) { this.data[name] = node diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index d2e1b864..175c4d87 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -27,7 +27,7 @@ class TaskBuilder(val name: String) { val localData = if (from.isEmpty()) { node } else { - node.getNode(from.toName()) ?: return null + node[from.toName()].node ?: return null } return transform(workspace.context, model, localData) } diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index fa8f6ddf..62fc97bd 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -2,7 +2,7 @@ package hep.dataforge.workspace import hep.dataforge.context.Context import hep.dataforge.data.Data -import hep.dataforge.data.task +import hep.dataforge.data.goal import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.io.IOFormat import hep.dataforge.io.JsonMetaFormat @@ -58,7 +58,7 @@ suspend fun Context.readData( } else { null } - val goal = task { + val goal = goal { withContext(Dispatchers.IO) { format.run { Files.newByteChannel(path, StandardOpenOption.READ) diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index d47f972f..51471525 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -92,13 +92,13 @@ class SimpleWorkspaceTest { fun testWorkspace() { val node = workspace.run("sum") val res = node.first() - assertEquals(328350, res.get()) + assertEquals(328350, res?.get()) } @Test fun testMetaPropagation() { val node = workspace.run("sum") { "testFlag" to true } - val res = node.first().get() + val res = node.first()?.get() } @Test From 43c18bcde7f0bb09845e6860644131c46cf7b1f5 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 7 Aug 2019 22:07:36 +0300 Subject: [PATCH 39/48] Some refactoring in goal mechanics --- build.gradle.kts | 2 +- .../kotlin/hep/dataforge/data/Action.kt | 17 -------------- .../kotlin/hep/dataforge/data/JoinAction.kt | 7 +++--- .../kotlin/hep/dataforge/data/PipeAction.kt | 11 +++++----- .../kotlin/hep/dataforge/data/SplitAction.kt | 7 +++--- .../kotlin/hep/dataforge/data/goals.kt | 22 +++++++++++-------- .../hep/dataforge/workspace/TaskBuilder.kt | 13 +++++++---- .../hep/dataforge/workspace/fileData.kt | 4 ++-- 8 files changed, 36 insertions(+), 47 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index ecaee7a6..758279ab 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,7 +3,7 @@ plugins { id("scientifik.publish") version "0.1.4" apply false } -val dataforgeVersion by extra("0.1.3-dev-10") +val dataforgeVersion by extra("0.1.3-dev-11") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Action.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Action.kt index 228522dc..cf030c75 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Action.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Action.kt @@ -1,7 +1,6 @@ package hep.dataforge.data import hep.dataforge.meta.Meta -import hep.dataforge.names.Name /** * A simple data transformation on a data node @@ -34,19 +33,3 @@ infix fun Action.then(action: Action): A } } - -///** -// * An action that performs the same transformation on each of input data nodes. Null results are ignored. -// * The transformation is non-suspending because it is lazy. -// */ -//class PipeAction(val transform: (Name, Data, Meta) -> Data?) : Action { -// override fun invoke(node: DataNode, meta: Meta): DataNode = DataNode.build { -// node.data().forEach { (name, data) -> -// val res = transform(name, data, meta) -// if (res != null) { -// set(name, res) -// } -// } -// } -//} - diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt index 0a78fa7e..636d4a87 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt @@ -6,9 +6,8 @@ import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.builder import hep.dataforge.names.Name import hep.dataforge.names.toName +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Deferred -import kotlin.coroutines.CoroutineContext -import kotlin.coroutines.EmptyCoroutineContext import kotlin.reflect.KClass @@ -79,7 +78,7 @@ class JoinGroupBuilder(val actionMeta: Meta) { class JoinAction( val inputType: KClass, val outputType: KClass, - val context: CoroutineContext = EmptyCoroutineContext, + val scope: CoroutineScope, private val action: JoinGroupBuilder.() -> Unit ) : Action { @@ -96,7 +95,7 @@ class JoinAction( val env = ActionEnv(groupName.toName(), laminate.builder()) - val goal = goalMap.join(context) { group.result.invoke(env, it) } + val goal = goalMap.join(scope) { group.result.invoke(env, it) } val res = Data.of(outputType, goal, env.meta) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt index 17f22911..7e2099bb 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt @@ -2,8 +2,7 @@ package hep.dataforge.data import hep.dataforge.meta.* import hep.dataforge.names.Name -import kotlin.coroutines.CoroutineContext -import kotlin.coroutines.EmptyCoroutineContext +import kotlinx.coroutines.CoroutineScope import kotlin.reflect.KClass class ActionEnv(val name: Name, val meta: Meta) @@ -27,7 +26,7 @@ class PipeBuilder(var name: Name, var meta: MetaBuilder) { class PipeAction( val inputType: KClass, val outputType: KClass, - val context: CoroutineContext = EmptyCoroutineContext, + val scope: CoroutineScope, private val block: PipeBuilder.() -> Unit ) : Action { @@ -47,7 +46,7 @@ class PipeAction( //getting new meta val newMeta = builder.meta.seal() //creating a goal with custom context if provided - val goal = data.task.pipe(context) { builder.result(env, it) } + val goal = data.task.pipe(scope) { builder.result(env, it) } //setting the data node this[newName] = Data.of(outputType, goal, newMeta) } @@ -57,9 +56,9 @@ class PipeAction( inline fun DataNode.pipe( meta: Meta, - context: CoroutineContext = EmptyCoroutineContext, + scope: CoroutineScope, noinline action: PipeBuilder.() -> Unit -): DataNode = PipeAction(T::class, R::class, context, action).invoke(this, meta) +): DataNode = PipeAction(T::class, R::class, scope, action).invoke(this, meta) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt index 5606bae5..d2545973 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt @@ -6,9 +6,8 @@ import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.builder import hep.dataforge.names.Name import hep.dataforge.names.toName +import kotlinx.coroutines.CoroutineScope import kotlin.collections.set -import kotlin.coroutines.CoroutineContext -import kotlin.coroutines.EmptyCoroutineContext import kotlin.reflect.KClass @@ -37,7 +36,7 @@ class SplitBuilder(val name: Name, val meta: Meta) { class SplitAction( val inputType: KClass, val outputType: KClass, - val context: CoroutineContext = EmptyCoroutineContext, + val scope: CoroutineScope, private val action: SplitBuilder.() -> Unit ) : Action { @@ -58,7 +57,7 @@ class SplitAction( rule(env) - val goal = data.task.pipe(context) { env.result(it) } + val goal = data.task.pipe(scope) { env.result(it) } val res = Data.of(outputType, goal, env.meta) set(env.name, res) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/goals.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/goals.kt index f4f200f0..82693c96 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/goals.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/goals.kt @@ -10,11 +10,12 @@ import kotlin.coroutines.EmptyCoroutineContext * **Important:** Unlike regular deferred, the [Deferred] is started lazily, so the actual calculation is called only when result is requested. */ fun goal( - context: CoroutineContext = EmptyCoroutineContext, + scope: CoroutineScope, + coroutineContext: CoroutineContext = EmptyCoroutineContext, dependencies: Collection = emptyList(), block: suspend CoroutineScope.() -> T -): Deferred = CoroutineScope(context).async( - CoroutineMonitor() + Dependencies(dependencies), +): Deferred = scope.async( + coroutineContext + CoroutineMonitor() + Dependencies(dependencies), start = CoroutineStart.LAZY ) { dependencies.forEach { job -> @@ -30,9 +31,10 @@ fun goal( * Create a one-to-one goal based on existing goal */ fun Deferred.pipe( - context: CoroutineContext = EmptyCoroutineContext, + scope: CoroutineScope, + coroutineContext: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(T) -> R -): Deferred = goal(this + context,listOf(this)) { +): Deferred = goal(scope, coroutineContext, listOf(this)) { block(await()) } @@ -41,9 +43,10 @@ fun Deferred.pipe( * @param scope the scope for resulting goal. By default use first goal in list */ fun Collection>.join( - context: CoroutineContext = EmptyCoroutineContext, + scope: CoroutineScope, + coroutineContext: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(Collection) -> R -): Deferred = goal(context, this) { +): Deferred = goal(scope, coroutineContext, this) { block(map { it.await() }) } @@ -54,9 +57,10 @@ fun Collection>.join( * @param R type of the result goal */ fun Map>.join( - context: CoroutineContext = EmptyCoroutineContext, + scope: CoroutineScope, + coroutineContext: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.(Map) -> R -): Deferred = goal(context, this.values) { +): Deferred = goal(scope, coroutineContext, this.values) { block(mapValues { it.value.await() }) } diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index 175c4d87..0b439032 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -85,7 +85,8 @@ class TaskBuilder(val name: String) { val context = this PipeAction( inputType = T::class, - outputType = R::class + outputType = R::class, + scope = context ) { block(context) } } } @@ -102,7 +103,8 @@ class TaskBuilder(val name: String) { val context = this PipeAction( inputType = T::class, - outputType = R::class + outputType = R::class, + scope = context ) { //TODO automatically append task meta result = { data -> @@ -123,7 +125,8 @@ class TaskBuilder(val name: String) { action(from, to) { JoinAction( inputType = T::class, - outputType = R::class + outputType = R::class, + scope = this ) { block(this@action) } } } @@ -141,6 +144,7 @@ class TaskBuilder(val name: String) { JoinAction( inputType = T::class, outputType = R::class, + scope = context, action = { result( actionMeta[TaskModel.MODEL_TARGET_KEY]?.string ?: "@anonymous" @@ -163,7 +167,8 @@ class TaskBuilder(val name: String) { action(from, to) { SplitAction( inputType = T::class, - outputType = R::class + outputType = R::class, + scope = this ) { block(this@action) } } } diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index 62fc97bd..9d2efc95 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -2,7 +2,6 @@ package hep.dataforge.workspace import hep.dataforge.context.Context import hep.dataforge.data.Data -import hep.dataforge.data.goal import hep.dataforge.descriptors.NodeDescriptor import hep.dataforge.io.IOFormat import hep.dataforge.io.JsonMetaFormat @@ -10,6 +9,7 @@ import hep.dataforge.io.MetaFormat import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.withContext import kotlinx.io.nio.asInput @@ -58,7 +58,7 @@ suspend fun Context.readData( } else { null } - val goal = goal { + val goal = async { withContext(Dispatchers.IO) { format.run { Files.newByteChannel(path, StandardOpenOption.READ) From 5921556254b800a86bcfe9f4d1547e6aafe2a7f4 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 9 Aug 2019 16:31:59 +0300 Subject: [PATCH 40/48] Reworked Data and Goal mechanics --- .../kotlin/hep/dataforge/data/Data.kt | 160 ++++++++++++++---- .../kotlin/hep/dataforge/data/DataNode.kt | 10 +- .../kotlin/hep/dataforge/data/Goal.kt | 117 +++++++++++++ .../kotlin/hep/dataforge/data/JoinAction.kt | 9 +- .../kotlin/hep/dataforge/data/PipeAction.kt | 10 +- .../kotlin/hep/dataforge/data/SplitAction.kt | 6 +- .../kotlin/hep/dataforge/data/goals.kt | 66 -------- .../kotlin/hep/dataforge/data/CastDataNode.kt | 17 +- .../kotlin/hep/dataforge/data/_data.kt | 2 +- .../hep/dataforge/workspace/TaskBuilder.kt | 13 +- .../hep/dataforge/workspace/fileData.kt | 4 +- 11 files changed, 276 insertions(+), 138 deletions(-) create mode 100644 dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt delete mode 100644 dataforge-data/src/commonMain/kotlin/hep/dataforge/data/goals.kt diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt index 6bbd06f8..9b0d9027 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt @@ -1,15 +1,17 @@ package hep.dataforge.data +import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaRepr -import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.Deferred +import kotlinx.coroutines.CoroutineScope +import kotlin.coroutines.CoroutineContext +import kotlin.coroutines.EmptyCoroutineContext import kotlin.reflect.KClass /** * A data element characterized by its meta */ -interface Data : MetaRepr { +interface Data : Goal, MetaRepr { /** * Type marker for the data. The type is known before the calculation takes place so it could be checked. */ @@ -19,52 +21,148 @@ interface Data : MetaRepr { */ val meta: Meta - /** - * Lazy data value - */ - val task: Deferred - override fun toMeta(): Meta = meta companion object { const val TYPE = "data" - fun of(type: KClass, goal: Deferred, meta: Meta): Data = DataImpl(type, goal, meta) + operator fun invoke( + type: KClass, + meta: Meta = EmptyMeta, + context: CoroutineContext = EmptyCoroutineContext, + dependencies: Collection> = emptyList(), + block: suspend CoroutineScope.() -> T + ): Data = DynamicData(type, meta, context, dependencies, block) - inline fun of(goal: Deferred, meta: Meta): Data = of(T::class, goal, meta) + operator inline fun invoke( + meta: Meta = EmptyMeta, + context: CoroutineContext = EmptyCoroutineContext, + dependencies: Collection> = emptyList(), + noinline block: suspend CoroutineScope.() -> T + ): Data = invoke(T::class, meta, context, dependencies, block) - fun of(name: String, type: KClass, goal: Deferred, meta: Meta): Data = - NamedData(name, of(type, goal, meta)) + operator fun invoke( + name: String, + type: KClass, + meta: Meta = EmptyMeta, + context: CoroutineContext = EmptyCoroutineContext, + dependencies: Collection> = emptyList(), + block: suspend CoroutineScope.() -> T + ): Data = NamedData(name, invoke(type, meta, context, dependencies, block)) - inline fun of(name: String, goal: Deferred, meta: Meta): Data = - of(name, T::class, goal, meta) + operator inline fun invoke( + name: String, + meta: Meta = EmptyMeta, + context: CoroutineContext = EmptyCoroutineContext, + dependencies: Collection> = emptyList(), + noinline block: suspend CoroutineScope.() -> T + ): Data = + invoke(name, T::class, meta, context, dependencies, block) - fun static(value: T, meta: Meta): Data = - DataImpl(value::class, CompletableDeferred(value), meta) + fun static(value: T, meta: Meta = EmptyMeta): Data = + StaticData(value, meta) + } +} + + +fun Data.cast(type: KClass): Data { + return object : Data by this { + override val type: KClass = type } } /** * Upcast a [Data] to a supertype */ -inline fun Data.cast(): Data { - return Data.of(R::class, task, meta) -} +inline fun Data.cast(): Data = cast(R::class) -fun Data.cast(type: KClass): Data { - return Data.of(type, task, meta) -} -suspend fun Data.await(): T = task.await() - -/** - * Generic Data implementation - */ -private class DataImpl( +class DynamicData( override val type: KClass, - override val task: Deferred, - override val meta: Meta -) : Data + override val meta: Meta = EmptyMeta, + context: CoroutineContext = EmptyCoroutineContext, + dependencies: Collection> = emptyList(), + block: suspend CoroutineScope.() -> T +) : Data, DynamicGoal(context, dependencies, block) + +class StaticData( + value: T, + override val meta: Meta = EmptyMeta +) : Data, StaticGoal(value) { + override val type: KClass get() = value::class +} class NamedData(val name: String, data: Data) : Data by data +fun Data.pipe( + outputType: KClass, + coroutineContext: CoroutineContext = EmptyCoroutineContext, + meta: Meta = this.meta, + block: suspend CoroutineScope.(T) -> R +): Data = DynamicData(outputType, meta, coroutineContext, listOf(this)) { + block(await(this)) +} + + +/** + * Create a data pipe + */ +inline fun Data.pipe( + coroutineContext: CoroutineContext = EmptyCoroutineContext, + meta: Meta = this.meta, + noinline block: suspend CoroutineScope.(T) -> R +): Data = DynamicData(R::class, meta, coroutineContext, listOf(this)) { + block(await(this)) +} + +/** + * Create a joined data. + */ +inline fun Collection>.join( + coroutineContext: CoroutineContext = EmptyCoroutineContext, + meta: Meta, + noinline block: suspend CoroutineScope.(Collection) -> R +): Data = DynamicData( + R::class, + meta, + coroutineContext, + this +) { + block(map { this.run { it.await(this) } }) +} + +fun Map>.join( + outputType: KClass, + coroutineContext: CoroutineContext = EmptyCoroutineContext, + meta: Meta, + block: suspend CoroutineScope.(Map) -> R +): DynamicData = DynamicData( + outputType, + meta, + coroutineContext, + this.values +) { + block(mapValues { it.value.await(this) }) +} + + +/** + * A joining of multiple data into a single one + * @param K type of the map key + * @param T type of the input goal + * @param R type of the result goal + */ +inline fun Map>.join( + coroutineContext: CoroutineContext = EmptyCoroutineContext, + meta: Meta, + noinline block: suspend CoroutineScope.(Map) -> R +): DynamicData = DynamicData( + R::class, + meta, + coroutineContext, + this.values +) { + block(mapValues { it.value.await(this) }) +} + + diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index 08c5af0c..a407b512 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -49,19 +49,19 @@ val DataItem?.data: Data? get() = (this as? DataItem.Leaf)?.v /** * Start computation for all goals in data node */ -fun DataNode<*>.startAll(): Unit = items.values.forEach { +fun DataNode<*>.startAll(scope: CoroutineScope): Unit = items.values.forEach { when (it) { - is DataItem.Node<*> -> it.value.startAll() - is DataItem.Leaf<*> -> it.value.task.start() + is DataItem.Node<*> -> it.value.startAll(scope) + is DataItem.Leaf<*> -> it.value.start(scope) } } fun DataNode<*>.joinAll(scope: CoroutineScope): Job = scope.launch { - startAll() + startAll(scope) items.forEach { when (val value = it.value) { is DataItem.Node -> value.value.joinAll(this).join() - is DataItem.Leaf -> value.value.task.await() + is DataItem.Leaf -> value.value.await(scope) } } } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt new file mode 100644 index 00000000..54bb743e --- /dev/null +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt @@ -0,0 +1,117 @@ +package hep.dataforge.data + +import kotlinx.coroutines.* +import kotlin.coroutines.CoroutineContext +import kotlin.coroutines.EmptyCoroutineContext + +interface Goal { + val dependencies: Collection> + /** + * Returns current running coroutine if the goal is started + */ + val result: Deferred? + + /** + * Get ongoing computation or start a new one. + * Does not guarantee thread safety. In case of multi-thread access, could create orphan computations. + */ + fun startAsync(scope: CoroutineScope): Deferred + + suspend fun CoroutineScope.await(): T = startAsync(this).await() + + /** + * Reset the computation + */ + fun reset() + + companion object { + + } +} + +fun Goal<*>.start(scope: CoroutineScope): Job = startAsync(scope) + +val Goal<*>.isComplete get() = result?.isCompleted ?: false + +suspend fun Goal.await(scope: CoroutineScope): T = scope.await() + +open class StaticGoal(val value: T) : Goal { + override val dependencies: Collection> get() = emptyList() + override val result: Deferred = CompletableDeferred(value) + + override fun startAsync(scope: CoroutineScope): Deferred = result + + override fun reset() { + //doNothing + } +} + +open class DynamicGoal( + val coroutineContext: CoroutineContext = EmptyCoroutineContext, + override val dependencies: Collection> = emptyList(), + val block: suspend CoroutineScope.() -> T +) : Goal { + + final override var result: Deferred? = null + private set + + /** + * Get ongoing computation or start a new one. + * Does not guarantee thread safety. In case of multi-thread access, could create orphan computations. + */ + override fun startAsync(scope: CoroutineScope): Deferred { + val startedDependencies = this.dependencies.map { goal -> + goal.startAsync(scope) + } + return result ?: scope.async(coroutineContext + CoroutineMonitor() + Dependencies(startedDependencies)) { + startedDependencies.forEach { deferred -> + deferred.invokeOnCompletion { error -> + if (error != null) cancel(CancellationException("Dependency $deferred failed with error: ${error.message}")) + } + } + block() + }.also { result = it } + } + + /** + * Reset the computation + */ + override fun reset() { + result?.cancel() + result = null + } +} + +/** + * Create a one-to-one goal based on existing goal + */ +fun Goal.pipe( + coroutineContext: CoroutineContext = EmptyCoroutineContext, + block: suspend CoroutineScope.(T) -> R +): Goal = DynamicGoal(coroutineContext, listOf(this)) { + block(await(this)) +} + +/** + * Create a joining goal. + */ +fun Collection>.join( + coroutineContext: CoroutineContext = EmptyCoroutineContext, + block: suspend CoroutineScope.(Collection) -> R +): Goal = DynamicGoal(coroutineContext, this) { + block(map { this.run { it.await(this) } }) +} + +/** + * A joining goal for a map + * @param K type of the map key + * @param T type of the input goal + * @param R type of the result goal + */ +fun Map>.join( + coroutineContext: CoroutineContext = EmptyCoroutineContext, + block: suspend CoroutineScope.(Map) -> R +): Goal = DynamicGoal(coroutineContext, this.values) { + block(mapValues { it.value.await(this) }) +} + diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt index 636d4a87..5f0f5845 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt @@ -6,8 +6,6 @@ import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.builder import hep.dataforge.names.Name import hep.dataforge.names.toName -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Deferred import kotlin.reflect.KClass @@ -78,7 +76,6 @@ class JoinGroupBuilder(val actionMeta: Meta) { class JoinAction( val inputType: KClass, val outputType: KClass, - val scope: CoroutineScope, private val action: JoinGroupBuilder.() -> Unit ) : Action { @@ -89,15 +86,13 @@ class JoinAction( val laminate = Laminate(group.meta, meta) - val goalMap: Map> = group.node.dataSequence().associate { it.first to it.second.task } + val dataMap = group.node.dataSequence().associate { it } val groupName: String = group.name; val env = ActionEnv(groupName.toName(), laminate.builder()) - val goal = goalMap.join(scope) { group.result.invoke(env, it) } - - val res = Data.of(outputType, goal, env.meta) + val res: DynamicData = dataMap.join(outputType, meta = laminate) { group.result.invoke(env, it) } set(env.name, res) } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt index 7e2099bb..c84e5a13 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/PipeAction.kt @@ -2,7 +2,6 @@ package hep.dataforge.data import hep.dataforge.meta.* import hep.dataforge.names.Name -import kotlinx.coroutines.CoroutineScope import kotlin.reflect.KClass class ActionEnv(val name: Name, val meta: Meta) @@ -26,7 +25,6 @@ class PipeBuilder(var name: Name, var meta: MetaBuilder) { class PipeAction( val inputType: KClass, val outputType: KClass, - val scope: CoroutineScope, private val block: PipeBuilder.() -> Unit ) : Action { @@ -45,10 +43,9 @@ class PipeAction( val newName = builder.name //getting new meta val newMeta = builder.meta.seal() - //creating a goal with custom context if provided - val goal = data.task.pipe(scope) { builder.result(env, it) } + val newData = data.pipe(outputType, meta = newMeta) { builder.result(env, it) } //setting the data node - this[newName] = Data.of(outputType, goal, newMeta) + this[newName] = newData } } } @@ -56,9 +53,8 @@ class PipeAction( inline fun DataNode.pipe( meta: Meta, - scope: CoroutineScope, noinline action: PipeBuilder.() -> Unit -): DataNode = PipeAction(T::class, R::class, scope, action).invoke(this, meta) +): DataNode = PipeAction(T::class, R::class, action).invoke(this, meta) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt index d2545973..be9764a6 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/SplitAction.kt @@ -6,7 +6,6 @@ import hep.dataforge.meta.MetaBuilder import hep.dataforge.meta.builder import hep.dataforge.names.Name import hep.dataforge.names.toName -import kotlinx.coroutines.CoroutineScope import kotlin.collections.set import kotlin.reflect.KClass @@ -36,7 +35,6 @@ class SplitBuilder(val name: Name, val meta: Meta) { class SplitAction( val inputType: KClass, val outputType: KClass, - val scope: CoroutineScope, private val action: SplitBuilder.() -> Unit ) : Action { @@ -57,9 +55,7 @@ class SplitAction( rule(env) - val goal = data.task.pipe(scope) { env.result(it) } - - val res = Data.of(outputType, goal, env.meta) + val res = data.pipe(outputType, meta = env.meta) { env.result(it) } set(env.name, res) } } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/goals.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/goals.kt deleted file mode 100644 index 82693c96..00000000 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/goals.kt +++ /dev/null @@ -1,66 +0,0 @@ -package hep.dataforge.data - -import kotlinx.coroutines.* -import kotlin.coroutines.CoroutineContext -import kotlin.coroutines.EmptyCoroutineContext - -/** - * Create a new [Deferred] with given [dependencies] and execution [block]. The block takes monitor as parameter. - * - * **Important:** Unlike regular deferred, the [Deferred] is started lazily, so the actual calculation is called only when result is requested. - */ -fun goal( - scope: CoroutineScope, - coroutineContext: CoroutineContext = EmptyCoroutineContext, - dependencies: Collection = emptyList(), - block: suspend CoroutineScope.() -> T -): Deferred = scope.async( - coroutineContext + CoroutineMonitor() + Dependencies(dependencies), - start = CoroutineStart.LAZY -) { - dependencies.forEach { job -> - job.start() - job.invokeOnCompletion { error -> - if (error != null) cancel(CancellationException("Dependency $job failed with error: ${error.message}")) - } - } - return@async block() -} - -/** - * Create a one-to-one goal based on existing goal - */ -fun Deferred.pipe( - scope: CoroutineScope, - coroutineContext: CoroutineContext = EmptyCoroutineContext, - block: suspend CoroutineScope.(T) -> R -): Deferred = goal(scope, coroutineContext, listOf(this)) { - block(await()) -} - -/** - * Create a joining goal. - * @param scope the scope for resulting goal. By default use first goal in list - */ -fun Collection>.join( - scope: CoroutineScope, - coroutineContext: CoroutineContext = EmptyCoroutineContext, - block: suspend CoroutineScope.(Collection) -> R -): Deferred = goal(scope, coroutineContext, this) { - block(map { it.await() }) -} - -/** - * A joining goal for a map - * @param K type of the map key - * @param T type of the input goal - * @param R type of the result goal - */ -fun Map>.join( - scope: CoroutineScope, - coroutineContext: CoroutineContext = EmptyCoroutineContext, - block: suspend CoroutineScope.(Map) -> R -): Deferred = goal(scope, coroutineContext, this.values) { - block(mapValues { it.value.await() }) -} - diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt index 512af4a1..324864fc 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/CastDataNode.kt @@ -1,14 +1,23 @@ package hep.dataforge.data +import hep.dataforge.meta.Meta import hep.dataforge.names.NameToken +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Deferred import kotlin.reflect.KClass import kotlin.reflect.full.isSubclassOf -fun Data.safeCast(type: KClass): Data? { - return if (type.isSubclassOf(type)) { - @Suppress("UNCHECKED_CAST") - Data.of(type, task as Deferred, meta) +@Suppress("UNCHECKED_CAST") +fun Data.safeCast(type: KClass): Data? { + return if (this.type.isSubclassOf(type)) { + return object : Data { + override val meta: Meta get() = this@safeCast.meta + override val dependencies: Collection> get() = this@safeCast.dependencies + override val result: Deferred? get() = this@safeCast.result as Deferred + override fun startAsync(scope: CoroutineScope): Deferred = this@safeCast.startAsync(scope) as Deferred + override fun reset() = this@safeCast.reset() + override val type: KClass = type + } } else { null } diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_data.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_data.kt index df6cc33e..00c9e656 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_data.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_data.kt @@ -5,4 +5,4 @@ import kotlinx.coroutines.runBlocking /** * Block the thread and get data content */ -fun Data.get(): T = runBlocking { task.await() } \ No newline at end of file +fun Data.get(): T = runBlocking { await() } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt index 0b439032..175c4d87 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt @@ -85,8 +85,7 @@ class TaskBuilder(val name: String) { val context = this PipeAction( inputType = T::class, - outputType = R::class, - scope = context + outputType = R::class ) { block(context) } } } @@ -103,8 +102,7 @@ class TaskBuilder(val name: String) { val context = this PipeAction( inputType = T::class, - outputType = R::class, - scope = context + outputType = R::class ) { //TODO automatically append task meta result = { data -> @@ -125,8 +123,7 @@ class TaskBuilder(val name: String) { action(from, to) { JoinAction( inputType = T::class, - outputType = R::class, - scope = this + outputType = R::class ) { block(this@action) } } } @@ -144,7 +141,6 @@ class TaskBuilder(val name: String) { JoinAction( inputType = T::class, outputType = R::class, - scope = context, action = { result( actionMeta[TaskModel.MODEL_TARGET_KEY]?.string ?: "@anonymous" @@ -167,8 +163,7 @@ class TaskBuilder(val name: String) { action(from, to) { SplitAction( inputType = T::class, - outputType = R::class, - scope = this + outputType = R::class ) { block(this@action) } } } diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index 9d2efc95..25350006 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -9,7 +9,6 @@ import hep.dataforge.io.MetaFormat import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.withContext import kotlinx.io.nio.asInput @@ -58,7 +57,7 @@ suspend fun Context.readData( } else { null } - val goal = async { + Data(type, externalMeta ?: EmptyMeta){ withContext(Dispatchers.IO) { format.run { Files.newByteChannel(path, StandardOpenOption.READ) @@ -67,6 +66,5 @@ suspend fun Context.readData( } } } - Data.of(type, goal, externalMeta ?: EmptyMeta) } } \ No newline at end of file From 56679cff23942a5a8386e962cf285d2d1c3d04c0 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 9 Aug 2019 20:53:08 +0300 Subject: [PATCH 41/48] Major rework of IO --- .../kotlin/hep/dataforge/data/_data.kt | 8 -- .../data/{checkType.kt => dataJVM.kt} | 6 + .../hep/dataforge/io/BinaryMetaFormat.kt | 2 +- .../kotlin/hep/dataforge/io/Envelope.kt | 18 ++- .../kotlin/hep/dataforge/io/IOFormat.kt | 8 +- .../kotlin/hep/dataforge/io/IOPlugin.kt | 10 +- .../kotlin/hep/dataforge/io/JsonMetaFormat.kt | 5 +- .../kotlin/hep/dataforge/io/MetaFormat.kt | 12 +- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 124 +++++++++--------- .../kotlin/hep/dataforge/io/FileBinary.kt | 1 + .../kotlin/hep/dataforge/io/FileEnvelope.kt | 22 +++- .../hep/dataforge/workspace/fileData.kt | 2 +- 12 files changed, 119 insertions(+), 99 deletions(-) delete mode 100644 dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_data.kt rename dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/{checkType.kt => dataJVM.kt} (70%) diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_data.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_data.kt deleted file mode 100644 index 00c9e656..00000000 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_data.kt +++ /dev/null @@ -1,8 +0,0 @@ -package hep.dataforge.data - -import kotlinx.coroutines.runBlocking - -/** - * Block the thread and get data content - */ -fun Data.get(): T = runBlocking { await() } \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/checkType.kt b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt similarity index 70% rename from dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/checkType.kt rename to dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt index 0b4c602f..f87c4155 100644 --- a/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/checkType.kt +++ b/dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/dataJVM.kt @@ -1,8 +1,14 @@ package hep.dataforge.data +import kotlinx.coroutines.runBlocking import kotlin.reflect.KClass import kotlin.reflect.full.isSuperclassOf +/** + * Block the thread and get data content + */ +fun Data.get(): T = runBlocking { await() } + /** * Check that node is compatible with given type meaning that each element could be cast to the type */ diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt index aa4be898..daf08756 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt @@ -80,7 +80,7 @@ object BinaryMetaFormat : MetaFormat { writeValue(item.value) } is MetaItem.NodeItem -> { - writeObject(item.node) + writeThis(item.node) } } } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt index 08fe44e5..bfbe40bf 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt @@ -1,6 +1,8 @@ package hep.dataforge.io +import hep.dataforge.context.Named import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE +import hep.dataforge.io.IOPlugin.Companion.defaultMetaFormats import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.string @@ -49,15 +51,23 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str */ val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string +/** + * A partially read envelope with meta, but without data + */ +@ExperimentalUnsignedTypes data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?) @Type(ENVELOPE_FORMAT_TYPE) -interface EnvelopeFormat : IOFormat{ - fun readPartial(input: Input): PartialEnvelope +interface EnvelopeFormat : IOFormat, Named { + fun Input.readPartial(formats: Collection = defaultMetaFormats): PartialEnvelope - fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat) + fun Input.readEnvelope(formats: Collection = defaultMetaFormats): Envelope - override fun Output.writeObject(obj: Envelope) = writeEnvelope(obj, JsonMetaFormat) + override fun Input.readThis(): Envelope = readEnvelope() + + fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat) + + override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj) companion object { const val ENVELOPE_FORMAT_TYPE = "envelopeFormat" diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index 9055b249..9cc9a584 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -6,9 +6,9 @@ import kotlinx.io.core.* * And interface for serialization facilities */ interface IOFormat { - fun Output.writeObject(obj: T) - fun Input.readObject(): T + fun Output.writeThis(obj: T) + fun Input.readThis(): T } -fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeObject(obj) } -fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeObject(obj) }.readBytes() \ No newline at end of file +fun IOFormat.writePacket(obj: T): ByteReadPacket = buildPacket { writeThis(obj) } +fun IOFormat.writeBytes(obj: T): ByteArray = buildPacket { writeThis(obj) }.readBytes() \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt index 6416e5c4..b65e4982 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt @@ -6,7 +6,6 @@ import hep.dataforge.context.PluginTag import hep.dataforge.context.content import hep.dataforge.meta.Meta import hep.dataforge.names.Name -import hep.dataforge.names.asName import kotlin.reflect.KClass class IOPlugin(meta: Meta) : AbstractPlugin(meta) { @@ -21,16 +20,15 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { override fun provideTop(target: String): Map { return when (target) { - MetaFormat.META_FORMAT_TYPE -> internalMetaFormats - EnvelopeFormat.ENVELOPE_FORMAT_TYPE -> mapOf( - TaggedEnvelopeFormat.VERSION.asName() to TaggedEnvelopeFormat(metaFormats) - ) + MetaFormat.META_FORMAT_TYPE -> defaultMetaFormats.toMap() + EnvelopeFormat.ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap() else -> super.provideTop(target) } } companion object : PluginFactory { - private val internalMetaFormats = listOf(JsonMetaFormat, BinaryMetaFormat).toMap() + val defaultMetaFormats: List = listOf(JsonMetaFormat, BinaryMetaFormat) + val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat) override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP) override val type: KClass = IOPlugin::class diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index 5dabf499..ab31f78c 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -56,15 +56,16 @@ fun Value.toJson(descriptor: ValueDescriptor? = null): JsonElement { //Use theese methods to customize JSON key mapping private fun NameToken.toJsonKey(descriptor: ItemDescriptor?) = toString() + private fun NodeDescriptor?.getDescriptor(key: String) = this?.items?.get(key) fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject { //TODO search for same name siblings and arrange them into arrays - val map = this.items.entries.associate {(name,item)-> + val map = this.items.entries.associate { (name, item) -> val itemDescriptor = descriptor?.items?.get(name.body) val key = name.toJsonKey(itemDescriptor) - val value = when (item) { + val value = when (item) { is MetaItem.ValueItem -> { item.value.toJson(itemDescriptor as? ValueDescriptor) } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index f658ca32..b0ecafa4 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -15,11 +15,11 @@ interface MetaFormat : IOFormat, Named { override val name: String val key: Short - override fun Output.writeObject(obj: Meta) { + override fun Output.writeThis(obj: Meta) { writeMeta(obj, null) } - override fun Input.readObject(): Meta = readMeta(null) + override fun Input.readThis(): Meta = readMeta(null) fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor? = null) fun Input.readMeta(descriptor: NodeDescriptor? = null): Meta @@ -30,20 +30,20 @@ interface MetaFormat : IOFormat, Named { } fun Meta.toString(format: MetaFormat = JsonMetaFormat): String = buildPacket { - format.run { writeObject(this@toString) } + format.run { writeThis(this@toString) } }.readText() fun Meta.toBytes(format: MetaFormat = JsonMetaFormat): ByteReadPacket = buildPacket { - format.run { writeObject(this@toBytes) } + format.run { writeThis(this@toBytes) } } fun MetaFormat.parse(str: String): Meta { - return ByteReadPacket(str.toByteArray()).readObject() + return ByteReadPacket(str.toByteArray()).readThis() } fun MetaFormat.fromBytes(packet: ByteReadPacket): Meta { - return packet.readObject() + return packet.readThis() } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index dbbe9c58..291f539d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -2,21 +2,75 @@ package hep.dataforge.io import kotlinx.io.core.* -class TaggedEnvelopeFormat(val metaFormats: Collection) : EnvelopeFormat { + +@ExperimentalUnsignedTypes +object TaggedEnvelopeFormat : EnvelopeFormat { + const val VERSION = "DF03" + private const val START_SEQUENCE = "#~" + private const val END_SEQUENCE = "~#\r\n" + private const val TAG_SIZE = 26u + + override val name: String get() = VERSION + + private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) { + writeText(START_SEQUENCE) + writeText(VERSION) + writeShort(metaFormatKey) + writeUInt(metaSize) + writeULong(dataSize) + writeText(END_SEQUENCE) + } + + private fun Input.readTag(): Tag { + val start = readTextExactBytes(2) + if (start != START_SEQUENCE) error("The input is not an envelope") + val version = readTextExactBytes(4) + if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version") + val metaFormatKey = readShort() + val metaLength = readUInt() + val dataLength = readULong() + return Tag(metaFormatKey, metaLength, dataLength) + } override fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat) { - write(this, envelope, format) + val metaBytes = format.writeBytes(envelope.meta) + val tag = Tag(format.key, metaBytes.size.toUInt(), envelope.data?.size ?: 0.toULong()) + writePacket(tag.toBytes()) + writeFully(metaBytes) + envelope.data?.read { copyTo(this@writeEnvelope) } } /** * Read an envelope from input into memory * * @param input an input to read from - * @param metaFormats a collection of meta formats to resolve + * @param formats a collection of meta formats to resolve */ - override fun Input.readObject(): Envelope = read(this, metaFormats) + override fun Input.readEnvelope(formats: Collection): Envelope { + val tag = readTag() - override fun readPartial(input: Input): PartialEnvelope = Companion.readPartial(input, metaFormats) + val metaFormat = formats.find { it.key == tag.metaFormatKey } + ?: error("Meta format with key ${tag.metaFormatKey} not found") + + val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt())) + val meta = metaFormat.run { metaPacket.readThis() } + + val dataBytes = readBytes(tag.dataSize.toInt()) + + return SimpleEnvelope(meta, ArrayBinary(dataBytes)) + } + + override fun Input.readPartial(formats: Collection): PartialEnvelope { + val tag = readTag() + + val metaFormat = formats.find { it.key == tag.metaFormatKey } + ?: error("Meta format with key ${tag.metaFormatKey} not found") + + val metaPacket = ByteReadPacket(readBytes(tag.metaSize.toInt())) + val meta = metaFormat.run { metaPacket.readThis() } + + return PartialEnvelope(meta, TAG_SIZE + tag.metaSize, tag.dataSize) + } private data class Tag( val metaFormatKey: Short, @@ -24,64 +78,4 @@ class TaggedEnvelopeFormat(val metaFormats: Collection) : EnvelopeFo val dataSize: ULong ) - companion object { - const val VERSION = "DF03" - private const val START_SEQUENCE = "#~" - private const val END_SEQUENCE = "~#\r\n" - private const val TAG_SIZE = 26u - - private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) { - writeText(START_SEQUENCE) - writeText(VERSION) - writeShort(metaFormatKey) - writeUInt(metaSize) - writeULong(dataSize) - writeText(END_SEQUENCE) - } - - private fun Input.readTag(): Tag { - val start = readTextExactBytes(2) - if (start != START_SEQUENCE) error("The input is not an envelope") - val version = readTextExactBytes(4) - if (version != VERSION) error("Wrong version of DataForge: expected $VERSION but found $version") - val metaFormatKey = readShort() - val metaLength = readUInt() - val dataLength = readULong() - return Tag(metaFormatKey, metaLength, dataLength) - } - - fun read(input: Input, metaFormats: Collection): Envelope { - val tag = input.readTag() - - val metaFormat = metaFormats.find { it.key == tag.metaFormatKey } - ?: error("Meta format with key ${tag.metaFormatKey} not found") - - val metaPacket = ByteReadPacket(input.readBytes(tag.metaSize.toInt())) - val meta = metaFormat.run { metaPacket.readObject() } - - val dataBytes = input.readBytes(tag.dataSize.toInt()) - - return SimpleEnvelope(meta, ArrayBinary(dataBytes)) - } - - fun readPartial(input: Input, metaFormats: Collection): PartialEnvelope { - val tag = input.readTag() - - val metaFormat = metaFormats.find { it.key == tag.metaFormatKey } - ?: error("Meta format with key ${tag.metaFormatKey} not found") - - val metaPacket = ByteReadPacket(input.readBytes(tag.metaSize.toInt())) - val meta = metaFormat.run { metaPacket.readObject() } - - return PartialEnvelope(meta, TAG_SIZE + tag.metaSize, tag.dataSize) - } - - fun write(out: Output, envelope: Envelope, metaFormat: MetaFormat) { - val metaBytes = metaFormat.writeBytes(envelope.meta) - val tag = Tag(metaFormat.key, metaBytes.size.toUInt(), envelope.data?.size ?: 0.toULong()) - out.writePacket(tag.toBytes()) - out.writeFully(metaBytes) - envelope.data?.read { copyTo(out) } - } - } } \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt index e68d002d..038281d4 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt @@ -7,6 +7,7 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption +@ExperimentalUnsignedTypes class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = null) : RandomAccessBinary { override val size: ULong = size ?: (Files.size(path).toULong() - offset).toULong() diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt index de926db8..fdc4acc0 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt @@ -2,6 +2,7 @@ package hep.dataforge.io import hep.dataforge.meta.Meta import kotlinx.io.nio.asInput +import kotlinx.io.nio.asOutput import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption @@ -13,7 +14,7 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm init { val input = Files.newByteChannel(path, StandardOpenOption.READ).asInput() - partialEnvelope = format.readPartial(input) + partialEnvelope = format.run { input.readPartial() } } override val meta: Meta get() = partialEnvelope.meta @@ -21,4 +22,21 @@ class FileEnvelope internal constructor(val path: Path, val format: EnvelopeForm override val data: Binary? = FileBinary(path, partialEnvelope.dataOffset, partialEnvelope.dataSize) } -fun Path.readEnvelope(format: EnvelopeFormat) = FileEnvelope(this,format) \ No newline at end of file +fun Path.readEnvelope(format: EnvelopeFormat) = FileEnvelope(this, format) + +fun Path.writeEnvelope( + envelope: Envelope, + format: EnvelopeFormat = TaggedEnvelopeFormat, + metaFormat: MetaFormat = JsonMetaFormat +) { + val output = Files.newByteChannel( + this, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ).asOutput() + + with(format) { + output.writeEnvelope(envelope, metaFormat) + } +} \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index 25350006..1367998d 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -62,7 +62,7 @@ suspend fun Context.readData( format.run { Files.newByteChannel(path, StandardOpenOption.READ) .asInput() - .readObject() + .readThis() } } } From d1061a6669c97451f02e8b945d2c1537132590e4 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 10 Aug 2019 17:28:27 +0300 Subject: [PATCH 42/48] Fix for value equality in JS --- .../commonMain/kotlin/hep/dataforge/values/Value.kt | 12 ++++++------ .../kotlin/hep/dataforge/output/html/HtmlOutput.kt | 4 +++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt index 5da65a10..cd68e040 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/values/Value.kt @@ -133,12 +133,12 @@ class NumberValue(override val number: Number) : Value { override fun equals(other: Any?): Boolean { if (other !is Value) return false return when (number) { - is Short -> number == other.number.toShort() - is Long -> number == other.number.toLong() - is Byte -> number == other.number.toByte() - is Int -> number == other.number.toInt() - is Float -> number == other.number.toFloat() - is Double -> number == other.number.toDouble() + is Short -> number.toShort() == other.number.toShort() + is Long -> number.toLong() == other.number.toLong() + is Byte -> number.toByte() == other.number.toByte() + is Int -> number.toInt() == other.number.toInt() + is Float -> number.toFloat() == other.number.toFloat() + is Double -> number.toDouble() == other.number.toDouble() else -> number.toString() == other.number.toString() } } diff --git a/dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt b/dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt index c7aea610..b54b7eb7 100644 --- a/dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt +++ b/dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt @@ -27,7 +27,8 @@ class HtmlOutput(override val context: Context, private val consumer: T } else { val value = cache[obj::class] if (value == null) { - val answer = context.top>(HTML_CONVERTER_TYPE).values.firstOrNull { it.type.isInstance(obj) } + val answer = + context.top>(HTML_CONVERTER_TYPE).values.firstOrNull { it.type.isInstance(obj) } if (answer != null) { cache[obj::class] = answer answer @@ -39,6 +40,7 @@ class HtmlOutput(override val context: Context, private val consumer: T } } context.launch(Dispatchers.Output) { + @Suppress("UNCHECKED_CAST") (builder as HtmlBuilder).run { consumer.render(obj) } } } From a8c91005392f95b1e08dfe16945ca043290e3050 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 10 Aug 2019 17:51:58 +0300 Subject: [PATCH 43/48] Fixed scripting test by randomly placing random strings in random places --- build.gradle.kts | 2 +- .../src/jvmMain/kotlin/hep/dataforge/scripting/Builders.kt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 758279ab..016a350e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,7 +3,7 @@ plugins { id("scientifik.publish") version "0.1.4" apply false } -val dataforgeVersion by extra("0.1.3-dev-11") +val dataforgeVersion by extra("0.1.3") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-scripting/src/jvmMain/kotlin/hep/dataforge/scripting/Builders.kt b/dataforge-scripting/src/jvmMain/kotlin/hep/dataforge/scripting/Builders.kt index 31f3a508..2df5a183 100644 --- a/dataforge-scripting/src/jvmMain/kotlin/hep/dataforge/scripting/Builders.kt +++ b/dataforge-scripting/src/jvmMain/kotlin/hep/dataforge/scripting/Builders.kt @@ -8,6 +8,7 @@ import hep.dataforge.workspace.WorkspaceBuilder import java.io.File import kotlin.script.experimental.api.* import kotlin.script.experimental.host.toScriptSource +import kotlin.script.experimental.jvm.defaultJvmScriptingHostConfiguration import kotlin.script.experimental.jvm.dependenciesFromCurrentContext import kotlin.script.experimental.jvm.jvm import kotlin.script.experimental.jvmhost.BasicJvmScriptingHost @@ -24,6 +25,7 @@ object Builders { jvm { dependenciesFromCurrentContext(wholeClasspath = true) } + hostConfiguration(defaultJvmScriptingHostConfiguration) } val evaluationConfiguration = ScriptEvaluationConfiguration { From 8c32eaeb6b66cc03501254b5d5c50686d08c6644 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 11 Aug 2019 18:35:03 +0300 Subject: [PATCH 44/48] Binary and file data update --- dataforge-data/build.gradle.kts | 2 - dataforge-io/build.gradle.kts | 4 +- .../kotlin/hep/dataforge/io/Binary.kt | 23 ++++++++- .../kotlin/hep/dataforge/io/Envelope.kt | 37 ++++++-------- .../kotlin/hep/dataforge/io/EnvelopeFormat.kt | 31 ++++++++++++ .../kotlin/hep/dataforge/io/FileEnvelope.kt | 3 +- .../hep/dataforge/workspace/dataUtils.kt | 14 ++++++ .../hep/dataforge/workspace/fileData.kt | 50 +++++++++++++------ 8 files changed, 121 insertions(+), 43 deletions(-) create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt create mode 100644 dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/dataUtils.kt diff --git a/dataforge-data/build.gradle.kts b/dataforge-data/build.gradle.kts index 1faff685..793f551b 100644 --- a/dataforge-data/build.gradle.kts +++ b/dataforge-data/build.gradle.kts @@ -5,8 +5,6 @@ plugins { val coroutinesVersion: String = Scientifik.coroutinesVersion kotlin { - jvm() - js() sourceSets { val commonMain by getting{ dependencies { diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index 3ed7ab62..b4306efc 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -12,12 +12,12 @@ scientifik{ kotlin { sourceSets { - val commonMain by getting{ + commonMain{ dependencies { api(project(":dataforge-context")) } } - val jsMain by getting{ + jsMain{ dependencies{ api(npm("text-encoding")) } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt index a0374968..d8fe9ad7 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt @@ -2,6 +2,7 @@ package hep.dataforge.io import kotlinx.io.core.ByteReadPacket import kotlinx.io.core.Input +import kotlinx.io.core.buildPacket import kotlinx.io.core.readBytes /** @@ -52,9 +53,29 @@ object EmptyBinary : RandomAccessBinary { } class ArrayBinary(val array: ByteArray) : RandomAccessBinary { - override val size: ULong = array.size.toULong() + override val size: ULong get() = array.size.toULong() override fun read(from: UInt, size: UInt, block: Input.() -> R): R { return ByteReadPacket(array, from.toInt(), size.toInt()).block() } +} + +/** + * Read given binary as object using given format + */ +fun Binary.readWith(format: IOFormat): T = format.run { + read { + readThis() + } +} + +/** + * Write this object to a binary + * TODO make a lazy binary that does not use intermediate array + */ +fun T.writeWith(format: IOFormat): Binary = format.run{ + val packet = buildPacket { + writeThis(this@writeWith) + } + return@run ArrayBinary(packet.readBytes()) } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt index bfbe40bf..c2abca21 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt @@ -1,14 +1,9 @@ package hep.dataforge.io -import hep.dataforge.context.Named -import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE -import hep.dataforge.io.IOPlugin.Companion.defaultMetaFormats +import hep.dataforge.meta.Laminate import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.string -import hep.dataforge.provider.Type -import kotlinx.io.core.Input -import kotlinx.io.core.Output interface Envelope { val meta: Meta @@ -52,24 +47,20 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string /** - * A partially read envelope with meta, but without data + * An envelope, which wraps existing envelope and adds one or several additional layers of meta */ -@ExperimentalUnsignedTypes -data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?) +class ProxyEnvelope(val source: Envelope, vararg meta: Meta) : Envelope { + override val meta: Laminate = Laminate(*meta, source.meta) + override val data: Binary? get() = source.data +} -@Type(ENVELOPE_FORMAT_TYPE) -interface EnvelopeFormat : IOFormat, Named { - fun Input.readPartial(formats: Collection = defaultMetaFormats): PartialEnvelope - - fun Input.readEnvelope(formats: Collection = defaultMetaFormats): Envelope - - override fun Input.readThis(): Envelope = readEnvelope() - - fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat) - - override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj) - - companion object { - const val ENVELOPE_FORMAT_TYPE = "envelopeFormat" +/** + * Add few meta layers to existing envelope + */ +fun Envelope.withMetaLayers(vararg layers: Meta): Envelope { + return when { + layers.isEmpty() -> this + this is ProxyEnvelope -> ProxyEnvelope(source, *layers, *this.meta.layers.toTypedArray()) + else -> ProxyEnvelope(this, *layers) } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt new file mode 100644 index 00000000..24217e14 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt @@ -0,0 +1,31 @@ +package hep.dataforge.io + +import hep.dataforge.context.Named +import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE +import hep.dataforge.meta.Meta +import hep.dataforge.provider.Type +import kotlinx.io.core.Input +import kotlinx.io.core.Output + +/** + * A partially read envelope with meta, but without data + */ +@ExperimentalUnsignedTypes +data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?) + +@Type(ENVELOPE_FORMAT_TYPE) +interface EnvelopeFormat : IOFormat, Named { + fun Input.readPartial(formats: Collection = IOPlugin.defaultMetaFormats): PartialEnvelope + + fun Input.readEnvelope(formats: Collection = IOPlugin.defaultMetaFormats): Envelope + + override fun Input.readThis(): Envelope = readEnvelope() + + fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat = JsonMetaFormat) + + override fun Output.writeThis(obj: Envelope) = writeEnvelope(obj) + + companion object { + const val ENVELOPE_FORMAT_TYPE = "envelopeFormat" + } +} \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt index fdc4acc0..0c54012c 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt @@ -39,4 +39,5 @@ fun Path.writeEnvelope( with(format) { output.writeEnvelope(envelope, metaFormat) } -} \ No newline at end of file +} + diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/dataUtils.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/dataUtils.kt new file mode 100644 index 00000000..f6d27774 --- /dev/null +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/dataUtils.kt @@ -0,0 +1,14 @@ +package hep.dataforge.workspace + +import hep.dataforge.data.Data +import hep.dataforge.io.Envelope +import hep.dataforge.io.IOFormat +import hep.dataforge.io.readWith +import kotlin.reflect.KClass + +/** + * Convert an [Envelope] to a data via given format. The actual parsing is done lazily. + */ +fun Envelope.toData(type: KClass, format: IOFormat): Data = Data(type, meta) { + data?.readWith(format) ?: error("Can't convert envelope without data to Data") +} \ No newline at end of file diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index 1367998d..f20e18cf 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -1,11 +1,8 @@ package hep.dataforge.workspace -import hep.dataforge.context.Context import hep.dataforge.data.Data import hep.dataforge.descriptors.NodeDescriptor -import hep.dataforge.io.IOFormat -import hep.dataforge.io.JsonMetaFormat -import hep.dataforge.io.MetaFormat +import hep.dataforge.io.* import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.coroutines.Dispatchers @@ -44,11 +41,19 @@ suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescripto } } -suspend fun Context.readData( +/** + * Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file. + * @param type explicit type of data read + * @param format binary format + * @param envelopeFormat the format of envelope. If null, file is read directly + * @param metaFile the relative file for optional meta override + * @param metaFileFormat the meta format for override + */ +suspend fun Path.readData( type: KClass, - path: Path, format: IOFormat, - metaFile: Path = path.resolveSibling("${path.fileName}.meta"), + envelopeFormat: EnvelopeFormat? = null, + metaFile: Path = resolveSibling("$fileName.meta"), metaFileFormat: MetaFormat = JsonMetaFormat ): Data { return coroutineScope { @@ -57,14 +62,31 @@ suspend fun Context.readData( } else { null } - Data(type, externalMeta ?: EmptyMeta){ - withContext(Dispatchers.IO) { - format.run { - Files.newByteChannel(path, StandardOpenOption.READ) - .asInput() - .readThis() + if (envelopeFormat == null) { + Data(type, externalMeta ?: EmptyMeta) { + withContext(Dispatchers.IO) { + format.run { + Files.newByteChannel(this@readData, StandardOpenOption.READ) + .asInput() + .readThis() + } } } + } else { + withContext(Dispatchers.IO) { + readEnvelope(envelopeFormat).let { + if (externalMeta == null) { + it + } else { + it.withMetaLayers(externalMeta) + } + }.toData(type, format) + } } } -} \ No newline at end of file +} + +//suspend fun Path.writeData( +// data: Data, +// format: IOFormat, +// ) \ No newline at end of file From 58a0584cae5db25581bedfbda9ad8b646be620a4 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 16 Aug 2019 15:08:56 +0300 Subject: [PATCH 45/48] Compatibility for Json top level arrays and nested arrays added. --- .../kotlin/hep/dataforge/data/GroupBuilder.kt | 75 ------------------- .../kotlin/hep/dataforge/data/GroupRule.kt | 68 +++++++++++++++++ .../kotlin/hep/dataforge/io/JsonMetaFormat.kt | 59 ++++++++++----- .../kotlin/hep/dataforge/io/MetaFormat.kt | 2 +- .../kotlin/hep/dataforge/io/MetaFormatTest.kt | 33 +++++++- 5 files changed, 137 insertions(+), 100 deletions(-) delete mode 100644 dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupBuilder.kt create mode 100644 dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupBuilder.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupBuilder.kt deleted file mode 100644 index 409be5bf..00000000 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupBuilder.kt +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2015 Alexander Nozik. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package hep.dataforge.data - -import hep.dataforge.meta.Meta -import hep.dataforge.meta.get -import hep.dataforge.meta.string - -interface GroupRule { - operator fun invoke(node: DataNode): Map> -} - -/** - * The class to builder groups of content with annotation defined rules - * - * @author Alexander Nozik - */ - -object GroupBuilder { - - /** - * Create grouping rule that creates groups for different values of value - * field with name [key] - * - * @param key - * @param defaultTagValue - * @return - */ - fun byValue(key: String, defaultTagValue: String): GroupRule = object : - GroupRule { - override fun invoke(node: DataNode): Map> { - val map = HashMap>() - - node.dataSequence().forEach { (name, data) -> - val tagValue = data.meta[key]?.string ?: defaultTagValue - map.getOrPut(tagValue) { DataNode.builder(node.type) }[name] = data - } - - return map.mapValues { it.value.build() } - } - } - - - // @ValueDef(key = "byValue", required = true, info = "The name of annotation value by which grouping should be made") -// @ValueDef( -// key = "defaultValue", -// def = "default", -// info = "Default value which should be used for content in which the grouping value is not presented" -// ) - fun byMeta(config: Meta): GroupRule { - //TODO expand grouping options - return config["byValue"]?.string?.let { - byValue( - it, - config["defaultValue"]?.string ?: "default" - ) - } - ?: object : GroupRule { - override fun invoke(node: DataNode): Map> = mapOf("" to node) - } - } -} diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt new file mode 100644 index 00000000..5cfc55e8 --- /dev/null +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/GroupRule.kt @@ -0,0 +1,68 @@ +/* + * Copyright 2015 Alexander Nozik. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hep.dataforge.data + +import hep.dataforge.meta.Meta +import hep.dataforge.meta.get +import hep.dataforge.meta.string + +interface GroupRule { + operator fun invoke(node: DataNode): Map> + + companion object{ + /** + * Create grouping rule that creates groups for different values of value + * field with name [key] + * + * @param key + * @param defaultTagValue + * @return + */ + fun byValue(key: String, defaultTagValue: String): GroupRule = object : + GroupRule { + override fun invoke(node: DataNode): Map> { + val map = HashMap>() + + node.dataSequence().forEach { (name, data) -> + val tagValue = data.meta[key]?.string ?: defaultTagValue + map.getOrPut(tagValue) { DataNode.builder(node.type) }[name] = data + } + + return map.mapValues { it.value.build() } + } + } + + + // @ValueDef(key = "byValue", required = true, info = "The name of annotation value by which grouping should be made") +// @ValueDef( +// key = "defaultValue", +// def = "default", +// info = "Default value which should be used for content in which the grouping value is not presented" +// ) + fun byMeta(config: Meta): GroupRule { + //TODO expand grouping options + return config["byValue"]?.string?.let { + byValue( + it, + config["defaultValue"]?.string ?: "default" + ) + } + ?: object : GroupRule { + override fun invoke(node: DataNode): Map> = mapOf("" to node) + } + } + } +} diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index ab31f78c..64b368e2 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -25,8 +25,8 @@ object JsonMetaFormat : MetaFormat { override val key: Short = 0x4a53//"JS" override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) { - val str = meta.toJson().toString() - writeText(str) + val json = meta.toJson(descriptor) + writeText(json.toString()) } override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { @@ -78,17 +78,46 @@ fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject { return JsonObject(map) } +fun JsonObject.toMeta(descriptor: NodeDescriptor? = null): JsonMeta = JsonMeta(this, descriptor) -fun JsonObject.toMeta(descriptor: NodeDescriptor? = null) = JsonMeta(this, descriptor) +fun JsonPrimitive.toValue(descriptor: ValueDescriptor?): Value { + return when (this) { + JsonNull -> Null + else -> this.content.parseValue() // Optimize number and boolean parsing + } +} -class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : MetaBase() { - - private fun JsonPrimitive.toValue(descriptor: ValueDescriptor?): Value { - return when (this) { - JsonNull -> Null - else -> this.content.parseValue() // Optimize number and boolean parsing +fun JsonElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem = when (this) { + is JsonPrimitive -> { + val value = this.toValue(descriptor as? ValueDescriptor) + MetaItem.ValueItem(value) + } + is JsonObject -> { + val meta = toMeta(descriptor as? NodeDescriptor) + MetaItem.NodeItem(meta) + } + is JsonArray -> { + if (this.all { it is JsonPrimitive }) { + val value = if (isEmpty()) { + Null + } else { + ListValue( + map { + //We already checked that all values are primitives + (it as JsonPrimitive).toValue(descriptor as? ValueDescriptor) + } + ) + } + MetaItem.ValueItem(value) + } else { + json { + "@value" to this@toMetaItem + }.toMetaItem(descriptor) } } +} + +class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : MetaBase() { @Suppress("UNCHECKED_CAST") private operator fun MutableMap>.set(key: String, value: JsonElement): Unit { @@ -114,17 +143,7 @@ class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : M this[name] = MetaItem.ValueItem(listValue) as MetaItem } else -> value.forEachIndexed { index, jsonElement -> - when (jsonElement) { - is JsonObject -> { - this["$name[$index]"] = - MetaItem.NodeItem(jsonElement.toMeta(itemDescriptor as? NodeDescriptor)) - } - is JsonPrimitive -> { - this["$name[$index]"] = - MetaItem.ValueItem(jsonElement.toValue(itemDescriptor as? ValueDescriptor)) - } - is JsonArray -> TODO("Nested arrays not supported") - } + this["$name[$index]"] = jsonElement.toMetaItem(itemDescriptor) } } } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index b0ecafa4..3185e29e 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -29,7 +29,7 @@ interface MetaFormat : IOFormat, Named { } } -fun Meta.toString(format: MetaFormat = JsonMetaFormat): String = buildPacket { +fun Meta.toString(format: MetaFormat): String = buildPacket { format.run { writeThis(this@toString) } }.readText() diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt index 8348f7b0..16d946e3 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt @@ -1,9 +1,9 @@ package hep.dataforge.io -import hep.dataforge.meta.Meta -import hep.dataforge.meta.buildMeta -import hep.dataforge.meta.get -import hep.dataforge.meta.seal +import hep.dataforge.meta.* +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.json +import kotlinx.serialization.json.jsonArray import kotlin.test.Test import kotlin.test.assertEquals @@ -45,4 +45,29 @@ class MetaFormatTest { assertEquals(meta, result) } + @Test + fun testJsonToMeta(){ + val json = jsonArray{ + //top level array + +jsonArray { + +JsonPrimitive(88) + +json{ + "c" to "aasdad" + "d" to true + } + } + +"value" + +jsonArray { + +JsonPrimitive(1.0) + +JsonPrimitive(2.0) + +JsonPrimitive(3.0) + } + } + val meta = json.toMetaItem().node!! + + assertEquals(true, meta["@value[0].@value[1].d"].boolean) + assertEquals("value", meta["@value[1]"].string) + assertEquals(listOf(1.0,2.0,3.0),meta["@value[2"].value?.list?.map{it.number.toDouble()}) + } + } \ No newline at end of file From 578b4ede21793dde81c51618445d6c8f2a8fac6d Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 16 Aug 2019 16:30:10 +0300 Subject: [PATCH 46/48] Meta and Name serialization tests --- .../hep/dataforge/io/MetaSerializerTest.kt | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt new file mode 100644 index 00000000..ea5854d2 --- /dev/null +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt @@ -0,0 +1,33 @@ +package hep.dataforge.io + +import hep.dataforge.meta.buildMeta +import hep.dataforge.names.toName +import kotlinx.serialization.json.Json +import kotlin.test.Test +import kotlin.test.assertEquals + +class MetaSerializerTest { + @Test + fun testMetaSerialization() { + val meta = buildMeta { + "a" to 22 + "node" to { + "b" to "DDD" + "c" to 11.1 + "array" to doubleArrayOf(1.0, 2.0, 3.0) + } + } + + val string = Json.indented.stringify(MetaSerializer, meta) + val restored = Json.plain.parse(MetaSerializer, string) + assertEquals(restored, meta) + } + + @Test + fun testNameSerialization() { + val name = "a.b.c".toName() + val string = Json.indented.stringify(NameSerializer, name) + val restored = Json.plain.parse(NameSerializer, string) + assertEquals(restored, name) + } +} \ No newline at end of file From 27c510f5d070609277ee498615ee5d2c29abf0f0 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 17 Aug 2019 10:32:58 +0300 Subject: [PATCH 47/48] Removed inline from Name due to problems with equality and serialization. --- .../kotlin/hep/dataforge/names/Name.kt | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt index 40502c8c..070a8f75 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt @@ -6,10 +6,9 @@ package hep.dataforge.names * The name is a dot separated list of strings like `token1.token2.token3`. * Each token could contain additional index in square brackets. */ -inline class Name constructor(val tokens: List) { +class Name(val tokens: List) { - val length - get() = tokens.size + val length get() = tokens.size /** * First token of the name or null if it is empty @@ -35,6 +34,23 @@ inline class Name constructor(val tokens: List) { override fun toString(): String = tokens.joinToString(separator = NAME_SEPARATOR) { it.toString() } + override fun equals(other: Any?): Boolean { + return when (other) { + is Name -> this.tokens == other.tokens + is NameToken -> this.length == 1 && this.tokens.first() == other + else -> false + } + } + + override fun hashCode(): Int { + return if (tokens.size == 1) { + tokens.first().hashCode() + } else { + tokens.hashCode() + } + } + + companion object { const val NAME_SEPARATOR = "." } From d16d051a1c54ddfdb36b2219d6c9d915c38293c6 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 17 Aug 2019 17:10:40 +0300 Subject: [PATCH 48/48] 0.1.3 release --- .../src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt | 2 +- dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt index 5f0f5845..4acae87f 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/JoinAction.kt @@ -29,7 +29,7 @@ class JoinGroupBuilder(val actionMeta: Meta) { */ fun byValue(tag: String, defaultTag: String = "@default", action: JoinGroup.() -> Unit) { groupRules += { node -> - GroupBuilder.byValue(tag, defaultTag).invoke(node).map { + GroupRule.byValue(tag, defaultTag).invoke(node).map { JoinGroup(it.key, it.value).apply(action) } } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt index d8fe9ad7..67c962dd 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Binary.kt @@ -25,6 +25,7 @@ interface Binary { /** * A [Binary] with addition random access functionality. It by default allows multiple [read] operations. */ +@ExperimentalUnsignedTypes interface RandomAccessBinary : Binary { /** * Read at most [size] of bytes starting at [from] offset from the beginning of the binary. @@ -39,10 +40,12 @@ fun Binary.readAll(): ByteReadPacket = read { ByteReadPacket(this.readBytes()) } +@ExperimentalUnsignedTypes fun RandomAccessBinary.readPacket(from: UInt, size: UInt): ByteReadPacket = read(from, size) { ByteReadPacket(this.readBytes()) } +@ExperimentalUnsignedTypes object EmptyBinary : RandomAccessBinary { override val size: ULong = 0.toULong() @@ -52,6 +55,7 @@ object EmptyBinary : RandomAccessBinary { } } +@ExperimentalUnsignedTypes class ArrayBinary(val array: ByteArray) : RandomAccessBinary { override val size: ULong get() = array.size.toULong()