From 45f158817551cf277688d42c3cdb8994f6401023 Mon Sep 17 00:00:00 2001 From: Iaroslav Postovalov Date: Tue, 15 Jun 2021 13:18:40 +0700 Subject: [PATCH] Generated benchmarking reports --- benchmarks/build.gradle.kts | 8 +- .../kmath/benchmarks/JafamaBenchmark.kt | 6 +- buildSrc/build.gradle.kts | 16 ++- .../kscience/kmath/benchmarks/JmhReport.kt | 60 ++++++++++ .../benchmarks/addBenchmarkProperties.kt | 105 ++++++++++++++++++ kmath-ast/build.gradle.kts | 12 +- kmath-jafama/README.md | 97 +++------------- kmath-jafama/docs/README-TEMPLATE.md | 93 +--------------- settings.gradle.kts | 2 +- 9 files changed, 210 insertions(+), 189 deletions(-) create mode 100644 buildSrc/src/main/kotlin/space/kscience/kmath/benchmarks/JmhReport.kt create mode 100644 buildSrc/src/main/kotlin/space/kscience/kmath/benchmarks/addBenchmarkProperties.kt diff --git a/benchmarks/build.gradle.kts b/benchmarks/build.gradle.kts index f171437d3..2198ac5d6 100644 --- a/benchmarks/build.gradle.kts +++ b/benchmarks/build.gradle.kts @@ -1,3 +1,7 @@ +@file:Suppress("UNUSED_VARIABLE") + +import space.kscience.kmath.benchmarks.addBenchmarkProperties + plugins { kotlin("multiplatform") kotlin("plugin.allopen") @@ -32,7 +36,7 @@ kotlin { implementation(project(":kmath-dimensions")) implementation(project(":kmath-for-real")) implementation(project(":kmath-jafama")) - implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:0.3.0") + implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:0.3.1") } } @@ -130,3 +134,5 @@ tasks.withType { readme { maturity = ru.mipt.npm.gradle.Maturity.EXPERIMENTAL } + +addBenchmarkProperties() diff --git a/benchmarks/src/jvmMain/kotlin/space/kscience/kmath/benchmarks/JafamaBenchmark.kt b/benchmarks/src/jvmMain/kotlin/space/kscience/kmath/benchmarks/JafamaBenchmark.kt index 260eea2b6..24a730375 100644 --- a/benchmarks/src/jvmMain/kotlin/space/kscience/kmath/benchmarks/JafamaBenchmark.kt +++ b/benchmarks/src/jvmMain/kotlin/space/kscience/kmath/benchmarks/JafamaBenchmark.kt @@ -18,17 +18,17 @@ import kotlin.random.Random @State(Scope.Benchmark) internal class JafamaBenchmark { @Benchmark - fun jafamaBench(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x -> + fun jafama(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x -> JafamaDoubleField { x * power(x, 4) * exp(x) / cos(x) + sin(x) } } @Benchmark - fun coreBench(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x -> + fun core(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x -> DoubleField { x * power(x, 4) * exp(x) / cos(x) + sin(x) } } @Benchmark - fun strictJafamaBench(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x -> + fun strictJafama(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x -> StrictJafamaDoubleField { x * power(x, 4) * exp(x) / cos(x) + sin(x) } } diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 7ca4df19d..8ff1c4369 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -1,5 +1,19 @@ plugins { `kotlin-dsl` + kotlin("plugin.serialization") version "1.4.31" } -repositories.mavenCentral() +repositories { + gradlePluginPortal() + mavenCentral() +} + +dependencies { + api("org.jetbrains.kotlinx:kotlinx-serialization-json:1.1.0") + api("ru.mipt.npm:gradle-tools:0.9.10") + api("org.jetbrains.kotlinx:kotlinx-benchmark-plugin:0.3.1") +} + +kotlin.sourceSets.all { + languageSettings.useExperimentalAnnotation("kotlin.ExperimentalStdlibApi") +} diff --git a/buildSrc/src/main/kotlin/space/kscience/kmath/benchmarks/JmhReport.kt b/buildSrc/src/main/kotlin/space/kscience/kmath/benchmarks/JmhReport.kt new file mode 100644 index 000000000..eaa0f59d8 --- /dev/null +++ b/buildSrc/src/main/kotlin/space/kscience/kmath/benchmarks/JmhReport.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2018-2021 KMath contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package space.kscience.kmath.benchmarks + +import kotlinx.serialization.Serializable + +@Serializable +data class JmhReport( + val jmhVersion: String, + val benchmark: String, + val mode: String, + val threads: Int, + val forks: Int, + val jvm: String, + val jvmArgs: List, + val jdkVersion: String, + val vmName: String, + val vmVersion: String, + val warmupIterations: Int, + val warmupTime: String, + val warmupBatchSize: Int, + val measurementIterations: Int, + val measurementTime: String, + val measurementBatchSize: Int, + val params: Map = emptyMap(), + val primaryMetric: PrimaryMetric, + val secondaryMetrics: Map, +) { + interface Metric { + val score: Double + val scoreError: Double + val scoreConfidence: List + val scorePercentiles: Map + val scoreUnit: String + } + + @Serializable + data class PrimaryMetric( + override val score: Double, + override val scoreError: Double, + override val scoreConfidence: List, + override val scorePercentiles: Map, + override val scoreUnit: String, + val rawDataHistogram: List>>>? = null, + val rawData: List>? = null, + ) : Metric + + @Serializable + data class SecondaryMetric( + override val score: Double, + override val scoreError: Double, + override val scoreConfidence: List, + override val scorePercentiles: Map, + override val scoreUnit: String, + val rawData: List>, + ) : Metric +} diff --git a/buildSrc/src/main/kotlin/space/kscience/kmath/benchmarks/addBenchmarkProperties.kt b/buildSrc/src/main/kotlin/space/kscience/kmath/benchmarks/addBenchmarkProperties.kt new file mode 100644 index 000000000..31d952157 --- /dev/null +++ b/buildSrc/src/main/kotlin/space/kscience/kmath/benchmarks/addBenchmarkProperties.kt @@ -0,0 +1,105 @@ +/* + * Copyright 2018-2021 KMath contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package space.kscience.kmath.benchmarks + +import kotlinx.serialization.* +import org.gradle.api.Project +import java.time.* +import java.time.format.* +import java.time.temporal.ChronoField.* + +private val ISO_DATE_TIME: DateTimeFormatter = DateTimeFormatterBuilder().run { + parseCaseInsensitive() + appendValue(YEAR, 4, 10, SignStyle.EXCEEDS_PAD) + appendLiteral('-') + appendValue(MONTH_OF_YEAR, 2) + appendLiteral('-') + appendValue(DAY_OF_MONTH, 2) + appendLiteral('T') + appendValue(HOUR_OF_DAY, 2) + appendLiteral('.') + appendValue(MINUTE_OF_HOUR, 2) + optionalStart() + appendLiteral('.') + appendValue(SECOND_OF_MINUTE, 2) + optionalStart() + appendFraction(NANO_OF_SECOND, 0, 9, true) + optionalStart() + appendOffsetId() + optionalStart() + appendLiteral('[') + parseCaseSensitive() + appendZoneRegionId() + appendLiteral(']') + toFormatter() +} + +private fun noun(number: Number, singular: String, plural: String) = if (number.toLong() == 1L) singular else plural + +fun Project.addBenchmarkProperties() { + val benchmarksProject = this + rootProject.subprojects.forEach { p -> + p.extensions.findByType(ru.mipt.npm.gradle.KScienceReadmeExtension::class.java)?.run { + benchmarksProject.extensions.findByType(kotlinx.benchmark.gradle.BenchmarksExtension::class.java)?.configurations?.forEach { cfg -> + // TODO remove this hack when https://github.com/mipt-npm/gradle-tools/pull/15 is merged + @Suppress("UNCHECKED_CAST") + (javaClass.getDeclaredField("properties") + .also { + it.isAccessible = true + }[this] as MutableMap Any?>)["benchmark${cfg.name.replaceFirstChar(Char::uppercase)}"] = + { + val launches = benchmarksProject.buildDir.resolve("reports/benchmarks/${cfg.name}") + + val resDirectory = launches.listFiles()?.maxByOrNull { + LocalDateTime.parse(it.name, ISO_DATE_TIME).atZone(ZoneId.systemDefault()).toInstant() + } + + if (resDirectory == null) { + "> **Can't find appropriate benchmark data. Try generating readme files after running benchmarks**." + } else { + val reports = + kotlinx.serialization.json.Json.decodeFromString>( + resDirectory.resolve("jvm.json").readText() + ) + + buildString { + appendLine("
") + appendLine("") + appendLine("Report for benchmark configuration ${cfg.name}") + appendLine("") + appendLine() + val first = reports.first() + + appendLine("* Run on ${first.vmName} (build ${first.vmVersion}) with Java process:") + appendLine() + appendLine("```") + appendLine("${first.jvm} ${ + first.jvmArgs.joinToString(" ") + }") + appendLine("```") + + appendLine("* JMH ${first.jmhVersion} was used in `${first.mode}` mode with ${first.warmupIterations} warmup ${ + noun(first.warmupIterations, "iteration", "iterations") + } by ${first.warmupTime} and ${first.measurementIterations} measurement ${ + noun(first.measurementIterations, "iteration", "iterations") + } by ${first.measurementTime}.") + + appendLine() + appendLine("| Benchmark | Score |") + appendLine("|:---------:|:-----:|") + + reports.forEach { report -> + appendLine("|`${report.benchmark}`|${report.primaryMetric.score} ± ${report.primaryMetric.scoreError} ${report.primaryMetric.scoreUnit}|") + } + + appendLine("
") + } + } + } + } + } + } +} diff --git a/kmath-ast/build.gradle.kts b/kmath-ast/build.gradle.kts index 89647c9e9..8209a0dad 100644 --- a/kmath-ast/build.gradle.kts +++ b/kmath-ast/build.gradle.kts @@ -62,25 +62,21 @@ readme { feature( id = "expression-language", - description = "Expression language and its parser", ref = "src/commonMain/kotlin/space/kscience/kmath/ast/parser.kt" - ) + ) { "Expression language and its parser" } feature( id = "mst-jvm-codegen", - description = "Dynamic MST to JVM bytecode compiler", ref = "src/jvmMain/kotlin/space/kscience/kmath/asm/asm.kt" - ) + ) { "Dynamic MST to JVM bytecode compiler" } feature( id = "mst-js-codegen", - description = "Dynamic MST to JS compiler", ref = "src/jsMain/kotlin/space/kscience/kmath/estree/estree.kt" - ) + ) { "Dynamic MST to JS compiler" } feature( id = "rendering", - description = "Extendable MST rendering", ref = "src/commonMain/kotlin/space/kscience/kmath/ast/rendering/MathRenderer.kt" - ) + ) { "Extendable MST rendering" } } diff --git a/kmath-jafama/README.md b/kmath-jafama/README.md index 9b439c5c2..ef8fcd352 100644 --- a/kmath-jafama/README.md +++ b/kmath-jafama/README.md @@ -50,95 +50,24 @@ fun main() { ## Performance -According to benchmarking data, on Hotspot Jafama functions are 20% faster than JDK math. On GraalVM, they are slower. +According to KMath benchmarks on GraalVM, Jafama functions are slower than JDK math; however, there are indications that on Hotspot Jafama is a bit faster.
-Raw data: + +Report for benchmark configuration jafamaDouble + -**Hotspot** +* Run on OpenJDK 64-Bit Server VM (build 11.0.11+8-jvmci-21.1-b05) with Java process: ``` -jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.coreBench - -Warm-up 1: 11.447 ops/s -Iteration 1: 13.354 ops/s -Iteration 2: 14.237 ops/s -Iteration 3: 14.708 ops/s -Iteration 4: 14.629 ops/s -Iteration 5: 14.692 ops/s - -14.324 ±(99.9%) 2.217 ops/s [Average] - (min, avg, max) = (13.354, 14.324, 14.708), stdev = 0.576 - CI (99.9%): [12.107, 16.541] (assumes normal distribution) - -jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.jafamaBench - -Warm-up 1: 15.628 ops/s -Iteration 1: 15.991 ops/s -Iteration 2: 16.633 ops/s -Iteration 3: 16.583 ops/s -Iteration 4: 16.716 ops/s -Iteration 5: 16.762 ops/s - -16.537 ±(99.9%) 1.205 ops/s [Average] - (min, avg, max) = (15.991, 16.537, 16.762), stdev = 0.313 - CI (99.9%): [15.332, 17.743] (assumes normal distribution) - -jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafamaBench - -Warm-up 1: 13.378 ops/s -Iteration 1: 15.049 ops/s -Iteration 2: 14.468 ops/s -Iteration 3: 14.469 ops/s -Iteration 4: 14.753 ops/s -Iteration 5: 14.958 ops/s - -14.739 ±(99.9%) 1.038 ops/s [Average] - (min, avg, max) = (14.468, 14.739, 15.049), stdev = 0.269 - CI (99.9%): [13.701, 15.777] (assumes normal distribution) -``` - -**GraalVM** - -``` -jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.coreBench - -Warm-up 1: 14.357 ops/s -Iteration 1: 14.768 ops/s -Iteration 2: 14.922 ops/s -Iteration 3: 14.966 ops/s -Iteration 4: 14.805 ops/s -Iteration 5: 14.520 ops/s - -14.796 ±(99.9%) 0.671 ops/s [Average] - (min, avg, max) = (14.520, 14.796, 14.966), stdev = 0.174 - CI (99.9%): [14.125, 15.468] (assumes normal distribution) - -jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.jafamaBench - -Warm-up 1: 11.592 ops/s -Iteration 1: 12.174 ops/s -Iteration 2: 11.734 ops/s -Iteration 3: 11.939 ops/s -Iteration 4: 12.026 ops/s -Iteration 5: 12.221 ops/s - -12.019 ±(99.9%) 0.752 ops/s [Average] - (min, avg, max) = (11.734, 12.019, 12.221), stdev = 0.195 - CI (99.9%): [11.267, 12.771] (assumes normal distribution) - -jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafamaBench - -Warm-up 1: 12.097 ops/s -Iteration 1: 13.072 ops/s -Iteration 2: 13.112 ops/s -Iteration 3: 13.103 ops/s -Iteration 4: 12.950 ops/s -Iteration 5: 13.011 ops/s - -13.049 ±(99.9%) 0.263 ops/s [Average] - (min, avg, max) = (12.950, 13.049, 13.112), stdev = 0.068 - CI (99.9%): [12.787, 13.312] (assumes normal distribution) +/home/commandertvis/graalvm-ce-java11/bin/java -XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UnlockExperimentalVMOptions -XX:ThreadPriorityPolicy=1 -javaagent:/home/commandertvis/.gradle/caches/modules-2/files-2.1/org.jetbrains.kotlinx/kotlinx-coroutines-core-jvm/1.5.0/d8cebccdcddd029022aa8646a5a953ff88b13ac8/kotlinx-coroutines-core-jvm-1.5.0.jar -Dfile.encoding=UTF-8 -Duser.country=US -Duser.language=en -Duser.variant -ea ``` +* JMH 1.21 was used in `thrpt` mode with 1 warmup iteration by 1000 ms and 5 measurement iterations by 1000 ms. +| Benchmark | Score | +|:---------:|:-----:| +|`space.kscience.kmath.benchmarks.JafamaBenchmark.core`|14.296120859512893 ± 0.36462633435888736 ops/s| +|`space.kscience.kmath.benchmarks.JafamaBenchmark.jafama`|11.431566395649781 ± 2.570896777898243 ops/s| +|`space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafama`|11.746020495694117 ± 6.205909559197869 ops/s|
+ diff --git a/kmath-jafama/docs/README-TEMPLATE.md b/kmath-jafama/docs/README-TEMPLATE.md index fa58f7697..54348467b 100644 --- a/kmath-jafama/docs/README-TEMPLATE.md +++ b/kmath-jafama/docs/README-TEMPLATE.md @@ -24,95 +24,6 @@ fun main() { ## Performance -According to benchmarking data, on Hotspot Jafama functions are 20% faster than JDK math. On GraalVM, they are slower. +According to KMath benchmarks on GraalVM, Jafama functions are slower than JDK math; however, there are indications that on Hotspot Jafama is a bit faster. -
-Raw data: - -**Hotspot** - -``` -jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.coreBench - -Warm-up 1: 11.447 ops/s -Iteration 1: 13.354 ops/s -Iteration 2: 14.237 ops/s -Iteration 3: 14.708 ops/s -Iteration 4: 14.629 ops/s -Iteration 5: 14.692 ops/s - -14.324 ±(99.9%) 2.217 ops/s [Average] - (min, avg, max) = (13.354, 14.324, 14.708), stdev = 0.576 - CI (99.9%): [12.107, 16.541] (assumes normal distribution) - -jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.jafamaBench - -Warm-up 1: 15.628 ops/s -Iteration 1: 15.991 ops/s -Iteration 2: 16.633 ops/s -Iteration 3: 16.583 ops/s -Iteration 4: 16.716 ops/s -Iteration 5: 16.762 ops/s - -16.537 ±(99.9%) 1.205 ops/s [Average] - (min, avg, max) = (15.991, 16.537, 16.762), stdev = 0.313 - CI (99.9%): [15.332, 17.743] (assumes normal distribution) - -jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafamaBench - -Warm-up 1: 13.378 ops/s -Iteration 1: 15.049 ops/s -Iteration 2: 14.468 ops/s -Iteration 3: 14.469 ops/s -Iteration 4: 14.753 ops/s -Iteration 5: 14.958 ops/s - -14.739 ±(99.9%) 1.038 ops/s [Average] - (min, avg, max) = (14.468, 14.739, 15.049), stdev = 0.269 - CI (99.9%): [13.701, 15.777] (assumes normal distribution) -``` - -**GraalVM** - -``` -jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.coreBench - -Warm-up 1: 14.357 ops/s -Iteration 1: 14.768 ops/s -Iteration 2: 14.922 ops/s -Iteration 3: 14.966 ops/s -Iteration 4: 14.805 ops/s -Iteration 5: 14.520 ops/s - -14.796 ±(99.9%) 0.671 ops/s [Average] - (min, avg, max) = (14.520, 14.796, 14.966), stdev = 0.174 - CI (99.9%): [14.125, 15.468] (assumes normal distribution) - -jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.jafamaBench - -Warm-up 1: 11.592 ops/s -Iteration 1: 12.174 ops/s -Iteration 2: 11.734 ops/s -Iteration 3: 11.939 ops/s -Iteration 4: 12.026 ops/s -Iteration 5: 12.221 ops/s - -12.019 ±(99.9%) 0.752 ops/s [Average] - (min, avg, max) = (11.734, 12.019, 12.221), stdev = 0.195 - CI (99.9%): [11.267, 12.771] (assumes normal distribution) - -jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafamaBench - -Warm-up 1: 12.097 ops/s -Iteration 1: 13.072 ops/s -Iteration 2: 13.112 ops/s -Iteration 3: 13.103 ops/s -Iteration 4: 12.950 ops/s -Iteration 5: 13.011 ops/s - -13.049 ±(99.9%) 0.263 ops/s [Average] - (min, avg, max) = (12.950, 13.049, 13.112), stdev = 0.068 - CI (99.9%): [12.787, 13.312] (assumes normal distribution) -``` - -
+${benchmarkJafamaDouble} diff --git a/settings.gradle.kts b/settings.gradle.kts index 15a6a176c..710017588 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -5,7 +5,7 @@ pluginManagement { gradlePluginPortal() } - val toolsVersion = "0.9.9" + val toolsVersion = "0.9.10" val kotlinVersion = "1.5.0" plugins {