Merge pull request #369 from mipt-npm/commandertvis/benchmarks

Generated benchmarking reports

benchmarks/build.gradle.kts

@@ -1,3 +1,7 @@
+@file:Suppress("UNUSED_VARIABLE")
+
+import space.kscience.kmath.benchmarks.addBenchmarkProperties
+
 plugins {
     kotlin("multiplatform")
     kotlin("plugin.allopen")
@@ -32,7 +36,7 @@ kotlin {
                 implementation(project(":kmath-dimensions"))
                 implementation(project(":kmath-for-real"))
                 implementation(project(":kmath-jafama"))
-                implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:0.3.0")
+                implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:0.3.1")
             }
         }
 
@@ -130,3 +134,5 @@ tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile> {
 readme {
     maturity = ru.mipt.npm.gradle.Maturity.EXPERIMENTAL
 }
+
+addBenchmarkProperties()

benchmarks/src/jvmMain/kotlin/space/kscience/kmath/benchmarks/JafamaBenchmark.kt

@@ -18,17 +18,17 @@ import kotlin.random.Random
 @State(Scope.Benchmark)
 internal class JafamaBenchmark {
     @Benchmark
-    fun jafamaBench(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
+    fun jafama(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
         JafamaDoubleField { x * power(x, 4) * exp(x) / cos(x) + sin(x) }
     }
 
     @Benchmark
-    fun coreBench(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
+    fun core(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
         DoubleField { x * power(x, 4) * exp(x) / cos(x) + sin(x) }
     }
 
     @Benchmark
-    fun strictJafamaBench(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
+    fun strictJafama(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
         StrictJafamaDoubleField { x * power(x, 4) * exp(x) / cos(x) + sin(x) }
     }
 

buildSrc/build.gradle.kts

@@ -1,5 +1,19 @@
 plugins {
     `kotlin-dsl`
+    kotlin("plugin.serialization") version "1.4.31"
 }
 
-repositories.mavenCentral()
+repositories {
+    gradlePluginPortal()
+    mavenCentral()
+}
+
+dependencies {
+    api("org.jetbrains.kotlinx:kotlinx-serialization-json:1.1.0")
+    api("ru.mipt.npm:gradle-tools:0.9.10")
+    api("org.jetbrains.kotlinx:kotlinx-benchmark-plugin:0.3.1")
+}
+
+kotlin.sourceSets.all {
+    languageSettings.useExperimentalAnnotation("kotlin.ExperimentalStdlibApi")
+}

buildSrc/src/main/kotlin/space/kscience/kmath/benchmarks/JmhReport.kt

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2018-2021 KMath contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package space.kscience.kmath.benchmarks
+
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class JmhReport(
+    val jmhVersion: String,
+    val benchmark: String,
+    val mode: String,
+    val threads: Int,
+    val forks: Int,
+    val jvm: String,
+    val jvmArgs: List<String>,
+    val jdkVersion: String,
+    val vmName: String,
+    val vmVersion: String,
+    val warmupIterations: Int,
+    val warmupTime: String,
+    val warmupBatchSize: Int,
+    val measurementIterations: Int,
+    val measurementTime: String,
+    val measurementBatchSize: Int,
+    val params: Map<String, String> = emptyMap(),
+    val primaryMetric: PrimaryMetric,
+    val secondaryMetrics: Map<String, SecondaryMetric>,
+) {
+    interface Metric {
+        val score: Double
+        val scoreError: Double
+        val scoreConfidence: List<Double>
+        val scorePercentiles: Map<Double, Double>
+        val scoreUnit: String
+    }
+
+    @Serializable
+    data class PrimaryMetric(
+        override val score: Double,
+        override val scoreError: Double,
+        override val scoreConfidence: List<Double>,
+        override val scorePercentiles: Map<Double, Double>,
+        override val scoreUnit: String,
+        val rawDataHistogram: List<List<List<List<Double>>>>? = null,
+        val rawData: List<List<Double>>? = null,
+    ) : Metric
+
+    @Serializable
+    data class SecondaryMetric(
+        override val score: Double,
+        override val scoreError: Double,
+        override val scoreConfidence: List<Double>,
+        override val scorePercentiles: Map<Double, Double>,
+        override val scoreUnit: String,
+        val rawData: List<List<Double>>,
+    ) : Metric
+}

buildSrc/src/main/kotlin/space/kscience/kmath/benchmarks/addBenchmarkProperties.kt

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2018-2021 KMath contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package space.kscience.kmath.benchmarks
+
+import kotlinx.serialization.*
+import org.gradle.api.Project
+import java.time.*
+import java.time.format.*
+import java.time.temporal.ChronoField.*
+
+private val ISO_DATE_TIME: DateTimeFormatter = DateTimeFormatterBuilder().run {
+    parseCaseInsensitive()
+    appendValue(YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
+    appendLiteral('-')
+    appendValue(MONTH_OF_YEAR, 2)
+    appendLiteral('-')
+    appendValue(DAY_OF_MONTH, 2)
+    appendLiteral('T')
+    appendValue(HOUR_OF_DAY, 2)
+    appendLiteral('.')
+    appendValue(MINUTE_OF_HOUR, 2)
+    optionalStart()
+    appendLiteral('.')
+    appendValue(SECOND_OF_MINUTE, 2)
+    optionalStart()
+    appendFraction(NANO_OF_SECOND, 0, 9, true)
+    optionalStart()
+    appendOffsetId()
+    optionalStart()
+    appendLiteral('[')
+    parseCaseSensitive()
+    appendZoneRegionId()
+    appendLiteral(']')
+    toFormatter()
+}
+
+private fun noun(number: Number, singular: String, plural: String) = if (number.toLong() == 1L) singular else plural
+
+fun Project.addBenchmarkProperties() {
+    val benchmarksProject = this
+    rootProject.subprojects.forEach { p ->
+        p.extensions.findByType(ru.mipt.npm.gradle.KScienceReadmeExtension::class.java)?.run {
+            benchmarksProject.extensions.findByType(kotlinx.benchmark.gradle.BenchmarksExtension::class.java)?.configurations?.forEach { cfg ->
+                // TODO remove this hack when https://github.com/mipt-npm/gradle-tools/pull/15 is merged
+                @Suppress("UNCHECKED_CAST")
+                (javaClass.getDeclaredField("properties")
+                    .also {
+                        it.isAccessible = true
+                    }[this] as MutableMap<String, () -> Any?>)["benchmark${cfg.name.replaceFirstChar(Char::uppercase)}"] =
+                    {
+                        val launches = benchmarksProject.buildDir.resolve("reports/benchmarks/${cfg.name}")
+
+                        val resDirectory = launches.listFiles()?.maxByOrNull {
+                            LocalDateTime.parse(it.name, ISO_DATE_TIME).atZone(ZoneId.systemDefault()).toInstant()
+                        }
+
+                        if (resDirectory == null) {
+                            "> **Can't find appropriate benchmark data. Try generating readme files after running benchmarks**."
+                        } else {
+                            val reports =
+                                kotlinx.serialization.json.Json.decodeFromString<List<JmhReport>>(
+                                    resDirectory.resolve("jvm.json").readText()
+                                )
+
+                            buildString {
+                                appendLine("<details>")
+                                appendLine("<summary>")
+                                appendLine("Report for benchmark configuration <code>${cfg.name}</code>")
+                                appendLine("</summary>")
+                                appendLine()
+                                val first = reports.first()
+
+                                appendLine("* Run on ${first.vmName} (build ${first.vmVersion}) with Java process:")
+                                appendLine()
+                                appendLine("```")
+                                appendLine("${first.jvm} ${
+                                    first.jvmArgs.joinToString(" ")
+                                }")
+                                appendLine("```")
+
+                                appendLine("* JMH ${first.jmhVersion} was used in `${first.mode}` mode with ${first.warmupIterations} warmup ${
+                                    noun(first.warmupIterations, "iteration", "iterations")
+                                } by ${first.warmupTime} and ${first.measurementIterations} measurement ${
+                                    noun(first.measurementIterations, "iteration", "iterations")
+                                } by ${first.measurementTime}.")
+
+                                appendLine()
+                                appendLine("| Benchmark | Score |")
+                                appendLine("|:---------:|:-----:|")
+
+                                reports.forEach { report ->
+                                    appendLine("|`${report.benchmark}`|${report.primaryMetric.score} &plusmn; ${report.primaryMetric.scoreError} ${report.primaryMetric.scoreUnit}|")
+                                }
+
+                                appendLine("</details>")
+                            }
+                        }
+                    }
+            }
+        }
+    }
+}

kmath-ast/build.gradle.kts

@@ -62,25 +62,21 @@ readme {
 
     feature(
         id = "expression-language",
-        description = "Expression language and its parser",
         ref = "src/commonMain/kotlin/space/kscience/kmath/ast/parser.kt"
-    )
+    ) { "Expression language and its parser" }
 
     feature(
         id = "mst-jvm-codegen",
-        description = "Dynamic MST to JVM bytecode compiler",
         ref = "src/jvmMain/kotlin/space/kscience/kmath/asm/asm.kt"
-    )
+    ) { "Dynamic MST to JVM bytecode compiler" }
 
     feature(
         id = "mst-js-codegen",
-        description = "Dynamic MST to JS compiler",
         ref = "src/jsMain/kotlin/space/kscience/kmath/estree/estree.kt"
-    )
+    ) { "Dynamic MST to JS compiler" }
 
     feature(
         id = "rendering",
-        description = "Extendable MST rendering",
         ref = "src/commonMain/kotlin/space/kscience/kmath/ast/rendering/MathRenderer.kt"
-    )
+    ) { "Extendable MST rendering" }
 }

kmath-jafama/README.md

@@ -50,95 +50,24 @@ fun main() {
 
 ## Performance
 
-According to benchmarking data, on Hotspot Jafama functions are 20% faster than JDK math. On GraalVM, they are slower.
+According to KMath benchmarks on GraalVM, Jafama functions are slower than JDK math; however, there are indications that on Hotspot Jafama is a bit faster.
 
 <details>
-<summary>Raw data:</summary>
+<summary>
+Report for benchmark configuration <code>jafamaDouble</code>
+</summary>
 
-**Hotspot**
+* Run on OpenJDK 64-Bit Server VM (build 11.0.11+8-jvmci-21.1-b05) with Java process:
 
 ```
-jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.coreBench
+/home/commandertvis/graalvm-ce-java11/bin/java -XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UnlockExperimentalVMOptions -XX:ThreadPriorityPolicy=1 -javaagent:/home/commandertvis/.gradle/caches/modules-2/files-2.1/org.jetbrains.kotlinx/kotlinx-coroutines-core-jvm/1.5.0/d8cebccdcddd029022aa8646a5a953ff88b13ac8/kotlinx-coroutines-core-jvm-1.5.0.jar -Dfile.encoding=UTF-8 -Duser.country=US -Duser.language=en -Duser.variant -ea
-
-Warm-up 1: 11.447 ops/s
-Iteration 1: 13.354 ops/s
-Iteration 2: 14.237 ops/s
-Iteration 3: 14.708 ops/s
-Iteration 4: 14.629 ops/s
-Iteration 5: 14.692 ops/s
-
-14.324 ±(99.9%) 2.217 ops/s [Average]
-  (min, avg, max) = (13.354, 14.324, 14.708), stdev = 0.576
-  CI (99.9%): [12.107, 16.541] (assumes normal distribution)
-
-jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.jafamaBench
-
-Warm-up 1: 15.628 ops/s
-Iteration 1: 15.991 ops/s
-Iteration 2: 16.633 ops/s
-Iteration 3: 16.583 ops/s
-Iteration 4: 16.716 ops/s
-Iteration 5: 16.762 ops/s
-
-16.537 ±(99.9%) 1.205 ops/s [Average]
-  (min, avg, max) = (15.991, 16.537, 16.762), stdev = 0.313
-  CI (99.9%): [15.332, 17.743] (assumes normal distribution)
-
-jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafamaBench
-
-Warm-up 1: 13.378 ops/s
-Iteration 1: 15.049 ops/s
-Iteration 2: 14.468 ops/s
-Iteration 3: 14.469 ops/s
-Iteration 4: 14.753 ops/s
-Iteration 5: 14.958 ops/s
-
-14.739 ±(99.9%) 1.038 ops/s [Average]
-  (min, avg, max) = (14.468, 14.739, 15.049), stdev = 0.269
-  CI (99.9%): [13.701, 15.777] (assumes normal distribution)
-```
-
-**GraalVM**
-
-```
-jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.coreBench
-
-Warm-up 1: 14.357 ops/s
-Iteration 1: 14.768 ops/s
-Iteration 2: 14.922 ops/s
-Iteration 3: 14.966 ops/s
-Iteration 4: 14.805 ops/s
-Iteration 5: 14.520 ops/s
-
-14.796 ±(99.9%) 0.671 ops/s [Average]
-  (min, avg, max) = (14.520, 14.796, 14.966), stdev = 0.174
-  CI (99.9%): [14.125, 15.468] (assumes normal distribution)
-
-jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.jafamaBench
-
-Warm-up 1: 11.592 ops/s
-Iteration 1: 12.174 ops/s
-Iteration 2: 11.734 ops/s
-Iteration 3: 11.939 ops/s
-Iteration 4: 12.026 ops/s
-Iteration 5: 12.221 ops/s
-
-12.019 ±(99.9%) 0.752 ops/s [Average]
-  (min, avg, max) = (11.734, 12.019, 12.221), stdev = 0.195
-  CI (99.9%): [11.267, 12.771] (assumes normal distribution)
-
-jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafamaBench
-
-Warm-up 1: 12.097 ops/s
-Iteration 1: 13.072 ops/s
-Iteration 2: 13.112 ops/s
-Iteration 3: 13.103 ops/s
-Iteration 4: 12.950 ops/s
-Iteration 5: 13.011 ops/s
-
-13.049 ±(99.9%) 0.263 ops/s [Average]
-  (min, avg, max) = (12.950, 13.049, 13.112), stdev = 0.068
-  CI (99.9%): [12.787, 13.312] (assumes normal distribution)
 ```
+* JMH 1.21 was used in `thrpt` mode with 1 warmup iteration by 1000 ms and 5 measurement iterations by 1000 ms.
 
+| Benchmark | Score |
+|:---------:|:-----:|
+|`space.kscience.kmath.benchmarks.JafamaBenchmark.core`|14.296120859512893 &plusmn; 0.36462633435888736 ops/s|
+|`space.kscience.kmath.benchmarks.JafamaBenchmark.jafama`|11.431566395649781 &plusmn; 2.570896777898243 ops/s|
+|`space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafama`|11.746020495694117 &plusmn; 6.205909559197869 ops/s|
 </details>
+

kmath-jafama/docs/README-TEMPLATE.md

@@ -24,95 +24,6 @@ fun main() {
 
 ## Performance
 
-According to benchmarking data, on Hotspot Jafama functions are 20% faster than JDK math. On GraalVM, they are slower.
+According to KMath benchmarks on GraalVM, Jafama functions are slower than JDK math; however, there are indications that on Hotspot Jafama is a bit faster.
 
-<details>
+${benchmarkJafamaDouble}
-<summary>Raw data:</summary>
-
-**Hotspot**
-
-```
-jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.coreBench
-
-Warm-up 1: 11.447 ops/s
-Iteration 1: 13.354 ops/s
-Iteration 2: 14.237 ops/s
-Iteration 3: 14.708 ops/s
-Iteration 4: 14.629 ops/s
-Iteration 5: 14.692 ops/s
-
-14.324 ±(99.9%) 2.217 ops/s [Average]
-  (min, avg, max) = (13.354, 14.324, 14.708), stdev = 0.576
-  CI (99.9%): [12.107, 16.541] (assumes normal distribution)
-
-jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.jafamaBench
-
-Warm-up 1: 15.628 ops/s
-Iteration 1: 15.991 ops/s
-Iteration 2: 16.633 ops/s
-Iteration 3: 16.583 ops/s
-Iteration 4: 16.716 ops/s
-Iteration 5: 16.762 ops/s
-
-16.537 ±(99.9%) 1.205 ops/s [Average]
-  (min, avg, max) = (15.991, 16.537, 16.762), stdev = 0.313
-  CI (99.9%): [15.332, 17.743] (assumes normal distribution)
-
-jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafamaBench
-
-Warm-up 1: 13.378 ops/s
-Iteration 1: 15.049 ops/s
-Iteration 2: 14.468 ops/s
-Iteration 3: 14.469 ops/s
-Iteration 4: 14.753 ops/s
-Iteration 5: 14.958 ops/s
-
-14.739 ±(99.9%) 1.038 ops/s [Average]
-  (min, avg, max) = (14.468, 14.739, 15.049), stdev = 0.269
-  CI (99.9%): [13.701, 15.777] (assumes normal distribution)
-```
-
-**GraalVM**
-
-```
-jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.coreBench
-
-Warm-up 1: 14.357 ops/s
-Iteration 1: 14.768 ops/s
-Iteration 2: 14.922 ops/s
-Iteration 3: 14.966 ops/s
-Iteration 4: 14.805 ops/s
-Iteration 5: 14.520 ops/s
-
-14.796 ±(99.9%) 0.671 ops/s [Average]
-  (min, avg, max) = (14.520, 14.796, 14.966), stdev = 0.174
-  CI (99.9%): [14.125, 15.468] (assumes normal distribution)
-
-jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.jafamaBench
-
-Warm-up 1: 11.592 ops/s
-Iteration 1: 12.174 ops/s
-Iteration 2: 11.734 ops/s
-Iteration 3: 11.939 ops/s
-Iteration 4: 12.026 ops/s
-Iteration 5: 12.221 ops/s
-
-12.019 ±(99.9%) 0.752 ops/s [Average]
-  (min, avg, max) = (11.734, 12.019, 12.221), stdev = 0.195
-  CI (99.9%): [11.267, 12.771] (assumes normal distribution)
-
-jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafamaBench
-
-Warm-up 1: 12.097 ops/s
-Iteration 1: 13.072 ops/s
-Iteration 2: 13.112 ops/s
-Iteration 3: 13.103 ops/s
-Iteration 4: 12.950 ops/s
-Iteration 5: 13.011 ops/s
-
-13.049 ±(99.9%) 0.263 ops/s [Average]
-  (min, avg, max) = (12.950, 13.049, 13.112), stdev = 0.068
-  CI (99.9%): [12.787, 13.312] (assumes normal distribution)
-```
-
-</details>

settings.gradle.kts

@@ -5,7 +5,7 @@ pluginManagement {
         gradlePluginPortal()
     }
 
-    val toolsVersion = "0.9.9"
+    val toolsVersion = "0.9.10"
     val kotlinVersion = "1.5.0"
 
     plugins {