Merge pull request #369 from mipt-npm/commandertvis/benchmarks
Generated benchmarking reports
This commit is contained in:
Iaroslav Postovalov
GitHub
commit
d133f54476
@ -1,3 +1,7 @@
|
||||
@file:Suppress("UNUSED_VARIABLE")
|
||||
|
||||
import space.kscience.kmath.benchmarks.addBenchmarkProperties
|
||||
|
||||
plugins {
|
||||
kotlin("multiplatform")
|
||||
kotlin("plugin.allopen")
|
||||
@ -32,7 +36,7 @@ kotlin {
|
||||
implementation(project(":kmath-dimensions"))
|
||||
implementation(project(":kmath-for-real"))
|
||||
implementation(project(":kmath-jafama"))
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:0.3.0")
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:0.3.1")
|
||||
}
|
||||
}
|
||||
|
||||
@ -130,3 +134,5 @@ tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile> {
|
||||
readme {
|
||||
maturity = ru.mipt.npm.gradle.Maturity.EXPERIMENTAL
|
||||
}
|
||||
|
||||
addBenchmarkProperties()
|
||||
|
||||
@ -18,17 +18,17 @@ import kotlin.random.Random
|
||||
@State(Scope.Benchmark)
|
||||
internal class JafamaBenchmark {
|
||||
@Benchmark
|
||||
fun jafamaBench(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
|
||||
fun jafama(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
|
||||
JafamaDoubleField { x * power(x, 4) * exp(x) / cos(x) + sin(x) }
|
||||
}
|
||||
|
||||
@Benchmark
|
||||
fun coreBench(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
|
||||
fun core(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
|
||||
DoubleField { x * power(x, 4) * exp(x) / cos(x) + sin(x) }
|
||||
}
|
||||
|
||||
@Benchmark
|
||||
fun strictJafamaBench(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
|
||||
fun strictJafama(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
|
||||
StrictJafamaDoubleField { x * power(x, 4) * exp(x) / cos(x) + sin(x) }
|
||||
}
|
||||
|
||||
|
||||
@ -1,5 +1,19 @@
|
||||
plugins {
|
||||
`kotlin-dsl`
|
||||
kotlin("plugin.serialization") version "1.4.31"
|
||||
}
|
||||
|
||||
repositories.mavenCentral()
|
||||
repositories {
|
||||
gradlePluginPortal()
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
api("org.jetbrains.kotlinx:kotlinx-serialization-json:1.1.0")
|
||||
api("ru.mipt.npm:gradle-tools:0.9.10")
|
||||
api("org.jetbrains.kotlinx:kotlinx-benchmark-plugin:0.3.1")
|
||||
}
|
||||
|
||||
kotlin.sourceSets.all {
|
||||
languageSettings.useExperimentalAnnotation("kotlin.ExperimentalStdlibApi")
|
||||
}
|
||||
|
||||
@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Copyright 2018-2021 KMath contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package space.kscience.kmath.benchmarks
|
||||
|
||||
import kotlinx.serialization.Serializable
|
||||
|
||||
@Serializable
|
||||
data class JmhReport(
|
||||
val jmhVersion: String,
|
||||
val benchmark: String,
|
||||
val mode: String,
|
||||
val threads: Int,
|
||||
val forks: Int,
|
||||
val jvm: String,
|
||||
val jvmArgs: List<String>,
|
||||
val jdkVersion: String,
|
||||
val vmName: String,
|
||||
val vmVersion: String,
|
||||
val warmupIterations: Int,
|
||||
val warmupTime: String,
|
||||
val warmupBatchSize: Int,
|
||||
val measurementIterations: Int,
|
||||
val measurementTime: String,
|
||||
val measurementBatchSize: Int,
|
||||
val params: Map<String, String> = emptyMap(),
|
||||
val primaryMetric: PrimaryMetric,
|
||||
val secondaryMetrics: Map<String, SecondaryMetric>,
|
||||
) {
|
||||
interface Metric {
|
||||
val score: Double
|
||||
val scoreError: Double
|
||||
val scoreConfidence: List<Double>
|
||||
val scorePercentiles: Map<Double, Double>
|
||||
val scoreUnit: String
|
||||
}
|
||||
|
||||
@Serializable
|
||||
data class PrimaryMetric(
|
||||
override val score: Double,
|
||||
override val scoreError: Double,
|
||||
override val scoreConfidence: List<Double>,
|
||||
override val scorePercentiles: Map<Double, Double>,
|
||||
override val scoreUnit: String,
|
||||
val rawDataHistogram: List<List<List<List<Double>>>>? = null,
|
||||
val rawData: List<List<Double>>? = null,
|
||||
) : Metric
|
||||
|
||||
@Serializable
|
||||
data class SecondaryMetric(
|
||||
override val score: Double,
|
||||
override val scoreError: Double,
|
||||
override val scoreConfidence: List<Double>,
|
||||
override val scorePercentiles: Map<Double, Double>,
|
||||
override val scoreUnit: String,
|
||||
val rawData: List<List<Double>>,
|
||||
) : Metric
|
||||
}
|
||||
@ -0,0 +1,105 @@
|
||||
/*
|
||||
* Copyright 2018-2021 KMath contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package space.kscience.kmath.benchmarks
|
||||
|
||||
import kotlinx.serialization.*
|
||||
import org.gradle.api.Project
|
||||
import java.time.*
|
||||
import java.time.format.*
|
||||
import java.time.temporal.ChronoField.*
|
||||
|
||||
private val ISO_DATE_TIME: DateTimeFormatter = DateTimeFormatterBuilder().run {
|
||||
parseCaseInsensitive()
|
||||
appendValue(YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
|
||||
appendLiteral('-')
|
||||
appendValue(MONTH_OF_YEAR, 2)
|
||||
appendLiteral('-')
|
||||
appendValue(DAY_OF_MONTH, 2)
|
||||
appendLiteral('T')
|
||||
appendValue(HOUR_OF_DAY, 2)
|
||||
appendLiteral('.')
|
||||
appendValue(MINUTE_OF_HOUR, 2)
|
||||
optionalStart()
|
||||
appendLiteral('.')
|
||||
appendValue(SECOND_OF_MINUTE, 2)
|
||||
optionalStart()
|
||||
appendFraction(NANO_OF_SECOND, 0, 9, true)
|
||||
optionalStart()
|
||||
appendOffsetId()
|
||||
optionalStart()
|
||||
appendLiteral('[')
|
||||
parseCaseSensitive()
|
||||
appendZoneRegionId()
|
||||
appendLiteral(']')
|
||||
toFormatter()
|
||||
}
|
||||
|
||||
private fun noun(number: Number, singular: String, plural: String) = if (number.toLong() == 1L) singular else plural
|
||||
|
||||
fun Project.addBenchmarkProperties() {
|
||||
val benchmarksProject = this
|
||||
rootProject.subprojects.forEach { p ->
|
||||
p.extensions.findByType(ru.mipt.npm.gradle.KScienceReadmeExtension::class.java)?.run {
|
||||
benchmarksProject.extensions.findByType(kotlinx.benchmark.gradle.BenchmarksExtension::class.java)?.configurations?.forEach { cfg ->
|
||||
// TODO remove this hack when https://github.com/mipt-npm/gradle-tools/pull/15 is merged
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
(javaClass.getDeclaredField("properties")
|
||||
.also {
|
||||
it.isAccessible = true
|
||||
}[this] as MutableMap<String, () -> Any?>)["benchmark${cfg.name.replaceFirstChar(Char::uppercase)}"] =
|
||||
{
|
||||
val launches = benchmarksProject.buildDir.resolve("reports/benchmarks/${cfg.name}")
|
||||
|
||||
val resDirectory = launches.listFiles()?.maxByOrNull {
|
||||
LocalDateTime.parse(it.name, ISO_DATE_TIME).atZone(ZoneId.systemDefault()).toInstant()
|
||||
}
|
||||
|
||||
if (resDirectory == null) {
|
||||
"> **Can't find appropriate benchmark data. Try generating readme files after running benchmarks**."
|
||||
} else {
|
||||
val reports =
|
||||
kotlinx.serialization.json.Json.decodeFromString<List<JmhReport>>(
|
||||
resDirectory.resolve("jvm.json").readText()
|
||||
)
|
||||
|
||||
buildString {
|
||||
appendLine("<details>")
|
||||
appendLine("<summary>")
|
||||
appendLine("Report for benchmark configuration <code>${cfg.name}</code>")
|
||||
appendLine("</summary>")
|
||||
appendLine()
|
||||
val first = reports.first()
|
||||
|
||||
appendLine("* Run on ${first.vmName} (build ${first.vmVersion}) with Java process:")
|
||||
appendLine()
|
||||
appendLine("```")
|
||||
appendLine("${first.jvm} ${
|
||||
first.jvmArgs.joinToString(" ")
|
||||
}")
|
||||
appendLine("```")
|
||||
|
||||
appendLine("* JMH ${first.jmhVersion} was used in `${first.mode}` mode with ${first.warmupIterations} warmup ${
|
||||
noun(first.warmupIterations, "iteration", "iterations")
|
||||
} by ${first.warmupTime} and ${first.measurementIterations} measurement ${
|
||||
noun(first.measurementIterations, "iteration", "iterations")
|
||||
} by ${first.measurementTime}.")
|
||||
|
||||
appendLine()
|
||||
appendLine("| Benchmark | Score |")
|
||||
appendLine("|:---------:|:-----:|")
|
||||
|
||||
reports.forEach { report ->
|
||||
appendLine("|`${report.benchmark}`|${report.primaryMetric.score} ± ${report.primaryMetric.scoreError} ${report.primaryMetric.scoreUnit}|")
|
||||
}
|
||||
|
||||
appendLine("</details>")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -62,25 +62,21 @@ readme {
|
||||
|
||||
feature(
|
||||
id = "expression-language",
|
||||
description = "Expression language and its parser",
|
||||
ref = "src/commonMain/kotlin/space/kscience/kmath/ast/parser.kt"
|
||||
)
|
||||
) { "Expression language and its parser" }
|
||||
|
||||
feature(
|
||||
id = "mst-jvm-codegen",
|
||||
description = "Dynamic MST to JVM bytecode compiler",
|
||||
ref = "src/jvmMain/kotlin/space/kscience/kmath/asm/asm.kt"
|
||||
)
|
||||
) { "Dynamic MST to JVM bytecode compiler" }
|
||||
|
||||
feature(
|
||||
id = "mst-js-codegen",
|
||||
description = "Dynamic MST to JS compiler",
|
||||
ref = "src/jsMain/kotlin/space/kscience/kmath/estree/estree.kt"
|
||||
)
|
||||
) { "Dynamic MST to JS compiler" }
|
||||
|
||||
feature(
|
||||
id = "rendering",
|
||||
description = "Extendable MST rendering",
|
||||
ref = "src/commonMain/kotlin/space/kscience/kmath/ast/rendering/MathRenderer.kt"
|
||||
)
|
||||
) { "Extendable MST rendering" }
|
||||
}
|
||||
|
||||
@ -50,95 +50,24 @@ fun main() {
|
||||
|
||||
## Performance
|
||||
|
||||
According to benchmarking data, on Hotspot Jafama functions are 20% faster than JDK math. On GraalVM, they are slower.
|
||||
According to KMath benchmarks on GraalVM, Jafama functions are slower than JDK math; however, there are indications that on Hotspot Jafama is a bit faster.
|
||||
|
||||
<details>
|
||||
<summary>Raw data:</summary>
|
||||
<summary>
|
||||
Report for benchmark configuration <code>jafamaDouble</code>
|
||||
</summary>
|
||||
|
||||
**Hotspot**
|
||||
* Run on OpenJDK 64-Bit Server VM (build 11.0.11+8-jvmci-21.1-b05) with Java process:
|
||||
|
||||
```
|
||||
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.coreBench
|
||||
|
||||
Warm-up 1: 11.447 ops/s
|
||||
Iteration 1: 13.354 ops/s
|
||||
Iteration 2: 14.237 ops/s
|
||||
Iteration 3: 14.708 ops/s
|
||||
Iteration 4: 14.629 ops/s
|
||||
Iteration 5: 14.692 ops/s
|
||||
|
||||
14.324 ±(99.9%) 2.217 ops/s [Average]
|
||||
(min, avg, max) = (13.354, 14.324, 14.708), stdev = 0.576
|
||||
CI (99.9%): [12.107, 16.541] (assumes normal distribution)
|
||||
|
||||
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.jafamaBench
|
||||
|
||||
Warm-up 1: 15.628 ops/s
|
||||
Iteration 1: 15.991 ops/s
|
||||
Iteration 2: 16.633 ops/s
|
||||
Iteration 3: 16.583 ops/s
|
||||
Iteration 4: 16.716 ops/s
|
||||
Iteration 5: 16.762 ops/s
|
||||
|
||||
16.537 ±(99.9%) 1.205 ops/s [Average]
|
||||
(min, avg, max) = (15.991, 16.537, 16.762), stdev = 0.313
|
||||
CI (99.9%): [15.332, 17.743] (assumes normal distribution)
|
||||
|
||||
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafamaBench
|
||||
|
||||
Warm-up 1: 13.378 ops/s
|
||||
Iteration 1: 15.049 ops/s
|
||||
Iteration 2: 14.468 ops/s
|
||||
Iteration 3: 14.469 ops/s
|
||||
Iteration 4: 14.753 ops/s
|
||||
Iteration 5: 14.958 ops/s
|
||||
|
||||
14.739 ±(99.9%) 1.038 ops/s [Average]
|
||||
(min, avg, max) = (14.468, 14.739, 15.049), stdev = 0.269
|
||||
CI (99.9%): [13.701, 15.777] (assumes normal distribution)
|
||||
```
|
||||
|
||||
**GraalVM**
|
||||
|
||||
```
|
||||
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.coreBench
|
||||
|
||||
Warm-up 1: 14.357 ops/s
|
||||
Iteration 1: 14.768 ops/s
|
||||
Iteration 2: 14.922 ops/s
|
||||
Iteration 3: 14.966 ops/s
|
||||
Iteration 4: 14.805 ops/s
|
||||
Iteration 5: 14.520 ops/s
|
||||
|
||||
14.796 ±(99.9%) 0.671 ops/s [Average]
|
||||
(min, avg, max) = (14.520, 14.796, 14.966), stdev = 0.174
|
||||
CI (99.9%): [14.125, 15.468] (assumes normal distribution)
|
||||
|
||||
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.jafamaBench
|
||||
|
||||
Warm-up 1: 11.592 ops/s
|
||||
Iteration 1: 12.174 ops/s
|
||||
Iteration 2: 11.734 ops/s
|
||||
Iteration 3: 11.939 ops/s
|
||||
Iteration 4: 12.026 ops/s
|
||||
Iteration 5: 12.221 ops/s
|
||||
|
||||
12.019 ±(99.9%) 0.752 ops/s [Average]
|
||||
(min, avg, max) = (11.734, 12.019, 12.221), stdev = 0.195
|
||||
CI (99.9%): [11.267, 12.771] (assumes normal distribution)
|
||||
|
||||
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafamaBench
|
||||
|
||||
Warm-up 1: 12.097 ops/s
|
||||
Iteration 1: 13.072 ops/s
|
||||
Iteration 2: 13.112 ops/s
|
||||
Iteration 3: 13.103 ops/s
|
||||
Iteration 4: 12.950 ops/s
|
||||
Iteration 5: 13.011 ops/s
|
||||
|
||||
13.049 ±(99.9%) 0.263 ops/s [Average]
|
||||
(min, avg, max) = (12.950, 13.049, 13.112), stdev = 0.068
|
||||
CI (99.9%): [12.787, 13.312] (assumes normal distribution)
|
||||
/home/commandertvis/graalvm-ce-java11/bin/java -XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UnlockExperimentalVMOptions -XX:ThreadPriorityPolicy=1 -javaagent:/home/commandertvis/.gradle/caches/modules-2/files-2.1/org.jetbrains.kotlinx/kotlinx-coroutines-core-jvm/1.5.0/d8cebccdcddd029022aa8646a5a953ff88b13ac8/kotlinx-coroutines-core-jvm-1.5.0.jar -Dfile.encoding=UTF-8 -Duser.country=US -Duser.language=en -Duser.variant -ea
|
||||
```
|
||||
* JMH 1.21 was used in `thrpt` mode with 1 warmup iteration by 1000 ms and 5 measurement iterations by 1000 ms.
|
||||
|
||||
| Benchmark | Score |
|
||||
|:---------:|:-----:|
|
||||
|`space.kscience.kmath.benchmarks.JafamaBenchmark.core`|14.296120859512893 ± 0.36462633435888736 ops/s|
|
||||
|`space.kscience.kmath.benchmarks.JafamaBenchmark.jafama`|11.431566395649781 ± 2.570896777898243 ops/s|
|
||||
|`space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafama`|11.746020495694117 ± 6.205909559197869 ops/s|
|
||||
</details>
|
||||
|
||||
|
||||
@ -24,95 +24,6 @@ fun main() {
|
||||
|
||||
## Performance
|
||||
|
||||
According to benchmarking data, on Hotspot Jafama functions are 20% faster than JDK math. On GraalVM, they are slower.
|
||||
According to KMath benchmarks on GraalVM, Jafama functions are slower than JDK math; however, there are indications that on Hotspot Jafama is a bit faster.
|
||||
|
||||
<details>
|
||||
<summary>Raw data:</summary>
|
||||
|
||||
**Hotspot**
|
||||
|
||||
```
|
||||
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.coreBench
|
||||
|
||||
Warm-up 1: 11.447 ops/s
|
||||
Iteration 1: 13.354 ops/s
|
||||
Iteration 2: 14.237 ops/s
|
||||
Iteration 3: 14.708 ops/s
|
||||
Iteration 4: 14.629 ops/s
|
||||
Iteration 5: 14.692 ops/s
|
||||
|
||||
14.324 ±(99.9%) 2.217 ops/s [Average]
|
||||
(min, avg, max) = (13.354, 14.324, 14.708), stdev = 0.576
|
||||
CI (99.9%): [12.107, 16.541] (assumes normal distribution)
|
||||
|
||||
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.jafamaBench
|
||||
|
||||
Warm-up 1: 15.628 ops/s
|
||||
Iteration 1: 15.991 ops/s
|
||||
Iteration 2: 16.633 ops/s
|
||||
Iteration 3: 16.583 ops/s
|
||||
Iteration 4: 16.716 ops/s
|
||||
Iteration 5: 16.762 ops/s
|
||||
|
||||
16.537 ±(99.9%) 1.205 ops/s [Average]
|
||||
(min, avg, max) = (15.991, 16.537, 16.762), stdev = 0.313
|
||||
CI (99.9%): [15.332, 17.743] (assumes normal distribution)
|
||||
|
||||
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafamaBench
|
||||
|
||||
Warm-up 1: 13.378 ops/s
|
||||
Iteration 1: 15.049 ops/s
|
||||
Iteration 2: 14.468 ops/s
|
||||
Iteration 3: 14.469 ops/s
|
||||
Iteration 4: 14.753 ops/s
|
||||
Iteration 5: 14.958 ops/s
|
||||
|
||||
14.739 ±(99.9%) 1.038 ops/s [Average]
|
||||
(min, avg, max) = (14.468, 14.739, 15.049), stdev = 0.269
|
||||
CI (99.9%): [13.701, 15.777] (assumes normal distribution)
|
||||
```
|
||||
|
||||
**GraalVM**
|
||||
|
||||
```
|
||||
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.coreBench
|
||||
|
||||
Warm-up 1: 14.357 ops/s
|
||||
Iteration 1: 14.768 ops/s
|
||||
Iteration 2: 14.922 ops/s
|
||||
Iteration 3: 14.966 ops/s
|
||||
Iteration 4: 14.805 ops/s
|
||||
Iteration 5: 14.520 ops/s
|
||||
|
||||
14.796 ±(99.9%) 0.671 ops/s [Average]
|
||||
(min, avg, max) = (14.520, 14.796, 14.966), stdev = 0.174
|
||||
CI (99.9%): [14.125, 15.468] (assumes normal distribution)
|
||||
|
||||
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.jafamaBench
|
||||
|
||||
Warm-up 1: 11.592 ops/s
|
||||
Iteration 1: 12.174 ops/s
|
||||
Iteration 2: 11.734 ops/s
|
||||
Iteration 3: 11.939 ops/s
|
||||
Iteration 4: 12.026 ops/s
|
||||
Iteration 5: 12.221 ops/s
|
||||
|
||||
12.019 ±(99.9%) 0.752 ops/s [Average]
|
||||
(min, avg, max) = (11.734, 12.019, 12.221), stdev = 0.195
|
||||
CI (99.9%): [11.267, 12.771] (assumes normal distribution)
|
||||
|
||||
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafamaBench
|
||||
|
||||
Warm-up 1: 12.097 ops/s
|
||||
Iteration 1: 13.072 ops/s
|
||||
Iteration 2: 13.112 ops/s
|
||||
Iteration 3: 13.103 ops/s
|
||||
Iteration 4: 12.950 ops/s
|
||||
Iteration 5: 13.011 ops/s
|
||||
|
||||
13.049 ±(99.9%) 0.263 ops/s [Average]
|
||||
(min, avg, max) = (12.950, 13.049, 13.112), stdev = 0.068
|
||||
CI (99.9%): [12.787, 13.312] (assumes normal distribution)
|
||||
```
|
||||
|
||||
</details>
|
||||
${benchmarkJafamaDouble}
|
||||
|
||||
@ -5,7 +5,7 @@ pluginManagement {
|
||||
gradlePluginPortal()
|
||||
}
|
||||
|
||||
val toolsVersion = "0.9.9"
|
||||
val toolsVersion = "0.9.10"
|
||||
val kotlinVersion = "1.5.0"
|
||||
|
||||
plugins {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user