kscience/kmath
9
0
Fork 1
Code Issues Pull Requests 6 Packages Projects Releases 16 Wiki Activity

Generated benchmarking reports

Browse Source
This commit is contained in:
Iaroslav Postovalov 2021-06-15 13:18:40 +07:00
parent cfe6c3ef4b
commit 45f1588175
9 changed files with 210 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
benchmarks/build.gradle.kts
View File

@ -1,3 +1,7 @@
@file:Suppress("UNUSED_VARIABLE")
import space.kscience.kmath.benchmarks.addBenchmarkProperties
plugins {
kotlin("multiplatform")
kotlin("plugin.allopen")
@ -32,7 +36,7 @@ kotlin {
implementation(project(":kmath-dimensions"))
implementation(project(":kmath-for-real"))
implementation(project(":kmath-jafama"))
implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:0.3.0")
implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:0.3.1")
}
}
@ -130,3 +134,5 @@ tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile> {
readme {
maturity = ru.mipt.npm.gradle.Maturity.EXPERIMENTAL
}
addBenchmarkProperties()

6
benchmarks/src/jvmMain/kotlin/space/kscience/kmath/benchmarks/JafamaBenchmark.kt
View File

@ -18,17 +18,17 @@ import kotlin.random.Random
@State(Scope.Benchmark)
internal class JafamaBenchmark {
@Benchmark
fun jafamaBench(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
fun jafama(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
JafamaDoubleField { x * power(x, 4) * exp(x) / cos(x) + sin(x) }
}
@Benchmark
fun coreBench(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
fun core(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
DoubleField { x * power(x, 4) * exp(x) / cos(x) + sin(x) }
}
@Benchmark
fun strictJafamaBench(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
fun strictJafama(blackhole: Blackhole) = invokeBenchmarks(blackhole) { x ->
StrictJafamaDoubleField { x * power(x, 4) * exp(x) / cos(x) + sin(x) }
}

16
buildSrc/build.gradle.kts
View File

@ -1,5 +1,19 @@
plugins {
`kotlin-dsl`
kotlin("plugin.serialization") version "1.4.31"
}
repositories.mavenCentral()
repositories {
gradlePluginPortal()
mavenCentral()
}
dependencies {
api("org.jetbrains.kotlinx:kotlinx-serialization-json:1.1.0")
api("ru.mipt.npm:gradle-tools:0.9.10")
api("org.jetbrains.kotlinx:kotlinx-benchmark-plugin:0.3.1")
}
kotlin.sourceSets.all {
languageSettings.useExperimentalAnnotation("kotlin.ExperimentalStdlibApi")
}

60
buildSrc/src/main/kotlin/space/kscience/kmath/benchmarks/JmhReport.kt Normal file
View File

@ -0,0 +1,60 @@
/*
* Copyright 2018-2021 KMath contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package space.kscience.kmath.benchmarks
import kotlinx.serialization.Serializable
@Serializable
data class JmhReport(
val jmhVersion: String,
val benchmark: String,
val mode: String,
val threads: Int,
val forks: Int,
val jvm: String,
val jvmArgs: List<String>,
val jdkVersion: String,
val vmName: String,
val vmVersion: String,
val warmupIterations: Int,
val warmupTime: String,
val warmupBatchSize: Int,
val measurementIterations: Int,
val measurementTime: String,
val measurementBatchSize: Int,
val params: Map<String, String> = emptyMap(),
val primaryMetric: PrimaryMetric,
val secondaryMetrics: Map<String, SecondaryMetric>,
) {
interface Metric {
val score: Double
val scoreError: Double
val scoreConfidence: List<Double>
val scorePercentiles: Map<Double, Double>
val scoreUnit: String
}
@Serializable
data class PrimaryMetric(
override val score: Double,
override val scoreError: Double,
override val scoreConfidence: List<Double>,
override val scorePercentiles: Map<Double, Double>,
override val scoreUnit: String,
val rawDataHistogram: List<List<List<List<Double>>>>? = null,
val rawData: List<List<Double>>? = null,
) : Metric
@Serializable
data class SecondaryMetric(
override val score: Double,
override val scoreError: Double,
override val scoreConfidence: List<Double>,
override val scorePercentiles: Map<Double, Double>,
override val scoreUnit: String,
val rawData: List<List<Double>>,
) : Metric
}

105
buildSrc/src/main/kotlin/space/kscience/kmath/benchmarks/addBenchmarkProperties.kt Normal file
View File

@ -0,0 +1,105 @@
/*
* Copyright 2018-2021 KMath contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package space.kscience.kmath.benchmarks
import kotlinx.serialization.*
import org.gradle.api.Project
import java.time.*
import java.time.format.*
import java.time.temporal.ChronoField.*
private val ISO_DATE_TIME: DateTimeFormatter = DateTimeFormatterBuilder().run {
parseCaseInsensitive()
appendValue(YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
appendLiteral('-')
appendValue(MONTH_OF_YEAR, 2)
appendLiteral('-')
appendValue(DAY_OF_MONTH, 2)
appendLiteral('T')
appendValue(HOUR_OF_DAY, 2)
appendLiteral('.')
appendValue(MINUTE_OF_HOUR, 2)
optionalStart()
appendLiteral('.')
appendValue(SECOND_OF_MINUTE, 2)
optionalStart()
appendFraction(NANO_OF_SECOND, 0, 9, true)
optionalStart()
appendOffsetId()
optionalStart()
appendLiteral('[')
parseCaseSensitive()
appendZoneRegionId()
appendLiteral(']')
toFormatter()
}
private fun noun(number: Number, singular: String, plural: String) = if (number.toLong() == 1L) singular else plural
fun Project.addBenchmarkProperties() {
val benchmarksProject = this
rootProject.subprojects.forEach { p ->
p.extensions.findByType(ru.mipt.npm.gradle.KScienceReadmeExtension::class.java)?.run {
benchmarksProject.extensions.findByType(kotlinx.benchmark.gradle.BenchmarksExtension::class.java)?.configurations?.forEach { cfg ->
// TODO remove this hack when https://github.com/mipt-npm/gradle-tools/pull/15 is merged
@Suppress("UNCHECKED_CAST")
(javaClass.getDeclaredField("properties")
.also {
it.isAccessible = true
}[this] as MutableMap<String, () -> Any?>)["benchmark${cfg.name.replaceFirstChar(Char::uppercase)}"] =
{
val launches = benchmarksProject.buildDir.resolve("reports/benchmarks/${cfg.name}")
val resDirectory = launches.listFiles()?.maxByOrNull {
LocalDateTime.parse(it.name, ISO_DATE_TIME).atZone(ZoneId.systemDefault()).toInstant()
}
if (resDirectory == null) {
"> **Can't find appropriate benchmark data. Try generating readme files after running benchmarks**."
} else {
val reports =
kotlinx.serialization.json.Json.decodeFromString<List<JmhReport>>(
resDirectory.resolve("jvm.json").readText()
)
buildString {
appendLine("<details>")
appendLine("<summary>")
appendLine("Report for benchmark configuration <code>${cfg.name}</code>")
appendLine("</summary>")
appendLine()
val first = reports.first()
appendLine("* Run on ${first.vmName} (build ${first.vmVersion}) with Java process:")
appendLine()
appendLine("```")
appendLine("${first.jvm} ${
first.jvmArgs.joinToString(" ")
}")
appendLine("```")
appendLine("* JMH ${first.jmhVersion} was used in `${first.mode}` mode with ${first.warmupIterations} warmup ${
noun(first.warmupIterations, "iteration", "iterations")
} by ${first.warmupTime} and ${first.measurementIterations} measurement ${
noun(first.measurementIterations, "iteration", "iterations")
} by ${first.measurementTime}.")
appendLine()
appendLine("| Benchmark | Score |")
appendLine("|:---------:|:-----:|")
reports.forEach { report ->
appendLine("|`${report.benchmark}`|${report.primaryMetric.score} &plusmn; ${report.primaryMetric.scoreError} ${report.primaryMetric.scoreUnit}|")
}
appendLine("</details>")
}
}
}
}
}
}
}

12
kmath-ast/build.gradle.kts
View File

@ -62,25 +62,21 @@ readme {
feature(
id = "expression-language",
description = "Expression language and its parser",
ref = "src/commonMain/kotlin/space/kscience/kmath/ast/parser.kt"
)
) { "Expression language and its parser" }
feature(
id = "mst-jvm-codegen",
description = "Dynamic MST to JVM bytecode compiler",
ref = "src/jvmMain/kotlin/space/kscience/kmath/asm/asm.kt"
)
) { "Dynamic MST to JVM bytecode compiler" }
feature(
id = "mst-js-codegen",
description = "Dynamic MST to JS compiler",
ref = "src/jsMain/kotlin/space/kscience/kmath/estree/estree.kt"
)
) { "Dynamic MST to JS compiler" }
feature(
id = "rendering",
description = "Extendable MST rendering",
ref = "src/commonMain/kotlin/space/kscience/kmath/ast/rendering/MathRenderer.kt"
)
) { "Extendable MST rendering" }
}

97
kmath-jafama/README.md
View File

@ -50,95 +50,24 @@ fun main() {
## Performance
According to benchmarking data, on Hotspot Jafama functions are 20% faster than JDK math. On GraalVM, they are slower.
According to KMath benchmarks on GraalVM, Jafama functions are slower than JDK math; however, there are indications that on Hotspot Jafama is a bit faster.
<details>
<summary>Raw data:</summary>
<summary>
Report for benchmark configuration <code>jafamaDouble</code>
</summary>
**Hotspot**
* Run on OpenJDK 64-Bit Server VM (build 11.0.11+8-jvmci-21.1-b05) with Java process:
```
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.coreBench
Warm-up 1: 11.447 ops/s
Iteration 1: 13.354 ops/s
Iteration 2: 14.237 ops/s
Iteration 3: 14.708 ops/s
Iteration 4: 14.629 ops/s
Iteration 5: 14.692 ops/s
14.324 ±(99.9%) 2.217 ops/s [Average]
(min, avg, max) = (13.354, 14.324, 14.708), stdev = 0.576
CI (99.9%): [12.107, 16.541] (assumes normal distribution)
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.jafamaBench
Warm-up 1: 15.628 ops/s
Iteration 1: 15.991 ops/s
Iteration 2: 16.633 ops/s
Iteration 3: 16.583 ops/s
Iteration 4: 16.716 ops/s
Iteration 5: 16.762 ops/s
16.537 ±(99.9%) 1.205 ops/s [Average]
(min, avg, max) = (15.991, 16.537, 16.762), stdev = 0.313
CI (99.9%): [15.332, 17.743] (assumes normal distribution)
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafamaBench
Warm-up 1: 13.378 ops/s
Iteration 1: 15.049 ops/s
Iteration 2: 14.468 ops/s
Iteration 3: 14.469 ops/s
Iteration 4: 14.753 ops/s
Iteration 5: 14.958 ops/s
14.739 ±(99.9%) 1.038 ops/s [Average]
(min, avg, max) = (14.468, 14.739, 15.049), stdev = 0.269
CI (99.9%): [13.701, 15.777] (assumes normal distribution)
```
**GraalVM**
```
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.coreBench
Warm-up 1: 14.357 ops/s
Iteration 1: 14.768 ops/s
Iteration 2: 14.922 ops/s
Iteration 3: 14.966 ops/s
Iteration 4: 14.805 ops/s
Iteration 5: 14.520 ops/s
14.796 ±(99.9%) 0.671 ops/s [Average]
(min, avg, max) = (14.520, 14.796, 14.966), stdev = 0.174
CI (99.9%): [14.125, 15.468] (assumes normal distribution)
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.jafamaBench
Warm-up 1: 11.592 ops/s
Iteration 1: 12.174 ops/s
Iteration 2: 11.734 ops/s
Iteration 3: 11.939 ops/s
Iteration 4: 12.026 ops/s
Iteration 5: 12.221 ops/s
12.019 ±(99.9%) 0.752 ops/s [Average]
(min, avg, max) = (11.734, 12.019, 12.221), stdev = 0.195
CI (99.9%): [11.267, 12.771] (assumes normal distribution)
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafamaBench
Warm-up 1: 12.097 ops/s
Iteration 1: 13.072 ops/s
Iteration 2: 13.112 ops/s
Iteration 3: 13.103 ops/s
Iteration 4: 12.950 ops/s
Iteration 5: 13.011 ops/s
13.049 ±(99.9%) 0.263 ops/s [Average]
(min, avg, max) = (12.950, 13.049, 13.112), stdev = 0.068
CI (99.9%): [12.787, 13.312] (assumes normal distribution)
/home/commandertvis/graalvm-ce-java11/bin/java -XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UnlockExperimentalVMOptions -XX:ThreadPriorityPolicy=1 -javaagent:/home/commandertvis/.gradle/caches/modules-2/files-2.1/org.jetbrains.kotlinx/kotlinx-coroutines-core-jvm/1.5.0/d8cebccdcddd029022aa8646a5a953ff88b13ac8/kotlinx-coroutines-core-jvm-1.5.0.jar -Dfile.encoding=UTF-8 -Duser.country=US -Duser.language=en -Duser.variant -ea
```
* JMH 1.21 was used in `thrpt` mode with 1 warmup iteration by 1000 ms and 5 measurement iterations by 1000 ms.
| Benchmark | Score |
|:---------:|:-----:|
|`space.kscience.kmath.benchmarks.JafamaBenchmark.core`|14.296120859512893 &plusmn; 0.36462633435888736 ops/s|
|`space.kscience.kmath.benchmarks.JafamaBenchmark.jafama`|11.431566395649781 &plusmn; 2.570896777898243 ops/s|
|`space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafama`|11.746020495694117 &plusmn; 6.205909559197869 ops/s|
</details>

93
kmath-jafama/docs/README-TEMPLATE.md
View File

@ -24,95 +24,6 @@ fun main() {
## Performance
According to benchmarking data, on Hotspot Jafama functions are 20% faster than JDK math. On GraalVM, they are slower.
According to KMath benchmarks on GraalVM, Jafama functions are slower than JDK math; however, there are indications that on Hotspot Jafama is a bit faster.
<details>
<summary>Raw data:</summary>
**Hotspot**
```
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.coreBench
Warm-up 1: 11.447 ops/s
Iteration 1: 13.354 ops/s
Iteration 2: 14.237 ops/s
Iteration 3: 14.708 ops/s
Iteration 4: 14.629 ops/s
Iteration 5: 14.692 ops/s
14.324 ±(99.9%) 2.217 ops/s [Average]
(min, avg, max) = (13.354, 14.324, 14.708), stdev = 0.576
CI (99.9%): [12.107, 16.541] (assumes normal distribution)
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.jafamaBench
Warm-up 1: 15.628 ops/s
Iteration 1: 15.991 ops/s
Iteration 2: 16.633 ops/s
Iteration 3: 16.583 ops/s
Iteration 4: 16.716 ops/s
Iteration 5: 16.762 ops/s
16.537 ±(99.9%) 1.205 ops/s [Average]
(min, avg, max) = (15.991, 16.537, 16.762), stdev = 0.313
CI (99.9%): [15.332, 17.743] (assumes normal distribution)
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafamaBench
Warm-up 1: 13.378 ops/s
Iteration 1: 15.049 ops/s
Iteration 2: 14.468 ops/s
Iteration 3: 14.469 ops/s
Iteration 4: 14.753 ops/s
Iteration 5: 14.958 ops/s
14.739 ±(99.9%) 1.038 ops/s [Average]
(min, avg, max) = (14.468, 14.739, 15.049), stdev = 0.269
CI (99.9%): [13.701, 15.777] (assumes normal distribution)
```
**GraalVM**
```
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.coreBench
Warm-up 1: 14.357 ops/s
Iteration 1: 14.768 ops/s
Iteration 2: 14.922 ops/s
Iteration 3: 14.966 ops/s
Iteration 4: 14.805 ops/s
Iteration 5: 14.520 ops/s
14.796 ±(99.9%) 0.671 ops/s [Average]
(min, avg, max) = (14.520, 14.796, 14.966), stdev = 0.174
CI (99.9%): [14.125, 15.468] (assumes normal distribution)
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.jafamaBench
Warm-up 1: 11.592 ops/s
Iteration 1: 12.174 ops/s
Iteration 2: 11.734 ops/s
Iteration 3: 11.939 ops/s
Iteration 4: 12.026 ops/s
Iteration 5: 12.221 ops/s
12.019 ±(99.9%) 0.752 ops/s [Average]
(min, avg, max) = (11.734, 12.019, 12.221), stdev = 0.195
CI (99.9%): [11.267, 12.771] (assumes normal distribution)
jvm: space.kscience.kmath.benchmarks.JafamaBenchmark.strictJafamaBench
Warm-up 1: 12.097 ops/s
Iteration 1: 13.072 ops/s
Iteration 2: 13.112 ops/s
Iteration 3: 13.103 ops/s
Iteration 4: 12.950 ops/s
Iteration 5: 13.011 ops/s
13.049 ±(99.9%) 0.263 ops/s [Average]
(min, avg, max) = (12.950, 13.049, 13.112), stdev = 0.068
CI (99.9%): [12.787, 13.312] (assumes normal distribution)
```
</details>
${benchmarkJafamaDouble}

2
settings.gradle.kts
View File

@ -5,7 +5,7 @@ pluginManagement {
gradlePluginPortal()
}
val toolsVersion = "0.9.9"
val toolsVersion = "0.9.10"
val kotlinVersion = "1.5.0"
plugins {