Fix benchmark results script

This commit is contained in:
Alexander Nozik 2024-08-26 13:37:49 +03:00
parent 222cdc2c14
commit ab16bd16ac
6 changed files with 170 additions and 51 deletions

View File

@ -1,4 +1,119 @@
# Module benchmarks # BenchmarksResult
## Report for benchmark configuration <code>main</code>
* Run on Java HotSpot(TM) 64-Bit Server VM (build 21.0.4+8-LTS-jvmci-23.1-b41) with Java process:
```
C:\Users\altavir\scoop\apps\graalvm-oracle-21jdk\current\bin\java.exe -XX:ThreadPriorityPolicy=1 -XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UnlockExperimentalVMOptions -Dfile.encoding=UTF-8 -Duser.country=US -Duser.language=en -Duser.variant
```
* JMH 1.21 was used in `thrpt` mode with 5 warmup iterations by 10 s and 5 measurement iterations by 10 s.
### [ArrayBenchmark](src/jvmMain/kotlin/space/kscience/kmath/benchmarks/ArrayBenchmark.kt)
| Benchmark | Score |
|:---------:|:-----:|
|`benchmarkArrayRead`|1.9E+07 &plusmn; 2.3E+05 ops/s|
|`benchmarkBufferRead`|1.4E+07 &plusmn; 8.7E+05 ops/s|
|`nativeBufferRead`|1.4E+07 &plusmn; 1.3E+06 ops/s|
### [BigIntBenchmark](src/jvmMain/kotlin/space/kscience/kmath/benchmarks/BigIntBenchmark.kt)
| Benchmark | Score |
|:---------:|:-----:|
|`jvmAdd`|5.1E+07 &plusmn; 1.3E+06 ops/s|
|`jvmAddLarge`|5.1E+04 &plusmn; 8.2E+02 ops/s|
|`jvmMultiply`|8.5E+07 &plusmn; 9.7E+06 ops/s|
|`jvmMultiplyLarge`|2.5E+02 &plusmn; 15 ops/s|
|`jvmParsing10`|8.7E+06 &plusmn; 5.1E+05 ops/s|
|`jvmParsing16`|6.4E+06 &plusmn; 1.8E+05 ops/s|
|`jvmPower`|28 &plusmn; 0.79 ops/s|
|`jvmSmallAdd`|7.0E+07 &plusmn; 4.3E+06 ops/s|
|`kmAdd`|4.8E+07 &plusmn; 2.2E+06 ops/s|
|`kmAddLarge`|3.5E+04 &plusmn; 3.7E+03 ops/s|
|`kmMultiply`|6.7E+07 &plusmn; 1.5E+07 ops/s|
|`kmMultiplyLarge`|54 &plusmn; 4.2 ops/s|
|`kmParsing10`|4.5E+06 &plusmn; 8.3E+04 ops/s|
|`kmParsing16`|4.9E+06 &plusmn; 1.1E+05 ops/s|
|`kmPower`|10 &plusmn; 0.96 ops/s|
|`kmSmallAdd`|4.1E+07 &plusmn; 5.9E+05 ops/s|
### [BufferBenchmark](src/jvmMain/kotlin/space/kscience/kmath/benchmarks/BufferBenchmark.kt)
| Benchmark | Score |
|:---------:|:-----:|
|`bufferViewReadWrite`|5.8E+06 &plusmn; 1.6E+05 ops/s|
|`bufferViewReadWriteSpecialized`|5.6E+06 &plusmn; 2.6E+05 ops/s|
|`complexBufferReadWrite`|6.6E+06 &plusmn; 2.7E+05 ops/s|
|`doubleArrayReadWrite`|7.5E+06 &plusmn; 1.0E+06 ops/s|
|`doubleBufferReadWrite`|8.0E+06 &plusmn; 6.7E+05 ops/s|
### [DotBenchmark](src/jvmMain/kotlin/space/kscience/kmath/benchmarks/DotBenchmark.kt)
| Benchmark | Score |
|:---------:|:-----:|
|`bufferedDot`|1.3 &plusmn; 0.020 ops/s|
|`cmDot`|0.47 &plusmn; 0.42 ops/s|
|`cmDotWithConversion`|0.76 &plusmn; 0.13 ops/s|
|`ejmlDot`|6.7 &plusmn; 0.091 ops/s|
|`ejmlDotWithConversion`|6.4 &plusmn; 0.82 ops/s|
|`multikDot`|40 &plusmn; 6.7 ops/s|
|`parallelDot`|12 &plusmn; 1.8 ops/s|
|`tensorDot`|1.2 &plusmn; 0.041 ops/s|
|`tfDot`|5.9 &plusmn; 0.49 ops/s|
### [ExpressionsInterpretersBenchmark](src/jvmMain/kotlin/space/kscience/kmath/benchmarks/ExpressionsInterpretersBenchmark.kt)
| Benchmark | Score |
|:---------:|:-----:|
|`asmGenericExpression`|29 &plusmn; 1.2 ops/s|
|`asmPrimitiveExpression`|43 &plusmn; 1.3 ops/s|
|`asmPrimitiveExpressionArray`|71 &plusmn; 0.38 ops/s|
|`functionalExpression`|5.6 &plusmn; 0.11 ops/s|
|`justCalculate`|69 &plusmn; 9.0 ops/s|
|`mstExpression`|7.1 &plusmn; 0.020 ops/s|
|`rawExpression`|41 &plusmn; 1.5 ops/s|
### [IntegrationBenchmark](src/jvmMain/kotlin/space/kscience/kmath/benchmarks/IntegrationBenchmark.kt)
| Benchmark | Score |
|:---------:|:-----:|
|`complexIntegration`|3.6E+03 &plusmn; 1.9E+02 ops/s|
|`doubleIntegration`|3.7E+03 &plusmn; 12 ops/s|
### [JafamaBenchmark](src/jvmMain/kotlin/space/kscience/kmath/benchmarks/JafamaBenchmark.kt)
| Benchmark | Score |
|:---------:|:-----:|
|`core`|38 &plusmn; 0.64 ops/s|
|`jafama`|52 &plusmn; 0.36 ops/s|
|`strictJafama`|52 &plusmn; 4.0 ops/s|
### [MatrixInverseBenchmark](src/jvmMain/kotlin/space/kscience/kmath/benchmarks/MatrixInverseBenchmark.kt)
| Benchmark | Score |
|:---------:|:-----:|
|`cmLUPInversion`|2.2E+03 &plusmn; 76 ops/s|
|`ejmlInverse`|1.3E+03 &plusmn; 5.7 ops/s|
|`kmathLupInversion`|9.5E+02 &plusmn; 1.8E+02 ops/s|
|`kmathParallelLupInversion`|9.1E+02 &plusmn; 1.4E+02 ops/s|
### [NDFieldBenchmark](src/jvmMain/kotlin/space/kscience/kmath/benchmarks/NDFieldBenchmark.kt)
| Benchmark | Score |
|:---------:|:-----:|
|`boxingFieldAdd`|7.7 &plusmn; 0.79 ops/s|
|`multikAdd`|6.5 &plusmn; 0.33 ops/s|
|`multikInPlaceAdd`|64 &plusmn; 0.79 ops/s|
|`specializedFieldAdd`|8.0 &plusmn; 0.090 ops/s|
|`tensorAdd`|9.2 &plusmn; 0.053 ops/s|
|`tensorInPlaceAdd`|17 &plusmn; 10 ops/s|
|`viktorAdd`|7.6 &plusmn; 1.2 ops/s|
### [ViktorBenchmark](src/jvmMain/kotlin/space/kscience/kmath/benchmarks/ViktorBenchmark.kt)
| Benchmark | Score |
|:---------:|:-----:|
|`doubleFieldAddition`|7.7 &plusmn; 0.34 ops/s|
|`rawViktor`|5.9 &plusmn; 1.1 ops/s|
|`viktorFieldAddition`|7.3 &plusmn; 1.1 ops/s|
### [ViktorLogBenchmark](src/jvmMain/kotlin/space/kscience/kmath/benchmarks/ViktorLogBenchmark.kt)
| Benchmark | Score |
|:---------:|:-----:|
|`rawViktorLog`|1.4 &plusmn; 0.076 ops/s|
|`realFieldLog`|1.3 &plusmn; 0.069 ops/s|
|`viktorFieldLog`|1.3 &plusmn; 0.032 ops/s|

View File

@ -3,10 +3,6 @@ import com.fasterxml.jackson.module.kotlin.readValue
import kotlinx.benchmark.gradle.BenchmarksExtension import kotlinx.benchmark.gradle.BenchmarksExtension
import java.time.LocalDateTime import java.time.LocalDateTime
import java.time.ZoneId import java.time.ZoneId
import java.time.format.DateTimeFormatter
import java.time.format.DateTimeFormatterBuilder
import java.time.format.SignStyle
import java.time.temporal.ChronoField.*
import java.util.* import java.util.*
plugins { plugins {
@ -220,41 +216,20 @@ readme {
val jsonMapper = jacksonObjectMapper() val jsonMapper = jacksonObjectMapper()
val ISO_DATE_TIME: DateTimeFormatter = DateTimeFormatterBuilder().run {
parseCaseInsensitive()
appendValue(YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
appendLiteral('-')
appendValue(MONTH_OF_YEAR, 2)
appendLiteral('-')
appendValue(DAY_OF_MONTH, 2)
appendLiteral('T')
appendValue(HOUR_OF_DAY, 2)
appendLiteral('.')
appendValue(MINUTE_OF_HOUR, 2)
optionalStart()
appendLiteral('.')
appendValue(SECOND_OF_MINUTE, 2)
optionalStart()
appendFraction(NANO_OF_SECOND, 0, 9, true)
optionalStart()
appendOffsetId()
optionalStart()
appendLiteral('[')
parseCaseSensitive()
appendZoneRegionId()
appendLiteral(']')
toFormatter()
}
fun noun(number: Number, singular: String, plural: String) = if (number.toLong() == 1L) singular else plural fun noun(number: Number, singular: String, plural: String) = if (number.toLong() == 1L) singular else plural
extensions.findByType(BenchmarksExtension::class.java)?.configurations?.forEach { cfg -> extensions.findByType(BenchmarksExtension::class.java)?.configurations?.forEach { cfg ->
property("benchmark${cfg.name.replaceFirstChar { if (it.isLowerCase()) it.titlecase(Locale.getDefault()) else it.toString() }}") { val propertyName =
val launches = layout.buildDirectory.dir("reports/benchmarks/${cfg.name}").get() "benchmark${cfg.name.replaceFirstChar { if (it.isLowerCase()) it.titlecase(Locale.getDefault()) else it.toString() }}"
val resDirectory = launches.files().maxByOrNull { logger.info("Processing benchmark data from benchmark ${cfg.name} into readme property $propertyName")
LocalDateTime.parse(it.name, ISO_DATE_TIME).atZone(ZoneId.systemDefault()).toInstant()
val launches = layout.buildDirectory.dir("reports/benchmarks/${cfg.name}").get().asFile
if (!launches.exists()) return@forEach
property(propertyName) {
val resDirectory = launches.listFiles()?.maxByOrNull {
LocalDateTime.parse(it.name).atZone(ZoneId.systemDefault()).toInstant()
} }
if (resDirectory == null || !(resDirectory.resolve("jvm.json")).exists()) { if (resDirectory == null || !(resDirectory.resolve("jvm.json")).exists()) {
@ -264,10 +239,7 @@ readme {
jsonMapper.readValue<List<JmhReport>>(resDirectory.resolve("jvm.json")) jsonMapper.readValue<List<JmhReport>>(resDirectory.resolve("jvm.json"))
buildString { buildString {
appendLine("<details>") appendLine("## Report for benchmark configuration <code>${cfg.name}</code>")
appendLine("<summary>")
appendLine("Report for benchmark configuration <code>${cfg.name}</code>")
appendLine("</summary>")
appendLine() appendLine()
val first = reports.first() val first = reports.first()
@ -289,15 +261,19 @@ readme {
} by ${first.measurementTime}." } by ${first.measurementTime}."
) )
appendLine() reports.groupBy { it.benchmark.substringBeforeLast(".") }.forEach { (cl, compare) ->
appendLine("| Benchmark | Score |") appendLine("### [${cl.substringAfterLast(".")}](src/jvmMain/kotlin/${cl.replace(".","/")}.kt)")
appendLine("|:---------:|:-----:|") appendLine()
appendLine("| Benchmark | Score |")
appendLine("|:---------:|:-----:|")
compare.forEach { report ->
val benchmarkName = report.benchmark.substringAfterLast(".")
val score = String.format("%.2G", report.primaryMetric.score)
val error = String.format("%.2G", report.primaryMetric.scoreError)
reports.forEach { report -> appendLine("|`$benchmarkName`|$score &plusmn; $error ${report.primaryMetric.scoreUnit}|")
appendLine("|`${report.benchmark}`|${report.primaryMetric.score} &plusmn; ${report.primaryMetric.scoreError} ${report.primaryMetric.scoreUnit}|") }
} }
appendLine("</details>")
} }
} }
} }

View File

@ -0,0 +1,5 @@
# BenchmarksResult
${benchmarkMain}

View File

@ -36,6 +36,26 @@ private suspend fun runKMathChained(): Duration {
return Duration.between(startTime, Instant.now()) return Duration.between(startTime, Instant.now())
} }
private fun runKMathBlocking(): Duration {
val generator = RandomGenerator.fromSource(RandomSource.MT, 123L)
val normal = GaussianSampler(7.0, 2.0)
val chain = normal.sample(generator)
val startTime = Instant.now()
var sum = 0.0
repeat(10000001) { counter ->
sum += chain.nextBlocking()
if (counter % 100000 == 0) {
val duration = Duration.between(startTime, Instant.now())
val meanValue = sum / counter
println("Chain sampler completed $counter elements in $duration: $meanValue")
}
}
return Duration.between(startTime, Instant.now())
}
private fun runCMDirect(): Duration { private fun runCMDirect(): Duration {
val rng = RandomSource.MT.create(123L) val rng = RandomSource.MT.create(123L)
@ -67,6 +87,8 @@ private fun runCMDirect(): Duration {
fun main(): Unit = runBlocking(Dispatchers.Default) { fun main(): Unit = runBlocking(Dispatchers.Default) {
val directJob = async { runCMDirect() } val directJob = async { runCMDirect() }
val chainJob = async { runKMathChained() } val chainJob = async { runKMathChained() }
val blockingJob = async { runKMathBlocking() }
println("KMath Chained: ${chainJob.await()}") println("KMath Chained: ${chainJob.await()}")
println("KMath Blocking: ${blockingJob.await()}")
println("Apache Direct: ${directJob.await()}") println("Apache Direct: ${directJob.await()}")
} }

View File

@ -67,13 +67,14 @@ public fun <T : Any, A : Ring<T>> MatrixBuilder<T, A>.symmetric(
): Matrix<T> { ): Matrix<T> {
require(columns == rows) { "In order to build symmetric matrix, number of rows $rows should be equal to number of columns $columns" } require(columns == rows) { "In order to build symmetric matrix, number of rows $rows should be equal to number of columns $columns" }
return with(BufferAccessor2D<T?>(rows, rows, MutableBufferFactory(type))) { return with(BufferAccessor2D<T?>(rows, rows, MutableBufferFactory(type))) {
val cache = factory(rows * rows) { null } val cache = HashMap<IntArray, T>()
linearSpace.buildMatrix(rows, rows) { i, j -> linearSpace.buildMatrix(rows, rows) { i, j ->
val cached = cache[i, j] val index = intArrayOf(i, j)
val cached = cache[index]
if (cached == null) { if (cached == null) {
val value = if (i <= j) builder(i, j) else builder(j, i) val value = if (i <= j) builder(i, j) else builder(j, i)
cache[i, j] = value cache[index] = value
cache[j, i] = value cache[index] = value
value value
} else { } else {
cached cached