Merge branch 'dev' into dev

.gitignore

@@ -4,6 +4,7 @@ out/
 
 .idea/
 .vscode/
+.fleet/
 
 
 # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored)

.space.kts

@@ -1,3 +1,48 @@
+import kotlin.io.path.readText
+
+val projectName = "kmath"
+
 job("Build") {
-    gradlew("openjdk:11", "build")
+    //Perform only jvm tests
+    gradlew("spc.registry.jetbrains.space/p/sci/containers/kotlin-ci:1.0.3", "test", "jvmTest")
+}
+
+job("Publish") {
+    startOn {
+        gitPush { enabled = false }
+    }
+    container("spc.registry.jetbrains.space/p/sci/containers/kotlin-ci:1.0.3") {
+        env["SPACE_USER"] = "{{ project:space_user }}"
+        env["SPACE_TOKEN"] = "{{ project:space_token }}"
+        kotlinScript { api ->
+
+            val spaceUser = System.getenv("SPACE_USER")
+            val spaceToken = System.getenv("SPACE_TOKEN")
+
+            // write the version to the build directory
+            api.gradlew("version")
+
+            //read the version from build file
+            val version = java.nio.file.Path.of("build/project-version.txt").readText()
+
+            val revisionSuffix = if (version.endsWith("SNAPSHOT")) {
+                "-" + api.gitRevision().take(7)
+            } else {
+                ""
+            }
+
+            api.space().projects.automation.deployments.start(
+                project = api.projectIdentifier(),
+                targetIdentifier = TargetIdentifier.Key(projectName),
+                version = version+revisionSuffix,
+                // automatically update deployment status based on the status of a job
+                syncWithAutomationJob = true
+            )
+            api.gradlew(
+                "publishAllPublicationsToSpaceRepository",
+                "-Ppublishing.space.user=\"$spaceUser\"",
+                "-Ppublishing.space.token=\"$spaceToken\"",
+            )
+        }
+    }
 }

CHANGELOG.md

@@ -14,7 +14,7 @@
 
 ### Security
 
-## 0.3.1-dev-RC - 2023-04-09
+## 0.3.1 - 2023-04-09
 
 ### Added
 - Wasm support for `memory`, `core`, `complex` and `functions` modules.

build.gradle.kts

@@ -15,7 +15,7 @@ allprojects {
     }
 
     group = "space.kscience"
-    version = "0.3.1-dev-RC"
+    version = "0.3.1"
 }
 
 subprojects {

kmath-ast/src/jsTest/kotlin/space/kscience/kmath/ast/utils.kt

@@ -3,8 +3,11 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
+@file:OptIn(UnstableKMathAPI::class)
+
 package space.kscience.kmath.ast
 
+import space.kscience.kmath.UnstableKMathAPI
 import space.kscience.kmath.expressions.Expression
 import space.kscience.kmath.expressions.MST
 import space.kscience.kmath.expressions.Symbol

kmath-ast/src/jsTest/kotlin/space/kscience/kmath/wasm/TestWasmSpecific.kt

@@ -5,6 +5,7 @@
 
 package space.kscience.kmath.wasm
 
+import space.kscience.kmath.UnstableKMathAPI
 import space.kscience.kmath.expressions.MstExtendedField
 import space.kscience.kmath.expressions.MstRing
 import space.kscience.kmath.expressions.invoke
@@ -15,6 +16,7 @@ import space.kscience.kmath.operations.invoke
 import kotlin.test.Test
 import kotlin.test.assertEquals
 
+@OptIn(UnstableKMathAPI::class)
 internal class TestWasmSpecific {
     @Test
     fun int() {

kmath-ast/src/jvmTest/kotlin/space/kscience/kmath/ast/utils.kt

@@ -5,6 +5,7 @@
 
 package space.kscience.kmath.ast
 
+import space.kscience.kmath.UnstableKMathAPI
 import space.kscience.kmath.expressions.Expression
 import space.kscience.kmath.expressions.MST
 import space.kscience.kmath.expressions.Symbol
@@ -30,6 +31,7 @@ private object GenericAsmCompilerTestContext : CompilerTestContext {
         asmCompile(algebra as Algebra<Double>, arguments)
 }
 
+@OptIn(UnstableKMathAPI::class)
 private object PrimitiveAsmCompilerTestContext : CompilerTestContext {
     override fun MST.compileToExpression(algebra: IntRing): Expression<Int> = asmCompileToExpression(algebra)
     override fun MST.compile(algebra: IntRing, arguments: Map<Symbol, Int>): Int = asmCompile(algebra, arguments)

kmath-core/src/commonMain/kotlin/space/kscience/kmath/misc/collections.kt

@@ -9,7 +9,7 @@ package space.kscience.kmath.misc
  * The same as [zipWithNext], but includes link between last and first element
  */
 public inline fun <T, R> List<T>.zipWithNextCircular(transform: (a: T, b: T) -> R): List<R> {
-    if (isEmpty()) return emptyList()
+    if (size < 2) return emptyList()
     return indices.map { i ->
         if (i == size - 1) {
             transform(last(), first())
@@ -19,4 +19,4 @@ public inline fun <T, R> List<T>.zipWithNextCircular(transform: (a: T, b: T) ->
     }
 }
 
-public fun <T> List<T>.zipWithNextCircular(): List<Pair<T,T>> = zipWithNextCircular { l, r -> l to r  }
+public fun <T> List<T>.zipWithNextCircular(): List<Pair<T, T>> = zipWithNextCircular { l, r -> l to r }

kmath-core/src/commonMain/kotlin/space/kscience/kmath/structures/bufferPrimitiveAccess.kt

@@ -5,7 +5,6 @@ import space.kscience.kmath.UnstableKMathAPI
 /**
  * Non-boxing access to primitive [Double]
  */
-
 @UnstableKMathAPI
 public fun Buffer<Double>.getDouble(index: Int): Double = if (this is BufferView) {
     val originIndex = originIndex(index)

kmath-kotlingrad/src/test/kotlin/space/kscience/kmath/kotlingrad/AdaptingTests.kt

@@ -6,6 +6,7 @@
 package space.kscience.kmath.kotlingrad
 
 import ai.hypergraph.kotlingrad.api.*
+import space.kscience.kmath.UnstableKMathAPI
 import space.kscience.kmath.asm.compileToExpression
 import space.kscience.kmath.ast.parseMath
 import space.kscience.kmath.expressions.MstNumericAlgebra
@@ -17,6 +18,7 @@ import kotlin.test.assertEquals
 import kotlin.test.assertTrue
 import kotlin.test.fail
 
+@OptIn(UnstableKMathAPI::class)
 internal class AdaptingTests {
     @Test
     fun symbol() {

kmath-memory/api/kmath-memory.api

@@ -1,3 +1,14 @@
+public abstract interface annotation class space/kscience/kmath/PerformancePitfall : java/lang/annotation/Annotation {
+	public abstract fun message ()Ljava/lang/String;
+}
+
+public abstract interface annotation class space/kscience/kmath/UnsafeKMathAPI : java/lang/annotation/Annotation {
+	public abstract fun message ()Ljava/lang/String;
+}
+
+public abstract interface annotation class space/kscience/kmath/UnstableKMathAPI : java/lang/annotation/Annotation {
+}
+
 public final class space/kscience/kmath/memory/ByteBufferMemory : space/kscience/kmath/memory/Memory {
 	public fun <init> (Ljava/nio/ByteBuffer;II)V
 	public synthetic fun <init> (Ljava/nio/ByteBuffer;IIILkotlin/jvm/internal/DefaultConstructorMarker;)V
@@ -36,7 +47,8 @@ public final class space/kscience/kmath/memory/MemoryKt {
 	public static final fun write (Lspace/kscience/kmath/memory/Memory;Lkotlin/jvm/functions/Function1;)V
 }
 
-public abstract interface class space/kscience/kmath/memory/MemoryReader {
+public abstract interface class space/kscience/kmath/memory/MemoryReader : java/lang/AutoCloseable {
+	public abstract fun close ()V
 	public abstract fun getMemory ()Lspace/kscience/kmath/memory/Memory;
 	public abstract fun readByte (I)B
 	public abstract fun readDouble (I)D
@@ -44,7 +56,6 @@ public abstract interface class space/kscience/kmath/memory/MemoryReader {
 	public abstract fun readInt (I)I
 	public abstract fun readLong (I)J
 	public abstract fun readShort (I)S
-	public abstract fun release ()V
 }
 
 public abstract interface class space/kscience/kmath/memory/MemorySpec {
@@ -59,9 +70,9 @@ public final class space/kscience/kmath/memory/MemorySpecKt {
 	public static final fun writeArray (Lspace/kscience/kmath/memory/MemoryWriter;Lspace/kscience/kmath/memory/MemorySpec;I[Ljava/lang/Object;)V
 }
 
-public abstract interface class space/kscience/kmath/memory/MemoryWriter {
+public abstract interface class space/kscience/kmath/memory/MemoryWriter : java/lang/AutoCloseable {
+	public abstract fun close ()V
 	public abstract fun getMemory ()Lspace/kscience/kmath/memory/Memory;
-	public abstract fun release ()V
 	public abstract fun writeByte (IB)V
 	public abstract fun writeDouble (ID)V
 	public abstract fun writeFloat (IF)V

kmath-stat/src/commonMain/kotlin/space/kscience/kmath/distributions/NormalDistribution.kt

@@ -6,6 +6,7 @@
 package space.kscience.kmath.distributions
 
 import space.kscience.kmath.chains.Chain
+import space.kscience.kmath.operations.DoubleField.pow
 import space.kscience.kmath.random.RandomGenerator
 import space.kscience.kmath.samplers.GaussianSampler
 import space.kscience.kmath.samplers.InternalErf
@@ -34,8 +35,23 @@ public class NormalDistribution(public val sampler: GaussianSampler) : Distribut
         }
     }
 
-    private companion object {
+    public companion object {
         private val SQRT2 = sqrt(2.0)
+
+        /**
+         * Zelen & Severo approximation for the standard normal CDF.
+         * The error upper boundary by 7.5 * 10e-8.
+         */
+        public fun zSNormalCDF(x: Double): Double {
+            val t = 1 / (1 + 0.2316419 * abs(x))
+            val sum = 0.319381530 * t -
+                    0.356563782 * t.pow(2) +
+                    1.781477937 * t.pow(3) -
+                    1.821255978 * t.pow(4) +
+                    1.330274429 * t.pow(5)
+            val temp = sum * exp(-abs(x).pow(2) / 2) / (2 * PI).pow(0.5)
+            return if (x >= 0) 1 - temp else temp
+        }
     }
 }
 

kmath-stat/src/commonMain/kotlin/space/kscience/kmath/series/SeriesAlgebra.kt

@@ -191,7 +191,7 @@ public open class SeriesAlgebra<T, out A : Ring<T>, out BA : BufferAlgebra<T, A>
         crossinline operation: A.(left: T, right: T) -> T,
     ): Series<T> {
         val newRange = offsetIndices.intersect(other.offsetIndices)
-        return seriesByOffset(startOffset = newRange.first, size = newRange.last - newRange.first) { offset ->
+        return seriesByOffset(startOffset = newRange.first, size = newRange.last + 1 - newRange.first) { offset ->
             elementAlgebra.operation(
                 getByOffset(offset),
                 other.getByOffset(offset)
@@ -199,12 +199,25 @@ public open class SeriesAlgebra<T, out A : Ring<T>, out BA : BufferAlgebra<T, A>
         }
     }
 
+    /**
+     * Zip buffer with itself, but shifted
+     * */
+    public inline fun Buffer<T>.zipWithShift(
+        shift: Int = 1,
+        crossinline operation: A.(left: T, right: T) -> T
+    ): Buffer<T> {
+        val shifted = this.moveBy(shift)
+        return zip(shifted, operation)
+    }
+
     override fun Buffer<T>.unaryMinus(): Buffer<T> = map { -it }
 
     override fun add(left: Buffer<T>, right: Buffer<T>): Series<T> = left.zip(right) { l, r -> l + r }
 
     override fun multiply(left: Buffer<T>, right: Buffer<T>): Buffer<T> = left.zip(right) { l, r -> l * r }
 
+    public fun Buffer<T>.difference(shift: Int=1): Buffer<T> = this.zipWithShift(shift) {l, r -> r - l}
+
     public companion object
 }
 

kmath-stat/src/commonMain/kotlin/space/kscience/kmath/series/VarianceRatioTest.kt

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2018-2023 KMath contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package space.kscience.kmath.series
+
+import space.kscience.kmath.distributions.NormalDistribution
+import space.kscience.kmath.operations.DoubleField.pow
+import space.kscience.kmath.operations.fold
+import kotlin.math.absoluteValue
+
+
+/**
+ * Container class for Variance Ratio Test result:
+ * ratio itself, corresponding Z-score, also it's p-value
+ */
+public data class VarianceRatioTestResult(
+    val varianceRatio: Double = 1.0,
+    val zScore: Double = 0.0,
+    val pValue: Double = 0.5,
+)
+
+
+/**
+ * Calculates the Z-statistic and the p-value for the Lo and MacKinlay's Variance Ratio test (1987)
+ * under Homoscedastic or Heteroscedstic assumptions
+ * with two-sided p-value test
+ * 	https://ssrn.com/abstract=346975
+ *
+ * 	@author https://github.com/mrFendel
+ */
+public fun SeriesAlgebra<Double, *, *, *>.varianceRatioTest(
+    series: Series<Double>,
+    shift: Int,
+    homoscedastic: Boolean = true,
+): VarianceRatioTestResult {
+
+    require(shift > 1) { "Shift must be greater than one" }
+    require(shift < series.size) { "Shift must be smaller than sample size" }
+    val sum = { x: Double, y: Double -> x + y }
+
+
+    val mean = series.fold(0.0, sum) / series.size
+    val demeanedSquares = series.map { (it - mean).pow(2) }
+    val variance = demeanedSquares.fold(0.0, sum)
+    if (variance == 0.0) return VarianceRatioTestResult()
+
+
+    var seriesAgg = series
+    for (i in 1..<shift) {
+        seriesAgg = seriesAgg.zip(series.moveTo(i)) { v1, v2 -> v1 + v2 }
+    }
+
+    val demeanedSquaresAgg = seriesAgg.map { (it - shift * mean).pow(2) }
+    val varianceAgg = demeanedSquaresAgg.fold(0.0, sum)
+
+    val varianceRatio =
+        varianceAgg * (series.size.toDouble() - 1) / variance / (series.size.toDouble() - shift.toDouble() + 1) / (1 - shift.toDouble() / series.size.toDouble()) / shift.toDouble()
+
+
+    // calculating asymptotic variance
+    val phi = if (homoscedastic) {  // under homoscedastic null hypothesis
+        2 * (2 * shift - 1.0) * (shift - 1.0) / (3 * shift * series.size)
+    } else { // under heteroscedastic null hypothesis
+        var accumulator = 0.0
+        for (j in 1..<shift) {
+            val temp = demeanedSquares
+            val delta = series.size * temp.zipWithShift(j) { v1, v2 -> v1 * v2 }.fold(0.0, sum) / variance.pow(2)
+            accumulator += delta * 4 * (shift - j).toDouble().pow(2) / shift.toDouble().pow(2)
+        }
+        accumulator
+    }
+
+    val zScore = (varianceRatio - 1) / phi.pow(0.5)
+    val pValue = 2 * (1 - NormalDistribution.zSNormalCDF(zScore.absoluteValue))
+    return VarianceRatioTestResult(varianceRatio, zScore, pValue)
+}
+
+
+
+

kmath-stat/src/commonTest/kotlin/space/kscience/kmath/series/TestVarianceRatioTest.kt

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2018-2023 KMath contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package space.kscience.kmath.series
+
+import space.kscience.kmath.operations.algebra
+import space.kscience.kmath.operations.bufferAlgebra
+import kotlin.math.PI
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class TestVarianceRatioTest {
+
+    @Test
+    fun monotonicData() {
+        with(Double.algebra.bufferAlgebra.seriesAlgebra()) {
+            val monotonicData = series(10) { it * 1.0 }
+            val resultHomo = varianceRatioTest(monotonicData, 2, homoscedastic = true)
+            assertEquals(1.818181, resultHomo.varianceRatio, 1e-6)
+            // homoscedastic zScore
+            assertEquals(2.587318, resultHomo.zScore, 1e-6)
+            assertEquals(.0096, resultHomo.pValue, 1e-4)
+            val resultHetero = varianceRatioTest(monotonicData, 2, homoscedastic = false)
+            // heteroscedastic zScore
+            assertEquals(0.819424, resultHetero.zScore, 1e-6)
+            assertEquals(.4125, resultHetero.pValue, 1e-4)
+        }
+    }
+
+    @Test
+    fun volatileData() {
+        with(Double.algebra.bufferAlgebra.seriesAlgebra()) {
+            val volatileData = series(10) { sin(PI * it + PI/2) + 1.0}
+            val resultHomo = varianceRatioTest(volatileData, 2)
+            assertEquals(0.0, resultHomo.varianceRatio, 1e-6)
+            // homoscedastic zScore
+            assertEquals(-3.162277, resultHomo.zScore, 1e-6)
+            assertEquals(.0015, resultHomo.pValue, 1e-4)
+            val resultHetero = varianceRatioTest(volatileData, 2, homoscedastic = false)
+            // heteroscedastic zScore
+            assertEquals(-1.0540925, resultHetero.zScore, 1e-6)
+            assertEquals(.2918, resultHetero.pValue, 1e-4)
+        }
+    }
+
+    @Test
+    fun negativeData() {
+        with(Double.algebra.bufferAlgebra.seriesAlgebra()) {
+            val negativeData = series(10) { sin(it * 1.2)}
+            val resultHomo = varianceRatioTest(negativeData, 3)
+            assertEquals(1.240031, resultHomo.varianceRatio, 1e-6)
+            // homoscedastic zScore
+            assertEquals(0.509183, resultHomo.zScore, 1e-6)
+            val resultHetero = varianceRatioTest(negativeData, 3, homoscedastic = false)
+            // heteroscedastic zScore
+            assertEquals(0.209202, resultHetero.zScore, 1e-6)
+        }
+    }
+
+    @Test
+    fun zeroVolatility() {
+        with(Double.algebra.bufferAlgebra.seriesAlgebra()) {
+            val zeroVolData = series(10) { 0.0 }
+            val result = varianceRatioTest(zeroVolData, 4)
+            assertEquals(1.0, result.varianceRatio, 1e-6)
+            assertEquals(0.0, result.zScore, 1e-6)
+            assertEquals(0.5, result.pValue, 1e-4)
+        }
+    }
+}

kmath-viktor/api/kmath-viktor.api

@@ -29,7 +29,7 @@ public class space/kscience/kmath/viktor/ViktorFieldND : space/kscience/kmath/vi
 	public synthetic fun getOne ()Ljava/lang/Object;
 	public synthetic fun getOne ()Lspace/kscience/kmath/nd/StructureND;
 	public fun getOne ()Lspace/kscience/kmath/viktor/ViktorStructureND;
-	public fun getShape ()[I
+	public fun getShape-IIYLAfE ()[I
 	public synthetic fun getZero ()Ljava/lang/Object;
 	public synthetic fun getZero ()Lspace/kscience/kmath/nd/StructureND;
 	public fun getZero ()Lspace/kscience/kmath/viktor/ViktorStructureND;
@@ -85,8 +85,8 @@ public class space/kscience/kmath/viktor/ViktorFieldOpsND : space/kscience/kmath
 	public fun sin (Lspace/kscience/kmath/nd/StructureND;)Lspace/kscience/kmath/viktor/ViktorStructureND;
 	public synthetic fun sinh (Ljava/lang/Object;)Ljava/lang/Object;
 	public fun sinh (Lspace/kscience/kmath/nd/StructureND;)Lspace/kscience/kmath/viktor/ViktorStructureND;
-	public synthetic fun structureND ([ILkotlin/jvm/functions/Function2;)Lspace/kscience/kmath/nd/StructureND;
-	public fun structureND ([ILkotlin/jvm/functions/Function2;)Lspace/kscience/kmath/viktor/ViktorStructureND;
+	public synthetic fun structureND-qL90JFI ([ILkotlin/jvm/functions/Function2;)Lspace/kscience/kmath/nd/StructureND;
+	public fun structureND-qL90JFI ([ILkotlin/jvm/functions/Function2;)Lspace/kscience/kmath/viktor/ViktorStructureND;
 	public synthetic fun tan (Ljava/lang/Object;)Ljava/lang/Object;
 	public fun tan (Lspace/kscience/kmath/nd/StructureND;)Lspace/kscience/kmath/viktor/ViktorStructureND;
 	public synthetic fun times (Ljava/lang/Object;Ljava/lang/Number;)Ljava/lang/Object;
@@ -112,7 +112,7 @@ public final class space/kscience/kmath/viktor/ViktorStructureND : space/kscienc
 	public fun get ([I)Ljava/lang/Double;
 	public synthetic fun get ([I)Ljava/lang/Object;
 	public final fun getF64Buffer ()Lorg/jetbrains/bio/viktor/F64Array;
-	public fun getShape ()[I
+	public fun getShape-IIYLAfE ()[I
 	public fun set ([ID)V
 	public synthetic fun set ([ILjava/lang/Object;)V
 }

test-utils/README.md

@@ -0,0 +1,4 @@
+# Module test-utils
+
+
+