Merge pull request #510 from SciProgCentre/dev

0.3.1-dev-11

.github/workflows/build.yml

@@ -7,26 +7,18 @@ on:
 
 jobs:
   build:
-    strategy:
+    runs-on: windows-latest
-      matrix:
+    timeout-minutes: 20
-        os: [ macOS-latest, windows-latest ]
-    runs-on: ${{matrix.os}}
-    timeout-minutes: 40
     steps:
-      - uses: actions/checkout@v3.0.0
+      - uses: actions/checkout@v3
-      - uses: actions/setup-java@v3.0.0
+      - uses: actions/setup-java@v3.5.1
         with:
-          java-version: 11
+          java-version: '11'
-          distribution: liberica
+          distribution: 'liberica'
-      - name: Cache konan
+          cache: 'gradle'
-        uses: actions/cache@v3.0.1
-        with:
-          path: ~/.konan
-          key: ${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts') }}
-          restore-keys: |
-            ${{ runner.os }}-gradle-
       - name: Gradle Wrapper Validation
         uses: gradle/wrapper-validation-action@v1.0.4
-      - uses: gradle/gradle-build-action@v2.1.5
+      - name: Gradle Build
+        uses: gradle/gradle-build-action@v2.3.2
         with:
-          arguments: build
+          arguments: test jvmTest

.github/workflows/publish.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ${{matrix.os}}
     steps:
       - uses: actions/checkout@v3.0.0
-      - uses: actions/setup-java@v3.0.0
+      - uses: actions/setup-java@v3.10.0
         with:
           java-version: 11
           distribution: liberica
@@ -26,26 +26,25 @@ jobs:
           key: ${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts') }}
           restore-keys: |
             ${{ runner.os }}-gradle-
-      - uses: gradle/wrapper-validation-action@v1.0.4
       - name: Publish Windows Artifacts
         if: matrix.os == 'windows-latest'
-        uses: gradle/gradle-build-action@v2.1.5
+        uses: gradle/gradle-build-action@v2.4.0
         with:
           arguments: |
-            releaseAll
+            publishAllPublicationsToSpaceRepository
-            -Ppublishing.enabled=true
+            -Ppublishing.targets=all
-            -Ppublishing.sonatype=false
             -Ppublishing.space.user=${{ secrets.SPACE_APP_ID }}
             -Ppublishing.space.token=${{ secrets.SPACE_APP_SECRET }}
       - name: Publish Mac Artifacts
         if: matrix.os == 'macOS-latest'
-        uses: gradle/gradle-build-action@v2.1.5
+        uses: gradle/gradle-build-action@v2.4.0
         with:
           arguments: |
-            releaseMacosX64
+            publishMacosX64PublicationToSpaceRepository
-            releaseIosArm64
+            publishMacosArm64PublicationToSpaceRepository
-            releaseIosX64
+            publishIosX64PublicationToSpaceRepository
-            -Ppublishing.enabled=true
+            publishIosArm64PublicationToSpaceRepository
-            -Ppublishing.sonatype=false            
+            publishIosSimulatorArm64PublicationToSpaceRepository
+            -Ppublishing.targets=all
             -Ppublishing.space.user=${{ secrets.SPACE_APP_ID }}
             -Ppublishing.space.token=${{ secrets.SPACE_APP_SECRET }}

.gitignore

@@ -3,10 +3,9 @@ build/
 out/
 
 .idea/
-
-
 .vscode/
 
+
 # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored)
 !gradle-wrapper.jar
 
@@ -19,4 +18,5 @@ out/
 
 !/.idea/copyright/
 !/.idea/scopes/
-/kotlin-js-store/yarn.lock
+/gradle/yarn.lock
+

CHANGELOG.md

@@ -2,11 +2,19 @@
 
 ## [Unreleased]
 ### Added
+- Generic builders for `BufferND` and `MutableBufferND`
+- `NamedMatrix` - matrix with symbol-based indexing
+- `Expression` with default arguments
+- Type-aliases for numbers like `Float64`
 - 2D optimal trajectory computation in a separate module `kmath-trajectory`
 - Autodiff for generic algebra elements in core!
 - Algebra now has an obligatory `bufferFactory` (#477).
 
 ### Changed
+- Trajectory use type-safe angles
+- Tensor operations switched to prefix notation
+- Row-wise and column-wise ND shapes in the core
+- Shape is read-only
 - Major refactor of tensors (only minor API changes)
 - Kotlin 1.7.20
 - `LazyStructure` `deffered` -> `async` to comply with coroutines code style
@@ -16,6 +24,8 @@
 ### Deprecated
 
 ### Removed
+- Trajectory moved to https://github.com/SciProgCentre/maps-kt
+- Polynomials moved to https://github.com/SciProgCentre/kmath-polynomial
 
 ### Fixed
 

benchmarks/build.gradle.kts

@@ -5,7 +5,7 @@ import space.kscience.kmath.benchmarks.addBenchmarkProperties
 
 plugins {
     kotlin("multiplatform")
-    kotlin("plugin.allopen")
+    alias(spclibs.plugins.kotlin.plugin.allopen)
     id("org.jetbrains.kotlinx.benchmark")
 }
 
@@ -44,7 +44,7 @@ kotlin {
                 implementation(project(":kmath-tensors"))
                 implementation(project(":kmath-multik"))
                 implementation("org.jetbrains.kotlinx:multik-default:$multikVersion")
-                implementation(npmlibs.kotlinx.benchmark.runtime)
+                implementation(spclibs.kotlinx.benchmark.runtime)
             }
         }
 
@@ -142,12 +142,10 @@ benchmark {
         commonConfiguration()
         include("ViktorLogBenchmark")
     }
-}
 
-// Fix kotlinx-benchmarks bug
+    configurations.register("integration") {
-afterEvaluate {
+        commonConfiguration()
-    val jvmBenchmarkJar by tasks.getting(org.gradle.jvm.tasks.Jar::class) {
+        include("IntegrationBenchmark")
-        duplicatesStrategy = DuplicatesStrategy.EXCLUDE
     }
 }
 

benchmarks/src/jvmMain/kotlin/space/kscience/kmath/benchmarks/BufferBenchmark.kt

@@ -6,34 +6,75 @@
 package space.kscience.kmath.benchmarks
 
 import kotlinx.benchmark.Benchmark
+import kotlinx.benchmark.Blackhole
 import kotlinx.benchmark.Scope
 import kotlinx.benchmark.State
 import space.kscience.kmath.complex.Complex
+import space.kscience.kmath.complex.ComplexField
 import space.kscience.kmath.complex.complex
+import space.kscience.kmath.operations.invoke
+import space.kscience.kmath.structures.Buffer
 import space.kscience.kmath.structures.DoubleBuffer
-import space.kscience.kmath.structures.MutableBuffer
+import space.kscience.kmath.structures.getDouble
+import space.kscience.kmath.structures.permute
 
 @State(Scope.Benchmark)
 internal class BufferBenchmark {
-    @Benchmark
-    fun genericDoubleBufferReadWrite() {
-        val buffer = DoubleBuffer(size) { it.toDouble() }
 
+    @Benchmark
+    fun doubleArrayReadWrite(blackhole: Blackhole) {
+        val buffer = DoubleArray(size) { it.toDouble() }
+        var res = 0.0
         (0 until size).forEach {
-            buffer[it]
+            res += buffer[it]
         }
+        blackhole.consume(res)
     }
 
     @Benchmark
-    fun complexBufferReadWrite() {
+    fun doubleBufferReadWrite(blackhole: Blackhole) {
-        val buffer = MutableBuffer.complex(size / 2) { Complex(it.toDouble(), -it.toDouble()) }
+        val buffer = DoubleBuffer(size) { it.toDouble() }
-
+        var res = 0.0
-        (0 until size / 2).forEach {
+        (0 until size).forEach {
-            buffer[it]
+            res += buffer[it]
         }
+        blackhole.consume(res)
+    }
+
+    @Benchmark
+    fun bufferViewReadWrite(blackhole: Blackhole) {
+        val buffer = DoubleBuffer(size) { it.toDouble() }.permute(reversedIndices)
+        var res = 0.0
+        (0 until size).forEach {
+            res += buffer[it]
+        }
+        blackhole.consume(res)
+    }
+
+    @Benchmark
+    fun bufferViewReadWriteSpecialized(blackhole: Blackhole) {
+        val buffer = DoubleBuffer(size) { it.toDouble() }.permute(reversedIndices)
+        var res = 0.0
+        (0 until size).forEach {
+            res += buffer.getDouble(it)
+        }
+        blackhole.consume(res)
+    }
+
+    @Benchmark
+    fun complexBufferReadWrite(blackhole: Blackhole) = ComplexField {
+        val buffer = Buffer.complex(size / 2) { Complex(it.toDouble(), -it.toDouble()) }
+
+        var res = zero
+        (0 until size / 2).forEach {
+            res += buffer[it]
+        }
+
+        blackhole.consume(res)
     }
 
     private companion object {
         private const val size = 100
+        private val reversedIndices = IntArray(size){it}.apply { reverse() }
     }
 }

benchmarks/src/jvmMain/kotlin/space/kscience/kmath/benchmarks/IntegrationBenchmark.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2018-2023 KMath contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package space.kscience.kmath.benchmarks
+
+import org.openjdk.jmh.annotations.Benchmark
+import org.openjdk.jmh.annotations.Scope
+import org.openjdk.jmh.annotations.State
+import org.openjdk.jmh.infra.Blackhole
+import space.kscience.kmath.complex.Complex
+import space.kscience.kmath.complex.algebra
+import space.kscience.kmath.integration.gaussIntegrator
+import space.kscience.kmath.integration.integrate
+import space.kscience.kmath.integration.value
+import space.kscience.kmath.operations.algebra
+
+
+@State(Scope.Benchmark)
+internal class IntegrationBenchmark {
+
+    @Benchmark
+    fun doubleIntegration(blackhole: Blackhole) {
+        val res = Double.algebra.gaussIntegrator.integrate(0.0..1.0, intervals = 1000) { x: Double ->
+            //sin(1 / x)
+            1/x
+        }.value
+        blackhole.consume(res)
+    }
+
+    @Benchmark
+    fun complexIntegration(blackhole: Blackhole) = with(Complex.algebra) {
+        val res = gaussIntegrator.integrate(0.0..1.0, intervals = 1000) { x: Double ->
+//            sin(1 / x) + i * cos(1 / x)
+            1/x - i/x
+        }.value
+        blackhole.consume(res)
+    }
+}

benchmarks/src/jvmMain/kotlin/space/kscience/kmath/benchmarks/NDFieldBenchmark.kt

@@ -13,10 +13,8 @@ import org.jetbrains.kotlinx.multik.api.Multik
 import org.jetbrains.kotlinx.multik.api.ones
 import org.jetbrains.kotlinx.multik.ndarray.data.DN
 import org.jetbrains.kotlinx.multik.ndarray.data.DataType
-import space.kscience.kmath.nd.BufferedFieldOpsND
+import space.kscience.kmath.misc.UnsafeKMathAPI
-import space.kscience.kmath.nd.StructureND
+import space.kscience.kmath.nd.*
-import space.kscience.kmath.nd.ndAlgebra
-import space.kscience.kmath.nd.one
 import space.kscience.kmath.nd4j.nd4j
 import space.kscience.kmath.operations.DoubleField
 import space.kscience.kmath.tensors.core.DoubleTensor
@@ -69,9 +67,10 @@ internal class NDFieldBenchmark {
         blackhole.consume(res)
     }
 
+    @OptIn(UnsafeKMathAPI::class)
     @Benchmark
     fun multikInPlaceAdd(blackhole: Blackhole) = with(multikAlgebra) {
-        val res = Multik.ones<Double, DN>(shape, DataType.DoubleDataType).wrap()
+        val res = Multik.ones<Double, DN>(shape.asArray(), DataType.DoubleDataType).wrap()
         repeat(n) { res += 1.0 }
         blackhole.consume(res)
     }
@@ -86,7 +85,7 @@ internal class NDFieldBenchmark {
     private companion object {
         private const val dim = 1000
         private const val n = 100
-        private val shape = intArrayOf(dim, dim)
+        private val shape = ShapeND(dim, dim)
         private val specializedField = DoubleField.ndAlgebra
         private val genericField = BufferedFieldOpsND(DoubleField)
         private val nd4jField = DoubleField.nd4j

benchmarks/src/jvmMain/kotlin/space/kscience/kmath/benchmarks/TensorAlgebraBenchmark.kt

@@ -13,6 +13,8 @@ import space.kscience.kmath.linear.linearSpace
 import space.kscience.kmath.linear.matrix
 import space.kscience.kmath.linear.symmetric
 import space.kscience.kmath.operations.DoubleField
+import space.kscience.kmath.tensors.core.symEigJacobi
+import space.kscience.kmath.tensors.core.symEigSvd
 import space.kscience.kmath.tensors.core.tensorAlgebra
 import kotlin.random.Random
 
@@ -27,11 +29,11 @@ internal class TensorAlgebraBenchmark {
 
     @Benchmark
     fun tensorSymEigSvd(blackhole: Blackhole) = with(Double.tensorAlgebra) {
-        blackhole.consume(matrix.symEigSvd(1e-10))
+        blackhole.consume(symEigSvd(matrix, 1e-10))
     }
 
     @Benchmark
     fun tensorSymEigJacobi(blackhole: Blackhole) = with(Double.tensorAlgebra) {
-        blackhole.consume(matrix.symEigJacobi(50, 1e-10))
+        blackhole.consume(symEigJacobi(matrix, 50, 1e-10))
     }
 }

benchmarks/src/jvmMain/kotlin/space/kscience/kmath/benchmarks/ViktorBenchmark.kt

@@ -10,7 +10,7 @@ import kotlinx.benchmark.Blackhole
 import kotlinx.benchmark.Scope
 import kotlinx.benchmark.State
 import org.jetbrains.bio.viktor.F64Array
-import space.kscience.kmath.nd.Shape
+import space.kscience.kmath.nd.ShapeND
 import space.kscience.kmath.nd.StructureND
 import space.kscience.kmath.nd.ndAlgebra
 import space.kscience.kmath.nd.one
@@ -49,7 +49,7 @@ internal class ViktorBenchmark {
     private companion object {
         private const val dim = 1000
         private const val n = 100
-        private val shape = Shape(dim, dim)
+        private val shape = ShapeND(dim, dim)
 
         // automatically build context most suited for given type.
         private val doubleField = DoubleField.ndAlgebra

benchmarks/src/jvmMain/kotlin/space/kscience/kmath/benchmarks/ViktorLogBenchmark.kt

@@ -10,7 +10,7 @@ import kotlinx.benchmark.Blackhole
 import kotlinx.benchmark.Scope
 import kotlinx.benchmark.State
 import org.jetbrains.bio.viktor.F64Array
-import space.kscience.kmath.nd.Shape
+import space.kscience.kmath.nd.ShapeND
 import space.kscience.kmath.nd.ndAlgebra
 import space.kscience.kmath.nd.one
 import space.kscience.kmath.operations.DoubleField
@@ -49,7 +49,7 @@ internal class ViktorLogBenchmark {
     private companion object {
         private const val dim = 1000
         private const val n = 100
-        private val shape = Shape(dim, dim)
+        private val shape = ShapeND(dim, dim)
 
         // automatically build context most suited for given type.
         private val doubleField = DoubleField.ndAlgebra

build.gradle.kts

@@ -15,7 +15,7 @@ allprojects {
     }
 
     group = "space.kscience"
-    version = "0.3.1-dev-4"
+    version = "0.3.1-dev-11"
 }
 
 subprojects {
@@ -79,9 +79,9 @@ ksciencePublish {
     github("kmath", "SciProgCentre")
     space(
         if (isInDevelopment) {
-            "https://maven.pkg.jetbrains.space/mipt-npm/p/sci/dev"
+            "https://maven.pkg.jetbrains.space/spc/p/sci/dev"
         } else {
-            "https://maven.pkg.jetbrains.space/mipt-npm/p/sci/release"
+            "https://maven.pkg.jetbrains.space/spc/p/sci/maven"
         }
     )
     sonatype()

buildSrc/build.gradle.kts

@@ -1,7 +1,6 @@
 plugins {
     `kotlin-dsl`
     `version-catalog`
-    kotlin("plugin.serialization") version "1.6.21"
 }
 
 java.targetCompatibility = JavaVersion.VERSION_11
@@ -13,18 +12,18 @@ repositories {
     gradlePluginPortal()
 }
 
-val toolsVersion = npmlibs.versions.tools.get()
+val toolsVersion = spclibs.versions.tools.get()
-val kotlinVersion = npmlibs.versions.kotlin.asProvider().get()
+val kotlinVersion = spclibs.versions.kotlin.asProvider().get()
-val benchmarksVersion = npmlibs.versions.kotlinx.benchmark.get()
+val benchmarksVersion = spclibs.versions.kotlinx.benchmark.get()
 
 dependencies {
     api("space.kscience:gradle-tools:$toolsVersion")
-    api(npmlibs.atomicfu.gradle)
     //plugins form benchmarks
-    api("org.jetbrains.kotlinx:kotlinx-benchmark-plugin:$benchmarksVersion")
+    api("org.jetbrains.kotlinx:kotlinx-benchmark-plugin:0.4.7")
-    api("org.jetbrains.kotlin:kotlin-allopen:$kotlinVersion")
+    //api("org.jetbrains.kotlin:kotlin-allopen:$kotlinVersion")
     //to be used inside build-script only
-    implementation(npmlibs.kotlinx.serialization.json)
+    //implementation(spclibs.kotlinx.serialization.json)
+    implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.14.+")
 }
 
 kotlin.sourceSets.all {

buildSrc/settings.gradle.kts

@@ -26,7 +26,7 @@ dependencyResolutionManagement {
     }
 
     versionCatalogs {
-        create("npmlibs") {
+        create("spclibs") {
             from("space.kscience:version-catalog:$toolsVersion")
         }
     }

buildSrc/src/main/kotlin/space/kscience/kmath/benchmarks/JmhReport.kt

@@ -5,9 +5,6 @@
 
 package space.kscience.kmath.benchmarks
 
-import kotlinx.serialization.Serializable
-
-@Serializable
 data class JmhReport(
     val jmhVersion: String,
     val benchmark: String,
@@ -37,7 +34,6 @@ data class JmhReport(
         val scoreUnit: String
     }
 
-    @Serializable
     data class PrimaryMetric(
         override val score: Double,
         override val scoreError: Double,
@@ -48,7 +44,6 @@ data class JmhReport(
         val rawData: List<List<Double>>? = null,
     ) : Metric
 
-    @Serializable
     data class SecondaryMetric(
         override val score: Double,
         override val scoreError: Double,

buildSrc/src/main/kotlin/space/kscience/kmath/benchmarks/addBenchmarkProperties.kt

@@ -6,8 +6,8 @@
 package space.kscience.kmath.benchmarks
 
 import kotlinx.benchmark.gradle.BenchmarksExtension
-import kotlinx.serialization.decodeFromString
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
-import kotlinx.serialization.json.Json
+import com.fasterxml.jackson.module.kotlin.readValue
 import org.gradle.api.Project
 import space.kscience.gradle.KScienceReadmeExtension
 import java.time.LocalDateTime
@@ -45,6 +45,8 @@ private val ISO_DATE_TIME: DateTimeFormatter = DateTimeFormatterBuilder().run {
 
 private fun noun(number: Number, singular: String, plural: String) = if (number.toLong() == 1L) singular else plural
 
+private val jsonMapper = jacksonObjectMapper()
+
 fun Project.addBenchmarkProperties() {
     val benchmarksProject = this
     rootProject.subprojects.forEach { p ->
@@ -60,8 +62,7 @@ fun Project.addBenchmarkProperties() {
                     if (resDirectory == null || !(resDirectory.resolve("jvm.json")).exists()) {
                         "> **Can't find appropriate benchmark data. Try generating readme files after running benchmarks**."
                     } else {
-                        val reports =
+                        val reports: List<JmhReport> = jsonMapper.readValue<List<JmhReport>>(resDirectory.resolve("jvm.json"))
-                            Json.decodeFromString<List<JmhReport>>(resDirectory.resolve("jvm.json").readText())
 
                         buildString {
                             appendLine("<details>")

docs/templates/ARTIFACT-TEMPLATE.md

@@ -3,10 +3,12 @@
 The Maven coordinates of this project are `${group}:${name}:${version}`.
 
 **Gradle:**
-```gradle
+```groovy
 repositories {
     maven { url 'https://repo.kotlin.link' }
     mavenCentral()
+    // development and snapshot versions
+    maven { url 'https://maven.pkg.jetbrains.space/spc/p/sci/dev' }
 }
 
 dependencies {
@@ -18,6 +20,8 @@ dependencies {
 repositories {
     maven("https://repo.kotlin.link")
     mavenCentral()
+    // development and snapshot versions
+    maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev")
 }
 
 dependencies {

examples/build.gradle.kts

@@ -1,3 +1,5 @@
+import org.jetbrains.kotlin.gradle.tasks.KotlinJvmCompile
+
 plugins {
     kotlin("jvm")
 }
@@ -18,7 +20,6 @@ dependencies {
     implementation(project(":kmath-commons"))
     implementation(project(":kmath-complex"))
     implementation(project(":kmath-functions"))
-    implementation(project(":kmath-polynomial"))
     implementation(project(":kmath-optimization"))
     implementation(project(":kmath-stat"))
     implementation(project(":kmath-viktor"))
@@ -34,6 +35,8 @@ dependencies {
     implementation(project(":kmath-multik"))
     implementation("org.jetbrains.kotlinx:multik-default:$multikVersion")
 
+    //datetime
+    implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.4.0")
 
     implementation("org.nd4j:nd4j-native:1.0.0-beta7")
 
@@ -47,25 +50,24 @@ dependencies {
 //    } else
     implementation("org.nd4j:nd4j-native-platform:1.0.0-beta7")
 
-    // multik implementation
-    implementation("org.jetbrains.kotlinx:multik-default:0.1.0")
-
     implementation("org.slf4j:slf4j-simple:1.7.32")
     // plotting
     implementation("space.kscience:plotlykt-server:0.5.0")
 }
 
-kotlin.sourceSets.all {
+kotlin {
+    jvmToolchain(11)
+    sourceSets.all {
         with(languageSettings) {
             optIn("kotlin.contracts.ExperimentalContracts")
             optIn("kotlin.ExperimentalUnsignedTypes")
             optIn("space.kscience.kmath.misc.UnstableKMathAPI")
         }
     }
+}
 
-tasks.withType<org.jetbrains.kotlin.gradle.dsl.KotlinJvmCompile> {
+tasks.withType<KotlinJvmCompile> {
     kotlinOptions {
-        jvmTarget = "11"
         freeCompilerArgs = freeCompilerArgs + "-Xjvm-default=all" + "-Xopt-in=kotlin.RequiresOptIn" + "-Xlambdas=indy"
     }
 }

examples/notebooks/Naive classifier.ipynb

@@ -0,0 +1,418 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "source": [
+    "%use kmath(0.3.1-dev-5)\n",
+    "%use plotly(0.5.0)\n",
+    "@file:DependsOn(\"space.kscience:kmath-commons:0.3.1-dev-5\")"
+   ],
+   "execution_count": null,
+   "outputs": [],
+   "metadata": {
+    "datalore": {
+     "node_id": "lQbSB87rNAn9lV6poArVWW",
+     "type": "CODE",
+     "hide_input_from_viewers": false,
+     "hide_output_from_viewers": false
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "//Uncomment to work in Jupyter classic or DataLore\n",
+    "//Plotly.jupyter.notebook()"
+   ],
+   "execution_count": null,
+   "outputs": [],
+   "metadata": {
+    "datalore": {
+     "node_id": "0UP158hfccGgjQtHz0wAi6",
+     "type": "CODE",
+     "hide_input_from_viewers": false,
+     "hide_output_from_viewers": false
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "# The model\n",
+    "\n",
+    "Defining the input data format, the statistic abstraction and the statistic implementation based on a weighted sum of elements."
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "class XYValues(val xValues: DoubleArray, val yValues: DoubleArray) {\n",
+    "    init {\n",
+    "        require(xValues.size == yValues.size)\n",
+    "    }\n",
+    "}\n",
+    "\n",
+    "fun interface XYStatistic {\n",
+    "    operator fun invoke(values: XYValues): Double\n",
+    "}\n",
+    "\n",
+    "class ConvolutionalXYStatistic(val weights: DoubleArray) : XYStatistic {\n",
+    "    override fun invoke(values: XYValues): Double {\n",
+    "        require(weights.size == values.yValues.size)\n",
+    "        val norm = values.yValues.sum()\n",
+    "        return values.yValues.zip(weights) { value, weight -> value * weight }.sum()/norm\n",
+    "    }\n",
+    "}"
+   ],
+   "execution_count": null,
+   "outputs": [],
+   "metadata": {
+    "datalore": {
+     "node_id": "Zhgz1Ui91PWz0meJiQpHol",
+     "type": "CODE",
+     "hide_input_from_viewers": false,
+     "hide_output_from_viewers": false
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "# Generator\n",
+    "Generate sample data for parabolas and hyperbolas"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "fun generateParabolas(xValues: DoubleArray, a: Double, b: Double, c: Double): XYValues {\n",
+    "    val yValues = xValues.map { x -> a * x * x + b * x + c }.toDoubleArray()\n",
+    "    return XYValues(xValues, yValues)\n",
+    "}\n",
+    "\n",
+    "fun generateHyperbols(xValues: DoubleArray, gamma: Double, x0: Double, y0: Double): XYValues {\n",
+    "    val yValues = xValues.map { x -> y0 + gamma / (x - x0) }.toDoubleArray()\n",
+    "    return XYValues(xValues, yValues)\n",
+    "}"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "val xValues = (1.0..10.0).step(1.0).toDoubleArray()\n",
+    "\n",
+    "val xy = generateHyperbols(xValues, 1.0, 0.0, 0.0)\n",
+    "\n",
+    "Plotly.plot {\n",
+    "    scatter {\n",
+    "        this.x.doubles = xValues\n",
+    "        this.y.doubles = xy.yValues\n",
+    "    }\n",
+    "}"
+   ],
+   "execution_count": null,
+   "outputs": [],
+   "metadata": {
+    "datalore": {
+     "node_id": "ZE2atNvFzQsCvpAF8KK4ch",
+     "type": "CODE",
+     "hide_input_from_viewers": false,
+     "hide_output_from_viewers": false
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "Create a default statistic with uniform weights"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "val statistic = ConvolutionalXYStatistic(DoubleArray(xValues.size){1.0})\n",
+    "statistic(xy)"
+   ],
+   "execution_count": null,
+   "outputs": [],
+   "metadata": {
+    "datalore": {
+     "node_id": "EA5HaydTddRKYrtAUwd29h",
+     "type": "CODE",
+     "hide_input_from_viewers": false,
+     "hide_output_from_viewers": false
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "import kotlin.random.Random\n",
+    "\n",
+    "val random = Random(1288)\n",
+    "\n",
+    "val parabolas = buildList{\n",
+    "    repeat(500){\n",
+    "        add(\n",
+    "            generateParabolas(\n",
+    "                xValues, \n",
+    "                random.nextDouble(), \n",
+    "                random.nextDouble(), \n",
+    "                random.nextDouble()\n",
+    "            )\n",
+    "        )\n",
+    "    }\n",
+    "}\n",
+    "\n",
+    "val hyperbolas: List<XYValues> =  buildList{\n",
+    "    repeat(500){\n",
+    "        add(\n",
+    "            generateHyperbols(\n",
+    "                xValues, \n",
+    "                random.nextDouble()*10, \n",
+    "                random.nextDouble(), \n",
+    "                random.nextDouble()\n",
+    "            )\n",
+    "        )\n",
+    "    }\n",
+    "}"
+   ],
+   "execution_count": null,
+   "outputs": [],
+   "metadata": {
+    "datalore": {
+     "node_id": "t5t6IYmD7Q1ykeo9uijFfQ",
+     "type": "CODE",
+     "hide_input_from_viewers": false,
+     "hide_output_from_viewers": false
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "Plotly.plot { \n",
+    "    scatter { \n",
+    "        x.doubles = xValues\n",
+    "        y.doubles = parabolas[257].yValues\n",
+    "    }\n",
+    "    scatter { \n",
+    "        x.doubles = xValues\n",
+    "        y.doubles = hyperbolas[252].yValues\n",
+    "    }\n",
+    " }"
+   ],
+   "execution_count": null,
+   "outputs": [],
+   "metadata": {
+    "datalore": {
+     "node_id": "oXB8lmju7YVYjMRXITKnhO",
+     "type": "CODE",
+     "hide_input_from_viewers": false,
+     "hide_output_from_viewers": false
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "Plotly.plot { \n",
+    "    histogram { \n",
+    "        name = \"parabolae\"\n",
+    "        x.numbers = parabolas.map { statistic(it) }\n",
+    "    }\n",
+    "    histogram { \n",
+    "        name = \"hyperbolae\"\n",
+    "        x.numbers = hyperbolas.map { statistic(it) }\n",
+    "    }\n",
+    "}"
+   ],
+   "execution_count": null,
+   "outputs": [],
+   "metadata": {
+    "datalore": {
+     "node_id": "8EIIecUZrt2NNrOkhxG5P0",
+     "type": "CODE",
+     "hide_input_from_viewers": false,
+     "hide_output_from_viewers": false
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "val lossFunction: (XYStatistic) -> Double = { statistic ->\n",
+    "    - abs(parabolas.sumOf { statistic(it) } - hyperbolas.sumOf { statistic(it) })\n",
+    "}"
+   ],
+   "execution_count": null,
+   "outputs": [],
+   "metadata": {
+    "datalore": {
+     "node_id": "h7UmglJW5zXkAfKHK40oIL",
+     "type": "CODE",
+     "hide_input_from_viewers": false,
+     "hide_output_from_viewers": false
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "Using commons-math optimizer to optimize weights"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "import org.apache.commons.math3.optim.*\n",
+    "import org.apache.commons.math3.optim.nonlinear.scalar.*\n",
+    "import org.apache.commons.math3.optim.nonlinear.scalar.noderiv.*\n",
+    "\n",
+    "val optimizer = SimplexOptimizer(1e-1, Double.MAX_VALUE)\n",
+    "\n",
+    "val result = optimizer.optimize(\n",
+    "    ObjectiveFunction { point ->\n",
+    "        lossFunction(ConvolutionalXYStatistic(point))\n",
+    "    },\n",
+    "    NelderMeadSimplex(xValues.size),\n",
+    "    InitialGuess(DoubleArray(xValues.size){ 1.0 }),\n",
+    "    GoalType.MINIMIZE,\n",
+    "    MaxEval(100000)\n",
+    ")"
+   ],
+   "execution_count": null,
+   "outputs": [],
+   "metadata": {
+    "datalore": {
+     "node_id": "0EG3K4aCUciMlgGQKPvJ57",
+     "type": "CODE",
+     "hide_input_from_viewers": false,
+     "hide_output_from_viewers": false
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "Print resulting weights of optimization"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "result.point"
+   ],
+   "execution_count": null,
+   "outputs": [],
+   "metadata": {
+    "datalore": {
+     "node_id": "LelUlY0ZSlJEO9yC6SLk5B",
+     "type": "CODE",
+     "hide_input_from_viewers": false,
+     "hide_output_from_viewers": false
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "Plotly.plot { \n",
+    "    scatter { \n",
+    "        y.doubles = result.point\n",
+    "     }\n",
+    "}"
+   ],
+   "execution_count": null,
+   "outputs": [],
+   "metadata": {
+    "datalore": {
+     "node_id": "AuFOq5t9KpOIkGrOLsVXNf",
+     "type": "CODE",
+     "hide_input_from_viewers": false,
+     "hide_output_from_viewers": false
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "# The resulting statistic distribution"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "val resultStatistic = ConvolutionalXYStatistic(result.point)\n",
+    "Plotly.plot { \n",
+    "    histogram { \n",
+    "        name = \"parabolae\"\n",
+    "        x.numbers = parabolas.map { resultStatistic(it) }\n",
+    "    }\n",
+    "    histogram { \n",
+    "        name = \"hyperbolae\"\n",
+    "        x.numbers = hyperbolas.map { resultStatistic(it) }\n",
+    "    }\n",
+    "}"
+   ],
+   "execution_count": null,
+   "outputs": [],
+   "metadata": {
+    "datalore": {
+     "node_id": "zvmq42DRdM5mZ3SpzviHwI",
+     "type": "CODE",
+     "hide_input_from_viewers": false,
+     "hide_output_from_viewers": false
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [],
+   "metadata": {
+    "collapsed": false
+   }
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Kotlin",
+   "language": "kotlin",
+   "name": "kotlin"
+  },
+  "datalore": {
+   "version": 1,
+   "computation_mode": "JUPYTER",
+   "package_manager": "pip",
+   "base_environment": "default",
+   "packages": []
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/src/main/kotlin/space/kscience/kmath/fit/chiSquared.kt

@@ -7,10 +7,9 @@ package space.kscience.kmath.fit
 
 import kotlinx.html.br
 import kotlinx.html.h3
-import space.kscience.kmath.commons.expressions.DSProcessor
 import space.kscience.kmath.commons.optimization.CMOptimizer
 import space.kscience.kmath.distributions.NormalDistribution
-import space.kscience.kmath.expressions.chiSquaredExpression
+import space.kscience.kmath.expressions.autodiff
 import space.kscience.kmath.expressions.symbol
 import space.kscience.kmath.operations.asIterable
 import space.kscience.kmath.operations.toList
@@ -18,10 +17,11 @@ import space.kscience.kmath.optimization.FunctionOptimizationTarget
 import space.kscience.kmath.optimization.optimizeWith
 import space.kscience.kmath.optimization.resultPoint
 import space.kscience.kmath.optimization.resultValue
+import space.kscience.kmath.random.RandomGenerator
 import space.kscience.kmath.real.DoubleVector
 import space.kscience.kmath.real.map
 import space.kscience.kmath.real.step
-import space.kscience.kmath.stat.RandomGenerator
+import space.kscience.kmath.stat.chiSquaredExpression
 import space.kscience.plotly.*
 import space.kscience.plotly.models.ScatterMode
 import space.kscience.plotly.models.TraceValues
@@ -67,7 +67,7 @@ suspend fun main() {
     val yErr = y.map { sqrt(it) }//RealVector.same(x.size, sigma)
 
     // compute differentiable chi^2 sum for given model ax^2 + bx + c
-    val chi2 = DSProcessor.chiSquaredExpression(x, y, yErr) { arg ->
+    val chi2 = Double.autodiff.chiSquaredExpression(x, y, yErr) { arg ->
         //bind variables to autodiff context
         val a = bindSymbol(a)
         val b = bindSymbol(b)

examples/src/main/kotlin/space/kscience/kmath/fit/qowFit.kt

@@ -7,21 +7,18 @@ package space.kscience.kmath.fit
 
 import kotlinx.html.br
 import kotlinx.html.h3
-import space.kscience.kmath.commons.expressions.DSProcessor
 import space.kscience.kmath.data.XYErrorColumnarData
 import space.kscience.kmath.distributions.NormalDistribution
 import space.kscience.kmath.expressions.Symbol
+import space.kscience.kmath.expressions.autodiff
 import space.kscience.kmath.expressions.binding
 import space.kscience.kmath.expressions.symbol
 import space.kscience.kmath.operations.asIterable
 import space.kscience.kmath.operations.toList
-import space.kscience.kmath.optimization.QowOptimizer
+import space.kscience.kmath.optimization.*
-import space.kscience.kmath.optimization.chiSquaredOrNull
+import space.kscience.kmath.random.RandomGenerator
-import space.kscience.kmath.optimization.fitWith
-import space.kscience.kmath.optimization.resultPoint
 import space.kscience.kmath.real.map
 import space.kscience.kmath.real.step
-import space.kscience.kmath.stat.RandomGenerator
 import space.kscience.plotly.*
 import space.kscience.plotly.models.ScatterMode
 import kotlin.math.abs
@@ -32,6 +29,8 @@ import kotlin.math.sqrt
 private val a by symbol
 private val b by symbol
 private val c by symbol
+private val d by symbol
+private val e by symbol
 
 
 /**
@@ -63,17 +62,23 @@ suspend fun main() {
 
     val result = XYErrorColumnarData.of(x, y, yErr).fitWith(
         QowOptimizer,
-        DSProcessor,
+        Double.autodiff,
-        mapOf(a to 0.9, b to 1.2, c to 2.0)
+        mapOf(a to 0.9, b to 1.2, c to 2.0, e to 1.0, d to 1.0, e to 0.0),
+        OptimizationParameters(a, b, c, d)
     ) { arg ->
         //bind variables to autodiff context
         val a by binding
         val b by binding
         //Include default value for c if it is not provided as a parameter
         val c = bindSymbolOrNull(c) ?: one
-        a * arg.pow(2) + b * arg + c
+        val d by binding
+        val e by binding
+
+        a * arg.pow(2) + b * arg + c + d * arg.pow(3) + e / arg
     }
 
+    println("Resulting chi2/dof: ${result.chiSquaredOrNull}/${result.dof}")
+
     //display a page with plot and numerical results
     val page = Plotly.page {
         plot {
@@ -89,7 +94,7 @@ suspend fun main() {
             scatter {
                 mode = ScatterMode.lines
                 x(x)
-                y(x.map { result.model(result.resultPoint + (Symbol.x to it)) })
+                y(x.map { result.model(result.startPoint + result.resultPoint + (Symbol.x to it)) })
                 name = "fit"
             }
         }
@@ -98,7 +103,7 @@ suspend fun main() {
             +"Fit result: ${result.resultPoint}"
         }
         h3 {
-            +"Chi2/dof = ${result.chiSquaredOrNull!! / (x.size - 3)}"
+            +"Chi2/dof = ${result.chiSquaredOrNull!! / result.dof}"
         }
     }
 

examples/src/main/kotlin/space/kscience/kmath/functions/integrate.kt

@@ -5,6 +5,11 @@
 
 package space.kscience.kmath.functions
 
+import space.kscience.kmath.complex.Complex
+import space.kscience.kmath.complex.ComplexField
+import space.kscience.kmath.complex.ComplexField.div
+import space.kscience.kmath.complex.ComplexField.minus
+import space.kscience.kmath.complex.algebra
 import space.kscience.kmath.integration.gaussIntegrator
 import space.kscience.kmath.integration.integrate
 import space.kscience.kmath.integration.value
@@ -20,4 +25,12 @@ fun main() {
 
     //the value is nullable because in some cases the integration could not succeed
     println(result.value)
+
+
+    repeat(100000) {
+        Complex.algebra.gaussIntegrator.integrate(0.0..1.0, intervals = 1000) { x: Double ->
+//            sin(1 / x) + i * cos(1 / x)
+            1 / x - ComplexField.i / x
+        }.value
+    }
 }

examples/src/main/kotlin/space/kscience/kmath/functions/matrixIntegration.kt

@@ -12,10 +12,9 @@ import space.kscience.kmath.nd.StructureND
 import space.kscience.kmath.nd.structureND
 import space.kscience.kmath.nd.withNdAlgebra
 import space.kscience.kmath.operations.algebra
-import space.kscience.kmath.operations.invoke
+import kotlin.math.pow
 
-fun main(): Unit = Double.algebra {
+fun main(): Unit = Double.algebra.withNdAlgebra(2, 2) {
-    withNdAlgebra(2, 2) {
 
     //Produce a diagonal StructureND
     fun diagonal(v: Double) = structureND { (i, j) ->
@@ -31,4 +30,3 @@ fun main(): Unit = Double.algebra {
     //the value is nullable because in some cases the integration could not succeed
     println(result.value)
 }
-}

examples/src/main/kotlin/space/kscience/kmath/functions/polynomials.kt

@@ -1,399 +0,0 @@
-/*
- * Copyright 2018-2022 KMath contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-@file:Suppress("LocalVariableName")
-
-package space.kscience.kmath.functions
-
-import space.kscience.kmath.expressions.Symbol
-import space.kscience.kmath.expressions.symbol
-import space.kscience.kmath.operations.algebra
-import space.kscience.kmath.operations.invoke
-
-
-/**
- * Shows [ListPolynomial]s' and [ListRationalFunction]s' capabilities.
- */
-fun listPolynomialsExample() {
-    // [ListPolynomial] is a representation of a univariate polynomial as a list of coefficients from the least term to
-    // the greatest term. For example,
-    val polynomial1: ListPolynomial<Int> = ListPolynomial(listOf(2, -3, 1))
-    // represents polynomial 2 + (-3) x + x^2
-
-    // There are also shortcut fabrics:
-    val polynomial2: ListPolynomial<Int> = ListPolynomial(2, -3, 1)
-    println(polynomial1 == polynomial2) // true
-    // and even
-    val polynomial3: ListPolynomial<Int> = 57.asListPolynomial()
-    val polynomial4: ListPolynomial<Int> = ListPolynomial(listOf(57))
-    println(polynomial3 == polynomial4) // true
-
-    val polynomial5: ListPolynomial<Int> = ListPolynomial(3, -1)
-    // For every ring there can be provided a polynomial ring:
-    Int.algebra.listPolynomialSpace {
-        println(-polynomial5 == ListPolynomial(-3, 1)) // true
-        println(polynomial1 + polynomial5 == ListPolynomial(5, -4, 1)) // true
-        println(polynomial1 - polynomial5 == ListPolynomial(-1, -2, 1)) // true
-        println(polynomial1 * polynomial5 == ListPolynomial(6, -11, 6, -1)) // true
-    }
-    // You can even write
-    val x: ListPolynomial<Double> = ListPolynomial(0.0, 1.0)
-    val polynomial6: ListPolynomial<Double> = ListPolynomial(2.0, -3.0, 1.0)
-    Double.algebra.listPolynomialSpace {
-        println(2 - 3 * x + x * x == polynomial6)
-        println(2.0 - 3.0 * x + x * x == polynomial6)
-    }
-
-    // Also there are some utilities for polynomials:
-    println(polynomial1.substitute(Int.algebra, 1) == 0) // true, because 2 + (-3) * 1 + 1^2 = 0
-    println(polynomial1.substitute(Int.algebra, polynomial5) == polynomial1) // true, because 2 + (-3) * (3-x) + (3-x)^2 = 2 - 3x + x^2
-    println(polynomial1.derivative(Int.algebra) == ListPolynomial(-3, 2)) // true, (2 - 3x + x^2)' = -3 + 2x
-    println(polynomial1.nthDerivative(Int.algebra, 2) == 2.asListPolynomial()) // true, (2 - 3x + x^2)'' = 2
-
-    // Lastly, there are rational functions and some other utilities:
-    Double.algebra.listRationalFunctionSpace {
-        val rationalFunction1: ListRationalFunction<Double> = ListRationalFunction(listOf(2.0, -3.0, 1.0), listOf(3.0, -1.0))
-        // It's just (2 - 3x + x^2)/(3 - x)
-
-        val rationalFunction2 : ListRationalFunction<Double> = ListRationalFunction(listOf(5.0, -4.0, 1.0), listOf(3.0, -1.0))
-        // It's just (5 - 4x + x^2)/(3 - x)
-
-        println(rationalFunction1 + 1 == rationalFunction2)
-    }
-}
-
-/**
- * Shows [NumberedPolynomial]s' and [NumberedRationalFunction]s' capabilities.
- */
-fun numberedPolynomialsExample() {
-    // Consider polynomial
-    // 3 + 5 x_2 - 7 x_1^2 x_3
-    // Consider, for example, its term -7 x_1^2 x_3. -7 is a coefficient of the term, whereas (2, 0, 1, 0, 0, ...) is
-    // description of degrees of variables x_1, x_2, ... in the term. Such description with removed leading zeros
-    // [2, 0, 1] is called "signature" of the term -7 x_1^2 x_3.
-
-    val polynomial1: NumberedPolynomial<Int>
-    with(Int.algebra) {
-        // [NumberedPolynomial] is a representation of a multivariate polynomial, that stores terms in a map with terms'
-        // signatures as the map's keys and terms' coefficients as corresponding values. For example,
-        polynomial1 = NumberedPolynomial(
-            mapOf(
-                listOf<UInt>() to 3,
-                listOf(0u, 1u) to 5,
-                listOf(2u, 0u, 1u) to -7,
-            )
-        )
-        // represents polynomial 3 + 5 x_2 - 7 x_1^2 x_3
-
-        // This `NumberedPolynomial` function needs context of either ring of constant (as `Int.algebra` in this example)
-        // or space of NumberedPolynomials over it. To understand why it is like this see documentations of functions
-        // NumberedPolynomial and NumberedPolynomialWithoutCheck
-
-        // There are also shortcut fabrics:
-        val polynomial2: NumberedPolynomial<Int> = NumberedPolynomial(
-            listOf<UInt>() to 3,
-            listOf(0u, 1u) to 5,
-            listOf(2u, 0u, 1u) to -7,
-        )
-        println(polynomial1 == polynomial2) // true
-        // and even
-        val polynomial3: NumberedPolynomial<Int> = 57.asNumberedPolynomial() // This one actually does not algebraic context!
-        val polynomial4: NumberedPolynomial<Int> = NumberedPolynomial(listOf<UInt>() to 57)
-        println(polynomial3 == polynomial4) // true
-
-        numberedPolynomialSpace {
-            // Also there is DSL for constructing NumberedPolynomials:
-            val polynomial5: NumberedPolynomial<Int> = NumberedPolynomialDSL1 {
-                3 {}
-                5 { 1 inPowerOf 1u }
-                -7 with { 0 pow 2u; 2 pow 1u }
-                // `pow` and `inPowerOf` are the same
-                // `with` is omittable
-            }
-            println(polynomial1 == polynomial5) // true
-
-            // Unfortunately the DSL does not work good in bare context of constants' ring, so for now it's disabled and
-            // works only in NumberedPolynomialSpace and NumberedRationalFunctionSpace
-        }
-    }
-
-    val polynomial6: NumberedPolynomial<Int> = Int.algebra {
-        NumberedPolynomial(
-            listOf<UInt>() to 7,
-            listOf(0u, 1u) to -5,
-            listOf(2u, 0u, 1u) to 0,
-            listOf(0u, 0u, 0u, 4u) to 4,
-        )
-    }
-    // For every ring there can be provided a polynomial ring:
-    Int.algebra.numberedPolynomialSpace {
-        println(
-            -polynomial6 == NumberedPolynomial(
-                listOf<UInt>() to -7,
-                listOf(0u, 1u) to 5,
-                listOf(2u, 0u, 1u) to 0,
-                listOf(0u, 0u, 0u, 4u) to (-4),
-            )
-        ) // true
-        println(
-            polynomial1 + polynomial6 == NumberedPolynomial(
-                listOf<UInt>() to 10,
-                listOf(0u, 1u) to 0,
-                listOf(2u, 0u, 1u) to -7,
-                listOf(0u, 0u, 0u, 4u) to 4,
-            )
-        ) // true
-        println(
-            polynomial1 - polynomial6 == NumberedPolynomial(
-                listOf<UInt>() to -4,
-                listOf(0u, 1u) to 10,
-                listOf(2u, 0u, 1u) to -7,
-                listOf(0u, 0u, 0u, 4u) to -4,
-            )
-        ) // true
-
-        polynomial1 * polynomial6 // Multiplication works too
-    }
-
-    Double.algebra.numberedPolynomialSpace {
-        // You can even write
-        val x_1: NumberedPolynomial<Double> = NumberedPolynomial(listOf(1u) to 1.0)
-        val x_2: NumberedPolynomial<Double> = NumberedPolynomial(listOf(0u, 1u) to 1.0)
-        val x_3: NumberedPolynomial<Double> = NumberedPolynomial(listOf(0u, 0u, 1u) to 1.0)
-        val polynomial7: NumberedPolynomial<Double> = NumberedPolynomial(
-            listOf<UInt>() to 3.0,
-            listOf(0u, 1u) to 5.0,
-            listOf(2u, 0u, 1u) to -7.0,
-        )
-        Double.algebra.listPolynomialSpace {
-            println(3 + 5 * x_2 - 7 * x_1 * x_1 * x_3 == polynomial7)
-            println(3.0 + 5.0 * x_2 - 7.0 * x_1 * x_1 * x_3 == polynomial7)
-        }
-    }
-
-    Int.algebra.numberedPolynomialSpace {
-        val x_4: NumberedPolynomial<Int> = NumberedPolynomial(listOf(0u, 0u, 0u, 4u) to 1)
-        // Also there are some utilities for polynomials:
-        println(polynomial1.substitute(mapOf(0 to 1, 1 to -2, 2 to -1)) == 0.asNumberedPolynomial()) // true,
-            // because it's substitution x_1 -> 1, x_2 -> -2, x_3 -> -1,
-            // so 3 + 5 x_2 - 7 x_1^2 x_3 = 3 + 5 * (-2) - 7 * 1^2 * (-1) = 3 - 10 + 7 = 0
-        println(
-            polynomial1.substitute(mapOf(1 to x_4)) == NumberedPolynomial(
-                listOf<UInt>() to 3,
-                listOf(0u, 1u) to 5,
-                listOf(2u, 0u, 1u) to -7,
-            )
-        ) // true, because it's substitution x_2 -> x_4, so result is 3 + 5 x_4 - 7 x_1^2 x_3
-        println(
-            polynomial1.derivativeWithRespectTo(Int.algebra, 1) ==
-                    NumberedPolynomial(listOf<UInt>() to 5)
-        ) // true, d/dx_2 (3 + 5 x_2 - 7 x_1^2 x_3) = 5
-    }
-
-    // Lastly, there are rational functions and some other utilities:
-    Double.algebra.numberedRationalFunctionSpace {
-        val rationalFunction1: NumberedRationalFunction<Double> = NumberedRationalFunction(
-            NumberedPolynomial(
-                listOf<UInt>() to 2.0,
-                listOf(1u) to -3.0,
-                listOf(2u) to 1.0,
-            ),
-            NumberedPolynomial(
-                listOf<UInt>() to 3.0,
-                listOf(1u) to -1.0,
-            )
-        )
-        // It's just (2 - 3x + x^2)/(3 - x) where x = x_1
-
-        val rationalFunction2: NumberedRationalFunction<Double> = NumberedRationalFunction(
-            NumberedPolynomial(
-                listOf<UInt>() to 5.0,
-                listOf(1u) to -4.0,
-                listOf(2u) to 1.0,
-            ),
-            NumberedPolynomial(
-                listOf<UInt>() to 3.0,
-                listOf(1u) to -1.0,
-            )
-        )
-        // It's just (5 - 4x + x^2)/(3 - x) where x = x_1
-
-        println(rationalFunction1 + 1 == rationalFunction2)
-    }
-}
-
-/**
- * Shows [LabeledPolynomial]s' and [LabeledRationalFunction]s' capabilities.
- */
-fun labeledPolynomialsExample() {
-    val x by symbol
-    val y by symbol
-    val z by symbol
-    val t by symbol
-
-    // Consider polynomial
-    // 3 + 5 y - 7 x^2 z
-    // Consider, for example, its term -7 x^2 z. -7 is a coefficient of the term, whereas matching (x -> 2, z -> 3) is
-    // description of degrees of variables x_1, x_2, ... in the term. Such description is called "signature" of the
-    // term -7 x_1^2 x_3.
-
-    val polynomial1: LabeledPolynomial<Int>
-    with(Int.algebra) {
-        // [LabeledPolynomial] is a representation of a multivariate polynomial, that stores terms in a map with terms'
-        // signatures as the map's keys and terms' coefficients as corresponding values. For example,
-        polynomial1 = LabeledPolynomial(
-            mapOf(
-                mapOf<Symbol, UInt>() to 3,
-                mapOf(y to 1u) to 5,
-                mapOf(x to 2u, z to 1u) to -7,
-            )
-        )
-        // represents polynomial 3 + 5 y - 7 x^2 z
-
-        // This `LabeledPolynomial` function needs context of either ring of constant (as `Int.algebra` in this example)
-        // or space of LabeledPolynomials over it. To understand why it is like this see documentations of functions
-        // LabeledPolynomial and LabeledPolynomialWithoutCheck
-
-        // There are also shortcut fabrics:
-        val polynomial2: LabeledPolynomial<Int> = LabeledPolynomial(
-            mapOf<Symbol, UInt>() to 3,
-            mapOf(y to 1u) to 5,
-            mapOf(x to 2u, z to 1u) to -7,
-        )
-        println(polynomial1 == polynomial2) // true
-        // and even
-        val polynomial3: LabeledPolynomial<Int> = 57.asLabeledPolynomial() // This one actually does not algebraic context!
-        val polynomial4: LabeledPolynomial<Int> = LabeledPolynomial(mapOf<Symbol, UInt>() to 57)
-        println(polynomial3 == polynomial4) // true
-
-        labeledPolynomialSpace {
-            // Also there is DSL for constructing NumberedPolynomials:
-            val polynomial5: LabeledPolynomial<Int> = LabeledPolynomialDSL1 {
-                3 {}
-                5 { y inPowerOf 1u }
-                -7 with { x pow 2u; z pow 1u }
-                // `pow` and `inPowerOf` are the same
-                // `with` is omittable
-            }
-            println(polynomial1 == polynomial5) // true
-
-            // Unfortunately the DSL does not work good in bare context of constants' ring, so for now it's disabled and
-            // works only in NumberedPolynomialSpace and NumberedRationalFunctionSpace
-        }
-    }
-
-    val polynomial6: LabeledPolynomial<Int> = Int.algebra {
-        LabeledPolynomial(
-            mapOf<Symbol, UInt>() to 7,
-            mapOf(y to 1u) to -5,
-            mapOf(x to 2u, z to 1u) to 0,
-            mapOf(t to 4u) to 4,
-        )
-    }
-    // For every ring there can be provided a polynomial ring:
-    Int.algebra.labeledPolynomialSpace {
-        println(
-            -polynomial6 == LabeledPolynomial(
-                mapOf<Symbol, UInt>() to -7,
-                mapOf(y to 1u) to 5,
-                mapOf(x to 2u, z to 1u) to 0,
-                mapOf(t to 4u) to -4,
-            )
-        ) // true
-        println(
-            polynomial1 + polynomial6 == LabeledPolynomial(
-                mapOf<Symbol, UInt>() to 10,
-                mapOf(y to 1u) to 0,
-                mapOf(x to 2u, z to 1u) to -7,
-                mapOf(t to 4u) to 4,
-            )
-        ) // true
-        println(
-            polynomial1 - polynomial6 == LabeledPolynomial(
-                mapOf<Symbol, UInt>() to -4,
-                mapOf(y to 1u) to 10,
-                mapOf(x to 2u, z to 1u) to -7,
-                mapOf(t to 4u) to -4,
-            )
-        ) // true
-
-        polynomial1 * polynomial6 // Multiplication works too
-    }
-
-    Double.algebra.labeledPolynomialSpace {
-        // You can even write
-        val polynomial7: LabeledPolynomial<Double> = LabeledPolynomial(
-            mapOf<Symbol, UInt>() to 3.0,
-            mapOf(y to 1u) to 5.0,
-            mapOf(x to 2u, z to 1u) to -7.0,
-        )
-        Double.algebra.listPolynomialSpace {
-            println(3 + 5 * y - 7 * x * x * z == polynomial7)
-            println(3.0 + 5.0 * y - 7.0 * x * x * z == polynomial7)
-        }
-    }
-
-    Int.algebra.labeledPolynomialSpace {
-        // Also there are some utilities for polynomials:
-        println(polynomial1.substitute(mapOf(x to 1, y to -2, z to -1)) == 0.asLabeledPolynomial()) // true,
-        // because it's substitution x -> 1, y -> -2, z -> -1,
-        // so 3 + 5 y - 7 x^2 z = 3 + 5 * (-2) - 7 * 1^2 * (-1) = 3 - 10 + 7 = 0
-        println(
-            polynomial1.substitute(mapOf(y to t.asPolynomial())) == LabeledPolynomial(
-                mapOf<Symbol, UInt>() to 3,
-                mapOf(t to 1u) to 5,
-                mapOf(x to 2u, z to 1u) to -7,
-                )
-        ) // true, because it's substitution y -> t, so result is 3 + 5 t - 7 x^2 z
-        println(
-            polynomial1.derivativeWithRespectTo(Int.algebra, y) == LabeledPolynomial(mapOf<Symbol, UInt>() to 5)
-        ) // true, d/dy (3 + 5 y - 7 x^2 z) = 5
-    }
-
-    // Lastly, there are rational functions and some other utilities:
-    Double.algebra.labeledRationalFunctionSpace {
-        val rationalFunction1: LabeledRationalFunction<Double> = LabeledRationalFunction(
-            LabeledPolynomial(
-                mapOf<Symbol, UInt>() to 2.0,
-                mapOf(x to 1u) to -3.0,
-                mapOf(x to 2u) to 1.0,
-            ),
-            LabeledPolynomial(
-                mapOf<Symbol, UInt>() to 3.0,
-                mapOf(x to 1u) to -1.0,
-            )
-        )
-        // It's just (2 - 3x + x^2)/(3 - x)
-
-        val rationalFunction2: LabeledRationalFunction<Double> = LabeledRationalFunction(
-            LabeledPolynomial(
-                mapOf<Symbol, UInt>() to 5.0,
-                mapOf(x to 1u) to -4.0,
-                mapOf(x to 2u) to 1.0,
-            ),
-            LabeledPolynomial(
-                mapOf<Symbol, UInt>() to 3.0,
-                mapOf(x to 1u) to -1.0,
-            )
-        )
-        // It's just (5 - 4x + x^2)/(3 - x)
-
-        println(rationalFunction1 + 1 == rationalFunction2)
-    }
-}
-
-fun main() {
-    println("ListPolynomials:")
-    listPolynomialsExample()
-    println()
-
-    println("NumberedPolynomials:")
-    numberedPolynomialsExample()
-    println()
-
-    println("ListPolynomials:")
-    labeledPolynomialsExample()
-    println()
-}

examples/src/main/kotlin/space/kscience/kmath/linear/dotPerformance.kt

@@ -7,8 +7,10 @@ package space.kscience.kmath.linear
 
 import space.kscience.kmath.operations.algebra
 import kotlin.random.Random
-import kotlin.system.measureTimeMillis
+import kotlin.time.ExperimentalTime
+import kotlin.time.measureTime
 
+@OptIn(ExperimentalTime::class)
 fun main() {
     val random = Random(12224)
     val dim = 1000
@@ -21,7 +23,7 @@ fun main() {
         if (i <= j) random.nextDouble() else 0.0
     }
 
-    val time = measureTimeMillis {
+    val time = measureTime {
         with(Double.algebra.linearSpace) {
             repeat(10) {
                 matrix1 dot matrix2

examples/src/main/kotlin/space/kscience/kmath/operations/mixedNDOperations.kt

@@ -8,14 +8,14 @@ package space.kscience.kmath.operations
 import space.kscience.kmath.commons.linear.CMLinearSpace
 import space.kscience.kmath.linear.matrix
 import space.kscience.kmath.nd.DoubleBufferND
-import space.kscience.kmath.nd.Shape
+import space.kscience.kmath.nd.ShapeND
 import space.kscience.kmath.nd.Structure2D
 import space.kscience.kmath.nd.ndAlgebra
 import space.kscience.kmath.viktor.ViktorStructureND
 import space.kscience.kmath.viktor.viktorAlgebra
 
 fun main() {
-    val viktorStructure: ViktorStructureND = DoubleField.viktorAlgebra.structureND(Shape(2, 2)) { (i, j) ->
+    val viktorStructure: ViktorStructureND = DoubleField.viktorAlgebra.structureND(ShapeND(2, 2)) { (i, j) ->
         if (i == j) 2.0 else 0.0
     }
 

examples/src/main/kotlin/space/kscience/kmath/series/analyzeDif.kt

@@ -0,0 +1,49 @@
+package space.kscience.kmath.series
+
+
+import kotlinx.html.FlowContent
+import kotlinx.html.h1
+import space.kscience.kmath.operations.DoubleBufferOps
+import space.kscience.kmath.operations.algebra
+import space.kscience.kmath.operations.bufferAlgebra
+import space.kscience.kmath.operations.toList
+import space.kscience.kmath.stat.KMComparisonResult
+import space.kscience.kmath.stat.ksComparisonStatistic
+import space.kscience.kmath.structures.Buffer
+import space.kscience.kmath.structures.slice
+import space.kscience.plotly.*
+import kotlin.math.PI
+
+fun main() = with(Double.algebra.bufferAlgebra.seriesAlgebra()) {
+    fun FlowContent.plotSeries(buffer: Buffer<Double>) {
+        val ls = buffer.labels
+        plot {
+            scatter {
+                x.numbers = ls
+                y.numbers = buffer.toList()
+            }
+            layout {
+                xaxis {
+                    range(0.0..100.0)
+                }
+            }
+        }
+    }
+
+
+    val s1 = series(100) { sin(2 * PI * it / 100) + 1.0 }
+    val s2 = s1.slice(20..50).moveTo(40)
+
+    val s3: Buffer<Double> = s1.zip(s2) { l, r -> l + r } //s1 + s2
+    val s4 = DoubleBufferOps.ln(s3)
+
+    @Suppress("UNUSED_VARIABLE") val kmTest: KMComparisonResult<Double> = ksComparisonStatistic(s1, s2)
+
+    Plotly.page {
+        h1 { +"This is my plot" }
+        plotSeries(s1)
+        plotSeries(s2)
+        plotSeries(s4)
+    }.makeFile()
+
+}

examples/src/main/kotlin/space/kscience/kmath/stat/DateTimeSeries.kt

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2018-2023 KMath contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package space.kscience.kmath.stat
+
+import kotlinx.datetime.Instant
+import space.kscience.kmath.operations.algebra
+import space.kscience.kmath.operations.bufferAlgebra
+import space.kscience.kmath.series.MonotonicSeriesAlgebra
+import space.kscience.kmath.series.SeriesAlgebra
+import kotlin.time.Duration
+
+fun SeriesAlgebra.Companion.time(zero: Instant, step: Duration) = MonotonicSeriesAlgebra(
+    bufferAlgebra = Double.algebra.bufferAlgebra,
+    offsetToLabel = { zero + step * it },
+    labelToOffset = { (it - zero) / step }
+)

examples/src/main/kotlin/space/kscience/kmath/stat/DistributionBenchmark.kt

@@ -10,6 +10,7 @@ import kotlinx.coroutines.async
 import kotlinx.coroutines.runBlocking
 import org.apache.commons.rng.sampling.distribution.BoxMullerNormalizedGaussianSampler
 import org.apache.commons.rng.simple.RandomSource
+import space.kscience.kmath.random.RandomGenerator
 import space.kscience.kmath.samplers.GaussianSampler
 import java.time.Duration
 import java.time.Instant
@@ -35,7 +36,7 @@ private suspend fun runKMathChained(): Duration {
     return Duration.between(startTime, Instant.now())
 }
 
-private fun runApacheDirect(): Duration {
+private fun runCMDirect(): Duration {
     val rng = RandomSource.create(RandomSource.MT, 123L)
 
     val sampler = CMGaussianSampler.of(
@@ -64,7 +65,7 @@ private fun runApacheDirect(): Duration {
  * Comparing chain sampling performance with direct sampling performance
  */
 fun main(): Unit = runBlocking(Dispatchers.Default) {
-    val directJob = async { runApacheDirect() }
+    val directJob = async { runCMDirect() }
     val chainJob = async { runKMathChained() }
     println("KMath Chained: ${chainJob.await()}")
     println("Apache Direct: ${directJob.await()}")

examples/src/main/kotlin/space/kscience/kmath/stat/DistributionDemo.kt

@@ -9,6 +9,7 @@ import kotlinx.coroutines.runBlocking
 import space.kscience.kmath.chains.Chain
 import space.kscience.kmath.chains.combineWithState
 import space.kscience.kmath.distributions.NormalDistribution
+import space.kscience.kmath.random.RandomGenerator
 
 private data class AveragingChainState(var num: Int = 0, var value: Double = 0.0)
 

examples/src/main/kotlin/space/kscience/kmath/structures/NDField.kt

@@ -29,7 +29,7 @@ fun main() {
     Nd4j.zeros(0)
     val dim = 1000
     val n = 1000
-    val shape = Shape(dim, dim)
+    val shape = ShapeND(dim, dim)
 
 
     // specialized nd-field for Double. It works as generic Double field as well.

examples/src/main/kotlin/space/kscience/kmath/structures/StreamDoubleFieldND.kt

@@ -17,11 +17,11 @@ import java.util.stream.IntStream
  * A demonstration implementation of NDField over Real using Java [java.util.stream.DoubleStream] for parallel
  * execution.
  */
-class StreamDoubleFieldND(override val shape: IntArray) : FieldND<Double, DoubleField>,
+class StreamDoubleFieldND(override val shape: ShapeND) : FieldND<Double, DoubleField>,
     NumbersAddOps<StructureND<Double>>,
     ExtendedField<StructureND<Double>> {
 
-    private val strides = DefaultStrides(shape)
+    private val strides = ColumnStrides(shape)
     override val elementAlgebra: DoubleField get() = DoubleField
     override val zero: BufferND<Double> by lazy { structureND(shape) { zero } }
     override val one: BufferND<Double> by lazy { structureND(shape) { one } }
@@ -31,17 +31,19 @@ class StreamDoubleFieldND(override val shape: IntArray) : FieldND<Double, Double
         return structureND(shape) { d }
     }
 
+    @OptIn(PerformancePitfall::class)
     private val StructureND<Double>.buffer: DoubleBuffer
         get() = when {
             !shape.contentEquals(this@StreamDoubleFieldND.shape) -> throw ShapeMismatchException(
                 this@StreamDoubleFieldND.shape,
                 shape
             )
-            this is BufferND && this.indices == this@StreamDoubleFieldND.strides -> this.buffer as DoubleBuffer
+
+            this is BufferND && indices == this@StreamDoubleFieldND.strides -> this.buffer as DoubleBuffer
             else -> DoubleBuffer(strides.linearSize) { offset -> get(strides.index(offset)) }
         }
 
-    override fun structureND(shape: Shape, initializer: DoubleField.(IntArray) -> Double): BufferND<Double> {
+    override fun structureND(shape: ShapeND, initializer: DoubleField.(IntArray) -> Double): BufferND<Double> {
         val array = IntStream.range(0, strides.linearSize).parallel().mapToDouble { offset ->
             val index = strides.index(offset)
             DoubleField.initializer(index)
@@ -109,4 +111,4 @@ class StreamDoubleFieldND(override val shape: IntArray) : FieldND<Double, Double
     override fun atanh(arg: StructureND<Double>): BufferND<Double> = arg.map { atanh(it) }
 }
 
-fun DoubleField.ndStreaming(vararg shape: Int): StreamDoubleFieldND = StreamDoubleFieldND(shape)
+fun DoubleField.ndStreaming(vararg shape: Int): StreamDoubleFieldND = StreamDoubleFieldND(ShapeND(shape))

examples/src/main/kotlin/space/kscience/kmath/structures/StructureReadBenchmark.kt

@@ -5,16 +5,19 @@
 
 package space.kscience.kmath.structures
 
+import space.kscience.kmath.misc.PerformancePitfall
 import space.kscience.kmath.nd.BufferND
-import space.kscience.kmath.nd.DefaultStrides
+import space.kscience.kmath.nd.ColumnStrides
+import space.kscience.kmath.nd.ShapeND
 import kotlin.system.measureTimeMillis
 
 @Suppress("ASSIGNED_BUT_NEVER_ACCESSED_VARIABLE")
+@OptIn(PerformancePitfall::class)
 fun main() {
     val n = 6000
     val array = DoubleArray(n * n) { 1.0 }
     val buffer = DoubleBuffer(array)
-    val strides = DefaultStrides(intArrayOf(n, n))
+    val strides = ColumnStrides(ShapeND(n, n))
     val structure = BufferND(strides, buffer)
 
     measureTimeMillis {

examples/src/main/kotlin/space/kscience/kmath/structures/StructureWriteBenchmark.kt

@@ -5,16 +5,23 @@
 
 package space.kscience.kmath.structures
 
+import space.kscience.kmath.nd.BufferND
+import space.kscience.kmath.nd.ShapeND
 import space.kscience.kmath.nd.StructureND
-import space.kscience.kmath.nd.mapToBuffer
+import space.kscience.kmath.operations.mapToBuffer
 import kotlin.system.measureTimeMillis
 
+private inline fun <T, reified R : Any> BufferND<T>.mapToBufferND(
+    bufferFactory: BufferFactory<R> = BufferFactory.auto(),
+    crossinline block: (T) -> R,
+): BufferND<R> = BufferND(indices, buffer.mapToBuffer(bufferFactory, block))
+
 @Suppress("UNUSED_VARIABLE")
 fun main() {
     val n = 6000
-    val structure = StructureND.buffered(intArrayOf(n, n), Buffer.Companion::auto) { 1.0 }
+    val structure = StructureND.buffered(ShapeND(n, n), Buffer.Companion::auto) { 1.0 }
-    structure.mapToBuffer { it + 1 } // warm-up
+    structure.mapToBufferND { it + 1 } // warm-up
-    val time1 = measureTimeMillis { val res = structure.mapToBuffer { it + 1 } }
+    val time1 = measureTimeMillis { val res = structure.mapToBufferND { it + 1 } }
     println("Structure mapping finished in $time1 millis")
     val array = DoubleArray(n * n) { 1.0 }
 

examples/src/main/kotlin/space/kscience/kmath/tensors/OLSWithSVD.kt

@@ -5,16 +5,17 @@
 
 package space.kscience.kmath.tensors
 
-import space.kscience.kmath.misc.PerformancePitfall
+import space.kscience.kmath.nd.ShapeND
+import space.kscience.kmath.nd.contentEquals
 import space.kscience.kmath.operations.invoke
 import space.kscience.kmath.tensors.core.DoubleTensor
 import space.kscience.kmath.tensors.core.DoubleTensorAlgebra
-
+import space.kscience.kmath.tensors.core.randomNormal
+import space.kscience.kmath.tensors.core.randomNormalLike
 import kotlin.math.abs
 
 // OLS estimator using SVD
 
-@OptIn(PerformancePitfall::class)
 fun main() {
     //seed for random
     val randSeed = 100500L
@@ -23,10 +24,10 @@ fun main() {
     DoubleTensorAlgebra {
         // take coefficient vector from normal distribution
         val alpha = randomNormal(
-            intArrayOf(5),
+            ShapeND(5),
             randSeed
         ) + fromArray(
-            intArrayOf(5),
+            ShapeND(5),
             doubleArrayOf(1.0, 2.5, 3.4, 5.0, 10.1)
         )
 
@@ -34,16 +35,16 @@ fun main() {
 
         // also take sample of size 20 from normal distribution for x
         val x = randomNormal(
-            intArrayOf(20, 5),
+            ShapeND(20, 5),
             randSeed
         )
 
         // calculate y and add gaussian noise (N(0, 0.05))
         val y = x dot alpha
-        y += y.randomNormalLike(randSeed) * 0.05
+        y += randomNormalLike(y, randSeed) * 0.05
 
         // now restore the coefficient vector with OSL estimator with SVD
-        val (u, singValues, v) = x.svd()
+        val (u, singValues, v) = svd(x)
 
         // we have to make sure the singular values of the matrix are not close to zero
         println("Singular values:\n$singValues")
@@ -53,8 +54,10 @@ fun main() {
         val sigma = diagonalEmbedding(singValues.map { if (abs(it) < 1e-3) 0.0 else 1.0 / it })
 
         val alphaOLS = v dot sigma dot u.transposed() dot y
-        println("Estimated alpha:\n" +
+        println(
-                "$alphaOLS")
+            "Estimated alpha:\n" +
+                    "$alphaOLS"
+        )
 
         // figure out MSE of approximation
         fun mse(yTrue: DoubleTensor, yPred: DoubleTensor): Double {
@@ -62,7 +65,7 @@ fun main() {
             require(yTrue.shape contentEquals yPred.shape)
 
             val diff = yTrue - yPred
-            return diff.dot(diff).sqrt().value()
+            return sqrt(diff.dot(diff)).value()
         }
 
         println("MSE: ${mse(alpha, alphaOLS)}")

examples/src/main/kotlin/space/kscience/kmath/tensors/PCA.kt

@@ -5,8 +5,8 @@
 
 package space.kscience.kmath.tensors
 
-import space.kscience.kmath.tensors.core.tensorAlgebra
+import space.kscience.kmath.nd.ShapeND
-import space.kscience.kmath.tensors.core.withBroadcast
+import space.kscience.kmath.tensors.core.*
 
 
 // simple PCA
@@ -16,12 +16,12 @@ fun main(): Unit = Double.tensorAlgebra.withBroadcast {  // work in context with
 
     // assume x is range from 0 until 10
     val x = fromArray(
-        intArrayOf(10),
+        ShapeND(10),
         DoubleArray(10) { it.toDouble() }
     )
 
     // take y dependent on x with noise
-    val y = 2.0 * x + (3.0 + x.randomNormalLike(seed) * 1.5)
+    val y = 2.0 * x + (3.0 + randomNormalLike(x, seed) * 1.5)
 
     println("x:\n$x")
     println("y:\n$y")
@@ -30,34 +30,34 @@ fun main(): Unit = Double.tensorAlgebra.withBroadcast {  // work in context with
     val dataset = stack(listOf(x, y)).transposed()
 
     // normalize both x and y
-    val xMean = x.mean()
+    val xMean = mean(x)
-    val yMean = y.mean()
+    val yMean = mean(y)
 
-    val xStd = x.std()
+    val xStd = std(x)
-    val yStd = y.std()
+    val yStd = std(y)
 
-    val xScaled = (x - xMean) / xStd
+    val xScaled: DoubleTensor = (x - xMean) / xStd
-    val yScaled = (y - yMean) / yStd
+    val yScaled: DoubleTensor = (y - yMean) / yStd
 
     // save means ans standard deviations for further recovery
     val mean = fromArray(
-        intArrayOf(2),
+        ShapeND(2),
         doubleArrayOf(xMean, yMean)
     )
     println("Means:\n$mean")
 
     val std = fromArray(
-        intArrayOf(2),
+        ShapeND(2),
         doubleArrayOf(xStd, yStd)
     )
     println("Standard deviations:\n$std")
 
     // calculate the covariance matrix of scaled x and y
-    val covMatrix = cov(listOf(xScaled, yScaled))
+    val covMatrix = covariance(listOf(xScaled.asDoubleTensor1D(), yScaled.asDoubleTensor1D()))
     println("Covariance matrix:\n$covMatrix")
 
     // and find out eigenvector of it
-    val (_, evecs) = covMatrix.symEig()
+    val (_, evecs) = symEig(covMatrix)
     val v = evecs.getTensor(0)
     println("Eigenvector:\n$v")
 
@@ -68,7 +68,7 @@ fun main(): Unit = Double.tensorAlgebra.withBroadcast {  // work in context with
     // we can restore original data from reduced data;
     // for example, find 7th element of dataset.
     val n = 7
-    val restored = (datasetReduced.getTensor(n) dot v.view(intArrayOf(1, 2))) * std + mean
+    val restored = (datasetReduced.getTensor(n) dot v.view(ShapeND(1, 2))) * std + mean
     println("Original value:\n${dataset.getTensor(n)}")
     println("Restored value:\n$restored")
 }

examples/src/main/kotlin/space/kscience/kmath/tensors/dataSetNormalization.kt

@@ -5,6 +5,8 @@
 
 package space.kscience.kmath.tensors
 
+import space.kscience.kmath.nd.ShapeND
+import space.kscience.kmath.tensors.core.randomNormal
 import space.kscience.kmath.tensors.core.tensorAlgebra
 import space.kscience.kmath.tensors.core.withBroadcast
 
@@ -13,17 +15,17 @@ import space.kscience.kmath.tensors.core.withBroadcast
 
 fun main() = Double.tensorAlgebra.withBroadcast {  // work in context with broadcast methods
     // take dataset of 5-element vectors from normal distribution
-    val dataset = randomNormal(intArrayOf(100, 5)) * 1.5 // all elements from N(0, 1.5)
+    val dataset = randomNormal(ShapeND(100, 5)) * 1.5 // all elements from N(0, 1.5)
 
     dataset += fromArray(
-        intArrayOf(5),
+        ShapeND(5),
         doubleArrayOf(0.0, 1.0, 1.5, 3.0, 5.0) // row means
     )
 
 
     // find out mean and standard deviation of each column
-    val mean = dataset.mean(0, false)
+    val mean = mean(dataset, 0, false)
-    val std = dataset.std(0, false)
+    val std = std(dataset, 0, false)
 
     println("Mean:\n$mean")
     println("Standard deviation:\n$std")
@@ -35,8 +37,8 @@ fun main() = Double.tensorAlgebra.withBroadcast {  // work in context with broad
     // now we can scale dataset with mean normalization
     val datasetScaled = (dataset - mean) / std
 
-    // find out mean and std of scaled dataset
+    // find out mean and standardDiviation of scaled dataset
 
-    println("Mean of scaled:\n${datasetScaled.mean(0, false)}")
+    println("Mean of scaled:\n${mean(datasetScaled, 0, false)}")
-    println("Mean of scaled:\n${datasetScaled.std(0, false)}")
+    println("Mean of scaled:\n${std(datasetScaled, 0, false)}")
 }

examples/src/main/kotlin/space/kscience/kmath/tensors/linearSystemSolvingWithLUP.kt

@@ -5,6 +5,7 @@
 
 package space.kscience.kmath.tensors
 
+import space.kscience.kmath.nd.ShapeND
 import space.kscience.kmath.tensors.core.DoubleTensor
 import space.kscience.kmath.tensors.core.tensorAlgebra
 import space.kscience.kmath.tensors.core.withBroadcast
@@ -15,13 +16,13 @@ fun main() = Double.tensorAlgebra.withBroadcast {// work in context with linear
 
     // set true value of x
     val trueX = fromArray(
-        intArrayOf(4),
+        ShapeND(4),
         doubleArrayOf(-2.0, 1.5, 6.8, -2.4)
     )
 
     // and A matrix
     val a = fromArray(
-        intArrayOf(4, 4),
+        ShapeND(4, 4),
         doubleArrayOf(
             0.5, 10.5, 4.5, 1.0,
             8.5, 0.9, 12.8, 0.1,
@@ -40,7 +41,7 @@ fun main() = Double.tensorAlgebra.withBroadcast {// work in context with linear
     // solve `Ax = b` system using LUP decomposition
 
     // get P, L, U such that PA = LU
-    val (p, l, u) = a.lu()
+    val (p, l, u) = lu(a)
 
     // check P is permutation matrix
     println("P:\n$p")
@@ -64,7 +65,7 @@ fun main() = Double.tensorAlgebra.withBroadcast {// work in context with linear
     // this function returns solution x of a system lx = b, l should be lower triangular
     fun solveLT(l: DoubleTensor, b: DoubleTensor): DoubleTensor {
         val n = l.shape[0]
-        val x = zeros(intArrayOf(n))
+        val x = zeros(ShapeND(n))
         for (i in 0 until n) {
             x[intArrayOf(i)] = (b[intArrayOf(i)] - l.getTensor(i).dot(x).value()) / l[intArrayOf(i, i)]
         }

examples/src/main/kotlin/space/kscience/kmath/tensors/neuralNetwork.kt

@@ -5,12 +5,11 @@
 
 package space.kscience.kmath.tensors
 
+import space.kscience.kmath.nd.ShapeND
+import space.kscience.kmath.nd.contentEquals
 import space.kscience.kmath.operations.asIterable
 import space.kscience.kmath.operations.invoke
-import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra
+import space.kscience.kmath.tensors.core.*
-import space.kscience.kmath.tensors.core.DoubleTensor
-import space.kscience.kmath.tensors.core.DoubleTensorAlgebra
-import space.kscience.kmath.tensors.core.toDoubleTensor
 import kotlin.math.sqrt
 
 const val seed = 100500L
@@ -49,7 +48,7 @@ fun reluDer(x: DoubleTensor): DoubleTensor = DoubleTensorAlgebra {
 class ReLU : Activation(::relu, ::reluDer)
 
 fun sigmoid(x: DoubleTensor): DoubleTensor = DoubleTensorAlgebra {
-    1.0 / (1.0 + (-x).exp())
+    1.0 / (1.0 + exp((-x)))
 }
 
 fun sigmoidDer(x: DoubleTensor): DoubleTensor = DoubleTensorAlgebra {
@@ -68,12 +67,12 @@ class Dense(
 
     private val weights: DoubleTensor = DoubleTensorAlgebra {
         randomNormal(
-            intArrayOf(inputUnits, outputUnits),
+            ShapeND(inputUnits, outputUnits),
             seed
         ) * sqrt(2.0 / (inputUnits + outputUnits))
     }
 
-    private val bias: DoubleTensor = DoubleTensorAlgebra { zeros(intArrayOf(outputUnits)) }
+    private val bias: DoubleTensor = DoubleTensorAlgebra { zeros(ShapeND(outputUnits)) }
 
     override fun forward(input: DoubleTensor): DoubleTensor = BroadcastDoubleTensorAlgebra {
         (input dot weights) + bias
@@ -83,7 +82,7 @@ class Dense(
         val gradInput = outputError dot weights.transposed()
 
         val gradW = input.transposed() dot outputError
-        val gradBias = outputError.mean(dim = 0, keepDim = false) * input.shape[0].toDouble()
+        val gradBias = mean(structureND = outputError, dim = 0, keepDim = false) * input.shape[0].toDouble()
 
         weights -= learningRate * gradW
         bias -= learningRate * gradBias
@@ -116,7 +115,7 @@ class NeuralNetwork(private val layers: List<Layer>) {
             onesForAnswers[intArrayOf(index, label)] = 1.0
         }
 
-        val softmaxValue = yPred.exp() / yPred.exp().sum(dim = 1, keepDim = true)
+        val softmaxValue = exp(yPred) / exp(yPred).sum(dim = 1, keepDim = true)
 
         (-onesForAnswers + softmaxValue) / (yPred.shape[0].toDouble())
     }
@@ -174,7 +173,6 @@ class NeuralNetwork(private val layers: List<Layer>) {
 }
 
 
-@OptIn(ExperimentalStdlibApi::class)
 fun main() = BroadcastDoubleTensorAlgebra {
     val features = 5
     val sampleSize = 250
@@ -182,17 +180,17 @@ fun main() = BroadcastDoubleTensorAlgebra {
     //val testSize = sampleSize - trainSize
 
     // take sample of features from normal distribution
-    val x = randomNormal(intArrayOf(sampleSize, features), seed) * 2.5
+    val x = randomNormal(ShapeND(sampleSize, features), seed) * 2.5
 
     x += fromArray(
-        intArrayOf(5),
+        ShapeND(5),
         doubleArrayOf(0.0, -1.0, -2.5, -3.0, 5.5) // row means
     )
 
 
     // define class like '1' if the sum of features > 0 and '0' otherwise
     val y = fromArray(
-        intArrayOf(sampleSize, 1),
+        ShapeND(sampleSize, 1),
         DoubleArray(sampleSize) { i ->
             if (x.getTensor(i).sum() > 0.0) {
                 1.0

gradle.properties

@@ -3,13 +3,14 @@
 # Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
 #
 kotlin.code.style=official
-kotlin.jupyter.add.scanner=false
 kotlin.mpp.stability.nowarn=true
 kotlin.native.ignoreDisabledTargets=true
-kotlin.incremental.js.ir=true
 
 org.gradle.configureondemand=true
-org.gradle.parallel=true
 org.gradle.jvmargs=-Xmx4096m
 
-toolsVersion=0.13.0-kotlin-1.7.20-Beta
+toolsVersion=0.14.6-kotlin-1.8.20
+
+
+org.gradle.parallel=true
+org.gradle.workers.max=4

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

kmath-ast/build.gradle.kts

@@ -3,10 +3,33 @@ plugins {
 }
 
 kscience{
+    jvm()
+    js()
     native()
+
+    dependencies {
+        api(projects.kmathCore)
+        api("com.github.h0tk3y.betterParse:better-parse:0.4.4")
     }
 
-kotlin.js {
+    testDependencies {
+        implementation(projects.kmathComplex)
+    }
+
+    dependencies(jsMain) {
+        implementation(npm("astring", "1.7.5"))
+        implementation(npm("binaryen", "101.0.0"))
+        implementation(npm("js-base64", "3.6.1"))
+    }
+
+    dependencies(jvmMain){
+        implementation("org.ow2.asm:asm-commons:9.2")
+    }
+
+}
+
+kotlin {
+    js {
         nodejs {
             testTask {
                 useMocha().timeout = "0"
@@ -20,48 +43,18 @@ kotlin.js {
         }
     }
 
-kotlin.sourceSets {
+    sourceSets {
         filter { it.name.contains("test", true) }
             .map(org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet::languageSettings)
             .forEach { it.optIn("space.kscience.kmath.misc.UnstableKMathAPI") }
-
-    commonMain {
-        dependencies {
-            api("com.github.h0tk3y.betterParse:better-parse:0.4.4")
-            api(project(":kmath-core"))
     }
 }
 
-    commonTest {
+if (System.getProperty("space.kscience.kmath.ast.dump.generated.classes") == "1") {
-        dependencies {
+    tasks.withType<org.jetbrains.kotlin.gradle.targets.jvm.tasks.KotlinJvmTest> {
-            implementation(project(":kmath-complex"))
-        }
-    }
-
-    jsMain {
-        dependencies {
-            implementation(npm("astring", "1.7.5"))
-            implementation(npm("binaryen", "101.0.0"))
-            implementation(npm("js-base64", "3.6.1"))
-        }
-    }
-
-    jvmMain {
-        dependencies {
-            implementation("org.ow2.asm:asm-commons:9.2")
-        }
-    }
-}
-
-//Workaround for https://github.com/Kotlin/dokka/issues/1455
-tasks.dokkaHtml {
-    dependsOn(tasks.build)
-}
-
-if (System.getProperty("space.kscience.kmath.ast.dump.generated.classes") == "1")
-    tasks.jvmTest {
         jvmArgs("-Dspace.kscience.kmath.ast.dump.generated.classes=1")
     }
+}
 
 readme {
     maturity = space.kscience.gradle.Maturity.EXPERIMENTAL

kmath-commons/src/main/kotlin/space/kscience/kmath/commons/expressions/CmDsExpression.kt

@@ -3,6 +3,8 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
+@file:Suppress("DEPRECATION")
+
 package space.kscience.kmath.commons.expressions
 
 import org.apache.commons.math3.analysis.differentiation.DerivativeStructure
@@ -18,7 +20,8 @@ import space.kscience.kmath.operations.NumbersAddOps
  * @param bindings The map of bindings values. All bindings are considered free parameters
  */
 @OptIn(UnstableKMathAPI::class)
-public class DerivativeStructureField(
+@Deprecated("Use generic DSAlgebra from the core")
+public class CmDsField(
     public val order: Int,
     bindings: Map<Symbol, Double>,
 ) : ExtendedField<DerivativeStructure>, ExpressionAlgebra<Double, DerivativeStructure>,
@@ -108,25 +111,27 @@ public class DerivativeStructureField(
 /**
  * Auto-diff processor based on Commons-math [DerivativeStructure]
  */
-public object DSProcessor : AutoDiffProcessor<Double, DerivativeStructure, DerivativeStructureField> {
+@Deprecated("Use generic DSAlgebra from the core")
+public object CmDsProcessor : AutoDiffProcessor<Double, DerivativeStructure, CmDsField> {
      override fun differentiate(
-        function: DerivativeStructureField.() -> DerivativeStructure,
+         function: CmDsField.() -> DerivativeStructure,
-    ): DerivativeStructureExpression = DerivativeStructureExpression(function)
+    ): CmDsExpression = CmDsExpression(function)
 }
 
 /**
  * A constructs that creates a derivative structure with required order on-demand
  */
-public class DerivativeStructureExpression(
+@Deprecated("Use generic DSAlgebra from the core")
-    public val function: DerivativeStructureField.() -> DerivativeStructure,
+public class CmDsExpression(
+    public val function: CmDsField.() -> DerivativeStructure,
 ) : DifferentiableExpression<Double> {
     override operator fun invoke(arguments: Map<Symbol, Double>): Double =
-        DerivativeStructureField(0, arguments).function().value
+        CmDsField(0, arguments).function().value
 
     /**
      * Get the derivative expression with given orders
      */
     override fun derivativeOrNull(symbols: List<Symbol>): Expression<Double> = Expression { arguments ->
-        with(DerivativeStructureField(symbols.size, arguments)) { function().derivative(symbols) }
+        with(CmDsField(symbols.size, arguments)) { function().derivative(symbols) }
     }
 }

kmath-commons/src/main/kotlin/space/kscience/kmath/commons/random/CMRandomGeneratorWrapper.kt

@@ -6,12 +6,12 @@
 package space.kscience.kmath.commons.random
 
 import kotlinx.coroutines.runBlocking
-import space.kscience.kmath.misc.PerformancePitfall
-import space.kscience.kmath.samplers.GaussianSampler
 import space.kscience.kmath.misc.toIntExact
-import space.kscience.kmath.stat.RandomGenerator
+import space.kscience.kmath.random.RandomGenerator
+import space.kscience.kmath.samplers.GaussianSampler
 import space.kscience.kmath.stat.next
 
+
 public class CMRandomGeneratorWrapper(
     public val factory: (IntArray) -> RandomGenerator,
 ) : org.apache.commons.math3.random.RandomGenerator {

kmath-commons/src/main/kotlin/space/kscience/kmath/commons/transform/Transformations.kt

@@ -10,28 +10,18 @@ import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.map
 import org.apache.commons.math3.transform.*
 import space.kscience.kmath.complex.Complex
-import space.kscience.kmath.operations.SuspendBufferTransform
+import space.kscience.kmath.operations.BufferTransform
 import space.kscience.kmath.streaming.chunked
 import space.kscience.kmath.streaming.spread
-import space.kscience.kmath.structures.Buffer
+import space.kscience.kmath.structures.*
-import space.kscience.kmath.structures.DoubleBuffer
-import space.kscience.kmath.structures.VirtualBuffer
-import space.kscience.kmath.structures.asBuffer
-
 
 /**
- * Streaming and buffer transformations
+ * Streaming and buffer transformations with Commons-math algorithms
  */
 public object Transformations {
-    private fun Buffer<Complex>.toArray(): Array<org.apache.commons.math3.complex.Complex> =
+    private fun Buffer<Complex>.toCmComplexArray(): Array<org.apache.commons.math3.complex.Complex> =
         Array(size) { org.apache.commons.math3.complex.Complex(get(it).re, get(it).im) }
 
-    private fun Buffer<Double>.asArray() = if (this is DoubleBuffer) {
-        array
-    } else {
-        DoubleArray(size) { i -> get(i) }
-    }
-
     /**
      * Create a virtual buffer on top of array
      */
@@ -43,70 +33,67 @@ public object Transformations {
     public fun fourier(
         normalization: DftNormalization = DftNormalization.STANDARD,
         direction: TransformType = TransformType.FORWARD,
-    ): SuspendBufferTransform<Complex, Complex> = {
+    ): BufferTransform<Complex, Complex> = BufferTransform {
-        FastFourierTransformer(normalization).transform(it.toArray(), direction).asBuffer()
+        FastFourierTransformer(normalization).transform(it.toCmComplexArray(), direction).asBuffer()
     }
 
     public fun realFourier(
         normalization: DftNormalization = DftNormalization.STANDARD,
         direction: TransformType = TransformType.FORWARD,
-    ): SuspendBufferTransform<Double, Complex> = {
+    ): BufferTransform<Double, Complex> = BufferTransform {
-        FastFourierTransformer(normalization).transform(it.asArray(), direction).asBuffer()
+        FastFourierTransformer(normalization).transform(it.toDoubleArray(), direction).asBuffer()
     }
 
     public fun sine(
         normalization: DstNormalization = DstNormalization.STANDARD_DST_I,
         direction: TransformType = TransformType.FORWARD,
-    ): SuspendBufferTransform<Double, Double> = {
+    ): BufferTransform<Double, Double> = DoubleBufferTransform {
-        FastSineTransformer(normalization).transform(it.asArray(), direction).asBuffer()
+        FastSineTransformer(normalization).transform(it.array, direction).asBuffer()
     }
 
     public fun cosine(
         normalization: DctNormalization = DctNormalization.STANDARD_DCT_I,
         direction: TransformType = TransformType.FORWARD,
-    ): SuspendBufferTransform<Double, Double> = {
+    ): BufferTransform<Double, Double> = BufferTransform {
-        FastCosineTransformer(normalization).transform(it.asArray(), direction).asBuffer()
+        FastCosineTransformer(normalization).transform(it.toDoubleArray(), direction).asBuffer()
     }
 
     public fun hadamard(
         direction: TransformType = TransformType.FORWARD,
-    ): SuspendBufferTransform<Double, Double> = {
+    ): BufferTransform<Double, Double> = DoubleBufferTransform {
-        FastHadamardTransformer().transform(it.asArray(), direction).asBuffer()
+        FastHadamardTransformer().transform(it.array, direction).asBuffer()
     }
 }
 
 /**
  * Process given [Flow] with commons-math fft transformation
  */
-@FlowPreview
+public fun Flow<Buffer<Complex>>.fft(
-public fun Flow<Buffer<Complex>>.FFT(
     normalization: DftNormalization = DftNormalization.STANDARD,
     direction: TransformType = TransformType.FORWARD,
 ): Flow<Buffer<Complex>> {
     val transform = Transformations.fourier(normalization, direction)
-    return map { transform(it) }
+    return map(transform::transform)
 }
 
-@FlowPreview
 @JvmName("realFFT")
-public fun Flow<Buffer<Double>>.FFT(
+public fun Flow<Buffer<Double>>.fft(
     normalization: DftNormalization = DftNormalization.STANDARD,
     direction: TransformType = TransformType.FORWARD,
 ): Flow<Buffer<Complex>> {
     val transform = Transformations.realFourier(normalization, direction)
-    return map(transform)
+    return map(transform::transform)
 }
 
 /**
  * Process a continuous flow of real numbers in FFT splitting it in chunks of [bufferSize].
  */
-@FlowPreview
 @JvmName("realFFT")
-public fun Flow<Double>.FFT(
+public fun Flow<Double>.fft(
     bufferSize: Int = Int.MAX_VALUE,
     normalization: DftNormalization = DftNormalization.STANDARD,
     direction: TransformType = TransformType.FORWARD,
-): Flow<Complex> = chunked(bufferSize).FFT(normalization, direction).spread()
+): Flow<Complex> = chunked(bufferSize).fft(normalization, direction).spread()
 
 /**
  * Map a complex flow into real flow by taking real part of each number

kmath-commons/src/test/kotlin/space/kscience/kmath/commons/expressions/DerivativeStructureExpressionTest.kt

@@ -3,6 +3,8 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
+@file:Suppress("DEPRECATION")
+
 package space.kscience.kmath.commons.expressions
 
 import space.kscience.kmath.expressions.*
@@ -15,10 +17,10 @@ import kotlin.test.assertFails
 internal inline fun diff(
     order: Int,
     vararg parameters: Pair<Symbol, Double>,
-    block: DerivativeStructureField.() -> Unit,
+    block: CmDsField.() -> Unit,
 ) {
     contract { callsInPlace(block, InvocationKind.EXACTLY_ONCE) }
-    DerivativeStructureField(order, mapOf(*parameters)).run(block)
+    CmDsField(order, mapOf(*parameters)).run(block)
 }
 
 internal class AutoDiffTest {
@@ -41,7 +43,7 @@ internal class AutoDiffTest {
 
     @Test
     fun autoDifTest() {
-        val f = DerivativeStructureExpression {
+        val f = CmDsExpression {
             val x by binding
             val y by binding
             x.pow(2) + 2 * x * y + y.pow(2) + 1

kmath-commons/src/test/kotlin/space/kscience/kmath/commons/optimization/OptimizeTest.kt

@@ -6,23 +6,25 @@
 package space.kscience.kmath.commons.optimization
 
 import kotlinx.coroutines.runBlocking
-import space.kscience.kmath.commons.expressions.DSProcessor
-import space.kscience.kmath.commons.expressions.DerivativeStructureExpression
 import space.kscience.kmath.distributions.NormalDistribution
+import space.kscience.kmath.expressions.DSFieldExpression
 import space.kscience.kmath.expressions.Symbol.Companion.x
 import space.kscience.kmath.expressions.Symbol.Companion.y
-import space.kscience.kmath.expressions.chiSquaredExpression
+import space.kscience.kmath.expressions.autodiff
 import space.kscience.kmath.expressions.symbol
-import space.kscience.kmath.operations.map
+import space.kscience.kmath.misc.UnstableKMathAPI
+import space.kscience.kmath.operations.DoubleBufferOps.Companion.map
+import space.kscience.kmath.operations.DoubleField
 import space.kscience.kmath.optimization.*
-import space.kscience.kmath.stat.RandomGenerator
+import space.kscience.kmath.random.RandomGenerator
+import space.kscience.kmath.stat.chiSquaredExpression
 import space.kscience.kmath.structures.DoubleBuffer
 import space.kscience.kmath.structures.asBuffer
-import kotlin.math.pow
 import kotlin.test.Test
 
+@OptIn(UnstableKMathAPI::class)
 internal class OptimizeTest {
-    val normal = DerivativeStructureExpression {
+    val normal = DSFieldExpression(DoubleField) {
         exp(-bindSymbol(x).pow(2) / 2) + exp(-bindSymbol(y).pow(2) / 2)
     }
 
@@ -61,7 +63,7 @@ internal class OptimizeTest {
 
         val yErr = DoubleBuffer(x.size) { sigma }
 
-        val chi2 = DSProcessor.chiSquaredExpression(
+        val chi2 = Double.autodiff.chiSquaredExpression(
             x, y, yErr
         ) { arg ->
             val cWithDefault = bindSymbolOrNull(c) ?: one

kmath-complex/build.gradle.kts

@@ -3,14 +3,12 @@ plugins {
 }
 
 kscience {
+    jvm()
+    js()
     native()
-}
 
-kotlin.sourceSets {
-    commonMain {
     dependencies {
-            api(project(":kmath-core"))
+        api(projects.kmathCore)
-        }
     }
 }
 

kmath-complex/src/commonMain/kotlin/space/kscience/kmath/complex/Complex.kt

@@ -80,6 +80,8 @@ public object ComplexField :
     override fun add(left: Complex, right: Complex): Complex = Complex(left.re + right.re, left.im + right.im)
 //    override fun multiply(a: Complex, k: Number): Complex = Complex(a.re * k.toDouble(), a.im * k.toDouble())
 
+//    override fun Complex.minus(arg: Complex): Complex = Complex(re - arg.re, im - arg.im)
+
     override fun multiply(left: Complex, right: Complex): Complex =
         Complex(left.re * right.re - left.im * right.im, left.re * right.im + left.im * right.re)
 
@@ -193,7 +195,6 @@ public object ComplexField :
  * @property re The real part.
  * @property im The imaginary part.
  */
-@OptIn(UnstableKMathAPI::class)
 public data class Complex(val re: Double, val im: Double) {
     public constructor(re: Number, im: Number) : this(re.toDouble(), im.toDouble())
     public constructor(re: Number) : this(re.toDouble(), 0.0)

kmath-complex/src/commonMain/kotlin/space/kscience/kmath/complex/ComplexFieldND.kt

@@ -5,6 +5,7 @@
 
 package space.kscience.kmath.complex
 
+import space.kscience.kmath.misc.PerformancePitfall
 import space.kscience.kmath.misc.UnstableKMathAPI
 import space.kscience.kmath.nd.*
 import space.kscience.kmath.operations.*
@@ -20,6 +21,7 @@ import kotlin.contracts.contract
 public sealed class ComplexFieldOpsND : BufferedFieldOpsND<Complex, ComplexField>(ComplexField.bufferAlgebra),
     ScaleOperations<StructureND<Complex>>, ExtendedFieldOps<StructureND<Complex>>, PowerOperations<StructureND<Complex>> {
 
+    @OptIn(PerformancePitfall::class)
     override fun StructureND<Complex>.toBufferND(): BufferND<Complex> = when (this) {
         is BufferND -> this
         else -> {
@@ -57,7 +59,7 @@ public sealed class ComplexFieldOpsND : BufferedFieldOpsND<Complex, ComplexField
 }
 
 @OptIn(UnstableKMathAPI::class)
-public class ComplexFieldND(override val shape: Shape) :
+public class ComplexFieldND(override val shape: ShapeND) :
     ComplexFieldOpsND(), FieldND<Complex, ComplexField>,
     NumbersAddOps<StructureND<Complex>> {
 
@@ -69,12 +71,12 @@ public class ComplexFieldND(override val shape: Shape) :
 
 public val ComplexField.ndAlgebra: ComplexFieldOpsND get() = ComplexFieldOpsND
 
-public fun ComplexField.ndAlgebra(vararg shape: Int): ComplexFieldND = ComplexFieldND(shape)
+public fun ComplexField.ndAlgebra(vararg shape: Int): ComplexFieldND = ComplexFieldND(ShapeND(shape))
 
 /**
  * Produce a context for n-dimensional operations inside this real field
  */
 public inline fun <R> ComplexField.withNdAlgebra(vararg shape: Int, action: ComplexFieldND.() -> R): R {
     contract { callsInPlace(action, InvocationKind.EXACTLY_ONCE) }
-    return ComplexFieldND(shape).action()
+    return ComplexFieldND(ShapeND(shape)).action()
 }

kmath-core/build.gradle.kts

@@ -3,10 +3,12 @@ plugins {
 }
 
 kscience{
+    jvm()
+    js()
     native()
 
     dependencies {
-        api(project(":kmath-memory"))
+        api(projects.kmathMemory)
     }
 }
 

kmath-core/src/commonMain/kotlin/space/kscience/kmath/expressions/DSAlgebra.kt

@@ -80,7 +80,6 @@ public abstract class DSAlgebra<T, A : Ring<T>>(
     public val algebra: A,
     public val order: Int,
     bindings: Map<Symbol, T>,
-    public val valueBufferFactory: MutableBufferFactory<T> = algebra.bufferFactory,
 ) : ExpressionAlgebra<T, DS<T, A>>, SymbolIndexer {
 
     /**
@@ -116,7 +115,6 @@ public abstract class DSAlgebra<T, A : Ring<T>>(
 
                     newCache[p][o] = DSCompiler(
                         algebra,
-                        valueBufferFactory,
                         p,
                         o,
                         valueCompiler,
@@ -141,7 +139,7 @@ public abstract class DSAlgebra<T, A : Ring<T>>(
     override val symbols: List<Symbol> = bindings.map { it.key }
 
     private fun bufferForVariable(index: Int, value: T): Buffer<T> {
-        val buffer = valueBufferFactory(compiler.size) { algebra.zero }
+        val buffer = algebra.bufferFactory(compiler.size) { algebra.zero }
         buffer[0] = value
         if (compiler.order > 0) {
             // the derivative of the variable with respect to itself is 1.
@@ -207,7 +205,7 @@ public abstract class DSAlgebra<T, A : Ring<T>>(
     }
 
     public override fun const(value: T): DS<T, A> {
-        val buffer = valueBufferFactory(compiler.size) { algebra.zero }
+        val buffer = algebra.bufferFactory(compiler.size) { algebra.zero }
         buffer[0] = value
 
         return DS(buffer)
@@ -245,11 +243,14 @@ public open class DSRing<T, A>(
     algebra: A,
     order: Int,
     bindings: Map<Symbol, T>,
-    valueBufferFactory: MutableBufferFactory<T>,
+) : DSAlgebra<T, A>(algebra, order, bindings),
-) : DSAlgebra<T, A>(algebra, order, bindings, valueBufferFactory),
+    Ring<DS<T, A>>,
-    Ring<DS<T, A>>, ScaleOperations<DS<T, A>>,
+    ScaleOperations<DS<T, A>>,
     NumericAlgebra<DS<T, A>>,
-    NumbersAddOps<DS<T, A>> where A : Ring<T>, A : NumericAlgebra<T>, A : ScaleOperations<T> {
+    NumbersAddOps<DS<T, A>>
+        where A : Ring<T>, A : NumericAlgebra<T>, A : ScaleOperations<T> {
+
+    public val elementBufferFactory: MutableBufferFactory<T> = algebra.bufferFactory
 
     override fun bindSymbolOrNull(value: String): DSSymbol? =
         super<DSAlgebra>.bindSymbolOrNull(value)
@@ -261,14 +262,14 @@ public open class DSRing<T, A>(
      */
     protected inline fun DS<T, A>.transformDataBuffer(block: A.(MutableBuffer<T>) -> Unit): DS<T, A> {
         require(derivativeAlgebra == this@DSRing) { "All derivative operations should be done in the same algebra" }
-        val newData = valueBufferFactory(compiler.size) { data[it] }
+        val newData = elementBufferFactory(compiler.size) { data[it] }
         algebra.block(newData)
         return DS(newData)
     }
 
     protected fun DS<T, A>.mapData(block: A.(T) -> T): DS<T, A> {
         require(derivativeAlgebra == this@DSRing) { "All derivative operations should be done in the same algebra" }
-        val newData: Buffer<T> = data.map(valueBufferFactory) {
+        val newData: Buffer<T> = data.mapToBuffer(elementBufferFactory) {
             algebra.block(it)
         }
         return DS(newData)
@@ -276,7 +277,7 @@ public open class DSRing<T, A>(
 
     protected fun DS<T, A>.mapDataIndexed(block: (Int, T) -> T): DS<T, A> {
         require(derivativeAlgebra == this@DSRing) { "All derivative operations should be done in the same algebra" }
-        val newData: Buffer<T> = data.mapIndexed(valueBufferFactory, block)
+        val newData: Buffer<T> = data.mapIndexedToBuffer(elementBufferFactory, block)
         return DS(newData)
     }
 
@@ -329,22 +330,21 @@ public class DerivativeStructureRingExpression<T, A>(
     public val function: DSRing<T, A>.() -> DS<T, A>,
 ) : DifferentiableExpression<T> where A : Ring<T>, A : ScaleOperations<T>, A : NumericAlgebra<T> {
     override operator fun invoke(arguments: Map<Symbol, T>): T =
-        DSRing(algebra, 0, arguments, elementBufferFactory).function().value
+        DSRing(algebra, 0, arguments).function().value
 
     override fun derivativeOrNull(symbols: List<Symbol>): Expression<T> = Expression { arguments ->
         with(
             DSRing(
                 algebra,
                 symbols.size,
-                arguments,
+                arguments
-                elementBufferFactory
             )
         ) { function().derivative(symbols) }
     }
 }
 
 /**
- * A field over commons-math [DerivativeStructure].
+ * A field over [DS].
  *
  * @property order The derivation order.
  * @param bindings The map of bindings values. All bindings are considered free parameters.
@@ -354,8 +354,7 @@ public class DSField<T, A : ExtendedField<T>>(
     algebra: A,
     order: Int,
     bindings: Map<Symbol, T>,
-    valueBufferFactory: MutableBufferFactory<T>,
+) : DSRing<T, A>(algebra, order, bindings), ExtendedField<DS<T, A>> {
-) : DSRing<T, A>(algebra, order, bindings, valueBufferFactory), ExtendedField<DS<T, A>> {
     override fun number(value: Number): DS<T, A> = const(algebra.number(value))
 
     override fun divide(left: DS<T, A>, right: DS<T, A>): DS<T, A> = left.transformDataBuffer { result ->
@@ -414,6 +413,7 @@ public class DSField<T, A : ExtendedField<T>>(
         is Int -> arg.transformDataBuffer { result ->
             compiler.pow(arg.data, 0, pow, result, 0)
         }
+
         else -> arg.transformDataBuffer { result ->
             compiler.pow(arg.data, 0, pow.toDouble(), result, 0)
         }
@@ -439,18 +439,29 @@ public class DSField<T, A : ExtendedField<T>>(
 @UnstableKMathAPI
 public class DSFieldExpression<T, A : ExtendedField<T>>(
     public val algebra: A,
-    private val valueBufferFactory: MutableBufferFactory<T> = algebra.bufferFactory,
     public val function: DSField<T, A>.() -> DS<T, A>,
 ) : DifferentiableExpression<T> {
     override operator fun invoke(arguments: Map<Symbol, T>): T =
-        DSField(algebra, 0, arguments, valueBufferFactory).function().value
+        DSField(algebra, 0, arguments).function().value
 
     override fun derivativeOrNull(symbols: List<Symbol>): Expression<T> = Expression { arguments ->
         DSField(
             algebra,
             symbols.size,
             arguments,
-            valueBufferFactory,
         ).run { function().derivative(symbols) }
     }
 }
+
+
+@UnstableKMathAPI
+public class DSFieldProcessor<T, A : ExtendedField<T>>(
+    public val algebra: A,
+) : AutoDiffProcessor<T, DS<T, A>, DSField<T, A>> {
+    override fun differentiate(
+        function: DSField<T, A>.() -> DS<T, A>,
+    ): DifferentiableExpression<T> = DSFieldExpression(algebra, function)
+}
+
+@UnstableKMathAPI
+public val Double.Companion.autodiff: DSFieldProcessor<Double, DoubleField> get() = DSFieldProcessor(DoubleField)

kmath-core/src/commonMain/kotlin/space/kscience/kmath/expressions/DSCompiler.kt

@@ -9,7 +9,6 @@ package space.kscience.kmath.expressions
 import space.kscience.kmath.operations.*
 import space.kscience.kmath.structures.Buffer
 import space.kscience.kmath.structures.MutableBuffer
-import space.kscience.kmath.structures.MutableBufferFactory
 import kotlin.math.min
 
 internal fun <T> MutableBuffer<T>.fill(element: T, fromIndex: Int = 0, toIndex: Int = size) {
@@ -56,7 +55,6 @@ internal fun <T> MutableBuffer<T>.fill(element: T, fromIndex: Int = 0, toIndex:
  */
 public class DSCompiler<T, out A : Algebra<T>> internal constructor(
     public val algebra: A,
-    public val bufferFactory: MutableBufferFactory<T>,
     public val freeParameters: Int,
     public val order: Int,
     valueCompiler: DSCompiler<T, A>?,

kmath-core/src/commonMain/kotlin/space/kscience/kmath/expressions/Expression.kt

@@ -48,6 +48,10 @@ public interface DoubleExpression : Expression<Double> {
      * @return the value.
      */
     public operator fun invoke(arguments: DoubleArray): Double
+
+    public companion object{
+        internal val EMPTY_DOUBLE_ARRAY = DoubleArray(0)
+    }
 }
 
 /**
@@ -73,6 +77,10 @@ public interface IntExpression : Expression<Int> {
      * @return the value.
      */
     public operator fun invoke(arguments: IntArray): Int
+
+    public companion object{
+        internal val EMPTY_INT_ARRAY = IntArray(0)
+    }
 }
 
 /**
@@ -98,6 +106,10 @@ public interface LongExpression : Expression<Long> {
      * @return the value.
      */
     public operator fun invoke(arguments: LongArray): Long
+
+    public companion object{
+        internal val EMPTY_LONG_ARRAY = LongArray(0)
+    }
 }
 
 /**
@@ -145,7 +157,7 @@ public operator fun <T> Expression<T>.invoke(vararg pairs: Pair<String, T>): T =
     }
 )
 
-private val EMPTY_DOUBLE_ARRAY = DoubleArray(0)
+
 
 /**
  * Calls this expression without providing any arguments.
@@ -153,7 +165,7 @@ private val EMPTY_DOUBLE_ARRAY = DoubleArray(0)
  * @return a value.
  */
 @UnstableKMathAPI
-public operator fun DoubleExpression.invoke(): Double = this(EMPTY_DOUBLE_ARRAY)
+public operator fun DoubleExpression.invoke(): Double = this(DoubleExpression.EMPTY_DOUBLE_ARRAY)
 
 /**
  * Calls this expression from arguments.
@@ -164,15 +176,13 @@ public operator fun DoubleExpression.invoke(): Double = this(EMPTY_DOUBLE_ARRAY)
 @UnstableKMathAPI
 public operator fun DoubleExpression.invoke(vararg arguments: Double): Double = this(arguments)
 
-private val EMPTY_INT_ARRAY = IntArray(0)
-
 /**
  * Calls this expression without providing any arguments.
  *
  * @return a value.
  */
 @UnstableKMathAPI
-public operator fun IntExpression.invoke(): Int = this(EMPTY_INT_ARRAY)
+public operator fun IntExpression.invoke(): Int = this(IntExpression.EMPTY_INT_ARRAY)
 
 /**
  * Calls this expression from arguments.
@@ -183,15 +193,13 @@ public operator fun IntExpression.invoke(): Int = this(EMPTY_INT_ARRAY)
 @UnstableKMathAPI
 public operator fun IntExpression.invoke(vararg arguments: Int): Int = this(arguments)
 
-private val EMPTY_LONG_ARRAY = LongArray(0)
-
 /**
  * Calls this expression without providing any arguments.
  *
  * @return a value.
  */
 @UnstableKMathAPI
-public operator fun LongExpression.invoke(): Long = this(EMPTY_LONG_ARRAY)
+public operator fun LongExpression.invoke(): Long = this(LongExpression.EMPTY_LONG_ARRAY)
 
 /**
  * Calls this expression from arguments.

kmath-core/src/commonMain/kotlin/space/kscience/kmath/expressions/ExpressionWithDefault.kt

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2018-2023 KMath contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package space.kscience.kmath.expressions
+
+public class ExpressionWithDefault<T>(
+    private val origin: Expression<T>,
+    private val defaultArgs: Map<Symbol, T>,
+) : Expression<T> {
+    override fun invoke(arguments: Map<Symbol, T>): T = origin.invoke(defaultArgs + arguments)
+}
+
+public fun <T> Expression<T>.withDefaultArgs(defaultArgs: Map<Symbol, T>): ExpressionWithDefault<T> =
+    ExpressionWithDefault(this, defaultArgs)
+
+
+public class DiffExpressionWithDefault<T>(
+    private val origin: DifferentiableExpression<T>,
+    private val defaultArgs: Map<Symbol, T>,
+) : DifferentiableExpression<T> {
+
+    override fun invoke(arguments: Map<Symbol, T>): T = origin.invoke(defaultArgs + arguments)
+
+    override fun derivativeOrNull(symbols: List<Symbol>): Expression<T>? =
+        origin.derivativeOrNull(symbols)?.withDefaultArgs(defaultArgs)
+}
+
+public fun <T> DifferentiableExpression<T>.withDefaultArgs(defaultArgs: Map<Symbol, T>): DiffExpressionWithDefault<T> =
+    DiffExpressionWithDefault(this, defaultArgs)

kmath-core/src/commonMain/kotlin/space/kscience/kmath/expressions/NamedMatrix.kt

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2018-2023 KMath contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+@file:OptIn(UnstableKMathAPI::class)
+
+package space.kscience.kmath.expressions
+
+import space.kscience.kmath.linear.Matrix
+import space.kscience.kmath.misc.PerformancePitfall
+import space.kscience.kmath.misc.UnstableKMathAPI
+import space.kscience.kmath.structures.getOrNull
+
+public class NamedMatrix<T>(public val values: Matrix<T>, public val indexer: SymbolIndexer) : Matrix<T> by values {
+    public operator fun get(i: Symbol, j: Symbol): T = get(indexer.indexOf(i), indexer.indexOf(j))
+
+    public companion object {
+
+        @OptIn(PerformancePitfall::class)
+        public fun toStringWithSymbols(values: Matrix<*>, indexer: SymbolIndexer): String = buildString {
+            appendLine(indexer.symbols.joinToString(separator = "\t", prefix = "\t\t"))
+            indexer.symbols.forEach { i ->
+                append(i.identity + "\t")
+                values.rows.getOrNull(indexer.indexOf(i))?.let { row ->
+                    indexer.symbols.forEach { j ->
+                        append(row.getOrNull(indexer.indexOf(j)).toString())
+                        append("\t")
+                    }
+                    appendLine()
+                }
+            }
+        }
+    }
+}
+
+public fun <T> Matrix<T>.named(indexer: SymbolIndexer): NamedMatrix<T> = NamedMatrix(this, indexer)
+
+public fun <T> Matrix<T>.named(symbols: List<Symbol>): NamedMatrix<T> = named(SimpleSymbolIndexer(symbols))

kmath-core/src/commonMain/kotlin/space/kscience/kmath/linear/BufferedLinearSpace.kt

@@ -6,9 +6,7 @@
 package space.kscience.kmath.linear
 
 import space.kscience.kmath.misc.PerformancePitfall
-import space.kscience.kmath.nd.BufferedRingOpsND
+import space.kscience.kmath.nd.*
-import space.kscience.kmath.nd.as2D
-import space.kscience.kmath.nd.asND
 import space.kscience.kmath.operations.*
 import space.kscience.kmath.structures.Buffer
 import space.kscience.kmath.structures.VirtualBuffer
@@ -23,7 +21,7 @@ public class BufferedLinearSpace<T, out A : Ring<T>>(
     private val ndAlgebra = BufferedRingOpsND(bufferAlgebra)
 
     override fun buildMatrix(rows: Int, columns: Int, initializer: A.(i: Int, j: Int) -> T): Matrix<T> =
-        ndAlgebra.structureND(intArrayOf(rows, columns)) { (i, j) -> elementAlgebra.initializer(i, j) }.as2D()
+        ndAlgebra.structureND(ShapeND(rows, columns)) { (i, j) -> elementAlgebra.initializer(i, j) }.as2D()
 
     override fun buildVector(size: Int, initializer: A.(Int) -> T): Point<T> =
         bufferAlgebra.buffer(size) { elementAlgebra.initializer(it) }

kmath-core/src/commonMain/kotlin/space/kscience/kmath/linear/DoubleLinearSpace.kt

@@ -6,9 +6,7 @@
 package space.kscience.kmath.linear
 
 import space.kscience.kmath.misc.PerformancePitfall
-import space.kscience.kmath.nd.DoubleFieldOpsND
+import space.kscience.kmath.nd.*
-import space.kscience.kmath.nd.as2D
-import space.kscience.kmath.nd.asND
 import space.kscience.kmath.operations.DoubleBufferOps
 import space.kscience.kmath.operations.DoubleField
 import space.kscience.kmath.operations.invoke
@@ -23,7 +21,7 @@ public object DoubleLinearSpace : LinearSpace<Double, DoubleField> {
         rows: Int,
         columns: Int,
         initializer: DoubleField.(i: Int, j: Int) -> Double
-    ): Matrix<Double> = DoubleFieldOpsND.structureND(intArrayOf(rows, columns)) { (i, j) ->
+    ): Matrix<Double> = DoubleFieldOpsND.structureND(ShapeND(rows, columns)) { (i, j) ->
         DoubleField.initializer(i, j)
     }.as2D()
 

kmath-core/src/commonMain/kotlin/space/kscience/kmath/linear/VirtualMatrix.kt

@@ -5,6 +5,9 @@
 
 package space.kscience.kmath.linear
 
+import space.kscience.kmath.nd.ShapeND
+
+
 /**
  * The matrix where each element is evaluated each time when is being accessed.
  *
@@ -16,7 +19,7 @@ public class VirtualMatrix<out T : Any>(
     public val generator: (i: Int, j: Int) -> T,
 ) : Matrix<T> {
 
-    override val shape: IntArray get() = intArrayOf(rowNum, colNum)
+    override val shape: ShapeND get() = ShapeND(rowNum, colNum)
 
     override operator fun get(i: Int, j: Int): T = generator(i, j)
 }

kmath-core/src/commonMain/kotlin/space/kscience/kmath/misc/Featured.kt

@@ -9,7 +9,7 @@ import kotlin.jvm.JvmInline
 import kotlin.reflect.KClass
 
 /**
- * A entity that contains a set of features defined by their types
+ * An entity that contains a set of features defined by their types
  */
 public interface Featured<F : Any> {
     public fun <T : F> getFeature(type: FeatureKey<T>): T?

kmath-core/src/commonMain/kotlin/space/kscience/kmath/misc/annotations.kt

@@ -29,3 +29,16 @@ public annotation class UnstableKMathAPI
 public annotation class PerformancePitfall(
     val message: String = "Potential performance problem",
 )
+
+/**
+ * Marks API that is public, but should not be used without clear understanding what it does.
+ */
+@MustBeDocumented
+@Retention(value = AnnotationRetention.BINARY)
+@RequiresOptIn(
+    "This API is unsafe and should be used carefully",
+    RequiresOptIn.Level.ERROR,
+)
+public annotation class UnsafeKMathAPI(
+    val message: String = "Unsafe API",
+)

kmath-core/src/commonMain/kotlin/space/kscience/kmath/misc/collections.kt

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2018-2023 KMath contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package space.kscience.kmath.misc
+
+/**
+ * The same as [zipWithNext], but includes link between last and first element
+ */
+public inline fun <T, R> List<T>.zipWithNextCircular(transform: (a: T, b: T) -> R): List<R> {
+    if (isEmpty()) return emptyList()
+    return indices.map { i ->
+        if (i == size - 1) {
+            transform(last(), first())
+        } else {
+            transform(get(i), get(i + 1))
+        }
+    }
+}
+
+public inline fun <T> List<T>.zipWithNextCircular(): List<Pair<T,T>> = zipWithNextCircular { l, r -> l to r  }

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/AlgebraND.kt

@@ -25,7 +25,7 @@ public interface AlgebraND<T, out C : Algebra<T>>: Algebra<StructureND<T>> {
     /**
      * Produces a new [StructureND] using given initializer function.
      */
-    public fun structureND(shape: Shape, initializer: C.(IntArray) -> T): StructureND<T>
+    public fun structureND(shape: ShapeND, initializer: C.(IntArray) -> T): StructureND<T>
 
     /**
      * Maps elements from one structure to another one by applying [transform] to them.

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/BufferAlgebraND.kt

@@ -12,11 +12,11 @@ import space.kscience.kmath.misc.UnstableKMathAPI
 import space.kscience.kmath.operations.*
 
 public interface BufferAlgebraND<T, out A : Algebra<T>> : AlgebraND<T, A> {
-    public val indexerBuilder: (IntArray) -> ShapeIndexer
+    public val indexerBuilder: (ShapeND) -> ShapeIndexer
     public val bufferAlgebra: BufferAlgebra<T, A>
     override val elementAlgebra: A get() = bufferAlgebra.elementAlgebra
 
-    override fun structureND(shape: Shape, initializer: A.(IntArray) -> T): BufferND<T> {
+    override fun structureND(shape: ShapeND, initializer: A.(IntArray) -> T): BufferND<T> {
         val indexer = indexerBuilder(shape)
         return BufferND(
             indexer,
@@ -26,6 +26,7 @@ public interface BufferAlgebraND<T, out A : Algebra<T>> : AlgebraND<T, A> {
         )
     }
 
+    @OptIn(PerformancePitfall::class)
     public fun StructureND<T>.toBufferND(): BufferND<T> = when (this) {
         is BufferND -> this
         else -> {
@@ -46,7 +47,7 @@ public interface BufferAlgebraND<T, out A : Algebra<T>> : AlgebraND<T, A> {
         zipInline(left.toBufferND(), right.toBufferND(), transform)
 
     public companion object {
-        public val defaultIndexerBuilder: (IntArray) -> ShapeIndexer = ::Strides
+        public val defaultIndexerBuilder: (ShapeND) -> ShapeIndexer = ::Strides
     }
 }
 
@@ -98,24 +99,24 @@ internal inline fun <T, A : Algebra<T>> BufferAlgebraND<T, A>.zipInline(
 @OptIn(PerformancePitfall::class)
 public open class BufferedGroupNDOps<T, out A : Group<T>>(
     override val bufferAlgebra: BufferAlgebra<T, A>,
-    override val indexerBuilder: (IntArray) -> ShapeIndexer = BufferAlgebraND.defaultIndexerBuilder,
+    override val indexerBuilder: (ShapeND) -> ShapeIndexer = BufferAlgebraND.defaultIndexerBuilder,
 ) : GroupOpsND<T, A>, BufferAlgebraND<T, A> {
     override fun StructureND<T>.unaryMinus(): StructureND<T> = map { -it }
 }
 
 public open class BufferedRingOpsND<T, out A : Ring<T>>(
     bufferAlgebra: BufferAlgebra<T, A>,
-    indexerBuilder: (IntArray) -> ShapeIndexer = BufferAlgebraND.defaultIndexerBuilder,
+    indexerBuilder: (ShapeND) -> ShapeIndexer = BufferAlgebraND.defaultIndexerBuilder,
 ) : BufferedGroupNDOps<T, A>(bufferAlgebra, indexerBuilder), RingOpsND<T, A>
 
 public open class BufferedFieldOpsND<T, out A : Field<T>>(
     bufferAlgebra: BufferAlgebra<T, A>,
-    indexerBuilder: (IntArray) -> ShapeIndexer = BufferAlgebraND.defaultIndexerBuilder,
+    indexerBuilder: (ShapeND) -> ShapeIndexer = BufferAlgebraND.defaultIndexerBuilder,
 ) : BufferedRingOpsND<T, A>(bufferAlgebra, indexerBuilder), FieldOpsND<T, A> {
 
     public constructor(
         elementAlgebra: A,
-        indexerBuilder: (IntArray) -> ShapeIndexer = BufferAlgebraND.defaultIndexerBuilder,
+        indexerBuilder: (ShapeND) -> ShapeIndexer = BufferAlgebraND.defaultIndexerBuilder,
     ) : this(BufferFieldOps(elementAlgebra), indexerBuilder)
 
     @OptIn(PerformancePitfall::class)
@@ -130,7 +131,7 @@ public val <T, A : Field<T>> BufferAlgebra<T, A>.nd: BufferedFieldOpsND<T, A> ge
 public fun <T, A : Algebra<T>> BufferAlgebraND<T, A>.structureND(
     vararg shape: Int,
     initializer: A.(IntArray) -> T,
-): BufferND<T> = structureND(shape, initializer)
+): BufferND<T> = structureND(ShapeND(shape), initializer)
 
 public fun <T, EA : Algebra<T>, A> A.structureND(
     initializer: EA.(IntArray) -> T,

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/BufferND.kt

@@ -5,6 +5,7 @@
 
 package space.kscience.kmath.nd
 
+import space.kscience.kmath.misc.PerformancePitfall
 import space.kscience.kmath.structures.Buffer
 import space.kscience.kmath.structures.BufferFactory
 import space.kscience.kmath.structures.MutableBuffer
@@ -22,32 +23,45 @@ public open class BufferND<out T>(
     public open val buffer: Buffer<T>,
 ) : StructureND<T> {
 
+    @PerformancePitfall
     override operator fun get(index: IntArray): T = buffer[indices.offset(index)]
 
-    override val shape: IntArray get() = indices.shape
+    override val shape: ShapeND get() = indices.shape
 
     override fun toString(): String = StructureND.toString(this)
 }
 
 /**
- * Transform structure to a new structure using provided [BufferFactory] and optimizing if argument is [BufferND]
+ * Create a generic [BufferND] using provided [initializer]
  */
-public inline fun <T, R : Any> StructureND<T>.mapToBuffer(
+public fun <T> BufferND(
-    factory: BufferFactory<R>,
+    shape: ShapeND,
-    crossinline transform: (T) -> R,
+    bufferFactory: BufferFactory<T> = BufferFactory.boxing(),
-): BufferND<R> = if (this is BufferND<T>)
+    initializer: (IntArray) -> T,
-    BufferND(this.indices, factory.invoke(indices.linearSize) { transform(buffer[it]) })
+): BufferND<T> {
-else {
+    val strides = Strides(shape)
-    val strides = DefaultStrides(shape)
+    return BufferND(strides, bufferFactory(strides.linearSize) { initializer(strides.index(it)) })
-    BufferND(strides, factory.invoke(strides.linearSize) { transform(get(strides.index(it))) })
 }
 
-/**
+///**
- * Transform structure to a new structure using inferred [BufferFactory]
+// * Transform structure to a new structure using provided [BufferFactory] and optimizing if argument is [BufferND]
- */
+// */
-public inline fun <T, reified R : Any> StructureND<T>.mapToBuffer(
+//public inline fun <T, R : Any> StructureND<T>.mapToBuffer(
-    crossinline transform: (T) -> R,
+//    factory: BufferFactory<R>,
-): BufferND<R> = mapToBuffer(Buffer.Companion::auto, transform)
+//    crossinline transform: (T) -> R,
+//): BufferND<R> = if (this is BufferND<T>)
+//    BufferND(this.indices, factory.invoke(indices.linearSize) { transform(buffer[it]) })
+//else {
+//    val strides = ColumnStrides(shape)
+//    BufferND(strides, factory.invoke(strides.linearSize) { transform(get(strides.index(it))) })
+//}
+//
+///**
+// * Transform structure to a new structure using inferred [BufferFactory]
+// */
+//public inline fun <T, reified R : Any> StructureND<T>.mapToBuffer(
+//    crossinline transform: (T) -> R,
+//): BufferND<R> = mapToBuffer(Buffer.Companion::auto, transform)
 
 /**
  * Represents [MutableStructureND] over [MutableBuffer].
@@ -60,22 +74,36 @@ public open class MutableBufferND<T>(
     strides: ShapeIndexer,
     override val buffer: MutableBuffer<T>,
 ) : MutableStructureND<T>, BufferND<T>(strides, buffer) {
+
+    @PerformancePitfall
     override fun set(index: IntArray, value: T) {
         buffer[indices.offset(index)] = value
     }
 }
 
 /**
- * Transform structure to a new structure using provided [MutableBufferFactory] and optimizing if argument is [MutableBufferND]
+ * Create a generic [BufferND] using provided [initializer]
  */
-public inline fun <T, reified R : Any> MutableStructureND<T>.mapToMutableBuffer(
+public fun <T> MutableBufferND(
-    factory: MutableBufferFactory<R> = MutableBufferFactory(MutableBuffer.Companion::auto),
+    shape: ShapeND,
-    crossinline transform: (T) -> R,
+    bufferFactory: MutableBufferFactory<T> = MutableBufferFactory.boxing(),
-): MutableBufferND<R> {
+    initializer: (IntArray) -> T,
-    return if (this is MutableBufferND<T>)
+): MutableBufferND<T> {
-        MutableBufferND(this.indices, factory.invoke(indices.linearSize) { transform(buffer[it]) })
+    val strides = Strides(shape)
-    else {
+    return MutableBufferND(strides, bufferFactory(strides.linearSize) { initializer(strides.index(it)) })
-        val strides = DefaultStrides(shape)
-        MutableBufferND(strides, factory.invoke(strides.linearSize) { transform(get(strides.index(it))) })
-    }
 }
+
+///**
+// * Transform structure to a new structure using provided [MutableBufferFactory] and optimizing if argument is [MutableBufferND]
+// */
+//public inline fun <T, reified R : Any> MutableStructureND<T>.mapToMutableBuffer(
+//    factory: MutableBufferFactory<R> = MutableBufferFactory(MutableBuffer.Companion::auto),
+//    crossinline transform: (T) -> R,
+//): MutableBufferND<R> {
+//    return if (this is MutableBufferND<T>)
+//        MutableBufferND(this.indices, factory.invoke(indices.linearSize) { transform(buffer[it]) })
+//    else {
+//        val strides = ColumnStrides(shape)
+//        MutableBufferND(strides, factory.invoke(strides.linearSize) { transform(get(strides.index(it))) })
+//    }
+//}

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/DoubleFieldND.kt

@@ -14,15 +14,25 @@ import kotlin.contracts.contract
 import kotlin.math.pow
 import kotlin.math.pow as kpow
 
+/**
+ * A simple mutable [StructureND] of doubles
+ */
 public class DoubleBufferND(
     indexes: ShapeIndexer,
     override val buffer: DoubleBuffer,
-) : MutableBufferND<Double>(indexes, buffer)
+) : MutableBufferND<Double>(indexes, buffer), MutableStructureNDOfDouble{
+    override fun getDouble(index: IntArray): Double = buffer[indices.offset(index)]
+
+    override fun setDouble(index: IntArray, value: Double) {
+        buffer[indices.offset(index)] = value
+    }
+}
 
 
 public sealed class DoubleFieldOpsND : BufferedFieldOpsND<Double, DoubleField>(DoubleField.bufferAlgebra),
     ScaleOperations<StructureND<Double>>, ExtendedFieldOps<StructureND<Double>> {
 
+    @OptIn(PerformancePitfall::class)
     override fun StructureND<Double>.toBufferND(): DoubleBufferND = when (this) {
         is DoubleBufferND -> this
         else -> {
@@ -64,7 +74,7 @@ public sealed class DoubleFieldOpsND : BufferedFieldOpsND<Double, DoubleField>(D
         transform: DoubleField.(Double, Double) -> Double,
     ): BufferND<Double> = zipInline(left.toBufferND(), right.toBufferND()) { l, r -> DoubleField.transform(l, r) }
 
-    override fun structureND(shape: Shape, initializer: DoubleField.(IntArray) -> Double): DoubleBufferND {
+    override fun structureND(shape: ShapeND, initializer: DoubleField.(IntArray) -> Double): DoubleBufferND {
         val indexer = indexerBuilder(shape)
         return DoubleBufferND(
             indexer,
@@ -179,7 +189,7 @@ public sealed class DoubleFieldOpsND : BufferedFieldOpsND<Double, DoubleField>(D
 }
 
 @OptIn(UnstableKMathAPI::class)
-public class DoubleFieldND(override val shape: Shape) :
+public class DoubleFieldND(override val shape: ShapeND) :
     DoubleFieldOpsND(), FieldND<Double, DoubleField>, NumbersAddOps<StructureND<Double>>,
     ExtendedField<StructureND<Double>> {
 
@@ -221,7 +231,8 @@ public class DoubleFieldND(override val shape: Shape) :
 
 public val DoubleField.ndAlgebra: DoubleFieldOpsND get() = DoubleFieldOpsND
 
-public fun DoubleField.ndAlgebra(vararg shape: Int): DoubleFieldND = DoubleFieldND(shape)
+public fun DoubleField.ndAlgebra(vararg shape: Int): DoubleFieldND = DoubleFieldND(ShapeND(shape))
+public fun DoubleField.ndAlgebra(shape: ShapeND): DoubleFieldND = DoubleFieldND(shape)
 
 /**
  * Produce a context for n-dimensional operations inside this real field
@@ -229,5 +240,5 @@ public fun DoubleField.ndAlgebra(vararg shape: Int): DoubleFieldND = DoubleField
 @UnstableKMathAPI
 public inline fun <R> DoubleField.withNdAlgebra(vararg shape: Int, action: DoubleFieldND.() -> R): R {
     contract { callsInPlace(action, InvocationKind.EXACTLY_ONCE) }
-    return DoubleFieldND(shape).run(action)
+    return DoubleFieldND(ShapeND(shape)).run(action)
 }

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/IntRingND.kt

@@ -20,7 +20,7 @@ public class IntBufferND(
 
 public sealed class IntRingOpsND : BufferedRingOpsND<Int, IntRing>(IntRing.bufferAlgebra) {
 
-    override fun structureND(shape: Shape, initializer: IntRing.(IntArray) -> Int): IntBufferND {
+    override fun structureND(shape: ShapeND, initializer: IntRing.(IntArray) -> Int): IntBufferND {
         val indexer = indexerBuilder(shape)
         return IntBufferND(
             indexer,
@@ -35,7 +35,7 @@ public sealed class IntRingOpsND : BufferedRingOpsND<Int, IntRing>(IntRing.buffe
 
 @OptIn(UnstableKMathAPI::class)
 public class IntRingND(
-    override val shape: Shape
+    override val shape: ShapeND
 ) : IntRingOpsND(), RingND<Int, IntRing>, NumbersAddOps<StructureND<Int>> {
 
     override fun number(value: Number): BufferND<Int> {
@@ -46,5 +46,5 @@ public class IntRingND(
 
 public inline fun <R> IntRing.withNdAlgebra(vararg shape: Int, action: IntRingND.() -> R): R {
     contract { callsInPlace(action, InvocationKind.EXACTLY_ONCE) }
-    return IntRingND(shape).run(action)
+    return IntRingND(ShapeND(shape)).run(action)
 }

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/PermutedStructureND.kt

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2018-2022 KMath contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package space.kscience.kmath.nd
+
+import space.kscience.kmath.misc.PerformancePitfall
+
+
+public class PermutedStructureND<T>(
+    public val origin: StructureND<T>,
+    public val permutation: (IntArray) -> IntArray,
+) : StructureND<T> {
+
+    override val shape: ShapeND
+        get() = origin.shape
+
+    @OptIn(PerformancePitfall::class)
+    override fun get(index: IntArray): T {
+        return origin[permutation(index)]
+    }
+}
+
+public fun <T> StructureND<T>.permute(
+    permutation: (IntArray) -> IntArray,
+): PermutedStructureND<T> = PermutedStructureND(this, permutation)
+
+public class PermutedMutableStructureND<T>(
+    public val origin: MutableStructureND<T>,
+    override val shape: ShapeND = origin.shape,
+    public val permutation: (IntArray) -> IntArray,
+) : MutableStructureND<T> {
+
+
+    @OptIn(PerformancePitfall::class)
+    override fun set(index: IntArray, value: T) {
+        origin[permutation(index)] = value
+    }
+
+    @OptIn(PerformancePitfall::class)
+    override fun get(index: IntArray): T {
+        return origin[permutation(index)]
+    }
+}
+
+public fun <T> MutableStructureND<T>.permute(
+    newShape: ShapeND = shape,
+    permutation: (IntArray) -> IntArray,
+): PermutedMutableStructureND<T> = PermutedMutableStructureND(this, newShape, permutation)

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/Shape.kt

@@ -1,35 +0,0 @@
-/*
- * Copyright 2018-2022 KMath contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package space.kscience.kmath.nd
-
-/**
- * An exception is thrown when the expected and actual shape of NDArray differ.
- *
- * @property expected the expected shape.
- * @property actual the actual shape.
- */
-public class ShapeMismatchException(public val expected: IntArray, public val actual: IntArray) :
-    RuntimeException("Shape ${actual.contentToString()} doesn't fit in expected shape ${expected.contentToString()}.")
-
-public class IndexOutOfShapeException(public val shape: Shape, public val index: IntArray) :
-    RuntimeException("Index ${index.contentToString()} is out of shape ${shape.contentToString()}")
-
-public typealias Shape = IntArray
-
-public fun Shape(shapeFirst: Int, vararg shapeRest: Int): Shape = intArrayOf(shapeFirst, *shapeRest)
-
-public interface WithShape {
-    public val shape: Shape
-
-    public val indices: ShapeIndexer get() = DefaultStrides(shape)
-}
-
-internal fun requireIndexInShape(index: IntArray, shape: Shape) {
-    if (index.size != shape.size) throw IndexOutOfShapeException(index, shape)
-    shape.forEachIndexed { axis, axisShape ->
-        if (index[axis] !in 0 until axisShape) throw IndexOutOfShapeException(index, shape)
-    }
-}

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/ShapeIndexer.kt

@@ -1,127 +0,0 @@
-/*
- * Copyright 2018-2022 KMath contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package space.kscience.kmath.nd
-
-import kotlin.native.concurrent.ThreadLocal
-
-/**
- * A converter from linear index to multivariate index
- */
-public interface ShapeIndexer : Iterable<IntArray> {
-    public val shape: Shape
-
-    /**
-     * Get linear index from multidimensional index
-     */
-    public fun offset(index: IntArray): Int
-
-    /**
-     * Get multidimensional from linear
-     */
-    public fun index(offset: Int): IntArray
-
-    /**
-     * The size of linear buffer to accommodate all elements of ND-structure corresponding to strides
-     */
-    public val linearSize: Int
-
-    // TODO introduce a fast way to calculate index of the next element?
-
-    /**
-     * Iterate over ND indices in a natural order
-     */
-    public fun asSequence(): Sequence<IntArray>
-
-    override fun iterator(): Iterator<IntArray> = asSequence().iterator()
-
-    override fun equals(other: Any?): Boolean
-    override fun hashCode(): Int
-}
-
-/**
- * Linear transformation of indexes
- */
-public abstract class Strides : ShapeIndexer {
-    /**
-     * Array strides
-     */
-    public abstract val strides: IntArray
-
-    public override fun offset(index: IntArray): Int = index.mapIndexed { i, value ->
-        if (value < 0 || value >= shape[i]) throw IndexOutOfBoundsException("Index $value out of shape bounds: (0,${this.shape[i]})")
-        value * strides[i]
-    }.sum()
-
-    // TODO introduce a fast way to calculate index of the next element?
-
-    /**
-     * Iterate over ND indices in a natural order
-     */
-    public override fun asSequence(): Sequence<IntArray> = (0 until linearSize).asSequence().map(::index)
-}
-
-/**
- * Simple implementation of [Strides].
- */
-public class DefaultStrides(override val shape: IntArray) : Strides() {
-    override val linearSize: Int get() = strides[shape.size]
-
-    /**
-     * Strides for memory access
-     */
-    override val strides: IntArray by lazy {
-        sequence {
-            var current = 1
-            yield(1)
-
-            shape.forEach {
-                current *= it
-                yield(current)
-            }
-        }.toList().toIntArray()
-    }
-
-    override fun index(offset: Int): IntArray {
-        val res = IntArray(shape.size)
-        var current = offset
-        var strideIndex = strides.size - 2
-
-        while (strideIndex >= 0) {
-            res[strideIndex] = (current / strides[strideIndex])
-            current %= strides[strideIndex]
-            strideIndex--
-        }
-
-        return res
-    }
-
-    override fun equals(other: Any?): Boolean {
-        if (this === other) return true
-        if (other !is DefaultStrides) return false
-        if (!shape.contentEquals(other.shape)) return false
-        return true
-    }
-
-    override fun hashCode(): Int = shape.contentHashCode()
-
-
-    public companion object {
-        /**
-         * Cached builder for default strides
-         */
-        @Deprecated("Replace by Strides(shape)")
-        public operator fun invoke(shape: IntArray): Strides =
-            defaultStridesCache.getOrPut(shape) { DefaultStrides(shape) }
-    }
-}
-
-@ThreadLocal
-private val defaultStridesCache = HashMap<IntArray, Strides>()
-
-/**
- * Cached builder for default strides
- */
-public fun Strides(shape: IntArray): Strides = defaultStridesCache.getOrPut(shape) { DefaultStrides(shape) }

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/ShapeIndices.kt

@@ -0,0 +1,174 @@
+/*
+ * Copyright 2018-2022 KMath contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package space.kscience.kmath.nd
+
+import kotlin.math.max
+import kotlin.native.concurrent.ThreadLocal
+
+/**
+ * A converter from linear index to multivariate index
+ */
+public interface ShapeIndexer : Iterable<IntArray> {
+    public val shape: ShapeND
+
+    /**
+     * Get linear index from multidimensional index
+     */
+    public fun offset(index: IntArray): Int
+
+    /**
+     * Get multidimensional from linear
+     */
+    public fun index(offset: Int): IntArray
+
+    /**
+     * The size of linear buffer to accommodate all elements of ND-structure corresponding to strides
+     */
+    public val linearSize: Int
+
+    // TODO introduce a fast way to calculate index of the next element?
+
+    /**
+     * Iterate over ND indices in a natural order
+     */
+    public fun asSequence(): Sequence<IntArray>
+
+    override fun iterator(): Iterator<IntArray> = asSequence().iterator()
+
+    override fun equals(other: Any?): Boolean
+    override fun hashCode(): Int
+}
+
+/**
+ * Linear transformation of indexes
+ */
+public abstract class Strides : ShapeIndexer {
+    /**
+     * Array strides
+     */
+    internal abstract val strides: IntArray
+
+    public override fun offset(index: IntArray): Int {
+        var res = 0
+        index.forEachIndexed { i, value ->
+            if (value !in 0 until shape[i]) throw IndexOutOfBoundsException("Index $value out of shape bounds: (0, ${this.shape[i]})")
+            res += value * strides[i]
+
+        }
+        return res
+    }
+
+    // TODO introduce a fast way to calculate index of the next element?
+
+    /**
+     * Iterate over ND indices in a natural order
+     */
+    public override fun asSequence(): Sequence<IntArray> = (0 until linearSize).asSequence().map(::index)
+
+}
+
+/**
+ * Column-first [Strides]. Columns are represented as continuous arrays
+ */
+public class ColumnStrides(override val shape: ShapeND) : Strides() {
+    override val linearSize: Int get() = strides[shape.size]
+
+    /**
+     * Strides for memory access
+     */
+    override val strides: IntArray = sequence {
+        var current = 1
+        yield(1)
+
+        shape.forEach {
+            current *= it
+            yield(current)
+        }
+    }.toList().toIntArray()
+
+    override fun index(offset: Int): IntArray {
+        val res = IntArray(shape.size)
+        var current = offset
+        var strideIndex = strides.size - 2
+
+        while (strideIndex >= 0) {
+            res[strideIndex] = (current / strides[strideIndex])
+            current %= strides[strideIndex]
+            strideIndex--
+        }
+
+        return res
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other !is ColumnStrides) return false
+        return shape.contentEquals(other.shape)
+    }
+
+    override fun hashCode(): Int = shape.contentHashCode()
+
+
+    public companion object
+}
+
+/**
+ * This [Strides] implementation follows the last dimension first convention
+ * For more information: https://numpy.org/doc/stable/reference/generated/numpy.ndarray.strides.html
+ *
+ * @param shape the shape of the tensor.
+ */
+public class RowStrides(override val shape: ShapeND) : Strides() {
+
+    override val strides: IntArray = run {
+        val nDim = shape.size
+        val res = IntArray(nDim)
+        if (nDim == 0) return@run res
+
+        var current = nDim - 1
+        res[current] = 1
+
+        while (current > 0) {
+            res[current - 1] = max(1, shape[current]) * res[current]
+            current--
+        }
+        res
+    }
+
+    override fun index(offset: Int): IntArray {
+        val res = IntArray(shape.size)
+        var current = offset
+        var strideIndex = 0
+
+        while (strideIndex < shape.size) {
+            res[strideIndex] = (current / strides[strideIndex])
+            current %= strides[strideIndex]
+            strideIndex++
+        }
+        return res
+    }
+
+    override val linearSize: Int get() = shape.linearSize
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other !is RowStrides) return false
+        return shape.contentEquals(other.shape)
+    }
+
+    override fun hashCode(): Int = shape.contentHashCode()
+
+    public companion object
+
+}
+
+@ThreadLocal
+private val defaultStridesCache = HashMap<ShapeND, Strides>()
+
+/**
+ * Cached builder for default strides
+ */
+public fun Strides(shape: ShapeND): Strides = defaultStridesCache.getOrPut(shape) { RowStrides(shape) }

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/ShapeND.kt

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2018-2022 KMath contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package space.kscience.kmath.nd
+
+import space.kscience.kmath.misc.UnsafeKMathAPI
+import kotlin.jvm.JvmInline
+
+/**
+ * A read-only ND shape
+ */
+@JvmInline
+public value class ShapeND(@PublishedApi internal val array: IntArray) {
+    public val size: Int get() = array.size
+    public operator fun get(index: Int): Int = array[index]
+    override fun toString(): String = array.contentToString()
+}
+
+public inline fun ShapeND.forEach(block: (value: Int) -> Unit): Unit = array.forEach(block)
+
+public inline fun ShapeND.forEachIndexed(block: (index: Int, value: Int) -> Unit): Unit = array.forEachIndexed(block)
+
+public infix fun ShapeND.contentEquals(other: ShapeND): Boolean = array.contentEquals(other.array)
+
+public fun ShapeND.contentHashCode(): Int = array.contentHashCode()
+
+public val ShapeND.indices: IntRange get() = array.indices
+public val ShapeND.linearSize: Int get() = array.reduce(Int::times)
+
+public fun ShapeND.slice(range: IntRange): ShapeND = ShapeND(array.sliceArray(range))
+
+public fun ShapeND.last(): Int = array.last()
+
+/**
+ * A shape including last [n] dimensions of this shape
+ */
+public fun ShapeND.last(n: Int): ShapeND = ShapeND(array.copyOfRange(size - n, size))
+
+public fun ShapeND.first(): Int = array.first()
+
+/**
+ * A shape including first [n] dimensions of this shape
+ */
+public fun ShapeND.first(n: Int): ShapeND = ShapeND(array.copyOfRange(0, n))
+
+public operator fun ShapeND.plus(add: IntArray): ShapeND = ShapeND(array + add)
+
+public operator fun ShapeND.plus(add: ShapeND): ShapeND = ShapeND(array + add.array)
+
+public fun ShapeND.isEmpty(): Boolean = size == 0
+public fun ShapeND.isNotEmpty(): Boolean = size > 0
+
+public fun ShapeND.transposed(i: Int, j: Int): ShapeND = ShapeND(array.copyOf().apply {
+    val ith = get(i)
+    val jth = get(j)
+    set(i, jth)
+    set(j, ith)
+})
+
+public operator fun ShapeND.component1(): Int = get(0)
+public operator fun ShapeND.component2(): Int = get(1)
+public operator fun ShapeND.component3(): Int = get(2)
+
+/**
+ * Convert to array with protective copy
+ */
+public fun ShapeND.toArray(): IntArray = array.copyOf()
+
+@UnsafeKMathAPI
+public fun ShapeND.asArray(): IntArray = array
+
+public fun ShapeND.asList(): List<Int> = array.asList()
+
+
+/**
+ * An exception is thrown when the expected and actual shape of NDArray differ.
+ *
+ * @property expected the expected shape.
+ * @property actual the actual shape.
+ */
+public class ShapeMismatchException(public val expected: ShapeND, public val actual: ShapeND) :
+    RuntimeException("Shape $actual doesn't fit in expected shape ${expected}.")
+
+public class IndexOutOfShapeException(public val shape: ShapeND, public val index: IntArray) :
+    RuntimeException("Index ${index.contentToString()} is out of shape ${shape}")
+
+public fun ShapeND(shapeFirst: Int, vararg shapeRest: Int): ShapeND = ShapeND(intArrayOf(shapeFirst, *shapeRest))
+
+public interface WithShape {
+    public val shape: ShapeND
+
+    public val indices: ShapeIndexer get() = ColumnStrides(shape)
+}
+
+internal fun requireIndexInShape(index: IntArray, shape: ShapeND) {
+    if (index.size != shape.size) throw IndexOutOfShapeException(shape, index)
+    shape.forEachIndexed { axis, axisShape ->
+        if (index[axis] !in 0 until axisShape) throw IndexOutOfShapeException(shape, index)
+    }
+}

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/ShortRingND.kt

@@ -18,7 +18,7 @@ public sealed class ShortRingOpsND : BufferedRingOpsND<Short, ShortRing>(ShortRi
 
 @OptIn(UnstableKMathAPI::class)
 public class ShortRingND(
-    override val shape: Shape
+    override val shape: ShapeND
 ) : ShortRingOpsND(), RingND<Short, ShortRing>, NumbersAddOps<StructureND<Short>> {
 
     override fun number(value: Number): BufferND<Short> {
@@ -30,5 +30,5 @@ public class ShortRingND(
 
 public inline fun <R> ShortRing.withNdAlgebra(vararg shape: Int, action: ShortRingND.() -> R): R {
     contract { callsInPlace(action, InvocationKind.EXACTLY_ONCE) }
-    return ShortRingND(shape).run(action)
+    return ShortRingND(ShapeND(shape)).run(action)
 }

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/Structure1D.kt

@@ -18,6 +18,7 @@ import kotlin.jvm.JvmInline
 public interface Structure1D<out T> : StructureND<T>, Buffer<T> {
     override val dimension: Int get() = 1
 
+    @PerformancePitfall
     override operator fun get(index: IntArray): T {
         require(index.size == 1) { "Index dimension mismatch. Expected 1 but found ${index.size}" }
         return get(index[0])
@@ -32,6 +33,8 @@ public interface Structure1D<out T> : StructureND<T>, Buffer<T> {
  * A mutable structure that is guaranteed to be one-dimensional
  */
 public interface MutableStructure1D<T> : Structure1D<T>, MutableStructureND<T>, MutableBuffer<T> {
+
+    @PerformancePitfall
     override operator fun set(index: IntArray, value: T) {
         require(index.size == 1) { "Index dimension mismatch. Expected 1 but found ${index.size}" }
         set(index[0], value)
@@ -43,9 +46,10 @@ public interface MutableStructure1D<T> : Structure1D<T>, MutableStructureND<T>,
  */
 @JvmInline
 private value class Structure1DWrapper<out T>(val structure: StructureND<T>) : Structure1D<T> {
-    override val shape: IntArray get() = structure.shape
+    override val shape: ShapeND get() = structure.shape
     override val size: Int get() = structure.shape[0]
 
+    @PerformancePitfall
     override operator fun get(index: Int): T = structure[index]
 
     @PerformancePitfall
@@ -56,13 +60,16 @@ private value class Structure1DWrapper<out T>(val structure: StructureND<T>) : S
  * A 1D wrapper for a mutable nd-structure
  */
 private class MutableStructure1DWrapper<T>(val structure: MutableStructureND<T>) : MutableStructure1D<T> {
-    override val shape: IntArray get() = structure.shape
+    override val shape: ShapeND get() = structure.shape
     override val size: Int get() = structure.shape[0]
 
     @PerformancePitfall
     override fun elements(): Sequence<Pair<IntArray, T>> = structure.elements()
 
+    @PerformancePitfall
     override fun get(index: Int): T = structure[index]
+
+    @PerformancePitfall
     override fun set(index: Int, value: T) {
         structure[intArrayOf(index)] = value
     }
@@ -83,7 +90,7 @@ private class MutableStructure1DWrapper<T>(val structure: MutableStructureND<T>)
  */
 @JvmInline
 private value class Buffer1DWrapper<out T>(val buffer: Buffer<T>) : Structure1D<T> {
-    override val shape: IntArray get() = intArrayOf(buffer.size)
+    override val shape: ShapeND get() = ShapeND(buffer.size)
     override val size: Int get() = buffer.size
 
     @PerformancePitfall
@@ -95,7 +102,7 @@ private value class Buffer1DWrapper<out T>(val buffer: Buffer<T>) : Structure1D<
 }
 
 internal class MutableBuffer1DWrapper<T>(val buffer: MutableBuffer<T>) : MutableStructure1D<T> {
-    override val shape: IntArray get() = intArrayOf(buffer.size)
+    override val shape: ShapeND get() = ShapeND(buffer.size)
     override val size: Int get() = buffer.size
 
     @PerformancePitfall

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/Structure2D.kt

@@ -7,6 +7,7 @@ package space.kscience.kmath.nd
 
 import space.kscience.kmath.misc.PerformancePitfall
 import space.kscience.kmath.structures.Buffer
+import space.kscience.kmath.structures.MutableBuffer
 import space.kscience.kmath.structures.MutableListBuffer
 import space.kscience.kmath.structures.VirtualBuffer
 import kotlin.jvm.JvmInline
@@ -28,7 +29,7 @@ public interface Structure2D<out T> : StructureND<T> {
      */
     public val colNum: Int
 
-    override val shape: IntArray get() = intArrayOf(rowNum, colNum)
+    override val shape: ShapeND get() = ShapeND(rowNum, colNum)
 
     /**
      * The buffer of rows of this structure. It gets elements from the structure dynamically.
@@ -53,6 +54,7 @@ public interface Structure2D<out T> : StructureND<T> {
      */
     public operator fun get(i: Int, j: Int): T
 
+    @PerformancePitfall
     override operator fun get(index: IntArray): T {
         require(index.size == 2) { "Index dimension mismatch. Expected 2 but found ${index.size}" }
         return get(index[0], index[1])
@@ -84,15 +86,15 @@ public interface MutableStructure2D<T> : Structure2D<T>, MutableStructureND<T> {
      * The buffer of rows of this structure. It gets elements from the structure dynamically.
      */
     @PerformancePitfall
-    override val rows: List<MutableStructure1D<T>>
+    override val rows: List<MutableBuffer<T>>
-        get() = List(rowNum) { i -> MutableBuffer1DWrapper(MutableListBuffer(colNum) { j -> get(i, j) }) }
+        get() = List(rowNum) { i -> MutableListBuffer(colNum) { j -> get(i, j) } }
 
     /**
      * The buffer of columns of this structure. It gets elements from the structure dynamically.
      */
     @PerformancePitfall
-    override val columns: List<MutableStructure1D<T>>
+    override val columns: List<MutableBuffer<T>>
-        get() = List(colNum) { j -> MutableBuffer1DWrapper(MutableListBuffer(rowNum) { i -> get(i, j) }) }
+        get() = List(colNum) { j -> MutableListBuffer(rowNum) { i -> get(i, j) } }
 }
 
 /**
@@ -100,11 +102,12 @@ public interface MutableStructure2D<T> : Structure2D<T>, MutableStructureND<T> {
  */
 @JvmInline
 private value class Structure2DWrapper<out T>(val structure: StructureND<T>) : Structure2D<T> {
-    override val shape: Shape get() = structure.shape
+    override val shape: ShapeND get() = structure.shape
 
     override val rowNum: Int get() = shape[0]
     override val colNum: Int get() = shape[1]
 
+    @PerformancePitfall
     override operator fun get(i: Int, j: Int): T = structure[i, j]
 
     override fun <F : StructureFeature> getFeature(type: KClass<out F>): F? = structure.getFeature(type)
@@ -117,17 +120,20 @@ private value class Structure2DWrapper<out T>(val structure: StructureND<T>) : S
  * A 2D wrapper for a mutable nd-structure
  */
 private class MutableStructure2DWrapper<T>(val structure: MutableStructureND<T>) : MutableStructure2D<T> {
-    override val shape: Shape get() = structure.shape
+    override val shape: ShapeND get() = structure.shape
 
     override val rowNum: Int get() = shape[0]
     override val colNum: Int get() = shape[1]
 
+    @PerformancePitfall
     override operator fun get(i: Int, j: Int): T = structure[i, j]
 
+    @PerformancePitfall
     override fun set(index: IntArray, value: T) {
         structure[index] = value
     }
 
+    @PerformancePitfall
     override operator fun set(i: Int, j: Int, value: T) {
         structure[intArrayOf(i, j)] = value
     }

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/StructureND.kt

@@ -33,7 +33,7 @@ public interface StructureND<out T> : Featured<StructureFeature>, WithShape {
      * The shape of structure i.e., non-empty sequence of non-negative integers that specify sizes of dimensions of
      * this structure.
      */
-    override val shape: Shape
+    override val shape: ShapeND
 
     /**
      * The count of dimensions in this structure. It should be equal to size of [shape].
@@ -46,6 +46,7 @@ public interface StructureND<out T> : Featured<StructureFeature>, WithShape {
      * @param index the indices.
      * @return the value.
      */
+    @PerformancePitfall
     public operator fun get(index: IntArray): T
 
     /**
@@ -97,6 +98,7 @@ public interface StructureND<out T> : Featured<StructureFeature>, WithShape {
         /**
          * Debug output to string
          */
+        @OptIn(PerformancePitfall::class)
         public fun toString(structure: StructureND<*>): String {
             val bufferRepr: String = when (structure.shape.size) {
                 1 -> (0 until structure.shape[0]).map { structure[it] }
@@ -116,7 +118,7 @@ public interface StructureND<out T> : Featured<StructureFeature>, WithShape {
             }
             val className = structure::class.simpleName ?: "StructureND"
 
-            return "$className(shape=${structure.shape.contentToString()}, buffer=$bufferRepr)"
+            return "$className(shape=${structure.shape}, buffer=$bufferRepr)"
         }
 
         /**
@@ -145,28 +147,28 @@ public interface StructureND<out T> : Featured<StructureFeature>, WithShape {
         ): BufferND<T> = BufferND(strides, Buffer.auto(type, strides.linearSize) { i -> initializer(strides.index(i)) })
 
         public fun <T> buffered(
-            shape: IntArray,
+            shape: ShapeND,
             bufferFactory: BufferFactory<T> = BufferFactory.boxing(),
             initializer: (IntArray) -> T,
-        ): BufferND<T> = buffered(DefaultStrides(shape), bufferFactory, initializer)
+        ): BufferND<T> = buffered(ColumnStrides(shape), bufferFactory, initializer)
 
         public inline fun <reified T : Any> auto(
-            shape: IntArray,
+            shape: ShapeND,
             crossinline initializer: (IntArray) -> T,
-        ): BufferND<T> = auto(DefaultStrides(shape), initializer)
+        ): BufferND<T> = auto(ColumnStrides(shape), initializer)
 
         @JvmName("autoVarArg")
         public inline fun <reified T : Any> auto(
             vararg shape: Int,
             crossinline initializer: (IntArray) -> T,
         ): BufferND<T> =
-            auto(DefaultStrides(shape), initializer)
+            auto(ColumnStrides(ShapeND(shape)), initializer)
 
         public inline fun <T : Any> auto(
             type: KClass<T>,
             vararg shape: Int,
             crossinline initializer: (IntArray) -> T,
-        ): BufferND<T> = auto(type, DefaultStrides(shape), initializer)
+        ): BufferND<T> = auto(type, ColumnStrides(ShapeND(shape)), initializer)
     }
 }
 
@@ -214,8 +216,13 @@ public fun <T : Comparable<T>> LinearSpace<T, Ring<T>>.contentEquals(
  * @param index the indices.
  * @return the value.
  */
+@PerformancePitfall
 public operator fun <T> StructureND<T>.get(vararg index: Int): T = get(index)
 
+public operator fun StructureND<Double>.get(vararg index: Int): Double = getDouble(index)
+
+public operator fun StructureND<Int>.get(vararg index: Int): Int = getInt(index)
+
 //@UnstableKMathAPI
 //public inline fun <reified T : StructureFeature> StructureND<*>.getFeature(): T? = getFeature(T::class)
 
@@ -229,12 +236,14 @@ public interface MutableStructureND<T> : StructureND<T> {
      * @param index the indices.
      * @param value the value.
      */
+    @PerformancePitfall
     public operator fun set(index: IntArray, value: T)
 }
 
 /**
  * Set value at specified indices
  */
+@PerformancePitfall
 public operator fun <T> MutableStructureND<T>.set(vararg index: Int, value: T) {
     set(index, value)
 }

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/VirtualStructureND.kt

@@ -5,12 +5,15 @@
 
 package space.kscience.kmath.nd
 
+import space.kscience.kmath.misc.PerformancePitfall
 import space.kscience.kmath.misc.UnstableKMathAPI
 
 public open class VirtualStructureND<T>(
-    override val shape: Shape,
+    override val shape: ShapeND,
     public val producer: (IntArray) -> T,
 ) : StructureND<T> {
+
+    @PerformancePitfall
     override fun get(index: IntArray): T {
         requireIndexInShape(index, shape)
         return producer(index)
@@ -19,12 +22,12 @@ public open class VirtualStructureND<T>(
 
 @UnstableKMathAPI
 public class VirtualDoubleStructureND(
-    shape: Shape,
+    shape: ShapeND,
     producer: (IntArray) -> Double,
 ) : VirtualStructureND<Double>(shape, producer)
 
 @UnstableKMathAPI
 public class VirtualIntStructureND(
-    shape: Shape,
+    shape: ShapeND,
     producer: (IntArray) -> Int,
 ) : VirtualStructureND<Int>(shape, producer)

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/algebraNDExtentions.kt

@@ -15,9 +15,9 @@ public fun <T, A : Algebra<T>> AlgebraND<T, A>.structureND(
     shapeFirst: Int,
     vararg shapeRest: Int,
     initializer: A.(IntArray) -> T
-): StructureND<T> = structureND(Shape(shapeFirst, *shapeRest), initializer)
+): StructureND<T> = structureND(ShapeND(shapeFirst, *shapeRest), initializer)
 
-public fun <T, A : Group<T>> AlgebraND<T, A>.zero(shape: Shape): StructureND<T> = structureND(shape) { zero }
+public fun <T, A : Group<T>> AlgebraND<T, A>.zero(shape: ShapeND): StructureND<T> = structureND(shape) { zero }
 
 @JvmName("zeroVarArg")
 public fun <T, A : Group<T>> AlgebraND<T, A>.zero(
@@ -25,7 +25,7 @@ public fun <T, A : Group<T>> AlgebraND<T, A>.zero(
     vararg shapeRest: Int,
 ): StructureND<T> = structureND(shapeFirst, *shapeRest) { zero }
 
-public fun <T, A : Ring<T>> AlgebraND<T, A>.one(shape: Shape): StructureND<T> = structureND(shape) { one }
+public fun <T, A : Ring<T>> AlgebraND<T, A>.one(shape: ShapeND): StructureND<T> = structureND(shape) { one }
 
 @JvmName("oneVarArg")
 public fun <T, A : Ring<T>> AlgebraND<T, A>.one(

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/operationsND.kt

@@ -5,6 +5,9 @@
 
 package space.kscience.kmath.nd
 
+import space.kscience.kmath.misc.PerformancePitfall
+
+@OptIn(PerformancePitfall::class)
 public fun <T> StructureND<T>.roll(axis: Int, step: Int = 1): StructureND<T> {
     require(axis in shape.indices) { "Axis $axis is outside of shape dimensions: [0, ${shape.size})" }
     return VirtualStructureND(shape) { index ->
@@ -19,6 +22,7 @@ public fun <T> StructureND<T>.roll(axis: Int, step: Int = 1): StructureND<T> {
     }
 }
 
+@OptIn(PerformancePitfall::class)
 public fun <T> StructureND<T>.roll(pair: Pair<Int, Int>, vararg others: Pair<Int, Int>): StructureND<T> {
     val axisMap: Map<Int, Int> = mapOf(pair, *others)
     require(axisMap.keys.all { it in shape.indices }) { "Some of axes ${axisMap.keys} is outside of shape dimensions: [0, ${shape.size})" }

kmath-core/src/commonMain/kotlin/space/kscience/kmath/nd/primitiveStructureND.kt

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2018-2022 KMath contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package space.kscience.kmath.nd
+
+import space.kscience.kmath.misc.PerformancePitfall
+
+public interface StructureNDOfDouble : StructureND<Double> {
+    /**
+     * Guaranteed non-blocking access to content
+     */
+    public fun getDouble(index: IntArray): Double
+}
+
+/**
+ * Optimized method to access primitive without boxing if possible
+ */
+@OptIn(PerformancePitfall::class)
+public fun StructureND<Double>.getDouble(index: IntArray): Double =
+    if (this is StructureNDOfDouble) getDouble(index) else get(index)
+
+public interface MutableStructureNDOfDouble : StructureNDOfDouble, MutableStructureND<Double> {
+    /**
+     * Guaranteed non-blocking access to content
+     */
+    public fun setDouble(index: IntArray, value: Double)
+}
+
+@OptIn(PerformancePitfall::class)
+public fun MutableStructureND<Double>.getDouble(index: IntArray): Double =
+    if (this is StructureNDOfDouble) getDouble(index) else get(index)
+
+
+public interface StructureNDOfInt : StructureND<Int> {
+    /**
+     * Guaranteed non-blocking access to content
+     */
+    public fun getInt(index: IntArray): Int
+}
+
+@OptIn(PerformancePitfall::class)
+public fun StructureND<Int>.getInt(index: IntArray): Int =
+    if (this is StructureNDOfInt) getInt(index) else get(index)

kmath-core/src/commonMain/kotlin/space/kscience/kmath/operations/DoubleBufferField.kt

@@ -5,8 +5,6 @@
 
 package space.kscience.kmath.operations
 
-import space.kscience.kmath.misc.UnstableKMathAPI
-import space.kscience.kmath.operations.DoubleField.pow
 import space.kscience.kmath.structures.Buffer
 import space.kscience.kmath.structures.DoubleBuffer
 
@@ -32,9 +30,9 @@ public class DoubleBufferField(public val size: Int) : ExtendedField<Buffer<Doub
     override fun atanh(arg: Buffer<Double>): DoubleBuffer = super<DoubleBufferOps>.atanh(arg)
 
     override fun power(arg: Buffer<Double>, pow: Number): DoubleBuffer = if (pow.isInteger()) {
-        arg.mapInline { it.pow(pow.toInt()) }
+        arg.map { it.pow(pow.toInt()) }
     } else {
-        arg.mapInline {
+        arg.map {
             if(it<0) throw IllegalArgumentException("Negative argument $it could not be raised to the fractional power")
             it.pow(pow.toDouble())
         }
@@ -42,103 +40,4 @@ public class DoubleBufferField(public val size: Int) : ExtendedField<Buffer<Doub
 
     override fun unaryOperationFunction(operation: String): (arg: Buffer<Double>) -> Buffer<Double> =
         super<ExtendedField>.unaryOperationFunction(operation)
-
-    //    override fun number(value: Number): Buffer<Double> = DoubleBuffer(size) { value.toDouble() }
-//
-//    override fun Buffer<Double>.unaryMinus(): Buffer<Double> = DoubleBufferOperations.run {
-//        -this@unaryMinus
-//    }
-//
-//    override fun add(a: Buffer<Double>, b: Buffer<Double>): DoubleBuffer {
-//        require(a.size == size) { "The buffer size ${a.size} does not match context size $size" }
-//        return DoubleBufferOperations.add(a, b)
-//    }
-//
-
-//
-//    override fun multiply(a: Buffer<Double>, b: Buffer<Double>): DoubleBuffer {
-//        require(a.size == size) { "The buffer size ${a.size} does not match context size $size" }
-//        return DoubleBufferOperations.multiply(a, b)
-//    }
-//
-//    override fun divide(a: Buffer<Double>, b: Buffer<Double>): DoubleBuffer {
-//        require(a.size == size) { "The buffer size ${a.size} does not match context size $size" }
-//        return DoubleBufferOperations.divide(a, b)
-//    }
-//
-//    override fun sin(arg: Buffer<Double>): DoubleBuffer {
-//        require(arg.size == size) { "The buffer size ${arg.size} does not match context size $size" }
-//        return DoubleBufferOperations.sin(arg)
-//    }
-//
-//    override fun cos(arg: Buffer<Double>): DoubleBuffer {
-//        require(arg.size == size) { "The buffer size ${arg.size} does not match context size $size" }
-//        return DoubleBufferOperations.cos(arg)
-//    }
-//
-//    override fun tan(arg: Buffer<Double>): DoubleBuffer {
-//        require(arg.size == size) { "The buffer size ${arg.size} does not match context size $size" }
-//        return DoubleBufferOperations.tan(arg)
-//    }
-//
-//    override fun asin(arg: Buffer<Double>): DoubleBuffer {
-//        require(arg.size == size) { "The buffer size ${arg.size} does not match context size $size" }
-//        return DoubleBufferOperations.asin(arg)
-//    }
-//
-//    override fun acos(arg: Buffer<Double>): DoubleBuffer {
-//        require(arg.size == size) { "The buffer size ${arg.size} does not match context size $size" }
-//        return DoubleBufferOperations.acos(arg)
-//    }
-//
-//    override fun atan(arg: Buffer<Double>): DoubleBuffer {
-//        require(arg.size == size) { "The buffer size ${arg.size} does not match context size $size" }
-//        return DoubleBufferOperations.atan(arg)
-//    }
-//
-//    override fun sinh(arg: Buffer<Double>): DoubleBuffer {
-//        require(arg.size == size) { "The buffer size ${arg.size} does not match context size $size" }
-//        return DoubleBufferOperations.sinh(arg)
-//    }
-//
-//    override fun cosh(arg: Buffer<Double>): DoubleBuffer {
-//        require(arg.size == size) { "The buffer size ${arg.size} does not match context size $size" }
-//        return DoubleBufferOperations.cosh(arg)
-//    }
-//
-//    override fun tanh(arg: Buffer<Double>): DoubleBuffer {
-//        require(arg.size == size) { "The buffer size ${arg.size} does not match context size $size" }
-//        return DoubleBufferOperations.tanh(arg)
-//    }
-//
-//    override fun asinh(arg: Buffer<Double>): DoubleBuffer {
-//        require(arg.size == size) { "The buffer size ${arg.size} does not match context size $size" }
-//        return DoubleBufferOperations.asinh(arg)
-//    }
-//
-//    override fun acosh(arg: Buffer<Double>): DoubleBuffer {
-//        require(arg.size == size) { "The buffer size ${arg.size} does not match context size $size" }
-//        return DoubleBufferOperations.acosh(arg)
-//    }
-//
-//    override fun atanh(arg: Buffer<Double>): DoubleBuffer {
-//        require(arg.size == size) { "The buffer size ${arg.size} does not match context size $size" }
-//        return DoubleBufferOperations.atanh(arg)
-//    }
-//
-//    override fun power(arg: Buffer<Double>, pow: Number): DoubleBuffer {
-//        require(arg.size == size) { "The buffer size ${arg.size} does not match context size $size" }
-//        return DoubleBufferOperations.power(arg, pow)
-//    }
-//
-//    override fun exp(arg: Buffer<Double>): DoubleBuffer {
-//        require(arg.size == size) { "The buffer size ${arg.size} does not match context size $size" }
-//        return DoubleBufferOperations.exp(arg)
-//    }
-//
-//    override fun ln(arg: Buffer<Double>): DoubleBuffer {
-//        require(arg.size == size) { "The buffer size ${arg.size} does not match context size $size" }
-//        return DoubleBufferOperations.ln(arg)
-//    }
-
 }

kmath-core/src/commonMain/kotlin/space/kscience/kmath/operations/DoubleBufferOps.kt

@@ -6,10 +6,8 @@
 package space.kscience.kmath.operations
 
 import space.kscience.kmath.linear.Point
-import space.kscience.kmath.structures.Buffer
+import space.kscience.kmath.misc.UnstableKMathAPI
-import space.kscience.kmath.structures.DoubleBuffer
+import space.kscience.kmath.structures.*
-import space.kscience.kmath.structures.MutableBufferFactory
-import space.kscience.kmath.structures.asBuffer
 import kotlin.math.*
 
 /**
@@ -19,10 +17,29 @@ public abstract class DoubleBufferOps : BufferAlgebra<Double, DoubleField>, Exte
     Norm<Buffer<Double>, Double> {
 
     override val elementAlgebra: DoubleField get() = DoubleField
+
     override val elementBufferFactory: MutableBufferFactory<Double> get() = elementAlgebra.bufferFactory
 
-    override fun Buffer<Double>.map(block: DoubleField.(Double) -> Double): DoubleBuffer =
+    @Suppress("OVERRIDE_BY_INLINE")
-        mapInline { DoubleField.block(it) }
+    @OptIn(UnstableKMathAPI::class)
+    final override inline fun Buffer<Double>.map(block: DoubleField.(Double) -> Double): DoubleBuffer =
+        DoubleArray(size) { DoubleField.block(getDouble(it)) }.asBuffer()
+
+
+    @OptIn(UnstableKMathAPI::class)
+    @Suppress("OVERRIDE_BY_INLINE")
+    final override inline fun Buffer<Double>.mapIndexed(block: DoubleField.(index: Int, arg: Double) -> Double): DoubleBuffer =
+        DoubleBuffer(size) { DoubleField.block(it, getDouble(it)) }
+
+    @OptIn(UnstableKMathAPI::class)
+    @Suppress("OVERRIDE_BY_INLINE")
+    final override inline fun Buffer<Double>.zip(
+        other: Buffer<Double>,
+        block: DoubleField.(left: Double, right: Double) -> Double,
+    ): DoubleBuffer {
+        require(size == other.size) { "Incompatible buffer sizes. left: ${size}, right: ${other.size}" }
+        return DoubleBuffer(size) { DoubleField.block(getDouble(it), other.getDouble(it)) }
+    }
 
     override fun unaryOperationFunction(operation: String): (arg: Buffer<Double>) -> Buffer<Double> =
         super<ExtendedFieldOps>.unaryOperationFunction(operation)
@@ -30,7 +47,7 @@ public abstract class DoubleBufferOps : BufferAlgebra<Double, DoubleField>, Exte
     override fun binaryOperationFunction(operation: String): (left: Buffer<Double>, right: Buffer<Double>) -> Buffer<Double> =
         super<ExtendedFieldOps>.binaryOperationFunction(operation)
 
-    override fun Buffer<Double>.unaryMinus(): DoubleBuffer = mapInline { -it }
+    override fun Buffer<Double>.unaryMinus(): DoubleBuffer = map { -it }
 
     override fun add(left: Buffer<Double>, right: Buffer<Double>): DoubleBuffer {
         require(right.size == left.size) {
@@ -77,6 +94,7 @@ public abstract class DoubleBufferOps : BufferAlgebra<Double, DoubleField>, Exte
 //        } else RealBuffer(DoubleArray(a.size) { a[it] / kValue })
 //    }
 
+    @UnstableKMathAPI
     override fun multiply(left: Buffer<Double>, right: Buffer<Double>): DoubleBuffer {
         require(right.size == left.size) {
             "The size of the first buffer ${left.size} should be the same as for second one: ${right.size} "
@@ -101,55 +119,83 @@ public abstract class DoubleBufferOps : BufferAlgebra<Double, DoubleField>, Exte
         } else DoubleBuffer(DoubleArray(left.size) { left[it] / right[it] })
     }
 
-    override fun sin(arg: Buffer<Double>): DoubleBuffer = arg.mapInline(::sin)
+    override fun sin(arg: Buffer<Double>): DoubleBuffer = arg.map { sin(it) }
 
-    override fun cos(arg: Buffer<Double>): DoubleBuffer = arg.mapInline(::cos)
+    override fun cos(arg: Buffer<Double>): DoubleBuffer = arg.map { cos(it) }
 
-    override fun tan(arg: Buffer<Double>): DoubleBuffer = arg.mapInline(::tan)
+    override fun tan(arg: Buffer<Double>): DoubleBuffer = arg.map { tan(it) }
 
-    override fun asin(arg: Buffer<Double>): DoubleBuffer = arg.mapInline(::asin)
+    override fun asin(arg: Buffer<Double>): DoubleBuffer = arg.map { asin(it) }
 
-    override fun acos(arg: Buffer<Double>): DoubleBuffer = arg.mapInline(::acos)
+    override fun acos(arg: Buffer<Double>): DoubleBuffer = arg.map { acos(it) }
 
-    override fun atan(arg: Buffer<Double>): DoubleBuffer = arg.mapInline(::atan)
+    override fun atan(arg: Buffer<Double>): DoubleBuffer = arg.map { atan(it) }
 
-    override fun sinh(arg: Buffer<Double>): DoubleBuffer = arg.mapInline(::sinh)
+    override fun sinh(arg: Buffer<Double>): DoubleBuffer = arg.map { sinh(it) }
 
-    override fun cosh(arg: Buffer<Double>): DoubleBuffer = arg.mapInline(::cosh)
+    override fun cosh(arg: Buffer<Double>): DoubleBuffer = arg.map { cosh(it) }
 
-    override fun tanh(arg: Buffer<Double>): DoubleBuffer = arg.mapInline(::tanh)
+    override fun tanh(arg: Buffer<Double>): DoubleBuffer = arg.map { tanh(it) }
 
-    override fun asinh(arg: Buffer<Double>): DoubleBuffer = arg.mapInline(::asinh)
+    override fun asinh(arg: Buffer<Double>): DoubleBuffer = arg.map { asinh(it) }
 
-    override fun acosh(arg: Buffer<Double>): DoubleBuffer = arg.mapInline(::acosh)
+    override fun acosh(arg: Buffer<Double>): DoubleBuffer = arg.map { acosh(it) }
 
-    override fun atanh(arg: Buffer<Double>): DoubleBuffer = arg.mapInline(::atanh)
+    override fun atanh(arg: Buffer<Double>): DoubleBuffer = arg.map { atanh(it) }
 
-    override fun exp(arg: Buffer<Double>): DoubleBuffer = arg.mapInline(::exp)
+    override fun exp(arg: Buffer<Double>): DoubleBuffer = arg.map { exp(it) }
 
-    override fun ln(arg: Buffer<Double>): DoubleBuffer = arg.mapInline(::ln)
+    override fun ln(arg: Buffer<Double>): DoubleBuffer = arg.map { ln(it) }
 
     override fun norm(arg: Buffer<Double>): Double = DoubleL2Norm.norm(arg)
 
-    override fun scale(a: Buffer<Double>, value: Double): DoubleBuffer = a.mapInline { it * value }
+    override fun scale(a: Buffer<Double>, value: Double): DoubleBuffer = a.map { it * value }
 
     override fun power(arg: Buffer<Double>, pow: Number): Buffer<Double> = if (pow is Int) {
-        arg.mapInline { it.pow(pow) }
+        arg.map { it.pow(pow) }
     } else {
-        arg.mapInline { it.pow(pow.toDouble()) }
+        arg.map { it.pow(pow.toDouble()) }
     }
 
-    public companion object : DoubleBufferOps() {
+    public companion object : DoubleBufferOps()
-        public inline fun Buffer<Double>.mapInline(block: (Double) -> Double): DoubleBuffer =
-            if (this is DoubleBuffer) {
-                DoubleArray(size) { block(array[it]) }.asBuffer()
-            } else {
-                DoubleArray(size) { block(get(it)) }.asBuffer()
-            }
-    }
 }
 
 public object DoubleL2Norm : Norm<Point<Double>, Double> {
     override fun norm(arg: Point<Double>): Double = sqrt(arg.fold(0.0) { acc: Double, d: Double -> acc + d.pow(2) })
 }
 
+public fun DoubleBufferOps.sum(buffer: Buffer<Double>): Double = buffer.reduce(Double::plus)
+
+/**
+ * Sum of elements using given [conversion]
+ */
+public inline fun <T> DoubleBufferOps.sumOf(buffer: Buffer<T>, conversion: (T) -> Double): Double =
+    buffer.fold(0.0) { acc, value -> acc + conversion(value) }
+
+public fun DoubleBufferOps.average(buffer: Buffer<Double>): Double = sum(buffer) / buffer.size
+
+/**
+ * Average of elements using given [conversion]
+ */
+public inline fun <T> DoubleBufferOps.averageOf(buffer: Buffer<T>, conversion: (T) -> Double): Double =
+    sumOf(buffer, conversion) / buffer.size
+
+public fun DoubleBufferOps.dispersion(buffer: Buffer<Double>): Double {
+    val av = average(buffer)
+    return buffer.fold(0.0) { acc, value -> acc + (value - av).pow(2) } / buffer.size
+}
+
+public fun DoubleBufferOps.std(buffer: Buffer<Double>): Double = sqrt(dispersion(buffer))
+
+public fun DoubleBufferOps.covariance(x: Buffer<Double>, y: Buffer<Double>): Double {
+    require(x.size == y.size) { "Expected buffers of the same size, but x.size == ${x.size} and y.size == ${y.size}" }
+    val xMean = average(x)
+    val yMean = average(y)
+    var sum = 0.0
+    x.indices.forEach {
+        sum += (x[it] - xMean) * (y[it] - yMean)
+    }
+    return sum / (x.size - 1)
+}
+
+

kmath-core/src/commonMain/kotlin/space/kscience/kmath/operations/algebraExtensions.kt

@@ -5,6 +5,21 @@
 
 package space.kscience.kmath.operations
 
+import space.kscience.kmath.misc.PerformancePitfall
+import space.kscience.kmath.structures.Buffer
+
+/**
+ * Returns the sum of all elements in the iterable in this [Group].
+ *
+ * @receiver the algebra that provides addition.
+ * @param data the iterable to sum up.
+ * @return the sum.
+ */
+@PerformancePitfall("Potential boxing access to buffer elements")
+public fun <T> Group<T>.sum(data: Buffer<T>): T = data.fold(zero) { left, right ->
+    add(left, right)
+}
+
 /**
  * Returns the sum of all elements in the iterable in this [Group].
  *
@@ -29,6 +44,18 @@ public fun <T> Group<T>.sum(data: Sequence<T>): T = data.fold(zero) { left, righ
     add(left, right)
 }
 
+/**
+ * Returns an average value of elements in the iterable in this [Group].
+ *
+ * @receiver the algebra that provides addition and division.
+ * @param data the iterable to find average.
+ * @return the average value.
+ * @author Iaroslav Postovalov
+ */
+@PerformancePitfall("Potential boxing access to buffer elements")
+public fun <T, S> S.average(data: Buffer<T>): T where S : Group<T>, S : ScaleOperations<T> =
+    sum(data) / data.size
+
 /**
  * Returns an average value of elements in the iterable in this [Group].
  *
@@ -65,6 +92,17 @@ public fun <T : Comparable<T>> Group<T>.abs(value: T): T = if (value > zero) val
  */
 public fun <T> Iterable<T>.sumWith(group: Group<T>): T = group.sum(this)
 
+/**
+ * Sum extracted elements of [Iterable] with given [group]
+ *
+ * @receiver the collection to sum up.
+ * @param group tha algebra that provides addition
+ * @param extractor the (inline) lambda function to extract value
+ */
+public inline fun <T, R> Iterable<T>.sumWithGroupOf(group: Group<R>, extractor: (T) -> R): R = this.fold(group.zero) { left: R, right: T ->
+    group.add(left, extractor(right))
+}
+
 /**
  * Returns the sum of all elements in the sequence in provided space.
  *
@@ -95,4 +133,3 @@ public fun <T, S> Iterable<T>.averageWith(space: S): T where S : Group<T>, S : S
  */
 public fun <T, S> Sequence<T>.averageWith(space: S): T where S : Group<T>, S : ScaleOperations<T> =
     space.average(this)
-

kmath-core/src/commonMain/kotlin/space/kscience/kmath/structures/Buffer.kt

@@ -5,6 +5,7 @@
 
 package space.kscience.kmath.structures
 
+import space.kscience.kmath.operations.WithSize
 import space.kscience.kmath.operations.asSequence
 import kotlin.jvm.JvmInline
 import kotlin.reflect.KClass
@@ -50,11 +51,11 @@ public fun interface MutableBufferFactory<T> : BufferFactory<T> {
  *
  * @param T the type of elements contained in the buffer.
  */
-public interface Buffer<out T> {
+public interface Buffer<out T> : WithSize {
     /**
      * The size of this buffer.
      */
-    public val size: Int
+    override val size: Int
 
     /**
      * Gets element at given index.
@@ -64,7 +65,7 @@ public interface Buffer<out T> {
     /**
      * Iterates over all elements.
      */
-    public operator fun iterator(): Iterator<T>
+    public operator fun iterator(): Iterator<T> = indices.asSequence().map(::get).iterator()
 
     override fun toString(): String
 
@@ -122,7 +123,14 @@ public interface Buffer<out T> {
 /**
  * Returns an [IntRange] of the valid indices for this [Buffer].
  */
-public val Buffer<*>.indices: IntRange get() = 0 until size
+public val <T> Buffer<T>.indices: IntRange get() = 0 until size
+
+public operator fun <T> Buffer<T>.get(index: UInt): T = get(index.toInt())
+
+/**
+ * if index is in range of buffer, return the value. Otherwise, return null.
+ */
+public fun <T> Buffer<T>.getOrNull(index: Int): T? = if (index in indices) get(index) else null
 
 public fun <T> Buffer<T>.first(): T {
     require(size > 0) { "Can't get the first element of empty buffer" }

kmath-core/src/commonMain/kotlin/space/kscience/kmath/structures/BufferAccessor2D.kt

@@ -5,10 +5,7 @@
 
 package space.kscience.kmath.structures
 
-import space.kscience.kmath.nd.DefaultStrides
+import space.kscience.kmath.nd.*
-import space.kscience.kmath.nd.Structure2D
-import space.kscience.kmath.nd.StructureND
-import space.kscience.kmath.nd.as2D
 
 /**
  * A context that allows to operate on a [MutableBuffer] as on 2d array
@@ -31,7 +28,7 @@ internal class BufferAccessor2D<T>(
 
     //TODO optimize wrapper
     fun MutableBuffer<T>.collect(): Structure2D<T> = StructureND.buffered(
-        DefaultStrides(intArrayOf(rowNum, colNum)),
+        ColumnStrides(ShapeND(rowNum, colNum)),
         factory
     ) { (i, j) ->
         get(i, j)

kmath-core/src/commonMain/kotlin/space/kscience/kmath/structures/BufferView.kt

@@ -0,0 +1,192 @@
+package space.kscience.kmath.structures
+
+import space.kscience.kmath.misc.UnstableKMathAPI
+
+/**
+ * A buffer that wraps an original buffer
+ */
+public interface BufferView<T> : Buffer<T> {
+    public val origin: Buffer<T>
+
+    /**
+     * Get the index in [origin] buffer from index in this buffer.
+     * Return -1 if element not present in the original buffer
+     * This method should be used internally to optimize non-boxing access.
+     */
+    @UnstableKMathAPI
+    public fun originIndex(index: Int): Int
+
+    @OptIn(UnstableKMathAPI::class)
+    override fun get(index: Int): T = origin[originIndex(index)]
+}
+
+/**
+ * A zero-copy buffer that "sees" only part of original buffer. Slice can't go beyond original buffer borders.
+ */
+public class BufferSlice<T>(
+    override val origin: Buffer<T>,
+    public val offset: Int = 0,
+    override val size: Int,
+) : BufferView<T> {
+
+    init {
+        require(size > 0) { "Size must be positive" }
+        require(offset + size <= origin.size) {
+            "End of buffer ${offset + size} is beyond the end of origin buffer size ${origin.size}"
+        }
+    }
+
+    override fun get(index: Int): T = if (index >= size) {
+        throw IndexOutOfBoundsException("$index is out of ${0 until size} rage")
+    } else {
+        origin[index + offset]
+    }
+
+    override fun iterator(): Iterator<T> =
+        (offset until (offset + size)).asSequence().map { origin[it] }.iterator()
+
+    @UnstableKMathAPI
+    override fun originIndex(index: Int): Int = if (index >= size) -1 else index - offset
+
+    override fun toString(): String = "$origin[$offset..${offset + size}"
+}
+
+/**
+ * An expanded buffer that could include the whole initial buffer or its part and fills all space beyond it borders with [defaultValue].
+ *
+ * The [offset] parameter shows the shift of expanded buffer start relative to origin start and could be both positive and negative.
+ */
+public class BufferExpanded<T>(
+    override val origin: Buffer<T>,
+    private val defaultValue: T,
+    public val offset: Int = 0,
+    override val size: Int = origin.size,
+) : BufferView<T> {
+
+    init {
+        require(size > 0) { "Size must be positive" }
+    }
+
+    override fun get(index: Int): T = when (index) {
+        !in 0 until size -> throw IndexOutOfBoundsException("Index $index is not in $indices")
+        in -offset until origin.size - offset -> origin[index + offset]
+        else -> defaultValue
+    }
+
+    @UnstableKMathAPI
+    override fun originIndex(index: Int): Int = if (index in -offset until origin.size - offset) index + offset else -1
+
+    override fun toString(): String = "$origin[$offset..${offset + size}]"
+}
+
+/**
+ * Zero-copy select a slice inside the original buffer
+ */
+public fun <T> Buffer<T>.slice(range: IntRange): BufferView<T> = if (this is BufferSlice) {
+    BufferSlice(
+        origin,
+        this.offset + range.first,
+        (range.last - range.first) + 1
+    )
+} else {
+    BufferSlice(
+        this,
+        range.first,
+        (range.last - range.first) + 1
+    )
+}
+
+/**
+ * Resize original buffer to a given range using given [range], filling additional segments with [defaultValue].
+ * Range left border could be negative to designate adding new blank segment to the beginning of the buffer
+ */
+public fun <T> Buffer<T>.expand(
+    range: IntRange,
+    defaultValue: T,
+): BufferView<T> = if (range.first >= 0 && range.last < size) {
+    BufferSlice(
+        this,
+        range.first,
+        (range.last - range.first) + 1
+    )
+} else {
+    BufferExpanded(
+        this,
+        defaultValue,
+        range.first,
+        (range.last - range.first) + 1
+    )
+}
+
+/**
+ * A [BufferView] that overrides indexing of the original buffer
+ */
+public class PermutedBuffer<T>(
+    override val origin: Buffer<T>,
+    private val permutations: IntArray,
+) : BufferView<T> {
+    init {
+        permutations.forEach { index ->
+            if (index !in origin.indices) {
+                throw IndexOutOfBoundsException("Index $index is not in ${origin.indices}")
+            }
+        }
+    }
+
+    override val size: Int get() = permutations.size
+
+    override fun get(index: Int): T = origin[permutations[index]]
+
+    override fun iterator(): Iterator<T> = permutations.asSequence().map { origin[it] }.iterator()
+
+    @UnstableKMathAPI
+    override fun originIndex(index: Int): Int = if (index in permutations.indices) permutations[index] else -1
+
+    override fun toString(): String = Buffer.toString(this)
+}
+
+/**
+ * Created a permuted view of given buffer using provided [indices]
+ */
+public fun <T> Buffer<T>.permute(indices: IntArray): PermutedBuffer<T> =
+    PermutedBuffer(this, indices)
+
+/**
+ * A [BufferView] that overrides indexing of the original buffer
+ */
+public class PermutedMutableBuffer<T>(
+    override val origin: MutableBuffer<T>,
+    private val permutations: IntArray,
+) : BufferView<T>, MutableBuffer<T> {
+    init {
+        permutations.forEach { index ->
+            if (index !in origin.indices) {
+                throw IndexOutOfBoundsException("Index $index is not in ${origin.indices}")
+            }
+        }
+    }
+
+    override val size: Int get() = permutations.size
+
+    override fun get(index: Int): T = origin[permutations[index]]
+
+    override fun set(index: Int, value: T) {
+        origin[permutations[index]] = value
+    }
+
+    override fun copy(): MutableBuffer<T> = PermutedMutableBuffer(origin.copy(), permutations)
+    //TODO Probably could be optimized
+
+    override fun iterator(): Iterator<T> = permutations.asSequence().map { origin[it] }.iterator()
+
+    @UnstableKMathAPI
+    override fun originIndex(index: Int): Int = if (index in permutations.indices) permutations[index] else -1
+
+    override fun toString(): String = Buffer.toString(this)
+}
+
+/**
+ * Created a permuted mutable view of given buffer using provided [indices]
+ */
+public fun <T> MutableBuffer<T>.permute(indices: IntArray): PermutedMutableBuffer<T> =
+    PermutedMutableBuffer(this, indices)

kmath-core/src/commonMain/kotlin/space/kscience/kmath/structures/DoubleBuffer.kt

@@ -5,6 +5,7 @@
 
 package space.kscience.kmath.structures
 
+import space.kscience.kmath.operations.BufferTransform
 import kotlin.jvm.JvmInline
 
 /**
@@ -13,7 +14,7 @@ import kotlin.jvm.JvmInline
  * @property array the underlying array.
  */
 @JvmInline
-public value class DoubleBuffer(public val array: DoubleArray) : MutableBuffer<Double> {
+public value class DoubleBuffer(public val array: DoubleArray) : PrimitiveBuffer<Double> {
     override val size: Int get() = array.size
 
     override operator fun get(index: Int): Double = array[index]
@@ -40,7 +41,8 @@ public value class DoubleBuffer(public val array: DoubleArray) : MutableBuffer<D
  * The function [init] is called for each array element sequentially starting from the first one.
  * It should return the value for a buffer element given its index.
  */
-public inline fun DoubleBuffer(size: Int, init: (Int) -> Double): DoubleBuffer = DoubleBuffer(DoubleArray(size) { init(it) })
+public inline fun DoubleBuffer(size: Int, init: (Int) -> Double): DoubleBuffer =
+    DoubleBuffer(DoubleArray(size) { init(it) })
 
 /**
  * Returns a new [DoubleBuffer] of given elements.
@@ -51,10 +53,18 @@ public fun DoubleBuffer(vararg doubles: Double): DoubleBuffer = DoubleBuffer(dou
  * Returns a new [DoubleArray] containing all the elements of this [Buffer].
  */
 public fun Buffer<Double>.toDoubleArray(): DoubleArray = when (this) {
-    is DoubleBuffer -> array.copyOf()
+    is DoubleBuffer -> array
     else -> DoubleArray(size, ::get)
 }
 
+/**
+ * Represent this buffer as [DoubleBuffer]. Does not guarantee that changes in the original buffer are reflected on this buffer.
+ */
+public fun Buffer<Double>.toDoubleBuffer(): DoubleBuffer = when (this) {
+    is DoubleBuffer -> this
+    else -> DoubleArray(size, ::get).asBuffer()
+}
+
 /**
  * Returns [DoubleBuffer] over this array.
  *
@@ -62,3 +72,10 @@ public fun Buffer<Double>.toDoubleArray(): DoubleArray = when (this) {
  * @return the new buffer.
  */
 public fun DoubleArray.asBuffer(): DoubleBuffer = DoubleBuffer(this)
+
+
+public fun interface DoubleBufferTransform : BufferTransform<Double, Double> {
+    public fun transform(arg: DoubleBuffer): DoubleBuffer
+
+    override fun transform(arg: Buffer<Double>): DoubleBuffer = arg.toDoubleBuffer()
+}

kmath-core/src/commonMain/kotlin/space/kscience/kmath/structures/FloatBuffer.kt

@@ -14,7 +14,7 @@ import kotlin.jvm.JvmInline
  * @author Iaroslav Postovalov
  */
 @JvmInline
-public value class FloatBuffer(public val array: FloatArray) : MutableBuffer<Float> {
+public value class FloatBuffer(public val array: FloatArray) : PrimitiveBuffer<Float> {
     override val size: Int get() = array.size
 
     override operator fun get(index: Int): Float = array[index]

kmath-core/src/commonMain/kotlin/space/kscience/kmath/structures/IntBuffer.kt

@@ -13,7 +13,7 @@ import kotlin.jvm.JvmInline
  * @property array the underlying array.
  */
 @JvmInline
-public value class IntBuffer(public val array: IntArray) : MutableBuffer<Int> {
+public value class IntBuffer(public val array: IntArray) : PrimitiveBuffer<Int> {
     override val size: Int get() = array.size
 
     override operator fun get(index: Int): Int = array[index]

kmath-core/src/commonMain/kotlin/space/kscience/kmath/structures/LongBuffer.kt

@@ -13,7 +13,7 @@ import kotlin.jvm.JvmInline
  * @property array the underlying array.
  */
 @JvmInline
-public value class LongBuffer(public val array: LongArray) : MutableBuffer<Long> {
+public value class LongBuffer(public val array: LongArray) : PrimitiveBuffer<Long> {
     override val size: Int get() = array.size
 
     override operator fun get(index: Int): Long = array[index]

kmath-core/src/commonMain/kotlin/space/kscience/kmath/structures/MutableBuffer.kt

@@ -95,3 +95,6 @@ public interface MutableBuffer<T> : Buffer<T> {
             auto(T::class, size, initializer)
     }
 }
+
+
+public sealed interface PrimitiveBuffer<T>: MutableBuffer<T>

kmath-core/src/commonMain/kotlin/space/kscience/kmath/structures/bufferExtensions.kt

@@ -11,12 +11,16 @@ import space.kscience.kmath.structures.*
 /**
  * Type alias for buffer transformations.
  */
-public typealias BufferTransform<T, R> = (Buffer<T>) -> Buffer<R>
+public fun interface BufferTransform<T, R> {
+    public fun transform(arg: Buffer<T>): Buffer<R>
+}
 
-/**
+///**
- * Typealias for buffer transformations with suspend function.
+// * Type alias for buffer transformations with suspend function.
- */
+// */
-public typealias SuspendBufferTransform<T, R> = suspend (Buffer<T>) -> Buffer<R>
+//public fun interface SuspendBufferTransform<T, R>{
+//    public suspend fun transform(arg: Buffer<T>): Buffer<R>
+//}
 
 
 /**
@@ -57,18 +61,18 @@ public fun <T> Buffer<T>.toMutableList(): MutableList<T> = when (this) {
  */
 @UnstableKMathAPI
 public inline fun <reified T> Buffer<T>.toTypedArray(): Array<T> = Array(size, ::get)
-
+//
-/**
+///**
- * Create a new buffer from this one with the given mapping function and using [Buffer.Companion.auto] buffer factory.
+// * Create a new buffer from this one with the given mapping function and using [Buffer.Companion.auto] buffer factory.
- */
+// */
-public inline fun <T, reified R : Any> Buffer<T>.map(block: (T) -> R): Buffer<R> =
+//public inline fun <T, reified R : Any> Buffer<T>.map(block: (T) -> R): Buffer<R> =
-    Buffer.auto(size) { block(get(it)) }
+//    Buffer.auto(size) { block(get(it)) }
 
 /**
  * Create a new buffer from this one with the given mapping function.
  * Provided [bufferFactory] is used to construct the new buffer.
  */
-public inline fun <T, R> Buffer<T>.map(
+public inline fun <T, R> Buffer<T>.mapToBuffer(
     bufferFactory: BufferFactory<R>,
     crossinline block: (T) -> R,
 ): Buffer<R> = bufferFactory(size) { block(get(it)) }
@@ -77,23 +81,24 @@ public inline fun <T, R> Buffer<T>.map(
  * Create a new buffer from this one with the given mapping (indexed) function.
  * Provided [bufferFactory] is used to construct the new buffer.
  */
-public inline fun <T, R> Buffer<T>.mapIndexed(
+public inline fun <T, R> Buffer<T>.mapIndexedToBuffer(
     bufferFactory: BufferFactory<R>,
     crossinline block: (index: Int, value: T) -> R,
 ): Buffer<R> = bufferFactory(size) { block(it, get(it)) }
-
+//
-/**
+///**
- * Create a new buffer from this one with the given indexed mapping function.
+// * Create a new buffer from this one with the given indexed mapping function.
- * Provided [BufferFactory] is used to construct the new buffer.
+// * Provided [BufferFactory] is used to construct the new buffer.
- */
+// */
-public inline fun <T, reified R : Any> Buffer<T>.mapIndexed(
+//public inline fun <T, reified R : Any> Buffer<T>.mapIndexed(
-    crossinline block: (index: Int, value: T) -> R,
+//    crossinline block: (index: Int, value: T) -> R,
-): Buffer<R> = Buffer.auto(size) { block(it, get(it)) }
+//): Buffer<R> = Buffer.auto(size) { block(it, get(it)) }
 
 /**
  * Fold given buffer according to [operation]
  */
 public inline fun <T, R> Buffer<T>.fold(initial: R, operation: (acc: R, T) -> R): R {
+    if (size == 0) return initial
     var accumulator = initial
     for (index in this.indices) accumulator = operation(accumulator, get(index))
     return accumulator
@@ -103,18 +108,31 @@ public inline fun <T, R> Buffer<T>.fold(initial: R, operation: (acc: R, T) -> R)
  * Fold given buffer according to indexed [operation]
  */
 public inline fun <T : Any, R> Buffer<T>.foldIndexed(initial: R, operation: (index: Int, acc: R, T) -> R): R {
+    if (size == 0) return initial
     var accumulator = initial
     for (index in this.indices) accumulator = operation(index, accumulator, get(index))
     return accumulator
 }
 
+/**
+ * Reduce a buffer from left to right according to [operation]
+ */
+public inline fun <T> Buffer<T>.reduce(operation: (left: T, value: T) -> T): T {
+    require(size > 0) { "Buffer must have elements" }
+    var current = get(0)
+    for (i in 1 until size) {
+        current = operation(current, get(i))
+    }
+    return current
+}
+
 /**
  * Zip two buffers using given [transform].
  */
 @UnstableKMathAPI
-public inline fun <T1, T2 : Any, reified R : Any> Buffer<T1>.zip(
+public inline fun <T1, T2, R> Buffer<T1>.combineToBuffer(
     other: Buffer<T2>,
-    bufferFactory: BufferFactory<R> = BufferFactory.auto(),
+    bufferFactory: BufferFactory<R>,
     crossinline transform: (T1, T2) -> R,
 ): Buffer<R> {
     require(size == other.size) { "Buffer size mismatch in zip: expected $size but found ${other.size}" }

kmath-core/src/commonMain/kotlin/space/kscience/kmath/structures/bufferPrimitiveAccess.kt

@@ -0,0 +1,38 @@
+package space.kscience.kmath.structures
+
+import space.kscience.kmath.misc.UnstableKMathAPI
+
+/**
+ * Non-boxing access to primitive [Double]
+ */
+
+@UnstableKMathAPI
+public fun Buffer<Double>.getDouble(index: Int): Double = if (this is BufferView) {
+    val originIndex = originIndex(index)
+    if (originIndex >= 0) {
+        origin.getDouble(originIndex)
+    } else {
+        get(index)
+    }
+} else if (this is DoubleBuffer) {
+    array[index]
+} else {
+    get(index)
+}
+
+/**
+ * Non-boxing access to primitive [Int]
+ */
+@UnstableKMathAPI
+public fun Buffer<Int>.getInt(index: Int): Int = if (this is BufferView) {
+    val originIndex = originIndex(index)
+    if (originIndex >= 0) {
+        origin.getInt(originIndex)
+    } else {
+        get(index)
+    }
+} else if (this is IntBuffer) {
+    array[index]
+} else {
+    get(index)
+}

kmath-core/src/commonMain/kotlin/space/kscience/kmath/structures/types.kt

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2018-2022 KMath contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package space.kscience.kmath.structures
+
+
+public typealias Float32 = Float
+public typealias Float64 = Double
+
+public typealias Int8 = Byte
+public typealias Int16 = Short
+public typealias Int32 = Int
+public typealias Int64 = Long
+
+public typealias UInt8 = UByte
+public typealias UInt16 = UShort
+public typealias UInt32 = UInt
+public typealias UInt64 = ULong

kmath-core/src/commonTest/kotlin/space/kscience/kmath/expressions/DSTest.kt

@@ -9,7 +9,6 @@ package space.kscience.kmath.expressions
 
 import space.kscience.kmath.misc.UnstableKMathAPI
 import space.kscience.kmath.operations.DoubleField
-import space.kscience.kmath.structures.DoubleBuffer
 import kotlin.contracts.InvocationKind
 import kotlin.contracts.contract
 import kotlin.test.Test
@@ -22,7 +21,7 @@ internal inline fun diff(
     block: DSField<Double, DoubleField>.() -> Unit,
 ) {
     contract { callsInPlace(block, InvocationKind.EXACTLY_ONCE) }
-    DSField(DoubleField, order, mapOf(*parameters), ::DoubleBuffer).block()
+    DSField(DoubleField, order, mapOf(*parameters)).block()
 }
 
 internal class DSTest {
@@ -45,7 +44,7 @@ internal class DSTest {
 
     @Test
     fun dsExpressionTest() {
-        val f = DSFieldExpression(DoubleField, ::DoubleBuffer) {
+        val f = DSFieldExpression(DoubleField) {
             val x by binding
             val y by binding
             x.pow(2) + 2 * x * y + y.pow(2) + 1

kmath-core/src/commonTest/kotlin/space/kscience/kmath/nd/StridesTest.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2018-2022 KMath contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package space.kscience.kmath.nd
+
+import kotlin.test.Test
+
+class StridesTest {
+    @Test
+    fun checkRowBasedStrides() {
+        val strides = RowStrides(ShapeND(3, 3))
+        var counter = 0
+        for(i in 0..2){
+            for(j in 0..2){
+//                print(strides.offset(intArrayOf(i,j)).toString() + "\t")
+                require(strides.offset(intArrayOf(i,j)) == counter)
+                counter++
+            }
+            println()
+        }
+    }
+
+    @Test
+    fun checkColumnBasedStrides() {
+        val strides = ColumnStrides(ShapeND(3, 3))
+        var counter = 0
+        for(i in 0..2){
+            for(j in 0..2){
+//                print(strides.offset(intArrayOf(i,j)).toString() + "\t")
+                require(strides.offset(intArrayOf(j,i)) == counter)
+                counter++
+            }
+            println()
+        }
+    }
+}

kmath-core/src/commonTest/kotlin/space/kscience/kmath/structures/BufferExpandedTest.kt

@@ -0,0 +1,27 @@
+package space.kscience.kmath.structures
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertFails
+
+internal class BufferExpandedTest {
+    private val buffer = (0..100).toList().asBuffer()
+
+    @Test
+    fun shrink(){
+        val view = buffer.slice(20..30)
+        assertEquals(20, view[0])
+        assertEquals(30, view[10])
+        assertFails { view[11] }
+    }
+
+    @Test
+    fun expandNegative(){
+        val view: BufferView<Int> = buffer.expand(-20..113,0)
+        assertEquals(0,view[4])
+        assertEquals(0,view[123])
+        assertEquals(100, view[120])
+        assertFails { view[-2] }
+        assertFails { view[134] }
+    }
+}

kmath-core/src/commonTest/kotlin/space/kscience/kmath/structures/NumberNDFieldTest.kt

@@ -88,7 +88,7 @@ class NumberNDFieldTest {
     @Test
     fun testInternalContext() {
         algebra {
-            (DoubleField.ndAlgebra(*array1.shape)) { with(L2Norm) { 1 + norm(array1) + exp(array2) } }
+            (DoubleField.ndAlgebra(array1.shape)) { with(L2Norm) { 1 + norm(array1) + exp(array2) } }
         }
     }
 }

kmath-core/src/jvmTest/kotlin/space/kscience/kmath/misc/JBigTest.kt

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2018-2022 KMath contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package space.kscience.kmath.misc
+
+import org.junit.jupiter.api.Test
+import space.kscience.kmath.operations.JBigDecimalField
+import kotlin.test.assertEquals
+import kotlin.test.assertNotEquals
+
+class JBigTest {
+
+    @Test
+    fun testExact() = with(JBigDecimalField) {
+        assertNotEquals(0.3, 0.1 + 0.2)
+        assertEquals(one * 0.3, one * 0.1 + one * 0.2)
+    }
+}

kmath-coroutines/build.gradle.kts

@@ -3,24 +3,13 @@ plugins {
 }
 
 kscience {
+    jvm()
+    js()
     native()
-}
-
-kotlin.sourceSets {
-    all {
-        with(languageSettings) {
-            optIn("kotlinx.coroutines.InternalCoroutinesApi")
-            optIn("kotlinx.coroutines.ExperimentalCoroutinesApi")
-            optIn("kotlinx.coroutines.FlowPreview")
-        }
-    }
-
-    commonMain {
     dependencies {
         api(project(":kmath-core"))
         api(project(":kmath-complex"))
-            api("org.jetbrains.kotlinx:kotlinx-coroutines-core:${space.kscience.gradle.KScienceVersions.coroutinesVersion}")
+        api(spclibs.kotlinx.coroutines.core)
-        }
     }
 }