From 10f84bd630ae6654fb03898e1fd2d50300976db7 Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Wed, 3 May 2023 21:14:29 +0300 Subject: [PATCH 01/26] added function solve --- .../kmath/tensors/api/LinearOpsTensorAlgebra.kt | 14 ++++++++++++++ .../kmath/tensors/core/DoubleTensorAlgebra.kt | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt index faff2eb80..60865ecba 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt @@ -5,8 +5,15 @@ package space.kscience.kmath.tensors.api +import space.kscience.kmath.nd.MutableStructure2D import space.kscience.kmath.nd.StructureND +import space.kscience.kmath.nd.as2D import space.kscience.kmath.operations.Field +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.dot +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.map +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.transposed +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra /** * Common linear algebra operations. Operates on [Tensor]. @@ -103,4 +110,11 @@ public interface LinearOpsTensorAlgebra> : TensorPartialDivision */ public fun symEig(structureND: StructureND): Pair, StructureND> + /** Returns the solution to the equation Ax = B for the square matrix A as `input1` and + * for the square matrix B as `input2`. + * + * @receiver the `input1` and the `input2`. + * @return the square matrix x which is the solution of the equation. + */ + public fun solve(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D } diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt index 70a3ef7e2..1571eb7f1 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt @@ -711,6 +711,12 @@ public open class DoubleTensorAlgebra : override fun symEig(structureND: StructureND): Pair = symEigJacobi(structureND = structureND, maxIteration = 50, epsilon = 1e-15) + override fun solve(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D { + val aSvd = DoubleTensorAlgebra.svd(a) + val s = BroadcastDoubleTensorAlgebra.diagonalEmbedding(aSvd.second.map {1.0 / it}) + val aInverse = aSvd.third.dot(s).dot(aSvd.first.transposed()) + return aInverse.dot(b).as2D() + } } public val Double.Companion.tensorAlgebra: DoubleTensorAlgebra get() = DoubleTensorAlgebra From 19c1af18743b899e566c254a121cd4a961d0ca59 Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Wed, 3 May 2023 21:25:30 +0300 Subject: [PATCH 02/26] added helper functions for levenberg-marquardt algorithm --- .../kmath/tensors/core/internal/linUtils.kt | 228 ++++++++++++++++++ 1 file changed, 228 insertions(+) diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/internal/linUtils.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/internal/linUtils.kt index cf3697e76..c559803ec 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/internal/linUtils.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/internal/linUtils.kt @@ -12,7 +12,14 @@ import space.kscience.kmath.structures.IntBuffer import space.kscience.kmath.structures.asBuffer import space.kscience.kmath.structures.indices import space.kscience.kmath.tensors.core.* +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.div +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.dot +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.minus +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.times +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.transposed +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.plus import kotlin.math.abs +import kotlin.math.max import kotlin.math.min import kotlin.math.sqrt @@ -308,3 +315,224 @@ internal fun DoubleTensorAlgebra.svdHelper( matrixV.source[i] = vBuffer[i] } } + +data class LMSettings ( + var iteration:Int, + var func_calls: Int, + var example_number:Int +) + +/* matrix -> column of all elemnets */ +fun make_column(tensor: MutableStructure2D) : MutableStructure2D { + val shape = intArrayOf(tensor.shape.component1() * tensor.shape.component2(), 1) + var buffer = DoubleArray(tensor.shape.component1() * tensor.shape.component2()) + for (i in 0 until tensor.shape.component1()) { + for (j in 0 until tensor.shape.component2()) { + buffer[i * tensor.shape.component2() + j] = tensor[i, j] + } + } + var column = BroadcastDoubleTensorAlgebra.fromArray(ShapeND(shape), buffer).as2D() + return column +} + +/* column length */ +fun length(column: MutableStructure2D) : Int { + return column.shape.component1() +} + +fun MutableStructure2D.abs() { + for (i in 0 until this.shape.component1()) { + for (j in 0 until this.shape.component2()) { + this[i, j] = abs(this[i, j]) + } + } +} + +fun abs(input: MutableStructure2D): MutableStructure2D { + val tensor = BroadcastDoubleTensorAlgebra.ones( + ShapeND( + intArrayOf( + input.shape.component1(), + input.shape.component2() + ) + ) + ).as2D() + for (i in 0 until tensor.shape.component1()) { + for (j in 0 until tensor.shape.component2()) { + tensor[i, j] = abs(input[i, j]) + } + } + return tensor +} + +fun diag(input: MutableStructure2D): MutableStructure2D { + val tensor = BroadcastDoubleTensorAlgebra.ones(ShapeND(intArrayOf(input.shape.component1(), 1))).as2D() + for (i in 0 until tensor.shape.component1()) { + tensor[i, 0] = input[i, i] + } + return tensor +} + +fun make_matrx_with_diagonal(column: MutableStructure2D): MutableStructure2D { + val size = column.shape.component1() + val tensor = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(size, size))).as2D() + for (i in 0 until size) { + tensor[i, i] = column[i, 0] + } + return tensor +} + +fun lm_eye(size: Int): MutableStructure2D { + val column = BroadcastDoubleTensorAlgebra.ones(ShapeND(intArrayOf(size, 1))).as2D() + return make_matrx_with_diagonal(column) +} + +fun largest_element_comparison(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D { + val a_sizeX = a.shape.component1() + val a_sizeY = a.shape.component2() + val b_sizeX = b.shape.component1() + val b_sizeY = b.shape.component2() + val tensor = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(max(a_sizeX, b_sizeX), max(a_sizeY, b_sizeY)))).as2D() + for (i in 0 until tensor.shape.component1()) { + for (j in 0 until tensor.shape.component2()) { + if (i < a_sizeX && i < b_sizeX && j < a_sizeY && j < b_sizeY) { + tensor[i, j] = max(a[i, j], b[i, j]) + } + else if (i < a_sizeX && j < a_sizeY) { + tensor[i, j] = a[i, j] + } + else { + tensor[i, j] = b[i, j] + } + } + } + return tensor +} + +fun smallest_element_comparison(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D { + val a_sizeX = a.shape.component1() + val a_sizeY = a.shape.component2() + val b_sizeX = b.shape.component1() + val b_sizeY = b.shape.component2() + val tensor = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(max(a_sizeX, b_sizeX), max(a_sizeY, b_sizeY)))).as2D() + for (i in 0 until tensor.shape.component1()) { + for (j in 0 until tensor.shape.component2()) { + if (i < a_sizeX && i < b_sizeX && j < a_sizeY && j < b_sizeY) { + tensor[i, j] = min(a[i, j], b[i, j]) + } + else if (i < a_sizeX && j < a_sizeY) { + tensor[i, j] = a[i, j] + } + else { + tensor[i, j] = b[i, j] + } + } + } + return tensor +} + +fun get_zero_indices(column: MutableStructure2D, epsilon: Double = 0.000001): MutableStructure2D? { + var idx = emptyArray() + for (i in 0 until column.shape.component1()) { + if (abs(column[i, 0]) > epsilon) { + idx += (i + 1.0) + } + } + if (idx.size > 0) { + return BroadcastDoubleTensorAlgebra.fromArray(ShapeND(intArrayOf(idx.size, 1)), idx.toDoubleArray()).as2D() + } + return null +} + +fun feval(func: (MutableStructure2D, MutableStructure2D, LMSettings) -> MutableStructure2D, + t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings) + : MutableStructure2D +{ + return func(t, p, settings) +} + +fun lm_matx(func: (MutableStructure2D, MutableStructure2D, LMSettings) -> MutableStructure2D, + t: MutableStructure2D, p_old: MutableStructure2D, y_old: MutableStructure2D, + dX2: Int, J_input: MutableStructure2D, p: MutableStructure2D, + y_dat: MutableStructure2D, weight: MutableStructure2D, dp:MutableStructure2D, settings:LMSettings) : Array> +{ + // default: dp = 0.001 + + val Npnt = length(y_dat) // number of data points + val Npar = length(p) // number of parameters + + val y_hat = feval(func, t, p, settings) // evaluate model using parameters 'p' + settings.func_calls += 1 + + var J = J_input + + if (settings.iteration % (2 * Npar) == 0 || dX2 > 0) { + J = lm_FD_J(func, t, p, y_hat, dp, settings).as2D() // finite difference + } + else { + J = lm_Broyden_J(p_old, y_old, J, p, y_hat).as2D() // rank-1 update + } + + val delta_y = y_dat.minus(y_hat) + + val Chi_sq = delta_y.transposed().dot( delta_y.times(weight) ).as2D() + val JtWJ = J.transposed().dot ( J.times( weight.dot(BroadcastDoubleTensorAlgebra.ones(ShapeND(intArrayOf(1, Npar)))) ) ).as2D() + val JtWdy = J.transposed().dot( weight.times(delta_y) ).as2D() + + return arrayOf(JtWJ,JtWdy,Chi_sq,y_hat,J) +} + +fun lm_Broyden_J(p_old: MutableStructure2D, y_old: MutableStructure2D, J_input: MutableStructure2D, + p: MutableStructure2D, y: MutableStructure2D): MutableStructure2D { + var J = J_input.copyToTensor() + + val h = p.minus(p_old) + val increase = y.minus(y_old).minus( J.dot(h) ).dot(h.transposed()).div( (h.transposed().dot(h)).as2D()[0, 0] ) + J = J.plus(increase) + + return J.as2D() +} + +fun lm_FD_J(func: (MutableStructure2D, MutableStructure2D, settings: LMSettings) -> MutableStructure2D, + t: MutableStructure2D, p: MutableStructure2D, y: MutableStructure2D, + dp: MutableStructure2D, settings: LMSettings): MutableStructure2D { + // default: dp = 0.001 * ones(1,n) + + val m = length(y) // number of data points + val n = length(p) // number of parameters + + val ps = p.copyToTensor().as2D() + val J = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(m, n))).as2D() // initialize Jacobian to Zero + val del = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(n, 1))).as2D() + + for (j in 0 until n) { + + del[j, 0] = dp[j, 0] * (1 + abs(p[j, 0])) // parameter perturbation + p[j, 0] = ps[j, 0] + del[j, 0] // perturb parameter p(j) + + val epsilon = 0.0000001 + if (kotlin.math.abs(del[j, 0]) > epsilon) { + val y1 = feval(func, t, p, settings) + settings.func_calls += 1 + + if (dp[j, 0] < 0) { // backwards difference + for (i in 0 until J.shape.component1()) { + J[i, j] = (y1.as2D().minus(y).as2D())[i, 0] / del[j, 0] + } + } + else { + // Do tests for it + println("Potential mistake") + p[j, 0] = ps[j, 0] - del[j, 0] // central difference, additional func call + for (i in 0 until J.shape.component1()) { + J[i, j] = (y1.as2D().minus(feval(func, t, p, settings)).as2D())[i, 0] / (2 * del[j, 0]) + } + settings.func_calls += 1 + } + } + + p[j, 0] = ps[j, 0] // restore p(j) + } + + return J.as2D() +} From 89a5522144b2052278c252f833ec678628918b06 Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Thu, 4 May 2023 00:44:18 +0300 Subject: [PATCH 03/26] added new svd algorithm (Golub Kahan) and used by default for svd --- .../space/kscience/kmath/ejml/_generated.kt | 2 +- .../kmath/tensors/core/DoubleTensorAlgebra.kt | 2 +- .../kmath/tensors/core/internal/linUtils.kt | 301 +++++++++++++++++- .../kscience/kmath/tensors/core/tensorOps.kt | 30 ++ 4 files changed, 329 insertions(+), 6 deletions(-) diff --git a/kmath-ejml/src/main/kotlin/space/kscience/kmath/ejml/_generated.kt b/kmath-ejml/src/main/kotlin/space/kscience/kmath/ejml/_generated.kt index c56583fa8..8ad7f7293 100644 --- a/kmath-ejml/src/main/kotlin/space/kscience/kmath/ejml/_generated.kt +++ b/kmath-ejml/src/main/kotlin/space/kscience/kmath/ejml/_generated.kt @@ -19,9 +19,9 @@ import org.ejml.sparse.csc.factory.DecompositionFactory_DSCC import org.ejml.sparse.csc.factory.DecompositionFactory_FSCC import org.ejml.sparse.csc.factory.LinearSolverFactory_DSCC import org.ejml.sparse.csc.factory.LinearSolverFactory_FSCC -import space.kscience.kmath.UnstableKMathAPI import space.kscience.kmath.linear.* import space.kscience.kmath.linear.Matrix +import space.kscience.kmath.UnstableKMathAPI import space.kscience.kmath.nd.StructureFeature import space.kscience.kmath.operations.DoubleField import space.kscience.kmath.operations.FloatField diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt index 1571eb7f1..5325fe19e 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt @@ -706,7 +706,7 @@ public open class DoubleTensorAlgebra : override fun svd( structureND: StructureND, ): Triple, StructureND, StructureND> = - svd(structureND = structureND, epsilon = 1e-10) + svdGolubKahan(structureND = structureND, epsilon = 1e-10) override fun symEig(structureND: StructureND): Pair = symEigJacobi(structureND = structureND, maxIteration = 50, epsilon = 1e-15) diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/internal/linUtils.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/internal/linUtils.kt index c559803ec..6e5456c62 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/internal/linUtils.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/internal/linUtils.kt @@ -7,10 +7,7 @@ package space.kscience.kmath.tensors.core.internal import space.kscience.kmath.nd.* import space.kscience.kmath.operations.invoke -import space.kscience.kmath.structures.DoubleBuffer -import space.kscience.kmath.structures.IntBuffer -import space.kscience.kmath.structures.asBuffer -import space.kscience.kmath.structures.indices +import space.kscience.kmath.structures.* import space.kscience.kmath.tensors.core.* import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.div import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.dot @@ -316,6 +313,302 @@ internal fun DoubleTensorAlgebra.svdHelper( } } +private fun pythag(a: Double, b: Double): Double { + val at: Double = abs(a) + val bt: Double = abs(b) + val ct: Double + val result: Double + if (at > bt) { + ct = bt / at + result = at * sqrt(1.0 + ct * ct) + } else if (bt > 0.0) { + ct = at / bt + result = bt * sqrt(1.0 + ct * ct) + } else result = 0.0 + return result +} + +private fun SIGN(a: Double, b: Double): Double { + if (b >= 0.0) + return abs(a) + else + return -abs(a) +} + +internal fun MutableStructure2D.svdGolubKahanHelper(u: MutableStructure2D, w: BufferedTensor, + v: MutableStructure2D, iterations: Int, epsilon: Double) { + val shape = this.shape + val m = shape.component1() + val n = shape.component2() + var f = 0.0 + val rv1 = DoubleArray(n) + var s = 0.0 + var scale = 0.0 + var anorm = 0.0 + var g = 0.0 + var l = 0 + + val wStart = 0 + val wBuffer = w.source + + for (i in 0 until n) { + /* left-hand reduction */ + l = i + 1 + rv1[i] = scale * g + g = 0.0 + s = 0.0 + scale = 0.0 + if (i < m) { + for (k in i until m) { + scale += abs(this[k, i]); + } + if (abs(scale) > epsilon) { + for (k in i until m) { + this[k, i] = (this[k, i] / scale) + s += this[k, i] * this[k, i] + } + f = this[i, i] + if (f >= 0) { + g = (-1) * abs(sqrt(s)) + } else { + g = abs(sqrt(s)) + } + val h = f * g - s + this[i, i] = f - g + if (i != n - 1) { + for (j in l until n) { + s = 0.0 + for (k in i until m) { + s += this[k, i] * this[k, j] + } + f = s / h + for (k in i until m) { + this[k, j] += f * this[k, i] + } + } + } + for (k in i until m) { + this[k, i] = this[k, i] * scale + } + } + } + + wBuffer[wStart + i] = scale * g + /* right-hand reduction */ + g = 0.0 + s = 0.0 + scale = 0.0 + if (i < m && i != n - 1) { + for (k in l until n) { + scale += abs(this[i, k]) + } + if (abs(scale) > epsilon) { + for (k in l until n) { + this[i, k] = this[i, k] / scale + s += this[i, k] * this[i, k] + } + f = this[i, l] + if (f >= 0) { + g = (-1) * abs(sqrt(s)) + } else { + g = abs(sqrt(s)) + } + val h = f * g - s + this[i, l] = f - g + for (k in l until n) { + rv1[k] = this[i, k] / h + } + if (i != m - 1) { + for (j in l until m) { + s = 0.0 + for (k in l until n) { + s += this[j, k] * this[i, k] + } + for (k in l until n) { + this[j, k] += s * rv1[k] + } + } + } + for (k in l until n) { + this[i, k] = this[i, k] * scale + } + } + } + anorm = max(anorm, (abs(wBuffer[wStart + i]) + abs(rv1[i]))); + } + + for (i in n - 1 downTo 0) { + if (i < n - 1) { + if (abs(g) > epsilon) { + for (j in l until n) { + v[j, i] = (this[i, j] / this[i, l]) / g + } + for (j in l until n) { + s = 0.0 + for (k in l until n) + s += this[i, k] * v[k, j] + for (k in l until n) + v[k, j] += s * v[k, i] + } + } + for (j in l until n) { + v[i, j] = 0.0 + v[j, i] = 0.0 + } + } + v[i, i] = 1.0 + g = rv1[i] + l = i + } + + for (i in min(n, m) - 1 downTo 0) { + l = i + 1 + g = wBuffer[wStart + i] + for (j in l until n) { + this[i, j] = 0.0 + } + if (abs(g) > epsilon) { + g = 1.0 / g + for (j in l until n) { + s = 0.0 + for (k in l until m) { + s += this[k, i] * this[k, j] + } + f = (s / this[i, i]) * g + for (k in i until m) { + this[k, j] += f * this[k, i] + } + } + for (j in i until m) { + this[j, i] *= g + } + } else { + for (j in i until m) { + this[j, i] = 0.0 + } + } + this[i, i] += 1.0 + } + + var flag = 0 + var nm = 0 + var c = 0.0 + var h = 0.0 + var y = 0.0 + var z = 0.0 + var x = 0.0 + for (k in n - 1 downTo 0) { + for (its in 1 until iterations) { + flag = 1 + for (newl in k downTo 0) { + nm = newl - 1 + if (abs(rv1[newl]) + anorm == anorm) { + flag = 0 + l = newl + break + } + if (abs(wBuffer[wStart + nm]) + anorm == anorm) { + l = newl + break + } + } + + if (flag != 0) { + c = 0.0 + s = 1.0 + for (i in l until k + 1) { + f = s * rv1[i] + rv1[i] = c * rv1[i] + if (abs(f) + anorm == anorm) { + break + } + g = wBuffer[wStart + i] + h = pythag(f, g) + wBuffer[wStart + i] = h + h = 1.0 / h + c = g * h + s = (-f) * h + for (j in 0 until m) { + y = this[j, nm] + z = this[j, i] + this[j, nm] = y * c + z * s + this[j, i] = z * c - y * s + } + } + } + + z = wBuffer[wStart + k] + if (l == k) { + if (z < 0.0) { + wBuffer[wStart + k] = -z + for (j in 0 until n) + v[j, k] = -v[j, k] + } + break + } + + x = wBuffer[wStart + l] + nm = k - 1 + y = wBuffer[wStart + nm] + g = rv1[nm] + h = rv1[k] + f = ((y - z) * (y + z) + (g - h) * (g + h)) / (2.0 * h * y) + g = pythag(f, 1.0) + f = ((x - z) * (x + z) + h * ((y / (f + SIGN(g, f))) - h)) / x + c = 1.0 + s = 1.0 + + var i = 0 + for (j in l until nm + 1) { + i = j + 1 + g = rv1[i] + y = wBuffer[wStart + i] + h = s * g + g = c * g + z = pythag(f, h) + rv1[j] = z + c = f / z + s = h / z + f = x * c + g * s + g = g * c - x * s + h = y * s + y *= c + + for (jj in 0 until n) { + x = v[jj, j]; + z = v[jj, i]; + v[jj, j] = x * c + z * s; + v[jj, i] = z * c - x * s; + } + z = pythag(f, h) + wBuffer[wStart + j] = z + if (abs(z) > epsilon) { + z = 1.0 / z + c = f * z + s = h * z + } + f = c * g + s * y + x = c * y - s * g + for (jj in 0 until m) { + y = this[jj, j] + z = this[jj, i] + this[jj, j] = y * c + z * s + this[jj, i] = z * c - y * s + } + } + rv1[l] = 0.0 + rv1[k] = f + wBuffer[wStart + k] = x + } + } + + for (i in 0 until n) { + for (j in 0 until m) { + u[j, i] = this[j, i] + } + } +} + data class LMSettings ( var iteration:Int, var func_calls: Int, diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/tensorOps.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/tensorOps.kt index e5dc55f68..88ffb0bfe 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/tensorOps.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/tensorOps.kt @@ -212,6 +212,36 @@ public fun DoubleTensorAlgebra.svd( return Triple(uTensor.transposed(), sTensor, vTensor.transposed()) } +public fun DoubleTensorAlgebra.svdGolubKahan( + structureND: StructureND, + iterations: Int = 30, epsilon: Double = 1e-10 +): Triple { + val size = structureND.dimension + val commonShape = structureND.shape.slice(0 until size - 2) + val (n, m) = structureND.shape.slice(size - 2 until size) + val uTensor = zeros(commonShape + intArrayOf(n, m)) + val sTensor = zeros(commonShape + intArrayOf(m)) + val vTensor = zeros(commonShape + intArrayOf(m, m)) + + val matrices = structureND.asDoubleTensor().matrices + val uTensors = uTensor.matrices + val sTensorVectors = sTensor.vectors + val vTensors = vTensor.matrices + + for (index in matrices.indices) { + val matrix = matrices[index] + val matrixSize = matrix.shape.linearSize + val curMatrix = DoubleTensor( + matrix.shape, + matrix.source.view(0, matrixSize).copy() + ) + curMatrix.as2D().svdGolubKahanHelper(uTensors[index].as2D(), sTensorVectors[index], vTensors[index].as2D(), + iterations, epsilon) + } + + return Triple(uTensor, sTensor, vTensor) +} + /** * Returns eigenvalues and eigenvectors of a real symmetric matrix input or a batch of real symmetric matrices, * represented by a pair `eigenvalues to eigenvectors`. From b526f9a476c3df4d65b945130beebba9f578e76d Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Thu, 4 May 2023 20:05:32 +0300 Subject: [PATCH 04/26] added Levenberg-Marquardt algorithm + test --- .../tensors/api/LinearOpsTensorAlgebra.kt | 8 + .../kmath/tensors/core/DoubleTensorAlgebra.kt | 369 +++++++++++++++++- .../tensors/core/TestDoubleTensorAlgebra.kt | 85 +++- 3 files changed, 455 insertions(+), 7 deletions(-) diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt index 60865ecba..5cf0081fb 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt @@ -14,6 +14,8 @@ import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.dot import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.map import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.transposed import space.kscience.kmath.tensors.core.DoubleTensorAlgebra +import space.kscience.kmath.tensors.core.internal.LMSettings +import kotlin.reflect.KFunction3 /** * Common linear algebra operations. Operates on [Tensor]. @@ -117,4 +119,10 @@ public interface LinearOpsTensorAlgebra> : TensorPartialDivision * @return the square matrix x which is the solution of the equation. */ public fun solve(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D + + public fun lm( + func: KFunction3, MutableStructure2D, LMSettings, MutableStructure2D>, + p_input: MutableStructure2D, t_input: MutableStructure2D, y_dat_input: MutableStructure2D, + weight_input: MutableStructure2D, dp_input: MutableStructure2D, p_min_input: MutableStructure2D, p_max_input: MutableStructure2D, + c_input: MutableStructure2D, opts_input: DoubleArray, nargin: Int, example_number: Int): Double } diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt index 5325fe19e..e7e22afa9 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt @@ -9,6 +9,7 @@ package space.kscience.kmath.tensors.core import space.kscience.kmath.PerformancePitfall +import space.kscience.kmath.linear.transpose import space.kscience.kmath.nd.* import space.kscience.kmath.operations.DoubleBufferOps import space.kscience.kmath.operations.DoubleField @@ -17,10 +18,8 @@ import space.kscience.kmath.tensors.api.AnalyticTensorAlgebra import space.kscience.kmath.tensors.api.LinearOpsTensorAlgebra import space.kscience.kmath.tensors.api.Tensor import space.kscience.kmath.tensors.core.internal.* -import kotlin.math.abs -import kotlin.math.ceil -import kotlin.math.floor -import kotlin.math.sqrt +import kotlin.math.* +import kotlin.reflect.KFunction3 /** * Implementation of basic operations over double tensors and basic algebra operations on them. @@ -717,6 +716,368 @@ public open class DoubleTensorAlgebra : val aInverse = aSvd.third.dot(s).dot(aSvd.first.transposed()) return aInverse.dot(b).as2D() } + + override fun lm( + func: KFunction3, MutableStructure2D, LMSettings, MutableStructure2D>, + p_input: MutableStructure2D, t_input: MutableStructure2D, y_dat_input: MutableStructure2D, + weight_input: MutableStructure2D, dp_input: MutableStructure2D, p_min_input: MutableStructure2D, p_max_input: MutableStructure2D, + c_input: MutableStructure2D, opts_input: DoubleArray, nargin: Int, example_number: Int): Double { + + var result_chi_sq = 0.0 + + val tensor_parameter = 0 + val eps:Double = 2.2204e-16 + + var settings = LMSettings(0, 0, example_number) + settings.func_calls = 0 // running count of function evaluations + + var p = p_input + val y_dat = y_dat_input + val t = t_input + + val Npar = length(p) // number of parameters + val Npnt = length(y_dat) // number of data points + var p_old = zeros(ShapeND(intArrayOf(Npar, 1))).as2D() // previous set of parameters + var y_old = zeros(ShapeND(intArrayOf(Npnt, 1))).as2D() // previous model, y_old = y_hat(t;p_old) + var X2 = 1e-3 / eps // a really big initial Chi-sq value + var X2_old = 1e-3 / eps // a really big initial Chi-sq value + var J = zeros(ShapeND(intArrayOf(Npnt, Npar))).as2D() // Jacobian matrix + val DoF = Npnt - Npar + 1 // statistical degrees of freedom + + var corr_p = 0 + var sigma_p = 0 + var sigma_y = 0 + var R_sq = 0 + var cvg_hist = 0 + + if (length(t) != length(y_dat)) { + println("lm.m error: the length of t must equal the length of y_dat") + val length_t = length(t) + val length_y_dat = length(y_dat) + X2 = 0.0 + + corr_p = 0 + sigma_p = 0 + sigma_y = 0 + R_sq = 0 + cvg_hist = 0 + +// if (tensor_parameter != 0) { // Зачем эта проверка? +// return +// } + } + + var weight = weight_input + if (nargin < 5) { + weight = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf((y_dat.transpose().dot(y_dat)).as1D()[0])).as2D() + } + + var dp = dp_input + if (nargin < 6) { + dp = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.001)).as2D() + } + + var p_min = p_min_input + if (nargin < 7) { + p_min = p + p_min.abs() + p_min = p_min.div(-100.0).as2D() + } + + var p_max = p_max_input + if (nargin < 8) { + p_max = p + p_max.abs() + p_max = p_max.div(100.0).as2D() + } + + var c = c_input + if (nargin < 9) { + c = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf(1.0)).as2D() + } + + var opts = opts_input + if (nargin < 10) { + opts = doubleArrayOf(3.0, 10.0 * Npar, 1e-3, 1e-3, 1e-1, 1e-1, 1e-2, 11.0, 9.0, 1.0) + } + + val prnt = opts[0] // >1 intermediate results; >2 plots + val MaxIter = opts[1].toInt() // maximum number of iterations + val epsilon_1 = opts[2] // convergence tolerance for gradient + val epsilon_2 = opts[3] // convergence tolerance for parameters + val epsilon_3 = opts[4] // convergence tolerance for Chi-square + val epsilon_4 = opts[5] // determines acceptance of a L-M step + val lambda_0 = opts[6] // initial value of damping paramter, lambda + val lambda_UP_fac = opts[7] // factor for increasing lambda + val lambda_DN_fac = opts[8] // factor for decreasing lambda + val Update_Type = opts[9].toInt() // 1: Levenberg-Marquardt lambda update + // 2: Quadratic update + // 3: Nielsen's lambda update equations + + val plotcmd = "figure(102); plot(t(:,1),y_init,''-k'',t(:,1),y_hat,''-b'',t(:,1),y_dat,''o'',''color'',[0,0.6,0],''MarkerSize'',4); title(sprintf(''\\chi^2_\\nu = %f'',X2/DoF)); drawnow" + + p_min = make_column(p_min) + p_max = make_column(p_max) + + if (length(make_column(dp)) == 1) { + dp = ones(ShapeND(intArrayOf(Npar, 1))).div(1 / dp[0, 0]).as2D() + } + + val idx = get_zero_indices(dp) // indices of the parameters to be fit + val Nfit = idx?.shape?.component1() // number of parameters to fit + var stop = false // termination flag + val y_init = feval(func, t, p, settings) // residual error using p_try + + if (weight.shape.component1() == 1 || variance(weight) == 0.0) { // identical weights vector + weight = ones(ShapeND(intArrayOf(Npnt, 1))).div(1 / kotlin.math.abs(weight[0, 0])).as2D() // !!! need to check + println("using uniform weights for error analysis") + } + else { + weight = make_column(weight) + weight.abs() + } + + // initialize Jacobian with finite difference calculation + var lm_matx_ans = lm_matx(func, t, p_old, y_old,1, J, p, y_dat, weight, dp, settings) + var JtWJ = lm_matx_ans[0] + var JtWdy = lm_matx_ans[1] + X2 = lm_matx_ans[2][0, 0] + var y_hat = lm_matx_ans[3] + J = lm_matx_ans[4] + + if ( abs(JtWdy).max()!! < epsilon_1 ) { + println(" *** Your Initial Guess is Extremely Close to Optimal ***\n") + println(" *** epsilon_1 = %e\n$epsilon_1") + stop = true + } + + var lambda = 1.0 + var nu = 1 + when (Update_Type) { + 1 -> lambda = lambda_0 // Marquardt: init'l lambda + else -> { // Quadratic and Nielsen + lambda = lambda_0 * (diag(JtWJ)).max()!! + nu = 2 + } + } + + X2_old = X2 // previous value of X2 + var cvg_hst = ones(ShapeND(intArrayOf(MaxIter, Npar + 3))) // initialize convergence history + + var h = JtWJ.copyToTensor() + var dX2 = X2 + while (!stop && settings.iteration <= MaxIter) { //--- Start Main Loop + settings.iteration += 1 + + // incremental change in parameters + h = when (Update_Type) { + 1 -> { // Marquardt + val solve = solve(JtWJ.plus(make_matrx_with_diagonal(diag(JtWJ)).div(1 / lambda)).as2D(), JtWdy) + solve.asDoubleTensor() + } + + else -> { // Quadratic and Nielsen + val solve = solve(JtWJ.plus(lm_eye(Npar).div(1 / lambda)).as2D(), JtWdy) + solve.asDoubleTensor() + } + } + + // big = max(abs(h./p)) > 2; % this is a big step + + // --- Are parameters [p+h] much better than [p] ? + + var p_try = (p + h).as2D() // update the [idx] elements + p_try = smallest_element_comparison(largest_element_comparison(p_min, p_try.as2D()), p_max) // apply constraints + + var delta_y = y_dat.minus(feval(func, t, p_try, settings)) // residual error using p_try + + // TODO + //if ~all(isfinite(delta_y)) // floating point error; break + // stop = 1; + // break + //end + + settings.func_calls += 1 + + val tmp = delta_y.times(weight) + var X2_try = delta_y.as2D().transpose().dot(tmp) // Chi-squared error criteria + + val alpha = 1.0 + if (Update_Type == 2) { // Quadratic + // One step of quadratic line update in the h direction for minimum X2 + +// TODO +// val alpha = JtWdy.transpose().dot(h) / ((X2_try.minus(X2)).div(2.0).plus(2 * JtWdy.transpose().dot(h))) +// alpha = JtWdy'*h / ( (X2_try - X2)/2 + 2*JtWdy'*h ) ; +// h = alpha * h; +// +// p_try = p + h(idx); % update only [idx] elements +// p_try = min(max(p_min,p_try),p_max); % apply constraints +// +// delta_y = y_dat - feval(func,t,p_try,c); % residual error using p_try +// func_calls = func_calls + 1; +// тX2_try = delta_y' * ( delta_y .* weight ); % Chi-squared error criteria + } + + val rho = when (Update_Type) { // Nielsen + 1 -> { + val tmp = h.transposed().dot(make_matrx_with_diagonal(diag(JtWJ)).div(1 / lambda).dot(h).plus(JtWdy)) + X2.minus(X2_try).as2D()[0, 0] / abs(tmp.as2D()).as2D()[0, 0] + } + else -> { + val tmp = h.transposed().dot(h.div(1 / lambda).plus(JtWdy)) + X2.minus(X2_try).as2D()[0, 0] / abs(tmp.as2D()).as2D()[0, 0] + } + } + + println() + println("rho = " + rho) + + if (rho > epsilon_4) { // it IS significantly better + val dX2 = X2.minus(X2_old) + X2_old = X2 + p_old = p.copyToTensor().as2D() + y_old = y_hat.copyToTensor().as2D() + p = make_column(p_try) // accept p_try + + lm_matx_ans = lm_matx(func, t, p_old, y_old, dX2.toInt(), J, p, y_dat, weight, dp, settings) + // decrease lambda ==> Gauss-Newton method + + JtWJ = lm_matx_ans[0] + JtWdy = lm_matx_ans[1] + X2 = lm_matx_ans[2][0, 0] + y_hat = lm_matx_ans[3] + J = lm_matx_ans[4] + + lambda = when (Update_Type) { + 1 -> { // Levenberg + max(lambda / lambda_DN_fac, 1e-7); + } + 2 -> { // Quadratic + max( lambda / (1 + alpha) , 1e-7 ); + } + else -> { // Nielsen + nu = 2 + lambda * max( 1.0 / 3, 1 - (2 * rho - 1).pow(3) ) + } + } + + // if (prnt > 2) { + // eval(plotcmd) + // } + } + else { // it IS NOT better + X2 = X2_old // do not accept p_try + if (settings.iteration % (2 * Npar) == 0 ) { // rank-1 update of Jacobian + lm_matx_ans = lm_matx(func, t, p_old, y_old,-1, J, p, y_dat, weight, dp, settings) + JtWJ = lm_matx_ans[0] + JtWdy = lm_matx_ans[1] + dX2 = lm_matx_ans[2][0, 0] + y_hat = lm_matx_ans[3] + J = lm_matx_ans[4] + } + + // increase lambda ==> gradient descent method + lambda = when (Update_Type) { + 1 -> { // Levenberg + min(lambda * lambda_UP_fac, 1e7) + } + 2 -> { // Quadratic + lambda + abs(((X2_try.as2D()[0, 0] - X2) / 2) / alpha) + } + else -> { // Nielsen + nu *= 2 + lambda * (nu / 2) + } + } + } + + if (prnt > 1) { + val chi_sq = X2 / DoF + println("Iteration $settings.iteration, func_calls $settings.func_calls | chi_sq=$chi_sq | lambda=$lambda") + print("param: ") + for (pn in 0 until Npar) { + print(p[pn, 0].toString() + " ") + } + print("\ndp/p: ") + for (pn in 0 until Npar) { + print((h.as2D()[pn, 0] / p[pn, 0]).toString() + " ") + } + result_chi_sq = chi_sq + } + + // update convergence history ... save _reduced_ Chi-square + // cvg_hst(iteration,:) = [ func_calls p' X2/DoF lambda ]; + + if (abs(JtWdy).max()!! < epsilon_1 && settings.iteration > 2) { + println(" **** Convergence in r.h.s. (\"JtWdy\") ****") + println(" **** epsilon_1 = $epsilon_1") + stop = true + } + if ((abs(h.as2D()).div(abs(p) + 1e-12)).max() < epsilon_2 && settings.iteration > 2) { + println(" **** Convergence in Parameters ****") + println(" **** epsilon_2 = $epsilon_2") + stop = true + } + if (X2 / DoF < epsilon_3 && settings.iteration > 2) { + println(" **** Convergence in reduced Chi-square **** ") + println(" **** epsilon_3 = $epsilon_3") + stop = true + } + if (settings.iteration == MaxIter) { + println(" !! Maximum Number of Iterations Reached Without Convergence !!") + stop = true + } + } // --- End of Main Loop + + // --- convergence achieved, find covariance and confidence intervals + + // ---- Error Analysis ---- + +// if (weight.shape.component1() == 1 || weight.variance() == 0.0) { +// weight = DoF / (delta_y.transpose().dot(delta_y)) * ones(intArrayOf(Npt, 1)) +// } + +// if (nargout > 1) { +// val redX2 = X2 / DoF +// } +// +// lm_matx_ans = lm_matx(func, t, p_old, y_old, -1, J, p, y_dat, weight, dp) +// JtWJ = lm_matx_ans[0] +// JtWdy = lm_matx_ans[1] +// X2 = lm_matx_ans[2][0, 0] +// y_hat = lm_matx_ans[3] +// J = lm_matx_ans[4] +// +// if (nargout > 2) { // standard error of parameters +// covar_p = inv(JtWJ); +// siif nagma_p = sqrt(diag(covar_p)); +// } +// +// if (nargout > 3) { // standard error of the fit +// /// sigma_y = sqrt(diag(J * covar_p * J')); // slower version of below +// sigma_y = zeros(Npnt,1); +// for i=1:Npnt +// sigma_y(i) = J(i,:) * covar_p * J(i,:)'; +// end +// sigma_y = sqrt(sigma_y); +// } +// +// if (nargout > 4) { // parameter correlation matrix +// corr_p = covar_p ./ [sigma_p*sigma_p']; +// } +// +// if (nargout > 5) { // coefficient of multiple determination +// R_sq = corr([y_dat y_hat]); +// R_sq = R_sq(1,2).^2; +// } +// +// if (nargout > 6) { // convergence history +// cvg_hst = cvg_hst(1:iteration,:); +// } + + return result_chi_sq + } } public val Double.Companion.tensorAlgebra: DoubleTensorAlgebra get() = DoubleTensorAlgebra diff --git a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt index cae01bed8..2c3b3a231 100644 --- a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt +++ b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt @@ -6,11 +6,11 @@ package space.kscience.kmath.tensors.core -import space.kscience.kmath.nd.ShapeND -import space.kscience.kmath.nd.contentEquals -import space.kscience.kmath.nd.get +import space.kscience.kmath.nd.* import space.kscience.kmath.operations.invoke +import space.kscience.kmath.tensors.core.internal.LMSettings import space.kscience.kmath.testutils.assertBufferEquals +import kotlin.math.roundToInt import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFalse @@ -207,4 +207,83 @@ internal class TestDoubleTensorAlgebra { assertTrue { ShapeND(5, 5) contentEquals res.shape } assertEquals(2.0, res[4, 4]) } + + @Test + fun testLM() = DoubleTensorAlgebra { + fun lm_func(t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings): MutableStructure2D { + val m = t.shape.component1() + var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(m, 1))) + + if (settings.example_number == 1) { + y_hat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))).times(p[0, 0]) + t.times(p[2, 0]).times( + DoubleTensorAlgebra.exp((t.times(-1.0 / p[3, 0]))) + ) + } + else if (settings.example_number == 2) { + val mt = t.max() + y_hat = (t.times(1.0 / mt)).times(p[0, 0]) + + (t.times(1.0 / mt)).pow(2).times(p[1, 0]) + + (t.times(1.0 / mt)).pow(3).times(p[2, 0]) + + (t.times(1.0 / mt)).pow(4).times(p[3, 0]) + } + else if (settings.example_number == 3) { + y_hat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))) + .times(p[0, 0]) + DoubleTensorAlgebra.sin((t.times(1.0 / p[3, 0]))).times(p[2, 0]) + } + + return y_hat.as2D() + } + + val lm_matx_y_dat = doubleArrayOf( + 19.6594, 18.6096, 17.6792, 17.2747, 16.3065, 17.1458, 16.0467, 16.7023, 15.7809, 15.9807, + 14.7620, 15.1128, 16.0973, 15.1934, 15.8636, 15.4763, 15.6860, 15.1895, 15.3495, 16.6054, + 16.2247, 15.9854, 16.1421, 17.0960, 16.7769, 17.1997, 17.2767, 17.5882, 17.5378, 16.7894, + 17.7648, 18.2512, 18.1581, 16.7037, 17.8475, 17.9081, 18.3067, 17.9632, 18.2817, 19.1427, + 18.8130, 18.5658, 18.0056, 18.4607, 18.5918, 18.2544, 18.3731, 18.7511, 19.3181, 17.3066, + 17.9632, 19.0513, 18.7528, 18.2928, 18.5967, 17.8567, 17.7859, 18.4016, 18.9423, 18.4959, + 17.8000, 18.4251, 17.7829, 17.4645, 17.5221, 17.3517, 17.4637, 17.7563, 16.8471, 17.4558, + 17.7447, 17.1487, 17.3183, 16.8312, 17.7551, 17.0942, 15.6093, 16.4163, 15.3755, 16.6725, + 16.2332, 16.2316, 16.2236, 16.5361, 15.3721, 15.3347, 15.5815, 15.6319, 14.4538, 14.6044, + 14.7665, 13.3718, 15.0587, 13.8320, 14.7873, 13.6824, 14.2579, 14.2154, 13.5818, 13.8157 + ) + + var example_number = 1 + val p_init = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(4, 1)), doubleArrayOf(5.0, 2.0, 0.2, 10.0) + ).as2D() + + var t = ones(ShapeND(intArrayOf(100, 1))).as2D() + for (i in 0 until 100) { + t[i, 0] = t[i, 0] * (i + 1) + } + + val y_dat = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(100, 1)), lm_matx_y_dat + ).as2D() + + val weight = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 4.0 } + ).as2D() + + val dp = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } + ).as2D() + + val p_min = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(4, 1)), doubleArrayOf(-50.0, -20.0, -2.0, -100.0) + ).as2D() + + val p_max = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(4, 1)), doubleArrayOf(50.0, 20.0, 2.0, 100.0) + ).as2D() + + val consts = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.0) + ).as2D() + + val opts = doubleArrayOf(3.0, 100.0, 1e-3, 1e-3, 1e-1, 1e-1, 1e-2, 11.0, 9.0, 1.0) + + val chi_sq = lm(::lm_func, p_init, t, y_dat, weight, dp, p_min, p_max, consts, opts, 10, example_number) + assertEquals(0.9131, (chi_sq * 10000).roundToInt() / 10000.0) + } } From 64e563340a04b1f8f1ccb3e0c78921ef479aeb7c Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Sun, 7 May 2023 17:26:59 +0300 Subject: [PATCH 05/26] fixed error for chi_sq and added more complete output for lm --- .../tensors/api/LinearOpsTensorAlgebra.kt | 11 +++++- .../kmath/tensors/core/DoubleTensorAlgebra.kt | 38 ++++++++----------- .../tensors/core/TestDoubleTensorAlgebra.kt | 9 ++++- 3 files changed, 32 insertions(+), 26 deletions(-) diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt index 5cf0081fb..3c74263ec 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt @@ -120,9 +120,18 @@ public interface LinearOpsTensorAlgebra> : TensorPartialDivision */ public fun solve(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D + data class LMResultInfo ( + var iterations:Int, + var func_calls: Int, + var example_number: Int, + var result_chi_sq: Double, + var result_lambda: Double, + var result_parameters: MutableStructure2D + ) + public fun lm( func: KFunction3, MutableStructure2D, LMSettings, MutableStructure2D>, p_input: MutableStructure2D, t_input: MutableStructure2D, y_dat_input: MutableStructure2D, weight_input: MutableStructure2D, dp_input: MutableStructure2D, p_min_input: MutableStructure2D, p_max_input: MutableStructure2D, - c_input: MutableStructure2D, opts_input: DoubleArray, nargin: Int, example_number: Int): Double + c_input: MutableStructure2D, opts_input: DoubleArray, nargin: Int, example_number: Int): LMResultInfo } diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt index e7e22afa9..21a034676 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt @@ -721,9 +721,9 @@ public open class DoubleTensorAlgebra : func: KFunction3, MutableStructure2D, LMSettings, MutableStructure2D>, p_input: MutableStructure2D, t_input: MutableStructure2D, y_dat_input: MutableStructure2D, weight_input: MutableStructure2D, dp_input: MutableStructure2D, p_min_input: MutableStructure2D, p_max_input: MutableStructure2D, - c_input: MutableStructure2D, opts_input: DoubleArray, nargin: Int, example_number: Int): Double { + c_input: MutableStructure2D, opts_input: DoubleArray, nargin: Int, example_number: Int): LinearOpsTensorAlgebra.LMResultInfo { - var result_chi_sq = 0.0 + var resultInfo = LinearOpsTensorAlgebra.LMResultInfo(0, 0, example_number, 0.0, 0.0, p_input) val tensor_parameter = 0 val eps:Double = 2.2204e-16 @@ -742,7 +742,7 @@ public open class DoubleTensorAlgebra : var X2 = 1e-3 / eps // a really big initial Chi-sq value var X2_old = 1e-3 / eps // a really big initial Chi-sq value var J = zeros(ShapeND(intArrayOf(Npnt, Npar))).as2D() // Jacobian matrix - val DoF = Npnt - Npar + 1 // statistical degrees of freedom + val DoF = Npnt - Npar // statistical degrees of freedom var corr_p = 0 var sigma_p = 0 @@ -811,10 +811,8 @@ public open class DoubleTensorAlgebra : val lambda_UP_fac = opts[7] // factor for increasing lambda val lambda_DN_fac = opts[8] // factor for decreasing lambda val Update_Type = opts[9].toInt() // 1: Levenberg-Marquardt lambda update - // 2: Quadratic update - // 3: Nielsen's lambda update equations - - val plotcmd = "figure(102); plot(t(:,1),y_init,''-k'',t(:,1),y_hat,''-b'',t(:,1),y_dat,''o'',''color'',[0,0.6,0],''MarkerSize'',4); title(sprintf(''\\chi^2_\\nu = %f'',X2/DoF)); drawnow" + // 2: Quadratic update + // 3: Nielsen's lambda update equations p_min = make_column(p_min) p_max = make_column(p_max) @@ -829,7 +827,7 @@ public open class DoubleTensorAlgebra : val y_init = feval(func, t, p, settings) // residual error using p_try if (weight.shape.component1() == 1 || variance(weight) == 0.0) { // identical weights vector - weight = ones(ShapeND(intArrayOf(Npnt, 1))).div(1 / kotlin.math.abs(weight[0, 0])).as2D() // !!! need to check + weight = ones(ShapeND(intArrayOf(Npnt, 1))).div(1 / abs(weight[0, 0])).as2D() // !!! need to check println("using uniform weights for error analysis") } else { @@ -864,7 +862,7 @@ public open class DoubleTensorAlgebra : X2_old = X2 // previous value of X2 var cvg_hst = ones(ShapeND(intArrayOf(MaxIter, Npar + 3))) // initialize convergence history - var h = JtWJ.copyToTensor() + var h: DoubleTensor var dX2 = X2 while (!stop && settings.iteration <= MaxIter) { //--- Start Main Loop settings.iteration += 1 @@ -882,10 +880,6 @@ public open class DoubleTensorAlgebra : } } - // big = max(abs(h./p)) > 2; % this is a big step - - // --- Are parameters [p+h] much better than [p] ? - var p_try = (p + h).as2D() // update the [idx] elements p_try = smallest_element_comparison(largest_element_comparison(p_min, p_try.as2D()), p_max) // apply constraints @@ -961,10 +955,6 @@ public open class DoubleTensorAlgebra : lambda * max( 1.0 / 3, 1 - (2 * rho - 1).pow(3) ) } } - - // if (prnt > 2) { - // eval(plotcmd) - // } } else { // it IS NOT better X2 = X2_old // do not accept p_try @@ -994,7 +984,7 @@ public open class DoubleTensorAlgebra : if (prnt > 1) { val chi_sq = X2 / DoF - println("Iteration $settings.iteration, func_calls $settings.func_calls | chi_sq=$chi_sq | lambda=$lambda") + println("Iteration $settings | chi_sq=$chi_sq | lambda=$lambda") print("param: ") for (pn in 0 until Npar) { print(p[pn, 0].toString() + " ") @@ -1003,7 +993,11 @@ public open class DoubleTensorAlgebra : for (pn in 0 until Npar) { print((h.as2D()[pn, 0] / p[pn, 0]).toString() + " ") } - result_chi_sq = chi_sq + resultInfo.iterations = settings.iteration + resultInfo.func_calls = settings.func_calls + resultInfo.result_chi_sq = chi_sq + resultInfo.result_lambda = lambda + resultInfo.result_parameters = p } // update convergence history ... save _reduced_ Chi-square @@ -1076,11 +1070,9 @@ public open class DoubleTensorAlgebra : // cvg_hst = cvg_hst(1:iteration,:); // } - return result_chi_sq + return resultInfo } } public val Double.Companion.tensorAlgebra: DoubleTensorAlgebra get() = DoubleTensorAlgebra -public val DoubleField.tensorAlgebra: DoubleTensorAlgebra get() = DoubleTensorAlgebra - - +public val DoubleField.tensorAlgebra: DoubleTensorAlgebra get() = DoubleTensorAlgebra \ No newline at end of file diff --git a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt index 2c3b3a231..e5c35f8fd 100644 --- a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt +++ b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt @@ -11,6 +11,7 @@ import space.kscience.kmath.operations.invoke import space.kscience.kmath.tensors.core.internal.LMSettings import space.kscience.kmath.testutils.assertBufferEquals import kotlin.math.roundToInt +import kotlin.math.roundToLong import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFalse @@ -283,7 +284,11 @@ internal class TestDoubleTensorAlgebra { val opts = doubleArrayOf(3.0, 100.0, 1e-3, 1e-3, 1e-1, 1e-1, 1e-2, 11.0, 9.0, 1.0) - val chi_sq = lm(::lm_func, p_init, t, y_dat, weight, dp, p_min, p_max, consts, opts, 10, example_number) - assertEquals(0.9131, (chi_sq * 10000).roundToInt() / 10000.0) + val result = lm(::lm_func, p_init, t, y_dat, weight, dp, p_min, p_max, consts, opts, 10, example_number) + assertEquals(13, result.iterations) + assertEquals(31, result.func_calls) + assertEquals(1, result.example_number) + assertEquals(0.9131368192633, (result.result_chi_sq * 1e13).roundToLong() / 1e13) + assertEquals(3.7790980 * 1e-7, (result.result_lambda * 1e13).roundToLong() / 1e13) } } From cfe8e9bfee19974d083bfd653a25ed1e8f301dd9 Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Sun, 7 May 2023 21:34:20 +0300 Subject: [PATCH 06/26] done TODOs, deleted prints and added type of convergence to output of lm --- .../tensors/api/LinearOpsTensorAlgebra.kt | 13 ++- .../kmath/tensors/core/DoubleTensorAlgebra.kt | 95 ++++++++++--------- .../tensors/core/TestDoubleTensorAlgebra.kt | 2 + 3 files changed, 62 insertions(+), 48 deletions(-) diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt index 3c74263ec..7964656f1 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt @@ -120,13 +120,22 @@ public interface LinearOpsTensorAlgebra> : TensorPartialDivision */ public fun solve(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D - data class LMResultInfo ( + public enum class TypeOfConvergence{ + inRHS_JtWdy, + inParameters, + inReducedChi_square, + noConvergence + } + + public data class LMResultInfo ( var iterations:Int, var func_calls: Int, var example_number: Int, var result_chi_sq: Double, var result_lambda: Double, - var result_parameters: MutableStructure2D + var result_parameters: MutableStructure2D, + var typeOfConvergence: TypeOfConvergence, + var epsilon: Double ) public fun lm( diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt index 21a034676..43f097564 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt @@ -723,9 +723,9 @@ public open class DoubleTensorAlgebra : weight_input: MutableStructure2D, dp_input: MutableStructure2D, p_min_input: MutableStructure2D, p_max_input: MutableStructure2D, c_input: MutableStructure2D, opts_input: DoubleArray, nargin: Int, example_number: Int): LinearOpsTensorAlgebra.LMResultInfo { - var resultInfo = LinearOpsTensorAlgebra.LMResultInfo(0, 0, example_number, 0.0, 0.0, p_input) + var resultInfo = LinearOpsTensorAlgebra.LMResultInfo(0, 0, example_number, 0.0, + 0.0, p_input, LinearOpsTensorAlgebra.TypeOfConvergence.noConvergence, 0.0) - val tensor_parameter = 0 val eps:Double = 2.2204e-16 var settings = LMSettings(0, 0, example_number) @@ -751,7 +751,7 @@ public open class DoubleTensorAlgebra : var cvg_hist = 0 if (length(t) != length(y_dat)) { - println("lm.m error: the length of t must equal the length of y_dat") + // println("lm.m error: the length of t must equal the length of y_dat") val length_t = length(t) val length_y_dat = length(y_dat) X2 = 0.0 @@ -761,10 +761,6 @@ public open class DoubleTensorAlgebra : sigma_y = 0 R_sq = 0 cvg_hist = 0 - -// if (tensor_parameter != 0) { // Зачем эта проверка? -// return -// } } var weight = weight_input @@ -827,8 +823,8 @@ public open class DoubleTensorAlgebra : val y_init = feval(func, t, p, settings) // residual error using p_try if (weight.shape.component1() == 1 || variance(weight) == 0.0) { // identical weights vector - weight = ones(ShapeND(intArrayOf(Npnt, 1))).div(1 / abs(weight[0, 0])).as2D() // !!! need to check - println("using uniform weights for error analysis") + weight = ones(ShapeND(intArrayOf(Npnt, 1))).div(1 / abs(weight[0, 0])).as2D() + // println("using uniform weights for error analysis") } else { weight = make_column(weight) @@ -844,8 +840,8 @@ public open class DoubleTensorAlgebra : J = lm_matx_ans[4] if ( abs(JtWdy).max()!! < epsilon_1 ) { - println(" *** Your Initial Guess is Extremely Close to Optimal ***\n") - println(" *** epsilon_1 = %e\n$epsilon_1") +// println(" *** Your Initial Guess is Extremely Close to Optimal ***\n") +// println(" *** epsilon_1 = %e\n$epsilon_1") stop = true } @@ -885,11 +881,14 @@ public open class DoubleTensorAlgebra : var delta_y = y_dat.minus(feval(func, t, p_try, settings)) // residual error using p_try - // TODO - //if ~all(isfinite(delta_y)) // floating point error; break - // stop = 1; - // break - //end + for (i in 0 until delta_y.shape.component1()) { // floating point error; break + for (j in 0 until delta_y.shape.component2()) { + if (delta_y[i, j] == Double.POSITIVE_INFINITY || delta_y[i, j] == Double.NEGATIVE_INFINITY) { + stop = true + break + } + } + } settings.func_calls += 1 @@ -900,17 +899,16 @@ public open class DoubleTensorAlgebra : if (Update_Type == 2) { // Quadratic // One step of quadratic line update in the h direction for minimum X2 -// TODO -// val alpha = JtWdy.transpose().dot(h) / ((X2_try.minus(X2)).div(2.0).plus(2 * JtWdy.transpose().dot(h))) -// alpha = JtWdy'*h / ( (X2_try - X2)/2 + 2*JtWdy'*h ) ; -// h = alpha * h; -// -// p_try = p + h(idx); % update only [idx] elements -// p_try = min(max(p_min,p_try),p_max); % apply constraints -// -// delta_y = y_dat - feval(func,t,p_try,c); % residual error using p_try -// func_calls = func_calls + 1; -// тX2_try = delta_y' * ( delta_y .* weight ); % Chi-squared error criteria + val alpha = JtWdy.transpose().dot(h) / ( (X2_try.minus(X2)).div(2.0).plus(2 * JtWdy.transpose().dot(h)) ) + h = h.dot(alpha) + p_try = p.plus(h).as2D() // update only [idx] elements + p_try = smallest_element_comparison(largest_element_comparison(p_min, p_try), p_max) // apply constraints + + var delta_y = y_dat.minus(feval(func, t, p_try, settings)) // residual error using p_try + settings.func_calls += 1 + + val tmp = delta_y.times(weight) + X2_try = delta_y.as2D().transpose().dot(tmp) // Chi-squared error criteria } val rho = when (Update_Type) { // Nielsen @@ -924,9 +922,6 @@ public open class DoubleTensorAlgebra : } } - println() - println("rho = " + rho) - if (rho > epsilon_4) { // it IS significantly better val dX2 = X2.minus(X2_old) X2_old = X2 @@ -984,15 +979,15 @@ public open class DoubleTensorAlgebra : if (prnt > 1) { val chi_sq = X2 / DoF - println("Iteration $settings | chi_sq=$chi_sq | lambda=$lambda") - print("param: ") - for (pn in 0 until Npar) { - print(p[pn, 0].toString() + " ") - } - print("\ndp/p: ") - for (pn in 0 until Npar) { - print((h.as2D()[pn, 0] / p[pn, 0]).toString() + " ") - } +// println("Iteration $settings | chi_sq=$chi_sq | lambda=$lambda") +// print("param: ") +// for (pn in 0 until Npar) { +// print(p[pn, 0].toString() + " ") +// } +// print("\ndp/p: ") +// for (pn in 0 until Npar) { +// print((h.as2D()[pn, 0] / p[pn, 0]).toString() + " ") +// } resultInfo.iterations = settings.iteration resultInfo.func_calls = settings.func_calls resultInfo.result_chi_sq = chi_sq @@ -1004,22 +999,30 @@ public open class DoubleTensorAlgebra : // cvg_hst(iteration,:) = [ func_calls p' X2/DoF lambda ]; if (abs(JtWdy).max()!! < epsilon_1 && settings.iteration > 2) { - println(" **** Convergence in r.h.s. (\"JtWdy\") ****") - println(" **** epsilon_1 = $epsilon_1") +// println(" **** Convergence in r.h.s. (\"JtWdy\") ****") +// println(" **** epsilon_1 = $epsilon_1") + resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.inRHS_JtWdy + resultInfo.epsilon = epsilon_1 stop = true } if ((abs(h.as2D()).div(abs(p) + 1e-12)).max() < epsilon_2 && settings.iteration > 2) { - println(" **** Convergence in Parameters ****") - println(" **** epsilon_2 = $epsilon_2") +// println(" **** Convergence in Parameters ****") +// println(" **** epsilon_2 = $epsilon_2") + resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.inParameters + resultInfo.epsilon = epsilon_2 stop = true } if (X2 / DoF < epsilon_3 && settings.iteration > 2) { - println(" **** Convergence in reduced Chi-square **** ") - println(" **** epsilon_3 = $epsilon_3") +// println(" **** Convergence in reduced Chi-square **** ") +// println(" **** epsilon_3 = $epsilon_3") + resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.inReducedChi_square + resultInfo.epsilon = epsilon_3 stop = true } if (settings.iteration == MaxIter) { - println(" !! Maximum Number of Iterations Reached Without Convergence !!") +// println(" !! Maximum Number of Iterations Reached Without Convergence !!") + resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.noConvergence + resultInfo.epsilon = 0.0 stop = true } } // --- End of Main Loop diff --git a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt index e5c35f8fd..85b81524b 100644 --- a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt +++ b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt @@ -8,6 +8,7 @@ package space.kscience.kmath.tensors.core import space.kscience.kmath.nd.* import space.kscience.kmath.operations.invoke +import space.kscience.kmath.tensors.api.LinearOpsTensorAlgebra import space.kscience.kmath.tensors.core.internal.LMSettings import space.kscience.kmath.testutils.assertBufferEquals import kotlin.math.roundToInt @@ -290,5 +291,6 @@ internal class TestDoubleTensorAlgebra { assertEquals(1, result.example_number) assertEquals(0.9131368192633, (result.result_chi_sq * 1e13).roundToLong() / 1e13) assertEquals(3.7790980 * 1e-7, (result.result_lambda * 1e13).roundToLong() / 1e13) + assertEquals(result.typeOfConvergence, LinearOpsTensorAlgebra.TypeOfConvergence.inParameters) } } From a18fa01100e5f2689ef0842a9190f08e28f0dd5a Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Fri, 26 May 2023 21:53:50 +0300 Subject: [PATCH 07/26] added parameter check in tests --- .../kmath/tensors/core/TestDoubleTensorAlgebra.kt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt index 85b81524b..5aea9b879 100644 --- a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt +++ b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt @@ -292,5 +292,17 @@ internal class TestDoubleTensorAlgebra { assertEquals(0.9131368192633, (result.result_chi_sq * 1e13).roundToLong() / 1e13) assertEquals(3.7790980 * 1e-7, (result.result_lambda * 1e13).roundToLong() / 1e13) assertEquals(result.typeOfConvergence, LinearOpsTensorAlgebra.TypeOfConvergence.inParameters) + val expectedParameters = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(4, 1)), doubleArrayOf(20.527230909086, 9.833627103230, 0.997571256572, 50.174445822506) + ).as2D() + result.result_parameters = result.result_parameters.map { x -> (x * 1e12).toLong() / 1e12}.as2D() + val receivedParameters = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(4, 1)), doubleArrayOf(result.result_parameters[0, 0], result.result_parameters[1, 0], + result.result_parameters[2, 0], result.result_parameters[3, 0]) + ).as2D() + assertEquals(expectedParameters[0, 0], receivedParameters[0, 0]) + assertEquals(expectedParameters[1, 0], receivedParameters[1, 0]) + assertEquals(expectedParameters[2, 0], receivedParameters[2, 0]) + assertEquals(expectedParameters[3, 0], receivedParameters[3, 0]) } } From ce169461055a1372baed6baae80e662320150b60 Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Sat, 27 May 2023 01:16:43 +0300 Subject: [PATCH 08/26] added streaming version of LM --- .../StreamingLm/functionsToOptimize.kt | 109 ++++++++++++++++++ .../tensors/StreamingLm/streamingLmMain.kt | 75 ++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/functionsToOptimize.kt create mode 100644 examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/streamingLmMain.kt diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/functionsToOptimize.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/functionsToOptimize.kt new file mode 100644 index 000000000..1d0ce7deb --- /dev/null +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/functionsToOptimize.kt @@ -0,0 +1,109 @@ +/* + * Copyright 2018-2023 KMath contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package space.kscience.kmath.tensors.StreamingLm + +import space.kscience.kmath.nd.MutableStructure2D +import space.kscience.kmath.nd.ShapeND +import space.kscience.kmath.nd.as2D +import space.kscience.kmath.nd.component1 +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.max +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.plus +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.pow +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.times +import space.kscience.kmath.tensors.core.internal.LMSettings + +public data class StartDataLm ( + var lm_matx_y_dat: MutableStructure2D, + var example_number: Int, + var p_init: MutableStructure2D, + var t: MutableStructure2D, + var y_dat: MutableStructure2D, + var weight: MutableStructure2D, + var dp: MutableStructure2D, + var p_min: MutableStructure2D, + var p_max: MutableStructure2D, + var consts: MutableStructure2D, + var opts: DoubleArray +) + +fun func1ForLm(t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings): MutableStructure2D { + val m = t.shape.component1() + var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf (m, 1))) + + if (settings.example_number == 1) { + y_hat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))).times(p[0, 0]) + t.times(p[2, 0]).times( + DoubleTensorAlgebra.exp((t.times(-1.0 / p[3, 0]))) + ) + } + else if (settings.example_number == 2) { + val mt = t.max() + y_hat = (t.times(1.0 / mt)).times(p[0, 0]) + + (t.times(1.0 / mt)).pow(2).times(p[1, 0]) + + (t.times(1.0 / mt)).pow(3).times(p[2, 0]) + + (t.times(1.0 / mt)).pow(4).times(p[3, 0]) + } + else if (settings.example_number == 3) { + y_hat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))) + .times(p[0, 0]) + DoubleTensorAlgebra.sin((t.times(1.0 / p[3, 0]))).times(p[2, 0]) + } + + return y_hat.as2D() +} + +fun getStartDataForFunc1(): StartDataLm { + val lm_matx_y_dat = doubleArrayOf( + 19.6594, 18.6096, 17.6792, 17.2747, 16.3065, 17.1458, 16.0467, 16.7023, 15.7809, 15.9807, + 14.7620, 15.1128, 16.0973, 15.1934, 15.8636, 15.4763, 15.6860, 15.1895, 15.3495, 16.6054, + 16.2247, 15.9854, 16.1421, 17.0960, 16.7769, 17.1997, 17.2767, 17.5882, 17.5378, 16.7894, + 17.7648, 18.2512, 18.1581, 16.7037, 17.8475, 17.9081, 18.3067, 17.9632, 18.2817, 19.1427, + 18.8130, 18.5658, 18.0056, 18.4607, 18.5918, 18.2544, 18.3731, 18.7511, 19.3181, 17.3066, + 17.9632, 19.0513, 18.7528, 18.2928, 18.5967, 17.8567, 17.7859, 18.4016, 18.9423, 18.4959, + 17.8000, 18.4251, 17.7829, 17.4645, 17.5221, 17.3517, 17.4637, 17.7563, 16.8471, 17.4558, + 17.7447, 17.1487, 17.3183, 16.8312, 17.7551, 17.0942, 15.6093, 16.4163, 15.3755, 16.6725, + 16.2332, 16.2316, 16.2236, 16.5361, 15.3721, 15.3347, 15.5815, 15.6319, 14.4538, 14.6044, + 14.7665, 13.3718, 15.0587, 13.8320, 14.7873, 13.6824, 14.2579, 14.2154, 13.5818, 13.8157 + ) + + var example_number = 1 + val p_init = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(4, 1)), doubleArrayOf(5.0, 2.0, 0.2, 10.0) + ).as2D() + + var t = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(100, 1))).as2D() + for (i in 0 until 100) { + t[i, 0] = t[i, 0] * (i + 1) + } + + val y_dat = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(100, 1)), lm_matx_y_dat + ).as2D() + + val weight = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 4.0 } + ).as2D() + + val dp = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } + ).as2D() + + val p_min = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(4, 1)), doubleArrayOf(-50.0, -20.0, -2.0, -100.0) + ).as2D() + + val p_max = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(4, 1)), doubleArrayOf(50.0, 20.0, 2.0, 100.0) + ).as2D() + + val consts = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.0) + ).as2D() + + val opts = doubleArrayOf(3.0, 100.0, 1e-3, 1e-3, 1e-1, 1e-1, 1e-2, 11.0, 9.0, 1.0) + + return StartDataLm(y_dat, example_number, p_init, t, y_dat, weight, dp, p_min, p_max, consts, opts) +} \ No newline at end of file diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/streamingLmMain.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/streamingLmMain.kt new file mode 100644 index 000000000..e33cfac80 --- /dev/null +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/streamingLmMain.kt @@ -0,0 +1,75 @@ +/* + * Copyright 2018-2023 KMath contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package space.kscience.kmath.tensors.StreamingLm + +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.* +import space.kscience.kmath.nd.* +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.zeros +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra +import space.kscience.kmath.tensors.core.internal.LMSettings +import kotlin.math.roundToInt +import kotlin.random.Random +import kotlin.reflect.KFunction3 + +fun streamLm(lm_func: KFunction3, MutableStructure2D, LMSettings, MutableStructure2D>, + startData: StartDataLm, launchFrequencyInMs: Long): Flow> = flow{ + + var example_number = startData.example_number + var p_init = startData.p_init + var t = startData.t + val y_dat = startData.y_dat + val weight = startData.weight + val dp = startData.dp + val p_min = startData.p_min + val p_max = startData.p_max + val consts = startData.consts + val opts = startData.opts + + while (true) { + val result = DoubleTensorAlgebra.lm( + lm_func, + p_init, + t, + y_dat, + weight, + dp, + p_min, + p_max, + consts, + opts, + 10, + example_number + ) + emit(result.result_parameters) + delay(launchFrequencyInMs) + p_init = generateNewParameters(p_init, 0.1) + } +} + +fun generateNewParameters(p: MutableStructure2D, delta: Double): MutableStructure2D{ + val n = p.shape.component1() + val p_new = zeros(ShapeND(intArrayOf(n, 1))).as2D() + for (i in 0 until n) { + val randomEps = Random.nextDouble(delta + delta) - delta + p_new[i, 0] = p[i, 0] + randomEps + } + return p_new +} + +suspend fun main(){ + val startData = getStartDataForFunc1() + // Создание потока: + val numberFlow = streamLm(::func1ForLm, startData, 1000) + // Запуск потока + numberFlow.collect { parameters -> + for (i in 0 until parameters.shape.component1()) { + val x = (parameters[i, 0] * 10000).roundToInt() / 10000.0 + print("$x ") + if (i == parameters.shape.component1() - 1) println() + } + } +} \ No newline at end of file From e738fbc86d27597dcc9521756837026910b46648 Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Sat, 27 May 2023 01:24:37 +0300 Subject: [PATCH 09/26] typo fixed --- .../kscience/kmath/tensors/StreamingLm/streamingLmMain.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/streamingLmMain.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/streamingLmMain.kt index e33cfac80..06105fb9f 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/streamingLmMain.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/streamingLmMain.kt @@ -63,9 +63,9 @@ fun generateNewParameters(p: MutableStructure2D, delta: Double): Mutable suspend fun main(){ val startData = getStartDataForFunc1() // Создание потока: - val numberFlow = streamLm(::func1ForLm, startData, 1000) + val lmFlow = streamLm(::func1ForLm, startData, 1000) // Запуск потока - numberFlow.collect { parameters -> + lmFlow.collect { parameters -> for (i in 0 until parameters.shape.component1()) { val x = (parameters[i, 0] * 10000).roundToInt() / 10000.0 print("$x ") From 20c20a30e8b7f670f7586efffbb8cf63a4e9e917 Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Sat, 27 May 2023 16:07:13 +0300 Subject: [PATCH 10/26] y_dat added generation --- .../kmath/tensors/StreamingLm/streamingLmMain.kt | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/streamingLmMain.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/streamingLmMain.kt index 06105fb9f..0fa934955 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/streamingLmMain.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/streamingLmMain.kt @@ -21,7 +21,7 @@ fun streamLm(lm_func: KFunction3, MutableStructure2D< var example_number = startData.example_number var p_init = startData.p_init var t = startData.t - val y_dat = startData.y_dat + var y_dat = startData.y_dat val weight = startData.weight val dp = startData.dp val p_min = startData.p_min @@ -46,18 +46,19 @@ fun streamLm(lm_func: KFunction3, MutableStructure2D< ) emit(result.result_parameters) delay(launchFrequencyInMs) - p_init = generateNewParameters(p_init, 0.1) + p_init = result.result_parameters + y_dat = generateNewYDat(y_dat, 0.1) } } -fun generateNewParameters(p: MutableStructure2D, delta: Double): MutableStructure2D{ - val n = p.shape.component1() - val p_new = zeros(ShapeND(intArrayOf(n, 1))).as2D() +fun generateNewYDat(y_dat: MutableStructure2D, delta: Double): MutableStructure2D{ + val n = y_dat.shape.component1() + val y_dat_new = zeros(ShapeND(intArrayOf(n, 1))).as2D() for (i in 0 until n) { val randomEps = Random.nextDouble(delta + delta) - delta - p_new[i, 0] = p[i, 0] + randomEps + y_dat_new[i, 0] = y_dat[i, 0] + randomEps } - return p_new + return y_dat_new } suspend fun main(){ From 33cb317ceee1256f3d271dd6723b8d55c57b814f Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Sun, 28 May 2023 23:07:01 +0300 Subject: [PATCH 11/26] added examples and tests --- .../StaticLm/staticDifficultTest.kt | 92 ++++++ .../StaticLm/staticEasyTest.kt | 56 ++++ .../StaticLm/staticMiddleTest.kt | 91 ++++++ .../StreamingLm/streamLm.kt} | 26 +- .../StreamingLm/streamingLmTest.kt | 25 ++ .../functionsToOptimize.kt | 39 ++- .../kmath/tensors/core/DoubleTensorAlgebra.kt | 6 +- .../kmath/tensors/core/TestLmAlgorithm.kt | 267 ++++++++++++++++++ 8 files changed, 579 insertions(+), 23 deletions(-) create mode 100644 examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt create mode 100644 examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt create mode 100644 examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt rename examples/src/main/kotlin/space/kscience/kmath/tensors/{StreamingLm/streamingLmMain.kt => LevenbergMarquardt/StreamingLm/streamLm.kt} (72%) create mode 100644 examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamingLmTest.kt rename examples/src/main/kotlin/space/kscience/kmath/tensors/{StreamingLm => LevenbergMarquardt}/functionsToOptimize.kt (77%) create mode 100644 kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt new file mode 100644 index 000000000..621943aea --- /dev/null +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2018-2023 KMath contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package space.kscience.kmath.tensors.LevenbergMarquardt.StaticLm + +import space.kscience.kmath.nd.ShapeND +import space.kscience.kmath.nd.as2D +import space.kscience.kmath.nd.component1 +import space.kscience.kmath.tensors.LevenbergMarquardt.funcDifficultForLm +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.div +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra +import space.kscience.kmath.tensors.core.internal.LMSettings +import kotlin.math.roundToInt + +fun main() { + val NData = 200 + var t_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(NData, 1))).as2D() + for (i in 0 until NData) { + t_example[i, 0] = t_example[i, 0] * (i + 1) - 104 + } + + val Nparams = 15 + var p_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))).as2D() + for (i in 0 until Nparams) { + p_example[i, 0] = p_example[i, 0] + i - 25 + } + + val settings = LMSettings(0, 0, 1) + + var y_hat = funcDifficultForLm(t_example, p_example, settings) + + var p_init = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() + for (i in 0 until Nparams) { + p_init[i, 0] = (p_example[i, 0] + 0.9) + } + + var t = t_example + val y_dat = y_hat + val weight = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 1.0 / Nparams * 1.0 - 0.085 } + ).as2D() + val dp = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } + ).as2D() + var p_min = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) + p_min = p_min.div(1.0 / -50.0) + val p_max = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) + p_min = p_min.div(1.0 / 50.0) + val consts = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.0) + ).as2D() + val opts = doubleArrayOf(3.0, 10000.0, 1e-2, 0.015, 1e-2, 1e-2, 1e-2, 11.0, 9.0, 1.0) +// val opts = doubleArrayOf(3.0, 10000.0, 1e-5, 1e-5, 1e-5, 1e-5, 1e-3, 11.0, 9.0, 1.0) + + val result = DoubleTensorAlgebra.lm( + ::funcDifficultForLm, + p_init.as2D(), + t, + y_dat, + weight, + dp, + p_min.as2D(), + p_max.as2D(), + consts, + opts, + 10, + 1 + ) + + println("Parameters:") + for (i in 0 until result.result_parameters.shape.component1()) { + val x = (result.result_parameters[i, 0] * 10000).roundToInt() / 10000.0 + print("$x ") + } + println() + + println("Y true and y received:") + var y_hat_after = funcDifficultForLm(t_example, result.result_parameters, settings) + for (i in 0 until y_hat.shape.component1()) { + val x = (y_hat[i, 0] * 10000).roundToInt() / 10000.0 + val y = (y_hat_after[i, 0] * 10000).roundToInt() / 10000.0 + println("$x $y") + } + + println("Сhi_sq:") + println(result.result_chi_sq) + println("Number of iterations:") + println(result.iterations) +} \ No newline at end of file diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt new file mode 100644 index 000000000..bae5a674f --- /dev/null +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2018-2023 KMath contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package space.kscience.kmath.tensors.LevenbergMarquardt.StaticLm + +import space.kscience.kmath.nd.ShapeND +import space.kscience.kmath.nd.as2D +import space.kscience.kmath.nd.component1 +import space.kscience.kmath.tensors.LevenbergMarquardt.funcDifficultForLm +import space.kscience.kmath.tensors.LevenbergMarquardt.funcEasyForLm +import space.kscience.kmath.tensors.LevenbergMarquardt.getStartDataForFuncEasy +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra +import space.kscience.kmath.tensors.core.internal.LMSettings +import kotlin.math.roundToInt + +fun main() { + val startedData = getStartDataForFuncEasy() + + val result = DoubleTensorAlgebra.lm( + ::funcEasyForLm, + DoubleTensorAlgebra.ones(ShapeND(intArrayOf(4, 1))).as2D(), + startedData.t, + startedData.y_dat, + startedData.weight, + startedData.dp, + startedData.p_min, + startedData.p_max, + startedData.consts, + startedData.opts, + 10, + startedData.example_number + ) + + println("Parameters:") + for (i in 0 until result.result_parameters.shape.component1()) { + val x = (result.result_parameters[i, 0] * 10000).roundToInt() / 10000.0 + print("$x ") + } + println() + + println("Y true and y received:") + var y_hat_after = funcDifficultForLm(startedData.t, result.result_parameters, LMSettings(0, 0, startedData.example_number)) + for (i in 0 until startedData.y_dat.shape.component1()) { + val x = (startedData.y_dat[i, 0] * 10000).roundToInt() / 10000.0 + val y = (y_hat_after[i, 0] * 10000).roundToInt() / 10000.0 + println("$x $y") + } + + println("Сhi_sq:") + println(result.result_chi_sq) + println("Number of iterations:") + println(result.iterations) +} \ No newline at end of file diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt new file mode 100644 index 000000000..a39572858 --- /dev/null +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt @@ -0,0 +1,91 @@ +/* + * Copyright 2018-2023 KMath contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package space.kscience.kmath.tensors.LevenbergMarquardt.StaticLm + +import space.kscience.kmath.nd.ShapeND +import space.kscience.kmath.nd.as2D +import space.kscience.kmath.nd.component1 +import space.kscience.kmath.tensors.LevenbergMarquardt.funcMiddleForLm +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.div +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra +import space.kscience.kmath.tensors.core.internal.LMSettings +import kotlin.math.roundToInt +fun main() { + val NData = 100 + var t_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(NData, 1))).as2D() + for (i in 0 until NData) { + t_example[i, 0] = t_example[i, 0] * (i + 1) + } + + val Nparams = 20 + var p_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))).as2D() + for (i in 0 until Nparams) { + p_example[i, 0] = p_example[i, 0] + i - 25 + } + + val settings = LMSettings(0, 0, 1) + + var y_hat = funcMiddleForLm(t_example, p_example, settings) + + var p_init = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() + for (i in 0 until Nparams) { + p_init[i, 0] = (p_example[i, 0] + 0.9) + } +// val p_init = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) +// val p_init = p_example + var t = t_example + val y_dat = y_hat + val weight = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 1.0 } + ).as2D() + val dp = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } + ).as2D() + var p_min = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) + p_min = p_min.div(1.0 / -50.0) + val p_max = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) + p_min = p_min.div(1.0 / 50.0) + val consts = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.0) + ).as2D() + val opts = doubleArrayOf(3.0, 10000.0, 1e-5, 1e-5, 1e-5, 1e-5, 1e-5, 11.0, 9.0, 1.0) + + val result = DoubleTensorAlgebra.lm( + ::funcMiddleForLm, + p_init.as2D(), + t, + y_dat, + weight, + dp, + p_min.as2D(), + p_max.as2D(), + consts, + opts, + 10, + 1 + ) + + println("Parameters:") + for (i in 0 until result.result_parameters.shape.component1()) { + val x = (result.result_parameters[i, 0] * 10000).roundToInt() / 10000.0 + print("$x ") + } + println() + + println("Y true and y received:") + var y_hat_after = funcMiddleForLm(t_example, result.result_parameters, settings) + for (i in 0 until y_hat.shape.component1()) { + val x = (y_hat[i, 0] * 10000).roundToInt() / 10000.0 + val y = (y_hat_after[i, 0] * 10000).roundToInt() / 10000.0 + println("$x $y") + } + + println("Сhi_sq:") + println(result.result_chi_sq) + println("Number of iterations:") + println(result.iterations) +} \ No newline at end of file diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/streamingLmMain.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt similarity index 72% rename from examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/streamingLmMain.kt rename to examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt index 0fa934955..5a1037618 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/streamingLmMain.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt @@ -3,20 +3,20 @@ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ -package space.kscience.kmath.tensors.StreamingLm +package space.kscience.kmath.tensors.LevenbergMarquardt.StreamingLm import kotlinx.coroutines.delay import kotlinx.coroutines.flow.* import space.kscience.kmath.nd.* +import space.kscience.kmath.tensors.LevenbergMarquardt.StartDataLm import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.zeros import space.kscience.kmath.tensors.core.DoubleTensorAlgebra import space.kscience.kmath.tensors.core.internal.LMSettings -import kotlin.math.roundToInt import kotlin.random.Random import kotlin.reflect.KFunction3 fun streamLm(lm_func: KFunction3, MutableStructure2D, LMSettings, MutableStructure2D>, - startData: StartDataLm, launchFrequencyInMs: Long): Flow> = flow{ + startData: StartDataLm, launchFrequencyInMs: Long, numberOfLaunches: Int): Flow> = flow{ var example_number = startData.example_number var p_init = startData.p_init @@ -29,7 +29,10 @@ fun streamLm(lm_func: KFunction3, MutableStructure2D< val consts = startData.consts val opts = startData.opts - while (true) { + var steps = numberOfLaunches + val isEndless = (steps <= 0) + + while (isEndless || steps > 0) { val result = DoubleTensorAlgebra.lm( lm_func, p_init, @@ -48,6 +51,7 @@ fun streamLm(lm_func: KFunction3, MutableStructure2D< delay(launchFrequencyInMs) p_init = result.result_parameters y_dat = generateNewYDat(y_dat, 0.1) + if (!isEndless) steps -= 1 } } @@ -59,18 +63,4 @@ fun generateNewYDat(y_dat: MutableStructure2D, delta: Double): MutableSt y_dat_new[i, 0] = y_dat[i, 0] + randomEps } return y_dat_new -} - -suspend fun main(){ - val startData = getStartDataForFunc1() - // Создание потока: - val lmFlow = streamLm(::func1ForLm, startData, 1000) - // Запуск потока - lmFlow.collect { parameters -> - for (i in 0 until parameters.shape.component1()) { - val x = (parameters[i, 0] * 10000).roundToInt() / 10000.0 - print("$x ") - if (i == parameters.shape.component1() - 1) println() - } - } } \ No newline at end of file diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamingLmTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamingLmTest.kt new file mode 100644 index 000000000..f81b048e5 --- /dev/null +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamingLmTest.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2018-2023 KMath contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package space.kscience.kmath.tensors.LevenbergMarquardt.StreamingLm + +import space.kscience.kmath.nd.* +import space.kscience.kmath.tensors.LevenbergMarquardt.funcEasyForLm +import space.kscience.kmath.tensors.LevenbergMarquardt.getStartDataForFuncEasy +import kotlin.math.roundToInt + +suspend fun main(){ + val startData = getStartDataForFuncEasy() + // Создание потока: + val lmFlow = streamLm(::funcEasyForLm, startData, 1000, 10) + // Запуск потока + lmFlow.collect { parameters -> + for (i in 0 until parameters.shape.component1()) { + val x = (parameters[i, 0] * 10000).roundToInt() / 10000.0 + print("$x ") + if (i == parameters.shape.component1() - 1) println() + } + } +} \ No newline at end of file diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/functionsToOptimize.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt similarity index 77% rename from examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/functionsToOptimize.kt rename to examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt index 1d0ce7deb..191dc5c67 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/StreamingLm/functionsToOptimize.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt @@ -3,7 +3,7 @@ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ -package space.kscience.kmath.tensors.StreamingLm +package space.kscience.kmath.tensors.LevenbergMarquardt import space.kscience.kmath.nd.MutableStructure2D import space.kscience.kmath.nd.ShapeND @@ -15,6 +15,7 @@ import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.max import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.plus import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.pow import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.times +import space.kscience.kmath.tensors.core.asDoubleTensor import space.kscience.kmath.tensors.core.internal.LMSettings public data class StartDataLm ( @@ -31,7 +32,39 @@ public data class StartDataLm ( var opts: DoubleArray ) -fun func1ForLm(t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings): MutableStructure2D { +fun funcDifficultForLm(t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings): MutableStructure2D { + val m = t.shape.component1() + var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf (m, 1))) + + val mt = t.max() + for(i in 0 until p.shape.component1()){ + y_hat = y_hat.plus( (t.times(1.0 / mt)).times(p[i, 0]) ) + } + + for(i in 0 until 4){ + y_hat = funcEasyForLm((y_hat.as2D() + t).as2D(), p, settings).asDoubleTensor() + } + + return y_hat.as2D() +} + +fun funcMiddleForLm(t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings): MutableStructure2D { + val m = t.shape.component1() + var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf (m, 1))) + + val mt = t.max() + for(i in 0 until p.shape.component1()){ + y_hat += (t.times(1.0 / mt)).times(p[i, 0]) + } + + for(i in 0 until 5){ + y_hat = funcEasyForLm(y_hat.as2D(), p, settings).asDoubleTensor() + } + + return y_hat.as2D() +} + +fun funcEasyForLm(t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings): MutableStructure2D { val m = t.shape.component1() var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf (m, 1))) @@ -55,7 +88,7 @@ fun func1ForLm(t: MutableStructure2D, p: MutableStructure2D, set return y_hat.as2D() } -fun getStartDataForFunc1(): StartDataLm { +fun getStartDataForFuncEasy(): StartDataLm { val lm_matx_y_dat = doubleArrayOf( 19.6594, 18.6096, 17.6792, 17.2747, 16.3065, 17.1458, 16.0467, 16.7023, 15.7809, 15.9807, 14.7620, 15.1128, 16.0973, 15.1934, 15.8636, 15.4763, 15.6860, 15.1895, 15.3495, 16.6054, diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt index 43f097564..2b8bf84c0 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt @@ -765,12 +765,12 @@ public open class DoubleTensorAlgebra : var weight = weight_input if (nargin < 5) { - weight = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf((y_dat.transpose().dot(y_dat)).as1D()[0])).as2D() + fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf(1.0)).as2D() } var dp = dp_input if (nargin < 6) { - dp = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.001)).as2D() + dp = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf(-0.001)).as2D() } var p_min = p_min_input @@ -1023,6 +1023,8 @@ public open class DoubleTensorAlgebra : // println(" !! Maximum Number of Iterations Reached Without Convergence !!") resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.noConvergence resultInfo.epsilon = 0.0 + print("noConvergence, MaxIter = ") + println(MaxIter) stop = true } } // --- End of Main Loop diff --git a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt new file mode 100644 index 000000000..29accbbfa --- /dev/null +++ b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt @@ -0,0 +1,267 @@ +/* + * Copyright 2018-2023 KMath contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package space.kscience.kmath.tensors.core + +import space.kscience.kmath.nd.MutableStructure2D +import space.kscience.kmath.nd.ShapeND +import space.kscience.kmath.nd.as2D +import space.kscience.kmath.nd.component1 +import space.kscience.kmath.operations.invoke +import space.kscience.kmath.tensors.api.LinearOpsTensorAlgebra +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.max +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.plus +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.pow +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.times +import space.kscience.kmath.tensors.core.internal.LMSettings +import kotlin.math.roundToLong +import kotlin.test.Test +import kotlin.test.assertEquals + +class TestLmAlgorithm { + companion object { + fun funcEasyForLm(t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings): MutableStructure2D { + val m = t.shape.component1() + var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(m, 1))) + + if (settings.example_number == 1) { + y_hat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))).times(p[0, 0]) + t.times(p[2, 0]).times( + DoubleTensorAlgebra.exp((t.times(-1.0 / p[3, 0]))) + ) + } + else if (settings.example_number == 2) { + val mt = t.max() + y_hat = (t.times(1.0 / mt)).times(p[0, 0]) + + (t.times(1.0 / mt)).pow(2).times(p[1, 0]) + + (t.times(1.0 / mt)).pow(3).times(p[2, 0]) + + (t.times(1.0 / mt)).pow(4).times(p[3, 0]) + } + else if (settings.example_number == 3) { + y_hat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))) + .times(p[0, 0]) + DoubleTensorAlgebra.sin((t.times(1.0 / p[3, 0]))).times(p[2, 0]) + } + + return y_hat.as2D() + } + + fun funcMiddleForLm(t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings): MutableStructure2D { + val m = t.shape.component1() + var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf (m, 1))) + + val mt = t.max() + for(i in 0 until p.shape.component1()){ + y_hat += (t.times(1.0 / mt)).times(p[i, 0]) + } + + for(i in 0 until 5){ + y_hat = funcEasyForLm(y_hat.as2D(), p, settings).asDoubleTensor() + } + + return y_hat.as2D() + } + + fun funcDifficultForLm(t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings): MutableStructure2D { + val m = t.shape.component1() + var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf (m, 1))) + + val mt = t.max() + for(i in 0 until p.shape.component1()){ + y_hat = y_hat.plus( (t.times(1.0 / mt)).times(p[i, 0]) ) + } + + for(i in 0 until 4){ + y_hat = funcEasyForLm((y_hat.as2D() + t).as2D(), p, settings).asDoubleTensor() + } + + return y_hat.as2D() + } + + } + @Test + fun testLMEasy() = DoubleTensorAlgebra { + val lm_matx_y_dat = doubleArrayOf( + 19.6594, 18.6096, 17.6792, 17.2747, 16.3065, 17.1458, 16.0467, 16.7023, 15.7809, 15.9807, + 14.7620, 15.1128, 16.0973, 15.1934, 15.8636, 15.4763, 15.6860, 15.1895, 15.3495, 16.6054, + 16.2247, 15.9854, 16.1421, 17.0960, 16.7769, 17.1997, 17.2767, 17.5882, 17.5378, 16.7894, + 17.7648, 18.2512, 18.1581, 16.7037, 17.8475, 17.9081, 18.3067, 17.9632, 18.2817, 19.1427, + 18.8130, 18.5658, 18.0056, 18.4607, 18.5918, 18.2544, 18.3731, 18.7511, 19.3181, 17.3066, + 17.9632, 19.0513, 18.7528, 18.2928, 18.5967, 17.8567, 17.7859, 18.4016, 18.9423, 18.4959, + 17.8000, 18.4251, 17.7829, 17.4645, 17.5221, 17.3517, 17.4637, 17.7563, 16.8471, 17.4558, + 17.7447, 17.1487, 17.3183, 16.8312, 17.7551, 17.0942, 15.6093, 16.4163, 15.3755, 16.6725, + 16.2332, 16.2316, 16.2236, 16.5361, 15.3721, 15.3347, 15.5815, 15.6319, 14.4538, 14.6044, + 14.7665, 13.3718, 15.0587, 13.8320, 14.7873, 13.6824, 14.2579, 14.2154, 13.5818, 13.8157 + ) + + var example_number = 1 + val p_init = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(4, 1)), doubleArrayOf(5.0, 2.0, 0.2, 10.0) + ).as2D() + + var t = ones(ShapeND(intArrayOf(100, 1))).as2D() + for (i in 0 until 100) { + t[i, 0] = t[i, 0] * (i + 1) + } + + val y_dat = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(100, 1)), lm_matx_y_dat + ).as2D() + + val weight = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 4.0 } + ).as2D() + + val dp = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } + ).as2D() + + val p_min = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(4, 1)), doubleArrayOf(-50.0, -20.0, -2.0, -100.0) + ).as2D() + + val p_max = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(4, 1)), doubleArrayOf(50.0, 20.0, 2.0, 100.0) + ).as2D() + + val consts = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.0) + ).as2D() + + val opts = doubleArrayOf(3.0, 100.0, 1e-3, 1e-3, 1e-1, 1e-1, 1e-2, 11.0, 9.0, 1.0) + + val result = lm(::funcEasyForLm, p_init, t, y_dat, weight, dp, p_min, p_max, consts, opts, 10, example_number) + assertEquals(13, result.iterations) + assertEquals(31, result.func_calls) + assertEquals(1, result.example_number) + assertEquals(0.9131368192633, (result.result_chi_sq * 1e13).roundToLong() / 1e13) + assertEquals(3.7790980 * 1e-7, (result.result_lambda * 1e13).roundToLong() / 1e13) + assertEquals(result.typeOfConvergence, LinearOpsTensorAlgebra.TypeOfConvergence.inParameters) + val expectedParameters = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(4, 1)), doubleArrayOf(20.527230909086, 9.833627103230, 0.997571256572, 50.174445822506) + ).as2D() + result.result_parameters = result.result_parameters.map { x -> (x * 1e12).toLong() / 1e12}.as2D() + val receivedParameters = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(4, 1)), doubleArrayOf(result.result_parameters[0, 0], result.result_parameters[1, 0], + result.result_parameters[2, 0], result.result_parameters[3, 0]) + ).as2D() + assertEquals(expectedParameters[0, 0], receivedParameters[0, 0]) + assertEquals(expectedParameters[1, 0], receivedParameters[1, 0]) + assertEquals(expectedParameters[2, 0], receivedParameters[2, 0]) + assertEquals(expectedParameters[3, 0], receivedParameters[3, 0]) + } + + @Test + fun TestLMMiddle() = DoubleTensorAlgebra { + val NData = 100 + var t_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(NData, 1))).as2D() + for (i in 0 until NData) { + t_example[i, 0] = t_example[i, 0] * (i + 1) + } + + val Nparams = 20 + var p_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))).as2D() + for (i in 0 until Nparams) { + p_example[i, 0] = p_example[i, 0] + i - 25 + } + + val settings = LMSettings(0, 0, 1) + + var y_hat = funcMiddleForLm(t_example, p_example, settings) + + var p_init = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() + for (i in 0 until Nparams) { + p_init[i, 0] = (p_example[i, 0] + 0.9) + } + + var t = t_example + val y_dat = y_hat + val weight = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 1.0 } + ).as2D() + val dp = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } + ).as2D() + var p_min = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) + p_min = p_min.div(1.0 / -50.0) + val p_max = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) + p_min = p_min.div(1.0 / 50.0) + val consts = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.0) + ).as2D() + val opts = doubleArrayOf(3.0, 7000.0, 1e-5, 1e-5, 1e-5, 1e-5, 1e-5, 11.0, 9.0, 1.0) + + val result = DoubleTensorAlgebra.lm( + ::funcMiddleForLm, + p_init.as2D(), + t, + y_dat, + weight, + dp, + p_min.as2D(), + p_max.as2D(), + consts, + opts, + 10, + 1 + ) + } + + @Test + fun TestLMDifficult() = DoubleTensorAlgebra { + val NData = 200 + var t_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(NData, 1))).as2D() + for (i in 0 until NData) { + t_example[i, 0] = t_example[i, 0] * (i + 1) - 104 + } + + val Nparams = 15 + var p_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))).as2D() + for (i in 0 until Nparams) { + p_example[i, 0] = p_example[i, 0] + i - 25 + } + + val settings = LMSettings(0, 0, 1) + + var y_hat = funcDifficultForLm(t_example, p_example, settings) + + var p_init = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() + for (i in 0 until Nparams) { + p_init[i, 0] = (p_example[i, 0] + 0.9) + } + + var t = t_example + val y_dat = y_hat + val weight = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 1.0 / Nparams * 1.0 - 0.085 } + ).as2D() + val dp = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } + ).as2D() + var p_min = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) + p_min = p_min.div(1.0 / -50.0) + val p_max = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) + p_min = p_min.div(1.0 / 50.0) + val consts = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.0) + ).as2D() + val opts = doubleArrayOf(3.0, 7000.0, 1e-2, 1e-1, 1e-2, 1e-2, 1e-2, 11.0, 9.0, 1.0) + + val result = DoubleTensorAlgebra.lm( + ::funcDifficultForLm, + p_init.as2D(), + t, + y_dat, + weight, + dp, + p_min.as2D(), + p_max.as2D(), + consts, + opts, + 10, + 1 + ) + +// assertEquals(1.15, (result.result_chi_sq * 1e2).roundToLong() / 1e2) + } +} \ No newline at end of file From 1afb0d0a4c3f88a03e04358e636745937aeab32d Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Mon, 29 May 2023 15:13:13 +0300 Subject: [PATCH 12/26] fixed time for js tests for lm --- kmath-tensors/build.gradle.kts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/kmath-tensors/build.gradle.kts b/kmath-tensors/build.gradle.kts index d27faeeef..fdd841bc3 100644 --- a/kmath-tensors/build.gradle.kts +++ b/kmath-tensors/build.gradle.kts @@ -4,7 +4,13 @@ plugins { kscience{ jvm() - js() + js { + browser { + testTask { + useMocha().timeout = "100000" + } + } + } native() dependencies { From 47600dff23a63e6d11341e509a698288114cb057 Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Tue, 6 Jun 2023 00:39:19 +0300 Subject: [PATCH 13/26] tests changed --- .../StaticLm/staticDifficultTest.kt | 4 +- .../StaticLm/staticMiddleTest.kt | 5 +- .../StreamingLm/streamingLmTest.kt | 15 +++- .../LevenbergMarquardt/functionsToOptimize.kt | 87 +++++++++++++++++++ kmath-tensors/build.gradle.kts | 2 +- .../kmath/tensors/core/TestLmAlgorithm.kt | 4 +- 6 files changed, 105 insertions(+), 12 deletions(-) diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt index 621943aea..0a502afa8 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt @@ -52,8 +52,8 @@ fun main() { val consts = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.0) ).as2D() - val opts = doubleArrayOf(3.0, 10000.0, 1e-2, 0.015, 1e-2, 1e-2, 1e-2, 11.0, 9.0, 1.0) -// val opts = doubleArrayOf(3.0, 10000.0, 1e-5, 1e-5, 1e-5, 1e-5, 1e-3, 11.0, 9.0, 1.0) + val opts = doubleArrayOf(3.0, 10000.0, 1e-6, 1e-6, 1e-6, 1e-6, 1e-2, 11.0, 9.0, 1.0) +// val opts = doubleArrayOf(3.0, 10000.0, 1e-6, 1e-6, 1e-6, 1e-6, 1e-3, 11.0, 9.0, 1.0) val result = DoubleTensorAlgebra.lm( ::funcDifficultForLm, diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt index a39572858..02917caf2 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt @@ -12,6 +12,7 @@ import space.kscience.kmath.tensors.LevenbergMarquardt.funcMiddleForLm import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.div import space.kscience.kmath.tensors.core.DoubleTensorAlgebra +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.times import space.kscience.kmath.tensors.core.internal.LMSettings import kotlin.math.roundToInt fun main() { @@ -52,7 +53,7 @@ fun main() { val consts = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.0) ).as2D() - val opts = doubleArrayOf(3.0, 10000.0, 1e-5, 1e-5, 1e-5, 1e-5, 1e-5, 11.0, 9.0, 1.0) + val opts = doubleArrayOf(3.0, 10000.0, 1e-3, 1e-3, 1e-3, 1e-3, 1e-15, 11.0, 9.0, 1.0) val result = DoubleTensorAlgebra.lm( ::funcMiddleForLm, @@ -76,7 +77,7 @@ fun main() { } println() - println("Y true and y received:") + var y_hat_after = funcMiddleForLm(t_example, result.result_parameters, settings) for (i in 0 until y_hat.shape.component1()) { val x = (y_hat[i, 0] * 10000).roundToInt() / 10000.0 diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamingLmTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamingLmTest.kt index f81b048e5..c9dd5029e 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamingLmTest.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamingLmTest.kt @@ -6,20 +6,27 @@ package space.kscience.kmath.tensors.LevenbergMarquardt.StreamingLm import space.kscience.kmath.nd.* -import space.kscience.kmath.tensors.LevenbergMarquardt.funcEasyForLm -import space.kscience.kmath.tensors.LevenbergMarquardt.getStartDataForFuncEasy +import space.kscience.kmath.tensors.LevenbergMarquardt.* import kotlin.math.roundToInt suspend fun main(){ - val startData = getStartDataForFuncEasy() + val startData = getStartDataForFuncDifficult() // Создание потока: - val lmFlow = streamLm(::funcEasyForLm, startData, 1000, 10) + val lmFlow = streamLm(::funcDifficultForLm, startData, 0, 100) + var initialTime = System.currentTimeMillis() + var lastTime: Long + val launches = mutableListOf() // Запуск потока lmFlow.collect { parameters -> + lastTime = System.currentTimeMillis() + launches.add(lastTime - initialTime) + initialTime = lastTime for (i in 0 until parameters.shape.component1()) { val x = (parameters[i, 0] * 10000).roundToInt() / 10000.0 print("$x ") if (i == parameters.shape.component1() - 1) println() } } + + println("Average without first is: ${launches.subList(1, launches.size - 1).average()}") } \ No newline at end of file diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt index 191dc5c67..5b194ab6b 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt @@ -10,6 +10,7 @@ import space.kscience.kmath.nd.ShapeND import space.kscience.kmath.nd.as2D import space.kscience.kmath.nd.component1 import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.div import space.kscience.kmath.tensors.core.DoubleTensorAlgebra import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.max import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.plus @@ -17,6 +18,7 @@ import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.pow import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.times import space.kscience.kmath.tensors.core.asDoubleTensor import space.kscience.kmath.tensors.core.internal.LMSettings +import kotlin.math.roundToInt public data class StartDataLm ( var lm_matx_y_dat: MutableStructure2D, @@ -88,6 +90,91 @@ fun funcEasyForLm(t: MutableStructure2D, p: MutableStructure2D, return y_hat.as2D() } +fun getStartDataForFuncDifficult(): StartDataLm { + val NData = 200 + var t_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(NData, 1))).as2D() + for (i in 0 until NData) { + t_example[i, 0] = t_example[i, 0] * (i + 1) - 104 + } + + val Nparams = 15 + var p_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))).as2D() + for (i in 0 until Nparams) { + p_example[i, 0] = p_example[i, 0] + i - 25 + } + + val settings = LMSettings(0, 0, 1) + + var y_hat = funcDifficultForLm(t_example, p_example, settings) + + var p_init = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() + for (i in 0 until Nparams) { + p_init[i, 0] = (p_example[i, 0] + 0.9) + } + + var t = t_example + val y_dat = y_hat + val weight = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 1.0 / Nparams * 1.0 - 0.085 } + ).as2D() + val dp = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } + ).as2D() + var p_min = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) + p_min = p_min.div(1.0 / -50.0) + val p_max = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) + p_min = p_min.div(1.0 / 50.0) + val consts = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.0) + ).as2D() + val opts = doubleArrayOf(3.0, 10000.0, 1e-2, 1e-3, 1e-2, 1e-2, 1e-2, 11.0, 9.0, 1.0) + + return StartDataLm(y_dat, 1, p_init, t, y_dat, weight, dp, p_min.as2D(), p_max.as2D(), consts, opts) +} + +fun getStartDataForFuncMiddle(): StartDataLm { + val NData = 100 + var t_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(NData, 1))).as2D() + for (i in 0 until NData) { + t_example[i, 0] = t_example[i, 0] * (i + 1) + } + + val Nparams = 20 + var p_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))).as2D() + for (i in 0 until Nparams) { + p_example[i, 0] = p_example[i, 0] + i - 25 + } + + val settings = LMSettings(0, 0, 1) + + var y_hat = funcMiddleForLm(t_example, p_example, settings) + + var p_init = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() + for (i in 0 until Nparams) { + p_init[i, 0] = (p_example[i, 0] + 10.0) + } + var t = t_example + val y_dat = y_hat + val weight = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 1.0 } + ).as2D() + val dp = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } + ).as2D() + var p_min = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) + p_min = p_min.div(1.0 / -50.0) + val p_max = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) + p_min = p_min.div(1.0 / 50.0) + val consts = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.0) + ).as2D() + val opts = doubleArrayOf(3.0, 10000.0, 1e-5, 1e-5, 1e-5, 1e-5, 1e-5, 11.0, 9.0, 1.0) + + var example_number = 1 + + return StartDataLm(y_dat, example_number, p_init, t, y_dat, weight, dp, p_min.as2D(), p_max.as2D(), consts, opts) +} + fun getStartDataForFuncEasy(): StartDataLm { val lm_matx_y_dat = doubleArrayOf( 19.6594, 18.6096, 17.6792, 17.2747, 16.3065, 17.1458, 16.0467, 16.7023, 15.7809, 15.9807, diff --git a/kmath-tensors/build.gradle.kts b/kmath-tensors/build.gradle.kts index fdd841bc3..5e82835a7 100644 --- a/kmath-tensors/build.gradle.kts +++ b/kmath-tensors/build.gradle.kts @@ -7,7 +7,7 @@ kscience{ js { browser { testTask { - useMocha().timeout = "100000" + useMocha().timeout = "0" } } } diff --git a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt index 29accbbfa..f554a742c 100644 --- a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt +++ b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt @@ -245,7 +245,7 @@ class TestLmAlgorithm { val consts = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.0) ).as2D() - val opts = doubleArrayOf(3.0, 7000.0, 1e-2, 1e-1, 1e-2, 1e-2, 1e-2, 11.0, 9.0, 1.0) + val opts = doubleArrayOf(3.0, 7000.0, 1e-2, 1e-3, 1e-2, 1e-2, 1e-2, 11.0, 9.0, 1.0) val result = DoubleTensorAlgebra.lm( ::funcDifficultForLm, @@ -261,7 +261,5 @@ class TestLmAlgorithm { 10, 1 ) - -// assertEquals(1.15, (result.result_chi_sq * 1e2).roundToLong() / 1e2) } } \ No newline at end of file From 8d81d2d8d5a39d0309dbd2c4b70b516092f5bfea Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Tue, 6 Jun 2023 01:41:08 +0300 Subject: [PATCH 14/26] move lm-algorithm from DoubleTensorAlgebra as extension --- .../tensors/api/LinearOpsTensorAlgebra.kt | 14 - .../kmath/tensors/core/DoubleTensorAlgebra.kt | 363 ------------ .../core/LevenbergMarquardtAlgorithm.kt | 553 ++++++++++++++++++ .../kmath/tensors/core/internal/linUtils.kt | 229 +------- .../tensors/core/TestDoubleTensorAlgebra.kt | 100 ---- .../kmath/tensors/core/TestLmAlgorithm.kt | 1 - 6 files changed, 554 insertions(+), 706 deletions(-) create mode 100644 kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt index 7964656f1..6b3859316 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt @@ -7,15 +7,7 @@ package space.kscience.kmath.tensors.api import space.kscience.kmath.nd.MutableStructure2D import space.kscience.kmath.nd.StructureND -import space.kscience.kmath.nd.as2D import space.kscience.kmath.operations.Field -import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra -import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.dot -import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.map -import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.transposed -import space.kscience.kmath.tensors.core.DoubleTensorAlgebra -import space.kscience.kmath.tensors.core.internal.LMSettings -import kotlin.reflect.KFunction3 /** * Common linear algebra operations. Operates on [Tensor]. @@ -137,10 +129,4 @@ public interface LinearOpsTensorAlgebra> : TensorPartialDivision var typeOfConvergence: TypeOfConvergence, var epsilon: Double ) - - public fun lm( - func: KFunction3, MutableStructure2D, LMSettings, MutableStructure2D>, - p_input: MutableStructure2D, t_input: MutableStructure2D, y_dat_input: MutableStructure2D, - weight_input: MutableStructure2D, dp_input: MutableStructure2D, p_min_input: MutableStructure2D, p_max_input: MutableStructure2D, - c_input: MutableStructure2D, opts_input: DoubleArray, nargin: Int, example_number: Int): LMResultInfo } diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt index 2b8bf84c0..c8cf56888 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/DoubleTensorAlgebra.kt @@ -9,7 +9,6 @@ package space.kscience.kmath.tensors.core import space.kscience.kmath.PerformancePitfall -import space.kscience.kmath.linear.transpose import space.kscience.kmath.nd.* import space.kscience.kmath.operations.DoubleBufferOps import space.kscience.kmath.operations.DoubleField @@ -19,7 +18,6 @@ import space.kscience.kmath.tensors.api.LinearOpsTensorAlgebra import space.kscience.kmath.tensors.api.Tensor import space.kscience.kmath.tensors.core.internal.* import kotlin.math.* -import kotlin.reflect.KFunction3 /** * Implementation of basic operations over double tensors and basic algebra operations on them. @@ -716,367 +714,6 @@ public open class DoubleTensorAlgebra : val aInverse = aSvd.third.dot(s).dot(aSvd.first.transposed()) return aInverse.dot(b).as2D() } - - override fun lm( - func: KFunction3, MutableStructure2D, LMSettings, MutableStructure2D>, - p_input: MutableStructure2D, t_input: MutableStructure2D, y_dat_input: MutableStructure2D, - weight_input: MutableStructure2D, dp_input: MutableStructure2D, p_min_input: MutableStructure2D, p_max_input: MutableStructure2D, - c_input: MutableStructure2D, opts_input: DoubleArray, nargin: Int, example_number: Int): LinearOpsTensorAlgebra.LMResultInfo { - - var resultInfo = LinearOpsTensorAlgebra.LMResultInfo(0, 0, example_number, 0.0, - 0.0, p_input, LinearOpsTensorAlgebra.TypeOfConvergence.noConvergence, 0.0) - - val eps:Double = 2.2204e-16 - - var settings = LMSettings(0, 0, example_number) - settings.func_calls = 0 // running count of function evaluations - - var p = p_input - val y_dat = y_dat_input - val t = t_input - - val Npar = length(p) // number of parameters - val Npnt = length(y_dat) // number of data points - var p_old = zeros(ShapeND(intArrayOf(Npar, 1))).as2D() // previous set of parameters - var y_old = zeros(ShapeND(intArrayOf(Npnt, 1))).as2D() // previous model, y_old = y_hat(t;p_old) - var X2 = 1e-3 / eps // a really big initial Chi-sq value - var X2_old = 1e-3 / eps // a really big initial Chi-sq value - var J = zeros(ShapeND(intArrayOf(Npnt, Npar))).as2D() // Jacobian matrix - val DoF = Npnt - Npar // statistical degrees of freedom - - var corr_p = 0 - var sigma_p = 0 - var sigma_y = 0 - var R_sq = 0 - var cvg_hist = 0 - - if (length(t) != length(y_dat)) { - // println("lm.m error: the length of t must equal the length of y_dat") - val length_t = length(t) - val length_y_dat = length(y_dat) - X2 = 0.0 - - corr_p = 0 - sigma_p = 0 - sigma_y = 0 - R_sq = 0 - cvg_hist = 0 - } - - var weight = weight_input - if (nargin < 5) { - fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf(1.0)).as2D() - } - - var dp = dp_input - if (nargin < 6) { - dp = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf(-0.001)).as2D() - } - - var p_min = p_min_input - if (nargin < 7) { - p_min = p - p_min.abs() - p_min = p_min.div(-100.0).as2D() - } - - var p_max = p_max_input - if (nargin < 8) { - p_max = p - p_max.abs() - p_max = p_max.div(100.0).as2D() - } - - var c = c_input - if (nargin < 9) { - c = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf(1.0)).as2D() - } - - var opts = opts_input - if (nargin < 10) { - opts = doubleArrayOf(3.0, 10.0 * Npar, 1e-3, 1e-3, 1e-1, 1e-1, 1e-2, 11.0, 9.0, 1.0) - } - - val prnt = opts[0] // >1 intermediate results; >2 plots - val MaxIter = opts[1].toInt() // maximum number of iterations - val epsilon_1 = opts[2] // convergence tolerance for gradient - val epsilon_2 = opts[3] // convergence tolerance for parameters - val epsilon_3 = opts[4] // convergence tolerance for Chi-square - val epsilon_4 = opts[5] // determines acceptance of a L-M step - val lambda_0 = opts[6] // initial value of damping paramter, lambda - val lambda_UP_fac = opts[7] // factor for increasing lambda - val lambda_DN_fac = opts[8] // factor for decreasing lambda - val Update_Type = opts[9].toInt() // 1: Levenberg-Marquardt lambda update - // 2: Quadratic update - // 3: Nielsen's lambda update equations - - p_min = make_column(p_min) - p_max = make_column(p_max) - - if (length(make_column(dp)) == 1) { - dp = ones(ShapeND(intArrayOf(Npar, 1))).div(1 / dp[0, 0]).as2D() - } - - val idx = get_zero_indices(dp) // indices of the parameters to be fit - val Nfit = idx?.shape?.component1() // number of parameters to fit - var stop = false // termination flag - val y_init = feval(func, t, p, settings) // residual error using p_try - - if (weight.shape.component1() == 1 || variance(weight) == 0.0) { // identical weights vector - weight = ones(ShapeND(intArrayOf(Npnt, 1))).div(1 / abs(weight[0, 0])).as2D() - // println("using uniform weights for error analysis") - } - else { - weight = make_column(weight) - weight.abs() - } - - // initialize Jacobian with finite difference calculation - var lm_matx_ans = lm_matx(func, t, p_old, y_old,1, J, p, y_dat, weight, dp, settings) - var JtWJ = lm_matx_ans[0] - var JtWdy = lm_matx_ans[1] - X2 = lm_matx_ans[2][0, 0] - var y_hat = lm_matx_ans[3] - J = lm_matx_ans[4] - - if ( abs(JtWdy).max()!! < epsilon_1 ) { -// println(" *** Your Initial Guess is Extremely Close to Optimal ***\n") -// println(" *** epsilon_1 = %e\n$epsilon_1") - stop = true - } - - var lambda = 1.0 - var nu = 1 - when (Update_Type) { - 1 -> lambda = lambda_0 // Marquardt: init'l lambda - else -> { // Quadratic and Nielsen - lambda = lambda_0 * (diag(JtWJ)).max()!! - nu = 2 - } - } - - X2_old = X2 // previous value of X2 - var cvg_hst = ones(ShapeND(intArrayOf(MaxIter, Npar + 3))) // initialize convergence history - - var h: DoubleTensor - var dX2 = X2 - while (!stop && settings.iteration <= MaxIter) { //--- Start Main Loop - settings.iteration += 1 - - // incremental change in parameters - h = when (Update_Type) { - 1 -> { // Marquardt - val solve = solve(JtWJ.plus(make_matrx_with_diagonal(diag(JtWJ)).div(1 / lambda)).as2D(), JtWdy) - solve.asDoubleTensor() - } - - else -> { // Quadratic and Nielsen - val solve = solve(JtWJ.plus(lm_eye(Npar).div(1 / lambda)).as2D(), JtWdy) - solve.asDoubleTensor() - } - } - - var p_try = (p + h).as2D() // update the [idx] elements - p_try = smallest_element_comparison(largest_element_comparison(p_min, p_try.as2D()), p_max) // apply constraints - - var delta_y = y_dat.minus(feval(func, t, p_try, settings)) // residual error using p_try - - for (i in 0 until delta_y.shape.component1()) { // floating point error; break - for (j in 0 until delta_y.shape.component2()) { - if (delta_y[i, j] == Double.POSITIVE_INFINITY || delta_y[i, j] == Double.NEGATIVE_INFINITY) { - stop = true - break - } - } - } - - settings.func_calls += 1 - - val tmp = delta_y.times(weight) - var X2_try = delta_y.as2D().transpose().dot(tmp) // Chi-squared error criteria - - val alpha = 1.0 - if (Update_Type == 2) { // Quadratic - // One step of quadratic line update in the h direction for minimum X2 - - val alpha = JtWdy.transpose().dot(h) / ( (X2_try.minus(X2)).div(2.0).plus(2 * JtWdy.transpose().dot(h)) ) - h = h.dot(alpha) - p_try = p.plus(h).as2D() // update only [idx] elements - p_try = smallest_element_comparison(largest_element_comparison(p_min, p_try), p_max) // apply constraints - - var delta_y = y_dat.minus(feval(func, t, p_try, settings)) // residual error using p_try - settings.func_calls += 1 - - val tmp = delta_y.times(weight) - X2_try = delta_y.as2D().transpose().dot(tmp) // Chi-squared error criteria - } - - val rho = when (Update_Type) { // Nielsen - 1 -> { - val tmp = h.transposed().dot(make_matrx_with_diagonal(diag(JtWJ)).div(1 / lambda).dot(h).plus(JtWdy)) - X2.minus(X2_try).as2D()[0, 0] / abs(tmp.as2D()).as2D()[0, 0] - } - else -> { - val tmp = h.transposed().dot(h.div(1 / lambda).plus(JtWdy)) - X2.minus(X2_try).as2D()[0, 0] / abs(tmp.as2D()).as2D()[0, 0] - } - } - - if (rho > epsilon_4) { // it IS significantly better - val dX2 = X2.minus(X2_old) - X2_old = X2 - p_old = p.copyToTensor().as2D() - y_old = y_hat.copyToTensor().as2D() - p = make_column(p_try) // accept p_try - - lm_matx_ans = lm_matx(func, t, p_old, y_old, dX2.toInt(), J, p, y_dat, weight, dp, settings) - // decrease lambda ==> Gauss-Newton method - - JtWJ = lm_matx_ans[0] - JtWdy = lm_matx_ans[1] - X2 = lm_matx_ans[2][0, 0] - y_hat = lm_matx_ans[3] - J = lm_matx_ans[4] - - lambda = when (Update_Type) { - 1 -> { // Levenberg - max(lambda / lambda_DN_fac, 1e-7); - } - 2 -> { // Quadratic - max( lambda / (1 + alpha) , 1e-7 ); - } - else -> { // Nielsen - nu = 2 - lambda * max( 1.0 / 3, 1 - (2 * rho - 1).pow(3) ) - } - } - } - else { // it IS NOT better - X2 = X2_old // do not accept p_try - if (settings.iteration % (2 * Npar) == 0 ) { // rank-1 update of Jacobian - lm_matx_ans = lm_matx(func, t, p_old, y_old,-1, J, p, y_dat, weight, dp, settings) - JtWJ = lm_matx_ans[0] - JtWdy = lm_matx_ans[1] - dX2 = lm_matx_ans[2][0, 0] - y_hat = lm_matx_ans[3] - J = lm_matx_ans[4] - } - - // increase lambda ==> gradient descent method - lambda = when (Update_Type) { - 1 -> { // Levenberg - min(lambda * lambda_UP_fac, 1e7) - } - 2 -> { // Quadratic - lambda + abs(((X2_try.as2D()[0, 0] - X2) / 2) / alpha) - } - else -> { // Nielsen - nu *= 2 - lambda * (nu / 2) - } - } - } - - if (prnt > 1) { - val chi_sq = X2 / DoF -// println("Iteration $settings | chi_sq=$chi_sq | lambda=$lambda") -// print("param: ") -// for (pn in 0 until Npar) { -// print(p[pn, 0].toString() + " ") -// } -// print("\ndp/p: ") -// for (pn in 0 until Npar) { -// print((h.as2D()[pn, 0] / p[pn, 0]).toString() + " ") -// } - resultInfo.iterations = settings.iteration - resultInfo.func_calls = settings.func_calls - resultInfo.result_chi_sq = chi_sq - resultInfo.result_lambda = lambda - resultInfo.result_parameters = p - } - - // update convergence history ... save _reduced_ Chi-square - // cvg_hst(iteration,:) = [ func_calls p' X2/DoF lambda ]; - - if (abs(JtWdy).max()!! < epsilon_1 && settings.iteration > 2) { -// println(" **** Convergence in r.h.s. (\"JtWdy\") ****") -// println(" **** epsilon_1 = $epsilon_1") - resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.inRHS_JtWdy - resultInfo.epsilon = epsilon_1 - stop = true - } - if ((abs(h.as2D()).div(abs(p) + 1e-12)).max() < epsilon_2 && settings.iteration > 2) { -// println(" **** Convergence in Parameters ****") -// println(" **** epsilon_2 = $epsilon_2") - resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.inParameters - resultInfo.epsilon = epsilon_2 - stop = true - } - if (X2 / DoF < epsilon_3 && settings.iteration > 2) { -// println(" **** Convergence in reduced Chi-square **** ") -// println(" **** epsilon_3 = $epsilon_3") - resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.inReducedChi_square - resultInfo.epsilon = epsilon_3 - stop = true - } - if (settings.iteration == MaxIter) { -// println(" !! Maximum Number of Iterations Reached Without Convergence !!") - resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.noConvergence - resultInfo.epsilon = 0.0 - print("noConvergence, MaxIter = ") - println(MaxIter) - stop = true - } - } // --- End of Main Loop - - // --- convergence achieved, find covariance and confidence intervals - - // ---- Error Analysis ---- - -// if (weight.shape.component1() == 1 || weight.variance() == 0.0) { -// weight = DoF / (delta_y.transpose().dot(delta_y)) * ones(intArrayOf(Npt, 1)) -// } - -// if (nargout > 1) { -// val redX2 = X2 / DoF -// } -// -// lm_matx_ans = lm_matx(func, t, p_old, y_old, -1, J, p, y_dat, weight, dp) -// JtWJ = lm_matx_ans[0] -// JtWdy = lm_matx_ans[1] -// X2 = lm_matx_ans[2][0, 0] -// y_hat = lm_matx_ans[3] -// J = lm_matx_ans[4] -// -// if (nargout > 2) { // standard error of parameters -// covar_p = inv(JtWJ); -// siif nagma_p = sqrt(diag(covar_p)); -// } -// -// if (nargout > 3) { // standard error of the fit -// /// sigma_y = sqrt(diag(J * covar_p * J')); // slower version of below -// sigma_y = zeros(Npnt,1); -// for i=1:Npnt -// sigma_y(i) = J(i,:) * covar_p * J(i,:)'; -// end -// sigma_y = sqrt(sigma_y); -// } -// -// if (nargout > 4) { // parameter correlation matrix -// corr_p = covar_p ./ [sigma_p*sigma_p']; -// } -// -// if (nargout > 5) { // coefficient of multiple determination -// R_sq = corr([y_dat y_hat]); -// R_sq = R_sq(1,2).^2; -// } -// -// if (nargout > 6) { // convergence history -// cvg_hst = cvg_hst(1:iteration,:); -// } - - return resultInfo - } } public val Double.Companion.tensorAlgebra: DoubleTensorAlgebra get() = DoubleTensorAlgebra diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt new file mode 100644 index 000000000..f4b50626a --- /dev/null +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt @@ -0,0 +1,553 @@ +/* + * Copyright 2018-2023 KMath contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package space.kscience.kmath.tensors.core + +import space.kscience.kmath.linear.transpose +import space.kscience.kmath.nd.* +import space.kscience.kmath.tensors.api.LinearOpsTensorAlgebra +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.div +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.dot +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.minus +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.times +import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.transposed +import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.plus +import kotlin.math.max +import kotlin.math.min +import kotlin.math.pow +import kotlin.reflect.KFunction3 + +public fun DoubleTensorAlgebra.lm( + func: KFunction3, MutableStructure2D, LMSettings, MutableStructure2D>, + p_input: MutableStructure2D, t_input: MutableStructure2D, y_dat_input: MutableStructure2D, + weight_input: MutableStructure2D, dp_input: MutableStructure2D, p_min_input: MutableStructure2D, p_max_input: MutableStructure2D, + c_input: MutableStructure2D, opts_input: DoubleArray, nargin: Int, example_number: Int): LinearOpsTensorAlgebra.LMResultInfo { + + val resultInfo = LinearOpsTensorAlgebra.LMResultInfo(0, 0, example_number, 0.0, + 0.0, p_input, LinearOpsTensorAlgebra.TypeOfConvergence.noConvergence, 0.0) + + val eps:Double = 2.2204e-16 + + val settings = LMSettings(0, 0, example_number) + settings.func_calls = 0 // running count of function evaluations + + var p = p_input + val y_dat = y_dat_input + val t = t_input + + val Npar = length(p) // number of parameters + val Npnt = length(y_dat) // number of data points + var p_old = zeros(ShapeND(intArrayOf(Npar, 1))).as2D() // previous set of parameters + var y_old = zeros(ShapeND(intArrayOf(Npnt, 1))).as2D() // previous model, y_old = y_hat(t;p_old) + var X2 = 1e-3 / eps // a really big initial Chi-sq value + var X2_old = 1e-3 / eps // a really big initial Chi-sq value + var J = zeros(ShapeND(intArrayOf(Npnt, Npar))).as2D() // Jacobian matrix + val DoF = Npnt - Npar // statistical degrees of freedom + + var corr_p = 0 + var sigma_p = 0 + var sigma_y = 0 + var R_sq = 0 + var cvg_hist = 0 + + if (length(t) != length(y_dat)) { + // println("lm.m error: the length of t must equal the length of y_dat") + val length_t = length(t) + val length_y_dat = length(y_dat) + X2 = 0.0 + + corr_p = 0 + sigma_p = 0 + sigma_y = 0 + R_sq = 0 + cvg_hist = 0 + } + + var weight = weight_input + if (nargin < 5) { + weight = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf((y_dat.transpose().dot(y_dat)).as1D()[0])).as2D() + } + + var dp = dp_input + if (nargin < 6) { + dp = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.001)).as2D() + } + + var p_min = p_min_input + if (nargin < 7) { + p_min = p + p_min.abs() + p_min = p_min.div(-100.0).as2D() + } + + var p_max = p_max_input + if (nargin < 8) { + p_max = p + p_max.abs() + p_max = p_max.div(100.0).as2D() + } + + var c = c_input + if (nargin < 9) { + c = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf(1.0)).as2D() + } + + var opts = opts_input + if (nargin < 10) { + opts = doubleArrayOf(3.0, 10.0 * Npar, 1e-3, 1e-3, 1e-1, 1e-1, 1e-2, 11.0, 9.0, 1.0) + } + + val prnt = opts[0] // >1 intermediate results; >2 plots + val MaxIter = opts[1].toInt() // maximum number of iterations + val epsilon_1 = opts[2] // convergence tolerance for gradient + val epsilon_2 = opts[3] // convergence tolerance for parameters + val epsilon_3 = opts[4] // convergence tolerance for Chi-square + val epsilon_4 = opts[5] // determines acceptance of a L-M step + val lambda_0 = opts[6] // initial value of damping paramter, lambda + val lambda_UP_fac = opts[7] // factor for increasing lambda + val lambda_DN_fac = opts[8] // factor for decreasing lambda + val Update_Type = opts[9].toInt() // 1: Levenberg-Marquardt lambda update + // 2: Quadratic update + // 3: Nielsen's lambda update equations + + p_min = make_column(p_min) + p_max = make_column(p_max) + + if (length(make_column(dp)) == 1) { + dp = ones(ShapeND(intArrayOf(Npar, 1))).div(1 / dp[0, 0]).as2D() + } + + val idx = get_zero_indices(dp) // indices of the parameters to be fit + val Nfit = idx?.shape?.component1() // number of parameters to fit + var stop = false // termination flag + val y_init = feval(func, t, p, settings) // residual error using p_try + + if (weight.shape.component1() == 1 || variance(weight) == 0.0) { // identical weights vector + weight = ones(ShapeND(intArrayOf(Npnt, 1))).div(1 / kotlin.math.abs(weight[0, 0])).as2D() + // println("using uniform weights for error analysis") + } + else { + weight = make_column(weight) + weight.abs() + } + + // initialize Jacobian with finite difference calculation + var lm_matx_ans = lm_matx(func, t, p_old, y_old,1, J, p, y_dat, weight, dp, settings) + var JtWJ = lm_matx_ans[0] + var JtWdy = lm_matx_ans[1] + X2 = lm_matx_ans[2][0, 0] + var y_hat = lm_matx_ans[3] + J = lm_matx_ans[4] + + if ( abs(JtWdy).max()!! < epsilon_1 ) { +// println(" *** Your Initial Guess is Extremely Close to Optimal ***\n") +// println(" *** epsilon_1 = %e\n$epsilon_1") + stop = true + } + + var lambda = 1.0 + var nu = 1 + when (Update_Type) { + 1 -> lambda = lambda_0 // Marquardt: init'l lambda + else -> { // Quadratic and Nielsen + lambda = lambda_0 * (diag(JtWJ)).max()!! + nu = 2 + } + } + + X2_old = X2 // previous value of X2 + var cvg_hst = ones(ShapeND(intArrayOf(MaxIter, Npar + 3))) // initialize convergence history + + var h: DoubleTensor + var dX2 = X2 + while (!stop && settings.iteration <= MaxIter) { //--- Start Main Loop + settings.iteration += 1 + + // incremental change in parameters + h = when (Update_Type) { + 1 -> { // Marquardt + val solve = solve(JtWJ.plus(make_matrx_with_diagonal(diag(JtWJ)).div(1 / lambda)).as2D(), JtWdy) + solve.asDoubleTensor() + } + + else -> { // Quadratic and Nielsen + val solve = solve(JtWJ.plus(lm_eye(Npar).div(1 / lambda)).as2D(), JtWdy) + solve.asDoubleTensor() + } + } + + var p_try = (p + h).as2D() // update the [idx] elements + p_try = smallest_element_comparison(largest_element_comparison(p_min, p_try.as2D()), p_max) // apply constraints + + var delta_y = y_dat.minus(feval(func, t, p_try, settings)) // residual error using p_try + + for (i in 0 until delta_y.shape.component1()) { // floating point error; break + for (j in 0 until delta_y.shape.component2()) { + if (delta_y[i, j] == Double.POSITIVE_INFINITY || delta_y[i, j] == Double.NEGATIVE_INFINITY) { + stop = true + break + } + } + } + + settings.func_calls += 1 + + val tmp = delta_y.times(weight) + var X2_try = delta_y.as2D().transpose().dot(tmp) // Chi-squared error criteria + + val alpha = 1.0 + if (Update_Type == 2) { // Quadratic + // One step of quadratic line update in the h direction for minimum X2 + + val alpha = JtWdy.transpose().dot(h) / ( (X2_try.minus(X2)).div(2.0).plus(2 * JtWdy.transpose().dot(h)) ) + h = h.dot(alpha) + p_try = p.plus(h).as2D() // update only [idx] elements + p_try = smallest_element_comparison(largest_element_comparison(p_min, p_try), p_max) // apply constraints + + var delta_y = y_dat.minus(feval(func, t, p_try, settings)) // residual error using p_try + settings.func_calls += 1 + + val tmp = delta_y.times(weight) + X2_try = delta_y.as2D().transpose().dot(tmp) // Chi-squared error criteria + } + + val rho = when (Update_Type) { // Nielsen + 1 -> { + val tmp = h.transposed().dot(make_matrx_with_diagonal(diag(JtWJ)).div(1 / lambda).dot(h).plus(JtWdy)) + X2.minus(X2_try).as2D()[0, 0] / abs(tmp.as2D()).as2D()[0, 0] + } + else -> { + val tmp = h.transposed().dot(h.div(1 / lambda).plus(JtWdy)) + X2.minus(X2_try).as2D()[0, 0] / abs(tmp.as2D()).as2D()[0, 0] + } + } + + if (rho > epsilon_4) { // it IS significantly better + val dX2 = X2.minus(X2_old) + X2_old = X2 + p_old = p.copyToTensor().as2D() + y_old = y_hat.copyToTensor().as2D() + p = make_column(p_try) // accept p_try + + lm_matx_ans = lm_matx(func, t, p_old, y_old, dX2.toInt(), J, p, y_dat, weight, dp, settings) + // decrease lambda ==> Gauss-Newton method + + JtWJ = lm_matx_ans[0] + JtWdy = lm_matx_ans[1] + X2 = lm_matx_ans[2][0, 0] + y_hat = lm_matx_ans[3] + J = lm_matx_ans[4] + + lambda = when (Update_Type) { + 1 -> { // Levenberg + max(lambda / lambda_DN_fac, 1e-7); + } + 2 -> { // Quadratic + max( lambda / (1 + alpha) , 1e-7 ); + } + else -> { // Nielsen + nu = 2 + lambda * max( 1.0 / 3, 1 - (2 * rho - 1).pow(3) ) + } + } + } + else { // it IS NOT better + X2 = X2_old // do not accept p_try + if (settings.iteration % (2 * Npar) == 0 ) { // rank-1 update of Jacobian + lm_matx_ans = lm_matx(func, t, p_old, y_old,-1, J, p, y_dat, weight, dp, settings) + JtWJ = lm_matx_ans[0] + JtWdy = lm_matx_ans[1] + dX2 = lm_matx_ans[2][0, 0] + y_hat = lm_matx_ans[3] + J = lm_matx_ans[4] + } + + // increase lambda ==> gradient descent method + lambda = when (Update_Type) { + 1 -> { // Levenberg + min(lambda * lambda_UP_fac, 1e7) + } + 2 -> { // Quadratic + lambda + kotlin.math.abs(((X2_try.as2D()[0, 0] - X2) / 2) / alpha) + } + else -> { // Nielsen + nu *= 2 + lambda * (nu / 2) + } + } + } + + if (prnt > 1) { + val chi_sq = X2 / DoF +// println("Iteration $settings | chi_sq=$chi_sq | lambda=$lambda") +// print("param: ") +// for (pn in 0 until Npar) { +// print(p[pn, 0].toString() + " ") +// } +// print("\ndp/p: ") +// for (pn in 0 until Npar) { +// print((h.as2D()[pn, 0] / p[pn, 0]).toString() + " ") +// } + resultInfo.iterations = settings.iteration + resultInfo.func_calls = settings.func_calls + resultInfo.result_chi_sq = chi_sq + resultInfo.result_lambda = lambda + resultInfo.result_parameters = p + } + + // update convergence history ... save _reduced_ Chi-square + // cvg_hst(iteration,:) = [ func_calls p' X2/DoF lambda ]; + + if (abs(JtWdy).max()!! < epsilon_1 && settings.iteration > 2) { +// println(" **** Convergence in r.h.s. (\"JtWdy\") ****") +// println(" **** epsilon_1 = $epsilon_1") + resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.inRHS_JtWdy + resultInfo.epsilon = epsilon_1 + stop = true + } + if ((abs(h.as2D()).div(abs(p) + 1e-12)).max() < epsilon_2 && settings.iteration > 2) { +// println(" **** Convergence in Parameters ****") +// println(" **** epsilon_2 = $epsilon_2") + resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.inParameters + resultInfo.epsilon = epsilon_2 + stop = true + } + if (X2 / DoF < epsilon_3 && settings.iteration > 2) { +// println(" **** Convergence in reduced Chi-square **** ") +// println(" **** epsilon_3 = $epsilon_3") + resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.inReducedChi_square + resultInfo.epsilon = epsilon_3 + stop = true + } + if (settings.iteration == MaxIter) { +// println(" !! Maximum Number of Iterations Reached Without Convergence !!") + resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.noConvergence + resultInfo.epsilon = 0.0 + stop = true + } + } // --- End of Main Loop + return resultInfo +} + +public data class LMSettings ( + var iteration:Int, + var func_calls: Int, + var example_number:Int +) + +/* matrix -> column of all elemnets */ +public fun make_column(tensor: MutableStructure2D) : MutableStructure2D { + val shape = intArrayOf(tensor.shape.component1() * tensor.shape.component2(), 1) + val buffer = DoubleArray(tensor.shape.component1() * tensor.shape.component2()) + for (i in 0 until tensor.shape.component1()) { + for (j in 0 until tensor.shape.component2()) { + buffer[i * tensor.shape.component2() + j] = tensor[i, j] + } + } + val column = BroadcastDoubleTensorAlgebra.fromArray(ShapeND(shape), buffer).as2D() + return column +} + +/* column length */ +public fun length(column: MutableStructure2D) : Int { + return column.shape.component1() +} + +public fun MutableStructure2D.abs() { + for (i in 0 until this.shape.component1()) { + for (j in 0 until this.shape.component2()) { + this[i, j] = kotlin.math.abs(this[i, j]) + } + } +} + +public fun abs(input: MutableStructure2D): MutableStructure2D { + val tensor = BroadcastDoubleTensorAlgebra.ones( + ShapeND( + intArrayOf( + input.shape.component1(), + input.shape.component2() + ) + ) + ).as2D() + for (i in 0 until tensor.shape.component1()) { + for (j in 0 until tensor.shape.component2()) { + tensor[i, j] = kotlin.math.abs(input[i, j]) + } + } + return tensor +} + +public fun diag(input: MutableStructure2D): MutableStructure2D { + val tensor = BroadcastDoubleTensorAlgebra.ones(ShapeND(intArrayOf(input.shape.component1(), 1))).as2D() + for (i in 0 until tensor.shape.component1()) { + tensor[i, 0] = input[i, i] + } + return tensor +} + +public fun make_matrx_with_diagonal(column: MutableStructure2D): MutableStructure2D { + val size = column.shape.component1() + val tensor = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(size, size))).as2D() + for (i in 0 until size) { + tensor[i, i] = column[i, 0] + } + return tensor +} + +public fun lm_eye(size: Int): MutableStructure2D { + val column = BroadcastDoubleTensorAlgebra.ones(ShapeND(intArrayOf(size, 1))).as2D() + return make_matrx_with_diagonal(column) +} + +public fun largest_element_comparison(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D { + val a_sizeX = a.shape.component1() + val a_sizeY = a.shape.component2() + val b_sizeX = b.shape.component1() + val b_sizeY = b.shape.component2() + val tensor = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(max(a_sizeX, b_sizeX), max(a_sizeY, b_sizeY)))).as2D() + for (i in 0 until tensor.shape.component1()) { + for (j in 0 until tensor.shape.component2()) { + if (i < a_sizeX && i < b_sizeX && j < a_sizeY && j < b_sizeY) { + tensor[i, j] = max(a[i, j], b[i, j]) + } + else if (i < a_sizeX && j < a_sizeY) { + tensor[i, j] = a[i, j] + } + else { + tensor[i, j] = b[i, j] + } + } + } + return tensor +} + +public fun smallest_element_comparison(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D { + val a_sizeX = a.shape.component1() + val a_sizeY = a.shape.component2() + val b_sizeX = b.shape.component1() + val b_sizeY = b.shape.component2() + val tensor = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(max(a_sizeX, b_sizeX), max(a_sizeY, b_sizeY)))).as2D() + for (i in 0 until tensor.shape.component1()) { + for (j in 0 until tensor.shape.component2()) { + if (i < a_sizeX && i < b_sizeX && j < a_sizeY && j < b_sizeY) { + tensor[i, j] = min(a[i, j], b[i, j]) + } + else if (i < a_sizeX && j < a_sizeY) { + tensor[i, j] = a[i, j] + } + else { + tensor[i, j] = b[i, j] + } + } + } + return tensor +} + +public fun get_zero_indices(column: MutableStructure2D, epsilon: Double = 0.000001): MutableStructure2D? { + var idx = emptyArray() + for (i in 0 until column.shape.component1()) { + if (kotlin.math.abs(column[i, 0]) > epsilon) { + idx += (i + 1.0) + } + } + if (idx.size > 0) { + return BroadcastDoubleTensorAlgebra.fromArray(ShapeND(intArrayOf(idx.size, 1)), idx.toDoubleArray()).as2D() + } + return null +} + +public fun feval(func: (MutableStructure2D, MutableStructure2D, LMSettings) -> MutableStructure2D, + t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings) + : MutableStructure2D +{ + return func(t, p, settings) +} + +public fun lm_matx(func: (MutableStructure2D, MutableStructure2D, LMSettings) -> MutableStructure2D, + t: MutableStructure2D, p_old: MutableStructure2D, y_old: MutableStructure2D, + dX2: Int, J_input: MutableStructure2D, p: MutableStructure2D, + y_dat: MutableStructure2D, weight: MutableStructure2D, dp:MutableStructure2D, settings:LMSettings) : Array> +{ + // default: dp = 0.001 + + val Npnt = length(y_dat) // number of data points + val Npar = length(p) // number of parameters + + val y_hat = feval(func, t, p, settings) // evaluate model using parameters 'p' + settings.func_calls += 1 + + var J = J_input + + if (settings.iteration % (2 * Npar) == 0 || dX2 > 0) { + J = lm_FD_J(func, t, p, y_hat, dp, settings).as2D() // finite difference + } + else { + J = lm_Broyden_J(p_old, y_old, J, p, y_hat).as2D() // rank-1 update + } + + val delta_y = y_dat.minus(y_hat) + + val Chi_sq = delta_y.transposed().dot( delta_y.times(weight) ).as2D() + val JtWJ = J.transposed().dot ( J.times( weight.dot(BroadcastDoubleTensorAlgebra.ones(ShapeND(intArrayOf(1, Npar)))) ) ).as2D() + val JtWdy = J.transposed().dot( weight.times(delta_y) ).as2D() + + return arrayOf(JtWJ,JtWdy,Chi_sq,y_hat,J) +} + +public fun lm_Broyden_J(p_old: MutableStructure2D, y_old: MutableStructure2D, J_input: MutableStructure2D, + p: MutableStructure2D, y: MutableStructure2D): MutableStructure2D { + var J = J_input.copyToTensor() + + val h = p.minus(p_old) + val increase = y.minus(y_old).minus( J.dot(h) ).dot(h.transposed()).div( (h.transposed().dot(h)).as2D()[0, 0] ) + J = J.plus(increase) + + return J.as2D() +} + +public fun lm_FD_J(func: (MutableStructure2D, MutableStructure2D, settings: LMSettings) -> MutableStructure2D, + t: MutableStructure2D, p: MutableStructure2D, y: MutableStructure2D, + dp: MutableStructure2D, settings: LMSettings): MutableStructure2D { + // default: dp = 0.001 * ones(1,n) + + val m = length(y) // number of data points + val n = length(p) // number of parameters + + val ps = p.copyToTensor().as2D() + val J = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(m, n))).as2D() // initialize Jacobian to Zero + val del = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(n, 1))).as2D() + + for (j in 0 until n) { + + del[j, 0] = dp[j, 0] * (1 + kotlin.math.abs(p[j, 0])) // parameter perturbation + p[j, 0] = ps[j, 0] + del[j, 0] // perturb parameter p(j) + + val epsilon = 0.0000001 + if (kotlin.math.abs(del[j, 0]) > epsilon) { + val y1 = feval(func, t, p, settings) + settings.func_calls += 1 + + if (dp[j, 0] < 0) { // backwards difference + for (i in 0 until J.shape.component1()) { + J[i, j] = (y1.as2D().minus(y).as2D())[i, 0] / del[j, 0] + } + } + else { + // Do tests for it + println("Potential mistake") + p[j, 0] = ps[j, 0] - del[j, 0] // central difference, additional func call + for (i in 0 until J.shape.component1()) { + J[i, j] = (y1.as2D().minus(feval(func, t, p, settings)).as2D())[i, 0] / (2 * del[j, 0]) + } + settings.func_calls += 1 + } + } + + p[j, 0] = ps[j, 0] // restore p(j) + } + + return J.as2D() +} diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/internal/linUtils.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/internal/linUtils.kt index 6e5456c62..086c69e49 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/internal/linUtils.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/internal/linUtils.kt @@ -9,12 +9,6 @@ import space.kscience.kmath.nd.* import space.kscience.kmath.operations.invoke import space.kscience.kmath.structures.* import space.kscience.kmath.tensors.core.* -import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.div -import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.dot -import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.minus -import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.times -import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.transposed -import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.plus import kotlin.math.abs import kotlin.math.max import kotlin.math.min @@ -607,225 +601,4 @@ internal fun MutableStructure2D.svdGolubKahanHelper(u: MutableStructure2 u[j, i] = this[j, i] } } -} - -data class LMSettings ( - var iteration:Int, - var func_calls: Int, - var example_number:Int -) - -/* matrix -> column of all elemnets */ -fun make_column(tensor: MutableStructure2D) : MutableStructure2D { - val shape = intArrayOf(tensor.shape.component1() * tensor.shape.component2(), 1) - var buffer = DoubleArray(tensor.shape.component1() * tensor.shape.component2()) - for (i in 0 until tensor.shape.component1()) { - for (j in 0 until tensor.shape.component2()) { - buffer[i * tensor.shape.component2() + j] = tensor[i, j] - } - } - var column = BroadcastDoubleTensorAlgebra.fromArray(ShapeND(shape), buffer).as2D() - return column -} - -/* column length */ -fun length(column: MutableStructure2D) : Int { - return column.shape.component1() -} - -fun MutableStructure2D.abs() { - for (i in 0 until this.shape.component1()) { - for (j in 0 until this.shape.component2()) { - this[i, j] = abs(this[i, j]) - } - } -} - -fun abs(input: MutableStructure2D): MutableStructure2D { - val tensor = BroadcastDoubleTensorAlgebra.ones( - ShapeND( - intArrayOf( - input.shape.component1(), - input.shape.component2() - ) - ) - ).as2D() - for (i in 0 until tensor.shape.component1()) { - for (j in 0 until tensor.shape.component2()) { - tensor[i, j] = abs(input[i, j]) - } - } - return tensor -} - -fun diag(input: MutableStructure2D): MutableStructure2D { - val tensor = BroadcastDoubleTensorAlgebra.ones(ShapeND(intArrayOf(input.shape.component1(), 1))).as2D() - for (i in 0 until tensor.shape.component1()) { - tensor[i, 0] = input[i, i] - } - return tensor -} - -fun make_matrx_with_diagonal(column: MutableStructure2D): MutableStructure2D { - val size = column.shape.component1() - val tensor = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(size, size))).as2D() - for (i in 0 until size) { - tensor[i, i] = column[i, 0] - } - return tensor -} - -fun lm_eye(size: Int): MutableStructure2D { - val column = BroadcastDoubleTensorAlgebra.ones(ShapeND(intArrayOf(size, 1))).as2D() - return make_matrx_with_diagonal(column) -} - -fun largest_element_comparison(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D { - val a_sizeX = a.shape.component1() - val a_sizeY = a.shape.component2() - val b_sizeX = b.shape.component1() - val b_sizeY = b.shape.component2() - val tensor = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(max(a_sizeX, b_sizeX), max(a_sizeY, b_sizeY)))).as2D() - for (i in 0 until tensor.shape.component1()) { - for (j in 0 until tensor.shape.component2()) { - if (i < a_sizeX && i < b_sizeX && j < a_sizeY && j < b_sizeY) { - tensor[i, j] = max(a[i, j], b[i, j]) - } - else if (i < a_sizeX && j < a_sizeY) { - tensor[i, j] = a[i, j] - } - else { - tensor[i, j] = b[i, j] - } - } - } - return tensor -} - -fun smallest_element_comparison(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D { - val a_sizeX = a.shape.component1() - val a_sizeY = a.shape.component2() - val b_sizeX = b.shape.component1() - val b_sizeY = b.shape.component2() - val tensor = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(max(a_sizeX, b_sizeX), max(a_sizeY, b_sizeY)))).as2D() - for (i in 0 until tensor.shape.component1()) { - for (j in 0 until tensor.shape.component2()) { - if (i < a_sizeX && i < b_sizeX && j < a_sizeY && j < b_sizeY) { - tensor[i, j] = min(a[i, j], b[i, j]) - } - else if (i < a_sizeX && j < a_sizeY) { - tensor[i, j] = a[i, j] - } - else { - tensor[i, j] = b[i, j] - } - } - } - return tensor -} - -fun get_zero_indices(column: MutableStructure2D, epsilon: Double = 0.000001): MutableStructure2D? { - var idx = emptyArray() - for (i in 0 until column.shape.component1()) { - if (abs(column[i, 0]) > epsilon) { - idx += (i + 1.0) - } - } - if (idx.size > 0) { - return BroadcastDoubleTensorAlgebra.fromArray(ShapeND(intArrayOf(idx.size, 1)), idx.toDoubleArray()).as2D() - } - return null -} - -fun feval(func: (MutableStructure2D, MutableStructure2D, LMSettings) -> MutableStructure2D, - t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings) - : MutableStructure2D -{ - return func(t, p, settings) -} - -fun lm_matx(func: (MutableStructure2D, MutableStructure2D, LMSettings) -> MutableStructure2D, - t: MutableStructure2D, p_old: MutableStructure2D, y_old: MutableStructure2D, - dX2: Int, J_input: MutableStructure2D, p: MutableStructure2D, - y_dat: MutableStructure2D, weight: MutableStructure2D, dp:MutableStructure2D, settings:LMSettings) : Array> -{ - // default: dp = 0.001 - - val Npnt = length(y_dat) // number of data points - val Npar = length(p) // number of parameters - - val y_hat = feval(func, t, p, settings) // evaluate model using parameters 'p' - settings.func_calls += 1 - - var J = J_input - - if (settings.iteration % (2 * Npar) == 0 || dX2 > 0) { - J = lm_FD_J(func, t, p, y_hat, dp, settings).as2D() // finite difference - } - else { - J = lm_Broyden_J(p_old, y_old, J, p, y_hat).as2D() // rank-1 update - } - - val delta_y = y_dat.minus(y_hat) - - val Chi_sq = delta_y.transposed().dot( delta_y.times(weight) ).as2D() - val JtWJ = J.transposed().dot ( J.times( weight.dot(BroadcastDoubleTensorAlgebra.ones(ShapeND(intArrayOf(1, Npar)))) ) ).as2D() - val JtWdy = J.transposed().dot( weight.times(delta_y) ).as2D() - - return arrayOf(JtWJ,JtWdy,Chi_sq,y_hat,J) -} - -fun lm_Broyden_J(p_old: MutableStructure2D, y_old: MutableStructure2D, J_input: MutableStructure2D, - p: MutableStructure2D, y: MutableStructure2D): MutableStructure2D { - var J = J_input.copyToTensor() - - val h = p.minus(p_old) - val increase = y.minus(y_old).minus( J.dot(h) ).dot(h.transposed()).div( (h.transposed().dot(h)).as2D()[0, 0] ) - J = J.plus(increase) - - return J.as2D() -} - -fun lm_FD_J(func: (MutableStructure2D, MutableStructure2D, settings: LMSettings) -> MutableStructure2D, - t: MutableStructure2D, p: MutableStructure2D, y: MutableStructure2D, - dp: MutableStructure2D, settings: LMSettings): MutableStructure2D { - // default: dp = 0.001 * ones(1,n) - - val m = length(y) // number of data points - val n = length(p) // number of parameters - - val ps = p.copyToTensor().as2D() - val J = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(m, n))).as2D() // initialize Jacobian to Zero - val del = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(n, 1))).as2D() - - for (j in 0 until n) { - - del[j, 0] = dp[j, 0] * (1 + abs(p[j, 0])) // parameter perturbation - p[j, 0] = ps[j, 0] + del[j, 0] // perturb parameter p(j) - - val epsilon = 0.0000001 - if (kotlin.math.abs(del[j, 0]) > epsilon) { - val y1 = feval(func, t, p, settings) - settings.func_calls += 1 - - if (dp[j, 0] < 0) { // backwards difference - for (i in 0 until J.shape.component1()) { - J[i, j] = (y1.as2D().minus(y).as2D())[i, 0] / del[j, 0] - } - } - else { - // Do tests for it - println("Potential mistake") - p[j, 0] = ps[j, 0] - del[j, 0] // central difference, additional func call - for (i in 0 until J.shape.component1()) { - J[i, j] = (y1.as2D().minus(feval(func, t, p, settings)).as2D())[i, 0] / (2 * del[j, 0]) - } - settings.func_calls += 1 - } - } - - p[j, 0] = ps[j, 0] // restore p(j) - } - - return J.as2D() -} +} \ No newline at end of file diff --git a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt index 5aea9b879..7222fc7a6 100644 --- a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt +++ b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestDoubleTensorAlgebra.kt @@ -8,11 +8,7 @@ package space.kscience.kmath.tensors.core import space.kscience.kmath.nd.* import space.kscience.kmath.operations.invoke -import space.kscience.kmath.tensors.api.LinearOpsTensorAlgebra -import space.kscience.kmath.tensors.core.internal.LMSettings import space.kscience.kmath.testutils.assertBufferEquals -import kotlin.math.roundToInt -import kotlin.math.roundToLong import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFalse @@ -209,100 +205,4 @@ internal class TestDoubleTensorAlgebra { assertTrue { ShapeND(5, 5) contentEquals res.shape } assertEquals(2.0, res[4, 4]) } - - @Test - fun testLM() = DoubleTensorAlgebra { - fun lm_func(t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings): MutableStructure2D { - val m = t.shape.component1() - var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(m, 1))) - - if (settings.example_number == 1) { - y_hat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))).times(p[0, 0]) + t.times(p[2, 0]).times( - DoubleTensorAlgebra.exp((t.times(-1.0 / p[3, 0]))) - ) - } - else if (settings.example_number == 2) { - val mt = t.max() - y_hat = (t.times(1.0 / mt)).times(p[0, 0]) + - (t.times(1.0 / mt)).pow(2).times(p[1, 0]) + - (t.times(1.0 / mt)).pow(3).times(p[2, 0]) + - (t.times(1.0 / mt)).pow(4).times(p[3, 0]) - } - else if (settings.example_number == 3) { - y_hat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))) - .times(p[0, 0]) + DoubleTensorAlgebra.sin((t.times(1.0 / p[3, 0]))).times(p[2, 0]) - } - - return y_hat.as2D() - } - - val lm_matx_y_dat = doubleArrayOf( - 19.6594, 18.6096, 17.6792, 17.2747, 16.3065, 17.1458, 16.0467, 16.7023, 15.7809, 15.9807, - 14.7620, 15.1128, 16.0973, 15.1934, 15.8636, 15.4763, 15.6860, 15.1895, 15.3495, 16.6054, - 16.2247, 15.9854, 16.1421, 17.0960, 16.7769, 17.1997, 17.2767, 17.5882, 17.5378, 16.7894, - 17.7648, 18.2512, 18.1581, 16.7037, 17.8475, 17.9081, 18.3067, 17.9632, 18.2817, 19.1427, - 18.8130, 18.5658, 18.0056, 18.4607, 18.5918, 18.2544, 18.3731, 18.7511, 19.3181, 17.3066, - 17.9632, 19.0513, 18.7528, 18.2928, 18.5967, 17.8567, 17.7859, 18.4016, 18.9423, 18.4959, - 17.8000, 18.4251, 17.7829, 17.4645, 17.5221, 17.3517, 17.4637, 17.7563, 16.8471, 17.4558, - 17.7447, 17.1487, 17.3183, 16.8312, 17.7551, 17.0942, 15.6093, 16.4163, 15.3755, 16.6725, - 16.2332, 16.2316, 16.2236, 16.5361, 15.3721, 15.3347, 15.5815, 15.6319, 14.4538, 14.6044, - 14.7665, 13.3718, 15.0587, 13.8320, 14.7873, 13.6824, 14.2579, 14.2154, 13.5818, 13.8157 - ) - - var example_number = 1 - val p_init = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(4, 1)), doubleArrayOf(5.0, 2.0, 0.2, 10.0) - ).as2D() - - var t = ones(ShapeND(intArrayOf(100, 1))).as2D() - for (i in 0 until 100) { - t[i, 0] = t[i, 0] * (i + 1) - } - - val y_dat = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(100, 1)), lm_matx_y_dat - ).as2D() - - val weight = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 4.0 } - ).as2D() - - val dp = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } - ).as2D() - - val p_min = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(4, 1)), doubleArrayOf(-50.0, -20.0, -2.0, -100.0) - ).as2D() - - val p_max = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(4, 1)), doubleArrayOf(50.0, 20.0, 2.0, 100.0) - ).as2D() - - val consts = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.0) - ).as2D() - - val opts = doubleArrayOf(3.0, 100.0, 1e-3, 1e-3, 1e-1, 1e-1, 1e-2, 11.0, 9.0, 1.0) - - val result = lm(::lm_func, p_init, t, y_dat, weight, dp, p_min, p_max, consts, opts, 10, example_number) - assertEquals(13, result.iterations) - assertEquals(31, result.func_calls) - assertEquals(1, result.example_number) - assertEquals(0.9131368192633, (result.result_chi_sq * 1e13).roundToLong() / 1e13) - assertEquals(3.7790980 * 1e-7, (result.result_lambda * 1e13).roundToLong() / 1e13) - assertEquals(result.typeOfConvergence, LinearOpsTensorAlgebra.TypeOfConvergence.inParameters) - val expectedParameters = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(4, 1)), doubleArrayOf(20.527230909086, 9.833627103230, 0.997571256572, 50.174445822506) - ).as2D() - result.result_parameters = result.result_parameters.map { x -> (x * 1e12).toLong() / 1e12}.as2D() - val receivedParameters = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(4, 1)), doubleArrayOf(result.result_parameters[0, 0], result.result_parameters[1, 0], - result.result_parameters[2, 0], result.result_parameters[3, 0]) - ).as2D() - assertEquals(expectedParameters[0, 0], receivedParameters[0, 0]) - assertEquals(expectedParameters[1, 0], receivedParameters[1, 0]) - assertEquals(expectedParameters[2, 0], receivedParameters[2, 0]) - assertEquals(expectedParameters[3, 0], receivedParameters[3, 0]) - } } diff --git a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt index f554a742c..db8bc9d20 100644 --- a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt +++ b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt @@ -15,7 +15,6 @@ import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.max import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.plus import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.pow import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.times -import space.kscience.kmath.tensors.core.internal.LMSettings import kotlin.math.roundToLong import kotlin.test.Test import kotlin.test.assertEquals From 963e14b00a273c55049f66dcb96a93e1c158d7c4 Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Tue, 6 Jun 2023 20:07:42 +0300 Subject: [PATCH 15/26] move enums --- kmath-symja/build.gradle.kts | 2 +- .../tensors/api/LinearOpsTensorAlgebra.kt | 18 ----------- .../core/LevenbergMarquardtAlgorithm.kt | 32 +++++++++++++++---- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/kmath-symja/build.gradle.kts b/kmath-symja/build.gradle.kts index 8741de2ae..a996f3bec 100644 --- a/kmath-symja/build.gradle.kts +++ b/kmath-symja/build.gradle.kts @@ -10,7 +10,7 @@ plugins { description = "Symja integration module" dependencies { - api("org.matheclipse:matheclipse-core:2.0.0-SNAPSHOT") { + api("org.matheclipse:matheclipse-core:2.0.0") { // Incorrect transitive dependencies exclude("org.apfloat", "apfloat") exclude("org.hipparchus", "hipparchus-clustering") diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt index 6b3859316..f2c7f1821 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/api/LinearOpsTensorAlgebra.kt @@ -111,22 +111,4 @@ public interface LinearOpsTensorAlgebra> : TensorPartialDivision * @return the square matrix x which is the solution of the equation. */ public fun solve(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D - - public enum class TypeOfConvergence{ - inRHS_JtWdy, - inParameters, - inReducedChi_square, - noConvergence - } - - public data class LMResultInfo ( - var iterations:Int, - var func_calls: Int, - var example_number: Int, - var result_chi_sq: Double, - var result_lambda: Double, - var result_parameters: MutableStructure2D, - var typeOfConvergence: TypeOfConvergence, - var epsilon: Double - ) } diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt index f4b50626a..4323a86a3 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt @@ -19,14 +19,32 @@ import kotlin.math.min import kotlin.math.pow import kotlin.reflect.KFunction3 +public enum class TypeOfConvergence{ + inRHS_JtWdy, + inParameters, + inReducedChi_square, + noConvergence +} + +public data class LMResultInfo ( + var iterations:Int, + var func_calls: Int, + var example_number: Int, + var result_chi_sq: Double, + var result_lambda: Double, + var result_parameters: MutableStructure2D, + var typeOfConvergence: TypeOfConvergence, + var epsilon: Double +) + public fun DoubleTensorAlgebra.lm( func: KFunction3, MutableStructure2D, LMSettings, MutableStructure2D>, p_input: MutableStructure2D, t_input: MutableStructure2D, y_dat_input: MutableStructure2D, weight_input: MutableStructure2D, dp_input: MutableStructure2D, p_min_input: MutableStructure2D, p_max_input: MutableStructure2D, - c_input: MutableStructure2D, opts_input: DoubleArray, nargin: Int, example_number: Int): LinearOpsTensorAlgebra.LMResultInfo { + c_input: MutableStructure2D, opts_input: DoubleArray, nargin: Int, example_number: Int): LMResultInfo { - val resultInfo = LinearOpsTensorAlgebra.LMResultInfo(0, 0, example_number, 0.0, - 0.0, p_input, LinearOpsTensorAlgebra.TypeOfConvergence.noConvergence, 0.0) + val resultInfo = LMResultInfo(0, 0, example_number, 0.0, + 0.0, p_input, TypeOfConvergence.noConvergence, 0.0) val eps:Double = 2.2204e-16 @@ -303,27 +321,27 @@ public fun DoubleTensorAlgebra.lm( if (abs(JtWdy).max()!! < epsilon_1 && settings.iteration > 2) { // println(" **** Convergence in r.h.s. (\"JtWdy\") ****") // println(" **** epsilon_1 = $epsilon_1") - resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.inRHS_JtWdy + resultInfo.typeOfConvergence = TypeOfConvergence.inRHS_JtWdy resultInfo.epsilon = epsilon_1 stop = true } if ((abs(h.as2D()).div(abs(p) + 1e-12)).max() < epsilon_2 && settings.iteration > 2) { // println(" **** Convergence in Parameters ****") // println(" **** epsilon_2 = $epsilon_2") - resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.inParameters + resultInfo.typeOfConvergence = TypeOfConvergence.inParameters resultInfo.epsilon = epsilon_2 stop = true } if (X2 / DoF < epsilon_3 && settings.iteration > 2) { // println(" **** Convergence in reduced Chi-square **** ") // println(" **** epsilon_3 = $epsilon_3") - resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.inReducedChi_square + resultInfo.typeOfConvergence = TypeOfConvergence.inReducedChi_square resultInfo.epsilon = epsilon_3 stop = true } if (settings.iteration == MaxIter) { // println(" !! Maximum Number of Iterations Reached Without Convergence !!") - resultInfo.typeOfConvergence = LinearOpsTensorAlgebra.TypeOfConvergence.noConvergence + resultInfo.typeOfConvergence = TypeOfConvergence.noConvergence resultInfo.epsilon = 0.0 stop = true } From 29d392a8a05aa8bb27b342a53b5920607644217e Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Tue, 6 Jun 2023 20:31:15 +0300 Subject: [PATCH 16/26] fix problem with imports --- .../LevenbergMarquardt/StaticLm/staticDifficultTest.kt | 3 ++- .../tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt | 4 ++-- .../tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt | 3 ++- .../kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt | 3 ++- .../kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt | 3 +-- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt index 0a502afa8..e996b7f7e 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt @@ -12,7 +12,8 @@ import space.kscience.kmath.tensors.LevenbergMarquardt.funcDifficultForLm import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.div import space.kscience.kmath.tensors.core.DoubleTensorAlgebra -import space.kscience.kmath.tensors.core.internal.LMSettings +import space.kscience.kmath.tensors.core.LMSettings +import space.kscience.kmath.tensors.core.lm import kotlin.math.roundToInt fun main() { diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt index bae5a674f..d9e5f350e 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt @@ -11,9 +11,9 @@ import space.kscience.kmath.nd.component1 import space.kscience.kmath.tensors.LevenbergMarquardt.funcDifficultForLm import space.kscience.kmath.tensors.LevenbergMarquardt.funcEasyForLm import space.kscience.kmath.tensors.LevenbergMarquardt.getStartDataForFuncEasy -import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra import space.kscience.kmath.tensors.core.DoubleTensorAlgebra -import space.kscience.kmath.tensors.core.internal.LMSettings +import space.kscience.kmath.tensors.core.LMSettings +import space.kscience.kmath.tensors.core.lm import kotlin.math.roundToInt fun main() { diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt index 02917caf2..b5553a930 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt @@ -13,7 +13,8 @@ import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.div import space.kscience.kmath.tensors.core.DoubleTensorAlgebra import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.times -import space.kscience.kmath.tensors.core.internal.LMSettings +import space.kscience.kmath.tensors.core.LMSettings +import space.kscience.kmath.tensors.core.lm import kotlin.math.roundToInt fun main() { val NData = 100 diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt index 5a1037618..46bda40c6 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt @@ -11,7 +11,8 @@ import space.kscience.kmath.nd.* import space.kscience.kmath.tensors.LevenbergMarquardt.StartDataLm import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.zeros import space.kscience.kmath.tensors.core.DoubleTensorAlgebra -import space.kscience.kmath.tensors.core.internal.LMSettings +import space.kscience.kmath.tensors.core.LMSettings +import space.kscience.kmath.tensors.core.lm import kotlin.random.Random import kotlin.reflect.KFunction3 diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt index 5b194ab6b..a22de71d8 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt @@ -16,9 +16,8 @@ import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.max import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.plus import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.pow import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.times +import space.kscience.kmath.tensors.core.LMSettings import space.kscience.kmath.tensors.core.asDoubleTensor -import space.kscience.kmath.tensors.core.internal.LMSettings -import kotlin.math.roundToInt public data class StartDataLm ( var lm_matx_y_dat: MutableStructure2D, From 1ed40cd8ce5bd0f6d70da097b328267c2a49203c Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Tue, 6 Jun 2023 20:43:59 +0300 Subject: [PATCH 17/26] fix problem with imports --- .../space/kscience/kmath/tensors/core/TestLmAlgorithm.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt index db8bc9d20..dae3006c9 100644 --- a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt +++ b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt @@ -10,7 +10,6 @@ import space.kscience.kmath.nd.ShapeND import space.kscience.kmath.nd.as2D import space.kscience.kmath.nd.component1 import space.kscience.kmath.operations.invoke -import space.kscience.kmath.tensors.api.LinearOpsTensorAlgebra import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.max import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.plus import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.pow @@ -135,7 +134,7 @@ class TestLmAlgorithm { assertEquals(1, result.example_number) assertEquals(0.9131368192633, (result.result_chi_sq * 1e13).roundToLong() / 1e13) assertEquals(3.7790980 * 1e-7, (result.result_lambda * 1e13).roundToLong() / 1e13) - assertEquals(result.typeOfConvergence, LinearOpsTensorAlgebra.TypeOfConvergence.inParameters) + assertEquals(result.typeOfConvergence, TypeOfConvergence.inParameters) val expectedParameters = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(4, 1)), doubleArrayOf(20.527230909086, 9.833627103230, 0.997571256572, 50.174445822506) ).as2D() From 0c7f5697da6e8a52176ca5f90f780b37ed918f07 Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Wed, 7 Jun 2023 00:50:27 +0300 Subject: [PATCH 18/26] add documentation for enum TypeOfConvergence --- .../core/LevenbergMarquardtAlgorithm.kt | 66 +++++++++---------- .../kmath/tensors/core/TestLmAlgorithm.kt | 2 +- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt index 4323a86a3..deb7ee300 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt @@ -7,7 +7,6 @@ package space.kscience.kmath.tensors.core import space.kscience.kmath.linear.transpose import space.kscience.kmath.nd.* -import space.kscience.kmath.tensors.api.LinearOpsTensorAlgebra import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.div import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.dot import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.minus @@ -19,11 +18,26 @@ import kotlin.math.min import kotlin.math.pow import kotlin.reflect.KFunction3 +/** + * Type of convergence achieved as a result of executing the Levenberg-Marquardt algorithm + * + * InGradient: gradient convergence achieved + * (max(J^T W dy) < epsilon1 = opts[2], + * where J - Jacobi matrix (dy^/dp) for the current approximation y^, + * W - weight matrix from input, dy = (y - y^(p))) + * InParameters: convergence in parameters achieved + * (max(h_i / p_i) < epsilon2 = opts[3], + * where h_i - offset for parameter p_i on the current iteration) + * InReducedChiSquare: chi-squared convergence achieved + * (chi squared value divided by (m - n + 1) < epsilon2 = opts[4], + * where n - number of parameters, m - amount of points + * NoConvergence: the maximum number of iterations has been reached without reaching convergence + */ public enum class TypeOfConvergence{ - inRHS_JtWdy, - inParameters, - inReducedChi_square, - noConvergence + InGradient, + InParameters, + InReducedChiSquare, + NoConvergence } public data class LMResultInfo ( @@ -37,6 +51,12 @@ public data class LMResultInfo ( var epsilon: Double ) +public data class LMSettings ( + var iteration:Int, + var func_calls: Int, + var example_number:Int +) + public fun DoubleTensorAlgebra.lm( func: KFunction3, MutableStructure2D, LMSettings, MutableStructure2D>, p_input: MutableStructure2D, t_input: MutableStructure2D, y_dat_input: MutableStructure2D, @@ -44,7 +64,7 @@ public fun DoubleTensorAlgebra.lm( c_input: MutableStructure2D, opts_input: DoubleArray, nargin: Int, example_number: Int): LMResultInfo { val resultInfo = LMResultInfo(0, 0, example_number, 0.0, - 0.0, p_input, TypeOfConvergence.noConvergence, 0.0) + 0.0, p_input, TypeOfConvergence.NoConvergence, 0.0) val eps:Double = 2.2204e-16 @@ -299,15 +319,6 @@ public fun DoubleTensorAlgebra.lm( if (prnt > 1) { val chi_sq = X2 / DoF -// println("Iteration $settings | chi_sq=$chi_sq | lambda=$lambda") -// print("param: ") -// for (pn in 0 until Npar) { -// print(p[pn, 0].toString() + " ") -// } -// print("\ndp/p: ") -// for (pn in 0 until Npar) { -// print((h.as2D()[pn, 0] / p[pn, 0]).toString() + " ") -// } resultInfo.iterations = settings.iteration resultInfo.func_calls = settings.func_calls resultInfo.result_chi_sq = chi_sq @@ -319,42 +330,29 @@ public fun DoubleTensorAlgebra.lm( // cvg_hst(iteration,:) = [ func_calls p' X2/DoF lambda ]; if (abs(JtWdy).max()!! < epsilon_1 && settings.iteration > 2) { -// println(" **** Convergence in r.h.s. (\"JtWdy\") ****") -// println(" **** epsilon_1 = $epsilon_1") - resultInfo.typeOfConvergence = TypeOfConvergence.inRHS_JtWdy + resultInfo.typeOfConvergence = TypeOfConvergence.InGradient resultInfo.epsilon = epsilon_1 stop = true } if ((abs(h.as2D()).div(abs(p) + 1e-12)).max() < epsilon_2 && settings.iteration > 2) { -// println(" **** Convergence in Parameters ****") -// println(" **** epsilon_2 = $epsilon_2") - resultInfo.typeOfConvergence = TypeOfConvergence.inParameters + resultInfo.typeOfConvergence = TypeOfConvergence.InParameters resultInfo.epsilon = epsilon_2 stop = true } if (X2 / DoF < epsilon_3 && settings.iteration > 2) { -// println(" **** Convergence in reduced Chi-square **** ") -// println(" **** epsilon_3 = $epsilon_3") - resultInfo.typeOfConvergence = TypeOfConvergence.inReducedChi_square + resultInfo.typeOfConvergence = TypeOfConvergence.InReducedChiSquare resultInfo.epsilon = epsilon_3 stop = true } if (settings.iteration == MaxIter) { -// println(" !! Maximum Number of Iterations Reached Without Convergence !!") - resultInfo.typeOfConvergence = TypeOfConvergence.noConvergence + resultInfo.typeOfConvergence = TypeOfConvergence.NoConvergence resultInfo.epsilon = 0.0 stop = true } - } // --- End of Main Loop + } return resultInfo } -public data class LMSettings ( - var iteration:Int, - var func_calls: Int, - var example_number:Int -) - /* matrix -> column of all elemnets */ public fun make_column(tensor: MutableStructure2D) : MutableStructure2D { val shape = intArrayOf(tensor.shape.component1() * tensor.shape.component2(), 1) @@ -564,7 +562,7 @@ public fun lm_FD_J(func: (MutableStructure2D, MutableStructure2D } } - p[j, 0] = ps[j, 0] // restore p(j) + p[j, 0] = ps[j, 0] } return J.as2D() diff --git a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt index dae3006c9..1112043fc 100644 --- a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt +++ b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt @@ -134,7 +134,7 @@ class TestLmAlgorithm { assertEquals(1, result.example_number) assertEquals(0.9131368192633, (result.result_chi_sq * 1e13).roundToLong() / 1e13) assertEquals(3.7790980 * 1e-7, (result.result_lambda * 1e13).roundToLong() / 1e13) - assertEquals(result.typeOfConvergence, TypeOfConvergence.inParameters) + assertEquals(result.typeOfConvergence, TypeOfConvergence.InParameters) val expectedParameters = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(4, 1)), doubleArrayOf(20.527230909086, 9.833627103230, 0.997571256572, 50.174445822506) ).as2D() From cac5b513f30515e38b4f7706b7161905733781e3 Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Wed, 7 Jun 2023 01:55:38 +0300 Subject: [PATCH 19/26] made class for settings private and removed settings as input from a custom function --- .../StaticLm/staticDifficultTest.kt | 13 +-- .../StaticLm/staticEasyTest.kt | 9 +- .../StaticLm/staticMiddleTest.kt | 19 ++-- .../StreamingLm/streamLm.kt | 7 +- .../LevenbergMarquardt/functionsToOptimize.kt | 60 +++++----- .../core/LevenbergMarquardtAlgorithm.kt | 107 +++++++++--------- .../kmath/tensors/core/TestLmAlgorithm.kt | 38 +++---- 7 files changed, 125 insertions(+), 128 deletions(-) diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt index e996b7f7e..24cfa955a 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt @@ -12,7 +12,6 @@ import space.kscience.kmath.tensors.LevenbergMarquardt.funcDifficultForLm import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.div import space.kscience.kmath.tensors.core.DoubleTensorAlgebra -import space.kscience.kmath.tensors.core.LMSettings import space.kscience.kmath.tensors.core.lm import kotlin.math.roundToInt @@ -29,9 +28,9 @@ fun main() { p_example[i, 0] = p_example[i, 0] + i - 25 } - val settings = LMSettings(0, 0, 1) + val exampleNumber = 1 - var y_hat = funcDifficultForLm(t_example, p_example, settings) + var y_hat = funcDifficultForLm(t_example, p_example, exampleNumber) var p_init = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() for (i in 0 until Nparams) { @@ -72,14 +71,14 @@ fun main() { ) println("Parameters:") - for (i in 0 until result.result_parameters.shape.component1()) { - val x = (result.result_parameters[i, 0] * 10000).roundToInt() / 10000.0 + for (i in 0 until result.resultParameters.shape.component1()) { + val x = (result.resultParameters[i, 0] * 10000).roundToInt() / 10000.0 print("$x ") } println() println("Y true and y received:") - var y_hat_after = funcDifficultForLm(t_example, result.result_parameters, settings) + var y_hat_after = funcDifficultForLm(t_example, result.resultParameters, exampleNumber) for (i in 0 until y_hat.shape.component1()) { val x = (y_hat[i, 0] * 10000).roundToInt() / 10000.0 val y = (y_hat_after[i, 0] * 10000).roundToInt() / 10000.0 @@ -87,7 +86,7 @@ fun main() { } println("Сhi_sq:") - println(result.result_chi_sq) + println(result.resultChiSq) println("Number of iterations:") println(result.iterations) } \ No newline at end of file diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt index d9e5f350e..c7b2b0def 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt @@ -12,7 +12,6 @@ import space.kscience.kmath.tensors.LevenbergMarquardt.funcDifficultForLm import space.kscience.kmath.tensors.LevenbergMarquardt.funcEasyForLm import space.kscience.kmath.tensors.LevenbergMarquardt.getStartDataForFuncEasy import space.kscience.kmath.tensors.core.DoubleTensorAlgebra -import space.kscience.kmath.tensors.core.LMSettings import space.kscience.kmath.tensors.core.lm import kotlin.math.roundToInt @@ -35,14 +34,14 @@ fun main() { ) println("Parameters:") - for (i in 0 until result.result_parameters.shape.component1()) { - val x = (result.result_parameters[i, 0] * 10000).roundToInt() / 10000.0 + for (i in 0 until result.resultParameters.shape.component1()) { + val x = (result.resultParameters[i, 0] * 10000).roundToInt() / 10000.0 print("$x ") } println() println("Y true and y received:") - var y_hat_after = funcDifficultForLm(startedData.t, result.result_parameters, LMSettings(0, 0, startedData.example_number)) + var y_hat_after = funcDifficultForLm(startedData.t, result.resultParameters, startedData.example_number) for (i in 0 until startedData.y_dat.shape.component1()) { val x = (startedData.y_dat[i, 0] * 10000).roundToInt() / 10000.0 val y = (y_hat_after[i, 0] * 10000).roundToInt() / 10000.0 @@ -50,7 +49,7 @@ fun main() { } println("Сhi_sq:") - println(result.result_chi_sq) + println(result.resultChiSq) println("Number of iterations:") println(result.iterations) } \ No newline at end of file diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt index b5553a930..471143102 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt @@ -12,8 +12,6 @@ import space.kscience.kmath.tensors.LevenbergMarquardt.funcMiddleForLm import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.div import space.kscience.kmath.tensors.core.DoubleTensorAlgebra -import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.times -import space.kscience.kmath.tensors.core.LMSettings import space.kscience.kmath.tensors.core.lm import kotlin.math.roundToInt fun main() { @@ -29,16 +27,15 @@ fun main() { p_example[i, 0] = p_example[i, 0] + i - 25 } - val settings = LMSettings(0, 0, 1) + val exampleNumber = 1 - var y_hat = funcMiddleForLm(t_example, p_example, settings) + var y_hat = funcMiddleForLm(t_example, p_example, exampleNumber) var p_init = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() for (i in 0 until Nparams) { p_init[i, 0] = (p_example[i, 0] + 0.9) } -// val p_init = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) -// val p_init = p_example + var t = t_example val y_dat = y_hat val weight = BroadcastDoubleTensorAlgebra.fromArray( @@ -54,7 +51,7 @@ fun main() { val consts = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.0) ).as2D() - val opts = doubleArrayOf(3.0, 10000.0, 1e-3, 1e-3, 1e-3, 1e-3, 1e-15, 11.0, 9.0, 1.0) + val opts = doubleArrayOf(3.0, 7000.0, 1e-5, 1e-5, 1e-5, 1e-5, 1e-5, 11.0, 9.0, 1.0) val result = DoubleTensorAlgebra.lm( ::funcMiddleForLm, @@ -72,14 +69,14 @@ fun main() { ) println("Parameters:") - for (i in 0 until result.result_parameters.shape.component1()) { - val x = (result.result_parameters[i, 0] * 10000).roundToInt() / 10000.0 + for (i in 0 until result.resultParameters.shape.component1()) { + val x = (result.resultParameters[i, 0] * 10000).roundToInt() / 10000.0 print("$x ") } println() - var y_hat_after = funcMiddleForLm(t_example, result.result_parameters, settings) + var y_hat_after = funcMiddleForLm(t_example, result.resultParameters, exampleNumber) for (i in 0 until y_hat.shape.component1()) { val x = (y_hat[i, 0] * 10000).roundToInt() / 10000.0 val y = (y_hat_after[i, 0] * 10000).roundToInt() / 10000.0 @@ -87,7 +84,7 @@ fun main() { } println("Сhi_sq:") - println(result.result_chi_sq) + println(result.resultChiSq) println("Number of iterations:") println(result.iterations) } \ No newline at end of file diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt index 46bda40c6..f031d82bf 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt @@ -11,12 +11,11 @@ import space.kscience.kmath.nd.* import space.kscience.kmath.tensors.LevenbergMarquardt.StartDataLm import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.zeros import space.kscience.kmath.tensors.core.DoubleTensorAlgebra -import space.kscience.kmath.tensors.core.LMSettings import space.kscience.kmath.tensors.core.lm import kotlin.random.Random import kotlin.reflect.KFunction3 -fun streamLm(lm_func: KFunction3, MutableStructure2D, LMSettings, MutableStructure2D>, +fun streamLm(lm_func: KFunction3, MutableStructure2D, Int, MutableStructure2D>, startData: StartDataLm, launchFrequencyInMs: Long, numberOfLaunches: Int): Flow> = flow{ var example_number = startData.example_number @@ -48,9 +47,9 @@ fun streamLm(lm_func: KFunction3, MutableStructure2D< 10, example_number ) - emit(result.result_parameters) + emit(result.resultParameters) delay(launchFrequencyInMs) - p_init = result.result_parameters + p_init = result.resultParameters y_dat = generateNewYDat(y_dat, 0.1) if (!isEndless) steps -= 1 } diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt index a22de71d8..537b86da3 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt @@ -16,7 +16,6 @@ import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.max import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.plus import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.pow import space.kscience.kmath.tensors.core.DoubleTensorAlgebra.Companion.times -import space.kscience.kmath.tensors.core.LMSettings import space.kscience.kmath.tensors.core.asDoubleTensor public data class StartDataLm ( @@ -33,23 +32,31 @@ public data class StartDataLm ( var opts: DoubleArray ) -fun funcDifficultForLm(t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings): MutableStructure2D { +fun funcEasyForLm(t: MutableStructure2D, p: MutableStructure2D, exampleNumber: Int): MutableStructure2D { val m = t.shape.component1() - var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf (m, 1))) + var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(m, 1))) - val mt = t.max() - for(i in 0 until p.shape.component1()){ - y_hat = y_hat.plus( (t.times(1.0 / mt)).times(p[i, 0]) ) + if (exampleNumber == 1) { + y_hat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))).times(p[0, 0]) + t.times(p[2, 0]).times( + DoubleTensorAlgebra.exp((t.times(-1.0 / p[3, 0]))) + ) } - - for(i in 0 until 4){ - y_hat = funcEasyForLm((y_hat.as2D() + t).as2D(), p, settings).asDoubleTensor() + else if (exampleNumber == 2) { + val mt = t.max() + y_hat = (t.times(1.0 / mt)).times(p[0, 0]) + + (t.times(1.0 / mt)).pow(2).times(p[1, 0]) + + (t.times(1.0 / mt)).pow(3).times(p[2, 0]) + + (t.times(1.0 / mt)).pow(4).times(p[3, 0]) + } + else if (exampleNumber == 3) { + y_hat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))) + .times(p[0, 0]) + DoubleTensorAlgebra.sin((t.times(1.0 / p[3, 0]))).times(p[2, 0]) } return y_hat.as2D() } -fun funcMiddleForLm(t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings): MutableStructure2D { +fun funcMiddleForLm(t: MutableStructure2D, p: MutableStructure2D, exampleNumber: Int): MutableStructure2D { val m = t.shape.component1() var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf (m, 1))) @@ -59,36 +66,29 @@ fun funcMiddleForLm(t: MutableStructure2D, p: MutableStructure2D } for(i in 0 until 5){ - y_hat = funcEasyForLm(y_hat.as2D(), p, settings).asDoubleTensor() + y_hat = funcEasyForLm(y_hat.as2D(), p, exampleNumber).asDoubleTensor() } return y_hat.as2D() } -fun funcEasyForLm(t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings): MutableStructure2D { +fun funcDifficultForLm(t: MutableStructure2D, p: MutableStructure2D, exampleNumber: Int): MutableStructure2D { val m = t.shape.component1() var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf (m, 1))) - if (settings.example_number == 1) { - y_hat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))).times(p[0, 0]) + t.times(p[2, 0]).times( - DoubleTensorAlgebra.exp((t.times(-1.0 / p[3, 0]))) - ) + val mt = t.max() + for(i in 0 until p.shape.component1()){ + y_hat = y_hat.plus( (t.times(1.0 / mt)).times(p[i, 0]) ) } - else if (settings.example_number == 2) { - val mt = t.max() - y_hat = (t.times(1.0 / mt)).times(p[0, 0]) + - (t.times(1.0 / mt)).pow(2).times(p[1, 0]) + - (t.times(1.0 / mt)).pow(3).times(p[2, 0]) + - (t.times(1.0 / mt)).pow(4).times(p[3, 0]) - } - else if (settings.example_number == 3) { - y_hat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))) - .times(p[0, 0]) + DoubleTensorAlgebra.sin((t.times(1.0 / p[3, 0]))).times(p[2, 0]) + + for(i in 0 until 4){ + y_hat = funcEasyForLm((y_hat.as2D() + t).as2D(), p, exampleNumber).asDoubleTensor() } return y_hat.as2D() } + fun getStartDataForFuncDifficult(): StartDataLm { val NData = 200 var t_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(NData, 1))).as2D() @@ -102,9 +102,9 @@ fun getStartDataForFuncDifficult(): StartDataLm { p_example[i, 0] = p_example[i, 0] + i - 25 } - val settings = LMSettings(0, 0, 1) + val exampleNumber = 1 - var y_hat = funcDifficultForLm(t_example, p_example, settings) + var y_hat = funcDifficultForLm(t_example, p_example, exampleNumber) var p_init = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() for (i in 0 until Nparams) { @@ -144,9 +144,9 @@ fun getStartDataForFuncMiddle(): StartDataLm { p_example[i, 0] = p_example[i, 0] + i - 25 } - val settings = LMSettings(0, 0, 1) + val exampleNumber = 1 - var y_hat = funcMiddleForLm(t_example, p_example, settings) + var y_hat = funcMiddleForLm(t_example, p_example, exampleNumber) var p_init = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() for (i in 0 until Nparams) { diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt index deb7ee300..7fbf6ecd8 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt @@ -40,36 +40,39 @@ public enum class TypeOfConvergence{ NoConvergence } +/** + * Class for the data obtained as a result of the execution of the Levenberg-Marquardt algorithm + * + * iterations: number of completed iterations + * funcCalls: the number of evaluations of the input function during execution + * resultChiSq: chi squared value on final parameters + * resultLambda: final lambda parameter used to calculate the offset + * resultParameters: final parameters + * typeOfConvergence: type of convergence + */ public data class LMResultInfo ( var iterations:Int, - var func_calls: Int, - var example_number: Int, - var result_chi_sq: Double, - var result_lambda: Double, - var result_parameters: MutableStructure2D, + var funcCalls: Int, + var resultChiSq: Double, + var resultLambda: Double, + var resultParameters: MutableStructure2D, var typeOfConvergence: TypeOfConvergence, - var epsilon: Double -) - -public data class LMSettings ( - var iteration:Int, - var func_calls: Int, - var example_number:Int ) public fun DoubleTensorAlgebra.lm( - func: KFunction3, MutableStructure2D, LMSettings, MutableStructure2D>, + func: KFunction3, MutableStructure2D, Int, MutableStructure2D>, p_input: MutableStructure2D, t_input: MutableStructure2D, y_dat_input: MutableStructure2D, weight_input: MutableStructure2D, dp_input: MutableStructure2D, p_min_input: MutableStructure2D, p_max_input: MutableStructure2D, c_input: MutableStructure2D, opts_input: DoubleArray, nargin: Int, example_number: Int): LMResultInfo { - val resultInfo = LMResultInfo(0, 0, example_number, 0.0, - 0.0, p_input, TypeOfConvergence.NoConvergence, 0.0) + + val resultInfo = LMResultInfo(0, 0, 0.0, + 0.0, p_input, TypeOfConvergence.NoConvergence) val eps:Double = 2.2204e-16 val settings = LMSettings(0, 0, example_number) - settings.func_calls = 0 // running count of function evaluations + settings.funcCalls = 0 // running count of function evaluations var p = p_input val y_dat = y_dat_input @@ -160,7 +163,7 @@ public fun DoubleTensorAlgebra.lm( val idx = get_zero_indices(dp) // indices of the parameters to be fit val Nfit = idx?.shape?.component1() // number of parameters to fit var stop = false // termination flag - val y_init = feval(func, t, p, settings) // residual error using p_try + val y_init = feval(func, t, p, example_number) // residual error using p_try if (weight.shape.component1() == 1 || variance(weight) == 0.0) { // identical weights vector weight = ones(ShapeND(intArrayOf(Npnt, 1))).div(1 / kotlin.math.abs(weight[0, 0])).as2D() @@ -219,7 +222,7 @@ public fun DoubleTensorAlgebra.lm( var p_try = (p + h).as2D() // update the [idx] elements p_try = smallest_element_comparison(largest_element_comparison(p_min, p_try.as2D()), p_max) // apply constraints - var delta_y = y_dat.minus(feval(func, t, p_try, settings)) // residual error using p_try + var delta_y = y_dat.minus(feval(func, t, p_try, example_number)) // residual error using p_try for (i in 0 until delta_y.shape.component1()) { // floating point error; break for (j in 0 until delta_y.shape.component2()) { @@ -230,7 +233,7 @@ public fun DoubleTensorAlgebra.lm( } } - settings.func_calls += 1 + settings.funcCalls += 1 val tmp = delta_y.times(weight) var X2_try = delta_y.as2D().transpose().dot(tmp) // Chi-squared error criteria @@ -244,8 +247,8 @@ public fun DoubleTensorAlgebra.lm( p_try = p.plus(h).as2D() // update only [idx] elements p_try = smallest_element_comparison(largest_element_comparison(p_min, p_try), p_max) // apply constraints - var delta_y = y_dat.minus(feval(func, t, p_try, settings)) // residual error using p_try - settings.func_calls += 1 + var delta_y = y_dat.minus(feval(func, t, p_try, example_number)) // residual error using p_try + settings.funcCalls += 1 val tmp = delta_y.times(weight) X2_try = delta_y.as2D().transpose().dot(tmp) // Chi-squared error criteria @@ -320,10 +323,10 @@ public fun DoubleTensorAlgebra.lm( if (prnt > 1) { val chi_sq = X2 / DoF resultInfo.iterations = settings.iteration - resultInfo.func_calls = settings.func_calls - resultInfo.result_chi_sq = chi_sq - resultInfo.result_lambda = lambda - resultInfo.result_parameters = p + resultInfo.funcCalls = settings.funcCalls + resultInfo.resultChiSq = chi_sq + resultInfo.resultLambda = lambda + resultInfo.resultParameters = p } // update convergence history ... save _reduced_ Chi-square @@ -331,30 +334,32 @@ public fun DoubleTensorAlgebra.lm( if (abs(JtWdy).max()!! < epsilon_1 && settings.iteration > 2) { resultInfo.typeOfConvergence = TypeOfConvergence.InGradient - resultInfo.epsilon = epsilon_1 stop = true } if ((abs(h.as2D()).div(abs(p) + 1e-12)).max() < epsilon_2 && settings.iteration > 2) { resultInfo.typeOfConvergence = TypeOfConvergence.InParameters - resultInfo.epsilon = epsilon_2 stop = true } if (X2 / DoF < epsilon_3 && settings.iteration > 2) { resultInfo.typeOfConvergence = TypeOfConvergence.InReducedChiSquare - resultInfo.epsilon = epsilon_3 stop = true } if (settings.iteration == MaxIter) { resultInfo.typeOfConvergence = TypeOfConvergence.NoConvergence - resultInfo.epsilon = 0.0 stop = true } } return resultInfo } +private data class LMSettings ( + var iteration:Int, + var funcCalls: Int, + var exampleNumber:Int +) + /* matrix -> column of all elemnets */ -public fun make_column(tensor: MutableStructure2D) : MutableStructure2D { +private fun make_column(tensor: MutableStructure2D) : MutableStructure2D { val shape = intArrayOf(tensor.shape.component1() * tensor.shape.component2(), 1) val buffer = DoubleArray(tensor.shape.component1() * tensor.shape.component2()) for (i in 0 until tensor.shape.component1()) { @@ -367,11 +372,11 @@ public fun make_column(tensor: MutableStructure2D) : MutableStructure2D< } /* column length */ -public fun length(column: MutableStructure2D) : Int { +private fun length(column: MutableStructure2D) : Int { return column.shape.component1() } -public fun MutableStructure2D.abs() { +private fun MutableStructure2D.abs() { for (i in 0 until this.shape.component1()) { for (j in 0 until this.shape.component2()) { this[i, j] = kotlin.math.abs(this[i, j]) @@ -379,7 +384,7 @@ public fun MutableStructure2D.abs() { } } -public fun abs(input: MutableStructure2D): MutableStructure2D { +private fun abs(input: MutableStructure2D): MutableStructure2D { val tensor = BroadcastDoubleTensorAlgebra.ones( ShapeND( intArrayOf( @@ -396,7 +401,7 @@ public fun abs(input: MutableStructure2D): MutableStructure2D { return tensor } -public fun diag(input: MutableStructure2D): MutableStructure2D { +private fun diag(input: MutableStructure2D): MutableStructure2D { val tensor = BroadcastDoubleTensorAlgebra.ones(ShapeND(intArrayOf(input.shape.component1(), 1))).as2D() for (i in 0 until tensor.shape.component1()) { tensor[i, 0] = input[i, i] @@ -404,7 +409,7 @@ public fun diag(input: MutableStructure2D): MutableStructure2D { return tensor } -public fun make_matrx_with_diagonal(column: MutableStructure2D): MutableStructure2D { +private fun make_matrx_with_diagonal(column: MutableStructure2D): MutableStructure2D { val size = column.shape.component1() val tensor = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(size, size))).as2D() for (i in 0 until size) { @@ -413,12 +418,12 @@ public fun make_matrx_with_diagonal(column: MutableStructure2D): Mutable return tensor } -public fun lm_eye(size: Int): MutableStructure2D { +private fun lm_eye(size: Int): MutableStructure2D { val column = BroadcastDoubleTensorAlgebra.ones(ShapeND(intArrayOf(size, 1))).as2D() return make_matrx_with_diagonal(column) } -public fun largest_element_comparison(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D { +private fun largest_element_comparison(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D { val a_sizeX = a.shape.component1() val a_sizeY = a.shape.component2() val b_sizeX = b.shape.component1() @@ -440,7 +445,7 @@ public fun largest_element_comparison(a: MutableStructure2D, b: MutableS return tensor } -public fun smallest_element_comparison(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D { +private fun smallest_element_comparison(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D { val a_sizeX = a.shape.component1() val a_sizeY = a.shape.component2() val b_sizeX = b.shape.component1() @@ -462,7 +467,7 @@ public fun smallest_element_comparison(a: MutableStructure2D, b: Mutable return tensor } -public fun get_zero_indices(column: MutableStructure2D, epsilon: Double = 0.000001): MutableStructure2D? { +private fun get_zero_indices(column: MutableStructure2D, epsilon: Double = 0.000001): MutableStructure2D? { var idx = emptyArray() for (i in 0 until column.shape.component1()) { if (kotlin.math.abs(column[i, 0]) > epsilon) { @@ -475,14 +480,14 @@ public fun get_zero_indices(column: MutableStructure2D, epsilon: Double return null } -public fun feval(func: (MutableStructure2D, MutableStructure2D, LMSettings) -> MutableStructure2D, - t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings) +private fun feval(func: (MutableStructure2D, MutableStructure2D, Int) -> MutableStructure2D, + t: MutableStructure2D, p: MutableStructure2D, exampleNumber: Int) : MutableStructure2D { - return func(t, p, settings) + return func(t, p, exampleNumber) } -public fun lm_matx(func: (MutableStructure2D, MutableStructure2D, LMSettings) -> MutableStructure2D, +private fun lm_matx(func: (MutableStructure2D, MutableStructure2D, Int) -> MutableStructure2D, t: MutableStructure2D, p_old: MutableStructure2D, y_old: MutableStructure2D, dX2: Int, J_input: MutableStructure2D, p: MutableStructure2D, y_dat: MutableStructure2D, weight: MutableStructure2D, dp:MutableStructure2D, settings:LMSettings) : Array> @@ -492,8 +497,8 @@ public fun lm_matx(func: (MutableStructure2D, MutableStructure2D val Npnt = length(y_dat) // number of data points val Npar = length(p) // number of parameters - val y_hat = feval(func, t, p, settings) // evaluate model using parameters 'p' - settings.func_calls += 1 + val y_hat = feval(func, t, p, settings.exampleNumber) // evaluate model using parameters 'p' + settings.funcCalls += 1 var J = J_input @@ -513,7 +518,7 @@ public fun lm_matx(func: (MutableStructure2D, MutableStructure2D return arrayOf(JtWJ,JtWdy,Chi_sq,y_hat,J) } -public fun lm_Broyden_J(p_old: MutableStructure2D, y_old: MutableStructure2D, J_input: MutableStructure2D, +private fun lm_Broyden_J(p_old: MutableStructure2D, y_old: MutableStructure2D, J_input: MutableStructure2D, p: MutableStructure2D, y: MutableStructure2D): MutableStructure2D { var J = J_input.copyToTensor() @@ -524,7 +529,7 @@ public fun lm_Broyden_J(p_old: MutableStructure2D, y_old: MutableStructu return J.as2D() } -public fun lm_FD_J(func: (MutableStructure2D, MutableStructure2D, settings: LMSettings) -> MutableStructure2D, +private fun lm_FD_J(func: (MutableStructure2D, MutableStructure2D, exampleNumber: Int) -> MutableStructure2D, t: MutableStructure2D, p: MutableStructure2D, y: MutableStructure2D, dp: MutableStructure2D, settings: LMSettings): MutableStructure2D { // default: dp = 0.001 * ones(1,n) @@ -543,8 +548,8 @@ public fun lm_FD_J(func: (MutableStructure2D, MutableStructure2D val epsilon = 0.0000001 if (kotlin.math.abs(del[j, 0]) > epsilon) { - val y1 = feval(func, t, p, settings) - settings.func_calls += 1 + val y1 = feval(func, t, p, settings.exampleNumber) + settings.funcCalls += 1 if (dp[j, 0] < 0) { // backwards difference for (i in 0 until J.shape.component1()) { @@ -556,9 +561,9 @@ public fun lm_FD_J(func: (MutableStructure2D, MutableStructure2D println("Potential mistake") p[j, 0] = ps[j, 0] - del[j, 0] // central difference, additional func call for (i in 0 until J.shape.component1()) { - J[i, j] = (y1.as2D().minus(feval(func, t, p, settings)).as2D())[i, 0] / (2 * del[j, 0]) + J[i, j] = (y1.as2D().minus(feval(func, t, p, settings.exampleNumber)).as2D())[i, 0] / (2 * del[j, 0]) } - settings.func_calls += 1 + settings.funcCalls += 1 } } diff --git a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt index 1112043fc..c3e5fa16f 100644 --- a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt +++ b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt @@ -20,23 +20,23 @@ import kotlin.test.assertEquals class TestLmAlgorithm { companion object { - fun funcEasyForLm(t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings): MutableStructure2D { + fun funcEasyForLm(t: MutableStructure2D, p: MutableStructure2D, exampleNumber: Int): MutableStructure2D { val m = t.shape.component1() var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(m, 1))) - if (settings.example_number == 1) { + if (exampleNumber == 1) { y_hat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))).times(p[0, 0]) + t.times(p[2, 0]).times( DoubleTensorAlgebra.exp((t.times(-1.0 / p[3, 0]))) ) } - else if (settings.example_number == 2) { + else if (exampleNumber == 2) { val mt = t.max() y_hat = (t.times(1.0 / mt)).times(p[0, 0]) + (t.times(1.0 / mt)).pow(2).times(p[1, 0]) + (t.times(1.0 / mt)).pow(3).times(p[2, 0]) + (t.times(1.0 / mt)).pow(4).times(p[3, 0]) } - else if (settings.example_number == 3) { + else if (exampleNumber == 3) { y_hat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))) .times(p[0, 0]) + DoubleTensorAlgebra.sin((t.times(1.0 / p[3, 0]))).times(p[2, 0]) } @@ -44,7 +44,7 @@ class TestLmAlgorithm { return y_hat.as2D() } - fun funcMiddleForLm(t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings): MutableStructure2D { + fun funcMiddleForLm(t: MutableStructure2D, p: MutableStructure2D, exampleNumber: Int): MutableStructure2D { val m = t.shape.component1() var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf (m, 1))) @@ -54,13 +54,13 @@ class TestLmAlgorithm { } for(i in 0 until 5){ - y_hat = funcEasyForLm(y_hat.as2D(), p, settings).asDoubleTensor() + y_hat = funcEasyForLm(y_hat.as2D(), p, exampleNumber).asDoubleTensor() } return y_hat.as2D() } - fun funcDifficultForLm(t: MutableStructure2D, p: MutableStructure2D, settings: LMSettings): MutableStructure2D { + fun funcDifficultForLm(t: MutableStructure2D, p: MutableStructure2D, exampleNumber: Int): MutableStructure2D { val m = t.shape.component1() var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf (m, 1))) @@ -70,12 +70,11 @@ class TestLmAlgorithm { } for(i in 0 until 4){ - y_hat = funcEasyForLm((y_hat.as2D() + t).as2D(), p, settings).asDoubleTensor() + y_hat = funcEasyForLm((y_hat.as2D() + t).as2D(), p, exampleNumber).asDoubleTensor() } return y_hat.as2D() } - } @Test fun testLMEasy() = DoubleTensorAlgebra { @@ -130,18 +129,17 @@ class TestLmAlgorithm { val result = lm(::funcEasyForLm, p_init, t, y_dat, weight, dp, p_min, p_max, consts, opts, 10, example_number) assertEquals(13, result.iterations) - assertEquals(31, result.func_calls) - assertEquals(1, result.example_number) - assertEquals(0.9131368192633, (result.result_chi_sq * 1e13).roundToLong() / 1e13) - assertEquals(3.7790980 * 1e-7, (result.result_lambda * 1e13).roundToLong() / 1e13) + assertEquals(31, result.funcCalls) + assertEquals(0.9131368192633, (result.resultChiSq * 1e13).roundToLong() / 1e13) + assertEquals(3.7790980 * 1e-7, (result.resultLambda * 1e13).roundToLong() / 1e13) assertEquals(result.typeOfConvergence, TypeOfConvergence.InParameters) val expectedParameters = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(4, 1)), doubleArrayOf(20.527230909086, 9.833627103230, 0.997571256572, 50.174445822506) ).as2D() - result.result_parameters = result.result_parameters.map { x -> (x * 1e12).toLong() / 1e12}.as2D() + result.resultParameters = result.resultParameters.map { x -> (x * 1e12).toLong() / 1e12}.as2D() val receivedParameters = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(4, 1)), doubleArrayOf(result.result_parameters[0, 0], result.result_parameters[1, 0], - result.result_parameters[2, 0], result.result_parameters[3, 0]) + ShapeND(intArrayOf(4, 1)), doubleArrayOf(result.resultParameters[0, 0], result.resultParameters[1, 0], + result.resultParameters[2, 0], result.resultParameters[3, 0]) ).as2D() assertEquals(expectedParameters[0, 0], receivedParameters[0, 0]) assertEquals(expectedParameters[1, 0], receivedParameters[1, 0]) @@ -163,9 +161,9 @@ class TestLmAlgorithm { p_example[i, 0] = p_example[i, 0] + i - 25 } - val settings = LMSettings(0, 0, 1) + val exampleNumber = 1 - var y_hat = funcMiddleForLm(t_example, p_example, settings) + var y_hat = funcMiddleForLm(t_example, p_example, exampleNumber) var p_init = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() for (i in 0 until Nparams) { @@ -219,9 +217,9 @@ class TestLmAlgorithm { p_example[i, 0] = p_example[i, 0] + i - 25 } - val settings = LMSettings(0, 0, 1) + val exampleNumber = 1 - var y_hat = funcDifficultForLm(t_example, p_example, settings) + var y_hat = funcDifficultForLm(t_example, p_example, exampleNumber) var p_init = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() for (i in 0 until Nparams) { From 162e37cb2fe8a3b27df82cbde91ff3442f4394de Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Wed, 7 Jun 2023 02:52:00 +0300 Subject: [PATCH 20/26] removed extra comments, unnecessary variables, renaming variables and secondary functions --- .../StaticLm/staticDifficultTest.kt | 4 - .../StaticLm/staticEasyTest.kt | 1 - .../StaticLm/staticMiddleTest.kt | 4 - .../StreamingLm/streamLm.kt | 2 - .../core/LevenbergMarquardtAlgorithm.kt | 363 ++++++++---------- .../kmath/tensors/core/TestLmAlgorithm.kt | 14 +- 6 files changed, 165 insertions(+), 223 deletions(-) diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt index 24cfa955a..95a62e21e 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt @@ -49,9 +49,6 @@ fun main() { p_min = p_min.div(1.0 / -50.0) val p_max = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) p_min = p_min.div(1.0 / 50.0) - val consts = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.0) - ).as2D() val opts = doubleArrayOf(3.0, 10000.0, 1e-6, 1e-6, 1e-6, 1e-6, 1e-2, 11.0, 9.0, 1.0) // val opts = doubleArrayOf(3.0, 10000.0, 1e-6, 1e-6, 1e-6, 1e-6, 1e-3, 11.0, 9.0, 1.0) @@ -64,7 +61,6 @@ fun main() { dp, p_min.as2D(), p_max.as2D(), - consts, opts, 10, 1 diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt index c7b2b0def..0b26838a0 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt @@ -27,7 +27,6 @@ fun main() { startedData.dp, startedData.p_min, startedData.p_max, - startedData.consts, startedData.opts, 10, startedData.example_number diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt index 471143102..b60ea1897 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt @@ -48,9 +48,6 @@ fun main() { p_min = p_min.div(1.0 / -50.0) val p_max = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) p_min = p_min.div(1.0 / 50.0) - val consts = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.0) - ).as2D() val opts = doubleArrayOf(3.0, 7000.0, 1e-5, 1e-5, 1e-5, 1e-5, 1e-5, 11.0, 9.0, 1.0) val result = DoubleTensorAlgebra.lm( @@ -62,7 +59,6 @@ fun main() { dp, p_min.as2D(), p_max.as2D(), - consts, opts, 10, 1 diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt index f031d82bf..99fb97923 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt @@ -26,7 +26,6 @@ fun streamLm(lm_func: KFunction3, MutableStructure2D< val dp = startData.dp val p_min = startData.p_min val p_max = startData.p_max - val consts = startData.consts val opts = startData.opts var steps = numberOfLaunches @@ -42,7 +41,6 @@ fun streamLm(lm_func: KFunction3, MutableStructure2D< dp, p_min, p_max, - consts, opts, 10, example_number diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt index 7fbf6ecd8..af41a7a00 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt @@ -31,7 +31,7 @@ import kotlin.reflect.KFunction3 * InReducedChiSquare: chi-squared convergence achieved * (chi squared value divided by (m - n + 1) < epsilon2 = opts[4], * where n - number of parameters, m - amount of points - * NoConvergence: the maximum number of iterations has been reached without reaching convergence + * NoConvergence: the maximum number of iterations has been reached without reaching any convergence */ public enum class TypeOfConvergence{ InGradient, @@ -61,172 +61,141 @@ public data class LMResultInfo ( public fun DoubleTensorAlgebra.lm( func: KFunction3, MutableStructure2D, Int, MutableStructure2D>, - p_input: MutableStructure2D, t_input: MutableStructure2D, y_dat_input: MutableStructure2D, - weight_input: MutableStructure2D, dp_input: MutableStructure2D, p_min_input: MutableStructure2D, p_max_input: MutableStructure2D, - c_input: MutableStructure2D, opts_input: DoubleArray, nargin: Int, example_number: Int): LMResultInfo { + pInput: MutableStructure2D, tInput: MutableStructure2D, yDatInput: MutableStructure2D, + weightInput: MutableStructure2D, dpInput: MutableStructure2D, pMinInput: MutableStructure2D, + pMaxInput: MutableStructure2D, optsInput: DoubleArray, nargin: Int, exampleNumber: Int): LMResultInfo { val resultInfo = LMResultInfo(0, 0, 0.0, - 0.0, p_input, TypeOfConvergence.NoConvergence) + 0.0, pInput, TypeOfConvergence.NoConvergence) - val eps:Double = 2.2204e-16 + val eps = 2.2204e-16 - val settings = LMSettings(0, 0, example_number) + val settings = LMSettings(0, 0, exampleNumber) settings.funcCalls = 0 // running count of function evaluations - var p = p_input - val y_dat = y_dat_input - val t = t_input + var p = pInput + val t = tInput val Npar = length(p) // number of parameters - val Npnt = length(y_dat) // number of data points - var p_old = zeros(ShapeND(intArrayOf(Npar, 1))).as2D() // previous set of parameters - var y_old = zeros(ShapeND(intArrayOf(Npnt, 1))).as2D() // previous model, y_old = y_hat(t;p_old) + val Npnt = length(yDatInput) // number of data points + var pOld = zeros(ShapeND(intArrayOf(Npar, 1))).as2D() // previous set of parameters + var yOld = zeros(ShapeND(intArrayOf(Npnt, 1))).as2D() // previous model, y_old = y_hat(t;p_old) var X2 = 1e-3 / eps // a really big initial Chi-sq value - var X2_old = 1e-3 / eps // a really big initial Chi-sq value + var X2Old = 1e-3 / eps // a really big initial Chi-sq value var J = zeros(ShapeND(intArrayOf(Npnt, Npar))).as2D() // Jacobian matrix val DoF = Npnt - Npar // statistical degrees of freedom - var corr_p = 0 - var sigma_p = 0 - var sigma_y = 0 - var R_sq = 0 - var cvg_hist = 0 - - if (length(t) != length(y_dat)) { - // println("lm.m error: the length of t must equal the length of y_dat") - val length_t = length(t) - val length_y_dat = length(y_dat) - X2 = 0.0 - - corr_p = 0 - sigma_p = 0 - sigma_y = 0 - R_sq = 0 - cvg_hist = 0 - } - - var weight = weight_input + var weight = weightInput if (nargin < 5) { - weight = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf((y_dat.transpose().dot(y_dat)).as1D()[0])).as2D() + weight = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf((yDatInput.transpose().dot(yDatInput)).as1D()[0])).as2D() } - var dp = dp_input + var dp = dpInput if (nargin < 6) { dp = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.001)).as2D() } - var p_min = p_min_input + var pMin = pMinInput if (nargin < 7) { - p_min = p - p_min.abs() - p_min = p_min.div(-100.0).as2D() + pMin = p + pMin.abs() + pMin = pMin.div(-100.0).as2D() } - var p_max = p_max_input + var pMax = pMaxInput if (nargin < 8) { - p_max = p - p_max.abs() - p_max = p_max.div(100.0).as2D() + pMax = p + pMax.abs() + pMax = pMax.div(100.0).as2D() } - var c = c_input - if (nargin < 9) { - c = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf(1.0)).as2D() - } - - var opts = opts_input + var opts = optsInput if (nargin < 10) { opts = doubleArrayOf(3.0, 10.0 * Npar, 1e-3, 1e-3, 1e-1, 1e-1, 1e-2, 11.0, 9.0, 1.0) } val prnt = opts[0] // >1 intermediate results; >2 plots - val MaxIter = opts[1].toInt() // maximum number of iterations - val epsilon_1 = opts[2] // convergence tolerance for gradient - val epsilon_2 = opts[3] // convergence tolerance for parameters - val epsilon_3 = opts[4] // convergence tolerance for Chi-square - val epsilon_4 = opts[5] // determines acceptance of a L-M step - val lambda_0 = opts[6] // initial value of damping paramter, lambda - val lambda_UP_fac = opts[7] // factor for increasing lambda - val lambda_DN_fac = opts[8] // factor for decreasing lambda - val Update_Type = opts[9].toInt() // 1: Levenberg-Marquardt lambda update - // 2: Quadratic update - // 3: Nielsen's lambda update equations + val maxIterations = opts[1].toInt() // maximum number of iterations + val epsilon1 = opts[2] // convergence tolerance for gradient + val epsilon2 = opts[3] // convergence tolerance for parameters + val epsilon3 = opts[4] // convergence tolerance for Chi-square + val epsilon4 = opts[5] // determines acceptance of a L-M step + val lambda0 = opts[6] // initial value of damping paramter, lambda + val lambdaUpFac = opts[7] // factor for increasing lambda + val lambdaDnFac = opts[8] // factor for decreasing lambda + val updateType = opts[9].toInt() // 1: Levenberg-Marquardt lambda update + // 2: Quadratic update + // 3: Nielsen's lambda update equations - p_min = make_column(p_min) - p_max = make_column(p_max) + pMin = makeColumn(pMin) + pMax = makeColumn(pMax) - if (length(make_column(dp)) == 1) { + if (length(makeColumn(dp)) == 1) { dp = ones(ShapeND(intArrayOf(Npar, 1))).div(1 / dp[0, 0]).as2D() } - val idx = get_zero_indices(dp) // indices of the parameters to be fit - val Nfit = idx?.shape?.component1() // number of parameters to fit var stop = false // termination flag - val y_init = feval(func, t, p, example_number) // residual error using p_try if (weight.shape.component1() == 1 || variance(weight) == 0.0) { // identical weights vector weight = ones(ShapeND(intArrayOf(Npnt, 1))).div(1 / kotlin.math.abs(weight[0, 0])).as2D() - // println("using uniform weights for error analysis") } else { - weight = make_column(weight) + weight = makeColumn(weight) weight.abs() } // initialize Jacobian with finite difference calculation - var lm_matx_ans = lm_matx(func, t, p_old, y_old,1, J, p, y_dat, weight, dp, settings) - var JtWJ = lm_matx_ans[0] - var JtWdy = lm_matx_ans[1] - X2 = lm_matx_ans[2][0, 0] - var y_hat = lm_matx_ans[3] - J = lm_matx_ans[4] + var lmMatxAns = lmMatx(func, t, pOld, yOld, 1, J, p, yDatInput, weight, dp, settings) + var JtWJ = lmMatxAns[0] + var JtWdy = lmMatxAns[1] + X2 = lmMatxAns[2][0, 0] + var yHat = lmMatxAns[3] + J = lmMatxAns[4] - if ( abs(JtWdy).max()!! < epsilon_1 ) { -// println(" *** Your Initial Guess is Extremely Close to Optimal ***\n") -// println(" *** epsilon_1 = %e\n$epsilon_1") + if ( abs(JtWdy).max() < epsilon1 ) { stop = true } var lambda = 1.0 var nu = 1 - when (Update_Type) { - 1 -> lambda = lambda_0 // Marquardt: init'l lambda + when (updateType) { + 1 -> lambda = lambda0 // Marquardt: init'l lambda else -> { // Quadratic and Nielsen - lambda = lambda_0 * (diag(JtWJ)).max()!! + lambda = lambda0 * (makeColumnFromDiagonal(JtWJ)).max()!! nu = 2 } } - X2_old = X2 // previous value of X2 - var cvg_hst = ones(ShapeND(intArrayOf(MaxIter, Npar + 3))) // initialize convergence history + X2Old = X2 // previous value of X2 var h: DoubleTensor - var dX2 = X2 - while (!stop && settings.iteration <= MaxIter) { //--- Start Main Loop + + while (!stop && settings.iteration <= maxIterations) { //--- Start Main Loop settings.iteration += 1 // incremental change in parameters - h = when (Update_Type) { + h = when (updateType) { 1 -> { // Marquardt - val solve = solve(JtWJ.plus(make_matrx_with_diagonal(diag(JtWJ)).div(1 / lambda)).as2D(), JtWdy) + val solve = + solve(JtWJ.plus(makeMatrixWithDiagonal(makeColumnFromDiagonal(JtWJ)).div(1 / lambda)).as2D(), JtWdy) solve.asDoubleTensor() } else -> { // Quadratic and Nielsen - val solve = solve(JtWJ.plus(lm_eye(Npar).div(1 / lambda)).as2D(), JtWdy) + val solve = solve(JtWJ.plus(lmEye(Npar).div(1 / lambda)).as2D(), JtWdy) solve.asDoubleTensor() } } - var p_try = (p + h).as2D() // update the [idx] elements - p_try = smallest_element_comparison(largest_element_comparison(p_min, p_try.as2D()), p_max) // apply constraints + var pTry = (p + h).as2D() // update the [idx] elements + pTry = smallestElementComparison(largestElementComparison(pMin, pTry.as2D()), pMax) // apply constraints - var delta_y = y_dat.minus(feval(func, t, p_try, example_number)) // residual error using p_try + var deltaY = yDatInput.minus(evaluateFunction(func, t, pTry, exampleNumber)) // residual error using p_try - for (i in 0 until delta_y.shape.component1()) { // floating point error; break - for (j in 0 until delta_y.shape.component2()) { - if (delta_y[i, j] == Double.POSITIVE_INFINITY || delta_y[i, j] == Double.NEGATIVE_INFINITY) { + for (i in 0 until deltaY.shape.component1()) { // floating point error; break + for (j in 0 until deltaY.shape.component2()) { + if (deltaY[i, j] == Double.POSITIVE_INFINITY || deltaY[i, j] == Double.NEGATIVE_INFINITY) { stop = true break } @@ -235,84 +204,87 @@ public fun DoubleTensorAlgebra.lm( settings.funcCalls += 1 - val tmp = delta_y.times(weight) - var X2_try = delta_y.as2D().transpose().dot(tmp) // Chi-squared error criteria + val tmp = deltaY.times(weight) + var X2Try = deltaY.as2D().transpose().dot(tmp) // Chi-squared error criteria val alpha = 1.0 - if (Update_Type == 2) { // Quadratic + if (updateType == 2) { // Quadratic // One step of quadratic line update in the h direction for minimum X2 - val alpha = JtWdy.transpose().dot(h) / ( (X2_try.minus(X2)).div(2.0).plus(2 * JtWdy.transpose().dot(h)) ) + val alpha = JtWdy.transpose().dot(h) / ((X2Try.minus(X2)).div(2.0).plus(2 * JtWdy.transpose().dot(h))) h = h.dot(alpha) - p_try = p.plus(h).as2D() // update only [idx] elements - p_try = smallest_element_comparison(largest_element_comparison(p_min, p_try), p_max) // apply constraints + pTry = p.plus(h).as2D() // update only [idx] elements + pTry = smallestElementComparison(largestElementComparison(pMin, pTry), pMax) // apply constraints - var delta_y = y_dat.minus(feval(func, t, p_try, example_number)) // residual error using p_try + deltaY = yDatInput.minus(evaluateFunction(func, t, pTry, exampleNumber)) // residual error using p_try settings.funcCalls += 1 - val tmp = delta_y.times(weight) - X2_try = delta_y.as2D().transpose().dot(tmp) // Chi-squared error criteria + X2Try = deltaY.as2D().transpose().dot(deltaY.times(weight)) // Chi-squared error criteria } - val rho = when (Update_Type) { // Nielsen + val rho = when (updateType) { // Nielsen 1 -> { - val tmp = h.transposed().dot(make_matrx_with_diagonal(diag(JtWJ)).div(1 / lambda).dot(h).plus(JtWdy)) - X2.minus(X2_try).as2D()[0, 0] / abs(tmp.as2D()).as2D()[0, 0] + val tmp = h.transposed() + .dot(makeMatrixWithDiagonal(makeColumnFromDiagonal(JtWJ)).div(1 / lambda).dot(h).plus(JtWdy)) + X2.minus(X2Try).as2D()[0, 0] / abs(tmp.as2D()).as2D()[0, 0] } + else -> { val tmp = h.transposed().dot(h.div(1 / lambda).plus(JtWdy)) - X2.minus(X2_try).as2D()[0, 0] / abs(tmp.as2D()).as2D()[0, 0] + X2.minus(X2Try).as2D()[0, 0] / abs(tmp.as2D()).as2D()[0, 0] } } - if (rho > epsilon_4) { // it IS significantly better - val dX2 = X2.minus(X2_old) - X2_old = X2 - p_old = p.copyToTensor().as2D() - y_old = y_hat.copyToTensor().as2D() - p = make_column(p_try) // accept p_try + if (rho > epsilon4) { // it IS significantly better + val dX2 = X2.minus(X2Old) + X2Old = X2 + pOld = p.copyToTensor().as2D() + yOld = yHat.copyToTensor().as2D() + p = makeColumn(pTry) // accept p_try - lm_matx_ans = lm_matx(func, t, p_old, y_old, dX2.toInt(), J, p, y_dat, weight, dp, settings) + lmMatxAns = lmMatx(func, t, pOld, yOld, dX2.toInt(), J, p, yDatInput, weight, dp, settings) // decrease lambda ==> Gauss-Newton method - JtWJ = lm_matx_ans[0] - JtWdy = lm_matx_ans[1] - X2 = lm_matx_ans[2][0, 0] - y_hat = lm_matx_ans[3] - J = lm_matx_ans[4] + JtWJ = lmMatxAns[0] + JtWdy = lmMatxAns[1] + X2 = lmMatxAns[2][0, 0] + yHat = lmMatxAns[3] + J = lmMatxAns[4] - lambda = when (Update_Type) { + lambda = when (updateType) { 1 -> { // Levenberg - max(lambda / lambda_DN_fac, 1e-7); + max(lambda / lambdaDnFac, 1e-7); } + 2 -> { // Quadratic - max( lambda / (1 + alpha) , 1e-7 ); + max(lambda / (1 + alpha), 1e-7); } + else -> { // Nielsen nu = 2 - lambda * max( 1.0 / 3, 1 - (2 * rho - 1).pow(3) ) + lambda * max(1.0 / 3, 1 - (2 * rho - 1).pow(3)) } } - } - else { // it IS NOT better - X2 = X2_old // do not accept p_try - if (settings.iteration % (2 * Npar) == 0 ) { // rank-1 update of Jacobian - lm_matx_ans = lm_matx(func, t, p_old, y_old,-1, J, p, y_dat, weight, dp, settings) - JtWJ = lm_matx_ans[0] - JtWdy = lm_matx_ans[1] - dX2 = lm_matx_ans[2][0, 0] - y_hat = lm_matx_ans[3] - J = lm_matx_ans[4] + } else { // it IS NOT better + X2 = X2Old // do not accept p_try + if (settings.iteration % (2 * Npar) == 0) { // rank-1 update of Jacobian + lmMatxAns = lmMatx(func, t, pOld, yOld, -1, J, p, yDatInput, weight, dp, settings) + JtWJ = lmMatxAns[0] + JtWdy = lmMatxAns[1] + yHat = lmMatxAns[3] + J = lmMatxAns[4] } // increase lambda ==> gradient descent method - lambda = when (Update_Type) { + lambda = when (updateType) { 1 -> { // Levenberg - min(lambda * lambda_UP_fac, 1e7) + min(lambda * lambdaUpFac, 1e7) } + 2 -> { // Quadratic - lambda + kotlin.math.abs(((X2_try.as2D()[0, 0] - X2) / 2) / alpha) + lambda + kotlin.math.abs(((X2Try.as2D()[0, 0] - X2) / 2) / alpha) } + else -> { // Nielsen nu *= 2 lambda * (nu / 2) @@ -321,30 +293,27 @@ public fun DoubleTensorAlgebra.lm( } if (prnt > 1) { - val chi_sq = X2 / DoF + val chiSq = X2 / DoF resultInfo.iterations = settings.iteration resultInfo.funcCalls = settings.funcCalls - resultInfo.resultChiSq = chi_sq + resultInfo.resultChiSq = chiSq resultInfo.resultLambda = lambda resultInfo.resultParameters = p } - // update convergence history ... save _reduced_ Chi-square - // cvg_hst(iteration,:) = [ func_calls p' X2/DoF lambda ]; - - if (abs(JtWdy).max()!! < epsilon_1 && settings.iteration > 2) { + if (abs(JtWdy).max() < epsilon1 && settings.iteration > 2) { resultInfo.typeOfConvergence = TypeOfConvergence.InGradient stop = true } - if ((abs(h.as2D()).div(abs(p) + 1e-12)).max() < epsilon_2 && settings.iteration > 2) { + if ((abs(h.as2D()).div(abs(p) + 1e-12)).max() < epsilon2 && settings.iteration > 2) { resultInfo.typeOfConvergence = TypeOfConvergence.InParameters stop = true } - if (X2 / DoF < epsilon_3 && settings.iteration > 2) { + if (X2 / DoF < epsilon3 && settings.iteration > 2) { resultInfo.typeOfConvergence = TypeOfConvergence.InReducedChiSquare stop = true } - if (settings.iteration == MaxIter) { + if (settings.iteration == maxIterations) { resultInfo.typeOfConvergence = TypeOfConvergence.NoConvergence stop = true } @@ -358,8 +327,8 @@ private data class LMSettings ( var exampleNumber:Int ) -/* matrix -> column of all elemnets */ -private fun make_column(tensor: MutableStructure2D) : MutableStructure2D { +/* matrix -> column of all elements */ +private fun makeColumn(tensor: MutableStructure2D): MutableStructure2D { val shape = intArrayOf(tensor.shape.component1() * tensor.shape.component2(), 1) val buffer = DoubleArray(tensor.shape.component1() * tensor.shape.component2()) for (i in 0 until tensor.shape.component1()) { @@ -367,8 +336,7 @@ private fun make_column(tensor: MutableStructure2D) : MutableStructure2D buffer[i * tensor.shape.component2() + j] = tensor[i, j] } } - val column = BroadcastDoubleTensorAlgebra.fromArray(ShapeND(shape), buffer).as2D() - return column + return BroadcastDoubleTensorAlgebra.fromArray(ShapeND(shape), buffer).as2D() } /* column length */ @@ -401,7 +369,7 @@ private fun abs(input: MutableStructure2D): MutableStructure2D { return tensor } -private fun diag(input: MutableStructure2D): MutableStructure2D { +private fun makeColumnFromDiagonal(input: MutableStructure2D): MutableStructure2D { val tensor = BroadcastDoubleTensorAlgebra.ones(ShapeND(intArrayOf(input.shape.component1(), 1))).as2D() for (i in 0 until tensor.shape.component1()) { tensor[i, 0] = input[i, i] @@ -409,7 +377,7 @@ private fun diag(input: MutableStructure2D): MutableStructure2D return tensor } -private fun make_matrx_with_diagonal(column: MutableStructure2D): MutableStructure2D { +private fun makeMatrixWithDiagonal(column: MutableStructure2D): MutableStructure2D { val size = column.shape.component1() val tensor = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(size, size))).as2D() for (i in 0 until size) { @@ -418,23 +386,23 @@ private fun make_matrx_with_diagonal(column: MutableStructure2D): Mutabl return tensor } -private fun lm_eye(size: Int): MutableStructure2D { +private fun lmEye(size: Int): MutableStructure2D { val column = BroadcastDoubleTensorAlgebra.ones(ShapeND(intArrayOf(size, 1))).as2D() - return make_matrx_with_diagonal(column) + return makeMatrixWithDiagonal(column) } -private fun largest_element_comparison(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D { - val a_sizeX = a.shape.component1() - val a_sizeY = a.shape.component2() - val b_sizeX = b.shape.component1() - val b_sizeY = b.shape.component2() - val tensor = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(max(a_sizeX, b_sizeX), max(a_sizeY, b_sizeY)))).as2D() +private fun largestElementComparison(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D { + val aSizeX = a.shape.component1() + val aSizeY = a.shape.component2() + val bSizeX = b.shape.component1() + val bSizeY = b.shape.component2() + val tensor = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(max(aSizeX, bSizeX), max(aSizeY, bSizeY)))).as2D() for (i in 0 until tensor.shape.component1()) { for (j in 0 until tensor.shape.component2()) { - if (i < a_sizeX && i < b_sizeX && j < a_sizeY && j < b_sizeY) { + if (i < aSizeX && i < bSizeX && j < aSizeY && j < bSizeY) { tensor[i, j] = max(a[i, j], b[i, j]) } - else if (i < a_sizeX && j < a_sizeY) { + else if (i < aSizeX && j < aSizeY) { tensor[i, j] = a[i, j] } else { @@ -445,18 +413,18 @@ private fun largest_element_comparison(a: MutableStructure2D, b: Mutable return tensor } -private fun smallest_element_comparison(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D { - val a_sizeX = a.shape.component1() - val a_sizeY = a.shape.component2() - val b_sizeX = b.shape.component1() - val b_sizeY = b.shape.component2() - val tensor = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(max(a_sizeX, b_sizeX), max(a_sizeY, b_sizeY)))).as2D() +private fun smallestElementComparison(a: MutableStructure2D, b: MutableStructure2D): MutableStructure2D { + val aSizeX = a.shape.component1() + val aSizeY = a.shape.component2() + val bSizeX = b.shape.component1() + val bSizeY = b.shape.component2() + val tensor = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(max(aSizeX, bSizeX), max(aSizeY, bSizeY)))).as2D() for (i in 0 until tensor.shape.component1()) { for (j in 0 until tensor.shape.component2()) { - if (i < a_sizeX && i < b_sizeX && j < a_sizeY && j < b_sizeY) { + if (i < aSizeX && i < bSizeX && j < aSizeY && j < bSizeY) { tensor[i, j] = min(a[i, j], b[i, j]) } - else if (i < a_sizeX && j < a_sizeY) { + else if (i < aSizeX && j < aSizeY) { tensor[i, j] = a[i, j] } else { @@ -467,71 +435,69 @@ private fun smallest_element_comparison(a: MutableStructure2D, b: Mutabl return tensor } -private fun get_zero_indices(column: MutableStructure2D, epsilon: Double = 0.000001): MutableStructure2D? { +private fun getZeroIndices(column: MutableStructure2D, epsilon: Double = 0.000001): MutableStructure2D? { var idx = emptyArray() for (i in 0 until column.shape.component1()) { if (kotlin.math.abs(column[i, 0]) > epsilon) { idx += (i + 1.0) } } - if (idx.size > 0) { + if (idx.isNotEmpty()) { return BroadcastDoubleTensorAlgebra.fromArray(ShapeND(intArrayOf(idx.size, 1)), idx.toDoubleArray()).as2D() } return null } -private fun feval(func: (MutableStructure2D, MutableStructure2D, Int) -> MutableStructure2D, - t: MutableStructure2D, p: MutableStructure2D, exampleNumber: Int) +private fun evaluateFunction(func: (MutableStructure2D, MutableStructure2D, Int) -> MutableStructure2D, + t: MutableStructure2D, p: MutableStructure2D, exampleNumber: Int) : MutableStructure2D { return func(t, p, exampleNumber) } -private fun lm_matx(func: (MutableStructure2D, MutableStructure2D, Int) -> MutableStructure2D, - t: MutableStructure2D, p_old: MutableStructure2D, y_old: MutableStructure2D, - dX2: Int, J_input: MutableStructure2D, p: MutableStructure2D, - y_dat: MutableStructure2D, weight: MutableStructure2D, dp:MutableStructure2D, settings:LMSettings) : Array> +private fun lmMatx(func: (MutableStructure2D, MutableStructure2D, Int) -> MutableStructure2D, + t: MutableStructure2D, pOld: MutableStructure2D, yOld: MutableStructure2D, + dX2: Int, JInput: MutableStructure2D, p: MutableStructure2D, + yDat: MutableStructure2D, weight: MutableStructure2D, dp:MutableStructure2D, settings:LMSettings) : Array> { // default: dp = 0.001 - - val Npnt = length(y_dat) // number of data points val Npar = length(p) // number of parameters - val y_hat = feval(func, t, p, settings.exampleNumber) // evaluate model using parameters 'p' + val yHat = evaluateFunction(func, t, p, settings.exampleNumber) // evaluate model using parameters 'p' settings.funcCalls += 1 - var J = J_input + var J = JInput - if (settings.iteration % (2 * Npar) == 0 || dX2 > 0) { - J = lm_FD_J(func, t, p, y_hat, dp, settings).as2D() // finite difference + J = if (settings.iteration % (2 * Npar) == 0 || dX2 > 0) { + lmFdJ(func, t, p, yHat, dp, settings).as2D() // finite difference } else { - J = lm_Broyden_J(p_old, y_old, J, p, y_hat).as2D() // rank-1 update + lmBroydenJ(pOld, yOld, J, p, yHat).as2D() // rank-1 update } - val delta_y = y_dat.minus(y_hat) + val deltaY = yDat.minus(yHat) - val Chi_sq = delta_y.transposed().dot( delta_y.times(weight) ).as2D() + val chiSq = deltaY.transposed().dot( deltaY.times(weight) ).as2D() val JtWJ = J.transposed().dot ( J.times( weight.dot(BroadcastDoubleTensorAlgebra.ones(ShapeND(intArrayOf(1, Npar)))) ) ).as2D() - val JtWdy = J.transposed().dot( weight.times(delta_y) ).as2D() + val JtWdy = J.transposed().dot( weight.times(deltaY) ).as2D() - return arrayOf(JtWJ,JtWdy,Chi_sq,y_hat,J) + return arrayOf(JtWJ,JtWdy,chiSq,yHat,J) } -private fun lm_Broyden_J(p_old: MutableStructure2D, y_old: MutableStructure2D, J_input: MutableStructure2D, - p: MutableStructure2D, y: MutableStructure2D): MutableStructure2D { - var J = J_input.copyToTensor() +private fun lmBroydenJ(pOld: MutableStructure2D, yOld: MutableStructure2D, JInput: MutableStructure2D, + p: MutableStructure2D, y: MutableStructure2D): MutableStructure2D { + var J = JInput.copyToTensor() - val h = p.minus(p_old) - val increase = y.minus(y_old).minus( J.dot(h) ).dot(h.transposed()).div( (h.transposed().dot(h)).as2D()[0, 0] ) + val h = p.minus(pOld) + val increase = y.minus(yOld).minus( J.dot(h) ).dot(h.transposed()).div( (h.transposed().dot(h)).as2D()[0, 0] ) J = J.plus(increase) return J.as2D() } -private fun lm_FD_J(func: (MutableStructure2D, MutableStructure2D, exampleNumber: Int) -> MutableStructure2D, - t: MutableStructure2D, p: MutableStructure2D, y: MutableStructure2D, - dp: MutableStructure2D, settings: LMSettings): MutableStructure2D { +private fun lmFdJ(func: (MutableStructure2D, MutableStructure2D, exampleNumber: Int) -> MutableStructure2D, + t: MutableStructure2D, p: MutableStructure2D, y: MutableStructure2D, + dp: MutableStructure2D, settings: LMSettings): MutableStructure2D { // default: dp = 0.001 * ones(1,n) val m = length(y) // number of data points @@ -548,7 +514,7 @@ private fun lm_FD_J(func: (MutableStructure2D, MutableStructure2D epsilon) { - val y1 = feval(func, t, p, settings.exampleNumber) + val y1 = evaluateFunction(func, t, p, settings.exampleNumber) settings.funcCalls += 1 if (dp[j, 0] < 0) { // backwards difference @@ -558,10 +524,9 @@ private fun lm_FD_J(func: (MutableStructure2D, MutableStructure2D Date: Wed, 7 Jun 2023 05:25:32 +0300 Subject: [PATCH 21/26] the input data is placed in a separate class, to which the documentation is written --- .../StaticLm/staticDifficultTest.kt | 20 +- .../StaticLm/staticEasyTest.kt | 17 +- .../StaticLm/staticMiddleTest.kt | 20 +- .../StreamingLm/streamLm.kt | 32 +-- .../LevenbergMarquardt/functionsToOptimize.kt | 14 +- .../core/LevenbergMarquardtAlgorithm.kt | 193 +++++++++++------- .../kmath/tensors/core/TestLmAlgorithm.kt | 45 ++-- 7 files changed, 197 insertions(+), 144 deletions(-) diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt index 95a62e21e..e6f575262 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticDifficultTest.kt @@ -12,7 +12,8 @@ import space.kscience.kmath.tensors.LevenbergMarquardt.funcDifficultForLm import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.div import space.kscience.kmath.tensors.core.DoubleTensorAlgebra -import space.kscience.kmath.tensors.core.lm +import space.kscience.kmath.tensors.core.LMInput +import space.kscience.kmath.tensors.core.levenbergMarquardt import kotlin.math.roundToInt fun main() { @@ -39,9 +40,7 @@ fun main() { var t = t_example val y_dat = y_hat - val weight = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 1.0 / Nparams * 1.0 - 0.085 } - ).as2D() + val weight = 1.0 / Nparams * 1.0 - 0.085 val dp = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } ).as2D() @@ -52,8 +51,7 @@ fun main() { val opts = doubleArrayOf(3.0, 10000.0, 1e-6, 1e-6, 1e-6, 1e-6, 1e-2, 11.0, 9.0, 1.0) // val opts = doubleArrayOf(3.0, 10000.0, 1e-6, 1e-6, 1e-6, 1e-6, 1e-3, 11.0, 9.0, 1.0) - val result = DoubleTensorAlgebra.lm( - ::funcDifficultForLm, + val inputData = LMInput(::funcDifficultForLm, p_init.as2D(), t, y_dat, @@ -61,10 +59,14 @@ fun main() { dp, p_min.as2D(), p_max.as2D(), - opts, + opts[1].toInt(), + doubleArrayOf(opts[2], opts[3], opts[4], opts[5]), + doubleArrayOf(opts[6], opts[7], opts[8]), + opts[9].toInt(), 10, - 1 - ) + 1) + + val result = DoubleTensorAlgebra.levenbergMarquardt(inputData) println("Parameters:") for (i in 0 until result.resultParameters.shape.component1()) { diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt index 0b26838a0..507943031 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticEasyTest.kt @@ -12,14 +12,13 @@ import space.kscience.kmath.tensors.LevenbergMarquardt.funcDifficultForLm import space.kscience.kmath.tensors.LevenbergMarquardt.funcEasyForLm import space.kscience.kmath.tensors.LevenbergMarquardt.getStartDataForFuncEasy import space.kscience.kmath.tensors.core.DoubleTensorAlgebra -import space.kscience.kmath.tensors.core.lm +import space.kscience.kmath.tensors.core.LMInput +import space.kscience.kmath.tensors.core.levenbergMarquardt import kotlin.math.roundToInt fun main() { val startedData = getStartDataForFuncEasy() - - val result = DoubleTensorAlgebra.lm( - ::funcEasyForLm, + val inputData = LMInput(::funcEasyForLm, DoubleTensorAlgebra.ones(ShapeND(intArrayOf(4, 1))).as2D(), startedData.t, startedData.y_dat, @@ -27,10 +26,14 @@ fun main() { startedData.dp, startedData.p_min, startedData.p_max, - startedData.opts, + startedData.opts[1].toInt(), + doubleArrayOf(startedData.opts[2], startedData.opts[3], startedData.opts[4], startedData.opts[5]), + doubleArrayOf(startedData.opts[6], startedData.opts[7], startedData.opts[8]), + startedData.opts[9].toInt(), 10, - startedData.example_number - ) + startedData.example_number) + + val result = DoubleTensorAlgebra.levenbergMarquardt(inputData) println("Parameters:") for (i in 0 until result.resultParameters.shape.component1()) { diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt index b60ea1897..0659db103 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StaticLm/staticMiddleTest.kt @@ -12,7 +12,8 @@ import space.kscience.kmath.tensors.LevenbergMarquardt.funcMiddleForLm import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.div import space.kscience.kmath.tensors.core.DoubleTensorAlgebra -import space.kscience.kmath.tensors.core.lm +import space.kscience.kmath.tensors.core.LMInput +import space.kscience.kmath.tensors.core.levenbergMarquardt import kotlin.math.roundToInt fun main() { val NData = 100 @@ -38,9 +39,7 @@ fun main() { var t = t_example val y_dat = y_hat - val weight = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 1.0 } - ).as2D() + val weight = 1.0 val dp = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } ).as2D() @@ -50,8 +49,7 @@ fun main() { p_min = p_min.div(1.0 / 50.0) val opts = doubleArrayOf(3.0, 7000.0, 1e-5, 1e-5, 1e-5, 1e-5, 1e-5, 11.0, 9.0, 1.0) - val result = DoubleTensorAlgebra.lm( - ::funcMiddleForLm, + val inputData = LMInput(::funcMiddleForLm, p_init.as2D(), t, y_dat, @@ -59,10 +57,14 @@ fun main() { dp, p_min.as2D(), p_max.as2D(), - opts, + opts[1].toInt(), + doubleArrayOf(opts[2], opts[3], opts[4], opts[5]), + doubleArrayOf(opts[6], opts[7], opts[8]), + opts[9].toInt(), 10, - 1 - ) + 1) + + val result = DoubleTensorAlgebra.levenbergMarquardt(inputData) println("Parameters:") for (i in 0 until result.resultParameters.shape.component1()) { diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt index 99fb97923..fe96b2fe9 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt @@ -11,7 +11,8 @@ import space.kscience.kmath.nd.* import space.kscience.kmath.tensors.LevenbergMarquardt.StartDataLm import space.kscience.kmath.tensors.core.BroadcastDoubleTensorAlgebra.zeros import space.kscience.kmath.tensors.core.DoubleTensorAlgebra -import space.kscience.kmath.tensors.core.lm +import space.kscience.kmath.tensors.core.LMInput +import space.kscience.kmath.tensors.core.levenbergMarquardt import kotlin.random.Random import kotlin.reflect.KFunction3 @@ -31,20 +32,23 @@ fun streamLm(lm_func: KFunction3, MutableStructure2D< var steps = numberOfLaunches val isEndless = (steps <= 0) + val inputData = LMInput(lm_func, + p_init, + t, + y_dat, + weight, + dp, + p_min, + p_max, + opts[1].toInt(), + doubleArrayOf(opts[2], opts[3], opts[4], opts[5]), + doubleArrayOf(opts[6], opts[7], opts[8]), + opts[9].toInt(), + 10, + example_number) + while (isEndless || steps > 0) { - val result = DoubleTensorAlgebra.lm( - lm_func, - p_init, - t, - y_dat, - weight, - dp, - p_min, - p_max, - opts, - 10, - example_number - ) + val result = DoubleTensorAlgebra.levenbergMarquardt(inputData) emit(result.resultParameters) delay(launchFrequencyInMs) p_init = result.resultParameters diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt index 537b86da3..7ccb37ed0 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/functionsToOptimize.kt @@ -24,7 +24,7 @@ public data class StartDataLm ( var p_init: MutableStructure2D, var t: MutableStructure2D, var y_dat: MutableStructure2D, - var weight: MutableStructure2D, + var weight: Double, var dp: MutableStructure2D, var p_min: MutableStructure2D, var p_max: MutableStructure2D, @@ -113,9 +113,7 @@ fun getStartDataForFuncDifficult(): StartDataLm { var t = t_example val y_dat = y_hat - val weight = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 1.0 / Nparams * 1.0 - 0.085 } - ).as2D() + val weight = 1.0 / Nparams * 1.0 - 0.085 val dp = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } ).as2D() @@ -154,9 +152,7 @@ fun getStartDataForFuncMiddle(): StartDataLm { } var t = t_example val y_dat = y_hat - val weight = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 1.0 } - ).as2D() + val weight = 1.0 val dp = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } ).as2D() @@ -202,9 +198,7 @@ fun getStartDataForFuncEasy(): StartDataLm { ShapeND(intArrayOf(100, 1)), lm_matx_y_dat ).as2D() - val weight = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 4.0 } - ).as2D() + val weight = 4.0 val dp = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt index af41a7a00..ec9d92c88 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt @@ -19,21 +19,21 @@ import kotlin.math.pow import kotlin.reflect.KFunction3 /** - * Type of convergence achieved as a result of executing the Levenberg-Marquardt algorithm + * Type of convergence achieved as a result of executing the Levenberg-Marquardt algorithm. * * InGradient: gradient convergence achieved - * (max(J^T W dy) < epsilon1 = opts[2], + * (max(J^T W dy) < epsilon1, * where J - Jacobi matrix (dy^/dp) for the current approximation y^, - * W - weight matrix from input, dy = (y - y^(p))) + * W - weight matrix from input, dy = (y - y^(p))). * InParameters: convergence in parameters achieved - * (max(h_i / p_i) < epsilon2 = opts[3], - * where h_i - offset for parameter p_i on the current iteration) + * (max(h_i / p_i) < epsilon2, + * where h_i - offset for parameter p_i on the current iteration). * InReducedChiSquare: chi-squared convergence achieved - * (chi squared value divided by (m - n + 1) < epsilon2 = opts[4], - * where n - number of parameters, m - amount of points - * NoConvergence: the maximum number of iterations has been reached without reaching any convergence + * (chi squared value divided by (m - n + 1) < epsilon2, + * where n - number of parameters, m - amount of points). + * NoConvergence: the maximum number of iterations has been reached without reaching any convergence. */ -public enum class TypeOfConvergence{ +public enum class TypeOfConvergence { InGradient, InParameters, InReducedChiSquare, @@ -41,14 +41,14 @@ public enum class TypeOfConvergence{ } /** - * Class for the data obtained as a result of the execution of the Levenberg-Marquardt algorithm + * The data obtained as a result of the execution of the Levenberg-Marquardt algorithm. * - * iterations: number of completed iterations - * funcCalls: the number of evaluations of the input function during execution - * resultChiSq: chi squared value on final parameters - * resultLambda: final lambda parameter used to calculate the offset - * resultParameters: final parameters - * typeOfConvergence: type of convergence + * iterations: number of completed iterations. + * funcCalls: the number of evaluations of the input function during execution. + * resultChiSq: chi squared value on final parameters. + * resultLambda: final lambda parameter used to calculate the offset. + * resultParameters: final parameters. + * typeOfConvergence: type of convergence. */ public data class LMResultInfo ( var iterations:Int, @@ -59,26 +59,65 @@ public data class LMResultInfo ( var typeOfConvergence: TypeOfConvergence, ) -public fun DoubleTensorAlgebra.lm( - func: KFunction3, MutableStructure2D, Int, MutableStructure2D>, - pInput: MutableStructure2D, tInput: MutableStructure2D, yDatInput: MutableStructure2D, - weightInput: MutableStructure2D, dpInput: MutableStructure2D, pMinInput: MutableStructure2D, - pMaxInput: MutableStructure2D, optsInput: DoubleArray, nargin: Int, exampleNumber: Int): LMResultInfo { - +/** + * Input data for the Levenberg-Marquardt function. + * + * func: function of n independent variables x, m parameters an example number, + * rotating a vector of n values y, in which each of the y_i is calculated at its x_i with the given parameters. + * startParameters: starting parameters. + * independentVariables: independent variables, for each of which the real value is known. + * realValues: real values obtained with given independent variables but unknown parameters. + * weight: measurement error for realValues (denominator in each term of sum of weighted squared errors). + * pDelta: delta when calculating the derivative with respect to parameters. + * minParameters: the lower bound of parameter values. + * maxParameters: upper limit of parameter values. + * maxIterations: maximum allowable number of iterations. + * epsilons: epsilon1 - convergence tolerance for gradient, + * epsilon2 - convergence tolerance for parameters, + * epsilon3 - convergence tolerance for reduced chi-square, + * epsilon4 - determines acceptance of a step. + * lambdas: lambda0 - starting lambda value for parameter offset count, + * lambdaUp - factor for increasing lambda, + * lambdaDown - factor for decreasing lambda. + * updateType: 1: Levenberg-Marquardt lambda update, + * 2: Quadratic update, + * 3: Nielsen's lambda update equations. + * nargin: a value that determines which options to use by default + * (<5 - use weight by default, <6 - use pDelta by default, <7 - use minParameters by default, + * <8 - use maxParameters by default, <9 - use updateType by default). + * exampleNumber: a parameter for a function with which you can choose its behavior. + */ +public data class LMInput ( + var func: KFunction3, MutableStructure2D, Int, MutableStructure2D>, + var startParameters: MutableStructure2D, + var independentVariables: MutableStructure2D, + var realValues: MutableStructure2D, + var weight: Double, + var pDelta: MutableStructure2D, + var minParameters: MutableStructure2D, + var maxParameters: MutableStructure2D, + var maxIterations: Int, + var epsilons: DoubleArray, + var lambdas: DoubleArray, + var updateType: Int, + var nargin: Int, + var exampleNumber: Int +) +public fun DoubleTensorAlgebra.levenbergMarquardt(inputData: LMInput): LMResultInfo { val resultInfo = LMResultInfo(0, 0, 0.0, - 0.0, pInput, TypeOfConvergence.NoConvergence) + 0.0, inputData.startParameters, TypeOfConvergence.NoConvergence) val eps = 2.2204e-16 - val settings = LMSettings(0, 0, exampleNumber) + val settings = LMSettings(0, 0, inputData.exampleNumber) settings.funcCalls = 0 // running count of function evaluations - var p = pInput - val t = tInput + var p = inputData.startParameters + val t = inputData.independentVariables val Npar = length(p) // number of parameters - val Npnt = length(yDatInput) // number of data points + val Npnt = length(inputData.realValues) // number of data points var pOld = zeros(ShapeND(intArrayOf(Npar, 1))).as2D() // previous set of parameters var yOld = zeros(ShapeND(intArrayOf(Npnt, 1))).as2D() // previous model, y_old = y_hat(t;p_old) var X2 = 1e-3 / eps // a really big initial Chi-sq value @@ -86,50 +125,55 @@ public fun DoubleTensorAlgebra.lm( var J = zeros(ShapeND(intArrayOf(Npnt, Npar))).as2D() // Jacobian matrix val DoF = Npnt - Npar // statistical degrees of freedom - var weight = weightInput - if (nargin < 5) { - weight = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf((yDatInput.transpose().dot(yDatInput)).as1D()[0])).as2D() + var weight = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf(inputData.weight)).as2D() + if (inputData.nargin < 5) { + weight = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf((inputData.realValues.transpose().dot(inputData.realValues)).as1D()[0])).as2D() } - var dp = dpInput - if (nargin < 6) { + var dp = inputData.pDelta + if (inputData.nargin < 6) { dp = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf(0.001)).as2D() } - var pMin = pMinInput - if (nargin < 7) { - pMin = p - pMin.abs() - pMin = pMin.div(-100.0).as2D() + var minParameters = inputData.minParameters + if (inputData.nargin < 7) { + minParameters = p + minParameters.abs() + minParameters = minParameters.div(-100.0).as2D() } - var pMax = pMaxInput - if (nargin < 8) { - pMax = p - pMax.abs() - pMax = pMax.div(100.0).as2D() + var maxParameters = inputData.maxParameters + if (inputData.nargin < 8) { + maxParameters = p + maxParameters.abs() + maxParameters = maxParameters.div(100.0).as2D() } - var opts = optsInput - if (nargin < 10) { - opts = doubleArrayOf(3.0, 10.0 * Npar, 1e-3, 1e-3, 1e-1, 1e-1, 1e-2, 11.0, 9.0, 1.0) + var maxIterations = inputData.maxIterations + var epsilon1 = inputData.epsilons[0] // convergence tolerance for gradient + var epsilon2 = inputData.epsilons[1] // convergence tolerance for parameters + var epsilon3 = inputData.epsilons[2] // convergence tolerance for Chi-square + var epsilon4 = inputData.epsilons[3] // determines acceptance of a L-M step + var lambda0 = inputData.lambdas[0] // initial value of damping paramter, lambda + var lambdaUpFac = inputData.lambdas[1] // factor for increasing lambda + var lambdaDnFac = inputData.lambdas[2] // factor for decreasing lambda + var updateType = inputData.updateType // 1: Levenberg-Marquardt lambda update + // 2: Quadratic update + // 3: Nielsen's lambda update equations + if (inputData.nargin < 9) { + maxIterations = 10 * Npar + epsilon1 = 1e-3 + epsilon2 = 1e-3 + epsilon3 = 1e-1 + epsilon4 = 1e-1 + lambda0 = 1e-2 + lambdaUpFac = 11.0 + lambdaDnFac = 9.0 + updateType = 1 } - val prnt = opts[0] // >1 intermediate results; >2 plots - val maxIterations = opts[1].toInt() // maximum number of iterations - val epsilon1 = opts[2] // convergence tolerance for gradient - val epsilon2 = opts[3] // convergence tolerance for parameters - val epsilon3 = opts[4] // convergence tolerance for Chi-square - val epsilon4 = opts[5] // determines acceptance of a L-M step - val lambda0 = opts[6] // initial value of damping paramter, lambda - val lambdaUpFac = opts[7] // factor for increasing lambda - val lambdaDnFac = opts[8] // factor for decreasing lambda - val updateType = opts[9].toInt() // 1: Levenberg-Marquardt lambda update - // 2: Quadratic update - // 3: Nielsen's lambda update equations - - pMin = makeColumn(pMin) - pMax = makeColumn(pMax) + minParameters = makeColumn(minParameters) + maxParameters = makeColumn(maxParameters) if (length(makeColumn(dp)) == 1) { dp = ones(ShapeND(intArrayOf(Npar, 1))).div(1 / dp[0, 0]).as2D() @@ -146,7 +190,7 @@ public fun DoubleTensorAlgebra.lm( } // initialize Jacobian with finite difference calculation - var lmMatxAns = lmMatx(func, t, pOld, yOld, 1, J, p, yDatInput, weight, dp, settings) + var lmMatxAns = lmMatx(inputData.func, t, pOld, yOld, 1, J, p, inputData.realValues, weight, dp, settings) var JtWJ = lmMatxAns[0] var JtWdy = lmMatxAns[1] X2 = lmMatxAns[2][0, 0] @@ -189,9 +233,9 @@ public fun DoubleTensorAlgebra.lm( } var pTry = (p + h).as2D() // update the [idx] elements - pTry = smallestElementComparison(largestElementComparison(pMin, pTry.as2D()), pMax) // apply constraints + pTry = smallestElementComparison(largestElementComparison(minParameters, pTry.as2D()), maxParameters) // apply constraints - var deltaY = yDatInput.minus(evaluateFunction(func, t, pTry, exampleNumber)) // residual error using p_try + var deltaY = inputData.realValues.minus(evaluateFunction(inputData.func, t, pTry, inputData.exampleNumber)) // residual error using p_try for (i in 0 until deltaY.shape.component1()) { // floating point error; break for (j in 0 until deltaY.shape.component2()) { @@ -214,9 +258,9 @@ public fun DoubleTensorAlgebra.lm( val alpha = JtWdy.transpose().dot(h) / ((X2Try.minus(X2)).div(2.0).plus(2 * JtWdy.transpose().dot(h))) h = h.dot(alpha) pTry = p.plus(h).as2D() // update only [idx] elements - pTry = smallestElementComparison(largestElementComparison(pMin, pTry), pMax) // apply constraints + pTry = smallestElementComparison(largestElementComparison(minParameters, pTry), maxParameters) // apply constraints - deltaY = yDatInput.minus(evaluateFunction(func, t, pTry, exampleNumber)) // residual error using p_try + deltaY = inputData.realValues.minus(evaluateFunction(inputData.func, t, pTry, inputData.exampleNumber)) // residual error using p_try settings.funcCalls += 1 X2Try = deltaY.as2D().transpose().dot(deltaY.times(weight)) // Chi-squared error criteria @@ -242,7 +286,7 @@ public fun DoubleTensorAlgebra.lm( yOld = yHat.copyToTensor().as2D() p = makeColumn(pTry) // accept p_try - lmMatxAns = lmMatx(func, t, pOld, yOld, dX2.toInt(), J, p, yDatInput, weight, dp, settings) + lmMatxAns = lmMatx(inputData.func, t, pOld, yOld, dX2.toInt(), J, p, inputData.realValues, weight, dp, settings) // decrease lambda ==> Gauss-Newton method JtWJ = lmMatxAns[0] @@ -268,7 +312,7 @@ public fun DoubleTensorAlgebra.lm( } else { // it IS NOT better X2 = X2Old // do not accept p_try if (settings.iteration % (2 * Npar) == 0) { // rank-1 update of Jacobian - lmMatxAns = lmMatx(func, t, pOld, yOld, -1, J, p, yDatInput, weight, dp, settings) + lmMatxAns = lmMatx(inputData.func, t, pOld, yOld, -1, J, p, inputData.realValues, weight, dp, settings) JtWJ = lmMatxAns[0] JtWdy = lmMatxAns[1] yHat = lmMatxAns[3] @@ -292,14 +336,13 @@ public fun DoubleTensorAlgebra.lm( } } - if (prnt > 1) { - val chiSq = X2 / DoF - resultInfo.iterations = settings.iteration - resultInfo.funcCalls = settings.funcCalls - resultInfo.resultChiSq = chiSq - resultInfo.resultLambda = lambda - resultInfo.resultParameters = p - } + val chiSq = X2 / DoF + resultInfo.iterations = settings.iteration + resultInfo.funcCalls = settings.funcCalls + resultInfo.resultChiSq = chiSq + resultInfo.resultLambda = lambda + resultInfo.resultParameters = p + if (abs(JtWdy).max() < epsilon1 && settings.iteration > 2) { resultInfo.typeOfConvergence = TypeOfConvergence.InGradient diff --git a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt index 114e5f879..4e1df3b08 100644 --- a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt +++ b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt @@ -105,9 +105,7 @@ class TestLmAlgorithm { ShapeND(intArrayOf(100, 1)), lm_matx_y_dat ).as2D() - val weight = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 4.0 } - ).as2D() + val weight = 4.0 val dp = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } @@ -123,7 +121,12 @@ class TestLmAlgorithm { val opts = doubleArrayOf(3.0, 100.0, 1e-3, 1e-3, 1e-1, 1e-1, 1e-2, 11.0, 9.0, 1.0) - val result = lm(::funcEasyForLm, p_init, t, y_dat, weight, dp, p_min, p_max, opts, 10, example_number) + val inputData = LMInput(::funcEasyForLm, p_init, t, y_dat, weight, dp, p_min, p_max, opts[1].toInt(), + doubleArrayOf(opts[2], opts[3], opts[4], opts[5]), + doubleArrayOf(opts[6], opts[7], opts[8]), + opts[9].toInt(), 10, example_number) + + val result = levenbergMarquardt(inputData) assertEquals(13, result.iterations) assertEquals(31, result.funcCalls) assertEquals(0.9131368192633, (result.resultChiSq * 1e13).roundToLong() / 1e13) @@ -168,9 +171,7 @@ class TestLmAlgorithm { var t = t_example val y_dat = y_hat - val weight = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 1.0 } - ).as2D() + val weight = 1.0 val dp = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } ).as2D() @@ -180,8 +181,7 @@ class TestLmAlgorithm { p_min = p_min.div(1.0 / 50.0) val opts = doubleArrayOf(3.0, 7000.0, 1e-5, 1e-5, 1e-5, 1e-5, 1e-5, 11.0, 9.0, 1.0) - val result = DoubleTensorAlgebra.lm( - ::funcMiddleForLm, + val inputData = LMInput(::funcMiddleForLm, p_init.as2D(), t, y_dat, @@ -189,10 +189,14 @@ class TestLmAlgorithm { dp, p_min.as2D(), p_max.as2D(), - opts, + opts[1].toInt(), + doubleArrayOf(opts[2], opts[3], opts[4], opts[5]), + doubleArrayOf(opts[6], opts[7], opts[8]), + opts[9].toInt(), 10, - 1 - ) + 1) + + val result = DoubleTensorAlgebra.levenbergMarquardt(inputData) } @Test @@ -220,9 +224,7 @@ class TestLmAlgorithm { var t = t_example val y_dat = y_hat - val weight = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(1, 1)), DoubleArray(1) { 1.0 / Nparams * 1.0 - 0.085 } - ).as2D() + val weight = 1.0 / Nparams * 1.0 - 0.085 val dp = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } ).as2D() @@ -232,8 +234,7 @@ class TestLmAlgorithm { p_min = p_min.div(1.0 / 50.0) val opts = doubleArrayOf(3.0, 7000.0, 1e-2, 1e-3, 1e-2, 1e-2, 1e-2, 11.0, 9.0, 1.0) - val result = DoubleTensorAlgebra.lm( - ::funcDifficultForLm, + val inputData = LMInput(::funcDifficultForLm, p_init.as2D(), t, y_dat, @@ -241,9 +242,13 @@ class TestLmAlgorithm { dp, p_min.as2D(), p_max.as2D(), - opts, + opts[1].toInt(), + doubleArrayOf(opts[2], opts[3], opts[4], opts[5]), + doubleArrayOf(opts[6], opts[7], opts[8]), + opts[9].toInt(), 10, - 1 - ) + 1) + + val result = DoubleTensorAlgebra.levenbergMarquardt(inputData) } } \ No newline at end of file From 0655642933a67bf971c043b6583877ea07eb791e Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Wed, 7 Jun 2023 06:00:58 +0300 Subject: [PATCH 22/26] add documentation to the main function levenbergMarquardt --- .../tensors/core/LevenbergMarquardtAlgorithm.kt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt index ec9d92c88..b0caf987a 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt @@ -104,6 +104,21 @@ public data class LMInput ( var exampleNumber: Int ) + +/** + * Levenberg-Marquardt optimization. + * + * An optimization method that iteratively searches for the optimal function parameters + * that best describe the dataset. The 'input' is the function being optimized, a set of real data + * (calculated with independent variables, but with an unknown set of parameters), a set of + * independent variables, and variables for adjusting the algorithm, described in the documentation for the LMInput class. + * The function returns number of completed iterations, the number of evaluations of the input function during execution, + * chi squared value on final parameters, final lambda parameter used to calculate the offset, final parameters + * and type of convergence in the 'output'. + * + * @receiver the `input`. + * @return the 'output'. + */ public fun DoubleTensorAlgebra.levenbergMarquardt(inputData: LMInput): LMResultInfo { val resultInfo = LMResultInfo(0, 0, 0.0, 0.0, inputData.startParameters, TypeOfConvergence.NoConvergence) From 346e2e97f24c27a896ff96b4689c9dae242e9bc7 Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Wed, 7 Jun 2023 06:14:05 +0300 Subject: [PATCH 23/26] add minor fixes --- .../core/LevenbergMarquardtAlgorithm.kt | 90 +++++++++---------- 1 file changed, 41 insertions(+), 49 deletions(-) diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt index b0caf987a..d9c282fb6 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt @@ -131,14 +131,14 @@ public fun DoubleTensorAlgebra.levenbergMarquardt(inputData: LMInput): LMResultI var p = inputData.startParameters val t = inputData.independentVariables - val Npar = length(p) // number of parameters - val Npnt = length(inputData.realValues) // number of data points - var pOld = zeros(ShapeND(intArrayOf(Npar, 1))).as2D() // previous set of parameters - var yOld = zeros(ShapeND(intArrayOf(Npnt, 1))).as2D() // previous model, y_old = y_hat(t;p_old) - var X2 = 1e-3 / eps // a really big initial Chi-sq value - var X2Old = 1e-3 / eps // a really big initial Chi-sq value - var J = zeros(ShapeND(intArrayOf(Npnt, Npar))).as2D() // Jacobian matrix - val DoF = Npnt - Npar // statistical degrees of freedom + val Npar = length(p) // number of parameters + val Npnt = length(inputData.realValues) // number of data points + var pOld = zeros(ShapeND(intArrayOf(Npar, 1))).as2D() // previous set of parameters + var yOld = zeros(ShapeND(intArrayOf(Npnt, 1))).as2D() // previous model, y_old = y_hat(t;p_old) + var X2 = 1e-3 / eps // a really big initial Chi-sq value + var X2Old = 1e-3 / eps // a really big initial Chi-sq value + var J = zeros(ShapeND(intArrayOf(Npnt, Npar))).as2D() // Jacobian matrix + val DoF = Npnt - Npar // statistical degrees of freedom var weight = fromArray(ShapeND(intArrayOf(1, 1)), doubleArrayOf(inputData.weight)).as2D() if (inputData.nargin < 5) { @@ -165,16 +165,15 @@ public fun DoubleTensorAlgebra.levenbergMarquardt(inputData: LMInput): LMResultI } var maxIterations = inputData.maxIterations - var epsilon1 = inputData.epsilons[0] // convergence tolerance for gradient - var epsilon2 = inputData.epsilons[1] // convergence tolerance for parameters - var epsilon3 = inputData.epsilons[2] // convergence tolerance for Chi-square - var epsilon4 = inputData.epsilons[3] // determines acceptance of a L-M step - var lambda0 = inputData.lambdas[0] // initial value of damping paramter, lambda - var lambdaUpFac = inputData.lambdas[1] // factor for increasing lambda - var lambdaDnFac = inputData.lambdas[2] // factor for decreasing lambda - var updateType = inputData.updateType // 1: Levenberg-Marquardt lambda update - // 2: Quadratic update - // 3: Nielsen's lambda update equations + var epsilon1 = inputData.epsilons[0] + var epsilon2 = inputData.epsilons[1] + var epsilon3 = inputData.epsilons[2] + var epsilon4 = inputData.epsilons[3] + var lambda0 = inputData.lambdas[0] + var lambdaUpFac = inputData.lambdas[1] + var lambdaDnFac = inputData.lambdas[2] + var updateType = inputData.updateType + if (inputData.nargin < 9) { maxIterations = 10 * Npar epsilon1 = 1e-3 @@ -194,7 +193,7 @@ public fun DoubleTensorAlgebra.levenbergMarquardt(inputData: LMInput): LMResultI dp = ones(ShapeND(intArrayOf(Npar, 1))).div(1 / dp[0, 0]).as2D() } - var stop = false // termination flag + var stop = false // termination flag if (weight.shape.component1() == 1 || variance(weight) == 0.0) { // identical weights vector weight = ones(ShapeND(intArrayOf(Npnt, 1))).div(1 / kotlin.math.abs(weight[0, 0])).as2D() @@ -218,39 +217,35 @@ public fun DoubleTensorAlgebra.levenbergMarquardt(inputData: LMInput): LMResultI var lambda = 1.0 var nu = 1 - when (updateType) { - 1 -> lambda = lambda0 // Marquardt: init'l lambda - else -> { // Quadratic and Nielsen - lambda = lambda0 * (makeColumnFromDiagonal(JtWJ)).max()!! - nu = 2 - } + + if (updateType == 1) { + lambda = lambda0 // Marquardt: init'l lambda + } + else { + lambda = lambda0 * (makeColumnFromDiagonal(JtWJ)).max() + nu = 2 } X2Old = X2 // previous value of X2 var h: DoubleTensor - while (!stop && settings.iteration <= maxIterations) { //--- Start Main Loop + while (!stop && settings.iteration <= maxIterations) { settings.iteration += 1 // incremental change in parameters - h = when (updateType) { - 1 -> { // Marquardt - val solve = - solve(JtWJ.plus(makeMatrixWithDiagonal(makeColumnFromDiagonal(JtWJ)).div(1 / lambda)).as2D(), JtWdy) - solve.asDoubleTensor() - } - - else -> { // Quadratic and Nielsen - val solve = solve(JtWJ.plus(lmEye(Npar).div(1 / lambda)).as2D(), JtWdy) - solve.asDoubleTensor() - } + h = if (updateType == 1) { // Marquardt + val solve = solve(JtWJ.plus(makeMatrixWithDiagonal(makeColumnFromDiagonal(JtWJ)).div(1 / lambda)).as2D(), JtWdy) + solve.asDoubleTensor() + } else { // Quadratic and Nielsen + val solve = solve(JtWJ.plus(lmEye(Npar).div(1 / lambda)).as2D(), JtWdy) + solve.asDoubleTensor() } var pTry = (p + h).as2D() // update the [idx] elements - pTry = smallestElementComparison(largestElementComparison(minParameters, pTry.as2D()), maxParameters) // apply constraints + pTry = smallestElementComparison(largestElementComparison(minParameters, pTry.as2D()), maxParameters) // apply constraints - var deltaY = inputData.realValues.minus(evaluateFunction(inputData.func, t, pTry, inputData.exampleNumber)) // residual error using p_try + var deltaY = inputData.realValues.minus(evaluateFunction(inputData.func, t, pTry, inputData.exampleNumber)) // residual error using p_try for (i in 0 until deltaY.shape.component1()) { // floating point error; break for (j in 0 until deltaY.shape.component2()) { @@ -264,21 +259,20 @@ public fun DoubleTensorAlgebra.levenbergMarquardt(inputData: LMInput): LMResultI settings.funcCalls += 1 val tmp = deltaY.times(weight) - var X2Try = deltaY.as2D().transpose().dot(tmp) // Chi-squared error criteria + var X2Try = deltaY.as2D().transpose().dot(tmp) // Chi-squared error criteria val alpha = 1.0 if (updateType == 2) { // Quadratic // One step of quadratic line update in the h direction for minimum X2 - val alpha = JtWdy.transpose().dot(h) / ((X2Try.minus(X2)).div(2.0).plus(2 * JtWdy.transpose().dot(h))) h = h.dot(alpha) pTry = p.plus(h).as2D() // update only [idx] elements pTry = smallestElementComparison(largestElementComparison(minParameters, pTry), maxParameters) // apply constraints - deltaY = inputData.realValues.minus(evaluateFunction(inputData.func, t, pTry, inputData.exampleNumber)) // residual error using p_try + deltaY = inputData.realValues.minus(evaluateFunction(inputData.func, t, pTry, inputData.exampleNumber)) // residual error using p_try settings.funcCalls += 1 - X2Try = deltaY.as2D().transpose().dot(deltaY.times(weight)) // Chi-squared error criteria + X2Try = deltaY.as2D().transpose().dot(deltaY.times(weight)) // Chi-squared error criteria } val rho = when (updateType) { // Nielsen @@ -287,7 +281,6 @@ public fun DoubleTensorAlgebra.levenbergMarquardt(inputData: LMInput): LMResultI .dot(makeMatrixWithDiagonal(makeColumnFromDiagonal(JtWJ)).div(1 / lambda).dot(h).plus(JtWdy)) X2.minus(X2Try).as2D()[0, 0] / abs(tmp.as2D()).as2D()[0, 0] } - else -> { val tmp = h.transposed().dot(h.div(1 / lambda).plus(JtWdy)) X2.minus(X2Try).as2D()[0, 0] / abs(tmp.as2D()).as2D()[0, 0] @@ -303,7 +296,6 @@ public fun DoubleTensorAlgebra.levenbergMarquardt(inputData: LMInput): LMResultI lmMatxAns = lmMatx(inputData.func, t, pOld, yOld, dX2.toInt(), J, p, inputData.realValues, weight, dp, settings) // decrease lambda ==> Gauss-Newton method - JtWJ = lmMatxAns[0] JtWdy = lmMatxAns[1] X2 = lmMatxAns[2][0, 0] @@ -519,7 +511,7 @@ private fun lmMatx(func: (MutableStructure2D, MutableStructure2D yDat: MutableStructure2D, weight: MutableStructure2D, dp:MutableStructure2D, settings:LMSettings) : Array> { // default: dp = 0.001 - val Npar = length(p) // number of parameters + val Npar = length(p) // number of parameters val yHat = evaluateFunction(func, t, p, settings.exampleNumber) // evaluate model using parameters 'p' settings.funcCalls += 1 @@ -558,8 +550,8 @@ private fun lmFdJ(func: (MutableStructure2D, MutableStructure2D, dp: MutableStructure2D, settings: LMSettings): MutableStructure2D { // default: dp = 0.001 * ones(1,n) - val m = length(y) // number of data points - val n = length(p) // number of parameters + val m = length(y) // number of data points + val n = length(p) // number of parameters val ps = p.copyToTensor().as2D() val J = BroadcastDoubleTensorAlgebra.zeros(ShapeND(intArrayOf(m, n))).as2D() // initialize Jacobian to Zero @@ -568,7 +560,7 @@ private fun lmFdJ(func: (MutableStructure2D, MutableStructure2D, for (j in 0 until n) { del[j, 0] = dp[j, 0] * (1 + kotlin.math.abs(p[j, 0])) // parameter perturbation - p[j, 0] = ps[j, 0] + del[j, 0] // perturb parameter p(j) + p[j, 0] = ps[j, 0] + del[j, 0] // perturb parameter p(j) val epsilon = 0.0000001 if (kotlin.math.abs(del[j, 0]) > epsilon) { From f91b018d4f2a330697bc00dd9dd94f71a150dcb7 Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Wed, 7 Jun 2023 07:24:47 +0300 Subject: [PATCH 24/26] add assertEquals to middle and difficult test --- .../kmath/tensors/core/TestLmAlgorithm.kt | 144 +++++++++++------- 1 file changed, 85 insertions(+), 59 deletions(-) diff --git a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt index 4e1df3b08..4b031eb11 100644 --- a/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt +++ b/kmath-tensors/src/commonTest/kotlin/space/kscience/kmath/tensors/core/TestLmAlgorithm.kt @@ -22,63 +22,63 @@ class TestLmAlgorithm { companion object { fun funcEasyForLm(t: MutableStructure2D, p: MutableStructure2D, exampleNumber: Int): MutableStructure2D { val m = t.shape.component1() - var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(m, 1))) + var yHat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(m, 1))) if (exampleNumber == 1) { - y_hat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))).times(p[0, 0]) + t.times(p[2, 0]).times( + yHat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))).times(p[0, 0]) + t.times(p[2, 0]).times( DoubleTensorAlgebra.exp((t.times(-1.0 / p[3, 0]))) ) } else if (exampleNumber == 2) { val mt = t.max() - y_hat = (t.times(1.0 / mt)).times(p[0, 0]) + + yHat = (t.times(1.0 / mt)).times(p[0, 0]) + (t.times(1.0 / mt)).pow(2).times(p[1, 0]) + (t.times(1.0 / mt)).pow(3).times(p[2, 0]) + (t.times(1.0 / mt)).pow(4).times(p[3, 0]) } else if (exampleNumber == 3) { - y_hat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))) + yHat = DoubleTensorAlgebra.exp((t.times(-1.0 / p[1, 0]))) .times(p[0, 0]) + DoubleTensorAlgebra.sin((t.times(1.0 / p[3, 0]))).times(p[2, 0]) } - return y_hat.as2D() + return yHat.as2D() } fun funcMiddleForLm(t: MutableStructure2D, p: MutableStructure2D, exampleNumber: Int): MutableStructure2D { val m = t.shape.component1() - var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf (m, 1))) + var yHat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf (m, 1))) val mt = t.max() for(i in 0 until p.shape.component1()){ - y_hat += (t.times(1.0 / mt)).times(p[i, 0]) + yHat += (t.times(1.0 / mt)).times(p[i, 0]) } for(i in 0 until 5){ - y_hat = funcEasyForLm(y_hat.as2D(), p, exampleNumber).asDoubleTensor() + yHat = funcEasyForLm(yHat.as2D(), p, exampleNumber).asDoubleTensor() } - return y_hat.as2D() + return yHat.as2D() } fun funcDifficultForLm(t: MutableStructure2D, p: MutableStructure2D, exampleNumber: Int): MutableStructure2D { val m = t.shape.component1() - var y_hat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf (m, 1))) + var yHat = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf (m, 1))) val mt = t.max() for(i in 0 until p.shape.component1()){ - y_hat = y_hat.plus( (t.times(1.0 / mt)).times(p[i, 0]) ) + yHat = yHat.plus( (t.times(1.0 / mt)).times(p[i, 0]) ) } for(i in 0 until 4){ - y_hat = funcEasyForLm((y_hat.as2D() + t).as2D(), p, exampleNumber).asDoubleTensor() + yHat = funcEasyForLm((yHat.as2D() + t).as2D(), p, exampleNumber).asDoubleTensor() } - return y_hat.as2D() + return yHat.as2D() } } @Test fun testLMEasy() = DoubleTensorAlgebra { - val lm_matx_y_dat = doubleArrayOf( + val lmMatxYDat = doubleArrayOf( 19.6594, 18.6096, 17.6792, 17.2747, 16.3065, 17.1458, 16.0467, 16.7023, 15.7809, 15.9807, 14.7620, 15.1128, 16.0973, 15.1934, 15.8636, 15.4763, 15.6860, 15.1895, 15.3495, 16.6054, 16.2247, 15.9854, 16.1421, 17.0960, 16.7769, 17.1997, 17.2767, 17.5882, 17.5378, 16.7894, @@ -91,7 +91,7 @@ class TestLmAlgorithm { 14.7665, 13.3718, 15.0587, 13.8320, 14.7873, 13.6824, 14.2579, 14.2154, 13.5818, 13.8157 ) - var example_number = 1 + var exampleNumber = 1 val p_init = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(4, 1)), doubleArrayOf(5.0, 2.0, 0.2, 10.0) ).as2D() @@ -101,8 +101,8 @@ class TestLmAlgorithm { t[i, 0] = t[i, 0] * (i + 1) } - val y_dat = BroadcastDoubleTensorAlgebra.fromArray( - ShapeND(intArrayOf(100, 1)), lm_matx_y_dat + val yDat = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(100, 1)), lmMatxYDat ).as2D() val weight = 4.0 @@ -111,20 +111,16 @@ class TestLmAlgorithm { ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } ).as2D() - val p_min = BroadcastDoubleTensorAlgebra.fromArray( + val pMin = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(4, 1)), doubleArrayOf(-50.0, -20.0, -2.0, -100.0) ).as2D() - val p_max = BroadcastDoubleTensorAlgebra.fromArray( + val pMax = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(4, 1)), doubleArrayOf(50.0, 20.0, 2.0, 100.0) ).as2D() - val opts = doubleArrayOf(3.0, 100.0, 1e-3, 1e-3, 1e-1, 1e-1, 1e-2, 11.0, 9.0, 1.0) - - val inputData = LMInput(::funcEasyForLm, p_init, t, y_dat, weight, dp, p_min, p_max, opts[1].toInt(), - doubleArrayOf(opts[2], opts[3], opts[4], opts[5]), - doubleArrayOf(opts[6], opts[7], opts[8]), - opts[9].toInt(), 10, example_number) + val inputData = LMInput(::funcEasyForLm, p_init, t, yDat, weight, dp, pMin, pMax, 100, + doubleArrayOf(1e-3, 1e-3, 1e-1, 1e-1), doubleArrayOf(1e-2, 11.0, 9.0), 1, 10, exampleNumber) val result = levenbergMarquardt(inputData) assertEquals(13, result.iterations) @@ -149,46 +145,46 @@ class TestLmAlgorithm { @Test fun TestLMMiddle() = DoubleTensorAlgebra { val NData = 100 - var t_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(NData, 1))).as2D() + val tExample = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(NData, 1))).as2D() for (i in 0 until NData) { - t_example[i, 0] = t_example[i, 0] * (i + 1) + tExample[i, 0] = tExample[i, 0] * (i + 1) } val Nparams = 20 - var p_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))).as2D() + val pExample = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))).as2D() for (i in 0 until Nparams) { - p_example[i, 0] = p_example[i, 0] + i - 25 + pExample[i, 0] = pExample[i, 0] + i - 25 } val exampleNumber = 1 - var y_hat = funcMiddleForLm(t_example, p_example, exampleNumber) + val yHat = funcMiddleForLm(tExample, pExample, exampleNumber) - var p_init = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() + val pInit = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() for (i in 0 until Nparams) { - p_init[i, 0] = (p_example[i, 0] + 0.9) + pInit[i, 0] = (pExample[i, 0] + 0.9) } - var t = t_example - val y_dat = y_hat + val t = tExample + val yDat = yHat val weight = 1.0 val dp = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } ).as2D() - var p_min = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) - p_min = p_min.div(1.0 / -50.0) - val p_max = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) - p_min = p_min.div(1.0 / 50.0) + var pMin = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) + pMin = pMin.div(1.0 / -50.0) + val pMax = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) + pMin = pMin.div(1.0 / 50.0) val opts = doubleArrayOf(3.0, 7000.0, 1e-5, 1e-5, 1e-5, 1e-5, 1e-5, 11.0, 9.0, 1.0) val inputData = LMInput(::funcMiddleForLm, - p_init.as2D(), + pInit.as2D(), t, - y_dat, + yDat, weight, dp, - p_min.as2D(), - p_max.as2D(), + pMin.as2D(), + pMax.as2D(), opts[1].toInt(), doubleArrayOf(opts[2], opts[3], opts[4], opts[5]), doubleArrayOf(opts[6], opts[7], opts[8]), @@ -197,51 +193,67 @@ class TestLmAlgorithm { 1) val result = DoubleTensorAlgebra.levenbergMarquardt(inputData) + + assertEquals(46, result.iterations) + assertEquals(113, result.funcCalls) + assertEquals(0.000005977, (result.resultChiSq * 1e9).roundToLong() / 1e9) + assertEquals(1.0 * 1e-7, (result.resultLambda * 1e13).roundToLong() / 1e13) + assertEquals(result.typeOfConvergence, TypeOfConvergence.InReducedChiSquare) + val expectedParameters = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(Nparams, 1)), doubleArrayOf( -23.9717, -18.6686, -21.7971, + -20.9681, -22.086, -20.5859, -19.0384, -17.4957, -15.9991, -14.576, -13.2441, - + 12.0201, -10.9256, -9.9878, -9.2309, -8.6589, -8.2365, -7.8783, -7.4598, -6.8511)).as2D() + result.resultParameters = result.resultParameters.map { x -> (x * 1e4).roundToLong() / 1e4}.as2D() + val receivedParameters = zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() + for (i in 0 until Nparams) { + receivedParameters[i, 0] = result.resultParameters[i, 0] + assertEquals(expectedParameters[i, 0], result.resultParameters[i, 0]) + } } @Test fun TestLMDifficult() = DoubleTensorAlgebra { val NData = 200 - var t_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(NData, 1))).as2D() + var tExample = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(NData, 1))).as2D() for (i in 0 until NData) { - t_example[i, 0] = t_example[i, 0] * (i + 1) - 104 + tExample[i, 0] = tExample[i, 0] * (i + 1) - 104 } val Nparams = 15 - var p_example = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))).as2D() + var pExample = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))).as2D() for (i in 0 until Nparams) { - p_example[i, 0] = p_example[i, 0] + i - 25 + pExample[i, 0] = pExample[i, 0] + i - 25 } val exampleNumber = 1 - var y_hat = funcDifficultForLm(t_example, p_example, exampleNumber) + var yHat = funcDifficultForLm(tExample, pExample, exampleNumber) - var p_init = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() + var pInit = DoubleTensorAlgebra.zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() for (i in 0 until Nparams) { - p_init[i, 0] = (p_example[i, 0] + 0.9) + pInit[i, 0] = (pExample[i, 0] + 0.9) } - var t = t_example - val y_dat = y_hat + var t = tExample + val yDat = yHat val weight = 1.0 / Nparams * 1.0 - 0.085 val dp = BroadcastDoubleTensorAlgebra.fromArray( ShapeND(intArrayOf(1, 1)), DoubleArray(1) { -0.01 } ).as2D() - var p_min = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) - p_min = p_min.div(1.0 / -50.0) - val p_max = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) - p_min = p_min.div(1.0 / 50.0) + var pMin = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) + pMin = pMin.div(1.0 / -50.0) + val pMax = DoubleTensorAlgebra.ones(ShapeND(intArrayOf(Nparams, 1))) + pMin = pMin.div(1.0 / 50.0) val opts = doubleArrayOf(3.0, 7000.0, 1e-2, 1e-3, 1e-2, 1e-2, 1e-2, 11.0, 9.0, 1.0) val inputData = LMInput(::funcDifficultForLm, - p_init.as2D(), + pInit.as2D(), t, - y_dat, + yDat, weight, dp, - p_min.as2D(), - p_max.as2D(), + pMin.as2D(), + pMax.as2D(), opts[1].toInt(), doubleArrayOf(opts[2], opts[3], opts[4], opts[5]), doubleArrayOf(opts[6], opts[7], opts[8]), @@ -250,5 +262,19 @@ class TestLmAlgorithm { 1) val result = DoubleTensorAlgebra.levenbergMarquardt(inputData) + + assertEquals(2375, result.iterations) + assertEquals(4858, result.funcCalls) + assertEquals(5.14347, (result.resultLambda * 1e5).roundToLong() / 1e5) + assertEquals(result.typeOfConvergence, TypeOfConvergence.InParameters) + val expectedParameters = BroadcastDoubleTensorAlgebra.fromArray( + ShapeND(intArrayOf(Nparams, 1)), doubleArrayOf(-23.6412, -16.7402, -21.5705, -21.0464, + -17.2852, -17.2959, -17.298, 0.9999, -17.2885, -17.3008, -17.2941, -17.2923, -17.2976, -17.3028, -17.2891)).as2D() + result.resultParameters = result.resultParameters.map { x -> (x * 1e4).roundToLong() / 1e4}.as2D() + val receivedParameters = zeros(ShapeND(intArrayOf(Nparams, 1))).as2D() + for (i in 0 until Nparams) { + receivedParameters[i, 0] = result.resultParameters[i, 0] + assertEquals(expectedParameters[i, 0], result.resultParameters[i, 0]) + } } } \ No newline at end of file From ef4335bc410fd7fdd82e88bea960ede05a41e2be Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Wed, 7 Jun 2023 15:24:01 +0300 Subject: [PATCH 25/26] use function types for input func --- .../kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt | 2 +- .../kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt index fe96b2fe9..f052279ae 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt @@ -16,7 +16,7 @@ import space.kscience.kmath.tensors.core.levenbergMarquardt import kotlin.random.Random import kotlin.reflect.KFunction3 -fun streamLm(lm_func: KFunction3, MutableStructure2D, Int, MutableStructure2D>, +fun streamLm(lm_func: (MutableStructure2D, MutableStructure2D, Int) -> (MutableStructure2D), startData: StartDataLm, launchFrequencyInMs: Long, numberOfLaunches: Int): Flow> = flow{ var example_number = startData.example_number diff --git a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt index d9c282fb6..3cb485d7d 100644 --- a/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt +++ b/kmath-tensors/src/commonMain/kotlin/space/kscience/kmath/tensors/core/LevenbergMarquardtAlgorithm.kt @@ -88,7 +88,7 @@ public data class LMResultInfo ( * exampleNumber: a parameter for a function with which you can choose its behavior. */ public data class LMInput ( - var func: KFunction3, MutableStructure2D, Int, MutableStructure2D>, + var func: (MutableStructure2D, MutableStructure2D, Int) -> (MutableStructure2D), var startParameters: MutableStructure2D, var independentVariables: MutableStructure2D, var realValues: MutableStructure2D, From 5f2690309b16d27f1fbf10760ff1a8e609149583 Mon Sep 17 00:00:00 2001 From: Margarita Lashina Date: Tue, 13 Jun 2023 03:06:55 +0300 Subject: [PATCH 26/26] fix mistake in streaming version --- .../kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt index f052279ae..b2818ef2a 100644 --- a/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt +++ b/examples/src/main/kotlin/space/kscience/kmath/tensors/LevenbergMarquardt/StreamingLm/streamLm.kt @@ -51,8 +51,8 @@ fun streamLm(lm_func: (MutableStructure2D, MutableStructure2D, I val result = DoubleTensorAlgebra.levenbergMarquardt(inputData) emit(result.resultParameters) delay(launchFrequencyInMs) - p_init = result.resultParameters - y_dat = generateNewYDat(y_dat, 0.1) + inputData.realValues = generateNewYDat(y_dat, 0.1) + inputData.startParameters = result.resultParameters if (!isEndless) steps -= 1 } }