From 25e8e03494898428b34a229c766d142f8d943a0f Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 20 Feb 2019 12:54:39 +0300 Subject: [PATCH] Memory on JVM and JS --- benchmarks/build.gradle | 1 + .../kmath/structures/NDFieldBenchmark.kt | 2 +- .../kmath/structures/ComplexNDBenchmark.kt | 6 +- .../kmath/structures/NDFieldBenchmark.kt | 4 +- build.gradle.kts | 2 +- kmath-core/build.gradle.kts | 1 + .../scientifik/kmath/operations/Complex.kt | 28 +++++- .../kmath/structures/ComplexNDField.kt | 5 +- .../scientifik/kmath/structures/NDAlgebra.kt | 7 +- .../scientifik/kmath/structures/NDElement.kt | 4 +- .../kmath/structures/ObjectBuffer.kt | 48 ++++++++++ .../kmath/structures/ComplexBufferSpecTest.kt | 5 +- .../kmath/structures/NumberNDFieldTest.kt | 2 +- .../scientifik/kmath/structures/BufferSpec.kt | 44 --------- .../kmath/structures/ComplexBufferSpec.kt | 46 --------- .../kmath/structures/ObjectBuffer.kt | 39 -------- .../kmath/structures/RealBufferSpec.kt | 28 ------ kmath-memory/build.gradle.kts | 50 ++++++++++ .../kotlin/scientifik/memory/Memory.kt | 71 ++++++++++++++ .../kotlin/scientifik/memory/MemorySpec.kt | 34 +++++++ .../scientifik/memory/DataViewMemory.kt | 91 ++++++++++++++++++ .../scientifik/memory/ByteBufferMemory.kt | 93 +++++++++++++++++++ .../kmath/sequential/BufferStreaming.kt | 4 + settings.gradle.kts | 3 +- 24 files changed, 441 insertions(+), 177 deletions(-) rename kmath-core/src/{jvmMain => commonMain}/kotlin/scientifik/kmath/structures/ComplexNDField.kt (97%) create mode 100644 kmath-core/src/commonMain/kotlin/scientifik/kmath/structures/ObjectBuffer.kt rename kmath-core/src/{jvmTest => commonTest}/kotlin/scientifik/kmath/structures/ComplexBufferSpecTest.kt (61%) delete mode 100644 kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/BufferSpec.kt delete mode 100644 kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/ComplexBufferSpec.kt delete mode 100644 kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/ObjectBuffer.kt delete mode 100644 kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/RealBufferSpec.kt create mode 100644 kmath-memory/build.gradle.kts create mode 100644 kmath-memory/src/commonMain/kotlin/scientifik/memory/Memory.kt create mode 100644 kmath-memory/src/commonMain/kotlin/scientifik/memory/MemorySpec.kt create mode 100644 kmath-memory/src/jsMain/kotlin/scientifik/memory/DataViewMemory.kt create mode 100644 kmath-memory/src/jvmMain/kotlin/scientifik/memory/ByteBufferMemory.kt diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle index c57d84fe4..989590397 100644 --- a/benchmarks/build.gradle +++ b/benchmarks/build.gradle @@ -16,6 +16,7 @@ dependencies { implementation project(":kmath-commons") implementation project(":kmath-koma") implementation group: "com.kyonifer", name:"koma-core-ejml", version: "0.12" + implementation "org.jetbrains.kotlinx:kotlinx-io-jvm:0.1.5" //compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8" //jmh project(':kmath-core') } diff --git a/benchmarks/src/jmh/kotlin/scientifik/kmath/structures/NDFieldBenchmark.kt b/benchmarks/src/jmh/kotlin/scientifik/kmath/structures/NDFieldBenchmark.kt index ad180b727..0cef4bdb4 100644 --- a/benchmarks/src/jmh/kotlin/scientifik/kmath/structures/NDFieldBenchmark.kt +++ b/benchmarks/src/jmh/kotlin/scientifik/kmath/structures/NDFieldBenchmark.kt @@ -61,7 +61,7 @@ open class NDFieldBenchmark { val dim = 1000 val n = 100 - val bufferedField = NDField.auto(intArrayOf(dim, dim), RealField) + val bufferedField = NDField.auto(RealField, intArrayOf(dim, dim)) val specializedField = NDField.real(intArrayOf(dim, dim)) val genericField = NDField.buffered(intArrayOf(dim, dim), RealField) val lazyNDField = NDField.lazy(intArrayOf(dim, dim), RealField) diff --git a/benchmarks/src/main/kotlin/scientifik/kmath/structures/ComplexNDBenchmark.kt b/benchmarks/src/main/kotlin/scientifik/kmath/structures/ComplexNDBenchmark.kt index 02f19cd55..46b1b56dd 100644 --- a/benchmarks/src/main/kotlin/scientifik/kmath/structures/ComplexNDBenchmark.kt +++ b/benchmarks/src/main/kotlin/scientifik/kmath/structures/ComplexNDBenchmark.kt @@ -1,15 +1,13 @@ package scientifik.kmath.structures -import scientifik.kmath.operations.Complex -import scientifik.kmath.operations.toComplex import kotlin.system.measureTimeMillis fun main() { val dim = 1000 val n = 1000 - val realField = NDField.real(intArrayOf(dim, dim)) - val complexField = NDField.complex(intArrayOf(dim, dim)) + val realField = NDField.real(dim, dim) + val complexField = NDField.complex(dim, dim) val realTime = measureTimeMillis { diff --git a/benchmarks/src/main/kotlin/scientifik/kmath/structures/NDFieldBenchmark.kt b/benchmarks/src/main/kotlin/scientifik/kmath/structures/NDFieldBenchmark.kt index 170320d22..53a23c2bc 100644 --- a/benchmarks/src/main/kotlin/scientifik/kmath/structures/NDFieldBenchmark.kt +++ b/benchmarks/src/main/kotlin/scientifik/kmath/structures/NDFieldBenchmark.kt @@ -9,9 +9,9 @@ fun main(args: Array) { val n = 1000 // automatically build context most suited for given type. - val autoField = NDField.auto(intArrayOf(dim, dim), RealField) + val autoField = NDField.auto(RealField, dim, dim) // specialized nd-field for Double. It works as generic Double field as well - val specializedField = NDField.real(intArrayOf(dim, dim)) + val specializedField = NDField.real(dim, dim) //A generic boxing field. It should be used for objects, not primitives. val genericField = NDField.buffered(intArrayOf(dim, dim), RealField) diff --git a/build.gradle.kts b/build.gradle.kts index 702373b1e..dadb99422 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -2,7 +2,7 @@ import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension buildscript { val kotlinVersion: String by rootProject.extra("1.3.21") - val ioVersion: String by rootProject.extra("0.1.4") + val ioVersion: String by rootProject.extra("0.1.5") val coroutinesVersion: String by rootProject.extra("1.1.1") val atomicfuVersion: String by rootProject.extra("0.12.1") diff --git a/kmath-core/build.gradle.kts b/kmath-core/build.gradle.kts index f6d10875c..bd125b373 100644 --- a/kmath-core/build.gradle.kts +++ b/kmath-core/build.gradle.kts @@ -12,6 +12,7 @@ kotlin { sourceSets { val commonMain by getting { dependencies { + api(project(":kmath-memory")) api(kotlin("stdlib")) } } diff --git a/kmath-core/src/commonMain/kotlin/scientifik/kmath/operations/Complex.kt b/kmath-core/src/commonMain/kotlin/scientifik/kmath/operations/Complex.kt index 3003ce1bc..4b4002cbb 100644 --- a/kmath-core/src/commonMain/kotlin/scientifik/kmath/operations/Complex.kt +++ b/kmath-core/src/commonMain/kotlin/scientifik/kmath/operations/Complex.kt @@ -1,5 +1,11 @@ package scientifik.kmath.operations +import scientifik.kmath.structures.Buffer +import scientifik.kmath.structures.MutableBuffer +import scientifik.kmath.structures.ObjectBuffer +import scientifik.memory.MemoryReader +import scientifik.memory.MemorySpec +import scientifik.memory.MemoryWriter import kotlin.math.* /** @@ -67,7 +73,25 @@ data class Complex(val re: Double, val im: Double) : FieldElement { + override val objectSize: Int = 16 + + override fun MemoryReader.read(offset: Int): Complex = + Complex(readDouble(offset), readDouble(offset + 8)) + + override fun MemoryWriter.write(offset: Int, value: Complex) { + writeDouble(offset, value.re) + writeDouble(offset + 8, value.im) + } + } } -fun Double.toComplex() = Complex(this, 0.0) \ No newline at end of file +fun Double.toComplex() = Complex(this, 0.0) + +fun Buffer.Companion.complex(size: Int, init: (Int) -> Complex): Buffer { + return ObjectBuffer.create(Complex, size, init) +} + +fun MutableBuffer.Companion.complex(size: Int, init: (Int) -> Complex): Buffer { + return ObjectBuffer.create(Complex, size, init) +} diff --git a/kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/ComplexNDField.kt b/kmath-core/src/commonMain/kotlin/scientifik/kmath/structures/ComplexNDField.kt similarity index 97% rename from kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/ComplexNDField.kt rename to kmath-core/src/commonMain/kotlin/scientifik/kmath/structures/ComplexNDField.kt index a8b31af3e..6444f667d 100644 --- a/kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/ComplexNDField.kt +++ b/kmath-core/src/commonMain/kotlin/scientifik/kmath/structures/ComplexNDField.kt @@ -3,6 +3,7 @@ package scientifik.kmath.structures import scientifik.kmath.operations.Complex import scientifik.kmath.operations.ComplexField import scientifik.kmath.operations.FieldElement +import scientifik.kmath.operations.complex typealias ComplexNDElement = BufferedNDFieldElement @@ -128,4 +129,6 @@ operator fun ComplexNDElement.plus(arg: Double) = map { it + arg } operator fun ComplexNDElement.minus(arg: Double) = - map { it - arg } \ No newline at end of file + map { it - arg } + +fun NDField.Companion.complex(vararg shape: Int) = ComplexNDField(shape) \ No newline at end of file diff --git a/kmath-core/src/commonMain/kotlin/scientifik/kmath/structures/NDAlgebra.kt b/kmath-core/src/commonMain/kotlin/scientifik/kmath/structures/NDAlgebra.kt index 7ea768c63..887a480bb 100644 --- a/kmath-core/src/commonMain/kotlin/scientifik/kmath/structures/NDAlgebra.kt +++ b/kmath-core/src/commonMain/kotlin/scientifik/kmath/structures/NDAlgebra.kt @@ -3,6 +3,7 @@ package scientifik.kmath.structures import scientifik.kmath.operations.Field import scientifik.kmath.operations.Ring import scientifik.kmath.operations.Space +import kotlin.jvm.JvmName /** @@ -118,7 +119,7 @@ interface NDField, N : NDStructure> : Field, NDRing, N : NDStructure> : Field, NDRing> auto(shape: IntArray, field: F): BufferedNDField = + inline fun > auto(field: F, vararg shape: Int): BufferedNDField = when { - T::class == Double::class -> real(shape) as BufferedNDField + T::class == Double::class -> real(*shape) as BufferedNDField else -> BoxingNDField(shape, field, Buffer.Companion::auto) } } diff --git a/kmath-core/src/commonMain/kotlin/scientifik/kmath/structures/NDElement.kt b/kmath-core/src/commonMain/kotlin/scientifik/kmath/structures/NDElement.kt index c97f959f3..59cf98df3 100644 --- a/kmath-core/src/commonMain/kotlin/scientifik/kmath/structures/NDElement.kt +++ b/kmath-core/src/commonMain/kotlin/scientifik/kmath/structures/NDElement.kt @@ -24,7 +24,7 @@ interface NDElement> : NDStructure { * Create a optimized NDArray of doubles */ fun real(shape: IntArray, initializer: RealField.(IntArray) -> Double = { 0.0 }) = - NDField.real(shape).produce(initializer) + NDField.real(*shape).produce(initializer) fun real1D(dim: Int, initializer: (Int) -> Double = { _ -> 0.0 }) = @@ -55,7 +55,7 @@ interface NDElement> : NDStructure { field: F, noinline initializer: F.(IntArray) -> T ): BufferedNDFieldElement { - val ndField = NDField.auto(shape, field) + val ndField = NDField.auto(field, *shape) return BufferedNDFieldElement(ndField, ndField.produce(initializer).buffer) } } diff --git a/kmath-core/src/commonMain/kotlin/scientifik/kmath/structures/ObjectBuffer.kt b/kmath-core/src/commonMain/kotlin/scientifik/kmath/structures/ObjectBuffer.kt new file mode 100644 index 000000000..badf8a483 --- /dev/null +++ b/kmath-core/src/commonMain/kotlin/scientifik/kmath/structures/ObjectBuffer.kt @@ -0,0 +1,48 @@ +package scientifik.kmath.structures + +import scientifik.memory.* + +/** + * A non-boxing buffer based on [ByteBuffer] storage + */ +open class ObjectBuffer(protected val memory: Memory, protected val spec: MemorySpec) : Buffer { + + override val size: Int get() = memory.size / spec.objectSize + + private val reader = memory.reader() + + override fun get(index: Int): T = reader.read(spec, spec.objectSize * index) + + override fun iterator(): Iterator = (0 until size).asSequence().map { get(it) }.iterator() + + + companion object { + fun create(spec: MemorySpec, size: Int) = + ObjectBuffer(Memory.allocate(size * spec.objectSize), spec) + + inline fun create( + spec: MemorySpec, + size: Int, + crossinline initializer: (Int) -> T + ): ObjectBuffer = + MutableObjectBuffer(Memory.allocate(size * spec.objectSize), spec).also { buffer -> + (0 until size).forEach { + buffer[it] = initializer(it) + } + } + } +} + +class MutableObjectBuffer(memory: Memory, spec: MemorySpec) : ObjectBuffer(memory, spec), + MutableBuffer { + + private val writer = memory.writer() + + override fun set(index: Int, value: T) = writer.write(spec, spec.objectSize * index, value) + + override fun copy(): MutableBuffer = MutableObjectBuffer(memory.copy(), spec) + + companion object { + + } +} \ No newline at end of file diff --git a/kmath-core/src/jvmTest/kotlin/scientifik/kmath/structures/ComplexBufferSpecTest.kt b/kmath-core/src/commonTest/kotlin/scientifik/kmath/structures/ComplexBufferSpecTest.kt similarity index 61% rename from kmath-core/src/jvmTest/kotlin/scientifik/kmath/structures/ComplexBufferSpecTest.kt rename to kmath-core/src/commonTest/kotlin/scientifik/kmath/structures/ComplexBufferSpecTest.kt index a07f2167e..454683dac 100644 --- a/kmath-core/src/jvmTest/kotlin/scientifik/kmath/structures/ComplexBufferSpecTest.kt +++ b/kmath-core/src/commonTest/kotlin/scientifik/kmath/structures/ComplexBufferSpecTest.kt @@ -1,13 +1,14 @@ package scientifik.kmath.structures -import org.junit.Test import scientifik.kmath.operations.Complex +import scientifik.kmath.operations.complex +import kotlin.test.Test import kotlin.test.assertEquals class ComplexBufferSpecTest { @Test fun testComplexBuffer() { - val buffer = MutableBuffer.complex(20){Complex(it.toDouble(), -it.toDouble())} + val buffer = Buffer.complex(20) { Complex(it.toDouble(), -it.toDouble()) } assertEquals(Complex(5.0, -5.0), buffer[5]) } } \ No newline at end of file diff --git a/kmath-core/src/commonTest/kotlin/scientifik/kmath/structures/NumberNDFieldTest.kt b/kmath-core/src/commonTest/kotlin/scientifik/kmath/structures/NumberNDFieldTest.kt index 3c4160329..60f1f9979 100644 --- a/kmath-core/src/commonTest/kotlin/scientifik/kmath/structures/NumberNDFieldTest.kt +++ b/kmath-core/src/commonTest/kotlin/scientifik/kmath/structures/NumberNDFieldTest.kt @@ -63,7 +63,7 @@ class NumberNDFieldTest { @Test fun testInternalContext() { - NDField.real(array1.shape).run { + NDField.real(*array1.shape).run { with(L2Norm) { 1 + norm(array1) + exp(array2) } diff --git a/kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/BufferSpec.kt b/kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/BufferSpec.kt deleted file mode 100644 index ce87a1298..000000000 --- a/kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/BufferSpec.kt +++ /dev/null @@ -1,44 +0,0 @@ -package scientifik.kmath.structures - -import java.nio.ByteBuffer - - -/** - * A specification for serialization and deserialization objects to buffer (at current buffer position) - */ -interface BufferSpec { - /** - * Read an object from buffer in current position - */ - fun ByteBuffer.readObject(): T - - /** - * Write object to [ByteBuffer] in current buffer position - */ - fun ByteBuffer.writeObject(value: T) -} - -/** - * A [BufferSpec] with fixed unit size. Allows storage of any object without boxing. - */ -interface FixedSizeBufferSpec : BufferSpec { - val unitSize: Int - - - /** - * Read an object from buffer in given index (not buffer position - */ - fun ByteBuffer.readObject(index: Int): T { - position(index * unitSize) - return readObject() - } - - - /** - * Put an object in given index - */ - fun ByteBuffer.writeObject(index: Int, obj: T) { - position(index * unitSize) - writeObject(obj) - } -} diff --git a/kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/ComplexBufferSpec.kt b/kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/ComplexBufferSpec.kt deleted file mode 100644 index 34f21cf75..000000000 --- a/kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/ComplexBufferSpec.kt +++ /dev/null @@ -1,46 +0,0 @@ -package scientifik.kmath.structures - -import scientifik.kmath.operations.Complex -import scientifik.kmath.operations.ComplexField -import java.nio.ByteBuffer - -/** - * A serialization specification for complex numbers - */ -object ComplexBufferSpec : FixedSizeBufferSpec { - - override val unitSize: Int = 16 - - override fun ByteBuffer.readObject(): Complex { - val re = double - val im = double - return Complex(re, im) - } - - override fun ByteBuffer.writeObject(value: Complex) { - putDouble(value.re) - putDouble(value.im) - } -} - -/** - * Create a read-only/mutable buffer which ignores boxing - */ -fun Buffer.Companion.complex(size: Int): Buffer = - ObjectBuffer.create(ComplexBufferSpec, size) - -inline fun Buffer.Companion.complex(size: Int, crossinline initializer: (Int) -> Complex): Buffer = - ObjectBuffer.create(ComplexBufferSpec, size, initializer) - -fun MutableBuffer.Companion.complex(size: Int) = - ObjectBuffer.create(ComplexBufferSpec, size) - -inline fun MutableBuffer.Companion.complex(size: Int, crossinline initializer: (Int) -> Complex) = - ObjectBuffer.create(ComplexBufferSpec, size, initializer) - -fun NDField.Companion.complex(shape: IntArray) = ComplexNDField(shape) - -fun NDElement.Companion.complex(shape: IntArray, initializer: ComplexField.(IntArray) -> Complex) = - NDField.complex(shape).produce(initializer) - - diff --git a/kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/ObjectBuffer.kt b/kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/ObjectBuffer.kt deleted file mode 100644 index e95c54b2c..000000000 --- a/kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/ObjectBuffer.kt +++ /dev/null @@ -1,39 +0,0 @@ -package scientifik.kmath.structures - -import java.nio.ByteBuffer - -/** - * A non-boxing buffer based on [ByteBuffer] storage - */ -class ObjectBuffer(private val buffer: ByteBuffer, private val spec: FixedSizeBufferSpec) : - MutableBuffer { - override val size: Int - get() = buffer.limit() / spec.unitSize - - override fun get(index: Int): T = with(spec) { buffer.readObject(index) } - - override fun iterator(): Iterator = (0 until size).asSequence().map { get(it) }.iterator() - - override fun set(index: Int, value: T) = with(spec) { buffer.writeObject(index, value) } - - override fun copy(): MutableBuffer { - val dup = buffer.duplicate() - val copy = ByteBuffer.allocate(dup.capacity()) - dup.rewind() - copy.put(dup) - copy.flip() - return ObjectBuffer(copy, spec) - } - - companion object { - fun create(spec: FixedSizeBufferSpec, size: Int) = - ObjectBuffer(ByteBuffer.allocate(size * spec.unitSize), spec) - - inline fun create(spec: FixedSizeBufferSpec, size: Int, crossinline initializer: (Int) -> T) = - ObjectBuffer(ByteBuffer.allocate(size * spec.unitSize), spec).also { buffer -> - (0 until size).forEach { - buffer[it] = initializer(it) - } - } - } -} \ No newline at end of file diff --git a/kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/RealBufferSpec.kt b/kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/RealBufferSpec.kt deleted file mode 100644 index fdd9e92b7..000000000 --- a/kmath-core/src/jvmMain/kotlin/scientifik/kmath/structures/RealBufferSpec.kt +++ /dev/null @@ -1,28 +0,0 @@ -package scientifik.kmath.structures - -import scientifik.kmath.operations.Real -import java.nio.ByteBuffer - -object RealBufferSpec : FixedSizeBufferSpec { - override val unitSize: Int = 8 - - override fun ByteBuffer.readObject(): Real = Real(double) - - override fun ByteBuffer.writeObject(value: Real) { - putDouble(value.value) - } -} - -object DoubleBufferSpec : FixedSizeBufferSpec { - override val unitSize: Int = 8 - - override fun ByteBuffer.readObject() = double - - override fun ByteBuffer.writeObject(value: Double) { - putDouble(value) - } - -} - -fun Double.Companion.createBuffer(size: Int) = ObjectBuffer.create(DoubleBufferSpec, size) -fun Real.Companion.createBuffer(size: Int) = ObjectBuffer.create(RealBufferSpec, size) \ No newline at end of file diff --git a/kmath-memory/build.gradle.kts b/kmath-memory/build.gradle.kts new file mode 100644 index 000000000..f6d10875c --- /dev/null +++ b/kmath-memory/build.gradle.kts @@ -0,0 +1,50 @@ +plugins { + kotlin("multiplatform") +} + +val ioVersion: String by rootProject.extra + + +kotlin { + jvm() + js() + + sourceSets { + val commonMain by getting { + dependencies { + api(kotlin("stdlib")) + } + } + val commonTest by getting { + dependencies { + implementation(kotlin("test-common")) + implementation(kotlin("test-annotations-common")) + } + } + val jvmMain by getting { + dependencies { + api(kotlin("stdlib-jdk8")) + } + } + val jvmTest by getting { + dependencies { + implementation(kotlin("test")) + implementation(kotlin("test-junit")) + } + } + val jsMain by getting { + dependencies { + api(kotlin("stdlib-js")) + } + } + val jsTest by getting { + dependencies { + implementation(kotlin("test-js")) + } + } +// mingwMain { +// } +// mingwTest { +// } + } +} \ No newline at end of file diff --git a/kmath-memory/src/commonMain/kotlin/scientifik/memory/Memory.kt b/kmath-memory/src/commonMain/kotlin/scientifik/memory/Memory.kt new file mode 100644 index 000000000..f9f938dcc --- /dev/null +++ b/kmath-memory/src/commonMain/kotlin/scientifik/memory/Memory.kt @@ -0,0 +1,71 @@ +package scientifik.memory + +interface Memory { + val size: Int + + /** + * Get a projection of this memory (it reflects the changes in the parent memory block) + */ + fun view(offset: Int, length: Int): Memory + + /** + * Create a copy of this memory, which does not know anything about this memory + */ + fun copy(): Memory + + /** + * Create and possibly register a new reader + */ + fun reader(): MemoryReader + + fun writer(): MemoryWriter + + companion object { + + } +} + +interface MemoryReader { + val memory: Memory + + fun readDouble(offset: Int): Double + fun readFloat(offset: Int): Float + fun readByte(offset: Int): Byte + fun readShort(offset: Int): Short + fun readInt(offset: Int): Int + fun readLong(offset: Int): Long + + fun release() +} + +/** + * Use the memory for read then release the reader + */ +inline fun Memory.read(block: MemoryReader.() -> Unit) { + reader().apply(block).apply { release() } +} + +interface MemoryWriter { + val memory: Memory + + fun writeDouble(offset: Int, value: Double) + fun writeFloat(offset: Int, value: Float) + fun writeByte(offset: Int, value: Byte) + fun writeShort(offset: Int, value: Short) + fun writeInt(offset: Int, value: Int) + fun writeLong(offset: Int, value: Long) + + fun release() +} + +/** + * Use the memory for write then release the writer + */ +inline fun Memory.write(block: MemoryWriter.() -> Unit) { + writer().apply(block).apply { release() } +} + +/** + * Allocate the most effective platform-specific memory + */ +expect fun Memory.Companion.allocate(length: Int): Memory diff --git a/kmath-memory/src/commonMain/kotlin/scientifik/memory/MemorySpec.kt b/kmath-memory/src/commonMain/kotlin/scientifik/memory/MemorySpec.kt new file mode 100644 index 000000000..a3166ecd7 --- /dev/null +++ b/kmath-memory/src/commonMain/kotlin/scientifik/memory/MemorySpec.kt @@ -0,0 +1,34 @@ +package scientifik.memory + +/** + * A specification to read or write custom objects with fixed size in bytes + */ +interface MemorySpec { + /** + * Size of [T] in bytes after serialization + */ + val objectSize: Int + + fun MemoryReader.read(offset: Int): T + fun MemoryWriter.write(offset: Int, value: T) +} + +fun MemoryReader.read(spec: MemorySpec, offset: Int): T = spec.run { read(offset) } +fun MemoryWriter.write(spec: MemorySpec, offset: Int, value: T) = spec.run { write(offset, value) } + +inline fun MemoryReader.readArray(spec: MemorySpec, offset: Int, size: Int) = + Array(size) { i -> + spec.run { + read(offset + i * objectSize) + } + } + +fun MemoryWriter.writeArray(spec: MemorySpec, offset: Int, array: Array) { + spec.run { + for (i in 0 until array.size) { + write(offset + i * objectSize, array[i]) + } + } +} + +//TODO It is possible to add elastic MemorySpec with unknown object size \ No newline at end of file diff --git a/kmath-memory/src/jsMain/kotlin/scientifik/memory/DataViewMemory.kt b/kmath-memory/src/jsMain/kotlin/scientifik/memory/DataViewMemory.kt new file mode 100644 index 000000000..843464ab9 --- /dev/null +++ b/kmath-memory/src/jsMain/kotlin/scientifik/memory/DataViewMemory.kt @@ -0,0 +1,91 @@ +package scientifik.memory + +import org.khronos.webgl.ArrayBuffer +import org.khronos.webgl.DataView + +/** + * Allocate the most effective platform-specific memory + */ +actual fun Memory.Companion.allocate(length: Int): Memory { + val buffer = ArrayBuffer(length) + return DataViewMemory(DataView(buffer, 0, length)) +} + +class DataViewMemory(val view: DataView) : Memory { + + override val size: Int get() = view.byteLength + + override fun view(offset: Int, length: Int): Memory { + require(offset >= 0) { "offset shouldn't be negative: $offset" } + require(length >= 0) { "length shouldn't be negative: $length" } + if (offset + length > size) { + throw IndexOutOfBoundsException("offset + length > size: $offset + $length > $size") + } + return DataViewMemory(DataView(view.buffer, view.byteOffset + offset, length)) + } + + + override fun copy(): Memory { + TODO("not implemented") //To change body of created functions use File | Settings | File Templates. + } + + private val reader = object : MemoryReader { + override val memory: Memory get() = this@DataViewMemory + + override fun readDouble(offset: Int): Double = view.getFloat64(offset, false) + + override fun readFloat(offset: Int): Float = view.getFloat32(offset, false) + + override fun readByte(offset: Int): Byte = view.getInt8(offset) + + override fun readShort(offset: Int): Short = view.getInt16(offset, false) + + override fun readInt(offset: Int): Int = view.getInt32(offset, false) + + override fun readLong(offset: Int): Long = (view.getInt32(offset, false).toLong() shl 32) or + view.getInt32(offset + 4, false).toLong() + + override fun release() { + // does nothing on JS because of GC + } + } + + override fun reader(): MemoryReader = reader + + private val writer = object : MemoryWriter { + override val memory: Memory get() = this@DataViewMemory + + override fun writeDouble(offset: Int, value: Double) { + view.setFloat64(offset, value, false) + } + + override fun writeFloat(offset: Int, value: Float) { + view.setFloat32(offset, value, false) + } + + override fun writeByte(offset: Int, value: Byte) { + view.setInt8(offset, value) + } + + override fun writeShort(offset: Int, value: Short) { + view.setUint16(offset, value, false) + } + + override fun writeInt(offset: Int, value: Int) { + view.setInt32(offset, value, false) + } + + override fun writeLong(offset: Int, value: Long) { + view.setInt32(offset, (value shr 32).toInt(), littleEndian = false) + view.setInt32(offset + 4, (value and 0xffffffffL).toInt(), littleEndian = false) + } + + override fun release() { + //does nothing on JS + } + + } + + override fun writer(): MemoryWriter = writer + +} \ No newline at end of file diff --git a/kmath-memory/src/jvmMain/kotlin/scientifik/memory/ByteBufferMemory.kt b/kmath-memory/src/jvmMain/kotlin/scientifik/memory/ByteBufferMemory.kt new file mode 100644 index 000000000..30a87228a --- /dev/null +++ b/kmath-memory/src/jvmMain/kotlin/scientifik/memory/ByteBufferMemory.kt @@ -0,0 +1,93 @@ +package scientifik.memory + +import java.nio.ByteBuffer + + +/** + * Allocate the most effective platform-specific memory + */ +actual fun Memory.Companion.allocate(length: Int): Memory { + val buffer = ByteBuffer.allocate(length) + return ByteBufferMemory(buffer) +} + +class ByteBufferMemory( + val buffer: ByteBuffer, + val startOffset: Int = 0, + override val size: Int = buffer.limit() +) : Memory { + + + @Suppress("NOTHING_TO_INLINE") + private inline fun position(o: Int): Int = startOffset + o + + override fun view(offset: Int, length: Int): Memory { + if (offset + length > size) error("Selecting a Memory view outside of memory range") + return ByteBufferMemory(buffer, position(offset), length) + } + + override fun copy(): Memory { + val copy = ByteBuffer.allocate(buffer.capacity()) + buffer.rewind() + copy.put(buffer) + copy.flip() + return ByteBufferMemory(copy) + + } + + private val reader = object : MemoryReader { + override val memory: Memory get() = this@ByteBufferMemory + + override fun readDouble(offset: Int) = buffer.getDouble(position(offset)) + + override fun readFloat(offset: Int) = buffer.getFloat(position(offset)) + + override fun readByte(offset: Int) = buffer.get(position(offset)) + + override fun readShort(offset: Int) = buffer.getShort(position(offset)) + + override fun readInt(offset: Int) = buffer.getInt(position(offset)) + + override fun readLong(offset: Int) = buffer.getLong(position(offset)) + + override fun release() { + //does nothing on JVM + } + } + + override fun reader(): MemoryReader = reader + + private val writer = object : MemoryWriter { + override val memory: Memory get() = this@ByteBufferMemory + + override fun writeDouble(offset: Int, value: Double) { + buffer.putDouble(position(offset), value) + } + + override fun writeFloat(offset: Int, value: Float) { + buffer.putFloat(position(offset), value) + } + + override fun writeByte(offset: Int, value: Byte) { + buffer.put(position(offset), value) + } + + override fun writeShort(offset: Int, value: Short) { + buffer.putShort(position(offset), value) + } + + override fun writeInt(offset: Int, value: Int) { + buffer.putInt(position(offset), value) + } + + override fun writeLong(offset: Int, value: Long) { + buffer.putLong(position(offset), value) + } + + override fun release() { + //does nothing on JVM + } + } + + override fun writer(): MemoryWriter = writer +} \ No newline at end of file diff --git a/kmath-sequential/src/commonMain/kotlin/scientifik/kmath/sequential/BufferStreaming.kt b/kmath-sequential/src/commonMain/kotlin/scientifik/kmath/sequential/BufferStreaming.kt index ad9e4f259..b98f48186 100644 --- a/kmath-sequential/src/commonMain/kotlin/scientifik/kmath/sequential/BufferStreaming.kt +++ b/kmath-sequential/src/commonMain/kotlin/scientifik/kmath/sequential/BufferStreaming.kt @@ -1,6 +1,7 @@ package scientifik.kmath.sequential import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.channels.produce import kotlinx.coroutines.isActive @@ -12,6 +13,7 @@ import scientifik.kmath.structures.BufferFactory /** * A processor that collects incoming elements into fixed size buffers */ +@ExperimentalCoroutinesApi class JoinProcessor( scope: CoroutineScope, bufferSize: Int, @@ -68,9 +70,11 @@ class SplitProcessor(scope: CoroutineScope) : AbstractProcessor, T> } } +@ExperimentalCoroutinesApi fun Producer.chunked(chunkSize: Int, bufferFactory: BufferFactory) = JoinProcessor(this, chunkSize, bufferFactory).also { connect(it) } +@ExperimentalCoroutinesApi inline fun Producer.chunked(chunkSize: Int) = JoinProcessor(this, chunkSize, Buffer.Companion::auto).also { connect(it) } diff --git a/settings.gradle.kts b/settings.gradle.kts index 7d8731532..8684de3ee 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -15,10 +15,11 @@ pluginManagement { } } -//enableFeaturePreview("GRADLE_METADATA") +enableFeaturePreview("GRADLE_METADATA") rootProject.name = "kmath" include( + ":kmath-memory", ":kmath-core", // ":kmath-io", ":kmath-coroutines",