Document memory module and several ND objects

This commit is contained in:
Iaroslav 2020-08-08 04:21:59 +07:00
parent 0a8044ddb3
commit c8cd6cd288
No known key found for this signature in database
GPG Key ID: 46E15E4A31B3BCD7
12 changed files with 217 additions and 67 deletions

View File

@ -3,7 +3,6 @@ package scientifik.kmath.structures
import scientifik.kmath.operations.Field import scientifik.kmath.operations.Field
import scientifik.kmath.operations.FieldElement import scientifik.kmath.operations.FieldElement
class BoxingNDField<T, F : Field<T>>( class BoxingNDField<T, F : Field<T>>(
override val shape: IntArray, override val shape: IntArray,
override val elementContext: F, override val elementContext: F,

View File

@ -11,7 +11,8 @@ interface BufferedNDAlgebra<T, C> : NDAlgebra<T, C, NDBuffer<T>> {
/** /**
* Convert any [NDStructure] to buffered structure using strides from this context. * Convert any [NDStructure] to buffered structure using strides from this context.
* If the structure is already [NDBuffer], conversion is free. If not, it could be expensive because iteration over indexes * If the structure is already [NDBuffer], conversion is free. If not, it could be expensive because iteration over
* indices.
* *
* If the argument is [NDBuffer] with different strides structure, the new element will be produced. * If the argument is [NDBuffer] with different strides structure, the new element will be produced.
*/ */

View File

@ -98,13 +98,13 @@ inline fun BufferedNDField<Complex, ComplexField>.produceInline(crossinline init
} }
/** /**
* Map one [ComplexNDElement] using function with indexes * Map one [ComplexNDElement] using function with indices.
*/ */
inline fun ComplexNDElement.mapIndexed(crossinline transform: ComplexField.(index: IntArray, Complex) -> Complex): ComplexNDElement = inline fun ComplexNDElement.mapIndexed(crossinline transform: ComplexField.(index: IntArray, Complex) -> Complex): ComplexNDElement =
context.produceInline { offset -> transform(strides.index(offset), buffer[offset]) } context.produceInline { offset -> transform(strides.index(offset), buffer[offset]) }
/** /**
* Map one [ComplexNDElement] using function without indexes * Map one [ComplexNDElement] using function without indices.
*/ */
inline fun ComplexNDElement.map(crossinline transform: ComplexField.(Complex) -> Complex): ComplexNDElement { inline fun ComplexNDElement.map(crossinline transform: ComplexField.(Complex) -> Complex): ComplexNDElement {
val buffer = Buffer.complex(strides.linearSize) { offset -> ComplexField.transform(buffer[offset]) } val buffer = Buffer.complex(strides.linearSize) { offset -> ComplexField.transform(buffer[offset]) }

View File

@ -2,6 +2,13 @@ package scientifik.kmath.structures
import scientifik.kmath.operations.ExtendedField import scientifik.kmath.operations.ExtendedField
/**
* [ExtendedField] over [NDStructure].
*
* @param T the type of the element contained in ND structure.
* @param N the type of ND structure.
* @param F the extended field of structure elements.
*/
interface ExtendedNDField<T : Any, F : ExtendedField<T>, N : NDStructure<T>> : NDField<T, F, N>, ExtendedField<N> interface ExtendedNDField<T : Any, F : ExtendedField<T>, N : NDStructure<T>> : NDField<T, F, N>, ExtendedField<N>
///** ///**

View File

@ -10,7 +10,6 @@ import scientifik.memory.*
* @property spec the spec of [T] type. * @property spec the spec of [T] type.
*/ */
open class MemoryBuffer<T : Any>(protected val memory: Memory, protected val spec: MemorySpec<T>) : Buffer<T> { open class MemoryBuffer<T : Any>(protected val memory: Memory, protected val spec: MemorySpec<T>) : Buffer<T> {
override val size: Int get() = memory.size / spec.objectSize override val size: Int get() = memory.size / spec.objectSize
private val reader: MemoryReader = memory.reader() private val reader: MemoryReader = memory.reader()

View File

@ -105,10 +105,11 @@ interface NDRing<T, R : Ring<T>, N : NDStructure<T>> : Ring<N>, NDSpace<T, R, N>
} }
/** /**
* Field for n-dimensional structures. * Field of [NDStructure].
* *
* @param T the type of the element contained in ND structure * @param T the type of the element contained in ND structure.
* @param F field of structure elements * @param N the type of ND structure.
* @param F field of structure elements.
*/ */
interface NDField<T, F : Field<T>, N : NDStructure<T>> : Field<N>, NDRing<T, F, N> { interface NDField<T, F : Field<T>, N : NDStructure<T>> : Field<N>, NDRing<T, F, N> {

View File

@ -3,15 +3,38 @@ package scientifik.kmath.structures
import kotlin.jvm.JvmName import kotlin.jvm.JvmName
import kotlin.reflect.KClass import kotlin.reflect.KClass
/**
* Represents n-dimensional structure, i.e. multidimensional container of items of the same type and size. The number
* of dimensions and items in an array is defined by its shape, which is a sequence of non-negative integers that
* specify the sizes of each dimension.
*
* @param T the type of items.
*/
interface NDStructure<T> { interface NDStructure<T> {
/**
* The shape of structure, i.e. non-empty sequence of non-negative integers that specify sizes of dimensions of
* this structure.
*/
val shape: IntArray val shape: IntArray
/**
* The count of dimensions in this structure. It should be equal to size of [shape].
*/
val dimension: Int get() = shape.size val dimension: Int get() = shape.size
/**
* Returns the value at the specified indices.
*
* @param index the indices.
* @return the value.
*/
operator fun get(index: IntArray): T operator fun get(index: IntArray): T
/**
* Returns the sequence of all the elements associated by their indices.
*
* @return the lazy sequence of pairs of indices to values.
*/
fun elements(): Sequence<Pair<IntArray, T>> fun elements(): Sequence<Pair<IntArray, T>>
override fun equals(other: Any?): Boolean override fun equals(other: Any?): Boolean
@ -19,6 +42,9 @@ interface NDStructure<T> {
override fun hashCode(): Int override fun hashCode(): Int
companion object { companion object {
/**
* Indicates whether some [NDStructure] is equal to another one.
*/
fun equals(st1: NDStructure<*>, st2: NDStructure<*>): Boolean { fun equals(st1: NDStructure<*>, st2: NDStructure<*>): Boolean {
if (st1 === st2) return true if (st1 === st2) return true
@ -36,9 +62,9 @@ interface NDStructure<T> {
} }
/** /**
* Create a NDStructure with explicit buffer factory * Creates a NDStructure with explicit buffer factory.
* *
* Strides should be reused if possible * Strides should be reused if possible.
*/ */
fun <T> build( fun <T> build(
strides: Strides, strides: Strides,
@ -91,9 +117,24 @@ interface NDStructure<T> {
} }
} }
/**
* Returns the value at the specified indices.
*
* @param index the indices.
* @return the value.
*/
operator fun <T> NDStructure<T>.get(vararg index: Int): T = get(index) operator fun <T> NDStructure<T>.get(vararg index: Int): T = get(index)
/**
* Represents mutable [NDStructure].
*/
interface MutableNDStructure<T> : NDStructure<T> { interface MutableNDStructure<T> : NDStructure<T> {
/**
* Inserts an item at the specified indices.
*
* @param index the indices.
* @param value the value.
*/
operator fun set(index: IntArray, value: T) operator fun set(index: IntArray, value: T)
} }
@ -104,7 +145,7 @@ inline fun <T> MutableNDStructure<T>.mapInPlace(action: (IntArray, T) -> T) {
} }
/** /**
* A way to convert ND index to linear one and back * A way to convert ND index to linear one and back.
*/ */
interface Strides { interface Strides {
/** /**
@ -141,6 +182,9 @@ interface Strides {
} }
} }
/**
* Simple implementation of [Strides].
*/
class DefaultStrides private constructor(override val shape: IntArray) : Strides { class DefaultStrides private constructor(override val shape: IntArray) : Strides {
/** /**
* Strides for memory access * Strides for memory access
@ -180,19 +224,14 @@ class DefaultStrides private constructor(override val shape: IntArray) : Strides
override val linearSize: Int override val linearSize: Int
get() = strides[shape.size] get() = strides[shape.size]
override fun equals(other: Any?): Boolean { override fun equals(other: Any?): Boolean {
if (this === other) return true if (this === other) return true
if (other !is DefaultStrides) return false if (other !is DefaultStrides) return false
if (!shape.contentEquals(other.shape)) return false if (!shape.contentEquals(other.shape)) return false
return true return true
} }
override fun hashCode(): Int { override fun hashCode(): Int = shape.contentHashCode()
return shape.contentHashCode()
}
companion object { companion object {
private val defaultStridesCache = HashMap<IntArray, Strides>() private val defaultStridesCache = HashMap<IntArray, Strides>()
@ -204,8 +243,20 @@ class DefaultStrides private constructor(override val shape: IntArray) : Strides
} }
} }
/**
* Represents [NDStructure] over [Buffer].
*
* @param T the type of items.
*/
abstract class NDBuffer<T> : NDStructure<T> { abstract class NDBuffer<T> : NDStructure<T> {
/**
* The underlying buffer.
*/
abstract val buffer: Buffer<T> abstract val buffer: Buffer<T>
/**
* The strides to access elements of [Buffer] by linear indices.
*/
abstract val strides: Strides abstract val strides: Strides
override fun get(index: IntArray): T = buffer[strides.offset(index)] override fun get(index: IntArray): T = buffer[strides.offset(index)]
@ -263,8 +314,8 @@ class MutableBufferNDStructure<T>(
) : NDBuffer<T>(), MutableNDStructure<T> { ) : NDBuffer<T>(), MutableNDStructure<T> {
init { init {
if (strides.linearSize != buffer.size) { require(strides.linearSize == buffer.size) {
error("Expected buffer side of ${strides.linearSize}, but found ${buffer.size}") "Expected buffer side of ${strides.linearSize}, but found ${buffer.size}"
} }
} }

View File

@ -93,13 +93,13 @@ inline fun BufferedNDField<Double, RealField>.produceInline(crossinline initiali
} }
/** /**
* Map one [RealNDElement] using function with indexes * Map one [RealNDElement] using function with indices.
*/ */
inline fun RealNDElement.mapIndexed(crossinline transform: RealField.(index: IntArray, Double) -> Double): RealNDElement = inline fun RealNDElement.mapIndexed(crossinline transform: RealField.(index: IntArray, Double) -> Double): RealNDElement =
context.produceInline { offset -> transform(strides.index(offset), buffer[offset]) } context.produceInline { offset -> transform(strides.index(offset), buffer[offset]) }
/** /**
* Map one [RealNDElement] using function without indexes * Map one [RealNDElement] using function without indices.
*/ */
inline fun RealNDElement.map(crossinline transform: RealField.(Double) -> Double): RealNDElement { inline fun RealNDElement.map(crossinline transform: RealField.(Double) -> Double): RealNDElement {
val array = DoubleArray(strides.linearSize) { offset -> RealField.transform(buffer[offset]) } val array = DoubleArray(strides.linearSize) { offset -> RealField.transform(buffer[offset]) }

View File

@ -1,77 +1,148 @@
package scientifik.memory package scientifik.memory
/**
* Represents a display of certain memory structure.
*/
interface Memory { interface Memory {
/**
* The length of this memory in bytes.
*/
val size: Int val size: Int
/** /**
* Get a projection of this memory (it reflects the changes in the parent memory block) * Get a projection of this memory (it reflects the changes in the parent memory block).
*/ */
fun view(offset: Int, length: Int): Memory fun view(offset: Int, length: Int): Memory
/** /**
* Create a copy of this memory, which does not know anything about this memory * Creates an independent copy of this memory.
*/ */
fun copy(): Memory fun copy(): Memory
/** /**
* Create and possibly register a new reader * Gets or creates a reader of this memory.
*/ */
fun reader(): MemoryReader fun reader(): MemoryReader
/**
* Gets or creates a writer of this memory.
*/
fun writer(): MemoryWriter fun writer(): MemoryWriter
companion object { companion object
}
} }
/**
* The interface to read primitive types in this memory.
*/
interface MemoryReader { interface MemoryReader {
/**
* The underlying memory.
*/
val memory: Memory val memory: Memory
/**
* Reads [Double] at certain [offset].
*/
fun readDouble(offset: Int): Double fun readDouble(offset: Int): Double
/**
* Reads [Float] at certain [offset].
*/
fun readFloat(offset: Int): Float fun readFloat(offset: Int): Float
/**
* Reads [Byte] at certain [offset].
*/
fun readByte(offset: Int): Byte fun readByte(offset: Int): Byte
/**
* Reads [Short] at certain [offset].
*/
fun readShort(offset: Int): Short fun readShort(offset: Int): Short
/**
* Reads [Int] at certain [offset].
*/
fun readInt(offset: Int): Int fun readInt(offset: Int): Int
/**
* Reads [Long] at certain [offset].
*/
fun readLong(offset: Int): Long fun readLong(offset: Int): Long
/**
* Disposes this reader if needed.
*/
fun release() fun release()
} }
/** /**
* Use the memory for read then release the reader * Uses the memory for read then releases the reader.
*/ */
inline fun Memory.read(block: MemoryReader.() -> Unit) { inline fun Memory.read(block: MemoryReader.() -> Unit) {
reader().apply(block).apply { release() } reader().apply(block).release()
} }
/**
* The interface to write primitive types into this memory.
*/
interface MemoryWriter { interface MemoryWriter {
/**
* The underlying memory.
*/
val memory: Memory val memory: Memory
/**
* Writes [Double] at certain [offset].
*/
fun writeDouble(offset: Int, value: Double) fun writeDouble(offset: Int, value: Double)
/**
* Writes [Float] at certain [offset].
*/
fun writeFloat(offset: Int, value: Float) fun writeFloat(offset: Int, value: Float)
/**
* Writes [Byte] at certain [offset].
*/
fun writeByte(offset: Int, value: Byte) fun writeByte(offset: Int, value: Byte)
/**
* Writes [Short] at certain [offset].
*/
fun writeShort(offset: Int, value: Short) fun writeShort(offset: Int, value: Short)
/**
* Writes [Int] at certain [offset].
*/
fun writeInt(offset: Int, value: Int) fun writeInt(offset: Int, value: Int)
/**
* Writes [Long] at certain [offset].
*/
fun writeLong(offset: Int, value: Long) fun writeLong(offset: Int, value: Long)
/**
* Disposes this writer if needed.
*/
fun release() fun release()
} }
/** /**
* Use the memory for write then release the writer * Uses the memory for write then releases the writer.
*/ */
inline fun Memory.write(block: MemoryWriter.() -> Unit) { inline fun Memory.write(block: MemoryWriter.() -> Unit) {
writer().apply(block).apply { release() } writer().apply(block).release()
} }
/** /**
* Allocate the most effective platform-specific memory * Allocates the most effective platform-specific memory.
*/ */
expect fun Memory.Companion.allocate(length: Int): Memory expect fun Memory.Companion.allocate(length: Int): Memory
/** /**
* Wrap a [Memory] around existing [ByteArray]. This operation is unsafe since the array is not copied * Wraps a [Memory] around existing [ByteArray]. This operation is unsafe since the array is not copied
* and could be mutated independently from the resulting [Memory] * and could be mutated independently from the resulting [Memory].
*/ */
expect fun Memory.Companion.wrap(array: ByteArray): Memory expect fun Memory.Companion.wrap(array: ByteArray): Memory

View File

@ -7,7 +7,7 @@ package scientifik.memory
*/ */
interface MemorySpec<T : Any> { interface MemorySpec<T : Any> {
/** /**
* Size of [T] in bytes after serialization * Size of [T] in bytes after serialization.
*/ */
val objectSize: Int val objectSize: Int
@ -24,9 +24,19 @@ interface MemorySpec<T : Any> {
fun MemoryWriter.write(offset: Int, value: T) fun MemoryWriter.write(offset: Int, value: T)
} }
/**
* Reads the object with [spec] starting from [offset].
*/
fun <T : Any> MemoryReader.read(spec: MemorySpec<T>, offset: Int): T = with(spec) { read(offset) } fun <T : Any> MemoryReader.read(spec: MemorySpec<T>, offset: Int): T = with(spec) { read(offset) }
/**
* Writes the object [value] with [spec] starting from [offset].
*/
fun <T : Any> MemoryWriter.write(spec: MemorySpec<T>, offset: Int, value: T): Unit = with(spec) { write(offset, value) } fun <T : Any> MemoryWriter.write(spec: MemorySpec<T>, offset: Int, value: T): Unit = with(spec) { write(offset, value) }
/**
* Reads array of [size] objects mapped by [spec] at certain [offset].
*/
inline fun <reified T : Any> MemoryReader.readArray(spec: MemorySpec<T>, offset: Int, size: Int): Array<T> = inline fun <reified T : Any> MemoryReader.readArray(spec: MemorySpec<T>, offset: Int, size: Int): Array<T> =
Array(size) { i -> Array(size) { i ->
spec.run { spec.run {
@ -34,6 +44,9 @@ inline fun <reified T : Any> MemoryReader.readArray(spec: MemorySpec<T>, offset:
} }
} }
/**
* Writes [array] of objects mapped by [spec] at certain [offset].
*/
fun <T : Any> MemoryWriter.writeArray(spec: MemorySpec<T>, offset: Int, array: Array<T>): Unit = fun <T : Any> MemoryWriter.writeArray(spec: MemorySpec<T>, offset: Int, array: Array<T>): Unit =
with(spec) { array.indices.forEach { i -> write(offset + i * objectSize, array[i]) } } with(spec) { array.indices.forEach { i -> write(offset + i * objectSize, array[i]) } }

View File

@ -4,13 +4,13 @@ import org.khronos.webgl.ArrayBuffer
import org.khronos.webgl.DataView import org.khronos.webgl.DataView
import org.khronos.webgl.Int8Array import org.khronos.webgl.Int8Array
class DataViewMemory(val view: DataView) : Memory { private class DataViewMemory(val view: DataView) : Memory {
override val size: Int get() = view.byteLength override val size: Int get() = view.byteLength
override fun view(offset: Int, length: Int): Memory { override fun view(offset: Int, length: Int): Memory {
require(offset >= 0) { "offset shouldn't be negative: $offset" } require(offset >= 0) { "offset shouldn't be negative: $offset" }
require(length >= 0) { "length shouldn't be negative: $length" } require(length >= 0) { "length shouldn't be negative: $length" }
require(offset + length <= size) { "Can't view memory outside the parent region." }
if (offset + length > size) if (offset + length > size)
throw IndexOutOfBoundsException("offset + length > size: $offset + $length > $size") throw IndexOutOfBoundsException("offset + length > size: $offset + $length > $size")
@ -33,11 +33,11 @@ class DataViewMemory(val view: DataView) : Memory {
override fun readInt(offset: Int): Int = view.getInt32(offset, false) override fun readInt(offset: Int): Int = view.getInt32(offset, false)
override fun readLong(offset: Int): Long = (view.getInt32(offset, false).toLong() shl 32) or override fun readLong(offset: Int): Long =
view.getInt32(offset + 4, false).toLong() view.getInt32(offset, false).toLong() shl 32 or view.getInt32(offset + 4, false).toLong()
override fun release() { override fun release() {
// does nothing on JS because of GC // does nothing on JS
} }
} }
@ -81,13 +81,17 @@ class DataViewMemory(val view: DataView) : Memory {
} }
/** /**
* Allocate the most effective platform-specific memory * Allocates memory based on a [DataView].
*/ */
actual fun Memory.Companion.allocate(length: Int): Memory { actual fun Memory.Companion.allocate(length: Int): Memory {
val buffer = ArrayBuffer(length) val buffer = ArrayBuffer(length)
return DataViewMemory(DataView(buffer, 0, length)) return DataViewMemory(DataView(buffer, 0, length))
} }
/**
* Wraps a [Memory] around existing [ByteArray]. This operation is unsafe since the array is not copied
* and could be mutated independently from the resulting [Memory].
*/
actual fun Memory.Companion.wrap(array: ByteArray): Memory { actual fun Memory.Companion.wrap(array: ByteArray): Memory {
@Suppress("CAST_NEVER_SUCCEEDS") val int8Array = array as Int8Array @Suppress("CAST_NEVER_SUCCEEDS") val int8Array = array as Int8Array
return DataViewMemory(DataView(int8Array.buffer, int8Array.byteOffset, int8Array.length)) return DataViewMemory(DataView(int8Array.buffer, int8Array.byteOffset, int8Array.length))

View File

@ -6,19 +6,18 @@ import java.nio.file.Files
import java.nio.file.Path import java.nio.file.Path
import java.nio.file.StandardOpenOption import java.nio.file.StandardOpenOption
private class ByteBufferMemory( private class ByteBufferMemory(
val buffer: ByteBuffer, val buffer: ByteBuffer,
val startOffset: Int = 0, val startOffset: Int = 0,
override val size: Int = buffer.limit() override val size: Int = buffer.limit()
) : Memory { ) : Memory {
@Suppress("NOTHING_TO_INLINE") @Suppress("NOTHING_TO_INLINE")
private inline fun position(o: Int): Int = startOffset + o private inline fun position(o: Int): Int = startOffset + o
override fun view(offset: Int, length: Int): Memory { override fun view(offset: Int, length: Int): Memory {
if (offset + length > size) error("Selecting a Memory view outside of memory range") require(offset >= 0) { "offset shouldn't be negative: $offset" }
require(length >= 0) { "length shouldn't be negative: $length" }
require(offset + length <= size) { "Can't view memory outside the parent region." }
return ByteBufferMemory(buffer, position(offset), length) return ByteBufferMemory(buffer, position(offset), length)
} }
@ -28,10 +27,9 @@ private class ByteBufferMemory(
copy.put(buffer) copy.put(buffer)
copy.flip() copy.flip()
return ByteBufferMemory(copy) return ByteBufferMemory(copy)
} }
private val reader = object : MemoryReader { private val reader: MemoryReader = object : MemoryReader {
override val memory: Memory get() = this@ByteBufferMemory override val memory: Memory get() = this@ByteBufferMemory
override fun readDouble(offset: Int) = buffer.getDouble(position(offset)) override fun readDouble(offset: Int) = buffer.getDouble(position(offset))
@ -53,7 +51,7 @@ private class ByteBufferMemory(
override fun reader(): MemoryReader = reader override fun reader(): MemoryReader = reader
private val writer = object : MemoryWriter { private val writer: MemoryWriter = object : MemoryWriter {
override val memory: Memory get() = this@ByteBufferMemory override val memory: Memory get() = this@ByteBufferMemory
override fun writeDouble(offset: Int, value: Double) { override fun writeDouble(offset: Int, value: Double) {
@ -89,26 +87,32 @@ private class ByteBufferMemory(
} }
/** /**
* Allocate the most effective platform-specific memory * Allocates memory based on a [ByteBuffer].
*/ */
actual fun Memory.Companion.allocate(length: Int): Memory { actual fun Memory.Companion.allocate(length: Int): Memory =
val buffer = ByteBuffer.allocate(length) ByteBufferMemory(checkNotNull(ByteBuffer.allocate(length)))
return ByteBufferMemory(buffer)
}
actual fun Memory.Companion.wrap(array: ByteArray): Memory { /**
val buffer = ByteBuffer.wrap(array) * Wraps a [Memory] around existing [ByteArray]. This operation is unsafe since the array is not copied
return ByteBufferMemory(buffer) * and could be mutated independently from the resulting [Memory].
} */
actual fun Memory.Companion.wrap(array: ByteArray): Memory = ByteBufferMemory(checkNotNull(ByteBuffer.wrap(array)))
/**
* Wraps this [ByteBuffer] to [Memory] object.
*
* @receiver the byte buffer.
* @param startOffset the start offset.
* @param size the size of memory to map.
* @return the [Memory] object.
*/
fun ByteBuffer.asMemory(startOffset: Int = 0, size: Int = limit()): Memory = fun ByteBuffer.asMemory(startOffset: Int = 0, size: Int = limit()): Memory =
ByteBufferMemory(this, startOffset, size) ByteBufferMemory(this, startOffset, size)
/** /**
* Use direct memory-mapped buffer from file to read something and close it afterwards. * Uses direct memory-mapped buffer from file to read something and close it afterwards.
*/ */
fun <R> Path.readAsMemory(position: Long = 0, size: Long = Files.size(this), block: Memory.() -> R): R { fun <R> Path.readAsMemory(position: Long = 0, size: Long = Files.size(this), block: Memory.() -> R): R =
return FileChannel.open(this, StandardOpenOption.READ).use { FileChannel.open(this, StandardOpenOption.READ).use {
ByteBufferMemory(it.map(FileChannel.MapMode.READ_ONLY, position, size)).block() ByteBufferMemory(it.map(FileChannel.MapMode.READ_ONLY, position, size)).block()
} }
}