Document memory module and several ND objects

This commit is contained in:
Iaroslav 2020-08-08 04:21:59 +07:00
parent 0a8044ddb3
commit c8cd6cd288
No known key found for this signature in database
GPG Key ID: 46E15E4A31B3BCD7
12 changed files with 217 additions and 67 deletions

View File

@ -3,7 +3,6 @@ package scientifik.kmath.structures
import scientifik.kmath.operations.Field
import scientifik.kmath.operations.FieldElement
class BoxingNDField<T, F : Field<T>>(
override val shape: IntArray,
override val elementContext: F,

View File

@ -11,7 +11,8 @@ interface BufferedNDAlgebra<T, C> : NDAlgebra<T, C, NDBuffer<T>> {
/**
* Convert any [NDStructure] to buffered structure using strides from this context.
* If the structure is already [NDBuffer], conversion is free. If not, it could be expensive because iteration over indexes
* If the structure is already [NDBuffer], conversion is free. If not, it could be expensive because iteration over
* indices.
*
* If the argument is [NDBuffer] with different strides structure, the new element will be produced.
*/

View File

@ -98,13 +98,13 @@ inline fun BufferedNDField<Complex, ComplexField>.produceInline(crossinline init
}
/**
* Map one [ComplexNDElement] using function with indexes
* Map one [ComplexNDElement] using function with indices.
*/
inline fun ComplexNDElement.mapIndexed(crossinline transform: ComplexField.(index: IntArray, Complex) -> Complex): ComplexNDElement =
context.produceInline { offset -> transform(strides.index(offset), buffer[offset]) }
/**
* Map one [ComplexNDElement] using function without indexes
* Map one [ComplexNDElement] using function without indices.
*/
inline fun ComplexNDElement.map(crossinline transform: ComplexField.(Complex) -> Complex): ComplexNDElement {
val buffer = Buffer.complex(strides.linearSize) { offset -> ComplexField.transform(buffer[offset]) }

View File

@ -2,6 +2,13 @@ package scientifik.kmath.structures
import scientifik.kmath.operations.ExtendedField
/**
* [ExtendedField] over [NDStructure].
*
* @param T the type of the element contained in ND structure.
* @param N the type of ND structure.
* @param F the extended field of structure elements.
*/
interface ExtendedNDField<T : Any, F : ExtendedField<T>, N : NDStructure<T>> : NDField<T, F, N>, ExtendedField<N>
///**

View File

@ -10,7 +10,6 @@ import scientifik.memory.*
* @property spec the spec of [T] type.
*/
open class MemoryBuffer<T : Any>(protected val memory: Memory, protected val spec: MemorySpec<T>) : Buffer<T> {
override val size: Int get() = memory.size / spec.objectSize
private val reader: MemoryReader = memory.reader()

View File

@ -105,10 +105,11 @@ interface NDRing<T, R : Ring<T>, N : NDStructure<T>> : Ring<N>, NDSpace<T, R, N>
}
/**
* Field for n-dimensional structures.
* Field of [NDStructure].
*
* @param T the type of the element contained in ND structure
* @param F field of structure elements
* @param T the type of the element contained in ND structure.
* @param N the type of ND structure.
* @param F field of structure elements.
*/
interface NDField<T, F : Field<T>, N : NDStructure<T>> : Field<N>, NDRing<T, F, N> {

View File

@ -3,15 +3,38 @@ package scientifik.kmath.structures
import kotlin.jvm.JvmName
import kotlin.reflect.KClass
/**
* Represents n-dimensional structure, i.e. multidimensional container of items of the same type and size. The number
* of dimensions and items in an array is defined by its shape, which is a sequence of non-negative integers that
* specify the sizes of each dimension.
*
* @param T the type of items.
*/
interface NDStructure<T> {
/**
* The shape of structure, i.e. non-empty sequence of non-negative integers that specify sizes of dimensions of
* this structure.
*/
val shape: IntArray
/**
* The count of dimensions in this structure. It should be equal to size of [shape].
*/
val dimension: Int get() = shape.size
/**
* Returns the value at the specified indices.
*
* @param index the indices.
* @return the value.
*/
operator fun get(index: IntArray): T
/**
* Returns the sequence of all the elements associated by their indices.
*
* @return the lazy sequence of pairs of indices to values.
*/
fun elements(): Sequence<Pair<IntArray, T>>
override fun equals(other: Any?): Boolean
@ -19,6 +42,9 @@ interface NDStructure<T> {
override fun hashCode(): Int
companion object {
/**
* Indicates whether some [NDStructure] is equal to another one.
*/
fun equals(st1: NDStructure<*>, st2: NDStructure<*>): Boolean {
if (st1 === st2) return true
@ -36,9 +62,9 @@ interface NDStructure<T> {
}
/**
* Create a NDStructure with explicit buffer factory
* Creates a NDStructure with explicit buffer factory.
*
* Strides should be reused if possible
* Strides should be reused if possible.
*/
fun <T> build(
strides: Strides,
@ -91,9 +117,24 @@ interface NDStructure<T> {
}
}
/**
* Returns the value at the specified indices.
*
* @param index the indices.
* @return the value.
*/
operator fun <T> NDStructure<T>.get(vararg index: Int): T = get(index)
/**
* Represents mutable [NDStructure].
*/
interface MutableNDStructure<T> : NDStructure<T> {
/**
* Inserts an item at the specified indices.
*
* @param index the indices.
* @param value the value.
*/
operator fun set(index: IntArray, value: T)
}
@ -104,7 +145,7 @@ inline fun <T> MutableNDStructure<T>.mapInPlace(action: (IntArray, T) -> T) {
}
/**
* A way to convert ND index to linear one and back
* A way to convert ND index to linear one and back.
*/
interface Strides {
/**
@ -141,6 +182,9 @@ interface Strides {
}
}
/**
* Simple implementation of [Strides].
*/
class DefaultStrides private constructor(override val shape: IntArray) : Strides {
/**
* Strides for memory access
@ -180,19 +224,14 @@ class DefaultStrides private constructor(override val shape: IntArray) : Strides
override val linearSize: Int
get() = strides[shape.size]
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other !is DefaultStrides) return false
if (!shape.contentEquals(other.shape)) return false
return true
}
override fun hashCode(): Int {
return shape.contentHashCode()
}
override fun hashCode(): Int = shape.contentHashCode()
companion object {
private val defaultStridesCache = HashMap<IntArray, Strides>()
@ -204,8 +243,20 @@ class DefaultStrides private constructor(override val shape: IntArray) : Strides
}
}
/**
* Represents [NDStructure] over [Buffer].
*
* @param T the type of items.
*/
abstract class NDBuffer<T> : NDStructure<T> {
/**
* The underlying buffer.
*/
abstract val buffer: Buffer<T>
/**
* The strides to access elements of [Buffer] by linear indices.
*/
abstract val strides: Strides
override fun get(index: IntArray): T = buffer[strides.offset(index)]
@ -263,8 +314,8 @@ class MutableBufferNDStructure<T>(
) : NDBuffer<T>(), MutableNDStructure<T> {
init {
if (strides.linearSize != buffer.size) {
error("Expected buffer side of ${strides.linearSize}, but found ${buffer.size}")
require(strides.linearSize == buffer.size) {
"Expected buffer side of ${strides.linearSize}, but found ${buffer.size}"
}
}

View File

@ -93,13 +93,13 @@ inline fun BufferedNDField<Double, RealField>.produceInline(crossinline initiali
}
/**
* Map one [RealNDElement] using function with indexes
* Map one [RealNDElement] using function with indices.
*/
inline fun RealNDElement.mapIndexed(crossinline transform: RealField.(index: IntArray, Double) -> Double): RealNDElement =
context.produceInline { offset -> transform(strides.index(offset), buffer[offset]) }
/**
* Map one [RealNDElement] using function without indexes
* Map one [RealNDElement] using function without indices.
*/
inline fun RealNDElement.map(crossinline transform: RealField.(Double) -> Double): RealNDElement {
val array = DoubleArray(strides.linearSize) { offset -> RealField.transform(buffer[offset]) }

View File

@ -1,77 +1,148 @@
package scientifik.memory
/**
* Represents a display of certain memory structure.
*/
interface Memory {
/**
* The length of this memory in bytes.
*/
val size: Int
/**
* Get a projection of this memory (it reflects the changes in the parent memory block)
* Get a projection of this memory (it reflects the changes in the parent memory block).
*/
fun view(offset: Int, length: Int): Memory
/**
* Create a copy of this memory, which does not know anything about this memory
* Creates an independent copy of this memory.
*/
fun copy(): Memory
/**
* Create and possibly register a new reader
* Gets or creates a reader of this memory.
*/
fun reader(): MemoryReader
/**
* Gets or creates a writer of this memory.
*/
fun writer(): MemoryWriter
companion object {
}
companion object
}
/**
* The interface to read primitive types in this memory.
*/
interface MemoryReader {
/**
* The underlying memory.
*/
val memory: Memory
/**
* Reads [Double] at certain [offset].
*/
fun readDouble(offset: Int): Double
/**
* Reads [Float] at certain [offset].
*/
fun readFloat(offset: Int): Float
/**
* Reads [Byte] at certain [offset].
*/
fun readByte(offset: Int): Byte
/**
* Reads [Short] at certain [offset].
*/
fun readShort(offset: Int): Short
/**
* Reads [Int] at certain [offset].
*/
fun readInt(offset: Int): Int
/**
* Reads [Long] at certain [offset].
*/
fun readLong(offset: Int): Long
/**
* Disposes this reader if needed.
*/
fun release()
}
/**
* Use the memory for read then release the reader
* Uses the memory for read then releases the reader.
*/
inline fun Memory.read(block: MemoryReader.() -> Unit) {
reader().apply(block).apply { release() }
reader().apply(block).release()
}
/**
* The interface to write primitive types into this memory.
*/
interface MemoryWriter {
/**
* The underlying memory.
*/
val memory: Memory
/**
* Writes [Double] at certain [offset].
*/
fun writeDouble(offset: Int, value: Double)
/**
* Writes [Float] at certain [offset].
*/
fun writeFloat(offset: Int, value: Float)
/**
* Writes [Byte] at certain [offset].
*/
fun writeByte(offset: Int, value: Byte)
/**
* Writes [Short] at certain [offset].
*/
fun writeShort(offset: Int, value: Short)
/**
* Writes [Int] at certain [offset].
*/
fun writeInt(offset: Int, value: Int)
/**
* Writes [Long] at certain [offset].
*/
fun writeLong(offset: Int, value: Long)
/**
* Disposes this writer if needed.
*/
fun release()
}
/**
* Use the memory for write then release the writer
* Uses the memory for write then releases the writer.
*/
inline fun Memory.write(block: MemoryWriter.() -> Unit) {
writer().apply(block).apply { release() }
writer().apply(block).release()
}
/**
* Allocate the most effective platform-specific memory
* Allocates the most effective platform-specific memory.
*/
expect fun Memory.Companion.allocate(length: Int): Memory
/**
* Wrap a [Memory] around existing [ByteArray]. This operation is unsafe since the array is not copied
* and could be mutated independently from the resulting [Memory]
* Wraps a [Memory] around existing [ByteArray]. This operation is unsafe since the array is not copied
* and could be mutated independently from the resulting [Memory].
*/
expect fun Memory.Companion.wrap(array: ByteArray): Memory

View File

@ -7,7 +7,7 @@ package scientifik.memory
*/
interface MemorySpec<T : Any> {
/**
* Size of [T] in bytes after serialization
* Size of [T] in bytes after serialization.
*/
val objectSize: Int
@ -24,9 +24,19 @@ interface MemorySpec<T : Any> {
fun MemoryWriter.write(offset: Int, value: T)
}
/**
* Reads the object with [spec] starting from [offset].
*/
fun <T : Any> MemoryReader.read(spec: MemorySpec<T>, offset: Int): T = with(spec) { read(offset) }
/**
* Writes the object [value] with [spec] starting from [offset].
*/
fun <T : Any> MemoryWriter.write(spec: MemorySpec<T>, offset: Int, value: T): Unit = with(spec) { write(offset, value) }
/**
* Reads array of [size] objects mapped by [spec] at certain [offset].
*/
inline fun <reified T : Any> MemoryReader.readArray(spec: MemorySpec<T>, offset: Int, size: Int): Array<T> =
Array(size) { i ->
spec.run {
@ -34,7 +44,10 @@ inline fun <reified T : Any> MemoryReader.readArray(spec: MemorySpec<T>, offset:
}
}
/**
* Writes [array] of objects mapped by [spec] at certain [offset].
*/
fun <T : Any> MemoryWriter.writeArray(spec: MemorySpec<T>, offset: Int, array: Array<T>): Unit =
with(spec) { array.indices.forEach { i -> write(offset + i * objectSize, array[i]) } }
//TODO It is possible to add elastic MemorySpec with unknown object size
// TODO It is possible to add elastic MemorySpec with unknown object size

View File

@ -4,13 +4,13 @@ import org.khronos.webgl.ArrayBuffer
import org.khronos.webgl.DataView
import org.khronos.webgl.Int8Array
class DataViewMemory(val view: DataView) : Memory {
private class DataViewMemory(val view: DataView) : Memory {
override val size: Int get() = view.byteLength
override fun view(offset: Int, length: Int): Memory {
require(offset >= 0) { "offset shouldn't be negative: $offset" }
require(length >= 0) { "length shouldn't be negative: $length" }
require(offset + length <= size) { "Can't view memory outside the parent region." }
if (offset + length > size)
throw IndexOutOfBoundsException("offset + length > size: $offset + $length > $size")
@ -33,11 +33,11 @@ class DataViewMemory(val view: DataView) : Memory {
override fun readInt(offset: Int): Int = view.getInt32(offset, false)
override fun readLong(offset: Int): Long = (view.getInt32(offset, false).toLong() shl 32) or
view.getInt32(offset + 4, false).toLong()
override fun readLong(offset: Int): Long =
view.getInt32(offset, false).toLong() shl 32 or view.getInt32(offset + 4, false).toLong()
override fun release() {
// does nothing on JS because of GC
// does nothing on JS
}
}
@ -72,7 +72,7 @@ class DataViewMemory(val view: DataView) : Memory {
}
override fun release() {
//does nothing on JS
// does nothing on JS
}
}
@ -81,13 +81,17 @@ class DataViewMemory(val view: DataView) : Memory {
}
/**
* Allocate the most effective platform-specific memory
* Allocates memory based on a [DataView].
*/
actual fun Memory.Companion.allocate(length: Int): Memory {
val buffer = ArrayBuffer(length)
return DataViewMemory(DataView(buffer, 0, length))
}
/**
* Wraps a [Memory] around existing [ByteArray]. This operation is unsafe since the array is not copied
* and could be mutated independently from the resulting [Memory].
*/
actual fun Memory.Companion.wrap(array: ByteArray): Memory {
@Suppress("CAST_NEVER_SUCCEEDS") val int8Array = array as Int8Array
return DataViewMemory(DataView(int8Array.buffer, int8Array.byteOffset, int8Array.length))

View File

@ -6,19 +6,18 @@ import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardOpenOption
private class ByteBufferMemory(
val buffer: ByteBuffer,
val startOffset: Int = 0,
override val size: Int = buffer.limit()
) : Memory {
@Suppress("NOTHING_TO_INLINE")
private inline fun position(o: Int): Int = startOffset + o
override fun view(offset: Int, length: Int): Memory {
if (offset + length > size) error("Selecting a Memory view outside of memory range")
require(offset >= 0) { "offset shouldn't be negative: $offset" }
require(length >= 0) { "length shouldn't be negative: $length" }
require(offset + length <= size) { "Can't view memory outside the parent region." }
return ByteBufferMemory(buffer, position(offset), length)
}
@ -28,10 +27,9 @@ private class ByteBufferMemory(
copy.put(buffer)
copy.flip()
return ByteBufferMemory(copy)
}
private val reader = object : MemoryReader {
private val reader: MemoryReader = object : MemoryReader {
override val memory: Memory get() = this@ByteBufferMemory
override fun readDouble(offset: Int) = buffer.getDouble(position(offset))
@ -47,13 +45,13 @@ private class ByteBufferMemory(
override fun readLong(offset: Int) = buffer.getLong(position(offset))
override fun release() {
//does nothing on JVM
// does nothing on JVM
}
}
override fun reader(): MemoryReader = reader
private val writer = object : MemoryWriter {
private val writer: MemoryWriter = object : MemoryWriter {
override val memory: Memory get() = this@ByteBufferMemory
override fun writeDouble(offset: Int, value: Double) {
@ -81,7 +79,7 @@ private class ByteBufferMemory(
}
override fun release() {
//does nothing on JVM
// does nothing on JVM
}
}
@ -89,26 +87,32 @@ private class ByteBufferMemory(
}
/**
* Allocate the most effective platform-specific memory
* Allocates memory based on a [ByteBuffer].
*/
actual fun Memory.Companion.allocate(length: Int): Memory {
val buffer = ByteBuffer.allocate(length)
return ByteBufferMemory(buffer)
}
actual fun Memory.Companion.allocate(length: Int): Memory =
ByteBufferMemory(checkNotNull(ByteBuffer.allocate(length)))
actual fun Memory.Companion.wrap(array: ByteArray): Memory {
val buffer = ByteBuffer.wrap(array)
return ByteBufferMemory(buffer)
}
/**
* Wraps a [Memory] around existing [ByteArray]. This operation is unsafe since the array is not copied
* and could be mutated independently from the resulting [Memory].
*/
actual fun Memory.Companion.wrap(array: ByteArray): Memory = ByteBufferMemory(checkNotNull(ByteBuffer.wrap(array)))
/**
* Wraps this [ByteBuffer] to [Memory] object.
*
* @receiver the byte buffer.
* @param startOffset the start offset.
* @param size the size of memory to map.
* @return the [Memory] object.
*/
fun ByteBuffer.asMemory(startOffset: Int = 0, size: Int = limit()): Memory =
ByteBufferMemory(this, startOffset, size)
/**
* Use direct memory-mapped buffer from file to read something and close it afterwards.
* Uses direct memory-mapped buffer from file to read something and close it afterwards.
*/
fun <R> Path.readAsMemory(position: Long = 0, size: Long = Files.size(this), block: Memory.() -> R): R {
return FileChannel.open(this, StandardOpenOption.READ).use {
fun <R> Path.readAsMemory(position: Long = 0, size: Long = Files.size(this), block: Memory.() -> R): R =
FileChannel.open(this, StandardOpenOption.READ).use {
ByteBufferMemory(it.map(FileChannel.MapMode.READ_ONLY, position, size)).block()
}
}