Hidden channels in streaming blocks.
This commit is contained in:
parent
c0bdacecb3
commit
28695148e9
@ -1,16 +1,17 @@
|
|||||||
package scientifik.kmath.sequential
|
package scientifik.kmath.sequential
|
||||||
|
|
||||||
|
import kotlinx.atomicfu.atomic
|
||||||
|
import kotlinx.atomicfu.update
|
||||||
import kotlinx.coroutines.CoroutineScope
|
import kotlinx.coroutines.CoroutineScope
|
||||||
import kotlinx.coroutines.channels.ReceiveChannel
|
import kotlinx.coroutines.channels.*
|
||||||
import kotlinx.coroutines.channels.SendChannel
|
import kotlinx.coroutines.launch
|
||||||
import kotlinx.coroutines.channels.toChannel
|
|
||||||
|
|
||||||
interface DoubleProducer : Producer<Double> {
|
interface DoubleProducer : Producer<Double> {
|
||||||
val arrayOutput: ReceiveChannel<DoubleArray>
|
suspend fun receiveArray(): DoubleArray
|
||||||
}
|
}
|
||||||
|
|
||||||
interface DoubleConsumer : Consumer<Double> {
|
interface DoubleConsumer : Consumer<Double> {
|
||||||
val arrayInput: SendChannel<DoubleArray>
|
suspend fun sendArray(): DoubleArray
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -19,7 +20,7 @@ abstract class AbstractDoubleProducer(scope: CoroutineScope) : AbstractProducer<
|
|||||||
if (consumer is DoubleConsumer) {
|
if (consumer is DoubleConsumer) {
|
||||||
arrayOutput.toChannel(consumer.arrayInput)
|
arrayOutput.toChannel(consumer.arrayInput)
|
||||||
} else {
|
} else {
|
||||||
super.connectOutput(consumer)
|
connectOutput(super, consumer)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -41,7 +42,7 @@ abstract class AbstractDoubleProcessor(scope: CoroutineScope) : AbstractProcesso
|
|||||||
if (consumer is DoubleConsumer) {
|
if (consumer is DoubleConsumer) {
|
||||||
arrayOutput.toChannel(consumer.arrayInput)
|
arrayOutput.toChannel(consumer.arrayInput)
|
||||||
} else {
|
} else {
|
||||||
super.connectOutput(consumer)
|
connectOutput(super, consumer)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -53,3 +54,26 @@ abstract class AbstractDoubleProcessor(scope: CoroutineScope) : AbstractProcesso
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
class DoubleReducer<S>(
|
||||||
|
scope: CoroutineScope,
|
||||||
|
initialState: S,
|
||||||
|
fold: suspend (S, DoubleArray) -> S
|
||||||
|
) : AbstractDoubleConsumer(scope) {
|
||||||
|
private val state = atomic(initialState)
|
||||||
|
|
||||||
|
val value: S = state.value
|
||||||
|
|
||||||
|
override val arrayInput: SendChannel<DoubleArray> by lazy {
|
||||||
|
//create a channel and start process of reading all elements into aggregator
|
||||||
|
Channel<DoubleArray>(capacity = Channel.RENDEZVOUS).also {
|
||||||
|
launch {
|
||||||
|
it.consumeEach { value -> state.update { fold(it, value) } }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override val input: SendChannel<DoubleArray> = object :Abstr
|
||||||
|
|
||||||
|
|
||||||
|
}
|
@ -5,6 +5,7 @@ import kotlinx.atomicfu.update
|
|||||||
import kotlinx.coroutines.CoroutineScope
|
import kotlinx.coroutines.CoroutineScope
|
||||||
import kotlinx.coroutines.GlobalScope
|
import kotlinx.coroutines.GlobalScope
|
||||||
import kotlinx.coroutines.channels.*
|
import kotlinx.coroutines.channels.*
|
||||||
|
import kotlinx.coroutines.isActive
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
import kotlinx.coroutines.sync.Mutex
|
import kotlinx.coroutines.sync.Mutex
|
||||||
import kotlinx.coroutines.sync.withLock
|
import kotlinx.coroutines.sync.withLock
|
||||||
@ -20,24 +21,30 @@ import kotlin.coroutines.CoroutineContext
|
|||||||
* Manually putting elements to connected block could lead to undetermined behavior and must be avoided.
|
* Manually putting elements to connected block could lead to undetermined behavior and must be avoided.
|
||||||
*/
|
*/
|
||||||
interface Producer<T> : CoroutineScope {
|
interface Producer<T> : CoroutineScope {
|
||||||
val output: ReceiveChannel<T>
|
|
||||||
fun connect(consumer: Consumer<T>)
|
fun connect(consumer: Consumer<T>)
|
||||||
|
|
||||||
|
suspend fun receive(): T
|
||||||
|
|
||||||
val consumer: Consumer<T>?
|
val consumer: Consumer<T>?
|
||||||
|
|
||||||
val outputIsConnected: Boolean get() = consumer != null
|
val outputIsConnected: Boolean get() = consumer != null
|
||||||
|
|
||||||
|
//fun close()
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Terminal chain block. Could consume an element sequence and be connected to signle [Producer]
|
* Terminal chain block. Could consume an element sequence and be connected to signle [Producer]
|
||||||
*/
|
*/
|
||||||
interface Consumer<T> : CoroutineScope {
|
interface Consumer<T> : CoroutineScope {
|
||||||
val input: SendChannel<T>
|
|
||||||
fun connect(producer: Producer<T>)
|
fun connect(producer: Producer<T>)
|
||||||
|
|
||||||
|
suspend fun send(value: T)
|
||||||
|
|
||||||
val producer: Producer<T>?
|
val producer: Producer<T>?
|
||||||
|
|
||||||
val inputIsConnected: Boolean get() = producer != null
|
val inputIsConnected: Boolean get() = producer != null
|
||||||
|
|
||||||
|
//fun close()
|
||||||
}
|
}
|
||||||
|
|
||||||
interface Processor<T, R> : Consumer<T>, Producer<R>
|
interface Processor<T, R> : Consumer<T>, Producer<R>
|
||||||
@ -56,17 +63,19 @@ abstract class AbstractProducer<T>(scope: CoroutineScope) : Producer<T> {
|
|||||||
this.consumer = consumer
|
this.consumer = consumer
|
||||||
if (consumer.producer != null) {
|
if (consumer.producer != null) {
|
||||||
//No need to save the job, it will be canceled on scope cancel
|
//No need to save the job, it will be canceled on scope cancel
|
||||||
launch {
|
|
||||||
connectOutput(consumer)
|
connectOutput(consumer)
|
||||||
}
|
|
||||||
// connect back, consumer is already set so no circular reference
|
// connect back, consumer is already set so no circular reference
|
||||||
consumer.connect(this)
|
consumer.connect(this)
|
||||||
} else error("Unreachable statement")
|
} else error("Unreachable statement")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected open suspend fun connectOutput(consumer: Consumer<T>) {
|
protected open fun connectOutput(consumer: Consumer<T>) {
|
||||||
output.toChannel(consumer.input)
|
launch {
|
||||||
|
while (this.isActive) {
|
||||||
|
consumer.send(receive())
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -84,17 +93,19 @@ abstract class AbstractConsumer<T>(scope: CoroutineScope) : Consumer<T> {
|
|||||||
this.producer = producer
|
this.producer = producer
|
||||||
//No need to save the job, it will be canceled on scope cancel
|
//No need to save the job, it will be canceled on scope cancel
|
||||||
if (producer.consumer != null) {
|
if (producer.consumer != null) {
|
||||||
launch {
|
|
||||||
connectInput(producer)
|
connectInput(producer)
|
||||||
}
|
|
||||||
// connect back
|
// connect back
|
||||||
producer.connect(this)
|
producer.connect(this)
|
||||||
} else error("Unreachable statement")
|
} else error("Unreachable statement")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected open suspend fun connectInput(producer: Producer<T>) {
|
protected open fun connectInput(producer: Producer<T>) {
|
||||||
producer.output.toChannel(input)
|
launch {
|
||||||
|
while (isActive) {
|
||||||
|
send(producer.receive())
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -111,17 +122,19 @@ abstract class AbstractProcessor<T, R>(scope: CoroutineScope) : Processor<T, R>,
|
|||||||
this.producer = producer
|
this.producer = producer
|
||||||
//No need to save the job, it will be canceled on scope cancel
|
//No need to save the job, it will be canceled on scope cancel
|
||||||
if (producer.consumer != null) {
|
if (producer.consumer != null) {
|
||||||
launch {
|
|
||||||
connectInput(producer)
|
connectInput(producer)
|
||||||
}
|
|
||||||
// connect back
|
// connect back
|
||||||
producer.connect(this)
|
producer.connect(this)
|
||||||
} else error("Unreachable statement")
|
} else error("Unreachable statement")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected open suspend fun connectInput(producer: Producer<T>) {
|
protected open fun connectInput(producer: Producer<T>) {
|
||||||
producer.output.toChannel(input)
|
launch {
|
||||||
|
while (isActive) {
|
||||||
|
send(producer.receive())
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -133,8 +146,10 @@ class GenericProducer<T>(
|
|||||||
capacity: Int = Channel.UNLIMITED,
|
capacity: Int = Channel.UNLIMITED,
|
||||||
block: suspend ProducerScope<T>.() -> Unit
|
block: suspend ProducerScope<T>.() -> Unit
|
||||||
) : AbstractProducer<T>(scope) {
|
) : AbstractProducer<T>(scope) {
|
||||||
//The generation begins on first request to output
|
|
||||||
override val output: ReceiveChannel<T> by lazy { produce(capacity = capacity, block = block) }
|
private val channel: ReceiveChannel<T> by lazy { produce(capacity = capacity, block = block) }
|
||||||
|
|
||||||
|
override suspend fun receive(): T = channel.receive()
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -146,9 +161,14 @@ class PipeProcessor<T, R>(
|
|||||||
process: suspend (T) -> R
|
process: suspend (T) -> R
|
||||||
) : AbstractProcessor<T, R>(scope) {
|
) : AbstractProcessor<T, R>(scope) {
|
||||||
|
|
||||||
private val _input = Channel<T>(capacity)
|
private val input = Channel<T>(capacity)
|
||||||
override val input: SendChannel<T> get() = _input
|
private val output: ReceiveChannel<R> = input.map(coroutineContext, process)
|
||||||
override val output: ReceiveChannel<R> = _input.map(coroutineContext, process)
|
|
||||||
|
override suspend fun receive(): R = output.receive()
|
||||||
|
|
||||||
|
override suspend fun send(value: T) {
|
||||||
|
input.send(value)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -160,41 +180,45 @@ class ChunkProcessor<T, R>(
|
|||||||
process: suspend (List<T>) -> R
|
process: suspend (List<T>) -> R
|
||||||
) : AbstractProcessor<T, R>(scope) {
|
) : AbstractProcessor<T, R>(scope) {
|
||||||
|
|
||||||
private val _input = Channel<T>(chunkSize)
|
private val input = Channel<T>(chunkSize)
|
||||||
|
|
||||||
override val input: SendChannel<T> get() = _input
|
|
||||||
|
|
||||||
private val chunked = produce<List<T>>(coroutineContext) {
|
private val chunked = produce<List<T>>(coroutineContext) {
|
||||||
val list = ArrayList<T>(chunkSize)
|
val list = ArrayList<T>(chunkSize)
|
||||||
repeat(chunkSize) {
|
repeat(chunkSize) {
|
||||||
list.add(_input.receive())
|
list.add(input.receive())
|
||||||
}
|
}
|
||||||
send(list)
|
send(list)
|
||||||
}
|
}
|
||||||
|
|
||||||
override val output: ReceiveChannel<R> = chunked.map(coroutineContext, process)
|
private val output: ReceiveChannel<R> = chunked.map(coroutineContext, process)
|
||||||
|
|
||||||
|
override suspend fun receive(): R = output.receive()
|
||||||
|
|
||||||
|
override suspend fun send(value: T) {
|
||||||
|
input.send(value)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A moving window [Processor]
|
* A moving window [Processor] with circular buffer
|
||||||
*/
|
*/
|
||||||
class WindowProcessor<T, R>(
|
class WindowedProcessor<T, R>(
|
||||||
scope: CoroutineScope,
|
scope: CoroutineScope,
|
||||||
window: Int,
|
window: Int,
|
||||||
process: suspend (List<T>) -> R
|
process: suspend (List<T>) -> R
|
||||||
) : AbstractProcessor<T, R>(scope) {
|
) : AbstractProcessor<T, R>(scope) {
|
||||||
|
|
||||||
|
override suspend fun receive(): R {
|
||||||
|
TODO("not implemented") //To change body of created functions use File | Settings | File Templates.
|
||||||
|
}
|
||||||
|
|
||||||
override val output: ReceiveChannel<R>
|
override suspend fun send(value: T) {
|
||||||
get() = TODO("not implemented") //To change initializer of created properties use File | Settings | File Templates.
|
TODO("not implemented") //To change body of created functions use File | Settings | File Templates.
|
||||||
override val input: SendChannel<T>
|
}
|
||||||
get() = TODO("not implemented") //To change initializer of created properties use File | Settings | File Templates.
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//TODO add circular buffer processor
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Thread-safe aggregator of values from input. The aggregator does not store all incoming values, it uses fold procedure
|
* Thread-safe aggregator of values from input. The aggregator does not store all incoming values, it uses fold procedure
|
||||||
* to incorporate them into state on-arrival.
|
* to incorporate them into state on-arrival.
|
||||||
@ -212,7 +236,7 @@ class Reducer<T, S>(
|
|||||||
|
|
||||||
val value: S = state.value
|
val value: S = state.value
|
||||||
|
|
||||||
override val input: SendChannel<T> by lazy {
|
private val input: SendChannel<T> by lazy {
|
||||||
//create a channel and start process of reading all elements into aggregator
|
//create a channel and start process of reading all elements into aggregator
|
||||||
Channel<T>(capacity = Channel.RENDEZVOUS).also {
|
Channel<T>(capacity = Channel.RENDEZVOUS).also {
|
||||||
launch {
|
launch {
|
||||||
@ -220,6 +244,8 @@ class Reducer<T, S>(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
override suspend fun send(value: T) = input.send(value)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -231,7 +257,7 @@ class Collector<T>(scope: CoroutineScope) : AbstractConsumer<T>(scope) {
|
|||||||
private val mutex = Mutex()
|
private val mutex = Mutex()
|
||||||
val list: List<T> get() = _list
|
val list: List<T> get() = _list
|
||||||
|
|
||||||
override val input: SendChannel<T> by lazy {
|
private val input: SendChannel<T> by lazy {
|
||||||
//create a channel and start process of reading all elements into aggregator
|
//create a channel and start process of reading all elements into aggregator
|
||||||
Channel<T>(capacity = Channel.RENDEZVOUS).also {
|
Channel<T>(capacity = Channel.RENDEZVOUS).also {
|
||||||
launch {
|
launch {
|
||||||
@ -243,6 +269,8 @@ class Collector<T>(scope: CoroutineScope) : AbstractConsumer<T>(scope) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
override suspend fun send(value: T) = input.send(value)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
Loading…
Reference in New Issue
Block a user