Hidden channels in streaming blocks.

This commit is contained in:
Alexander Nozik 2019-02-06 22:24:03 +03:00
parent c0bdacecb3
commit 28695148e9
2 changed files with 96 additions and 44 deletions

View File

@ -1,16 +1,17 @@
package scientifik.kmath.sequential package scientifik.kmath.sequential
import kotlinx.atomicfu.atomic
import kotlinx.atomicfu.update
import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.channels.ReceiveChannel import kotlinx.coroutines.channels.*
import kotlinx.coroutines.channels.SendChannel import kotlinx.coroutines.launch
import kotlinx.coroutines.channels.toChannel
interface DoubleProducer : Producer<Double> { interface DoubleProducer : Producer<Double> {
val arrayOutput: ReceiveChannel<DoubleArray> suspend fun receiveArray(): DoubleArray
} }
interface DoubleConsumer : Consumer<Double> { interface DoubleConsumer : Consumer<Double> {
val arrayInput: SendChannel<DoubleArray> suspend fun sendArray(): DoubleArray
} }
@ -19,7 +20,7 @@ abstract class AbstractDoubleProducer(scope: CoroutineScope) : AbstractProducer<
if (consumer is DoubleConsumer) { if (consumer is DoubleConsumer) {
arrayOutput.toChannel(consumer.arrayInput) arrayOutput.toChannel(consumer.arrayInput)
} else { } else {
super.connectOutput(consumer) connectOutput(super, consumer)
} }
} }
} }
@ -41,7 +42,7 @@ abstract class AbstractDoubleProcessor(scope: CoroutineScope) : AbstractProcesso
if (consumer is DoubleConsumer) { if (consumer is DoubleConsumer) {
arrayOutput.toChannel(consumer.arrayInput) arrayOutput.toChannel(consumer.arrayInput)
} else { } else {
super.connectOutput(consumer) connectOutput(super, consumer)
} }
} }
@ -53,3 +54,26 @@ abstract class AbstractDoubleProcessor(scope: CoroutineScope) : AbstractProcesso
} }
} }
} }
class DoubleReducer<S>(
scope: CoroutineScope,
initialState: S,
fold: suspend (S, DoubleArray) -> S
) : AbstractDoubleConsumer(scope) {
private val state = atomic(initialState)
val value: S = state.value
override val arrayInput: SendChannel<DoubleArray> by lazy {
//create a channel and start process of reading all elements into aggregator
Channel<DoubleArray>(capacity = Channel.RENDEZVOUS).also {
launch {
it.consumeEach { value -> state.update { fold(it, value) } }
}
}
}
override val input: SendChannel<DoubleArray> = object :Abstr
}

View File

@ -5,6 +5,7 @@ import kotlinx.atomicfu.update
import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.channels.* import kotlinx.coroutines.channels.*
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.sync.withLock
@ -20,24 +21,30 @@ import kotlin.coroutines.CoroutineContext
* Manually putting elements to connected block could lead to undetermined behavior and must be avoided. * Manually putting elements to connected block could lead to undetermined behavior and must be avoided.
*/ */
interface Producer<T> : CoroutineScope { interface Producer<T> : CoroutineScope {
val output: ReceiveChannel<T>
fun connect(consumer: Consumer<T>) fun connect(consumer: Consumer<T>)
suspend fun receive(): T
val consumer: Consumer<T>? val consumer: Consumer<T>?
val outputIsConnected: Boolean get() = consumer != null val outputIsConnected: Boolean get() = consumer != null
//fun close()
} }
/** /**
* Terminal chain block. Could consume an element sequence and be connected to signle [Producer] * Terminal chain block. Could consume an element sequence and be connected to signle [Producer]
*/ */
interface Consumer<T> : CoroutineScope { interface Consumer<T> : CoroutineScope {
val input: SendChannel<T>
fun connect(producer: Producer<T>) fun connect(producer: Producer<T>)
suspend fun send(value: T)
val producer: Producer<T>? val producer: Producer<T>?
val inputIsConnected: Boolean get() = producer != null val inputIsConnected: Boolean get() = producer != null
//fun close()
} }
interface Processor<T, R> : Consumer<T>, Producer<R> interface Processor<T, R> : Consumer<T>, Producer<R>
@ -56,17 +63,19 @@ abstract class AbstractProducer<T>(scope: CoroutineScope) : Producer<T> {
this.consumer = consumer this.consumer = consumer
if (consumer.producer != null) { if (consumer.producer != null) {
//No need to save the job, it will be canceled on scope cancel //No need to save the job, it will be canceled on scope cancel
launch {
connectOutput(consumer) connectOutput(consumer)
}
// connect back, consumer is already set so no circular reference // connect back, consumer is already set so no circular reference
consumer.connect(this) consumer.connect(this)
} else error("Unreachable statement") } else error("Unreachable statement")
} }
} }
protected open suspend fun connectOutput(consumer: Consumer<T>) { protected open fun connectOutput(consumer: Consumer<T>) {
output.toChannel(consumer.input) launch {
while (this.isActive) {
consumer.send(receive())
}
}
} }
} }
@ -84,17 +93,19 @@ abstract class AbstractConsumer<T>(scope: CoroutineScope) : Consumer<T> {
this.producer = producer this.producer = producer
//No need to save the job, it will be canceled on scope cancel //No need to save the job, it will be canceled on scope cancel
if (producer.consumer != null) { if (producer.consumer != null) {
launch {
connectInput(producer) connectInput(producer)
}
// connect back // connect back
producer.connect(this) producer.connect(this)
} else error("Unreachable statement") } else error("Unreachable statement")
} }
} }
protected open suspend fun connectInput(producer: Producer<T>) { protected open fun connectInput(producer: Producer<T>) {
producer.output.toChannel(input) launch {
while (isActive) {
send(producer.receive())
}
}
} }
} }
@ -111,17 +122,19 @@ abstract class AbstractProcessor<T, R>(scope: CoroutineScope) : Processor<T, R>,
this.producer = producer this.producer = producer
//No need to save the job, it will be canceled on scope cancel //No need to save the job, it will be canceled on scope cancel
if (producer.consumer != null) { if (producer.consumer != null) {
launch {
connectInput(producer) connectInput(producer)
}
// connect back // connect back
producer.connect(this) producer.connect(this)
} else error("Unreachable statement") } else error("Unreachable statement")
} }
} }
protected open suspend fun connectInput(producer: Producer<T>) { protected open fun connectInput(producer: Producer<T>) {
producer.output.toChannel(input) launch {
while (isActive) {
send(producer.receive())
}
}
} }
} }
@ -133,8 +146,10 @@ class GenericProducer<T>(
capacity: Int = Channel.UNLIMITED, capacity: Int = Channel.UNLIMITED,
block: suspend ProducerScope<T>.() -> Unit block: suspend ProducerScope<T>.() -> Unit
) : AbstractProducer<T>(scope) { ) : AbstractProducer<T>(scope) {
//The generation begins on first request to output
override val output: ReceiveChannel<T> by lazy { produce(capacity = capacity, block = block) } private val channel: ReceiveChannel<T> by lazy { produce(capacity = capacity, block = block) }
override suspend fun receive(): T = channel.receive()
} }
/** /**
@ -146,9 +161,14 @@ class PipeProcessor<T, R>(
process: suspend (T) -> R process: suspend (T) -> R
) : AbstractProcessor<T, R>(scope) { ) : AbstractProcessor<T, R>(scope) {
private val _input = Channel<T>(capacity) private val input = Channel<T>(capacity)
override val input: SendChannel<T> get() = _input private val output: ReceiveChannel<R> = input.map(coroutineContext, process)
override val output: ReceiveChannel<R> = _input.map(coroutineContext, process)
override suspend fun receive(): R = output.receive()
override suspend fun send(value: T) {
input.send(value)
}
} }
/** /**
@ -160,41 +180,45 @@ class ChunkProcessor<T, R>(
process: suspend (List<T>) -> R process: suspend (List<T>) -> R
) : AbstractProcessor<T, R>(scope) { ) : AbstractProcessor<T, R>(scope) {
private val _input = Channel<T>(chunkSize) private val input = Channel<T>(chunkSize)
override val input: SendChannel<T> get() = _input
private val chunked = produce<List<T>>(coroutineContext) { private val chunked = produce<List<T>>(coroutineContext) {
val list = ArrayList<T>(chunkSize) val list = ArrayList<T>(chunkSize)
repeat(chunkSize) { repeat(chunkSize) {
list.add(_input.receive()) list.add(input.receive())
} }
send(list) send(list)
} }
override val output: ReceiveChannel<R> = chunked.map(coroutineContext, process) private val output: ReceiveChannel<R> = chunked.map(coroutineContext, process)
override suspend fun receive(): R = output.receive()
override suspend fun send(value: T) {
input.send(value)
}
} }
/** /**
* A moving window [Processor] * A moving window [Processor] with circular buffer
*/ */
class WindowProcessor<T, R>( class WindowedProcessor<T, R>(
scope: CoroutineScope, scope: CoroutineScope,
window: Int, window: Int,
process: suspend (List<T>) -> R process: suspend (List<T>) -> R
) : AbstractProcessor<T, R>(scope) { ) : AbstractProcessor<T, R>(scope) {
override suspend fun receive(): R {
TODO("not implemented") //To change body of created functions use File | Settings | File Templates.
}
override val output: ReceiveChannel<R> override suspend fun send(value: T) {
get() = TODO("not implemented") //To change initializer of created properties use File | Settings | File Templates. TODO("not implemented") //To change body of created functions use File | Settings | File Templates.
override val input: SendChannel<T> }
get() = TODO("not implemented") //To change initializer of created properties use File | Settings | File Templates.
} }
//TODO add circular buffer processor
/** /**
* Thread-safe aggregator of values from input. The aggregator does not store all incoming values, it uses fold procedure * Thread-safe aggregator of values from input. The aggregator does not store all incoming values, it uses fold procedure
* to incorporate them into state on-arrival. * to incorporate them into state on-arrival.
@ -212,7 +236,7 @@ class Reducer<T, S>(
val value: S = state.value val value: S = state.value
override val input: SendChannel<T> by lazy { private val input: SendChannel<T> by lazy {
//create a channel and start process of reading all elements into aggregator //create a channel and start process of reading all elements into aggregator
Channel<T>(capacity = Channel.RENDEZVOUS).also { Channel<T>(capacity = Channel.RENDEZVOUS).also {
launch { launch {
@ -220,6 +244,8 @@ class Reducer<T, S>(
} }
} }
} }
override suspend fun send(value: T) = input.send(value)
} }
/** /**
@ -231,7 +257,7 @@ class Collector<T>(scope: CoroutineScope) : AbstractConsumer<T>(scope) {
private val mutex = Mutex() private val mutex = Mutex()
val list: List<T> get() = _list val list: List<T> get() = _list
override val input: SendChannel<T> by lazy { private val input: SendChannel<T> by lazy {
//create a channel and start process of reading all elements into aggregator //create a channel and start process of reading all elements into aggregator
Channel<T>(capacity = Channel.RENDEZVOUS).also { Channel<T>(capacity = Channel.RENDEZVOUS).also {
launch { launch {
@ -243,6 +269,8 @@ class Collector<T>(scope: CoroutineScope) : AbstractConsumer<T>(scope) {
} }
} }
} }
override suspend fun send(value: T) = input.send(value)
} }
/** /**