Finish migration to 1.4.20

This commit is contained in:
Alexander Nozik 2020-09-28 15:38:29 +03:00
parent acfb070938
commit 01c209d1d6
21 changed files with 102 additions and 309 deletions

View File

@ -1,11 +1,8 @@
plugins {
id("ru.mipt.npm.publish") apply false
id("org.jetbrains.changelog") version "0.4.0"
id("ru.mipt.npm.project")
}
apply(plugin = "org.jetbrains.dokka")
val dataforgeVersion by extra("0.2.0-dev-2")
val dataforgeVersion by extra("0.2.0-dev-3")
val bintrayRepo by extra("dataforge")
val githubProject by extra("dataforge-core")
@ -15,6 +12,8 @@ allprojects {
group = "hep.dataforge"
version = dataforgeVersion
apply(plugin = "org.jetbrains.dokka")
repositories {
mavenLocal()
}
@ -22,5 +21,4 @@ allprojects {
subprojects {
apply(plugin = "ru.mipt.npm.publish")
apply(plugin = "org.jetbrains.dokka")
}

View File

@ -1,6 +1,5 @@
plugins {
id("ru.mipt.npm.mpp")
id("ru.mipt.npm.node")
id("ru.mipt.npm.native")
}

View File

@ -19,24 +19,24 @@ import java.util.*
import kotlin.reflect.KClass
import kotlin.reflect.full.cast
class ClassLoaderPlugin(val classLoader: ClassLoader) : AbstractPlugin() {
public class ClassLoaderPlugin(private val classLoader: ClassLoader) : AbstractPlugin() {
override val tag: PluginTag = PluginTag("classLoader", PluginTag.DATAFORGE_GROUP)
private val serviceCache: MutableMap<Class<*>, ServiceLoader<*>> = HashMap()
fun <T : Any> services(type: KClass<T>): Sequence<T> {
public fun <T : Any> services(type: KClass<T>): Sequence<T> {
return serviceCache.getOrPut(type.java) { ServiceLoader.load(type.java, classLoader) }.asSequence()
.map { type.cast(it) }
}
companion object {
val DEFAULT = ClassLoaderPlugin(Global::class.java.classLoader)
public companion object {
public val DEFAULT: ClassLoaderPlugin = ClassLoaderPlugin(Global::class.java.classLoader)
}
}
val Context.classLoaderPlugin get() = this.plugins.get() ?: ClassLoaderPlugin.DEFAULT
public val Context.classLoaderPlugin: ClassLoaderPlugin get() = this.plugins.get() ?: ClassLoaderPlugin.DEFAULT
inline fun <reified T : Any> Context.services() = classLoaderPlugin.services(T::class)
public inline fun <reified T : Any> Context.services(): Sequence<T> = classLoaderPlugin.services(T::class)
//open class JVMContext(

View File

@ -1,6 +1,5 @@
plugins {
id("ru.mipt.npm.mpp")
id("ru.mipt.npm.node")
id("ru.mipt.npm.native")
}

View File

@ -1,6 +1,5 @@
plugins {
id("ru.mipt.npm.mpp")
id("ru.mipt.npm.node")
id("ru.mipt.npm.native")
}

View File

@ -64,14 +64,19 @@ public class FrontMatterEnvelopeFormat(
return SimpleEnvelope(meta, data)
}
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
val metaFormat = metaFormatFactory(formatMeta, io.context)
writeRawString("$SEPARATOR\r\n")
metaFormat.run { writeObject(this@writeEnvelope, envelope.meta) }
writeRawString("$SEPARATOR\r\n")
override fun writeEnvelope(
output: Output,
envelope: Envelope,
metaFormatFactory: MetaFormatFactory,
formatMeta: Meta,
) {
val metaFormat = metaFormatFactory(formatMeta, this@FrontMatterEnvelopeFormat.io.context)
output.writeRawString("${hep.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR}\r\n")
metaFormat.run { this.writeObject(output, envelope.meta) }
output.writeRawString("${hep.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR}\r\n")
//Printing data
envelope.data?.let { data ->
writeBinary(data)
output.writeBinary(data)
}
}
@ -105,11 +110,13 @@ public class FrontMatterEnvelopeFormat(
override fun readPartial(input: Input): PartialEnvelope =
default.readPartial(input)
override fun Output.writeEnvelope(
override fun writeEnvelope(
output: Output,
envelope: Envelope,
metaFormatFactory: MetaFormatFactory,
formatMeta: Meta,
): Unit = default.run { writeEnvelope(envelope, metaFormatFactory, formatMeta) }
): Unit = default.writeEnvelope(output, envelope, metaFormatFactory, formatMeta)
override fun readObject(input: Input): Envelope = default.readObject(input)

View File

@ -20,15 +20,16 @@ public interface EnvelopeFormat : IOFormat<Envelope> {
public fun readPartial(input: Input): PartialEnvelope
public fun Output.writeEnvelope(
public fun writeEnvelope(
output: Output,
envelope: Envelope,
metaFormatFactory: MetaFormatFactory = defaultMetaFormat,
formatMeta: Meta = Meta.EMPTY
formatMeta: Meta = Meta.EMPTY,
)
override fun readObject(input: Input): Envelope
override fun writeObject(output: Output, obj: Envelope): Unit = output.writeEnvelope(obj)
override fun writeObject(output: Output, obj: Envelope): Unit = writeEnvelope(output, obj)
}
public fun EnvelopeFormat.read(input: Input): Envelope = readObject(input)

View File

@ -41,18 +41,23 @@ public class TaggedEnvelopeFormat(
writeRawString(END_SEQUENCE)
}
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
val metaFormat = metaFormatFactory.invoke(formatMeta, io.context)
override fun writeEnvelope(
output: Output,
envelope: Envelope,
metaFormatFactory: MetaFormatFactory,
formatMeta: Meta,
) {
val metaFormat = metaFormatFactory.invoke(formatMeta, this@TaggedEnvelopeFormat.io.context)
val metaBytes = metaFormat.toBinary(envelope.meta)
val actualSize: ULong = (envelope.data?.size ?: 0).toULong()
val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, actualSize)
writeBinary(tag.toBinary())
writeBinary(metaBytes)
writeRawString("\r\n")
output.writeBinary(tag.toBinary())
output.writeBinary(metaBytes)
output.writeRawString("\r\n")
envelope.data?.let {
writeBinary(it)
output.writeBinary(it)
}
flush()
output.flush()
}
/**
@ -158,11 +163,19 @@ public class TaggedEnvelopeFormat(
override fun readPartial(input: Input): PartialEnvelope =
default.run { readPartial(input) }
override fun Output.writeEnvelope(
override fun writeEnvelope(
output: Output,
envelope: Envelope,
metaFormatFactory: MetaFormatFactory,
formatMeta: Meta,
): Unit = default.run { writeEnvelope(envelope, metaFormatFactory, formatMeta) }
): Unit = default.run {
writeEnvelope(
output,
envelope,
metaFormatFactory,
formatMeta
)
}
override fun readObject(input: Input): Envelope = default.readObject(input)
}

View File

@ -30,32 +30,39 @@ public class TaglessEnvelopeFormat(
writeUtf8String("#? $key: $value;\r\n")
}
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
val metaFormat = metaFormatFactory(formatMeta, io.context)
override fun writeEnvelope(
output: Output,
envelope: Envelope,
metaFormatFactory: MetaFormatFactory,
formatMeta: Meta
) {
val metaFormat = metaFormatFactory(formatMeta, this.io.context)
//printing header
writeRawString(TAGLESS_ENVELOPE_HEADER + "\r\n")
output.writeRawString(TAGLESS_ENVELOPE_HEADER + "\r\n")
//printing all properties
writeProperty(META_TYPE_PROPERTY, metaFormatFactory.shortName)
output.writeProperty(META_TYPE_PROPERTY,
metaFormatFactory.shortName)
//TODO add optional metaFormat properties
val actualSize: Int = envelope.data?.size ?: 0
writeProperty(DATA_LENGTH_PROPERTY, actualSize)
output.writeProperty(DATA_LENGTH_PROPERTY, actualSize)
//Printing meta
if (!envelope.meta.isEmpty()) {
val metaBytes = metaFormat.toBinary(envelope.meta)
writeProperty(META_LENGTH_PROPERTY, metaBytes.size + 2)
writeUtf8String(metaStart + "\r\n")
writeBinary(metaBytes)
writeRawString("\r\n")
output.writeProperty(META_LENGTH_PROPERTY,
metaBytes.size + 2)
output.writeUtf8String(this.metaStart + "\r\n")
output.writeBinary(metaBytes)
output.writeRawString("\r\n")
}
//Printing data
envelope.data?.let { data ->
writeUtf8String(dataStart + "\r\n")
writeBinary(data)
output.writeUtf8String(this.dataStart + "\r\n")
output.writeBinary(data)
}
}
@ -169,7 +176,7 @@ public class TaglessEnvelopeFormat(
public companion object : EnvelopeFormatFactory {
private val propertyPattern = "#\\?\\s*(?<key>[\\w.]*)\\s*:\\s*(?<value>[^;]*);?".toRegex()
private val propertyPattern = "#\\?\\s*([\\w.]*)\\s*:\\s*([^;]*);?".toRegex()
public const val META_TYPE_PROPERTY: String = "metaType"
public const val META_LENGTH_PROPERTY: String = "metaLength"
@ -197,11 +204,19 @@ public class TaglessEnvelopeFormat(
override fun readPartial(input: Input): PartialEnvelope =
default.run { readPartial(input) }
override fun Output.writeEnvelope(
override fun writeEnvelope(
output: Output,
envelope: Envelope,
metaFormatFactory: MetaFormatFactory,
formatMeta: Meta,
): Unit = default.run { writeEnvelope(envelope, metaFormatFactory, formatMeta) }
): Unit = default.run {
writeEnvelope(
output,
envelope,
metaFormatFactory,
formatMeta
)
}
override fun readObject(input: Input): Envelope = default.readObject(input)

View File

@ -204,9 +204,7 @@ public fun IOPlugin.writeEnvelopeFile(
metaFormat: MetaFormatFactory? = null,
) {
path.rewrite {
with(envelopeFormat) {
writeEnvelope(envelope, metaFormat ?: envelopeFormat.defaultMetaFormat)
}
envelopeFormat.writeEnvelope(this, envelope, metaFormat ?: envelopeFormat.defaultMetaFormat)
}
}

View File

@ -1,4 +1,4 @@
package hep.dataforge.io.tcp
package hep.dataforge.io
import kotlinx.io.*
import kotlinx.io.buffer.Buffer
@ -30,16 +30,16 @@ private class BlockingStreamInput(val source: InputStream) : Input() {
}
}
fun <R> InputStream.read(size: Int, block: Input.() -> R): R {
public fun <R> InputStream.read(size: Int, block: Input.() -> R): R {
val buffer = ByteArray(size)
read(buffer)
return buffer.asBinary().read(block = block)
}
fun <R> InputStream.read(block: Input.() -> R): R = asInput().block()
public fun <R> InputStream.read(block: Input.() -> R): R = asInput().block()
fun <R> InputStream.readBlocking(block: Input.() -> R): R = BlockingStreamInput(this).block()
public fun <R> InputStream.readBlocking(block: Input.() -> R): R = BlockingStreamInput(this).block()
inline fun OutputStream.write(block: Output.() -> Unit) {
public inline fun OutputStream.write(block: Output.() -> Unit) {
asOutput().block()
}

View File

@ -1,67 +0,0 @@
package hep.dataforge.io.tcp
import hep.dataforge.context.Context
import hep.dataforge.context.ContextAware
import hep.dataforge.io.*
import hep.dataforge.meta.Meta
import kotlinx.coroutines.asCoroutineDispatcher
import kotlinx.coroutines.withContext
import java.net.Socket
import java.util.concurrent.Executors
import kotlin.time.ExperimentalTime
@ExperimentalTime
@Deprecated("To be replaced by flow-based client")
public class EnvelopeClient(
override val context: Context,
public val host: String,
public val port: Int,
formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
formatMeta: Meta = Meta.EMPTY
) : Responder, ContextAware {
private val dispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher()
private val format = formatFactory(formatMeta, context = context)
// private var socket: SocketChannel? = null
//
// private fun getSocket(): Socket {
// val socket = socket ?: Socket(host, port).also { this.socket = it }
// return if (socket.isConnected) {
// socket
// } else {
// Socket(host, port).also { this.socket = it }
// }
// }
public suspend fun close() {
try {
respond(
Envelope {
type = EnvelopeServer.SHUTDOWN_ENVELOPE_TYPE
}
)
} catch (ex: Exception) {
logger.error { ex }
}
}
@Suppress("BlockingMethodInNonBlockingContext")
override suspend fun respond(request: Envelope): Envelope = withContext(dispatcher) {
//val address = InetSocketAddress(host,port)
val socket = Socket(host, port)
val inputStream = socket.getInputStream()
val outputStream = socket.getOutputStream()
format.run {
outputStream.write {
writeObject(this, request)
}
logger.debug { "Sent request with type ${request.type} to ${socket.remoteSocketAddress}" }
val res = inputStream.readBlocking { readObject(this) }
logger.debug { "Received response with type ${res.type} from ${socket.remoteSocketAddress}" }
return@withContext res
}
}
}

View File

@ -1,104 +0,0 @@
package hep.dataforge.io.tcp
import hep.dataforge.context.Context
import hep.dataforge.context.ContextAware
import hep.dataforge.io.EnvelopeFormatFactory
import hep.dataforge.io.Responder
import hep.dataforge.io.TaggedEnvelopeFormat
import hep.dataforge.io.type
import hep.dataforge.meta.Meta
import kotlinx.coroutines.*
import java.net.ServerSocket
import java.net.Socket
import kotlin.concurrent.thread
@Deprecated("To be replaced by flow-based server")
public class EnvelopeServer(
override val context: Context,
val port: Int,
val responder: Responder,
val scope: CoroutineScope,
formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
formatMeta: Meta = Meta.EMPTY
) : ContextAware {
private var job: Job? = null
private val format = formatFactory(formatMeta, context = context)
public fun start() {
if (job == null) {
logger.info { "Starting envelope server on port $port" }
job = scope.launch(Dispatchers.IO) {
val serverSocket = ServerSocket(port)
//TODO add handshake and format negotiation
while (isActive && !serverSocket.isClosed) {
val socket = serverSocket.accept()
logger.info { "Accepted connection from ${socket.remoteSocketAddress}" }
readSocket(socket)
}
}
}
}
public fun stop() {
logger.info { "Stopping envelope server on port $port" }
job?.cancel()
job = null
}
// private fun CoroutineScope.readSocket(socket: Socket) {
// launch(Dispatchers.IO) {
// val input = socket.getInputStream().asInput()
// val output = socket.getOutputStream().asOutput()
// format.run {
// while (isActive && socket.isConnected) {
// val request = input.readThis()
// logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" }
// if (request.type == SHUTDOWN_ENVELOPE_TYPE) {
// //Echo shutdown command
// logger.info { "Accepted graceful shutdown signal from ${socket.inetAddress}" }
// socket.close()
// cancel("Graceful connection shutdown requested by client")
// }
// val response = responder.respond(request)
// output.writeThis(response)
// }
// }
// }
// }
private fun readSocket(socket: Socket) {
thread {
val inputStream = socket.getInputStream()
val outputStream = socket.getOutputStream()
format.run {
while (socket.isConnected) {
val request = inputStream.readBlocking { readObject(this) }
logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" }
if (request.type == SHUTDOWN_ENVELOPE_TYPE) {
//Echo shutdown command
outputStream.write {
writeObject(this, request)
}
logger.info { "Accepted graceful shutdown signal from ${socket.inetAddress}" }
socket.close()
return@thread
// cancel("Graceful connection shutdown requested by client")
}
runBlocking {
val response = responder.respond(request)
outputStream.write {
writeObject(this, response)
}
logger.debug { "Sent response with type ${response.type} to ${socket.remoteSocketAddress}" }
}
}
}
}
}
public companion object {
public const val SHUTDOWN_ENVELOPE_TYPE = "@shutdown"
}
}

View File

@ -1,70 +0,0 @@
package hep.dataforge.io.tcp
import hep.dataforge.context.Global
import hep.dataforge.io.Envelope
import hep.dataforge.io.Responder
import hep.dataforge.io.TaggedEnvelopeFormat
import hep.dataforge.io.writeToByteArray
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.runBlocking
import kotlinx.io.writeDouble
import org.junit.jupiter.api.AfterAll
import org.junit.jupiter.api.BeforeAll
import org.junit.jupiter.api.Timeout
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.time.ExperimentalTime
@ExperimentalStdlibApi
object EchoResponder : Responder {
override suspend fun respond(request: Envelope): Envelope {
val string = TaggedEnvelopeFormat().run { writeToByteArray(request).decodeToString() }
println("ECHO:")
println(string)
return request
}
}
@ExperimentalTime
@ExperimentalStdlibApi
class EnvelopeServerTest {
companion object {
@JvmStatic
val echoEnvelopeServer = EnvelopeServer(Global, 7778, EchoResponder, GlobalScope)
@BeforeAll
@JvmStatic
fun start() {
echoEnvelopeServer.start()
}
@AfterAll
@JvmStatic
fun close() {
echoEnvelopeServer.stop()
}
}
@Test
@Timeout(1)
fun doEchoTest() {
val request = Envelope {
type = "test.echo"
meta {
"test.value" put 22
}
data {
writeDouble(22.7)
}
}
val client = EnvelopeClient(Global, host = "localhost", port = 7778)
runBlocking {
val response = client.respond(request)
assertEquals(request.meta, response.meta)
// assertEquals(request.data?.toBytes()?.decodeToString(), response.data?.toBytes()?.decodeToString())
client.close()
}
}
}

View File

@ -1,6 +1,7 @@
import ru.mipt.npm.gradle.KScienceVersions
plugins {
id("ru.mipt.npm.mpp")
id("ru.mipt.npm.node")
id("ru.mipt.npm.native")
}
@ -8,4 +9,8 @@ kscience {
useSerialization()
}
description = "Meta definition and basic operations on meta"
description = "Meta definition and basic operations on meta"
dependencies{
commonMainApi("org.jetbrains.kotlinx:kotlinx-serialization-json:${KScienceVersions.serializationVersion}")
}

View File

@ -8,7 +8,7 @@ import hep.dataforge.values.isList
//TODO add Meta wrapper for dynamic
fun Value.toDynamic(): dynamic {
public fun Value.toDynamic(): dynamic {
return if (isList()) {
list.map { it.toDynamic() }.toTypedArray().asDynamic()
} else {
@ -19,7 +19,7 @@ fun Value.toDynamic(): dynamic {
/**
* Represent or copy this [Meta] to dynamic object to be passed to JS libraries
*/
fun Meta.toDynamic(): dynamic {
public fun Meta.toDynamic(): dynamic {
if (this is DynamicMeta) return this.obj
fun MetaItem<*>.toDynamic(): dynamic = when (this) {
@ -38,8 +38,8 @@ fun Meta.toDynamic(): dynamic {
return res
}
public class DynamicMeta(public val obj: dynamic) : MetaBase() {
private fun keys() = js("Object.keys(this.obj)") as Array<String>
public class DynamicMeta(internal val obj: dynamic) : MetaBase() {
private fun keys(): Array<String> = js("Object").keys(obj)
private fun isArray(@Suppress("UNUSED_PARAMETER") obj: dynamic): Boolean =
js("Array.isArray(obj)") as Boolean

View File

@ -1,6 +1,5 @@
plugins {
id("ru.mipt.npm.mpp")
id("ru.mipt.npm.node")
id("ru.mipt.npm.native")
}

View File

@ -1,6 +1,5 @@
plugins {
id("ru.mipt.npm.mpp")
id("ru.mipt.npm.node")
id("ru.mipt.npm.native")
}

View File

@ -1,6 +1,5 @@
plugins {
id("ru.mipt.npm.mpp")
id("ru.mipt.npm.node")
id("ru.mipt.npm.native")
}

View File

@ -1,6 +1,8 @@
kotlin.code.style=official
kotlin.parallel.tasks.in.project=true
kotlin.mpp.enableGranularSourceSetsMetadata=true
kotlin.native.enableDependencyPropagation=false
kotlin.mpp.stability.nowarn=true
org.gradle.jvmargs=-XX:MaxMetaspaceSize=512m
org.gradle.parallel=true

View File

@ -10,10 +10,11 @@ pluginManagement {
maven("https://dl.bintray.com/mipt-npm/dev")
}
val toolsVersion = "0.6.0-dev-5"
val kotlinVersion = "1.4.20-dev-3898-14"
val toolsVersion = "0.6.1-dev-1.4.20-M1"
val kotlinVersion = "1.4.20-M1"
plugins {
id("ru.mipt.npm.project") version toolsVersion
id("ru.mipt.npm.mpp") version toolsVersion
id("ru.mipt.npm.jvm") version toolsVersion
id("ru.mipt.npm.js") version toolsVersion