kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Finish migration to 1.4.20

Browse Source
This commit is contained in:
Alexander Nozik 2020-09-28 15:38:29 +03:00
parent acfb070938
commit 01c209d1d6
21 changed files with 102 additions and 309 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
build.gradle.kts
View File

@ -1,11 +1,8 @@
plugins {
id("ru.mipt.npm.publish") apply false
id("org.jetbrains.changelog") version "0.4.0"
id("ru.mipt.npm.project")
}
apply(plugin = "org.jetbrains.dokka")
val dataforgeVersion by extra("0.2.0-dev-2")
val dataforgeVersion by extra("0.2.0-dev-3")
val bintrayRepo by extra("dataforge")
val githubProject by extra("dataforge-core")
@ -15,6 +12,8 @@ allprojects {
group = "hep.dataforge"
version = dataforgeVersion
apply(plugin = "org.jetbrains.dokka")
repositories {
mavenLocal()
}
@ -22,5 +21,4 @@ allprojects {
subprojects {
apply(plugin = "ru.mipt.npm.publish")
apply(plugin = "org.jetbrains.dokka")
}

1
dataforge-context/build.gradle.kts
View File

@ -1,6 +1,5 @@
plugins {
id("ru.mipt.npm.mpp")
id("ru.mipt.npm.node")
id("ru.mipt.npm.native")
}

12
dataforge-context/src/jvmMain/kotlin/hep/dataforge/context/ClassLoaderPlugin.kt
View File

@ -19,24 +19,24 @@ import java.util.*
import kotlin.reflect.KClass
import kotlin.reflect.full.cast
class ClassLoaderPlugin(val classLoader: ClassLoader) : AbstractPlugin() {
public class ClassLoaderPlugin(private val classLoader: ClassLoader) : AbstractPlugin() {
override val tag: PluginTag = PluginTag("classLoader", PluginTag.DATAFORGE_GROUP)
private val serviceCache: MutableMap<Class<*>, ServiceLoader<*>> = HashMap()
fun <T : Any> services(type: KClass<T>): Sequence<T> {
public fun <T : Any> services(type: KClass<T>): Sequence<T> {
return serviceCache.getOrPut(type.java) { ServiceLoader.load(type.java, classLoader) }.asSequence()
.map { type.cast(it) }
}
companion object {
val DEFAULT = ClassLoaderPlugin(Global::class.java.classLoader)
public companion object {
public val DEFAULT: ClassLoaderPlugin = ClassLoaderPlugin(Global::class.java.classLoader)
}
}
val Context.classLoaderPlugin get() = this.plugins.get() ?: ClassLoaderPlugin.DEFAULT
public val Context.classLoaderPlugin: ClassLoaderPlugin get() = this.plugins.get() ?: ClassLoaderPlugin.DEFAULT
inline fun <reified T : Any> Context.services() = classLoaderPlugin.services(T::class)
public inline fun <reified T : Any> Context.services(): Sequence<T> = classLoaderPlugin.services(T::class)
//open class JVMContext(

1
dataforge-data/build.gradle.kts
View File

@ -1,6 +1,5 @@
plugins {
id("ru.mipt.npm.mpp")
id("ru.mipt.npm.node")
id("ru.mipt.npm.native")
}

1
dataforge-io/build.gradle.kts
View File

@ -1,6 +1,5 @@
plugins {
id("ru.mipt.npm.mpp")
id("ru.mipt.npm.node")
id("ru.mipt.npm.native")
}

23
dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
View File

@ -64,14 +64,19 @@ public class FrontMatterEnvelopeFormat(
return SimpleEnvelope(meta, data)
}
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
val metaFormat = metaFormatFactory(formatMeta, io.context)
writeRawString("$SEPARATOR\r\n")
metaFormat.run { writeObject(this@writeEnvelope, envelope.meta) }
writeRawString("$SEPARATOR\r\n")
override fun writeEnvelope(
output: Output,
envelope: Envelope,
metaFormatFactory: MetaFormatFactory,
formatMeta: Meta,
) {
val metaFormat = metaFormatFactory(formatMeta, this@FrontMatterEnvelopeFormat.io.context)
output.writeRawString("${hep.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR}\r\n")
metaFormat.run { this.writeObject(output, envelope.meta) }
output.writeRawString("${hep.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR}\r\n")
//Printing data
envelope.data?.let { data ->
writeBinary(data)
output.writeBinary(data)
}
}
@ -105,11 +110,13 @@ public class FrontMatterEnvelopeFormat(
override fun readPartial(input: Input): PartialEnvelope =
default.readPartial(input)
override fun Output.writeEnvelope(
override fun writeEnvelope(
output: Output,
envelope: Envelope,
metaFormatFactory: MetaFormatFactory,
formatMeta: Meta,
): Unit = default.run { writeEnvelope(envelope, metaFormatFactory, formatMeta) }
): Unit = default.writeEnvelope(output, envelope, metaFormatFactory, formatMeta)
override fun readObject(input: Input): Envelope = default.readObject(input)

7
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt
View File

@ -20,15 +20,16 @@ public interface EnvelopeFormat : IOFormat<Envelope> {
public fun readPartial(input: Input): PartialEnvelope
public fun Output.writeEnvelope(
public fun writeEnvelope(
output: Output,
envelope: Envelope,
metaFormatFactory: MetaFormatFactory = defaultMetaFormat,
formatMeta: Meta = Meta.EMPTY
formatMeta: Meta = Meta.EMPTY,
)
override fun readObject(input: Input): Envelope
override fun writeObject(output: Output, obj: Envelope): Unit = output.writeEnvelope(obj)
override fun writeObject(output: Output, obj: Envelope): Unit = writeEnvelope(output, obj)
}
public fun EnvelopeFormat.read(input: Input): Envelope = readObject(input)

31
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt
View File

@ -41,18 +41,23 @@ public class TaggedEnvelopeFormat(
writeRawString(END_SEQUENCE)
}
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
val metaFormat = metaFormatFactory.invoke(formatMeta, io.context)
override fun writeEnvelope(
output: Output,
envelope: Envelope,
metaFormatFactory: MetaFormatFactory,
formatMeta: Meta,
) {
val metaFormat = metaFormatFactory.invoke(formatMeta, this@TaggedEnvelopeFormat.io.context)
val metaBytes = metaFormat.toBinary(envelope.meta)
val actualSize: ULong = (envelope.data?.size ?: 0).toULong()
val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, actualSize)
writeBinary(tag.toBinary())
writeBinary(metaBytes)
writeRawString("\r\n")
output.writeBinary(tag.toBinary())
output.writeBinary(metaBytes)
output.writeRawString("\r\n")
envelope.data?.let {
writeBinary(it)
output.writeBinary(it)
}
flush()
output.flush()
}
/**
@ -158,11 +163,19 @@ public class TaggedEnvelopeFormat(
override fun readPartial(input: Input): PartialEnvelope =
default.run { readPartial(input) }
override fun Output.writeEnvelope(
override fun writeEnvelope(
output: Output,
envelope: Envelope,
metaFormatFactory: MetaFormatFactory,
formatMeta: Meta,
): Unit = default.run { writeEnvelope(envelope, metaFormatFactory, formatMeta) }
): Unit = default.run {
writeEnvelope(
output,
envelope,
metaFormatFactory,
formatMeta
)
}
override fun readObject(input: Input): Envelope = default.readObject(input)
}

43
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt
View File

@ -30,32 +30,39 @@ public class TaglessEnvelopeFormat(
writeUtf8String("#? $key: $value;\r\n")
}
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
val metaFormat = metaFormatFactory(formatMeta, io.context)
override fun writeEnvelope(
output: Output,
envelope: Envelope,
metaFormatFactory: MetaFormatFactory,
formatMeta: Meta
) {
val metaFormat = metaFormatFactory(formatMeta, this.io.context)
//printing header
writeRawString(TAGLESS_ENVELOPE_HEADER + "\r\n")
output.writeRawString(TAGLESS_ENVELOPE_HEADER + "\r\n")
//printing all properties
writeProperty(META_TYPE_PROPERTY, metaFormatFactory.shortName)
output.writeProperty(META_TYPE_PROPERTY,
metaFormatFactory.shortName)
//TODO add optional metaFormat properties
val actualSize: Int = envelope.data?.size ?: 0
writeProperty(DATA_LENGTH_PROPERTY, actualSize)
output.writeProperty(DATA_LENGTH_PROPERTY, actualSize)
//Printing meta
if (!envelope.meta.isEmpty()) {
val metaBytes = metaFormat.toBinary(envelope.meta)
writeProperty(META_LENGTH_PROPERTY, metaBytes.size + 2)
writeUtf8String(metaStart + "\r\n")
writeBinary(metaBytes)
writeRawString("\r\n")
output.writeProperty(META_LENGTH_PROPERTY,
metaBytes.size + 2)
output.writeUtf8String(this.metaStart + "\r\n")
output.writeBinary(metaBytes)
output.writeRawString("\r\n")
}
//Printing data
envelope.data?.let { data ->
writeUtf8String(dataStart + "\r\n")
writeBinary(data)
output.writeUtf8String(this.dataStart + "\r\n")
output.writeBinary(data)
}
}
@ -169,7 +176,7 @@ public class TaglessEnvelopeFormat(
public companion object : EnvelopeFormatFactory {
private val propertyPattern = "#\\?\\s*(?<key>[\\w.]*)\\s*:\\s*(?<value>[^;]*);?".toRegex()
private val propertyPattern = "#\\?\\s*([\\w.]*)\\s*:\\s*([^;]*);?".toRegex()
public const val META_TYPE_PROPERTY: String = "metaType"
public const val META_LENGTH_PROPERTY: String = "metaLength"
@ -197,11 +204,19 @@ public class TaglessEnvelopeFormat(
override fun readPartial(input: Input): PartialEnvelope =
default.run { readPartial(input) }
override fun Output.writeEnvelope(
override fun writeEnvelope(
output: Output,
envelope: Envelope,
metaFormatFactory: MetaFormatFactory,
formatMeta: Meta,
): Unit = default.run { writeEnvelope(envelope, metaFormatFactory, formatMeta) }
): Unit = default.run {
writeEnvelope(
output,
envelope,
metaFormatFactory,
formatMeta
)
}
override fun readObject(input: Input): Envelope = default.readObject(input)

4
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt
View File

@ -204,9 +204,7 @@ public fun IOPlugin.writeEnvelopeFile(
metaFormat: MetaFormatFactory? = null,
) {
path.rewrite {
with(envelopeFormat) {
writeEnvelope(envelope, metaFormat ?: envelopeFormat.defaultMetaFormat)
}
envelopeFormat.writeEnvelope(this, envelope, metaFormat ?: envelopeFormat.defaultMetaFormat)
}
}

10
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/streams.kt → dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/streamsIO.kt
View File

@ -1,4 +1,4 @@
package hep.dataforge.io.tcp
package hep.dataforge.io
import kotlinx.io.*
import kotlinx.io.buffer.Buffer
@ -30,16 +30,16 @@ private class BlockingStreamInput(val source: InputStream) : Input() {
}
}
fun <R> InputStream.read(size: Int, block: Input.() -> R): R {
public fun <R> InputStream.read(size: Int, block: Input.() -> R): R {
val buffer = ByteArray(size)
read(buffer)
return buffer.asBinary().read(block = block)
}
fun <R> InputStream.read(block: Input.() -> R): R = asInput().block()
public fun <R> InputStream.read(block: Input.() -> R): R = asInput().block()
fun <R> InputStream.readBlocking(block: Input.() -> R): R = BlockingStreamInput(this).block()
public fun <R> InputStream.readBlocking(block: Input.() -> R): R = BlockingStreamInput(this).block()
inline fun OutputStream.write(block: Output.() -> Unit) {
public inline fun OutputStream.write(block: Output.() -> Unit) {
asOutput().block()
}

67
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt
View File

@ -1,67 +0,0 @@
package hep.dataforge.io.tcp
import hep.dataforge.context.Context
import hep.dataforge.context.ContextAware
import hep.dataforge.io.*
import hep.dataforge.meta.Meta
import kotlinx.coroutines.asCoroutineDispatcher
import kotlinx.coroutines.withContext
import java.net.Socket
import java.util.concurrent.Executors
import kotlin.time.ExperimentalTime
@ExperimentalTime
@Deprecated("To be replaced by flow-based client")
public class EnvelopeClient(
override val context: Context,
public val host: String,
public val port: Int,
formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
formatMeta: Meta = Meta.EMPTY
) : Responder, ContextAware {
private val dispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher()
private val format = formatFactory(formatMeta, context = context)
// private var socket: SocketChannel? = null
//
// private fun getSocket(): Socket {
// val socket = socket ?: Socket(host, port).also { this.socket = it }
// return if (socket.isConnected) {
// socket
// } else {
// Socket(host, port).also { this.socket = it }
// }
// }
public suspend fun close() {
try {
respond(
Envelope {
type = EnvelopeServer.SHUTDOWN_ENVELOPE_TYPE
}
)
} catch (ex: Exception) {
logger.error { ex }
}
}
@Suppress("BlockingMethodInNonBlockingContext")
override suspend fun respond(request: Envelope): Envelope = withContext(dispatcher) {
//val address = InetSocketAddress(host,port)
val socket = Socket(host, port)
val inputStream = socket.getInputStream()
val outputStream = socket.getOutputStream()
format.run {
outputStream.write {
writeObject(this, request)
}
logger.debug { "Sent request with type ${request.type} to ${socket.remoteSocketAddress}" }
val res = inputStream.readBlocking { readObject(this) }
logger.debug { "Received response with type ${res.type} from ${socket.remoteSocketAddress}" }
return@withContext res
}
}
}

104
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt
View File

@ -1,104 +0,0 @@
package hep.dataforge.io.tcp
import hep.dataforge.context.Context
import hep.dataforge.context.ContextAware
import hep.dataforge.io.EnvelopeFormatFactory
import hep.dataforge.io.Responder
import hep.dataforge.io.TaggedEnvelopeFormat
import hep.dataforge.io.type
import hep.dataforge.meta.Meta
import kotlinx.coroutines.*
import java.net.ServerSocket
import java.net.Socket
import kotlin.concurrent.thread
@Deprecated("To be replaced by flow-based server")
public class EnvelopeServer(
override val context: Context,
val port: Int,
val responder: Responder,
val scope: CoroutineScope,
formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
formatMeta: Meta = Meta.EMPTY
) : ContextAware {
private var job: Job? = null
private val format = formatFactory(formatMeta, context = context)
public fun start() {
if (job == null) {
logger.info { "Starting envelope server on port $port" }
job = scope.launch(Dispatchers.IO) {
val serverSocket = ServerSocket(port)
//TODO add handshake and format negotiation
while (isActive && !serverSocket.isClosed) {
val socket = serverSocket.accept()
logger.info { "Accepted connection from ${socket.remoteSocketAddress}" }
readSocket(socket)
}
}
}
}
public fun stop() {
logger.info { "Stopping envelope server on port $port" }
job?.cancel()
job = null
}
// private fun CoroutineScope.readSocket(socket: Socket) {
// launch(Dispatchers.IO) {
// val input = socket.getInputStream().asInput()
// val output = socket.getOutputStream().asOutput()
// format.run {
// while (isActive && socket.isConnected) {
// val request = input.readThis()
// logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" }
// if (request.type == SHUTDOWN_ENVELOPE_TYPE) {
// //Echo shutdown command
// logger.info { "Accepted graceful shutdown signal from ${socket.inetAddress}" }
// socket.close()
// cancel("Graceful connection shutdown requested by client")
// }
// val response = responder.respond(request)
// output.writeThis(response)
// }
// }
// }
// }
private fun readSocket(socket: Socket) {
thread {
val inputStream = socket.getInputStream()
val outputStream = socket.getOutputStream()
format.run {
while (socket.isConnected) {
val request = inputStream.readBlocking { readObject(this) }
logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" }
if (request.type == SHUTDOWN_ENVELOPE_TYPE) {
//Echo shutdown command
outputStream.write {
writeObject(this, request)
}
logger.info { "Accepted graceful shutdown signal from ${socket.inetAddress}" }
socket.close()
return@thread
// cancel("Graceful connection shutdown requested by client")
}
runBlocking {
val response = responder.respond(request)
outputStream.write {
writeObject(this, response)
}
logger.debug { "Sent response with type ${response.type} to ${socket.remoteSocketAddress}" }
}
}
}
}
}
public companion object {
public const val SHUTDOWN_ENVELOPE_TYPE = "@shutdown"
}
}

70
dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt
View File

@ -1,70 +0,0 @@
package hep.dataforge.io.tcp
import hep.dataforge.context.Global
import hep.dataforge.io.Envelope
import hep.dataforge.io.Responder
import hep.dataforge.io.TaggedEnvelopeFormat
import hep.dataforge.io.writeToByteArray
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.runBlocking
import kotlinx.io.writeDouble
import org.junit.jupiter.api.AfterAll
import org.junit.jupiter.api.BeforeAll
import org.junit.jupiter.api.Timeout
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.time.ExperimentalTime
@ExperimentalStdlibApi
object EchoResponder : Responder {
override suspend fun respond(request: Envelope): Envelope {
val string = TaggedEnvelopeFormat().run { writeToByteArray(request).decodeToString() }
println("ECHO:")
println(string)
return request
}
}
@ExperimentalTime
@ExperimentalStdlibApi
class EnvelopeServerTest {
companion object {
@JvmStatic
val echoEnvelopeServer = EnvelopeServer(Global, 7778, EchoResponder, GlobalScope)
@BeforeAll
@JvmStatic
fun start() {
echoEnvelopeServer.start()
}
@AfterAll
@JvmStatic
fun close() {
echoEnvelopeServer.stop()
}
}
@Test
@Timeout(1)
fun doEchoTest() {
val request = Envelope {
type = "test.echo"
meta {
"test.value" put 22
}
data {
writeDouble(22.7)
}
}
val client = EnvelopeClient(Global, host = "localhost", port = 7778)
runBlocking {
val response = client.respond(request)
assertEquals(request.meta, response.meta)
// assertEquals(request.data?.toBytes()?.decodeToString(), response.data?.toBytes()?.decodeToString())
client.close()
}
}
}

7
dataforge-meta/build.gradle.kts
View File

@ -1,6 +1,7 @@
import ru.mipt.npm.gradle.KScienceVersions
plugins {
id("ru.mipt.npm.mpp")
id("ru.mipt.npm.node")
id("ru.mipt.npm.native")
}
@ -9,3 +10,7 @@ kscience {
}
description = "Meta definition and basic operations on meta"
dependencies{
commonMainApi("org.jetbrains.kotlinx:kotlinx-serialization-json:${KScienceVersions.serializationVersion}")
}

8
dataforge-meta/src/jsMain/kotlin/hep/dataforge/meta/DynamicMeta.kt
View File

@ -8,7 +8,7 @@ import hep.dataforge.values.isList
//TODO add Meta wrapper for dynamic
fun Value.toDynamic(): dynamic {
public fun Value.toDynamic(): dynamic {
return if (isList()) {
list.map { it.toDynamic() }.toTypedArray().asDynamic()
} else {
@ -19,7 +19,7 @@ fun Value.toDynamic(): dynamic {
/**
* Represent or copy this [Meta] to dynamic object to be passed to JS libraries
*/
fun Meta.toDynamic(): dynamic {
public fun Meta.toDynamic(): dynamic {
if (this is DynamicMeta) return this.obj
fun MetaItem<*>.toDynamic(): dynamic = when (this) {
@ -38,8 +38,8 @@ fun Meta.toDynamic(): dynamic {
return res
}
public class DynamicMeta(public val obj: dynamic) : MetaBase() {
private fun keys() = js("Object.keys(this.obj)") as Array<String>
public class DynamicMeta(internal val obj: dynamic) : MetaBase() {
private fun keys(): Array<String> = js("Object").keys(obj)
private fun isArray(@Suppress("UNUSED_PARAMETER") obj: dynamic): Boolean =
js("Array.isArray(obj)") as Boolean

1
dataforge-output/build.gradle.kts
View File

@ -1,6 +1,5 @@
plugins {
id("ru.mipt.npm.mpp")
id("ru.mipt.npm.node")
id("ru.mipt.npm.native")
}

1
dataforge-tables/build.gradle.kts
View File

@ -1,6 +1,5 @@
plugins {
id("ru.mipt.npm.mpp")
id("ru.mipt.npm.node")
id("ru.mipt.npm.native")
}

1
dataforge-workspace/build.gradle.kts
View File

@ -1,6 +1,5 @@
plugins {
id("ru.mipt.npm.mpp")
id("ru.mipt.npm.node")
id("ru.mipt.npm.native")
}

2
gradle.properties
View File

@ -1,6 +1,8 @@
kotlin.code.style=official
kotlin.parallel.tasks.in.project=true
kotlin.mpp.enableGranularSourceSetsMetadata=true
kotlin.native.enableDependencyPropagation=false
kotlin.mpp.stability.nowarn=true
org.gradle.jvmargs=-XX:MaxMetaspaceSize=512m
org.gradle.parallel=true

5
settings.gradle.kts
View File

@ -10,10 +10,11 @@ pluginManagement {
maven("https://dl.bintray.com/mipt-npm/dev")
}
val toolsVersion = "0.6.0-dev-5"
val kotlinVersion = "1.4.20-dev-3898-14"
val toolsVersion = "0.6.1-dev-1.4.20-M1"
val kotlinVersion = "1.4.20-M1"
plugins {
id("ru.mipt.npm.project") version toolsVersion
id("ru.mipt.npm.mpp") version toolsVersion
id("ru.mipt.npm.jvm") version toolsVersion
id("ru.mipt.npm.js") version toolsVersion