kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

IO refactoring

Browse Source
This commit is contained in:
Alexander Nozik 2020-09-07 13:44:02 +03:00
parent a767f279a3
commit eeb4643d93
34 changed files with 371 additions and 413 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
build.gradle.kts
View File

@ -3,7 +3,9 @@ plugins {
id("org.jetbrains.changelog") version "0.4.0"
}
val dataforgeVersion by extra("0.1.9-dev-5")
apply(plugin = "org.jetbrains.dokka")
val dataforgeVersion by extra("0.2.0-dev-1")
val bintrayRepo by extra("dataforge")
val githubProject by extra("dataforge-core")

2
dataforge-context/build.gradle.kts
View File

@ -1,7 +1,7 @@
plugins {
id("ru.mipt.npm.mpp")
id("ru.mipt.npm.node")
// id("ru.mipt.npm.native")
id("ru.mipt.npm.native")
}
description = "Context and provider definitions"

4
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt
View File

@ -6,7 +6,7 @@ import kotlin.properties.ReadOnlyProperty
import kotlin.reflect.KClass
import kotlin.reflect.KProperty
abstract class AbstractPlugin(override val meta: Meta = Meta.EMPTY) : Plugin {
public abstract class AbstractPlugin(override val meta: Meta = Meta.EMPTY) : Plugin {
private var _context: Context? = null
private val dependencies = ArrayList<PluginFactory<*>>()
@ -32,7 +32,7 @@ abstract class AbstractPlugin(override val meta: Meta = Meta.EMPTY) : Plugin {
}
}
fun <T : Named> Collection<T>.toMap(): Map<Name, T> = associate { it.name to it }
public fun <T : Named> Collection<T>.toMap(): Map<Name, T> = associate { it.name to it }
private class PluginDependencyDelegate<P : Plugin>(val type: KClass<out P>) : ReadOnlyProperty<AbstractPlugin, P> {
override fun getValue(thisRef: AbstractPlugin, property: KProperty<*>): P {

75
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt
View File

@ -2,13 +2,11 @@ package hep.dataforge.context
import hep.dataforge.meta.*
import hep.dataforge.names.Name
import hep.dataforge.names.asName
import hep.dataforge.names.plus
import hep.dataforge.provider.Provider
import hep.dataforge.provider.top
import hep.dataforge.values.Value
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.SupervisorJob
import mu.KLogger
@ -28,9 +26,9 @@ import kotlin.jvm.JvmName
* Since plugins could contain mutable state, context has two states: active and inactive. No changes are allowed to active context.
* @author Alexander Nozik
*/
open class Context(
public open class Context(
final override val name: Name,
val parent: Context? = Global
public val parent: Context? = Global,
) : Named, MetaRepr, Provider, CoroutineScope {
private val config = Config()
@ -38,7 +36,7 @@ open class Context(
/**
* Context properties. Working as substitute for environment variables
*/
val properties: Meta = if (parent == null) {
private val properties: Meta = if (parent == null) {
config
} else {
Laminate(config, parent.properties)
@ -47,19 +45,19 @@ open class Context(
/**
* Context logger
*/
val logger: KLogger = KotlinLogging.logger(name.toString())
public val logger: KLogger = KotlinLogging.logger(name.toString())
/**
* A [PluginManager] for current context
*/
val plugins: PluginManager by lazy { PluginManager(this) }
public val plugins: PluginManager by lazy { PluginManager(this) }
private val activators = HashSet<Any>()
/**
* Defines if context is used in any kind of active computations. Active context properties and plugins could not be changed
*/
val isActive: Boolean = activators.isNotEmpty()
public val isActive: Boolean = activators.isNotEmpty()
override val defaultTarget: String get() = Plugin.PLUGIN_TARGET
@ -74,21 +72,21 @@ open class Context(
/**
* Mark context as active and used by [activator]
*/
fun activate(activator: Any) {
public fun activate(activator: Any) {
activators.add(activator)
}
/**
* Mark context unused by [activator]
*/
fun deactivate(activator: Any) {
public fun deactivate(activator: Any) {
activators.remove(activator)
}
/**
* Change the properties of the context. If active, throw an exception
*/
fun configure(action: Config.() -> Unit) {
public fun configure(action: Config.() -> Unit) {
if (isActive) error("Can't configure active context")
config.action()
}
@ -102,7 +100,7 @@ open class Context(
/**
* Detach all plugins and terminate context
*/
open fun close() {
public open fun close() {
if (isActive) error("Can't close active context")
//detach all plugins
plugins.forEach { it.detach() }
@ -115,71 +113,30 @@ open class Context(
}
}
fun Context.content(target: String): Map<Name, Any> = content<Any>(target)
/**
* A map of all objects provided by plugins with given target and type
*/
@JvmName("typedContent")
inline fun <reified T : Any> Context.content(target: String): Map<Name, T> =
plugins.flatMap { plugin ->
public inline fun <reified T : Any> Context.resolve(target: String): Map<Name, T> = plugins.flatMap { plugin ->
plugin.top<T>(target).entries.map { (plugin.name + it.key) to it.value }
}.associate { it }
}.associate { it }
/**
* A global root context. Closing [Global] terminates the framework.
*/
object Global : Context("GLOBAL".asName(), null) {
override val coroutineContext: CoroutineContext = GlobalScope.coroutineContext + SupervisorJob()
/**
* Closing all contexts
*
* @throws Exception
*/
override fun close() {
logger.info { "Shutting down GLOBAL" }
for (ctx in contextRegistry.values) {
ctx.close()
}
super.close()
}
private val contextRegistry = HashMap<String, Context>()
/**
* Get previously built context
*
* @param name
* @return
*/
fun getContext(name: String): Context? {
return contextRegistry[name]
}
fun context(name: String, parent: Context = this, block: ContextBuilder.() -> Unit = {}): Context =
ContextBuilder(name, parent).apply(block).build()
}
public fun Context.resolve(target: String): Map<Name, Any> = resolve<Any>(target)
/**
* The interface for something that encapsulated in context
*
* @author Alexander Nozik
* @version $Id: $Id
*/
interface ContextAware {
public interface ContextAware {
/**
* Get context for this object
*
* @return
*/
val context: Context
public val context: Context
val logger: KLogger
public val logger: KLogger
get() = if (this is Named) {
KotlinLogging.logger((context.name + this.name).toString())
} else {

19
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt
View File

@ -9,31 +9,34 @@ import hep.dataforge.names.toName
* A convenience builder for context
*/
@DFBuilder
class ContextBuilder(var name: String = "@anonymous", val parent: Context = Global) {
public class ContextBuilder(private val parent: Context = Global, public var name: String = "@anonymous") {
private val plugins = ArrayList<Plugin>()
private var meta = MetaBuilder()
fun properties(action: MetaBuilder.() -> Unit) {
public fun properties(action: MetaBuilder.() -> Unit) {
meta.action()
}
fun plugin(plugin: Plugin) {
public fun plugin(plugin: Plugin) {
plugins.add(plugin)
}
fun plugin(tag: PluginTag, action: MetaBuilder.() -> Unit = {}) {
plugins.add(PluginRepository.fetch(tag, Meta(action)))
public fun plugin(tag: PluginTag, action: MetaBuilder.() -> Unit = {}) {
val factory = parent.resolve<PluginFactory<*>>(PluginFactory.TYPE).values
.find { it.tag.matches(tag) } ?: error("Can't resolve plugin factory for $tag")
val plugin = factory.invoke(Meta(action), parent)
plugins.add(plugin)
}
fun plugin(builder: PluginFactory<*>, action: MetaBuilder.() -> Unit = {}) {
public fun plugin(builder: PluginFactory<*>, action: MetaBuilder.() -> Unit = {}) {
plugins.add(builder.invoke(Meta(action)))
}
fun plugin(name: String, group: String = "", version: String = "", action: MetaBuilder.() -> Unit = {}) {
public fun plugin(name: String, group: String = "", version: String = "", action: MetaBuilder.() -> Unit = {}) {
plugin(PluginTag(name, group, version), action)
}
fun build(): Context {
public fun build(): Context {
return Context(name.toName(), parent).apply {
this@ContextBuilder.plugins.forEach {
plugins.load(it)

43
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Global.kt Normal file
View File

@ -0,0 +1,43 @@
package hep.dataforge.context
import hep.dataforge.names.asName
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.SupervisorJob
import kotlin.coroutines.CoroutineContext
/**
* A global root context. Closing [Global] terminates the framework.
*/
public object Global : Context("GLOBAL".asName(), null) {
override val coroutineContext: CoroutineContext = GlobalScope.coroutineContext + SupervisorJob()
/**
* Closing all contexts
*
* @throws Exception
*/
override fun close() {
logger.info { "Shutting down GLOBAL" }
for (ctx in contextRegistry.values) {
ctx.close()
}
super.close()
}
private val contextRegistry = HashMap<String, Context>()
/**
* Get previously built context
*
* @param name
* @return
*/
public fun getContext(name: String): Context? {
return contextRegistry[name]
}
public fun context(name: String, parent: Context = this, block: ContextBuilder.() -> Unit = {}): Context =
ContextBuilder(parent, name).apply(block).build()
}

11
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt
View File

@ -4,6 +4,17 @@ import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBuilder
import kotlin.reflect.KClass
interface PluginFactory<T : Plugin> : Factory<T> {
val tag: PluginTag
val type: KClass<out T>
companion object{
const val TYPE = "pluginFactory"
}
}
/**
* The manager for plugin system. Should monitor plugin dependencies and locks.
*

48
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt
View File

@ -1,48 +0,0 @@
package hep.dataforge.context
import hep.dataforge.meta.Meta
import kotlin.reflect.KClass
interface PluginFactory<T : Plugin> : Factory<T> {
val tag: PluginTag
val type: KClass<out T>
}
expect object PluginRepository {
fun register(factory: PluginFactory<*>)
/**
* List plugins available in the repository
*/
fun list(): Sequence<PluginFactory<*>>
}
/**
* Fetch specific plugin and instantiate it with given meta
*/
fun PluginRepository.fetch(tag: PluginTag, meta: Meta = Meta.EMPTY): Plugin =
list().find { it.tag.matches(tag) }?.invoke(meta = meta)
?: error("Plugin with tag $tag not found in the repository")
fun <T : Plugin> PluginRepository.register(
tag: PluginTag,
type: KClass<out T>,
constructor: (Context, Meta) -> T
): PluginFactory<T> {
val factory = object : PluginFactory<T> {
override val tag: PluginTag = tag
override val type: KClass<out T> = type
override fun invoke(meta: Meta, context: Context): T = constructor(context, meta)
}
register(factory)
return factory
}
inline fun <reified T : Plugin> PluginRepository.register(tag: PluginTag, noinline constructor: (Context, Meta) -> T) =
register(tag, T::class, constructor)
fun PluginRepository.register(plugin: Plugin) = register(plugin.tag, plugin::class) { _, _ -> plugin }

2
dataforge-context/src/commonTest/kotlin/hep/dataforge/context/ContextTest.kt
View File

@ -22,7 +22,7 @@ class ContextTest {
@Test
fun testPluginManager() {
Global.plugins.load(DummyPlugin())
val members = Global.content<Name>("test")
val members = Global.resolve<Name>("test")
assertEquals(3, members.count())
members.forEach {
assertEquals(it.key, it.value.appendLeft("test"))

16
dataforge-context/src/jsMain/kotlin/hep/dataforge/context/PluginRepository.kt
View File

@ -1,16 +0,0 @@
package hep.dataforge.context
actual object PluginRepository {
private val factories: MutableSet<PluginFactory<*>> = HashSet()
actual fun register(factory: PluginFactory<*>) {
factories.add(factory)
}
/**
* List plugins available in the repository
*/
actual fun list(): Sequence<PluginFactory<*>> = factories.asSequence()
}

17
dataforge-context/src/jvmMain/kotlin/hep/dataforge/context/PluginRepository.kt
View File

@ -1,17 +0,0 @@
package hep.dataforge.context
actual object PluginRepository {
private val factories: MutableSet<PluginFactory<*>> = HashSet()
actual fun register(factory: PluginFactory<*>) {
factories.add(factory)
}
/**
* List plugins available in the repository
*/
actual fun list(): Sequence<PluginFactory<*>> =
factories.asSequence() + Global.services()
}

4
dataforge-context/src/jvmMain/kotlin/hep/dataforge/provider/Types.kt
View File

@ -1,7 +1,7 @@
package hep.dataforge.provider
import hep.dataforge.context.Context
import hep.dataforge.context.content
import hep.dataforge.context.resolve
import hep.dataforge.names.Name
import kotlin.reflect.KClass
import kotlin.reflect.full.findAnnotation
@ -40,5 +40,5 @@ inline fun <reified T : Any> Provider.top(): Map<Name, T> {
/**
* A sequences of all objects provided by plugins with given target and type
*/
inline fun <reified T : Any> Context.content(): Map<Name, T> = content<T>(Types[T::class])
inline fun <reified T : Any> Context.content(): Map<Name, T> = resolve<T>(Types[T::class])

37
dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
View File

@ -13,14 +13,14 @@ import kotlinx.io.text.writeUtf8String
@DFExperimental
class FrontMatterEnvelopeFormat(
val io: IOPlugin,
val meta: Meta = Meta.EMPTY
val meta: Meta = Meta.EMPTY,
) : EnvelopeFormat {
override fun Input.readPartial(): PartialEnvelope {
override fun readPartial(input: Input): PartialEnvelope {
var line = ""
var offset = 0u
do {
line = readUtf8Line() //?: error("Input does not contain front matter separator")
line = input.readUtf8Line() //?: error("Input does not contain front matter separator")
offset += line.toByteArray().size.toUInt()
} while (!line.startsWith(SEPARATOR))
@ -31,22 +31,21 @@ class FrontMatterEnvelopeFormat(
//TODO replace by preview
val meta = Binary {
do {
line = readUtf8Line()
line = input.readUtf8Line()
writeUtf8String(line + "\r\n")
offset += line.toByteArray().size.toUInt()
} while (!line.startsWith(SEPARATOR))
}.read {
readMetaFormat.run {
readMeta()
}
readMetaFormat.readMeta(input)
}
return PartialEnvelope(meta, offset, null)
}
override fun Input.readObject(): Envelope {
override fun readObject(input: Input): Envelope {
var line = ""
do {
line = readUtf8Line() //?: error("Input does not contain front matter separator")
line = input.readUtf8Line() //?: error("Input does not contain front matter separator")
} while (!line.startsWith(SEPARATOR))
val readMetaFormat =
@ -55,14 +54,12 @@ class FrontMatterEnvelopeFormat(
val meta = Binary {
do {
writeUtf8String(readUtf8Line() + "\r\n")
writeUtf8String(input.readUtf8Line() + "\r\n")
} while (!line.startsWith(SEPARATOR))
}.read {
readMetaFormat.run {
readMeta()
readMetaFormat.readMeta(input)
}
}
val bytes = readByteArray()
val bytes = input.readByteArray()
val data = bytes.asBinary()
return SimpleEnvelope(meta, data)
}
@ -70,7 +67,7 @@ class FrontMatterEnvelopeFormat(
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
val metaFormat = metaFormatFactory(formatMeta, io.context)
writeRawString("$SEPARATOR\r\n")
metaFormat.run { writeObject(envelope.meta) }
metaFormat.run { writeObject(this@writeEnvelope, envelope.meta) }
writeRawString("$SEPARATOR\r\n")
//Printing data
envelope.data?.let { data ->
@ -105,14 +102,14 @@ class FrontMatterEnvelopeFormat(
private val default by lazy { invoke() }
override fun Input.readPartial(): PartialEnvelope =
default.run { readPartial() }
override fun readPartial(input: Input): PartialEnvelope =
default.readPartial(input)
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) =
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta): Unit =
default.run { writeEnvelope(envelope, metaFormatFactory, formatMeta) }
override fun Input.readObject(): Envelope =
default.run { readObject() }
override fun readObject(input: Input): Envelope =
default.readObject(input)
}
}

25
dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt
View File

@ -16,17 +16,20 @@ import kotlinx.io.asInputStream
import kotlinx.io.text.writeUtf8String
import org.yaml.snakeyaml.Yaml
/**
* Represent meta as Yaml
*/
@DFExperimental
class YamlMetaFormat(val meta: Meta) : MetaFormat {
public class YamlMetaFormat(private val meta: Meta) : MetaFormat {
private val yaml = Yaml()
override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) {
override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?) {
val string = yaml.dump(meta.toMap(descriptor))
writeUtf8String(string)
output.writeUtf8String(string)
}
override fun Input.readMeta(descriptor: NodeDescriptor?): Meta {
val map: Map<String, Any?> = yaml.load(asInputStream())
override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta {
val map: Map<String, Any?> = yaml.load(input.asInputStream())
return map.toMeta(descriptor)
}
@ -35,19 +38,19 @@ class YamlMetaFormat(val meta: Meta) : MetaFormat {
META_KEY put meta
}
companion object : MetaFormatFactory {
public companion object : MetaFormatFactory {
override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta)
override val shortName = "yaml"
override val shortName: String = "yaml"
override val key: Short = 0x594d //YM
private val default = YamlMetaFormat()
override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) =
default.run { writeMeta(meta, descriptor) }
override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?): Unit =
default.writeMeta(output, meta, descriptor)
override fun Input.readMeta(descriptor: NodeDescriptor?): Meta =
default.run { readMeta(descriptor) }
override fun readMeta(input: kotlinx.io.Input, descriptor: NodeDescriptor?): Meta =
default.readMeta(input, descriptor)
}
}

30
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/BinaryMetaFormat.kt
View File

@ -11,14 +11,18 @@ import kotlinx.io.*
import kotlinx.io.text.readUtf8String
import kotlinx.io.text.writeUtf8String
object BinaryMetaFormat : MetaFormat, MetaFormatFactory {
/**
* A DataForge-specific simplified binary format for meta
* TODO add description
*/
public object BinaryMetaFormat : MetaFormat, MetaFormatFactory {
override val shortName: String = "bin"
override val key: Short = 0x4249//BI
override fun invoke(meta: Meta, context: Context): MetaFormat = this
override fun Input.readMeta(descriptor: NodeDescriptor?): Meta {
return (readMetaItem() as MetaItem.NodeItem).node
override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta {
return (input.readMetaItem() as MetaItem.NodeItem).node
}
private fun Output.writeChar(char: Char) = writeByte(char.toByte())
@ -28,7 +32,7 @@ object BinaryMetaFormat : MetaFormat, MetaFormatFactory {
writeUtf8String(str)
}
fun Output.writeValue(value: Value) {
public fun Output.writeValue(value: Value) {
if (value.isList()) {
writeChar('L')
writeInt(value.list.size)
@ -75,17 +79,21 @@ object BinaryMetaFormat : MetaFormat, MetaFormatFactory {
}
}
override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) {
writeChar('M')
writeInt(meta.items.size)
override fun writeMeta(
output: kotlinx.io.Output,
meta: hep.dataforge.meta.Meta,
descriptor: hep.dataforge.meta.descriptors.NodeDescriptor?
) {
output.writeChar('M')
output.writeInt(meta.items.size)
meta.items.forEach { (key, item) ->
writeString(key.toString())
output.writeString(key.toString())
when (item) {
is MetaItem.ValueItem -> {
writeValue(item.value)
output.writeValue(item.value)
}
is MetaItem.NodeItem -> {
writeObject(item.node)
writeObject(output, item.node)
}
}
}
@ -97,7 +105,7 @@ object BinaryMetaFormat : MetaFormat, MetaFormatFactory {
}
@Suppress("UNCHECKED_CAST")
fun Input.readMetaItem(): MetaItem<MetaBuilder> {
public fun Input.readMetaItem(): MetaItem<MetaBuilder> {
return when (val keyChar = readByte().toChar()) {
'S' -> MetaItem.ValueItem(StringValue(readString()))
'N' -> MetaItem.ValueItem(Null)

44
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt
View File

@ -4,64 +4,66 @@ import hep.dataforge.meta.Laminate
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.string
import hep.dataforge.names.Name
import hep.dataforge.names.asName
import hep.dataforge.names.plus
import kotlinx.io.Binary
interface Envelope {
val meta: Meta
val data: Binary?
public interface Envelope {
public val meta: Meta
public val data: Binary?
companion object {
public companion object {
/**
* meta keys
*/
val ENVELOPE_NODE_KEY = "@envelope".asName()
val ENVELOPE_TYPE_KEY = ENVELOPE_NODE_KEY + "type"
val ENVELOPE_DATA_TYPE_KEY = ENVELOPE_NODE_KEY + "dataType"
val ENVELOPE_DATA_ID_KEY = ENVELOPE_NODE_KEY + "dataID"
val ENVELOPE_DESCRIPTION_KEY = ENVELOPE_NODE_KEY + "description"
val ENVELOPE_NAME_KEY = ENVELOPE_NODE_KEY + "name"
public val ENVELOPE_NODE_KEY: Name = "@envelope".asName()
public val ENVELOPE_TYPE_KEY: Name = ENVELOPE_NODE_KEY + "type"
public val ENVELOPE_DATA_TYPE_KEY: Name = ENVELOPE_NODE_KEY + "dataType"
public val ENVELOPE_DATA_ID_KEY: Name = ENVELOPE_NODE_KEY + "dataID"
public val ENVELOPE_DESCRIPTION_KEY: Name = ENVELOPE_NODE_KEY + "description"
public val ENVELOPE_NAME_KEY: Name = ENVELOPE_NODE_KEY + "name"
//const val ENVELOPE_TIME_KEY = "@envelope.time"
/**
* Build a static envelope using provided builder
*/
inline operator fun invoke(block: EnvelopeBuilder.() -> Unit) = EnvelopeBuilder().apply(block).seal()
public inline operator fun invoke(block: EnvelopeBuilder.() -> Unit): Envelope =
EnvelopeBuilder().apply(block).seal()
}
}
class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope
public class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope
/**
* The purpose of the envelope
*
*/
val Envelope.type: String? get() = meta[Envelope.ENVELOPE_TYPE_KEY].string
public val Envelope.type: String? get() = meta[Envelope.ENVELOPE_TYPE_KEY].string
/**
* The type of data encoding
*
*/
val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].string
public val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].string
/**
* Textual user friendly description
*
*/
val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string
public val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string
/**
* An optional unique identifier that is used for data comparison. Data without identifier could not be compared to another data.
*/
val Envelope.dataID: String? get() = meta[Envelope.ENVELOPE_DATA_ID_KEY].string
public val Envelope.dataID: String? get() = meta[Envelope.ENVELOPE_DATA_ID_KEY].string
fun Envelope.metaEquals(other: Envelope): Boolean = this.meta == other.meta
public fun Envelope.metaEquals(other: Envelope): Boolean = this.meta == other.meta
fun Envelope.dataEquals(other: Envelope): Boolean = this.dataID != null && this.dataID == other.dataID
public fun Envelope.dataEquals(other: Envelope): Boolean = this.dataID != null && this.dataID == other.dataID
fun Envelope.contentEquals(other: Envelope): Boolean {
public fun Envelope.contentEquals(other: Envelope): Boolean {
return (this === other || (metaEquals(other) && dataEquals(other)))
}
@ -69,7 +71,7 @@ fun Envelope.contentEquals(other: Envelope): Boolean {
/**
* An envelope, which wraps existing envelope and adds one or several additional layers of meta
*/
class ProxyEnvelope(val source: Envelope, vararg meta: Meta) : Envelope {
public class ProxyEnvelope(public val source: Envelope, vararg meta: Meta) : Envelope {
override val meta: Laminate = Laminate(*meta, source.meta)
override val data: Binary? get() = source.data
}
@ -77,7 +79,7 @@ class ProxyEnvelope(val source: Envelope, vararg meta: Meta) : Envelope {
/**
* Add few meta layers to existing envelope (on top of existing meta)
*/
fun Envelope.withMetaLayers(vararg layers: Meta): Envelope {
public fun Envelope.withMetaLayers(vararg layers: Meta): Envelope {
return when {
layers.isEmpty() -> this
this is ProxyEnvelope -> ProxyEnvelope(source, *layers, *this.meta.layers.toTypedArray())

27
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt
View File

@ -13,31 +13,28 @@ import kotlin.reflect.KClass
/**
* A partially read envelope with meta, but without data
*/
@ExperimentalUnsignedTypes
data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
public data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
interface EnvelopeFormat : IOFormat<Envelope> {
val defaultMetaFormat: MetaFormatFactory get() = JsonMetaFormat
public interface EnvelopeFormat : IOFormat<Envelope> {
public val defaultMetaFormat: MetaFormatFactory get() = JsonMetaFormat
fun Input.readPartial(): PartialEnvelope
public fun readPartial(input: Input): PartialEnvelope
fun Output.writeEnvelope(
public fun Output.writeEnvelope(
envelope: Envelope,
metaFormatFactory: MetaFormatFactory = defaultMetaFormat,
formatMeta: Meta = Meta.EMPTY
)
override fun Input.readObject(): Envelope
override fun readObject(input: Input): Envelope
override fun Output.writeObject(obj: Envelope): Unit = writeEnvelope(obj)
override fun writeObject(output: Output, obj: Envelope): Unit = output.writeEnvelope(obj)
}
fun EnvelopeFormat.readPartial(input: Input) = input.readPartial()
fun EnvelopeFormat.read(input: Input) = input.readObject()
public fun EnvelopeFormat.read(input: Input): Envelope = readObject(input)
@Type(ENVELOPE_FORMAT_TYPE)
interface EnvelopeFormatFactory : IOFormatFactory<Envelope>, EnvelopeFormat {
public interface EnvelopeFormatFactory : IOFormatFactory<Envelope>, EnvelopeFormat {
override val name: Name get() = "envelope".asName()
override val type: KClass<out Envelope> get() = Envelope::class
@ -47,9 +44,9 @@ interface EnvelopeFormatFactory : IOFormatFactory<Envelope>, EnvelopeFormat {
* Try to infer specific format from input and return null if the attempt is failed.
* This method does **not** return Input into initial state.
*/
fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat?
public fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat?
companion object {
const val ENVELOPE_FORMAT_TYPE = "io.format.envelope"
public companion object {
public const val ENVELOPE_FORMAT_TYPE: String = "io.format.envelope"
}
}

16
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt
View File

@ -31,11 +31,11 @@ private class PartDescriptor : Scheme() {
}
}
data class EnvelopePart(val binary: Binary, val description: Meta?)
public data class EnvelopePart(val binary: Binary, val description: Meta?)
typealias EnvelopeParts = List<EnvelopePart>
public typealias EnvelopeParts = List<EnvelopePart>
fun EnvelopeBuilder.multipart(
public fun EnvelopeBuilder.multipart(
parts: EnvelopeParts,
separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR
) {
@ -69,7 +69,7 @@ fun EnvelopeBuilder.multipart(
}
}
fun EnvelopeBuilder.envelopes(
public fun EnvelopeBuilder.envelopes(
envelopes: List<Envelope>,
format: EnvelopeFormat = TaggedEnvelopeFormat,
separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR
@ -84,7 +84,7 @@ fun EnvelopeBuilder.envelopes(
multipart(parts, separator)
}
fun Envelope.parts(): EnvelopeParts {
public fun Envelope.parts(): EnvelopeParts {
if (data == null) return emptyList()
//TODO add zip folder reader
val parts = meta.getIndexed(PARTS_KEY).values.mapNotNull { it.node }.map {
@ -101,14 +101,14 @@ fun Envelope.parts(): EnvelopeParts {
}
}
fun EnvelopePart.envelope(format: EnvelopeFormat): Envelope = binary.readWith(format)
public fun EnvelopePart.envelope(format: EnvelopeFormat): Envelope = binary.readWith(format)
val EnvelopePart.name: String? get() = description?.get("name").string
public val EnvelopePart.name: String? get() = description?.get("name").string
/**
* Represent envelope part by an envelope
*/
fun EnvelopePart.envelope(plugin: IOPlugin): Envelope {
public fun EnvelopePart.envelope(plugin: IOPlugin): Envelope {
val formatItem = description?.get(PART_FORMAT_KEY)
return if (formatItem != null) {
val format: EnvelopeFormat = plugin.resolveEnvelopeFormat(formatItem)

67
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt
View File

@ -20,41 +20,42 @@ import kotlin.reflect.KClass
/**
* And interface for reading and writing objects into with IO streams
*/
interface IOFormat<T : Any> : MetaRepr {
fun Output.writeObject(obj: T)
fun Input.readObject(): T
public interface IOFormat<T : Any> : MetaRepr {
public fun writeObject(output: Output, obj: T)
public fun readObject(input: Input): T
companion object{
val NAME_KEY = "name".asName()
val META_KEY = "meta".asName()
public companion object {
public val NAME_KEY: Name = "name".asName()
public val META_KEY: Name = "meta".asName()
}
}
fun <T : Any> Input.readWith(format: IOFormat<T>): T = format.run { readObject() }
public fun <T : Any> Input.readWith(format: IOFormat<T>): T = format.run { readObject(this@readWith) }
/**
* Read given binary as object using given format
*/
fun <T : Any> Binary.readWith(format: IOFormat<T>): T = read {
public fun <T : Any> Binary.readWith(format: IOFormat<T>): T = read {
readWith(format)
}
fun <T : Any> Output.writeWith(format: IOFormat<T>, obj: T) = format.run { writeObject(obj) }
public fun <T : Any> Output.writeWith(format: IOFormat<T>, obj: T): Unit =
format.run { writeObject(this@writeWith, obj) }
class ListIOFormat<T : Any>(val format: IOFormat<T>) : IOFormat<List<T>> {
override fun Output.writeObject(obj: List<T>) {
writeInt(obj.size)
format.run {
public class ListIOFormat<T : Any>(public val format: IOFormat<T>) : IOFormat<List<T>> {
override fun writeObject(output: Output, obj: List<T>) {
output.writeInt(obj.size)
this.format.run {
obj.forEach {
writeObject(it)
writeObject(output, it)
}
}
}
override fun Input.readObject(): List<T> {
val size = readInt()
override fun readObject(input: Input): List<T> {
val size = input.readInt()
return format.run {
List(size) { readObject() }
List(size) { readObject(input) }
}
}
@ -64,9 +65,9 @@ class ListIOFormat<T : Any>(val format: IOFormat<T>) : IOFormat<List<T>> {
}
}
val <T : Any> IOFormat<T>.list get() = ListIOFormat(this)
//public val <T : Any> IOFormat<T>.list: ListIOFormat<T> get() = ListIOFormat(this)
fun ObjectPool<Buffer>.fill(block: Buffer.() -> Unit): Buffer {
public fun ObjectPool<Buffer>.fill(block: Buffer.() -> Unit): Buffer {
val buffer = borrow()
return try {
buffer.apply(block)
@ -77,50 +78,50 @@ fun ObjectPool<Buffer>.fill(block: Buffer.() -> Unit): Buffer {
}
@Type(IO_FORMAT_TYPE)
interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named, MetaRepr {
public interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named, MetaRepr {
/**
* Explicit type for dynamic type checks
*/
val type: KClass<out T>
public val type: KClass<out T>
override fun toMeta(): Meta = Meta {
NAME_KEY put name.toString()
}
companion object {
const val IO_FORMAT_TYPE = "io.format"
public companion object {
public const val IO_FORMAT_TYPE: String = "io.format"
}
}
fun <T : Any> IOFormat<T>.toBinary(obj: T): Binary = Binary { writeObject(obj) }
public fun <T : Any> IOFormat<T>.toBinary(obj: T): Binary = Binary { writeObject(this, obj) }
object DoubleIOFormat : IOFormat<Double>, IOFormatFactory<Double> {
public object DoubleIOFormat : IOFormat<Double>, IOFormatFactory<Double> {
override fun invoke(meta: Meta, context: Context): IOFormat<Double> = this
override val name: Name = "double".asName()
override val type: KClass<out Double> get() = Double::class
override fun Output.writeObject(obj: Double) {
writeDouble(obj)
override fun writeObject(output: Output, obj: kotlin.Double) {
output.writeDouble(obj)
}
override fun Input.readObject(): Double = readDouble()
override fun readObject(input: Input): Double = input.readDouble()
}
object ValueIOFormat : IOFormat<Value>, IOFormatFactory<Value> {
public object ValueIOFormat : IOFormat<Value>, IOFormatFactory<Value> {
override fun invoke(meta: Meta, context: Context): IOFormat<Value> = this
override val name: Name = "value".asName()
override val type: KClass<out Value> get() = Value::class
override fun Output.writeObject(obj: Value) {
BinaryMetaFormat.run { writeValue(obj) }
override fun writeObject(output: Output, obj: Value) {
BinaryMetaFormat.run { output.writeValue(obj) }
}
override fun Input.readObject(): Value {
return (BinaryMetaFormat.run { readMetaItem() } as? MetaItem.ValueItem)?.value
override fun readObject(input: Input): Value {
return (BinaryMetaFormat.run { input.readMetaItem() } as? MetaItem.ValueItem)?.value
?: error("The item is not a value")
}
}

32
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt
View File

@ -11,14 +11,14 @@ import hep.dataforge.names.Name
import hep.dataforge.names.toName
import kotlin.reflect.KClass
class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
public class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
override val tag: PluginTag get() = Companion.tag
val ioFormatFactories by lazy {
context.content<IOFormatFactory<*>>(IO_FORMAT_TYPE).values
public val ioFormatFactories: Collection<IOFormatFactory<*>> by lazy {
context.resolve<IOFormatFactory<*>>(IO_FORMAT_TYPE).values
}
fun <T : Any> resolveIOFormat(item: MetaItem<*>, type: KClass<out T>): IOFormat<T>? {
public fun <T : Any> resolveIOFormat(item: MetaItem<*>, type: KClass<out T>): IOFormat<T>? {
val key = item.string ?: item.node[NAME_KEY]?.string ?: error("Format name not defined")
val name = key.toName()
return ioFormatFactories.find { it.name == name }?.let {
@ -29,24 +29,24 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
}
val metaFormatFactories by lazy {
context.content<MetaFormatFactory>(META_FORMAT_TYPE).values
public val metaFormatFactories: Collection<MetaFormatFactory> by lazy {
context.resolve<MetaFormatFactory>(META_FORMAT_TYPE).values
}
fun resolveMetaFormat(key: Short, meta: Meta = Meta.EMPTY): MetaFormat? =
public fun resolveMetaFormat(key: Short, meta: Meta = Meta.EMPTY): MetaFormat? =
metaFormatFactories.find { it.key == key }?.invoke(meta)
fun resolveMetaFormat(name: String, meta: Meta = Meta.EMPTY): MetaFormat? =
public fun resolveMetaFormat(name: String, meta: Meta = Meta.EMPTY): MetaFormat? =
metaFormatFactories.find { it.shortName == name }?.invoke(meta)
val envelopeFormatFactories by lazy {
context.content<EnvelopeFormatFactory>(ENVELOPE_FORMAT_TYPE).values
public val envelopeFormatFactories: Collection<EnvelopeFormatFactory> by lazy {
context.resolve<EnvelopeFormatFactory>(ENVELOPE_FORMAT_TYPE).values
}
fun resolveEnvelopeFormat(name: Name, meta: Meta = Meta.EMPTY): EnvelopeFormat? =
private fun resolveEnvelopeFormat(name: Name, meta: Meta = Meta.EMPTY): EnvelopeFormat? =
envelopeFormatFactories.find { it.name == name }?.invoke(meta, context)
fun resolveEnvelopeFormat(item: MetaItem<*>): EnvelopeFormat? {
public fun resolveEnvelopeFormat(item: MetaItem<*>): EnvelopeFormat? {
val name = item.string ?: item.node[NAME_KEY]?.string ?: error("Envelope format name not defined")
val meta = item.node[META_KEY].node ?: Meta.EMPTY
return resolveEnvelopeFormat(name.toName(), meta)
@ -60,9 +60,9 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
}
}
companion object : PluginFactory<IOPlugin> {
val defaultMetaFormats: List<MetaFormatFactory> = listOf(JsonMetaFormat, BinaryMetaFormat)
val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat, TaglessEnvelopeFormat)
public companion object : PluginFactory<IOPlugin> {
public val defaultMetaFormats: List<MetaFormatFactory> = listOf(JsonMetaFormat, BinaryMetaFormat)
public val defaultEnvelopeFormats: List<EnvelopeFormatFactory> = listOf(TaggedEnvelopeFormat, TaglessEnvelopeFormat)
override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP)
@ -71,4 +71,4 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
}
}
val Context.io: IOPlugin get() = plugins.fetch(IOPlugin)
public val Context.io: IOPlugin get() = plugins.fetch(IOPlugin)

28
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt
View File

@ -17,39 +17,41 @@ import kotlinx.io.text.writeUtf8String
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
/**
* A Json format for Meta representation
*/
public class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat {
class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat {
override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) {
override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?) {
val jsonObject = meta.toJson(descriptor)
writeUtf8String(json.encodeToString(JsonObject.serializer(), jsonObject))
output.writeUtf8String(this.json.encodeToString(JsonObject.serializer(), jsonObject))
}
override fun toMeta(): Meta = Meta {
NAME_KEY put name.toString()
}
override fun Input.readMeta(descriptor: NodeDescriptor?): Meta {
val str = readByteArray().decodeToString()
override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta {
val str = input.readByteArray().decodeToString()
val jsonElement = json.parseToJsonElement(str)
val item = jsonElement.toMetaItem(descriptor)
return item.node ?: Meta.EMPTY
}
companion object : MetaFormatFactory {
val DEFAULT_JSON = Json { prettyPrint = true }
public companion object : MetaFormatFactory {
public val DEFAULT_JSON: Json = Json { prettyPrint = true }
override fun invoke(meta: Meta, context: Context): MetaFormat = default
override val shortName = "json"
override val shortName: String = "json"
override val key: Short = 0x4a53//"JS"
private val default = JsonMetaFormat()
override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) =
default.run { writeMeta(meta, descriptor) }
override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?): Unit =
default.run { writeMeta(output, meta, descriptor) }
override fun Input.readMeta(descriptor: NodeDescriptor?): Meta =
default.run { readMeta(descriptor) }
override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta =
default.run { readMeta(input, descriptor) }
}
}

40
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt
View File

@ -17,46 +17,50 @@ import kotlin.reflect.KClass
/**
* A format for meta serialization
*/
public interface MetaFormat : IOFormat<Meta> {
interface MetaFormat : IOFormat<Meta> {
override fun Output.writeObject(obj: Meta) {
writeMeta(obj, null)
override fun writeObject(output: Output, obj: Meta) {
writeMeta(output, obj, null)
}
override fun Input.readObject(): Meta = readMeta()
override fun readObject(input: Input): Meta = readMeta(input)
fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor? = null)
fun Input.readMeta(descriptor: NodeDescriptor? = null): Meta
public fun writeMeta(
output: Output,
meta: Meta,
descriptor: NodeDescriptor? = null,
)
public fun readMeta(input: Input, descriptor: NodeDescriptor? = null): Meta
}
@Type(META_FORMAT_TYPE)
interface MetaFormatFactory : IOFormatFactory<Meta>, MetaFormat {
val shortName: String
public interface MetaFormatFactory : IOFormatFactory<Meta>, MetaFormat {
public val shortName: String
override val name: Name get() = "meta".asName() + shortName
override val type: KClass<out Meta> get() = Meta::class
val key: Short get() = name.hashCode().toShort()
public val key: Short get() = name.hashCode().toShort()
override operator fun invoke(meta: Meta, context: Context): MetaFormat
companion object {
const val META_FORMAT_TYPE = "io.format.meta"
public companion object {
public const val META_FORMAT_TYPE: String = "io.format.meta"
}
}
fun Meta.toString(format: MetaFormat): String = buildByteArray {
format.run { writeObject(this@toString) }
public fun Meta.toString(format: MetaFormat): String = buildByteArray {
format.run { writeObject(this@buildByteArray, this@toString) }
}.decodeToString()
fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory())
public fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory())
fun MetaFormat.parse(str: String): Meta {
return ByteArrayInput(str.encodeToByteArray()).use { it.readObject() }
public fun MetaFormat.parse(str: String): Meta {
return ByteArrayInput(str.encodeToByteArray()).use { readObject(it) }
}
fun MetaFormatFactory.parse(str: String, formatMeta: Meta): Meta = invoke(formatMeta).parse(str)
public fun MetaFormatFactory.parse(str: String, formatMeta: Meta): Meta = invoke(formatMeta).parse(str)

42
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt
View File

@ -12,9 +12,13 @@ import hep.dataforge.names.plus
import hep.dataforge.names.toName
import kotlinx.io.*
class TaggedEnvelopeFormat(
val io: IOPlugin,
val version: VERSION = VERSION.DF02
/**
* A streaming-friendly envelope format with a short binary tag.
* TODO add description
*/
public class TaggedEnvelopeFormat(
public val io: IOPlugin,
public val version: VERSION = VERSION.DF02
) : EnvelopeFormat {
// private val metaFormat = io.metaFormat(metaFormatKey)
@ -57,32 +61,32 @@ class TaggedEnvelopeFormat(
* @param input an input to read from
* @param formats a collection of meta formats to resolve
*/
override fun Input.readObject(): Envelope {
val tag = readTag(version)
override fun readObject(input: Input): Envelope {
val tag = input.readTag(this.version)
val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
?: error("Meta format with key ${tag.metaFormatKey} not found")
val meta: Meta = limit(tag.metaSize.toInt()).run {
val meta: Meta = input.limit(tag.metaSize.toInt()).run {
metaFormat.run {
readObject()
readObject(input)
}
}
val data = readBinary(tag.dataSize.toInt())
val data = input.readBinary(tag.dataSize.toInt())
return SimpleEnvelope(meta, data)
}
override fun Input.readPartial(): PartialEnvelope {
val tag = readTag(version)
override fun readPartial(input: Input): PartialEnvelope {
val tag = input.readTag(this.version)
val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
?: error("Meta format with key ${tag.metaFormatKey} not found")
val meta: Meta = limit(tag.metaSize.toInt()).run {
val meta: Meta = input.limit(tag.metaSize.toInt()).run {
metaFormat.run {
readObject()
readObject(input)
}
}
@ -95,7 +99,7 @@ class TaggedEnvelopeFormat(
val dataSize: ULong
)
enum class VERSION(val tagSize: UInt) {
public enum class VERSION(public val tagSize: UInt) {
DF02(20u),
DF03(24u)
}
@ -107,7 +111,7 @@ class TaggedEnvelopeFormat(
}
}
companion object : EnvelopeFormatFactory {
public companion object : EnvelopeFormatFactory {
private const val START_SEQUENCE = "#~"
private const val END_SEQUENCE = "~#\r\n"
@ -158,14 +162,14 @@ class TaggedEnvelopeFormat(
private val default by lazy { invoke() }
override fun Input.readPartial(): PartialEnvelope =
default.run { readPartial() }
override fun readPartial(input: Input): PartialEnvelope =
default.run { readPartial(input) }
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) =
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta): Unit =
default.run { writeEnvelope(envelope, metaFormatFactory, formatMeta) }
override fun Input.readObject(): Envelope =
default.run { readObject() }
override fun readObject(input: Input): Envelope =
default.run { readObject(input) }
}

77
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt
View File

@ -7,15 +7,20 @@ import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.isEmpty
import hep.dataforge.meta.string
import hep.dataforge.names.Name
import hep.dataforge.names.asName
import kotlinx.io.*
import kotlinx.io.text.readUtf8Line
import kotlinx.io.text.writeUtf8String
import kotlin.collections.set
class TaglessEnvelopeFormat(
val io: IOPlugin,
val meta: Meta = Meta.EMPTY
/**
* A text envelope format with human-readable tag.
* TODO add description
*/
public class TaglessEnvelopeFormat(
public val io: IOPlugin,
public val meta: Meta = Meta.EMPTY
) : EnvelopeFormat {
private val metaStart = meta[META_START_PROPERTY].string ?: DEFAULT_META_START
@ -54,10 +59,10 @@ class TaglessEnvelopeFormat(
}
}
override fun Input.readObject(): Envelope {
override fun readObject(input: Input): Envelope {
var line: String
do {
line = readUtf8Line() // ?: error("Input does not contain tagless envelope header")
line = input.readUtf8Line() // ?: error("Input does not contain tagless envelope header")
} while (!line.startsWith(TAGLESS_ENVELOPE_HEADER))
val properties = HashMap<String, String>()
@ -70,8 +75,8 @@ class TaglessEnvelopeFormat(
properties[key] = value
}
//If can't read line, return envelope without data
if (exhausted()) return SimpleEnvelope(Meta.EMPTY, null)
line = readUtf8Line()
if (input.exhausted()) return SimpleEnvelope(Meta.EMPTY, null)
line = input.readUtf8Line()
}
var meta: Meta = Meta.EMPTY
@ -80,19 +85,19 @@ class TaglessEnvelopeFormat(
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat
val metaSize = properties[META_LENGTH_PROPERTY]?.toInt()
meta = if (metaSize != null) {
limit(metaSize).run {
metaFormat.run { readObject() }
input.limit(metaSize).run {
metaFormat.run { readObject(input) }
}
} else {
metaFormat.run {
readObject()
readObject(input)
}
}
}
do {
try {
line = readUtf8Line()
line = input.readUtf8Line()
} catch (ex: EOFException) {
//returning an Envelope without data if end of input is reached
return SimpleEnvelope(meta, null)
@ -100,24 +105,24 @@ class TaglessEnvelopeFormat(
} while (!line.startsWith(dataStart))
val data: Binary? = if (properties.containsKey(DATA_LENGTH_PROPERTY)) {
readBinary(properties[DATA_LENGTH_PROPERTY]!!.toInt())
input.readBinary(properties[DATA_LENGTH_PROPERTY]!!.toInt())
// val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt())
// readByteArray(bytes)
// bytes.asBinary()
} else {
Binary {
copyTo(this)
input.copyTo(this)
}
}
return SimpleEnvelope(meta, data)
}
override fun Input.readPartial(): PartialEnvelope {
override fun readPartial(input: Input): PartialEnvelope {
var offset = 0u
var line: String
do {
line = readUtf8Line()// ?: error("Input does not contain tagless envelope header")
line = input.readUtf8Line()// ?: error("Input does not contain tagless envelope header")
offset += line.encodeToByteArray().size.toUInt()
} while (!line.startsWith(TAGLESS_ENVELOPE_HEADER))
val properties = HashMap<String, String>()
@ -131,7 +136,7 @@ class TaglessEnvelopeFormat(
properties[key] = value
}
try {
line = readUtf8Line()
line = input.readUtf8Line()
offset += line.encodeToByteArray().size.toUInt()
} catch (ex: EOFException) {
return PartialEnvelope(Meta.EMPTY, offset.toUInt(), 0.toULong())
@ -145,8 +150,8 @@ class TaglessEnvelopeFormat(
val metaSize = properties[META_LENGTH_PROPERTY]?.toInt()
meta = if (metaSize != null) {
offset += metaSize.toUInt()
limit(metaSize).run {
metaFormat.run { readObject() }
input.limit(metaSize).run {
metaFormat.run { readObject(input) }
}
} else {
error("Can't partially read an envelope with undefined meta size")
@ -154,7 +159,7 @@ class TaglessEnvelopeFormat(
}
do {
line = readUtf8Line() //?: return PartialEnvelope(Meta.EMPTY, offset.toUInt(), 0.toULong())
line = input.readUtf8Line() //?: return PartialEnvelope(Meta.EMPTY, offset.toUInt(), 0.toULong())
offset += line.encodeToByteArray().size.toUInt()
//returning an Envelope without data if end of input is reached
} while (!line.startsWith(dataStart))
@ -168,26 +173,26 @@ class TaglessEnvelopeFormat(
META_KEY put meta
}
companion object : EnvelopeFormatFactory {
public companion object : EnvelopeFormatFactory {
private val propertyPattern = "#\\?\\s*(?<key>[\\w.]*)\\s*:\\s*(?<value>[^;]*);?".toRegex()
const val META_TYPE_PROPERTY = "metaType"
const val META_LENGTH_PROPERTY = "metaLength"
const val DATA_LENGTH_PROPERTY = "dataLength"
public const val META_TYPE_PROPERTY: String = "metaType"
public const val META_LENGTH_PROPERTY: String = "metaLength"
public const val DATA_LENGTH_PROPERTY: String = "dataLength"
const val TAGLESS_ENVELOPE_TYPE = "tagless"
public const val TAGLESS_ENVELOPE_TYPE: String = "tagless"
const val TAGLESS_ENVELOPE_HEADER = "#~DFTL~#"
const val META_START_PROPERTY = "metaSeparator"
const val DEFAULT_META_START = "#~META~#"
const val DATA_START_PROPERTY = "dataSeparator"
const val DEFAULT_DATA_START = "#~DATA~#"
public const val TAGLESS_ENVELOPE_HEADER: String = "#~DFTL~#"
public const val META_START_PROPERTY: String = "metaSeparator"
public const val DEFAULT_META_START: String = "#~META~#"
public const val DATA_START_PROPERTY: String = "dataSeparator"
public const val DEFAULT_DATA_START: String = "#~DATA~#"
const val code: Int = 0x4446544c //DFTL
public const val code: Int = 0x4446544c //DFTL
override val name = TAGLESS_ENVELOPE_TYPE.asName()
override val name: Name = TAGLESS_ENVELOPE_TYPE.asName()
override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
return TaglessEnvelopeFormat(context.io, meta)
@ -195,14 +200,14 @@ class TaglessEnvelopeFormat(
private val default by lazy { invoke() }
override fun Input.readPartial(): PartialEnvelope =
default.run { readPartial() }
override fun readPartial(input: Input): PartialEnvelope =
default.run { readPartial(input) }
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) =
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta): Unit =
default.run { writeEnvelope(envelope, metaFormatFactory, formatMeta) }
override fun Input.readObject(): Envelope =
default.run { readObject() }
override fun readObject(input: Input): Envelope =
default.run { readObject(input) }
override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
return try {

4
dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt
View File

@ -8,11 +8,11 @@ import kotlin.test.Test
import kotlin.test.assertEquals
fun Meta.toByteArray(format: MetaFormat = JsonMetaFormat) = buildByteArray {
format.run { writeObject(this@toByteArray) }
format.writeObject(this@buildByteArray, this@toByteArray)
}
fun MetaFormat.fromByteArray(packet: ByteArray): Meta {
return packet.asBinary().read { readObject() }
return packet.asBinary().read { readObject(this) }
}
class MetaFormatTest {

4
dataforge-io/src/commonTest/kotlin/hep/dataforge/io/ioTestUtils.kt
View File

@ -3,5 +3,5 @@ package hep.dataforge.io
import kotlinx.io.ByteArrayInput
import kotlinx.io.use
fun <T : Any> IOFormat<T>.toByteArray(obj: T): ByteArray = buildByteArray { writeObject(obj) }
fun <T : Any> IOFormat<T>.readByteArray(array: ByteArray): T = ByteArrayInput(array).use { it.readObject() }
fun <T : Any> IOFormat<T>.toByteArray(obj: T): ByteArray = buildByteArray { writeObject(this, obj) }
fun <T : Any> IOFormat<T>.readByteArray(array: ByteArray): T = ByteArrayInput(array).use { readObject(it) }

36
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt
View File

@ -11,12 +11,12 @@ import java.nio.file.StandardOpenOption
import kotlin.reflect.full.isSuperclassOf
import kotlin.streams.asSequence
fun <R> Path.read(block: Input.() -> R): R = asBinary().read(block = block)
public fun <R> Path.read(block: Input.() -> R): R = asBinary().read(block = block)
/**
* Write a live output to a newly created file. If file does not exist, throws error
*/
fun Path.write(block: Output.() -> Unit): Unit {
public fun Path.write(block: Output.() -> Unit): Unit {
val stream = Files.newOutputStream(this, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
stream.asOutput().use(block)
}
@ -24,7 +24,7 @@ fun Path.write(block: Output.() -> Unit): Unit {
/**
* Create a new file or append to exiting one with given output [block]
*/
fun Path.append(block: Output.() -> Unit): Unit {
public fun Path.append(block: Output.() -> Unit): Unit {
val stream = Files.newOutputStream(
this,
StandardOpenOption.WRITE, StandardOpenOption.APPEND, StandardOpenOption.CREATE
@ -35,7 +35,7 @@ fun Path.append(block: Output.() -> Unit): Unit {
/**
* Create a new file or replace existing one using given output [block]
*/
fun Path.rewrite(block: Output.() -> Unit): Unit {
public fun Path.rewrite(block: Output.() -> Unit): Unit {
val stream = Files.newOutputStream(
this,
StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE
@ -43,9 +43,9 @@ fun Path.rewrite(block: Output.() -> Unit): Unit {
stream.asOutput().use(block)
}
fun Path.readEnvelope(format: EnvelopeFormat): Envelope {
public fun Path.readEnvelope(format: EnvelopeFormat): Envelope {
val partialEnvelope: PartialEnvelope = asBinary().read {
format.run { readPartial() }
format.run { readPartial(this@read) }
}
val offset: Int = partialEnvelope.dataOffset.toInt()
val size: Int = partialEnvelope.dataSize?.toInt() ?: (Files.size(this).toInt() - offset)
@ -58,7 +58,7 @@ fun Path.readEnvelope(format: EnvelopeFormat): Envelope {
*/
@Suppress("UNCHECKED_CAST")
@DFExperimental
inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? {
public inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? {
return ioFormatFactories.find { it.type.isSuperclassOf(T::class) } as IOFormat<T>?
}
@ -66,7 +66,7 @@ inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? {
* Read file containing meta using given [formatOverride] or file extension to infer meta type.
* If [path] is a directory search for file starting with `meta` in it
*/
fun IOPlugin.readMetaFile(path: Path, formatOverride: MetaFormat? = null, descriptor: NodeDescriptor? = null): Meta {
public fun IOPlugin.readMetaFile(path: Path, formatOverride: MetaFormat? = null, descriptor: NodeDescriptor? = null): Meta {
if (!Files.exists(path)) error("Meta file $path does not exist")
val actualPath: Path = if (Files.isDirectory(path)) {
@ -80,7 +80,7 @@ fun IOPlugin.readMetaFile(path: Path, formatOverride: MetaFormat? = null, descri
val metaFormat = formatOverride ?: resolveMetaFormat(extension) ?: error("Can't resolve meta format $extension")
return metaFormat.run {
actualPath.read {
readMeta(descriptor)
readMeta(this, descriptor)
}
}
}
@ -89,7 +89,7 @@ fun IOPlugin.readMetaFile(path: Path, formatOverride: MetaFormat? = null, descri
* Write meta to file using [metaFormat]. If [path] is a directory, write a file with name equals name of [metaFormat].
* Like "meta.json"
*/
fun IOPlugin.writeMetaFile(
public fun IOPlugin.writeMetaFile(
path: Path,
meta: Meta,
metaFormat: MetaFormatFactory = JsonMetaFormat,
@ -102,7 +102,7 @@ fun IOPlugin.writeMetaFile(
}
metaFormat.run {
actualPath.write {
writeMeta(meta, descriptor)
writeMeta(this, meta, descriptor)
}
}
}
@ -111,7 +111,7 @@ fun IOPlugin.writeMetaFile(
* Return inferred [EnvelopeFormat] if only one format could read given file. If no format accepts file, return null. If
* multiple formats accepts file, throw an error.
*/
fun IOPlugin.peekBinaryFormat(path: Path): EnvelopeFormat? {
public fun IOPlugin.peekBinaryFormat(path: Path): EnvelopeFormat? {
val binary = path.asBinary()
val formats = envelopeFormatFactories.mapNotNull { factory ->
binary.read {
@ -126,8 +126,8 @@ fun IOPlugin.peekBinaryFormat(path: Path): EnvelopeFormat? {
}
}
val IOPlugin.Companion.META_FILE_NAME: String get() = "@meta"
val IOPlugin.Companion.DATA_FILE_NAME: String get() = "@data"
public val IOPlugin.Companion.META_FILE_NAME: String get() = "@meta"
public val IOPlugin.Companion.DATA_FILE_NAME: String get() = "@data"
/**
* Read and envelope from file if the file exists, return null if file does not exist.
@ -143,7 +143,7 @@ val IOPlugin.Companion.DATA_FILE_NAME: String get() = "@data"
* Return null otherwise.
*/
@DFExperimental
fun IOPlugin.readEnvelopeFile(
public fun IOPlugin.readEnvelopeFile(
path: Path,
readNonEnvelopes: Boolean = false,
formatPeeker: IOPlugin.(Path) -> EnvelopeFormat? = IOPlugin::peekBinaryFormat
@ -182,9 +182,9 @@ fun IOPlugin.readEnvelopeFile(
/**
* Write a binary into file. Throws an error if file already exists
*/
fun <T : Any> IOFormat<T>.writeToFile(path: Path, obj: T) {
public fun <T : Any> IOFormat<T>.writeToFile(path: Path, obj: T) {
path.write {
writeObject(obj)
writeObject(this, obj)
}
}
@ -192,7 +192,7 @@ fun <T : Any> IOFormat<T>.writeToFile(path: Path, obj: T) {
* Write envelope file to given [path] using [envelopeFormat] and optional [metaFormat]
*/
@DFExperimental
fun IOPlugin.writeEnvelopeFile(
public fun IOPlugin.writeEnvelopeFile(
path: Path,
envelope: Envelope,
envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat,

4
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt
View File

@ -56,10 +56,10 @@ public class EnvelopeClient(
val outputStream = socket.getOutputStream()
format.run {
outputStream.write {
writeObject(request)
writeObject(this, request)
}
logger.debug { "Sent request with type ${request.type} to ${socket.remoteSocketAddress}" }
val res = inputStream.readBlocking { readObject() }
val res = inputStream.readBlocking { readObject(this) }
logger.debug { "Received response with type ${res.type} from ${socket.remoteSocketAddress}" }
return@withContext res
}

6
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt
View File

@ -74,12 +74,12 @@ public class EnvelopeServer(
val outputStream = socket.getOutputStream()
format.run {
while (socket.isConnected) {
val request = inputStream.readBlocking { readObject() }
val request = inputStream.readBlocking { readObject(this) }
logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" }
if (request.type == SHUTDOWN_ENVELOPE_TYPE) {
//Echo shutdown command
outputStream.write {
writeObject(request)
writeObject(this, request)
}
logger.info { "Accepted graceful shutdown signal from ${socket.inetAddress}" }
socket.close()
@ -89,7 +89,7 @@ public class EnvelopeServer(
runBlocking {
val response = responder.respond(request)
outputStream.write {
writeObject(response)
writeObject(this, response)
}
logger.debug { "Sent response with type ${response.type} to ${socket.remoteSocketAddress}" }
}

4
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/SimpleWorkspace.kt
View File

@ -1,7 +1,7 @@
package hep.dataforge.workspace
import hep.dataforge.context.Context
import hep.dataforge.context.content
import hep.dataforge.context.resolve
import hep.dataforge.context.toMap
import hep.dataforge.data.DataNode
import hep.dataforge.meta.Meta
@ -19,7 +19,7 @@ class SimpleWorkspace(
) : Workspace {
override val tasks: Map<Name, Task<*>> by lazy {
context.content<Task<*>>(Task.TYPE) + tasks.toMap()
context.resolve<Task<*>>(Task.TYPE) + tasks.toMap()
}
companion object {

2
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/WorkspaceBuilder.kt
View File

@ -27,7 +27,7 @@ interface WorkspaceBuilder {
* Set the context for future workspcace
*/
fun WorkspaceBuilder.context(name: String = "WORKSPACE", block: ContextBuilder.() -> Unit = {}) {
context = ContextBuilder(name, parentContext).apply(block).build()
context = ContextBuilder(parentContext, name).apply(block).build()
}
inline fun <reified T : Any> WorkspaceBuilder.data(

2
dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt
View File

@ -168,7 +168,7 @@ private suspend fun <T : Any> ZipOutputStream.writeNode(
val entry = ZipEntry(name)
putNextEntry(entry)
envelopeFormat.run {
asOutput().writeObject(envelope)
writeObject(asOutput(), envelope)
}
}
is DataItem.Node -> {

8
dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt
View File

@ -30,12 +30,12 @@ class FileDataTest {
}
object StringIOFormat : IOFormat<String> {
override fun Output.writeObject(obj: String) {
writeUtf8String(obj)
override fun writeObject(output: Output, obj: String) {
output.writeUtf8String(obj)
}
override fun Input.readObject(): String {
return readUtf8String()
override fun readObject(input: Input): String {
return input.readUtf8String()
}
override fun toMeta(): Meta = Meta {

2
settings.gradle.kts
View File

@ -8,7 +8,7 @@ pluginManagement {
maven("https://dl.bintray.com/mipt-npm/dev")
}
val toolsVersion = "0.6.0-dev-3"
val toolsVersion = "0.6.0-dev-4"
val kotlinVersion = "1.4.0"
plugins {