Fix zip writer and name refactoring
This commit is contained in:
parent
eebfe534cc
commit
6450696157
@ -1,9 +1,7 @@
|
|||||||
package hep.dataforge.context
|
package hep.dataforge.context
|
||||||
|
|
||||||
import hep.dataforge.meta.EmptyMeta
|
|
||||||
import hep.dataforge.meta.Meta
|
import hep.dataforge.meta.Meta
|
||||||
import hep.dataforge.meta.MetaBuilder
|
import hep.dataforge.meta.MetaBuilder
|
||||||
import hep.dataforge.meta.buildMeta
|
|
||||||
import kotlin.reflect.KClass
|
import kotlin.reflect.KClass
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -98,7 +96,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
|
|||||||
/**
|
/**
|
||||||
* Load a plugin using its factory
|
* Load a plugin using its factory
|
||||||
*/
|
*/
|
||||||
fun <T : Plugin> load(factory: PluginFactory<T>, meta: Meta = EmptyMeta): T =
|
fun <T : Plugin> load(factory: PluginFactory<T>, meta: Meta = Meta.EMPTY): T =
|
||||||
load(factory(meta, context))
|
load(factory(meta, context))
|
||||||
|
|
||||||
fun <T : Plugin> load(factory: PluginFactory<T>, metaBuilder: MetaBuilder.() -> Unit): T =
|
fun <T : Plugin> load(factory: PluginFactory<T>, metaBuilder: MetaBuilder.() -> Unit): T =
|
||||||
@ -121,7 +119,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable<Plug
|
|||||||
* Get an existing plugin with given meta or load new one using provided factory
|
* Get an existing plugin with given meta or load new one using provided factory
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
fun <T : Plugin> fetch(factory: PluginFactory<T>, recursive: Boolean = true, meta: Meta = EmptyMeta): T {
|
fun <T : Plugin> fetch(factory: PluginFactory<T>, recursive: Boolean = true, meta: Meta = Meta.EMPTY): T {
|
||||||
val loaded = get(factory.type, factory.tag, recursive)
|
val loaded = get(factory.type, factory.tag, recursive)
|
||||||
return when {
|
return when {
|
||||||
loaded == null -> load(factory(meta, context))
|
loaded == null -> load(factory(meta, context))
|
||||||
|
@ -1,6 +1,8 @@
|
|||||||
package hep.dataforge.data
|
package hep.dataforge.data
|
||||||
|
|
||||||
import hep.dataforge.meta.*
|
import hep.dataforge.meta.Meta
|
||||||
|
import hep.dataforge.meta.MetaRepr
|
||||||
|
import hep.dataforge.meta.isEmpty
|
||||||
import kotlinx.coroutines.CoroutineScope
|
import kotlinx.coroutines.CoroutineScope
|
||||||
import kotlin.coroutines.CoroutineContext
|
import kotlin.coroutines.CoroutineContext
|
||||||
import kotlin.coroutines.EmptyCoroutineContext
|
import kotlin.coroutines.EmptyCoroutineContext
|
||||||
@ -31,14 +33,14 @@ interface Data<out T : Any> : Goal<T>, MetaRepr{
|
|||||||
|
|
||||||
operator fun <T : Any> invoke(
|
operator fun <T : Any> invoke(
|
||||||
type: KClass<out T>,
|
type: KClass<out T>,
|
||||||
meta: Meta = EmptyMeta,
|
meta: Meta = Meta.EMPTY,
|
||||||
context: CoroutineContext = EmptyCoroutineContext,
|
context: CoroutineContext = EmptyCoroutineContext,
|
||||||
dependencies: Collection<Data<*>> = emptyList(),
|
dependencies: Collection<Data<*>> = emptyList(),
|
||||||
block: suspend CoroutineScope.() -> T
|
block: suspend CoroutineScope.() -> T
|
||||||
): Data<T> = DynamicData(type, meta, context, dependencies, block)
|
): Data<T> = DynamicData(type, meta, context, dependencies, block)
|
||||||
|
|
||||||
inline operator fun <reified T : Any> invoke(
|
inline operator fun <reified T : Any> invoke(
|
||||||
meta: Meta = EmptyMeta,
|
meta: Meta = Meta.EMPTY,
|
||||||
context: CoroutineContext = EmptyCoroutineContext,
|
context: CoroutineContext = EmptyCoroutineContext,
|
||||||
dependencies: Collection<Data<*>> = emptyList(),
|
dependencies: Collection<Data<*>> = emptyList(),
|
||||||
noinline block: suspend CoroutineScope.() -> T
|
noinline block: suspend CoroutineScope.() -> T
|
||||||
@ -47,7 +49,7 @@ interface Data<out T : Any> : Goal<T>, MetaRepr{
|
|||||||
operator fun <T : Any> invoke(
|
operator fun <T : Any> invoke(
|
||||||
name: String,
|
name: String,
|
||||||
type: KClass<out T>,
|
type: KClass<out T>,
|
||||||
meta: Meta = EmptyMeta,
|
meta: Meta = Meta.EMPTY,
|
||||||
context: CoroutineContext = EmptyCoroutineContext,
|
context: CoroutineContext = EmptyCoroutineContext,
|
||||||
dependencies: Collection<Data<*>> = emptyList(),
|
dependencies: Collection<Data<*>> = emptyList(),
|
||||||
block: suspend CoroutineScope.() -> T
|
block: suspend CoroutineScope.() -> T
|
||||||
@ -55,14 +57,14 @@ interface Data<out T : Any> : Goal<T>, MetaRepr{
|
|||||||
|
|
||||||
inline operator fun <reified T : Any> invoke(
|
inline operator fun <reified T : Any> invoke(
|
||||||
name: String,
|
name: String,
|
||||||
meta: Meta = EmptyMeta,
|
meta: Meta = Meta.EMPTY,
|
||||||
context: CoroutineContext = EmptyCoroutineContext,
|
context: CoroutineContext = EmptyCoroutineContext,
|
||||||
dependencies: Collection<Data<*>> = emptyList(),
|
dependencies: Collection<Data<*>> = emptyList(),
|
||||||
noinline block: suspend CoroutineScope.() -> T
|
noinline block: suspend CoroutineScope.() -> T
|
||||||
): Data<T> =
|
): Data<T> =
|
||||||
invoke(name, T::class, meta, context, dependencies, block)
|
invoke(name, T::class, meta, context, dependencies, block)
|
||||||
|
|
||||||
fun <T : Any> static(value: T, meta: Meta = EmptyMeta): Data<T> =
|
fun <T : Any> static(value: T, meta: Meta = Meta.EMPTY): Data<T> =
|
||||||
StaticData(value, meta)
|
StaticData(value, meta)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -70,7 +72,7 @@ interface Data<out T : Any> : Goal<T>, MetaRepr{
|
|||||||
|
|
||||||
class DynamicData<T : Any>(
|
class DynamicData<T : Any>(
|
||||||
override val type: KClass<out T>,
|
override val type: KClass<out T>,
|
||||||
override val meta: Meta = EmptyMeta,
|
override val meta: Meta = Meta.EMPTY,
|
||||||
context: CoroutineContext = EmptyCoroutineContext,
|
context: CoroutineContext = EmptyCoroutineContext,
|
||||||
dependencies: Collection<Data<*>> = emptyList(),
|
dependencies: Collection<Data<*>> = emptyList(),
|
||||||
block: suspend CoroutineScope.() -> T
|
block: suspend CoroutineScope.() -> T
|
||||||
@ -78,7 +80,7 @@ class DynamicData<T : Any>(
|
|||||||
|
|
||||||
class StaticData<T : Any>(
|
class StaticData<T : Any>(
|
||||||
value: T,
|
value: T,
|
||||||
override val meta: Meta = EmptyMeta
|
override val meta: Meta = Meta.EMPTY
|
||||||
) : Data<T>, StaticGoal<T>(value) {
|
) : Data<T>, StaticGoal<T>(value) {
|
||||||
override val type: KClass<out T> get() = value::class
|
override val type: KClass<out T> get() = value::class
|
||||||
}
|
}
|
||||||
|
@ -250,7 +250,7 @@ fun <T : Any> DataTreeBuilder<T>.datum(name: String, data: Data<T>) {
|
|||||||
this[name.toName()] = data
|
this[name.toName()] = data
|
||||||
}
|
}
|
||||||
|
|
||||||
fun <T : Any> DataTreeBuilder<T>.static(name: Name, data: T, meta: Meta = EmptyMeta) {
|
fun <T : Any> DataTreeBuilder<T>.static(name: Name, data: T, meta: Meta = Meta.EMPTY) {
|
||||||
this[name] = Data.static(data, meta)
|
this[name] = Data.static(data, meta)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -11,7 +11,7 @@ serialization(sourceSet = TEST){
|
|||||||
cbor()
|
cbor()
|
||||||
}
|
}
|
||||||
|
|
||||||
val ioVersion by rootProject.extra("0.2.0-npm-dev-6")
|
val ioVersion by rootProject.extra("0.2.0-npm-dev-7")
|
||||||
|
|
||||||
kotlin {
|
kotlin {
|
||||||
sourceSets {
|
sourceSets {
|
||||||
|
@ -2,6 +2,8 @@ package hep.dataforge.io.yaml
|
|||||||
|
|
||||||
import hep.dataforge.context.Context
|
import hep.dataforge.context.Context
|
||||||
import hep.dataforge.io.*
|
import hep.dataforge.io.*
|
||||||
|
import hep.dataforge.io.IOFormat.Companion.META_KEY
|
||||||
|
import hep.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||||
import hep.dataforge.meta.DFExperimental
|
import hep.dataforge.meta.DFExperimental
|
||||||
import hep.dataforge.meta.Meta
|
import hep.dataforge.meta.Meta
|
||||||
import kotlinx.io.*
|
import kotlinx.io.*
|
||||||
@ -78,8 +80,8 @@ class FrontMatterEnvelopeFormat(
|
|||||||
}
|
}
|
||||||
|
|
||||||
override fun toMeta(): Meta = Meta {
|
override fun toMeta(): Meta = Meta {
|
||||||
IOPlugin.IO_FORMAT_NAME_KEY put name.toString()
|
NAME_KEY put name.toString()
|
||||||
IOPlugin.IO_FORMAT_META_KEY put meta
|
META_KEY put meta
|
||||||
}
|
}
|
||||||
|
|
||||||
companion object : EnvelopeFormatFactory {
|
companion object : EnvelopeFormatFactory {
|
||||||
|
@ -1,7 +1,8 @@
|
|||||||
package hep.dataforge.io.yaml
|
package hep.dataforge.io.yaml
|
||||||
|
|
||||||
import hep.dataforge.context.Context
|
import hep.dataforge.context.Context
|
||||||
import hep.dataforge.io.IOPlugin
|
import hep.dataforge.io.IOFormat.Companion.META_KEY
|
||||||
|
import hep.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||||
import hep.dataforge.io.MetaFormat
|
import hep.dataforge.io.MetaFormat
|
||||||
import hep.dataforge.io.MetaFormatFactory
|
import hep.dataforge.io.MetaFormatFactory
|
||||||
import hep.dataforge.meta.DFExperimental
|
import hep.dataforge.meta.DFExperimental
|
||||||
@ -30,8 +31,8 @@ class YamlMetaFormat(val meta: Meta) : MetaFormat {
|
|||||||
}
|
}
|
||||||
|
|
||||||
override fun toMeta(): Meta = Meta{
|
override fun toMeta(): Meta = Meta{
|
||||||
IOPlugin.IO_FORMAT_NAME_KEY put FrontMatterEnvelopeFormat.name.toString()
|
NAME_KEY put FrontMatterEnvelopeFormat.name.toString()
|
||||||
IOPlugin.IO_FORMAT_META_KEY put meta
|
META_KEY put meta
|
||||||
}
|
}
|
||||||
|
|
||||||
companion object : MetaFormatFactory {
|
companion object : MetaFormatFactory {
|
||||||
|
@ -2,6 +2,7 @@ package hep.dataforge.io.yaml
|
|||||||
|
|
||||||
import hep.dataforge.io.parse
|
import hep.dataforge.io.parse
|
||||||
import hep.dataforge.io.toString
|
import hep.dataforge.io.toString
|
||||||
|
import hep.dataforge.meta.DFExperimental
|
||||||
import hep.dataforge.meta.Meta
|
import hep.dataforge.meta.Meta
|
||||||
import hep.dataforge.meta.get
|
import hep.dataforge.meta.get
|
||||||
import hep.dataforge.meta.seal
|
import hep.dataforge.meta.seal
|
||||||
@ -9,6 +10,7 @@ import kotlin.test.Test
|
|||||||
import kotlin.test.assertEquals
|
import kotlin.test.assertEquals
|
||||||
|
|
||||||
|
|
||||||
|
@DFExperimental
|
||||||
class YamlMetaFormatTest {
|
class YamlMetaFormatTest {
|
||||||
@Test
|
@Test
|
||||||
fun testYamlMetaFormat() {
|
fun testYamlMetaFormat() {
|
||||||
|
@ -3,8 +3,8 @@ package hep.dataforge.io
|
|||||||
import hep.dataforge.context.Context
|
import hep.dataforge.context.Context
|
||||||
import hep.dataforge.context.Factory
|
import hep.dataforge.context.Factory
|
||||||
import hep.dataforge.context.Named
|
import hep.dataforge.context.Named
|
||||||
|
import hep.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||||
import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
|
import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
|
||||||
import hep.dataforge.io.IOPlugin.Companion.IO_FORMAT_NAME_KEY
|
|
||||||
import hep.dataforge.meta.Meta
|
import hep.dataforge.meta.Meta
|
||||||
import hep.dataforge.meta.MetaItem
|
import hep.dataforge.meta.MetaItem
|
||||||
import hep.dataforge.meta.MetaRepr
|
import hep.dataforge.meta.MetaRepr
|
||||||
@ -23,6 +23,11 @@ import kotlin.reflect.KClass
|
|||||||
interface IOFormat<T : Any> : MetaRepr {
|
interface IOFormat<T : Any> : MetaRepr {
|
||||||
fun Output.writeObject(obj: T)
|
fun Output.writeObject(obj: T)
|
||||||
fun Input.readObject(): T
|
fun Input.readObject(): T
|
||||||
|
|
||||||
|
companion object{
|
||||||
|
val NAME_KEY = "name".asName()
|
||||||
|
val META_KEY = "meta".asName()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun <T : Any> Input.readWith(format: IOFormat<T>): T = format.run { readObject() }
|
fun <T : Any> Input.readWith(format: IOFormat<T>): T = format.run { readObject() }
|
||||||
@ -54,7 +59,7 @@ class ListIOFormat<T : Any>(val format: IOFormat<T>) : IOFormat<List<T>> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
override fun toMeta(): Meta = Meta {
|
override fun toMeta(): Meta = Meta {
|
||||||
IO_FORMAT_NAME_KEY put "list"
|
NAME_KEY put "list"
|
||||||
"contentFormat" put format.toMeta()
|
"contentFormat" put format.toMeta()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -79,7 +84,7 @@ interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named, MetaRepr {
|
|||||||
val type: KClass<out T>
|
val type: KClass<out T>
|
||||||
|
|
||||||
override fun toMeta(): Meta = Meta {
|
override fun toMeta(): Meta = Meta {
|
||||||
IO_FORMAT_NAME_KEY put name.toString()
|
NAME_KEY put name.toString()
|
||||||
}
|
}
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
|
@ -2,11 +2,12 @@ package hep.dataforge.io
|
|||||||
|
|
||||||
import hep.dataforge.context.*
|
import hep.dataforge.context.*
|
||||||
import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE
|
import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE
|
||||||
|
import hep.dataforge.io.IOFormat.Companion.META_KEY
|
||||||
|
import hep.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||||
import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
|
import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
|
||||||
import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE
|
import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE
|
||||||
import hep.dataforge.meta.*
|
import hep.dataforge.meta.*
|
||||||
import hep.dataforge.names.Name
|
import hep.dataforge.names.Name
|
||||||
import hep.dataforge.names.asName
|
|
||||||
import hep.dataforge.names.toName
|
import hep.dataforge.names.toName
|
||||||
import kotlin.reflect.KClass
|
import kotlin.reflect.KClass
|
||||||
|
|
||||||
@ -18,12 +19,12 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fun <T : Any> resolveIOFormat(item: MetaItem<*>, type: KClass<out T>): IOFormat<T>? {
|
fun <T : Any> resolveIOFormat(item: MetaItem<*>, type: KClass<out T>): IOFormat<T>? {
|
||||||
val key = item.string ?: item.node[IO_FORMAT_NAME_KEY]?.string ?: error("Format name not defined")
|
val key = item.string ?: item.node[NAME_KEY]?.string ?: error("Format name not defined")
|
||||||
val name = key.toName()
|
val name = key.toName()
|
||||||
return ioFormatFactories.find { it.name == name }?.let {
|
return ioFormatFactories.find { it.name == name }?.let {
|
||||||
@Suppress("UNCHECKED_CAST")
|
@Suppress("UNCHECKED_CAST")
|
||||||
if (it.type != type) error("Format type ${it.type} is not the same as requested type $type")
|
if (it.type != type) error("Format type ${it.type} is not the same as requested type $type")
|
||||||
else it.invoke(item.node[IO_FORMAT_META_KEY].node ?: Meta.EMPTY, context) as IOFormat<T>
|
else it.invoke(item.node[META_KEY].node ?: Meta.EMPTY, context) as IOFormat<T>
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -46,8 +47,8 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
|
|||||||
envelopeFormatFactories.find { it.name == name }?.invoke(meta, context)
|
envelopeFormatFactories.find { it.name == name }?.invoke(meta, context)
|
||||||
|
|
||||||
fun resolveEnvelopeFormat(item: MetaItem<*>): EnvelopeFormat? {
|
fun resolveEnvelopeFormat(item: MetaItem<*>): EnvelopeFormat? {
|
||||||
val name = item.string ?: item.node[IO_FORMAT_NAME_KEY]?.string ?: error("Envelope format name not defined")
|
val name = item.string ?: item.node[NAME_KEY]?.string ?: error("Envelope format name not defined")
|
||||||
val meta = item.node[IO_FORMAT_META_KEY].node ?: Meta.EMPTY
|
val meta = item.node[META_KEY].node ?: Meta.EMPTY
|
||||||
return resolveEnvelopeFormat(name.toName(), meta)
|
return resolveEnvelopeFormat(name.toName(), meta)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -60,9 +61,6 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
companion object : PluginFactory<IOPlugin> {
|
companion object : PluginFactory<IOPlugin> {
|
||||||
val IO_FORMAT_NAME_KEY = "name".asName()
|
|
||||||
val IO_FORMAT_META_KEY = "meta".asName()
|
|
||||||
|
|
||||||
val defaultMetaFormats: List<MetaFormatFactory> = listOf(JsonMetaFormat, BinaryMetaFormat)
|
val defaultMetaFormats: List<MetaFormatFactory> = listOf(JsonMetaFormat, BinaryMetaFormat)
|
||||||
val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat, TaglessEnvelopeFormat)
|
val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat, TaglessEnvelopeFormat)
|
||||||
|
|
||||||
|
@ -4,6 +4,7 @@ package hep.dataforge.io
|
|||||||
|
|
||||||
|
|
||||||
import hep.dataforge.context.Context
|
import hep.dataforge.context.Context
|
||||||
|
import hep.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||||
import hep.dataforge.meta.Meta
|
import hep.dataforge.meta.Meta
|
||||||
import hep.dataforge.meta.descriptors.NodeDescriptor
|
import hep.dataforge.meta.descriptors.NodeDescriptor
|
||||||
import hep.dataforge.meta.node
|
import hep.dataforge.meta.node
|
||||||
@ -26,7 +27,7 @@ class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat {
|
|||||||
}
|
}
|
||||||
|
|
||||||
override fun toMeta(): Meta = Meta{
|
override fun toMeta(): Meta = Meta{
|
||||||
IOPlugin.IO_FORMAT_NAME_KEY put name.toString()
|
NAME_KEY put name.toString()
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun Input.readMeta(descriptor: NodeDescriptor?): Meta {
|
override fun Input.readMeta(descriptor: NodeDescriptor?): Meta {
|
||||||
|
@ -1,6 +1,8 @@
|
|||||||
package hep.dataforge.io
|
package hep.dataforge.io
|
||||||
|
|
||||||
import hep.dataforge.context.Context
|
import hep.dataforge.context.Context
|
||||||
|
import hep.dataforge.io.IOFormat.Companion.META_KEY
|
||||||
|
import hep.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||||
import hep.dataforge.meta.Meta
|
import hep.dataforge.meta.Meta
|
||||||
import hep.dataforge.meta.enum
|
import hep.dataforge.meta.enum
|
||||||
import hep.dataforge.meta.get
|
import hep.dataforge.meta.get
|
||||||
@ -99,8 +101,8 @@ class TaggedEnvelopeFormat(
|
|||||||
}
|
}
|
||||||
|
|
||||||
override fun toMeta(): Meta = Meta {
|
override fun toMeta(): Meta = Meta {
|
||||||
IOPlugin.IO_FORMAT_NAME_KEY put name.toString()
|
NAME_KEY put name.toString()
|
||||||
IOPlugin.IO_FORMAT_META_KEY put {
|
META_KEY put {
|
||||||
"version" put version
|
"version" put version
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,6 +1,8 @@
|
|||||||
package hep.dataforge.io
|
package hep.dataforge.io
|
||||||
|
|
||||||
import hep.dataforge.context.Context
|
import hep.dataforge.context.Context
|
||||||
|
import hep.dataforge.io.IOFormat.Companion.META_KEY
|
||||||
|
import hep.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||||
import hep.dataforge.meta.Meta
|
import hep.dataforge.meta.Meta
|
||||||
import hep.dataforge.meta.get
|
import hep.dataforge.meta.get
|
||||||
import hep.dataforge.meta.isEmpty
|
import hep.dataforge.meta.isEmpty
|
||||||
@ -162,8 +164,8 @@ class TaglessEnvelopeFormat(
|
|||||||
}
|
}
|
||||||
|
|
||||||
override fun toMeta(): Meta = Meta {
|
override fun toMeta(): Meta = Meta {
|
||||||
IOPlugin.IO_FORMAT_NAME_KEY put name.toString()
|
NAME_KEY put name.toString()
|
||||||
IOPlugin.IO_FORMAT_META_KEY put meta
|
META_KEY put meta
|
||||||
}
|
}
|
||||||
|
|
||||||
companion object : EnvelopeFormatFactory {
|
companion object : EnvelopeFormatFactory {
|
||||||
|
@ -19,9 +19,6 @@ inline fun buildByteArray(expectedSize: Int = 16, block: Output.() -> Unit): Byt
|
|||||||
inline fun Binary(expectedSize: Int = 16, block: Output.() -> Unit): Binary =
|
inline fun Binary(expectedSize: Int = 16, block: Output.() -> Unit): Binary =
|
||||||
buildByteArray(expectedSize, block).asBinary()
|
buildByteArray(expectedSize, block).asBinary()
|
||||||
|
|
||||||
@Deprecated("To be replaced by Binary.EMPTY", level = DeprecationLevel.WARNING)
|
|
||||||
val EmptyBinary = ByteArrayBinary(ByteArray(0))
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* View section of a [Binary] as an independent binary
|
* View section of a [Binary] as an independent binary
|
||||||
*/
|
*/
|
||||||
|
@ -1,12 +1,9 @@
|
|||||||
package hep.dataforge.io
|
package hep.dataforge.io
|
||||||
|
|
||||||
import hep.dataforge.meta.Meta
|
import hep.dataforge.meta.*
|
||||||
import hep.dataforge.meta.MetaItem
|
|
||||||
import hep.dataforge.meta.MetaSerializer
|
|
||||||
import hep.dataforge.names.Name
|
import hep.dataforge.names.Name
|
||||||
import hep.dataforge.names.toName
|
import hep.dataforge.names.toName
|
||||||
import kotlinx.serialization.cbor.Cbor
|
import kotlinx.serialization.cbor.Cbor
|
||||||
import kotlinx.serialization.json.Json
|
|
||||||
import kotlin.test.Test
|
import kotlin.test.Test
|
||||||
import kotlin.test.assertEquals
|
import kotlin.test.assertEquals
|
||||||
|
|
||||||
@ -22,8 +19,8 @@ class MetaSerializerTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun testMetaSerialization() {
|
fun testMetaSerialization() {
|
||||||
val string = Json.indented.stringify(MetaSerializer, meta)
|
val string = JSON_PRETTY.stringify(MetaSerializer, meta)
|
||||||
val restored = Json.plain.parse(MetaSerializer, string)
|
val restored = JSON_PLAIN.parse(MetaSerializer, string)
|
||||||
assertEquals(restored, meta)
|
assertEquals(restored, meta)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -38,8 +35,8 @@ class MetaSerializerTest {
|
|||||||
@Test
|
@Test
|
||||||
fun testNameSerialization() {
|
fun testNameSerialization() {
|
||||||
val name = "a.b.c".toName()
|
val name = "a.b.c".toName()
|
||||||
val string = Json.indented.stringify(Name.serializer(), name)
|
val string = JSON_PRETTY.stringify(Name.serializer(), name)
|
||||||
val restored = Json.plain.parse(Name.serializer(), string)
|
val restored = JSON_PLAIN.parse(Name.serializer(), string)
|
||||||
assertEquals(restored, name)
|
assertEquals(restored, name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3,7 +3,6 @@ package hep.dataforge.io.tcp
|
|||||||
import hep.dataforge.context.Context
|
import hep.dataforge.context.Context
|
||||||
import hep.dataforge.context.ContextAware
|
import hep.dataforge.context.ContextAware
|
||||||
import hep.dataforge.io.*
|
import hep.dataforge.io.*
|
||||||
import hep.dataforge.meta.EmptyMeta
|
|
||||||
import hep.dataforge.meta.Meta
|
import hep.dataforge.meta.Meta
|
||||||
import kotlinx.coroutines.asCoroutineDispatcher
|
import kotlinx.coroutines.asCoroutineDispatcher
|
||||||
import kotlinx.coroutines.withContext
|
import kotlinx.coroutines.withContext
|
||||||
@ -17,7 +16,7 @@ class EnvelopeClient(
|
|||||||
val host: String,
|
val host: String,
|
||||||
val port: Int,
|
val port: Int,
|
||||||
formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
|
formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
|
||||||
formatMeta: Meta = EmptyMeta
|
formatMeta: Meta = Meta.EMPTY
|
||||||
) : Responder, ContextAware {
|
) : Responder, ContextAware {
|
||||||
|
|
||||||
private val dispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher()
|
private val dispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher()
|
||||||
|
@ -6,7 +6,6 @@ import hep.dataforge.io.EnvelopeFormatFactory
|
|||||||
import hep.dataforge.io.Responder
|
import hep.dataforge.io.Responder
|
||||||
import hep.dataforge.io.TaggedEnvelopeFormat
|
import hep.dataforge.io.TaggedEnvelopeFormat
|
||||||
import hep.dataforge.io.type
|
import hep.dataforge.io.type
|
||||||
import hep.dataforge.meta.EmptyMeta
|
|
||||||
import hep.dataforge.meta.Meta
|
import hep.dataforge.meta.Meta
|
||||||
import kotlinx.coroutines.*
|
import kotlinx.coroutines.*
|
||||||
import java.net.ServerSocket
|
import java.net.ServerSocket
|
||||||
@ -19,7 +18,7 @@ class EnvelopeServer(
|
|||||||
val responder: Responder,
|
val responder: Responder,
|
||||||
val scope: CoroutineScope,
|
val scope: CoroutineScope,
|
||||||
formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
|
formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat,
|
||||||
formatMeta: Meta = EmptyMeta
|
formatMeta: Meta = Meta.EMPTY
|
||||||
) : ContextAware {
|
) : ContextAware {
|
||||||
|
|
||||||
private var job: Job? = null
|
private var job: Job? = null
|
||||||
|
@ -188,7 +188,7 @@ abstract class MetaBase : Meta {
|
|||||||
|
|
||||||
override fun hashCode(): Int = items.hashCode()
|
override fun hashCode(): Int = items.hashCode()
|
||||||
|
|
||||||
override fun toString(): String = PRETTY_JSON.stringify(MetaSerializer, this)
|
override fun toString(): String = JSON_PRETTY.stringify(MetaSerializer, this)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -66,4 +66,5 @@ inline fun Encoder.encodeStructure(
|
|||||||
encoder.endStructure(desc)
|
encoder.endStructure(desc)
|
||||||
}
|
}
|
||||||
|
|
||||||
val PRETTY_JSON = Json(JsonConfiguration(prettyPrint = true, useArrayPolymorphism = true))
|
val JSON_PRETTY = Json(JsonConfiguration(prettyPrint = true, useArrayPolymorphism = true))
|
||||||
|
val JSON_PLAIN = Json(JsonConfiguration(prettyPrint = true, useArrayPolymorphism = true))
|
@ -26,7 +26,7 @@ class DynamicMetaTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun testMetaToDynamic(){
|
fun testMetaToDynamic(){
|
||||||
val meta = buildMeta {
|
val meta = Meta {
|
||||||
"a" put 22
|
"a" put 22
|
||||||
"array" put listOf(1, 2, 3)
|
"array" put listOf(1, 2, 3)
|
||||||
"b" put "myString"
|
"b" put "myString"
|
||||||
|
@ -2,7 +2,6 @@ package hep.dataforge.output
|
|||||||
|
|
||||||
import hep.dataforge.context.*
|
import hep.dataforge.context.*
|
||||||
import hep.dataforge.context.PluginTag.Companion.DATAFORGE_GROUP
|
import hep.dataforge.context.PluginTag.Companion.DATAFORGE_GROUP
|
||||||
import hep.dataforge.meta.EmptyMeta
|
|
||||||
import hep.dataforge.meta.Meta
|
import hep.dataforge.meta.Meta
|
||||||
import hep.dataforge.names.Name
|
import hep.dataforge.names.Name
|
||||||
import kotlinx.coroutines.CoroutineDispatcher
|
import kotlinx.coroutines.CoroutineDispatcher
|
||||||
@ -24,7 +23,7 @@ interface OutputManager {
|
|||||||
type: KClass<out T>,
|
type: KClass<out T>,
|
||||||
name: Name,
|
name: Name,
|
||||||
stage: Name = Name.EMPTY,
|
stage: Name = Name.EMPTY,
|
||||||
meta: Meta = EmptyMeta
|
meta: Meta = Meta.EMPTY
|
||||||
): Renderer<T>
|
): Renderer<T>
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -39,7 +38,7 @@ val Context.output: OutputManager get() = plugins.get() ?: ConsoleOutputManager(
|
|||||||
inline operator fun <reified T : Any> OutputManager.get(
|
inline operator fun <reified T : Any> OutputManager.get(
|
||||||
name: Name,
|
name: Name,
|
||||||
stage: Name = Name.EMPTY,
|
stage: Name = Name.EMPTY,
|
||||||
meta: Meta = EmptyMeta
|
meta: Meta = Meta.EMPTY
|
||||||
): Renderer<T> {
|
): Renderer<T> {
|
||||||
return get(T::class, name, stage, meta)
|
return get(T::class, name, stage, meta)
|
||||||
}
|
}
|
||||||
@ -47,7 +46,7 @@ inline operator fun <reified T : Any> OutputManager.get(
|
|||||||
/**
|
/**
|
||||||
* Directly render an object using the most suitable renderer
|
* Directly render an object using the most suitable renderer
|
||||||
*/
|
*/
|
||||||
fun OutputManager.render(obj: Any, name: Name, stage: Name = Name.EMPTY, meta: Meta = EmptyMeta) =
|
fun OutputManager.render(obj: Any, name: Name, stage: Name = Name.EMPTY, meta: Meta = Meta.EMPTY) =
|
||||||
get(obj::class, name, stage).render(obj, meta)
|
get(obj::class, name, stage).render(obj, meta)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -1,7 +1,6 @@
|
|||||||
package hep.dataforge.output
|
package hep.dataforge.output
|
||||||
|
|
||||||
import hep.dataforge.context.ContextAware
|
import hep.dataforge.context.ContextAware
|
||||||
import hep.dataforge.meta.EmptyMeta
|
|
||||||
import hep.dataforge.meta.Meta
|
import hep.dataforge.meta.Meta
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -18,5 +17,5 @@ interface Renderer<in T : Any> : ContextAware {
|
|||||||
* By convention actual render is called in asynchronous mode, so this method should never
|
* By convention actual render is called in asynchronous mode, so this method should never
|
||||||
* block execution
|
* block execution
|
||||||
*/
|
*/
|
||||||
fun render(obj: T, meta: Meta = EmptyMeta)
|
fun render(obj: T, meta: Meta = Meta.EMPTY)
|
||||||
}
|
}
|
||||||
|
@ -1,12 +1,11 @@
|
|||||||
package hep.dataforge.tables.io
|
package hep.dataforge.tables.io
|
||||||
|
|
||||||
import hep.dataforge.io.Binary
|
|
||||||
import hep.dataforge.io.EmptyBinary
|
|
||||||
import hep.dataforge.io.Envelope
|
import hep.dataforge.io.Envelope
|
||||||
import hep.dataforge.meta.*
|
import hep.dataforge.meta.*
|
||||||
import hep.dataforge.tables.SimpleColumnHeader
|
import hep.dataforge.tables.SimpleColumnHeader
|
||||||
import hep.dataforge.tables.Table
|
import hep.dataforge.tables.Table
|
||||||
import hep.dataforge.values.Value
|
import hep.dataforge.values.Value
|
||||||
|
import kotlinx.io.Binary
|
||||||
import kotlinx.io.ByteArrayOutput
|
import kotlinx.io.ByteArrayOutput
|
||||||
import kotlinx.io.ExperimentalIoApi
|
import kotlinx.io.ExperimentalIoApi
|
||||||
import kotlinx.io.asBinary
|
import kotlinx.io.asBinary
|
||||||
@ -39,5 +38,5 @@ fun TextRows.Companion.readEnvelope(envelope: Envelope): TextRows {
|
|||||||
.map { (_, item) ->
|
.map { (_, item) ->
|
||||||
SimpleColumnHeader(item.node["name"].string!!, Value::class, item.node["meta"].node ?: Meta.EMPTY)
|
SimpleColumnHeader(item.node["name"].string!!, Value::class, item.node["meta"].node ?: Meta.EMPTY)
|
||||||
}
|
}
|
||||||
return TextRows(header, envelope.data ?: EmptyBinary)
|
return TextRows(header, envelope.data ?: Binary.EMPTY)
|
||||||
}
|
}
|
@ -124,7 +124,7 @@ fun TaskDependencyContainer.allData(to: Name = Name.EMPTY) = AllDataDependency(t
|
|||||||
/**
|
/**
|
||||||
* A builder for [TaskModel]
|
* A builder for [TaskModel]
|
||||||
*/
|
*/
|
||||||
class TaskModelBuilder(val name: Name, meta: Meta = EmptyMeta) : TaskDependencyContainer {
|
class TaskModelBuilder(val name: Name, meta: Meta = Meta.EMPTY) : TaskDependencyContainer {
|
||||||
/**
|
/**
|
||||||
* Meta for current task. By default uses the whole input meta
|
* Meta for current task. By default uses the whole input meta
|
||||||
*/
|
*/
|
||||||
|
@ -6,16 +6,19 @@ import hep.dataforge.io.*
|
|||||||
import hep.dataforge.meta.*
|
import hep.dataforge.meta.*
|
||||||
import kotlinx.coroutines.Dispatchers
|
import kotlinx.coroutines.Dispatchers
|
||||||
import kotlinx.coroutines.withContext
|
import kotlinx.coroutines.withContext
|
||||||
|
import kotlinx.io.asOutput
|
||||||
import java.nio.file.FileSystem
|
import java.nio.file.FileSystem
|
||||||
import java.nio.file.Files
|
import java.nio.file.Files
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
import java.nio.file.StandardCopyOption
|
import java.nio.file.StandardOpenOption
|
||||||
import java.nio.file.spi.FileSystemProvider
|
import java.nio.file.spi.FileSystemProvider
|
||||||
|
import java.util.zip.ZipEntry
|
||||||
|
import java.util.zip.ZipOutputStream
|
||||||
import kotlin.reflect.KClass
|
import kotlin.reflect.KClass
|
||||||
|
|
||||||
typealias FileFormatResolver<T> = (Path, Meta) -> IOFormat<T>
|
typealias FileFormatResolver<T> = (Path, Meta) -> IOFormat<T>
|
||||||
|
|
||||||
//private val zipFSProvider = ZipFileSystemProvider()
|
|
||||||
|
|
||||||
private fun newZFS(path: Path): FileSystem {
|
private fun newZFS(path: Path): FileSystem {
|
||||||
val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" }
|
val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" }
|
||||||
@ -150,12 +153,43 @@ suspend fun <T : Any> IOPlugin.writeDataDirectory(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private suspend fun <T : Any> ZipOutputStream.writeNode(
|
||||||
|
name: String,
|
||||||
|
item: DataItem<T>,
|
||||||
|
dataFormat: IOFormat<T>,
|
||||||
|
envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat
|
||||||
|
) {
|
||||||
|
withContext(Dispatchers.IO) {
|
||||||
|
when (item) {
|
||||||
|
is DataItem.Leaf -> {
|
||||||
|
//TODO add directory-based envelope writer
|
||||||
|
val envelope = item.data.toEnvelope(dataFormat)
|
||||||
|
val entry = ZipEntry(name)
|
||||||
|
putNextEntry(entry)
|
||||||
|
envelopeFormat.run {
|
||||||
|
asOutput().writeObject(envelope)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
is DataItem.Node -> {
|
||||||
|
val entry = ZipEntry("$name/")
|
||||||
|
putNextEntry(entry)
|
||||||
|
closeEntry()
|
||||||
|
item.node.items.forEach { (token, item) ->
|
||||||
|
val childName = "$name/$token"
|
||||||
|
writeNode(childName, item, dataFormat, envelopeFormat)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@DFExperimental
|
||||||
suspend fun <T : Any> IOPlugin.writeZip(
|
suspend fun <T : Any> IOPlugin.writeZip(
|
||||||
path: Path,
|
path: Path,
|
||||||
node: DataNode<T>,
|
node: DataNode<T>,
|
||||||
format: IOFormat<T>,
|
format: IOFormat<T>,
|
||||||
envelopeFormat: EnvelopeFormat? = null,
|
envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat
|
||||||
metaFormat: MetaFormatFactory? = null
|
|
||||||
) {
|
) {
|
||||||
withContext(Dispatchers.IO) {
|
withContext(Dispatchers.IO) {
|
||||||
val actualFile = if (path.toString().endsWith(".zip")) {
|
val actualFile = if (path.toString().endsWith(".zip")) {
|
||||||
@ -163,21 +197,27 @@ suspend fun <T : Any> IOPlugin.writeZip(
|
|||||||
} else {
|
} else {
|
||||||
path.resolveSibling(path.fileName.toString() + ".zip")
|
path.resolveSibling(path.fileName.toString() + ".zip")
|
||||||
}
|
}
|
||||||
if (Files.exists(actualFile) && Files.size(path) == 0.toLong()) {
|
val fos = Files.newOutputStream(actualFile, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)
|
||||||
Files.delete(path)
|
val zos = ZipOutputStream(fos)
|
||||||
}
|
zos.use {
|
||||||
//Files.createFile(actualFile)
|
it.writeNode("", DataItem.Node(node), format, envelopeFormat)
|
||||||
newZFS(actualFile).use { zipfs ->
|
|
||||||
val internalTargetPath = zipfs.getPath("/")
|
|
||||||
Files.createDirectories(internalTargetPath)
|
|
||||||
val tmp = Files.createTempDirectory("df_zip")
|
|
||||||
writeDataDirectory(tmp, node, format, envelopeFormat, metaFormat)
|
|
||||||
Files.list(tmp).forEach { sourcePath ->
|
|
||||||
val targetPath = sourcePath.fileName.toString()
|
|
||||||
val internalTargetPath = internalTargetPath.resolve(targetPath)
|
|
||||||
Files.copy(sourcePath, internalTargetPath, StandardCopyOption.REPLACE_EXISTING)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// if (Files.exists(actualFile) && Files.size(path) == 0.toLong()) {
|
||||||
|
// Files.delete(path)
|
||||||
|
// }
|
||||||
|
// //Files.createFile(actualFile)
|
||||||
|
// newZFS(actualFile).use { zipfs ->
|
||||||
|
// val zipRootPath = zipfs.getPath("/")
|
||||||
|
// Files.createDirectories(zipRootPath)
|
||||||
|
// val tmp = Files.createTempDirectory("df_zip")
|
||||||
|
// writeDataDirectory(tmp, node, format, envelopeFormat, metaFormat)
|
||||||
|
// Files.list(tmp).forEach { sourcePath ->
|
||||||
|
// val targetPath = sourcePath.fileName.toString()
|
||||||
|
// val internalTargetPath = zipRootPath.resolve(targetPath)
|
||||||
|
// Files.copy(sourcePath, internalTargetPath, StandardCopyOption.REPLACE_EXISTING)
|
||||||
|
// }
|
||||||
|
// }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -5,16 +5,17 @@ import hep.dataforge.data.*
|
|||||||
import hep.dataforge.io.IOFormat
|
import hep.dataforge.io.IOFormat
|
||||||
import hep.dataforge.io.io
|
import hep.dataforge.io.io
|
||||||
import hep.dataforge.meta.DFExperimental
|
import hep.dataforge.meta.DFExperimental
|
||||||
|
import hep.dataforge.meta.Meta
|
||||||
import kotlinx.coroutines.runBlocking
|
import kotlinx.coroutines.runBlocking
|
||||||
import kotlinx.io.Input
|
import kotlinx.io.Input
|
||||||
import kotlinx.io.Output
|
import kotlinx.io.Output
|
||||||
import kotlinx.io.text.readUtf8String
|
import kotlinx.io.text.readUtf8String
|
||||||
import kotlinx.io.text.writeUtf8String
|
import kotlinx.io.text.writeUtf8String
|
||||||
import java.nio.file.Files
|
import java.nio.file.Files
|
||||||
import kotlin.test.Ignore
|
|
||||||
import kotlin.test.Test
|
import kotlin.test.Test
|
||||||
import kotlin.test.assertEquals
|
import kotlin.test.assertEquals
|
||||||
|
|
||||||
|
|
||||||
class FileDataTest {
|
class FileDataTest {
|
||||||
val dataNode = DataNode<String> {
|
val dataNode = DataNode<String> {
|
||||||
node("dir") {
|
node("dir") {
|
||||||
@ -37,6 +38,10 @@ class FileDataTest {
|
|||||||
return readUtf8String()
|
return readUtf8String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
override fun toMeta(): Meta = Meta {
|
||||||
|
IOFormat.NAME_KEY put "string"
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@ -56,7 +61,7 @@ class FileDataTest {
|
|||||||
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@Ignore
|
@DFExperimental
|
||||||
fun testZipWriteRead() {
|
fun testZipWriteRead() {
|
||||||
Global.io.run {
|
Global.io.run {
|
||||||
val zip = Files.createTempFile("df_data_node", ".zip")
|
val zip = Files.createTempFile("df_data_node", ".zip")
|
||||||
|
Loading…
Reference in New Issue
Block a user