From 417c292507073f6f675bb0ddaf29bdedc6611216 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 25 Mar 2020 10:41:27 +0300 Subject: [PATCH 01/11] Refactor inner workings of descriptors --- build.gradle.kts | 2 +- .../meta/descriptors/ItemDescriptor.kt | 180 ++++++------------ .../kotlin/hep/dataforge/meta/metaMatcher.kt | 8 +- .../meta/scheme/ConfigurableDelegate.kt | 9 +- .../hep/dataforge/meta/scheme/Scheme.kt | 4 +- .../dataforge/meta/scheme/Specification.kt | 6 +- .../kotlin/hep/dataforge/names/Name.kt | 12 +- .../hep/dataforge/meta/MetaDelegateTest.kt | 2 +- .../meta/descriptors/DescriptorTest.kt | 2 +- .../hep/dataforge/tables/ColumnScheme.kt | 3 +- 10 files changed, 88 insertions(+), 140 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 1fe59eff..ad2766a4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -6,7 +6,7 @@ plugins { id("scientifik.publish") version toolsVersion apply false } -val dataforgeVersion by extra("0.1.5") +val dataforgeVersion by extra("0.1.6-dev") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt index 84d0d5c0..54a01ace 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt @@ -2,15 +2,13 @@ package hep.dataforge.meta.descriptors import hep.dataforge.meta.* import hep.dataforge.meta.scheme.* -import hep.dataforge.names.Name -import hep.dataforge.names.NameToken -import hep.dataforge.names.asName -import hep.dataforge.names.isEmpty +import hep.dataforge.names.* import hep.dataforge.values.False import hep.dataforge.values.True import hep.dataforge.values.Value import hep.dataforge.values.ValueType +@DFBuilder sealed class ItemDescriptor : Scheme() { /** @@ -68,8 +66,7 @@ fun ItemDescriptor.validateItem(item: MetaItem<*>?): Boolean { * @author Alexander Nozik */ @DFBuilder -class NodeDescriptor : ItemDescriptor() { - +class NodeDescriptor private constructor() : ItemDescriptor() { /** * True if the node is required * @@ -84,86 +81,98 @@ class NodeDescriptor : ItemDescriptor() { */ var default by node() + val items: Map + get() = config.getIndexed(ITEM_KEY).mapValues { (_, item) -> + val node = item.node ?: error("Node descriptor must be a node") + if (node[IS_NODE_KEY].boolean == true) { + NodeDescriptor.wrap(node) + } else { + ValueDescriptor.wrap(node) + } + } + /** * The map of children node descriptors */ + @Suppress("UNCHECKED_CAST") val nodes: Map - get() = config.getIndexed(NODE_KEY.asName()).entries.associate { (name, node) -> - name to wrap(node.node ?: error("Node descriptor must be a node")) + get() = config.getIndexed(ITEM_KEY).entries.filter { + it.value.node[IS_NODE_KEY].boolean == true + }.associate { (name, item) -> + val node = item.node ?: error("Node descriptor must be a node") + name to NodeDescriptor.wrap(node) } /** - * Define a child item descriptor for this node + * The list of value descriptors */ - fun defineItem(name: String, descriptor: ItemDescriptor) { - if (items.keys.contains(name)) error("The key $name already exists in descriptor") - val token = when (descriptor) { - is NodeDescriptor -> NameToken(NODE_KEY, name) - is ValueDescriptor -> NameToken(VALUE_KEY, name) + val values: Map + get() = config.getIndexed(ITEM_KEY).entries.filter { + it.value.node[IS_NODE_KEY].boolean != true + }.associate { (name, item) -> + val node = item.node ?: error("Node descriptor must be a node") + name to ValueDescriptor.wrap(node) } - config[token] = descriptor.config - - } - - - fun defineNode(name: String, block: NodeDescriptor.() -> Unit) { - val token = NameToken(NODE_KEY, name) - if (config[token] == null) { - config[token] = NodeDescriptor(block) - } else { - NodeDescriptor.update(config[token].node ?: error("Node expected"), block) - } - } private fun buildNode(name: Name): NodeDescriptor { return when (name.length) { 0 -> this 1 -> { - val token = NameToken(NODE_KEY, name.toString()) - val config: Config = config[token].node ?: Config().also { config[token] = it } + val token = NameToken(ITEM_KEY.toString(), name.toString()) + val config: Config = config[token].node ?: Config().also { + it[IS_NODE_KEY] = true + config[token] = it + } wrap(config) } else -> buildNode(name.first()?.asName()!!).buildNode(name.cutFirst()) } } - fun defineNode(name: Name, block: NodeDescriptor.() -> Unit) { - buildNode(name).apply(block) + /** + * Define a child item descriptor for this node + */ + private fun newItem(key: String, descriptor: ItemDescriptor) { + if (items.keys.contains(key)) error("The key $key already exists in descriptor") + val token = ITEM_KEY.withIndex(key) + config[token] = descriptor.config } - /** - * The list of value descriptors - */ - val values: Map - get() = config.getIndexed(VALUE_KEY.asName()).entries.associate { (name, node) -> - name to ValueDescriptor.wrap(node.node ?: error("Value descriptor must be a node")) - } + fun defineItem(name: Name, descriptor: ItemDescriptor) { + buildNode(name.cutLast()).newItem(name.last().toString(), descriptor) + } + fun defineItem(name: String, descriptor: ItemDescriptor) { + defineItem(name.toName(), descriptor) + } - /** - * Add a value descriptor using block for - */ - fun defineValue(name: String, block: ValueDescriptor.() -> Unit) { - defineItem(name, ValueDescriptor(block)) + fun defineNode(name: Name, block: NodeDescriptor.() -> Unit) { + defineItem(name, NodeDescriptor(block)) + } + + fun defineNode(name: String, block: NodeDescriptor.() -> Unit) { + defineNode(name.toName(), block) } fun defineValue(name: Name, block: ValueDescriptor.() -> Unit) { require(name.length >= 1) { "Name length for value descriptor must be non-empty" } - buildNode(name.cutLast()).defineValue(name.last().toString(), block) + defineItem(name, ValueDescriptor(block)) } - val items: Map get() = nodes + values - - -//override val descriptor: NodeDescriptor = empty("descriptor") + fun defineValue(name: String, block: ValueDescriptor.() -> Unit) { + defineValue(name.toName(), block) + } companion object : SchemeSpec(::NodeDescriptor) { - // const val ITEM_KEY = "item" - const val NODE_KEY = "node" - const val VALUE_KEY = "value" + val ITEM_KEY = "item".asName() + val IS_NODE_KEY = "@isNode".asName() - //override fun wrap(config: Config): NodeDescriptor = NodeDescriptor(config) + override fun empty(): NodeDescriptor { + return super.empty().apply { + config[IS_NODE_KEY] = true + } + } //TODO infer descriptor from spec } @@ -187,9 +196,9 @@ operator fun ItemDescriptor.get(name: Name): ItemDescriptor? { * * @author Alexander Nozik */ +@DFBuilder class ValueDescriptor : ItemDescriptor() { - /** * True if the value is required * @@ -255,68 +264,5 @@ class ValueDescriptor : ItemDescriptor() { this.allowedValues = v.map { Value.of(it) } } - companion object : SchemeSpec(::ValueDescriptor) { -// inline fun > enum(name: String) = ValueDescriptor { -// type(ValueType.STRING) -// this.allowedValues = enumValues().map { Value.of(it.name) } -// } - -// /** -// * Build a value descriptor from annotation -// */ -// fun build(def: ValueDef): ValueDescriptor { -// val builder = MetaBuilder("value") -// .setValue("name", def.key) -// -// if (def.type.isNotEmpty()) { -// builder.setValue("type", def.type) -// } -// -// if (def.multiple) { -// builder.setValue("multiple", def.multiple) -// } -// -// if (!def.info.isEmpty()) { -// builder.setValue("info", def.info) -// } -// -// if (def.allowed.isNotEmpty()) { -// builder.setValue("allowedValues", def.allowed) -// } else if (def.enumeration != Any::class) { -// if (def.enumeration.java.isEnum) { -// val values = def.enumeration.java.enumConstants -// builder.setValue("allowedValues", values.map { it.toString() }) -// } else { -// throw RuntimeException("Only enumeration classes are allowed in 'enumeration' annotation property") -// } -// } -// -// if (def.def.isNotEmpty()) { -// builder.setValue("default", def.def) -// } else if (!def.required) { -// builder.setValue("required", def.required) -// } -// -// if (def.tags.isNotEmpty()) { -// builder.setValue("tags", def.tags) -// } -// return ValueDescriptor(builder) -// } -// -// /** -// * Build empty value descriptor -// */ -// fun empty(valueName: String): ValueDescriptor { -// val builder = MetaBuilder("value") -// .setValue("name", valueName) -// return ValueDescriptor(builder) -// } -// -// /** -// * Merge two separate value descriptors -// */ -// fun merge(primary: ValueDescriptor, secondary: ValueDescriptor): ValueDescriptor { -// return ValueDescriptor(Laminate(primary.meta, secondary.meta)) -// } - } + companion object : SchemeSpec(::ValueDescriptor) } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt index 263483a2..03df113d 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt @@ -6,12 +6,11 @@ import hep.dataforge.names.toName /** * Get all items matching given name. */ -@DFExperimental fun Meta.getIndexed(name: Name): Map> { val root = when (name.length) { - 0 -> error("Can't use empty name for that") + 0 -> error("Can't use empty name for 'getIndexed'") 1 -> this - else -> (this[name.cutLast()] as? MetaItem.NodeItem<*>)?.node + else -> this[name.cutLast()].node } val (body, index) = name.last()!! @@ -23,16 +22,13 @@ fun Meta.getIndexed(name: Name): Map> { ?: emptyMap() } -@DFExperimental fun Meta.getIndexed(name: String): Map> = this@getIndexed.getIndexed(name.toName()) /** * Get all items matching given name. */ @Suppress("UNCHECKED_CAST") -@DFExperimental fun > M.getIndexed(name: Name): Map> = (this as Meta).getIndexed(name) as Map> -@DFExperimental fun > M.getIndexed(name: String): Map> = getIndexed(name.toName()) \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt index c582282b..ae6722cf 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt @@ -168,9 +168,10 @@ fun Configurable.float(default: Float, key: Name? = null): ReadWriteProperty> Configurable.enum( - default: E, key: Name? = null, resolve: MetaItem<*>.() -> E? -): ReadWriteProperty = item(default, key).transform { it?.resolve() ?: default } +inline fun > Configurable.enum( + default: E, key: Name? = null +): ReadWriteProperty = + item(default, key).transform { item -> item?.string?.let { enumValueOf(it) } ?: default } /* * Extra delegates for special cases @@ -225,7 +226,7 @@ fun Configurable.spec( ): ReadWriteProperty = object : ReadWriteProperty { override fun getValue(thisRef: Any?, property: KProperty<*>): T { val name = key ?: property.name.asName() - return config[name].node?.let { spec.wrap(it) }?:default + return config[name].node?.let { spec.wrap(it) } ?: default } override fun setValue(thisRef: Any?, property: KProperty<*>, value: T) { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Scheme.kt index b8d6257f..be954bbe 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Scheme.kt @@ -60,8 +60,10 @@ inline operator fun T.invoke(block: T.() -> Unit) = apply(block) * A specification for simplified generation of wrappers */ open class SchemeSpec(val builder: () -> T) : Specification { + override fun empty(): T = builder() + override fun wrap(config: Config, defaultProvider: (Name) -> MetaItem<*>?): T { - return builder().apply { + return empty().apply { this.config = config this.defaultProvider = defaultProvider } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Specification.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Specification.kt index 1783e841..c2c6f8bb 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Specification.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Specification.kt @@ -17,7 +17,7 @@ interface Specification { return wrap(config).apply(action) } - operator fun invoke(action: T.() -> Unit) = update(Config(), action) + operator fun invoke(action: T.() -> Unit) = empty().apply(action) fun empty() = wrap() @@ -34,9 +34,7 @@ interface Specification { /** * Wrap a configuration using static meta as default */ - fun wrap(default: Meta): T = wrap( - Config() - ) { default[it] } + fun wrap(default: Meta): T = wrap(Config()) { default[it] } } /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt index 2b8908ed..141af13c 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt @@ -59,7 +59,8 @@ class Name(val tokens: List) { val EMPTY = Name(emptyList()) - override val descriptor: SerialDescriptor = PrimitiveDescriptor("hep.dataforge.names.Name", PrimitiveKind.STRING) + override val descriptor: SerialDescriptor = + PrimitiveDescriptor("hep.dataforge.names.Name", PrimitiveKind.STRING) override fun deserialize(decoder: Decoder): Name { return decoder.decodeString().toName() @@ -99,7 +100,8 @@ data class NameToken(val body: String, val index: String = "") { @Serializer(NameToken::class) companion object : KSerializer { - override val descriptor: SerialDescriptor = PrimitiveDescriptor("hep.dataforge.names.NameToken", PrimitiveKind.STRING) + override val descriptor: SerialDescriptor = + PrimitiveDescriptor("hep.dataforge.names.NameToken", PrimitiveKind.STRING) override fun deserialize(decoder: Decoder): NameToken { return decoder.decodeString().toName().first()!! @@ -188,8 +190,12 @@ fun Name.isEmpty(): Boolean = this.length == 0 * Set or replace last token index */ fun Name.withIndex(index: String): Name { - val tokens = ArrayList(tokens) val last = NameToken(tokens.last().body, index) + if (length == 0) error("Can't add index to empty name") + if (length == 1) { + return last.asName() + } + val tokens = ArrayList(tokens) tokens.removeAt(tokens.size - 1) tokens.add(last) return Name(tokens) diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaDelegateTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaDelegateTest.kt index 277c2a6c..cb359a51 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaDelegateTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaDelegateTest.kt @@ -20,7 +20,7 @@ class MetaDelegateTest { class TestScheme : Scheme() { var myValue by string() var safeValue by double(2.2) - var enumValue by enum(TestEnum.YES) { enum() } + var enumValue by enum(TestEnum.YES) var inner by spec(InnerSpec) companion object : SchemeSpec(::TestScheme) diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/descriptors/DescriptorTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/descriptors/DescriptorTest.kt index 1fa07382..77e670c8 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/descriptors/DescriptorTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/descriptors/DescriptorTest.kt @@ -26,6 +26,6 @@ class DescriptorTest { @Test fun testAllowedValues() { val allowed = descriptor.nodes["aNode"]?.values?.get("b")?.allowedValues - assertEquals(allowed, emptyList()) + assertEquals(emptyList(), allowed) } } \ No newline at end of file diff --git a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnScheme.kt b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnScheme.kt index 7d364784..2b65b234 100644 --- a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnScheme.kt +++ b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnScheme.kt @@ -1,6 +1,5 @@ package hep.dataforge.tables -import hep.dataforge.meta.enum import hep.dataforge.meta.scheme.Scheme import hep.dataforge.meta.scheme.SchemeSpec import hep.dataforge.meta.scheme.enum @@ -14,5 +13,5 @@ open class ColumnScheme : Scheme() { } class ValueColumnScheme : ColumnScheme() { - var valueType by enum(ValueType.STRING){enum()} + var valueType by enum(ValueType.STRING) } \ No newline at end of file From e0aab579537fda9f48e068a16f2ca7eff57760e6 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 25 Mar 2020 18:52:58 +0300 Subject: [PATCH 02/11] Indexed access test --- .../kotlin/hep/dataforge/meta/mapMeta.kt | 2 -- .../kotlin/hep/dataforge/meta/metaMatcher.kt | 3 ++- .../kotlin/hep/dataforge/meta/MetaTest.kt | 17 +++++++++++++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt index df3e97a5..921fe0c2 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt @@ -6,7 +6,6 @@ import hep.dataforge.values.Value /** * Convert meta to map of maps */ -@DFExperimental fun Meta.toMap(descriptor: NodeDescriptor? = null): Map { return items.entries.associate { (token, item) -> token.toString() to when (item) { @@ -19,7 +18,6 @@ fun Meta.toMap(descriptor: NodeDescriptor? = null): Map { /** * Convert map of maps to meta */ -@DFExperimental fun Map.toMeta(descriptor: NodeDescriptor? = null): Meta = Meta { entries.forEach { (key, value) -> @Suppress("UNCHECKED_CAST") diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt index 03df113d..f2f747db 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/metaMatcher.kt @@ -4,7 +4,8 @@ import hep.dataforge.names.Name import hep.dataforge.names.toName /** - * Get all items matching given name. + * Get all items matching given name. The index of the last element, if present is used as a [Regex], + * against which indexes of elements are matched. */ fun Meta.getIndexed(name: Name): Map> { val root = when (name.length) { diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt index c55cd4ad..55cc461d 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt @@ -32,7 +32,7 @@ class MetaTest { } @Test - fun metaToMap(){ + fun metaToMap() { val meta = Meta { "a" put 22 "b" put { @@ -47,6 +47,19 @@ class MetaTest { val map = meta.toMap() val reconstructed = map.toMeta() - assertEquals(meta,reconstructed) + assertEquals(meta, reconstructed) + } + + @Test + fun indexed() { + val meta = Meta { + (0..20).forEach { + set("a[$it]", it) + } + } + val indexed = meta.getIndexed("a[1.]") + assertEquals(10, indexed.size) + assertEquals(null, indexed["8"]) + assertEquals(12, indexed["12"].int) } } \ No newline at end of file From 5c3d51de581d293ef6f8115b067cf0e4428b0c90 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 25 Mar 2020 19:06:08 +0300 Subject: [PATCH 03/11] Fix enum delegate bug (KT-37503) --- build.gradle.kts | 2 +- .../kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index ad2766a4..e3972ef4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -6,7 +6,7 @@ plugins { id("scientifik.publish") version toolsVersion apply false } -val dataforgeVersion by extra("0.1.6-dev") +val dataforgeVersion by extra("0.1.6-dev-1") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt index ae6722cf..2ee2c282 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt @@ -171,7 +171,10 @@ fun Configurable.float(default: Float, key: Name? = null): ReadWriteProperty> Configurable.enum( default: E, key: Name? = null ): ReadWriteProperty = - item(default, key).transform { item -> item?.string?.let { enumValueOf(it) } ?: default } + item(default, key).transform { item -> item?.string?.let {str-> + @Suppress("USELESS_CAST") + enumValueOf(str) as E + } ?: default } /* * Extra delegates for special cases From 66cee2d42b3abb600544a99a3c3292d3159b76db Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Thu, 26 Mar 2020 21:11:23 +0300 Subject: [PATCH 04/11] Specification invoke made inline --- .../kotlin/hep/dataforge/meta/scheme/Specification.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Specification.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Specification.kt index c2c6f8bb..d4685926 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Specification.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Specification.kt @@ -17,8 +17,6 @@ interface Specification { return wrap(config).apply(action) } - operator fun invoke(action: T.() -> Unit) = empty().apply(action) - fun empty() = wrap() /** @@ -37,6 +35,8 @@ interface Specification { fun wrap(default: Meta): T = wrap(Config()) { default[it] } } +inline operator fun Specification.invoke(action: T.() -> Unit) = empty().apply(action) + /** * Apply specified configuration to configurable */ From 7f0ad68d9da5e79a9553d794644781cebc7c8066 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 28 Mar 2020 10:04:59 +0300 Subject: [PATCH 05/11] Meta to JSON SNS fix --- .../kotlin/hep/dataforge/data/DataFilter.kt | 6 +- .../kotlin/hep/dataforge/meta/JsonMeta.kt | 83 +++++++++++++++---- .../kotlin/hep/dataforge/meta/MutableMeta.kt | 2 +- .../hep/dataforge/meta/scheme/Scheme.kt | 3 + .../dataforge/meta/scheme/Specification.kt | 33 ++++---- .../kotlin/hep/dataforge/meta/JsonMetaTest.kt | 39 +++++++++ 6 files changed, 131 insertions(+), 35 deletions(-) create mode 100644 dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/JsonMetaTest.kt diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt index bc9caaec..e83dba96 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt @@ -1,9 +1,11 @@ package hep.dataforge.data -import hep.dataforge.meta.* +import hep.dataforge.meta.Meta +import hep.dataforge.meta.isEmpty import hep.dataforge.meta.scheme.Scheme import hep.dataforge.meta.scheme.SchemeSpec import hep.dataforge.meta.scheme.string +import hep.dataforge.meta.scheme.wrap import hep.dataforge.names.toName @@ -55,4 +57,4 @@ fun DataNode.filter(filter: Meta): DataNode = filter(DataFilter. * Filter data using [DataFilter] builder */ fun DataNode.filter(filterBuilder: DataFilter.() -> Unit): DataNode = - filter(DataFilter.invoke(filterBuilder)) \ No newline at end of file + filter(DataFilter(filterBuilder)) \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/JsonMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/JsonMeta.kt index ee906008..276d268e 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/JsonMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/JsonMeta.kt @@ -3,6 +3,7 @@ package hep.dataforge.meta import hep.dataforge.meta.descriptors.ItemDescriptor import hep.dataforge.meta.descriptors.NodeDescriptor import hep.dataforge.meta.descriptors.ValueDescriptor +import hep.dataforge.meta.scheme.getProperty import hep.dataforge.names.NameToken import hep.dataforge.names.toName import hep.dataforge.values.* @@ -26,27 +27,75 @@ fun Value.toJson(descriptor: ValueDescriptor? = null): JsonElement { } //Use these methods to customize JSON key mapping -private fun NameToken.toJsonKey(descriptor: ItemDescriptor?) = toString() +private fun String.toJsonKey(descriptor: ItemDescriptor?) = descriptor?.getProperty("jsonName").string ?: toString() //private fun NodeDescriptor?.getDescriptor(key: String) = this?.items?.get(key) -fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject { +/** + * Convert given [Meta] to [JsonObject]. Primitives and nodes are copied as is, same name siblings are treated as json arrays + */ +fun Meta.toJson(descriptor: NodeDescriptor? = null, index: String? = null): JsonObject { - //TODO search for same name siblings and arrange them into arrays - val map = this.items.entries.associate { (name, item) -> - val itemDescriptor = descriptor?.items?.get(name.body) - val key = name.toJsonKey(itemDescriptor) - val value = when (item) { - is MetaItem.ValueItem -> { - item.value.toJson(itemDescriptor as? ValueDescriptor) + val elementMap = HashMap() + + fun MetaItem<*>.toJsonElement(itemDescriptor: ItemDescriptor?, index: String? = null): JsonElement = when (this) { + is MetaItem.ValueItem -> { + value.toJson(itemDescriptor as? ValueDescriptor) + } + is MetaItem.NodeItem -> { + node.toJson(itemDescriptor as? NodeDescriptor, index) + } + } + + fun addElement(key: String) { + val itemDescriptor = descriptor?.items?.get(key) + val jsonKey = key.toJsonKey(itemDescriptor) + val items = getIndexed(key) + when (items.size) { + 0 -> { + //do nothing } - is MetaItem.NodeItem -> { - item.node.toJson(itemDescriptor as? NodeDescriptor) + 1 -> { + elementMap[jsonKey] = items.values.first().toJsonElement(itemDescriptor) + } + else -> { + val array = jsonArray { + items.forEach { (index, item) -> + +item.toJsonElement(itemDescriptor, index) + } + } + elementMap[jsonKey] = array } } - key to value } - return JsonObject(map) + + ((descriptor?.items?.keys ?: emptySet()) + items.keys.map { it.body }).forEach(::addElement) + + + if (index != null) { + elementMap["@index"] = JsonPrimitive(index) + } + + return JsonObject(elementMap) + +// // use descriptor keys in the order they are declared +// val keys = (descriptor?.items?.keys ?: emptySet()) + this.items.keys.map { it.body } +// +// //TODO search for same name siblings and arrange them into arrays +// val map = this.items.entries.associate { (name, item) -> +// val itemDescriptor = descriptor?.items?.get(name.body) +// val key = name.toJsonKey(itemDescriptor) +// val value = when (item) { +// is MetaItem.ValueItem -> { +// item.value.toJson(itemDescriptor as? ValueDescriptor) +// } +// is MetaItem.NodeItem -> { +// item.node.toJson(itemDescriptor as? NodeDescriptor) +// } +// } +// key to value +// } +// return JsonObject(map) } fun JsonObject.toMeta(descriptor: NodeDescriptor? = null): Meta = JsonMeta(this, descriptor) @@ -88,6 +137,9 @@ fun JsonElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem { - this[key] = MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor)) as MetaItem + this[key] = + MetaItem.ValueItem(value.toValue(itemDescriptor as? ValueDescriptor)) as MetaItem } is JsonObject -> { this[key] = MetaItem.NodeItem( @@ -125,7 +178,7 @@ class JsonMeta(val json: JsonObject, val descriptor: NodeDescriptor? = null) : M } override val items: Map> by lazy { - val map = HashMap>() + val map = LinkedHashMap>() json.forEach { (key, value) -> map[key] = value } map.mapKeys { it.key.toName().first()!! } } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt index 043283e3..8071d171 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt @@ -17,7 +17,7 @@ interface MutableMeta> : MetaNode { * Changes in Meta are not thread safe. */ abstract class AbstractMutableMeta> : AbstractMetaNode(), MutableMeta { - protected val _items: MutableMap> = HashMap() + protected val _items: MutableMap> = LinkedHashMap() override val items: Map> get() = _items diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Scheme.kt index be954bbe..6a13c930 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Scheme.kt @@ -68,6 +68,9 @@ open class SchemeSpec(val builder: () -> T) : Specification { this.defaultProvider = defaultProvider } } + + @Suppress("OVERRIDE_BY_INLINE") + final override inline operator fun invoke(action: T.() -> Unit) = empty().apply(action) } /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Specification.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Specification.kt index d4685926..c5c2c787 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Specification.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Specification.kt @@ -10,13 +10,6 @@ import kotlin.jvm.JvmName * */ interface Specification { - /** - * Update given configuration using given type as a builder - */ - fun update(config: Config, action: T.() -> Unit): T { - return wrap(config).apply(action) - } - fun empty() = wrap() /** @@ -24,18 +17,24 @@ interface Specification { */ fun wrap(config: Config = Config(), defaultProvider: (Name) -> MetaItem<*>? = { null }): T - /** - * Wrap a configuration using static meta as default - */ - fun wrap(config: Config = Config(), default: Meta): T = wrap(config) { default[it] } - - /** - * Wrap a configuration using static meta as default - */ - fun wrap(default: Meta): T = wrap(Config()) { default[it] } + operator fun invoke(action: T.() -> Unit) = empty().apply(action) } -inline operator fun Specification.invoke(action: T.() -> Unit) = empty().apply(action) +/** + * Update given configuration using given type as a builder + */ +fun Specification.update(config: Config, action: T.() -> Unit): T = wrap(config).apply(action) + +/** + * Wrap a configuration using static meta as default + */ +fun Specification.wrap(config: Config = Config(), default: Meta): T = wrap(config) { default[it] } + +/** + * Wrap a configuration using static meta as default + */ +fun Specification.wrap(default: Meta): T = wrap(Config()) { default[it] } + /** * Apply specified configuration to configurable diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/JsonMetaTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/JsonMetaTest.kt new file mode 100644 index 00000000..127649da --- /dev/null +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/JsonMetaTest.kt @@ -0,0 +1,39 @@ +package hep.dataforge.meta + +import kotlinx.serialization.json.int +import kotlinx.serialization.json.json +import kotlinx.serialization.json.jsonArray +import kotlin.test.Test +import kotlin.test.assertEquals + +class JsonMetaTest { + val json = json { + "firstValue" to "a" + "secondValue" to "b" + "array" to jsonArray { + +"1" + +"2" + +"3" + } + "nodeArray" to jsonArray { + +json { + "index" to 1 + } + +json { + "index" to 2 + } + +json { + "index" to 3 + } + } + } + + @Test + fun jsonMetaConversion() { + val meta = json.toMeta() + val reconstructed = meta.toJson() + println(json) + println(reconstructed) + assertEquals(2, reconstructed["nodeArray"]?.jsonArray?.get(1)?.jsonObject?.get("index")?.int) + } +} \ No newline at end of file From 01711c2b8c909b9110404229d00f368ba76daeeb Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 28 Mar 2020 10:25:24 +0300 Subject: [PATCH 06/11] Move scheme to root meta package to avoid import clutter --- .../dataforge/descriptors/reflectiveDescriptors.kt | 12 ------------ .../kotlin/hep/dataforge/data/DataFilter.kt | 7 +------ .../kotlin/hep/dataforge/io/EnvelopeBuilder.kt | 2 +- .../dataforge/io/functions/RemoteFunctionClient.kt | 1 - .../dataforge/io/functions/RemoteFunctionServer.kt | 1 - .../kotlin/hep/dataforge/io/MultipartTest.kt | 1 - .../hep/dataforge/meta/{scheme => }/Configurable.kt | 3 +-- .../meta/{scheme => }/ConfigurableDelegate.kt | 3 +-- .../commonMain/kotlin/hep/dataforge/meta/JsonMeta.kt | 3 ++- .../kotlin/hep/dataforge/meta/MutableMeta.kt | 1 - .../kotlin/hep/dataforge/meta/{scheme => }/Scheme.kt | 6 +++--- .../hep/dataforge/meta/{scheme => }/Specification.kt | 3 +-- .../hep/dataforge/meta/descriptors/DescriptorMeta.kt | 3 +++ .../hep/dataforge/meta/descriptors/ItemDescriptor.kt | 1 - .../commonMain/kotlin/hep/dataforge/meta/mapMeta.kt | 2 ++ .../kotlin/hep/dataforge/meta/MetaBuilderTest.kt | 4 ++-- .../kotlin/hep/dataforge/meta/MetaDelegateTest.kt | 1 - .../kotlin/hep/dataforge/meta/SchemeTest.kt | 2 -- .../kotlin/hep/dataforge/meta/SpecificationTest.kt | 3 --- .../kotlin/hep/dataforge/scripting/BuildersKtTest.kt | 1 - .../kotlin/hep/dataforge/tables/ColumnScheme.kt | 8 ++++---- .../hep/dataforge/workspace/SimpleWorkspaceTest.kt | 1 - 22 files changed, 21 insertions(+), 48 deletions(-) rename dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/{scheme => }/Configurable.kt (97%) rename dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/{scheme => }/ConfigurableDelegate.kt (99%) rename dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/{scheme => }/Scheme.kt (94%) rename dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/{scheme => }/Specification.kt (96%) diff --git a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/descriptors/reflectiveDescriptors.kt b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/descriptors/reflectiveDescriptors.kt index 0015436c..5df244dc 100644 --- a/dataforge-context/src/jvmMain/kotlin/hep/dataforge/descriptors/reflectiveDescriptors.kt +++ b/dataforge-context/src/jvmMain/kotlin/hep/dataforge/descriptors/reflectiveDescriptors.kt @@ -1,17 +1,5 @@ package hep.dataforge.descriptors -import hep.dataforge.meta.* -import hep.dataforge.meta.descriptors.ItemDescriptor -import hep.dataforge.meta.descriptors.NodeDescriptor -import hep.dataforge.meta.descriptors.attributes -import hep.dataforge.meta.scheme.ConfigurableDelegate -import hep.dataforge.meta.scheme.Scheme -import hep.dataforge.values.parseValue -import kotlin.reflect.KProperty1 -import kotlin.reflect.full.findAnnotation -import kotlin.reflect.full.isSubclassOf -import kotlin.reflect.full.memberProperties - //inline fun T.buildDescriptor(): NodeDescriptor = NodeDescriptor { // T::class.apply { diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt index e83dba96..b24f87a2 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataFilter.kt @@ -1,11 +1,6 @@ package hep.dataforge.data -import hep.dataforge.meta.Meta -import hep.dataforge.meta.isEmpty -import hep.dataforge.meta.scheme.Scheme -import hep.dataforge.meta.scheme.SchemeSpec -import hep.dataforge.meta.scheme.string -import hep.dataforge.meta.scheme.wrap +import hep.dataforge.meta.* import hep.dataforge.names.toName diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt index 94f4e59d..18c3fc04 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt @@ -34,7 +34,7 @@ class EnvelopeBuilder { /** * Construct a data binary from given builder */ - @ExperimentalIoApi + @OptIn(ExperimentalIoApi::class) fun data(block: Output.() -> Unit) { data = ArrayBinary.write(builder = block) } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt index 1df4b42d..7c294891 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionClient.kt @@ -6,7 +6,6 @@ import hep.dataforge.io.* import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.int -import hep.dataforge.meta.scheme.int import kotlin.reflect.KClass class RemoteFunctionClient(override val context: Context, val responder: Responder) : FunctionServer, ContextAware { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt index efd4351e..8252b1d3 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/functions/RemoteFunctionServer.kt @@ -8,7 +8,6 @@ import hep.dataforge.io.Responder import hep.dataforge.io.type import hep.dataforge.meta.get import hep.dataforge.meta.int -import hep.dataforge.meta.scheme.int class RemoteFunctionServer( override val context: Context, diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt index 04d097aa..0f60c077 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt @@ -3,7 +3,6 @@ package hep.dataforge.io import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.get import hep.dataforge.meta.int -import hep.dataforge.meta.scheme.int import kotlinx.io.text.writeRawString import kotlinx.io.text.writeUtf8String diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Configurable.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Configurable.kt similarity index 97% rename from dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Configurable.kt rename to dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Configurable.kt index 4ae5e180..b38b7d95 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Configurable.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Configurable.kt @@ -1,6 +1,5 @@ -package hep.dataforge.meta.scheme +package hep.dataforge.meta -import hep.dataforge.meta.* import hep.dataforge.meta.descriptors.* import hep.dataforge.names.Name import hep.dataforge.names.toName diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ConfigurableDelegate.kt similarity index 99% rename from dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt rename to dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ConfigurableDelegate.kt index 2ee2c282..1999ced4 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/ConfigurableDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/ConfigurableDelegate.kt @@ -1,6 +1,5 @@ -package hep.dataforge.meta.scheme +package hep.dataforge.meta -import hep.dataforge.meta.* import hep.dataforge.names.Name import hep.dataforge.names.asName import hep.dataforge.values.* diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/JsonMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/JsonMeta.kt index 276d268e..89ab1b3e 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/JsonMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/JsonMeta.kt @@ -1,9 +1,10 @@ +@file:Suppress("UNUSED_PARAMETER") + package hep.dataforge.meta import hep.dataforge.meta.descriptors.ItemDescriptor import hep.dataforge.meta.descriptors.NodeDescriptor import hep.dataforge.meta.descriptors.ValueDescriptor -import hep.dataforge.meta.scheme.getProperty import hep.dataforge.names.NameToken import hep.dataforge.names.toName import hep.dataforge.values.* diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt index 8071d171..bb280734 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMeta.kt @@ -1,6 +1,5 @@ package hep.dataforge.meta -import hep.dataforge.meta.scheme.Configurable import hep.dataforge.names.* import hep.dataforge.values.Value diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Scheme.kt similarity index 94% rename from dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Scheme.kt rename to dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Scheme.kt index 6a13c930..0229da8b 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Scheme.kt @@ -1,6 +1,5 @@ -package hep.dataforge.meta.scheme +package hep.dataforge.meta -import hep.dataforge.meta.* import hep.dataforge.meta.descriptors.* import hep.dataforge.names.Name import hep.dataforge.names.NameToken @@ -59,7 +58,8 @@ inline operator fun T.invoke(block: T.() -> Unit) = apply(block) /** * A specification for simplified generation of wrappers */ -open class SchemeSpec(val builder: () -> T) : Specification { +open class SchemeSpec(val builder: () -> T) : + Specification { override fun empty(): T = builder() override fun wrap(config: Config, defaultProvider: (Name) -> MetaItem<*>?): T { diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Specification.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt similarity index 96% rename from dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Specification.kt rename to dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt index c5c2c787..3724ce38 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/scheme/Specification.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt @@ -1,6 +1,5 @@ -package hep.dataforge.meta.scheme +package hep.dataforge.meta -import hep.dataforge.meta.* import hep.dataforge.names.Name import kotlin.jvm.JvmName diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/DescriptorMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/DescriptorMeta.kt index f95069d6..c0b27925 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/DescriptorMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/DescriptorMeta.kt @@ -5,6 +5,9 @@ import hep.dataforge.meta.MetaItem import hep.dataforge.names.NameToken import hep.dataforge.values.Null +/** + * A [Meta] that wraps a descriptor node + */ class DescriptorMeta(val descriptor: NodeDescriptor) : MetaBase() { override val items: Map> get() = descriptor.items.entries.associate { entry -> diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt index 54a01ace..e025cbb1 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/descriptors/ItemDescriptor.kt @@ -1,7 +1,6 @@ package hep.dataforge.meta.descriptors import hep.dataforge.meta.* -import hep.dataforge.meta.scheme.* import hep.dataforge.names.* import hep.dataforge.values.False import hep.dataforge.values.True diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt index 921fe0c2..0bd792af 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt @@ -1,3 +1,5 @@ +@file:Suppress("UNUSED_PARAMETER") + package hep.dataforge.meta import hep.dataforge.meta.descriptors.NodeDescriptor diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaBuilderTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaBuilderTest.kt index 3fe10ceb..593401d1 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaBuilderTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaBuilderTest.kt @@ -8,7 +8,7 @@ import kotlin.test.assertEquals class MetaBuilderTest { @Test fun testBuilder() { - val meta = buildMeta { + val meta = Meta { "a" put 22 "b" put listOf(1, 2, 3) this["c"] = "myValue".asValue() @@ -25,7 +25,7 @@ class MetaBuilderTest { @Test fun testSNS(){ - val meta = buildMeta { + val meta = Meta { repeat(10){ "b.a[$it]" put it } diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaDelegateTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaDelegateTest.kt index cb359a51..925ee36c 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaDelegateTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaDelegateTest.kt @@ -1,6 +1,5 @@ package hep.dataforge.meta -import hep.dataforge.meta.scheme.* import kotlin.test.Test import kotlin.test.assertEquals diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SchemeTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SchemeTest.kt index bcebedc6..939b8bb2 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SchemeTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SchemeTest.kt @@ -1,7 +1,5 @@ package hep.dataforge.meta -import hep.dataforge.meta.scheme.asScheme -import hep.dataforge.meta.scheme.getProperty import kotlin.test.Test import kotlin.test.assertEquals diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SpecificationTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SpecificationTest.kt index 6d99101a..f21c5b2c 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SpecificationTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/SpecificationTest.kt @@ -1,8 +1,5 @@ package hep.dataforge.meta -import hep.dataforge.meta.scheme.Scheme -import hep.dataforge.meta.scheme.Specification -import hep.dataforge.meta.scheme.numberList import hep.dataforge.names.Name import kotlin.test.Test import kotlin.test.assertEquals diff --git a/dataforge-scripting/src/jvmTest/kotlin/hep/dataforge/scripting/BuildersKtTest.kt b/dataforge-scripting/src/jvmTest/kotlin/hep/dataforge/scripting/BuildersKtTest.kt index 9fb1c919..6dd61105 100644 --- a/dataforge-scripting/src/jvmTest/kotlin/hep/dataforge/scripting/BuildersKtTest.kt +++ b/dataforge-scripting/src/jvmTest/kotlin/hep/dataforge/scripting/BuildersKtTest.kt @@ -3,7 +3,6 @@ package hep.dataforge.scripting import hep.dataforge.context.Global import hep.dataforge.meta.get import hep.dataforge.meta.int -import hep.dataforge.meta.scheme.int import hep.dataforge.workspace.SimpleWorkspaceBuilder import hep.dataforge.workspace.context import hep.dataforge.workspace.target diff --git a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnScheme.kt b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnScheme.kt index 2b65b234..c71ee53f 100644 --- a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnScheme.kt +++ b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/ColumnScheme.kt @@ -1,9 +1,9 @@ package hep.dataforge.tables -import hep.dataforge.meta.scheme.Scheme -import hep.dataforge.meta.scheme.SchemeSpec -import hep.dataforge.meta.scheme.enum -import hep.dataforge.meta.scheme.string +import hep.dataforge.meta.Scheme +import hep.dataforge.meta.SchemeSpec +import hep.dataforge.meta.enum +import hep.dataforge.meta.string import hep.dataforge.values.ValueType open class ColumnScheme : Scheme() { diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt index 8bd02c35..a4df6a4b 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/SimpleWorkspaceTest.kt @@ -6,7 +6,6 @@ import hep.dataforge.meta.boolean import hep.dataforge.meta.builder import hep.dataforge.meta.get import hep.dataforge.meta.int -import hep.dataforge.meta.scheme.int import hep.dataforge.names.plus import kotlin.test.Test import kotlin.test.assertEquals From 56e7d5545066649eeec7bd3f90eaeb394dcdc3e8 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 28 Mar 2020 21:23:44 +0300 Subject: [PATCH 07/11] Add a Config generator to MetaTransformation --- build.gradle.kts | 2 +- .../dataforge/io/yaml/YamlMetaFormatTest.kt | 2 +- .../kotlin/hep/dataforge/meta/mapMeta.kt | 29 ++++++++++++++----- .../transformations/MetaTransformation.kt | 18 ++++++++++-- .../kotlin/hep/dataforge/meta/MetaTest.kt | 1 + 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index e3972ef4..b5772512 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -6,7 +6,7 @@ plugins { id("scientifik.publish") version toolsVersion apply false } -val dataforgeVersion by extra("0.1.6-dev-1") +val dataforgeVersion by extra("0.1.6-dev-2") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt b/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt index 24fb6593..0bb1c5d0 100644 --- a/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt +++ b/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt @@ -34,7 +34,7 @@ class YamlMetaFormatTest { assertEquals(meta, meta.seal()) meta.items.keys.forEach { - if (meta[it] != result[it]) error("${meta[it]} != ${result[it]}") + assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}") } assertEquals(meta, result) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt index 0bd792af..ed77f7f0 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/mapMeta.kt @@ -1,8 +1,8 @@ -@file:Suppress("UNUSED_PARAMETER") - package hep.dataforge.meta import hep.dataforge.meta.descriptors.NodeDescriptor +import hep.dataforge.names.toName +import hep.dataforge.values.ListValue import hep.dataforge.values.Value /** @@ -18,14 +18,29 @@ fun Meta.toMap(descriptor: NodeDescriptor? = null): Map { } /** - * Convert map of maps to meta + * Convert map of maps to meta. This method will recognize [MetaItem], [Map] and [List] of all mentioned above as value. + * All other values will be converted to values. */ +@DFExperimental fun Map.toMeta(descriptor: NodeDescriptor? = null): Meta = Meta { + @Suppress("UNCHECKED_CAST") + fun toItem(value: Any?): MetaItem<*> = when (value) { + is MetaItem<*> -> value + is Meta -> MetaItem.NodeItem(value) + is Map<*, *> -> MetaItem.NodeItem((value as Map).toMeta()) + else -> MetaItem.ValueItem(Value.of(value)) + } + entries.forEach { (key, value) -> - @Suppress("UNCHECKED_CAST") - when (value) { - is Map<*, *> -> setNode(key, (value as Map).toMeta()) - else -> setValue(key, Value.of(value)) + if (value is List<*>) { + val items = value.map { toItem(it) } + if (items.all { it is MetaItem.ValueItem }) { + setValue(key, ListValue(items.map { it.value!! })) + } else { + setIndexedItems(key.toName(), value.map { toItem(it) }) + } + } else { + setItem(key, toItem(value)) } } } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt index d6f3bedf..f02545fe 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/transformations/MetaTransformation.kt @@ -22,7 +22,7 @@ interface TransformationRule { meta.sequence().filter { matches(it.first, it.second) }.map { it.first } /** - * Apply transformation for a single item (Node or Value) and return resulting tree with absolute path + * Apply transformation for a single item (Node or Value) to the target */ fun > transformItem(name: Name, item: MetaItem<*>?, target: M): Unit } @@ -89,7 +89,7 @@ inline class MetaTransformation(val transformations: Collection rule.selectItems(source).forEach { name -> @@ -98,6 +98,20 @@ inline class MetaTransformation(val transformations: Collection + rule.selectItems(source).forEach { name -> + rule.transformItem(name, source[name], this) + } + } + + bind(source, this) + } + /** * Transform a meta, replacing all elements found in rules with transformed entries */ diff --git a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt index 55cc461d..12569bc3 100644 --- a/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/hep/dataforge/meta/MetaTest.kt @@ -31,6 +31,7 @@ class MetaTest { assertEquals(meta1, meta2) } + @OptIn(DFExperimental::class) @Test fun metaToMap() { val meta = Meta { From 7efa19920be48254b07c93478e49bfcc655d02f2 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 3 Apr 2020 12:10:24 +0300 Subject: [PATCH 08/11] Migrate to io-dev-6 --- build.gradle.kts | 2 +- dataforge-io/build.gradle.kts | 2 +- .../io/yaml/FrontMatterEnvelopeFormat.kt | 8 +-- .../hep/dataforge/io/yaml/YamlMetaFormat.kt | 16 +----- .../hep/dataforge/io/EnvelopeBuilder.kt | 9 ++- .../kotlin/hep/dataforge/io/EnvelopeParts.kt | 2 - .../kotlin/hep/dataforge/io/IOFormat.kt | 6 +- .../kotlin/hep/dataforge/io/JsonMetaFormat.kt | 3 +- .../kotlin/hep/dataforge/io/MetaFormat.kt | 13 +++-- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 16 ++---- .../hep/dataforge/io/TaglessEnvelopeFormat.kt | 18 ++---- .../kotlin/hep/dataforge/io/ioMisc.kt | 22 +++++++ .../hep/dataforge/io/EnvelopeFormatTest.kt | 4 +- .../kotlin/hep/dataforge/io/MetaFormatTest.kt | 23 ++++---- .../kotlin/hep/dataforge/io/MultipartTest.kt | 5 +- .../kotlin/hep/dataforge/io/ioTestUtils.kt | 7 +++ .../kotlin/hep/dataforge/io/FileEnvelope.kt | 22 ------- .../jvmMain/kotlin/hep/dataforge/io/fileIO.kt | 55 ++++++++++++++++-- .../kotlin/hep/dataforge/io/tcp/streams.kt | 57 +++++++------------ .../dataforge/io/tcp/EnvelopeServerTest.kt | 4 +- .../kotlin/hep/dataforge/meta/Scheme.kt | 12 +--- .../hep/dataforge/tables/io/TextRows.kt | 5 +- .../dataforge/tables/io/textTableEnvelope.kt | 3 +- .../hep/dataforge/workspace/envelopeData.kt | 10 +--- 24 files changed, 157 insertions(+), 167 deletions(-) create mode 100644 dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt create mode 100644 dataforge-io/src/commonTest/kotlin/hep/dataforge/io/ioTestUtils.kt delete mode 100644 dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt diff --git a/build.gradle.kts b/build.gradle.kts index b5772512..95da230e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,6 +1,6 @@ plugins { - val toolsVersion = "0.4.0" + val toolsVersion = "0.4.1" id("scientifik.mpp") version toolsVersion apply false id("scientifik.jvm") version toolsVersion apply false id("scientifik.publish") version toolsVersion apply false diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index 1760f613..5a94bae5 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -11,7 +11,7 @@ serialization(sourceSet = TEST){ cbor() } -val ioVersion by rootProject.extra("0.2.0-npm-dev-4") +val ioVersion by rootProject.extra("0.2.0-npm-dev-6") kotlin { sourceSets { diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt index 6361c5dd..bac63c9d 100644 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -7,7 +7,6 @@ import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.io.* import kotlinx.io.text.readUtf8Line -import kotlinx.io.text.writeRawString import kotlinx.io.text.writeUtf8String import kotlinx.serialization.toUtf8Bytes @@ -29,7 +28,8 @@ class FrontMatterEnvelopeFormat( metaTypeRegex.matchEntire(line)?.groupValues?.first() ?.let { io.metaFormat(it) } ?: YamlMetaFormat - val meta = buildBytes { + //TODO replace by preview + val meta = Binary { do { line = readUtf8Line() writeUtf8String(line + "\r\n") @@ -53,7 +53,7 @@ class FrontMatterEnvelopeFormat( metaTypeRegex.matchEntire(line)?.groupValues?.first() ?.let { io.metaFormat(it) } ?: YamlMetaFormat - val meta = buildBytes { + val meta = Binary { do { writeUtf8String(readUtf8Line() + "\r\n") } while (!line.startsWith(SEPARATOR)) @@ -62,7 +62,7 @@ class FrontMatterEnvelopeFormat( readMeta() } } - val bytes = readRemaining() + val bytes = readByteArray() val data = bytes.asBinary() return SimpleEnvelope(meta, data) } diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt index d1ab09e4..71be6352 100644 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt @@ -10,24 +10,12 @@ import hep.dataforge.meta.toMap import hep.dataforge.meta.toMeta import kotlinx.io.Input import kotlinx.io.Output +import kotlinx.io.asInputStream import kotlinx.io.readUByte import kotlinx.io.text.writeUtf8String import org.yaml.snakeyaml.Yaml import java.io.InputStream -private class InputAsStream(val input: Input) : InputStream() { - override fun read(): Int { - if (input.eof()) return -1 - return input.readUByte().toInt() - } - - override fun close() { - input.close() - } -} - -private fun Input.asStream() = InputAsStream(this) - @DFExperimental class YamlMetaFormat(val meta: Meta) : MetaFormat { private val yaml = Yaml() @@ -38,7 +26,7 @@ class YamlMetaFormat(val meta: Meta) : MetaFormat { } override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { - val map: Map = yaml.load(asStream()) + val map: Map = yaml.load(asInputStream()) return map.toMeta(descriptor) } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt index 18c3fc04..c37f7c38 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt @@ -1,10 +1,7 @@ package hep.dataforge.io import hep.dataforge.meta.* -import kotlinx.io.ArrayBinary -import kotlinx.io.Binary -import kotlinx.io.ExperimentalIoApi -import kotlinx.io.Output +import kotlinx.io.* class EnvelopeBuilder { private val metaBuilder = MetaBuilder() @@ -36,7 +33,9 @@ class EnvelopeBuilder { */ @OptIn(ExperimentalIoApi::class) fun data(block: Output.() -> Unit) { - data = ArrayBinary.write(builder = block) + val arrayBuilder = ByteArrayOutput() + arrayBuilder.block() + data = arrayBuilder.toByteArray().asBinary() } fun build() = SimpleEnvelope(metaBuilder.seal(), data) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt index 9541c8fb..4780a71a 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt @@ -11,8 +11,6 @@ import hep.dataforge.meta.* import hep.dataforge.names.asName import hep.dataforge.names.plus import hep.dataforge.names.toName -import kotlinx.io.text.readRawString -import kotlinx.io.text.writeRawString object EnvelopeParts { val MULTIPART_KEY = "multipart".asName() diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index 44f68738..1300afe7 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -68,11 +68,7 @@ interface IOFormatFactory : Factory>, Named { } } -fun IOFormat.writeBytes(obj: T): Bytes = buildBytes { writeObject(obj) } - - -fun IOFormat.writeByteArray(obj: T): ByteArray = buildBytes { writeObject(obj) }.toByteArray() -fun IOFormat.readByteArray(array: ByteArray): T = array.asBinary().read { readObject() } +fun IOFormat.toBinary(obj: T): Binary = Binary { writeObject(obj) } object DoubleIOFormat : IOFormat, IOFormatFactory { override fun invoke(meta: Meta, context: Context): IOFormat = this diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index 01606e6d..39902338 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -11,6 +11,7 @@ import hep.dataforge.meta.toJson import hep.dataforge.meta.toMetaItem import kotlinx.io.Input import kotlinx.io.Output +import kotlinx.io.readByteArray import kotlinx.io.text.readUtf8String import kotlinx.io.text.writeUtf8String import kotlinx.serialization.UnstableDefault @@ -26,7 +27,7 @@ class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat { } override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { - val str = readUtf8String() + val str = readByteArray().decodeToString() val jsonElement = json.parseJson(str) val item = jsonElement.toMetaItem(descriptor) return item.node ?: Meta.EMPTY diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt index 4776e015..4b893a4a 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt @@ -1,14 +1,17 @@ package hep.dataforge.io import hep.dataforge.context.Context -import hep.dataforge.meta.descriptors.NodeDescriptor import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE import hep.dataforge.meta.Meta +import hep.dataforge.meta.descriptors.NodeDescriptor import hep.dataforge.names.Name import hep.dataforge.names.asName import hep.dataforge.names.plus import hep.dataforge.provider.Type -import kotlinx.io.* +import kotlinx.io.ByteArrayInput +import kotlinx.io.Input +import kotlinx.io.Output +import kotlinx.io.use import kotlin.reflect.KClass /** @@ -44,14 +47,14 @@ interface MetaFormatFactory : IOFormatFactory, MetaFormat { } } -fun Meta.toString(format: MetaFormat): String = buildBytes { +fun Meta.toString(format: MetaFormat): String = buildByteArray { format.run { writeObject(this@toString) } -}.toByteArray().decodeToString() +}.decodeToString() fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory()) fun MetaFormat.parse(str: String): Meta { - return str.encodeToByteArray().read { readObject() } + return ByteArrayInput(str.encodeToByteArray()).use { it.readObject() } } fun MetaFormatFactory.parse(str: String, formatMeta: Meta): Meta = invoke(formatMeta).parse(str) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index 0e18fa35..69c33bbd 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -8,8 +8,6 @@ import hep.dataforge.names.Name import hep.dataforge.names.plus import hep.dataforge.names.toName import kotlinx.io.* -import kotlinx.io.text.readRawString -import kotlinx.io.text.writeRawString @ExperimentalIoApi class TaggedEnvelopeFormat( @@ -21,7 +19,7 @@ class TaggedEnvelopeFormat( // ?: error("Meta format with key $metaFormatKey could not be resolved in $io") - private fun Tag.toBytes() = buildBytes(24) { + private fun Tag.toBinary() = Binary(24) { writeRawString(START_SEQUENCE) writeRawString(version.name) writeShort(metaFormatKey) @@ -39,14 +37,10 @@ class TaggedEnvelopeFormat( override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) { val metaFormat = metaFormatFactory.invoke(formatMeta, io.context) - val metaBytes = metaFormat.writeBytes(envelope.meta) - val actualSize: ULong = if (envelope.data == null) { - 0 - } else { - envelope.data?.size ?: Binary.INFINITE - }.toULong() + val metaBytes = metaFormat.toBinary(envelope.meta) + val actualSize: ULong = (envelope.data?.size ?: 0).toULong() val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, actualSize) - writeBinary(tag.toBytes()) + writeBinary(tag.toBinary()) writeBinary(metaBytes) writeRawString("\r\n") envelope.data?.let { @@ -73,7 +67,7 @@ class TaggedEnvelopeFormat( } } - val data = ByteArray(tag.dataSize.toInt()).also { readArray(it) }.asBinary() + val data = ByteArray(tag.dataSize.toInt()).also { readByteArray(it) }.asBinary() return SimpleEnvelope(meta, data) } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt index 7023ce1a..f29132c8 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt @@ -4,9 +4,7 @@ import hep.dataforge.context.Context import hep.dataforge.meta.* import hep.dataforge.names.asName import kotlinx.io.* -import kotlinx.io.text.readRawString import kotlinx.io.text.readUtf8Line -import kotlinx.io.text.writeRawString import kotlinx.io.text.writeUtf8String @ExperimentalIoApi @@ -31,17 +29,13 @@ class TaglessEnvelopeFormat( //printing all properties writeProperty(META_TYPE_PROPERTY, metaFormatFactory.shortName) //TODO add optional metaFormat properties - val actualSize: Int = if (envelope.data == null) { - 0 - } else { - envelope.data?.size ?: Binary.INFINITE - } + val actualSize: Int = envelope.data?.size ?: 0 writeProperty(DATA_LENGTH_PROPERTY, actualSize) //Printing meta if (!envelope.meta.isEmpty()) { - val metaBytes = metaFormat.writeBytes(envelope.meta) + val metaBytes = metaFormat.toBinary(envelope.meta) writeProperty(META_LENGTH_PROPERTY, metaBytes.size + 2) writeUtf8String(metaStart + "\r\n") writeBinary(metaBytes) @@ -71,7 +65,7 @@ class TaglessEnvelopeFormat( properties[key] = value } //If can't read line, return envelope without data - if (eof()) return SimpleEnvelope(Meta.EMPTY, null) + if (exhausted()) return SimpleEnvelope(Meta.EMPTY, null) line = readUtf8Line() } @@ -102,11 +96,11 @@ class TaglessEnvelopeFormat( val data: Binary? = if (properties.containsKey(DATA_LENGTH_PROPERTY)) { val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt()) - readArray(bytes) + readByteArray(bytes) bytes.asBinary() } else { - ArrayBinary.write { - writeInput(this@readObject) + Binary { + copyTo(this) } } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt new file mode 100644 index 00000000..eb5341d6 --- /dev/null +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt @@ -0,0 +1,22 @@ +package hep.dataforge.io + +import kotlinx.io.* + +fun Output.writeRawString(str: String) { + str.forEach { writeByte(it.toByte()) } +} + +fun Input.readRawString(size: Int): String { + val array = CharArray(size) { readByte().toChar() } + return String(array) +} + +inline fun buildByteArray(expectedSize: Int = 16, block: Output.() -> Unit): ByteArray = + ByteArrayOutput(expectedSize).apply(block).toByteArray() + +@Suppress("FunctionName") +inline fun Binary(expectedSize: Int = 16, block: Output.() -> Unit): Binary = + buildByteArray(expectedSize, block).asBinary() + +@Deprecated("To be replaced by Binary.EMPTY",level = DeprecationLevel.WARNING) +val EmptyBinary = ByteArrayBinary(ByteArray(0)) \ No newline at end of file diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt index 0851d6df..0fd09abf 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopeFormatTest.kt @@ -23,7 +23,7 @@ class EnvelopeFormatTest { @Test fun testTaggedFormat(){ TaggedEnvelopeFormat.run { - val byteArray = this.writeByteArray(envelope) + val byteArray = this.toByteArray(envelope) //println(byteArray.decodeToString()) val res = readByteArray(byteArray) assertEquals(envelope.meta,res.meta) @@ -37,7 +37,7 @@ class EnvelopeFormatTest { @Test fun testTaglessFormat(){ TaglessEnvelopeFormat.run { - val byteArray = writeByteArray(envelope) + val byteArray = toByteArray(envelope) //println(byteArray.decodeToString()) val res = readByteArray(byteArray) assertEquals(envelope.meta,res.meta) diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt index 6fc801ec..dcfc2265 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaFormatTest.kt @@ -1,20 +1,19 @@ package hep.dataforge.io import hep.dataforge.meta.* -import kotlinx.io.Bytes -import kotlinx.io.buildBytes +import kotlinx.io.asBinary import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.json import kotlinx.serialization.json.jsonArray import kotlin.test.Test import kotlin.test.assertEquals -fun Meta.toBytes(format: MetaFormat = JsonMetaFormat): Bytes = buildBytes { - format.run { writeObject(this@toBytes) } +fun Meta.toByteArray(format: MetaFormat = JsonMetaFormat) = buildByteArray { + format.run { writeObject(this@toByteArray) } } -fun MetaFormat.fromBytes(packet: Bytes): Meta { - return packet.read { readObject() } +fun MetaFormat.fromByteArray(packet: ByteArray): Meta { + return packet.asBinary().read { readObject() } } class MetaFormatTest { @@ -28,8 +27,8 @@ class MetaFormatTest { "array" put doubleArrayOf(1.0, 2.0, 3.0) } } - val bytes = meta.toBytes(BinaryMetaFormat) - val result = BinaryMetaFormat.fromBytes(bytes) + val bytes = meta.toByteArray(BinaryMetaFormat) + val result = BinaryMetaFormat.fromByteArray(bytes) assertEquals(meta, result) } @@ -56,12 +55,12 @@ class MetaFormatTest { } @Test - fun testJsonToMeta(){ - val json = jsonArray{ + fun testJsonToMeta() { + val json = jsonArray { //top level array +jsonArray { +JsonPrimitive(88) - +json{ + +json { "c" to "aasdad" "d" to true } @@ -77,7 +76,7 @@ class MetaFormatTest { assertEquals(true, meta["@value[0].@value[1].d"].boolean) assertEquals("value", meta["@value[1]"].string) - assertEquals(listOf(1.0,2.0,3.0),meta["@value[2"].value?.list?.map{it.number.toDouble()}) + assertEquals(listOf(1.0, 2.0, 3.0), meta["@value[2"].value?.list?.map { it.number.toDouble() }) } } \ No newline at end of file diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt index 0f60c077..344edf59 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt @@ -3,7 +3,6 @@ package hep.dataforge.io import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.get import hep.dataforge.meta.int -import kotlinx.io.text.writeRawString import kotlinx.io.text.writeUtf8String import kotlin.test.Test @@ -33,9 +32,9 @@ class MultipartTest { @Test fun testParts() { TaggedEnvelopeFormat.run { - val singleEnvelopeData = writeBytes(envelopes[0]) + val singleEnvelopeData = toBinary(envelopes[0]) val singleEnvelopeSize = singleEnvelopeData.size - val bytes = writeBytes(partsEnvelope) + val bytes = toBinary(partsEnvelope) assertTrue(5*singleEnvelopeSize < bytes.size) val reconstructed = bytes.readWith(this) val parts = reconstructed.parts()?.toList() ?: emptyList() diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/ioTestUtils.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/ioTestUtils.kt new file mode 100644 index 00000000..2694ff8f --- /dev/null +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/ioTestUtils.kt @@ -0,0 +1,7 @@ +package hep.dataforge.io + +import kotlinx.io.ByteArrayInput +import kotlinx.io.use + +fun IOFormat.toByteArray(obj: T): ByteArray = buildByteArray { writeObject(obj) } +fun IOFormat.readByteArray(array: ByteArray): T = ByteArrayInput(array).use { it.readObject() } \ No newline at end of file diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt deleted file mode 100644 index 21cca102..00000000 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt +++ /dev/null @@ -1,22 +0,0 @@ -package hep.dataforge.io - -import hep.dataforge.meta.Meta -import kotlinx.io.Binary -import kotlinx.io.ExperimentalIoApi -import kotlinx.io.FileBinary -import kotlinx.io.read -import java.nio.file.Path - -@ExperimentalIoApi -class FileEnvelope internal constructor(val path: Path, val format: EnvelopeFormat) : Envelope { - //TODO do not like this constructor. Hope to replace it later - - private val partialEnvelope: PartialEnvelope = path.read { - format.run { readPartial() } - } - - override val meta: Meta get() = partialEnvelope.meta - - override val data: Binary? = FileBinary(path, partialEnvelope.dataOffset.toInt(), partialEnvelope.dataSize?.toInt()) -} - diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt index c9b4f1bf..94b9b433 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt @@ -8,9 +8,52 @@ import hep.dataforge.meta.isEmpty import kotlinx.io.* import java.nio.file.Files import java.nio.file.Path +import java.nio.file.StandardOpenOption import kotlin.reflect.full.isSuperclassOf import kotlin.streams.asSequence +fun Path.read(block: Input.() -> R): R = asBinary().read(block = block) + +/** + * Write a live output to a newly created file. If file does not exist, throws error + */ +fun Path.write(block: Output.() -> Unit): Unit { + val stream = Files.newOutputStream(this, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW) + stream.asOutput().use(block) +} + +/** + * Create a new file or append to exiting one with given output [block] + */ +fun Path.append(block: Output.() -> Unit): Unit { + val stream = Files.newOutputStream( + this, + StandardOpenOption.WRITE, StandardOpenOption.APPEND, StandardOpenOption.CREATE + ) + stream.asOutput().use(block) +} + +/** + * Create a new file or replace existing one using given output [block] + */ +fun Path.rewrite(block: Output.() -> Unit): Unit { + val stream = Files.newOutputStream( + this, + StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE + ) + stream.asOutput().use(block) +} + +fun Path.readEnvelope(format: EnvelopeFormat): Envelope { + val partialEnvelope: PartialEnvelope = asBinary().read { + format.run { readPartial() } + } + val offset: Int = partialEnvelope.dataOffset.toInt() + val size: Int = partialEnvelope.dataSize?.toInt() ?: (Files.size(this).toInt() - offset) + val binary = FileBinary(this, offset, size) + return SimpleEnvelope(partialEnvelope.meta, binary) +} + /** * Resolve IOFormat based on type */ @@ -37,7 +80,7 @@ fun IOPlugin.readMetaFile(path: Path, formatOverride: MetaFormat? = null, descri val metaFormat = formatOverride ?: metaFormat(extension) ?: error("Can't resolve meta format $extension") return metaFormat.run { - actualPath.read{ + actualPath.read { readMeta(descriptor) } } @@ -59,7 +102,7 @@ fun IOPlugin.writeMetaFile( path } metaFormat.run { - actualPath.write{ + actualPath.write { writeMeta(meta, descriptor) } } @@ -131,7 +174,7 @@ fun IOPlugin.readEnvelopeFile( } return formatPeeker(path)?.let { format -> - FileEnvelope(path, format) + path.readEnvelope(format) } ?: if (readNonEnvelopes) { // if no format accepts file, read it as binary SimpleEnvelope(Meta.EMPTY, path.asBinary()) } else null @@ -156,7 +199,7 @@ fun IOPlugin.writeEnvelopeFile( envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat, metaFormat: MetaFormatFactory? = null ) { - path.write { + path.rewrite { with(envelopeFormat) { writeEnvelope(envelope, metaFormat ?: envelopeFormat.defaultMetaFormat) } @@ -184,8 +227,8 @@ fun IOPlugin.writeEnvelopeDirectory( val dataFile = path.resolve(IOPlugin.DATA_FILE_NAME) dataFile.write { envelope.data?.read { - val copied = writeInput(this) - if (envelope.data?.size != Binary.INFINITE && copied != envelope.data?.size) { + val copied = copyTo(this@write) + if (copied != envelope.data?.size) { error("The number of copied bytes does not equal data size") } } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/streams.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/streams.kt index 2c240f77..ec79eb22 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/streams.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/streams.kt @@ -1,62 +1,45 @@ package hep.dataforge.io.tcp -import kotlinx.io.Input -import kotlinx.io.Output -import kotlinx.io.asBinary +import kotlinx.io.* import kotlinx.io.buffer.Buffer -import kotlinx.io.buffer.get import kotlinx.io.buffer.set import java.io.InputStream import java.io.OutputStream -private class InputStreamInput(val source: InputStream, val waitForInput: Boolean = false) : Input() { +private class BlockingStreamInput(val source: InputStream) : Input() { override fun closeSource() { source.close() } - override fun fill(buffer: Buffer): Int { - if (waitForInput) { - while (source.available() == 0) { - //block until input is available - } + override fun fill(buffer: Buffer, startIndex: Int, endIndex: Int): Int { + while (source.available() == 0) { + //block until input is available } - var bufferPos = 0 - do { + // Zero-copy attempt + if (buffer.buffer.hasArray()) { + val result = source.read(buffer.buffer.array(), startIndex, endIndex - startIndex) + return result.coerceAtLeast(0) // -1 when IS is closed + } + + for (i in startIndex until endIndex) { val byte = source.read() - buffer[bufferPos] = byte.toByte() - bufferPos++ - } while (byte > 0 && bufferPos < buffer.size && source.available() > 0) - return bufferPos - } -} - -private class OutputStreamOutput(val out: OutputStream) : Output() { - override fun flush(source: Buffer, length: Int) { - for (i in 0..length) { - out.write(source[i].toInt()) + if (byte == -1) return (i - startIndex) + buffer[i] = byte.toByte() } - out.flush() - } - - override fun closeSource() { - out.flush() - out.close() + return endIndex - startIndex } } - fun InputStream.read(size: Int, block: Input.() -> R): R { val buffer = ByteArray(size) read(buffer) - return buffer.asBinary().read(block) + return buffer.asBinary().read(block = block) } -fun InputStream.read(block: Input.() -> R): R = - InputStreamInput(this, false).block() +fun InputStream.read(block: Input.() -> R): R = asInput().block() -fun InputStream.readBlocking(block: Input.() -> R): R = - InputStreamInput(this, true).block() +fun InputStream.readBlocking(block: Input.() -> R): R = BlockingStreamInput(this).block() -fun OutputStream.write(block: Output.() -> Unit) { - OutputStreamOutput(this).block() +inline fun OutputStream.write(block: Output.() -> Unit) { + asOutput().block() } \ No newline at end of file diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt index de1d35ff..5847f46d 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt @@ -4,7 +4,7 @@ import hep.dataforge.context.Global import hep.dataforge.io.Envelope import hep.dataforge.io.Responder import hep.dataforge.io.TaggedEnvelopeFormat -import hep.dataforge.io.writeByteArray +import hep.dataforge.io.toByteArray import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.runBlocking import kotlinx.io.writeDouble @@ -17,7 +17,7 @@ import kotlin.time.ExperimentalTime @ExperimentalStdlibApi object EchoResponder : Responder { override suspend fun respond(request: Envelope): Envelope { - val string = TaggedEnvelopeFormat().run { writeByteArray(request).decodeToString() } + val string = TaggedEnvelopeFormat().run { toByteArray(request).decodeToString() } println("ECHO:") println(string) return request diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Scheme.kt index 0229da8b..deec949f 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Scheme.kt @@ -23,9 +23,6 @@ open class Scheme() : Configurable, Described, MetaRepr { var defaultProvider: (Name) -> MetaItem<*>? = { null } internal set - final override var descriptor: NodeDescriptor? = null - internal set - override fun getDefaultItem(name: Name): MetaItem<*>? { return defaultProvider(name) ?: descriptor?.get(name)?.defaultItem() } @@ -78,15 +75,10 @@ open class SchemeSpec(val builder: () -> T) : */ open class MetaScheme( val meta: Meta, - descriptor: NodeDescriptor? = null, + override val descriptor: NodeDescriptor? = null, config: Config = Config() ) : Scheme(config, meta::get) { - init { - this.descriptor = descriptor - } - - override val defaultLayer: Meta - get() = Laminate(meta, descriptor?.defaultItem().node) + override val defaultLayer: Meta get() = Laminate(meta, descriptor?.defaultItem().node) } fun Meta.asScheme() = diff --git a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/TextRows.kt b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/TextRows.kt index ae3c386e..00715844 100644 --- a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/TextRows.kt +++ b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/TextRows.kt @@ -9,7 +9,6 @@ import kotlinx.coroutines.flow.toList import kotlinx.io.Binary import kotlinx.io.ExperimentalIoApi import kotlinx.io.Output -import kotlinx.io.RandomAccessBinary import kotlinx.io.text.forEachUtf8Line import kotlinx.io.text.readUtf8Line import kotlinx.io.text.readUtf8StringUntilDelimiter @@ -76,7 +75,7 @@ suspend fun TextRows.buildRowIndex(): List = indexFlow().toList() @ExperimentalIoApi class TextTable( override val header: ValueTableHeader, - val binary: RandomAccessBinary, + val binary: Binary, val index: List ) : Table { @@ -99,7 +98,7 @@ class TextTable( } companion object { - suspend operator fun invoke(header: ValueTableHeader, binary: RandomAccessBinary): TextTable { + suspend operator fun invoke(header: ValueTableHeader, binary: Binary): TextTable { val index = TextRows(header, binary).buildRowIndex() return TextTable(header, binary, index) } diff --git a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt index 1180ca23..87f2e29d 100644 --- a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt +++ b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt @@ -1,12 +1,13 @@ package hep.dataforge.tables.io +import hep.dataforge.io.Binary +import hep.dataforge.io.EmptyBinary import hep.dataforge.io.Envelope import hep.dataforge.meta.* import hep.dataforge.tables.SimpleColumnHeader import hep.dataforge.tables.Table import hep.dataforge.values.Value import kotlinx.io.ByteArrayOutput -import kotlinx.io.EmptyBinary import kotlinx.io.ExperimentalIoApi import kotlinx.io.asBinary diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/envelopeData.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/envelopeData.kt index d378726f..248e819c 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/envelopeData.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/envelopeData.kt @@ -2,11 +2,7 @@ package hep.dataforge.workspace import hep.dataforge.data.Data import hep.dataforge.data.await -import hep.dataforge.io.Envelope -import hep.dataforge.io.IOFormat -import hep.dataforge.io.SimpleEnvelope -import hep.dataforge.io.readWith -import kotlinx.io.ArrayBinary +import hep.dataforge.io.* import kotlin.reflect.KClass /** @@ -18,8 +14,6 @@ fun Envelope.toData(type: KClass, format: IOFormat): Data suspend fun Data.toEnvelope(format: IOFormat): Envelope { val obj = await() - val binary = ArrayBinary.write { - format.run { writeObject(obj) } - } + val binary = format.toBinary(obj) return SimpleEnvelope(meta, binary) } \ No newline at end of file From eebfe534cc5d9caf11b6168132f7c83b31a31359 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 5 Apr 2020 21:12:56 +0300 Subject: [PATCH 09/11] Fix Envelope IO with binaries --- .../hep/dataforge/context/AbstractPlugin.kt | 3 +- .../kotlin/hep/dataforge/context/Factory.kt | 3 +- .../hep/dataforge/context/PluginRepository.kt | 3 +- .../io/yaml/FrontMatterEnvelopeFormat.kt | 26 ++- .../hep/dataforge/io/yaml/YamlMetaFormat.kt | 8 +- .../kotlin/hep/dataforge/io/EnvelopeFormat.kt | 3 +- .../kotlin/hep/dataforge/io/EnvelopeParts.kt | 184 +++++++++--------- .../kotlin/hep/dataforge/io/IOFormat.kt | 32 +-- .../kotlin/hep/dataforge/io/IOPlugin.kt | 46 +++-- .../kotlin/hep/dataforge/io/JsonMetaFormat.kt | 5 +- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 31 ++- .../hep/dataforge/io/TaglessEnvelopeFormat.kt | 41 ++-- .../kotlin/hep/dataforge/io/ioMisc.kt | 24 ++- .../kotlin/hep/dataforge/io/BinaryTest.kt | 20 ++ .../kotlin/hep/dataforge/io/MultipartTest.kt | 18 +- .../jvmMain/kotlin/hep/dataforge/io/fileIO.kt | 7 +- .../kotlin/hep/dataforge/io/functionsJVM.kt | 2 +- .../hep/dataforge/io/FileEnvelopeTest.kt | 2 + .../dataforge/io/tcp/EnvelopeServerTest.kt | 2 +- .../kotlin/hep/dataforge/meta/Meta.kt | 7 +- .../hep/dataforge/meta/MutableMetaDelegate.kt | 4 +- .../hep/dataforge/meta/Specification.kt | 10 +- .../hep/dataforge/meta/serializationUtils.kt | 6 +- 23 files changed, 295 insertions(+), 192 deletions(-) create mode 100644 dataforge-io/src/commonTest/kotlin/hep/dataforge/io/BinaryTest.kt diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt index 232c7cd4..e1041935 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt @@ -1,13 +1,12 @@ package hep.dataforge.context -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.names.Name import kotlin.properties.ReadOnlyProperty import kotlin.reflect.KClass import kotlin.reflect.KProperty -abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin { +abstract class AbstractPlugin(override val meta: Meta = Meta.EMPTY) : Plugin { private var _context: Context? = null private val dependencies = ArrayList>() diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt index b20672c2..b01334e6 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Factory.kt @@ -1,8 +1,7 @@ package hep.dataforge.context -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta interface Factory { - operator fun invoke(meta: Meta = EmptyMeta, context: Context = Global): T + operator fun invoke(meta: Meta = Meta.EMPTY, context: Context = Global): T } \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt index 06c1132b..248b239c 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginRepository.kt @@ -1,6 +1,5 @@ package hep.dataforge.context -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlin.reflect.KClass @@ -23,7 +22,7 @@ expect object PluginRepository { /** * Fetch specific plugin and instantiate it with given meta */ -fun PluginRepository.fetch(tag: PluginTag, meta: Meta = EmptyMeta): Plugin = +fun PluginRepository.fetch(tag: PluginTag, meta: Meta = Meta.EMPTY): Plugin = list().find { it.tag.matches(tag) }?.invoke(meta = meta) ?: error("Plugin with tag $tag not found in the repository") diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt index bac63c9d..8a837d74 100644 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -3,7 +3,6 @@ package hep.dataforge.io.yaml import hep.dataforge.context.Context import hep.dataforge.io.* import hep.dataforge.meta.DFExperimental -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.io.* import kotlinx.io.text.readUtf8Line @@ -13,7 +12,7 @@ import kotlinx.serialization.toUtf8Bytes @DFExperimental class FrontMatterEnvelopeFormat( val io: IOPlugin, - meta: Meta = EmptyMeta + val meta: Meta = Meta.EMPTY ) : EnvelopeFormat { override fun Input.readPartial(): PartialEnvelope { @@ -26,7 +25,7 @@ class FrontMatterEnvelopeFormat( val readMetaFormat = metaTypeRegex.matchEntire(line)?.groupValues?.first() - ?.let { io.metaFormat(it) } ?: YamlMetaFormat + ?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat //TODO replace by preview val meta = Binary { @@ -51,11 +50,11 @@ class FrontMatterEnvelopeFormat( val readMetaFormat = metaTypeRegex.matchEntire(line)?.groupValues?.first() - ?.let { io.metaFormat(it) } ?: YamlMetaFormat + ?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat val meta = Binary { do { - writeUtf8String(readUtf8Line() + "\r\n") + writeUtf8String(readUtf8Line() + "\r\n") } while (!line.startsWith(SEPARATOR)) }.read { readMetaFormat.run { @@ -78,6 +77,11 @@ class FrontMatterEnvelopeFormat( } } + override fun toMeta(): Meta = Meta { + IOPlugin.IO_FORMAT_NAME_KEY put name.toString() + IOPlugin.IO_FORMAT_META_KEY put meta + } + companion object : EnvelopeFormatFactory { const val SEPARATOR = "---" @@ -88,11 +92,13 @@ class FrontMatterEnvelopeFormat( } override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { - val line = input.readUtf8Line() - return if (line.startsWith("---")) { - invoke() - } else { - null + return input.preview { + val line = readUtf8Line() + return@preview if (line.startsWith("---")) { + invoke() + } else { + null + } } } diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt index 71be6352..1799d072 100644 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt @@ -1,6 +1,7 @@ package hep.dataforge.io.yaml import hep.dataforge.context.Context +import hep.dataforge.io.IOPlugin import hep.dataforge.io.MetaFormat import hep.dataforge.io.MetaFormatFactory import hep.dataforge.meta.DFExperimental @@ -11,10 +12,8 @@ import hep.dataforge.meta.toMeta import kotlinx.io.Input import kotlinx.io.Output import kotlinx.io.asInputStream -import kotlinx.io.readUByte import kotlinx.io.text.writeUtf8String import org.yaml.snakeyaml.Yaml -import java.io.InputStream @DFExperimental class YamlMetaFormat(val meta: Meta) : MetaFormat { @@ -30,6 +29,11 @@ class YamlMetaFormat(val meta: Meta) : MetaFormat { return map.toMeta(descriptor) } + override fun toMeta(): Meta = Meta{ + IOPlugin.IO_FORMAT_NAME_KEY put FrontMatterEnvelopeFormat.name.toString() + IOPlugin.IO_FORMAT_META_KEY put meta + } + companion object : MetaFormatFactory { override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt index bf5b85f5..c634d696 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeFormat.kt @@ -2,7 +2,6 @@ package hep.dataforge.io import hep.dataforge.context.Context import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.names.Name import hep.dataforge.names.asName @@ -26,7 +25,7 @@ interface EnvelopeFormat : IOFormat { fun Output.writeEnvelope( envelope: Envelope, metaFormatFactory: MetaFormatFactory = defaultMetaFormat, - formatMeta: Meta = EmptyMeta + formatMeta: Meta = Meta.EMPTY ) override fun Input.readObject(): Envelope diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt index 4780a71a..19ccb458 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt @@ -1,123 +1,121 @@ package hep.dataforge.io -import hep.dataforge.context.Global -import hep.dataforge.io.EnvelopeParts.FORMAT_META_KEY -import hep.dataforge.io.EnvelopeParts.FORMAT_NAME_KEY -import hep.dataforge.io.EnvelopeParts.INDEX_KEY -import hep.dataforge.io.EnvelopeParts.MULTIPART_DATA_SEPARATOR -import hep.dataforge.io.EnvelopeParts.MULTIPART_DATA_TYPE -import hep.dataforge.io.EnvelopeParts.SIZE_KEY +import hep.dataforge.io.Envelope.Companion.ENVELOPE_NODE_KEY +import hep.dataforge.io.PartDescriptor.Companion.DEFAULT_MULTIPART_DATA_SEPARATOR +import hep.dataforge.io.PartDescriptor.Companion.MULTIPART_DATA_TYPE +import hep.dataforge.io.PartDescriptor.Companion.MULTIPART_KEY +import hep.dataforge.io.PartDescriptor.Companion.PARTS_KEY +import hep.dataforge.io.PartDescriptor.Companion.PART_FORMAT_KEY +import hep.dataforge.io.PartDescriptor.Companion.SEPARATOR_KEY import hep.dataforge.meta.* import hep.dataforge.names.asName import hep.dataforge.names.plus -import hep.dataforge.names.toName +import kotlinx.io.Binary +import kotlinx.io.writeBinary -object EnvelopeParts { - val MULTIPART_KEY = "multipart".asName() - val SIZE_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "size" - val INDEX_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "index" - val FORMAT_NAME_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "format" - val FORMAT_META_KEY = Envelope.ENVELOPE_NODE_KEY + MULTIPART_KEY + "meta" +private class PartDescriptor : Scheme() { + var offset by int(0) + var size by int(0) + var meta by node() - const val MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n" + companion object : SchemeSpec(::PartDescriptor) { + val MULTIPART_KEY = ENVELOPE_NODE_KEY + "multipart" + val PARTS_KEY = MULTIPART_KEY + "parts" + val SEPARATOR_KEY = MULTIPART_KEY + "separator" - const val MULTIPART_DATA_TYPE = "envelope.multipart" + const val DEFAULT_MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n" + + val PART_FORMAT_KEY = "format".asName() + + const val MULTIPART_DATA_TYPE = "envelope.multipart" + } } -/** - * Append multiple serialized envelopes to the data block. Previous data is erased if it was present - */ -@DFExperimental +data class EnvelopePart(val binary: Binary, val description: Meta?) + +typealias EnvelopeParts = List + fun EnvelopeBuilder.multipart( - envelopes: Collection, - format: EnvelopeFormatFactory, - formatMeta: Meta = EmptyMeta + parts: EnvelopeParts, + separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR ) { dataType = MULTIPART_DATA_TYPE - meta { - SIZE_KEY put envelopes.size - FORMAT_NAME_KEY put format.name.toString() - if (!formatMeta.isEmpty()) { - FORMAT_META_KEY put formatMeta + + var offsetCounter = 0 + val separatorSize = separator.length + val partDescriptors = parts.map { (binary, description) -> + offsetCounter += separatorSize + PartDescriptor { + offset = offsetCounter + size = binary.size + meta = description + }.also { + offsetCounter += binary.size } } + + meta { + if (separator != DEFAULT_MULTIPART_DATA_SEPARATOR) { + SEPARATOR_KEY put separator + } + setIndexed(PARTS_KEY, partDescriptors.map { it.toMeta() }) + } + data { - format(formatMeta).run { - envelopes.forEach { - writeRawString(MULTIPART_DATA_SEPARATOR) - writeEnvelope(it) - } + parts.forEach { + writeRawString(separator) + writeBinary(it.binary) } } } -/** - * Create a multipart partition in the envelope adding additional name-index mapping in meta - */ -@DFExperimental -fun EnvelopeBuilder.multipart( - envelopes: Map, - format: EnvelopeFormatFactory, - formatMeta: Meta = EmptyMeta +fun EnvelopeBuilder.envelopes( + envelopes: List, + format: EnvelopeFormat = TaggedEnvelopeFormat, + separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR ) { - dataType = MULTIPART_DATA_TYPE - meta { - SIZE_KEY put envelopes.size - FORMAT_NAME_KEY put format.name.toString() - if (!formatMeta.isEmpty()) { - FORMAT_META_KEY put formatMeta - } + val parts = envelopes.map { + val binary = format.toBinary(it) + EnvelopePart(binary, null) } - data { - format.run { - var counter = 0 - envelopes.forEach { (key, envelope) -> - writeRawString(MULTIPART_DATA_SEPARATOR) - writeEnvelope(envelope) - meta { - append(INDEX_KEY, Meta { - "key" put key - "index" put counter - }) - } - counter++ - } + meta{ + set(MULTIPART_KEY + PART_FORMAT_KEY, format.toMeta()) + } + multipart(parts, separator) +} + +fun Envelope.parts(): EnvelopeParts { + if (data == null) return emptyList() + //TODO add zip folder reader + val parts = meta.getIndexed(PARTS_KEY).values.mapNotNull { it.node }.map { + PartDescriptor.wrap(it) + } + return if (parts.isEmpty()) { + listOf(EnvelopePart(data!!, meta[MULTIPART_KEY].node)) + } else { + parts.map { + val binary = data!!.view(it.offset, it.size) + val meta = Laminate(it.meta, meta[MULTIPART_KEY].node) + EnvelopePart(binary, meta) } } } -@DFExperimental -fun EnvelopeBuilder.multipart( - formatFactory: EnvelopeFormatFactory, - formatMeta: Meta = EmptyMeta, - builder: suspend SequenceScope.() -> Unit -) = multipart(sequence(builder).toList(), formatFactory, formatMeta) +fun EnvelopePart.envelope(format: EnvelopeFormat): Envelope = binary.readWith(format) + +val EnvelopePart.name: String? get() = description?.get("name").string /** - * If given envelope supports multipart data, return a sequence of those parts (could be empty). Otherwise return null. + * Represent envelope part by an envelope */ -@DFExperimental -fun Envelope.parts(io: IOPlugin = Global.plugins.fetch(IOPlugin)): Sequence? { - return when (dataType) { - MULTIPART_DATA_TYPE -> { - val size = meta[SIZE_KEY].int ?: error("Unsized parts not supported yet") - val formatName = meta[FORMAT_NAME_KEY].string?.toName() - ?: error("Inferring parts format is not supported at the moment") - val formatMeta = meta[FORMAT_META_KEY].node ?: EmptyMeta - val format = io.envelopeFormat(formatName, formatMeta) - ?: error("Format $formatName is not resolved by $io") - return format.run { - data?.read { - sequence { - repeat(size) { - val separator = readRawString(MULTIPART_DATA_SEPARATOR.length) - if(separator!= MULTIPART_DATA_SEPARATOR) error("Separator is expected, but $separator found") - yield(readObject()) - } - } - } ?: emptySequence() - } - } - else -> null +fun EnvelopePart.envelope(plugin: IOPlugin): Envelope { + val formatItem = description?.get(PART_FORMAT_KEY) + return if (formatItem != null) { + val format: EnvelopeFormat = plugin.resolveEnvelopeFormat(formatItem) + ?: error("Envelope format for $formatItem is not resolved") + binary.readWith(format) + } else { + error("Envelope description not found") + //SimpleEnvelope(description ?: Meta.EMPTY, binary) } -} +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index 1300afe7..474d882e 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -4,8 +4,10 @@ import hep.dataforge.context.Context import hep.dataforge.context.Factory import hep.dataforge.context.Named import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE +import hep.dataforge.io.IOPlugin.Companion.IO_FORMAT_NAME_KEY import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaItem +import hep.dataforge.meta.MetaRepr import hep.dataforge.names.Name import hep.dataforge.names.asName import hep.dataforge.provider.Type @@ -18,12 +20,20 @@ import kotlin.reflect.KClass /** * And interface for reading and writing objects into with IO streams */ -interface IOFormat { +interface IOFormat : MetaRepr { fun Output.writeObject(obj: T) fun Input.readObject(): T } fun Input.readWith(format: IOFormat): T = format.run { readObject() } + +/** + * Read given binary as object using given format + */ +fun Binary.readWith(format: IOFormat): T = read { + readWith(format) +} + fun Output.writeWith(format: IOFormat, obj: T) = format.run { writeObject(obj) } class ListIOFormat(val format: IOFormat) : IOFormat> { @@ -42,6 +52,11 @@ class ListIOFormat(val format: IOFormat) : IOFormat> { List(size) { readObject() } } } + + override fun toMeta(): Meta = Meta { + IO_FORMAT_NAME_KEY put "list" + "contentFormat" put format.toMeta() + } } val IOFormat.list get() = ListIOFormat(this) @@ -57,12 +72,16 @@ fun ObjectPool.fill(block: Buffer.() -> Unit): Buffer { } @Type(IO_FORMAT_TYPE) -interface IOFormatFactory : Factory>, Named { +interface IOFormatFactory : Factory>, Named, MetaRepr { /** * Explicit type for dynamic type checks */ val type: KClass + override fun toMeta(): Meta = Meta { + IO_FORMAT_NAME_KEY put name.toString() + } + companion object { const val IO_FORMAT_TYPE = "io.format" } @@ -99,13 +118,4 @@ object ValueIOFormat : IOFormat, IOFormatFactory { return (BinaryMetaFormat.run { readMetaItem() } as? MetaItem.ValueItem)?.value ?: error("The item is not a value") } -} - -/** - * Read given binary as object using given format - */ -fun Binary.readWith(format: IOFormat): T = format.run { - read { - readObject() - } } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt index 6144a211..d31e4264 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt @@ -6,29 +6,51 @@ import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE import hep.dataforge.meta.* import hep.dataforge.names.Name -import hep.dataforge.names.get +import hep.dataforge.names.asName +import hep.dataforge.names.toName import kotlin.reflect.KClass class IOPlugin(meta: Meta) : AbstractPlugin(meta) { override val tag: PluginTag get() = Companion.tag + val ioFormatFactories by lazy { + context.content>(IO_FORMAT_TYPE).values + } + + fun resolveIOFormat(item: MetaItem<*>, type: KClass): IOFormat? { + val key = item.string ?: item.node[IO_FORMAT_NAME_KEY]?.string ?: error("Format name not defined") + val name = key.toName() + return ioFormatFactories.find { it.name == name }?.let { + @Suppress("UNCHECKED_CAST") + if (it.type != type) error("Format type ${it.type} is not the same as requested type $type") + else it.invoke(item.node[IO_FORMAT_META_KEY].node ?: Meta.EMPTY, context) as IOFormat + } + } + + val metaFormatFactories by lazy { context.content(META_FORMAT_TYPE).values } - fun metaFormat(key: Short, meta: Meta = EmptyMeta): MetaFormat? = + fun resolveMetaFormat(key: Short, meta: Meta = Meta.EMPTY): MetaFormat? = metaFormatFactories.find { it.key == key }?.invoke(meta) - fun metaFormat(name: String, meta: Meta = EmptyMeta): MetaFormat? = + fun resolveMetaFormat(name: String, meta: Meta = Meta.EMPTY): MetaFormat? = metaFormatFactories.find { it.shortName == name }?.invoke(meta) val envelopeFormatFactories by lazy { context.content(ENVELOPE_FORMAT_TYPE).values } - fun envelopeFormat(name: Name, meta: Meta = EmptyMeta) = + fun resolveEnvelopeFormat(name: Name, meta: Meta = Meta.EMPTY): EnvelopeFormat? = envelopeFormatFactories.find { it.name == name }?.invoke(meta, context) + fun resolveEnvelopeFormat(item: MetaItem<*>): EnvelopeFormat? { + val name = item.string ?: item.node[IO_FORMAT_NAME_KEY]?.string ?: error("Envelope format name not defined") + val meta = item.node[IO_FORMAT_META_KEY].node ?: Meta.EMPTY + return resolveEnvelopeFormat(name.toName(), meta) + } + override fun provideTop(target: String): Map { return when (target) { META_FORMAT_TYPE -> defaultMetaFormats.toMap() @@ -37,20 +59,10 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { } } - val ioFormats: Map> by lazy { - context.content>(IO_FORMAT_TYPE) - } - - fun resolveIOFormat(item: MetaItem<*>, type: KClass): IOFormat? { - val key = item.string ?: item.node["name"]?.string ?: error("Format name not defined") - return ioFormats[key]?.let { - @Suppress("UNCHECKED_CAST") - if (it.type != type) error("Format type ${it.type} is not the same as requested type $type") - else it.invoke(item.node["meta"].node ?: EmptyMeta, context) as IOFormat - } - } - companion object : PluginFactory { + val IO_FORMAT_NAME_KEY = "name".asName() + val IO_FORMAT_META_KEY = "meta".asName() + val defaultMetaFormats: List = listOf(JsonMetaFormat, BinaryMetaFormat) val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat, TaglessEnvelopeFormat) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index 39902338..ac32d314 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -12,7 +12,6 @@ import hep.dataforge.meta.toMetaItem import kotlinx.io.Input import kotlinx.io.Output import kotlinx.io.readByteArray -import kotlinx.io.text.readUtf8String import kotlinx.io.text.writeUtf8String import kotlinx.serialization.UnstableDefault import kotlinx.serialization.json.Json @@ -26,6 +25,10 @@ class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat { writeUtf8String(json.stringify(JsonObjectSerializer, jsonObject)) } + override fun toMeta(): Meta = Meta{ + IOPlugin.IO_FORMAT_NAME_KEY put name.toString() + } + override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { val str = readByteArray().decodeToString() val jsonElement = json.parseJson(str) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index 69c33bbd..10bbdab9 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -2,6 +2,7 @@ package hep.dataforge.io import hep.dataforge.context.Context import hep.dataforge.meta.Meta +import hep.dataforge.meta.enum import hep.dataforge.meta.get import hep.dataforge.meta.string import hep.dataforge.names.Name @@ -9,7 +10,6 @@ import hep.dataforge.names.plus import hep.dataforge.names.toName import kotlinx.io.* -@ExperimentalIoApi class TaggedEnvelopeFormat( val io: IOPlugin, val version: VERSION = VERSION.DF02 @@ -58,7 +58,7 @@ class TaggedEnvelopeFormat( override fun Input.readObject(): Envelope { val tag = readTag(version) - val metaFormat = io.metaFormat(tag.metaFormatKey) + val metaFormat = io.resolveMetaFormat(tag.metaFormatKey) ?: error("Meta format with key ${tag.metaFormatKey} not found") val meta: Meta = limit(tag.metaSize.toInt()).run { @@ -67,7 +67,7 @@ class TaggedEnvelopeFormat( } } - val data = ByteArray(tag.dataSize.toInt()).also { readByteArray(it) }.asBinary() + val data = readBinary(tag.dataSize.toInt()) return SimpleEnvelope(meta, data) } @@ -75,7 +75,7 @@ class TaggedEnvelopeFormat( override fun Input.readPartial(): PartialEnvelope { val tag = readTag(version) - val metaFormat = io.metaFormat(tag.metaFormatKey) + val metaFormat = io.resolveMetaFormat(tag.metaFormatKey) ?: error("Meta format with key ${tag.metaFormatKey} not found") val meta: Meta = limit(tag.metaSize.toInt()).run { @@ -98,6 +98,13 @@ class TaggedEnvelopeFormat( DF03(24u) } + override fun toMeta(): Meta = Meta { + IOPlugin.IO_FORMAT_NAME_KEY put name.toString() + IOPlugin.IO_FORMAT_META_KEY put { + "version" put version + } + } + companion object : EnvelopeFormatFactory { private const val START_SEQUENCE = "#~" private const val END_SEQUENCE = "~#\r\n" @@ -111,7 +118,9 @@ class TaggedEnvelopeFormat( //Check if appropriate factory exists io.metaFormatFactories.find { it.name == metaFormatName } ?: error("Meta format could not be resolved") - return TaggedEnvelopeFormat(io) + val version: VERSION = meta["version"].enum() ?: VERSION.DF02 + + return TaggedEnvelopeFormat(io, version) } private fun Input.readTag(version: VERSION): Tag { @@ -132,11 +141,13 @@ class TaggedEnvelopeFormat( override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { return try { - val header = input.readRawString(6) - when (header.substring(2..5)) { - VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02) - VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03) - else -> null + input.preview { + val header = readRawString(6) + return@preview when (header.substring(2..5)) { + VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02) + VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03) + else -> null + } } } catch (ex: Exception) { null diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt index f29132c8..2d86f814 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt @@ -1,16 +1,19 @@ package hep.dataforge.io import hep.dataforge.context.Context -import hep.dataforge.meta.* +import hep.dataforge.meta.Meta +import hep.dataforge.meta.get +import hep.dataforge.meta.isEmpty +import hep.dataforge.meta.string import hep.dataforge.names.asName import kotlinx.io.* import kotlinx.io.text.readUtf8Line import kotlinx.io.text.writeUtf8String +import kotlin.collections.set -@ExperimentalIoApi class TaglessEnvelopeFormat( val io: IOPlugin, - meta: Meta = EmptyMeta + val meta: Meta = Meta.EMPTY ) : EnvelopeFormat { private val metaStart = meta[META_START_PROPERTY].string ?: DEFAULT_META_START @@ -69,10 +72,10 @@ class TaglessEnvelopeFormat( line = readUtf8Line() } - var meta: Meta = EmptyMeta + var meta: Meta = Meta.EMPTY if (line.startsWith(metaStart)) { - val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat + val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat val metaSize = properties[META_LENGTH_PROPERTY]?.toInt() meta = if (metaSize != null) { limit(metaSize).run { @@ -95,9 +98,10 @@ class TaglessEnvelopeFormat( } while (!line.startsWith(dataStart)) val data: Binary? = if (properties.containsKey(DATA_LENGTH_PROPERTY)) { - val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt()) - readByteArray(bytes) - bytes.asBinary() + readBinary(properties[DATA_LENGTH_PROPERTY]!!.toInt()) +// val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt()) +// readByteArray(bytes) +// bytes.asBinary() } else { Binary { copyTo(this) @@ -132,10 +136,10 @@ class TaglessEnvelopeFormat( } } - var meta: Meta = EmptyMeta + var meta: Meta = Meta.EMPTY if (line.startsWith(metaStart)) { - val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat + val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat val metaSize = properties[META_LENGTH_PROPERTY]?.toInt() meta = if (metaSize != null) { offset += metaSize.toUInt() @@ -157,6 +161,11 @@ class TaglessEnvelopeFormat( return PartialEnvelope(meta, offset, dataSize) } + override fun toMeta(): Meta = Meta { + IOPlugin.IO_FORMAT_NAME_KEY put name.toString() + IOPlugin.IO_FORMAT_META_KEY put meta + } + companion object : EnvelopeFormatFactory { private val propertyPattern = "#\\?\\s*(?[\\w.]*)\\s*:\\s*(?[^;]*);?".toRegex() @@ -195,11 +204,13 @@ class TaglessEnvelopeFormat( override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? { return try { - val string = input.readRawString(TAGLESS_ENVELOPE_HEADER.length) - return if (string == TAGLESS_ENVELOPE_HEADER) { - TaglessEnvelopeFormat(io) - } else { - null + input.preview { + val string = readRawString(TAGLESS_ENVELOPE_HEADER.length) + return@preview if (string == TAGLESS_ENVELOPE_HEADER) { + TaglessEnvelopeFormat(io) + } else { + null + } } } catch (ex: Exception) { null diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt index eb5341d6..eddb7b40 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt @@ -1,6 +1,7 @@ package hep.dataforge.io import kotlinx.io.* +import kotlin.math.min fun Output.writeRawString(str: String) { str.forEach { writeByte(it.toByte()) } @@ -18,5 +19,24 @@ inline fun buildByteArray(expectedSize: Int = 16, block: Output.() -> Unit): Byt inline fun Binary(expectedSize: Int = 16, block: Output.() -> Unit): Binary = buildByteArray(expectedSize, block).asBinary() -@Deprecated("To be replaced by Binary.EMPTY",level = DeprecationLevel.WARNING) -val EmptyBinary = ByteArrayBinary(ByteArray(0)) \ No newline at end of file +@Deprecated("To be replaced by Binary.EMPTY", level = DeprecationLevel.WARNING) +val EmptyBinary = ByteArrayBinary(ByteArray(0)) + +/** + * View section of a [Binary] as an independent binary + */ +class BinaryView(private val source: Binary, private val start: Int, override val size: Int) : Binary { + + init { + require(start > 0) + require(start + size <= source.size) { "View boundary is outside source binary size" } + } + + override fun read(offset: Int, atMost: Int, block: Input.() -> R): R { + return source.read(start + offset, min(size, atMost), block) + } +} + +fun Binary.view(start: Int, size: Int) = BinaryView(this, start, size) + +operator fun Binary.get(range: IntRange) = view(range.first, range.last - range.first) \ No newline at end of file diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/BinaryTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/BinaryTest.kt new file mode 100644 index 00000000..157b695f --- /dev/null +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/BinaryTest.kt @@ -0,0 +1,20 @@ +package hep.dataforge.io + +import kotlinx.io.asBinary +import kotlinx.io.readByte +import kotlinx.io.readInt +import kotlin.test.Test +import kotlin.test.assertEquals + +class BinaryTest { + @Test + fun testBinaryAccess(){ + val binary = ByteArray(128){it.toByte()}.asBinary() + + binary[3..12].read { + readInt() + val res = readByte() + assertEquals(7, res) + } + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt index 344edf59..1b239edd 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MultipartTest.kt @@ -1,17 +1,19 @@ package hep.dataforge.io +import hep.dataforge.context.Global import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.get import hep.dataforge.meta.int import kotlinx.io.text.writeUtf8String - import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @DFExperimental class MultipartTest { - val envelopes = (0..5).map { + val io: IOPlugin = Global.io + + val envelopes = (0 until 5).map { Envelope { meta { "value" put it @@ -26,19 +28,21 @@ class MultipartTest { } val partsEnvelope = Envelope { - multipart(envelopes, TaggedEnvelopeFormat) + envelopes(envelopes, TaglessEnvelopeFormat) } @Test fun testParts() { - TaggedEnvelopeFormat.run { + TaglessEnvelopeFormat.run { val singleEnvelopeData = toBinary(envelopes[0]) val singleEnvelopeSize = singleEnvelopeData.size val bytes = toBinary(partsEnvelope) - assertTrue(5*singleEnvelopeSize < bytes.size) + assertTrue(envelopes.size * singleEnvelopeSize < bytes.size) val reconstructed = bytes.readWith(this) - val parts = reconstructed.parts()?.toList() ?: emptyList() - assertEquals(2, parts[2].meta["value"].int) + println(reconstructed.meta) + val parts = reconstructed.parts() + val envelope = parts[2].envelope(io) + assertEquals(2, envelope.meta["value"].int) println(reconstructed.data!!.size) } } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt index 94b9b433..044cf932 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/fileIO.kt @@ -1,7 +1,6 @@ package hep.dataforge.io import hep.dataforge.meta.DFExperimental -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.meta.descriptors.NodeDescriptor import hep.dataforge.meta.isEmpty @@ -60,7 +59,7 @@ fun Path.readEnvelope(format: EnvelopeFormat): Envelope { @Suppress("UNCHECKED_CAST") @DFExperimental inline fun IOPlugin.resolveIOFormat(): IOFormat? { - return ioFormats.values.find { it.type.isSuperclassOf(T::class) } as IOFormat? + return ioFormatFactories.find { it.type.isSuperclassOf(T::class) } as IOFormat? } /** @@ -78,7 +77,7 @@ fun IOPlugin.readMetaFile(path: Path, formatOverride: MetaFormat? = null, descri } val extension = actualPath.fileName.toString().substringAfterLast('.') - val metaFormat = formatOverride ?: metaFormat(extension) ?: error("Can't resolve meta format $extension") + val metaFormat = formatOverride ?: resolveMetaFormat(extension) ?: error("Can't resolve meta format $extension") return metaFormat.run { actualPath.read { readMeta(descriptor) @@ -157,7 +156,7 @@ fun IOPlugin.readEnvelopeFile( .singleOrNull { it.fileName.toString().startsWith(IOPlugin.META_FILE_NAME) } val meta = if (metaFile == null) { - EmptyMeta + Meta.EMPTY } else { readMetaFile(metaFile) } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/functionsJVM.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/functionsJVM.kt index ffb924ef..ccd57c55 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/functionsJVM.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/functionsJVM.kt @@ -9,7 +9,7 @@ import kotlin.reflect.full.isSuperclassOf fun IOPlugin.resolveIOFormatName(type: KClass<*>): Name { - return ioFormats.entries.find { it.value.type.isSuperclassOf(type) }?.key + return ioFormatFactories.find { it.type.isSuperclassOf(type) }?.name ?: error("Can't resolve IOFormat for type $type") } diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt index edee906b..8db2bf62 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileEnvelopeTest.kt @@ -1,12 +1,14 @@ package hep.dataforge.io import hep.dataforge.context.Global +import hep.dataforge.meta.DFExperimental import kotlinx.io.writeDouble import java.nio.file.Files import kotlin.test.Test import kotlin.test.assertTrue +@DFExperimental class FileEnvelopeTest { val envelope = Envelope { meta { diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt index 5847f46d..3e545421 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt @@ -46,7 +46,7 @@ class EnvelopeServerTest { @Test(timeout = 1000) fun doEchoTest() { - val request = Envelope.invoke { + val request = Envelope { type = "test.echo" meta { "test.value" put 22 diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 672a1922..75cae67d 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -98,7 +98,7 @@ interface Meta : MetaRepr { */ const val VALUE_KEY = "@value" - val EMPTY: EmptyMeta = EmptyMeta + val EMPTY = EmptyMeta } } @@ -188,7 +188,7 @@ abstract class MetaBase : Meta { override fun hashCode(): Int = items.hashCode() - override fun toString(): String = toJson().toString() + override fun toString(): String = PRETTY_JSON.stringify(MetaSerializer, this) } /** @@ -216,6 +216,7 @@ fun MetaItem<*>.seal(): MetaItem = when (this) { is NodeItem -> NodeItem(node.seal()) } +@Deprecated("Use Meta.EMPTY instead", replaceWith = ReplaceWith("Meta.EMPTY")) object EmptyMeta : MetaBase() { override val items: Map> = emptyMap() } @@ -251,4 +252,4 @@ val MetaItem?.node: M? is NodeItem -> node } -fun Meta.isEmpty() = this === EmptyMeta || this.items.isEmpty() \ No newline at end of file +fun Meta.isEmpty() = this === Meta.EMPTY || this.items.isEmpty() \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaDelegate.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaDelegate.kt index 3fcdc9da..0cffb3a3 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaDelegate.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MutableMetaDelegate.kt @@ -34,9 +34,7 @@ class ReadWriteDelegateWrapper( val reader: (T) -> R, val writer: (R) -> T ) : ReadWriteProperty { - override fun getValue(thisRef: Any?, property: KProperty<*>): R { - return reader(delegate.getValue(thisRef, property)) - } + override fun getValue(thisRef: Any?, property: KProperty<*>): R = reader(delegate.getValue(thisRef, property)) override fun setValue(thisRef: Any?, property: KProperty<*>, value: R) { delegate.setValue(thisRef, property, writer(value)) diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt index 3724ce38..85cf7b96 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Specification.kt @@ -16,7 +16,7 @@ interface Specification { */ fun wrap(config: Config = Config(), defaultProvider: (Name) -> MetaItem<*>? = { null }): T - operator fun invoke(action: T.() -> Unit) = empty().apply(action) + operator fun invoke(action: T.() -> Unit): T = empty().apply(action) } /** @@ -27,12 +27,16 @@ fun Specification.update(config: Config, action: T.() -> U /** * Wrap a configuration using static meta as default */ -fun Specification.wrap(config: Config = Config(), default: Meta): T = wrap(config) { default[it] } +fun Specification.wrap(config: Config = Config(), default: Meta = Meta.EMPTY): T = + wrap(config) { default[it] } /** * Wrap a configuration using static meta as default */ -fun Specification.wrap(default: Meta): T = wrap(Config()) { default[it] } +fun Specification.wrap(source: Meta): T { + val default = source.seal() + return wrap(source.asConfig(), default) +} /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt index 16c58bdc..0ac825b9 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt @@ -3,6 +3,8 @@ package hep.dataforge.meta import kotlinx.serialization.* import kotlinx.serialization.builtins.DoubleArraySerializer import kotlinx.serialization.builtins.serializer +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonConfiguration fun SerialDescriptorBuilder.boolean(name: String, isOptional: Boolean = false, vararg annotations: Annotation) = element(name, Boolean.serializer().descriptor, isOptional = isOptional, annotations = annotations.toList()) @@ -62,4 +64,6 @@ inline fun Encoder.encodeStructure( val encoder = beginStructure(desc, *typeParams) encoder.block() encoder.endStructure(desc) -} \ No newline at end of file +} + +val PRETTY_JSON = Json(JsonConfiguration(prettyPrint = true, useArrayPolymorphism = true)) \ No newline at end of file From 6450696157998ad65922b9c164dac4a9f9915567 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 6 Apr 2020 22:50:51 +0300 Subject: [PATCH 10/11] Fix zip writer and name refactoring --- .../hep/dataforge/context/PluginManager.kt | 6 +- .../kotlin/hep/dataforge/data/Data.kt | 18 +++-- .../kotlin/hep/dataforge/data/DataNode.kt | 2 +- dataforge-io/build.gradle.kts | 2 +- .../io/yaml/FrontMatterEnvelopeFormat.kt | 6 +- .../hep/dataforge/io/yaml/YamlMetaFormat.kt | 7 +- .../dataforge/io/yaml/YamlMetaFormatTest.kt | 2 + .../kotlin/hep/dataforge/io/IOFormat.kt | 11 ++- .../kotlin/hep/dataforge/io/IOPlugin.kt | 14 ++-- .../kotlin/hep/dataforge/io/JsonMetaFormat.kt | 3 +- .../hep/dataforge/io/TaggedEnvelopeFormat.kt | 6 +- .../hep/dataforge/io/TaglessEnvelopeFormat.kt | 6 +- .../kotlin/hep/dataforge/io/ioMisc.kt | 3 - .../hep/dataforge/io/MetaSerializerTest.kt | 13 ++-- .../hep/dataforge/io/tcp/EnvelopeClient.kt | 3 +- .../hep/dataforge/io/tcp/EnvelopeServer.kt | 5 +- .../kotlin/hep/dataforge/meta/Meta.kt | 2 +- .../hep/dataforge/meta/serializationUtils.kt | 3 +- .../hep/dataforge/meta/DynamicMetaTest.kt | 2 +- .../hep/dataforge/output/OutputManager.kt | 7 +- .../kotlin/hep/dataforge/output/Renderer.kt | 3 +- .../dataforge/tables/io/textTableEnvelope.kt | 5 +- .../hep/dataforge/workspace/TaskModel.kt | 2 +- .../hep/dataforge/workspace/fileData.kt | 76 ++++++++++++++----- .../hep/dataforge/workspace/FileDataTest.kt | 9 ++- 25 files changed, 132 insertions(+), 84 deletions(-) diff --git a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt index 5f35d876..e6e8464e 100644 --- a/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt +++ b/dataforge-context/src/commonMain/kotlin/hep/dataforge/context/PluginManager.kt @@ -1,9 +1,7 @@ package hep.dataforge.context -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaBuilder -import hep.dataforge.meta.buildMeta import kotlin.reflect.KClass /** @@ -98,7 +96,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable load(factory: PluginFactory, meta: Meta = EmptyMeta): T = + fun load(factory: PluginFactory, meta: Meta = Meta.EMPTY): T = load(factory(meta, context)) fun load(factory: PluginFactory, metaBuilder: MetaBuilder.() -> Unit): T = @@ -121,7 +119,7 @@ class PluginManager(override val context: Context) : ContextAware, Iterable fetch(factory: PluginFactory, recursive: Boolean = true, meta: Meta = EmptyMeta): T { + fun fetch(factory: PluginFactory, recursive: Boolean = true, meta: Meta = Meta.EMPTY): T { val loaded = get(factory.type, factory.tag, recursive) return when { loaded == null -> load(factory(meta, context)) diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt index 4d787b69..233ea164 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Data.kt @@ -1,6 +1,8 @@ package hep.dataforge.data -import hep.dataforge.meta.* +import hep.dataforge.meta.Meta +import hep.dataforge.meta.MetaRepr +import hep.dataforge.meta.isEmpty import kotlinx.coroutines.CoroutineScope import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext @@ -31,14 +33,14 @@ interface Data : Goal, MetaRepr{ operator fun invoke( type: KClass, - meta: Meta = EmptyMeta, + meta: Meta = Meta.EMPTY, context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), block: suspend CoroutineScope.() -> T ): Data = DynamicData(type, meta, context, dependencies, block) inline operator fun invoke( - meta: Meta = EmptyMeta, + meta: Meta = Meta.EMPTY, context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), noinline block: suspend CoroutineScope.() -> T @@ -47,7 +49,7 @@ interface Data : Goal, MetaRepr{ operator fun invoke( name: String, type: KClass, - meta: Meta = EmptyMeta, + meta: Meta = Meta.EMPTY, context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), block: suspend CoroutineScope.() -> T @@ -55,14 +57,14 @@ interface Data : Goal, MetaRepr{ inline operator fun invoke( name: String, - meta: Meta = EmptyMeta, + meta: Meta = Meta.EMPTY, context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), noinline block: suspend CoroutineScope.() -> T ): Data = invoke(name, T::class, meta, context, dependencies, block) - fun static(value: T, meta: Meta = EmptyMeta): Data = + fun static(value: T, meta: Meta = Meta.EMPTY): Data = StaticData(value, meta) } } @@ -70,7 +72,7 @@ interface Data : Goal, MetaRepr{ class DynamicData( override val type: KClass, - override val meta: Meta = EmptyMeta, + override val meta: Meta = Meta.EMPTY, context: CoroutineContext = EmptyCoroutineContext, dependencies: Collection> = emptyList(), block: suspend CoroutineScope.() -> T @@ -78,7 +80,7 @@ class DynamicData( class StaticData( value: T, - override val meta: Meta = EmptyMeta + override val meta: Meta = Meta.EMPTY ) : Data, StaticGoal(value) { override val type: KClass get() = value::class } diff --git a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt index ee8d9c4d..246d9796 100644 --- a/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt +++ b/dataforge-data/src/commonMain/kotlin/hep/dataforge/data/DataNode.kt @@ -250,7 +250,7 @@ fun DataTreeBuilder.datum(name: String, data: Data) { this[name.toName()] = data } -fun DataTreeBuilder.static(name: Name, data: T, meta: Meta = EmptyMeta) { +fun DataTreeBuilder.static(name: Name, data: T, meta: Meta = Meta.EMPTY) { this[name] = Data.static(data, meta) } diff --git a/dataforge-io/build.gradle.kts b/dataforge-io/build.gradle.kts index 5a94bae5..221d0d09 100644 --- a/dataforge-io/build.gradle.kts +++ b/dataforge-io/build.gradle.kts @@ -11,7 +11,7 @@ serialization(sourceSet = TEST){ cbor() } -val ioVersion by rootProject.extra("0.2.0-npm-dev-6") +val ioVersion by rootProject.extra("0.2.0-npm-dev-7") kotlin { sourceSets { diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt index 8a837d74..0fffdb7e 100644 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -2,6 +2,8 @@ package hep.dataforge.io.yaml import hep.dataforge.context.Context import hep.dataforge.io.* +import hep.dataforge.io.IOFormat.Companion.META_KEY +import hep.dataforge.io.IOFormat.Companion.NAME_KEY import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta import kotlinx.io.* @@ -78,8 +80,8 @@ class FrontMatterEnvelopeFormat( } override fun toMeta(): Meta = Meta { - IOPlugin.IO_FORMAT_NAME_KEY put name.toString() - IOPlugin.IO_FORMAT_META_KEY put meta + NAME_KEY put name.toString() + META_KEY put meta } companion object : EnvelopeFormatFactory { diff --git a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt index 1799d072..5806abf8 100644 --- a/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt @@ -1,7 +1,8 @@ package hep.dataforge.io.yaml import hep.dataforge.context.Context -import hep.dataforge.io.IOPlugin +import hep.dataforge.io.IOFormat.Companion.META_KEY +import hep.dataforge.io.IOFormat.Companion.NAME_KEY import hep.dataforge.io.MetaFormat import hep.dataforge.io.MetaFormatFactory import hep.dataforge.meta.DFExperimental @@ -30,8 +31,8 @@ class YamlMetaFormat(val meta: Meta) : MetaFormat { } override fun toMeta(): Meta = Meta{ - IOPlugin.IO_FORMAT_NAME_KEY put FrontMatterEnvelopeFormat.name.toString() - IOPlugin.IO_FORMAT_META_KEY put meta + NAME_KEY put FrontMatterEnvelopeFormat.name.toString() + META_KEY put meta } companion object : MetaFormatFactory { diff --git a/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt b/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt index 0bb1c5d0..83f6d3a6 100644 --- a/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt +++ b/dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt @@ -2,6 +2,7 @@ package hep.dataforge.io.yaml import hep.dataforge.io.parse import hep.dataforge.io.toString +import hep.dataforge.meta.DFExperimental import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.seal @@ -9,6 +10,7 @@ import kotlin.test.Test import kotlin.test.assertEquals +@DFExperimental class YamlMetaFormatTest { @Test fun testYamlMetaFormat() { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt index 474d882e..7946c536 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOFormat.kt @@ -3,8 +3,8 @@ package hep.dataforge.io import hep.dataforge.context.Context import hep.dataforge.context.Factory import hep.dataforge.context.Named +import hep.dataforge.io.IOFormat.Companion.NAME_KEY import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE -import hep.dataforge.io.IOPlugin.Companion.IO_FORMAT_NAME_KEY import hep.dataforge.meta.Meta import hep.dataforge.meta.MetaItem import hep.dataforge.meta.MetaRepr @@ -23,6 +23,11 @@ import kotlin.reflect.KClass interface IOFormat : MetaRepr { fun Output.writeObject(obj: T) fun Input.readObject(): T + + companion object{ + val NAME_KEY = "name".asName() + val META_KEY = "meta".asName() + } } fun Input.readWith(format: IOFormat): T = format.run { readObject() } @@ -54,7 +59,7 @@ class ListIOFormat(val format: IOFormat) : IOFormat> { } override fun toMeta(): Meta = Meta { - IO_FORMAT_NAME_KEY put "list" + NAME_KEY put "list" "contentFormat" put format.toMeta() } } @@ -79,7 +84,7 @@ interface IOFormatFactory : Factory>, Named, MetaRepr { val type: KClass override fun toMeta(): Meta = Meta { - IO_FORMAT_NAME_KEY put name.toString() + NAME_KEY put name.toString() } companion object { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt index d31e4264..73fd26f1 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt @@ -2,11 +2,12 @@ package hep.dataforge.io import hep.dataforge.context.* import hep.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE +import hep.dataforge.io.IOFormat.Companion.META_KEY +import hep.dataforge.io.IOFormat.Companion.NAME_KEY import hep.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE import hep.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE import hep.dataforge.meta.* import hep.dataforge.names.Name -import hep.dataforge.names.asName import hep.dataforge.names.toName import kotlin.reflect.KClass @@ -18,12 +19,12 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { } fun resolveIOFormat(item: MetaItem<*>, type: KClass): IOFormat? { - val key = item.string ?: item.node[IO_FORMAT_NAME_KEY]?.string ?: error("Format name not defined") + val key = item.string ?: item.node[NAME_KEY]?.string ?: error("Format name not defined") val name = key.toName() return ioFormatFactories.find { it.name == name }?.let { @Suppress("UNCHECKED_CAST") if (it.type != type) error("Format type ${it.type} is not the same as requested type $type") - else it.invoke(item.node[IO_FORMAT_META_KEY].node ?: Meta.EMPTY, context) as IOFormat + else it.invoke(item.node[META_KEY].node ?: Meta.EMPTY, context) as IOFormat } } @@ -46,8 +47,8 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { envelopeFormatFactories.find { it.name == name }?.invoke(meta, context) fun resolveEnvelopeFormat(item: MetaItem<*>): EnvelopeFormat? { - val name = item.string ?: item.node[IO_FORMAT_NAME_KEY]?.string ?: error("Envelope format name not defined") - val meta = item.node[IO_FORMAT_META_KEY].node ?: Meta.EMPTY + val name = item.string ?: item.node[NAME_KEY]?.string ?: error("Envelope format name not defined") + val meta = item.node[META_KEY].node ?: Meta.EMPTY return resolveEnvelopeFormat(name.toName(), meta) } @@ -60,9 +61,6 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) { } companion object : PluginFactory { - val IO_FORMAT_NAME_KEY = "name".asName() - val IO_FORMAT_META_KEY = "meta".asName() - val defaultMetaFormats: List = listOf(JsonMetaFormat, BinaryMetaFormat) val defaultEnvelopeFormats = listOf(TaggedEnvelopeFormat, TaglessEnvelopeFormat) diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt index ac32d314..e19a9f7d 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/JsonMetaFormat.kt @@ -4,6 +4,7 @@ package hep.dataforge.io import hep.dataforge.context.Context +import hep.dataforge.io.IOFormat.Companion.NAME_KEY import hep.dataforge.meta.Meta import hep.dataforge.meta.descriptors.NodeDescriptor import hep.dataforge.meta.node @@ -26,7 +27,7 @@ class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat { } override fun toMeta(): Meta = Meta{ - IOPlugin.IO_FORMAT_NAME_KEY put name.toString() + NAME_KEY put name.toString() } override fun Input.readMeta(descriptor: NodeDescriptor?): Meta { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt index 10bbdab9..17fbc589 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt @@ -1,6 +1,8 @@ package hep.dataforge.io import hep.dataforge.context.Context +import hep.dataforge.io.IOFormat.Companion.META_KEY +import hep.dataforge.io.IOFormat.Companion.NAME_KEY import hep.dataforge.meta.Meta import hep.dataforge.meta.enum import hep.dataforge.meta.get @@ -99,8 +101,8 @@ class TaggedEnvelopeFormat( } override fun toMeta(): Meta = Meta { - IOPlugin.IO_FORMAT_NAME_KEY put name.toString() - IOPlugin.IO_FORMAT_META_KEY put { + NAME_KEY put name.toString() + META_KEY put { "version" put version } } diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt index 2d86f814..c800d878 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaglessEnvelopeFormat.kt @@ -1,6 +1,8 @@ package hep.dataforge.io import hep.dataforge.context.Context +import hep.dataforge.io.IOFormat.Companion.META_KEY +import hep.dataforge.io.IOFormat.Companion.NAME_KEY import hep.dataforge.meta.Meta import hep.dataforge.meta.get import hep.dataforge.meta.isEmpty @@ -162,8 +164,8 @@ class TaglessEnvelopeFormat( } override fun toMeta(): Meta = Meta { - IOPlugin.IO_FORMAT_NAME_KEY put name.toString() - IOPlugin.IO_FORMAT_META_KEY put meta + NAME_KEY put name.toString() + META_KEY put meta } companion object : EnvelopeFormatFactory { diff --git a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt index eddb7b40..9b214c5c 100644 --- a/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt +++ b/dataforge-io/src/commonMain/kotlin/hep/dataforge/io/ioMisc.kt @@ -19,9 +19,6 @@ inline fun buildByteArray(expectedSize: Int = 16, block: Output.() -> Unit): Byt inline fun Binary(expectedSize: Int = 16, block: Output.() -> Unit): Binary = buildByteArray(expectedSize, block).asBinary() -@Deprecated("To be replaced by Binary.EMPTY", level = DeprecationLevel.WARNING) -val EmptyBinary = ByteArrayBinary(ByteArray(0)) - /** * View section of a [Binary] as an independent binary */ diff --git a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt index 86a569a6..dc8e5016 100644 --- a/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt +++ b/dataforge-io/src/commonTest/kotlin/hep/dataforge/io/MetaSerializerTest.kt @@ -1,12 +1,9 @@ package hep.dataforge.io -import hep.dataforge.meta.Meta -import hep.dataforge.meta.MetaItem -import hep.dataforge.meta.MetaSerializer +import hep.dataforge.meta.* import hep.dataforge.names.Name import hep.dataforge.names.toName import kotlinx.serialization.cbor.Cbor -import kotlinx.serialization.json.Json import kotlin.test.Test import kotlin.test.assertEquals @@ -22,8 +19,8 @@ class MetaSerializerTest { @Test fun testMetaSerialization() { - val string = Json.indented.stringify(MetaSerializer, meta) - val restored = Json.plain.parse(MetaSerializer, string) + val string = JSON_PRETTY.stringify(MetaSerializer, meta) + val restored = JSON_PLAIN.parse(MetaSerializer, string) assertEquals(restored, meta) } @@ -38,8 +35,8 @@ class MetaSerializerTest { @Test fun testNameSerialization() { val name = "a.b.c".toName() - val string = Json.indented.stringify(Name.serializer(), name) - val restored = Json.plain.parse(Name.serializer(), string) + val string = JSON_PRETTY.stringify(Name.serializer(), name) + val restored = JSON_PLAIN.parse(Name.serializer(), string) assertEquals(restored, name) } diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt index 9562d146..156acd89 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeClient.kt @@ -3,7 +3,6 @@ package hep.dataforge.io.tcp import hep.dataforge.context.Context import hep.dataforge.context.ContextAware import hep.dataforge.io.* -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.coroutines.asCoroutineDispatcher import kotlinx.coroutines.withContext @@ -17,7 +16,7 @@ class EnvelopeClient( val host: String, val port: Int, formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat, - formatMeta: Meta = EmptyMeta + formatMeta: Meta = Meta.EMPTY ) : Responder, ContextAware { private val dispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher() diff --git a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt index 10c5b712..fc61ee49 100644 --- a/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt +++ b/dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/tcp/EnvelopeServer.kt @@ -6,7 +6,6 @@ import hep.dataforge.io.EnvelopeFormatFactory import hep.dataforge.io.Responder import hep.dataforge.io.TaggedEnvelopeFormat import hep.dataforge.io.type -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import kotlinx.coroutines.* import java.net.ServerSocket @@ -19,7 +18,7 @@ class EnvelopeServer( val responder: Responder, val scope: CoroutineScope, formatFactory: EnvelopeFormatFactory = TaggedEnvelopeFormat, - formatMeta: Meta = EmptyMeta + formatMeta: Meta = Meta.EMPTY ) : ContextAware { private var job: Job? = null @@ -78,7 +77,7 @@ class EnvelopeServer( logger.debug { "Accepted request with type ${request.type} from ${socket.remoteSocketAddress}" } if (request.type == SHUTDOWN_ENVELOPE_TYPE) { //Echo shutdown command - outputStream.write{ + outputStream.write { writeObject(request) } logger.info { "Accepted graceful shutdown signal from ${socket.inetAddress}" } diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 75cae67d..1afbb183 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -188,7 +188,7 @@ abstract class MetaBase : Meta { override fun hashCode(): Int = items.hashCode() - override fun toString(): String = PRETTY_JSON.stringify(MetaSerializer, this) + override fun toString(): String = JSON_PRETTY.stringify(MetaSerializer, this) } /** diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt index 0ac825b9..41c33c7c 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt @@ -66,4 +66,5 @@ inline fun Encoder.encodeStructure( encoder.endStructure(desc) } -val PRETTY_JSON = Json(JsonConfiguration(prettyPrint = true, useArrayPolymorphism = true)) \ No newline at end of file +val JSON_PRETTY = Json(JsonConfiguration(prettyPrint = true, useArrayPolymorphism = true)) +val JSON_PLAIN = Json(JsonConfiguration(prettyPrint = true, useArrayPolymorphism = true)) \ No newline at end of file diff --git a/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt b/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt index 71eca4f5..b546ae6d 100644 --- a/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt +++ b/dataforge-meta/src/jsTest/kotlin/hep/dataforge/meta/DynamicMetaTest.kt @@ -26,7 +26,7 @@ class DynamicMetaTest { @Test fun testMetaToDynamic(){ - val meta = buildMeta { + val meta = Meta { "a" put 22 "array" put listOf(1, 2, 3) "b" put "myString" diff --git a/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt b/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt index e88b29a5..a8b1697d 100644 --- a/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt +++ b/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt @@ -2,7 +2,6 @@ package hep.dataforge.output import hep.dataforge.context.* import hep.dataforge.context.PluginTag.Companion.DATAFORGE_GROUP -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta import hep.dataforge.names.Name import kotlinx.coroutines.CoroutineDispatcher @@ -24,7 +23,7 @@ interface OutputManager { type: KClass, name: Name, stage: Name = Name.EMPTY, - meta: Meta = EmptyMeta + meta: Meta = Meta.EMPTY ): Renderer } @@ -39,7 +38,7 @@ val Context.output: OutputManager get() = plugins.get() ?: ConsoleOutputManager( inline operator fun OutputManager.get( name: Name, stage: Name = Name.EMPTY, - meta: Meta = EmptyMeta + meta: Meta = Meta.EMPTY ): Renderer { return get(T::class, name, stage, meta) } @@ -47,7 +46,7 @@ inline operator fun OutputManager.get( /** * Directly render an object using the most suitable renderer */ -fun OutputManager.render(obj: Any, name: Name, stage: Name = Name.EMPTY, meta: Meta = EmptyMeta) = +fun OutputManager.render(obj: Any, name: Name, stage: Name = Name.EMPTY, meta: Meta = Meta.EMPTY) = get(obj::class, name, stage).render(obj, meta) /** diff --git a/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/Renderer.kt b/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/Renderer.kt index c1bcf6e5..9f8009d1 100644 --- a/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/Renderer.kt +++ b/dataforge-output/src/commonMain/kotlin/hep/dataforge/output/Renderer.kt @@ -1,7 +1,6 @@ package hep.dataforge.output import hep.dataforge.context.ContextAware -import hep.dataforge.meta.EmptyMeta import hep.dataforge.meta.Meta /** @@ -18,5 +17,5 @@ interface Renderer : ContextAware { * By convention actual render is called in asynchronous mode, so this method should never * block execution */ - fun render(obj: T, meta: Meta = EmptyMeta) + fun render(obj: T, meta: Meta = Meta.EMPTY) } diff --git a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt index 87f2e29d..e634346c 100644 --- a/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt +++ b/dataforge-tables/src/commonMain/kotlin/hep/dataforge/tables/io/textTableEnvelope.kt @@ -1,12 +1,11 @@ package hep.dataforge.tables.io -import hep.dataforge.io.Binary -import hep.dataforge.io.EmptyBinary import hep.dataforge.io.Envelope import hep.dataforge.meta.* import hep.dataforge.tables.SimpleColumnHeader import hep.dataforge.tables.Table import hep.dataforge.values.Value +import kotlinx.io.Binary import kotlinx.io.ByteArrayOutput import kotlinx.io.ExperimentalIoApi import kotlinx.io.asBinary @@ -39,5 +38,5 @@ fun TextRows.Companion.readEnvelope(envelope: Envelope): TextRows { .map { (_, item) -> SimpleColumnHeader(item.node["name"].string!!, Value::class, item.node["meta"].node ?: Meta.EMPTY) } - return TextRows(header, envelope.data ?: EmptyBinary) + return TextRows(header, envelope.data ?: Binary.EMPTY) } \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt index a811f428..2f7b6998 100644 --- a/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt +++ b/dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt @@ -124,7 +124,7 @@ fun TaskDependencyContainer.allData(to: Name = Name.EMPTY) = AllDataDependency(t /** * A builder for [TaskModel] */ -class TaskModelBuilder(val name: Name, meta: Meta = EmptyMeta) : TaskDependencyContainer { +class TaskModelBuilder(val name: Name, meta: Meta = Meta.EMPTY) : TaskDependencyContainer { /** * Meta for current task. By default uses the whole input meta */ diff --git a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt index 2b6d5454..6d16c37d 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt @@ -6,16 +6,19 @@ import hep.dataforge.io.* import hep.dataforge.meta.* import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import kotlinx.io.asOutput import java.nio.file.FileSystem import java.nio.file.Files import java.nio.file.Path -import java.nio.file.StandardCopyOption +import java.nio.file.StandardOpenOption import java.nio.file.spi.FileSystemProvider +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream import kotlin.reflect.KClass typealias FileFormatResolver = (Path, Meta) -> IOFormat -//private val zipFSProvider = ZipFileSystemProvider() + private fun newZFS(path: Path): FileSystem { val fsProvider = FileSystemProvider.installedProviders().find { it.scheme == "jar" } @@ -150,12 +153,43 @@ suspend fun IOPlugin.writeDataDirectory( } } + +private suspend fun ZipOutputStream.writeNode( + name: String, + item: DataItem, + dataFormat: IOFormat, + envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat +) { + withContext(Dispatchers.IO) { + when (item) { + is DataItem.Leaf -> { + //TODO add directory-based envelope writer + val envelope = item.data.toEnvelope(dataFormat) + val entry = ZipEntry(name) + putNextEntry(entry) + envelopeFormat.run { + asOutput().writeObject(envelope) + } + } + is DataItem.Node -> { + val entry = ZipEntry("$name/") + putNextEntry(entry) + closeEntry() + item.node.items.forEach { (token, item) -> + val childName = "$name/$token" + writeNode(childName, item, dataFormat, envelopeFormat) + } + } + } + } +} + +@DFExperimental suspend fun IOPlugin.writeZip( path: Path, node: DataNode, format: IOFormat, - envelopeFormat: EnvelopeFormat? = null, - metaFormat: MetaFormatFactory? = null + envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat ) { withContext(Dispatchers.IO) { val actualFile = if (path.toString().endsWith(".zip")) { @@ -163,21 +197,27 @@ suspend fun IOPlugin.writeZip( } else { path.resolveSibling(path.fileName.toString() + ".zip") } - if (Files.exists(actualFile) && Files.size(path) == 0.toLong()) { - Files.delete(path) - } - //Files.createFile(actualFile) - newZFS(actualFile).use { zipfs -> - val internalTargetPath = zipfs.getPath("/") - Files.createDirectories(internalTargetPath) - val tmp = Files.createTempDirectory("df_zip") - writeDataDirectory(tmp, node, format, envelopeFormat, metaFormat) - Files.list(tmp).forEach { sourcePath -> - val targetPath = sourcePath.fileName.toString() - val internalTargetPath = internalTargetPath.resolve(targetPath) - Files.copy(sourcePath, internalTargetPath, StandardCopyOption.REPLACE_EXISTING) - } + val fos = Files.newOutputStream(actualFile, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING) + val zos = ZipOutputStream(fos) + zos.use { + it.writeNode("", DataItem.Node(node), format, envelopeFormat) } + +// if (Files.exists(actualFile) && Files.size(path) == 0.toLong()) { +// Files.delete(path) +// } +// //Files.createFile(actualFile) +// newZFS(actualFile).use { zipfs -> +// val zipRootPath = zipfs.getPath("/") +// Files.createDirectories(zipRootPath) +// val tmp = Files.createTempDirectory("df_zip") +// writeDataDirectory(tmp, node, format, envelopeFormat, metaFormat) +// Files.list(tmp).forEach { sourcePath -> +// val targetPath = sourcePath.fileName.toString() +// val internalTargetPath = zipRootPath.resolve(targetPath) +// Files.copy(sourcePath, internalTargetPath, StandardCopyOption.REPLACE_EXISTING) +// } +// } } } diff --git a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt index 4fc9e9a4..6c9537cc 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt @@ -5,16 +5,17 @@ import hep.dataforge.data.* import hep.dataforge.io.IOFormat import hep.dataforge.io.io import hep.dataforge.meta.DFExperimental +import hep.dataforge.meta.Meta import kotlinx.coroutines.runBlocking import kotlinx.io.Input import kotlinx.io.Output import kotlinx.io.text.readUtf8String import kotlinx.io.text.writeUtf8String import java.nio.file.Files -import kotlin.test.Ignore import kotlin.test.Test import kotlin.test.assertEquals + class FileDataTest { val dataNode = DataNode { node("dir") { @@ -37,6 +38,10 @@ class FileDataTest { return readUtf8String() } + override fun toMeta(): Meta = Meta { + IOFormat.NAME_KEY put "string" + } + } @Test @@ -56,7 +61,7 @@ class FileDataTest { @Test - @Ignore + @DFExperimental fun testZipWriteRead() { Global.io.run { val zip = Files.createTempFile("df_data_node", ".zip") From b0950f84158d62aad7d604de0972dcdb4b0ea8dc Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 7 Apr 2020 12:53:00 +0300 Subject: [PATCH 11/11] 0.1.7 release --- build.gradle.kts | 4 ++-- .../kotlin/hep/dataforge/io/FileBinaryTest.kt | 4 +++- .../hep/dataforge/io/tcp/EnvelopeServerTest.kt | 12 +++++++----- .../src/commonMain/kotlin/hep/dataforge/meta/Meta.kt | 9 +++------ .../kotlin/hep/dataforge/meta/serializationUtils.kt | 2 ++ 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 95da230e..d73b0e60 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,12 +1,12 @@ plugins { - val toolsVersion = "0.4.1" + val toolsVersion = "0.4.2" id("scientifik.mpp") version toolsVersion apply false id("scientifik.jvm") version toolsVersion apply false id("scientifik.publish") version toolsVersion apply false } -val dataforgeVersion by extra("0.1.6-dev-2") +val dataforgeVersion by extra("0.1.7") val bintrayRepo by extra("dataforge") val githubProject by extra("dataforge-core") diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt index 685342cf..7ccc65e7 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/FileBinaryTest.kt @@ -1,6 +1,7 @@ package hep.dataforge.io import hep.dataforge.context.Global +import hep.dataforge.meta.DFExperimental import kotlinx.io.asBinary import kotlinx.io.toByteArray import kotlinx.io.writeDouble @@ -46,6 +47,7 @@ class FileBinaryTest { } + @DFExperimental @Test fun testFileDataSizeRewriting() { println(System.getProperty("user.dir")) @@ -53,6 +55,6 @@ class FileBinaryTest { Global.io.writeEnvelopeFile(tmpPath, envelope) val binary = Global.io.readEnvelopeFile(tmpPath)?.data!! - assertEquals(binary.size.toInt(), binary.toByteArray().size) + assertEquals(binary.size, binary.toByteArray().size) } } \ No newline at end of file diff --git a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt index 3e545421..48c108d4 100644 --- a/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt +++ b/dataforge-io/src/jvmTest/kotlin/hep/dataforge/io/tcp/EnvelopeServerTest.kt @@ -8,8 +8,9 @@ import hep.dataforge.io.toByteArray import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.runBlocking import kotlinx.io.writeDouble -import org.junit.AfterClass -import org.junit.BeforeClass +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Timeout import kotlin.test.Test import kotlin.test.assertEquals import kotlin.time.ExperimentalTime @@ -31,20 +32,21 @@ class EnvelopeServerTest { @JvmStatic val echoEnvelopeServer = EnvelopeServer(Global, 7778, EchoResponder, GlobalScope) - @BeforeClass + @BeforeAll @JvmStatic fun start() { echoEnvelopeServer.start() } - @AfterClass + @AfterAll @JvmStatic fun close() { echoEnvelopeServer.stop() } } - @Test(timeout = 1000) + @Test + @Timeout(1) fun doEchoTest() { val request = Envelope { type = "test.echo" diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt index 1afbb183..9650b04e 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Meta.kt @@ -98,7 +98,9 @@ interface Meta : MetaRepr { */ const val VALUE_KEY = "@value" - val EMPTY = EmptyMeta + val EMPTY: Meta = object: MetaBase() { + override val items: Map> = emptyMap() + } } } @@ -216,11 +218,6 @@ fun MetaItem<*>.seal(): MetaItem = when (this) { is NodeItem -> NodeItem(node.seal()) } -@Deprecated("Use Meta.EMPTY instead", replaceWith = ReplaceWith("Meta.EMPTY")) -object EmptyMeta : MetaBase() { - override val items: Map> = emptyMap() -} - /** * Unsafe methods to access values and nodes directly from [MetaItem] */ diff --git a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt index 41c33c7c..f06bb06a 100644 --- a/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt +++ b/dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/serializationUtils.kt @@ -66,5 +66,7 @@ inline fun Encoder.encodeStructure( encoder.endStructure(desc) } +@OptIn(UnstableDefault::class) val JSON_PRETTY = Json(JsonConfiguration(prettyPrint = true, useArrayPolymorphism = true)) +@OptIn(UnstableDefault::class) val JSON_PLAIN = Json(JsonConfiguration(prettyPrint = true, useArrayPolymorphism = true)) \ No newline at end of file