Compare commits

...

4 Commits

18 changed files with 365 additions and 59 deletions

View File

@ -5,12 +5,16 @@
### Added
### Changed
- Simplify inheritance logic in `MutableTypedMeta`
### Deprecated
- MetaProvider `spec` is replaced by `readable`. `listOfSpec` replaced with `listOfReadable`
### Removed
### Fixed
- Fixed NameToken parsing.
- Top level string list meta conversion.
### Security

View File

@ -9,7 +9,7 @@ plugins {
allprojects {
group = "space.kscience"
version = "0.9.0"
version = "0.9.1-dev-1"
}
subprojects {

View File

@ -1,8 +1,5 @@
package space.kscience.dataforge.data
import kotlinx.coroutines.flow.collect
import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.flow.take
import kotlinx.coroutines.test.runTest
import space.kscience.dataforge.actions.Action
import space.kscience.dataforge.actions.invoke
@ -16,27 +13,28 @@ import kotlin.time.Duration.Companion.milliseconds
internal class ActionsTest {
@Test
fun testStaticMapAction() = runTest(timeout = 500.milliseconds) {
val plusOne = Action.mapping<Int, Int> {
result { it + 1 }
}
val data: DataTree<Int> = DataTree {
repeat(10) {
putValue(it.toString(), it)
}
}
val plusOne = Action.mapping<Int, Int> {
result { it + 1 }
}
val result = plusOne(data)
assertEquals(2, result["1"]?.await())
}
@Test
fun testDynamicMapAction() = runTest(timeout = 500.milliseconds) {
val source: MutableDataTree<Int> = MutableDataTree()
val plusOne = Action.mapping<Int, Int> {
result { it + 1 }
}
val source: MutableDataTree<Int> = MutableDataTree()
val result = plusOne(source)
@ -44,7 +42,7 @@ internal class ActionsTest {
source.updateValue(it.toString(), it)
}
result.updates.take(10).onEach { println(it.name) }.collect()
// result.updates.take(10).onEach { println(it.name) }.collect()
assertEquals(2, result["1"]?.await())
}

View File

@ -0,0 +1,32 @@
plugins {
id("space.kscience.gradle.mpp")
id("com.squareup.wire") version "4.9.9"
}
description = "ProtoBuf meta IO"
kscience {
jvm()
dependencies {
api(projects.dataforgeIo)
api("com.squareup.wire:wire-runtime:4.9.9")
}
useSerialization {
protobuf()
}
}
wire {
kotlin {
sourcePath {
srcDir("src/commonMain/proto")
}
}
}
readme {
maturity = space.kscience.gradle.Maturity.PROTOTYPE
description = """
ProtoBuf Meta representation
""".trimIndent()
}

View File

@ -0,0 +1,19 @@
syntax = "proto3";
package space.kscience.dataforge.io.proto;
message ProtoMeta {
message ProtoValue {
oneof value {
string stringValue = 2;
bool booleanValue = 3;
double doubleValue = 4;
float floatValue = 5;
int32 int32Value = 6;
int64 int64Value = 7;
bytes bytesValue = 8;
}
}
repeated ProtoValue value = 1;
map<string, ProtoMeta> items = 2;
}

View File

@ -0,0 +1,92 @@
package space.kscience.dataforge.io.proto
import kotlinx.io.Sink
import kotlinx.io.Source
import kotlinx.io.asInputStream
import kotlinx.io.asOutputStream
import org.slf4j.LoggerFactory
import space.kscience.dataforge.io.MetaFormat
import space.kscience.dataforge.meta.*
import space.kscience.dataforge.meta.descriptors.MetaDescriptor
import space.kscience.dataforge.names.NameToken
internal class ProtoMetaWrapper(private val proto: ProtoMeta) : Meta {
private fun ProtoMeta.ProtoValue.toValue(): Value = when {
stringValue != null -> stringValue.asValue()
booleanValue != null -> booleanValue.asValue()
doubleValue != null -> doubleValue.asValue()
floatValue != null -> floatValue.asValue()
int32Value != null -> int32Value.asValue()
int64Value != null -> int64Value.asValue()
bytesValue != null -> bytesValue.toByteArray().asValue()
else -> Null
}
override val value: Value?
get() = when (proto.value_.size) {
0 -> null
1 -> proto.value_[0].toValue()
else -> proto.value_.map { it.toValue() }.asValue()
}
override val items: Map<NameToken, Meta>
get() = proto.items.entries.associate { NameToken.parse(it.key) to ProtoMetaWrapper(it.value) }
override fun toString(): String = Meta.toString(this)
override fun equals(other: Any?): Boolean = Meta.equals(this, other as? Meta)
override fun hashCode(): Int = Meta.hashCode(this)
}
internal fun Meta.toProto(): ProtoMeta {
fun MutableList<ProtoMeta.ProtoValue>.appendProtoValues(value: Value): Unit {
when (value.type) {
ValueType.NULL -> {
//do nothing
}
ValueType.NUMBER -> when (value.value) {
is Int, is Short, is Byte -> add(ProtoMeta.ProtoValue(int32Value = value.int))
is Long -> add(ProtoMeta.ProtoValue(int64Value = value.long))
is Float -> add(ProtoMeta.ProtoValue(floatValue = value.float))
else -> {
LoggerFactory.getLogger(ProtoMeta::class.java)
.warn("Unknown number type ${value.value} encoded as Double")
add(ProtoMeta.ProtoValue(doubleValue = value.double))
}
}
ValueType.STRING -> add(ProtoMeta.ProtoValue(stringValue = value.string))
ValueType.BOOLEAN -> add(ProtoMeta.ProtoValue(booleanValue = value.boolean))
ValueType.LIST -> {
value.list.forEach {
if (it.type == ValueType.LIST) {
error("Nested lists are not supported")
} else {
appendProtoValues(it)
}
}
}
}
}
return ProtoMeta(
value_ = buildList { value?.let { appendProtoValues(it) } },
items.entries.associate { it.key.toString() to it.value.toProto() }
)
}
public object ProtoMetaFormat : MetaFormat {
override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) {
ProtoMeta.ADAPTER.encode(sink.asOutputStream(), meta.toProto())
}
override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta =
ProtoMetaWrapper(ProtoMeta.ADAPTER.decode(source.asInputStream()))
}

View File

@ -0,0 +1,39 @@
package space.kscience.dataforge.io.proto
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import kotlin.test.Test
import kotlin.test.assertEquals
class ProtoBufTest {
@Test
fun testProtoBufMetaFormat(){
val meta = Meta {
"a" put 22
"node" put {
"b" put "DDD"
"c" put 11.1
"d" put {
"d1" put {
"d11" put "aaa"
"d12" put "bbb"
}
"d2" put 2
}
"array" put doubleArrayOf(1.0, 2.0, 3.0)
}
}
val buffer = kotlinx.io.Buffer()
ProtoMetaFormat.writeTo(buffer,meta)
val result = ProtoMetaFormat.readFrom(buffer)
println(result["a"]?.value)
meta.items.keys.forEach {
assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}")
}
assertEquals(meta, result)
}
}

View File

@ -117,8 +117,11 @@ private fun MutableMap<NameToken, SealedMeta>.addJsonElement(
} else {
val indexKey = descriptor?.indexKey ?: Meta.INDEX_KEY
element.forEachIndexed { serial, childElement ->
val index = (childElement as? JsonObject)?.get(indexKey)?.jsonPrimitive?.content
val index = (childElement as? JsonObject)
?.get(indexKey)?.jsonPrimitive?.content
?: serial.toString()
val child: SealedMeta = when (childElement) {
is JsonObject -> childElement.toMeta(descriptor)
is JsonArray -> {
@ -160,11 +163,15 @@ public fun JsonObject.toMeta(descriptor: MetaDescriptor? = null): SealedMeta {
public fun JsonElement.toMeta(descriptor: MetaDescriptor? = null): SealedMeta = when (this) {
is JsonPrimitive -> Meta(toValue(descriptor))
is JsonObject -> toMeta(descriptor)
is JsonArray -> SealedMeta(null,
linkedMapOf<NameToken, SealedMeta>().apply {
addJsonElement(Meta.JSON_ARRAY_KEY, this@toMeta, null)
}
)
is JsonArray -> if (any { it is JsonObject }) {
SealedMeta(null,
linkedMapOf<NameToken, SealedMeta>().apply {
addJsonElement(Meta.JSON_ARRAY_KEY, this@toMeta, null)
}
)
} else{
Meta(map { it.toValueOrNull(descriptor) ?: kotlin.error("Unreachable: should not contain objects") }.asValue())
}
}
//

View File

@ -11,7 +11,7 @@ import space.kscience.dataforge.misc.DFExperimental
/**
* A converter of generic object to and from [Meta]
*/
public interface MetaConverter<T>: MetaReader<T> {
public interface MetaConverter<T> : MetaReader<T> {
/**
* A descriptor for resulting meta
@ -116,6 +116,12 @@ public interface MetaConverter<T>: MetaReader<T> {
override fun convert(obj: E): Meta = Meta(obj.asValue())
}
public val stringList: MetaConverter<List<String>> = object : MetaConverter<List<String>> {
override fun convert(obj: List<String>): Meta = Meta(obj.map { it.asValue() }.asValue())
override fun readOrNull(source: Meta): List<String>? = source.stringList
}
public fun <T> valueList(
writer: (T) -> Value = { Value.of(it) },
reader: (Value) -> T,

View File

@ -24,20 +24,45 @@ public fun MetaProvider.node(
}
}
/**
* Use [metaReader] to read the Meta node
* Use [reader] to read the Meta node
*/
public fun <T> MetaProvider.spec(
metaReader: MetaReader<T>,
public fun <T> MetaProvider.readable(
reader: MetaReader<T>,
key: Name? = null,
): MetaDelegate<T?> = object : MetaDelegate<T?> {
override val descriptor: MetaDescriptor? get() = metaReader.descriptor
override val descriptor: MetaDescriptor? get() = reader.descriptor
override fun getValue(thisRef: Any?, property: KProperty<*>): T? {
return get(key ?: property.name.asName())?.let { metaReader.read(it) }
return get(key ?: property.name.asName())?.let { reader.read(it) }
}
}
/**
* Use [reader] to read the Meta node or return [default] if node does not exist
*/
public fun <T> MetaProvider.readable(
reader: MetaReader<T>,
default: T,
key: Name? = null,
): MetaDelegate<T> = object : MetaDelegate<T> {
override val descriptor: MetaDescriptor? get() = reader.descriptor
override fun getValue(thisRef: Any?, property: KProperty<*>): T {
return get(key ?: property.name.asName())?.let { reader.read(it) } ?: default
}
}
/**
* Use [reader] to read the Meta node
*/
@Deprecated("Replace with reading", ReplaceWith("reading(metaReader, key)"))
public fun <T> MetaProvider.spec(
reader: MetaReader<T>,
key: Name? = null,
): MetaDelegate<T?> = readable(reader, key)
/**
* Use object serializer to transform it to Meta and back
*/
@ -45,34 +70,51 @@ public fun <T> MetaProvider.spec(
public inline fun <reified T> MetaProvider.serializable(
key: Name? = null,
descriptor: MetaDescriptor? = null,
): MetaDelegate<T?> = spec(MetaConverter.serializable(descriptor), key)
): MetaDelegate<T?> = readable(MetaConverter.serializable(descriptor), key)
@DFExperimental
public inline fun <reified T> MetaProvider.serializable(
key: Name? = null,
default: T,
descriptor: MetaDescriptor? = null,
): MetaDelegate<T> = readable(MetaConverter.serializable(descriptor), default, key)
@Deprecated("Use convertable", ReplaceWith("convertable(converter, key)"))
public fun <T> MetaProvider.node(
key: Name? = null,
converter: MetaReader<T>,
): ReadOnlyProperty<Any?, T?> = spec(converter, key)
): ReadOnlyProperty<Any?, T?> = readable(converter, key)
/**
* Use [converter] to convert a list of same name siblings meta to object
* Use [reader] to convert a list of same name siblings meta to object
*/
public fun <T> Meta.listOfSpec(
converter: MetaReader<T>,
public fun <T> Meta.listOfReadable(
reader: MetaReader<T>,
key: Name? = null,
): MetaDelegate<List<T>> = object : MetaDelegate<List<T>> {
override fun getValue(thisRef: Any?, property: KProperty<*>): List<T> {
val name = key ?: property.name.asName()
return getIndexed(name).values.map { converter.read(it) }
return getIndexed(name).values.map { reader.read(it) }
}
override val descriptor: MetaDescriptor? = converter.descriptor?.copy(multiple = true)
override val descriptor: MetaDescriptor? = reader.descriptor?.copy(multiple = true)
}
/**
* Use [converter] to convert a list of same name siblings meta to object
*/
@Deprecated("Replace with readingList", ReplaceWith("readingList(converter, key)"))
public fun <T> Meta.listOfSpec(
converter: MetaReader<T>,
key: Name? = null,
): MetaDelegate<List<T>> = listOfReadable(converter, key)
@DFExperimental
public inline fun <reified T> Meta.listOfSerializable(
key: Name? = null,
descriptor: MetaDescriptor? = null,
): MetaDelegate<List<T>> = listOfSpec(MetaConverter.serializable(descriptor), key)
): MetaDelegate<List<T>> = listOfReadable(MetaConverter.serializable(descriptor), key)
/**
* A property delegate that uses custom key

View File

@ -159,7 +159,17 @@ public interface MutableTypedMeta<M : MutableTypedMeta<M>> : TypedMeta<M>, Mutab
*/
@DFExperimental
public fun attach(name: Name, node: M)
override fun get(name: Name): M?
override fun get(name: Name): M? {
tailrec fun M.find(name: Name): M? = if (name.isEmpty()) {
self
} else {
items[name.firstOrNull()!!]?.find(name.cutFirst())
}
return self.find(name)
}
override fun getOrCreate(name: Name): M
}
@ -388,7 +398,7 @@ public fun MutableMeta.reset(newMeta: Meta) {
(items.keys - newMeta.items.keys).forEach {
remove(it.asName())
}
newMeta.items.forEach { (token, item)->
newMeta.items.forEach { (token, item) ->
set(token, item)
}
}

View File

@ -54,9 +54,25 @@ public fun <T> MutableMetaProvider.convertable(
}
}
@Deprecated("Use convertable", ReplaceWith("convertable(converter, key)"))
public fun <T> MutableMetaProvider.node(key: Name? = null, converter: MetaConverter<T>): MutableMetaDelegate<T?> =
convertable(converter, key)
public fun <T> MutableMetaProvider.convertable(
converter: MetaConverter<T>,
default: T,
key: Name? = null,
): MutableMetaDelegate<T> = object : MutableMetaDelegate<T> {
override val descriptor: MetaDescriptor? get() = converter.descriptor
override fun getValue(thisRef: Any?, property: KProperty<*>): T {
val name = key ?: property.name.asName()
return get(name)?.let { converter.read(it) } ?: default
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: T) {
val name = key ?: property.name.asName()
set(name, value?.let { converter.convert(it) })
}
}
/**
* Use object serializer to transform it to Meta and back.
@ -66,7 +82,14 @@ public fun <T> MutableMetaProvider.node(key: Name? = null, converter: MetaConver
public inline fun <reified T> MutableMetaProvider.serializable(
descriptor: MetaDescriptor? = null,
key: Name? = null,
): MutableMetaDelegate<T?> = convertable(MetaConverter.serializable(descriptor), key)
): MutableMetaDelegate<T?> = convertable<T>(MetaConverter.serializable(descriptor), key)
@DFExperimental
public inline fun <reified T> MutableMetaProvider.serializable(
descriptor: MetaDescriptor? = null,
default: T,
key: Name? = null,
): MutableMetaDelegate<T> = convertable(MetaConverter.serializable(descriptor), default, key)
/**
* Use [converter] to convert a list of same name siblings meta to object and back.

View File

@ -2,9 +2,6 @@ package space.kscience.dataforge.meta
import space.kscience.dataforge.misc.ThreadSafe
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.cutFirst
import space.kscience.dataforge.names.firstOrNull
import space.kscience.dataforge.names.isEmpty
internal data class MetaListener(
@ -39,20 +36,7 @@ public interface ObservableMeta : Meta {
* A [Meta] which is both observable and mutable
*/
public interface ObservableMutableMeta : ObservableMeta, MutableMeta, MutableTypedMeta<ObservableMutableMeta> {
override val self: ObservableMutableMeta get() = this
override fun getOrCreate(name: Name): ObservableMutableMeta
override fun get(name: Name): ObservableMutableMeta? {
tailrec fun ObservableMutableMeta.find(name: Name): ObservableMutableMeta? = if (name.isEmpty()) {
this
} else {
items[name.firstOrNull()!!]?.find(name.cutFirst())
}
return find(name)
}
}
internal abstract class AbstractObservableMeta : ObservableMeta {

View File

@ -221,13 +221,14 @@ public fun <T : Scheme> Configurable.updateWith(
/**
* A delegate that uses a [MetaReader] to wrap a child of this provider
*/
public fun <T : Scheme> MutableMeta.scheme(
public fun <T : Scheme> MutableMetaProvider.scheme(
spec: SchemeSpec<T>,
key: Name? = null,
): ReadWriteProperty<Any?, T> = object : ReadWriteProperty<Any?, T> {
override fun getValue(thisRef: Any?, property: KProperty<*>): T {
val name = key ?: property.name.asName()
return spec.write(getOrCreate(name))
val node = get(name)?: MutableMeta().also { set(name,it) }
return spec.write(node)
}
override fun setValue(thisRef: Any?, property: KProperty<*>, value: T) {

View File

@ -67,10 +67,29 @@ public class NameToken(public val body: String, public val index: String? = null
* Parse name token from a string
*/
public fun parse(string: String): NameToken {
val body = string.substringBefore('[')
val index = string.substringAfter('[', "")
if (index.isNotEmpty() && !index.endsWith(']')) error("NameToken with index must end with ']'")
return NameToken(body, index.removeSuffix("]"))
var indexStart = -1
var indexEnd = -1
string.forEachIndexed { index, c ->
when (c) {
'[' -> when {
indexStart >= 0 -> error("Second opening bracket not allowed in NameToken: $string")
else -> indexStart = index
}
']' -> when {
indexStart < 0 -> error("Closing index bracket could not be used before opening bracket in NameToken: $string")
indexEnd >= 0 -> error("Second closing bracket not allowed in NameToken: $string")
else -> indexEnd = index
}
else -> if(indexEnd>=0) error("Symbols not allowed after index in NameToken: $string")
}
}
if(indexStart>=0 && indexEnd<0) error("Opening bracket without closing bracket not allowed in NameToken: $string")
return NameToken(
if(indexStart>=0) string.substring(0, indexStart) else string,
if(indexStart>=0) string.substring(indexStart + 1, indexEnd) else null
)
}
}
}

View File

@ -0,0 +1,17 @@
package space.kscience.dataforge.meta
import kotlin.test.Test
import kotlin.test.assertEquals
class ConvertersTest {
@Test
fun stringListConversion() {
val list = listOf("A", "B", "C")
val meta = MetaConverter.stringList.convert(list)
val json = meta.toJson()
val reconstructedMeta = json.toMeta()
val reconstructed = MetaConverter.stringList.read(reconstructedMeta)
assertEquals(list,reconstructed)
}
}

View File

@ -56,10 +56,22 @@ class NameTest {
val token2 = NameToken.parse("token-body")
assertEquals("token-body", token2.body)
assertEquals("", token2.index)
assertEquals(null, token2.index)
// val token3 = NameToken.parse("[token-index]")
// assertEquals("", token3.body)
// assertEquals("token-index", token3.index)
assertFails{
NameToken.parse("[token-index]")
}
assertFails {
NameToken.parse("token[22")
}
assertFails {
NameToken.parse("token[22]ddd")
}
}
}

View File

@ -43,6 +43,7 @@ include(
":dataforge-meta",
":dataforge-io",
":dataforge-io:dataforge-io-yaml",
":dataforge-io:dataforge-io-proto",
":dataforge-context",
":dataforge-data",
":dataforge-workspace",