Add observability via MetaRef

dataforge-data/src/jvmTest/kotlin/space/kscience/dataforge/data/ActionsTest.kt

@@ -1,6 +1,11 @@
 package space.kscience.dataforge.data
 
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.ExperimentalCoroutinesApi
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.test.advanceUntilIdle
 import kotlinx.coroutines.test.runTest
+import kotlinx.coroutines.withContext
 import space.kscience.dataforge.actions.Action
 import space.kscience.dataforge.actions.invoke
 import space.kscience.dataforge.actions.mapping
@@ -9,7 +14,7 @@ import kotlin.test.Test
 import kotlin.test.assertEquals
 import kotlin.time.Duration.Companion.milliseconds
 
-@OptIn(DFExperimental::class)
+@OptIn(DFExperimental::class, ExperimentalCoroutinesApi::class)
 internal class ActionsTest {
     @Test
     fun testStaticMapAction() = runTest(timeout = 500.milliseconds) {
@@ -24,6 +29,8 @@ internal class ActionsTest {
         }
 
         val result = plusOne(data)
+
+        advanceUntilIdle()
         assertEquals(2, result["1"]?.await())
     }
 
@@ -38,8 +45,12 @@ internal class ActionsTest {
         val result = plusOne(source)
 
 
-        repeat(10) {
-            source.updateValue(it.toString(), it)
+        withContext(Dispatchers.Default) {
+            repeat(10) {
+                source.updateValue(it.toString(), it)
+            }
+
+            delay(50)
         }
 
 //        result.updates.take(10).onEach { println(it.name) }.collect()

dataforge-io/build.gradle.kts

@@ -4,7 +4,7 @@ plugins {
 
 description = "IO module"
 
-val ioVersion = "0.3.1"
+val ioVersion = "0.4.0"
 
 kscience {
     jvm()

dataforge-io/dataforge-io-proto/build.gradle.kts

@@ -7,6 +7,7 @@ description = "ProtoBuf meta IO"
 
 kscience {
     jvm()
+//    js()
     dependencies {
         api(projects.dataforgeIo)
         api("com.squareup.wire:wire-runtime:4.9.9")

dataforge-io/dataforge-io-proto/src/commonMain/kotlin/ProtoEnvelopeFormat.kt

@@ -0,0 +1,32 @@
+package pace.kscience.dataforge.io.proto
+
+import kotlinx.io.Sink
+import kotlinx.io.Source
+import kotlinx.io.readByteArray
+import okio.ByteString
+import okio.ByteString.Companion.toByteString
+import space.kscience.dataforge.io.Envelope
+import space.kscience.dataforge.io.EnvelopeFormat
+import space.kscience.dataforge.io.asBinary
+import space.kscience.dataforge.io.proto.ProtoEnvelope
+import space.kscience.dataforge.io.toByteArray
+import space.kscience.dataforge.meta.Meta
+
+
+public object ProtoEnvelopeFormat : EnvelopeFormat {
+    override fun readFrom(source: Source): Envelope {
+        val protoEnvelope = ProtoEnvelope.ADAPTER.decode(source.readByteArray())
+        return Envelope(
+            meta = protoEnvelope.meta?.let { ProtoMetaWrapper(it) } ?: Meta.EMPTY,
+            data = protoEnvelope.dataBytes.toByteArray().asBinary()
+        )
+    }
+
+    override fun writeTo(sink: Sink, obj: Envelope) {
+        val protoEnvelope = ProtoEnvelope(
+            obj.meta.toProto(),
+            obj.data?.toByteArray()?.toByteString() ?: ByteString.EMPTY
+        )
+        sink.write(ProtoEnvelope.ADAPTER.encode(protoEnvelope))
+    }
+}

dataforge-io/dataforge-io-proto/src/commonMain/kotlin/ProtoMetaFormat.kt

@@ -1,11 +1,10 @@
-package space.kscience.dataforge.io.proto
+package pace.kscience.dataforge.io.proto
 
 import kotlinx.io.Sink
 import kotlinx.io.Source
-import kotlinx.io.asInputStream
-import kotlinx.io.asOutputStream
-import org.slf4j.LoggerFactory
+import kotlinx.io.readByteArray
 import space.kscience.dataforge.io.MetaFormat
+import space.kscience.dataforge.io.proto.ProtoMeta
 import space.kscience.dataforge.meta.*
 import space.kscience.dataforge.meta.descriptors.MetaDescriptor
 import space.kscience.dataforge.names.NameToken
@@ -50,8 +49,8 @@ internal fun Meta.toProto(): ProtoMeta {
             is Long -> ProtoMeta.ProtoValue(int64Value = long)
             is Float -> ProtoMeta.ProtoValue(floatValue = float)
             else -> {
-                LoggerFactory.getLogger(ProtoMeta::class.java)
-                    .warn("Unknown number type ${value} encoded as Double")
+//                LoggerFactory.getLogger(ProtoMeta::class.java)
+//                    .warn("Unknown number type ${value} encoded as Double")
                 ProtoMeta.ProtoValue(doubleValue = double)
             }
         }
@@ -67,12 +66,11 @@ internal fun Meta.toProto(): ProtoMeta {
     )
 }
 
-
 public object ProtoMetaFormat : MetaFormat {
     override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) {
-        ProtoMeta.ADAPTER.encode(sink.asOutputStream(), meta.toProto())
+        sink.write(ProtoMeta.ADAPTER.encode(meta.toProto()))
     }
 
     override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta =
-        ProtoMetaWrapper(ProtoMeta.ADAPTER.decode(source.asInputStream()))
+        ProtoMetaWrapper(ProtoMeta.ADAPTER.decode(source.readByteArray()))
 }

dataforge-io/dataforge-io-proto/src/commonMain/proto/meta.proto

@@ -27,4 +27,9 @@ message ProtoMeta {
   ProtoValue protoValue = 1;
 
   map<string, ProtoMeta> items = 2;
+}
+
+message ProtoEnvelope{
+  ProtoMeta meta = 1;
+  bytes dataBytes = 2;
 }

dataforge-io/dataforge-io-proto/src/commonTest/kotlin/ProtoBufTest.kt

@@ -0,0 +1,83 @@
+package pace.kscience.dataforge.io.proto
+
+import kotlinx.io.writeString
+import space.kscience.dataforge.io.Envelope
+import space.kscience.dataforge.io.toByteArray
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.asValue
+import space.kscience.dataforge.meta.get
+import kotlin.test.Test
+import kotlin.test.assertContentEquals
+import kotlin.test.assertEquals
+
+class ProtoBufTest {
+
+    @Test
+    fun testProtoBufMetaFormat(){
+        val meta = Meta {
+            "a" put 22
+            "node" put {
+                "b" put "DDD"
+                "c" put 11.1
+                "d" put {
+                    "d1" put {
+                        "d11" put "aaa"
+                        "d12" put "bbb"
+                    }
+                    "d2" put 2
+                }
+                "array" put doubleArrayOf(1.0, 2.0, 3.0)
+                "array2d" put listOf(
+                    doubleArrayOf(1.0, 2.0, 3.0).asValue(),
+                    doubleArrayOf(1.0, 2.0, 3.0).asValue()
+                ).asValue()
+            }
+        }
+        val buffer = kotlinx.io.Buffer()
+        ProtoMetaFormat.writeTo(buffer,meta)
+        val result = ProtoMetaFormat.readFrom(buffer)
+
+//        println(result["a"]?.value)
+
+        meta.items.keys.forEach {
+            assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}")
+        }
+
+        assertEquals(meta, result)
+    }
+
+    @Test
+    fun testProtoBufEnvelopeFormat(){
+        val envelope = Envelope{
+            meta {
+                "a" put 22
+                "node" put {
+                    "b" put "DDD"
+                    "c" put 11.1
+                    "d" put {
+                        "d1" put {
+                            "d11" put "aaa"
+                            "d12" put "bbb"
+                        }
+                        "d2" put 2
+                    }
+                    "array" put doubleArrayOf(1.0, 2.0, 3.0)
+                    "array2d" put listOf(
+                        doubleArrayOf(1.0, 2.0, 3.0).asValue(),
+                        doubleArrayOf(1.0, 2.0, 3.0).asValue()
+                    ).asValue()
+                }
+            }
+            data {
+                writeString("Hello world!")
+            }
+        }
+
+        val buffer = kotlinx.io.Buffer()
+        ProtoEnvelopeFormat.writeTo(buffer,envelope)
+        val result = ProtoEnvelopeFormat.readFrom(buffer)
+
+        assertEquals(envelope.meta, result.meta)
+        assertContentEquals(envelope.data?.toByteArray(), result.data?.toByteArray())
+    }
+}

dataforge-io/dataforge-io-proto/src/jvmMain/kotlin/performanceComparison.kt

@@ -0,0 +1,51 @@
+package pace.kscience.dataforge.io.proto
+
+import kotlinx.io.writeString
+import space.kscience.dataforge.io.Envelope
+import space.kscience.dataforge.meta.asValue
+import kotlin.concurrent.thread
+import kotlin.time.measureTime
+
+public fun main() {
+    val envelope = Envelope {
+        meta {
+            "a" put 22
+            "node" put {
+                "b" put "DDD"
+                "c" put 11.1
+                "d" put {
+                    "d1" put {
+                        "d11" put "aaa"
+                        "d12" put "bbb"
+                    }
+                    "d2" put 2
+                }
+                "array" put doubleArrayOf(1.0, 2.0, 3.0)
+                "array2d" put listOf(
+                    doubleArrayOf(1.0, 2.0, 3.0).asValue(),
+                    doubleArrayOf(1.0, 2.0, 3.0).asValue()
+                ).asValue()
+            }
+        }
+        data {
+            writeString("Hello world!")
+        }
+    }
+
+    val format = ProtoEnvelopeFormat
+
+    measureTime {
+        val threads = List(100) {
+            thread {
+                repeat(100000) {
+                    val buffer = kotlinx.io.Buffer()
+                    format.writeTo(buffer, envelope)
+//                    println(buffer.size)
+                    val r = format.readFrom(buffer)
+                }
+            }
+        }
+
+        threads.forEach { it.join() }
+    }.also { println(it) }
+}

dataforge-io/dataforge-io-proto/src/jvmTest/kotlin/ProtoBufTest.kt

@@ -1,44 +0,0 @@
-package space.kscience.dataforge.io.proto
-
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.asValue
-import space.kscience.dataforge.meta.get
-import kotlin.test.Test
-import kotlin.test.assertEquals
-
-class ProtoBufTest {
-
-    @Test
-    fun testProtoBufMetaFormat(){
-        val meta = Meta {
-            "a" put 22
-            "node" put {
-                "b" put "DDD"
-                "c" put 11.1
-                "d" put {
-                    "d1" put {
-                        "d11" put "aaa"
-                        "d12" put "bbb"
-                    }
-                    "d2" put 2
-                }
-                "array" put doubleArrayOf(1.0, 2.0, 3.0)
-                "array2d" put listOf(
-                    doubleArrayOf(1.0, 2.0, 3.0).asValue(),
-                    doubleArrayOf(1.0, 2.0, 3.0).asValue()
-                ).asValue()
-            }
-        }
-        val buffer = kotlinx.io.Buffer()
-        ProtoMetaFormat.writeTo(buffer,meta)
-        val result = ProtoMetaFormat.readFrom(buffer)
-
-//        println(result["a"]?.value)
-
-        meta.items.keys.forEach {
-            assertEquals(meta[it],result[it],"${meta[it]} != ${result[it]}")
-        }
-
-        assertEquals(meta, result)
-    }
-}

dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Envelope.kt

@@ -2,7 +2,6 @@ package space.kscience.dataforge.io
 
 import space.kscience.dataforge.meta.Laminate
 import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.get
 import space.kscience.dataforge.meta.string
 import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.asName
@@ -34,7 +33,9 @@ public interface Envelope {
     }
 }
 
-internal class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope
+internal class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope{
+    override fun toString(): String  = "Envelope(meta=$meta, data=$data)"
+}
 
 public fun Envelope(meta: Meta, data: Binary?): Envelope = SimpleEnvelope(meta, data)
 

dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/JsonMeta.kt

@@ -34,9 +34,9 @@ private fun Meta.toJsonWithIndex(descriptor: MetaDescriptor?, index: String?): J
         val childDescriptor = descriptor?.nodes?.get(body)
         if (list.size == 1) {
             val (token, element) = list.first()
-                //do not add an empty element
-                val child: JsonElement = element.toJsonWithIndex(childDescriptor, token.index)
-            if(token.index == null) {
+            //do not add an empty element
+            val child: JsonElement = element.toJsonWithIndex(childDescriptor, token.index)
+            if (token.index == null) {
                 body to child
             } else {
                 body to JsonArray(listOf(child))
@@ -106,7 +106,7 @@ private fun JsonElement.toValueOrNull(descriptor: MetaDescriptor?): Value? = whe
 private fun MutableMap<NameToken, SealedMeta>.addJsonElement(
     key: String,
     element: JsonElement,
-    descriptor: MetaDescriptor?
+    descriptor: MetaDescriptor?,
 ) {
     when (element) {
         is JsonPrimitive -> put(NameToken(key), Meta(element.toValue(descriptor)))
@@ -136,12 +136,14 @@ private fun MutableMap<NameToken, SealedMeta>.addJsonElement(
                                 Meta(childValue)
                             }
                         }
+
                         is JsonPrimitive -> Meta(childElement.toValue(null))
                     }
                     put(NameToken(key, index), child)
                 }
             }
         }
+
         is JsonObject -> {
             val indexKey = descriptor?.indexKey ?: Meta.INDEX_KEY
             val index = element[indexKey]?.jsonPrimitive?.content
@@ -163,14 +165,14 @@ public fun JsonObject.toMeta(descriptor: MetaDescriptor? = null): SealedMeta {
 public fun JsonElement.toMeta(descriptor: MetaDescriptor? = null): SealedMeta = when (this) {
     is JsonPrimitive -> Meta(toValue(descriptor))
     is JsonObject -> toMeta(descriptor)
-    is JsonArray -> if (any { it is JsonObject }) {
+    is JsonArray -> if (all { it is JsonPrimitive }) {
+        Meta(map { it.toValueOrNull(descriptor) ?: error("Unreachable: should not contain objects") }.asValue())
+    } else {
         SealedMeta(null,
             linkedMapOf<NameToken, SealedMeta>().apply {
                 addJsonElement(Meta.JSON_ARRAY_KEY, this@toMeta, null)
             }
         )
-    } else{
-        Meta(map { it.toValueOrNull(descriptor) ?: kotlin.error("Unreachable: should not contain objects") }.asValue())
     }
 }
 

dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MetaRef.kt

@@ -1,17 +1,19 @@
 package space.kscience.dataforge.meta
 
+import kotlinx.serialization.json.Json
 import space.kscience.dataforge.meta.descriptors.Described
 import space.kscience.dataforge.meta.descriptors.MetaDescriptor
 import space.kscience.dataforge.meta.descriptors.MetaDescriptorBuilder
 import space.kscience.dataforge.misc.DFExperimental
 import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.asName
+import space.kscience.dataforge.names.startsWith
 import kotlin.properties.PropertyDelegateProvider
 import kotlin.properties.ReadOnlyProperty
 
 
 /**
- * A reference to a read-only value of type [T] inside [MetaProvider]
+ * A reference to a read-only value of type [T] inside [MetaProvider] or writable value in [MutableMetaProvider]
  */
 @DFExperimental
 public data class MetaRef<T>(
@@ -20,28 +22,77 @@ public data class MetaRef<T>(
     override val descriptor: MetaDescriptor? = converter.descriptor,
 ) : Described
 
+/**
+ * Get a value from provider by [ref] or return null if node with given name is missing
+ */
 @DFExperimental
 public operator fun <T> MetaProvider.get(ref: MetaRef<T>): T? = get(ref.name)?.let { ref.converter.readOrNull(it) }
 
+/**
+ * Set a value in a mutable provider by [ref]
+ */
 @DFExperimental
 public operator fun <T> MutableMetaProvider.set(ref: MetaRef<T>, value: T) {
     set(ref.name, ref.converter.convert(value))
 }
 
+/**
+ * Observe changes to specific property via given [ref].
+ *
+ * This listener should be removed in a same way as [ObservableMeta.onChange].
+ *
+ * @param callback an action to be performed on each change of item. Null means that the item is not present or malformed.
+ */
 @DFExperimental
-public class MetaSpec(
-    private val configuration: MetaDescriptorBuilder.() -> Unit = {},
-) : Described {
-    private val refs: MutableList<MetaRef<*>> = mutableListOf()
+public fun <T: Any> ObservableMeta.onValueChange(owner: Any?, ref: MetaRef<T>, callback: (T?) -> Unit) {
+    onChange(owner) { name ->
+        if (name.startsWith(ref.name)) {
+            get(name)?.let { value ->
+                callback(ref.converter.readOrNull(value))
+            }
+        }
+    }
+}
 
-    private fun registerRef(ref: MetaRef<*>) {
-        refs.add(ref)
+/**
+ * Remove a node corresponding to [ref] from a mutable provider if it exists
+ */
+@DFExperimental
+public fun MutableMetaProvider.remove(ref: MetaRef<*>) {
+    remove(ref.name)
+}
+
+/**
+ * Base storage of [MetaRef]
+ */
+@OptIn(DFExperimental::class)
+public interface MetaRefStore : Described {
+    public val refs: List<MetaRef<*>>
+}
+
+/**
+ * A base class for [Meta] specification that stores references to meta nodes.
+ */
+@DFExperimental
+public abstract class MetaSpec : MetaRefStore {
+    private val _refs: MutableList<MetaRef<*>> = mutableListOf()
+    override val refs: List<MetaRef<*>> get() = _refs
+
+    /**
+     * Register a ref in this specification
+     */
+    protected fun registerRef(ref: MetaRef<*>) {
+        _refs.add(ref)
     }
 
+    /**
+     * Create and register a ref by property name and provided converter.
+     * By default, uses descriptor from the converter
+     */
     public fun <T> item(
         converter: MetaConverter<T>,
-        descriptor: MetaDescriptor? = converter.descriptor,
         key: Name? = null,
+        descriptor: MetaDescriptor? = converter.descriptor,
     ): PropertyDelegateProvider<MetaSpec, ReadOnlyProperty<MetaSpec, MetaRef<T>>> =
         PropertyDelegateProvider { _, property ->
             val ref = MetaRef(key ?: property.name.asName(), converter, descriptor)
@@ -51,6 +102,11 @@ public class MetaSpec(
             }
         }
 
+    /**
+     * Override to provide custom [MetaDescriptor]
+     */
+    protected open fun MetaDescriptorBuilder.buildDescriptor(): Unit = Unit
+
     override val descriptor: MetaDescriptor by lazy {
         MetaDescriptor {
             refs.forEach { ref ->
@@ -58,7 +114,108 @@ public class MetaSpec(
                     node(ref.name, ref.descriptor)
                 }
             }
-            configuration()
+            buildDescriptor()
         }
     }
-}
+}
+
+/**
+ * Register an item using a [descriptorBuilder] to customize descriptor
+ */
+@DFExperimental
+public fun <T> MetaSpec.item(
+    converter: MetaConverter<T>,
+    key: Name? = null,
+    descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {},
+): PropertyDelegateProvider<MetaSpec, ReadOnlyProperty<MetaSpec, MetaRef<T>>> = item(converter, key, MetaDescriptor {
+    converter.descriptor?.let { from(it) }
+    descriptorBuilder()
+})
+
+//utility methods to add different nodes
+
+@DFExperimental
+public fun MetaSpec.metaItem(
+    key: Name? = null,
+    descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {},
+): PropertyDelegateProvider<MetaSpec, ReadOnlyProperty<MetaSpec, MetaRef<Meta>>> =
+    item(MetaConverter.meta, key, descriptorBuilder)
+
+@DFExperimental
+public fun MetaSpec.string(
+    key: Name? = null,
+    descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {},
+): PropertyDelegateProvider<MetaSpec, ReadOnlyProperty<MetaSpec, MetaRef<String>>> =
+    item(MetaConverter.string, key, descriptorBuilder)
+
+@DFExperimental
+public fun MetaSpec.boolean(
+    key: Name? = null,
+    descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {},
+): PropertyDelegateProvider<MetaSpec, ReadOnlyProperty<MetaSpec, MetaRef<Boolean>>> =
+    item(MetaConverter.boolean, key, descriptorBuilder)
+
+@DFExperimental
+public fun MetaSpec.stringList(
+    key: Name? = null,
+    descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {},
+): PropertyDelegateProvider<MetaSpec, ReadOnlyProperty<MetaSpec, MetaRef<List<String>>>> =
+    item(MetaConverter.stringList, key, descriptorBuilder)
+
+@DFExperimental
+public fun MetaSpec.float(
+    key: Name? = null,
+    descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {},
+): PropertyDelegateProvider<MetaSpec, ReadOnlyProperty<MetaSpec, MetaRef<Float>>> =
+    item(MetaConverter.float, key, descriptorBuilder)
+
+@DFExperimental
+public fun MetaSpec.double(
+    key: Name? = null,
+    descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {},
+): PropertyDelegateProvider<MetaSpec, ReadOnlyProperty<MetaSpec, MetaRef<Double>>> =
+    item(MetaConverter.double, key, descriptorBuilder)
+
+@DFExperimental
+public fun MetaSpec.int(
+    key: Name? = null,
+    descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {},
+): PropertyDelegateProvider<MetaSpec, ReadOnlyProperty<MetaSpec, MetaRef<Int>>> =
+    item(MetaConverter.int, key, descriptorBuilder)
+
+@DFExperimental
+public fun MetaSpec.long(
+    key: Name? = null,
+    descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {},
+): PropertyDelegateProvider<MetaSpec, ReadOnlyProperty<MetaSpec, MetaRef<Long>>> =
+    item(MetaConverter.long, key, descriptorBuilder)
+
+
+@DFExperimental
+public fun MetaSpec.doubleArray(
+    key: Name? = null,
+    descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {},
+): PropertyDelegateProvider<MetaSpec, ReadOnlyProperty<MetaSpec, MetaRef<DoubleArray>>> =
+    item(MetaConverter.doubleArray, key, descriptorBuilder)
+
+@DFExperimental
+public fun MetaSpec.byteArray(
+    key: Name? = null,
+    descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {},
+): PropertyDelegateProvider<MetaSpec, ReadOnlyProperty<MetaSpec, MetaRef<ByteArray>>> =
+    item(MetaConverter.byteArray, key, descriptorBuilder)
+
+@DFExperimental
+public inline fun <reified E : Enum<E>> MetaSpec.enum(
+    key: Name? = null,
+    noinline descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {},
+): PropertyDelegateProvider<MetaSpec, ReadOnlyProperty<MetaSpec, MetaRef<E>>> =
+    item(MetaConverter.enum(), key, descriptorBuilder)
+
+@DFExperimental
+public inline fun <reified T> MetaSpec.serializable(
+    key: Name? = null,
+    jsonEncoder: Json = Json,
+    noinline descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {},
+): PropertyDelegateProvider<MetaSpec, ReadOnlyProperty<MetaSpec, MetaRef<T>>> =
+    item(MetaConverter.serializable(jsonEncoder = jsonEncoder), key, descriptorBuilder)

dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ValueSerializer.kt

@@ -8,6 +8,9 @@ import kotlinx.serialization.descriptors.element
 import kotlinx.serialization.encoding.Decoder
 import kotlinx.serialization.encoding.Encoder
 
+/**
+ * A serializer for [Value]
+ */
 public object ValueSerializer : KSerializer<Value> {
     private val listSerializer by lazy { ListSerializer(ValueSerializer) }
 

dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/exoticValues.kt

@@ -21,6 +21,9 @@ public class LazyParsedValue(public val string: String) : Value {
     override fun hashCode(): Int = string.hashCode()
 }
 
+/**
+ * Read this string as lazily parsed value
+ */
 public fun String.lazyParseValue(): LazyParsedValue = LazyParsedValue(this)
 
 /**
@@ -47,6 +50,10 @@ public class DoubleArrayValue(override val value: DoubleArray) : Value, Iterable
     override fun iterator(): Iterator<Double> = value.iterator()
 }
 
+
+/**
+ * A zero-copy wrapping of this [DoubleArray] in a [Value]
+ */
 public fun DoubleArray.asValue(): Value = if (isEmpty()) Null else DoubleArrayValue(this)
 
 public val Value.doubleArray: DoubleArray
@@ -75,7 +82,17 @@ public fun MutableMetaProvider.doubleArray(
     reader = { it?.doubleArray ?: doubleArrayOf(*default) },
 )
 
+private object DoubleArrayMetaConverter : MetaConverter<DoubleArray> {
+    override fun readOrNull(source: Meta): DoubleArray? = source.doubleArray
 
+    override fun convert(obj: DoubleArray): Meta = Meta(obj.asValue())
+}
+
+public val MetaConverter.Companion.doubleArray: MetaConverter<DoubleArray> get() = DoubleArrayMetaConverter
+
+/**
+ * A [Value] wrapping a [ByteArray]
+ */
 public class ByteArrayValue(override val value: ByteArray) : Value, Iterable<Byte> {
     override val type: ValueType get() = ValueType.LIST
     override val list: List<Value> get() = value.map { NumberValue(it) }
@@ -123,4 +140,12 @@ public fun MutableMetaProvider.byteArray(
     key,
     writer = { ByteArrayValue(it) },
     reader = { it?.byteArray ?: byteArrayOf(*default) },
-)
+)
+
+private object ByteArrayMetaConverter : MetaConverter<ByteArray> {
+    override fun readOrNull(source: Meta): ByteArray? = source.byteArray
+
+    override fun convert(obj: ByteArray): Meta = Meta(obj.asValue())
+}
+
+public val MetaConverter.Companion.byteArray: MetaConverter<ByteArray> get() = ByteArrayMetaConverter

dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaRefTest.kt

@@ -0,0 +1,35 @@
+package space.kscience.dataforge.meta
+
+import kotlinx.serialization.Serializable
+import space.kscience.dataforge.misc.DFExperimental
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+@DFExperimental
+internal class MetaRefTest {
+
+    @Serializable
+    data class XY(val x: Double, val y: Double)
+
+    object TestMetaSpec : MetaSpec() {
+        val integer by int { description = "Integer value" }
+        val string by string { description = "String value" }
+        val custom by item(MetaConverter.serializable<XY>()) { description = "custom value" }
+    }
+
+    @Test
+    fun specWriteRead()  = with(TestMetaSpec){
+        val meta = MutableMeta()
+
+        meta[integer] = 22
+        meta[string] = "33"
+        val xy = XY(33.0, -33.0)
+        meta[custom] = xy
+
+        val sealed = meta.seal()
+
+        assertEquals(22, sealed[integer])
+        assertEquals("33", sealed[string])
+        assertEquals(xy, sealed[custom])
+    }
+}