Meta and Evelope format factories now implement Meta and Envelope formats (default representation).
This commit is contained in:
parent
ff59c14c17
commit
3e9cb3915c
@ -64,21 +64,13 @@ val <T : Any> DataItem<T>?.node: DataNode<T>? get() = (this as? DataItem.Node<T>
|
|||||||
val <T : Any> DataItem<T>?.data: Data<T>? get() = (this as? DataItem.Leaf<T>)?.value
|
val <T : Any> DataItem<T>?.data: Data<T>? get() = (this as? DataItem.Leaf<T>)?.value
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Start computation for all goals in data node
|
* Start computation for all goals in data node and return a job for the whole node
|
||||||
*/
|
*/
|
||||||
fun DataNode<*>.startAll(scope: CoroutineScope): Unit = items.values.forEach {
|
fun DataNode<*>.launchAll(scope: CoroutineScope): Job = scope.launch {
|
||||||
when (it) {
|
items.values.forEach {
|
||||||
is DataItem.Node<*> -> it.value.startAll(scope)
|
when (it) {
|
||||||
is DataItem.Leaf<*> -> it.value.start(scope)
|
is DataItem.Node<*> -> it.value.launchAll(scope)
|
||||||
}
|
is DataItem.Leaf<*> -> it.value.start(scope)
|
||||||
}
|
|
||||||
|
|
||||||
fun DataNode<*>.joinAll(scope: CoroutineScope): Job = scope.launch {
|
|
||||||
startAll(scope)
|
|
||||||
items.forEach {
|
|
||||||
when (val value = it.value) {
|
|
||||||
is DataItem.Node -> value.value.joinAll(this).join()
|
|
||||||
is DataItem.Leaf -> value.value.await(scope)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -24,7 +24,7 @@ class FrontMatterEnvelopeFormat(
|
|||||||
|
|
||||||
val readMetaFormat =
|
val readMetaFormat =
|
||||||
metaTypeRegex.matchEntire(line)?.groupValues?.first()
|
metaTypeRegex.matchEntire(line)?.groupValues?.first()
|
||||||
?.let { io.metaFormat(it) } ?: YamlMetaFormat.default
|
?.let { io.metaFormat(it) } ?: YamlMetaFormat
|
||||||
|
|
||||||
val metaBlock = buildPacket {
|
val metaBlock = buildPacket {
|
||||||
do {
|
do {
|
||||||
@ -45,7 +45,7 @@ class FrontMatterEnvelopeFormat(
|
|||||||
|
|
||||||
val readMetaFormat =
|
val readMetaFormat =
|
||||||
metaTypeRegex.matchEntire(line)?.groupValues?.first()
|
metaTypeRegex.matchEntire(line)?.groupValues?.first()
|
||||||
?.let { io.metaFormat(it) } ?: YamlMetaFormat.default
|
?.let { io.metaFormat(it) } ?: YamlMetaFormat
|
||||||
|
|
||||||
val metaBlock = buildPacket {
|
val metaBlock = buildPacket {
|
||||||
do {
|
do {
|
||||||
@ -72,7 +72,7 @@ class FrontMatterEnvelopeFormat(
|
|||||||
private val metaTypeRegex = "---(\\w*)\\s*".toRegex()
|
private val metaTypeRegex = "---(\\w*)\\s*".toRegex()
|
||||||
|
|
||||||
override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
|
override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
|
||||||
return FrontMatterEnvelopeFormat(context.io, meta)
|
return FrontMatterEnvelopeFormat(context.io, meta)
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
|
override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
|
||||||
@ -84,5 +84,16 @@ class FrontMatterEnvelopeFormat(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private val default by lazy { invoke() }
|
||||||
|
|
||||||
|
override fun Input.readPartial(): PartialEnvelope =
|
||||||
|
default.run { readPartial() }
|
||||||
|
|
||||||
|
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) =
|
||||||
|
default.run { writeEnvelope(envelope, metaFormatFactory, formatMeta) }
|
||||||
|
|
||||||
|
override fun Input.readObject(): Envelope =
|
||||||
|
default.run { readObject() }
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -45,12 +45,18 @@ class YamlMetaFormat(val meta: Meta) : MetaFormat {
|
|||||||
}
|
}
|
||||||
|
|
||||||
companion object : MetaFormatFactory {
|
companion object : MetaFormatFactory {
|
||||||
val default = YamlMetaFormat()
|
|
||||||
|
|
||||||
override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta)
|
override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta)
|
||||||
|
|
||||||
override val name: Name = super.name + "yaml"
|
override val name: Name = super.name + "yaml"
|
||||||
|
|
||||||
override val key: Short = 0x594d //YM
|
override val key: Short = 0x594d //YM
|
||||||
|
|
||||||
|
private val default = YamlMetaFormat()
|
||||||
|
|
||||||
|
override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) =
|
||||||
|
default.run { writeMeta(meta, descriptor) }
|
||||||
|
|
||||||
|
override fun Input.readMeta(descriptor: NodeDescriptor?): Meta =
|
||||||
|
default.run { readMeta(descriptor) }
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -8,7 +8,7 @@ import kotlin.math.min
|
|||||||
*/
|
*/
|
||||||
interface Binary {
|
interface Binary {
|
||||||
/**
|
/**
|
||||||
* The size of binary in bytes
|
* The size of binary in bytes. [ULong.MAX_VALUE] if size is not defined and input should be read until its end is reached
|
||||||
*/
|
*/
|
||||||
val size: ULong
|
val size: ULong
|
||||||
|
|
||||||
@ -18,6 +18,10 @@ interface Binary {
|
|||||||
* Some implementation may forbid this to be called twice. In this case second call will throw an exception.
|
* Some implementation may forbid this to be called twice. In this case second call will throw an exception.
|
||||||
*/
|
*/
|
||||||
fun <R> read(block: Input.() -> R): R
|
fun <R> read(block: Input.() -> R): R
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
val EMPTY = EmptyBinary
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -48,12 +52,11 @@ fun RandomAccessBinary.readPacket(from: UInt, size: UInt): ByteReadPacket = read
|
|||||||
@ExperimentalUnsignedTypes
|
@ExperimentalUnsignedTypes
|
||||||
object EmptyBinary : RandomAccessBinary {
|
object EmptyBinary : RandomAccessBinary {
|
||||||
|
|
||||||
override val size: ULong = 0.toULong()
|
override val size: ULong = 0u
|
||||||
|
|
||||||
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
|
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
|
||||||
error("The binary is empty")
|
error("The binary is empty")
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@ExperimentalUnsignedTypes
|
@ExperimentalUnsignedTypes
|
||||||
@ -79,9 +82,9 @@ fun <T : Any> Binary.readWith(format: IOFormat<T>): T = format.run {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun <T : Any> IOFormat<T>.writeBinary(obj: T): Binary {
|
//fun <T : Any> IOFormat<T>.writeBinary(obj: T): Binary {
|
||||||
val packet = buildPacket {
|
// val packet = buildPacket {
|
||||||
writeObject(obj)
|
// writeObject(obj)
|
||||||
}
|
// }
|
||||||
return ArrayBinary(packet.readBytes())
|
// return ArrayBinary(packet.readBytes())
|
||||||
}
|
//}
|
@ -31,7 +31,7 @@ interface EnvelopeFormat : IOFormat<Envelope> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Type(ENVELOPE_FORMAT_TYPE)
|
@Type(ENVELOPE_FORMAT_TYPE)
|
||||||
interface EnvelopeFormatFactory : IOFormatFactory<Envelope> {
|
interface EnvelopeFormatFactory : IOFormatFactory<Envelope>, EnvelopeFormat {
|
||||||
override val name: Name get() = "envelope".asName()
|
override val name: Name get() = "envelope".asName()
|
||||||
override val type: KClass<out Envelope> get() = Envelope::class
|
override val type: KClass<out Envelope> get() = Envelope::class
|
||||||
|
|
||||||
|
@ -38,12 +38,18 @@ class JsonMetaFormat(private val json: Json = Json.indented) : MetaFormat {
|
|||||||
}
|
}
|
||||||
|
|
||||||
companion object : MetaFormatFactory {
|
companion object : MetaFormatFactory {
|
||||||
val default = JsonMetaFormat()
|
|
||||||
|
|
||||||
override fun invoke(meta: Meta, context: Context): MetaFormat = default
|
override fun invoke(meta: Meta, context: Context): MetaFormat = default
|
||||||
|
|
||||||
override val name: Name = super.name + "json"
|
override val name: Name = super.name + "json"
|
||||||
override val key: Short = 0x4a53//"JS"
|
override val key: Short = 0x4a53//"JS"
|
||||||
|
|
||||||
|
private val default = JsonMetaFormat()
|
||||||
|
|
||||||
|
override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) =
|
||||||
|
default.run { writeMeta(meta,descriptor) }
|
||||||
|
|
||||||
|
override fun Input.readMeta(descriptor: NodeDescriptor?): Meta =
|
||||||
|
default.run { readMeta(descriptor) }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -90,7 +96,7 @@ fun Meta.toJson(descriptor: NodeDescriptor? = null): JsonObject {
|
|||||||
fun JsonElement.toMeta(descriptor: NodeDescriptor? = null): Meta {
|
fun JsonElement.toMeta(descriptor: NodeDescriptor? = null): Meta {
|
||||||
return when (val item = toMetaItem(descriptor)) {
|
return when (val item = toMetaItem(descriptor)) {
|
||||||
is MetaItem.NodeItem<*> -> item.node
|
is MetaItem.NodeItem<*> -> item.node
|
||||||
is MetaItem.ValueItem ->item.value.toMeta()
|
is MetaItem.ValueItem -> item.value.toMeta()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -27,7 +27,7 @@ interface MetaFormat : IOFormat<Meta> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Type(META_FORMAT_TYPE)
|
@Type(META_FORMAT_TYPE)
|
||||||
interface MetaFormatFactory : IOFormatFactory<Meta> {
|
interface MetaFormatFactory : IOFormatFactory<Meta>, MetaFormat {
|
||||||
override val name: Name get() = "meta".asName()
|
override val name: Name get() = "meta".asName()
|
||||||
|
|
||||||
override val type: KClass<out Meta> get() = Meta::class
|
override val type: KClass<out Meta> get() = Meta::class
|
||||||
@ -47,7 +47,7 @@ fun Meta.toString(format: MetaFormat): String = buildPacket {
|
|||||||
|
|
||||||
fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory())
|
fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory())
|
||||||
|
|
||||||
fun Meta.toBytes(format: MetaFormat = JsonMetaFormat.default): ByteReadPacket = buildPacket {
|
fun Meta.toBytes(format: MetaFormat = JsonMetaFormat): ByteReadPacket = buildPacket {
|
||||||
format.run { writeObject(this@toBytes) }
|
format.run { writeObject(this@toBytes) }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -39,7 +39,12 @@ class TaggedEnvelopeFormat(
|
|||||||
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
|
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
|
||||||
val metaFormat = metaFormatFactory.invoke(formatMeta, io.context)
|
val metaFormat = metaFormatFactory.invoke(formatMeta, io.context)
|
||||||
val metaBytes = metaFormat.writeBytes(envelope.meta)
|
val metaBytes = metaFormat.writeBytes(envelope.meta)
|
||||||
val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, envelope.data?.size ?: 0.toULong())
|
val actualSize: ULong = if (envelope.data == null) {
|
||||||
|
0u
|
||||||
|
} else {
|
||||||
|
envelope.data?.size ?: ULong.MAX_VALUE
|
||||||
|
}
|
||||||
|
val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, actualSize)
|
||||||
writePacket(tag.toBytes())
|
writePacket(tag.toBytes())
|
||||||
writeFully(metaBytes)
|
writeFully(metaBytes)
|
||||||
writeText("\r\n")
|
writeText("\r\n")
|
||||||
@ -134,7 +139,16 @@ class TaggedEnvelopeFormat(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
val default by lazy { invoke() }
|
private val default by lazy { invoke() }
|
||||||
|
|
||||||
|
override fun Input.readPartial(): PartialEnvelope =
|
||||||
|
default.run { readPartial() }
|
||||||
|
|
||||||
|
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) =
|
||||||
|
default.run { writeEnvelope(envelope, metaFormatFactory, formatMeta) }
|
||||||
|
|
||||||
|
override fun Input.readObject(): Envelope =
|
||||||
|
default.run { readObject() }
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
@ -27,7 +27,13 @@ class TaglessEnvelopeFormat(
|
|||||||
//printing all properties
|
//printing all properties
|
||||||
writeProperty(META_TYPE_PROPERTY, metaFormatFactory.type)
|
writeProperty(META_TYPE_PROPERTY, metaFormatFactory.type)
|
||||||
//TODO add optional metaFormat properties
|
//TODO add optional metaFormat properties
|
||||||
writeProperty(DATA_LENGTH_PROPERTY, envelope.data?.size ?: 0)
|
val actualSize: ULong = if (envelope.data == null) {
|
||||||
|
0u
|
||||||
|
} else {
|
||||||
|
envelope.data?.size ?: ULong.MAX_VALUE
|
||||||
|
}
|
||||||
|
|
||||||
|
writeProperty(DATA_LENGTH_PROPERTY, actualSize)
|
||||||
|
|
||||||
//Printing meta
|
//Printing meta
|
||||||
if (!envelope.meta.isEmpty()) {
|
if (!envelope.meta.isEmpty()) {
|
||||||
@ -66,7 +72,7 @@ class TaglessEnvelopeFormat(
|
|||||||
var meta: Meta = EmptyMeta
|
var meta: Meta = EmptyMeta
|
||||||
|
|
||||||
if (line.startsWith(metaStart)) {
|
if (line.startsWith(metaStart)) {
|
||||||
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default
|
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat
|
||||||
val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt()
|
val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt()
|
||||||
meta = if (metaSize != null) {
|
meta = if (metaSize != null) {
|
||||||
val metaPacket = buildPacket {
|
val metaPacket = buildPacket {
|
||||||
@ -121,7 +127,7 @@ class TaglessEnvelopeFormat(
|
|||||||
var meta: Meta = EmptyMeta
|
var meta: Meta = EmptyMeta
|
||||||
|
|
||||||
if (line.startsWith(metaStart)) {
|
if (line.startsWith(metaStart)) {
|
||||||
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat.default
|
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.metaFormat(it) } ?: JsonMetaFormat
|
||||||
|
|
||||||
val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt()
|
val metaSize = properties.get(META_LENGTH_PROPERTY)?.toInt()
|
||||||
meta = if (metaSize != null) {
|
meta = if (metaSize != null) {
|
||||||
@ -170,7 +176,16 @@ class TaglessEnvelopeFormat(
|
|||||||
return TaglessEnvelopeFormat(context.io, meta)
|
return TaglessEnvelopeFormat(context.io, meta)
|
||||||
}
|
}
|
||||||
|
|
||||||
val default by lazy { invoke() }
|
private val default by lazy { invoke() }
|
||||||
|
|
||||||
|
override fun Input.readPartial(): PartialEnvelope =
|
||||||
|
default.run { readPartial() }
|
||||||
|
|
||||||
|
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) =
|
||||||
|
default.run { writeEnvelope(envelope, metaFormatFactory, formatMeta) }
|
||||||
|
|
||||||
|
override fun Input.readObject(): Envelope =
|
||||||
|
default.run { readObject() }
|
||||||
|
|
||||||
override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
|
override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
|
||||||
return try {
|
return try {
|
||||||
|
@ -18,7 +18,7 @@ class EnvelopeFormatTest {
|
|||||||
@ExperimentalStdlibApi
|
@ExperimentalStdlibApi
|
||||||
@Test
|
@Test
|
||||||
fun testTaggedFormat(){
|
fun testTaggedFormat(){
|
||||||
TaggedEnvelopeFormat.default.run {
|
TaggedEnvelopeFormat.run {
|
||||||
val bytes = writeBytes(envelope)
|
val bytes = writeBytes(envelope)
|
||||||
println(bytes.decodeToString())
|
println(bytes.decodeToString())
|
||||||
val res = readBytes(bytes)
|
val res = readBytes(bytes)
|
||||||
@ -32,7 +32,7 @@ class EnvelopeFormatTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun testTaglessFormat(){
|
fun testTaglessFormat(){
|
||||||
TaglessEnvelopeFormat.default.run {
|
TaglessEnvelopeFormat.run {
|
||||||
val bytes = writeBytes(envelope)
|
val bytes = writeBytes(envelope)
|
||||||
println(bytes.decodeToString())
|
println(bytes.decodeToString())
|
||||||
val res = readBytes(bytes)
|
val res = readBytes(bytes)
|
||||||
|
@ -23,9 +23,9 @@ class EnvelopePartsTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun testParts() {
|
fun testParts() {
|
||||||
val bytes = TaggedEnvelopeFormat.default.writeBytes(partsEnvelope)
|
val bytes = TaggedEnvelopeFormat.writeBytes(partsEnvelope)
|
||||||
val reconstructed = TaggedEnvelopeFormat.default.readBytes(bytes)
|
val reconstructed = TaggedEnvelopeFormat.readBytes(bytes)
|
||||||
val parts = reconstructed.parts().toList()
|
val parts = reconstructed.parts()?.toList() ?: emptyList()
|
||||||
assertEquals(2, parts[2].meta["value"].int)
|
assertEquals(2, parts[2].meta["value"].int)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -0,0 +1,29 @@
|
|||||||
|
package hep.dataforge.io
|
||||||
|
|
||||||
|
import hep.dataforge.io.functions.FunctionServer
|
||||||
|
import hep.dataforge.io.functions.function
|
||||||
|
import hep.dataforge.meta.Meta
|
||||||
|
import hep.dataforge.meta.buildMeta
|
||||||
|
import hep.dataforge.names.Name
|
||||||
|
import kotlin.reflect.KClass
|
||||||
|
import kotlin.reflect.full.isSuperclassOf
|
||||||
|
|
||||||
|
|
||||||
|
fun IOPlugin.resolveIOFormatName(type: KClass<*>): Name {
|
||||||
|
return ioFormats.entries.find { it.value.type.isSuperclassOf(type) }?.key
|
||||||
|
?: error("Can't resolve IOFormat for type $type")
|
||||||
|
}
|
||||||
|
|
||||||
|
inline fun <reified T : Any, reified R : Any> IOPlugin.generateFunctionMeta(functionName: String): Meta = buildMeta {
|
||||||
|
FunctionServer.FUNCTION_NAME_KEY put functionName
|
||||||
|
FunctionServer.INPUT_FORMAT_KEY put resolveIOFormatName(T::class).toString()
|
||||||
|
FunctionServer.OUTPUT_FORMAT_KEY put resolveIOFormatName(R::class).toString()
|
||||||
|
}
|
||||||
|
|
||||||
|
inline fun <reified T : Any, reified R : Any> FunctionServer.function(
|
||||||
|
functionName: String
|
||||||
|
): (suspend (T) -> R) {
|
||||||
|
val plugin = context.plugins.get<IOPlugin>() ?: error("IO plugin not loaded")
|
||||||
|
val meta = plugin.generateFunctionMeta<T, R>(functionName)
|
||||||
|
return function(meta)
|
||||||
|
}
|
@ -1,51 +1,93 @@
|
|||||||
package hep.dataforge.io
|
package hep.dataforge.io
|
||||||
|
|
||||||
import hep.dataforge.descriptors.NodeDescriptor
|
import hep.dataforge.descriptors.NodeDescriptor
|
||||||
import hep.dataforge.io.functions.FunctionServer
|
import hep.dataforge.meta.DFExperimental
|
||||||
import hep.dataforge.io.functions.FunctionServer.Companion.FUNCTION_NAME_KEY
|
import hep.dataforge.meta.EmptyMeta
|
||||||
import hep.dataforge.io.functions.FunctionServer.Companion.INPUT_FORMAT_KEY
|
|
||||||
import hep.dataforge.io.functions.FunctionServer.Companion.OUTPUT_FORMAT_KEY
|
|
||||||
import hep.dataforge.io.functions.function
|
|
||||||
import hep.dataforge.meta.Meta
|
import hep.dataforge.meta.Meta
|
||||||
import hep.dataforge.meta.buildMeta
|
import kotlinx.io.nio.asInput
|
||||||
import hep.dataforge.names.Name
|
|
||||||
import kotlinx.io.nio.asOutput
|
import kotlinx.io.nio.asOutput
|
||||||
import java.nio.file.Files
|
import java.nio.file.Files
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
import java.nio.file.StandardOpenOption
|
import java.nio.file.StandardOpenOption
|
||||||
import kotlin.reflect.KClass
|
|
||||||
import kotlin.reflect.full.isSuperclassOf
|
import kotlin.reflect.full.isSuperclassOf
|
||||||
|
import kotlin.streams.asSequence
|
||||||
|
|
||||||
inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? {
|
inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? {
|
||||||
return ioFormats.values.find { it.type.isSuperclassOf(T::class) } as IOFormat<T>?
|
return ioFormats.values.find { it.type.isSuperclassOf(T::class) } as IOFormat<T>?
|
||||||
}
|
}
|
||||||
|
|
||||||
fun IOPlugin.resolveIOFormatName(type: KClass<*>): Name {
|
/**
|
||||||
return ioFormats.entries.find { it.value.type.isSuperclassOf(type) }?.key
|
* Read file containing meta using given [formatOverride] or file extension to infer meta type.
|
||||||
?: error("Can't resolve IOFormat for type $type")
|
*/
|
||||||
|
fun IOPlugin.readMetaFile(path: Path, formatOverride: MetaFormat? = null, descriptor: NodeDescriptor? = null): Meta {
|
||||||
|
if (!Files.exists(path)) error("Meta file $path does not exist")
|
||||||
|
val extension = path.fileName.toString().substringAfterLast('.')
|
||||||
|
|
||||||
|
val metaFormat = formatOverride ?: metaFormat(extension) ?: error("Can't resolve meta format $extension")
|
||||||
|
return metaFormat.run {
|
||||||
|
Files.newByteChannel(path, StandardOpenOption.READ).asInput().use { it.readMeta(descriptor) }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
inline fun <reified T : Any, reified R : Any> IOPlugin.generateFunctionMeta(functionName: String): Meta = buildMeta {
|
fun IOPlugin.writeMetaFile(
|
||||||
FUNCTION_NAME_KEY put functionName
|
path: Path,
|
||||||
INPUT_FORMAT_KEY put resolveIOFormatName(T::class).toString()
|
metaFormat: MetaFormat = JsonMetaFormat,
|
||||||
OUTPUT_FORMAT_KEY put resolveIOFormatName(R::class).toString()
|
descriptor: NodeDescriptor? = null
|
||||||
}
|
) {
|
||||||
|
metaFormat.run {
|
||||||
inline fun <reified T : Any, reified R : Any> FunctionServer.function(
|
Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW).asOutput().use {
|
||||||
functionName: String
|
it.writeMeta(meta, descriptor)
|
||||||
): (suspend (T) -> R) {
|
}
|
||||||
val plugin = context.plugins.get<IOPlugin>() ?: error("IO plugin not loaded")
|
}
|
||||||
val meta = plugin.generateFunctionMeta<T, R>(functionName)
|
|
||||||
return function(meta)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Write meta to file in a given [format]
|
* Read and envelope from file if the file exists, return null if file does not exist.
|
||||||
*/
|
*/
|
||||||
fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) {
|
@DFExperimental
|
||||||
format.run {
|
fun IOPlugin.readEnvelopeFromFile(path: Path, readNonEnvelopes: Boolean = false): Envelope? {
|
||||||
Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
|
if (!Files.exists(path)) return null
|
||||||
.asOutput()
|
|
||||||
.writeMeta(this@write, descriptor)
|
//read two-files directory
|
||||||
|
if (Files.isDirectory(path)) {
|
||||||
|
val metaFile = Files.list(path).asSequence()
|
||||||
|
.singleOrNull { it.fileName.toString().startsWith("meta") }
|
||||||
|
|
||||||
|
val meta = if (metaFile == null) {
|
||||||
|
EmptyMeta
|
||||||
|
} else {
|
||||||
|
readMetaFile(metaFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
val dataFile = path.resolve("data")
|
||||||
|
|
||||||
|
val data: Binary? = if (Files.exists(dataFile)) {
|
||||||
|
dataFile.asBinary()
|
||||||
|
} else {
|
||||||
|
null
|
||||||
|
}
|
||||||
|
|
||||||
|
return SimpleEnvelope(meta, data)
|
||||||
|
}
|
||||||
|
|
||||||
|
val binary = path.asBinary()
|
||||||
|
|
||||||
|
val formats = envelopeFormatFactories.mapNotNull { factory ->
|
||||||
|
binary.read {
|
||||||
|
factory.peekFormat(this@readEnvelopeFromFile, this@read)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return when (formats.size) {
|
||||||
|
0 -> if (readNonEnvelopes) {
|
||||||
|
SimpleEnvelope(Meta.empty, binary)
|
||||||
|
} else {
|
||||||
|
null
|
||||||
|
}
|
||||||
|
1 -> formats.first().run {
|
||||||
|
binary.read {
|
||||||
|
readObject()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else -> error("Envelope format file recognition clash")
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -39,7 +39,7 @@ class FileBinaryTest {
|
|||||||
}
|
}
|
||||||
val binary = envelopeFromFile.data!!
|
val binary = envelopeFromFile.data!!
|
||||||
println(binary.toBytes().size)
|
println(binary.toBytes().size)
|
||||||
assertEquals(binary.size.toInt(), binary.toBytes().size)
|
assertEquals(binary.size?.toInt(), binary.toBytes().size)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -50,7 +50,7 @@ class FileBinaryTest {
|
|||||||
Global.io.writeEnvelopeFile(tmpPath, envelope)
|
Global.io.writeEnvelopeFile(tmpPath, envelope)
|
||||||
|
|
||||||
val binary = Global.io.readEnvelopeFile(tmpPath).data!!
|
val binary = Global.io.readEnvelopeFile(tmpPath).data!!
|
||||||
assertEquals(binary.size.toInt(), binary.toBytes().size)
|
assertEquals(binary.size?.toInt(), binary.toBytes().size)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
@ -12,7 +12,7 @@ import hep.dataforge.names.toName
|
|||||||
import kotlin.jvm.JvmName
|
import kotlin.jvm.JvmName
|
||||||
import kotlin.reflect.KClass
|
import kotlin.reflect.KClass
|
||||||
|
|
||||||
@TaskBuildScope
|
@DFBuilder
|
||||||
interface WorkspaceBuilder {
|
interface WorkspaceBuilder {
|
||||||
val parentContext: Context
|
val parentContext: Context
|
||||||
var context: Context
|
var context: Context
|
||||||
|
@ -4,36 +4,16 @@ import hep.dataforge.data.Data
|
|||||||
import hep.dataforge.data.DataNode
|
import hep.dataforge.data.DataNode
|
||||||
import hep.dataforge.data.DataTreeBuilder
|
import hep.dataforge.data.DataTreeBuilder
|
||||||
import hep.dataforge.data.datum
|
import hep.dataforge.data.datum
|
||||||
import hep.dataforge.descriptors.NodeDescriptor
|
|
||||||
import hep.dataforge.io.*
|
import hep.dataforge.io.*
|
||||||
import hep.dataforge.meta.EmptyMeta
|
import hep.dataforge.meta.EmptyMeta
|
||||||
import hep.dataforge.meta.Meta
|
|
||||||
import kotlinx.coroutines.Dispatchers
|
import kotlinx.coroutines.Dispatchers
|
||||||
import kotlinx.coroutines.withContext
|
import kotlinx.coroutines.withContext
|
||||||
import kotlinx.io.nio.asInput
|
import kotlinx.io.nio.asInput
|
||||||
import kotlinx.io.nio.asOutput
|
|
||||||
import java.nio.file.Files
|
import java.nio.file.Files
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
import java.nio.file.StandardOpenOption
|
import java.nio.file.StandardOpenOption
|
||||||
import kotlin.reflect.KClass
|
import kotlin.reflect.KClass
|
||||||
|
|
||||||
/**
|
|
||||||
* Read meta from file in a given [MetaFormat]
|
|
||||||
*/
|
|
||||||
fun MetaFormat.readMetaFile(path: Path, descriptor: NodeDescriptor? = null): Meta {
|
|
||||||
return Files.newByteChannel(path, StandardOpenOption.READ)
|
|
||||||
.asInput()
|
|
||||||
.readMeta(descriptor)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Write meta to file using given [MetaFormat]
|
|
||||||
*/
|
|
||||||
fun MetaFormat.writeMetaFile(path: Path, meta: Meta, descriptor: NodeDescriptor? = null) {
|
|
||||||
return Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
|
|
||||||
.asOutput()
|
|
||||||
.writeMeta(meta, descriptor)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file.
|
* Read data with supported envelope format and binary format. If envelope format is null, then read binary directly from file.
|
||||||
@ -50,10 +30,10 @@ fun <T : Any> IOPlugin.readData(
|
|||||||
dataFormat: IOFormat<T>,
|
dataFormat: IOFormat<T>,
|
||||||
envelopeFormatFactory: EnvelopeFormatFactory? = null,
|
envelopeFormatFactory: EnvelopeFormatFactory? = null,
|
||||||
metaFile: Path = path.resolveSibling("${path.fileName}.meta"),
|
metaFile: Path = path.resolveSibling("${path.fileName}.meta"),
|
||||||
metaFileFormat: MetaFormat = JsonMetaFormat.default
|
metaFileFormat: MetaFormat = JsonMetaFormat
|
||||||
): Data<T> {
|
): Data<T> {
|
||||||
val externalMeta = if (Files.exists(metaFile)) {
|
val externalMeta = if (Files.exists(metaFile)) {
|
||||||
metaFileFormat.readMetaFile(metaFile)
|
readMetaFile(metaFile)
|
||||||
} else {
|
} else {
|
||||||
null
|
null
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user