kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Merge branch 'dev' into dev-zelenyy

Browse Source
This commit is contained in:
Mikhail Zelenyy 2019-11-06 15:46:53 +03:00
commit a0727e6a9b
12 changed files with 171 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
build.gradle.kts
View File

@ -1,10 +1,12 @@
import scientifik.ScientifikExtension
plugins {
id("scientifik.mpp") version "0.2.1" apply false
id("scientifik.jvm") version "0.2.1" apply false
id("scientifik.publish") version "0.2.1" apply false
id("scientifik.mpp") version "0.2.2" apply false
id("scientifik.jvm") version "0.2.2" apply false
id("scientifik.publish") version "0.2.2" apply false
}
val dataforgeVersion by extra("0.1.4")
val dataforgeVersion by extra("0.1.5-dev-1")
val bintrayRepo by extra("dataforge")
val githubProject by extra("dataforge-core")
@ -15,7 +17,8 @@ allprojects {
}
subprojects {
if (name.startsWith("dataforge")) {
apply(plugin = "scientifik.publish")
}
apply(plugin = "scientifik.publish")
afterEvaluate {
extensions.findByType<ScientifikExtension>()?.apply { withDokka() }
}
}

2
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/ContextBuilder.kt
View File

@ -1,5 +1,6 @@
package hep.dataforge.context
import hep.dataforge.meta.DFBuilder
import hep.dataforge.meta.MetaBuilder
import hep.dataforge.meta.buildMeta
import hep.dataforge.names.toName
@ -7,6 +8,7 @@ import hep.dataforge.names.toName
/**
* A convenience builder for context
*/
@DFBuilder
class ContextBuilder(var name: String = "@anonymous", val parent: Context = Global) {
private val plugins = ArrayList<Plugin>()
private var meta = MetaBuilder()

6
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/MapAction.kt
View File

@ -1,9 +1,6 @@
package hep.dataforge.data
import hep.dataforge.meta.Meta
import hep.dataforge.meta.MetaBuilder
import hep.dataforge.meta.builder
import hep.dataforge.meta.seal
import hep.dataforge.meta.*
import hep.dataforge.names.Name
import kotlin.reflect.KClass
@ -20,6 +17,7 @@ data class ActionEnv(
/**
* Action environment
*/
@DFBuilder
class MapActionBuilder<T, R>(var name: Name, var meta: MetaBuilder, val actionMeta: Meta) {
lateinit var result: suspend ActionEnv.(T) -> R

37
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt
View File

@ -1,11 +1,11 @@
package hep.dataforge.io
import hep.dataforge.meta.*
import hep.dataforge.meta.Laminate
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.string
import hep.dataforge.names.asName
import hep.dataforge.names.plus
import kotlinx.io.core.Output
import kotlinx.io.core.buildPacket
import kotlinx.io.core.readBytes
interface Envelope {
val meta: Meta
@ -83,32 +83,3 @@ fun Envelope.withMetaLayers(vararg layers: Meta): Envelope {
}
}
class EnvelopeBuilder {
private val metaBuilder = MetaBuilder()
var data: Binary? = null
fun meta(block: MetaBuilder.() -> Unit) {
metaBuilder.apply(block)
}
fun meta(meta: Meta) {
metaBuilder.update(meta)
}
var type by metaBuilder.string(key = Envelope.ENVELOPE_TYPE_KEY)
var dataType by metaBuilder.string(key = Envelope.ENVELOPE_DATA_TYPE_KEY)
var dataID by metaBuilder.string(key = Envelope.ENVELOPE_DATA_ID_KEY)
var description by metaBuilder.string(key = Envelope.ENVELOPE_DESCRIPTION_KEY)
/**
* Construct a binary and transform it into byte-array based buffer
*/
fun data(block: Output.() -> Unit) {
val bytes = buildPacket {
block()
}
data = ArrayBinary(bytes.readBytes())
}
internal fun build() = SimpleEnvelope(metaBuilder.seal(), data)
}

36
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeBuilder.kt Normal file
View File

@ -0,0 +1,36 @@
package hep.dataforge.io
import hep.dataforge.meta.*
import kotlinx.io.core.Output
import kotlinx.io.core.buildPacket
import kotlinx.io.core.readBytes
class EnvelopeBuilder {
private val metaBuilder = MetaBuilder()
var data: Binary? = null
fun meta(block: MetaBuilder.() -> Unit) {
metaBuilder.apply(block)
}
fun meta(meta: Meta) {
metaBuilder.update(meta)
}
var type by metaBuilder.string(key = Envelope.ENVELOPE_TYPE_KEY)
var dataType by metaBuilder.string(key = Envelope.ENVELOPE_DATA_TYPE_KEY)
var dataID by metaBuilder.string(key = Envelope.ENVELOPE_DATA_ID_KEY)
var description by metaBuilder.string(key = Envelope.ENVELOPE_DESCRIPTION_KEY)
/**
* Construct a binary and transform it into byte-array based buffer
*/
fun data(block: Output.() -> Unit) {
val bytes = buildPacket {
block()
}
data = ArrayBinary(bytes.readBytes())
}
internal fun build() = SimpleEnvelope(metaBuilder.seal(), data)
}

60
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/EnvelopeParts.kt Normal file
View File

@ -0,0 +1,60 @@
package hep.dataforge.io
import hep.dataforge.context.Global
import hep.dataforge.io.EnvelopeParts.FORMAT_META_KEY
import hep.dataforge.io.EnvelopeParts.FORMAT_NAME_KEY
import hep.dataforge.io.EnvelopeParts.PARTS_DATA_TYPE
import hep.dataforge.io.EnvelopeParts.SIZE_KEY
import hep.dataforge.meta.*
import hep.dataforge.names.plus
import hep.dataforge.names.toName
object EnvelopeParts {
val SIZE_KEY = Envelope.ENVELOPE_NODE_KEY + "parts" + "size"
val FORMAT_NAME_KEY = Envelope.ENVELOPE_NODE_KEY + "parts" + "format"
val FORMAT_META_KEY = Envelope.ENVELOPE_NODE_KEY + "parts" + "meta"
const val PARTS_DATA_TYPE = "envelope.parts"
}
fun EnvelopeBuilder.parts(formatFactory: EnvelopeFormatFactory, envelopes: Collection<Envelope>) {
dataType = PARTS_DATA_TYPE
meta {
SIZE_KEY put envelopes.size
FORMAT_NAME_KEY put formatFactory.name.toString()
}
val format = formatFactory()
data {
format.run {
envelopes.forEach {
writeObject(it)
}
}
}
}
fun EnvelopeBuilder.parts(formatFactory: EnvelopeFormatFactory, builder: suspend SequenceScope<Envelope>.() -> Unit) =
parts(formatFactory, sequence(builder).toList())
fun Envelope.parts(io: IOPlugin = Global.plugins.fetch(IOPlugin)): Sequence<Envelope> {
return when (dataType) {
PARTS_DATA_TYPE -> {
val size = meta[SIZE_KEY].int ?: error("Unsized parts not supported yet")
val formatName = meta[FORMAT_NAME_KEY].string?.toName()
?: error("Inferring parts format is not supported at the moment")
val formatMeta = meta[FORMAT_META_KEY].node ?: EmptyMeta
val format = io.envelopeFormat(formatName, formatMeta)
?: error("Format $formatName is not resolved by $io")
return format.run {
data?.read {
sequence {
repeat(size) {
yield(readObject())
}
}
} ?: emptySequence()
}
}
else -> emptySequence()
}
}

3
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt
View File

@ -26,6 +26,9 @@ class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
context.content<EnvelopeFormatFactory>(ENVELOPE_FORMAT_TYPE).values
}
fun envelopeFormat(name: Name, meta: Meta = EmptyMeta) =
envelopeFormatFactories.find { it.name == name }?.invoke(meta, context)
override fun provideTop(target: String): Map<Name, Any> {
return when (target) {
META_FORMAT_TYPE -> defaultMetaFormats.toMap()

26
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/serialization/serializationUtils.kt
View File

@ -1,9 +1,7 @@
package hep.dataforge.io.serialization
import kotlinx.serialization.CompositeDecoder
import kotlinx.serialization.Decoder
import kotlinx.serialization.KSerializer
import kotlinx.serialization.SerialDescriptor
import hep.dataforge.meta.DFExperimental
import kotlinx.serialization.*
import kotlinx.serialization.internal.*
/**
@ -71,10 +69,24 @@ inline fun <reified T : Any> KSerializer<T>.descriptor(
): SerialDescriptor =
SerialDescriptorBuilder(SerialClassDescImpl(name)).apply(block).build()
fun Decoder.decodeStructure(
@DFExperimental
inline fun <R> Decoder.decodeStructure(
desc: SerialDescriptor,
vararg typeParams: KSerializer<*> = emptyArray(),
block: CompositeDecoder.() -> Unit
crossinline block: CompositeDecoder.() -> R
): R {
val decoder = beginStructure(desc, *typeParams)
val res = decoder.block()
decoder.endStructure(desc)
return res
}
inline fun Encoder.encodeStructure(
desc: SerialDescriptor,
vararg typeParams: KSerializer<*> = emptyArray(),
block: CompositeEncoder.() -> Unit
) {
beginStructure(desc, *typeParams).apply(block).endStructure(desc)
val encoder = beginStructure(desc, *typeParams)
encoder.block()
encoder.endStructure(desc)
}

32
dataforge-io/src/commonTest/kotlin/hep/dataforge/io/EnvelopePartsTest.kt Normal file
View File

@ -0,0 +1,32 @@
package hep.dataforge.io
import hep.dataforge.meta.get
import hep.dataforge.meta.int
import kotlinx.io.core.writeText
import kotlin.test.Test
import kotlin.test.assertEquals
class EnvelopePartsTest {
val envelopes = (0..5).map {
Envelope {
meta {
"value" put it
}
data {
writeText("Hello World $it")
}
}
}
val partsEnvelope = Envelope {
parts(TaggedEnvelopeFormat, envelopes)
}
@Test
fun testParts() {
val bytes = TaggedEnvelopeFormat.default.writeBytes(partsEnvelope)
val reconstructed = TaggedEnvelopeFormat.default.readBytes(bytes)
val parts = reconstructed.parts().toList()
assertEquals(2, parts[2].meta["value"].int)
}
}

2
dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt
View File

@ -47,7 +47,7 @@ class HtmlOutput<T : Any>(override val context: Context, private val consumer: T
}
/**
* A text or binary renderer based on [kotlinx.io.core.Output]
* A text or binary renderer based on [Output]
*/
@Type(HTML_CONVERTER_TYPE)
interface HtmlBuilder<T : Any> {

3
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskBuilder.kt
View File

@ -3,6 +3,7 @@ package hep.dataforge.workspace
import hep.dataforge.context.Context
import hep.dataforge.data.*
import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.meta.DFBuilder
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.string
@ -13,7 +14,7 @@ import hep.dataforge.names.toName
import kotlin.jvm.JvmName
import kotlin.reflect.KClass
@TaskBuildScope
@DFBuilder
class TaskBuilder<R : Any>(val name: Name, val type: KClass<out R>) {
private var modelTransform: TaskModelBuilder.(Meta) -> Unit = { allData() }
// private val additionalDependencies = HashSet<Dependency>()

3
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/TaskModel.kt
View File

@ -58,9 +58,6 @@ fun TaskModel.buildInput(workspace: Workspace): DataTree<Any> {
}.build()
}
@DslMarker
annotation class TaskBuildScope
interface TaskDependencyContainer {
val defaultMeta: Meta
fun add(dependency: Dependency)