kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

A lot of minor fixes

Browse Source
This commit is contained in:
Alexander Nozik 2019-08-07 11:38:25 +03:00
parent a729d27d1c
commit 44737faa26
19 changed files with 288 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dataforge-context/build.gradle.kts
View File

@ -12,20 +12,20 @@ kotlin {
dependencies {
api(project(":dataforge-meta"))
api(kotlin("reflect"))
api("io.github.microutils:kotlin-logging-common:1.6.10")
api("io.github.microutils:kotlin-logging-common:1.7.2")
api("org.jetbrains.kotlinx:kotlinx-coroutines-core-common:$coroutinesVersion")
}
}
val jvmMain by getting {
dependencies {
api("io.github.microutils:kotlin-logging:1.6.10")
api("io.github.microutils:kotlin-logging:1.7.2")
api("ch.qos.logback:logback-classic:1.2.3")
api("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
}
}
val jsMain by getting {
dependencies {
api("io.github.microutils:kotlin-logging-js:1.6.10")
api("io.github.microutils:kotlin-logging-js:1.7.2")
api("org.jetbrains.kotlinx:kotlinx-coroutines-core-js:$coroutinesVersion")
}
}

5
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/AbstractPlugin.kt
View File

@ -3,6 +3,7 @@ package hep.dataforge.context
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import hep.dataforge.names.Name
import hep.dataforge.names.toName
abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin {
private var _context: Context? = null
@ -19,4 +20,8 @@ abstract class AbstractPlugin(override val meta: Meta = EmptyMeta) : Plugin {
}
override fun provideTop(target: String): Map<Name, Any> = emptyMap()
companion object{
fun <T: Named> Collection<T>.toMap(): Map<Name, T> = associate { it.name.toName() to it }
}
}

12
dataforge-context/src/commonMain/kotlin/hep/dataforge/context/Context.kt
View File

@ -26,8 +26,10 @@ import kotlin.jvm.JvmName
* Since plugins could contain mutable state, context has two states: active and inactive. No changes are allowed to active context.
* @author Alexander Nozik
*/
open class Context(final override val name: String, val parent: Context? = Global) : Named, MetaRepr, Provider,
CoroutineScope {
open class Context(
final override val name: String,
val parent: Context? = Global
) : Named, MetaRepr, Provider, CoroutineScope {
private val config = Config()
@ -60,10 +62,10 @@ open class Context(final override val name: String, val parent: Context? = Globa
override val defaultTarget: String get() = Plugin.PLUGIN_TARGET
override fun provideTop(target: String): Map<Name, Any> {
return when(target){
return when (target) {
Value.TYPE -> properties.sequence().toMap()
Plugin.PLUGIN_TARGET -> plugins.sequence(true).associateBy { it.name.toName() }
else-> emptyMap()
else -> emptyMap()
}
}
@ -111,7 +113,7 @@ open class Context(final override val name: String, val parent: Context? = Globa
fun Context.content(target: String): Map<Name, Any> = content<Any>(target)
/**
* A sequences of all objects provided by plugins with given target and type
* A map of all objects provided by plugins with given target and type
*/
@JvmName("typedContent")
inline fun <reified T : Any> Context.content(target: String): Map<Name, T> =

10
dataforge-data/src/commonMain/kotlin/hep/dataforge/data/Goal.kt
View File

@ -77,8 +77,8 @@ private class StaticGoalImpl<T>(override val scope: CoroutineScope, deferred: Co
*
* **Important:** Unlike regular deferred, the [Goal] is started lazily, so the actual calculation is called only when result is requested.
*/
fun <R> CoroutineScope.createGoal(
dependencies: Collection<Goal<*>>,
fun <R> CoroutineScope.goal(
dependencies: Collection<Goal<*>> = emptyList(),
context: CoroutineContext = EmptyCoroutineContext,
block: suspend CoroutineScope.() -> R
): Goal<R> {
@ -102,7 +102,7 @@ fun <R> CoroutineScope.createGoal(
fun <T, R> Goal<T>.pipe(
context: CoroutineContext = EmptyCoroutineContext,
block: suspend CoroutineScope.(T) -> R
): Goal<R> = createGoal(listOf(this), context) { block(await()) }
): Goal<R> = goal(listOf(this), context) { block(await()) }
/**
* Create a joining goal.
@ -112,7 +112,7 @@ fun <T, R> Collection<Goal<T>>.join(
scope: CoroutineScope = first(),
context: CoroutineContext = EmptyCoroutineContext,
block: suspend CoroutineScope.(Collection<T>) -> R
): Goal<R> = scope.createGoal(this, context) {
): Goal<R> = scope.goal(this, context) {
block(map { it.await() })
}
@ -126,6 +126,6 @@ fun <K, T, R> Map<K, Goal<T>>.join(
scope: CoroutineScope = values.first(),
context: CoroutineContext = EmptyCoroutineContext,
block: suspend CoroutineScope.(Map<K, T>) -> R
): Goal<R> = scope.createGoal(this.values, context) {
): Goal<R> = scope.goal(this.values, context) {
block(mapValues { it.value.await() })
}

0
dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/_Data.kt → dataforge-data/src/jvmMain/kotlin/hep/dataforge/data/data.kt
View File

2
dataforge-io/build.gradle.kts
View File

@ -14,7 +14,7 @@ kotlin {
sourceSets {
val commonMain by getting{
dependencies {
api(project(":dataforge-meta"))
api(project(":dataforge-context"))
}
}
val jsMain by getting{

18
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/Envelope.kt
View File

@ -1,9 +1,12 @@
package hep.dataforge.io
import hep.dataforge.io.EnvelopeFormat.Companion.ENVELOPE_FORMAT_TYPE
import hep.dataforge.meta.Meta
import hep.dataforge.meta.get
import hep.dataforge.meta.string
import hep.dataforge.provider.Type
import kotlinx.io.core.Input
import kotlinx.io.core.Output
interface Envelope {
val meta: Meta
@ -46,4 +49,17 @@ val Envelope.dataType: String? get() = meta[Envelope.ENVELOPE_DATA_TYPE_KEY].str
*/
val Envelope.description: String? get() = meta[Envelope.ENVELOPE_DESCRIPTION_KEY].string
typealias EnvelopeFormat = IOFormat<Envelope>
data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
@Type(ENVELOPE_FORMAT_TYPE)
interface EnvelopeFormat : IOFormat<Envelope>{
fun readPartial(input: Input): PartialEnvelope
fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat)
override fun Output.writeObject(obj: Envelope) = writeEnvelope(obj, JsonMetaFormat)
companion object {
const val ENVELOPE_FORMAT_TYPE = "envelopeFormat"
}
}

39
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/IOPlugin.kt Normal file
View File

@ -0,0 +1,39 @@
package hep.dataforge.io
import hep.dataforge.context.AbstractPlugin
import hep.dataforge.context.PluginFactory
import hep.dataforge.context.PluginTag
import hep.dataforge.context.content
import hep.dataforge.meta.Meta
import hep.dataforge.names.Name
import hep.dataforge.names.asName
import kotlin.reflect.KClass
class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
override val tag: PluginTag get() = Companion.tag
val metaFormats by lazy {
context.content<MetaFormat>(MetaFormat.META_FORMAT_TYPE).values
}
fun metaFormat(key: Short): MetaFormat? = metaFormats.find { it.key == key }
fun metaFormat(name: String): MetaFormat? = metaFormats.find { it.name == name }
override fun provideTop(target: String): Map<Name, Any> {
return when (target) {
MetaFormat.META_FORMAT_TYPE -> internalMetaFormats
EnvelopeFormat.ENVELOPE_FORMAT_TYPE -> mapOf(
TaggedEnvelopeFormat.VERSION.asName() to TaggedEnvelopeFormat(metaFormats)
)
else -> super.provideTop(target)
}
}
companion object : PluginFactory<IOPlugin> {
private val internalMetaFormats = listOf(JsonMetaFormat, BinaryMetaFormat).toMap()
override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP)
override val type: KClass<out IOPlugin> = IOPlugin::class
override fun invoke(meta: Meta): IOPlugin = IOPlugin(meta)
}
}

16
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaFormat.kt
View File

@ -1,14 +1,18 @@
package hep.dataforge.io
import hep.dataforge.context.Named
import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.io.MetaFormat.Companion.META_FORMAT_TYPE
import hep.dataforge.meta.Meta
import hep.dataforge.provider.Type
import kotlinx.io.core.*
/**
* A format for meta serialization
*/
interface MetaFormat : IOFormat<Meta> {
val name: String
@Type(META_FORMAT_TYPE)
interface MetaFormat : IOFormat<Meta>, Named {
override val name: String
val key: Short
override fun Output.writeObject(obj: Meta) {
@ -17,8 +21,12 @@ interface MetaFormat : IOFormat<Meta> {
override fun Input.readObject(): Meta = readMeta(null)
fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?)
fun Input.readMeta(descriptor: NodeDescriptor?): Meta
fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor? = null)
fun Input.readMeta(descriptor: NodeDescriptor? = null): Meta
companion object{
const val META_FORMAT_TYPE = "metaFormat"
}
}
fun Meta.toString(format: MetaFormat = JsonMetaFormat): String = buildPacket {

25
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/MetaSerializer.kt
View File

@ -3,15 +3,29 @@ package hep.dataforge.io
import hep.dataforge.meta.Config
import hep.dataforge.meta.Meta
import hep.dataforge.meta.toConfig
import kotlinx.serialization.Decoder
import kotlinx.serialization.Encoder
import kotlinx.serialization.KSerializer
import kotlinx.serialization.SerialDescriptor
import hep.dataforge.names.Name
import hep.dataforge.names.toName
import kotlinx.serialization.*
import kotlinx.serialization.internal.StringDescriptor
import kotlinx.serialization.json.JsonObjectSerializer
@Serializer(Name::class)
object NameSerializer : KSerializer<Name> {
override val descriptor: SerialDescriptor = StringDescriptor
override fun deserialize(decoder: Decoder): Name {
return decoder.decodeString().toName()
}
override fun serialize(encoder: Encoder, obj: Name) {
encoder.encodeString(obj.toString())
}
}
/**
* Serialized for meta
*/
@Serializer(Meta::class)
object MetaSerializer : KSerializer<Meta> {
override val descriptor: SerialDescriptor = JsonObjectSerializer.descriptor
@ -25,7 +39,8 @@ object MetaSerializer : KSerializer<Meta> {
}
}
object ConfigSerializer: KSerializer<Config>{
@Serializer(Config::class)
object ConfigSerializer : KSerializer<Config> {
override val descriptor: SerialDescriptor = JsonObjectSerializer.descriptor
override fun deserialize(decoder: Decoder): Config {

34
dataforge-io/src/commonMain/kotlin/hep/dataforge/io/TaggedEnvelopeFormat.kt
View File

@ -2,14 +2,10 @@ package hep.dataforge.io
import kotlinx.io.core.*
@ExperimentalUnsignedTypes
class TaggedEnvelopeFormat(
val metaFormats: Collection<MetaFormat>,
val outputMetaFormat: MetaFormat = metaFormats.first()
) : EnvelopeFormat {
class TaggedEnvelopeFormat(val metaFormats: Collection<MetaFormat>) : EnvelopeFormat {
override fun Output.writeObject(obj: Envelope) {
write(obj, this, outputMetaFormat)
override fun Output.writeEnvelope(envelope: Envelope, format: MetaFormat) {
write(this, envelope, format)
}
/**
@ -20,6 +16,7 @@ class TaggedEnvelopeFormat(
*/
override fun Input.readObject(): Envelope = read(this, metaFormats)
override fun readPartial(input: Input): PartialEnvelope = Companion.readPartial(input, metaFormats)
private data class Tag(
val metaFormatKey: Short,
@ -28,9 +25,10 @@ class TaggedEnvelopeFormat(
)
companion object {
private const val VERSION = "DF03"
const val VERSION = "DF03"
private const val START_SEQUENCE = "#~"
private const val END_SEQUENCE = "~#\r\n"
private const val TAG_SIZE = 26u
private fun Tag.toBytes(): ByteReadPacket = buildPacket(24) {
writeText(START_SEQUENCE)
@ -66,12 +64,24 @@ class TaggedEnvelopeFormat(
return SimpleEnvelope(meta, ArrayBinary(dataBytes))
}
fun write(obj: Envelope, out: Output, metaFormat: MetaFormat) {
val metaBytes = metaFormat.writeBytes(obj.meta)
val tag = Tag(metaFormat.key, metaBytes.size.toUInt(), obj.data?.size ?: 0.toULong())
fun readPartial(input: Input, metaFormats: Collection<MetaFormat>): PartialEnvelope {
val tag = input.readTag()
val metaFormat = metaFormats.find { it.key == tag.metaFormatKey }
?: error("Meta format with key ${tag.metaFormatKey} not found")
val metaPacket = ByteReadPacket(input.readBytes(tag.metaSize.toInt()))
val meta = metaFormat.run { metaPacket.readObject() }
return PartialEnvelope(meta, TAG_SIZE + tag.metaSize, tag.dataSize)
}
fun write(out: Output, envelope: Envelope, metaFormat: MetaFormat) {
val metaBytes = metaFormat.writeBytes(envelope.meta)
val tag = Tag(metaFormat.key, metaBytes.size.toUInt(), envelope.data?.size ?: 0.toULong())
out.writePacket(tag.toBytes())
out.writeFully(metaBytes)
obj.data?.read { copyTo(out) }
envelope.data?.read { copyTo(out) }
}
}
}

8
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileBinary.kt
View File

@ -7,15 +7,13 @@ import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardOpenOption
class FileBinary(val path: Path, private val offset: Int = 0) : RandomAccessBinary {
override val size: ULong
get() = (Files.size(path) - offset).toULong()
class FileBinary(val path: Path, private val offset: UInt = 0u, size: ULong? = null) : RandomAccessBinary {
override val size: ULong = size ?: (Files.size(path).toULong() - offset).toULong()
override fun <R> read(from: UInt, size: UInt, block: Input.() -> R): R {
FileChannel.open(path, StandardOpenOption.READ).use {
val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from.toLong() + offset), size.toLong())
val buffer = it.map(FileChannel.MapMode.READ_ONLY, (from + offset).toLong(), size.toLong())
return ByteReadPacket(buffer).block()
}
}

24
dataforge-io/src/jvmMain/kotlin/hep/dataforge/io/FileEnvelope.kt Normal file
View File

@ -0,0 +1,24 @@
package hep.dataforge.io
import hep.dataforge.meta.Meta
import kotlinx.io.nio.asInput
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardOpenOption
class FileEnvelope internal constructor(val path: Path, val format: EnvelopeFormat) : Envelope {
//TODO do not like this constructor. Hope to replace it later
private val partialEnvelope: PartialEnvelope
init {
val input = Files.newByteChannel(path, StandardOpenOption.READ).asInput()
partialEnvelope = format.readPartial(input)
}
override val meta: Meta get() = partialEnvelope.meta
override val data: Binary? = FileBinary(path, partialEnvelope.dataOffset, partialEnvelope.dataSize)
}
fun Path.readEnvelope(format: EnvelopeFormat) = FileEnvelope(this,format)

2
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/Laminate.kt
View File

@ -17,7 +17,7 @@ class Laminate(layers: List<Meta>) : Meta {
}
}
constructor(vararg layers: Meta) : this(layers.asList())
constructor(vararg layers: Meta?) : this(layers.filterNotNull())
override val items: Map<NameToken, MetaItem<Meta>>
get() = layers.map { it.items.keys }.flatten().associateWith { key ->

2
dataforge-meta/src/commonMain/kotlin/hep/dataforge/meta/MetaBuilder.kt
View File

@ -8,7 +8,7 @@ import hep.dataforge.values.Value
* DSL builder for meta. Is not intended to store mutable state
*/
class MetaBuilder : AbstractMutableMeta<MetaBuilder>() {
override fun wrapNode(meta: Meta): MetaBuilder = meta.builder()
override fun wrapNode(meta: Meta): MetaBuilder = if (meta is MetaBuilder) meta else meta.builder()
override fun empty(): MetaBuilder = MetaBuilder()
infix fun String.to(value: Any) {

51
dataforge-meta/src/commonMain/kotlin/hep/dataforge/names/Name.kt
View File

@ -51,32 +51,55 @@ data class NameToken(val body: String, val index: String = "") {
if (body.isEmpty()) error("Syntax error: Name token body is empty")
}
private fun String.escape() =
replace("\\", "\\\\")
.replace(".", "\\.")
.replace("[", "\\[")
.replace("]", "\\]")
override fun toString(): String = if (hasIndex()) {
"$body[$index]"
"${body.escape()}[$index]"
} else {
body
body.escape()
}
fun hasIndex() = index.isNotEmpty()
}
/**
* Convert a [String] to name parsing it and extracting name tokens and index syntax.
* This operation is rather heavy so it should be used with care in high performance code.
*/
fun String.toName(): Name {
if (isBlank()) return EmptyName
val tokens = sequence {
var bodyBuilder = StringBuilder()
var queryBuilder = StringBuilder()
var bracketCount: Int = 0
var escape: Boolean = false
fun queryOn() = bracketCount > 0
asSequence().forEach {
if (queryOn()) {
when (it) {
'[' -> bracketCount++
']' -> bracketCount--
for (it in this@toName) {
when {
escape -> {
if (queryOn()) {
queryBuilder.append(it)
} else {
bodyBuilder.append(it)
}
escape = false
}
if (queryOn()) queryBuilder.append(it)
} else {
when (it) {
it == '\\' -> {
escape = true
}
queryOn() -> {
when (it) {
'[' -> bracketCount++
']' -> bracketCount--
}
if (queryOn()) queryBuilder.append(it)
}
else -> when (it) {
'.' -> {
yield(NameToken(bodyBuilder.toString(), queryBuilder.toString()))
bodyBuilder = StringBuilder()
@ -96,6 +119,14 @@ fun String.toName(): Name {
return Name(tokens.toList())
}
/**
* Convert the [String] to a [Name] by simply wrapping it in a single name token without parsing.
* The input string could contain dots and braces, but they are just escaped, not parsed.
*/
fun String.asName(): Name {
return NameToken(this).asName()
}
operator fun NameToken.plus(other: Name): Name = Name(listOf(this) + other.tokens)
operator fun Name.plus(other: Name): Name = Name(this.tokens + other.tokens)

15
dataforge-meta/src/commonTest/kotlin/hep/dataforge/names/NameTest.kt
View File

@ -1,6 +1,9 @@
package hep.dataforge.names
import kotlin.test.*
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertTrue
class NameTest {
@Test
@ -25,4 +28,14 @@ class NameTest {
assertTrue { name1.endsWith(name3) }
assertFalse { name1.startsWith(name3) }
}
@Test
fun escapeTest(){
val escapedName = "token\\.one.token2".toName()
val unescapedName = "token\\.one.token2".asName()
assertEquals(2, escapedName.length)
assertEquals(1, unescapedName.length)
assertEquals(escapedName, escapedName.toString().toName())
}
}

1
dataforge-workspace/build.gradle.kts
View File

@ -10,6 +10,7 @@ kotlin {
dependencies {
api(project(":dataforge-context"))
api(project(":dataforge-data"))
api(project(":dataforge-output"))
}
}
}

72
dataforge-workspace/src/jvmMain/kotlin/hep/dataforge/workspace/fileData.kt Normal file
View File

@ -0,0 +1,72 @@
package hep.dataforge.workspace
import hep.dataforge.context.Context
import hep.dataforge.data.Data
import hep.dataforge.data.goal
import hep.dataforge.descriptors.NodeDescriptor
import hep.dataforge.io.IOFormat
import hep.dataforge.io.JsonMetaFormat
import hep.dataforge.io.MetaFormat
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.withContext
import kotlinx.io.nio.asInput
import kotlinx.io.nio.asOutput
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardOpenOption
import kotlin.reflect.KClass
/**
* Read meta from file in a given [format]
*/
suspend fun Path.readMeta(format: MetaFormat, descriptor: NodeDescriptor? = null): Meta {
return withContext(Dispatchers.IO) {
format.run {
Files.newByteChannel(this@readMeta, StandardOpenOption.READ)
.asInput()
.readMeta(descriptor)
}
}
}
/**
* Write meta to file in a given [format]
*/
suspend fun Meta.write(path: Path, format: MetaFormat, descriptor: NodeDescriptor? = null) {
withContext(Dispatchers.IO) {
format.run {
Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
.asOutput()
.writeMeta(this@write, descriptor)
}
}
}
suspend fun <T : Any> Context.readData(
type: KClass<out T>,
path: Path,
format: IOFormat<T>,
metaFile: Path = path.resolveSibling("${path.fileName}.meta"),
metaFileFormat: MetaFormat = JsonMetaFormat
): Data<T> {
return coroutineScope {
val externalMeta = if (Files.exists(metaFile)) {
metaFile.readMeta(metaFileFormat)
} else {
null
}
val goal = goal {
withContext(Dispatchers.IO) {
format.run {
Files.newByteChannel(path, StandardOpenOption.READ)
.asInput()
.readObject()
}
}
}
Data.of(type, goal, externalMeta ?: EmptyMeta)
}
}