kscience/dataforge-core
8
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 12 Wiki Activity

Fix build with new IO

Browse Source
This commit is contained in:
Alexander Nozik 2019-12-01 20:53:32 +03:00
parent ab47d7723e
commit c789dabdae
13 changed files with 86 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt
View File

@ -5,8 +5,10 @@ import hep.dataforge.io.*
import hep.dataforge.meta.DFExperimental
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
import kotlinx.io.Input
import kotlinx.io.Output
import kotlinx.io.*
import kotlinx.io.text.readUtf8Line
import kotlinx.io.text.writeRawString
import kotlinx.io.text.writeUtf8String
import kotlinx.serialization.toUtf8Bytes
@DFExperimental
@ -27,14 +29,17 @@ class FrontMatterEnvelopeFormat(
metaTypeRegex.matchEntire(line)?.groupValues?.first()
?.let { io.metaFormat(it) } ?: YamlMetaFormat
val metaBlock = buildPacket {
val meta = buildBytes {
do {
line = readUtf8Line() ?: error("Input does not contain closing front matter separator")
appendln(line)
line = readUtf8Line()
writeUtf8String(line + "\r\n")
offset += line.toUtf8Bytes().size.toUInt()
} while (!line.startsWith(SEPARATOR))
}.read {
readMetaFormat.run {
readMeta()
}
}
val meta = readMetaFormat.fromBytes(metaBlock)
return PartialEnvelope(meta, offset, null)
}
@ -48,23 +53,29 @@ class FrontMatterEnvelopeFormat(
metaTypeRegex.matchEntire(line)?.groupValues?.first()
?.let { io.metaFormat(it) } ?: YamlMetaFormat
val metaBlock = buildPacket {
val meta = buildBytes {
do {
appendln(readUtf8Line() ?: error("Input does not contain closing front matter separator"))
writeUtf8String(readUtf8Line() + "\r\n")
} while (!line.startsWith(SEPARATOR))
}.read {
readMetaFormat.run {
readMeta()
}
val meta = readMetaFormat.fromBytes(metaBlock)
val bytes = readBytes()
}
val bytes = readRemaining()
val data = bytes.asBinary()
return SimpleEnvelope(meta, data)
}
override fun Output.writeEnvelope(envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta) {
val metaFormat = metaFormatFactory(formatMeta, io.context)
writeText("$SEPARATOR\r\n")
writeRawString("$SEPARATOR\r\n")
metaFormat.run { writeObject(envelope.meta) }
writeText("$SEPARATOR\r\n")
envelope.data?.read { copyTo(this@writeEnvelope) }
writeRawString("$SEPARATOR\r\n")
//Printing data
envelope.data?.let { data ->
writeBinary(data)
}
}
companion object : EnvelopeFormatFactory {
@ -77,7 +88,7 @@ class FrontMatterEnvelopeFormat(
}
override fun peekFormat(io: IOPlugin, input: Input): EnvelopeFormat? {
val line = input.readUtf8Line(3, 30)
val line = input.readUtf8Line()
return if (line != null && line.startsWith("---")) {
invoke()
} else {

6
dataforge-io/dataforge-io-yaml/src/main/kotlin/hep/dataforge/io/yaml/YamlMetaFormat.kt
View File

@ -11,13 +11,13 @@ import hep.dataforge.meta.toMeta
import kotlinx.io.Input
import kotlinx.io.Output
import kotlinx.io.readUByte
import kotlinx.io.writeText
import kotlinx.io.text.writeUtf8String
import org.yaml.snakeyaml.Yaml
import java.io.InputStream
private class InputAsStream(val input: Input) : InputStream() {
override fun read(): Int {
if (input.endOfInput) return -1
if (input.eof()) return -1
return input.readUByte().toInt()
}
@ -34,7 +34,7 @@ class YamlMetaFormat(val meta: Meta) : MetaFormat {
override fun Output.writeMeta(meta: Meta, descriptor: NodeDescriptor?) {
val string = yaml.dump(meta.toMap(descriptor))
writeText(string)
writeUtf8String(string)
}
override fun Input.readMeta(descriptor: NodeDescriptor?): Meta {

2
dataforge-io/dataforge-io-yaml/src/test/kotlin/hep/dataforge/io/yaml/YamlMetaFormatTest.kt
View File

@ -6,7 +6,7 @@ import hep.dataforge.meta.Meta
import hep.dataforge.meta.buildMeta
import hep.dataforge.meta.get
import hep.dataforge.meta.seal
import org.junit.jupiter.api.Test
import kotlin.test.Test
import kotlin.test.assertEquals

9
dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlOutput.kt → dataforge-output-html/src/commonMain/kotlin/hep/dataforge/output/html/HtmlRenderer.kt
View File

@ -3,7 +3,8 @@ package hep.dataforge.output.html
import hep.dataforge.context.Context
import hep.dataforge.meta.Meta
import hep.dataforge.output.Output
import hep.dataforge.output.TextRenderer
import hep.dataforge.output.Renderer
import hep.dataforge.output.TextFormat
import hep.dataforge.output.html.HtmlBuilder.Companion.HTML_CONVERTER_TYPE
import hep.dataforge.provider.Type
import hep.dataforge.provider.top
@ -14,11 +15,11 @@ import kotlinx.html.p
import kotlin.reflect.KClass
class HtmlOutput<T : Any>(override val context: Context, private val consumer: TagConsumer<*>) : Output<T> {
class HtmlRenderer<T : Any>(override val context: Context, private val consumer: TagConsumer<*>) : Renderer<T> {
private val cache = HashMap<KClass<*>, HtmlBuilder<*>>()
/**
* Find the first [TextRenderer] matching the given object type.
* Find the first [TextFormat] matching the given object type.
*/
override fun render(obj: T, meta: Meta) {
@ -47,7 +48,7 @@ class HtmlOutput<T : Any>(override val context: Context, private val consumer: T
}
/**
* A text or binary renderer based on [Output]
* A text or binary renderer based on [Renderer]
*/
@Type(HTML_CONVERTER_TYPE)
interface HtmlBuilder<T : Any> {

24
dataforge-output/src/commonMain/kotlin/hep/dataforge/output/OutputManager.kt
View File

@ -1,9 +1,6 @@
package hep.dataforge.output
import hep.dataforge.context.AbstractPlugin
import hep.dataforge.context.Context
import hep.dataforge.context.PluginFactory
import hep.dataforge.context.PluginTag
import hep.dataforge.context.*
import hep.dataforge.context.PluginTag.Companion.DATAFORGE_GROUP
import hep.dataforge.meta.EmptyMeta
import hep.dataforge.meta.Meta
@ -22,14 +19,14 @@ interface OutputManager {
* Get an output specialized for given type, name and stage.
* @param stage represents the node or directory for the output. Empty means root node.
* @param name represents the name inside the node.
* @param meta configuration for [Output] (not for rendered object)
* @param meta configuration for [Renderer] (not for rendered object)
*/
operator fun <T : Any> get(
type: KClass<out T>,
name: Name,
stage: Name = EmptyName,
meta: Meta = EmptyMeta
): Output<T>
): Renderer<T>
}
/**
@ -44,7 +41,7 @@ inline operator fun <reified T : Any> OutputManager.get(
name: Name,
stage: Name = EmptyName,
meta: Meta = EmptyMeta
): Output<T> {
): Renderer<T> {
return get(T::class, name, stage, meta)
}
@ -56,14 +53,21 @@ fun OutputManager.render(obj: Any, name: Name, stage: Name = EmptyName, meta: Me
/**
* System console output.
* The [ConsoleOutput] is used when no other [OutputManager] is provided.
* The [CONSOLE_RENDERER] is used when no other [OutputManager] is provided.
*/
expect val ConsoleOutput: Output<Any>
val CONSOLE_RENDERER: Renderer<Any> = object : Renderer<Any> {
override fun render(obj: Any, meta: Meta) {
println(obj)
}
override val context: Context get() = Global
}
class ConsoleOutputManager : AbstractPlugin(), OutputManager {
override val tag: PluginTag get() = ConsoleOutputManager.tag
override fun <T : Any> get(type: KClass<out T>, name: Name, stage: Name, meta: Meta): Output<T> = ConsoleOutput
override fun <T : Any> get(type: KClass<out T>, name: Name, stage: Name, meta: Meta): Renderer<T> = CONSOLE_RENDERER
companion object : PluginFactory<ConsoleOutputManager> {
override val tag = PluginTag("output.console", group = DATAFORGE_GROUP)

4
dataforge-output/src/commonMain/kotlin/hep/dataforge/output/Output.kt → dataforge-output/src/commonMain/kotlin/hep/dataforge/output/Renderer.kt
View File

@ -7,11 +7,11 @@ import hep.dataforge.meta.Meta
/**
* A generic way to render any object in the output.
*
* An object could be rendered either in append or overlay mode. The mode is decided by the [Output]
* An object could be rendered either in append or overlay mode. The mode is decided by the [Renderer]
* based on its configuration and provided meta
*
*/
interface Output<in T : Any> : ContextAware {
interface Renderer<in T : Any> : ContextAware {
/**
* Render specific object with configuration.
*

33
dataforge-output/src/commonMain/kotlin/hep/dataforge/output/TextOutput.kt → dataforge-output/src/commonMain/kotlin/hep/dataforge/output/TextRenderer.kt
View File

@ -2,48 +2,50 @@ package hep.dataforge.output
import hep.dataforge.context.Context
import hep.dataforge.meta.Meta
import hep.dataforge.output.TextRenderer.Companion.TEXT_RENDERER_TYPE
import hep.dataforge.output.TextFormat.Companion.TEXT_RENDERER_TYPE
import hep.dataforge.provider.Type
import hep.dataforge.provider.top
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.io.Output
import kotlinx.io.text.writeUtf8String
import kotlin.reflect.KClass
class TextOutput(override val context: Context, private val output: kotlinx.io.Output) : Output<Any> {
private val cache = HashMap<KClass<*>, TextRenderer>()
class TextRenderer(override val context: Context, private val output: Output) : Renderer<Any> {
private val cache = HashMap<KClass<*>, TextFormat>()
/**
* Find the first [TextRenderer] matching the given object type.
* Find the first [TextFormat] matching the given object type.
*/
override fun render(obj: Any, meta: Meta) {
val renderer: TextRenderer = if (obj is CharSequence) {
DefaultTextRenderer
val format: TextFormat = if (obj is CharSequence) {
DefaultTextFormat
} else {
val value = cache[obj::class]
if (value == null) {
val answer =
context.top<TextRenderer>(TEXT_RENDERER_TYPE).values.firstOrNull { it.type.isInstance(obj) }
context.top<TextFormat>(TEXT_RENDERER_TYPE).values.firstOrNull { it.type.isInstance(obj) }
if (answer != null) {
cache[obj::class] = answer
answer
} else {
DefaultTextRenderer
DefaultTextFormat
}
} else {
value
}
}
context.launch(Dispatchers.Output) {
renderer.run { output.render(obj) }
format.run { output.render(obj) }
}
}
}
/**
* A text or binary renderer based on [kotlinx.io.Output]
* A text or binary renderer based on [Output]
*/
@Type(TEXT_RENDERER_TYPE)
interface TextRenderer {
interface TextFormat {
/**
* The priority of this renderer compared to other renderers
*/
@ -53,19 +55,18 @@ interface TextRenderer {
*/
val type: KClass<*>
suspend fun kotlinx.io.Output.render(obj: Any)
suspend fun Output.render(obj: Any)
companion object {
const val TEXT_RENDERER_TYPE = "dataforge.textRenderer"
}
}
object DefaultTextRenderer : TextRenderer {
object DefaultTextFormat : TextFormat {
override val priority: Int = Int.MAX_VALUE
override val type: KClass<*> = Any::class
override suspend fun kotlinx.io.Output.render(obj: Any) {
append(obj.toString())
append('\n')
override suspend fun Output.render(obj: Any) {
writeUtf8String(obj.toString() + "\n")
}
}

22
dataforge-output/src/jsMain/kotlin/hep/dataforge/output/ConsoleOutput.kt
View File

@ -1,22 +0,0 @@
package hep.dataforge.output
import hep.dataforge.context.Context
import hep.dataforge.context.Global
import hep.dataforge.meta.Meta
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.Dispatchers
/**
* System console output.
* The [ConsoleOutput] is used when no other [OutputManager] is provided.
*/
actual val ConsoleOutput: Output<Any> = object : Output<Any> {
override fun render(obj: Any, meta: Meta) {
println(obj)
}
override val context: Context get() = Global
}
actual val Dispatchers.Output: CoroutineDispatcher get() = Dispatchers.Default

7
dataforge-output/src/jsMain/kotlin/hep/dataforge/output/outputJS.kt Normal file
View File

@ -0,0 +1,7 @@
package hep.dataforge.output
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.Dispatchers
actual val Dispatchers.Output: CoroutineDispatcher get() = Dispatchers.Default

13
dataforge-output/src/jvmMain/kotlin/hep/dataforge/output/ConsoleOutput.kt
View File

@ -1,13 +0,0 @@
package hep.dataforge.output
import hep.dataforge.context.Global
import kotlinx.coroutines.Dispatchers
import kotlinx.io.streams.asOutput
/**
* System console output.
* The [ConsoleOutput] is used when no other [OutputManager] is provided.
*/
actual val ConsoleOutput: Output<Any> = TextOutput(Global, System.out.asOutput())
actual val Dispatchers.Output get() = Dispatchers.IO

5
dataforge-output/src/jvmMain/kotlin/hep/dataforge/output/outputJVM.kt Normal file
View File

@ -0,0 +1,5 @@
package hep.dataforge.output
import kotlinx.coroutines.Dispatchers
actual val Dispatchers.Output get() = Dispatchers.IO

12
dataforge-workspace/src/commonMain/kotlin/hep/dataforge/workspace/envelopeData.kt
View File

@ -7,8 +7,7 @@ import hep.dataforge.io.IOFormat
import hep.dataforge.io.SimpleEnvelope
import hep.dataforge.io.readWith
import kotlinx.coroutines.coroutineScope
import kotlinx.io.Input
import kotlinx.io.buildPacket
import kotlinx.io.ArrayBinary
import kotlin.reflect.KClass
/**
@ -22,15 +21,8 @@ suspend fun <T : Any> Data<T>.toEnvelope(format: IOFormat<T>): Envelope {
val obj = coroutineScope {
await(this)
}
val binary = object : Binary {
override fun <R> read(block: Input.() -> R): R {
//TODO optimize away additional copy by creating inputs that reads directly from output
val packet = buildPacket {
val binary = ArrayBinary.write {
format.run { writeObject(obj) }
}
return packet.block()
}
}
return SimpleEnvelope(meta, binary)
}

8
dataforge-workspace/src/jvmTest/kotlin/hep/dataforge/workspace/FileDataTest.kt
View File

@ -8,8 +8,8 @@ import hep.dataforge.meta.DFExperimental
import kotlinx.coroutines.runBlocking
import kotlinx.io.Input
import kotlinx.io.Output
import kotlinx.io.readText
import kotlinx.io.writeText
import kotlinx.io.text.readUtf8String
import kotlinx.io.text.writeUtf8String
import java.nio.file.Files
import kotlin.test.Ignore
import kotlin.test.Test
@ -30,11 +30,11 @@ class FileDataTest {
object StringIOFormat : IOFormat<String> {
override fun Output.writeObject(obj: String) {
writeText(obj)
writeUtf8String(obj)
}
override fun Input.readObject(): String {
return readText()
return readUtf8String()
}
}