Add pandoc-plugin and conversion to latex

Use common s3 api and '/'-separated keys to simulate directories

.dockerignore

@@ -0,0 +1,24 @@
+# .gitignore contents
+
+.gradle/
+build/
+.idea/
+/logs/
+
+!gradle/wrapper/gradle-wrapper.jar
+
+kotlin-js-store
+*.iml
+
+# addition to .gitignore
+
+README.md
+LICENSE
+gradlew.bat
+.dockerignore
+.gitignore
+Dockerfile
+
+tmp
+rundata
+credentials.json

.gitignore

@@ -2,5 +2,10 @@
 build/
 .idea/
 /logs/
+rundata/
 
 !gradle/wrapper/gradle-wrapper.jar
+
+kotlin-js-store
+*.iml
+*.json

Dockerfile

@@ -0,0 +1,25 @@
+FROM ubuntu:latest
+
+RUN apt-get update
+RUN apt-get install -y curl zip unzip
+
+ARG JAVA_VERSION="17.0.7-zulu"
+
+WORKDIR /app
+
+COPY ./snark-main/ci ./snark-main/ci
+
+RUN ./snark-main/ci/install_sdk.sh
+RUN ./snark-main/ci/install_java.sh "$JAVA_VERSION"
+
+COPY . .
+
+RUN ./requirements.sh
+
+RUN bash -c "source ~/.sdkman/bin/sdkman-init.sh && ./gradlew clean build"
+
+EXPOSE 8080
+
+RUN mkdir -p ~/.aws/ && ln -s /run/secrets/credentials.json ~/.aws/credentials.json
+
+CMD bash -c "source ~/.sdkman/bin/sdkman-init.sh && ./gradlew :snark-main:run_server"

docker-compose.yml

@@ -0,0 +1,19 @@
+version: '3'
+
+services:
+  snark:
+    build: .
+    ports:
+      - "8081:8080"
+    expose:
+      - 8081
+    volumes:
+      - storage:/app/rundata
+    secrets:
+      - credentials.json
+volumes:
+  storage:
+
+secrets:
+  credentials.json:
+    file: ./credentials.json

requirements.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -e
+
+apt-get update
+apt-get install -y sudo
+
+for dir in ./*/
+do
+    if [[ $dir == *'snark'* ]]
+    then
+        cd "$dir"
+
+        if [[ $(find -type d -name "ci") ]]
+        then
+
+            cd ci
+            if [[ -f "requirements.sh" ]]; then
+              echo "executing sub"
+              bash -c "./requirements.sh"
+            fi
+
+            cd ..
+        fi
+
+        cd ..
+    fi
+done
+

settings.gradle.kts

@@ -41,5 +41,9 @@ include(
     ":snark-gradle-plugin",
     ":snark-core",
     ":snark-html",
-    ":snark-ktor"
+    ":snark-ktor",
+    ":snark-storage-driver",
+    ":snark-document-builder",
+    ":snark-main",
+    ":snark-pandoc-plugin",
 )

snark-document-builder/build.gradle.kts

@@ -0,0 +1,21 @@
+plugins {
+    id("space.kscience.gradle.jvm")
+    `maven-publish`
+    id("kotlinx-serialization")
+}
+
+val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
+val jacksonVersion = "2.14.2"
+val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
+
+
+dependencies {
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
+    api("io.ktor:ktor-server-html-builder:$ktorVersion")
+
+    implementation(project(":snark-storage-driver"))
+    implementation(project(":snark-pandoc-plugin"))
+
+    implementation("com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion")
+    implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.3")
+}

snark-document-builder/ci/requirements.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+sudo apt-get install -y python3
+sudo apt-get install -y nodejs
+sudo apt-get install -y npm
+
+pushd ../src/main/nodejs
+npm install .
+popd

snark-document-builder/src/main/kotlin/Build.kt

@@ -0,0 +1,17 @@
+package documentBuilder
+
+public class GraphManager(public val graph: DependencyGraph) {
+    fun buildDocument(file: FileName) {
+        val list = graph.nodes[file]
+        if (list != null) {
+            for (element in list.dependencies) {
+                element.visit(this)
+            }
+        }
+    }
+
+    fun getAstRootDocument(file: FileName): MdAstRoot {
+        buildDocument(file)
+        return graph.nodes[file]!!.mdAst
+    }
+}

snark-document-builder/src/main/kotlin/DependencyGraph.kt

@@ -0,0 +1,71 @@
+package documentBuilder
+
+import kotlinx.coroutines.coroutineScope
+import kotlin.collections.MutableList
+
+
+public typealias FileName = String
+
+/**
+ * Node of dependency graph.
+ *
+ * One node represents one file and its dependencies
+ *
+ * @property mdAst - AST tree of current file.
+ * @property dependencies - list of tail end adjacent to this node (dependencies of current file to be resolved).
+ */
+public data class DependencyGraphNode(
+    val mdAst: MdAstRoot,
+    val dependencies: List<DependencyGraphEdge>
+)
+
+/**
+ * Interface of all dependency edges.
+ */
+public sealed interface DependencyGraphEdge {
+    public fun visit(graphManager: GraphManager)
+}
+
+/**
+ * Include dependency edge.
+ *
+ * @property parentNode - node inside AST tree, that is parent for dependent node.
+ * @property dependentNode - dependent node, i.e. node of part of document with include commands
+ * @property includeList - list of files to be included.
+ */
+public data class IncludeDependency(
+    val parentNode: MdAstParent,
+    val dependentNode: MdAstElement,
+    val includeList: List<FileName>
+) : DependencyGraphEdge {
+    override fun visit(graphManager: GraphManager) {
+        val parent = parentNode
+        val childs: MutableList<MdAstElement> = mutableListOf()
+        for (file in includeList) {
+            graphManager.buildDocument(file)
+            childs.addAll(graphManager.graph.nodes[file]!!.mdAst.children)
+        }
+        val elements: MutableList<MdAstElement> = parent.children.toMutableList()
+        val index = parent.children.indexOf(dependentNode)
+        elements.removeAt(index)
+        elements.addAll(index, childs)
+        parent.children = elements
+    }
+}
+
+// parent - List<MdAstElement> -------------------------------------
+//                |                                                  \
+//                 \                                                  \
+//                  |                                                  \
+// dependentNode - MdAstElement                                         \
+//                                                                       |
+// List<FileName> -> List<MdAstRoot> --> List<List<MdAstElement>> ===> List<MdAstElement>
+
+/**
+ * Whole dependency graph.
+ *
+ * @property nodes - map of nodes, where you can find DependencyGraphNode of file by its name.
+ */
+public data class DependencyGraph(
+    val nodes: Map<FileName, DependencyGraphNode>
+)

snark-document-builder/src/main/kotlin/DocumentBuilder.kt

@@ -0,0 +1,111 @@
+package documentBuilder
+
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import space.kscience.snark.pandoc.PandocCommandBuilder
+import space.kscience.snark.pandoc.PandocWrapper
+import space.kscience.snark.storage.*
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.io.path.*
+
+private val SNARK_HTML_RENDER = "snark-document-builder/src/main/nodejs/HtmlRenderer.js"
+private val SNARK_MD_RENDERER = "snark-document-builder/src/main/nodejs/MdRenderer.js"
+fun getHtml(ast_string: String): String
+{
+    return ProcessBuilder("node", SNARK_HTML_RENDER, ast_string)
+        .redirectOutput(ProcessBuilder.Redirect.PIPE)
+        .redirectError(ProcessBuilder.Redirect.INHERIT)
+        .start().inputStream.bufferedReader().readText()
+}
+
+fun getLatex(ast_string: String) : Path
+{
+    val outputMd = Files.createTempFile(Path.of("./data/"), "output", ".md")
+
+    val output = ProcessBuilder("node", SNARK_MD_RENDERER, ast_string)
+        .redirectOutput(ProcessBuilder.Redirect.PIPE)
+        .redirectError(ProcessBuilder.Redirect.INHERIT)
+        .start().inputStream.bufferedReader().readText()
+    outputMd.writeText(output)
+
+    val outputTex = Files.createTempFile(Path.of("./data/"), "output", ".tex")
+
+    val pandocWrapper = PandocWrapper()
+    pandocWrapper.use { p: PandocWrapper? ->
+        val command = PandocCommandBuilder(
+            listOf<Path>(outputMd),
+            outputTex
+        )
+        PandocWrapper.execute(command)
+    }
+
+    return outputTex
+}
+
+private val DEFAULT_DOCUMENT_ROOT = "main.md"
+
+public suspend fun buildDocument(root: Directory, path: Path): String {
+    val dependencyGraph = buildDependencyGraph(root, path)
+
+    val graphManage = GraphManager(dependencyGraph)
+
+    graphManage.buildDocument(path.toString())
+
+//    for ((key, value) in dependencyGraph.nodes) {
+//        println("Key ${key}")
+//        println("Value.mdAst ${value.mdAst}")
+//        println("Value.dependencies ${value.dependencies}")
+//    }
+
+    val root: MdAstRoot = dependencyGraph.nodes[path.toString()]!!.mdAst
+
+    return getHtml(jacksonObjectMapper().writeValueAsString(root))
+}
+
+public suspend fun buildLatex(root: Directory, path: Path) : Path {
+    val dependencyGraph = buildDependencyGraph(root, path)
+
+    val graphManage = GraphManager(dependencyGraph)
+
+    graphManage.buildDocument(path.toString())
+
+    val root: MdAstRoot = dependencyGraph.nodes[path.toString()]!!.mdAst
+
+    return getLatex(jacksonObjectMapper().writeValueAsString(root))
+}
+
+public suspend fun buildDependencyGraph(root: Directory, path: Path): DependencyGraph {
+    val nodes = HashMap<FileName, DependencyGraphNode>()
+
+    buildNodes(root, path, nodes)
+
+    return DependencyGraph(nodes)
+}
+
+private suspend fun buildNodes(root: Directory, path: Path, nodes: HashMap<FileName, DependencyGraphNode>) {
+    val pathString = path.toString()
+
+    assert(!nodes.containsKey(pathString))
+
+    val rootDcoument = (root / path).get(DEFAULT_DOCUMENT_ROOT)
+    nodes.put(pathString, buildDependencyGraphNode(rootDcoument.readAll(), path))
+
+    val dependencies = getDependencies(nodes.getValue(pathString))
+
+    for (dependency in dependencies) {
+        if (!nodes.containsKey(dependency))
+            buildNodes(root, Path(dependency), nodes)
+    }
+}
+
+public suspend fun getDependencies(node: DependencyGraphNode): Set<FileName> {
+    val dependencies = mutableListOf<FileName>()
+
+    for (dependency in node.dependencies) {
+        when (dependency) {
+            is IncludeDependency -> dependencies.addAll(dependency.includeList)
+        }
+    }
+
+    return dependencies.toSet()
+}

snark-document-builder/src/main/kotlin/MdAstElements.kt

@@ -0,0 +1,84 @@
+package documentBuilder
+
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.SerialName
+import com.fasterxml.jackson.annotation.JsonSubTypes
+import com.fasterxml.jackson.annotation.JsonTypeInfo
+
+@Serializable
+public data class Point(val line: Int, val column: Int, val offset: Int)
+
+@Serializable
+public data class Position(val start: Point, val end: Point)
+
+@JsonTypeInfo(
+    use = JsonTypeInfo.Id.NAME,
+    include = JsonTypeInfo.As.PROPERTY,
+    property = "type"
+)
+
+@JsonSubTypes(
+    JsonSubTypes.Type(value = MdAstRoot::class, name = "root"),
+    JsonSubTypes.Type(value = MdAstParagraph::class, name = "paragraph"),
+    JsonSubTypes.Type(value = MdAstText::class, name = "text"),
+    JsonSubTypes.Type(value = MdAstHeading::class, name = "heading"),
+    JsonSubTypes.Type(value = MdAstCode::class, name = "code"),
+    JsonSubTypes.Type(value = MdAstBlockquote::class, name = "blockquote")
+)
+
+@Serializable
+public sealed interface MdAstElement{
+    public abstract var position: Position
+}
+
+@Serializable
+public sealed interface MdAstParent: MdAstElement{
+    public var children: List<MdAstElement>
+}
+
+@Serializable
+@SerialName("root")
+public data class MdAstRoot(
+    override var children: List<MdAstElement>,
+    override var position: Position
+): MdAstParent
+
+
+@Serializable
+@SerialName("paragraph")
+public data class MdAstParagraph(
+    override var children: List<MdAstElement>,
+    override var position: Position
+): MdAstParent
+
+
+@Serializable
+@SerialName("text")
+public data class MdAstText(
+    val value: String,
+    override var position: Position
+): MdAstElement
+
+@Serializable
+@SerialName("heading")
+public data class MdAstHeading(
+    val depth: Int, 
+    override var children: List<MdAstElement>,
+    override var position: Position
+): MdAstParent
+
+@Serializable
+@SerialName("code")
+public data class MdAstCode(
+    var lang: String? = null,
+    var meta: String? = null,
+    var value: String,
+    override var position: Position,
+) : MdAstElement
+
+@Serializable
+@SerialName("blockquote")
+public data class MdAstBlockquote(
+    override var children: List<MdAstElement>,
+    override var position: Position
+): MdAstParent

snark-document-builder/src/main/kotlin/MdParser.kt

@@ -0,0 +1,60 @@
+package documentBuilder
+
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import com.fasterxml.jackson.module.kotlin.readValue
+import java.nio.file.Path
+
+// snark-main/build.gradle.kts depends on these pathes
+private val MARKDOWN_PARSER = "snark-document-builder/src/main/nodejs/MarkdownParser.js"
+private val SNARK_PARSER = "snark-document-builder/src/main/python/SnarkParser.py"
+
+public suspend fun parseMd(mdFile: ByteArray, parserPath: String = MARKDOWN_PARSER): MdAstRoot {
+    val process = ProcessBuilder("node", parserPath, String(mdFile))
+    val result = process
+        .redirectOutput(ProcessBuilder.Redirect.PIPE)
+        .redirectError(ProcessBuilder.Redirect.INHERIT)
+        .start().inputStream.bufferedReader().readText()
+
+    return jacksonObjectMapper().readValue<MdAstRoot>(result)
+}
+
+public suspend fun buildDependencyGraphNode(mdFile: ByteArray, path: Path): DependencyGraphNode {
+    val treeRoot = parseMd(mdFile)
+    val dependencies = mutableListOf<DependencyGraphEdge>()
+
+    fillDependencies(treeRoot, dependencies, path)
+
+    return DependencyGraphNode(treeRoot, dependencies)
+}
+
+internal suspend fun fillDependencies(
+        currentNode: MdAstElement,
+        dependencies: MutableList<DependencyGraphEdge>,
+        path: Path) {
+    when (currentNode) {
+        is MdAstParent -> {
+            for (child in currentNode.children) {
+                if (child is MdAstText) {
+                    val includeList = getIncludeFiles(child.value).toMutableList()
+
+                    if (includeList.size > 0) {
+                        includeList.replaceAll { path.toString() + "/" + it }
+
+                        dependencies += IncludeDependency(currentNode, child, includeList)
+                    }
+                } else {
+                    fillDependencies(child, dependencies, path)
+                }
+            }
+        }
+        else -> {}
+    }
+}
+
+public suspend fun getIncludeFiles(string: String): List<FileName> {
+    return jacksonObjectMapper()
+        .readValue<List<FileName>>(ProcessBuilder("python3", SNARK_PARSER, string)
+        .redirectOutput(ProcessBuilder.Redirect.PIPE)
+        .redirectError(ProcessBuilder.Redirect.INHERIT)
+        .start().inputStream.bufferedReader().readText())
+}

snark-document-builder/src/main/nodejs/.gitignore

@@ -0,0 +1 @@
+node_modules

snark-document-builder/src/main/nodejs/HtmlRenderer.js

@@ -0,0 +1,16 @@
+import {toHast} from 'mdast-util-to-hast'
+import {toHtml} from 'hast-util-to-html'
+
+main()
+
+function main()
+{
+    if (process.argv.length < 3)
+        throw "No input"
+
+    const md_ast = JSON.parse(process.argv[2])
+    const hast = toHast(md_ast)
+    const html = toHtml(hast)
+
+    console.log(html)
+}

snark-document-builder/src/main/nodejs/MarkdownParser.js

@@ -0,0 +1,14 @@
+import {fromMarkdown} from 'mdast-util-from-markdown'
+
+main()
+
+function main()
+{
+    if (process.argv.length < 3)
+        throw "No input"
+
+    const markdown_string = process.argv[2] 
+    const mdast = fromMarkdown(markdown_string)
+
+    console.log(JSON.stringify(mdast))
+}

snark-document-builder/src/main/nodejs/MdRenderer.js

@@ -0,0 +1,14 @@
+import {toMarkdown} from 'mdast-util-to-markdown'
+
+main()
+
+function main()
+{
+    if (process.argv.length < 3)
+        throw "No input"
+
+    const md_ast = JSON.parse(process.argv[2])
+    const markdown = toMarkdown(md_ast)
+
+    console.log(markdown)
+}

snark-document-builder/src/main/nodejs/package.json

@@ -0,0 +1,30 @@
+{
+
+  "type": "module",
+
+  "dependencies": {
+
+    "fs": "^0.0.1-security",
+
+    "hast-util-to-html": "^8.0.4",
+
+    "mdast-util-to-hast": "^12.3.0",
+
+    "node-fetch": "^3.3.1",
+
+    "remark-html": "^15.0.2",
+
+    "remark-parse": "^10.0.1",
+
+    "require": "^2.4.20",
+
+    "to-vfile": "^7.2.4",
+
+    "unified": "^10.1.2",
+
+    "whatwg-fetch": "^3.6.2"
+
+  }
+
+}
+

snark-document-builder/src/main/python/SnarkParser.py

@@ -0,0 +1,26 @@
+import sys
+import re
+import json
+
+assert(len(sys.argv) >= 2)
+
+string = sys.argv[1]
+
+outputfile = sys.argv[2] if len(sys.argv) >= 3 else None
+
+pattern = r'^([\n|\t| ]*@include\([a-z|0-9|.|_|\/]*\)[\n|\t| ]*)*$'
+
+files = []
+
+if re.search("@include", string, re.IGNORECASE):
+    if re.match(pattern, string):
+        matches = re.findall(r'@include\((.*?)\)', string)
+        files.extend(matches)
+    else:
+        sys.exit("Illformed string")
+
+if outputfile is None:
+    print(json.dumps(files))
+else:
+    with open(outputfile, 'w+') as f:
+        json.dump(files, f)

snark-document-builder/src/test/kotlin/DocumentBuilder.kt

@@ -0,0 +1,12 @@
+package documentBuilder
+
+import org.junit.jupiter.api.Test
+import kotlinx.coroutines.runBlocking
+import space.kscience.snark.storage.local.localStorage
+
+class SomeTest {
+    @Test
+    fun justWorks() = runBlocking {
+        // buildDocument(Directory("../example"))
+    }
+}

snark-document-builder/src/test/resources/no_dependencies_example/main.md

@@ -0,0 +1,3 @@
+# Hello
+
+I'm almost empty test document without any dependencies

snark-ktor/build.gradle.kts

@@ -12,6 +12,10 @@ dependencies {
     api("io.ktor:ktor-server-core:$ktorVersion")
     api("io.ktor:ktor-server-html-builder:$ktorVersion")
     api("io.ktor:ktor-server-host-common:$ktorVersion")
+    implementation("io.ktor:ktor-server-netty:2.3.0")
+    implementation(project(":snark-storage-driver"))
+    implementation("io.ktor:ktor-server-partial-content:$ktorVersion")
+    implementation("io.ktor:ktor-server-auto-head-response:$ktorVersion")
 
     testApi("io.ktor:ktor-server-tests:$ktorVersion")
 }

snark-ktor/snark-ktor.iml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src/main/kotlin" isTestSource="false" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="KotlinJavaRuntime" level="project" />
+  </component>
+</module>

snark-ktor/src/main/kotlin/space/kscience/snark/ktor/WebInterface.kt

@@ -0,0 +1,131 @@
+package space.kscience.snark.ktor
+
+import io.ktor.http.*
+import io.ktor.server.application.*
+import io.ktor.server.response.*
+import io.ktor.server.routing.*
+import io.ktor.http.content.*
+import io.ktor.server.engine.*
+import io.ktor.server.netty.*
+import io.ktor.server.html.*
+import io.ktor.server.request.*
+import kotlinx.html.*
+import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.unzip.unzip
+import java.nio.file.Path
+import kotlin.io.createTempFile
+import kotlin.io.writeBytes
+import kotlin.io.path.Path
+
+public interface DataHolder {
+    public suspend fun init(relativePath: Path) : Directory
+
+    public suspend fun represent(relativePath: Path): String
+    public suspend fun toPdf(relativePath: Path) : Path
+}
+
+public class SNARKServer(private val dataHolder: DataHolder, private val port: Int): Runnable {
+    private var relativePath = Path("")
+
+    private suspend fun receivePath(call: ApplicationCall) {
+        val pathString = call.receiveParameters()["path"]?:""
+        relativePath = Path(pathString.dropWhile{it == '/'})
+        call.respondRedirect("/")
+    }
+    private suspend fun renderGet(call: ApplicationCall) {
+        call.respondText(dataHolder.represent(relativePath), ContentType.Text.Html)
+    }
+    private suspend fun renderFile(call: ApplicationCall) {
+        call.response.header(
+            HttpHeaders.ContentDisposition,
+            ContentDisposition.Attachment.withParameter(ContentDisposition.Parameters.FileName, "output.tex")
+                .toString()
+        )
+        call.respondFile(dataHolder.toPdf(relativePath).toFile())
+        call.respondRedirect("/")
+
+    }
+    private suspend fun renderUpload(call: ApplicationCall) {
+        val multipartData = call.receiveMultipart()
+        val tmp = createTempFile(suffix=".zip")
+        multipartData.forEachPart { part ->
+            when (part) {
+                is PartData.FileItem -> {
+                    val fileBytes = part.streamProvider().readBytes()
+                    tmp.writeBytes(fileBytes)
+                }
+                else -> {
+                }
+            }
+            part.dispose()
+        }
+        unzip(tmp.toPath().toString(), dataHolder.init(relativePath))
+        call.respondRedirect("/")
+    }
+    private suspend fun renderMainPage(call: ApplicationCall)  {
+        call.respondHtml(HttpStatusCode.OK) {
+            head {
+                title {
+                    +"SNARK"
+                }
+            }
+            body {
+                h1 {
+                    +"SNARK"
+                }
+                p {
+                    +("Path: /" + relativePath.toString())
+                }
+            }
+            body {
+                postForm(action = "/changePath") {
+                    label {
+                        + "Enter new path:"
+                    }
+                    input(name = "path", type = InputType.text) {}
+                    button {
+                        +"Change path"
+                    }
+                }
+                postForm (action = "/upload", encType = FormEncType.multipartFormData)  {
+                    label {
+                        +"Choose zip archive: "
+                    }
+                    input (name = "file", type = InputType.file) {}
+                    button {
+                        +"Upload file"
+                    }
+                }
+                a("/data") {
+                    +"Show data\n"
+                }
+                getForm (action = "/download") {
+                    button {
+                        +"Download latex\n"
+                    }
+                }
+            }
+        }
+    }
+    override fun run() {
+        embeddedServer(Netty, port) {
+            routing {
+                get("/") {
+                    renderMainPage(call)
+                }
+                post("/changePath") {
+                    receivePath(call)
+                }
+                post("/upload") {
+                    renderUpload(call)
+                }
+                get("/data") {
+                    renderGet(call)
+                }
+                get("/download") {
+                    renderFile(call)
+                }
+            }
+        }.start(wait = true)
+    }
+}

snark-ktor/src/test/kotlin/space/kscience/snark/ktor/testServer.kt

@@ -0,0 +1,50 @@
+package space.kscience.snark.ktor
+
+import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.local.localStorage
+import java.nio.file.Path
+import kotlin.io.path.*
+import kotlin.io.path.createTempDirectory
+import kotlin.io.path.isDirectory
+import kotlin.io.path.listDirectoryEntries
+
+private class LocalDataHolder: DataHolder {
+    private var source: Path? = null
+    private var response: String = ""
+
+    private fun getPath(relativePath: Path) : Path {
+        return source!! / relativePath
+    }
+    override suspend fun init(relativePath: Path): Directory {
+        if (source == null) {
+            source = createTempDirectory()
+        }
+        val path = getPath(relativePath)
+        path.createDirectories()
+        path.toFile().deleteRecursively()
+        path.createDirectory()
+        return localStorage(path)
+    }
+    private fun buildResponse(from: Path, cur: Path) {
+        for (entry in cur.listDirectoryEntries()) {
+            if (entry.isDirectory()) {
+                buildResponse(from, entry)
+            } else {
+                response += from.relativize(entry).toString() + "<br>"
+            }
+        }
+    }
+    override suspend fun represent(relativePath: Path) : String =
+        if (source == null) {
+            "No data was loaded!"
+        } else {
+            response = "List of files:<br>"
+            val path = getPath(relativePath)
+            buildResponse(path, path)
+            response
+        }
+}
+
+fun main() {
+    SNARKServer(LocalDataHolder(), 9090).run()
+}

snark-main/build.gradle.kts

@@ -0,0 +1,27 @@
+plugins {
+    id("space.kscience.gradle.jvm")
+    `maven-publish`
+}
+
+val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
+
+
+dependencies {
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
+    implementation(project(":snark-ktor"))
+    implementation(project(":snark-storage-driver"))
+    implementation(project(":snark-document-builder"))
+
+    testImplementation(kotlin("test"))
+    testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
+}
+
+tasks.test {
+    useJUnitPlatform()
+}
+
+tasks.register<JavaExec>("run_server") {
+    classpath = sourceSets.main.get().runtimeClasspath
+    main = "space.kscience.snark.main.MainKt"
+    workingDir = File(workingDir.parent)
+}

snark-main/ci/install_java.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+JAVA_VERSION="$1"
+
+source ~/.sdkman/bin/sdkman-init.sh
+sdk install java "$JAVA_VERSION"

snark-main/ci/install_sdk.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+curl -s "https://get.sdkman.io" | bash

snark-main/src/main/kotlin/space/kscience/snark/main/Main.kt

@@ -0,0 +1,19 @@
+package space.kscience.snark.main
+
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+import space.kscience.snark.ktor.SNARKServer
+import space.kscience.snark.storage.local.localStorage
+import kotlin.io.path.Path
+
+// Entrypoint
+fun main(): Unit = runBlocking {
+    // Parse config, create driver
+    val port = 8080
+    val directory = localStorage(Path("./rundata"))
+    val server = SNARKServer(ServerDataHolder(directory), port)
+    launch {
+        server.run()
+    }
+}

snark-main/src/main/kotlin/space/kscience/snark/main/ServerDataHolder.kt

@@ -0,0 +1,20 @@
+package space.kscience.snark.main
+
+import space.kscience.snark.ktor.DataHolder
+import space.kscience.snark.storage.Directory
+import documentBuilder.*
+import java.nio.file.Path
+
+internal class ServerDataHolder(private val directory: Directory): DataHolder {
+
+    override suspend fun init(relativePath: Path): Directory = directory
+
+
+    override suspend fun represent(relativePath: Path): String {
+        return buildDocument(directory, relativePath)
+    }
+
+    override suspend fun toPdf(relativePath: Path) :  Path {
+        return buildLatex(directory, relativePath)
+    }
+}

snark-main/src/test/kotlin/space/kscience/snark/main/Test.kt

@@ -0,0 +1,13 @@
+package space.kscience.snark.main
+
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.runBlocking
+import org.junit.jupiter.api.Test
+
+class Test {
+    @Test
+    fun justWorks() = runBlocking {
+        delay(5)
+        //main()
+    }
+}

snark-pandoc-plugin/.gitignore

@@ -0,0 +1,2 @@
+.pandoc/
+

snark-pandoc-plugin/build.gradle.kts

@@ -0,0 +1,25 @@
+plugins {
+    id("java")
+}
+
+repositories {
+    mavenCentral()
+}
+
+java.sourceCompatibility = JavaVersion.VERSION_11
+
+dependencies {
+    implementation("commons-io:commons-io:2.7")
+    implementation("org.slf4j:slf4j-simple:2.0.6")
+    implementation("org.slf4j:slf4j-api:2.0.6")
+    implementation("org.apache.commons:commons-exec:1.3")
+    implementation("org.apache.commons:commons-compress:1.2")
+    implementation("org.apache.ant:ant:1.10.13")
+    implementation("com.fasterxml.jackson.core:jackson-databind:2.14.2")
+    testImplementation("org.junit.jupiter:junit-jupiter-api:5.8.1")
+    testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.8.1")
+}
+
+tasks.getByName<Test>("test") {
+    useJUnitPlatform()
+}

snark-pandoc-plugin/src/main/java/space/kscience/snark/pandoc/Installer.java

@@ -0,0 +1,367 @@
+package space.kscience.snark.pandoc;
+
+import java.io.BufferedInputStream;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipFile;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.exec.OS;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class Installer {
+
+    private static final Logger log
+            = LoggerFactory.getLogger(Installer.class);
+
+    public enum OSType {
+        WINDOWS("windows-x86_64.zip", "windows"),
+        MAC_OS_AMD("x86_64-macOS.zip", "mac.os.amd"),
+        MAC_OS_ARM("arm64-macOS.zip", "mac.os.arm"),
+        LINUX_ARM("linux-arm64", "linux.arm"),
+        LINUX_AMD("linux-amd64", "linux.amd");
+
+        private final String assetSuffix;
+        private final String propertySuffix;
+        OSType(String assetSuf, String propertySuf) {
+            assetSuffix = assetSuf;
+            propertySuffix = propertySuf;
+        }
+        public String getAssetSuffix() {
+            return assetSuffix;
+        }
+        public String getPropertySuffix() {
+            return propertySuffix;
+        }
+    }
+
+    private static class OSData {
+        private URL urlForInstalling;
+        private Path fileToInstall;
+        private Path pathToPandoc;
+        public OSData() {}
+        public void setUrlForInstalling(URL urlForInstalling) {
+            this.urlForInstalling = urlForInstalling;
+        }
+        public URL getUrlForInstalling() {
+            return urlForInstalling;
+        }
+        public void setFileToInstall(Path fileToInstall) {
+            this.fileToInstall = fileToInstall;
+        }
+        public Path getFileToInstall() {
+            return fileToInstall;
+        }
+        public Path getPathToPandoc() {
+            return pathToPandoc;
+        }
+        public void setPathToPandoc(Path pathToPandoc) {
+            this.pathToPandoc = pathToPandoc;
+        }
+    }
+
+    private static Map<OSType, OSData> dataForInstalling = new HashMap<>(OSType.values().length);
+    private static final Properties properties = new Properties();
+    private static Path pandocDir = Path.of("./pandoc").toAbsolutePath();
+    private static final int TIMEOUT_SECONDS = 2;
+    private static final int ATTEMPTS = 3;
+
+    Installer() throws IOException, InterruptedException {
+        try {
+            properties.load(new FileInputStream(
+                    Thread.currentThread().getContextClassLoader().getResource("installer.properties").getPath()
+            ));
+        } catch (Exception ex) {
+            log.error("Error during download properties, ex: {}", ex.getMessage(), ex);
+            throw ex;
+        }
+
+        initFiles();
+        var resp = getGithubUrls();
+        initUrls(resp);
+    }
+
+    private void initFiles() {
+        for (var os : OSType.values()) {
+            dataForInstalling.put(os, new OSData());
+            switch (os) {
+                case LINUX_AMD :
+                case LINUX_ARM :
+                        dataForInstalling.get(os)
+                                .setFileToInstall(Path.of(pandocDir.toString() + "/pandoc.tar.gz"));
+                        break;
+                default :
+                        dataForInstalling.get(os)
+                                .setFileToInstall(Path.of(pandocDir.toString() + "/pandoc.zip"));
+            }
+        }
+    }
+
+    private void initUrls(ResponseDto responseDto) throws IOException {
+
+        for (var os : OSType.values()) {
+            var asset = responseDto.getAssetByOsSuffix(os.getAssetSuffix());
+            var currUrl = asset.getBrowserDownloadUrl();
+
+            var currPath = properties.getProperty("path.to.pandoc." + os.getPropertySuffix()).replace("{version}",
+                    responseDto.getTagName());
+
+            dataForInstalling.get(os).setUrlForInstalling(URI.create(currUrl).toURL());
+            dataForInstalling.get(os).setPathToPandoc(Path.of(pandocDir.toString() + currPath));
+            log.info("Init {} url : {}, path to pandoc: {}", os, currUrl, dataForInstalling.get(os).getPathToPandoc());
+        }
+    }
+
+    /**
+     * Install last released pandoc from github
+     * @return path to executable pandoc
+     * @throws IOException in case incorrect github url or path of installation directory
+     */
+    Path installPandoc() throws IOException {
+        log.info("Start install");
+        Path res;
+        if (OS.isFamilyMac()) {
+            if (OS.isArch("aarch64")) {
+                res = installPandoc(OSType.MAC_OS_ARM);
+            } else {
+                res = installPandoc(OSType.MAC_OS_AMD);
+            }
+        } else if (OS.isFamilyUnix()) {
+            if (OS.isArch("aarch64")) {
+                res = installPandoc(OSType.LINUX_ARM);
+            } else {
+                res = installPandoc(OSType.LINUX_AMD);
+            }
+        } else if (OS.isFamilyWindows()) {
+            res = installPandoc(OSType.WINDOWS);
+        } else {
+            throw new RuntimeException("Got unexpected os, could not install pandoc");
+        }
+        return res;
+    }
+
+    private Path installPandoc(OSType os) throws IOException {
+        log.info(
+                "Start installing pandoc os: {}, url: {}, file: {}",
+                os,
+                dataForInstalling.get(os).getUrlForInstalling(),
+                dataForInstalling.get(os).getFileToInstall()
+        );
+
+        clearInstallingDirectory();
+
+        if (!handleSaving(os)) {
+            throw new RuntimeException("Could not save file from github");
+        }
+        if (!unarchive(os)) {
+            throw new RuntimeException("Could not unzip file");
+        }
+
+        if (!dataForInstalling.get(os).getPathToPandoc().toFile().setExecutable(true)) {
+            throw new RuntimeException("Could not make pandoc executable");
+        }
+
+        return dataForInstalling.get(os).getPathToPandoc();
+    }
+
+    /**
+     * Downloads from a (http/https) URL and saves to a file.
+     * @param file File to write. Parent directory will be created if necessary
+     * @param url  http/https url to connect
+     * @param secsConnectTimeout Seconds to wait for connection establishment
+     * @param secsReadTimeout Read timeout in seconds - trasmission will abort if it freezes more than this
+     * @return true if successfully save file and false if:
+     *     connection interrupted, timeout (but something was read)
+     *     server error (500...)
+     *     could not connect: connection timeout java.net.SocketTimeoutException
+     *     could not connect: java.net.ConnectException
+     *     could not resolve host (bad host, or no internet - no dns)
+     * @throws IOException Only if URL is malformed or if could not create the file
+     * @throws FileNotFoundException if did not find file for save
+     */
+    private boolean saveUrl(final Path file, final URL url,
+                              int secsConnectTimeout, int secsReadTimeout) throws IOException {
+        Files.createDirectories(file.getParent()); // make sure parent dir exists , this can throw exception
+        var conn = url.openConnection(); // can throw exception if bad url
+        if (secsConnectTimeout > 0) {
+            conn.setConnectTimeout(secsConnectTimeout * 1000);
+        }
+        if (secsReadTimeout > 0) {
+            conn.setReadTimeout(secsReadTimeout * 1000);
+        }
+        var ret = true;
+        boolean somethingRead = false;
+        try (var is = conn.getInputStream()) {
+            try (var in = new BufferedInputStream(is);
+                 var fout = Files.newOutputStream(file)) {
+                final byte data[] = new byte[8192];
+                int count;
+                while ((count = in.read(data)) > 0) {
+                    somethingRead = true;
+                    fout.write(data, 0, count);
+                }
+            }
+        } catch (java.io.IOException e) {
+            int httpcode = 999;
+            try {
+                httpcode = ((HttpURLConnection) conn).getResponseCode();
+            } catch (Exception ee) {}
+
+            if (e instanceof FileNotFoundException) {
+                throw new FileNotFoundException("Did not found file for install");
+            }
+
+            if (somethingRead && e instanceof java.net.SocketTimeoutException) {
+                log.error("Read something, but connection interrupted: {}", e.getMessage(), e);
+                ret = false;
+            } else if (httpcode >= 400 && httpcode < 600 ) {
+                log.error("Got server error, httpcode: {}", httpcode);
+                ret = false;
+            } else if (e instanceof java.net.SocketTimeoutException) {
+                log.error("Connection timeout: {}", e.getMessage(), e);
+                ret = false;
+            } else if (e instanceof java.net.ConnectException) {
+                log.error("Could not connect: {}", e.getMessage(), e);
+                ret = false;
+            } else if (e instanceof java.net.UnknownHostException ) {
+                log.error("Could not resolve host: {}", e.getMessage(), e);
+                ret = false;
+            } else {
+                throw e;
+            }
+        }
+        return ret;
+    }
+
+    private boolean handleSaving(OSType os) throws IOException {
+        var attempt = 0;
+        var saveFile = false;
+
+        while (attempt < ATTEMPTS && !saveFile) {
+            ++attempt;
+            saveFile = saveUrl(
+                    dataForInstalling.get(os).getFileToInstall(),
+                    dataForInstalling.get(os).getUrlForInstalling(),
+                    TIMEOUT_SECONDS,
+                    TIMEOUT_SECONDS);
+        }
+
+        return saveFile;
+    }
+
+    private boolean unarchive(OSType os) {
+        try {
+            switch (os) {
+                case LINUX_AMD:
+                case LINUX_ARM :
+                    unTarGz(dataForInstalling.get(os).getFileToInstall(), pandocDir);
+                    break;
+                default :
+                    unZip(dataForInstalling.get(os).getFileToInstall(), pandocDir);
+            }
+        } catch (IOException e) {
+            log.error("Could not perform unarchiving: {}", e.getMessage(), e);
+            return false;
+        }
+        return true;
+    }
+    private void unTarGz(Path pathInput, Path targetDir) throws IOException {
+        try (var tarIn =
+                new TarArchiveInputStream(
+                        new GzipCompressorInputStream(
+                                new BufferedInputStream(Files.newInputStream(pathInput))))) {
+            ArchiveEntry archiveEntry;
+            while ((archiveEntry = tarIn.getNextEntry()) != null) {
+                var pathEntryOutput = targetDir.resolve(archiveEntry.getName());
+                if (archiveEntry.isDirectory()) {
+                    Files.createDirectory(pathEntryOutput);
+                } else {
+                    Files.copy(tarIn, pathEntryOutput);
+                }
+            }
+        }
+    }
+
+    private void unZip(Path pathInput, Path targetDir) throws IOException {
+        ZipFile zipFile = new ZipFile(pathInput.toFile());
+        try {
+            Enumeration<ZipArchiveEntry> entries = zipFile.getEntries();
+            while (entries.hasMoreElements()) {
+                ZipArchiveEntry zipEntry = entries.nextElement();
+                var pathEntryOutput = targetDir.resolve(zipEntry.getName());
+                if (zipEntry.isDirectory()) {
+                    Files.createDirectories(pathEntryOutput);
+                } else {
+                    Files.createDirectories(pathEntryOutput.getParent());
+                    Files.copy(zipFile.getInputStream(zipEntry), pathEntryOutput);
+                }
+            }
+        } finally {
+            zipFile.close();
+        }
+    }
+
+
+    /**
+     * Clear installing directory
+     */
+    public static void clearInstallingDirectory() {
+        try {
+            FileUtils.cleanDirectory(pandocDir.toFile());
+        } catch (IOException e) {
+            log.error("Could not clean installing directory");
+        }
+    }
+
+    /**
+     * Set directory to install pandoc
+     * @param newDir
+     */
+    public void setInstallingDirectory(Path newDir) {
+        pandocDir = newDir.toAbsolutePath();
+    }
+
+    private ResponseDto getGithubUrls() throws IOException, InterruptedException {
+        var uri = URI.create(properties.getProperty("github.url"));
+        var client = HttpClient.newHttpClient();
+        var request = HttpRequest
+                .newBuilder()
+                .uri(uri)
+                .version(HttpClient.Version.HTTP_2)
+                .timeout(Duration.ofMinutes(1))
+                .header("Accept",  "application/vnd.github+json")
+                .GET()
+                .build();
+        var response = client.send(request, HttpResponse.BodyHandlers.ofString());
+        log.info("Got response from github, status: {}", response.statusCode());
+
+        var objectMapper = new ObjectMapper();
+        objectMapper.configure(DeserializationFeature.USE_JAVA_ARRAY_FOR_JSON_ARRAY, true);
+        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+        return objectMapper.readValue(response.body(), ResponseDto.class);
+
+    }
+}

snark-pandoc-plugin/src/main/java/space/kscience/snark/pandoc/PandocWrapper.java

@@ -0,0 +1,142 @@
+package space.kscience.snark.pandoc;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class PandocWrapper {
+
+    private static final Logger log
+            = LoggerFactory.getLogger(PandocWrapper.class);
+
+    private static final Installer installer;
+
+    static {
+        try {
+            installer = new Installer();
+        } catch (IOException | InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+    private static String pandocPath = "pandoc"; // got pandoc at PATH
+
+    public static String getPandocPath() {
+        return pandocPath;
+    }
+
+    /**
+     * Install pandoc if needed then perform block
+     * @param block
+     * @return block's return value
+     * @param <T>
+     */
+    public  <T> Object use(Function<PandocWrapper, T> block) {
+        if (!isPandocInstalled()) {
+            installPandoc();
+        }
+        return block.apply(this);
+    }
+
+    /**
+     * Check if pandoc is installed
+     * @return true if installed false otherwise
+     */
+    public static boolean isPandocInstalled() {
+        var pb = new PandocCommandBuilder().getVersion();
+        return execute(pb);
+    }
+    /**
+     * Call pandoc with options described by commandBuilder.
+     * @param commandBuilder
+     * @return true if successfully false otherwise
+     */
+    public static boolean execute(PandocCommandBuilder commandBuilder) {
+        return execute(commandBuilder, null, null);
+    }
+
+    /**
+     * Call pandoc with options described by commandBuilder and log output to outputFile
+     * @param commandBuilder
+     * @return true if successfully false otherwise
+     */
+    public static boolean execute(PandocCommandBuilder commandBuilder, Path outputFile) {
+        return execute(commandBuilder, outputFile, null);
+    }
+    /**
+     * Call pandoc with options described by commandBuilder and log output to outputFile and error to errorFile.
+     * In case errors write exit code to errorFile
+     * @param commandBuilder
+     * @return true if successfully false otherwise
+     */
+    public static boolean execute(PandocCommandBuilder commandBuilder, Path outputFile, Path errorFile) {
+        try {
+            Process pandoc = new ProcessBuilder(commandBuilder.build()).start();
+            pandoc.waitFor(1, TimeUnit.SECONDS);
+
+            BufferedReader inp = new BufferedReader(new InputStreamReader(pandoc.getInputStream()));
+            String currLine = inp.readLine();
+
+            log.info("log output from pandoc to: {}", outputFile);
+            do {
+                if (outputFile == null) {
+                    log.info(currLine);
+                } else {
+                    Files.writeString(outputFile, currLine + "\n", StandardOpenOption.CREATE, StandardOpenOption.APPEND);
+                }
+            } while ((currLine = inp.readLine()) != null);
+            inp.close();
+
+            if (pandoc.exitValue() == 0) {
+                log.info("Successfully execute");
+                return true;
+            } else {
+                log.error("Got problems with executing, pandoc exit error: {}", pandoc.exitValue());
+
+                BufferedReader input = new BufferedReader(new InputStreamReader(pandoc.getErrorStream()));
+                String line = input.readLine();
+                log.info("log error stream from pandoc to: {}", errorFile);
+
+                if (errorFile != null) {
+                    Files.writeString(errorFile, "exit code: " + pandoc.exitValue());
+                }
+                do {
+                    if (errorFile == null) {
+                        log.info(line);
+                    } else {
+                        Files.writeString(errorFile, currLine + "\n", StandardOpenOption.CREATE, StandardOpenOption.APPEND);
+                    }
+                } while ((line = input.readLine()) != null);
+                input.close();
+
+                return false;
+            }
+        } catch (Exception e) {
+            log.error("Got problems with executing: " + e.getMessage());
+            return false;
+        }
+    }
+
+    /**
+     * Install pandoc and set executable path.
+     * @return true if success false otherwise
+     */
+    public static boolean installPandoc() {
+        try {
+            pandocPath = installer.installPandoc().toString();
+            return true;
+        } catch (Exception e) {
+            log.error("Got error: {}", e.getMessage(), e);
+            return false;
+        }
+    }
+
+}

snark-pandoc-plugin/src/main/java/space/kscience/snark/pandoc/ResponseDto.java

@@ -0,0 +1,70 @@
+package space.kscience.snark.pandoc;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Response from github/releases/latest
+ */
+public class ResponseDto {
+
+    public AssetDto[] getAssets() {
+        return assets;
+    }
+
+    public void setAssets(AssetDto[] assets) {
+        this.assets = assets;
+    }
+
+    /**
+     * @param osSuffix
+     * @return asset appropriate to os
+     */
+    public AssetDto getAssetByOsSuffix(String osSuffix) {
+        for (var asset : assets) {
+            if (asset.getName().contains(osSuffix)) {
+                return asset;
+            }
+        }
+        throw new IllegalArgumentException("Unexpected osSuffix");
+    }
+
+
+    public static class AssetDto {
+
+        @JsonProperty("browser_download_url")
+        private String browserDownloadUrl;
+        private String name;
+
+        public String getBrowserDownloadUrl() {
+            return browserDownloadUrl;
+        }
+
+        public void setBrowserDownloadUrl(String browserDownloadUrl) {
+            this.browserDownloadUrl = browserDownloadUrl;
+        }
+
+        public AssetDto() {}
+
+        public String getName() {
+            return name;
+        }
+
+        public void setName(String name) {
+            this.name = name;
+        }
+    }
+    private AssetDto[] assets;
+
+    public String getTagName() {
+        return tagName;
+    }
+
+    public void setTagName(String tagName) {
+        this.tagName = tagName;
+    }
+
+    @JsonProperty("tag_name")
+    private String tagName;
+
+    public ResponseDto() {}
+}

snark-pandoc-plugin/src/main/resources/installer.properties

@@ -0,0 +1,8 @@
+path.to.pandoc.mac.os.arm=/pandoc-{version}-arm64/bin/pandoc
+path.to.pandoc.mac.os.amd=/pandoc-{version}-x86_64/bin/pandoc
+path.to.pandoc.windows=/pandoc-{version}/pandoc.exe
+path.to.pandoc.linux.amd=/pandoc-{version}/bin/pandoc
+path.to.pandoc.linux.arm=/pandoc-{version}/bin/pandoc
+
+github.url=https://api.github.com/repos/jgm/pandoc/releases/latest
+

snark-pandoc-plugin/src/test/java/PandocWrapperTest.java

@@ -0,0 +1,123 @@
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import space.kscience.snark.pandoc.Installer;
+import space.kscience.snark.pandoc.PandocCommandBuilder;
+import space.kscience.snark.pandoc.PandocWrapper;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+public class PandocWrapperTest {
+
+    private static final Path CORRECT_MD = Path.of("./src/test/testing_directory/first_test.md");
+    private static final Path TEX_PATH_TO = Path.of("./src/test/testing_directory/output1.tex");
+    private static final Path TESTING_DIRECTORY = Path.of("./src/test/testing_directory");
+    private final PandocWrapper pandocWrapper = new PandocWrapper();
+
+    @Test
+    public void when_gotPandocAndCorrectArgs_doConverting() {
+        try {
+            var res = pandocWrapper.use(p -> {
+                var command = new PandocCommandBuilder(List.of(CORRECT_MD), TEX_PATH_TO);
+                return PandocWrapper.execute(command);
+            });
+            assertTrue((Boolean) res);
+            assertTrue(TEX_PATH_TO.toFile().exists());
+
+            var reader = new BufferedReader(new FileReader(TEX_PATH_TO.toFile()));
+            String fileString = reader.lines().collect(Collectors.joining());
+
+            assertTrue(fileString.contains("Some simple text"));
+            assertTrue(fileString.contains("\\subsection{Copy elision}"));
+            assertTrue(fileString.contains("return"));
+
+            Files.delete(TEX_PATH_TO);
+
+        } catch (Exception ex) {
+            fail("Unexpected exception during test when_gotPandocAndCorrectArgs_doConverting()", ex);
+        }
+    }
+
+    @Test
+    public void when_gotPandocAndNotExistsFromFile_then_error() {
+        var notExistsFile = Path.of("./src/test/testing_directory/non_exists_test.md");
+        assertFalse(notExistsFile.toFile().exists());
+        var res = pandocWrapper.use(p -> {
+            var command = new PandocCommandBuilder(List.of(notExistsFile), TEX_PATH_TO);
+            return PandocWrapper.execute(command);
+        });
+        assertFalse((Boolean) res);
+    }
+
+    @Test
+    public void when_gotPandocAndPassDirectory_then_error() {
+        assertTrue(TESTING_DIRECTORY.toFile().isDirectory());
+        var res = pandocWrapper.use(p -> {
+            var command = new PandocCommandBuilder(List.of(TESTING_DIRECTORY), TEX_PATH_TO);
+            return PandocWrapper.execute(command);
+        });
+        assertFalse((Boolean) res);
+    }
+
+    @Test
+    public void when_askVersionToFile_then_Ok() throws IOException {
+        Path outputFile = Files.createTempFile(TESTING_DIRECTORY, "output", ".txt");
+
+        var res = pandocWrapper.use(p -> {
+            var command = new PandocCommandBuilder();
+            command.getVersion();
+            return PandocWrapper.execute(command, outputFile);
+        });
+
+        var reader = new BufferedReader(new FileReader(outputFile.toFile()));
+        String fileString = reader.lines().collect(Collectors.joining());
+        assertTrue(fileString.contains("pandoc"));
+        assertTrue(fileString.contains("This is free software"));
+        assertTrue((Boolean) res);
+
+        Files.delete(outputFile);
+    }
+
+    @Test
+    public void when_error_then_writeToErrorStream() throws IOException {
+        Path outputFile = Files.createTempFile(TESTING_DIRECTORY, "output", ".txt");
+        Path errorFile = Files.createTempFile(TESTING_DIRECTORY, "error", ".txt");
+
+        var res = pandocWrapper.use(p -> {
+            var command = new PandocCommandBuilder(List.of(Path.of("./simple.txt")), TEX_PATH_TO);
+            command.formatFrom("txt");
+            return PandocWrapper.execute(command, outputFile, errorFile);
+        });
+
+        var reader = new BufferedReader(new FileReader(errorFile.toFile()));
+        String fileString = reader.lines().collect(Collectors.joining());
+        assertFalse((Boolean) res);
+        assertTrue(fileString.contains("21"));
+
+        Files.delete(outputFile);
+        Files.delete(errorFile);
+    }
+
+
+
+    @Test
+    public void when_installPandoc_thenFindIt() {
+        Installer.clearInstallingDirectory();
+        assertTrue(PandocWrapper.installPandoc());
+        assertTrue(PandocWrapper.isPandocInstalled());
+    }
+
+    @AfterAll
+    public static void clear() {
+        Installer.clearInstallingDirectory();
+    }
+}

snark-pandoc-plugin/src/test/testing_directory/first_test.md

@@ -0,0 +1,15 @@
+
+
+## Copy elision
+### RVO/NRVO
+Some simple text
+```c++
+A f() {
+    return {5};
+}
+
+A g() {
+    A a(5);
+    return a;
+}
+```

snark-pandoc-plugin/src/test/testing_directory/simple.txt

@@ -0,0 +1 @@
+hello

snark-storage-driver/build.gradle.kts

@@ -0,0 +1,22 @@
+plugins {
+    id("space.kscience.gradle.jvm")
+    `maven-publish`
+}
+
+val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
+val awsSdkVersion = "0.+"
+
+
+dependencies {
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
+
+    // s3 Driver dependency
+    implementation("aws.sdk.kotlin:s3:$awsSdkVersion")
+
+    testImplementation(kotlin("test"))
+    testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
+}
+
+tasks.test {
+    useJUnitPlatform()
+}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Config.kt

@@ -0,0 +1,40 @@
+package space.kscience.snark.storage
+
+import aws.sdk.kotlin.services.s3.S3Client
+import space.kscience.snark.storage.local.localStorage
+import space.kscience.snark.storage.s3.s3Bucket
+import space.kscience.snark.storage.s3.s3Storage
+import java.nio.file.Path
+
+private const val DEFAULT_REGION = "arctic-vault"
+
+public sealed interface Config {
+    public fun build(): Directory
+}
+
+public data class LocalConfig(val path: Path) : Config {
+    override fun build(): Directory {
+        return localStorage(path)
+    }
+}
+
+/*
+ * ~/.aws/credentials.json file is required
+ */
+internal fun buildS3Client(regionSpec: String): S3Client {
+    return S3Client {
+        region = regionSpec
+    }
+}
+
+public data class S3BucketConfig(val bucketName: String, val region: String = DEFAULT_REGION) : Config {
+    override fun build(): Directory {
+        return s3Bucket(buildS3Client(region), bucketName)
+    }
+}
+
+public data class S3ServiceConfig(val region: String = DEFAULT_REGION) : Config {
+    override fun build(): Directory {
+        return s3Storage(buildS3Client(region))
+    }
+}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt

@@ -0,0 +1,46 @@
+package space.kscience.snark.storage
+
+import java.nio.file.Path
+import kotlin.io.path.Path
+
+public interface Directory : AutoCloseable {
+    // get file from subtree
+    public suspend fun get(filename: Path): FileReader
+
+    @Deprecated("Use put")
+    public suspend fun create(filename: String, ignoreIfExists: Boolean = false)
+
+    // put file to subtree
+    public suspend fun put(filename: Path): FileWriter
+
+    public suspend fun getSubdir(path: Path): Directory
+
+    @Deprecated("Directories are created on put")
+    public suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean = false): Directory
+
+    @Deprecated("Not a good idea")
+    public val path: Path
+
+    public companion object {
+        public fun fromConfig(config: Config): Directory {
+            return config.build()
+        }
+    }
+}
+
+
+public suspend fun Directory.get(filename: String): FileReader = get(Path(filename))
+
+public suspend fun Directory.put(filename: String): FileWriter = put(Path(filename))
+
+public suspend operator fun Directory.div(path: Path): Directory = getSubdir(path)
+
+public suspend operator fun Directory.div(path: String): Directory = getSubdir(Path(path))
+
+public interface FileReader : AutoCloseable {
+    public suspend fun readAll(): ByteArray
+}
+
+public interface FileWriter : AutoCloseable {
+    public suspend fun write(bytes: ByteArray)
+}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt

@@ -0,0 +1,71 @@
+package space.kscience.snark.storage.local
+
+import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.FileReader
+import space.kscience.snark.storage.FileWriter
+import java.nio.file.Path
+import kotlin.io.path.*
+
+public fun localStorage(rootPath: Path): Directory {
+    return LocalDirectory(rootPath, Path(""))
+}
+
+internal class LocalFile(private val path: Path) : FileReader, FileWriter {
+    override fun close() {}
+    override suspend fun readAll(): ByteArray = path.readBytes()
+
+    override suspend fun write(bytes: ByteArray) {
+        path.parent.createDirectories()
+        try {
+            path.createFile()
+        } catch (ex: Exception) {
+            // Do nothing
+        }
+        path.writeBytes(bytes)
+    }
+}
+
+internal class LocalDirectory(private val root: Path, private val currentDir: Path) : Directory {
+    @Deprecated("Use Path, not String")
+    private fun realpath(child: String): Path = root / currentDir / child
+    private fun realpath(child: Path): Path = root / currentDir / child
+
+    override fun close() {}
+
+    override suspend fun get(filename: Path): LocalFile = LocalFile(realpath(filename))
+
+    @Deprecated("Use put")
+    override suspend fun create(filename: String, ignoreIfExists: Boolean) {
+        val dir = realpath(filename)
+        dir.parent.createDirectories()
+        try {
+            realpath(filename).createFile()
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {
+            if (!ignoreIfExists) {
+                throw ex
+            }
+        }
+    }
+
+    override suspend fun put(filename: Path): LocalFile = get(filename)
+
+    override suspend fun getSubdir(path: Path): LocalDirectory = LocalDirectory(root, currentDir / path)
+
+    @Deprecated("Directories are created on put")
+    override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): LocalDirectory {
+        val dir = realpath(dirname)
+        dir.parent.createDirectories()
+        try {
+            dir.createDirectory()
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {
+            if (!ignoreIfExists) {
+                throw ex
+            }
+        }
+        return LocalDirectory(root, currentDir / dirname)
+    }
+
+    @Deprecated("Not a good idea")
+    override val path: Path
+        get() = currentDir
+}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt

@@ -0,0 +1,45 @@
+package space.kscience.snark.storage.s3
+
+import aws.sdk.kotlin.services.s3.S3Client
+import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.FileReader
+import space.kscience.snark.storage.FileWriter
+import java.nio.file.Path
+import kotlin.io.path.div
+
+internal class S3Directory(
+    private val client: S3Client,
+    private val bucketName: String,
+    private val currentDir: Path,
+) : Directory {
+    override suspend fun get(filename: Path): FileReader =
+        S3FileReader(client, bucketName, currentDir / filename)
+
+    @Deprecated("Use put")
+    override suspend fun create(filename: String, ignoreIfExists: Boolean) {
+        if (!ignoreIfExists) {
+            TODO("could not check if file exists")
+        }
+    }
+
+    override suspend fun put(filename: Path): FileWriter =
+        S3FileWriter(client, bucketName, currentDir / filename)
+
+    override suspend fun getSubdir(path: Path): S3Directory =
+        S3Directory(client, bucketName, currentDir / path)
+
+    @Deprecated("Directories are created on put")
+    override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): S3Directory =
+        if (!ignoreIfExists) {
+            TODO("could not check if directory exists")
+        } else {
+            S3Directory(client, bucketName, currentDir / dirname)
+        }
+
+    @Deprecated("Not a good idea")
+    override val path: Path
+        get() = currentDir
+
+    override fun close() {
+    }
+}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt

@@ -0,0 +1,41 @@
+package space.kscience.snark.storage.s3
+
+import aws.sdk.kotlin.services.s3.S3Client
+import aws.sdk.kotlin.services.s3.model.GetObjectRequest
+import aws.sdk.kotlin.services.s3.putObject
+import aws.smithy.kotlin.runtime.content.ByteStream
+import aws.smithy.kotlin.runtime.content.toByteArray
+import space.kscience.snark.storage.FileReader
+import space.kscience.snark.storage.FileWriter
+import java.nio.file.Path
+
+internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) :
+    FileReader {
+    override suspend fun readAll(): ByteArray {
+        val result = client.getObject(GetObjectRequest {
+            bucket = bucketName
+            key = path.toString()
+        }) {
+            it.body?.toByteArray() ?: ByteArray(0)
+        }
+        return result
+    }
+
+    override fun close() {
+    }
+}
+
+internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) :
+    FileWriter {
+    override suspend fun write(bytes: ByteArray) {
+        client.putObject {
+            bucket = bucketName
+            key = path.toString()
+            body = ByteStream.fromBytes(bytes)
+        }
+    }
+
+    override fun close() {
+    }
+
+}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt

@@ -0,0 +1,67 @@
+package space.kscience.snark.storage.s3
+
+import aws.sdk.kotlin.services.s3.S3Client
+import aws.sdk.kotlin.services.s3.createBucket
+import aws.sdk.kotlin.services.s3.headBucket
+import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.FileReader
+import space.kscience.snark.storage.FileWriter
+import java.nio.file.Path
+import kotlin.io.path.Path
+
+public fun s3Storage(client: S3Client): Directory =
+    S3Root(client)
+
+public fun s3Bucket(client: S3Client, bucket: String): Directory =
+    S3Directory(client, bucket, Path(""))
+
+internal fun splitPathIntoBucketAndPath(path: Path): Pair<String, Path> {
+    val bucket = path.getName(0)
+    val filePath = path.relativize(bucket)
+    return Pair(bucket.toString(), filePath)
+}
+
+internal class S3Root(private val client: S3Client) : Directory {
+
+    override suspend fun get(filename: Path): FileReader {
+        throw NoSuchFileException(filename.toFile())
+    }
+
+    @Deprecated("Use put")
+    override suspend fun create(filename: String, ignoreIfExists: Boolean) {
+        throw NoSuchFileException(Path(filename).toFile())
+    }
+
+    override suspend fun put(filename: Path): FileWriter {
+        throw NoSuchFileException(filename.toFile())
+    }
+
+    override suspend fun getSubdir(path: Path): Directory = try {
+        val (bucketName, filePath) = splitPathIntoBucketAndPath(path)
+        client.headBucket {
+            bucket = bucketName
+        }
+        S3Directory(client, bucketName, filePath)
+    } catch (ex: Exception) {
+        throw AccessDeniedException(path.toFile(), reason = ex.message)
+    }
+
+    @Deprecated("Directories are created on put")
+    override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try {
+        val (bucketName, filePath) = splitPathIntoBucketAndPath(Path(dirname))
+        client.createBucket {
+            bucket = bucketName
+        }
+        S3Directory(client, bucketName, filePath)
+    } catch (ex: Exception) {
+        throw AccessDeniedException(Path(dirname).toFile(), reason = ex.message)
+    }
+
+    @Deprecated("Not a good idea")
+    override val path: Path
+        get() = Path("")
+
+    override fun close() {
+    }
+
+}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt

@@ -0,0 +1,28 @@
+package space.kscience.snark.storage.unzip
+
+import space.kscience.snark.storage.*
+import java.io.FileInputStream
+import java.util.zip.ZipInputStream
+
+public suspend fun unzip(source_path: String, target: Directory) {
+    val zis = ZipInputStream(FileInputStream(source_path))
+    var zipEntry = zis.nextEntry
+    while (zipEntry != null) {
+        if (!zipEntry.isDirectory) {
+            val filename = zipEntry.name
+            target.create(filename, true)
+            val fos = target.put(filename)
+            var sz = zipEntry.size.toInt()
+            if (sz == -1) {
+                sz = 1024
+            }
+            val buffer = ByteArray(sz)
+            zis.read(buffer)
+            fos.write(buffer)
+            fos.close()
+        }
+        zipEntry = zis.nextEntry
+    }
+    zis.closeEntry()
+    zis.close()
+}

snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/JustCreates.kt

@@ -0,0 +1,25 @@
+package space.kscience.snark.storage
+
+import org.junit.jupiter.api.Test
+import kotlin.io.path.createTempDirectory
+
+class JustCreates {
+
+    @Test
+    fun s3Created() {
+        val dir = Directory.fromConfig(S3ServiceConfig())
+        dir.close()
+    }
+
+    @Test
+    fun s3BucketCreated() {
+        val dir = Directory.fromConfig(S3BucketConfig("snark-test"))
+        dir.close()
+    }
+
+    @Test
+    fun localCreated() {
+        val dir = Directory.fromConfig(LocalConfig(createTempDirectory("snark-test")))
+        dir.close()
+    }
+}

snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Example.kt

@@ -0,0 +1,47 @@
+package space.kscience.snark.storage.local
+
+import kotlinx.coroutines.runBlocking
+import org.junit.jupiter.api.BeforeAll
+import space.kscience.snark.storage.*
+import java.nio.file.Path
+import kotlin.io.path.createTempDirectory
+import kotlin.io.path.deleteExisting
+import kotlin.test.AfterTest
+import kotlin.test.BeforeTest
+import kotlin.test.Test
+import kotlin.io.path.*
+import kotlin.test.assertEquals
+
+class Example {
+    var tempDir: Path? = null
+    var somedir: Directory? = null
+
+    @BeforeTest
+    fun setUp() {
+        tempDir = createTempDirectory()
+        somedir = localStorage(tempDir!!)
+    }
+
+    @AfterTest
+    fun tearDown() {
+        tempDir!!.toFile().deleteRecursively()
+        somedir = null
+    }
+
+    @Test
+    fun exampleTest() = runBlocking {
+        somedir!!.put("somefile").write("hello".toByteArray())
+        assertEquals("hello", somedir!!.get("somefile").readAll().decodeToString())
+    }
+
+    @Test
+    fun subdirExample() = runBlocking {
+        val dir1 = somedir!! / "tmp1"
+        dir1.put("somefile").write("hello".toByteArray())
+
+        val dir2 = somedir!! / "tmp1"
+        val data = dir2.get("somefile").readAll()
+
+        assertEquals("hello", data.decodeToString())
+    }
+}

snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Tests.kt

@@ -0,0 +1,105 @@
+package space.kscience.snark.storage.local
+
+import kotlinx.coroutines.runBlocking
+
+import space.kscience.snark.storage.*
+import java.io.File
+import java.nio.file.Path
+import kotlin.io.path.*
+import kotlin.test.*
+
+internal class LocalDriverTests {
+
+    private var tempDir: Path? = null
+    private var testSample: Directory? = null
+    private val bytes = byteArrayOf(0, 1, 2, 3, 4, 5, 6, 7)
+
+    @BeforeTest
+    fun setUp() {
+        tempDir = createTempDirectory()
+        testSample = localStorage(tempDir!!)
+    }
+
+    @Test
+    fun testCreate() = runBlocking {
+        //folder is empty
+        assertEquals(0, tempDir!!.listDirectoryEntries().size)
+
+        //create first file
+        testSample!!.create("tmp1")
+        val entries = tempDir!!.listDirectoryEntries()
+        assertEquals(1, entries.size)
+        assertEquals(tempDir!! / Path("tmp1"), entries.first())
+        //assertTrue(!entries.first().isDirectory())
+
+        //create second file
+        testSample!!.create("tmp2")
+        assertEquals(2, tempDir!!.listDirectoryEntries().size)
+
+        //check exception after duplication
+        try {
+            testSample!!.create("tmp1")
+            fail("shouldn't ignore duplicates here")
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {}
+
+        //check ignorance
+        try {
+            testSample!!.create("tmp1", true)
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {
+            fail("should ignore duplicates here")
+        }
+    }
+
+    @Test
+    fun testPutGet() = runBlocking {
+        testSample!!.create("tmp")
+        testSample!!.put("tmp").write(bytes)
+        assertContentEquals(bytes, testSample!!.get("tmp").readAll())
+    }
+
+    @Test
+    fun testCreateSubdir() = runBlocking {
+        //folder is empty
+        assertEquals(0, tempDir!!.listDirectoryEntries().size)
+
+        //create first file
+        testSample!!.createSubdir("tmp1")
+        val entries = tempDir!!.listDirectoryEntries()
+        assertEquals(1, entries.size)
+        assertEquals(tempDir!! / Path("tmp1"), entries.first())
+        assertTrue (entries.first().isDirectory())
+
+        //create second file
+        testSample!!.createSubdir("tmp2")
+        assertEquals(2, tempDir!!.listDirectoryEntries().size)
+
+        //check exception after duplication
+        try {
+            testSample!!.createSubdir("tmp1")
+            fail("shouldn't ignore duplicates here")
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {}
+
+        //check ignorance
+        try {
+            testSample!!.createSubdir("tmp1", true)
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {
+            fail("should ignore duplicates here")
+        }
+        assertTrue {true}
+    }
+
+    @Test
+    fun testGetSubdir() = runBlocking {
+        testSample!!.createSubdir("tmp")
+        val pathStr = (Path("tmp") / "data.txt").toString()
+        testSample!!.create(pathStr)
+        testSample!!.put(pathStr).write(bytes)
+        val subdir = testSample!!.getSubdir(Path("tmp"))
+        assertContentEquals(bytes, subdir.get("data.txt").readAll())
+    }
+
+    @AfterTest
+    fun tearDown() {
+        tempDir!!.toFile().deleteRecursively()
+    }
+}

snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/unzip/Tests.kt

@@ -0,0 +1,108 @@
+package space.kscience.snark.storage.unzip
+
+import kotlinx.coroutines.runBlocking
+
+import space.kscience.snark.storage.*
+import space.kscience.snark.storage.local.localStorage
+import java.io.*
+import java.nio.file.Files
+import java.nio.file.Path
+import java.util.concurrent.TimeUnit
+import java.util.zip.ZipEntry
+import java.util.zip.ZipOutputStream
+import kotlin.io.path.*
+import kotlin.test.*
+
+internal class UnzipTests {
+
+    private var tempDir: Path? = null
+
+    @BeforeTest
+    fun setUp() {
+        tempDir = createTempDirectory()
+    }
+
+    private suspend fun makeFile(dir: Directory, filename: String, content: ByteArray) {
+        dir.create(filename)
+        dir.put(filename).write(content)
+    }
+
+    private fun zipAll(directory: String, zipFile: String) {
+        val sourceFile = File(directory)
+
+        ZipOutputStream(BufferedOutputStream( FileOutputStream(zipFile))).use {
+            zipFiles(it, sourceFile, File.separator)
+            it.closeEntry()
+            it.close()
+        }
+    }
+
+    private fun zipFiles(zipOut: ZipOutputStream, sourceFile: File, parentDirname: String) {
+
+        val data = ByteArray(2048)
+
+        for (f in sourceFile.listFiles()) {
+            if (f.isDirectory) {
+                zipFiles(zipOut, f, parentDirname + f.name + File.separator)
+            } else {
+                FileInputStream(f).use { fi ->
+                    BufferedInputStream(fi).use { origin ->
+                        var path = parentDirname + f.name
+                        val entry = ZipEntry(path.drop(1))
+                        entry.time = f.lastModified()
+                        entry.size = f.length()
+                        zipOut.putNextEntry(entry)
+                        while (true) {
+                            val readBytes = origin.read(data)
+                            if (readBytes == -1) {
+                                break
+                            }
+                            zipOut.write(data, 0, readBytes)
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+
+    @Test
+    fun testUnzip() = runBlocking {
+        val dir: Directory = localStorage(tempDir!!)
+        val source = dir.createSubdir("source")
+        val target = dir.createSubdir("target")
+        val bytes1 = byteArrayOf(0, 1, 2, 3)
+        val bytes2 = byteArrayOf(1, 0, 3, 2)
+        val bytes3 = byteArrayOf(3, 2, 1, 0)
+        makeFile(source, "tmp1", bytes1)
+        makeFile(source, "tmp2", bytes2);
+        makeFile(source, (Path("tdir") / "tmp3").toString(), bytes3)
+
+        dir.create("archive.zip")
+        val archive_path = (tempDir!! / Path("archive.zip")).toString()
+
+        zipAll((tempDir!! / Path("source")).toString(), archive_path)
+
+        unzip(archive_path, target)
+
+        val targetPath = tempDir!! / Path("target")
+        println(targetPath)
+        val entries = targetPath.listDirectoryEntries()
+
+        assertEquals(3, entries.size)
+        val exp_entries = listOf(
+            targetPath / Path("tmp1"),
+            targetPath / Path("tmp2"),
+            targetPath / Path("tdir"))
+        assertContentEquals(entries.sorted(), exp_entries.sorted())
+
+        val tdirEntries = (targetPath / Path("tdir")).listDirectoryEntries()
+        assertEquals(1, tdirEntries.size)
+        assertEquals(tdirEntries.first(), targetPath / Path("tdir") / Path("tmp3"))
+    }
+
+    @AfterTest
+    fun tearDown() {
+        tempDir!!.toFile().deleteRecursively()
+    }
+}