SNRK-98: merged main into SNRK-87

.dockerignore

@@ -0,0 +1,20 @@
+# .gitignore contents
+
+.gradle/
+build/
+.idea/
+/logs/
+
+!gradle/wrapper/gradle-wrapper.jar
+
+kotlin-js-store
+*.iml
+
+# addition to .gitignore
+
+README.md
+LICENSE
+gradlew.bat
+.dockerignore
+.gitignore
+Dockerfile

Dockerfile

@@ -0,0 +1,8 @@
+FROM ubuntu:latest
+
+WORKDIR Snark
+
+COPY . .
+
+RUN ./requirments.sh
+RUN ./gradlew build test

requirments.sh

@@ -0,0 +1,27 @@
+set -e
+
+apt-get update
+apt-get install -y sudo
+
+sudo apt-get install -y kotlin
+
+for dir in ./*/
+do
+    if [[ $dir == *'snark'* ]]
+    then
+        cd "$dir"
+
+        if [[ $(find -type d -name "ci") ]]
+        then
+
+            cd ci
+
+            ./requirments.sh
+
+            cd ..
+        fi
+
+        cd ..
+    fi
+done
+

settings.gradle.kts

@@ -43,4 +43,6 @@ include(
     ":snark-html",
     ":snark-ktor",
     ":snark-storage-driver",
+    ":snark-document-builder",
+    ":snark-main",
 )

snark-document-builder/build.gradle.kts

@@ -0,0 +1,19 @@
+plugins {
+    id("space.kscience.gradle.jvm")
+    `maven-publish`
+    id("kotlinx-serialization")
+}
+
+val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
+val jacksonVersion = "2.14.2"
+val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
+
+
+dependencies {
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
+    api("io.ktor:ktor-server-html-builder:$ktorVersion")
+
+    implementation(project(":snark-storage-driver"))
+    implementation("com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion")
+    implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.3")
+}

snark-document-builder/ci/requirments.sh

@@ -0,0 +1,6 @@
+sudo apt-get install -y python3
+sudo apt-get install -y nodejs
+sudo apt-get install -y npm
+
+cd ../src/main/nodejs
+sudo npm install .

snark-document-builder/src/main/kotlin/Build.kt

@@ -0,0 +1,17 @@
+package documentBuilder
+
+public class GraphManager(public val graph: DependencyGraph) {
+    fun buildDocument(file: FileName) {
+        val list = graph.nodes[file]
+        if (list != null) {
+            for (element in list.dependencies) {
+                element.visit(this)
+            }
+        }
+    }
+
+    fun getAstRootDocument(file: FileName): MdAstRoot {
+        buildDocument(file)
+        return graph.nodes[file]!!.mdAst
+    }
+}

snark-document-builder/src/main/kotlin/DependencyGraph.kt

@@ -0,0 +1,71 @@
+package documentBuilder
+
+import kotlinx.coroutines.coroutineScope
+import kotlin.collections.MutableList
+
+
+public typealias FileName = String
+
+/**
+ * Node of dependency graph.
+ *
+ * One node represents one file and its dependencies
+ *
+ * @property mdAst - AST tree of current file.
+ * @property dependencies - list of tail end adjacent to this node (dependencies of current file to be resolved).
+ */
+public data class DependencyGraphNode(
+    val mdAst: MdAstRoot,
+    val dependencies: List<DependencyGraphEdge>
+)
+
+/**
+ * Interface of all dependency edges.
+ */
+public sealed interface DependencyGraphEdge {
+    public fun visit(graphManager: GraphManager)
+}
+
+/**
+ * Include dependency edge.
+ *
+ * @property parentNode - node inside AST tree, that is parent for dependent node.
+ * @property dependentNode - dependent node, i.e. node of part of document with include commands
+ * @property includeList - list of files to be included.
+ */
+public data class IncludeDependency(
+    val parentNode: MdAstParent,
+    val dependentNode: MdAstElement,
+    val includeList: List<FileName>
+) : DependencyGraphEdge {
+    override fun visit(graphManager: GraphManager) {
+        val parent = parentNode
+        val childs: MutableList<MdAstElement> = mutableListOf()
+        for (file in includeList) {
+            graphManager.buildDocument(file)
+            childs.addAll(graphManager.graph.nodes[file]!!.mdAst.children)
+        }
+        val elements: MutableList<MdAstElement> = parent.children.toMutableList()
+        val index = parent.children.indexOf(dependentNode)
+        elements.removeAt(index)
+        elements.addAll(index, childs)
+        parent.children = elements
+    }
+}
+
+// parent - List<MdAstElement> -------------------------------------
+//                |                                                  \
+//                 \                                                  \
+//                  |                                                  \
+// dependentNode - MdAstElement                                         \
+//                                                                       |
+// List<FileName> -> List<MdAstRoot> --> List<List<MdAstElement>> ===> List<MdAstElement>
+
+/**
+ * Whole dependency graph.
+ *
+ * @property nodes - map of nodes, where you can find DependencyGraphNode of file by its name.
+ */
+public data class DependencyGraph(
+    val nodes: Map<FileName, DependencyGraphNode>
+)

snark-document-builder/src/main/kotlin/DocumentBuilder.kt

@@ -0,0 +1,67 @@
+package documentBuilder
+
+import com.fasterxml.jackson.core.io.BigDecimalParser
+import space.kscience.snark.storage.*
+import java.nio.file.Path
+import java.nio.file.Paths
+import kotlinx.html.*
+import kotlinx.html.dom.createHTMLDocument
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import com.fasterxml.jackson.module.kotlin.readValue
+import kotlinx.serialization.json.Json
+
+private val SNARK_HTML_RENDER = "snark-document-builder/src/main/nodejs/HtmlRenderer.js"
+fun getHtml(ast_string: String): String
+{
+    return ProcessBuilder("node", SNARK_HTML_RENDER, ast_string)
+        .redirectOutput(ProcessBuilder.Redirect.PIPE)
+        .redirectError(ProcessBuilder.Redirect.INHERIT)
+        .start().inputStream.bufferedReader().readText()
+}
+
+private val DEFAULT_DOCUMENT_ROOT = "main.md"
+
+public suspend fun buildDocument(documentDirectory: Directory): String {
+    val dependencyGraph = buildDependencyGraph(documentDirectory)
+
+    val root: MdAstRoot = dependencyGraph.nodes[""]!!.mdAst
+
+    return getHtml(jacksonObjectMapper().writeValueAsString(root))
+    // return jacksonObjectMapper().writeValueAsString(root)
+}
+
+public suspend fun buildDependencyGraph(root: Directory): DependencyGraph {
+    val nodes = HashMap<FileName, DependencyGraphNode>()
+
+    buildNodes(root, nodes)
+
+    return DependencyGraph(nodes)
+}
+
+private suspend fun buildNodes(folder: Directory, nodes: HashMap<FileName, DependencyGraphNode>) {
+    val pathString = folder.path.toString()
+
+    assert(!nodes.containsKey(pathString))
+
+    val rootDcoument = folder.get(DEFAULT_DOCUMENT_ROOT)
+    nodes.put(pathString, buildDependencyGraphNode(rootDcoument.readAll(), folder.path))
+
+    val dependencies = getDependencies(nodes.getValue(pathString))
+
+    for (dependency in dependencies) {
+        if (!nodes.containsKey(dependency))
+            buildNodes(folder.getSubdir(Paths.get(dependency)), nodes)
+    }
+}
+
+public suspend fun getDependencies(node: DependencyGraphNode): Set<FileName> {
+    val dependencies = mutableListOf<FileName>()
+
+    for (dependency in node.dependencies) {
+        when (dependency) {
+            is IncludeDependency -> dependencies.addAll(dependency.includeList)
+        }
+    }
+
+    return dependencies.toSet()
+}

snark-document-builder/src/main/kotlin/MdAstElements.kt

@@ -0,0 +1,84 @@
+package documentBuilder
+
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.SerialName
+import com.fasterxml.jackson.annotation.JsonSubTypes
+import com.fasterxml.jackson.annotation.JsonTypeInfo
+
+@Serializable
+public data class Point(val line: Int, val column: Int, val offset: Int)
+
+@Serializable
+public data class Position(val start: Point, val end: Point)
+
+@JsonTypeInfo(
+    use = JsonTypeInfo.Id.NAME,
+    include = JsonTypeInfo.As.PROPERTY,
+    property = "type"
+)
+
+@JsonSubTypes(
+    JsonSubTypes.Type(value = MdAstRoot::class, name = "root"),
+    JsonSubTypes.Type(value = MdAstParagraph::class, name = "paragraph"),
+    JsonSubTypes.Type(value = MdAstText::class, name = "text"),
+    JsonSubTypes.Type(value = MdAstHeading::class, name = "heading"),
+    JsonSubTypes.Type(value = MdAstCode::class, name = "code"),
+    JsonSubTypes.Type(value = MdAstBlockquote::class, name = "blockquote")
+)
+
+@Serializable
+public sealed interface MdAstElement{
+    public abstract var position: Position
+}
+
+@Serializable
+public sealed interface MdAstParent: MdAstElement{
+    public var children: List<MdAstElement>
+}
+
+@Serializable
+@SerialName("root")
+public data class MdAstRoot(
+    override var children: List<MdAstElement>,
+    override var position: Position
+): MdAstParent
+
+
+@Serializable
+@SerialName("paragraph")
+public data class MdAstParagraph(
+    override var children: List<MdAstElement>,
+    override var position: Position
+): MdAstParent
+
+
+@Serializable
+@SerialName("text")
+public data class MdAstText(
+    val value: String,
+    override var position: Position
+): MdAstElement
+
+@Serializable
+@SerialName("heading")
+public data class MdAstHeading(
+    val depth: Int, 
+    override var children: List<MdAstElement>,
+    override var position: Position
+): MdAstParent
+
+@Serializable
+@SerialName("code")
+public data class MdAstCode(
+    var lang: String? = null,
+    var meta: String? = null,
+    var value: String,
+    override var position: Position,
+) : MdAstElement
+
+@Serializable
+@SerialName("blockquote")
+public data class MdAstBlockquote(
+    override var children: List<MdAstElement>,
+    override var position: Position
+): MdAstParent

snark-document-builder/src/main/kotlin/MdParser.kt

@@ -0,0 +1,57 @@
+package documentBuilder
+
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import com.fasterxml.jackson.module.kotlin.readValue
+import java.nio.file.Path
+
+private val MARKDOWN_PARSER = "snark-document-builder/src/main/nodejs/MarkdownParser.js"
+private val SNARK_PARSER = "snark-document-builder/src/main/python/SnarkParser.py"
+
+public suspend fun parseMd(mdFile: ByteArray): MdAstRoot {
+    return jacksonObjectMapper()
+        .readValue<MdAstRoot>(ProcessBuilder("node", MARKDOWN_PARSER, mdFile.toString())
+        .redirectOutput(ProcessBuilder.Redirect.PIPE)
+        .redirectError(ProcessBuilder.Redirect.INHERIT)
+        .start().inputStream.bufferedReader().readText())
+}
+
+public suspend fun buildDependencyGraphNode(mdFile: ByteArray, path: Path): DependencyGraphNode {
+    val treeRoot = parseMd(mdFile)
+    val dependencies = mutableListOf<DependencyGraphEdge>()
+
+    fillDependencies(treeRoot, dependencies, path)
+
+    return DependencyGraphNode(treeRoot, dependencies)
+}
+
+internal suspend fun fillDependencies(
+        currentNode: MdAstElement,
+        dependencies: MutableList<DependencyGraphEdge>,
+        path: Path) {
+    when (currentNode) {
+        is MdAstParent -> {
+            for (child in currentNode.children) {
+                if (child is MdAstText) {
+                    val includeList = getIncludeFiles(child.value).toMutableList()
+
+                    if (includeList.size > 0) {
+                        includeList.replaceAll { path.toString() + "/" + it }
+
+                        dependencies += IncludeDependency(currentNode, child, includeList)
+                    }
+                } else {
+                    fillDependencies(child, dependencies, path)
+                }
+            }
+        }
+        else -> {}
+    }
+}
+
+public suspend fun getIncludeFiles(string: String): List<FileName> {
+    return jacksonObjectMapper()
+        .readValue<List<FileName>>(ProcessBuilder("python3", SNARK_PARSER, string)
+        .redirectOutput(ProcessBuilder.Redirect.PIPE)
+        .redirectError(ProcessBuilder.Redirect.INHERIT)
+        .start().inputStream.bufferedReader().readText())
+}

snark-document-builder/src/main/nodejs/.gitignore

@@ -0,0 +1 @@
+node_modules

snark-document-builder/src/main/nodejs/HtmlRenderer.js

@@ -0,0 +1,16 @@
+import {toHast} from 'mdast-util-to-hast'
+import {toHtml} from 'hast-util-to-html'
+
+main()
+
+function main()
+{
+    if (process.argv.length < 3)
+        throw "No input"
+
+    const md_ast = JSON.parse(process.argv[2])
+    const hast = toHast(md_ast)
+    const html = toHtml(hast)
+
+    console.log(html)
+}

snark-document-builder/src/main/nodejs/MarkdownParser.js

@@ -0,0 +1,14 @@
+import {fromMarkdown} from 'mdast-util-from-markdown'
+
+main()
+
+function main()
+{
+    if (process.argv.length < 3)
+        throw "No input"
+
+    const markdown_string = process.argv[2] 
+    const mdast = fromMarkdown(markdown_string)
+
+    console.log(JSON.stringify(mdast))
+}

snark-document-builder/src/main/nodejs/package.json

@@ -0,0 +1,30 @@
+{
+
+  "type": "module",
+
+  "dependencies": {
+
+    "fs": "^0.0.1-security",
+
+    "hast-util-to-html": "^8.0.4",
+
+    "mdast-util-to-hast": "^12.3.0",
+
+    "node-fetch": "^3.3.1",
+
+    "remark-html": "^15.0.2",
+
+    "remark-parse": "^10.0.1",
+
+    "require": "^2.4.20",
+
+    "to-vfile": "^7.2.4",
+
+    "unified": "^10.1.2",
+
+    "whatwg-fetch": "^3.6.2"
+
+  }
+
+}
+

snark-document-builder/src/main/python/SnarkParser.py

@@ -0,0 +1,26 @@
+import sys
+import re
+import json
+
+assert(len(sys.argv) >= 2)
+
+string = sys.argv[1]
+
+outputfile = sys.argv[2] if len(sys.argv) >= 3 else None
+
+pattern = r'^([\n|\t| ]*@include\([a-z|0-9|.|_]*.md\)[\n|\t| ]*)*$'
+
+files = []
+
+if re.search("@include", string, re.IGNORECASE):
+    if re.match(pattern, string):
+        matches = re.findall(r'@include\((.*?)\)', string)
+        files.extend(matches)
+    else:
+        sys.exit("Illformed string")
+
+if outputfile is None:
+    print(json.dumps(files))
+else:
+    with open(outputfile, 'w+') as f:
+        json.dump(files, f)

snark-document-builder/src/test/kotlin/DocumentBuilder.kt

@@ -0,0 +1,12 @@
+package documentBuilder
+
+import org.junit.jupiter.api.Test
+import kotlinx.coroutines.runBlocking
+import space.kscience.snark.storage.local.localStorage
+
+class SomeTest {
+    @Test
+    fun justWorks() = runBlocking {
+        // buildDocument(Directory("../example"))
+    }
+}

snark-document-builder/src/test/resources/no_dependencies_example/main.md

@@ -0,0 +1,3 @@
+# Hello
+
+I'm almost empty test document without any dependencies

snark-ktor/build.gradle.kts

@@ -13,7 +13,7 @@ dependencies {
     api("io.ktor:ktor-server-html-builder:$ktorVersion")
     api("io.ktor:ktor-server-host-common:$ktorVersion")
     implementation("io.ktor:ktor-server-netty:2.3.0")
-    implementation(project(mapOf("path" to ":snark-storage-driver")))
+    implementation(project(":snark-storage-driver"))
 
     testApi("io.ktor:ktor-server-tests:$ktorVersion")
 }

snark-ktor/src/main/kotlin/space/kscience/snark/ktor/WebInterface.kt

@@ -13,13 +13,15 @@ import io.ktor.server.routing.*
 import java.nio.file.Path
 import space.kscience.snark.storage.Directory
 import space.kscience.snark.storage.local.localStorage
+import kotlin.io.path.createTempDirectory
+import kotlin.io.path.isDirectory
+import kotlin.io.path.listDirectoryEntries
 import space.kscience.snark.storage.unzip.unzip
 import kotlin.io.createTempFile
 import kotlin.io.path.*
 import kotlin.io.writeBytes
 
 public interface DataHolder {
-
     public fun init(relativePath: String = "/") : Directory
 
     public fun represent(relativePath: String = "/"): String

snark-main/build.gradle.kts

@@ -0,0 +1,21 @@
+plugins {
+    id("space.kscience.gradle.jvm")
+    `maven-publish`
+}
+
+val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
+
+
+dependencies {
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
+    implementation(project(":snark-ktor"))
+    implementation(project(":snark-storage-driver"))
+    implementation(project(":snark-document-builder"))
+
+    testImplementation(kotlin("test"))
+    testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
+}
+
+tasks.test {
+    useJUnitPlatform()
+}

snark-main/src/main/kotlin/space/kscience/snark/main/Main.kt

@@ -0,0 +1,19 @@
+package space.kscience.snark.main
+
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+import space.kscience.snark.ktor.SNARKServer
+import space.kscience.snark.storage.local.localStorage
+import kotlin.io.path.Path
+
+// Entrypoint
+fun main(): Unit = runBlocking {
+    // Parse config, create driver
+    val port = 8080
+    val directory = localStorage(Path("./rundata"))
+    val server = SNARKServer(ServerDataHolder(directory), port)
+    launch {
+        server.run()
+    }
+}

snark-main/src/main/kotlin/space/kscience/snark/main/ServerDataHolder.kt

@@ -0,0 +1,15 @@
+package space.kscience.snark.main
+
+import space.kscience.snark.ktor.DataHolder
+import space.kscience.snark.storage.Directory
+import documentBuilder.*
+import kotlinx.html.HTML
+
+internal class ServerDataHolder(private val directory: Directory): DataHolder {
+    override fun init(): Directory =
+        directory
+
+    override suspend fun represent(): String {
+        return buildDocument(directory).toString()
+    }
+}

snark-main/src/test/kotlin/space/kscience/snark/main/Test.kt

@@ -0,0 +1,13 @@
+package space.kscience.snark.main
+
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.runBlocking
+import org.junit.jupiter.api.Test
+
+class Test {
+    @Test
+    fun justWorks() = runBlocking {
+        delay(5)
+        //main()
+    }
+}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt

@@ -1,35 +1,36 @@
 package space.kscience.snark.storage
 
 import java.nio.file.Path
+import kotlin.io.path.Path
 
 public interface Directory : AutoCloseable {
-    @Deprecated(
-        message = "Use Path, not String",
-        level = DeprecationLevel.WARNING,
-    )
-    public suspend fun get(filename: String): FileReader
     // get file from subtree
     public suspend fun get(filename: Path): FileReader
 
+    @Deprecated("Use put")
     public suspend fun create(filename: String, ignoreIfExists: Boolean = false)
-    @Deprecated(
-        message = "Use Path, not String",
-        level = DeprecationLevel.WARNING,
-    )
-    public suspend fun put(filename: String): FileWriter
+
     // put file to subtree
     public suspend fun put(filename: Path): FileWriter
 
     public suspend fun getSubdir(path: Path): Directory
+
+    @Deprecated("Directories are created on put")
     public suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean = false): Directory
 
-    @Deprecated(
-        message = "Not a good idea",
-        level = DeprecationLevel.WARNING,
-    )
+    @Deprecated("Not a good idea")
     public val path: Path
 }
 
+
+public suspend fun Directory.get(filename: String): FileReader = get(Path(filename))
+
+public suspend fun Directory.put(filename: String): FileWriter = put(Path(filename))
+
+public suspend operator fun Directory.div(path: Path): Directory = getSubdir(path)
+
+public suspend operator fun Directory.div(path: String): Directory = getSubdir(Path(path))
+
 public interface FileReader : AutoCloseable {
     public suspend fun readAll(): ByteArray
 }

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt

@@ -3,9 +3,7 @@ package space.kscience.snark.storage.local
 import space.kscience.snark.storage.Directory
 import space.kscience.snark.storage.FileReader
 import space.kscience.snark.storage.FileWriter
-import java.io.File
 import java.nio.file.Path
-import java.nio.file.attribute.PosixFilePermission
 import kotlin.io.path.*
 
 public fun localStorage(rootPath: Path): Directory {
@@ -16,24 +14,32 @@ internal class LocalFile(private val path: Path) : FileReader, FileWriter {
     override fun close() {}
     override suspend fun readAll(): ByteArray = path.readBytes()
 
-    override suspend fun write(bytes: ByteArray) = path.writeBytes(bytes)
+    override suspend fun write(bytes: ByteArray) {
+        path.parent.createDirectories()
+        try {
+            path.createFile()
+        } catch (ex: Exception) {
+            // Do nothing
+        }
+        path.writeBytes(bytes)
+    }
 }
 
 internal class LocalDirectory(private val root: Path, private val currentDir: Path) : Directory {
-    private fun child(child: String): Path = root / currentDir / child
-    private fun child(child: Path): Path = root / currentDir / child
+    @Deprecated("Use Path, not String")
+    private fun realpath(child: String): Path = root / currentDir / child
+    private fun realpath(child: Path): Path = root / currentDir / child
 
     override fun close() {}
 
-    override suspend fun get(filename: String): LocalFile = LocalFile(child(filename))
-
-    override suspend fun get(filename: Path): LocalFile = LocalFile(child(filename))
+    override suspend fun get(filename: Path): LocalFile = LocalFile(realpath(filename))
 
+    @Deprecated("Use put")
     override suspend fun create(filename: String, ignoreIfExists: Boolean) {
-        val dir = child(filename)
+        val dir = realpath(filename)
         dir.parent.createDirectories()
         try {
-            child(filename).createFile()
+            realpath(filename).createFile()
         } catch (ex: java.nio.file.FileAlreadyExistsException) {
             if (!ignoreIfExists) {
                 throw ex
@@ -41,13 +47,13 @@ internal class LocalDirectory(private val root: Path, private val currentDir: Pa
         }
     }
 
-    override suspend fun put(filename: String): LocalFile = get(filename)
-
     override suspend fun put(filename: Path): LocalFile = get(filename)
 
     override suspend fun getSubdir(path: Path): LocalDirectory = LocalDirectory(root, currentDir / path)
+
+    @Deprecated("Directories are created on put")
     override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): LocalDirectory {
-        val dir = child(dirname)
+        val dir = realpath(dirname)
         dir.parent.createDirectories()
         try {
             dir.createDirectory()
@@ -59,6 +65,7 @@ internal class LocalDirectory(private val root: Path, private val currentDir: Pa
         return LocalDirectory(root, currentDir / dirname)
     }
 
+    @Deprecated("Not a good idea")
     override val path: Path
         get() = currentDir
 }

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt

@@ -5,34 +5,30 @@ import space.kscience.snark.storage.Directory
 import space.kscience.snark.storage.FileReader
 import space.kscience.snark.storage.FileWriter
 import java.nio.file.Path
-import kotlin.io.path.*
+import kotlin.io.path.div
 
 internal class S3Directory(
     private val client: S3Client,
     private val bucketName: String,
     private val currentDir: Path,
 ) : Directory {
-    override suspend fun get(filename: String): FileReader =
-        S3FileReader(client, bucketName, currentDir / filename)
-
     override suspend fun get(filename: Path): FileReader =
         S3FileReader(client, bucketName, currentDir / filename)
 
+    @Deprecated("Use put")
     override suspend fun create(filename: String, ignoreIfExists: Boolean) {
         if (!ignoreIfExists) {
             TODO("could not check if file exists")
         }
     }
 
-    override suspend fun put(filename: String): FileWriter =
-        S3FileWriter(client, bucketName, currentDir / filename)
-
     override suspend fun put(filename: Path): FileWriter =
         S3FileWriter(client, bucketName, currentDir / filename)
 
     override suspend fun getSubdir(path: Path): S3Directory =
         S3Directory(client, bucketName, currentDir / path)
 
+    @Deprecated("Directories are created on put")
     override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): S3Directory =
         if (!ignoreIfExists) {
             TODO("could not check if directory exists")
@@ -40,6 +36,7 @@ internal class S3Directory(
             S3Directory(client, bucketName, currentDir / dirname)
         }
 
+    @Deprecated("Not a good idea")
     override val path: Path
         get() = currentDir
 

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt

@@ -8,11 +8,11 @@ import aws.smithy.kotlin.runtime.content.toByteArray
 import space.kscience.snark.storage.FileReader
 import space.kscience.snark.storage.FileWriter
 import java.nio.file.Path
-import kotlin.io.path.*
 
-internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) : FileReader {
+internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) :
+    FileReader {
     override suspend fun readAll(): ByteArray {
-        val result = client.getObject(GetObjectRequest{
+        val result = client.getObject(GetObjectRequest {
             bucket = bucketName
             key = path.toString()
         }) {
@@ -25,7 +25,8 @@ internal class S3FileReader(private val client: S3Client, private val bucketName
     }
 }
 
-internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) : FileWriter {
+internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) :
+    FileWriter {
     override suspend fun write(bytes: ByteArray) {
         client.putObject {
             bucket = bucketName

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt

@@ -1,12 +1,13 @@
 package space.kscience.snark.storage.s3
 
-import aws.sdk.kotlin.services.s3.*
+import aws.sdk.kotlin.services.s3.S3Client
+import aws.sdk.kotlin.services.s3.createBucket
+import aws.sdk.kotlin.services.s3.headBucket
 import space.kscience.snark.storage.Directory
 import space.kscience.snark.storage.FileReader
 import space.kscience.snark.storage.FileWriter
-import java.lang.Exception
 import java.nio.file.Path
-import kotlin.io.path.*
+import kotlin.io.path.Path
 
 public fun s3Storage(client: S3Client): Directory =
     S3Root(client)
@@ -21,22 +22,16 @@ internal fun splitPathIntoBucketAndPath(path: Path): Pair<String, Path> {
 }
 
 internal class S3Root(private val client: S3Client) : Directory {
-    override suspend fun get(filename: String): FileReader {
-        throw NoSuchFileException(Path(filename).toFile())
-    }
 
     override suspend fun get(filename: Path): FileReader {
         throw NoSuchFileException(filename.toFile())
     }
 
+    @Deprecated("Use put")
     override suspend fun create(filename: String, ignoreIfExists: Boolean) {
         throw NoSuchFileException(Path(filename).toFile())
     }
 
-    override suspend fun put(filename: String): FileWriter {
-        throw NoSuchFileException(Path(filename).toFile())
-    }
-
     override suspend fun put(filename: Path): FileWriter {
         throw NoSuchFileException(filename.toFile())
     }
@@ -51,6 +46,7 @@ internal class S3Root(private val client: S3Client) : Directory {
         throw AccessDeniedException(path.toFile(), reason = ex.message)
     }
 
+    @Deprecated("Directories are created on put")
     override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try {
         val (bucketName, filePath) = splitPathIntoBucketAndPath(Path(dirname))
         client.createBucket {
@@ -61,6 +57,7 @@ internal class S3Root(private val client: S3Client) : Directory {
         throw AccessDeniedException(Path(dirname).toFile(), reason = ex.message)
     }
 
+    @Deprecated("Not a good idea")
     override val path: Path
         get() = Path("")
 

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt

@@ -1,6 +1,6 @@
 package space.kscience.snark.storage.unzip
 
-import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.*
 import java.io.FileInputStream
 import java.util.zip.ZipInputStream
 

snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Example.kt

@@ -0,0 +1,47 @@
+package space.kscience.snark.storage.local
+
+import kotlinx.coroutines.runBlocking
+import org.junit.jupiter.api.BeforeAll
+import space.kscience.snark.storage.*
+import java.nio.file.Path
+import kotlin.io.path.createTempDirectory
+import kotlin.io.path.deleteExisting
+import kotlin.test.AfterTest
+import kotlin.test.BeforeTest
+import kotlin.test.Test
+import kotlin.io.path.*
+import kotlin.test.assertEquals
+
+class Example {
+    var tempDir: Path? = null
+    var somedir: Directory? = null
+
+    @BeforeTest
+    fun setUp() {
+        tempDir = createTempDirectory()
+        somedir = localStorage(tempDir!!)
+    }
+
+    @AfterTest
+    fun tearDown() {
+        tempDir!!.toFile().deleteRecursively()
+        somedir = null
+    }
+
+    @Test
+    fun exampleTest() = runBlocking {
+        somedir!!.put("somefile").write("hello".toByteArray())
+        assertEquals("hello", somedir!!.get("somefile").readAll().decodeToString())
+    }
+
+    @Test
+    fun subdirExample() = runBlocking {
+        val dir1 = somedir!! / "tmp1"
+        dir1.put("somefile").write("hello".toByteArray())
+
+        val dir2 = somedir!! / "tmp1"
+        val data = dir2.get("somefile").readAll()
+
+        assertEquals("hello", data.decodeToString())
+    }
+}

snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Tests.kt

@@ -2,7 +2,7 @@ package space.kscience.snark.storage.local
 
 import kotlinx.coroutines.runBlocking
 
-import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.*
 import java.io.File
 import java.nio.file.Path
 import kotlin.io.path.*

snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/unzip/Tests.kt

@@ -2,8 +2,7 @@ package space.kscience.snark.storage.unzip
 
 import kotlinx.coroutines.runBlocking
 
-import space.kscience.snark.storage.Directory
-import space.kscience.snark.storage.local.LocalDirectory
+import space.kscience.snark.storage.*
 import space.kscience.snark.storage.local.localStorage
 import java.io.*
 import java.nio.file.Files