SNRK-98: merged main into SNRK-87

This commit is contained in:
Anton Belyi 2023-05-17 14:10:44 +03:00
commit e609de95cb
34 changed files with 4806 additions and 55 deletions

20
.dockerignore Normal file
View File

@ -0,0 +1,20 @@
# .gitignore contents
.gradle/
build/
.idea/
/logs/
!gradle/wrapper/gradle-wrapper.jar
kotlin-js-store
*.iml
# addition to .gitignore
README.md
LICENSE
gradlew.bat
.dockerignore
.gitignore
Dockerfile

8
Dockerfile Normal file
View File

@ -0,0 +1,8 @@
FROM ubuntu:latest
WORKDIR Snark
COPY . .
RUN ./requirments.sh
RUN ./gradlew build test

27
requirments.sh Executable file
View File

@ -0,0 +1,27 @@
set -e
apt-get update
apt-get install -y sudo
sudo apt-get install -y kotlin
for dir in ./*/
do
if [[ $dir == *'snark'* ]]
then
cd "$dir"
if [[ $(find -type d -name "ci") ]]
then
cd ci
./requirments.sh
cd ..
fi
cd ..
fi
done

View File

@ -43,4 +43,6 @@ include(
":snark-html",
":snark-ktor",
":snark-storage-driver",
":snark-document-builder",
":snark-main",
)

View File

@ -0,0 +1,19 @@
plugins {
id("space.kscience.gradle.jvm")
`maven-publish`
id("kotlinx-serialization")
}
val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
val jacksonVersion = "2.14.2"
val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
dependencies {
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
api("io.ktor:ktor-server-html-builder:$ktorVersion")
implementation(project(":snark-storage-driver"))
implementation("com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion")
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.3")
}

View File

@ -0,0 +1,6 @@
sudo apt-get install -y python3
sudo apt-get install -y nodejs
sudo apt-get install -y npm
cd ../src/main/nodejs
sudo npm install .

View File

@ -0,0 +1,17 @@
package documentBuilder
public class GraphManager(public val graph: DependencyGraph) {
fun buildDocument(file: FileName) {
val list = graph.nodes[file]
if (list != null) {
for (element in list.dependencies) {
element.visit(this)
}
}
}
fun getAstRootDocument(file: FileName): MdAstRoot {
buildDocument(file)
return graph.nodes[file]!!.mdAst
}
}

View File

@ -0,0 +1,71 @@
package documentBuilder
import kotlinx.coroutines.coroutineScope
import kotlin.collections.MutableList
public typealias FileName = String
/**
* Node of dependency graph.
*
* One node represents one file and its dependencies
*
* @property mdAst - AST tree of current file.
* @property dependencies - list of tail end adjacent to this node (dependencies of current file to be resolved).
*/
public data class DependencyGraphNode(
val mdAst: MdAstRoot,
val dependencies: List<DependencyGraphEdge>
)
/**
* Interface of all dependency edges.
*/
public sealed interface DependencyGraphEdge {
public fun visit(graphManager: GraphManager)
}
/**
* Include dependency edge.
*
* @property parentNode - node inside AST tree, that is parent for dependent node.
* @property dependentNode - dependent node, i.e. node of part of document with include commands
* @property includeList - list of files to be included.
*/
public data class IncludeDependency(
val parentNode: MdAstParent,
val dependentNode: MdAstElement,
val includeList: List<FileName>
) : DependencyGraphEdge {
override fun visit(graphManager: GraphManager) {
val parent = parentNode
val childs: MutableList<MdAstElement> = mutableListOf()
for (file in includeList) {
graphManager.buildDocument(file)
childs.addAll(graphManager.graph.nodes[file]!!.mdAst.children)
}
val elements: MutableList<MdAstElement> = parent.children.toMutableList()
val index = parent.children.indexOf(dependentNode)
elements.removeAt(index)
elements.addAll(index, childs)
parent.children = elements
}
}
// parent - List<MdAstElement> -------------------------------------
// | \
// \ \
// | \
// dependentNode - MdAstElement \
// |
// List<FileName> -> List<MdAstRoot> --> List<List<MdAstElement>> ===> List<MdAstElement>
/**
* Whole dependency graph.
*
* @property nodes - map of nodes, where you can find DependencyGraphNode of file by its name.
*/
public data class DependencyGraph(
val nodes: Map<FileName, DependencyGraphNode>
)

View File

@ -0,0 +1,67 @@
package documentBuilder
import com.fasterxml.jackson.core.io.BigDecimalParser
import space.kscience.snark.storage.*
import java.nio.file.Path
import java.nio.file.Paths
import kotlinx.html.*
import kotlinx.html.dom.createHTMLDocument
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import kotlinx.serialization.json.Json
private val SNARK_HTML_RENDER = "snark-document-builder/src/main/nodejs/HtmlRenderer.js"
fun getHtml(ast_string: String): String
{
return ProcessBuilder("node", SNARK_HTML_RENDER, ast_string)
.redirectOutput(ProcessBuilder.Redirect.PIPE)
.redirectError(ProcessBuilder.Redirect.INHERIT)
.start().inputStream.bufferedReader().readText()
}
private val DEFAULT_DOCUMENT_ROOT = "main.md"
public suspend fun buildDocument(documentDirectory: Directory): String {
val dependencyGraph = buildDependencyGraph(documentDirectory)
val root: MdAstRoot = dependencyGraph.nodes[""]!!.mdAst
return getHtml(jacksonObjectMapper().writeValueAsString(root))
// return jacksonObjectMapper().writeValueAsString(root)
}
public suspend fun buildDependencyGraph(root: Directory): DependencyGraph {
val nodes = HashMap<FileName, DependencyGraphNode>()
buildNodes(root, nodes)
return DependencyGraph(nodes)
}
private suspend fun buildNodes(folder: Directory, nodes: HashMap<FileName, DependencyGraphNode>) {
val pathString = folder.path.toString()
assert(!nodes.containsKey(pathString))
val rootDcoument = folder.get(DEFAULT_DOCUMENT_ROOT)
nodes.put(pathString, buildDependencyGraphNode(rootDcoument.readAll(), folder.path))
val dependencies = getDependencies(nodes.getValue(pathString))
for (dependency in dependencies) {
if (!nodes.containsKey(dependency))
buildNodes(folder.getSubdir(Paths.get(dependency)), nodes)
}
}
public suspend fun getDependencies(node: DependencyGraphNode): Set<FileName> {
val dependencies = mutableListOf<FileName>()
for (dependency in node.dependencies) {
when (dependency) {
is IncludeDependency -> dependencies.addAll(dependency.includeList)
}
}
return dependencies.toSet()
}

View File

@ -0,0 +1,84 @@
package documentBuilder
import kotlinx.serialization.Serializable
import kotlinx.serialization.SerialName
import com.fasterxml.jackson.annotation.JsonSubTypes
import com.fasterxml.jackson.annotation.JsonTypeInfo
@Serializable
public data class Point(val line: Int, val column: Int, val offset: Int)
@Serializable
public data class Position(val start: Point, val end: Point)
@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include = JsonTypeInfo.As.PROPERTY,
property = "type"
)
@JsonSubTypes(
JsonSubTypes.Type(value = MdAstRoot::class, name = "root"),
JsonSubTypes.Type(value = MdAstParagraph::class, name = "paragraph"),
JsonSubTypes.Type(value = MdAstText::class, name = "text"),
JsonSubTypes.Type(value = MdAstHeading::class, name = "heading"),
JsonSubTypes.Type(value = MdAstCode::class, name = "code"),
JsonSubTypes.Type(value = MdAstBlockquote::class, name = "blockquote")
)
@Serializable
public sealed interface MdAstElement{
public abstract var position: Position
}
@Serializable
public sealed interface MdAstParent: MdAstElement{
public var children: List<MdAstElement>
}
@Serializable
@SerialName("root")
public data class MdAstRoot(
override var children: List<MdAstElement>,
override var position: Position
): MdAstParent
@Serializable
@SerialName("paragraph")
public data class MdAstParagraph(
override var children: List<MdAstElement>,
override var position: Position
): MdAstParent
@Serializable
@SerialName("text")
public data class MdAstText(
val value: String,
override var position: Position
): MdAstElement
@Serializable
@SerialName("heading")
public data class MdAstHeading(
val depth: Int,
override var children: List<MdAstElement>,
override var position: Position
): MdAstParent
@Serializable
@SerialName("code")
public data class MdAstCode(
var lang: String? = null,
var meta: String? = null,
var value: String,
override var position: Position,
) : MdAstElement
@Serializable
@SerialName("blockquote")
public data class MdAstBlockquote(
override var children: List<MdAstElement>,
override var position: Position
): MdAstParent

View File

@ -0,0 +1,57 @@
package documentBuilder
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import java.nio.file.Path
private val MARKDOWN_PARSER = "snark-document-builder/src/main/nodejs/MarkdownParser.js"
private val SNARK_PARSER = "snark-document-builder/src/main/python/SnarkParser.py"
public suspend fun parseMd(mdFile: ByteArray): MdAstRoot {
return jacksonObjectMapper()
.readValue<MdAstRoot>(ProcessBuilder("node", MARKDOWN_PARSER, mdFile.toString())
.redirectOutput(ProcessBuilder.Redirect.PIPE)
.redirectError(ProcessBuilder.Redirect.INHERIT)
.start().inputStream.bufferedReader().readText())
}
public suspend fun buildDependencyGraphNode(mdFile: ByteArray, path: Path): DependencyGraphNode {
val treeRoot = parseMd(mdFile)
val dependencies = mutableListOf<DependencyGraphEdge>()
fillDependencies(treeRoot, dependencies, path)
return DependencyGraphNode(treeRoot, dependencies)
}
internal suspend fun fillDependencies(
currentNode: MdAstElement,
dependencies: MutableList<DependencyGraphEdge>,
path: Path) {
when (currentNode) {
is MdAstParent -> {
for (child in currentNode.children) {
if (child is MdAstText) {
val includeList = getIncludeFiles(child.value).toMutableList()
if (includeList.size > 0) {
includeList.replaceAll { path.toString() + "/" + it }
dependencies += IncludeDependency(currentNode, child, includeList)
}
} else {
fillDependencies(child, dependencies, path)
}
}
}
else -> {}
}
}
public suspend fun getIncludeFiles(string: String): List<FileName> {
return jacksonObjectMapper()
.readValue<List<FileName>>(ProcessBuilder("python3", SNARK_PARSER, string)
.redirectOutput(ProcessBuilder.Redirect.PIPE)
.redirectError(ProcessBuilder.Redirect.INHERIT)
.start().inputStream.bufferedReader().readText())
}

View File

@ -0,0 +1 @@
node_modules

View File

@ -0,0 +1,16 @@
import {toHast} from 'mdast-util-to-hast'
import {toHtml} from 'hast-util-to-html'
main()
function main()
{
if (process.argv.length < 3)
throw "No input"
const md_ast = JSON.parse(process.argv[2])
const hast = toHast(md_ast)
const html = toHtml(hast)
console.log(html)
}

View File

@ -0,0 +1,14 @@
import {fromMarkdown} from 'mdast-util-from-markdown'
main()
function main()
{
if (process.argv.length < 3)
throw "No input"
const markdown_string = process.argv[2]
const mdast = fromMarkdown(markdown_string)
console.log(JSON.stringify(mdast))
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,30 @@
{
"type": "module",
"dependencies": {
"fs": "^0.0.1-security",
"hast-util-to-html": "^8.0.4",
"mdast-util-to-hast": "^12.3.0",
"node-fetch": "^3.3.1",
"remark-html": "^15.0.2",
"remark-parse": "^10.0.1",
"require": "^2.4.20",
"to-vfile": "^7.2.4",
"unified": "^10.1.2",
"whatwg-fetch": "^3.6.2"
}
}

View File

@ -0,0 +1,26 @@
import sys
import re
import json
assert(len(sys.argv) >= 2)
string = sys.argv[1]
outputfile = sys.argv[2] if len(sys.argv) >= 3 else None
pattern = r'^([\n|\t| ]*@include\([a-z|0-9|.|_]*.md\)[\n|\t| ]*)*$'
files = []
if re.search("@include", string, re.IGNORECASE):
if re.match(pattern, string):
matches = re.findall(r'@include\((.*?)\)', string)
files.extend(matches)
else:
sys.exit("Illformed string")
if outputfile is None:
print(json.dumps(files))
else:
with open(outputfile, 'w+') as f:
json.dump(files, f)

View File

@ -0,0 +1,12 @@
package documentBuilder
import org.junit.jupiter.api.Test
import kotlinx.coroutines.runBlocking
import space.kscience.snark.storage.local.localStorage
class SomeTest {
@Test
fun justWorks() = runBlocking {
// buildDocument(Directory("../example"))
}
}

View File

@ -0,0 +1,3 @@
# Hello
I'm almost empty test document without any dependencies

View File

@ -13,7 +13,7 @@ dependencies {
api("io.ktor:ktor-server-html-builder:$ktorVersion")
api("io.ktor:ktor-server-host-common:$ktorVersion")
implementation("io.ktor:ktor-server-netty:2.3.0")
implementation(project(mapOf("path" to ":snark-storage-driver")))
implementation(project(":snark-storage-driver"))
testApi("io.ktor:ktor-server-tests:$ktorVersion")
}

View File

@ -13,13 +13,15 @@ import io.ktor.server.routing.*
import java.nio.file.Path
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.local.localStorage
import kotlin.io.path.createTempDirectory
import kotlin.io.path.isDirectory
import kotlin.io.path.listDirectoryEntries
import space.kscience.snark.storage.unzip.unzip
import kotlin.io.createTempFile
import kotlin.io.path.*
import kotlin.io.writeBytes
public interface DataHolder {
public fun init(relativePath: String = "/") : Directory
public fun represent(relativePath: String = "/"): String
@ -147,4 +149,4 @@ public class SNARKServer(private val dataHolder: DataHolder, private val port: I
}
}.start(wait = true)
}
}
}

View File

@ -0,0 +1,21 @@
plugins {
id("space.kscience.gradle.jvm")
`maven-publish`
}
val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
dependencies {
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
implementation(project(":snark-ktor"))
implementation(project(":snark-storage-driver"))
implementation(project(":snark-document-builder"))
testImplementation(kotlin("test"))
testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
}
tasks.test {
useJUnitPlatform()
}

View File

@ -0,0 +1,19 @@
package space.kscience.snark.main
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import space.kscience.snark.ktor.SNARKServer
import space.kscience.snark.storage.local.localStorage
import kotlin.io.path.Path
// Entrypoint
fun main(): Unit = runBlocking {
// Parse config, create driver
val port = 8080
val directory = localStorage(Path("./rundata"))
val server = SNARKServer(ServerDataHolder(directory), port)
launch {
server.run()
}
}

View File

@ -0,0 +1,15 @@
package space.kscience.snark.main
import space.kscience.snark.ktor.DataHolder
import space.kscience.snark.storage.Directory
import documentBuilder.*
import kotlinx.html.HTML
internal class ServerDataHolder(private val directory: Directory): DataHolder {
override fun init(): Directory =
directory
override suspend fun represent(): String {
return buildDocument(directory).toString()
}
}

View File

@ -0,0 +1,13 @@
package space.kscience.snark.main
import kotlinx.coroutines.delay
import kotlinx.coroutines.runBlocking
import org.junit.jupiter.api.Test
class Test {
@Test
fun justWorks() = runBlocking {
delay(5)
//main()
}
}

View File

@ -1,35 +1,36 @@
package space.kscience.snark.storage
import java.nio.file.Path
import kotlin.io.path.Path
public interface Directory : AutoCloseable {
@Deprecated(
message = "Use Path, not String",
level = DeprecationLevel.WARNING,
)
public suspend fun get(filename: String): FileReader
// get file from subtree
public suspend fun get(filename: Path): FileReader
@Deprecated("Use put")
public suspend fun create(filename: String, ignoreIfExists: Boolean = false)
@Deprecated(
message = "Use Path, not String",
level = DeprecationLevel.WARNING,
)
public suspend fun put(filename: String): FileWriter
// put file to subtree
public suspend fun put(filename: Path): FileWriter
public suspend fun getSubdir(path: Path): Directory
@Deprecated("Directories are created on put")
public suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean = false): Directory
@Deprecated(
message = "Not a good idea",
level = DeprecationLevel.WARNING,
)
@Deprecated("Not a good idea")
public val path: Path
}
public suspend fun Directory.get(filename: String): FileReader = get(Path(filename))
public suspend fun Directory.put(filename: String): FileWriter = put(Path(filename))
public suspend operator fun Directory.div(path: Path): Directory = getSubdir(path)
public suspend operator fun Directory.div(path: String): Directory = getSubdir(Path(path))
public interface FileReader : AutoCloseable {
public suspend fun readAll(): ByteArray
}

View File

@ -3,9 +3,7 @@ package space.kscience.snark.storage.local
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.io.File
import java.nio.file.Path
import java.nio.file.attribute.PosixFilePermission
import kotlin.io.path.*
public fun localStorage(rootPath: Path): Directory {
@ -16,24 +14,32 @@ internal class LocalFile(private val path: Path) : FileReader, FileWriter {
override fun close() {}
override suspend fun readAll(): ByteArray = path.readBytes()
override suspend fun write(bytes: ByteArray) = path.writeBytes(bytes)
override suspend fun write(bytes: ByteArray) {
path.parent.createDirectories()
try {
path.createFile()
} catch (ex: Exception) {
// Do nothing
}
path.writeBytes(bytes)
}
}
internal class LocalDirectory(private val root: Path, private val currentDir: Path) : Directory {
private fun child(child: String): Path = root / currentDir / child
private fun child(child: Path): Path = root / currentDir / child
@Deprecated("Use Path, not String")
private fun realpath(child: String): Path = root / currentDir / child
private fun realpath(child: Path): Path = root / currentDir / child
override fun close() {}
override suspend fun get(filename: String): LocalFile = LocalFile(child(filename))
override suspend fun get(filename: Path): LocalFile = LocalFile(child(filename))
override suspend fun get(filename: Path): LocalFile = LocalFile(realpath(filename))
@Deprecated("Use put")
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
val dir = child(filename)
val dir = realpath(filename)
dir.parent.createDirectories()
try {
child(filename).createFile()
realpath(filename).createFile()
} catch (ex: java.nio.file.FileAlreadyExistsException) {
if (!ignoreIfExists) {
throw ex
@ -41,13 +47,13 @@ internal class LocalDirectory(private val root: Path, private val currentDir: Pa
}
}
override suspend fun put(filename: String): LocalFile = get(filename)
override suspend fun put(filename: Path): LocalFile = get(filename)
override suspend fun getSubdir(path: Path): LocalDirectory = LocalDirectory(root, currentDir / path)
@Deprecated("Directories are created on put")
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): LocalDirectory {
val dir = child(dirname)
val dir = realpath(dirname)
dir.parent.createDirectories()
try {
dir.createDirectory()
@ -59,6 +65,7 @@ internal class LocalDirectory(private val root: Path, private val currentDir: Pa
return LocalDirectory(root, currentDir / dirname)
}
@Deprecated("Not a good idea")
override val path: Path
get() = currentDir
}

View File

@ -5,34 +5,30 @@ import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.nio.file.Path
import kotlin.io.path.*
import kotlin.io.path.div
internal class S3Directory(
private val client: S3Client,
private val bucketName: String,
private val currentDir: Path,
) : Directory {
override suspend fun get(filename: String): FileReader =
S3FileReader(client, bucketName, currentDir / filename)
override suspend fun get(filename: Path): FileReader =
S3FileReader(client, bucketName, currentDir / filename)
@Deprecated("Use put")
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
if (!ignoreIfExists) {
TODO("could not check if file exists")
}
}
override suspend fun put(filename: String): FileWriter =
S3FileWriter(client, bucketName, currentDir / filename)
override suspend fun put(filename: Path): FileWriter =
S3FileWriter(client, bucketName, currentDir / filename)
override suspend fun getSubdir(path: Path): S3Directory =
S3Directory(client, bucketName, currentDir / path)
@Deprecated("Directories are created on put")
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): S3Directory =
if (!ignoreIfExists) {
TODO("could not check if directory exists")
@ -40,6 +36,7 @@ internal class S3Directory(
S3Directory(client, bucketName, currentDir / dirname)
}
@Deprecated("Not a good idea")
override val path: Path
get() = currentDir

View File

@ -8,11 +8,11 @@ import aws.smithy.kotlin.runtime.content.toByteArray
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.nio.file.Path
import kotlin.io.path.*
internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) : FileReader {
internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) :
FileReader {
override suspend fun readAll(): ByteArray {
val result = client.getObject(GetObjectRequest{
val result = client.getObject(GetObjectRequest {
bucket = bucketName
key = path.toString()
}) {
@ -25,7 +25,8 @@ internal class S3FileReader(private val client: S3Client, private val bucketName
}
}
internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) : FileWriter {
internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) :
FileWriter {
override suspend fun write(bytes: ByteArray) {
client.putObject {
bucket = bucketName

View File

@ -1,12 +1,13 @@
package space.kscience.snark.storage.s3
import aws.sdk.kotlin.services.s3.*
import aws.sdk.kotlin.services.s3.S3Client
import aws.sdk.kotlin.services.s3.createBucket
import aws.sdk.kotlin.services.s3.headBucket
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.lang.Exception
import java.nio.file.Path
import kotlin.io.path.*
import kotlin.io.path.Path
public fun s3Storage(client: S3Client): Directory =
S3Root(client)
@ -21,22 +22,16 @@ internal fun splitPathIntoBucketAndPath(path: Path): Pair<String, Path> {
}
internal class S3Root(private val client: S3Client) : Directory {
override suspend fun get(filename: String): FileReader {
throw NoSuchFileException(Path(filename).toFile())
}
override suspend fun get(filename: Path): FileReader {
throw NoSuchFileException(filename.toFile())
}
@Deprecated("Use put")
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
throw NoSuchFileException(Path(filename).toFile())
}
override suspend fun put(filename: String): FileWriter {
throw NoSuchFileException(Path(filename).toFile())
}
override suspend fun put(filename: Path): FileWriter {
throw NoSuchFileException(filename.toFile())
}
@ -51,6 +46,7 @@ internal class S3Root(private val client: S3Client) : Directory {
throw AccessDeniedException(path.toFile(), reason = ex.message)
}
@Deprecated("Directories are created on put")
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try {
val (bucketName, filePath) = splitPathIntoBucketAndPath(Path(dirname))
client.createBucket {
@ -61,6 +57,7 @@ internal class S3Root(private val client: S3Client) : Directory {
throw AccessDeniedException(Path(dirname).toFile(), reason = ex.message)
}
@Deprecated("Not a good idea")
override val path: Path
get() = Path("")

View File

@ -1,6 +1,6 @@
package space.kscience.snark.storage.unzip
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.*
import java.io.FileInputStream
import java.util.zip.ZipInputStream

View File

@ -0,0 +1,47 @@
package space.kscience.snark.storage.local
import kotlinx.coroutines.runBlocking
import org.junit.jupiter.api.BeforeAll
import space.kscience.snark.storage.*
import java.nio.file.Path
import kotlin.io.path.createTempDirectory
import kotlin.io.path.deleteExisting
import kotlin.test.AfterTest
import kotlin.test.BeforeTest
import kotlin.test.Test
import kotlin.io.path.*
import kotlin.test.assertEquals
class Example {
var tempDir: Path? = null
var somedir: Directory? = null
@BeforeTest
fun setUp() {
tempDir = createTempDirectory()
somedir = localStorage(tempDir!!)
}
@AfterTest
fun tearDown() {
tempDir!!.toFile().deleteRecursively()
somedir = null
}
@Test
fun exampleTest() = runBlocking {
somedir!!.put("somefile").write("hello".toByteArray())
assertEquals("hello", somedir!!.get("somefile").readAll().decodeToString())
}
@Test
fun subdirExample() = runBlocking {
val dir1 = somedir!! / "tmp1"
dir1.put("somefile").write("hello".toByteArray())
val dir2 = somedir!! / "tmp1"
val data = dir2.get("somefile").readAll()
assertEquals("hello", data.decodeToString())
}
}

View File

@ -2,7 +2,7 @@ package space.kscience.snark.storage.local
import kotlinx.coroutines.runBlocking
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.*
import java.io.File
import java.nio.file.Path
import kotlin.io.path.*

View File

@ -2,8 +2,7 @@ package space.kscience.snark.storage.unzip
import kotlinx.coroutines.runBlocking
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.local.LocalDirectory
import space.kscience.snark.storage.*
import space.kscience.snark.storage.local.localStorage
import java.io.*
import java.nio.file.Files