Merge branch 'main' into feature/SNRK-70/dependecy-graph

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt

@@ -0,0 +1,52 @@
+package space.kscience.snark.storage.local
+
+import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.FileReader
+import space.kscience.snark.storage.FileWriter
+import java.nio.file.Path
+import kotlin.io.path.*
+
+public fun localStorage(rootPath: Path): Directory {
+    return LocalDirectory(rootPath)
+}
+
+private class LocalFile(private val path: Path) : FileReader, FileWriter {
+    override fun close() {}
+    override suspend fun readAll(): ByteArray = path.readBytes()
+
+    override suspend fun write(bytes: ByteArray) = path.writeBytes(bytes)
+}
+
+private class LocalDirectory(private val path: Path) : Directory {
+    private fun child(child: String): Path = path / child
+    private fun child(child: Path): Path = path / child
+
+    override fun close() {}
+
+    override suspend fun get(filename: String): FileReader = LocalFile(child(filename))
+
+    override suspend fun create(filename: String, ignoreIfExists: Boolean) {
+        try {
+            child(filename).createFile()
+        } catch (ex: FileAlreadyExistsException) {
+            if (!ignoreIfExists) {
+                throw ex
+            }
+        }
+    }
+
+    override suspend fun put(filename: String): FileWriter = LocalFile(child(filename))
+
+    override suspend fun getSubdir(path: Path): Directory = LocalDirectory(child(path))
+    override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory {
+        val dir = child(dirname)
+        try {
+            dir.createDirectory()
+        } catch (ex: FileAlreadyExistsException) {
+            if (!ignoreIfExists) {
+                throw ex
+            }
+        }
+        return LocalDirectory(dir)
+    }
+}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/localDriver.kt

@@ -1,35 +0,0 @@
-package space.kscience.snark.storage.local
-
-import space.kscience.snark.storage.Directory
-import space.kscience.snark.storage.FileReader
-import space.kscience.snark.storage.FileWriter
-import java.io.File
-import java.nio.file.Path
-
-public class LocalFile(private val path: String) : FileReader, FileWriter {
-    override fun close() {}
-    override suspend fun readAll(): ByteArray = File(this.path).readBytes()
-
-    override suspend fun write(bytes: ByteArray) = File(this.path).writeBytes(bytes)
-}
-
-public class LocalDirectory(private val path: String) : Directory {
-    override fun close() {}
-    override suspend fun get(filename: String): FileReader = LocalFile("${this.path}/$filename")
-
-    override suspend fun create(filename: String, ignoreIfExists: Boolean) {
-        if (!File(filename).createNewFile() && !ignoreIfExists) {
-            throw UnsupportedOperationException("File already exists")
-        }
-    }
-
-    override suspend fun put(filename: String): FileWriter = LocalFile("${this.path}/$filename")
-
-    override suspend fun getSubdir(dirpath: Path): Directory = LocalDirectory("${this.path}/$dirpath")
-    override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory {
-        if (!File(dirname).mkdir() && !ignoreIfExists) {
-            throw UnsupportedOperationException("File already exists")
-        }
-        return this.getSubdir(File(dirname).toPath())
-    }
-}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt

@@ -0,0 +1,39 @@
+package space.kscience.snark.storage.s3
+
+import aws.sdk.kotlin.services.s3.S3Client
+import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.FileReader
+import space.kscience.snark.storage.FileWriter
+import java.nio.file.Path
+import kotlin.io.path.*
+
+internal class S3Directory(
+    private val client: S3Client,
+    private val bucketName: String,
+    private val currentDir: Path,
+) : Directory {
+    override suspend fun get(filename: String): FileReader =
+        S3FileReader(client, bucketName, currentDir / filename)
+
+    override suspend fun create(filename: String, ignoreIfExists: Boolean) {
+        if (!ignoreIfExists) {
+            TODO("could not check if file exists")
+        }
+    }
+
+    override suspend fun put(filename: String): FileWriter =
+        S3FileWriter(client, bucketName, currentDir / filename)
+
+    override suspend fun getSubdir(path: Path): S3Directory =
+        S3Directory(client, bucketName, currentDir / path)
+
+    override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): S3Directory =
+        if (!ignoreIfExists) {
+            TODO("could not check if directory exists")
+        } else {
+            S3Directory(client, bucketName, currentDir / dirname)
+        }
+
+    override fun close() {
+    }
+}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt

@@ -0,0 +1,40 @@
+package space.kscience.snark.storage.s3
+
+import aws.sdk.kotlin.services.s3.S3Client
+import aws.sdk.kotlin.services.s3.model.GetObjectRequest
+import aws.sdk.kotlin.services.s3.putObject
+import aws.smithy.kotlin.runtime.content.ByteStream
+import aws.smithy.kotlin.runtime.content.toByteArray
+import space.kscience.snark.storage.FileReader
+import space.kscience.snark.storage.FileWriter
+import java.nio.file.Path
+import kotlin.io.path.*
+
+internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) : FileReader {
+    override suspend fun readAll(): ByteArray {
+        val result = client.getObject(GetObjectRequest{
+            bucket = bucketName
+            key = path.toString()
+        }) {
+            it.body?.toByteArray() ?: ByteArray(0)
+        }
+        return result
+    }
+
+    override fun close() {
+    }
+}
+
+internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) : FileWriter {
+    override suspend fun write(bytes: ByteArray) {
+        client.putObject {
+            bucket = bucketName
+            key = path.toString()
+            body = ByteStream.fromBytes(bytes)
+        }
+    }
+
+    override fun close() {
+    }
+
+}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt

@@ -0,0 +1,59 @@
+package space.kscience.snark.storage.s3
+
+import aws.sdk.kotlin.services.s3.*
+import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.FileReader
+import space.kscience.snark.storage.FileWriter
+import java.lang.Exception
+import java.nio.file.Path
+import kotlin.io.path.*
+
+public fun s3Storage(client: S3Client): Directory =
+    S3Root(client)
+
+public fun s3Bucket(client: S3Client, bucket: String): Directory =
+    S3Directory(client, bucket, Path(""))
+
+internal fun splitPathIntoBucketAndPath(path: Path): Pair<String, Path> {
+    val bucket = path.getName(0)
+    val filePath = path.relativize(bucket)
+    return Pair(bucket.toString(), filePath)
+}
+
+internal class S3Root(private val client: S3Client) : Directory {
+    override suspend fun get(filename: String): FileReader {
+        throw NoSuchFileException(Path(filename).toFile())
+    }
+
+    override suspend fun create(filename: String, ignoreIfExists: Boolean) {
+        throw NoSuchFileException(Path(filename).toFile())
+    }
+
+    override suspend fun put(filename: String): FileWriter {
+        throw NoSuchFileException(Path(filename).toFile())
+    }
+
+    override suspend fun getSubdir(path: Path): Directory = try {
+        val (bucketName, filePath) = splitPathIntoBucketAndPath(path)
+        client.headBucket {
+            bucket = bucketName
+        }
+        S3Directory(client, bucketName, filePath)
+    } catch (ex: Exception) {
+        throw AccessDeniedException(path.toFile(), reason = ex.message)
+    }
+
+    override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try {
+        val (bucketName, filePath) = splitPathIntoBucketAndPath(Path(dirname))
+        client.createBucket {
+            bucket = bucketName
+        }
+        S3Directory(client, bucketName, filePath)
+    } catch (ex: Exception) {
+        throw AccessDeniedException(Path(dirname).toFile(), reason = ex.message)
+    }
+
+    override fun close() {
+    }
+
+}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt

@@ -0,0 +1,24 @@
+package space.kscience.snark.storage.unzip
+
+import space.kscience.snark.storage.Directory
+import java.io.FileInputStream
+import java.util.zip.ZipInputStream
+
+public suspend fun unzip(source_path: String, target: Directory) {
+    val zis = ZipInputStream(FileInputStream(source_path))
+    var zipEntry = zis.nextEntry
+    while (zipEntry != null) {
+        if (!zipEntry.isDirectory) {
+            val filename = zipEntry.name
+            target.create(filename, true)
+            val fos = target.put(filename)
+            val buffer = ByteArray(zipEntry.size.toInt())
+            zis.read(buffer)
+            fos.write(buffer)
+            fos.close()
+        }
+        zipEntry = zis.nextEntry
+    }
+    zis.closeEntry()
+    zis.close()
+}