Merge SNARK-MR-6: SNRK-48: Implement-s3-driver

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt

@@ -0,0 +1,39 @@
+package space.kscience.snark.storage.s3
+
+import aws.sdk.kotlin.services.s3.S3Client
+import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.FileReader
+import space.kscience.snark.storage.FileWriter
+import java.nio.file.Path
+import kotlin.io.path.*
+
+internal class S3Directory(
+    private val client: S3Client,
+    private val bucketName: String,
+    private val currentDir: Path,
+) : Directory {
+    override suspend fun get(filename: String): FileReader =
+        S3FileReader(client, bucketName, currentDir / filename)
+
+    override suspend fun create(filename: String, ignoreIfExists: Boolean) {
+        if (!ignoreIfExists) {
+            TODO("could not check if file exists")
+        }
+    }
+
+    override suspend fun put(filename: String): FileWriter =
+        S3FileWriter(client, bucketName, currentDir / filename)
+
+    override suspend fun getSubdir(path: Path): S3Directory =
+        S3Directory(client, bucketName, currentDir / path)
+
+    override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): S3Directory =
+        if (!ignoreIfExists) {
+            TODO("could not check if directory exists")
+        } else {
+            S3Directory(client, bucketName, currentDir / dirname)
+        }
+
+    override fun close() {
+    }
+}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt

@@ -0,0 +1,40 @@
+package space.kscience.snark.storage.s3
+
+import aws.sdk.kotlin.services.s3.S3Client
+import aws.sdk.kotlin.services.s3.model.GetObjectRequest
+import aws.sdk.kotlin.services.s3.putObject
+import aws.smithy.kotlin.runtime.content.ByteStream
+import aws.smithy.kotlin.runtime.content.toByteArray
+import space.kscience.snark.storage.FileReader
+import space.kscience.snark.storage.FileWriter
+import java.nio.file.Path
+import kotlin.io.path.*
+
+internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) : FileReader {
+    override suspend fun readAll(): ByteArray {
+        val result = client.getObject(GetObjectRequest{
+            bucket = bucketName
+            key = path.toString()
+        }) {
+            it.body?.toByteArray() ?: ByteArray(0)
+        }
+        return result
+    }
+
+    override fun close() {
+    }
+}
+
+internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) : FileWriter {
+    override suspend fun write(bytes: ByteArray) {
+        client.putObject {
+            bucket = bucketName
+            key = path.toString()
+            body = ByteStream.fromBytes(bytes)
+        }
+    }
+
+    override fun close() {
+    }
+
+}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt

@@ -0,0 +1,59 @@
+package space.kscience.snark.storage.s3
+
+import aws.sdk.kotlin.services.s3.*
+import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.FileReader
+import space.kscience.snark.storage.FileWriter
+import java.lang.Exception
+import java.nio.file.Path
+import kotlin.io.path.*
+
+public fun s3Storage(client: S3Client): Directory =
+    S3Root(client)
+
+public fun s3Bucket(client: S3Client, bucket: String): Directory =
+    S3Directory(client, bucket, Path(""))
+
+internal fun splitPathIntoBucketAndPath(path: Path): Pair<String, Path> {
+    val bucket = path.getName(0)
+    val filePath = path.relativize(bucket)
+    return Pair(bucket.toString(), filePath)
+}
+
+internal class S3Root(private val client: S3Client) : Directory {
+    override suspend fun get(filename: String): FileReader {
+        throw NoSuchFileException(Path(filename).toFile())
+    }
+
+    override suspend fun create(filename: String, ignoreIfExists: Boolean) {
+        throw NoSuchFileException(Path(filename).toFile())
+    }
+
+    override suspend fun put(filename: String): FileWriter {
+        throw NoSuchFileException(Path(filename).toFile())
+    }
+
+    override suspend fun getSubdir(path: Path): Directory = try {
+        val (bucketName, filePath) = splitPathIntoBucketAndPath(path)
+        client.headBucket {
+            bucket = bucketName
+        }
+        S3Directory(client, bucketName, filePath)
+    } catch (ex: Exception) {
+        throw AccessDeniedException(path.toFile(), reason = ex.message)
+    }
+
+    override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try {
+        val (bucketName, filePath) = splitPathIntoBucketAndPath(Path(dirname))
+        client.createBucket {
+            bucket = bucketName
+        }
+        S3Directory(client, bucketName, filePath)
+    } catch (ex: Exception) {
+        throw AccessDeniedException(Path(dirname).toFile(), reason = ex.message)
+    }
+
+    override fun close() {
+    }
+
+}