Merge branch 'main' into devops/SNRK-77/unit_testing

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt

@@ -3,13 +3,31 @@ package space.kscience.snark.storage
 import java.nio.file.Path
 
 public interface Directory : AutoCloseable {
+    @Deprecated(
+        message = "Use Path, not String",
+        level = DeprecationLevel.WARNING,
+    )
     public suspend fun get(filename: String): FileReader
+    // get file from subtree
+    public suspend fun get(filename: Path): FileReader
 
     public suspend fun create(filename: String, ignoreIfExists: Boolean = false)
+    @Deprecated(
+        message = "Use Path, not String",
+        level = DeprecationLevel.WARNING,
+    )
     public suspend fun put(filename: String): FileWriter
+    // put file to subtree
+    public suspend fun put(filename: Path): FileWriter
 
     public suspend fun getSubdir(path: Path): Directory
     public suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean = false): Directory
+
+    @Deprecated(
+        message = "Not a good idea",
+        level = DeprecationLevel.WARNING,
+    )
+    public val path: Path
 }
 
 public interface FileReader : AutoCloseable {

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt

@@ -9,7 +9,7 @@ import java.nio.file.attribute.PosixFilePermission
 import kotlin.io.path.*
 
 public fun localStorage(rootPath: Path): Directory {
-    return LocalDirectory(rootPath)
+    return LocalDirectory(rootPath, Path(""))
 }
 
 internal class LocalFile(private val path: Path) : FileReader, FileWriter {
@@ -19,14 +19,16 @@ internal class LocalFile(private val path: Path) : FileReader, FileWriter {
     override suspend fun write(bytes: ByteArray) = path.writeBytes(bytes)
 }
 
-internal class LocalDirectory(private val path: Path) : Directory {
-    private fun child(child: String): Path = path / child
-    private fun child(child: Path): Path = path / child
+private class LocalDirectory(private val root: Path, private val currentDir: Path) : Directory {
+    private fun child(child: String): Path = root / currentDir / child
+    private fun child(child: Path): Path = root / currentDir / child
 
     override fun close() {}
 
     override suspend fun get(filename: String): FileReader = LocalFile(child(filename))
 
+    override suspend fun get(filename: Path): FileReader = LocalFile(child(filename))
+
     override suspend fun create(filename: String, ignoreIfExists: Boolean) {
         child(filename).parent.createDirectories()
 
@@ -45,7 +47,10 @@ internal class LocalDirectory(private val path: Path) : Directory {
         return LocalFile(tmp)
     }
 
-    override suspend fun getSubdir(path: Path): LocalDirectory = LocalDirectory(child(path))
+
+    override suspend fun put(filename: Path): FileWriter = LocalFile(child(filename))
+
+    override suspend fun getSubdir(path: Path): LocalDirectory = LocalDirectory(root, child(path))
     override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): LocalDirectory {
         val dir = child(dirname)
         try {
@@ -55,6 +60,9 @@ internal class LocalDirectory(private val path: Path) : Directory {
                 throw ex
             }
         }
-        return LocalDirectory(dir)
+        return LocalDirectory(root, dir)
     }
+
+    override val path: Path
+        get() = currentDir
 }

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt

@@ -0,0 +1,48 @@
+package space.kscience.snark.storage.s3
+
+import aws.sdk.kotlin.services.s3.S3Client
+import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.FileReader
+import space.kscience.snark.storage.FileWriter
+import java.nio.file.Path
+import kotlin.io.path.*
+
+internal class S3Directory(
+    private val client: S3Client,
+    private val bucketName: String,
+    private val currentDir: Path,
+) : Directory {
+    override suspend fun get(filename: String): FileReader =
+        S3FileReader(client, bucketName, currentDir / filename)
+
+    override suspend fun get(filename: Path): FileReader =
+        S3FileReader(client, bucketName, currentDir / filename)
+
+    override suspend fun create(filename: String, ignoreIfExists: Boolean) {
+        if (!ignoreIfExists) {
+            TODO("could not check if file exists")
+        }
+    }
+
+    override suspend fun put(filename: String): FileWriter =
+        S3FileWriter(client, bucketName, currentDir / filename)
+
+    override suspend fun put(filename: Path): FileWriter =
+        S3FileWriter(client, bucketName, currentDir / filename)
+
+    override suspend fun getSubdir(path: Path): S3Directory =
+        S3Directory(client, bucketName, currentDir / path)
+
+    override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): S3Directory =
+        if (!ignoreIfExists) {
+            TODO("could not check if directory exists")
+        } else {
+            S3Directory(client, bucketName, currentDir / dirname)
+        }
+
+    override val path: Path
+        get() = currentDir
+
+    override fun close() {
+    }
+}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt

@@ -0,0 +1,40 @@
+package space.kscience.snark.storage.s3
+
+import aws.sdk.kotlin.services.s3.S3Client
+import aws.sdk.kotlin.services.s3.model.GetObjectRequest
+import aws.sdk.kotlin.services.s3.putObject
+import aws.smithy.kotlin.runtime.content.ByteStream
+import aws.smithy.kotlin.runtime.content.toByteArray
+import space.kscience.snark.storage.FileReader
+import space.kscience.snark.storage.FileWriter
+import java.nio.file.Path
+import kotlin.io.path.*
+
+internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) : FileReader {
+    override suspend fun readAll(): ByteArray {
+        val result = client.getObject(GetObjectRequest{
+            bucket = bucketName
+            key = path.toString()
+        }) {
+            it.body?.toByteArray() ?: ByteArray(0)
+        }
+        return result
+    }
+
+    override fun close() {
+    }
+}
+
+internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) : FileWriter {
+    override suspend fun write(bytes: ByteArray) {
+        client.putObject {
+            bucket = bucketName
+            key = path.toString()
+            body = ByteStream.fromBytes(bytes)
+        }
+    }
+
+    override fun close() {
+    }
+
+}

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt

@@ -0,0 +1,70 @@
+package space.kscience.snark.storage.s3
+
+import aws.sdk.kotlin.services.s3.*
+import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.FileReader
+import space.kscience.snark.storage.FileWriter
+import java.lang.Exception
+import java.nio.file.Path
+import kotlin.io.path.*
+
+public fun s3Storage(client: S3Client): Directory =
+    S3Root(client)
+
+public fun s3Bucket(client: S3Client, bucket: String): Directory =
+    S3Directory(client, bucket, Path(""))
+
+internal fun splitPathIntoBucketAndPath(path: Path): Pair<String, Path> {
+    val bucket = path.getName(0)
+    val filePath = path.relativize(bucket)
+    return Pair(bucket.toString(), filePath)
+}
+
+internal class S3Root(private val client: S3Client) : Directory {
+    override suspend fun get(filename: String): FileReader {
+        throw NoSuchFileException(Path(filename).toFile())
+    }
+
+    override suspend fun get(filename: Path): FileReader {
+        throw NoSuchFileException(filename.toFile())
+    }
+
+    override suspend fun create(filename: String, ignoreIfExists: Boolean) {
+        throw NoSuchFileException(Path(filename).toFile())
+    }
+
+    override suspend fun put(filename: String): FileWriter {
+        throw NoSuchFileException(Path(filename).toFile())
+    }
+
+    override suspend fun put(filename: Path): FileWriter {
+        throw NoSuchFileException(filename.toFile())
+    }
+
+    override suspend fun getSubdir(path: Path): Directory = try {
+        val (bucketName, filePath) = splitPathIntoBucketAndPath(path)
+        client.headBucket {
+            bucket = bucketName
+        }
+        S3Directory(client, bucketName, filePath)
+    } catch (ex: Exception) {
+        throw AccessDeniedException(path.toFile(), reason = ex.message)
+    }
+
+    override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try {
+        val (bucketName, filePath) = splitPathIntoBucketAndPath(Path(dirname))
+        client.createBucket {
+            bucket = bucketName
+        }
+        S3Directory(client, bucketName, filePath)
+    } catch (ex: Exception) {
+        throw AccessDeniedException(Path(dirname).toFile(), reason = ex.message)
+    }
+
+    override val path: Path
+        get() = Path("")
+
+    override fun close() {
+    }
+
+}