SNRK-48: Add s3 implementation

Use common s3 api and '/'-separated keys to simulate directories
This commit is contained in:
Kirill Grachev 2023-04-17 17:28:34 +03:00
parent a9c0cc10a6
commit a97a99d138
3 changed files with 125 additions and 0 deletions

View File

@ -0,0 +1,41 @@
package space.kscience.snark.storage.s3
import aws.sdk.kotlin.services.s3.S3Client
import space.kscience.snark.storage.Directory as Dir
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.nio.file.Path
public class Directory(
private val client: S3Client,
private val bucketName: String,
private val currentDir: String,
) : Dir {
override suspend fun get(filename: String): FileReader = run {
S3FileReader(client, bucketName, "$currentDir/$filename")
}
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
if (!ignoreIfExists) {
throw IllegalArgumentException("could not check if file exists")
}
}
override suspend fun put(filename: String): FileWriter = run {
S3FileWriter(client, bucketName, "$currentDir/$filename")
}
override suspend fun getSubdir(path: Path): Directory = run {
Directory(client, bucketName, "$currentDir/$path")
}
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = run {
if (!ignoreIfExists) {
throw IllegalArgumentException("could not check if directory exists")
}
Directory(client, bucketName, "$currentDir/$dirname")
}
override fun close() {
}
}

View File

@ -0,0 +1,38 @@
package space.kscience.snark.storage.s3
import aws.sdk.kotlin.services.s3.S3Client
import aws.sdk.kotlin.services.s3.model.GetObjectRequest
import aws.sdk.kotlin.services.s3.putObject
import aws.smithy.kotlin.runtime.content.ByteStream
import aws.smithy.kotlin.runtime.content.toByteArray
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
public class S3FileReader(private val client: S3Client, private val bucketName: String, private val fullQualifiedPath: String) : FileReader {
override suspend fun readAll(): ByteArray {
val result = client.getObject(GetObjectRequest{
bucket = bucketName
key = fullQualifiedPath
}) {
it.body?.toByteArray() ?: ByteArray(0)
}
return result
}
override fun close() {
}
}
public class S3FileWriter(private val client: S3Client, private val bucketName: String, private val fullQualifiedPath: String) : FileWriter {
override suspend fun write(bytes: ByteArray) {
client.putObject {
bucket = bucketName
key = fullQualifiedPath
body = ByteStream.fromBytes(bytes)
}
}
override fun close() {
}
}

View File

@ -0,0 +1,46 @@
package space.kscience.snark.storage.s3
import aws.sdk.kotlin.services.s3.*
import space.kscience.snark.storage.Directory as Dir
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.io.File
import java.lang.Exception
import java.nio.file.Path
public class Root(private val client: S3Client) : Dir {
override suspend fun get(filename: String): FileReader {
throw NoSuchFileException(File(filename))
}
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
throw IllegalCallerException()
}
override suspend fun put(filename: String): FileWriter {
throw NoSuchFileException(File(filename))
}
override suspend fun getSubdir(path: Path): Directory = try {
val bucketName = path.toString()
client.headBucket {
bucket = bucketName
}
Directory(client, bucketName, "")
} catch (ex: Exception) {
throw java.nio.file.AccessDeniedException(path.toString()).initCause(ex)
}
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try {
client.createBucket {
bucket = dirname
}
Directory(client, dirname, "")
} catch (ex: Exception) {
throw java.nio.file.AccessDeniedException(dirname).initCause(ex)
}
override fun close() {
}
}