From d292abc816d0c2f3c803db3386cc9a05b2cfacbd Mon Sep 17 00:00:00 2001 From: Kirill Grachev Date: Mon, 24 Apr 2023 21:12:21 +0300 Subject: [PATCH] SNRK-57: Fix internals --- .../s3/{Directory.kt => S3Directory.kt} | 25 ++++++++-------- .../snark/storage/s3/{File.kt => S3File.kt} | 10 ++++--- .../snark/storage/s3/{Root.kt => S3Root.kt} | 30 ++++++++++++++----- 3 files changed, 41 insertions(+), 24 deletions(-) rename snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/{Directory.kt => S3Directory.kt} (58%) rename snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/{File.kt => S3File.kt} (69%) rename snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/{Root.kt => S3Root.kt} (55%) diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/Directory.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt similarity index 58% rename from snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/Directory.kt rename to snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt index 27ece3b..d972fed 100644 --- a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/Directory.kt +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt @@ -1,18 +1,19 @@ package space.kscience.snark.storage.s3 import aws.sdk.kotlin.services.s3.S3Client -import space.kscience.snark.storage.Directory as Dir +import space.kscience.snark.storage.Directory import space.kscience.snark.storage.FileReader import space.kscience.snark.storage.FileWriter import java.nio.file.Path +import kotlin.io.path.* -public class Directory( +internal class S3Directory( private val client: S3Client, private val bucketName: String, - private val currentDir: String, -) : Dir { + private val currentDir: Path, +) : Directory { override suspend fun get(filename: String): FileReader = run { - S3FileReader(client, bucketName, "$currentDir/$filename") + S3FileReader(client, bucketName, currentDir / filename) } override suspend fun create(filename: String, ignoreIfExists: Boolean) { @@ -21,19 +22,17 @@ public class Directory( } } - override suspend fun put(filename: String): FileWriter = run { - S3FileWriter(client, bucketName, "$currentDir/$filename") - } + override suspend fun put(filename: String): FileWriter = + S3FileWriter(client, bucketName, currentDir / filename) - override suspend fun getSubdir(path: Path): Directory = run { - Directory(client, bucketName, "$currentDir/$path") - } + override suspend fun getSubdir(path: Path): S3Directory = + S3Directory(client, bucketName, currentDir / path) - override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = run { + override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): S3Directory = run { if (!ignoreIfExists) { throw IllegalArgumentException("could not check if directory exists") } - Directory(client, bucketName, "$currentDir/$dirname") + S3Directory(client, bucketName, currentDir / dirname) } override fun close() { diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/File.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt similarity index 69% rename from snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/File.kt rename to snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt index 43e872f..8255519 100644 --- a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/File.kt +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt @@ -7,12 +7,14 @@ import aws.smithy.kotlin.runtime.content.ByteStream import aws.smithy.kotlin.runtime.content.toByteArray import space.kscience.snark.storage.FileReader import space.kscience.snark.storage.FileWriter +import java.nio.file.Path +import kotlin.io.path.* -public class S3FileReader(private val client: S3Client, private val bucketName: String, private val fullQualifiedPath: String) : FileReader { +internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) : FileReader { override suspend fun readAll(): ByteArray { val result = client.getObject(GetObjectRequest{ bucket = bucketName - key = fullQualifiedPath + key = path.toString() }) { it.body?.toByteArray() ?: ByteArray(0) } @@ -23,11 +25,11 @@ public class S3FileReader(private val client: S3Client, private val bucketName: } } -public class S3FileWriter(private val client: S3Client, private val bucketName: String, private val fullQualifiedPath: String) : FileWriter { +internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) : FileWriter { override suspend fun write(bytes: ByteArray) { client.putObject { bucket = bucketName - key = fullQualifiedPath + key = path.toString() body = ByteStream.fromBytes(bytes) } } diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/Root.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt similarity index 55% rename from snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/Root.kt rename to snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt index de51731..4625a8c 100644 --- a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/Root.kt +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt @@ -1,20 +1,35 @@ package space.kscience.snark.storage.s3 import aws.sdk.kotlin.services.s3.* -import space.kscience.snark.storage.Directory as Dir +import space.kscience.snark.storage.Directory import space.kscience.snark.storage.FileReader import space.kscience.snark.storage.FileWriter import java.io.File import java.lang.Exception import java.nio.file.Path +import kotlin.io.path.* -public class Root(private val client: S3Client) : Dir { +public fun s3Storage(client: S3Client): Directory { + return S3Root(client) +} + +public fun s3Bucket(client: S3Client, bucket: String): Directory { + return S3Directory(client, bucket, Path("")) +} + +internal fun splitPathIntoBucketAndPath(path: Path): Pair { + val bucket = path.getName(0) + val recent = path.relativize(bucket) + return Pair(bucket.toString(), recent) +} + +internal class S3Root(private val client: S3Client) : Directory { override suspend fun get(filename: String): FileReader { throw NoSuchFileException(File(filename)) } override suspend fun create(filename: String, ignoreIfExists: Boolean) { - throw IllegalCallerException() + throw NoSuchFileException(File(filename)) } override suspend fun put(filename: String): FileWriter { @@ -22,20 +37,21 @@ public class Root(private val client: S3Client) : Dir { } override suspend fun getSubdir(path: Path): Directory = try { - val bucketName = path.toString() + val (bucketName, recentPath) = splitPathIntoBucketAndPath(path) client.headBucket { bucket = bucketName } - Directory(client, bucketName, "") + S3Directory(client, bucketName, recentPath) } catch (ex: Exception) { throw java.nio.file.AccessDeniedException(path.toString()).initCause(ex) } override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try { + val (bucketName, recentPath) = splitPathIntoBucketAndPath(Path(dirname)) client.createBucket { - bucket = dirname + bucket = bucketName } - Directory(client, dirname, "") + S3Directory(client, bucketName, recentPath) } catch (ex: Exception) { throw java.nio.file.AccessDeniedException(dirname).initCause(ex) }