SNRK-57: Fix internals

This commit is contained in:
Kirill Grachev 2023-04-24 21:12:21 +03:00
parent a97a99d138
commit d292abc816
3 changed files with 41 additions and 24 deletions

View File

@ -1,18 +1,19 @@
package space.kscience.snark.storage.s3 package space.kscience.snark.storage.s3
import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.S3Client
import space.kscience.snark.storage.Directory as Dir import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.FileReader import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter import space.kscience.snark.storage.FileWriter
import java.nio.file.Path import java.nio.file.Path
import kotlin.io.path.*
public class Directory( internal class S3Directory(
private val client: S3Client, private val client: S3Client,
private val bucketName: String, private val bucketName: String,
private val currentDir: String, private val currentDir: Path,
) : Dir { ) : Directory {
override suspend fun get(filename: String): FileReader = run { override suspend fun get(filename: String): FileReader = run {
S3FileReader(client, bucketName, "$currentDir/$filename") S3FileReader(client, bucketName, currentDir / filename)
} }
override suspend fun create(filename: String, ignoreIfExists: Boolean) { override suspend fun create(filename: String, ignoreIfExists: Boolean) {
@ -21,19 +22,17 @@ public class Directory(
} }
} }
override suspend fun put(filename: String): FileWriter = run { override suspend fun put(filename: String): FileWriter =
S3FileWriter(client, bucketName, "$currentDir/$filename") S3FileWriter(client, bucketName, currentDir / filename)
}
override suspend fun getSubdir(path: Path): Directory = run { override suspend fun getSubdir(path: Path): S3Directory =
Directory(client, bucketName, "$currentDir/$path") S3Directory(client, bucketName, currentDir / path)
}
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = run { override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): S3Directory = run {
if (!ignoreIfExists) { if (!ignoreIfExists) {
throw IllegalArgumentException("could not check if directory exists") throw IllegalArgumentException("could not check if directory exists")
} }
Directory(client, bucketName, "$currentDir/$dirname") S3Directory(client, bucketName, currentDir / dirname)
} }
override fun close() { override fun close() {

View File

@ -7,12 +7,14 @@ import aws.smithy.kotlin.runtime.content.ByteStream
import aws.smithy.kotlin.runtime.content.toByteArray import aws.smithy.kotlin.runtime.content.toByteArray
import space.kscience.snark.storage.FileReader import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter import space.kscience.snark.storage.FileWriter
import java.nio.file.Path
import kotlin.io.path.*
public class S3FileReader(private val client: S3Client, private val bucketName: String, private val fullQualifiedPath: String) : FileReader { internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) : FileReader {
override suspend fun readAll(): ByteArray { override suspend fun readAll(): ByteArray {
val result = client.getObject(GetObjectRequest{ val result = client.getObject(GetObjectRequest{
bucket = bucketName bucket = bucketName
key = fullQualifiedPath key = path.toString()
}) { }) {
it.body?.toByteArray() ?: ByteArray(0) it.body?.toByteArray() ?: ByteArray(0)
} }
@ -23,11 +25,11 @@ public class S3FileReader(private val client: S3Client, private val bucketName:
} }
} }
public class S3FileWriter(private val client: S3Client, private val bucketName: String, private val fullQualifiedPath: String) : FileWriter { internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) : FileWriter {
override suspend fun write(bytes: ByteArray) { override suspend fun write(bytes: ByteArray) {
client.putObject { client.putObject {
bucket = bucketName bucket = bucketName
key = fullQualifiedPath key = path.toString()
body = ByteStream.fromBytes(bytes) body = ByteStream.fromBytes(bytes)
} }
} }

View File

@ -1,20 +1,35 @@
package space.kscience.snark.storage.s3 package space.kscience.snark.storage.s3
import aws.sdk.kotlin.services.s3.* import aws.sdk.kotlin.services.s3.*
import space.kscience.snark.storage.Directory as Dir import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.FileReader import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter import space.kscience.snark.storage.FileWriter
import java.io.File import java.io.File
import java.lang.Exception import java.lang.Exception
import java.nio.file.Path import java.nio.file.Path
import kotlin.io.path.*
public class Root(private val client: S3Client) : Dir { public fun s3Storage(client: S3Client): Directory {
return S3Root(client)
}
public fun s3Bucket(client: S3Client, bucket: String): Directory {
return S3Directory(client, bucket, Path(""))
}
internal fun splitPathIntoBucketAndPath(path: Path): Pair<String, Path> {
val bucket = path.getName(0)
val recent = path.relativize(bucket)
return Pair(bucket.toString(), recent)
}
internal class S3Root(private val client: S3Client) : Directory {
override suspend fun get(filename: String): FileReader { override suspend fun get(filename: String): FileReader {
throw NoSuchFileException(File(filename)) throw NoSuchFileException(File(filename))
} }
override suspend fun create(filename: String, ignoreIfExists: Boolean) { override suspend fun create(filename: String, ignoreIfExists: Boolean) {
throw IllegalCallerException() throw NoSuchFileException(File(filename))
} }
override suspend fun put(filename: String): FileWriter { override suspend fun put(filename: String): FileWriter {
@ -22,20 +37,21 @@ public class Root(private val client: S3Client) : Dir {
} }
override suspend fun getSubdir(path: Path): Directory = try { override suspend fun getSubdir(path: Path): Directory = try {
val bucketName = path.toString() val (bucketName, recentPath) = splitPathIntoBucketAndPath(path)
client.headBucket { client.headBucket {
bucket = bucketName bucket = bucketName
} }
Directory(client, bucketName, "") S3Directory(client, bucketName, recentPath)
} catch (ex: Exception) { } catch (ex: Exception) {
throw java.nio.file.AccessDeniedException(path.toString()).initCause(ex) throw java.nio.file.AccessDeniedException(path.toString()).initCause(ex)
} }
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try { override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try {
val (bucketName, recentPath) = splitPathIntoBucketAndPath(Path(dirname))
client.createBucket { client.createBucket {
bucket = dirname bucket = bucketName
} }
Directory(client, dirname, "") S3Directory(client, bucketName, recentPath)
} catch (ex: Exception) { } catch (ex: Exception) {
throw java.nio.file.AccessDeniedException(dirname).initCause(ex) throw java.nio.file.AccessDeniedException(dirname).initCause(ex)
} }