diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt new file mode 100644 index 0000000..28e9487 --- /dev/null +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt @@ -0,0 +1,52 @@ +package space.kscience.snark.storage.local + +import space.kscience.snark.storage.Directory +import space.kscience.snark.storage.FileReader +import space.kscience.snark.storage.FileWriter +import java.nio.file.Path +import kotlin.io.path.* + +public fun localStorage(rootPath: Path): Directory { + return LocalDirectory(rootPath) +} + +private class LocalFile(private val path: Path) : FileReader, FileWriter { + override fun close() {} + override suspend fun readAll(): ByteArray = path.readBytes() + + override suspend fun write(bytes: ByteArray) = path.writeBytes(bytes) +} + +private class LocalDirectory(private val path: Path) : Directory { + private fun child(child: String): Path = path / child + private fun child(child: Path): Path = path / child + + override fun close() {} + + override suspend fun get(filename: String): FileReader = LocalFile(child(filename)) + + override suspend fun create(filename: String, ignoreIfExists: Boolean) { + try { + child(filename).createFile() + } catch (ex: FileAlreadyExistsException) { + if (!ignoreIfExists) { + throw ex + } + } + } + + override suspend fun put(filename: String): FileWriter = LocalFile(child(filename)) + + override suspend fun getSubdir(path: Path): Directory = LocalDirectory(child(path)) + override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory { + val dir = child(dirname) + try { + dir.createDirectory() + } catch (ex: FileAlreadyExistsException) { + if (!ignoreIfExists) { + throw ex + } + } + return LocalDirectory(dir) + } +} diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/localDriver.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/localDriver.kt deleted file mode 100644 index 554faf0..0000000 --- a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/localDriver.kt +++ /dev/null @@ -1,35 +0,0 @@ -package space.kscience.snark.storage.local - -import space.kscience.snark.storage.Directory -import space.kscience.snark.storage.FileReader -import space.kscience.snark.storage.FileWriter -import java.io.File -import java.nio.file.Path - -public class LocalFile(private val path: String) : FileReader, FileWriter { - override fun close() {} - override suspend fun readAll(): ByteArray = File(this.path).readBytes() - - override suspend fun write(bytes: ByteArray) = File(this.path).writeBytes(bytes) -} - -public class LocalDirectory(private val path: String) : Directory { - override fun close() {} - override suspend fun get(filename: String): FileReader = LocalFile("${this.path}/$filename") - - override suspend fun create(filename: String, ignoreIfExists: Boolean) { - if (!File(filename).createNewFile() && !ignoreIfExists) { - throw UnsupportedOperationException("File already exists") - } - } - - override suspend fun put(filename: String): FileWriter = LocalFile("${this.path}/$filename") - - override suspend fun getSubdir(dirpath: Path): Directory = LocalDirectory("${this.path}/$dirpath") - override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory { - if (!File(dirname).mkdir() && !ignoreIfExists) { - throw UnsupportedOperationException("File already exists") - } - return this.getSubdir(File(dirname).toPath()) - } -} diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt new file mode 100644 index 0000000..ed2c743 --- /dev/null +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt @@ -0,0 +1,39 @@ +package space.kscience.snark.storage.s3 + +import aws.sdk.kotlin.services.s3.S3Client +import space.kscience.snark.storage.Directory +import space.kscience.snark.storage.FileReader +import space.kscience.snark.storage.FileWriter +import java.nio.file.Path +import kotlin.io.path.* + +internal class S3Directory( + private val client: S3Client, + private val bucketName: String, + private val currentDir: Path, +) : Directory { + override suspend fun get(filename: String): FileReader = + S3FileReader(client, bucketName, currentDir / filename) + + override suspend fun create(filename: String, ignoreIfExists: Boolean) { + if (!ignoreIfExists) { + TODO("could not check if file exists") + } + } + + override suspend fun put(filename: String): FileWriter = + S3FileWriter(client, bucketName, currentDir / filename) + + override suspend fun getSubdir(path: Path): S3Directory = + S3Directory(client, bucketName, currentDir / path) + + override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): S3Directory = + if (!ignoreIfExists) { + TODO("could not check if directory exists") + } else { + S3Directory(client, bucketName, currentDir / dirname) + } + + override fun close() { + } +} \ No newline at end of file diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt new file mode 100644 index 0000000..8255519 --- /dev/null +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt @@ -0,0 +1,40 @@ +package space.kscience.snark.storage.s3 + +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.GetObjectRequest +import aws.sdk.kotlin.services.s3.putObject +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.content.toByteArray +import space.kscience.snark.storage.FileReader +import space.kscience.snark.storage.FileWriter +import java.nio.file.Path +import kotlin.io.path.* + +internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) : FileReader { + override suspend fun readAll(): ByteArray { + val result = client.getObject(GetObjectRequest{ + bucket = bucketName + key = path.toString() + }) { + it.body?.toByteArray() ?: ByteArray(0) + } + return result + } + + override fun close() { + } +} + +internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) : FileWriter { + override suspend fun write(bytes: ByteArray) { + client.putObject { + bucket = bucketName + key = path.toString() + body = ByteStream.fromBytes(bytes) + } + } + + override fun close() { + } + +} \ No newline at end of file diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt new file mode 100644 index 0000000..3f0037c --- /dev/null +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt @@ -0,0 +1,59 @@ +package space.kscience.snark.storage.s3 + +import aws.sdk.kotlin.services.s3.* +import space.kscience.snark.storage.Directory +import space.kscience.snark.storage.FileReader +import space.kscience.snark.storage.FileWriter +import java.lang.Exception +import java.nio.file.Path +import kotlin.io.path.* + +public fun s3Storage(client: S3Client): Directory = + S3Root(client) + +public fun s3Bucket(client: S3Client, bucket: String): Directory = + S3Directory(client, bucket, Path("")) + +internal fun splitPathIntoBucketAndPath(path: Path): Pair { + val bucket = path.getName(0) + val filePath = path.relativize(bucket) + return Pair(bucket.toString(), filePath) +} + +internal class S3Root(private val client: S3Client) : Directory { + override suspend fun get(filename: String): FileReader { + throw NoSuchFileException(Path(filename).toFile()) + } + + override suspend fun create(filename: String, ignoreIfExists: Boolean) { + throw NoSuchFileException(Path(filename).toFile()) + } + + override suspend fun put(filename: String): FileWriter { + throw NoSuchFileException(Path(filename).toFile()) + } + + override suspend fun getSubdir(path: Path): Directory = try { + val (bucketName, filePath) = splitPathIntoBucketAndPath(path) + client.headBucket { + bucket = bucketName + } + S3Directory(client, bucketName, filePath) + } catch (ex: Exception) { + throw AccessDeniedException(path.toFile(), reason = ex.message) + } + + override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try { + val (bucketName, filePath) = splitPathIntoBucketAndPath(Path(dirname)) + client.createBucket { + bucket = bucketName + } + S3Directory(client, bucketName, filePath) + } catch (ex: Exception) { + throw AccessDeniedException(Path(dirname).toFile(), reason = ex.message) + } + + override fun close() { + } + +} \ No newline at end of file diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt new file mode 100644 index 0000000..e7aff9c --- /dev/null +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt @@ -0,0 +1,24 @@ +package space.kscience.snark.storage.unzip + +import space.kscience.snark.storage.Directory +import java.io.FileInputStream +import java.util.zip.ZipInputStream + +public suspend fun unzip(source_path: String, target: Directory) { + val zis = ZipInputStream(FileInputStream(source_path)) + var zipEntry = zis.nextEntry + while (zipEntry != null) { + if (!zipEntry.isDirectory) { + val filename = zipEntry.name + target.create(filename, true) + val fos = target.put(filename) + val buffer = ByteArray(zipEntry.size.toInt()) + zis.read(buffer) + fos.write(buffer) + fos.close() + } + zipEntry = zis.nextEntry + } + zis.closeEntry() + zis.close() +} \ No newline at end of file