From 80deddc4123c54148cb8ba72444ee92726053e99 Mon Sep 17 00:00:00 2001 From: Kirill Grachev Date: Mon, 15 May 2023 17:59:53 +0300 Subject: [PATCH 1/5] SNRK-76: Add configs and method --- .../space/kscience/snark/storage/Config.kt | 35 +++++++++++++++++++ .../space/kscience/snark/storage/Driver.kt | 8 +++++ 2 files changed, 43 insertions(+) create mode 100644 snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Config.kt diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Config.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Config.kt new file mode 100644 index 0000000..0b870eb --- /dev/null +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Config.kt @@ -0,0 +1,35 @@ +package space.kscience.snark.storage + +import aws.sdk.kotlin.services.s3.S3Client +import space.kscience.snark.storage.local.localStorage +import space.kscience.snark.storage.s3.s3Bucket +import space.kscience.snark.storage.s3.s3Storage +import java.nio.file.Path + +public sealed interface Config { + public fun build(): Directory +} + +public data class LocalConfig(val path: Path): Config { + override fun build(): Directory { + return localStorage(path) + } +} + +internal fun buildS3Client(regionSpec: String): S3Client { + return S3Client { + region = regionSpec + } +} + +public data class S3BucketConfig(val bucketName: String): Config { + override fun build(): Directory { + return s3Bucket(buildS3Client("eu-east"), bucketName) + } +} + +public object S3ServiceConfig: Config { + override fun build(): Directory { + return s3Storage(buildS3Client("eu-east")) + } +} diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt index 902aa23..caccf06 100644 --- a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt @@ -1,5 +1,7 @@ package space.kscience.snark.storage +import space.kscience.snark.storage.local.localStorage +import space.kscience.snark.storage.s3.s3Bucket import java.nio.file.Path import kotlin.io.path.Path @@ -20,6 +22,12 @@ public interface Directory : AutoCloseable { @Deprecated("Not a good idea") public val path: Path + + public companion object { + public fun fromConfig(config: Config): Directory { + return config.build() + } + } } From f06182ee1d1536db1ac3f1796e6d33646a86dd1f Mon Sep 17 00:00:00 2001 From: Kirill Grachev Date: Mon, 15 May 2023 18:00:27 +0300 Subject: [PATCH 2/5] Add acceptance tests --- .../kscience/snark/storage/JustCreates.kt | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/JustCreates.kt diff --git a/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/JustCreates.kt b/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/JustCreates.kt new file mode 100644 index 0000000..ef9478a --- /dev/null +++ b/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/JustCreates.kt @@ -0,0 +1,25 @@ +package space.kscience.snark.storage + +import org.junit.jupiter.api.Test +import kotlin.io.path.createTempDirectory + +class JustCreates { + + @Test + fun s3Created() { + val dir = Directory.fromConfig(S3ServiceConfig) + dir.close() + } + + @Test + fun s3BucketCreated() { + val dir = Directory.fromConfig(S3BucketConfig("snark-test")) + dir.close() + } + + @Test + fun localCreated() { + val dir = Directory.fromConfig(LocalConfig(createTempDirectory("snark-test"))) + dir.close() + } +} \ No newline at end of file From 028cf631f45bc266b6013fd0e16a3b2fdaf0e34c Mon Sep 17 00:00:00 2001 From: Kirill Grachev Date: Mon, 15 May 2023 18:07:12 +0300 Subject: [PATCH 3/5] SNRK-76: Add regions --- .../kotlin/space/kscience/snark/storage/Config.kt | 13 +++++++++---- .../space/kscience/snark/storage/JustCreates.kt | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Config.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Config.kt index 0b870eb..5cf24de 100644 --- a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Config.kt +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Config.kt @@ -6,6 +6,8 @@ import space.kscience.snark.storage.s3.s3Bucket import space.kscience.snark.storage.s3.s3Storage import java.nio.file.Path +private const val DEFAULT_REGION = "arctic-vault" + public sealed interface Config { public fun build(): Directory } @@ -16,20 +18,23 @@ public data class LocalConfig(val path: Path): Config { } } +/* + * `~/.aws/credentials.json file is required + */ internal fun buildS3Client(regionSpec: String): S3Client { return S3Client { region = regionSpec } } -public data class S3BucketConfig(val bucketName: String): Config { +public data class S3BucketConfig(val bucketName: String, val region: String = DEFAULT_REGION): Config { override fun build(): Directory { - return s3Bucket(buildS3Client("eu-east"), bucketName) + return s3Bucket(buildS3Client(region), bucketName) } } -public object S3ServiceConfig: Config { +public data class S3ServiceConfig(val region: String = DEFAULT_REGION): Config { override fun build(): Directory { - return s3Storage(buildS3Client("eu-east")) + return s3Storage(buildS3Client(region)) } } diff --git a/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/JustCreates.kt b/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/JustCreates.kt index ef9478a..43e8144 100644 --- a/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/JustCreates.kt +++ b/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/JustCreates.kt @@ -7,7 +7,7 @@ class JustCreates { @Test fun s3Created() { - val dir = Directory.fromConfig(S3ServiceConfig) + val dir = Directory.fromConfig(S3ServiceConfig()) dir.close() } From 81c7bbb07397074b9e52ddd15061eb12207dfd98 Mon Sep 17 00:00:00 2001 From: Kirill Grachev Date: Mon, 15 May 2023 18:09:15 +0300 Subject: [PATCH 4/5] SNRK-76: Format files --- .../main/kotlin/space/kscience/snark/storage/Config.kt | 8 ++++---- .../main/kotlin/space/kscience/snark/storage/Driver.kt | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Config.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Config.kt index 5cf24de..452b12a 100644 --- a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Config.kt +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Config.kt @@ -12,14 +12,14 @@ public sealed interface Config { public fun build(): Directory } -public data class LocalConfig(val path: Path): Config { +public data class LocalConfig(val path: Path) : Config { override fun build(): Directory { return localStorage(path) } } /* - * `~/.aws/credentials.json file is required + * ~/.aws/credentials.json file is required */ internal fun buildS3Client(regionSpec: String): S3Client { return S3Client { @@ -27,13 +27,13 @@ internal fun buildS3Client(regionSpec: String): S3Client { } } -public data class S3BucketConfig(val bucketName: String, val region: String = DEFAULT_REGION): Config { +public data class S3BucketConfig(val bucketName: String, val region: String = DEFAULT_REGION) : Config { override fun build(): Directory { return s3Bucket(buildS3Client(region), bucketName) } } -public data class S3ServiceConfig(val region: String = DEFAULT_REGION): Config { +public data class S3ServiceConfig(val region: String = DEFAULT_REGION) : Config { override fun build(): Directory { return s3Storage(buildS3Client(region)) } diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt index caccf06..d0e2b6a 100644 --- a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt @@ -1,7 +1,5 @@ package space.kscience.snark.storage -import space.kscience.snark.storage.local.localStorage -import space.kscience.snark.storage.s3.s3Bucket import java.nio.file.Path import kotlin.io.path.Path From b463c492f14b1ba40b1d844fd23c90d8f8b54cac Mon Sep 17 00:00:00 2001 From: Kirill Grachev Date: Mon, 15 May 2023 18:36:14 +0300 Subject: [PATCH 5/5] SNRK-76: Ignore credentials and configs --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f06baf2..21511df 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ build/ !gradle/wrapper/gradle-wrapper.jar kotlin-js-store -*.iml \ No newline at end of file +*.iml +*.json