diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt index eaa3e6f..902aa23 100644 --- a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt @@ -1,35 +1,36 @@ package space.kscience.snark.storage import java.nio.file.Path +import kotlin.io.path.Path public interface Directory : AutoCloseable { - @Deprecated( - message = "Use Path, not String", - level = DeprecationLevel.WARNING, - ) - public suspend fun get(filename: String): FileReader // get file from subtree public suspend fun get(filename: Path): FileReader + @Deprecated("Use put") public suspend fun create(filename: String, ignoreIfExists: Boolean = false) - @Deprecated( - message = "Use Path, not String", - level = DeprecationLevel.WARNING, - ) - public suspend fun put(filename: String): FileWriter + // put file to subtree public suspend fun put(filename: Path): FileWriter public suspend fun getSubdir(path: Path): Directory + + @Deprecated("Directories are created on put") public suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean = false): Directory - @Deprecated( - message = "Not a good idea", - level = DeprecationLevel.WARNING, - ) + @Deprecated("Not a good idea") public val path: Path } + +public suspend fun Directory.get(filename: String): FileReader = get(Path(filename)) + +public suspend fun Directory.put(filename: String): FileWriter = put(Path(filename)) + +public suspend operator fun Directory.div(path: Path): Directory = getSubdir(path) + +public suspend operator fun Directory.div(path: String): Directory = getSubdir(Path(path)) + public interface FileReader : AutoCloseable { public suspend fun readAll(): ByteArray } diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt index 1b81fba..33292c4 100644 --- a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt @@ -3,9 +3,7 @@ package space.kscience.snark.storage.local import space.kscience.snark.storage.Directory import space.kscience.snark.storage.FileReader import space.kscience.snark.storage.FileWriter -import java.io.File import java.nio.file.Path -import java.nio.file.attribute.PosixFilePermission import kotlin.io.path.* public fun localStorage(rootPath: Path): Directory { @@ -16,24 +14,32 @@ internal class LocalFile(private val path: Path) : FileReader, FileWriter { override fun close() {} override suspend fun readAll(): ByteArray = path.readBytes() - override suspend fun write(bytes: ByteArray) = path.writeBytes(bytes) + override suspend fun write(bytes: ByteArray) { + path.parent.createDirectories() + try { + path.createFile() + } catch (ex: Exception) { + // Do nothing + } + path.writeBytes(bytes) + } } internal class LocalDirectory(private val root: Path, private val currentDir: Path) : Directory { - private fun child(child: String): Path = root / currentDir / child - private fun child(child: Path): Path = root / currentDir / child + @Deprecated("Use Path, not String") + private fun realpath(child: String): Path = root / currentDir / child + private fun realpath(child: Path): Path = root / currentDir / child override fun close() {} - override suspend fun get(filename: String): LocalFile = LocalFile(child(filename)) - - override suspend fun get(filename: Path): LocalFile = LocalFile(child(filename)) + override suspend fun get(filename: Path): LocalFile = LocalFile(realpath(filename)) + @Deprecated("Use put") override suspend fun create(filename: String, ignoreIfExists: Boolean) { - val dir = child(filename) + val dir = realpath(filename) dir.parent.createDirectories() try { - child(filename).createFile() + realpath(filename).createFile() } catch (ex: java.nio.file.FileAlreadyExistsException) { if (!ignoreIfExists) { throw ex @@ -41,13 +47,13 @@ internal class LocalDirectory(private val root: Path, private val currentDir: Pa } } - override suspend fun put(filename: String): LocalFile = get(filename) - override suspend fun put(filename: Path): LocalFile = get(filename) override suspend fun getSubdir(path: Path): LocalDirectory = LocalDirectory(root, currentDir / path) + + @Deprecated("Directories are created on put") override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): LocalDirectory { - val dir = child(dirname) + val dir = realpath(dirname) dir.parent.createDirectories() try { dir.createDirectory() @@ -59,6 +65,7 @@ internal class LocalDirectory(private val root: Path, private val currentDir: Pa return LocalDirectory(root, currentDir / dirname) } + @Deprecated("Not a good idea") override val path: Path get() = currentDir } diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt index 7dbb52b..2f858f7 100644 --- a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt @@ -5,34 +5,30 @@ import space.kscience.snark.storage.Directory import space.kscience.snark.storage.FileReader import space.kscience.snark.storage.FileWriter import java.nio.file.Path -import kotlin.io.path.* +import kotlin.io.path.div internal class S3Directory( private val client: S3Client, private val bucketName: String, private val currentDir: Path, ) : Directory { - override suspend fun get(filename: String): FileReader = - S3FileReader(client, bucketName, currentDir / filename) - override suspend fun get(filename: Path): FileReader = S3FileReader(client, bucketName, currentDir / filename) + @Deprecated("Use put") override suspend fun create(filename: String, ignoreIfExists: Boolean) { if (!ignoreIfExists) { TODO("could not check if file exists") } } - override suspend fun put(filename: String): FileWriter = - S3FileWriter(client, bucketName, currentDir / filename) - override suspend fun put(filename: Path): FileWriter = S3FileWriter(client, bucketName, currentDir / filename) override suspend fun getSubdir(path: Path): S3Directory = S3Directory(client, bucketName, currentDir / path) + @Deprecated("Directories are created on put") override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): S3Directory = if (!ignoreIfExists) { TODO("could not check if directory exists") @@ -40,6 +36,7 @@ internal class S3Directory( S3Directory(client, bucketName, currentDir / dirname) } + @Deprecated("Not a good idea") override val path: Path get() = currentDir diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt index 8255519..478f1d5 100644 --- a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt @@ -8,11 +8,11 @@ import aws.smithy.kotlin.runtime.content.toByteArray import space.kscience.snark.storage.FileReader import space.kscience.snark.storage.FileWriter import java.nio.file.Path -import kotlin.io.path.* -internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) : FileReader { +internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) : + FileReader { override suspend fun readAll(): ByteArray { - val result = client.getObject(GetObjectRequest{ + val result = client.getObject(GetObjectRequest { bucket = bucketName key = path.toString() }) { @@ -25,7 +25,8 @@ internal class S3FileReader(private val client: S3Client, private val bucketName } } -internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) : FileWriter { +internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) : + FileWriter { override suspend fun write(bytes: ByteArray) { client.putObject { bucket = bucketName diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt index 8f32607..be4a8ad 100644 --- a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt @@ -1,12 +1,13 @@ package space.kscience.snark.storage.s3 -import aws.sdk.kotlin.services.s3.* +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.createBucket +import aws.sdk.kotlin.services.s3.headBucket import space.kscience.snark.storage.Directory import space.kscience.snark.storage.FileReader import space.kscience.snark.storage.FileWriter -import java.lang.Exception import java.nio.file.Path -import kotlin.io.path.* +import kotlin.io.path.Path public fun s3Storage(client: S3Client): Directory = S3Root(client) @@ -21,22 +22,16 @@ internal fun splitPathIntoBucketAndPath(path: Path): Pair { } internal class S3Root(private val client: S3Client) : Directory { - override suspend fun get(filename: String): FileReader { - throw NoSuchFileException(Path(filename).toFile()) - } override suspend fun get(filename: Path): FileReader { throw NoSuchFileException(filename.toFile()) } + @Deprecated("Use put") override suspend fun create(filename: String, ignoreIfExists: Boolean) { throw NoSuchFileException(Path(filename).toFile()) } - override suspend fun put(filename: String): FileWriter { - throw NoSuchFileException(Path(filename).toFile()) - } - override suspend fun put(filename: Path): FileWriter { throw NoSuchFileException(filename.toFile()) } @@ -51,6 +46,7 @@ internal class S3Root(private val client: S3Client) : Directory { throw AccessDeniedException(path.toFile(), reason = ex.message) } + @Deprecated("Directories are created on put") override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try { val (bucketName, filePath) = splitPathIntoBucketAndPath(Path(dirname)) client.createBucket { @@ -61,6 +57,7 @@ internal class S3Root(private val client: S3Client) : Directory { throw AccessDeniedException(Path(dirname).toFile(), reason = ex.message) } + @Deprecated("Not a good idea") override val path: Path get() = Path("") diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt index 5201930..b60cc9c 100644 --- a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt @@ -1,6 +1,6 @@ package space.kscience.snark.storage.unzip -import space.kscience.snark.storage.Directory +import space.kscience.snark.storage.* import java.io.FileInputStream import java.util.zip.ZipInputStream diff --git a/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Example.kt b/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Example.kt new file mode 100644 index 0000000..e474fee --- /dev/null +++ b/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Example.kt @@ -0,0 +1,47 @@ +package space.kscience.snark.storage.local + +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.BeforeAll +import space.kscience.snark.storage.* +import java.nio.file.Path +import kotlin.io.path.createTempDirectory +import kotlin.io.path.deleteExisting +import kotlin.test.AfterTest +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.io.path.* +import kotlin.test.assertEquals + +class Example { + var tempDir: Path? = null + var somedir: Directory? = null + + @BeforeTest + fun setUp() { + tempDir = createTempDirectory() + somedir = localStorage(tempDir!!) + } + + @AfterTest + fun tearDown() { + tempDir!!.toFile().deleteRecursively() + somedir = null + } + + @Test + fun exampleTest() = runBlocking { + somedir!!.put("somefile").write("hello".toByteArray()) + assertEquals("hello", somedir!!.get("somefile").readAll().decodeToString()) + } + + @Test + fun subdirExample() = runBlocking { + val dir1 = somedir!! / "tmp1" + dir1.put("somefile").write("hello".toByteArray()) + + val dir2 = somedir!! / "tmp1" + val data = dir2.get("somefile").readAll() + + assertEquals("hello", data.decodeToString()) + } +} \ No newline at end of file diff --git a/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Tests.kt b/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Tests.kt index ec5765e..f47e7fa 100644 --- a/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Tests.kt +++ b/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Tests.kt @@ -2,7 +2,7 @@ package space.kscience.snark.storage.local import kotlinx.coroutines.runBlocking -import space.kscience.snark.storage.Directory +import space.kscience.snark.storage.* import java.io.File import java.nio.file.Path import kotlin.io.path.* diff --git a/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/unzip/Tests.kt b/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/unzip/Tests.kt index 3b6e054..b567948 100644 --- a/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/unzip/Tests.kt +++ b/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/unzip/Tests.kt @@ -2,8 +2,7 @@ package space.kscience.snark.storage.unzip import kotlinx.coroutines.runBlocking -import space.kscience.snark.storage.Directory -import space.kscience.snark.storage.local.LocalDirectory +import space.kscience.snark.storage.* import space.kscience.snark.storage.local.localStorage import java.io.* import java.nio.file.Files