Merge SNARK-MR-18: SNRK-68: Put creates file

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt

@@ -1,35 +1,36 @@
 package space.kscience.snark.storage
 
 import java.nio.file.Path
+import kotlin.io.path.Path
 
 public interface Directory : AutoCloseable {
-    @Deprecated(
-        message = "Use Path, not String",
-        level = DeprecationLevel.WARNING,
-    )
-    public suspend fun get(filename: String): FileReader
     // get file from subtree
     public suspend fun get(filename: Path): FileReader
 
+    @Deprecated("Use put")
     public suspend fun create(filename: String, ignoreIfExists: Boolean = false)
-    @Deprecated(
+
-        message = "Use Path, not String",
-        level = DeprecationLevel.WARNING,
-    )
-    public suspend fun put(filename: String): FileWriter
     // put file to subtree
     public suspend fun put(filename: Path): FileWriter
 
     public suspend fun getSubdir(path: Path): Directory
+
+    @Deprecated("Directories are created on put")
     public suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean = false): Directory
 
-    @Deprecated(
+    @Deprecated("Not a good idea")
-        message = "Not a good idea",
-        level = DeprecationLevel.WARNING,
-    )
     public val path: Path
 }
 
+
+public suspend fun Directory.get(filename: String): FileReader = get(Path(filename))
+
+public suspend fun Directory.put(filename: String): FileWriter = put(Path(filename))
+
+public suspend operator fun Directory.div(path: Path): Directory = getSubdir(path)
+
+public suspend operator fun Directory.div(path: String): Directory = getSubdir(Path(path))
+
 public interface FileReader : AutoCloseable {
     public suspend fun readAll(): ByteArray
 }

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt

@@ -3,9 +3,7 @@ package space.kscience.snark.storage.local
 import space.kscience.snark.storage.Directory
 import space.kscience.snark.storage.FileReader
 import space.kscience.snark.storage.FileWriter
-import java.io.File
 import java.nio.file.Path
-import java.nio.file.attribute.PosixFilePermission
 import kotlin.io.path.*
 
 public fun localStorage(rootPath: Path): Directory {
@@ -16,24 +14,32 @@ internal class LocalFile(private val path: Path) : FileReader, FileWriter {
     override fun close() {}
     override suspend fun readAll(): ByteArray = path.readBytes()
 
-    override suspend fun write(bytes: ByteArray) = path.writeBytes(bytes)
+    override suspend fun write(bytes: ByteArray) {
+        path.parent.createDirectories()
+        try {
+            path.createFile()
+        } catch (ex: Exception) {
+            // Do nothing
+        }
+        path.writeBytes(bytes)
+    }
 }
 
 internal class LocalDirectory(private val root: Path, private val currentDir: Path) : Directory {
-    private fun child(child: String): Path = root / currentDir / child
+    @Deprecated("Use Path, not String")
-    private fun child(child: Path): Path = root / currentDir / child
+    private fun realpath(child: String): Path = root / currentDir / child
+    private fun realpath(child: Path): Path = root / currentDir / child
 
     override fun close() {}
 
-    override suspend fun get(filename: String): LocalFile = LocalFile(child(filename))
+    override suspend fun get(filename: Path): LocalFile = LocalFile(realpath(filename))
-
-    override suspend fun get(filename: Path): LocalFile = LocalFile(child(filename))
 
+    @Deprecated("Use put")
     override suspend fun create(filename: String, ignoreIfExists: Boolean) {
-        val dir = child(filename)
+        val dir = realpath(filename)
         dir.parent.createDirectories()
         try {
-            child(filename).createFile()
+            realpath(filename).createFile()
         } catch (ex: java.nio.file.FileAlreadyExistsException) {
             if (!ignoreIfExists) {
                 throw ex
@@ -41,13 +47,13 @@ internal class LocalDirectory(private val root: Path, private val currentDir: Pa
         }
     }
 
-    override suspend fun put(filename: String): LocalFile = get(filename)
-
     override suspend fun put(filename: Path): LocalFile = get(filename)
 
     override suspend fun getSubdir(path: Path): LocalDirectory = LocalDirectory(root, currentDir / path)
+
+    @Deprecated("Directories are created on put")
     override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): LocalDirectory {
-        val dir = child(dirname)
+        val dir = realpath(dirname)
         dir.parent.createDirectories()
         try {
             dir.createDirectory()
@@ -59,6 +65,7 @@ internal class LocalDirectory(private val root: Path, private val currentDir: Pa
         return LocalDirectory(root, currentDir / dirname)
     }
 
+    @Deprecated("Not a good idea")
     override val path: Path
         get() = currentDir
 }

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt

@@ -5,34 +5,30 @@ import space.kscience.snark.storage.Directory
 import space.kscience.snark.storage.FileReader
 import space.kscience.snark.storage.FileWriter
 import java.nio.file.Path
-import kotlin.io.path.*
+import kotlin.io.path.div
 
 internal class S3Directory(
     private val client: S3Client,
     private val bucketName: String,
     private val currentDir: Path,
 ) : Directory {
-    override suspend fun get(filename: String): FileReader =
-        S3FileReader(client, bucketName, currentDir / filename)
-
     override suspend fun get(filename: Path): FileReader =
         S3FileReader(client, bucketName, currentDir / filename)
 
+    @Deprecated("Use put")
     override suspend fun create(filename: String, ignoreIfExists: Boolean) {
         if (!ignoreIfExists) {
             TODO("could not check if file exists")
         }
     }
 
-    override suspend fun put(filename: String): FileWriter =
-        S3FileWriter(client, bucketName, currentDir / filename)
-
     override suspend fun put(filename: Path): FileWriter =
         S3FileWriter(client, bucketName, currentDir / filename)
 
     override suspend fun getSubdir(path: Path): S3Directory =
         S3Directory(client, bucketName, currentDir / path)
 
+    @Deprecated("Directories are created on put")
     override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): S3Directory =
         if (!ignoreIfExists) {
             TODO("could not check if directory exists")
@@ -40,6 +36,7 @@ internal class S3Directory(
             S3Directory(client, bucketName, currentDir / dirname)
         }
 
+    @Deprecated("Not a good idea")
     override val path: Path
         get() = currentDir
 

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt

@@ -8,11 +8,11 @@ import aws.smithy.kotlin.runtime.content.toByteArray
 import space.kscience.snark.storage.FileReader
 import space.kscience.snark.storage.FileWriter
 import java.nio.file.Path
-import kotlin.io.path.*
 
-internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) : FileReader {
+internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) :
+    FileReader {
     override suspend fun readAll(): ByteArray {
-        val result = client.getObject(GetObjectRequest{
+        val result = client.getObject(GetObjectRequest {
             bucket = bucketName
             key = path.toString()
         }) {
@@ -25,7 +25,8 @@ internal class S3FileReader(private val client: S3Client, private val bucketName
     }
 }
 
-internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) : FileWriter {
+internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) :
+    FileWriter {
     override suspend fun write(bytes: ByteArray) {
         client.putObject {
             bucket = bucketName

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt

@@ -1,12 +1,13 @@
 package space.kscience.snark.storage.s3
 
-import aws.sdk.kotlin.services.s3.*
+import aws.sdk.kotlin.services.s3.S3Client
+import aws.sdk.kotlin.services.s3.createBucket
+import aws.sdk.kotlin.services.s3.headBucket
 import space.kscience.snark.storage.Directory
 import space.kscience.snark.storage.FileReader
 import space.kscience.snark.storage.FileWriter
-import java.lang.Exception
 import java.nio.file.Path
-import kotlin.io.path.*
+import kotlin.io.path.Path
 
 public fun s3Storage(client: S3Client): Directory =
     S3Root(client)
@@ -21,22 +22,16 @@ internal fun splitPathIntoBucketAndPath(path: Path): Pair<String, Path> {
 }
 
 internal class S3Root(private val client: S3Client) : Directory {
-    override suspend fun get(filename: String): FileReader {
-        throw NoSuchFileException(Path(filename).toFile())
-    }
 
     override suspend fun get(filename: Path): FileReader {
         throw NoSuchFileException(filename.toFile())
     }
 
+    @Deprecated("Use put")
     override suspend fun create(filename: String, ignoreIfExists: Boolean) {
         throw NoSuchFileException(Path(filename).toFile())
     }
 
-    override suspend fun put(filename: String): FileWriter {
-        throw NoSuchFileException(Path(filename).toFile())
-    }
-
     override suspend fun put(filename: Path): FileWriter {
         throw NoSuchFileException(filename.toFile())
     }
@@ -51,6 +46,7 @@ internal class S3Root(private val client: S3Client) : Directory {
         throw AccessDeniedException(path.toFile(), reason = ex.message)
     }
 
+    @Deprecated("Directories are created on put")
     override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try {
         val (bucketName, filePath) = splitPathIntoBucketAndPath(Path(dirname))
         client.createBucket {
@@ -61,6 +57,7 @@ internal class S3Root(private val client: S3Client) : Directory {
         throw AccessDeniedException(Path(dirname).toFile(), reason = ex.message)
     }
 
+    @Deprecated("Not a good idea")
     override val path: Path
         get() = Path("")
 

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt

@@ -1,6 +1,6 @@
 package space.kscience.snark.storage.unzip
 
-import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.*
 import java.io.FileInputStream
 import java.util.zip.ZipInputStream
 

snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Example.kt

@@ -0,0 +1,47 @@
+package space.kscience.snark.storage.local
+
+import kotlinx.coroutines.runBlocking
+import org.junit.jupiter.api.BeforeAll
+import space.kscience.snark.storage.*
+import java.nio.file.Path
+import kotlin.io.path.createTempDirectory
+import kotlin.io.path.deleteExisting
+import kotlin.test.AfterTest
+import kotlin.test.BeforeTest
+import kotlin.test.Test
+import kotlin.io.path.*
+import kotlin.test.assertEquals
+
+class Example {
+    var tempDir: Path? = null
+    var somedir: Directory? = null
+
+    @BeforeTest
+    fun setUp() {
+        tempDir = createTempDirectory()
+        somedir = localStorage(tempDir!!)
+    }
+
+    @AfterTest
+    fun tearDown() {
+        tempDir!!.toFile().deleteRecursively()
+        somedir = null
+    }
+
+    @Test
+    fun exampleTest() = runBlocking {
+        somedir!!.put("somefile").write("hello".toByteArray())
+        assertEquals("hello", somedir!!.get("somefile").readAll().decodeToString())
+    }
+
+    @Test
+    fun subdirExample() = runBlocking {
+        val dir1 = somedir!! / "tmp1"
+        dir1.put("somefile").write("hello".toByteArray())
+
+        val dir2 = somedir!! / "tmp1"
+        val data = dir2.get("somefile").readAll()
+
+        assertEquals("hello", data.decodeToString())
+    }
+}

snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Tests.kt

@@ -2,7 +2,7 @@ package space.kscience.snark.storage.local
 
 import kotlinx.coroutines.runBlocking
 
-import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.*
 import java.io.File
 import java.nio.file.Path
 import kotlin.io.path.*

snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/unzip/Tests.kt

@@ -2,8 +2,7 @@ package space.kscience.snark.storage.unzip
 
 import kotlinx.coroutines.runBlocking
 
-import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.*
-import space.kscience.snark.storage.local.LocalDirectory
 import space.kscience.snark.storage.local.localStorage
 import java.io.*
 import java.nio.file.Files