SPC/snark
5
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge SNARK-MR-18: SNRK-68: Put creates file

Browse Source
This commit is contained in:
Kirill Grachev 2023-05-14 15:20:15 +00:00 committed by Space Cloud
commit 64bfc75222
No known key found for this signature in database
GPG Key ID: 2F4D45726235F749
9 changed files with 101 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/Driver.kt
View File

@ -1,35 +1,36 @@
package space.kscience.snark.storage
import java.nio.file.Path
import kotlin.io.path.Path
public interface Directory : AutoCloseable {
@Deprecated(
message = "Use Path, not String",
level = DeprecationLevel.WARNING,
)
public suspend fun get(filename: String): FileReader
// get file from subtree
public suspend fun get(filename: Path): FileReader
@Deprecated("Use put")
public suspend fun create(filename: String, ignoreIfExists: Boolean = false)
@Deprecated(
message = "Use Path, not String",
level = DeprecationLevel.WARNING,
)
public suspend fun put(filename: String): FileWriter
// put file to subtree
public suspend fun put(filename: Path): FileWriter
public suspend fun getSubdir(path: Path): Directory
@Deprecated("Directories are created on put")
public suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean = false): Directory
@Deprecated(
message = "Not a good idea",
level = DeprecationLevel.WARNING,
)
@Deprecated("Not a good idea")
public val path: Path
}
public suspend fun Directory.get(filename: String): FileReader = get(Path(filename))
public suspend fun Directory.put(filename: String): FileWriter = put(Path(filename))
public suspend operator fun Directory.div(path: Path): Directory = getSubdir(path)
public suspend operator fun Directory.div(path: String): Directory = getSubdir(Path(path))
public interface FileReader : AutoCloseable {
public suspend fun readAll(): ByteArray
}

33
snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt
View File

@ -3,9 +3,7 @@ package space.kscience.snark.storage.local
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.io.File
import java.nio.file.Path
import java.nio.file.attribute.PosixFilePermission
import kotlin.io.path.*
public fun localStorage(rootPath: Path): Directory {
@ -16,24 +14,32 @@ internal class LocalFile(private val path: Path) : FileReader, FileWriter {
override fun close() {}
override suspend fun readAll(): ByteArray = path.readBytes()
override suspend fun write(bytes: ByteArray) = path.writeBytes(bytes)
override suspend fun write(bytes: ByteArray) {
path.parent.createDirectories()
try {
path.createFile()
} catch (ex: Exception) {
// Do nothing
}
path.writeBytes(bytes)
}
}
internal class LocalDirectory(private val root: Path, private val currentDir: Path) : Directory {
private fun child(child: String): Path = root / currentDir / child
private fun child(child: Path): Path = root / currentDir / child
@Deprecated("Use Path, not String")
private fun realpath(child: String): Path = root / currentDir / child
private fun realpath(child: Path): Path = root / currentDir / child
override fun close() {}
override suspend fun get(filename: String): LocalFile = LocalFile(child(filename))
override suspend fun get(filename: Path): LocalFile = LocalFile(child(filename))
override suspend fun get(filename: Path): LocalFile = LocalFile(realpath(filename))
@Deprecated("Use put")
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
val dir = child(filename)
val dir = realpath(filename)
dir.parent.createDirectories()
try {
child(filename).createFile()
realpath(filename).createFile()
} catch (ex: java.nio.file.FileAlreadyExistsException) {
if (!ignoreIfExists) {
throw ex
@ -41,13 +47,13 @@ internal class LocalDirectory(private val root: Path, private val currentDir: Pa
}
}
override suspend fun put(filename: String): LocalFile = get(filename)
override suspend fun put(filename: Path): LocalFile = get(filename)
override suspend fun getSubdir(path: Path): LocalDirectory = LocalDirectory(root, currentDir / path)
@Deprecated("Directories are created on put")
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): LocalDirectory {
val dir = child(dirname)
val dir = realpath(dirname)
dir.parent.createDirectories()
try {
dir.createDirectory()
@ -59,6 +65,7 @@ internal class LocalDirectory(private val root: Path, private val currentDir: Pa
return LocalDirectory(root, currentDir / dirname)
}
@Deprecated("Not a good idea")
override val path: Path
get() = currentDir
}

11
snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt
View File

@ -5,34 +5,30 @@ import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.nio.file.Path
import kotlin.io.path.*
import kotlin.io.path.div
internal class S3Directory(
private val client: S3Client,
private val bucketName: String,
private val currentDir: Path,
) : Directory {
override suspend fun get(filename: String): FileReader =
S3FileReader(client, bucketName, currentDir / filename)
override suspend fun get(filename: Path): FileReader =
S3FileReader(client, bucketName, currentDir / filename)
@Deprecated("Use put")
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
if (!ignoreIfExists) {
TODO("could not check if file exists")
}
}
override suspend fun put(filename: String): FileWriter =
S3FileWriter(client, bucketName, currentDir / filename)
override suspend fun put(filename: Path): FileWriter =
S3FileWriter(client, bucketName, currentDir / filename)
override suspend fun getSubdir(path: Path): S3Directory =
S3Directory(client, bucketName, currentDir / path)
@Deprecated("Directories are created on put")
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): S3Directory =
if (!ignoreIfExists) {
TODO("could not check if directory exists")
@ -40,6 +36,7 @@ internal class S3Directory(
S3Directory(client, bucketName, currentDir / dirname)
}
@Deprecated("Not a good idea")
override val path: Path
get() = currentDir

9
snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt
View File

@ -8,11 +8,11 @@ import aws.smithy.kotlin.runtime.content.toByteArray
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.nio.file.Path
import kotlin.io.path.*
internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) : FileReader {
internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) :
FileReader {
override suspend fun readAll(): ByteArray {
val result = client.getObject(GetObjectRequest{
val result = client.getObject(GetObjectRequest {
bucket = bucketName
key = path.toString()
}) {
@ -25,7 +25,8 @@ internal class S3FileReader(private val client: S3Client, private val bucketName
}
}
internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) : FileWriter {
internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) :
FileWriter {
override suspend fun write(bytes: ByteArray) {
client.putObject {
bucket = bucketName

17
snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt
View File

@ -1,12 +1,13 @@
package space.kscience.snark.storage.s3
import aws.sdk.kotlin.services.s3.*
import aws.sdk.kotlin.services.s3.S3Client
import aws.sdk.kotlin.services.s3.createBucket
import aws.sdk.kotlin.services.s3.headBucket
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.lang.Exception
import java.nio.file.Path
import kotlin.io.path.*
import kotlin.io.path.Path
public fun s3Storage(client: S3Client): Directory =
S3Root(client)
@ -21,22 +22,16 @@ internal fun splitPathIntoBucketAndPath(path: Path): Pair<String, Path> {
}
internal class S3Root(private val client: S3Client) : Directory {
override suspend fun get(filename: String): FileReader {
throw NoSuchFileException(Path(filename).toFile())
}
override suspend fun get(filename: Path): FileReader {
throw NoSuchFileException(filename.toFile())
}
@Deprecated("Use put")
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
throw NoSuchFileException(Path(filename).toFile())
}
override suspend fun put(filename: String): FileWriter {
throw NoSuchFileException(Path(filename).toFile())
}
override suspend fun put(filename: Path): FileWriter {
throw NoSuchFileException(filename.toFile())
}
@ -51,6 +46,7 @@ internal class S3Root(private val client: S3Client) : Directory {
throw AccessDeniedException(path.toFile(), reason = ex.message)
}
@Deprecated("Directories are created on put")
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try {
val (bucketName, filePath) = splitPathIntoBucketAndPath(Path(dirname))
client.createBucket {
@ -61,6 +57,7 @@ internal class S3Root(private val client: S3Client) : Directory {
throw AccessDeniedException(Path(dirname).toFile(), reason = ex.message)
}
@Deprecated("Not a good idea")
override val path: Path
get() = Path("")

2
snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt
View File

@ -1,6 +1,6 @@
package space.kscience.snark.storage.unzip
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.*
import java.io.FileInputStream
import java.util.zip.ZipInputStream

47
snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Example.kt Normal file
View File

@ -0,0 +1,47 @@
package space.kscience.snark.storage.local
import kotlinx.coroutines.runBlocking
import org.junit.jupiter.api.BeforeAll
import space.kscience.snark.storage.*
import java.nio.file.Path
import kotlin.io.path.createTempDirectory
import kotlin.io.path.deleteExisting
import kotlin.test.AfterTest
import kotlin.test.BeforeTest
import kotlin.test.Test
import kotlin.io.path.*
import kotlin.test.assertEquals
class Example {
var tempDir: Path? = null
var somedir: Directory? = null
@BeforeTest
fun setUp() {
tempDir = createTempDirectory()
somedir = localStorage(tempDir!!)
}
@AfterTest
fun tearDown() {
tempDir!!.toFile().deleteRecursively()
somedir = null
}
@Test
fun exampleTest() = runBlocking {
somedir!!.put("somefile").write("hello".toByteArray())
assertEquals("hello", somedir!!.get("somefile").readAll().decodeToString())
}
@Test
fun subdirExample() = runBlocking {
val dir1 = somedir!! / "tmp1"
dir1.put("somefile").write("hello".toByteArray())
val dir2 = somedir!! / "tmp1"
val data = dir2.get("somefile").readAll()
assertEquals("hello", data.decodeToString())
}
}

2
snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Tests.kt
View File

@ -2,7 +2,7 @@ package space.kscience.snark.storage.local
import kotlinx.coroutines.runBlocking
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.*
import java.io.File
import java.nio.file.Path
import kotlin.io.path.*

3
snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/unzip/Tests.kt
View File

@ -2,8 +2,7 @@ package space.kscience.snark.storage.unzip
import kotlinx.coroutines.runBlocking
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.local.LocalDirectory
import space.kscience.snark.storage.*
import space.kscience.snark.storage.local.localStorage
import java.io.*
import java.nio.file.Files