diff --git a/.gitignore b/.gitignore index 2e1f3e2..f06baf2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ build/ /logs/ !gradle/wrapper/gradle-wrapper.jar + +kotlin-js-store +*.iml \ No newline at end of file diff --git a/snark-storage-driver/build.gradle.kts b/snark-storage-driver/build.gradle.kts index addc596..6a2f967 100644 --- a/snark-storage-driver/build.gradle.kts +++ b/snark-storage-driver/build.gradle.kts @@ -12,4 +12,11 @@ dependencies { // s3 Driver dependency implementation("aws.sdk.kotlin:s3:$awsSdkVersion") + + testImplementation(kotlin("test")) + testImplementation("org.junit.jupiter:junit-jupiter:5.8.1") +} + +tasks.test { + useJUnitPlatform() } diff --git a/snark-storage-driver/snark-storage-driver.iml b/snark-storage-driver/snark-storage-driver.iml deleted file mode 100644 index 3200c88..0000000 --- a/snark-storage-driver/snark-storage-driver.iml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt index 4b26468..1b81fba 100644 --- a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt @@ -3,55 +3,60 @@ package space.kscience.snark.storage.local import space.kscience.snark.storage.Directory import space.kscience.snark.storage.FileReader import space.kscience.snark.storage.FileWriter +import java.io.File import java.nio.file.Path +import java.nio.file.attribute.PosixFilePermission import kotlin.io.path.* public fun localStorage(rootPath: Path): Directory { return LocalDirectory(rootPath, Path("")) } -private class LocalFile(private val path: Path) : FileReader, FileWriter { +internal class LocalFile(private val path: Path) : FileReader, FileWriter { override fun close() {} override suspend fun readAll(): ByteArray = path.readBytes() override suspend fun write(bytes: ByteArray) = path.writeBytes(bytes) } -private class LocalDirectory(private val root: Path, private val currentDir: Path) : Directory { +internal class LocalDirectory(private val root: Path, private val currentDir: Path) : Directory { private fun child(child: String): Path = root / currentDir / child private fun child(child: Path): Path = root / currentDir / child override fun close() {} - override suspend fun get(filename: String): FileReader = LocalFile(child(filename)) + override suspend fun get(filename: String): LocalFile = LocalFile(child(filename)) - override suspend fun get(filename: Path): FileReader = LocalFile(child(filename)) + override suspend fun get(filename: Path): LocalFile = LocalFile(child(filename)) override suspend fun create(filename: String, ignoreIfExists: Boolean) { + val dir = child(filename) + dir.parent.createDirectories() try { child(filename).createFile() - } catch (ex: FileAlreadyExistsException) { + } catch (ex: java.nio.file.FileAlreadyExistsException) { if (!ignoreIfExists) { throw ex } } } - override suspend fun put(filename: String): FileWriter = LocalFile(child(filename)) + override suspend fun put(filename: String): LocalFile = get(filename) - override suspend fun put(filename: Path): FileWriter = LocalFile(child(filename)) + override suspend fun put(filename: Path): LocalFile = get(filename) - override suspend fun getSubdir(path: Path): Directory = LocalDirectory(root, child(path)) - override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory { + override suspend fun getSubdir(path: Path): LocalDirectory = LocalDirectory(root, currentDir / path) + override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): LocalDirectory { val dir = child(dirname) + dir.parent.createDirectories() try { dir.createDirectory() - } catch (ex: FileAlreadyExistsException) { + } catch (ex: java.nio.file.FileAlreadyExistsException) { if (!ignoreIfExists) { throw ex } } - return LocalDirectory(root, dir) + return LocalDirectory(root, currentDir / dirname) } override val path: Path diff --git a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt index e7aff9c..5201930 100644 --- a/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt +++ b/snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt @@ -12,7 +12,11 @@ public suspend fun unzip(source_path: String, target: Directory) { val filename = zipEntry.name target.create(filename, true) val fos = target.put(filename) - val buffer = ByteArray(zipEntry.size.toInt()) + var sz = zipEntry.size.toInt() + if (sz == -1) { + sz = 1024 + } + val buffer = ByteArray(sz) zis.read(buffer) fos.write(buffer) fos.close() diff --git a/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Tests.kt b/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Tests.kt new file mode 100644 index 0000000..ec5765e --- /dev/null +++ b/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Tests.kt @@ -0,0 +1,105 @@ +package space.kscience.snark.storage.local + +import kotlinx.coroutines.runBlocking + +import space.kscience.snark.storage.Directory +import java.io.File +import java.nio.file.Path +import kotlin.io.path.* +import kotlin.test.* + +internal class LocalDriverTests { + + private var tempDir: Path? = null + private var testSample: Directory? = null + private val bytes = byteArrayOf(0, 1, 2, 3, 4, 5, 6, 7) + + @BeforeTest + fun setUp() { + tempDir = createTempDirectory() + testSample = localStorage(tempDir!!) + } + + @Test + fun testCreate() = runBlocking { + //folder is empty + assertEquals(0, tempDir!!.listDirectoryEntries().size) + + //create first file + testSample!!.create("tmp1") + val entries = tempDir!!.listDirectoryEntries() + assertEquals(1, entries.size) + assertEquals(tempDir!! / Path("tmp1"), entries.first()) + //assertTrue(!entries.first().isDirectory()) + + //create second file + testSample!!.create("tmp2") + assertEquals(2, tempDir!!.listDirectoryEntries().size) + + //check exception after duplication + try { + testSample!!.create("tmp1") + fail("shouldn't ignore duplicates here") + } catch (ex: java.nio.file.FileAlreadyExistsException) {} + + //check ignorance + try { + testSample!!.create("tmp1", true) + } catch (ex: java.nio.file.FileAlreadyExistsException) { + fail("should ignore duplicates here") + } + } + + @Test + fun testPutGet() = runBlocking { + testSample!!.create("tmp") + testSample!!.put("tmp").write(bytes) + assertContentEquals(bytes, testSample!!.get("tmp").readAll()) + } + + @Test + fun testCreateSubdir() = runBlocking { + //folder is empty + assertEquals(0, tempDir!!.listDirectoryEntries().size) + + //create first file + testSample!!.createSubdir("tmp1") + val entries = tempDir!!.listDirectoryEntries() + assertEquals(1, entries.size) + assertEquals(tempDir!! / Path("tmp1"), entries.first()) + assertTrue (entries.first().isDirectory()) + + //create second file + testSample!!.createSubdir("tmp2") + assertEquals(2, tempDir!!.listDirectoryEntries().size) + + //check exception after duplication + try { + testSample!!.createSubdir("tmp1") + fail("shouldn't ignore duplicates here") + } catch (ex: java.nio.file.FileAlreadyExistsException) {} + + //check ignorance + try { + testSample!!.createSubdir("tmp1", true) + } catch (ex: java.nio.file.FileAlreadyExistsException) { + fail("should ignore duplicates here") + } + assertTrue {true} + } + + @Test + fun testGetSubdir() = runBlocking { + testSample!!.createSubdir("tmp") + val pathStr = (Path("tmp") / "data.txt").toString() + testSample!!.create(pathStr) + testSample!!.put(pathStr).write(bytes) + val subdir = testSample!!.getSubdir(Path("tmp")) + assertContentEquals(bytes, subdir.get("data.txt").readAll()) + } + + @AfterTest + fun tearDown() { + tempDir!!.toFile().deleteRecursively() + } +} \ No newline at end of file diff --git a/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/unzip/Tests.kt b/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/unzip/Tests.kt new file mode 100644 index 0000000..3b6e054 --- /dev/null +++ b/snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/unzip/Tests.kt @@ -0,0 +1,109 @@ +package space.kscience.snark.storage.unzip + +import kotlinx.coroutines.runBlocking + +import space.kscience.snark.storage.Directory +import space.kscience.snark.storage.local.LocalDirectory +import space.kscience.snark.storage.local.localStorage +import java.io.* +import java.nio.file.Files +import java.nio.file.Path +import java.util.concurrent.TimeUnit +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream +import kotlin.io.path.* +import kotlin.test.* + +internal class UnzipTests { + + private var tempDir: Path? = null + + @BeforeTest + fun setUp() { + tempDir = createTempDirectory() + } + + private suspend fun makeFile(dir: Directory, filename: String, content: ByteArray) { + dir.create(filename) + dir.put(filename).write(content) + } + + private fun zipAll(directory: String, zipFile: String) { + val sourceFile = File(directory) + + ZipOutputStream(BufferedOutputStream( FileOutputStream(zipFile))).use { + zipFiles(it, sourceFile, File.separator) + it.closeEntry() + it.close() + } + } + + private fun zipFiles(zipOut: ZipOutputStream, sourceFile: File, parentDirname: String) { + + val data = ByteArray(2048) + + for (f in sourceFile.listFiles()) { + if (f.isDirectory) { + zipFiles(zipOut, f, parentDirname + f.name + File.separator) + } else { + FileInputStream(f).use { fi -> + BufferedInputStream(fi).use { origin -> + var path = parentDirname + f.name + val entry = ZipEntry(path.drop(1)) + entry.time = f.lastModified() + entry.size = f.length() + zipOut.putNextEntry(entry) + while (true) { + val readBytes = origin.read(data) + if (readBytes == -1) { + break + } + zipOut.write(data, 0, readBytes) + } + } + } + } + } + } + + + @Test + fun testUnzip() = runBlocking { + val dir: Directory = localStorage(tempDir!!) + val source = dir.createSubdir("source") + val target = dir.createSubdir("target") + val bytes1 = byteArrayOf(0, 1, 2, 3) + val bytes2 = byteArrayOf(1, 0, 3, 2) + val bytes3 = byteArrayOf(3, 2, 1, 0) + makeFile(source, "tmp1", bytes1) + makeFile(source, "tmp2", bytes2); + makeFile(source, (Path("tdir") / "tmp3").toString(), bytes3) + + dir.create("archive.zip") + val archive_path = (tempDir!! / Path("archive.zip")).toString() + + zipAll((tempDir!! / Path("source")).toString(), archive_path) + + unzip(archive_path, target) + + val targetPath = tempDir!! / Path("target") + println(targetPath) + val entries = targetPath.listDirectoryEntries() + + assertEquals(3, entries.size) + val exp_entries = listOf( + targetPath / Path("tmp1"), + targetPath / Path("tmp2"), + targetPath / Path("tdir")) + assertContentEquals(entries.sorted(), exp_entries.sorted()) + + val tdirEntries = (targetPath / Path("tdir")).listDirectoryEntries() + assertEquals(1, tdirEntries.size) + assertEquals(tdirEntries.first(), targetPath / Path("tdir") / Path("tmp3")) + } + + @AfterTest + fun tearDown() { + tempDir!!.toFile().deleteRecursively() + } +} \ No newline at end of file