Merge SNARK-MR-16: SNRK-77: unit_testing

.gitignore

@@ -4,3 +4,6 @@ build/
 /logs/
 
 !gradle/wrapper/gradle-wrapper.jar
+
+kotlin-js-store
+*.iml

snark-storage-driver/build.gradle.kts

@@ -12,4 +12,11 @@ dependencies {
 
     // s3 Driver dependency
     implementation("aws.sdk.kotlin:s3:$awsSdkVersion")
+
+    testImplementation(kotlin("test"))
+    testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
+}
+
+tasks.test {
+    useJUnitPlatform()
 }

snark-storage-driver/snark-storage-driver.iml

@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="JAVA_MODULE" version="4">
-  <component name="NewModuleRootManager" inherit-compiler-output="true">
-    <exclude-output />
-    <content url="file://$MODULE_DIR$">
-      <sourceFolder url="file://$MODULE_DIR$/src/main/kotlin" isTestSource="false" />
-    </content>
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-    <orderEntry type="library" name="KotlinJavaRuntime" level="project" />
-  </component>
-</module>

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt

@@ -3,55 +3,60 @@ package space.kscience.snark.storage.local
 import space.kscience.snark.storage.Directory
 import space.kscience.snark.storage.FileReader
 import space.kscience.snark.storage.FileWriter
+import java.io.File
 import java.nio.file.Path
+import java.nio.file.attribute.PosixFilePermission
 import kotlin.io.path.*
 
 public fun localStorage(rootPath: Path): Directory {
     return LocalDirectory(rootPath, Path(""))
 }
 
-private class LocalFile(private val path: Path) : FileReader, FileWriter {
+internal class LocalFile(private val path: Path) : FileReader, FileWriter {
     override fun close() {}
     override suspend fun readAll(): ByteArray = path.readBytes()
 
     override suspend fun write(bytes: ByteArray) = path.writeBytes(bytes)
 }
 
-private class LocalDirectory(private val root: Path, private val currentDir: Path) : Directory {
+internal class LocalDirectory(private val root: Path, private val currentDir: Path) : Directory {
     private fun child(child: String): Path = root / currentDir / child
     private fun child(child: Path): Path = root / currentDir / child
 
     override fun close() {}
 
-    override suspend fun get(filename: String): FileReader = LocalFile(child(filename))
+    override suspend fun get(filename: String): LocalFile = LocalFile(child(filename))
 
-    override suspend fun get(filename: Path): FileReader = LocalFile(child(filename))
+    override suspend fun get(filename: Path): LocalFile = LocalFile(child(filename))
 
     override suspend fun create(filename: String, ignoreIfExists: Boolean) {
+        val dir = child(filename)
+        dir.parent.createDirectories()
         try {
             child(filename).createFile()
-        } catch (ex: FileAlreadyExistsException) {
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {
             if (!ignoreIfExists) {
                 throw ex
             }
         }
     }
 
-    override suspend fun put(filename: String): FileWriter = LocalFile(child(filename))
+    override suspend fun put(filename: String): LocalFile = get(filename)
 
-    override suspend fun put(filename: Path): FileWriter = LocalFile(child(filename))
+    override suspend fun put(filename: Path): LocalFile = get(filename)
 
-    override suspend fun getSubdir(path: Path): Directory = LocalDirectory(root, child(path))
+    override suspend fun getSubdir(path: Path): LocalDirectory = LocalDirectory(root, currentDir / path)
-    override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory {
+    override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): LocalDirectory {
         val dir = child(dirname)
+        dir.parent.createDirectories()
         try {
             dir.createDirectory()
-        } catch (ex: FileAlreadyExistsException) {
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {
             if (!ignoreIfExists) {
                 throw ex
             }
         }
-        return LocalDirectory(root, dir)
+        return LocalDirectory(root, currentDir / dirname)
     }
 
     override val path: Path

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt

@@ -12,7 +12,11 @@ public suspend fun unzip(source_path: String, target: Directory) {
             val filename = zipEntry.name
             target.create(filename, true)
             val fos = target.put(filename)
-            val buffer = ByteArray(zipEntry.size.toInt())
+            var sz = zipEntry.size.toInt()
+            if (sz == -1) {
+                sz = 1024
+            }
+            val buffer = ByteArray(sz)
             zis.read(buffer)
             fos.write(buffer)
             fos.close()

snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Tests.kt

@@ -0,0 +1,105 @@
+package space.kscience.snark.storage.local
+
+import kotlinx.coroutines.runBlocking
+
+import space.kscience.snark.storage.Directory
+import java.io.File
+import java.nio.file.Path
+import kotlin.io.path.*
+import kotlin.test.*
+
+internal class LocalDriverTests {
+
+    private var tempDir: Path? = null
+    private var testSample: Directory? = null
+    private val bytes = byteArrayOf(0, 1, 2, 3, 4, 5, 6, 7)
+
+    @BeforeTest
+    fun setUp() {
+        tempDir = createTempDirectory()
+        testSample = localStorage(tempDir!!)
+    }
+
+    @Test
+    fun testCreate() = runBlocking {
+        //folder is empty
+        assertEquals(0, tempDir!!.listDirectoryEntries().size)
+
+        //create first file
+        testSample!!.create("tmp1")
+        val entries = tempDir!!.listDirectoryEntries()
+        assertEquals(1, entries.size)
+        assertEquals(tempDir!! / Path("tmp1"), entries.first())
+        //assertTrue(!entries.first().isDirectory())
+
+        //create second file
+        testSample!!.create("tmp2")
+        assertEquals(2, tempDir!!.listDirectoryEntries().size)
+
+        //check exception after duplication
+        try {
+            testSample!!.create("tmp1")
+            fail("shouldn't ignore duplicates here")
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {}
+
+        //check ignorance
+        try {
+            testSample!!.create("tmp1", true)
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {
+            fail("should ignore duplicates here")
+        }
+    }
+
+    @Test
+    fun testPutGet() = runBlocking {
+        testSample!!.create("tmp")
+        testSample!!.put("tmp").write(bytes)
+        assertContentEquals(bytes, testSample!!.get("tmp").readAll())
+    }
+
+    @Test
+    fun testCreateSubdir() = runBlocking {
+        //folder is empty
+        assertEquals(0, tempDir!!.listDirectoryEntries().size)
+
+        //create first file
+        testSample!!.createSubdir("tmp1")
+        val entries = tempDir!!.listDirectoryEntries()
+        assertEquals(1, entries.size)
+        assertEquals(tempDir!! / Path("tmp1"), entries.first())
+        assertTrue (entries.first().isDirectory())
+
+        //create second file
+        testSample!!.createSubdir("tmp2")
+        assertEquals(2, tempDir!!.listDirectoryEntries().size)
+
+        //check exception after duplication
+        try {
+            testSample!!.createSubdir("tmp1")
+            fail("shouldn't ignore duplicates here")
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {}
+
+        //check ignorance
+        try {
+            testSample!!.createSubdir("tmp1", true)
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {
+            fail("should ignore duplicates here")
+        }
+        assertTrue {true}
+    }
+
+    @Test
+    fun testGetSubdir() = runBlocking {
+        testSample!!.createSubdir("tmp")
+        val pathStr = (Path("tmp") / "data.txt").toString()
+        testSample!!.create(pathStr)
+        testSample!!.put(pathStr).write(bytes)
+        val subdir = testSample!!.getSubdir(Path("tmp"))
+        assertContentEquals(bytes, subdir.get("data.txt").readAll())
+    }
+
+    @AfterTest
+    fun tearDown() {
+        tempDir!!.toFile().deleteRecursively()
+    }
+}

snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/unzip/Tests.kt

@@ -0,0 +1,109 @@
+package space.kscience.snark.storage.unzip
+
+import kotlinx.coroutines.runBlocking
+
+import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.local.LocalDirectory
+import space.kscience.snark.storage.local.localStorage
+import java.io.*
+import java.nio.file.Files
+import java.nio.file.Path
+import java.util.concurrent.TimeUnit
+import java.util.zip.ZipEntry
+import java.util.zip.ZipOutputStream
+import kotlin.io.path.*
+import kotlin.test.*
+
+internal class UnzipTests {
+
+    private var tempDir: Path? = null
+
+    @BeforeTest
+    fun setUp() {
+        tempDir = createTempDirectory()
+    }
+
+    private suspend fun makeFile(dir: Directory, filename: String, content: ByteArray) {
+        dir.create(filename)
+        dir.put(filename).write(content)
+    }
+
+    private fun zipAll(directory: String, zipFile: String) {
+        val sourceFile = File(directory)
+
+        ZipOutputStream(BufferedOutputStream( FileOutputStream(zipFile))).use {
+            zipFiles(it, sourceFile, File.separator)
+            it.closeEntry()
+            it.close()
+        }
+    }
+
+    private fun zipFiles(zipOut: ZipOutputStream, sourceFile: File, parentDirname: String) {
+
+        val data = ByteArray(2048)
+
+        for (f in sourceFile.listFiles()) {
+            if (f.isDirectory) {
+                zipFiles(zipOut, f, parentDirname + f.name + File.separator)
+            } else {
+                FileInputStream(f).use { fi ->
+                    BufferedInputStream(fi).use { origin ->
+                        var path = parentDirname + f.name
+                        val entry = ZipEntry(path.drop(1))
+                        entry.time = f.lastModified()
+                        entry.size = f.length()
+                        zipOut.putNextEntry(entry)
+                        while (true) {
+                            val readBytes = origin.read(data)
+                            if (readBytes == -1) {
+                                break
+                            }
+                            zipOut.write(data, 0, readBytes)
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+
+    @Test
+    fun testUnzip() = runBlocking {
+        val dir: Directory = localStorage(tempDir!!)
+        val source = dir.createSubdir("source")
+        val target = dir.createSubdir("target")
+        val bytes1 = byteArrayOf(0, 1, 2, 3)
+        val bytes2 = byteArrayOf(1, 0, 3, 2)
+        val bytes3 = byteArrayOf(3, 2, 1, 0)
+        makeFile(source, "tmp1", bytes1)
+        makeFile(source, "tmp2", bytes2);
+        makeFile(source, (Path("tdir") / "tmp3").toString(), bytes3)
+
+        dir.create("archive.zip")
+        val archive_path = (tempDir!! / Path("archive.zip")).toString()
+
+        zipAll((tempDir!! / Path("source")).toString(), archive_path)
+
+        unzip(archive_path, target)
+
+        val targetPath = tempDir!! / Path("target")
+        println(targetPath)
+        val entries = targetPath.listDirectoryEntries()
+
+        assertEquals(3, entries.size)
+        val exp_entries = listOf(
+            targetPath / Path("tmp1"),
+            targetPath / Path("tmp2"),
+            targetPath / Path("tdir"))
+        assertContentEquals(entries.sorted(), exp_entries.sorted())
+
+        val tdirEntries = (targetPath / Path("tdir")).listDirectoryEntries()
+        assertEquals(1, tdirEntries.size)
+        assertEquals(tdirEntries.first(), targetPath / Path("tdir") / Path("tmp3"))
+    }
+
+    @AfterTest
+    fun tearDown() {
+        tempDir!!.toFile().deleteRecursively()
+    }
+}