SNRK-77: added tests and slightly changed some code

snark-storage-driver/build.gradle.kts

@@ -12,4 +12,11 @@ dependencies {
 
     // s3 Driver dependency
     implementation("aws.sdk.kotlin:s3:$awsSdkVersion")
+
+    testImplementation(kotlin("test"))
+    testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
+}
+
+tasks.test {
+    useJUnitPlatform()
 }

snark-storage-driver/snark-storage-driver.iml

@@ -8,5 +8,21 @@
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
     <orderEntry type="library" name="KotlinJavaRuntime" level="project" />
+    <orderEntry type="module-library" scope="TEST">
+      <library name="JUnit5.8.1">
+        <CLASSES>
+          <root url="jar://$MAVEN_REPOSITORY$/org/junit/jupiter/junit-jupiter/5.8.1/junit-jupiter-5.8.1.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/org/junit/jupiter/junit-jupiter-api/5.8.1/junit-jupiter-api-5.8.1.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/org/opentest4j/opentest4j/1.2.0/opentest4j-1.2.0.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/org/junit/platform/junit-platform-commons/1.8.1/junit-platform-commons-1.8.1.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/org/apiguardian/apiguardian-api/1.1.2/apiguardian-api-1.1.2.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/org/junit/jupiter/junit-jupiter-params/5.8.1/junit-jupiter-params-5.8.1.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/org/junit/jupiter/junit-jupiter-engine/5.8.1/junit-jupiter-engine-5.8.1.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/org/junit/platform/junit-platform-engine/1.8.1/junit-platform-engine-1.8.1.jar!/" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES />
+      </library>
+    </orderEntry>
   </component>
 </module>

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt

@@ -3,21 +3,23 @@ package space.kscience.snark.storage.local
 import space.kscience.snark.storage.Directory
 import space.kscience.snark.storage.FileReader
 import space.kscience.snark.storage.FileWriter
+import java.io.File
 import java.nio.file.Path
+import java.nio.file.attribute.PosixFilePermission
 import kotlin.io.path.*
 
 public fun localStorage(rootPath: Path): Directory {
     return LocalDirectory(rootPath)
 }
 
-private class LocalFile(private val path: Path) : FileReader, FileWriter {
+internal class LocalFile(private val path: Path) : FileReader, FileWriter {
     override fun close() {}
     override suspend fun readAll(): ByteArray = path.readBytes()
 
     override suspend fun write(bytes: ByteArray) = path.writeBytes(bytes)
 }
 
-private class LocalDirectory(private val path: Path) : Directory {
+internal class LocalDirectory(private val path: Path) : Directory {
     private fun child(child: String): Path = path / child
     private fun child(child: Path): Path = path / child
 
@@ -26,23 +28,38 @@ private class LocalDirectory(private val path: Path) : Directory {
     override suspend fun get(filename: String): FileReader = LocalFile(child(filename))
 
     override suspend fun create(filename: String, ignoreIfExists: Boolean) {
+        val parts = filename.split(File.separator)
+        var cdir = this
+        for (i in 0..(parts.size - 1)) {
+            cdir = cdir.createSubdir(parts[i], true)
+        }
         try {
-            child(filename).createFile()
+            val nfile = cdir.child(parts.last()).createFile()
-        } catch (ex: FileAlreadyExistsException) {
+            /*
+            nfile.toFile().setReadable(true)
+
+            nfile.toFile().setWritable(true)
+            nfile.toFile().setExecutable(true)
+            */
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {
             if (!ignoreIfExists) {
                 throw ex
             }
         }
     }
 
-    override suspend fun put(filename: String): FileWriter = LocalFile(child(filename))
+    override suspend fun put(filename: String): FileWriter {
+        val tmp = child(filename)
+        //tmp.toFile().setWritable(true)
+        return LocalFile(tmp)
+    }
 
     override suspend fun getSubdir(path: Path): Directory = LocalDirectory(child(path))
-    override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory {
+    override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): LocalDirectory {
         val dir = child(dirname)
         try {
             dir.createDirectory()
-        } catch (ex: FileAlreadyExistsException) {
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {
             if (!ignoreIfExists) {
                 throw ex
             }

snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt

@@ -12,7 +12,11 @@ public suspend fun unzip(source_path: String, target: Directory) {
             val filename = zipEntry.name
             target.create(filename, true)
             val fos = target.put(filename)
-            val buffer = ByteArray(zipEntry.size.toInt())
+            var sz = zipEntry.size.toInt()
+            if (sz == -1) {
+                sz = 1024
+            }
+            val buffer = ByteArray(sz)
             zis.read(buffer)
             fos.write(buffer)
             fos.close()

snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Tests.kt

@@ -0,0 +1,103 @@
+package space.kscience.snark.storage.local
+
+import kotlinx.coroutines.runBlocking
+
+import space.kscience.snark.storage.Directory
+import java.nio.file.Path
+import kotlin.io.path.*
+import kotlin.test.*
+
+internal class LocalDriverTests {
+
+    private var tempDir: Path? = null
+    private var testSample: Directory? = null
+    private val bytes = byteArrayOf(0, 1, 2, 3, 4, 5, 6, 7)
+
+    @BeforeTest
+    fun setUp() {
+        tempDir = createTempDirectory()
+        testSample = LocalDirectory(tempDir!!)
+    }
+
+    @Test
+    fun testCreate() = runBlocking {
+        //folder is empty
+        assertEquals(0, tempDir!!.listDirectoryEntries().size)
+
+        //create first file
+        testSample!!.create("tmp1")
+        val entries = tempDir!!.listDirectoryEntries()
+        assertEquals(1, entries.size)
+        assertEquals(tempDir!! / Path("tmp1"), entries.first())
+        assertTrue(!entries.first().isDirectory())
+
+        //create second file
+        testSample!!.create("tmp2")
+        assertEquals(2, tempDir!!.listDirectoryEntries().size)
+
+        //check exception after duplication
+        try {
+            testSample!!.create("tmp1")
+            fail("shouldn't ignore duplicates here")
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {}
+
+        //check ignorance
+        try {
+            testSample!!.create("tmp1", true)
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {
+            fail("should ignore duplicates here")
+        }
+    }
+
+    @Test
+    fun testPutGet() = runBlocking {
+        testSample!!.create("tmp")
+        testSample!!.put("tmp").write(bytes)
+        assertContentEquals(bytes, testSample!!.get("tmp").readAll())
+    }
+
+    @Test
+    fun testCreateSubdir() = runBlocking {
+        //folder is empty
+        assertEquals(0, tempDir!!.listDirectoryEntries().size)
+
+        //create first file
+        testSample!!.createSubdir("tmp1")
+        val entries = tempDir!!.listDirectoryEntries()
+        assertEquals(1, entries.size)
+        assertEquals(tempDir!! / Path("tmp1"), entries.first())
+        assertTrue (entries.first().isDirectory())
+
+        //create second file
+        testSample!!.createSubdir("tmp2")
+        assertEquals(2, tempDir!!.listDirectoryEntries().size)
+
+        //check exception after duplication
+        try {
+            testSample!!.createSubdir("tmp1")
+            fail("shouldn't ignore duplicates here")
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {}
+
+        //check ignorance
+        try {
+            testSample!!.createSubdir("tmp1", true)
+        } catch (ex: java.nio.file.FileAlreadyExistsException) {
+            fail("should ignore duplicates here")
+        }
+        assertTrue {true}
+    }
+
+    @Test
+    fun testGetSubdir() = runBlocking {
+        testSample!!.createSubdir("tmp")
+        testSample!!.create("tmp/data")
+        testSample!!.put("tmp/data").write(bytes)
+        val subdir = testSample!!.getSubdir(Path("tmp"))
+        assertContentEquals(bytes, subdir.get("data").readAll())
+    }
+
+    @AfterTest
+    fun tearDown() {
+        tempDir!!.toFile().deleteRecursively()
+    }
+}

snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/unzip/Tests.kt

@@ -0,0 +1,110 @@
+package space.kscience.snark.storage.unzip
+
+import kotlinx.coroutines.runBlocking
+
+import space.kscience.snark.storage.Directory
+import space.kscience.snark.storage.local.LocalDirectory
+import java.io.*
+import java.nio.file.Files
+import java.nio.file.Path
+import java.util.concurrent.TimeUnit
+import java.util.zip.ZipEntry
+import java.util.zip.ZipOutputStream
+import kotlin.io.path.*
+import kotlin.test.*
+
+internal class UnzipTests {
+
+    private var tempDir: Path? = null
+
+    @BeforeTest
+    fun setUp() {
+        tempDir = createTempDirectory()
+    }
+
+    private suspend fun makeFile(dir: Directory, filename: String, content: ByteArray) {
+        dir.create(filename)
+
+        val writter = dir.put(filename)
+        if (!(tempDir!! / Path("source") / Path(filename)).isRegularFile()) {
+            println("shit")
+        }
+        writter.write(content)
+    }
+
+    private fun zipAll(directory: String, zipFile: String) {
+        val sourceFile = File(directory)
+
+        ZipOutputStream(BufferedOutputStream( FileOutputStream(zipFile))).use {
+            it.use {
+                zipFiles(it, sourceFile, "")
+            }
+        }
+    }
+
+    private fun zipFiles(zipOut: ZipOutputStream, sourceFile: File, parentDirPath: String) {
+
+        val data = ByteArray(2048)
+
+        for (f in sourceFile.listFiles()) {
+
+            if (f.isDirectory) {
+                val entry = ZipEntry(f.name + File.separator)
+                entry.time = f.lastModified()
+                entry.isDirectory
+                entry.size = f.length()
+                zipOut.putNextEntry(entry)
+                zipFiles(zipOut, f, f.name)
+            } else {
+                if (!f.name.contains(".zip")) { //If folder contains a file with extension ".zip", skip it
+                    FileInputStream(f).use { fi ->
+                        BufferedInputStream(fi).use { origin ->
+                            val path = parentDirPath + File.separator + f.name
+                            val entry = ZipEntry(path)
+                            entry.time = f.lastModified()
+                            entry.isDirectory
+                            entry.size = f.length()
+                            zipOut.putNextEntry(entry)
+                            while (true) {
+                                val readBytes = origin.read(data)
+                                if (readBytes == -1) {
+                                    break
+                                }
+                                zipOut.write(data, 0, readBytes)
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+
+    @Test
+    fun testUnzip() = runBlocking {
+        val dir: Directory = LocalDirectory(tempDir!!)
+        val source = dir.createSubdir("source")
+        val target = dir.createSubdir("target")
+        val bytes1 = byteArrayOf(0, 1, 2, 3)
+        val bytes2 = byteArrayOf(1, 0, 3, 2)
+        val bytes3 = byteArrayOf(3, 2, 1, 0)
+        makeFile(source, "tmp1", bytes1)
+        makeFile(source, "tmp2", bytes2);
+        makeFile(source, "tdir${File.separator}tmp3", bytes3)
+
+        dir.create("archive.zip")
+        val archive_path = (tempDir!! / Path("archive.zip")).toString()
+
+        zipAll((tempDir!! / Path("source")).toString(), archive_path)
+
+        unzip(archive_path, target)
+
+        val entries = (tempDir!! / Path("target")).listDirectoryEntries()
+        assertEquals(3, entries.size)
+    }
+
+    @AfterTest
+    fun tearDown() {
+        tempDir!!.toFile().deleteRecursively()
+    }
+}