SPC/snark
6
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

SNRK-77: added tests and slightly changed some code

Browse Source
This commit is contained in:
Anton Belyi 2023-04-28 21:54:36 +03:00
parent e9eaa0f8c2
commit 3c07ee7dd5
7 changed files with 2299 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2034
kotlin-js-store/yarn.lock Normal file
View File

File diff suppressed because it is too large Load Diff

7
snark-storage-driver/build.gradle.kts
View File

@ -12,4 +12,11 @@ dependencies {
// s3 Driver dependency
implementation("aws.sdk.kotlin:s3:$awsSdkVersion")
testImplementation(kotlin("test"))
testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
}
tasks.test {
useJUnitPlatform()
}

16
snark-storage-driver/snark-storage-driver.iml
View File

@ -8,5 +8,21 @@
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="KotlinJavaRuntime" level="project" />
<orderEntry type="module-library" scope="TEST">
<library name="JUnit5.8.1">
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/org/junit/jupiter/junit-jupiter/5.8.1/junit-jupiter-5.8.1.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/org/junit/jupiter/junit-jupiter-api/5.8.1/junit-jupiter-api-5.8.1.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/org/opentest4j/opentest4j/1.2.0/opentest4j-1.2.0.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/org/junit/platform/junit-platform-commons/1.8.1/junit-platform-commons-1.8.1.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/org/apiguardian/apiguardian-api/1.1.2/apiguardian-api-1.1.2.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/org/junit/jupiter/junit-jupiter-params/5.8.1/junit-jupiter-params-5.8.1.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/org/junit/jupiter/junit-jupiter-engine/5.8.1/junit-jupiter-engine-5.8.1.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/org/junit/platform/junit-platform-engine/1.8.1/junit-platform-engine-1.8.1.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
</component>
</module>

31
snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt
View File

@ -3,21 +3,23 @@ package space.kscience.snark.storage.local
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.io.File
import java.nio.file.Path
import java.nio.file.attribute.PosixFilePermission
import kotlin.io.path.*
public fun localStorage(rootPath: Path): Directory {
return LocalDirectory(rootPath)
}
private class LocalFile(private val path: Path) : FileReader, FileWriter {
internal class LocalFile(private val path: Path) : FileReader, FileWriter {
override fun close() {}
override suspend fun readAll(): ByteArray = path.readBytes()
override suspend fun write(bytes: ByteArray) = path.writeBytes(bytes)
}
private class LocalDirectory(private val path: Path) : Directory {
internal class LocalDirectory(private val path: Path) : Directory {
private fun child(child: String): Path = path / child
private fun child(child: Path): Path = path / child
@ -26,23 +28,38 @@ private class LocalDirectory(private val path: Path) : Directory {
override suspend fun get(filename: String): FileReader = LocalFile(child(filename))
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
val parts = filename.split(File.separator)
var cdir = this
for (i in 0..(parts.size - 1)) {
cdir = cdir.createSubdir(parts[i], true)
}
try {
child(filename).createFile()
} catch (ex: FileAlreadyExistsException) {
val nfile = cdir.child(parts.last()).createFile()
/*
nfile.toFile().setReadable(true)
nfile.toFile().setWritable(true)
nfile.toFile().setExecutable(true)
*/
} catch (ex: java.nio.file.FileAlreadyExistsException) {
if (!ignoreIfExists) {
throw ex
}
}
}
override suspend fun put(filename: String): FileWriter = LocalFile(child(filename))
override suspend fun put(filename: String): FileWriter {
val tmp = child(filename)
//tmp.toFile().setWritable(true)
return LocalFile(tmp)
}
override suspend fun getSubdir(path: Path): Directory = LocalDirectory(child(path))
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory {
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): LocalDirectory {
val dir = child(dirname)
try {
dir.createDirectory()
} catch (ex: FileAlreadyExistsException) {
} catch (ex: java.nio.file.FileAlreadyExistsException) {
if (!ignoreIfExists) {
throw ex
}

6
snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt
View File

@ -12,7 +12,11 @@ public suspend fun unzip(source_path: String, target: Directory) {
val filename = zipEntry.name
target.create(filename, true)
val fos = target.put(filename)
val buffer = ByteArray(zipEntry.size.toInt())
var sz = zipEntry.size.toInt()
if (sz == -1) {
sz = 1024
}
val buffer = ByteArray(sz)
zis.read(buffer)
fos.write(buffer)
fos.close()

103
snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/local/Tests.kt Normal file
View File

@ -0,0 +1,103 @@
package space.kscience.snark.storage.local
import kotlinx.coroutines.runBlocking
import space.kscience.snark.storage.Directory
import java.nio.file.Path
import kotlin.io.path.*
import kotlin.test.*
internal class LocalDriverTests {
private var tempDir: Path? = null
private var testSample: Directory? = null
private val bytes = byteArrayOf(0, 1, 2, 3, 4, 5, 6, 7)
@BeforeTest
fun setUp() {
tempDir = createTempDirectory()
testSample = LocalDirectory(tempDir!!)
}
@Test
fun testCreate() = runBlocking {
//folder is empty
assertEquals(0, tempDir!!.listDirectoryEntries().size)
//create first file
testSample!!.create("tmp1")
val entries = tempDir!!.listDirectoryEntries()
assertEquals(1, entries.size)
assertEquals(tempDir!! / Path("tmp1"), entries.first())
assertTrue(!entries.first().isDirectory())
//create second file
testSample!!.create("tmp2")
assertEquals(2, tempDir!!.listDirectoryEntries().size)
//check exception after duplication
try {
testSample!!.create("tmp1")
fail("shouldn't ignore duplicates here")
} catch (ex: java.nio.file.FileAlreadyExistsException) {}
//check ignorance
try {
testSample!!.create("tmp1", true)
} catch (ex: java.nio.file.FileAlreadyExistsException) {
fail("should ignore duplicates here")
}
}
@Test
fun testPutGet() = runBlocking {
testSample!!.create("tmp")
testSample!!.put("tmp").write(bytes)
assertContentEquals(bytes, testSample!!.get("tmp").readAll())
}
@Test
fun testCreateSubdir() = runBlocking {
//folder is empty
assertEquals(0, tempDir!!.listDirectoryEntries().size)
//create first file
testSample!!.createSubdir("tmp1")
val entries = tempDir!!.listDirectoryEntries()
assertEquals(1, entries.size)
assertEquals(tempDir!! / Path("tmp1"), entries.first())
assertTrue (entries.first().isDirectory())
//create second file
testSample!!.createSubdir("tmp2")
assertEquals(2, tempDir!!.listDirectoryEntries().size)
//check exception after duplication
try {
testSample!!.createSubdir("tmp1")
fail("shouldn't ignore duplicates here")
} catch (ex: java.nio.file.FileAlreadyExistsException) {}
//check ignorance
try {
testSample!!.createSubdir("tmp1", true)
} catch (ex: java.nio.file.FileAlreadyExistsException) {
fail("should ignore duplicates here")
}
assertTrue {true}
}
@Test
fun testGetSubdir() = runBlocking {
testSample!!.createSubdir("tmp")
testSample!!.create("tmp/data")
testSample!!.put("tmp/data").write(bytes)
val subdir = testSample!!.getSubdir(Path("tmp"))
assertContentEquals(bytes, subdir.get("data").readAll())
}
@AfterTest
fun tearDown() {
tempDir!!.toFile().deleteRecursively()
}
}

110
snark-storage-driver/src/test/kotlin/space/kscience/snark/storage/unzip/Tests.kt Normal file
View File

@ -0,0 +1,110 @@
package space.kscience.snark.storage.unzip
import kotlinx.coroutines.runBlocking
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.local.LocalDirectory
import java.io.*
import java.nio.file.Files
import java.nio.file.Path
import java.util.concurrent.TimeUnit
import java.util.zip.ZipEntry
import java.util.zip.ZipOutputStream
import kotlin.io.path.*
import kotlin.test.*
internal class UnzipTests {
private var tempDir: Path? = null
@BeforeTest
fun setUp() {
tempDir = createTempDirectory()
}
private suspend fun makeFile(dir: Directory, filename: String, content: ByteArray) {
dir.create(filename)
val writter = dir.put(filename)
if (!(tempDir!! / Path("source") / Path(filename)).isRegularFile()) {
println("shit")
}
writter.write(content)
}
private fun zipAll(directory: String, zipFile: String) {
val sourceFile = File(directory)
ZipOutputStream(BufferedOutputStream( FileOutputStream(zipFile))).use {
it.use {
zipFiles(it, sourceFile, "")
}
}
}
private fun zipFiles(zipOut: ZipOutputStream, sourceFile: File, parentDirPath: String) {
val data = ByteArray(2048)
for (f in sourceFile.listFiles()) {
if (f.isDirectory) {
val entry = ZipEntry(f.name + File.separator)
entry.time = f.lastModified()
entry.isDirectory
entry.size = f.length()
zipOut.putNextEntry(entry)
zipFiles(zipOut, f, f.name)
} else {
if (!f.name.contains(".zip")) { //If folder contains a file with extension ".zip", skip it
FileInputStream(f).use { fi ->
BufferedInputStream(fi).use { origin ->
val path = parentDirPath + File.separator + f.name
val entry = ZipEntry(path)
entry.time = f.lastModified()
entry.isDirectory
entry.size = f.length()
zipOut.putNextEntry(entry)
while (true) {
val readBytes = origin.read(data)
if (readBytes == -1) {
break
}
zipOut.write(data, 0, readBytes)
}
}
}
}
}
}
}
@Test
fun testUnzip() = runBlocking {
val dir: Directory = LocalDirectory(tempDir!!)
val source = dir.createSubdir("source")
val target = dir.createSubdir("target")
val bytes1 = byteArrayOf(0, 1, 2, 3)
val bytes2 = byteArrayOf(1, 0, 3, 2)
val bytes3 = byteArrayOf(3, 2, 1, 0)
makeFile(source, "tmp1", bytes1)
makeFile(source, "tmp2", bytes2);
makeFile(source, "tdir${File.separator}tmp3", bytes3)
dir.create("archive.zip")
val archive_path = (tempDir!! / Path("archive.zip")).toString()
zipAll((tempDir!! / Path("source")).toString(), archive_path)
unzip(archive_path, target)
val entries = (tempDir!! / Path("target")).listDirectoryEntries()
assertEquals(3, entries.size)
}
@AfterTest
fun tearDown() {
tempDir!!.toFile().deleteRecursively()
}
}