SPC/snark
5
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'feature/SNRK-70/dependecy-graph' into feature/SNRK-71/document-builder-implementation

Browse Source
This commit is contained in:
liubar.pa 2023-05-04 02:46:51 +03:00
commit 088697da78
8 changed files with 218 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
snark-document-builder/src/main/kotlin/DocumentBuilder.kt
View File

@ -2,13 +2,11 @@ package documentBuilder
import com.fasterxml.jackson.core.io.BigDecimalParser
import space.kscience.snark.storage.*
import space.kscience.snark.storage.local.LocalDirectory
import java.nio.file.Path
val DEFAULT_DOCUMENT_ROOT = "main.md"
private val DEFAULT_DOCUMENT_ROOT = "main.md"
public suspend fun buildDocument(documentPath: String) {
val documentDirectory: Directory = LocalDirectory(documentPath)
public suspend fun buildDocument(documentDirectory: Directory) {
val dependencyGraph = buildDependencyGraph(documentDirectory)

4
snark-document-builder/src/test/kotlin/DocumentBuilder.kt
View File

@ -5,7 +5,7 @@ import kotlinx.coroutines.runBlocking
class SomeTest {
@Test
fun testEssential() = runBlocking {
buildDocument("../example")
fun justWorks() = runBlocking {
// buildDocument(Directory("../example"))
}
}

52
snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/LocalDriver.kt Normal file
View File

@ -0,0 +1,52 @@
package space.kscience.snark.storage.local
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.nio.file.Path
import kotlin.io.path.*
public fun localStorage(rootPath: Path): Directory {
return LocalDirectory(rootPath)
}
private class LocalFile(private val path: Path) : FileReader, FileWriter {
override fun close() {}
override suspend fun readAll(): ByteArray = path.readBytes()
override suspend fun write(bytes: ByteArray) = path.writeBytes(bytes)
}
private class LocalDirectory(private val path: Path) : Directory {
private fun child(child: String): Path = path / child
private fun child(child: Path): Path = path / child
override fun close() {}
override suspend fun get(filename: String): FileReader = LocalFile(child(filename))
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
try {
child(filename).createFile()
} catch (ex: FileAlreadyExistsException) {
if (!ignoreIfExists) {
throw ex
}
}
}
override suspend fun put(filename: String): FileWriter = LocalFile(child(filename))
override suspend fun getSubdir(path: Path): Directory = LocalDirectory(child(path))
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory {
val dir = child(dirname)
try {
dir.createDirectory()
} catch (ex: FileAlreadyExistsException) {
if (!ignoreIfExists) {
throw ex
}
}
return LocalDirectory(dir)
}
}

35
snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/local/localDriver.kt
View File

@ -1,35 +0,0 @@
package space.kscience.snark.storage.local
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.io.File
import java.nio.file.Path
public class LocalFile(private val path: String) : FileReader, FileWriter {
override fun close() {}
override suspend fun readAll(): ByteArray = File(this.path).readBytes()
override suspend fun write(bytes: ByteArray) = File(this.path).writeBytes(bytes)
}
public class LocalDirectory(private val path: String) : Directory {
override fun close() {}
override suspend fun get(filename: String): FileReader = LocalFile("${this.path}/$filename")
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
if (!File(filename).createNewFile() && !ignoreIfExists) {
throw UnsupportedOperationException("File already exists")
}
}
override suspend fun put(filename: String): FileWriter = LocalFile("${this.path}/$filename")
override suspend fun getSubdir(dirpath: Path): Directory = LocalDirectory("${this.path}/$dirpath")
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory {
if (!File(dirname).mkdir() && !ignoreIfExists) {
throw UnsupportedOperationException("File already exists")
}
return this.getSubdir(File(dirname).toPath())
}
}

39
snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Directory.kt Normal file
View File

@ -0,0 +1,39 @@
package space.kscience.snark.storage.s3
import aws.sdk.kotlin.services.s3.S3Client
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.nio.file.Path
import kotlin.io.path.*
internal class S3Directory(
private val client: S3Client,
private val bucketName: String,
private val currentDir: Path,
) : Directory {
override suspend fun get(filename: String): FileReader =
S3FileReader(client, bucketName, currentDir / filename)
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
if (!ignoreIfExists) {
TODO("could not check if file exists")
}
}
override suspend fun put(filename: String): FileWriter =
S3FileWriter(client, bucketName, currentDir / filename)
override suspend fun getSubdir(path: Path): S3Directory =
S3Directory(client, bucketName, currentDir / path)
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): S3Directory =
if (!ignoreIfExists) {
TODO("could not check if directory exists")
} else {
S3Directory(client, bucketName, currentDir / dirname)
}
override fun close() {
}
}

40
snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3File.kt Normal file
View File

@ -0,0 +1,40 @@
package space.kscience.snark.storage.s3
import aws.sdk.kotlin.services.s3.S3Client
import aws.sdk.kotlin.services.s3.model.GetObjectRequest
import aws.sdk.kotlin.services.s3.putObject
import aws.smithy.kotlin.runtime.content.ByteStream
import aws.smithy.kotlin.runtime.content.toByteArray
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.nio.file.Path
import kotlin.io.path.*
internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) : FileReader {
override suspend fun readAll(): ByteArray {
val result = client.getObject(GetObjectRequest{
bucket = bucketName
key = path.toString()
}) {
it.body?.toByteArray() ?: ByteArray(0)
}
return result
}
override fun close() {
}
}
internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) : FileWriter {
override suspend fun write(bytes: ByteArray) {
client.putObject {
bucket = bucketName
key = path.toString()
body = ByteStream.fromBytes(bytes)
}
}
override fun close() {
}
}

59
snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/s3/S3Root.kt Normal file
View File

@ -0,0 +1,59 @@
package space.kscience.snark.storage.s3
import aws.sdk.kotlin.services.s3.*
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.lang.Exception
import java.nio.file.Path
import kotlin.io.path.*
public fun s3Storage(client: S3Client): Directory =
S3Root(client)
public fun s3Bucket(client: S3Client, bucket: String): Directory =
S3Directory(client, bucket, Path(""))
internal fun splitPathIntoBucketAndPath(path: Path): Pair<String, Path> {
val bucket = path.getName(0)
val filePath = path.relativize(bucket)
return Pair(bucket.toString(), filePath)
}
internal class S3Root(private val client: S3Client) : Directory {
override suspend fun get(filename: String): FileReader {
throw NoSuchFileException(Path(filename).toFile())
}
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
throw NoSuchFileException(Path(filename).toFile())
}
override suspend fun put(filename: String): FileWriter {
throw NoSuchFileException(Path(filename).toFile())
}
override suspend fun getSubdir(path: Path): Directory = try {
val (bucketName, filePath) = splitPathIntoBucketAndPath(path)
client.headBucket {
bucket = bucketName
}
S3Directory(client, bucketName, filePath)
} catch (ex: Exception) {
throw AccessDeniedException(path.toFile(), reason = ex.message)
}
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try {
val (bucketName, filePath) = splitPathIntoBucketAndPath(Path(dirname))
client.createBucket {
bucket = bucketName
}
S3Directory(client, bucketName, filePath)
} catch (ex: Exception) {
throw AccessDeniedException(Path(dirname).toFile(), reason = ex.message)
}
override fun close() {
}
}

24
snark-storage-driver/src/main/kotlin/space/kscience/snark/storage/unzip/Unzip.kt Normal file
View File

@ -0,0 +1,24 @@
package space.kscience.snark.storage.unzip
import space.kscience.snark.storage.Directory
import java.io.FileInputStream
import java.util.zip.ZipInputStream
public suspend fun unzip(source_path: String, target: Directory) {
val zis = ZipInputStream(FileInputStream(source_path))
var zipEntry = zis.nextEntry
while (zipEntry != null) {
if (!zipEntry.isDirectory) {
val filename = zipEntry.name
target.create(filename, true)
val fos = target.put(filename)
val buffer = ByteArray(zipEntry.size.toInt())
zis.read(buffer)
fos.write(buffer)
fos.close()
}
zipEntry = zis.nextEntry
}
zis.closeEntry()
zis.close()
}