SNRK-98: merged main into SNRK-87
This commit is contained in:
commit
e609de95cb
20
.dockerignore
Normal file
20
.dockerignore
Normal file
@ -0,0 +1,20 @@
|
||||
# .gitignore contents
|
||||
|
||||
.gradle/
|
||||
build/
|
||||
.idea/
|
||||
/logs/
|
||||
|
||||
!gradle/wrapper/gradle-wrapper.jar
|
||||
|
||||
kotlin-js-store
|
||||
*.iml
|
||||
|
||||
# addition to .gitignore
|
||||
|
||||
README.md
|
||||
LICENSE
|
||||
gradlew.bat
|
||||
.dockerignore
|
||||
.gitignore
|
||||
Dockerfile
|
8
Dockerfile
Normal file
8
Dockerfile
Normal file
@ -0,0 +1,8 @@
|
||||
FROM ubuntu:latest
|
||||
|
||||
WORKDIR Snark
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN ./requirments.sh
|
||||
RUN ./gradlew build test
|
27
requirments.sh
Executable file
27
requirments.sh
Executable file
@ -0,0 +1,27 @@
|
||||
set -e
|
||||
|
||||
apt-get update
|
||||
apt-get install -y sudo
|
||||
|
||||
sudo apt-get install -y kotlin
|
||||
|
||||
for dir in ./*/
|
||||
do
|
||||
if [[ $dir == *'snark'* ]]
|
||||
then
|
||||
cd "$dir"
|
||||
|
||||
if [[ $(find -type d -name "ci") ]]
|
||||
then
|
||||
|
||||
cd ci
|
||||
|
||||
./requirments.sh
|
||||
|
||||
cd ..
|
||||
fi
|
||||
|
||||
cd ..
|
||||
fi
|
||||
done
|
||||
|
@ -43,4 +43,6 @@ include(
|
||||
":snark-html",
|
||||
":snark-ktor",
|
||||
":snark-storage-driver",
|
||||
":snark-document-builder",
|
||||
":snark-main",
|
||||
)
|
19
snark-document-builder/build.gradle.kts
Normal file
19
snark-document-builder/build.gradle.kts
Normal file
@ -0,0 +1,19 @@
|
||||
plugins {
|
||||
id("space.kscience.gradle.jvm")
|
||||
`maven-publish`
|
||||
id("kotlinx-serialization")
|
||||
}
|
||||
|
||||
val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
|
||||
val jacksonVersion = "2.14.2"
|
||||
val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
|
||||
|
||||
|
||||
dependencies {
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
|
||||
api("io.ktor:ktor-server-html-builder:$ktorVersion")
|
||||
|
||||
implementation(project(":snark-storage-driver"))
|
||||
implementation("com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion")
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.3")
|
||||
}
|
6
snark-document-builder/ci/requirments.sh
Executable file
6
snark-document-builder/ci/requirments.sh
Executable file
@ -0,0 +1,6 @@
|
||||
sudo apt-get install -y python3
|
||||
sudo apt-get install -y nodejs
|
||||
sudo apt-get install -y npm
|
||||
|
||||
cd ../src/main/nodejs
|
||||
sudo npm install .
|
17
snark-document-builder/src/main/kotlin/Build.kt
Normal file
17
snark-document-builder/src/main/kotlin/Build.kt
Normal file
@ -0,0 +1,17 @@
|
||||
package documentBuilder
|
||||
|
||||
public class GraphManager(public val graph: DependencyGraph) {
|
||||
fun buildDocument(file: FileName) {
|
||||
val list = graph.nodes[file]
|
||||
if (list != null) {
|
||||
for (element in list.dependencies) {
|
||||
element.visit(this)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun getAstRootDocument(file: FileName): MdAstRoot {
|
||||
buildDocument(file)
|
||||
return graph.nodes[file]!!.mdAst
|
||||
}
|
||||
}
|
71
snark-document-builder/src/main/kotlin/DependencyGraph.kt
Normal file
71
snark-document-builder/src/main/kotlin/DependencyGraph.kt
Normal file
@ -0,0 +1,71 @@
|
||||
package documentBuilder
|
||||
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlin.collections.MutableList
|
||||
|
||||
|
||||
public typealias FileName = String
|
||||
|
||||
/**
|
||||
* Node of dependency graph.
|
||||
*
|
||||
* One node represents one file and its dependencies
|
||||
*
|
||||
* @property mdAst - AST tree of current file.
|
||||
* @property dependencies - list of tail end adjacent to this node (dependencies of current file to be resolved).
|
||||
*/
|
||||
public data class DependencyGraphNode(
|
||||
val mdAst: MdAstRoot,
|
||||
val dependencies: List<DependencyGraphEdge>
|
||||
)
|
||||
|
||||
/**
|
||||
* Interface of all dependency edges.
|
||||
*/
|
||||
public sealed interface DependencyGraphEdge {
|
||||
public fun visit(graphManager: GraphManager)
|
||||
}
|
||||
|
||||
/**
|
||||
* Include dependency edge.
|
||||
*
|
||||
* @property parentNode - node inside AST tree, that is parent for dependent node.
|
||||
* @property dependentNode - dependent node, i.e. node of part of document with include commands
|
||||
* @property includeList - list of files to be included.
|
||||
*/
|
||||
public data class IncludeDependency(
|
||||
val parentNode: MdAstParent,
|
||||
val dependentNode: MdAstElement,
|
||||
val includeList: List<FileName>
|
||||
) : DependencyGraphEdge {
|
||||
override fun visit(graphManager: GraphManager) {
|
||||
val parent = parentNode
|
||||
val childs: MutableList<MdAstElement> = mutableListOf()
|
||||
for (file in includeList) {
|
||||
graphManager.buildDocument(file)
|
||||
childs.addAll(graphManager.graph.nodes[file]!!.mdAst.children)
|
||||
}
|
||||
val elements: MutableList<MdAstElement> = parent.children.toMutableList()
|
||||
val index = parent.children.indexOf(dependentNode)
|
||||
elements.removeAt(index)
|
||||
elements.addAll(index, childs)
|
||||
parent.children = elements
|
||||
}
|
||||
}
|
||||
|
||||
// parent - List<MdAstElement> -------------------------------------
|
||||
// | \
|
||||
// \ \
|
||||
// | \
|
||||
// dependentNode - MdAstElement \
|
||||
// |
|
||||
// List<FileName> -> List<MdAstRoot> --> List<List<MdAstElement>> ===> List<MdAstElement>
|
||||
|
||||
/**
|
||||
* Whole dependency graph.
|
||||
*
|
||||
* @property nodes - map of nodes, where you can find DependencyGraphNode of file by its name.
|
||||
*/
|
||||
public data class DependencyGraph(
|
||||
val nodes: Map<FileName, DependencyGraphNode>
|
||||
)
|
67
snark-document-builder/src/main/kotlin/DocumentBuilder.kt
Normal file
67
snark-document-builder/src/main/kotlin/DocumentBuilder.kt
Normal file
@ -0,0 +1,67 @@
|
||||
package documentBuilder
|
||||
|
||||
import com.fasterxml.jackson.core.io.BigDecimalParser
|
||||
import space.kscience.snark.storage.*
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
import kotlinx.html.*
|
||||
import kotlinx.html.dom.createHTMLDocument
|
||||
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
|
||||
import com.fasterxml.jackson.module.kotlin.readValue
|
||||
import kotlinx.serialization.json.Json
|
||||
|
||||
private val SNARK_HTML_RENDER = "snark-document-builder/src/main/nodejs/HtmlRenderer.js"
|
||||
fun getHtml(ast_string: String): String
|
||||
{
|
||||
return ProcessBuilder("node", SNARK_HTML_RENDER, ast_string)
|
||||
.redirectOutput(ProcessBuilder.Redirect.PIPE)
|
||||
.redirectError(ProcessBuilder.Redirect.INHERIT)
|
||||
.start().inputStream.bufferedReader().readText()
|
||||
}
|
||||
|
||||
private val DEFAULT_DOCUMENT_ROOT = "main.md"
|
||||
|
||||
public suspend fun buildDocument(documentDirectory: Directory): String {
|
||||
val dependencyGraph = buildDependencyGraph(documentDirectory)
|
||||
|
||||
val root: MdAstRoot = dependencyGraph.nodes[""]!!.mdAst
|
||||
|
||||
return getHtml(jacksonObjectMapper().writeValueAsString(root))
|
||||
// return jacksonObjectMapper().writeValueAsString(root)
|
||||
}
|
||||
|
||||
public suspend fun buildDependencyGraph(root: Directory): DependencyGraph {
|
||||
val nodes = HashMap<FileName, DependencyGraphNode>()
|
||||
|
||||
buildNodes(root, nodes)
|
||||
|
||||
return DependencyGraph(nodes)
|
||||
}
|
||||
|
||||
private suspend fun buildNodes(folder: Directory, nodes: HashMap<FileName, DependencyGraphNode>) {
|
||||
val pathString = folder.path.toString()
|
||||
|
||||
assert(!nodes.containsKey(pathString))
|
||||
|
||||
val rootDcoument = folder.get(DEFAULT_DOCUMENT_ROOT)
|
||||
nodes.put(pathString, buildDependencyGraphNode(rootDcoument.readAll(), folder.path))
|
||||
|
||||
val dependencies = getDependencies(nodes.getValue(pathString))
|
||||
|
||||
for (dependency in dependencies) {
|
||||
if (!nodes.containsKey(dependency))
|
||||
buildNodes(folder.getSubdir(Paths.get(dependency)), nodes)
|
||||
}
|
||||
}
|
||||
|
||||
public suspend fun getDependencies(node: DependencyGraphNode): Set<FileName> {
|
||||
val dependencies = mutableListOf<FileName>()
|
||||
|
||||
for (dependency in node.dependencies) {
|
||||
when (dependency) {
|
||||
is IncludeDependency -> dependencies.addAll(dependency.includeList)
|
||||
}
|
||||
}
|
||||
|
||||
return dependencies.toSet()
|
||||
}
|
84
snark-document-builder/src/main/kotlin/MdAstElements.kt
Normal file
84
snark-document-builder/src/main/kotlin/MdAstElements.kt
Normal file
@ -0,0 +1,84 @@
|
||||
package documentBuilder
|
||||
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.SerialName
|
||||
import com.fasterxml.jackson.annotation.JsonSubTypes
|
||||
import com.fasterxml.jackson.annotation.JsonTypeInfo
|
||||
|
||||
@Serializable
|
||||
public data class Point(val line: Int, val column: Int, val offset: Int)
|
||||
|
||||
@Serializable
|
||||
public data class Position(val start: Point, val end: Point)
|
||||
|
||||
@JsonTypeInfo(
|
||||
use = JsonTypeInfo.Id.NAME,
|
||||
include = JsonTypeInfo.As.PROPERTY,
|
||||
property = "type"
|
||||
)
|
||||
|
||||
@JsonSubTypes(
|
||||
JsonSubTypes.Type(value = MdAstRoot::class, name = "root"),
|
||||
JsonSubTypes.Type(value = MdAstParagraph::class, name = "paragraph"),
|
||||
JsonSubTypes.Type(value = MdAstText::class, name = "text"),
|
||||
JsonSubTypes.Type(value = MdAstHeading::class, name = "heading"),
|
||||
JsonSubTypes.Type(value = MdAstCode::class, name = "code"),
|
||||
JsonSubTypes.Type(value = MdAstBlockquote::class, name = "blockquote")
|
||||
)
|
||||
|
||||
@Serializable
|
||||
public sealed interface MdAstElement{
|
||||
public abstract var position: Position
|
||||
}
|
||||
|
||||
@Serializable
|
||||
public sealed interface MdAstParent: MdAstElement{
|
||||
public var children: List<MdAstElement>
|
||||
}
|
||||
|
||||
@Serializable
|
||||
@SerialName("root")
|
||||
public data class MdAstRoot(
|
||||
override var children: List<MdAstElement>,
|
||||
override var position: Position
|
||||
): MdAstParent
|
||||
|
||||
|
||||
@Serializable
|
||||
@SerialName("paragraph")
|
||||
public data class MdAstParagraph(
|
||||
override var children: List<MdAstElement>,
|
||||
override var position: Position
|
||||
): MdAstParent
|
||||
|
||||
|
||||
@Serializable
|
||||
@SerialName("text")
|
||||
public data class MdAstText(
|
||||
val value: String,
|
||||
override var position: Position
|
||||
): MdAstElement
|
||||
|
||||
@Serializable
|
||||
@SerialName("heading")
|
||||
public data class MdAstHeading(
|
||||
val depth: Int,
|
||||
override var children: List<MdAstElement>,
|
||||
override var position: Position
|
||||
): MdAstParent
|
||||
|
||||
@Serializable
|
||||
@SerialName("code")
|
||||
public data class MdAstCode(
|
||||
var lang: String? = null,
|
||||
var meta: String? = null,
|
||||
var value: String,
|
||||
override var position: Position,
|
||||
) : MdAstElement
|
||||
|
||||
@Serializable
|
||||
@SerialName("blockquote")
|
||||
public data class MdAstBlockquote(
|
||||
override var children: List<MdAstElement>,
|
||||
override var position: Position
|
||||
): MdAstParent
|
57
snark-document-builder/src/main/kotlin/MdParser.kt
Normal file
57
snark-document-builder/src/main/kotlin/MdParser.kt
Normal file
@ -0,0 +1,57 @@
|
||||
package documentBuilder
|
||||
|
||||
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
|
||||
import com.fasterxml.jackson.module.kotlin.readValue
|
||||
import java.nio.file.Path
|
||||
|
||||
private val MARKDOWN_PARSER = "snark-document-builder/src/main/nodejs/MarkdownParser.js"
|
||||
private val SNARK_PARSER = "snark-document-builder/src/main/python/SnarkParser.py"
|
||||
|
||||
public suspend fun parseMd(mdFile: ByteArray): MdAstRoot {
|
||||
return jacksonObjectMapper()
|
||||
.readValue<MdAstRoot>(ProcessBuilder("node", MARKDOWN_PARSER, mdFile.toString())
|
||||
.redirectOutput(ProcessBuilder.Redirect.PIPE)
|
||||
.redirectError(ProcessBuilder.Redirect.INHERIT)
|
||||
.start().inputStream.bufferedReader().readText())
|
||||
}
|
||||
|
||||
public suspend fun buildDependencyGraphNode(mdFile: ByteArray, path: Path): DependencyGraphNode {
|
||||
val treeRoot = parseMd(mdFile)
|
||||
val dependencies = mutableListOf<DependencyGraphEdge>()
|
||||
|
||||
fillDependencies(treeRoot, dependencies, path)
|
||||
|
||||
return DependencyGraphNode(treeRoot, dependencies)
|
||||
}
|
||||
|
||||
internal suspend fun fillDependencies(
|
||||
currentNode: MdAstElement,
|
||||
dependencies: MutableList<DependencyGraphEdge>,
|
||||
path: Path) {
|
||||
when (currentNode) {
|
||||
is MdAstParent -> {
|
||||
for (child in currentNode.children) {
|
||||
if (child is MdAstText) {
|
||||
val includeList = getIncludeFiles(child.value).toMutableList()
|
||||
|
||||
if (includeList.size > 0) {
|
||||
includeList.replaceAll { path.toString() + "/" + it }
|
||||
|
||||
dependencies += IncludeDependency(currentNode, child, includeList)
|
||||
}
|
||||
} else {
|
||||
fillDependencies(child, dependencies, path)
|
||||
}
|
||||
}
|
||||
}
|
||||
else -> {}
|
||||
}
|
||||
}
|
||||
|
||||
public suspend fun getIncludeFiles(string: String): List<FileName> {
|
||||
return jacksonObjectMapper()
|
||||
.readValue<List<FileName>>(ProcessBuilder("python3", SNARK_PARSER, string)
|
||||
.redirectOutput(ProcessBuilder.Redirect.PIPE)
|
||||
.redirectError(ProcessBuilder.Redirect.INHERIT)
|
||||
.start().inputStream.bufferedReader().readText())
|
||||
}
|
1
snark-document-builder/src/main/nodejs/.gitignore
vendored
Normal file
1
snark-document-builder/src/main/nodejs/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
node_modules
|
16
snark-document-builder/src/main/nodejs/HtmlRenderer.js
Normal file
16
snark-document-builder/src/main/nodejs/HtmlRenderer.js
Normal file
@ -0,0 +1,16 @@
|
||||
import {toHast} from 'mdast-util-to-hast'
|
||||
import {toHtml} from 'hast-util-to-html'
|
||||
|
||||
main()
|
||||
|
||||
function main()
|
||||
{
|
||||
if (process.argv.length < 3)
|
||||
throw "No input"
|
||||
|
||||
const md_ast = JSON.parse(process.argv[2])
|
||||
const hast = toHast(md_ast)
|
||||
const html = toHtml(hast)
|
||||
|
||||
console.log(html)
|
||||
}
|
14
snark-document-builder/src/main/nodejs/MarkdownParser.js
Normal file
14
snark-document-builder/src/main/nodejs/MarkdownParser.js
Normal file
@ -0,0 +1,14 @@
|
||||
import {fromMarkdown} from 'mdast-util-from-markdown'
|
||||
|
||||
main()
|
||||
|
||||
function main()
|
||||
{
|
||||
if (process.argv.length < 3)
|
||||
throw "No input"
|
||||
|
||||
const markdown_string = process.argv[2]
|
||||
const mdast = fromMarkdown(markdown_string)
|
||||
|
||||
console.log(JSON.stringify(mdast))
|
||||
}
|
4152
snark-document-builder/src/main/nodejs/package-lock.json
generated
Normal file
4152
snark-document-builder/src/main/nodejs/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
30
snark-document-builder/src/main/nodejs/package.json
Normal file
30
snark-document-builder/src/main/nodejs/package.json
Normal file
@ -0,0 +1,30 @@
|
||||
{
|
||||
|
||||
"type": "module",
|
||||
|
||||
"dependencies": {
|
||||
|
||||
"fs": "^0.0.1-security",
|
||||
|
||||
"hast-util-to-html": "^8.0.4",
|
||||
|
||||
"mdast-util-to-hast": "^12.3.0",
|
||||
|
||||
"node-fetch": "^3.3.1",
|
||||
|
||||
"remark-html": "^15.0.2",
|
||||
|
||||
"remark-parse": "^10.0.1",
|
||||
|
||||
"require": "^2.4.20",
|
||||
|
||||
"to-vfile": "^7.2.4",
|
||||
|
||||
"unified": "^10.1.2",
|
||||
|
||||
"whatwg-fetch": "^3.6.2"
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
26
snark-document-builder/src/main/python/SnarkParser.py
Normal file
26
snark-document-builder/src/main/python/SnarkParser.py
Normal file
@ -0,0 +1,26 @@
|
||||
import sys
|
||||
import re
|
||||
import json
|
||||
|
||||
assert(len(sys.argv) >= 2)
|
||||
|
||||
string = sys.argv[1]
|
||||
|
||||
outputfile = sys.argv[2] if len(sys.argv) >= 3 else None
|
||||
|
||||
pattern = r'^([\n|\t| ]*@include\([a-z|0-9|.|_]*.md\)[\n|\t| ]*)*$'
|
||||
|
||||
files = []
|
||||
|
||||
if re.search("@include", string, re.IGNORECASE):
|
||||
if re.match(pattern, string):
|
||||
matches = re.findall(r'@include\((.*?)\)', string)
|
||||
files.extend(matches)
|
||||
else:
|
||||
sys.exit("Illformed string")
|
||||
|
||||
if outputfile is None:
|
||||
print(json.dumps(files))
|
||||
else:
|
||||
with open(outputfile, 'w+') as f:
|
||||
json.dump(files, f)
|
12
snark-document-builder/src/test/kotlin/DocumentBuilder.kt
Normal file
12
snark-document-builder/src/test/kotlin/DocumentBuilder.kt
Normal file
@ -0,0 +1,12 @@
|
||||
package documentBuilder
|
||||
|
||||
import org.junit.jupiter.api.Test
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import space.kscience.snark.storage.local.localStorage
|
||||
|
||||
class SomeTest {
|
||||
@Test
|
||||
fun justWorks() = runBlocking {
|
||||
// buildDocument(Directory("../example"))
|
||||
}
|
||||
}
|
@ -0,0 +1,3 @@
|
||||
# Hello
|
||||
|
||||
I'm almost empty test document without any dependencies
|
@ -13,7 +13,7 @@ dependencies {
|
||||
api("io.ktor:ktor-server-html-builder:$ktorVersion")
|
||||
api("io.ktor:ktor-server-host-common:$ktorVersion")
|
||||
implementation("io.ktor:ktor-server-netty:2.3.0")
|
||||
implementation(project(mapOf("path" to ":snark-storage-driver")))
|
||||
implementation(project(":snark-storage-driver"))
|
||||
|
||||
testApi("io.ktor:ktor-server-tests:$ktorVersion")
|
||||
}
|
@ -13,13 +13,15 @@ import io.ktor.server.routing.*
|
||||
import java.nio.file.Path
|
||||
import space.kscience.snark.storage.Directory
|
||||
import space.kscience.snark.storage.local.localStorage
|
||||
import kotlin.io.path.createTempDirectory
|
||||
import kotlin.io.path.isDirectory
|
||||
import kotlin.io.path.listDirectoryEntries
|
||||
import space.kscience.snark.storage.unzip.unzip
|
||||
import kotlin.io.createTempFile
|
||||
import kotlin.io.path.*
|
||||
import kotlin.io.writeBytes
|
||||
|
||||
public interface DataHolder {
|
||||
|
||||
public fun init(relativePath: String = "/") : Directory
|
||||
|
||||
public fun represent(relativePath: String = "/"): String
|
||||
@ -147,4 +149,4 @@ public class SNARKServer(private val dataHolder: DataHolder, private val port: I
|
||||
}
|
||||
}.start(wait = true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
21
snark-main/build.gradle.kts
Normal file
21
snark-main/build.gradle.kts
Normal file
@ -0,0 +1,21 @@
|
||||
plugins {
|
||||
id("space.kscience.gradle.jvm")
|
||||
`maven-publish`
|
||||
}
|
||||
|
||||
val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
|
||||
|
||||
|
||||
dependencies {
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
|
||||
implementation(project(":snark-ktor"))
|
||||
implementation(project(":snark-storage-driver"))
|
||||
implementation(project(":snark-document-builder"))
|
||||
|
||||
testImplementation(kotlin("test"))
|
||||
testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
|
||||
}
|
||||
|
||||
tasks.test {
|
||||
useJUnitPlatform()
|
||||
}
|
19
snark-main/src/main/kotlin/space/kscience/snark/main/Main.kt
Normal file
19
snark-main/src/main/kotlin/space/kscience/snark/main/Main.kt
Normal file
@ -0,0 +1,19 @@
|
||||
package space.kscience.snark.main
|
||||
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import space.kscience.snark.ktor.SNARKServer
|
||||
import space.kscience.snark.storage.local.localStorage
|
||||
import kotlin.io.path.Path
|
||||
|
||||
// Entrypoint
|
||||
fun main(): Unit = runBlocking {
|
||||
// Parse config, create driver
|
||||
val port = 8080
|
||||
val directory = localStorage(Path("./rundata"))
|
||||
val server = SNARKServer(ServerDataHolder(directory), port)
|
||||
launch {
|
||||
server.run()
|
||||
}
|
||||
}
|
@ -0,0 +1,15 @@
|
||||
package space.kscience.snark.main
|
||||
|
||||
import space.kscience.snark.ktor.DataHolder
|
||||
import space.kscience.snark.storage.Directory
|
||||
import documentBuilder.*
|
||||
import kotlinx.html.HTML
|
||||
|
||||
internal class ServerDataHolder(private val directory: Directory): DataHolder {
|
||||
override fun init(): Directory =
|
||||
directory
|
||||
|
||||
override suspend fun represent(): String {
|
||||
return buildDocument(directory).toString()
|
||||
}
|
||||
}
|
13
snark-main/src/test/kotlin/space/kscience/snark/main/Test.kt
Normal file
13
snark-main/src/test/kotlin/space/kscience/snark/main/Test.kt
Normal file
@ -0,0 +1,13 @@
|
||||
package space.kscience.snark.main
|
||||
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class Test {
|
||||
@Test
|
||||
fun justWorks() = runBlocking {
|
||||
delay(5)
|
||||
//main()
|
||||
}
|
||||
}
|
@ -1,35 +1,36 @@
|
||||
package space.kscience.snark.storage
|
||||
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.Path
|
||||
|
||||
public interface Directory : AutoCloseable {
|
||||
@Deprecated(
|
||||
message = "Use Path, not String",
|
||||
level = DeprecationLevel.WARNING,
|
||||
)
|
||||
public suspend fun get(filename: String): FileReader
|
||||
// get file from subtree
|
||||
public suspend fun get(filename: Path): FileReader
|
||||
|
||||
@Deprecated("Use put")
|
||||
public suspend fun create(filename: String, ignoreIfExists: Boolean = false)
|
||||
@Deprecated(
|
||||
message = "Use Path, not String",
|
||||
level = DeprecationLevel.WARNING,
|
||||
)
|
||||
public suspend fun put(filename: String): FileWriter
|
||||
|
||||
// put file to subtree
|
||||
public suspend fun put(filename: Path): FileWriter
|
||||
|
||||
public suspend fun getSubdir(path: Path): Directory
|
||||
|
||||
@Deprecated("Directories are created on put")
|
||||
public suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean = false): Directory
|
||||
|
||||
@Deprecated(
|
||||
message = "Not a good idea",
|
||||
level = DeprecationLevel.WARNING,
|
||||
)
|
||||
@Deprecated("Not a good idea")
|
||||
public val path: Path
|
||||
}
|
||||
|
||||
|
||||
public suspend fun Directory.get(filename: String): FileReader = get(Path(filename))
|
||||
|
||||
public suspend fun Directory.put(filename: String): FileWriter = put(Path(filename))
|
||||
|
||||
public suspend operator fun Directory.div(path: Path): Directory = getSubdir(path)
|
||||
|
||||
public suspend operator fun Directory.div(path: String): Directory = getSubdir(Path(path))
|
||||
|
||||
public interface FileReader : AutoCloseable {
|
||||
public suspend fun readAll(): ByteArray
|
||||
}
|
||||
|
@ -3,9 +3,7 @@ package space.kscience.snark.storage.local
|
||||
import space.kscience.snark.storage.Directory
|
||||
import space.kscience.snark.storage.FileReader
|
||||
import space.kscience.snark.storage.FileWriter
|
||||
import java.io.File
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.attribute.PosixFilePermission
|
||||
import kotlin.io.path.*
|
||||
|
||||
public fun localStorage(rootPath: Path): Directory {
|
||||
@ -16,24 +14,32 @@ internal class LocalFile(private val path: Path) : FileReader, FileWriter {
|
||||
override fun close() {}
|
||||
override suspend fun readAll(): ByteArray = path.readBytes()
|
||||
|
||||
override suspend fun write(bytes: ByteArray) = path.writeBytes(bytes)
|
||||
override suspend fun write(bytes: ByteArray) {
|
||||
path.parent.createDirectories()
|
||||
try {
|
||||
path.createFile()
|
||||
} catch (ex: Exception) {
|
||||
// Do nothing
|
||||
}
|
||||
path.writeBytes(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
internal class LocalDirectory(private val root: Path, private val currentDir: Path) : Directory {
|
||||
private fun child(child: String): Path = root / currentDir / child
|
||||
private fun child(child: Path): Path = root / currentDir / child
|
||||
@Deprecated("Use Path, not String")
|
||||
private fun realpath(child: String): Path = root / currentDir / child
|
||||
private fun realpath(child: Path): Path = root / currentDir / child
|
||||
|
||||
override fun close() {}
|
||||
|
||||
override suspend fun get(filename: String): LocalFile = LocalFile(child(filename))
|
||||
|
||||
override suspend fun get(filename: Path): LocalFile = LocalFile(child(filename))
|
||||
override suspend fun get(filename: Path): LocalFile = LocalFile(realpath(filename))
|
||||
|
||||
@Deprecated("Use put")
|
||||
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
|
||||
val dir = child(filename)
|
||||
val dir = realpath(filename)
|
||||
dir.parent.createDirectories()
|
||||
try {
|
||||
child(filename).createFile()
|
||||
realpath(filename).createFile()
|
||||
} catch (ex: java.nio.file.FileAlreadyExistsException) {
|
||||
if (!ignoreIfExists) {
|
||||
throw ex
|
||||
@ -41,13 +47,13 @@ internal class LocalDirectory(private val root: Path, private val currentDir: Pa
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun put(filename: String): LocalFile = get(filename)
|
||||
|
||||
override suspend fun put(filename: Path): LocalFile = get(filename)
|
||||
|
||||
override suspend fun getSubdir(path: Path): LocalDirectory = LocalDirectory(root, currentDir / path)
|
||||
|
||||
@Deprecated("Directories are created on put")
|
||||
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): LocalDirectory {
|
||||
val dir = child(dirname)
|
||||
val dir = realpath(dirname)
|
||||
dir.parent.createDirectories()
|
||||
try {
|
||||
dir.createDirectory()
|
||||
@ -59,6 +65,7 @@ internal class LocalDirectory(private val root: Path, private val currentDir: Pa
|
||||
return LocalDirectory(root, currentDir / dirname)
|
||||
}
|
||||
|
||||
@Deprecated("Not a good idea")
|
||||
override val path: Path
|
||||
get() = currentDir
|
||||
}
|
||||
|
@ -5,34 +5,30 @@ import space.kscience.snark.storage.Directory
|
||||
import space.kscience.snark.storage.FileReader
|
||||
import space.kscience.snark.storage.FileWriter
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.*
|
||||
import kotlin.io.path.div
|
||||
|
||||
internal class S3Directory(
|
||||
private val client: S3Client,
|
||||
private val bucketName: String,
|
||||
private val currentDir: Path,
|
||||
) : Directory {
|
||||
override suspend fun get(filename: String): FileReader =
|
||||
S3FileReader(client, bucketName, currentDir / filename)
|
||||
|
||||
override suspend fun get(filename: Path): FileReader =
|
||||
S3FileReader(client, bucketName, currentDir / filename)
|
||||
|
||||
@Deprecated("Use put")
|
||||
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
|
||||
if (!ignoreIfExists) {
|
||||
TODO("could not check if file exists")
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun put(filename: String): FileWriter =
|
||||
S3FileWriter(client, bucketName, currentDir / filename)
|
||||
|
||||
override suspend fun put(filename: Path): FileWriter =
|
||||
S3FileWriter(client, bucketName, currentDir / filename)
|
||||
|
||||
override suspend fun getSubdir(path: Path): S3Directory =
|
||||
S3Directory(client, bucketName, currentDir / path)
|
||||
|
||||
@Deprecated("Directories are created on put")
|
||||
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): S3Directory =
|
||||
if (!ignoreIfExists) {
|
||||
TODO("could not check if directory exists")
|
||||
@ -40,6 +36,7 @@ internal class S3Directory(
|
||||
S3Directory(client, bucketName, currentDir / dirname)
|
||||
}
|
||||
|
||||
@Deprecated("Not a good idea")
|
||||
override val path: Path
|
||||
get() = currentDir
|
||||
|
||||
|
@ -8,11 +8,11 @@ import aws.smithy.kotlin.runtime.content.toByteArray
|
||||
import space.kscience.snark.storage.FileReader
|
||||
import space.kscience.snark.storage.FileWriter
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.*
|
||||
|
||||
internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) : FileReader {
|
||||
internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) :
|
||||
FileReader {
|
||||
override suspend fun readAll(): ByteArray {
|
||||
val result = client.getObject(GetObjectRequest{
|
||||
val result = client.getObject(GetObjectRequest {
|
||||
bucket = bucketName
|
||||
key = path.toString()
|
||||
}) {
|
||||
@ -25,7 +25,8 @@ internal class S3FileReader(private val client: S3Client, private val bucketName
|
||||
}
|
||||
}
|
||||
|
||||
internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) : FileWriter {
|
||||
internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) :
|
||||
FileWriter {
|
||||
override suspend fun write(bytes: ByteArray) {
|
||||
client.putObject {
|
||||
bucket = bucketName
|
||||
|
@ -1,12 +1,13 @@
|
||||
package space.kscience.snark.storage.s3
|
||||
|
||||
import aws.sdk.kotlin.services.s3.*
|
||||
import aws.sdk.kotlin.services.s3.S3Client
|
||||
import aws.sdk.kotlin.services.s3.createBucket
|
||||
import aws.sdk.kotlin.services.s3.headBucket
|
||||
import space.kscience.snark.storage.Directory
|
||||
import space.kscience.snark.storage.FileReader
|
||||
import space.kscience.snark.storage.FileWriter
|
||||
import java.lang.Exception
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.*
|
||||
import kotlin.io.path.Path
|
||||
|
||||
public fun s3Storage(client: S3Client): Directory =
|
||||
S3Root(client)
|
||||
@ -21,22 +22,16 @@ internal fun splitPathIntoBucketAndPath(path: Path): Pair<String, Path> {
|
||||
}
|
||||
|
||||
internal class S3Root(private val client: S3Client) : Directory {
|
||||
override suspend fun get(filename: String): FileReader {
|
||||
throw NoSuchFileException(Path(filename).toFile())
|
||||
}
|
||||
|
||||
override suspend fun get(filename: Path): FileReader {
|
||||
throw NoSuchFileException(filename.toFile())
|
||||
}
|
||||
|
||||
@Deprecated("Use put")
|
||||
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
|
||||
throw NoSuchFileException(Path(filename).toFile())
|
||||
}
|
||||
|
||||
override suspend fun put(filename: String): FileWriter {
|
||||
throw NoSuchFileException(Path(filename).toFile())
|
||||
}
|
||||
|
||||
override suspend fun put(filename: Path): FileWriter {
|
||||
throw NoSuchFileException(filename.toFile())
|
||||
}
|
||||
@ -51,6 +46,7 @@ internal class S3Root(private val client: S3Client) : Directory {
|
||||
throw AccessDeniedException(path.toFile(), reason = ex.message)
|
||||
}
|
||||
|
||||
@Deprecated("Directories are created on put")
|
||||
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try {
|
||||
val (bucketName, filePath) = splitPathIntoBucketAndPath(Path(dirname))
|
||||
client.createBucket {
|
||||
@ -61,6 +57,7 @@ internal class S3Root(private val client: S3Client) : Directory {
|
||||
throw AccessDeniedException(Path(dirname).toFile(), reason = ex.message)
|
||||
}
|
||||
|
||||
@Deprecated("Not a good idea")
|
||||
override val path: Path
|
||||
get() = Path("")
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
package space.kscience.snark.storage.unzip
|
||||
|
||||
import space.kscience.snark.storage.Directory
|
||||
import space.kscience.snark.storage.*
|
||||
import java.io.FileInputStream
|
||||
import java.util.zip.ZipInputStream
|
||||
|
||||
|
@ -0,0 +1,47 @@
|
||||
package space.kscience.snark.storage.local
|
||||
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.junit.jupiter.api.BeforeAll
|
||||
import space.kscience.snark.storage.*
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.createTempDirectory
|
||||
import kotlin.io.path.deleteExisting
|
||||
import kotlin.test.AfterTest
|
||||
import kotlin.test.BeforeTest
|
||||
import kotlin.test.Test
|
||||
import kotlin.io.path.*
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
class Example {
|
||||
var tempDir: Path? = null
|
||||
var somedir: Directory? = null
|
||||
|
||||
@BeforeTest
|
||||
fun setUp() {
|
||||
tempDir = createTempDirectory()
|
||||
somedir = localStorage(tempDir!!)
|
||||
}
|
||||
|
||||
@AfterTest
|
||||
fun tearDown() {
|
||||
tempDir!!.toFile().deleteRecursively()
|
||||
somedir = null
|
||||
}
|
||||
|
||||
@Test
|
||||
fun exampleTest() = runBlocking {
|
||||
somedir!!.put("somefile").write("hello".toByteArray())
|
||||
assertEquals("hello", somedir!!.get("somefile").readAll().decodeToString())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun subdirExample() = runBlocking {
|
||||
val dir1 = somedir!! / "tmp1"
|
||||
dir1.put("somefile").write("hello".toByteArray())
|
||||
|
||||
val dir2 = somedir!! / "tmp1"
|
||||
val data = dir2.get("somefile").readAll()
|
||||
|
||||
assertEquals("hello", data.decodeToString())
|
||||
}
|
||||
}
|
@ -2,7 +2,7 @@ package space.kscience.snark.storage.local
|
||||
|
||||
import kotlinx.coroutines.runBlocking
|
||||
|
||||
import space.kscience.snark.storage.Directory
|
||||
import space.kscience.snark.storage.*
|
||||
import java.io.File
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.*
|
||||
|
@ -2,8 +2,7 @@ package space.kscience.snark.storage.unzip
|
||||
|
||||
import kotlinx.coroutines.runBlocking
|
||||
|
||||
import space.kscience.snark.storage.Directory
|
||||
import space.kscience.snark.storage.local.LocalDirectory
|
||||
import space.kscience.snark.storage.*
|
||||
import space.kscience.snark.storage.local.localStorage
|
||||
import java.io.*
|
||||
import java.nio.file.Files
|
||||
|
Loading…
Reference in New Issue
Block a user