Compare commits
133 Commits
main
...
main-exper
Author | SHA1 | Date | |
---|---|---|---|
|
5659bd2f93 | ||
|
7cccb96023 | ||
|
0bc0f63131 | ||
|
156ef5ce0f | ||
|
9f6f713aee | ||
|
46658ef690 | ||
|
cc12584a8d | ||
|
bf4310375a | ||
|
afccae64ab | ||
|
652b865b89 | ||
|
7c5e2b7dbb | ||
|
9031638910 | ||
|
0df78294a1 | ||
|
1276275853 | ||
|
98e713c909 | ||
|
f8bcdcaeeb | ||
|
4ee51ebc0c | ||
|
45132e4355 | ||
|
e609de95cb | ||
|
f28c5c6263 | ||
|
44bac3b5ea | ||
|
ac8478678e | ||
|
62e8d859d8 | ||
|
b463c492f1 | ||
|
81c7bbb073 | ||
|
028cf631f4 | ||
|
f06182ee1d | ||
|
80deddc412 | ||
|
32f8a0d540 | ||
|
82cd925e35 | ||
|
167037e276 | ||
|
5c88ff5a00 | ||
|
bd8dcdca4d | ||
|
b0564e6539 | ||
|
64bfc75222 | ||
|
2755e2020d | ||
|
8795b49d38 | ||
|
f30b35e66a | ||
|
b0e3b5551a | ||
|
7fb778e483 | ||
|
8ffda3f42a | ||
|
fbd5237573 | ||
|
ef54c02c41 | ||
|
046bbda23a | ||
|
ffaa932127 | ||
|
8225501488 | ||
|
793218b6bc | ||
|
c6ceba2ed5 | ||
|
29d842b0bf | ||
|
4f9fba5237 | ||
|
9a41f67e29 | ||
|
8ed0cc0499 | ||
|
3358f8c856 | ||
|
1dfb5d8772 | ||
|
a13c4a8e8c | ||
|
14c1e650b9 | ||
|
a99e71d1b1 | ||
|
667688d46d | ||
|
363f63b8da | ||
|
33b4c56e4f | ||
|
19a023190f | ||
|
234dd279f0 | ||
|
778e58fc63 | ||
|
4019cbfe6b | ||
|
225b7346a8 | ||
|
cd85fcf329 | ||
|
fdcf9c9c5b | ||
|
ff84d2e87b | ||
|
f477af64e6 | ||
|
ea05f93cea | ||
|
7d681549b7 | ||
|
8e304377c6 | ||
|
e1c8be66db | ||
|
ff7b02d98a | ||
|
eda376d51d | ||
|
c112a74652 | ||
|
203aa52efe | ||
|
637a6810fc | ||
|
e0ef373c48 | ||
|
248677dad9 | ||
|
4441172ef7 | ||
|
b6db0ba479 | ||
|
634eb2d69d | ||
|
5b33419b37 | ||
|
e90c9728c1 | ||
|
859c53745f | ||
|
06bcdbd489 | ||
|
a411849d6f | ||
|
54828a6535 | ||
|
c76ecd592f | ||
|
088697da78 | ||
|
dee40ecdcc | ||
|
5458bd3677 | ||
|
6c9a08a14a | ||
|
cfe0f0740f | ||
|
7599fd5ffd | ||
|
1db2afce12 | ||
|
b4423e8bba | ||
|
fd45104a7e | ||
|
417d90b842 | ||
|
3c2ffb572b | ||
|
b7a439f711 | ||
|
7a57de989b | ||
|
efc3bc2537 | ||
|
ea4fef95b8 | ||
|
82f699070b | ||
|
3c07ee7dd5 | ||
|
a672db082f | ||
|
f71e5a3f1a | ||
|
52772cde7d | ||
|
256fb16748 | ||
|
d292abc816 | ||
|
e9eaa0f8c2 | ||
|
7c6157be96 | ||
|
b543735628 | ||
|
8069ea2f0b | ||
|
88510dbba8 | ||
|
9651ad1cd9 | ||
|
57df19f1bf | ||
|
0ab26812f8 | ||
|
49c350abad | ||
|
0ef269ec5c | ||
|
4465e68408 | ||
|
a92f9673ac | ||
|
0dad4cd923 | ||
|
a97a99d138 | ||
|
7b946c004f | ||
|
a9c0cc10a6 | ||
|
7127d3582d | ||
|
6e9eb88934 | ||
|
f8e7f6567f | ||
|
65fafe1400 | ||
|
07b2ecb35c |
24
.dockerignore
Normal file
24
.dockerignore
Normal file
@ -0,0 +1,24 @@
|
||||
# .gitignore contents
|
||||
|
||||
.gradle/
|
||||
build/
|
||||
.idea/
|
||||
/logs/
|
||||
|
||||
!gradle/wrapper/gradle-wrapper.jar
|
||||
|
||||
kotlin-js-store
|
||||
*.iml
|
||||
|
||||
# addition to .gitignore
|
||||
|
||||
README.md
|
||||
LICENSE
|
||||
gradlew.bat
|
||||
.dockerignore
|
||||
.gitignore
|
||||
Dockerfile
|
||||
|
||||
tmp
|
||||
rundata
|
||||
credentials.json
|
5
.gitignore
vendored
5
.gitignore
vendored
@ -2,5 +2,10 @@
|
||||
build/
|
||||
.idea/
|
||||
/logs/
|
||||
rundata/
|
||||
|
||||
!gradle/wrapper/gradle-wrapper.jar
|
||||
|
||||
kotlin-js-store
|
||||
*.iml
|
||||
*.json
|
||||
|
25
Dockerfile
Normal file
25
Dockerfile
Normal file
@ -0,0 +1,25 @@
|
||||
FROM ubuntu:latest
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y curl zip unzip
|
||||
|
||||
ARG JAVA_VERSION="17.0.7-zulu"
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY ./snark-main/ci ./snark-main/ci
|
||||
|
||||
RUN ./snark-main/ci/install_sdk.sh
|
||||
RUN ./snark-main/ci/install_java.sh "$JAVA_VERSION"
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN ./requirements.sh
|
||||
|
||||
RUN bash -c "source ~/.sdkman/bin/sdkman-init.sh && ./gradlew clean build"
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
RUN mkdir -p ~/.aws/ && ln -s /run/secrets/credentials.json ~/.aws/credentials.json
|
||||
|
||||
CMD bash -c "source ~/.sdkman/bin/sdkman-init.sh && ./gradlew :snark-main:run_server"
|
19
docker-compose.yml
Normal file
19
docker-compose.yml
Normal file
@ -0,0 +1,19 @@
|
||||
version: '3'
|
||||
|
||||
services:
|
||||
snark:
|
||||
build: .
|
||||
ports:
|
||||
- "8081:8080"
|
||||
expose:
|
||||
- 8081
|
||||
volumes:
|
||||
- storage:/app/rundata
|
||||
secrets:
|
||||
- credentials.json
|
||||
volumes:
|
||||
storage:
|
||||
|
||||
secrets:
|
||||
credentials.json:
|
||||
file: ./credentials.json
|
29
requirements.sh
Executable file
29
requirements.sh
Executable file
@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
apt-get update
|
||||
apt-get install -y sudo
|
||||
|
||||
for dir in ./*/
|
||||
do
|
||||
if [[ $dir == *'snark'* ]]
|
||||
then
|
||||
cd "$dir"
|
||||
|
||||
if [[ $(find -type d -name "ci") ]]
|
||||
then
|
||||
|
||||
cd ci
|
||||
if [[ -f "requirements.sh" ]]; then
|
||||
echo "executing sub"
|
||||
bash -c "./requirements.sh"
|
||||
fi
|
||||
|
||||
cd ..
|
||||
fi
|
||||
|
||||
cd ..
|
||||
fi
|
||||
done
|
||||
|
@ -41,5 +41,9 @@ include(
|
||||
":snark-gradle-plugin",
|
||||
":snark-core",
|
||||
":snark-html",
|
||||
":snark-ktor"
|
||||
":snark-ktor",
|
||||
":snark-storage-driver",
|
||||
":snark-document-builder",
|
||||
":snark-main",
|
||||
":snark-pandoc-plugin",
|
||||
)
|
21
snark-document-builder/build.gradle.kts
Normal file
21
snark-document-builder/build.gradle.kts
Normal file
@ -0,0 +1,21 @@
|
||||
plugins {
|
||||
id("space.kscience.gradle.jvm")
|
||||
`maven-publish`
|
||||
id("kotlinx-serialization")
|
||||
}
|
||||
|
||||
val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
|
||||
val jacksonVersion = "2.14.2"
|
||||
val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
|
||||
|
||||
|
||||
dependencies {
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
|
||||
api("io.ktor:ktor-server-html-builder:$ktorVersion")
|
||||
|
||||
implementation(project(":snark-storage-driver"))
|
||||
implementation(project(":snark-pandoc-plugin"))
|
||||
|
||||
implementation("com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion")
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.3")
|
||||
}
|
11
snark-document-builder/ci/requirements.sh
Executable file
11
snark-document-builder/ci/requirements.sh
Executable file
@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
sudo apt-get install -y python3
|
||||
sudo apt-get install -y nodejs
|
||||
sudo apt-get install -y npm
|
||||
|
||||
pushd ../src/main/nodejs
|
||||
npm install .
|
||||
popd
|
17
snark-document-builder/src/main/kotlin/Build.kt
Normal file
17
snark-document-builder/src/main/kotlin/Build.kt
Normal file
@ -0,0 +1,17 @@
|
||||
package documentBuilder
|
||||
|
||||
public class GraphManager(public val graph: DependencyGraph) {
|
||||
fun buildDocument(file: FileName) {
|
||||
val list = graph.nodes[file]
|
||||
if (list != null) {
|
||||
for (element in list.dependencies) {
|
||||
element.visit(this)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun getAstRootDocument(file: FileName): MdAstRoot {
|
||||
buildDocument(file)
|
||||
return graph.nodes[file]!!.mdAst
|
||||
}
|
||||
}
|
71
snark-document-builder/src/main/kotlin/DependencyGraph.kt
Normal file
71
snark-document-builder/src/main/kotlin/DependencyGraph.kt
Normal file
@ -0,0 +1,71 @@
|
||||
package documentBuilder
|
||||
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlin.collections.MutableList
|
||||
|
||||
|
||||
public typealias FileName = String
|
||||
|
||||
/**
|
||||
* Node of dependency graph.
|
||||
*
|
||||
* One node represents one file and its dependencies
|
||||
*
|
||||
* @property mdAst - AST tree of current file.
|
||||
* @property dependencies - list of tail end adjacent to this node (dependencies of current file to be resolved).
|
||||
*/
|
||||
public data class DependencyGraphNode(
|
||||
val mdAst: MdAstRoot,
|
||||
val dependencies: List<DependencyGraphEdge>
|
||||
)
|
||||
|
||||
/**
|
||||
* Interface of all dependency edges.
|
||||
*/
|
||||
public sealed interface DependencyGraphEdge {
|
||||
public fun visit(graphManager: GraphManager)
|
||||
}
|
||||
|
||||
/**
|
||||
* Include dependency edge.
|
||||
*
|
||||
* @property parentNode - node inside AST tree, that is parent for dependent node.
|
||||
* @property dependentNode - dependent node, i.e. node of part of document with include commands
|
||||
* @property includeList - list of files to be included.
|
||||
*/
|
||||
public data class IncludeDependency(
|
||||
val parentNode: MdAstParent,
|
||||
val dependentNode: MdAstElement,
|
||||
val includeList: List<FileName>
|
||||
) : DependencyGraphEdge {
|
||||
override fun visit(graphManager: GraphManager) {
|
||||
val parent = parentNode
|
||||
val childs: MutableList<MdAstElement> = mutableListOf()
|
||||
for (file in includeList) {
|
||||
graphManager.buildDocument(file)
|
||||
childs.addAll(graphManager.graph.nodes[file]!!.mdAst.children)
|
||||
}
|
||||
val elements: MutableList<MdAstElement> = parent.children.toMutableList()
|
||||
val index = parent.children.indexOf(dependentNode)
|
||||
elements.removeAt(index)
|
||||
elements.addAll(index, childs)
|
||||
parent.children = elements
|
||||
}
|
||||
}
|
||||
|
||||
// parent - List<MdAstElement> -------------------------------------
|
||||
// | \
|
||||
// \ \
|
||||
// | \
|
||||
// dependentNode - MdAstElement \
|
||||
// |
|
||||
// List<FileName> -> List<MdAstRoot> --> List<List<MdAstElement>> ===> List<MdAstElement>
|
||||
|
||||
/**
|
||||
* Whole dependency graph.
|
||||
*
|
||||
* @property nodes - map of nodes, where you can find DependencyGraphNode of file by its name.
|
||||
*/
|
||||
public data class DependencyGraph(
|
||||
val nodes: Map<FileName, DependencyGraphNode>
|
||||
)
|
111
snark-document-builder/src/main/kotlin/DocumentBuilder.kt
Normal file
111
snark-document-builder/src/main/kotlin/DocumentBuilder.kt
Normal file
@ -0,0 +1,111 @@
|
||||
package documentBuilder
|
||||
|
||||
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
|
||||
import space.kscience.snark.pandoc.PandocCommandBuilder
|
||||
import space.kscience.snark.pandoc.PandocWrapper
|
||||
import space.kscience.snark.storage.*
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.*
|
||||
|
||||
private val SNARK_HTML_RENDER = "snark-document-builder/src/main/nodejs/HtmlRenderer.js"
|
||||
private val SNARK_MD_RENDERER = "snark-document-builder/src/main/nodejs/MdRenderer.js"
|
||||
fun getHtml(ast_string: String): String
|
||||
{
|
||||
return ProcessBuilder("node", SNARK_HTML_RENDER, ast_string)
|
||||
.redirectOutput(ProcessBuilder.Redirect.PIPE)
|
||||
.redirectError(ProcessBuilder.Redirect.INHERIT)
|
||||
.start().inputStream.bufferedReader().readText()
|
||||
}
|
||||
|
||||
fun getLatex(ast_string: String) : Path
|
||||
{
|
||||
val outputMd = Files.createTempFile(Path.of("./data/"), "output", ".md")
|
||||
|
||||
val output = ProcessBuilder("node", SNARK_MD_RENDERER, ast_string)
|
||||
.redirectOutput(ProcessBuilder.Redirect.PIPE)
|
||||
.redirectError(ProcessBuilder.Redirect.INHERIT)
|
||||
.start().inputStream.bufferedReader().readText()
|
||||
outputMd.writeText(output)
|
||||
|
||||
val outputTex = Files.createTempFile(Path.of("./data/"), "output", ".tex")
|
||||
|
||||
val pandocWrapper = PandocWrapper()
|
||||
pandocWrapper.use { p: PandocWrapper? ->
|
||||
val command = PandocCommandBuilder(
|
||||
listOf<Path>(outputMd),
|
||||
outputTex
|
||||
)
|
||||
PandocWrapper.execute(command)
|
||||
}
|
||||
|
||||
return outputTex
|
||||
}
|
||||
|
||||
private val DEFAULT_DOCUMENT_ROOT = "main.md"
|
||||
|
||||
public suspend fun buildDocument(root: Directory, path: Path): String {
|
||||
val dependencyGraph = buildDependencyGraph(root, path)
|
||||
|
||||
val graphManage = GraphManager(dependencyGraph)
|
||||
|
||||
graphManage.buildDocument(path.toString())
|
||||
|
||||
// for ((key, value) in dependencyGraph.nodes) {
|
||||
// println("Key ${key}")
|
||||
// println("Value.mdAst ${value.mdAst}")
|
||||
// println("Value.dependencies ${value.dependencies}")
|
||||
// }
|
||||
|
||||
val root: MdAstRoot = dependencyGraph.nodes[path.toString()]!!.mdAst
|
||||
|
||||
return getHtml(jacksonObjectMapper().writeValueAsString(root))
|
||||
}
|
||||
|
||||
public suspend fun buildLatex(root: Directory, path: Path) : Path {
|
||||
val dependencyGraph = buildDependencyGraph(root, path)
|
||||
|
||||
val graphManage = GraphManager(dependencyGraph)
|
||||
|
||||
graphManage.buildDocument(path.toString())
|
||||
|
||||
val root: MdAstRoot = dependencyGraph.nodes[path.toString()]!!.mdAst
|
||||
|
||||
return getLatex(jacksonObjectMapper().writeValueAsString(root))
|
||||
}
|
||||
|
||||
public suspend fun buildDependencyGraph(root: Directory, path: Path): DependencyGraph {
|
||||
val nodes = HashMap<FileName, DependencyGraphNode>()
|
||||
|
||||
buildNodes(root, path, nodes)
|
||||
|
||||
return DependencyGraph(nodes)
|
||||
}
|
||||
|
||||
private suspend fun buildNodes(root: Directory, path: Path, nodes: HashMap<FileName, DependencyGraphNode>) {
|
||||
val pathString = path.toString()
|
||||
|
||||
assert(!nodes.containsKey(pathString))
|
||||
|
||||
val rootDcoument = (root / path).get(DEFAULT_DOCUMENT_ROOT)
|
||||
nodes.put(pathString, buildDependencyGraphNode(rootDcoument.readAll(), path))
|
||||
|
||||
val dependencies = getDependencies(nodes.getValue(pathString))
|
||||
|
||||
for (dependency in dependencies) {
|
||||
if (!nodes.containsKey(dependency))
|
||||
buildNodes(root, Path(dependency), nodes)
|
||||
}
|
||||
}
|
||||
|
||||
public suspend fun getDependencies(node: DependencyGraphNode): Set<FileName> {
|
||||
val dependencies = mutableListOf<FileName>()
|
||||
|
||||
for (dependency in node.dependencies) {
|
||||
when (dependency) {
|
||||
is IncludeDependency -> dependencies.addAll(dependency.includeList)
|
||||
}
|
||||
}
|
||||
|
||||
return dependencies.toSet()
|
||||
}
|
84
snark-document-builder/src/main/kotlin/MdAstElements.kt
Normal file
84
snark-document-builder/src/main/kotlin/MdAstElements.kt
Normal file
@ -0,0 +1,84 @@
|
||||
package documentBuilder
|
||||
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.SerialName
|
||||
import com.fasterxml.jackson.annotation.JsonSubTypes
|
||||
import com.fasterxml.jackson.annotation.JsonTypeInfo
|
||||
|
||||
@Serializable
|
||||
public data class Point(val line: Int, val column: Int, val offset: Int)
|
||||
|
||||
@Serializable
|
||||
public data class Position(val start: Point, val end: Point)
|
||||
|
||||
@JsonTypeInfo(
|
||||
use = JsonTypeInfo.Id.NAME,
|
||||
include = JsonTypeInfo.As.PROPERTY,
|
||||
property = "type"
|
||||
)
|
||||
|
||||
@JsonSubTypes(
|
||||
JsonSubTypes.Type(value = MdAstRoot::class, name = "root"),
|
||||
JsonSubTypes.Type(value = MdAstParagraph::class, name = "paragraph"),
|
||||
JsonSubTypes.Type(value = MdAstText::class, name = "text"),
|
||||
JsonSubTypes.Type(value = MdAstHeading::class, name = "heading"),
|
||||
JsonSubTypes.Type(value = MdAstCode::class, name = "code"),
|
||||
JsonSubTypes.Type(value = MdAstBlockquote::class, name = "blockquote")
|
||||
)
|
||||
|
||||
@Serializable
|
||||
public sealed interface MdAstElement{
|
||||
public abstract var position: Position
|
||||
}
|
||||
|
||||
@Serializable
|
||||
public sealed interface MdAstParent: MdAstElement{
|
||||
public var children: List<MdAstElement>
|
||||
}
|
||||
|
||||
@Serializable
|
||||
@SerialName("root")
|
||||
public data class MdAstRoot(
|
||||
override var children: List<MdAstElement>,
|
||||
override var position: Position
|
||||
): MdAstParent
|
||||
|
||||
|
||||
@Serializable
|
||||
@SerialName("paragraph")
|
||||
public data class MdAstParagraph(
|
||||
override var children: List<MdAstElement>,
|
||||
override var position: Position
|
||||
): MdAstParent
|
||||
|
||||
|
||||
@Serializable
|
||||
@SerialName("text")
|
||||
public data class MdAstText(
|
||||
val value: String,
|
||||
override var position: Position
|
||||
): MdAstElement
|
||||
|
||||
@Serializable
|
||||
@SerialName("heading")
|
||||
public data class MdAstHeading(
|
||||
val depth: Int,
|
||||
override var children: List<MdAstElement>,
|
||||
override var position: Position
|
||||
): MdAstParent
|
||||
|
||||
@Serializable
|
||||
@SerialName("code")
|
||||
public data class MdAstCode(
|
||||
var lang: String? = null,
|
||||
var meta: String? = null,
|
||||
var value: String,
|
||||
override var position: Position,
|
||||
) : MdAstElement
|
||||
|
||||
@Serializable
|
||||
@SerialName("blockquote")
|
||||
public data class MdAstBlockquote(
|
||||
override var children: List<MdAstElement>,
|
||||
override var position: Position
|
||||
): MdAstParent
|
60
snark-document-builder/src/main/kotlin/MdParser.kt
Normal file
60
snark-document-builder/src/main/kotlin/MdParser.kt
Normal file
@ -0,0 +1,60 @@
|
||||
package documentBuilder
|
||||
|
||||
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
|
||||
import com.fasterxml.jackson.module.kotlin.readValue
|
||||
import java.nio.file.Path
|
||||
|
||||
// snark-main/build.gradle.kts depends on these pathes
|
||||
private val MARKDOWN_PARSER = "snark-document-builder/src/main/nodejs/MarkdownParser.js"
|
||||
private val SNARK_PARSER = "snark-document-builder/src/main/python/SnarkParser.py"
|
||||
|
||||
public suspend fun parseMd(mdFile: ByteArray, parserPath: String = MARKDOWN_PARSER): MdAstRoot {
|
||||
val process = ProcessBuilder("node", parserPath, String(mdFile))
|
||||
val result = process
|
||||
.redirectOutput(ProcessBuilder.Redirect.PIPE)
|
||||
.redirectError(ProcessBuilder.Redirect.INHERIT)
|
||||
.start().inputStream.bufferedReader().readText()
|
||||
|
||||
return jacksonObjectMapper().readValue<MdAstRoot>(result)
|
||||
}
|
||||
|
||||
public suspend fun buildDependencyGraphNode(mdFile: ByteArray, path: Path): DependencyGraphNode {
|
||||
val treeRoot = parseMd(mdFile)
|
||||
val dependencies = mutableListOf<DependencyGraphEdge>()
|
||||
|
||||
fillDependencies(treeRoot, dependencies, path)
|
||||
|
||||
return DependencyGraphNode(treeRoot, dependencies)
|
||||
}
|
||||
|
||||
internal suspend fun fillDependencies(
|
||||
currentNode: MdAstElement,
|
||||
dependencies: MutableList<DependencyGraphEdge>,
|
||||
path: Path) {
|
||||
when (currentNode) {
|
||||
is MdAstParent -> {
|
||||
for (child in currentNode.children) {
|
||||
if (child is MdAstText) {
|
||||
val includeList = getIncludeFiles(child.value).toMutableList()
|
||||
|
||||
if (includeList.size > 0) {
|
||||
includeList.replaceAll { path.toString() + "/" + it }
|
||||
|
||||
dependencies += IncludeDependency(currentNode, child, includeList)
|
||||
}
|
||||
} else {
|
||||
fillDependencies(child, dependencies, path)
|
||||
}
|
||||
}
|
||||
}
|
||||
else -> {}
|
||||
}
|
||||
}
|
||||
|
||||
public suspend fun getIncludeFiles(string: String): List<FileName> {
|
||||
return jacksonObjectMapper()
|
||||
.readValue<List<FileName>>(ProcessBuilder("python3", SNARK_PARSER, string)
|
||||
.redirectOutput(ProcessBuilder.Redirect.PIPE)
|
||||
.redirectError(ProcessBuilder.Redirect.INHERIT)
|
||||
.start().inputStream.bufferedReader().readText())
|
||||
}
|
1
snark-document-builder/src/main/nodejs/.gitignore
vendored
Normal file
1
snark-document-builder/src/main/nodejs/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
node_modules
|
16
snark-document-builder/src/main/nodejs/HtmlRenderer.js
Normal file
16
snark-document-builder/src/main/nodejs/HtmlRenderer.js
Normal file
@ -0,0 +1,16 @@
|
||||
import {toHast} from 'mdast-util-to-hast'
|
||||
import {toHtml} from 'hast-util-to-html'
|
||||
|
||||
main()
|
||||
|
||||
function main()
|
||||
{
|
||||
if (process.argv.length < 3)
|
||||
throw "No input"
|
||||
|
||||
const md_ast = JSON.parse(process.argv[2])
|
||||
const hast = toHast(md_ast)
|
||||
const html = toHtml(hast)
|
||||
|
||||
console.log(html)
|
||||
}
|
14
snark-document-builder/src/main/nodejs/MarkdownParser.js
Normal file
14
snark-document-builder/src/main/nodejs/MarkdownParser.js
Normal file
@ -0,0 +1,14 @@
|
||||
import {fromMarkdown} from 'mdast-util-from-markdown'
|
||||
|
||||
main()
|
||||
|
||||
function main()
|
||||
{
|
||||
if (process.argv.length < 3)
|
||||
throw "No input"
|
||||
|
||||
const markdown_string = process.argv[2]
|
||||
const mdast = fromMarkdown(markdown_string)
|
||||
|
||||
console.log(JSON.stringify(mdast))
|
||||
}
|
14
snark-document-builder/src/main/nodejs/MdRenderer.js
Normal file
14
snark-document-builder/src/main/nodejs/MdRenderer.js
Normal file
@ -0,0 +1,14 @@
|
||||
import {toMarkdown} from 'mdast-util-to-markdown'
|
||||
|
||||
main()
|
||||
|
||||
function main()
|
||||
{
|
||||
if (process.argv.length < 3)
|
||||
throw "No input"
|
||||
|
||||
const md_ast = JSON.parse(process.argv[2])
|
||||
const markdown = toMarkdown(md_ast)
|
||||
|
||||
console.log(markdown)
|
||||
}
|
4152
snark-document-builder/src/main/nodejs/package-lock.json
generated
Normal file
4152
snark-document-builder/src/main/nodejs/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
30
snark-document-builder/src/main/nodejs/package.json
Normal file
30
snark-document-builder/src/main/nodejs/package.json
Normal file
@ -0,0 +1,30 @@
|
||||
{
|
||||
|
||||
"type": "module",
|
||||
|
||||
"dependencies": {
|
||||
|
||||
"fs": "^0.0.1-security",
|
||||
|
||||
"hast-util-to-html": "^8.0.4",
|
||||
|
||||
"mdast-util-to-hast": "^12.3.0",
|
||||
|
||||
"node-fetch": "^3.3.1",
|
||||
|
||||
"remark-html": "^15.0.2",
|
||||
|
||||
"remark-parse": "^10.0.1",
|
||||
|
||||
"require": "^2.4.20",
|
||||
|
||||
"to-vfile": "^7.2.4",
|
||||
|
||||
"unified": "^10.1.2",
|
||||
|
||||
"whatwg-fetch": "^3.6.2"
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
26
snark-document-builder/src/main/python/SnarkParser.py
Normal file
26
snark-document-builder/src/main/python/SnarkParser.py
Normal file
@ -0,0 +1,26 @@
|
||||
import sys
|
||||
import re
|
||||
import json
|
||||
|
||||
assert(len(sys.argv) >= 2)
|
||||
|
||||
string = sys.argv[1]
|
||||
|
||||
outputfile = sys.argv[2] if len(sys.argv) >= 3 else None
|
||||
|
||||
pattern = r'^([\n|\t| ]*@include\([a-z|0-9|.|_|\/]*\)[\n|\t| ]*)*$'
|
||||
|
||||
files = []
|
||||
|
||||
if re.search("@include", string, re.IGNORECASE):
|
||||
if re.match(pattern, string):
|
||||
matches = re.findall(r'@include\((.*?)\)', string)
|
||||
files.extend(matches)
|
||||
else:
|
||||
sys.exit("Illformed string")
|
||||
|
||||
if outputfile is None:
|
||||
print(json.dumps(files))
|
||||
else:
|
||||
with open(outputfile, 'w+') as f:
|
||||
json.dump(files, f)
|
12
snark-document-builder/src/test/kotlin/DocumentBuilder.kt
Normal file
12
snark-document-builder/src/test/kotlin/DocumentBuilder.kt
Normal file
@ -0,0 +1,12 @@
|
||||
package documentBuilder
|
||||
|
||||
import org.junit.jupiter.api.Test
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import space.kscience.snark.storage.local.localStorage
|
||||
|
||||
class SomeTest {
|
||||
@Test
|
||||
fun justWorks() = runBlocking {
|
||||
// buildDocument(Directory("../example"))
|
||||
}
|
||||
}
|
@ -0,0 +1,3 @@
|
||||
# Hello
|
||||
|
||||
I'm almost empty test document without any dependencies
|
@ -12,6 +12,10 @@ dependencies {
|
||||
api("io.ktor:ktor-server-core:$ktorVersion")
|
||||
api("io.ktor:ktor-server-html-builder:$ktorVersion")
|
||||
api("io.ktor:ktor-server-host-common:$ktorVersion")
|
||||
implementation("io.ktor:ktor-server-netty:2.3.0")
|
||||
implementation(project(":snark-storage-driver"))
|
||||
implementation("io.ktor:ktor-server-partial-content:$ktorVersion")
|
||||
implementation("io.ktor:ktor-server-auto-head-response:$ktorVersion")
|
||||
|
||||
testApi("io.ktor:ktor-server-tests:$ktorVersion")
|
||||
}
|
12
snark-ktor/snark-ktor.iml
Normal file
12
snark-ktor/snark-ktor.iml
Normal file
@ -0,0 +1,12 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="JAVA_MODULE" version="4">
|
||||
<component name="NewModuleRootManager" inherit-compiler-output="true">
|
||||
<exclude-output />
|
||||
<content url="file://$MODULE_DIR$">
|
||||
<sourceFolder url="file://$MODULE_DIR$/src/main/kotlin" isTestSource="false" />
|
||||
</content>
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
<orderEntry type="library" name="KotlinJavaRuntime" level="project" />
|
||||
</component>
|
||||
</module>
|
@ -0,0 +1,131 @@
|
||||
package space.kscience.snark.ktor
|
||||
|
||||
import io.ktor.http.*
|
||||
import io.ktor.server.application.*
|
||||
import io.ktor.server.response.*
|
||||
import io.ktor.server.routing.*
|
||||
import io.ktor.http.content.*
|
||||
import io.ktor.server.engine.*
|
||||
import io.ktor.server.netty.*
|
||||
import io.ktor.server.html.*
|
||||
import io.ktor.server.request.*
|
||||
import kotlinx.html.*
|
||||
import space.kscience.snark.storage.Directory
|
||||
import space.kscience.snark.storage.unzip.unzip
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.createTempFile
|
||||
import kotlin.io.writeBytes
|
||||
import kotlin.io.path.Path
|
||||
|
||||
public interface DataHolder {
|
||||
public suspend fun init(relativePath: Path) : Directory
|
||||
|
||||
public suspend fun represent(relativePath: Path): String
|
||||
public suspend fun toPdf(relativePath: Path) : Path
|
||||
}
|
||||
|
||||
public class SNARKServer(private val dataHolder: DataHolder, private val port: Int): Runnable {
|
||||
private var relativePath = Path("")
|
||||
|
||||
private suspend fun receivePath(call: ApplicationCall) {
|
||||
val pathString = call.receiveParameters()["path"]?:""
|
||||
relativePath = Path(pathString.dropWhile{it == '/'})
|
||||
call.respondRedirect("/")
|
||||
}
|
||||
private suspend fun renderGet(call: ApplicationCall) {
|
||||
call.respondText(dataHolder.represent(relativePath), ContentType.Text.Html)
|
||||
}
|
||||
private suspend fun renderFile(call: ApplicationCall) {
|
||||
call.response.header(
|
||||
HttpHeaders.ContentDisposition,
|
||||
ContentDisposition.Attachment.withParameter(ContentDisposition.Parameters.FileName, "output.tex")
|
||||
.toString()
|
||||
)
|
||||
call.respondFile(dataHolder.toPdf(relativePath).toFile())
|
||||
call.respondRedirect("/")
|
||||
|
||||
}
|
||||
private suspend fun renderUpload(call: ApplicationCall) {
|
||||
val multipartData = call.receiveMultipart()
|
||||
val tmp = createTempFile(suffix=".zip")
|
||||
multipartData.forEachPart { part ->
|
||||
when (part) {
|
||||
is PartData.FileItem -> {
|
||||
val fileBytes = part.streamProvider().readBytes()
|
||||
tmp.writeBytes(fileBytes)
|
||||
}
|
||||
else -> {
|
||||
}
|
||||
}
|
||||
part.dispose()
|
||||
}
|
||||
unzip(tmp.toPath().toString(), dataHolder.init(relativePath))
|
||||
call.respondRedirect("/")
|
||||
}
|
||||
private suspend fun renderMainPage(call: ApplicationCall) {
|
||||
call.respondHtml(HttpStatusCode.OK) {
|
||||
head {
|
||||
title {
|
||||
+"SNARK"
|
||||
}
|
||||
}
|
||||
body {
|
||||
h1 {
|
||||
+"SNARK"
|
||||
}
|
||||
p {
|
||||
+("Path: /" + relativePath.toString())
|
||||
}
|
||||
}
|
||||
body {
|
||||
postForm(action = "/changePath") {
|
||||
label {
|
||||
+ "Enter new path:"
|
||||
}
|
||||
input(name = "path", type = InputType.text) {}
|
||||
button {
|
||||
+"Change path"
|
||||
}
|
||||
}
|
||||
postForm (action = "/upload", encType = FormEncType.multipartFormData) {
|
||||
label {
|
||||
+"Choose zip archive: "
|
||||
}
|
||||
input (name = "file", type = InputType.file) {}
|
||||
button {
|
||||
+"Upload file"
|
||||
}
|
||||
}
|
||||
a("/data") {
|
||||
+"Show data\n"
|
||||
}
|
||||
getForm (action = "/download") {
|
||||
button {
|
||||
+"Download latex\n"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
override fun run() {
|
||||
embeddedServer(Netty, port) {
|
||||
routing {
|
||||
get("/") {
|
||||
renderMainPage(call)
|
||||
}
|
||||
post("/changePath") {
|
||||
receivePath(call)
|
||||
}
|
||||
post("/upload") {
|
||||
renderUpload(call)
|
||||
}
|
||||
get("/data") {
|
||||
renderGet(call)
|
||||
}
|
||||
get("/download") {
|
||||
renderFile(call)
|
||||
}
|
||||
}
|
||||
}.start(wait = true)
|
||||
}
|
||||
}
|
@ -0,0 +1,50 @@
|
||||
package space.kscience.snark.ktor
|
||||
|
||||
import space.kscience.snark.storage.Directory
|
||||
import space.kscience.snark.storage.local.localStorage
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.*
|
||||
import kotlin.io.path.createTempDirectory
|
||||
import kotlin.io.path.isDirectory
|
||||
import kotlin.io.path.listDirectoryEntries
|
||||
|
||||
private class LocalDataHolder: DataHolder {
|
||||
private var source: Path? = null
|
||||
private var response: String = ""
|
||||
|
||||
private fun getPath(relativePath: Path) : Path {
|
||||
return source!! / relativePath
|
||||
}
|
||||
override suspend fun init(relativePath: Path): Directory {
|
||||
if (source == null) {
|
||||
source = createTempDirectory()
|
||||
}
|
||||
val path = getPath(relativePath)
|
||||
path.createDirectories()
|
||||
path.toFile().deleteRecursively()
|
||||
path.createDirectory()
|
||||
return localStorage(path)
|
||||
}
|
||||
private fun buildResponse(from: Path, cur: Path) {
|
||||
for (entry in cur.listDirectoryEntries()) {
|
||||
if (entry.isDirectory()) {
|
||||
buildResponse(from, entry)
|
||||
} else {
|
||||
response += from.relativize(entry).toString() + "<br>"
|
||||
}
|
||||
}
|
||||
}
|
||||
override suspend fun represent(relativePath: Path) : String =
|
||||
if (source == null) {
|
||||
"No data was loaded!"
|
||||
} else {
|
||||
response = "List of files:<br>"
|
||||
val path = getPath(relativePath)
|
||||
buildResponse(path, path)
|
||||
response
|
||||
}
|
||||
}
|
||||
|
||||
fun main() {
|
||||
SNARKServer(LocalDataHolder(), 9090).run()
|
||||
}
|
27
snark-main/build.gradle.kts
Normal file
27
snark-main/build.gradle.kts
Normal file
@ -0,0 +1,27 @@
|
||||
plugins {
|
||||
id("space.kscience.gradle.jvm")
|
||||
`maven-publish`
|
||||
}
|
||||
|
||||
val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
|
||||
|
||||
|
||||
dependencies {
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
|
||||
implementation(project(":snark-ktor"))
|
||||
implementation(project(":snark-storage-driver"))
|
||||
implementation(project(":snark-document-builder"))
|
||||
|
||||
testImplementation(kotlin("test"))
|
||||
testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
|
||||
}
|
||||
|
||||
tasks.test {
|
||||
useJUnitPlatform()
|
||||
}
|
||||
|
||||
tasks.register<JavaExec>("run_server") {
|
||||
classpath = sourceSets.main.get().runtimeClasspath
|
||||
main = "space.kscience.snark.main.MainKt"
|
||||
workingDir = File(workingDir.parent)
|
||||
}
|
6
snark-main/ci/install_java.sh
Executable file
6
snark-main/ci/install_java.sh
Executable file
@ -0,0 +1,6 @@
|
||||
#!/bin/bash
|
||||
|
||||
JAVA_VERSION="$1"
|
||||
|
||||
source ~/.sdkman/bin/sdkman-init.sh
|
||||
sdk install java "$JAVA_VERSION"
|
3
snark-main/ci/install_sdk.sh
Executable file
3
snark-main/ci/install_sdk.sh
Executable file
@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
|
||||
curl -s "https://get.sdkman.io" | bash
|
19
snark-main/src/main/kotlin/space/kscience/snark/main/Main.kt
Normal file
19
snark-main/src/main/kotlin/space/kscience/snark/main/Main.kt
Normal file
@ -0,0 +1,19 @@
|
||||
package space.kscience.snark.main
|
||||
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import space.kscience.snark.ktor.SNARKServer
|
||||
import space.kscience.snark.storage.local.localStorage
|
||||
import kotlin.io.path.Path
|
||||
|
||||
// Entrypoint
|
||||
fun main(): Unit = runBlocking {
|
||||
// Parse config, create driver
|
||||
val port = 8080
|
||||
val directory = localStorage(Path("./rundata"))
|
||||
val server = SNARKServer(ServerDataHolder(directory), port)
|
||||
launch {
|
||||
server.run()
|
||||
}
|
||||
}
|
@ -0,0 +1,20 @@
|
||||
package space.kscience.snark.main
|
||||
|
||||
import space.kscience.snark.ktor.DataHolder
|
||||
import space.kscience.snark.storage.Directory
|
||||
import documentBuilder.*
|
||||
import java.nio.file.Path
|
||||
|
||||
internal class ServerDataHolder(private val directory: Directory): DataHolder {
|
||||
|
||||
override suspend fun init(relativePath: Path): Directory = directory
|
||||
|
||||
|
||||
override suspend fun represent(relativePath: Path): String {
|
||||
return buildDocument(directory, relativePath)
|
||||
}
|
||||
|
||||
override suspend fun toPdf(relativePath: Path) : Path {
|
||||
return buildLatex(directory, relativePath)
|
||||
}
|
||||
}
|
13
snark-main/src/test/kotlin/space/kscience/snark/main/Test.kt
Normal file
13
snark-main/src/test/kotlin/space/kscience/snark/main/Test.kt
Normal file
@ -0,0 +1,13 @@
|
||||
package space.kscience.snark.main
|
||||
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class Test {
|
||||
@Test
|
||||
fun justWorks() = runBlocking {
|
||||
delay(5)
|
||||
//main()
|
||||
}
|
||||
}
|
2
snark-pandoc-plugin/.gitignore
vendored
Normal file
2
snark-pandoc-plugin/.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
.pandoc/
|
||||
|
25
snark-pandoc-plugin/build.gradle.kts
Normal file
25
snark-pandoc-plugin/build.gradle.kts
Normal file
@ -0,0 +1,25 @@
|
||||
plugins {
|
||||
id("java")
|
||||
}
|
||||
|
||||
repositories {
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
java.sourceCompatibility = JavaVersion.VERSION_11
|
||||
|
||||
dependencies {
|
||||
implementation("commons-io:commons-io:2.7")
|
||||
implementation("org.slf4j:slf4j-simple:2.0.6")
|
||||
implementation("org.slf4j:slf4j-api:2.0.6")
|
||||
implementation("org.apache.commons:commons-exec:1.3")
|
||||
implementation("org.apache.commons:commons-compress:1.2")
|
||||
implementation("org.apache.ant:ant:1.10.13")
|
||||
implementation("com.fasterxml.jackson.core:jackson-databind:2.14.2")
|
||||
testImplementation("org.junit.jupiter:junit-jupiter-api:5.8.1")
|
||||
testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.8.1")
|
||||
}
|
||||
|
||||
tasks.getByName<Test>("test") {
|
||||
useJUnitPlatform()
|
||||
}
|
@ -0,0 +1,367 @@
|
||||
package space.kscience.snark.pandoc;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URI;
|
||||
import java.net.URL;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.Enumeration;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.apache.commons.compress.archivers.ArchiveEntry;
|
||||
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
|
||||
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
|
||||
import org.apache.commons.compress.archivers.zip.ZipFile;
|
||||
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
|
||||
import org.apache.commons.exec.OS;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
||||
public class Installer {
|
||||
|
||||
private static final Logger log
|
||||
= LoggerFactory.getLogger(Installer.class);
|
||||
|
||||
public enum OSType {
|
||||
WINDOWS("windows-x86_64.zip", "windows"),
|
||||
MAC_OS_AMD("x86_64-macOS.zip", "mac.os.amd"),
|
||||
MAC_OS_ARM("arm64-macOS.zip", "mac.os.arm"),
|
||||
LINUX_ARM("linux-arm64", "linux.arm"),
|
||||
LINUX_AMD("linux-amd64", "linux.amd");
|
||||
|
||||
private final String assetSuffix;
|
||||
private final String propertySuffix;
|
||||
OSType(String assetSuf, String propertySuf) {
|
||||
assetSuffix = assetSuf;
|
||||
propertySuffix = propertySuf;
|
||||
}
|
||||
public String getAssetSuffix() {
|
||||
return assetSuffix;
|
||||
}
|
||||
public String getPropertySuffix() {
|
||||
return propertySuffix;
|
||||
}
|
||||
}
|
||||
|
||||
private static class OSData {
|
||||
private URL urlForInstalling;
|
||||
private Path fileToInstall;
|
||||
private Path pathToPandoc;
|
||||
public OSData() {}
|
||||
public void setUrlForInstalling(URL urlForInstalling) {
|
||||
this.urlForInstalling = urlForInstalling;
|
||||
}
|
||||
public URL getUrlForInstalling() {
|
||||
return urlForInstalling;
|
||||
}
|
||||
public void setFileToInstall(Path fileToInstall) {
|
||||
this.fileToInstall = fileToInstall;
|
||||
}
|
||||
public Path getFileToInstall() {
|
||||
return fileToInstall;
|
||||
}
|
||||
public Path getPathToPandoc() {
|
||||
return pathToPandoc;
|
||||
}
|
||||
public void setPathToPandoc(Path pathToPandoc) {
|
||||
this.pathToPandoc = pathToPandoc;
|
||||
}
|
||||
}
|
||||
|
||||
private static Map<OSType, OSData> dataForInstalling = new HashMap<>(OSType.values().length);
|
||||
private static final Properties properties = new Properties();
|
||||
private static Path pandocDir = Path.of("./pandoc").toAbsolutePath();
|
||||
private static final int TIMEOUT_SECONDS = 2;
|
||||
private static final int ATTEMPTS = 3;
|
||||
|
||||
Installer() throws IOException, InterruptedException {
|
||||
try {
|
||||
properties.load(new FileInputStream(
|
||||
Thread.currentThread().getContextClassLoader().getResource("installer.properties").getPath()
|
||||
));
|
||||
} catch (Exception ex) {
|
||||
log.error("Error during download properties, ex: {}", ex.getMessage(), ex);
|
||||
throw ex;
|
||||
}
|
||||
|
||||
initFiles();
|
||||
var resp = getGithubUrls();
|
||||
initUrls(resp);
|
||||
}
|
||||
|
||||
private void initFiles() {
|
||||
for (var os : OSType.values()) {
|
||||
dataForInstalling.put(os, new OSData());
|
||||
switch (os) {
|
||||
case LINUX_AMD :
|
||||
case LINUX_ARM :
|
||||
dataForInstalling.get(os)
|
||||
.setFileToInstall(Path.of(pandocDir.toString() + "/pandoc.tar.gz"));
|
||||
break;
|
||||
default :
|
||||
dataForInstalling.get(os)
|
||||
.setFileToInstall(Path.of(pandocDir.toString() + "/pandoc.zip"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void initUrls(ResponseDto responseDto) throws IOException {
|
||||
|
||||
for (var os : OSType.values()) {
|
||||
var asset = responseDto.getAssetByOsSuffix(os.getAssetSuffix());
|
||||
var currUrl = asset.getBrowserDownloadUrl();
|
||||
|
||||
var currPath = properties.getProperty("path.to.pandoc." + os.getPropertySuffix()).replace("{version}",
|
||||
responseDto.getTagName());
|
||||
|
||||
dataForInstalling.get(os).setUrlForInstalling(URI.create(currUrl).toURL());
|
||||
dataForInstalling.get(os).setPathToPandoc(Path.of(pandocDir.toString() + currPath));
|
||||
log.info("Init {} url : {}, path to pandoc: {}", os, currUrl, dataForInstalling.get(os).getPathToPandoc());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Install last released pandoc from github
|
||||
* @return path to executable pandoc
|
||||
* @throws IOException in case incorrect github url or path of installation directory
|
||||
*/
|
||||
Path installPandoc() throws IOException {
|
||||
log.info("Start install");
|
||||
Path res;
|
||||
if (OS.isFamilyMac()) {
|
||||
if (OS.isArch("aarch64")) {
|
||||
res = installPandoc(OSType.MAC_OS_ARM);
|
||||
} else {
|
||||
res = installPandoc(OSType.MAC_OS_AMD);
|
||||
}
|
||||
} else if (OS.isFamilyUnix()) {
|
||||
if (OS.isArch("aarch64")) {
|
||||
res = installPandoc(OSType.LINUX_ARM);
|
||||
} else {
|
||||
res = installPandoc(OSType.LINUX_AMD);
|
||||
}
|
||||
} else if (OS.isFamilyWindows()) {
|
||||
res = installPandoc(OSType.WINDOWS);
|
||||
} else {
|
||||
throw new RuntimeException("Got unexpected os, could not install pandoc");
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
private Path installPandoc(OSType os) throws IOException {
|
||||
log.info(
|
||||
"Start installing pandoc os: {}, url: {}, file: {}",
|
||||
os,
|
||||
dataForInstalling.get(os).getUrlForInstalling(),
|
||||
dataForInstalling.get(os).getFileToInstall()
|
||||
);
|
||||
|
||||
clearInstallingDirectory();
|
||||
|
||||
if (!handleSaving(os)) {
|
||||
throw new RuntimeException("Could not save file from github");
|
||||
}
|
||||
if (!unarchive(os)) {
|
||||
throw new RuntimeException("Could not unzip file");
|
||||
}
|
||||
|
||||
if (!dataForInstalling.get(os).getPathToPandoc().toFile().setExecutable(true)) {
|
||||
throw new RuntimeException("Could not make pandoc executable");
|
||||
}
|
||||
|
||||
return dataForInstalling.get(os).getPathToPandoc();
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads from a (http/https) URL and saves to a file.
|
||||
* @param file File to write. Parent directory will be created if necessary
|
||||
* @param url http/https url to connect
|
||||
* @param secsConnectTimeout Seconds to wait for connection establishment
|
||||
* @param secsReadTimeout Read timeout in seconds - trasmission will abort if it freezes more than this
|
||||
* @return true if successfully save file and false if:
|
||||
* connection interrupted, timeout (but something was read)
|
||||
* server error (500...)
|
||||
* could not connect: connection timeout java.net.SocketTimeoutException
|
||||
* could not connect: java.net.ConnectException
|
||||
* could not resolve host (bad host, or no internet - no dns)
|
||||
* @throws IOException Only if URL is malformed or if could not create the file
|
||||
* @throws FileNotFoundException if did not find file for save
|
||||
*/
|
||||
private boolean saveUrl(final Path file, final URL url,
|
||||
int secsConnectTimeout, int secsReadTimeout) throws IOException {
|
||||
Files.createDirectories(file.getParent()); // make sure parent dir exists , this can throw exception
|
||||
var conn = url.openConnection(); // can throw exception if bad url
|
||||
if (secsConnectTimeout > 0) {
|
||||
conn.setConnectTimeout(secsConnectTimeout * 1000);
|
||||
}
|
||||
if (secsReadTimeout > 0) {
|
||||
conn.setReadTimeout(secsReadTimeout * 1000);
|
||||
}
|
||||
var ret = true;
|
||||
boolean somethingRead = false;
|
||||
try (var is = conn.getInputStream()) {
|
||||
try (var in = new BufferedInputStream(is);
|
||||
var fout = Files.newOutputStream(file)) {
|
||||
final byte data[] = new byte[8192];
|
||||
int count;
|
||||
while ((count = in.read(data)) > 0) {
|
||||
somethingRead = true;
|
||||
fout.write(data, 0, count);
|
||||
}
|
||||
}
|
||||
} catch (java.io.IOException e) {
|
||||
int httpcode = 999;
|
||||
try {
|
||||
httpcode = ((HttpURLConnection) conn).getResponseCode();
|
||||
} catch (Exception ee) {}
|
||||
|
||||
if (e instanceof FileNotFoundException) {
|
||||
throw new FileNotFoundException("Did not found file for install");
|
||||
}
|
||||
|
||||
if (somethingRead && e instanceof java.net.SocketTimeoutException) {
|
||||
log.error("Read something, but connection interrupted: {}", e.getMessage(), e);
|
||||
ret = false;
|
||||
} else if (httpcode >= 400 && httpcode < 600 ) {
|
||||
log.error("Got server error, httpcode: {}", httpcode);
|
||||
ret = false;
|
||||
} else if (e instanceof java.net.SocketTimeoutException) {
|
||||
log.error("Connection timeout: {}", e.getMessage(), e);
|
||||
ret = false;
|
||||
} else if (e instanceof java.net.ConnectException) {
|
||||
log.error("Could not connect: {}", e.getMessage(), e);
|
||||
ret = false;
|
||||
} else if (e instanceof java.net.UnknownHostException ) {
|
||||
log.error("Could not resolve host: {}", e.getMessage(), e);
|
||||
ret = false;
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
private boolean handleSaving(OSType os) throws IOException {
|
||||
var attempt = 0;
|
||||
var saveFile = false;
|
||||
|
||||
while (attempt < ATTEMPTS && !saveFile) {
|
||||
++attempt;
|
||||
saveFile = saveUrl(
|
||||
dataForInstalling.get(os).getFileToInstall(),
|
||||
dataForInstalling.get(os).getUrlForInstalling(),
|
||||
TIMEOUT_SECONDS,
|
||||
TIMEOUT_SECONDS);
|
||||
}
|
||||
|
||||
return saveFile;
|
||||
}
|
||||
|
||||
private boolean unarchive(OSType os) {
|
||||
try {
|
||||
switch (os) {
|
||||
case LINUX_AMD:
|
||||
case LINUX_ARM :
|
||||
unTarGz(dataForInstalling.get(os).getFileToInstall(), pandocDir);
|
||||
break;
|
||||
default :
|
||||
unZip(dataForInstalling.get(os).getFileToInstall(), pandocDir);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("Could not perform unarchiving: {}", e.getMessage(), e);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
private void unTarGz(Path pathInput, Path targetDir) throws IOException {
|
||||
try (var tarIn =
|
||||
new TarArchiveInputStream(
|
||||
new GzipCompressorInputStream(
|
||||
new BufferedInputStream(Files.newInputStream(pathInput))))) {
|
||||
ArchiveEntry archiveEntry;
|
||||
while ((archiveEntry = tarIn.getNextEntry()) != null) {
|
||||
var pathEntryOutput = targetDir.resolve(archiveEntry.getName());
|
||||
if (archiveEntry.isDirectory()) {
|
||||
Files.createDirectory(pathEntryOutput);
|
||||
} else {
|
||||
Files.copy(tarIn, pathEntryOutput);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void unZip(Path pathInput, Path targetDir) throws IOException {
|
||||
ZipFile zipFile = new ZipFile(pathInput.toFile());
|
||||
try {
|
||||
Enumeration<ZipArchiveEntry> entries = zipFile.getEntries();
|
||||
while (entries.hasMoreElements()) {
|
||||
ZipArchiveEntry zipEntry = entries.nextElement();
|
||||
var pathEntryOutput = targetDir.resolve(zipEntry.getName());
|
||||
if (zipEntry.isDirectory()) {
|
||||
Files.createDirectories(pathEntryOutput);
|
||||
} else {
|
||||
Files.createDirectories(pathEntryOutput.getParent());
|
||||
Files.copy(zipFile.getInputStream(zipEntry), pathEntryOutput);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
zipFile.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Clear installing directory
|
||||
*/
|
||||
public static void clearInstallingDirectory() {
|
||||
try {
|
||||
FileUtils.cleanDirectory(pandocDir.toFile());
|
||||
} catch (IOException e) {
|
||||
log.error("Could not clean installing directory");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set directory to install pandoc
|
||||
* @param newDir
|
||||
*/
|
||||
public void setInstallingDirectory(Path newDir) {
|
||||
pandocDir = newDir.toAbsolutePath();
|
||||
}
|
||||
|
||||
private ResponseDto getGithubUrls() throws IOException, InterruptedException {
|
||||
var uri = URI.create(properties.getProperty("github.url"));
|
||||
var client = HttpClient.newHttpClient();
|
||||
var request = HttpRequest
|
||||
.newBuilder()
|
||||
.uri(uri)
|
||||
.version(HttpClient.Version.HTTP_2)
|
||||
.timeout(Duration.ofMinutes(1))
|
||||
.header("Accept", "application/vnd.github+json")
|
||||
.GET()
|
||||
.build();
|
||||
var response = client.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
log.info("Got response from github, status: {}", response.statusCode());
|
||||
|
||||
var objectMapper = new ObjectMapper();
|
||||
objectMapper.configure(DeserializationFeature.USE_JAVA_ARRAY_FOR_JSON_ARRAY, true);
|
||||
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
|
||||
return objectMapper.readValue(response.body(), ResponseDto.class);
|
||||
|
||||
}
|
||||
}
|
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,142 @@
|
||||
package space.kscience.snark.pandoc;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Function;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
||||
public class PandocWrapper {
|
||||
|
||||
private static final Logger log
|
||||
= LoggerFactory.getLogger(PandocWrapper.class);
|
||||
|
||||
private static final Installer installer;
|
||||
|
||||
static {
|
||||
try {
|
||||
installer = new Installer();
|
||||
} catch (IOException | InterruptedException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
private static String pandocPath = "pandoc"; // got pandoc at PATH
|
||||
|
||||
public static String getPandocPath() {
|
||||
return pandocPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Install pandoc if needed then perform block
|
||||
* @param block
|
||||
* @return block's return value
|
||||
* @param <T>
|
||||
*/
|
||||
public <T> Object use(Function<PandocWrapper, T> block) {
|
||||
if (!isPandocInstalled()) {
|
||||
installPandoc();
|
||||
}
|
||||
return block.apply(this);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if pandoc is installed
|
||||
* @return true if installed false otherwise
|
||||
*/
|
||||
public static boolean isPandocInstalled() {
|
||||
var pb = new PandocCommandBuilder().getVersion();
|
||||
return execute(pb);
|
||||
}
|
||||
/**
|
||||
* Call pandoc with options described by commandBuilder.
|
||||
* @param commandBuilder
|
||||
* @return true if successfully false otherwise
|
||||
*/
|
||||
public static boolean execute(PandocCommandBuilder commandBuilder) {
|
||||
return execute(commandBuilder, null, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Call pandoc with options described by commandBuilder and log output to outputFile
|
||||
* @param commandBuilder
|
||||
* @return true if successfully false otherwise
|
||||
*/
|
||||
public static boolean execute(PandocCommandBuilder commandBuilder, Path outputFile) {
|
||||
return execute(commandBuilder, outputFile, null);
|
||||
}
|
||||
/**
|
||||
* Call pandoc with options described by commandBuilder and log output to outputFile and error to errorFile.
|
||||
* In case errors write exit code to errorFile
|
||||
* @param commandBuilder
|
||||
* @return true if successfully false otherwise
|
||||
*/
|
||||
public static boolean execute(PandocCommandBuilder commandBuilder, Path outputFile, Path errorFile) {
|
||||
try {
|
||||
Process pandoc = new ProcessBuilder(commandBuilder.build()).start();
|
||||
pandoc.waitFor(1, TimeUnit.SECONDS);
|
||||
|
||||
BufferedReader inp = new BufferedReader(new InputStreamReader(pandoc.getInputStream()));
|
||||
String currLine = inp.readLine();
|
||||
|
||||
log.info("log output from pandoc to: {}", outputFile);
|
||||
do {
|
||||
if (outputFile == null) {
|
||||
log.info(currLine);
|
||||
} else {
|
||||
Files.writeString(outputFile, currLine + "\n", StandardOpenOption.CREATE, StandardOpenOption.APPEND);
|
||||
}
|
||||
} while ((currLine = inp.readLine()) != null);
|
||||
inp.close();
|
||||
|
||||
if (pandoc.exitValue() == 0) {
|
||||
log.info("Successfully execute");
|
||||
return true;
|
||||
} else {
|
||||
log.error("Got problems with executing, pandoc exit error: {}", pandoc.exitValue());
|
||||
|
||||
BufferedReader input = new BufferedReader(new InputStreamReader(pandoc.getErrorStream()));
|
||||
String line = input.readLine();
|
||||
log.info("log error stream from pandoc to: {}", errorFile);
|
||||
|
||||
if (errorFile != null) {
|
||||
Files.writeString(errorFile, "exit code: " + pandoc.exitValue());
|
||||
}
|
||||
do {
|
||||
if (errorFile == null) {
|
||||
log.info(line);
|
||||
} else {
|
||||
Files.writeString(errorFile, currLine + "\n", StandardOpenOption.CREATE, StandardOpenOption.APPEND);
|
||||
}
|
||||
} while ((line = input.readLine()) != null);
|
||||
input.close();
|
||||
|
||||
return false;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Got problems with executing: " + e.getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Install pandoc and set executable path.
|
||||
* @return true if success false otherwise
|
||||
*/
|
||||
public static boolean installPandoc() {
|
||||
try {
|
||||
pandocPath = installer.installPandoc().toString();
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
log.error("Got error: {}", e.getMessage(), e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
@ -0,0 +1,70 @@
|
||||
package space.kscience.snark.pandoc;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
/**
|
||||
* Response from github/releases/latest
|
||||
*/
|
||||
public class ResponseDto {
|
||||
|
||||
public AssetDto[] getAssets() {
|
||||
return assets;
|
||||
}
|
||||
|
||||
public void setAssets(AssetDto[] assets) {
|
||||
this.assets = assets;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param osSuffix
|
||||
* @return asset appropriate to os
|
||||
*/
|
||||
public AssetDto getAssetByOsSuffix(String osSuffix) {
|
||||
for (var asset : assets) {
|
||||
if (asset.getName().contains(osSuffix)) {
|
||||
return asset;
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException("Unexpected osSuffix");
|
||||
}
|
||||
|
||||
|
||||
public static class AssetDto {
|
||||
|
||||
@JsonProperty("browser_download_url")
|
||||
private String browserDownloadUrl;
|
||||
private String name;
|
||||
|
||||
public String getBrowserDownloadUrl() {
|
||||
return browserDownloadUrl;
|
||||
}
|
||||
|
||||
public void setBrowserDownloadUrl(String browserDownloadUrl) {
|
||||
this.browserDownloadUrl = browserDownloadUrl;
|
||||
}
|
||||
|
||||
public AssetDto() {}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
}
|
||||
private AssetDto[] assets;
|
||||
|
||||
public String getTagName() {
|
||||
return tagName;
|
||||
}
|
||||
|
||||
public void setTagName(String tagName) {
|
||||
this.tagName = tagName;
|
||||
}
|
||||
|
||||
@JsonProperty("tag_name")
|
||||
private String tagName;
|
||||
|
||||
public ResponseDto() {}
|
||||
}
|
@ -0,0 +1,8 @@
|
||||
path.to.pandoc.mac.os.arm=/pandoc-{version}-arm64/bin/pandoc
|
||||
path.to.pandoc.mac.os.amd=/pandoc-{version}-x86_64/bin/pandoc
|
||||
path.to.pandoc.windows=/pandoc-{version}/pandoc.exe
|
||||
path.to.pandoc.linux.amd=/pandoc-{version}/bin/pandoc
|
||||
path.to.pandoc.linux.arm=/pandoc-{version}/bin/pandoc
|
||||
|
||||
github.url=https://api.github.com/repos/jgm/pandoc/releases/latest
|
||||
|
123
snark-pandoc-plugin/src/test/java/PandocWrapperTest.java
Normal file
123
snark-pandoc-plugin/src/test/java/PandocWrapperTest.java
Normal file
@ -0,0 +1,123 @@
|
||||
import java.io.BufferedReader;
|
||||
import java.io.FileReader;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import space.kscience.snark.pandoc.Installer;
|
||||
import space.kscience.snark.pandoc.PandocCommandBuilder;
|
||||
import space.kscience.snark.pandoc.PandocWrapper;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
public class PandocWrapperTest {
|
||||
|
||||
private static final Path CORRECT_MD = Path.of("./src/test/testing_directory/first_test.md");
|
||||
private static final Path TEX_PATH_TO = Path.of("./src/test/testing_directory/output1.tex");
|
||||
private static final Path TESTING_DIRECTORY = Path.of("./src/test/testing_directory");
|
||||
private final PandocWrapper pandocWrapper = new PandocWrapper();
|
||||
|
||||
@Test
|
||||
public void when_gotPandocAndCorrectArgs_doConverting() {
|
||||
try {
|
||||
var res = pandocWrapper.use(p -> {
|
||||
var command = new PandocCommandBuilder(List.of(CORRECT_MD), TEX_PATH_TO);
|
||||
return PandocWrapper.execute(command);
|
||||
});
|
||||
assertTrue((Boolean) res);
|
||||
assertTrue(TEX_PATH_TO.toFile().exists());
|
||||
|
||||
var reader = new BufferedReader(new FileReader(TEX_PATH_TO.toFile()));
|
||||
String fileString = reader.lines().collect(Collectors.joining());
|
||||
|
||||
assertTrue(fileString.contains("Some simple text"));
|
||||
assertTrue(fileString.contains("\\subsection{Copy elision}"));
|
||||
assertTrue(fileString.contains("return"));
|
||||
|
||||
Files.delete(TEX_PATH_TO);
|
||||
|
||||
} catch (Exception ex) {
|
||||
fail("Unexpected exception during test when_gotPandocAndCorrectArgs_doConverting()", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void when_gotPandocAndNotExistsFromFile_then_error() {
|
||||
var notExistsFile = Path.of("./src/test/testing_directory/non_exists_test.md");
|
||||
assertFalse(notExistsFile.toFile().exists());
|
||||
var res = pandocWrapper.use(p -> {
|
||||
var command = new PandocCommandBuilder(List.of(notExistsFile), TEX_PATH_TO);
|
||||
return PandocWrapper.execute(command);
|
||||
});
|
||||
assertFalse((Boolean) res);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void when_gotPandocAndPassDirectory_then_error() {
|
||||
assertTrue(TESTING_DIRECTORY.toFile().isDirectory());
|
||||
var res = pandocWrapper.use(p -> {
|
||||
var command = new PandocCommandBuilder(List.of(TESTING_DIRECTORY), TEX_PATH_TO);
|
||||
return PandocWrapper.execute(command);
|
||||
});
|
||||
assertFalse((Boolean) res);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void when_askVersionToFile_then_Ok() throws IOException {
|
||||
Path outputFile = Files.createTempFile(TESTING_DIRECTORY, "output", ".txt");
|
||||
|
||||
var res = pandocWrapper.use(p -> {
|
||||
var command = new PandocCommandBuilder();
|
||||
command.getVersion();
|
||||
return PandocWrapper.execute(command, outputFile);
|
||||
});
|
||||
|
||||
var reader = new BufferedReader(new FileReader(outputFile.toFile()));
|
||||
String fileString = reader.lines().collect(Collectors.joining());
|
||||
assertTrue(fileString.contains("pandoc"));
|
||||
assertTrue(fileString.contains("This is free software"));
|
||||
assertTrue((Boolean) res);
|
||||
|
||||
Files.delete(outputFile);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void when_error_then_writeToErrorStream() throws IOException {
|
||||
Path outputFile = Files.createTempFile(TESTING_DIRECTORY, "output", ".txt");
|
||||
Path errorFile = Files.createTempFile(TESTING_DIRECTORY, "error", ".txt");
|
||||
|
||||
var res = pandocWrapper.use(p -> {
|
||||
var command = new PandocCommandBuilder(List.of(Path.of("./simple.txt")), TEX_PATH_TO);
|
||||
command.formatFrom("txt");
|
||||
return PandocWrapper.execute(command, outputFile, errorFile);
|
||||
});
|
||||
|
||||
var reader = new BufferedReader(new FileReader(errorFile.toFile()));
|
||||
String fileString = reader.lines().collect(Collectors.joining());
|
||||
assertFalse((Boolean) res);
|
||||
assertTrue(fileString.contains("21"));
|
||||
|
||||
Files.delete(outputFile);
|
||||
Files.delete(errorFile);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void when_installPandoc_thenFindIt() {
|
||||
Installer.clearInstallingDirectory();
|
||||
assertTrue(PandocWrapper.installPandoc());
|
||||
assertTrue(PandocWrapper.isPandocInstalled());
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
public static void clear() {
|
||||
Installer.clearInstallingDirectory();
|
||||
}
|
||||
}
|
15
snark-pandoc-plugin/src/test/testing_directory/first_test.md
Normal file
15
snark-pandoc-plugin/src/test/testing_directory/first_test.md
Normal file
@ -0,0 +1,15 @@
|
||||
|
||||
|
||||
## Copy elision
|
||||
### RVO/NRVO
|
||||
Some simple text
|
||||
```c++
|
||||
A f() {
|
||||
return {5};
|
||||
}
|
||||
|
||||
A g() {
|
||||
A a(5);
|
||||
return a;
|
||||
}
|
||||
```
|
@ -0,0 +1 @@
|
||||
hello
|
22
snark-storage-driver/build.gradle.kts
Normal file
22
snark-storage-driver/build.gradle.kts
Normal file
@ -0,0 +1,22 @@
|
||||
plugins {
|
||||
id("space.kscience.gradle.jvm")
|
||||
`maven-publish`
|
||||
}
|
||||
|
||||
val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
|
||||
val awsSdkVersion = "0.+"
|
||||
|
||||
|
||||
dependencies {
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
|
||||
|
||||
// s3 Driver dependency
|
||||
implementation("aws.sdk.kotlin:s3:$awsSdkVersion")
|
||||
|
||||
testImplementation(kotlin("test"))
|
||||
testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
|
||||
}
|
||||
|
||||
tasks.test {
|
||||
useJUnitPlatform()
|
||||
}
|
@ -0,0 +1,40 @@
|
||||
package space.kscience.snark.storage
|
||||
|
||||
import aws.sdk.kotlin.services.s3.S3Client
|
||||
import space.kscience.snark.storage.local.localStorage
|
||||
import space.kscience.snark.storage.s3.s3Bucket
|
||||
import space.kscience.snark.storage.s3.s3Storage
|
||||
import java.nio.file.Path
|
||||
|
||||
private const val DEFAULT_REGION = "arctic-vault"
|
||||
|
||||
public sealed interface Config {
|
||||
public fun build(): Directory
|
||||
}
|
||||
|
||||
public data class LocalConfig(val path: Path) : Config {
|
||||
override fun build(): Directory {
|
||||
return localStorage(path)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* ~/.aws/credentials.json file is required
|
||||
*/
|
||||
internal fun buildS3Client(regionSpec: String): S3Client {
|
||||
return S3Client {
|
||||
region = regionSpec
|
||||
}
|
||||
}
|
||||
|
||||
public data class S3BucketConfig(val bucketName: String, val region: String = DEFAULT_REGION) : Config {
|
||||
override fun build(): Directory {
|
||||
return s3Bucket(buildS3Client(region), bucketName)
|
||||
}
|
||||
}
|
||||
|
||||
public data class S3ServiceConfig(val region: String = DEFAULT_REGION) : Config {
|
||||
override fun build(): Directory {
|
||||
return s3Storage(buildS3Client(region))
|
||||
}
|
||||
}
|
@ -0,0 +1,46 @@
|
||||
package space.kscience.snark.storage
|
||||
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.Path
|
||||
|
||||
public interface Directory : AutoCloseable {
|
||||
// get file from subtree
|
||||
public suspend fun get(filename: Path): FileReader
|
||||
|
||||
@Deprecated("Use put")
|
||||
public suspend fun create(filename: String, ignoreIfExists: Boolean = false)
|
||||
|
||||
// put file to subtree
|
||||
public suspend fun put(filename: Path): FileWriter
|
||||
|
||||
public suspend fun getSubdir(path: Path): Directory
|
||||
|
||||
@Deprecated("Directories are created on put")
|
||||
public suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean = false): Directory
|
||||
|
||||
@Deprecated("Not a good idea")
|
||||
public val path: Path
|
||||
|
||||
public companion object {
|
||||
public fun fromConfig(config: Config): Directory {
|
||||
return config.build()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public suspend fun Directory.get(filename: String): FileReader = get(Path(filename))
|
||||
|
||||
public suspend fun Directory.put(filename: String): FileWriter = put(Path(filename))
|
||||
|
||||
public suspend operator fun Directory.div(path: Path): Directory = getSubdir(path)
|
||||
|
||||
public suspend operator fun Directory.div(path: String): Directory = getSubdir(Path(path))
|
||||
|
||||
public interface FileReader : AutoCloseable {
|
||||
public suspend fun readAll(): ByteArray
|
||||
}
|
||||
|
||||
public interface FileWriter : AutoCloseable {
|
||||
public suspend fun write(bytes: ByteArray)
|
||||
}
|
@ -0,0 +1,71 @@
|
||||
package space.kscience.snark.storage.local
|
||||
|
||||
import space.kscience.snark.storage.Directory
|
||||
import space.kscience.snark.storage.FileReader
|
||||
import space.kscience.snark.storage.FileWriter
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.*
|
||||
|
||||
public fun localStorage(rootPath: Path): Directory {
|
||||
return LocalDirectory(rootPath, Path(""))
|
||||
}
|
||||
|
||||
internal class LocalFile(private val path: Path) : FileReader, FileWriter {
|
||||
override fun close() {}
|
||||
override suspend fun readAll(): ByteArray = path.readBytes()
|
||||
|
||||
override suspend fun write(bytes: ByteArray) {
|
||||
path.parent.createDirectories()
|
||||
try {
|
||||
path.createFile()
|
||||
} catch (ex: Exception) {
|
||||
// Do nothing
|
||||
}
|
||||
path.writeBytes(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
internal class LocalDirectory(private val root: Path, private val currentDir: Path) : Directory {
|
||||
@Deprecated("Use Path, not String")
|
||||
private fun realpath(child: String): Path = root / currentDir / child
|
||||
private fun realpath(child: Path): Path = root / currentDir / child
|
||||
|
||||
override fun close() {}
|
||||
|
||||
override suspend fun get(filename: Path): LocalFile = LocalFile(realpath(filename))
|
||||
|
||||
@Deprecated("Use put")
|
||||
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
|
||||
val dir = realpath(filename)
|
||||
dir.parent.createDirectories()
|
||||
try {
|
||||
realpath(filename).createFile()
|
||||
} catch (ex: java.nio.file.FileAlreadyExistsException) {
|
||||
if (!ignoreIfExists) {
|
||||
throw ex
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun put(filename: Path): LocalFile = get(filename)
|
||||
|
||||
override suspend fun getSubdir(path: Path): LocalDirectory = LocalDirectory(root, currentDir / path)
|
||||
|
||||
@Deprecated("Directories are created on put")
|
||||
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): LocalDirectory {
|
||||
val dir = realpath(dirname)
|
||||
dir.parent.createDirectories()
|
||||
try {
|
||||
dir.createDirectory()
|
||||
} catch (ex: java.nio.file.FileAlreadyExistsException) {
|
||||
if (!ignoreIfExists) {
|
||||
throw ex
|
||||
}
|
||||
}
|
||||
return LocalDirectory(root, currentDir / dirname)
|
||||
}
|
||||
|
||||
@Deprecated("Not a good idea")
|
||||
override val path: Path
|
||||
get() = currentDir
|
||||
}
|
@ -0,0 +1,45 @@
|
||||
package space.kscience.snark.storage.s3
|
||||
|
||||
import aws.sdk.kotlin.services.s3.S3Client
|
||||
import space.kscience.snark.storage.Directory
|
||||
import space.kscience.snark.storage.FileReader
|
||||
import space.kscience.snark.storage.FileWriter
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.div
|
||||
|
||||
internal class S3Directory(
|
||||
private val client: S3Client,
|
||||
private val bucketName: String,
|
||||
private val currentDir: Path,
|
||||
) : Directory {
|
||||
override suspend fun get(filename: Path): FileReader =
|
||||
S3FileReader(client, bucketName, currentDir / filename)
|
||||
|
||||
@Deprecated("Use put")
|
||||
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
|
||||
if (!ignoreIfExists) {
|
||||
TODO("could not check if file exists")
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun put(filename: Path): FileWriter =
|
||||
S3FileWriter(client, bucketName, currentDir / filename)
|
||||
|
||||
override suspend fun getSubdir(path: Path): S3Directory =
|
||||
S3Directory(client, bucketName, currentDir / path)
|
||||
|
||||
@Deprecated("Directories are created on put")
|
||||
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): S3Directory =
|
||||
if (!ignoreIfExists) {
|
||||
TODO("could not check if directory exists")
|
||||
} else {
|
||||
S3Directory(client, bucketName, currentDir / dirname)
|
||||
}
|
||||
|
||||
@Deprecated("Not a good idea")
|
||||
override val path: Path
|
||||
get() = currentDir
|
||||
|
||||
override fun close() {
|
||||
}
|
||||
}
|
@ -0,0 +1,41 @@
|
||||
package space.kscience.snark.storage.s3
|
||||
|
||||
import aws.sdk.kotlin.services.s3.S3Client
|
||||
import aws.sdk.kotlin.services.s3.model.GetObjectRequest
|
||||
import aws.sdk.kotlin.services.s3.putObject
|
||||
import aws.smithy.kotlin.runtime.content.ByteStream
|
||||
import aws.smithy.kotlin.runtime.content.toByteArray
|
||||
import space.kscience.snark.storage.FileReader
|
||||
import space.kscience.snark.storage.FileWriter
|
||||
import java.nio.file.Path
|
||||
|
||||
internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) :
|
||||
FileReader {
|
||||
override suspend fun readAll(): ByteArray {
|
||||
val result = client.getObject(GetObjectRequest {
|
||||
bucket = bucketName
|
||||
key = path.toString()
|
||||
}) {
|
||||
it.body?.toByteArray() ?: ByteArray(0)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
}
|
||||
}
|
||||
|
||||
internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) :
|
||||
FileWriter {
|
||||
override suspend fun write(bytes: ByteArray) {
|
||||
client.putObject {
|
||||
bucket = bucketName
|
||||
key = path.toString()
|
||||
body = ByteStream.fromBytes(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
}
|
||||
|
||||
}
|
@ -0,0 +1,67 @@
|
||||
package space.kscience.snark.storage.s3
|
||||
|
||||
import aws.sdk.kotlin.services.s3.S3Client
|
||||
import aws.sdk.kotlin.services.s3.createBucket
|
||||
import aws.sdk.kotlin.services.s3.headBucket
|
||||
import space.kscience.snark.storage.Directory
|
||||
import space.kscience.snark.storage.FileReader
|
||||
import space.kscience.snark.storage.FileWriter
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.Path
|
||||
|
||||
public fun s3Storage(client: S3Client): Directory =
|
||||
S3Root(client)
|
||||
|
||||
public fun s3Bucket(client: S3Client, bucket: String): Directory =
|
||||
S3Directory(client, bucket, Path(""))
|
||||
|
||||
internal fun splitPathIntoBucketAndPath(path: Path): Pair<String, Path> {
|
||||
val bucket = path.getName(0)
|
||||
val filePath = path.relativize(bucket)
|
||||
return Pair(bucket.toString(), filePath)
|
||||
}
|
||||
|
||||
internal class S3Root(private val client: S3Client) : Directory {
|
||||
|
||||
override suspend fun get(filename: Path): FileReader {
|
||||
throw NoSuchFileException(filename.toFile())
|
||||
}
|
||||
|
||||
@Deprecated("Use put")
|
||||
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
|
||||
throw NoSuchFileException(Path(filename).toFile())
|
||||
}
|
||||
|
||||
override suspend fun put(filename: Path): FileWriter {
|
||||
throw NoSuchFileException(filename.toFile())
|
||||
}
|
||||
|
||||
override suspend fun getSubdir(path: Path): Directory = try {
|
||||
val (bucketName, filePath) = splitPathIntoBucketAndPath(path)
|
||||
client.headBucket {
|
||||
bucket = bucketName
|
||||
}
|
||||
S3Directory(client, bucketName, filePath)
|
||||
} catch (ex: Exception) {
|
||||
throw AccessDeniedException(path.toFile(), reason = ex.message)
|
||||
}
|
||||
|
||||
@Deprecated("Directories are created on put")
|
||||
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try {
|
||||
val (bucketName, filePath) = splitPathIntoBucketAndPath(Path(dirname))
|
||||
client.createBucket {
|
||||
bucket = bucketName
|
||||
}
|
||||
S3Directory(client, bucketName, filePath)
|
||||
} catch (ex: Exception) {
|
||||
throw AccessDeniedException(Path(dirname).toFile(), reason = ex.message)
|
||||
}
|
||||
|
||||
@Deprecated("Not a good idea")
|
||||
override val path: Path
|
||||
get() = Path("")
|
||||
|
||||
override fun close() {
|
||||
}
|
||||
|
||||
}
|
@ -0,0 +1,28 @@
|
||||
package space.kscience.snark.storage.unzip
|
||||
|
||||
import space.kscience.snark.storage.*
|
||||
import java.io.FileInputStream
|
||||
import java.util.zip.ZipInputStream
|
||||
|
||||
public suspend fun unzip(source_path: String, target: Directory) {
|
||||
val zis = ZipInputStream(FileInputStream(source_path))
|
||||
var zipEntry = zis.nextEntry
|
||||
while (zipEntry != null) {
|
||||
if (!zipEntry.isDirectory) {
|
||||
val filename = zipEntry.name
|
||||
target.create(filename, true)
|
||||
val fos = target.put(filename)
|
||||
var sz = zipEntry.size.toInt()
|
||||
if (sz == -1) {
|
||||
sz = 1024
|
||||
}
|
||||
val buffer = ByteArray(sz)
|
||||
zis.read(buffer)
|
||||
fos.write(buffer)
|
||||
fos.close()
|
||||
}
|
||||
zipEntry = zis.nextEntry
|
||||
}
|
||||
zis.closeEntry()
|
||||
zis.close()
|
||||
}
|
@ -0,0 +1,25 @@
|
||||
package space.kscience.snark.storage
|
||||
|
||||
import org.junit.jupiter.api.Test
|
||||
import kotlin.io.path.createTempDirectory
|
||||
|
||||
class JustCreates {
|
||||
|
||||
@Test
|
||||
fun s3Created() {
|
||||
val dir = Directory.fromConfig(S3ServiceConfig())
|
||||
dir.close()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun s3BucketCreated() {
|
||||
val dir = Directory.fromConfig(S3BucketConfig("snark-test"))
|
||||
dir.close()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun localCreated() {
|
||||
val dir = Directory.fromConfig(LocalConfig(createTempDirectory("snark-test")))
|
||||
dir.close()
|
||||
}
|
||||
}
|
@ -0,0 +1,47 @@
|
||||
package space.kscience.snark.storage.local
|
||||
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.junit.jupiter.api.BeforeAll
|
||||
import space.kscience.snark.storage.*
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.createTempDirectory
|
||||
import kotlin.io.path.deleteExisting
|
||||
import kotlin.test.AfterTest
|
||||
import kotlin.test.BeforeTest
|
||||
import kotlin.test.Test
|
||||
import kotlin.io.path.*
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
class Example {
|
||||
var tempDir: Path? = null
|
||||
var somedir: Directory? = null
|
||||
|
||||
@BeforeTest
|
||||
fun setUp() {
|
||||
tempDir = createTempDirectory()
|
||||
somedir = localStorage(tempDir!!)
|
||||
}
|
||||
|
||||
@AfterTest
|
||||
fun tearDown() {
|
||||
tempDir!!.toFile().deleteRecursively()
|
||||
somedir = null
|
||||
}
|
||||
|
||||
@Test
|
||||
fun exampleTest() = runBlocking {
|
||||
somedir!!.put("somefile").write("hello".toByteArray())
|
||||
assertEquals("hello", somedir!!.get("somefile").readAll().decodeToString())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun subdirExample() = runBlocking {
|
||||
val dir1 = somedir!! / "tmp1"
|
||||
dir1.put("somefile").write("hello".toByteArray())
|
||||
|
||||
val dir2 = somedir!! / "tmp1"
|
||||
val data = dir2.get("somefile").readAll()
|
||||
|
||||
assertEquals("hello", data.decodeToString())
|
||||
}
|
||||
}
|
@ -0,0 +1,105 @@
|
||||
package space.kscience.snark.storage.local
|
||||
|
||||
import kotlinx.coroutines.runBlocking
|
||||
|
||||
import space.kscience.snark.storage.*
|
||||
import java.io.File
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.*
|
||||
import kotlin.test.*
|
||||
|
||||
internal class LocalDriverTests {
|
||||
|
||||
private var tempDir: Path? = null
|
||||
private var testSample: Directory? = null
|
||||
private val bytes = byteArrayOf(0, 1, 2, 3, 4, 5, 6, 7)
|
||||
|
||||
@BeforeTest
|
||||
fun setUp() {
|
||||
tempDir = createTempDirectory()
|
||||
testSample = localStorage(tempDir!!)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testCreate() = runBlocking {
|
||||
//folder is empty
|
||||
assertEquals(0, tempDir!!.listDirectoryEntries().size)
|
||||
|
||||
//create first file
|
||||
testSample!!.create("tmp1")
|
||||
val entries = tempDir!!.listDirectoryEntries()
|
||||
assertEquals(1, entries.size)
|
||||
assertEquals(tempDir!! / Path("tmp1"), entries.first())
|
||||
//assertTrue(!entries.first().isDirectory())
|
||||
|
||||
//create second file
|
||||
testSample!!.create("tmp2")
|
||||
assertEquals(2, tempDir!!.listDirectoryEntries().size)
|
||||
|
||||
//check exception after duplication
|
||||
try {
|
||||
testSample!!.create("tmp1")
|
||||
fail("shouldn't ignore duplicates here")
|
||||
} catch (ex: java.nio.file.FileAlreadyExistsException) {}
|
||||
|
||||
//check ignorance
|
||||
try {
|
||||
testSample!!.create("tmp1", true)
|
||||
} catch (ex: java.nio.file.FileAlreadyExistsException) {
|
||||
fail("should ignore duplicates here")
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testPutGet() = runBlocking {
|
||||
testSample!!.create("tmp")
|
||||
testSample!!.put("tmp").write(bytes)
|
||||
assertContentEquals(bytes, testSample!!.get("tmp").readAll())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testCreateSubdir() = runBlocking {
|
||||
//folder is empty
|
||||
assertEquals(0, tempDir!!.listDirectoryEntries().size)
|
||||
|
||||
//create first file
|
||||
testSample!!.createSubdir("tmp1")
|
||||
val entries = tempDir!!.listDirectoryEntries()
|
||||
assertEquals(1, entries.size)
|
||||
assertEquals(tempDir!! / Path("tmp1"), entries.first())
|
||||
assertTrue (entries.first().isDirectory())
|
||||
|
||||
//create second file
|
||||
testSample!!.createSubdir("tmp2")
|
||||
assertEquals(2, tempDir!!.listDirectoryEntries().size)
|
||||
|
||||
//check exception after duplication
|
||||
try {
|
||||
testSample!!.createSubdir("tmp1")
|
||||
fail("shouldn't ignore duplicates here")
|
||||
} catch (ex: java.nio.file.FileAlreadyExistsException) {}
|
||||
|
||||
//check ignorance
|
||||
try {
|
||||
testSample!!.createSubdir("tmp1", true)
|
||||
} catch (ex: java.nio.file.FileAlreadyExistsException) {
|
||||
fail("should ignore duplicates here")
|
||||
}
|
||||
assertTrue {true}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testGetSubdir() = runBlocking {
|
||||
testSample!!.createSubdir("tmp")
|
||||
val pathStr = (Path("tmp") / "data.txt").toString()
|
||||
testSample!!.create(pathStr)
|
||||
testSample!!.put(pathStr).write(bytes)
|
||||
val subdir = testSample!!.getSubdir(Path("tmp"))
|
||||
assertContentEquals(bytes, subdir.get("data.txt").readAll())
|
||||
}
|
||||
|
||||
@AfterTest
|
||||
fun tearDown() {
|
||||
tempDir!!.toFile().deleteRecursively()
|
||||
}
|
||||
}
|
@ -0,0 +1,108 @@
|
||||
package space.kscience.snark.storage.unzip
|
||||
|
||||
import kotlinx.coroutines.runBlocking
|
||||
|
||||
import space.kscience.snark.storage.*
|
||||
import space.kscience.snark.storage.local.localStorage
|
||||
import java.io.*
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.zip.ZipEntry
|
||||
import java.util.zip.ZipOutputStream
|
||||
import kotlin.io.path.*
|
||||
import kotlin.test.*
|
||||
|
||||
internal class UnzipTests {
|
||||
|
||||
private var tempDir: Path? = null
|
||||
|
||||
@BeforeTest
|
||||
fun setUp() {
|
||||
tempDir = createTempDirectory()
|
||||
}
|
||||
|
||||
private suspend fun makeFile(dir: Directory, filename: String, content: ByteArray) {
|
||||
dir.create(filename)
|
||||
dir.put(filename).write(content)
|
||||
}
|
||||
|
||||
private fun zipAll(directory: String, zipFile: String) {
|
||||
val sourceFile = File(directory)
|
||||
|
||||
ZipOutputStream(BufferedOutputStream( FileOutputStream(zipFile))).use {
|
||||
zipFiles(it, sourceFile, File.separator)
|
||||
it.closeEntry()
|
||||
it.close()
|
||||
}
|
||||
}
|
||||
|
||||
private fun zipFiles(zipOut: ZipOutputStream, sourceFile: File, parentDirname: String) {
|
||||
|
||||
val data = ByteArray(2048)
|
||||
|
||||
for (f in sourceFile.listFiles()) {
|
||||
if (f.isDirectory) {
|
||||
zipFiles(zipOut, f, parentDirname + f.name + File.separator)
|
||||
} else {
|
||||
FileInputStream(f).use { fi ->
|
||||
BufferedInputStream(fi).use { origin ->
|
||||
var path = parentDirname + f.name
|
||||
val entry = ZipEntry(path.drop(1))
|
||||
entry.time = f.lastModified()
|
||||
entry.size = f.length()
|
||||
zipOut.putNextEntry(entry)
|
||||
while (true) {
|
||||
val readBytes = origin.read(data)
|
||||
if (readBytes == -1) {
|
||||
break
|
||||
}
|
||||
zipOut.write(data, 0, readBytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
fun testUnzip() = runBlocking {
|
||||
val dir: Directory = localStorage(tempDir!!)
|
||||
val source = dir.createSubdir("source")
|
||||
val target = dir.createSubdir("target")
|
||||
val bytes1 = byteArrayOf(0, 1, 2, 3)
|
||||
val bytes2 = byteArrayOf(1, 0, 3, 2)
|
||||
val bytes3 = byteArrayOf(3, 2, 1, 0)
|
||||
makeFile(source, "tmp1", bytes1)
|
||||
makeFile(source, "tmp2", bytes2);
|
||||
makeFile(source, (Path("tdir") / "tmp3").toString(), bytes3)
|
||||
|
||||
dir.create("archive.zip")
|
||||
val archive_path = (tempDir!! / Path("archive.zip")).toString()
|
||||
|
||||
zipAll((tempDir!! / Path("source")).toString(), archive_path)
|
||||
|
||||
unzip(archive_path, target)
|
||||
|
||||
val targetPath = tempDir!! / Path("target")
|
||||
println(targetPath)
|
||||
val entries = targetPath.listDirectoryEntries()
|
||||
|
||||
assertEquals(3, entries.size)
|
||||
val exp_entries = listOf(
|
||||
targetPath / Path("tmp1"),
|
||||
targetPath / Path("tmp2"),
|
||||
targetPath / Path("tdir"))
|
||||
assertContentEquals(entries.sorted(), exp_entries.sorted())
|
||||
|
||||
val tdirEntries = (targetPath / Path("tdir")).listDirectoryEntries()
|
||||
assertEquals(1, tdirEntries.size)
|
||||
assertEquals(tdirEntries.first(), targetPath / Path("tdir") / Path("tmp3"))
|
||||
}
|
||||
|
||||
@AfterTest
|
||||
fun tearDown() {
|
||||
tempDir!!.toFile().deleteRecursively()
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user