Compare commits

...

133 Commits

Author SHA1 Message Date
Neledova Elizaveta
5659bd2f93 Add pandoc-plugin and conversion to latex 2023-06-11 21:55:45 +03:00
Kirill Grachev
7cccb96023 SNRK-100: Typo in "requirements" 2023-05-22 22:52:32 +03:00
Kirill Grachev
0bc0f63131 SNRK-100: Fix local problems with docker, Update scripts 2023-05-22 21:57:46 +03:00
Kirill Grachev
156ef5ce0f SNRK-100: Add necessary option to script 2023-05-22 20:18:58 +03:00
Kirill Grachev
9f6f713aee SNRK-95: Add secrets 2023-05-19 19:27:51 +03:00
Kirill Grachev
46658ef690 SNRK-100: Add docker-compose 2023-05-19 19:13:44 +03:00
Kirill Grachev
cc12584a8d SNRK-100: Add warning 2023-05-19 18:59:55 +03:00
Kirill Grachev
bf4310375a SNRK-100: Update dockerignore 2023-05-19 18:46:54 +03:00
Kirill Grachev
afccae64ab Set different workdir 2023-05-19 18:25:41 +03:00
Kirill Grachev
652b865b89 SNRK-100: Dockerfile as application 2023-05-19 16:31:43 +03:00
Leonid Pereverzin
7c5e2b7dbb SNRK-89: first buld document 2023-05-18 15:56:17 +03:00
Kirill Grachev
9031638910 Pass directory and root separately 2023-05-18 15:43:43 +03:00
Leonid Pereverzin
0df78294a1 SNRK-89: better regular expr 2023-05-18 15:15:26 +03:00
Leonid Pereverzin
1276275853 SNRK-89: better gitignore file and better path for build document 2023-05-18 14:42:59 +03:00
Leonid Pereverzin
98e713c909
Merge SNARK-MR-25: feature/SNRK-98/fix_snark_main 2023-05-18 11:19:26 +00:00
Kirill Grachev
f8bcdcaeeb
Merge SNARK-MR-24: Storage fabric method 2023-05-18 10:51:53 +00:00
Anton Belyi
4ee51ebc0c SNRK-98: fixed snark-main 2023-05-17 14:58:50 +03:00
Anton Belyi
45132e4355 SNRK-87: moved LocalDataHolder and changed drop to dropWhile 2023-05-17 14:17:28 +03:00
Anton Belyi
e609de95cb SNRK-98: merged main into SNRK-87 2023-05-17 14:10:44 +03:00
Anton Belyi
f28c5c6263 SNRK-98: minor fixes 2023-05-17 14:06:34 +03:00
liubar.pa
44bac3b5ea SNRK-99: bug should be fixed 2023-05-16 00:22:41 +03:00
Petr Lubar
ac8478678e
Merge SNARK-MR-22: Docker 2023-05-15 16:02:53 +00:00
Anton Belyi
62e8d859d8 SNRK-87: added redirect and resolved some conflicts 2023-05-15 18:51:46 +03:00
Kirill Grachev
b463c492f1 SNRK-76: Ignore credentials and configs 2023-05-15 18:36:14 +03:00
Kirill Grachev
81c7bbb073 SNRK-76: Format files 2023-05-15 18:09:15 +03:00
Kirill Grachev
028cf631f4 SNRK-76: Add regions 2023-05-15 18:07:12 +03:00
Kirill Grachev
f06182ee1d Add acceptance tests 2023-05-15 18:00:27 +03:00
Kirill Grachev
80deddc412 SNRK-76: Add configs and method 2023-05-15 17:59:53 +03:00
Anton Belyi
32f8a0d540 removed replace %2F 2023-05-15 17:10:33 +03:00
Anton Belyi
82cd925e35 SNRK-87: simple text-insertion of path 2023-05-15 16:47:35 +03:00
Leonid Pereverzin
167037e276 better 2023-05-14 20:33:26 +03:00
Leonid Pereverzin
5c88ff5a00 SNRK-89: ??? 2023-05-14 20:31:39 +03:00
liubar.pa
bd8dcdca4d SNRK-78: .gitignore is added to .dockerignore 2023-05-14 19:41:27 +03:00
Kirill Grachev
b0564e6539 Merge branch 'main' into release/SNRK-86/assembly 2023-05-14 18:23:39 +03:00
Kirill Grachev
64bfc75222
Merge SNARK-MR-18: SNRK-68: Put creates file 2023-05-14 15:20:15 +00:00
liubar.pa
2755e2020d SNRK-78: call of main is commented because test cannot end 2023-05-13 22:54:31 +03:00
liubar.pa
8795b49d38 SNRK-78: .dockerignore is added 2023-05-13 22:53:56 +03:00
liubar.pa
f30b35e66a SNRK-78: docker file is added 2023-05-13 21:48:25 +03:00
liubar.pa
b0e3b5551a SNRK-78: better 2023-05-13 21:45:44 +03:00
liubar.pa
7fb778e483 SNRK-78: requirments installation script is added 2023-05-13 21:20:53 +03:00
Anton Belyi
8ffda3f42a SNRK-90: return html as string 2023-05-12 18:06:57 +03:00
Leonid Pereverzin
fbd5237573 better naming 2023-05-07 18:06:12 +03:00
Leonid Pereverzin
ef54c02c41 delete debug print 2023-05-07 17:59:46 +03:00
Leonid Pereverzin
046bbda23a SNRK-84: better 2023-05-07 17:57:41 +03:00
Leonid Pereverzin
ffaa932127 CE fix 2023-05-07 16:06:26 +03:00
Kirill Grachev
8225501488 Format files 2023-05-06 21:32:17 +03:00
Kirill Grachev
793218b6bc SNRK-68: Update examples 2023-05-06 21:26:34 +03:00
Kirill Grachev
c6ceba2ed5 SNRK-68: Use extend functions to mark methods as final 2023-05-06 21:25:38 +03:00
Kirill Grachev
29d842b0bf SNRK-68: Implement new semantics 2023-05-06 20:56:54 +03:00
liubar.pa
4f9fba5237 Merge branch 'release/SNRK-86/requirments' into release/SNRK-86/assembly 2023-05-06 19:55:14 +03:00
Kirill Grachev
9a41f67e29 Add comments 2023-05-06 19:50:24 +03:00
liubar.pa
8ed0cc0499 SNRK-85: .gitignore to ignore node_modules folder is added 2023-05-06 19:49:30 +03:00
liubar.pa
3358f8c856 SNRK-85: better 2023-05-06 19:49:30 +03:00
liubar.pa
1dfb5d8772 SNRK-85: script to install dependencies is added 2023-05-06 19:49:30 +03:00
Kirill Grachev
a13c4a8e8c SNRK-82: Add code from another repository 2023-05-06 19:46:38 +03:00
Kirill Grachev
14c1e650b9 SNRK-86: Runs without fails 2023-05-06 19:25:38 +03:00
Kirill Grachev
a99e71d1b1 SNRK-86: Works 2023-05-06 19:14:44 +03:00
Kirill Grachev
667688d46d SNRK-86: Assembly parts 2023-05-06 18:57:55 +03:00
Kirill Grachev
363f63b8da Merge branch 'feature/SNRK-71/document-builder-implementation' into release/SNRK-86/assembly 2023-05-06 18:14:36 +03:00
liubar.pa
33b4c56e4f SNRK-71: subfolder for example is added 2023-05-06 18:05:36 +03:00
liubar.pa
19a023190f SNRK-71: better test document 2023-05-06 18:00:12 +03:00
liubar.pa
234dd279f0 SNARK-71: better folder naming 2023-05-06 17:59:43 +03:00
liubar.pa
778e58fc63 SNRK-71: getIncludeFiles is implemented 2023-05-06 01:33:26 +03:00
liubar.pa
4019cbfe6b SnarkParser.py is added 2023-05-06 00:17:33 +03:00
liubar.pa
225b7346a8 fillDependencies is implemented 2023-05-05 23:39:51 +03:00
liubar.pa
cd85fcf329 Merge branch 'feature/SNRK-70/dependecy-graph' into feature/SNRK-71/document-builder-implementation 2023-05-05 21:27:41 +03:00
liubar.pa
fdcf9c9c5b SNRK-70: IncludeDependency is changed 2023-05-05 21:25:43 +03:00
liubar.pa
ff84d2e87b SNRK-71: buildDependencyNode is implemented 2023-05-05 21:21:46 +03:00
Anton Belyi
f477af64e6
Merge SNARK-MR-16: SNRK-77: unit_testing 2023-05-05 13:57:48 +00:00
Anton Belyi
ea05f93cea SNRK-77: update gitignore 2023-05-05 16:55:41 +03:00
Anton Belyi
7d681549b7
Merge SNARK-MR-12: SNRK-69: added web interface 2023-05-05 13:47:53 +00:00
Anton Belyi
8e304377c6 fixed unzip tests and slightly modified LocalDriver 2023-05-05 16:41:32 +03:00
Anton Belyi
e1c8be66db Merge branch 'main' into devops/SNRK-77/unit_testing 2023-05-05 15:38:09 +03:00
Anton Belyi
ff7b02d98a SNRK-83: changed decomposition and moved main to test/ 2023-05-05 15:24:01 +03:00
liubar.pa
eda376d51d Merge remote-tracking branch 'origin/main' into feature/SNRK-71/document-builder-implementation 2023-05-05 00:21:45 +03:00
liubar.pa
c112a74652 SNRK-71: update after adding of path property 2023-05-05 00:21:20 +03:00
Kirill Grachev
203aa52efe SNRK-68: Implement missed 2023-05-05 00:17:56 +03:00
liubar.pa
637a6810fc Merge remote-tracking branch 'origin/main' into feature/SNRK-71/document-builder-implementation 2023-05-05 00:09:20 +03:00
liubar.pa
e0ef373c48 SNRK-71: parseMd is implemented 2023-05-05 00:07:59 +03:00
liubar.pa
248677dad9 SNRK-71: nodejs scripts for markdown parsing are added 2023-05-04 23:54:57 +03:00
Kirill Grachev
4441172ef7 SNRK-68: Add warning 2023-05-04 21:14:59 +03:00
Kirill Grachev
b6db0ba479 SNRK-68: Implement local storage 2023-05-04 21:12:23 +03:00
Kirill Grachev
634eb2d69d SNRK-68: Implement s3 storage 2023-05-04 21:11:56 +03:00
Kirill Grachev
5b33419b37 SNRK-68: Add methods 2023-05-04 21:11:29 +03:00
liubar.pa
e90c9728c1 SNRK-71: pathString replaced with toString() 2023-05-04 18:57:25 +03:00
liubar.pa
859c53745f SNARK-71: redundant else is removed 2023-05-04 18:54:21 +03:00
liubar.pa
06bcdbd489 SNARK-71: better path using 2023-05-04 18:53:55 +03:00
liubar.pa
a411849d6f SNARK-71: getDependencies are implemented 2023-05-04 09:47:00 +03:00
liubar.pa
54828a6535 SNARK-71: better implementation 2023-05-04 08:09:06 +03:00
liubar.pa
c76ecd592f SNARK-71: buildDependencyGraph is implemented 2023-05-04 07:55:48 +03:00
liubar.pa
088697da78 Merge branch 'feature/SNRK-70/dependecy-graph' into feature/SNRK-71/document-builder-implementation 2023-05-04 02:46:51 +03:00
liubar.pa
dee40ecdcc SNRK-70: visibility of DEFAULT_DICUMENT_ROOT is specified 2023-05-04 02:33:58 +03:00
liubar.pa
5458bd3677 SNRK-70: better arguments in buildDocument 2023-05-04 02:30:03 +03:00
liubar.pa
6c9a08a14a SNRK-70: better test naming 2023-05-04 02:18:37 +03:00
liubar.pa
cfe0f0740f Merge branch 'main' into feature/SNRK-70/dependecy-graph 2023-05-04 02:14:51 +03:00
Kirill Grachev
7599fd5ffd
Merge SNARK-MR-6: SNRK-48: Implement-s3-driver 2023-05-01 18:37:38 +00:00
liubar.pa
1db2afce12 SNRK-71: buildDependencyGraph scratch is added 2023-05-01 21:23:09 +03:00
Anton Belyi
b4423e8bba SNRK-69: added web interface 2023-05-01 19:19:17 +03:00
liubar.pa
fd45104a7e SNRK-70: visibility specificators are added 2023-05-01 14:48:34 +03:00
liubar.pa
417d90b842 SNRK-70: documentation is added 2023-04-30 23:17:08 +03:00
liubar.pa
3c2ffb572b SNRK-70: alias is added for names of files 2023-04-30 22:46:31 +03:00
liubar.pa
b7a439f711 SNRK-70: obsollete code is removed 2023-04-30 22:43:21 +03:00
liubar.pa
7a57de989b SNRK-70: better package placement 2023-04-30 22:40:53 +03:00
liubar.pa
efc3bc2537 SNRK-70: typo is removed 2023-04-30 22:34:17 +03:00
liubar.pa
ea4fef95b8 SNRK-70: MdAstElements and DependencyGraph moved to seperate files, return type of parseMd is changed to DepencyGraphVertex 2023-04-30 22:33:40 +03:00
Anton Belyi
82f699070b SNRK-77: fixed tests 2023-04-28 23:20:00 +03:00
Anton Belyi
3c07ee7dd5 SNRK-77: added tests and slightly changed some code 2023-04-28 23:19:22 +03:00
Kirill Grachev
a672db082f SNRK-57: Remove File from sources 2023-04-28 17:37:39 +03:00
Kirill Grachev
f71e5a3f1a SNRK-57: Format file 2023-04-28 17:07:54 +03:00
Kirill Grachev
52772cde7d SNRK-57: Remove run 2023-04-28 16:59:11 +03:00
Kirill Grachev
256fb16748 SNRK-57: Pretty exceptions 2023-04-24 21:45:42 +03:00
Kirill Grachev
d292abc816 SNRK-57: Fix internals 2023-04-24 21:12:21 +03:00
Kirill Grachev
e9eaa0f8c2 SNRK-67: Use kotlin.io.Path 2023-04-24 16:53:48 +03:00
Kirill Grachev
7c6157be96 SNRK-67: Use system-specific path separator 2023-04-24 16:35:50 +03:00
Kirill Grachev
b543735628 SNRK-67: Make implementation private 2023-04-24 16:34:47 +03:00
Kirill Grachev
8069ea2f0b SNRK-67: Add top-level function 2023-04-24 16:34:47 +03:00
Kirill Grachev
88510dbba8 SNRK-67: Fix pathes 2023-04-24 16:34:20 +03:00
Anton Belyi
9651ad1cd9 SNRK-60: slightly changed local driver and added unzip function 2023-04-24 15:32:29 +03:00
liubar.pa
57df19f1bf SNRK-66: better test 2023-04-24 07:27:44 +03:00
liubar.pa
0ab26812f8 SNRK-66: example is added 2023-04-24 07:27:32 +03:00
liubar.pa
49c350abad SNRK-66: parseMd is added 2023-04-24 07:27:07 +03:00
liubar.pa
0ef269ec5c SNRK-66: documentDependencies and dependencyGraph are added 2023-04-24 07:24:18 +03:00
liubar.pa
4465e68408 SNRK-66:buildDocument function scratch is added, AstNodes are copied from prototype 2023-04-24 07:01:31 +03:00
Kirill Grachev
a92f9673ac SNRK-58: Add gradle files to new subproject 2023-04-22 17:10:23 +03:00
Anton Belyi
0dad4cd923 SNRK-49: Made fields private 2023-04-17 18:14:47 +03:00
Kirill Grachev
a97a99d138 SNRK-48: Add s3 implementation
Use common s3 api and '/'-separated keys to simulate directories
2023-04-17 17:28:34 +03:00
Anton Belyi
7b946c004f SNRK-49: Added first version of local driver 2023-04-17 16:30:47 +03:00
Kirill Grachev
a9c0cc10a6 SNRK-48: Remove nullable types 2023-04-15 00:51:45 +03:00
Kirill Grachev
7127d3582d SNRK-48: Add method 2023-04-15 00:14:38 +03:00
Kirill Grachev
6e9eb88934 Add interface 2023-04-15 00:01:27 +03:00
Kirill Grachev
f8e7f6567f SNRK-48: Change dependency 2023-04-15 00:00:40 +03:00
Kirill Grachev
65fafe1400 SNRK-38: Add dependencies 2023-04-08 19:04:25 +03:00
Kirill Grachev
07b2ecb35c Chmod +x gradlew 2023-04-08 19:03:54 +03:00
55 changed files with 7709 additions and 1 deletions

24
.dockerignore Normal file
View File

@ -0,0 +1,24 @@
# .gitignore contents
.gradle/
build/
.idea/
/logs/
!gradle/wrapper/gradle-wrapper.jar
kotlin-js-store
*.iml
# addition to .gitignore
README.md
LICENSE
gradlew.bat
.dockerignore
.gitignore
Dockerfile
tmp
rundata
credentials.json

5
.gitignore vendored
View File

@ -2,5 +2,10 @@
build/
.idea/
/logs/
rundata/
!gradle/wrapper/gradle-wrapper.jar
kotlin-js-store
*.iml
*.json

25
Dockerfile Normal file
View File

@ -0,0 +1,25 @@
FROM ubuntu:latest
RUN apt-get update
RUN apt-get install -y curl zip unzip
ARG JAVA_VERSION="17.0.7-zulu"
WORKDIR /app
COPY ./snark-main/ci ./snark-main/ci
RUN ./snark-main/ci/install_sdk.sh
RUN ./snark-main/ci/install_java.sh "$JAVA_VERSION"
COPY . .
RUN ./requirements.sh
RUN bash -c "source ~/.sdkman/bin/sdkman-init.sh && ./gradlew clean build"
EXPOSE 8080
RUN mkdir -p ~/.aws/ && ln -s /run/secrets/credentials.json ~/.aws/credentials.json
CMD bash -c "source ~/.sdkman/bin/sdkman-init.sh && ./gradlew :snark-main:run_server"

19
docker-compose.yml Normal file
View File

@ -0,0 +1,19 @@
version: '3'
services:
snark:
build: .
ports:
- "8081:8080"
expose:
- 8081
volumes:
- storage:/app/rundata
secrets:
- credentials.json
volumes:
storage:
secrets:
credentials.json:
file: ./credentials.json

0
gradlew vendored Normal file → Executable file
View File

29
requirements.sh Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash
set -e
apt-get update
apt-get install -y sudo
for dir in ./*/
do
if [[ $dir == *'snark'* ]]
then
cd "$dir"
if [[ $(find -type d -name "ci") ]]
then
cd ci
if [[ -f "requirements.sh" ]]; then
echo "executing sub"
bash -c "./requirements.sh"
fi
cd ..
fi
cd ..
fi
done

View File

@ -41,5 +41,9 @@ include(
":snark-gradle-plugin",
":snark-core",
":snark-html",
":snark-ktor"
":snark-ktor",
":snark-storage-driver",
":snark-document-builder",
":snark-main",
":snark-pandoc-plugin",
)

View File

@ -0,0 +1,21 @@
plugins {
id("space.kscience.gradle.jvm")
`maven-publish`
id("kotlinx-serialization")
}
val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
val jacksonVersion = "2.14.2"
val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
dependencies {
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
api("io.ktor:ktor-server-html-builder:$ktorVersion")
implementation(project(":snark-storage-driver"))
implementation(project(":snark-pandoc-plugin"))
implementation("com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion")
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.3")
}

View File

@ -0,0 +1,11 @@
#!/bin/bash
set -e
sudo apt-get install -y python3
sudo apt-get install -y nodejs
sudo apt-get install -y npm
pushd ../src/main/nodejs
npm install .
popd

View File

@ -0,0 +1,17 @@
package documentBuilder
public class GraphManager(public val graph: DependencyGraph) {
fun buildDocument(file: FileName) {
val list = graph.nodes[file]
if (list != null) {
for (element in list.dependencies) {
element.visit(this)
}
}
}
fun getAstRootDocument(file: FileName): MdAstRoot {
buildDocument(file)
return graph.nodes[file]!!.mdAst
}
}

View File

@ -0,0 +1,71 @@
package documentBuilder
import kotlinx.coroutines.coroutineScope
import kotlin.collections.MutableList
public typealias FileName = String
/**
* Node of dependency graph.
*
* One node represents one file and its dependencies
*
* @property mdAst - AST tree of current file.
* @property dependencies - list of tail end adjacent to this node (dependencies of current file to be resolved).
*/
public data class DependencyGraphNode(
val mdAst: MdAstRoot,
val dependencies: List<DependencyGraphEdge>
)
/**
* Interface of all dependency edges.
*/
public sealed interface DependencyGraphEdge {
public fun visit(graphManager: GraphManager)
}
/**
* Include dependency edge.
*
* @property parentNode - node inside AST tree, that is parent for dependent node.
* @property dependentNode - dependent node, i.e. node of part of document with include commands
* @property includeList - list of files to be included.
*/
public data class IncludeDependency(
val parentNode: MdAstParent,
val dependentNode: MdAstElement,
val includeList: List<FileName>
) : DependencyGraphEdge {
override fun visit(graphManager: GraphManager) {
val parent = parentNode
val childs: MutableList<MdAstElement> = mutableListOf()
for (file in includeList) {
graphManager.buildDocument(file)
childs.addAll(graphManager.graph.nodes[file]!!.mdAst.children)
}
val elements: MutableList<MdAstElement> = parent.children.toMutableList()
val index = parent.children.indexOf(dependentNode)
elements.removeAt(index)
elements.addAll(index, childs)
parent.children = elements
}
}
// parent - List<MdAstElement> -------------------------------------
// | \
// \ \
// | \
// dependentNode - MdAstElement \
// |
// List<FileName> -> List<MdAstRoot> --> List<List<MdAstElement>> ===> List<MdAstElement>
/**
* Whole dependency graph.
*
* @property nodes - map of nodes, where you can find DependencyGraphNode of file by its name.
*/
public data class DependencyGraph(
val nodes: Map<FileName, DependencyGraphNode>
)

View File

@ -0,0 +1,111 @@
package documentBuilder
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import space.kscience.snark.pandoc.PandocCommandBuilder
import space.kscience.snark.pandoc.PandocWrapper
import space.kscience.snark.storage.*
import java.nio.file.Files
import java.nio.file.Path
import kotlin.io.path.*
private val SNARK_HTML_RENDER = "snark-document-builder/src/main/nodejs/HtmlRenderer.js"
private val SNARK_MD_RENDERER = "snark-document-builder/src/main/nodejs/MdRenderer.js"
fun getHtml(ast_string: String): String
{
return ProcessBuilder("node", SNARK_HTML_RENDER, ast_string)
.redirectOutput(ProcessBuilder.Redirect.PIPE)
.redirectError(ProcessBuilder.Redirect.INHERIT)
.start().inputStream.bufferedReader().readText()
}
fun getLatex(ast_string: String) : Path
{
val outputMd = Files.createTempFile(Path.of("./data/"), "output", ".md")
val output = ProcessBuilder("node", SNARK_MD_RENDERER, ast_string)
.redirectOutput(ProcessBuilder.Redirect.PIPE)
.redirectError(ProcessBuilder.Redirect.INHERIT)
.start().inputStream.bufferedReader().readText()
outputMd.writeText(output)
val outputTex = Files.createTempFile(Path.of("./data/"), "output", ".tex")
val pandocWrapper = PandocWrapper()
pandocWrapper.use { p: PandocWrapper? ->
val command = PandocCommandBuilder(
listOf<Path>(outputMd),
outputTex
)
PandocWrapper.execute(command)
}
return outputTex
}
private val DEFAULT_DOCUMENT_ROOT = "main.md"
public suspend fun buildDocument(root: Directory, path: Path): String {
val dependencyGraph = buildDependencyGraph(root, path)
val graphManage = GraphManager(dependencyGraph)
graphManage.buildDocument(path.toString())
// for ((key, value) in dependencyGraph.nodes) {
// println("Key ${key}")
// println("Value.mdAst ${value.mdAst}")
// println("Value.dependencies ${value.dependencies}")
// }
val root: MdAstRoot = dependencyGraph.nodes[path.toString()]!!.mdAst
return getHtml(jacksonObjectMapper().writeValueAsString(root))
}
public suspend fun buildLatex(root: Directory, path: Path) : Path {
val dependencyGraph = buildDependencyGraph(root, path)
val graphManage = GraphManager(dependencyGraph)
graphManage.buildDocument(path.toString())
val root: MdAstRoot = dependencyGraph.nodes[path.toString()]!!.mdAst
return getLatex(jacksonObjectMapper().writeValueAsString(root))
}
public suspend fun buildDependencyGraph(root: Directory, path: Path): DependencyGraph {
val nodes = HashMap<FileName, DependencyGraphNode>()
buildNodes(root, path, nodes)
return DependencyGraph(nodes)
}
private suspend fun buildNodes(root: Directory, path: Path, nodes: HashMap<FileName, DependencyGraphNode>) {
val pathString = path.toString()
assert(!nodes.containsKey(pathString))
val rootDcoument = (root / path).get(DEFAULT_DOCUMENT_ROOT)
nodes.put(pathString, buildDependencyGraphNode(rootDcoument.readAll(), path))
val dependencies = getDependencies(nodes.getValue(pathString))
for (dependency in dependencies) {
if (!nodes.containsKey(dependency))
buildNodes(root, Path(dependency), nodes)
}
}
public suspend fun getDependencies(node: DependencyGraphNode): Set<FileName> {
val dependencies = mutableListOf<FileName>()
for (dependency in node.dependencies) {
when (dependency) {
is IncludeDependency -> dependencies.addAll(dependency.includeList)
}
}
return dependencies.toSet()
}

View File

@ -0,0 +1,84 @@
package documentBuilder
import kotlinx.serialization.Serializable
import kotlinx.serialization.SerialName
import com.fasterxml.jackson.annotation.JsonSubTypes
import com.fasterxml.jackson.annotation.JsonTypeInfo
@Serializable
public data class Point(val line: Int, val column: Int, val offset: Int)
@Serializable
public data class Position(val start: Point, val end: Point)
@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include = JsonTypeInfo.As.PROPERTY,
property = "type"
)
@JsonSubTypes(
JsonSubTypes.Type(value = MdAstRoot::class, name = "root"),
JsonSubTypes.Type(value = MdAstParagraph::class, name = "paragraph"),
JsonSubTypes.Type(value = MdAstText::class, name = "text"),
JsonSubTypes.Type(value = MdAstHeading::class, name = "heading"),
JsonSubTypes.Type(value = MdAstCode::class, name = "code"),
JsonSubTypes.Type(value = MdAstBlockquote::class, name = "blockquote")
)
@Serializable
public sealed interface MdAstElement{
public abstract var position: Position
}
@Serializable
public sealed interface MdAstParent: MdAstElement{
public var children: List<MdAstElement>
}
@Serializable
@SerialName("root")
public data class MdAstRoot(
override var children: List<MdAstElement>,
override var position: Position
): MdAstParent
@Serializable
@SerialName("paragraph")
public data class MdAstParagraph(
override var children: List<MdAstElement>,
override var position: Position
): MdAstParent
@Serializable
@SerialName("text")
public data class MdAstText(
val value: String,
override var position: Position
): MdAstElement
@Serializable
@SerialName("heading")
public data class MdAstHeading(
val depth: Int,
override var children: List<MdAstElement>,
override var position: Position
): MdAstParent
@Serializable
@SerialName("code")
public data class MdAstCode(
var lang: String? = null,
var meta: String? = null,
var value: String,
override var position: Position,
) : MdAstElement
@Serializable
@SerialName("blockquote")
public data class MdAstBlockquote(
override var children: List<MdAstElement>,
override var position: Position
): MdAstParent

View File

@ -0,0 +1,60 @@
package documentBuilder
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import java.nio.file.Path
// snark-main/build.gradle.kts depends on these pathes
private val MARKDOWN_PARSER = "snark-document-builder/src/main/nodejs/MarkdownParser.js"
private val SNARK_PARSER = "snark-document-builder/src/main/python/SnarkParser.py"
public suspend fun parseMd(mdFile: ByteArray, parserPath: String = MARKDOWN_PARSER): MdAstRoot {
val process = ProcessBuilder("node", parserPath, String(mdFile))
val result = process
.redirectOutput(ProcessBuilder.Redirect.PIPE)
.redirectError(ProcessBuilder.Redirect.INHERIT)
.start().inputStream.bufferedReader().readText()
return jacksonObjectMapper().readValue<MdAstRoot>(result)
}
public suspend fun buildDependencyGraphNode(mdFile: ByteArray, path: Path): DependencyGraphNode {
val treeRoot = parseMd(mdFile)
val dependencies = mutableListOf<DependencyGraphEdge>()
fillDependencies(treeRoot, dependencies, path)
return DependencyGraphNode(treeRoot, dependencies)
}
internal suspend fun fillDependencies(
currentNode: MdAstElement,
dependencies: MutableList<DependencyGraphEdge>,
path: Path) {
when (currentNode) {
is MdAstParent -> {
for (child in currentNode.children) {
if (child is MdAstText) {
val includeList = getIncludeFiles(child.value).toMutableList()
if (includeList.size > 0) {
includeList.replaceAll { path.toString() + "/" + it }
dependencies += IncludeDependency(currentNode, child, includeList)
}
} else {
fillDependencies(child, dependencies, path)
}
}
}
else -> {}
}
}
public suspend fun getIncludeFiles(string: String): List<FileName> {
return jacksonObjectMapper()
.readValue<List<FileName>>(ProcessBuilder("python3", SNARK_PARSER, string)
.redirectOutput(ProcessBuilder.Redirect.PIPE)
.redirectError(ProcessBuilder.Redirect.INHERIT)
.start().inputStream.bufferedReader().readText())
}

View File

@ -0,0 +1 @@
node_modules

View File

@ -0,0 +1,16 @@
import {toHast} from 'mdast-util-to-hast'
import {toHtml} from 'hast-util-to-html'
main()
function main()
{
if (process.argv.length < 3)
throw "No input"
const md_ast = JSON.parse(process.argv[2])
const hast = toHast(md_ast)
const html = toHtml(hast)
console.log(html)
}

View File

@ -0,0 +1,14 @@
import {fromMarkdown} from 'mdast-util-from-markdown'
main()
function main()
{
if (process.argv.length < 3)
throw "No input"
const markdown_string = process.argv[2]
const mdast = fromMarkdown(markdown_string)
console.log(JSON.stringify(mdast))
}

View File

@ -0,0 +1,14 @@
import {toMarkdown} from 'mdast-util-to-markdown'
main()
function main()
{
if (process.argv.length < 3)
throw "No input"
const md_ast = JSON.parse(process.argv[2])
const markdown = toMarkdown(md_ast)
console.log(markdown)
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,30 @@
{
"type": "module",
"dependencies": {
"fs": "^0.0.1-security",
"hast-util-to-html": "^8.0.4",
"mdast-util-to-hast": "^12.3.0",
"node-fetch": "^3.3.1",
"remark-html": "^15.0.2",
"remark-parse": "^10.0.1",
"require": "^2.4.20",
"to-vfile": "^7.2.4",
"unified": "^10.1.2",
"whatwg-fetch": "^3.6.2"
}
}

View File

@ -0,0 +1,26 @@
import sys
import re
import json
assert(len(sys.argv) >= 2)
string = sys.argv[1]
outputfile = sys.argv[2] if len(sys.argv) >= 3 else None
pattern = r'^([\n|\t| ]*@include\([a-z|0-9|.|_|\/]*\)[\n|\t| ]*)*$'
files = []
if re.search("@include", string, re.IGNORECASE):
if re.match(pattern, string):
matches = re.findall(r'@include\((.*?)\)', string)
files.extend(matches)
else:
sys.exit("Illformed string")
if outputfile is None:
print(json.dumps(files))
else:
with open(outputfile, 'w+') as f:
json.dump(files, f)

View File

@ -0,0 +1,12 @@
package documentBuilder
import org.junit.jupiter.api.Test
import kotlinx.coroutines.runBlocking
import space.kscience.snark.storage.local.localStorage
class SomeTest {
@Test
fun justWorks() = runBlocking {
// buildDocument(Directory("../example"))
}
}

View File

@ -0,0 +1,3 @@
# Hello
I'm almost empty test document without any dependencies

View File

@ -12,6 +12,10 @@ dependencies {
api("io.ktor:ktor-server-core:$ktorVersion")
api("io.ktor:ktor-server-html-builder:$ktorVersion")
api("io.ktor:ktor-server-host-common:$ktorVersion")
implementation("io.ktor:ktor-server-netty:2.3.0")
implementation(project(":snark-storage-driver"))
implementation("io.ktor:ktor-server-partial-content:$ktorVersion")
implementation("io.ktor:ktor-server-auto-head-response:$ktorVersion")
testApi("io.ktor:ktor-server-tests:$ktorVersion")
}

12
snark-ktor/snark-ktor.iml Normal file
View File

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/kotlin" isTestSource="false" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="KotlinJavaRuntime" level="project" />
</component>
</module>

View File

@ -0,0 +1,131 @@
package space.kscience.snark.ktor
import io.ktor.http.*
import io.ktor.server.application.*
import io.ktor.server.response.*
import io.ktor.server.routing.*
import io.ktor.http.content.*
import io.ktor.server.engine.*
import io.ktor.server.netty.*
import io.ktor.server.html.*
import io.ktor.server.request.*
import kotlinx.html.*
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.unzip.unzip
import java.nio.file.Path
import kotlin.io.createTempFile
import kotlin.io.writeBytes
import kotlin.io.path.Path
public interface DataHolder {
public suspend fun init(relativePath: Path) : Directory
public suspend fun represent(relativePath: Path): String
public suspend fun toPdf(relativePath: Path) : Path
}
public class SNARKServer(private val dataHolder: DataHolder, private val port: Int): Runnable {
private var relativePath = Path("")
private suspend fun receivePath(call: ApplicationCall) {
val pathString = call.receiveParameters()["path"]?:""
relativePath = Path(pathString.dropWhile{it == '/'})
call.respondRedirect("/")
}
private suspend fun renderGet(call: ApplicationCall) {
call.respondText(dataHolder.represent(relativePath), ContentType.Text.Html)
}
private suspend fun renderFile(call: ApplicationCall) {
call.response.header(
HttpHeaders.ContentDisposition,
ContentDisposition.Attachment.withParameter(ContentDisposition.Parameters.FileName, "output.tex")
.toString()
)
call.respondFile(dataHolder.toPdf(relativePath).toFile())
call.respondRedirect("/")
}
private suspend fun renderUpload(call: ApplicationCall) {
val multipartData = call.receiveMultipart()
val tmp = createTempFile(suffix=".zip")
multipartData.forEachPart { part ->
when (part) {
is PartData.FileItem -> {
val fileBytes = part.streamProvider().readBytes()
tmp.writeBytes(fileBytes)
}
else -> {
}
}
part.dispose()
}
unzip(tmp.toPath().toString(), dataHolder.init(relativePath))
call.respondRedirect("/")
}
private suspend fun renderMainPage(call: ApplicationCall) {
call.respondHtml(HttpStatusCode.OK) {
head {
title {
+"SNARK"
}
}
body {
h1 {
+"SNARK"
}
p {
+("Path: /" + relativePath.toString())
}
}
body {
postForm(action = "/changePath") {
label {
+ "Enter new path:"
}
input(name = "path", type = InputType.text) {}
button {
+"Change path"
}
}
postForm (action = "/upload", encType = FormEncType.multipartFormData) {
label {
+"Choose zip archive: "
}
input (name = "file", type = InputType.file) {}
button {
+"Upload file"
}
}
a("/data") {
+"Show data\n"
}
getForm (action = "/download") {
button {
+"Download latex\n"
}
}
}
}
}
override fun run() {
embeddedServer(Netty, port) {
routing {
get("/") {
renderMainPage(call)
}
post("/changePath") {
receivePath(call)
}
post("/upload") {
renderUpload(call)
}
get("/data") {
renderGet(call)
}
get("/download") {
renderFile(call)
}
}
}.start(wait = true)
}
}

View File

@ -0,0 +1,50 @@
package space.kscience.snark.ktor
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.local.localStorage
import java.nio.file.Path
import kotlin.io.path.*
import kotlin.io.path.createTempDirectory
import kotlin.io.path.isDirectory
import kotlin.io.path.listDirectoryEntries
private class LocalDataHolder: DataHolder {
private var source: Path? = null
private var response: String = ""
private fun getPath(relativePath: Path) : Path {
return source!! / relativePath
}
override suspend fun init(relativePath: Path): Directory {
if (source == null) {
source = createTempDirectory()
}
val path = getPath(relativePath)
path.createDirectories()
path.toFile().deleteRecursively()
path.createDirectory()
return localStorage(path)
}
private fun buildResponse(from: Path, cur: Path) {
for (entry in cur.listDirectoryEntries()) {
if (entry.isDirectory()) {
buildResponse(from, entry)
} else {
response += from.relativize(entry).toString() + "<br>"
}
}
}
override suspend fun represent(relativePath: Path) : String =
if (source == null) {
"No data was loaded!"
} else {
response = "List of files:<br>"
val path = getPath(relativePath)
buildResponse(path, path)
response
}
}
fun main() {
SNARKServer(LocalDataHolder(), 9090).run()
}

View File

@ -0,0 +1,27 @@
plugins {
id("space.kscience.gradle.jvm")
`maven-publish`
}
val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
dependencies {
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
implementation(project(":snark-ktor"))
implementation(project(":snark-storage-driver"))
implementation(project(":snark-document-builder"))
testImplementation(kotlin("test"))
testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
}
tasks.test {
useJUnitPlatform()
}
tasks.register<JavaExec>("run_server") {
classpath = sourceSets.main.get().runtimeClasspath
main = "space.kscience.snark.main.MainKt"
workingDir = File(workingDir.parent)
}

6
snark-main/ci/install_java.sh Executable file
View File

@ -0,0 +1,6 @@
#!/bin/bash
JAVA_VERSION="$1"
source ~/.sdkman/bin/sdkman-init.sh
sdk install java "$JAVA_VERSION"

3
snark-main/ci/install_sdk.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/bash
curl -s "https://get.sdkman.io" | bash

View File

@ -0,0 +1,19 @@
package space.kscience.snark.main
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import space.kscience.snark.ktor.SNARKServer
import space.kscience.snark.storage.local.localStorage
import kotlin.io.path.Path
// Entrypoint
fun main(): Unit = runBlocking {
// Parse config, create driver
val port = 8080
val directory = localStorage(Path("./rundata"))
val server = SNARKServer(ServerDataHolder(directory), port)
launch {
server.run()
}
}

View File

@ -0,0 +1,20 @@
package space.kscience.snark.main
import space.kscience.snark.ktor.DataHolder
import space.kscience.snark.storage.Directory
import documentBuilder.*
import java.nio.file.Path
internal class ServerDataHolder(private val directory: Directory): DataHolder {
override suspend fun init(relativePath: Path): Directory = directory
override suspend fun represent(relativePath: Path): String {
return buildDocument(directory, relativePath)
}
override suspend fun toPdf(relativePath: Path) : Path {
return buildLatex(directory, relativePath)
}
}

View File

@ -0,0 +1,13 @@
package space.kscience.snark.main
import kotlinx.coroutines.delay
import kotlinx.coroutines.runBlocking
import org.junit.jupiter.api.Test
class Test {
@Test
fun justWorks() = runBlocking {
delay(5)
//main()
}
}

2
snark-pandoc-plugin/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
.pandoc/

View File

@ -0,0 +1,25 @@
plugins {
id("java")
}
repositories {
mavenCentral()
}
java.sourceCompatibility = JavaVersion.VERSION_11
dependencies {
implementation("commons-io:commons-io:2.7")
implementation("org.slf4j:slf4j-simple:2.0.6")
implementation("org.slf4j:slf4j-api:2.0.6")
implementation("org.apache.commons:commons-exec:1.3")
implementation("org.apache.commons:commons-compress:1.2")
implementation("org.apache.ant:ant:1.10.13")
implementation("com.fasterxml.jackson.core:jackson-databind:2.14.2")
testImplementation("org.junit.jupiter:junit-jupiter-api:5.8.1")
testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.8.1")
}
tasks.getByName<Test>("test") {
useJUnitPlatform()
}

View File

@ -0,0 +1,367 @@
package space.kscience.snark.pandoc;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipFile;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.exec.OS;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Installer {
private static final Logger log
= LoggerFactory.getLogger(Installer.class);
public enum OSType {
WINDOWS("windows-x86_64.zip", "windows"),
MAC_OS_AMD("x86_64-macOS.zip", "mac.os.amd"),
MAC_OS_ARM("arm64-macOS.zip", "mac.os.arm"),
LINUX_ARM("linux-arm64", "linux.arm"),
LINUX_AMD("linux-amd64", "linux.amd");
private final String assetSuffix;
private final String propertySuffix;
OSType(String assetSuf, String propertySuf) {
assetSuffix = assetSuf;
propertySuffix = propertySuf;
}
public String getAssetSuffix() {
return assetSuffix;
}
public String getPropertySuffix() {
return propertySuffix;
}
}
private static class OSData {
private URL urlForInstalling;
private Path fileToInstall;
private Path pathToPandoc;
public OSData() {}
public void setUrlForInstalling(URL urlForInstalling) {
this.urlForInstalling = urlForInstalling;
}
public URL getUrlForInstalling() {
return urlForInstalling;
}
public void setFileToInstall(Path fileToInstall) {
this.fileToInstall = fileToInstall;
}
public Path getFileToInstall() {
return fileToInstall;
}
public Path getPathToPandoc() {
return pathToPandoc;
}
public void setPathToPandoc(Path pathToPandoc) {
this.pathToPandoc = pathToPandoc;
}
}
private static Map<OSType, OSData> dataForInstalling = new HashMap<>(OSType.values().length);
private static final Properties properties = new Properties();
private static Path pandocDir = Path.of("./pandoc").toAbsolutePath();
private static final int TIMEOUT_SECONDS = 2;
private static final int ATTEMPTS = 3;
Installer() throws IOException, InterruptedException {
try {
properties.load(new FileInputStream(
Thread.currentThread().getContextClassLoader().getResource("installer.properties").getPath()
));
} catch (Exception ex) {
log.error("Error during download properties, ex: {}", ex.getMessage(), ex);
throw ex;
}
initFiles();
var resp = getGithubUrls();
initUrls(resp);
}
private void initFiles() {
for (var os : OSType.values()) {
dataForInstalling.put(os, new OSData());
switch (os) {
case LINUX_AMD :
case LINUX_ARM :
dataForInstalling.get(os)
.setFileToInstall(Path.of(pandocDir.toString() + "/pandoc.tar.gz"));
break;
default :
dataForInstalling.get(os)
.setFileToInstall(Path.of(pandocDir.toString() + "/pandoc.zip"));
}
}
}
private void initUrls(ResponseDto responseDto) throws IOException {
for (var os : OSType.values()) {
var asset = responseDto.getAssetByOsSuffix(os.getAssetSuffix());
var currUrl = asset.getBrowserDownloadUrl();
var currPath = properties.getProperty("path.to.pandoc." + os.getPropertySuffix()).replace("{version}",
responseDto.getTagName());
dataForInstalling.get(os).setUrlForInstalling(URI.create(currUrl).toURL());
dataForInstalling.get(os).setPathToPandoc(Path.of(pandocDir.toString() + currPath));
log.info("Init {} url : {}, path to pandoc: {}", os, currUrl, dataForInstalling.get(os).getPathToPandoc());
}
}
/**
* Install last released pandoc from github
* @return path to executable pandoc
* @throws IOException in case incorrect github url or path of installation directory
*/
Path installPandoc() throws IOException {
log.info("Start install");
Path res;
if (OS.isFamilyMac()) {
if (OS.isArch("aarch64")) {
res = installPandoc(OSType.MAC_OS_ARM);
} else {
res = installPandoc(OSType.MAC_OS_AMD);
}
} else if (OS.isFamilyUnix()) {
if (OS.isArch("aarch64")) {
res = installPandoc(OSType.LINUX_ARM);
} else {
res = installPandoc(OSType.LINUX_AMD);
}
} else if (OS.isFamilyWindows()) {
res = installPandoc(OSType.WINDOWS);
} else {
throw new RuntimeException("Got unexpected os, could not install pandoc");
}
return res;
}
private Path installPandoc(OSType os) throws IOException {
log.info(
"Start installing pandoc os: {}, url: {}, file: {}",
os,
dataForInstalling.get(os).getUrlForInstalling(),
dataForInstalling.get(os).getFileToInstall()
);
clearInstallingDirectory();
if (!handleSaving(os)) {
throw new RuntimeException("Could not save file from github");
}
if (!unarchive(os)) {
throw new RuntimeException("Could not unzip file");
}
if (!dataForInstalling.get(os).getPathToPandoc().toFile().setExecutable(true)) {
throw new RuntimeException("Could not make pandoc executable");
}
return dataForInstalling.get(os).getPathToPandoc();
}
/**
* Downloads from a (http/https) URL and saves to a file.
* @param file File to write. Parent directory will be created if necessary
* @param url http/https url to connect
* @param secsConnectTimeout Seconds to wait for connection establishment
* @param secsReadTimeout Read timeout in seconds - trasmission will abort if it freezes more than this
* @return true if successfully save file and false if:
* connection interrupted, timeout (but something was read)
* server error (500...)
* could not connect: connection timeout java.net.SocketTimeoutException
* could not connect: java.net.ConnectException
* could not resolve host (bad host, or no internet - no dns)
* @throws IOException Only if URL is malformed or if could not create the file
* @throws FileNotFoundException if did not find file for save
*/
private boolean saveUrl(final Path file, final URL url,
int secsConnectTimeout, int secsReadTimeout) throws IOException {
Files.createDirectories(file.getParent()); // make sure parent dir exists , this can throw exception
var conn = url.openConnection(); // can throw exception if bad url
if (secsConnectTimeout > 0) {
conn.setConnectTimeout(secsConnectTimeout * 1000);
}
if (secsReadTimeout > 0) {
conn.setReadTimeout(secsReadTimeout * 1000);
}
var ret = true;
boolean somethingRead = false;
try (var is = conn.getInputStream()) {
try (var in = new BufferedInputStream(is);
var fout = Files.newOutputStream(file)) {
final byte data[] = new byte[8192];
int count;
while ((count = in.read(data)) > 0) {
somethingRead = true;
fout.write(data, 0, count);
}
}
} catch (java.io.IOException e) {
int httpcode = 999;
try {
httpcode = ((HttpURLConnection) conn).getResponseCode();
} catch (Exception ee) {}
if (e instanceof FileNotFoundException) {
throw new FileNotFoundException("Did not found file for install");
}
if (somethingRead && e instanceof java.net.SocketTimeoutException) {
log.error("Read something, but connection interrupted: {}", e.getMessage(), e);
ret = false;
} else if (httpcode >= 400 && httpcode < 600 ) {
log.error("Got server error, httpcode: {}", httpcode);
ret = false;
} else if (e instanceof java.net.SocketTimeoutException) {
log.error("Connection timeout: {}", e.getMessage(), e);
ret = false;
} else if (e instanceof java.net.ConnectException) {
log.error("Could not connect: {}", e.getMessage(), e);
ret = false;
} else if (e instanceof java.net.UnknownHostException ) {
log.error("Could not resolve host: {}", e.getMessage(), e);
ret = false;
} else {
throw e;
}
}
return ret;
}
private boolean handleSaving(OSType os) throws IOException {
var attempt = 0;
var saveFile = false;
while (attempt < ATTEMPTS && !saveFile) {
++attempt;
saveFile = saveUrl(
dataForInstalling.get(os).getFileToInstall(),
dataForInstalling.get(os).getUrlForInstalling(),
TIMEOUT_SECONDS,
TIMEOUT_SECONDS);
}
return saveFile;
}
private boolean unarchive(OSType os) {
try {
switch (os) {
case LINUX_AMD:
case LINUX_ARM :
unTarGz(dataForInstalling.get(os).getFileToInstall(), pandocDir);
break;
default :
unZip(dataForInstalling.get(os).getFileToInstall(), pandocDir);
}
} catch (IOException e) {
log.error("Could not perform unarchiving: {}", e.getMessage(), e);
return false;
}
return true;
}
private void unTarGz(Path pathInput, Path targetDir) throws IOException {
try (var tarIn =
new TarArchiveInputStream(
new GzipCompressorInputStream(
new BufferedInputStream(Files.newInputStream(pathInput))))) {
ArchiveEntry archiveEntry;
while ((archiveEntry = tarIn.getNextEntry()) != null) {
var pathEntryOutput = targetDir.resolve(archiveEntry.getName());
if (archiveEntry.isDirectory()) {
Files.createDirectory(pathEntryOutput);
} else {
Files.copy(tarIn, pathEntryOutput);
}
}
}
}
private void unZip(Path pathInput, Path targetDir) throws IOException {
ZipFile zipFile = new ZipFile(pathInput.toFile());
try {
Enumeration<ZipArchiveEntry> entries = zipFile.getEntries();
while (entries.hasMoreElements()) {
ZipArchiveEntry zipEntry = entries.nextElement();
var pathEntryOutput = targetDir.resolve(zipEntry.getName());
if (zipEntry.isDirectory()) {
Files.createDirectories(pathEntryOutput);
} else {
Files.createDirectories(pathEntryOutput.getParent());
Files.copy(zipFile.getInputStream(zipEntry), pathEntryOutput);
}
}
} finally {
zipFile.close();
}
}
/**
* Clear installing directory
*/
public static void clearInstallingDirectory() {
try {
FileUtils.cleanDirectory(pandocDir.toFile());
} catch (IOException e) {
log.error("Could not clean installing directory");
}
}
/**
* Set directory to install pandoc
* @param newDir
*/
public void setInstallingDirectory(Path newDir) {
pandocDir = newDir.toAbsolutePath();
}
private ResponseDto getGithubUrls() throws IOException, InterruptedException {
var uri = URI.create(properties.getProperty("github.url"));
var client = HttpClient.newHttpClient();
var request = HttpRequest
.newBuilder()
.uri(uri)
.version(HttpClient.Version.HTTP_2)
.timeout(Duration.ofMinutes(1))
.header("Accept", "application/vnd.github+json")
.GET()
.build();
var response = client.send(request, HttpResponse.BodyHandlers.ofString());
log.info("Got response from github, status: {}", response.statusCode());
var objectMapper = new ObjectMapper();
objectMapper.configure(DeserializationFeature.USE_JAVA_ARRAY_FOR_JSON_ARRAY, true);
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
return objectMapper.readValue(response.body(), ResponseDto.class);
}
}

View File

@ -0,0 +1,142 @@
package space.kscience.snark.pandoc;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class PandocWrapper {
private static final Logger log
= LoggerFactory.getLogger(PandocWrapper.class);
private static final Installer installer;
static {
try {
installer = new Installer();
} catch (IOException | InterruptedException e) {
throw new RuntimeException(e);
}
}
private static String pandocPath = "pandoc"; // got pandoc at PATH
public static String getPandocPath() {
return pandocPath;
}
/**
* Install pandoc if needed then perform block
* @param block
* @return block's return value
* @param <T>
*/
public <T> Object use(Function<PandocWrapper, T> block) {
if (!isPandocInstalled()) {
installPandoc();
}
return block.apply(this);
}
/**
* Check if pandoc is installed
* @return true if installed false otherwise
*/
public static boolean isPandocInstalled() {
var pb = new PandocCommandBuilder().getVersion();
return execute(pb);
}
/**
* Call pandoc with options described by commandBuilder.
* @param commandBuilder
* @return true if successfully false otherwise
*/
public static boolean execute(PandocCommandBuilder commandBuilder) {
return execute(commandBuilder, null, null);
}
/**
* Call pandoc with options described by commandBuilder and log output to outputFile
* @param commandBuilder
* @return true if successfully false otherwise
*/
public static boolean execute(PandocCommandBuilder commandBuilder, Path outputFile) {
return execute(commandBuilder, outputFile, null);
}
/**
* Call pandoc with options described by commandBuilder and log output to outputFile and error to errorFile.
* In case errors write exit code to errorFile
* @param commandBuilder
* @return true if successfully false otherwise
*/
public static boolean execute(PandocCommandBuilder commandBuilder, Path outputFile, Path errorFile) {
try {
Process pandoc = new ProcessBuilder(commandBuilder.build()).start();
pandoc.waitFor(1, TimeUnit.SECONDS);
BufferedReader inp = new BufferedReader(new InputStreamReader(pandoc.getInputStream()));
String currLine = inp.readLine();
log.info("log output from pandoc to: {}", outputFile);
do {
if (outputFile == null) {
log.info(currLine);
} else {
Files.writeString(outputFile, currLine + "\n", StandardOpenOption.CREATE, StandardOpenOption.APPEND);
}
} while ((currLine = inp.readLine()) != null);
inp.close();
if (pandoc.exitValue() == 0) {
log.info("Successfully execute");
return true;
} else {
log.error("Got problems with executing, pandoc exit error: {}", pandoc.exitValue());
BufferedReader input = new BufferedReader(new InputStreamReader(pandoc.getErrorStream()));
String line = input.readLine();
log.info("log error stream from pandoc to: {}", errorFile);
if (errorFile != null) {
Files.writeString(errorFile, "exit code: " + pandoc.exitValue());
}
do {
if (errorFile == null) {
log.info(line);
} else {
Files.writeString(errorFile, currLine + "\n", StandardOpenOption.CREATE, StandardOpenOption.APPEND);
}
} while ((line = input.readLine()) != null);
input.close();
return false;
}
} catch (Exception e) {
log.error("Got problems with executing: " + e.getMessage());
return false;
}
}
/**
* Install pandoc and set executable path.
* @return true if success false otherwise
*/
public static boolean installPandoc() {
try {
pandocPath = installer.installPandoc().toString();
return true;
} catch (Exception e) {
log.error("Got error: {}", e.getMessage(), e);
return false;
}
}
}

View File

@ -0,0 +1,70 @@
package space.kscience.snark.pandoc;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* Response from github/releases/latest
*/
public class ResponseDto {
public AssetDto[] getAssets() {
return assets;
}
public void setAssets(AssetDto[] assets) {
this.assets = assets;
}
/**
* @param osSuffix
* @return asset appropriate to os
*/
public AssetDto getAssetByOsSuffix(String osSuffix) {
for (var asset : assets) {
if (asset.getName().contains(osSuffix)) {
return asset;
}
}
throw new IllegalArgumentException("Unexpected osSuffix");
}
public static class AssetDto {
@JsonProperty("browser_download_url")
private String browserDownloadUrl;
private String name;
public String getBrowserDownloadUrl() {
return browserDownloadUrl;
}
public void setBrowserDownloadUrl(String browserDownloadUrl) {
this.browserDownloadUrl = browserDownloadUrl;
}
public AssetDto() {}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}
private AssetDto[] assets;
public String getTagName() {
return tagName;
}
public void setTagName(String tagName) {
this.tagName = tagName;
}
@JsonProperty("tag_name")
private String tagName;
public ResponseDto() {}
}

View File

@ -0,0 +1,8 @@
path.to.pandoc.mac.os.arm=/pandoc-{version}-arm64/bin/pandoc
path.to.pandoc.mac.os.amd=/pandoc-{version}-x86_64/bin/pandoc
path.to.pandoc.windows=/pandoc-{version}/pandoc.exe
path.to.pandoc.linux.amd=/pandoc-{version}/bin/pandoc
path.to.pandoc.linux.arm=/pandoc-{version}/bin/pandoc
github.url=https://api.github.com/repos/jgm/pandoc/releases/latest

View File

@ -0,0 +1,123 @@
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.stream.Collectors;
import space.kscience.snark.pandoc.Installer;
import space.kscience.snark.pandoc.PandocCommandBuilder;
import space.kscience.snark.pandoc.PandocWrapper;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
public class PandocWrapperTest {
private static final Path CORRECT_MD = Path.of("./src/test/testing_directory/first_test.md");
private static final Path TEX_PATH_TO = Path.of("./src/test/testing_directory/output1.tex");
private static final Path TESTING_DIRECTORY = Path.of("./src/test/testing_directory");
private final PandocWrapper pandocWrapper = new PandocWrapper();
@Test
public void when_gotPandocAndCorrectArgs_doConverting() {
try {
var res = pandocWrapper.use(p -> {
var command = new PandocCommandBuilder(List.of(CORRECT_MD), TEX_PATH_TO);
return PandocWrapper.execute(command);
});
assertTrue((Boolean) res);
assertTrue(TEX_PATH_TO.toFile().exists());
var reader = new BufferedReader(new FileReader(TEX_PATH_TO.toFile()));
String fileString = reader.lines().collect(Collectors.joining());
assertTrue(fileString.contains("Some simple text"));
assertTrue(fileString.contains("\\subsection{Copy elision}"));
assertTrue(fileString.contains("return"));
Files.delete(TEX_PATH_TO);
} catch (Exception ex) {
fail("Unexpected exception during test when_gotPandocAndCorrectArgs_doConverting()", ex);
}
}
@Test
public void when_gotPandocAndNotExistsFromFile_then_error() {
var notExistsFile = Path.of("./src/test/testing_directory/non_exists_test.md");
assertFalse(notExistsFile.toFile().exists());
var res = pandocWrapper.use(p -> {
var command = new PandocCommandBuilder(List.of(notExistsFile), TEX_PATH_TO);
return PandocWrapper.execute(command);
});
assertFalse((Boolean) res);
}
@Test
public void when_gotPandocAndPassDirectory_then_error() {
assertTrue(TESTING_DIRECTORY.toFile().isDirectory());
var res = pandocWrapper.use(p -> {
var command = new PandocCommandBuilder(List.of(TESTING_DIRECTORY), TEX_PATH_TO);
return PandocWrapper.execute(command);
});
assertFalse((Boolean) res);
}
@Test
public void when_askVersionToFile_then_Ok() throws IOException {
Path outputFile = Files.createTempFile(TESTING_DIRECTORY, "output", ".txt");
var res = pandocWrapper.use(p -> {
var command = new PandocCommandBuilder();
command.getVersion();
return PandocWrapper.execute(command, outputFile);
});
var reader = new BufferedReader(new FileReader(outputFile.toFile()));
String fileString = reader.lines().collect(Collectors.joining());
assertTrue(fileString.contains("pandoc"));
assertTrue(fileString.contains("This is free software"));
assertTrue((Boolean) res);
Files.delete(outputFile);
}
@Test
public void when_error_then_writeToErrorStream() throws IOException {
Path outputFile = Files.createTempFile(TESTING_DIRECTORY, "output", ".txt");
Path errorFile = Files.createTempFile(TESTING_DIRECTORY, "error", ".txt");
var res = pandocWrapper.use(p -> {
var command = new PandocCommandBuilder(List.of(Path.of("./simple.txt")), TEX_PATH_TO);
command.formatFrom("txt");
return PandocWrapper.execute(command, outputFile, errorFile);
});
var reader = new BufferedReader(new FileReader(errorFile.toFile()));
String fileString = reader.lines().collect(Collectors.joining());
assertFalse((Boolean) res);
assertTrue(fileString.contains("21"));
Files.delete(outputFile);
Files.delete(errorFile);
}
@Test
public void when_installPandoc_thenFindIt() {
Installer.clearInstallingDirectory();
assertTrue(PandocWrapper.installPandoc());
assertTrue(PandocWrapper.isPandocInstalled());
}
@AfterAll
public static void clear() {
Installer.clearInstallingDirectory();
}
}

View File

@ -0,0 +1,15 @@
## Copy elision
### RVO/NRVO
Some simple text
```c++
A f() {
return {5};
}
A g() {
A a(5);
return a;
}
```

View File

@ -0,0 +1 @@
hello

View File

@ -0,0 +1,22 @@
plugins {
id("space.kscience.gradle.jvm")
`maven-publish`
}
val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
val awsSdkVersion = "0.+"
dependencies {
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
// s3 Driver dependency
implementation("aws.sdk.kotlin:s3:$awsSdkVersion")
testImplementation(kotlin("test"))
testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
}
tasks.test {
useJUnitPlatform()
}

View File

@ -0,0 +1,40 @@
package space.kscience.snark.storage
import aws.sdk.kotlin.services.s3.S3Client
import space.kscience.snark.storage.local.localStorage
import space.kscience.snark.storage.s3.s3Bucket
import space.kscience.snark.storage.s3.s3Storage
import java.nio.file.Path
private const val DEFAULT_REGION = "arctic-vault"
public sealed interface Config {
public fun build(): Directory
}
public data class LocalConfig(val path: Path) : Config {
override fun build(): Directory {
return localStorage(path)
}
}
/*
* ~/.aws/credentials.json file is required
*/
internal fun buildS3Client(regionSpec: String): S3Client {
return S3Client {
region = regionSpec
}
}
public data class S3BucketConfig(val bucketName: String, val region: String = DEFAULT_REGION) : Config {
override fun build(): Directory {
return s3Bucket(buildS3Client(region), bucketName)
}
}
public data class S3ServiceConfig(val region: String = DEFAULT_REGION) : Config {
override fun build(): Directory {
return s3Storage(buildS3Client(region))
}
}

View File

@ -0,0 +1,46 @@
package space.kscience.snark.storage
import java.nio.file.Path
import kotlin.io.path.Path
public interface Directory : AutoCloseable {
// get file from subtree
public suspend fun get(filename: Path): FileReader
@Deprecated("Use put")
public suspend fun create(filename: String, ignoreIfExists: Boolean = false)
// put file to subtree
public suspend fun put(filename: Path): FileWriter
public suspend fun getSubdir(path: Path): Directory
@Deprecated("Directories are created on put")
public suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean = false): Directory
@Deprecated("Not a good idea")
public val path: Path
public companion object {
public fun fromConfig(config: Config): Directory {
return config.build()
}
}
}
public suspend fun Directory.get(filename: String): FileReader = get(Path(filename))
public suspend fun Directory.put(filename: String): FileWriter = put(Path(filename))
public suspend operator fun Directory.div(path: Path): Directory = getSubdir(path)
public suspend operator fun Directory.div(path: String): Directory = getSubdir(Path(path))
public interface FileReader : AutoCloseable {
public suspend fun readAll(): ByteArray
}
public interface FileWriter : AutoCloseable {
public suspend fun write(bytes: ByteArray)
}

View File

@ -0,0 +1,71 @@
package space.kscience.snark.storage.local
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.nio.file.Path
import kotlin.io.path.*
public fun localStorage(rootPath: Path): Directory {
return LocalDirectory(rootPath, Path(""))
}
internal class LocalFile(private val path: Path) : FileReader, FileWriter {
override fun close() {}
override suspend fun readAll(): ByteArray = path.readBytes()
override suspend fun write(bytes: ByteArray) {
path.parent.createDirectories()
try {
path.createFile()
} catch (ex: Exception) {
// Do nothing
}
path.writeBytes(bytes)
}
}
internal class LocalDirectory(private val root: Path, private val currentDir: Path) : Directory {
@Deprecated("Use Path, not String")
private fun realpath(child: String): Path = root / currentDir / child
private fun realpath(child: Path): Path = root / currentDir / child
override fun close() {}
override suspend fun get(filename: Path): LocalFile = LocalFile(realpath(filename))
@Deprecated("Use put")
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
val dir = realpath(filename)
dir.parent.createDirectories()
try {
realpath(filename).createFile()
} catch (ex: java.nio.file.FileAlreadyExistsException) {
if (!ignoreIfExists) {
throw ex
}
}
}
override suspend fun put(filename: Path): LocalFile = get(filename)
override suspend fun getSubdir(path: Path): LocalDirectory = LocalDirectory(root, currentDir / path)
@Deprecated("Directories are created on put")
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): LocalDirectory {
val dir = realpath(dirname)
dir.parent.createDirectories()
try {
dir.createDirectory()
} catch (ex: java.nio.file.FileAlreadyExistsException) {
if (!ignoreIfExists) {
throw ex
}
}
return LocalDirectory(root, currentDir / dirname)
}
@Deprecated("Not a good idea")
override val path: Path
get() = currentDir
}

View File

@ -0,0 +1,45 @@
package space.kscience.snark.storage.s3
import aws.sdk.kotlin.services.s3.S3Client
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.nio.file.Path
import kotlin.io.path.div
internal class S3Directory(
private val client: S3Client,
private val bucketName: String,
private val currentDir: Path,
) : Directory {
override suspend fun get(filename: Path): FileReader =
S3FileReader(client, bucketName, currentDir / filename)
@Deprecated("Use put")
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
if (!ignoreIfExists) {
TODO("could not check if file exists")
}
}
override suspend fun put(filename: Path): FileWriter =
S3FileWriter(client, bucketName, currentDir / filename)
override suspend fun getSubdir(path: Path): S3Directory =
S3Directory(client, bucketName, currentDir / path)
@Deprecated("Directories are created on put")
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): S3Directory =
if (!ignoreIfExists) {
TODO("could not check if directory exists")
} else {
S3Directory(client, bucketName, currentDir / dirname)
}
@Deprecated("Not a good idea")
override val path: Path
get() = currentDir
override fun close() {
}
}

View File

@ -0,0 +1,41 @@
package space.kscience.snark.storage.s3
import aws.sdk.kotlin.services.s3.S3Client
import aws.sdk.kotlin.services.s3.model.GetObjectRequest
import aws.sdk.kotlin.services.s3.putObject
import aws.smithy.kotlin.runtime.content.ByteStream
import aws.smithy.kotlin.runtime.content.toByteArray
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.nio.file.Path
internal class S3FileReader(private val client: S3Client, private val bucketName: String, private val path: Path) :
FileReader {
override suspend fun readAll(): ByteArray {
val result = client.getObject(GetObjectRequest {
bucket = bucketName
key = path.toString()
}) {
it.body?.toByteArray() ?: ByteArray(0)
}
return result
}
override fun close() {
}
}
internal class S3FileWriter(private val client: S3Client, private val bucketName: String, private val path: Path) :
FileWriter {
override suspend fun write(bytes: ByteArray) {
client.putObject {
bucket = bucketName
key = path.toString()
body = ByteStream.fromBytes(bytes)
}
}
override fun close() {
}
}

View File

@ -0,0 +1,67 @@
package space.kscience.snark.storage.s3
import aws.sdk.kotlin.services.s3.S3Client
import aws.sdk.kotlin.services.s3.createBucket
import aws.sdk.kotlin.services.s3.headBucket
import space.kscience.snark.storage.Directory
import space.kscience.snark.storage.FileReader
import space.kscience.snark.storage.FileWriter
import java.nio.file.Path
import kotlin.io.path.Path
public fun s3Storage(client: S3Client): Directory =
S3Root(client)
public fun s3Bucket(client: S3Client, bucket: String): Directory =
S3Directory(client, bucket, Path(""))
internal fun splitPathIntoBucketAndPath(path: Path): Pair<String, Path> {
val bucket = path.getName(0)
val filePath = path.relativize(bucket)
return Pair(bucket.toString(), filePath)
}
internal class S3Root(private val client: S3Client) : Directory {
override suspend fun get(filename: Path): FileReader {
throw NoSuchFileException(filename.toFile())
}
@Deprecated("Use put")
override suspend fun create(filename: String, ignoreIfExists: Boolean) {
throw NoSuchFileException(Path(filename).toFile())
}
override suspend fun put(filename: Path): FileWriter {
throw NoSuchFileException(filename.toFile())
}
override suspend fun getSubdir(path: Path): Directory = try {
val (bucketName, filePath) = splitPathIntoBucketAndPath(path)
client.headBucket {
bucket = bucketName
}
S3Directory(client, bucketName, filePath)
} catch (ex: Exception) {
throw AccessDeniedException(path.toFile(), reason = ex.message)
}
@Deprecated("Directories are created on put")
override suspend fun createSubdir(dirname: String, ignoreIfExists: Boolean): Directory = try {
val (bucketName, filePath) = splitPathIntoBucketAndPath(Path(dirname))
client.createBucket {
bucket = bucketName
}
S3Directory(client, bucketName, filePath)
} catch (ex: Exception) {
throw AccessDeniedException(Path(dirname).toFile(), reason = ex.message)
}
@Deprecated("Not a good idea")
override val path: Path
get() = Path("")
override fun close() {
}
}

View File

@ -0,0 +1,28 @@
package space.kscience.snark.storage.unzip
import space.kscience.snark.storage.*
import java.io.FileInputStream
import java.util.zip.ZipInputStream
public suspend fun unzip(source_path: String, target: Directory) {
val zis = ZipInputStream(FileInputStream(source_path))
var zipEntry = zis.nextEntry
while (zipEntry != null) {
if (!zipEntry.isDirectory) {
val filename = zipEntry.name
target.create(filename, true)
val fos = target.put(filename)
var sz = zipEntry.size.toInt()
if (sz == -1) {
sz = 1024
}
val buffer = ByteArray(sz)
zis.read(buffer)
fos.write(buffer)
fos.close()
}
zipEntry = zis.nextEntry
}
zis.closeEntry()
zis.close()
}

View File

@ -0,0 +1,25 @@
package space.kscience.snark.storage
import org.junit.jupiter.api.Test
import kotlin.io.path.createTempDirectory
class JustCreates {
@Test
fun s3Created() {
val dir = Directory.fromConfig(S3ServiceConfig())
dir.close()
}
@Test
fun s3BucketCreated() {
val dir = Directory.fromConfig(S3BucketConfig("snark-test"))
dir.close()
}
@Test
fun localCreated() {
val dir = Directory.fromConfig(LocalConfig(createTempDirectory("snark-test")))
dir.close()
}
}

View File

@ -0,0 +1,47 @@
package space.kscience.snark.storage.local
import kotlinx.coroutines.runBlocking
import org.junit.jupiter.api.BeforeAll
import space.kscience.snark.storage.*
import java.nio.file.Path
import kotlin.io.path.createTempDirectory
import kotlin.io.path.deleteExisting
import kotlin.test.AfterTest
import kotlin.test.BeforeTest
import kotlin.test.Test
import kotlin.io.path.*
import kotlin.test.assertEquals
class Example {
var tempDir: Path? = null
var somedir: Directory? = null
@BeforeTest
fun setUp() {
tempDir = createTempDirectory()
somedir = localStorage(tempDir!!)
}
@AfterTest
fun tearDown() {
tempDir!!.toFile().deleteRecursively()
somedir = null
}
@Test
fun exampleTest() = runBlocking {
somedir!!.put("somefile").write("hello".toByteArray())
assertEquals("hello", somedir!!.get("somefile").readAll().decodeToString())
}
@Test
fun subdirExample() = runBlocking {
val dir1 = somedir!! / "tmp1"
dir1.put("somefile").write("hello".toByteArray())
val dir2 = somedir!! / "tmp1"
val data = dir2.get("somefile").readAll()
assertEquals("hello", data.decodeToString())
}
}

View File

@ -0,0 +1,105 @@
package space.kscience.snark.storage.local
import kotlinx.coroutines.runBlocking
import space.kscience.snark.storage.*
import java.io.File
import java.nio.file.Path
import kotlin.io.path.*
import kotlin.test.*
internal class LocalDriverTests {
private var tempDir: Path? = null
private var testSample: Directory? = null
private val bytes = byteArrayOf(0, 1, 2, 3, 4, 5, 6, 7)
@BeforeTest
fun setUp() {
tempDir = createTempDirectory()
testSample = localStorage(tempDir!!)
}
@Test
fun testCreate() = runBlocking {
//folder is empty
assertEquals(0, tempDir!!.listDirectoryEntries().size)
//create first file
testSample!!.create("tmp1")
val entries = tempDir!!.listDirectoryEntries()
assertEquals(1, entries.size)
assertEquals(tempDir!! / Path("tmp1"), entries.first())
//assertTrue(!entries.first().isDirectory())
//create second file
testSample!!.create("tmp2")
assertEquals(2, tempDir!!.listDirectoryEntries().size)
//check exception after duplication
try {
testSample!!.create("tmp1")
fail("shouldn't ignore duplicates here")
} catch (ex: java.nio.file.FileAlreadyExistsException) {}
//check ignorance
try {
testSample!!.create("tmp1", true)
} catch (ex: java.nio.file.FileAlreadyExistsException) {
fail("should ignore duplicates here")
}
}
@Test
fun testPutGet() = runBlocking {
testSample!!.create("tmp")
testSample!!.put("tmp").write(bytes)
assertContentEquals(bytes, testSample!!.get("tmp").readAll())
}
@Test
fun testCreateSubdir() = runBlocking {
//folder is empty
assertEquals(0, tempDir!!.listDirectoryEntries().size)
//create first file
testSample!!.createSubdir("tmp1")
val entries = tempDir!!.listDirectoryEntries()
assertEquals(1, entries.size)
assertEquals(tempDir!! / Path("tmp1"), entries.first())
assertTrue (entries.first().isDirectory())
//create second file
testSample!!.createSubdir("tmp2")
assertEquals(2, tempDir!!.listDirectoryEntries().size)
//check exception after duplication
try {
testSample!!.createSubdir("tmp1")
fail("shouldn't ignore duplicates here")
} catch (ex: java.nio.file.FileAlreadyExistsException) {}
//check ignorance
try {
testSample!!.createSubdir("tmp1", true)
} catch (ex: java.nio.file.FileAlreadyExistsException) {
fail("should ignore duplicates here")
}
assertTrue {true}
}
@Test
fun testGetSubdir() = runBlocking {
testSample!!.createSubdir("tmp")
val pathStr = (Path("tmp") / "data.txt").toString()
testSample!!.create(pathStr)
testSample!!.put(pathStr).write(bytes)
val subdir = testSample!!.getSubdir(Path("tmp"))
assertContentEquals(bytes, subdir.get("data.txt").readAll())
}
@AfterTest
fun tearDown() {
tempDir!!.toFile().deleteRecursively()
}
}

View File

@ -0,0 +1,108 @@
package space.kscience.snark.storage.unzip
import kotlinx.coroutines.runBlocking
import space.kscience.snark.storage.*
import space.kscience.snark.storage.local.localStorage
import java.io.*
import java.nio.file.Files
import java.nio.file.Path
import java.util.concurrent.TimeUnit
import java.util.zip.ZipEntry
import java.util.zip.ZipOutputStream
import kotlin.io.path.*
import kotlin.test.*
internal class UnzipTests {
private var tempDir: Path? = null
@BeforeTest
fun setUp() {
tempDir = createTempDirectory()
}
private suspend fun makeFile(dir: Directory, filename: String, content: ByteArray) {
dir.create(filename)
dir.put(filename).write(content)
}
private fun zipAll(directory: String, zipFile: String) {
val sourceFile = File(directory)
ZipOutputStream(BufferedOutputStream( FileOutputStream(zipFile))).use {
zipFiles(it, sourceFile, File.separator)
it.closeEntry()
it.close()
}
}
private fun zipFiles(zipOut: ZipOutputStream, sourceFile: File, parentDirname: String) {
val data = ByteArray(2048)
for (f in sourceFile.listFiles()) {
if (f.isDirectory) {
zipFiles(zipOut, f, parentDirname + f.name + File.separator)
} else {
FileInputStream(f).use { fi ->
BufferedInputStream(fi).use { origin ->
var path = parentDirname + f.name
val entry = ZipEntry(path.drop(1))
entry.time = f.lastModified()
entry.size = f.length()
zipOut.putNextEntry(entry)
while (true) {
val readBytes = origin.read(data)
if (readBytes == -1) {
break
}
zipOut.write(data, 0, readBytes)
}
}
}
}
}
}
@Test
fun testUnzip() = runBlocking {
val dir: Directory = localStorage(tempDir!!)
val source = dir.createSubdir("source")
val target = dir.createSubdir("target")
val bytes1 = byteArrayOf(0, 1, 2, 3)
val bytes2 = byteArrayOf(1, 0, 3, 2)
val bytes3 = byteArrayOf(3, 2, 1, 0)
makeFile(source, "tmp1", bytes1)
makeFile(source, "tmp2", bytes2);
makeFile(source, (Path("tdir") / "tmp3").toString(), bytes3)
dir.create("archive.zip")
val archive_path = (tempDir!! / Path("archive.zip")).toString()
zipAll((tempDir!! / Path("source")).toString(), archive_path)
unzip(archive_path, target)
val targetPath = tempDir!! / Path("target")
println(targetPath)
val entries = targetPath.listDirectoryEntries()
assertEquals(3, entries.size)
val exp_entries = listOf(
targetPath / Path("tmp1"),
targetPath / Path("tmp2"),
targetPath / Path("tdir"))
assertContentEquals(entries.sorted(), exp_entries.sorted())
val tdirEntries = (targetPath / Path("tdir")).listDirectoryEntries()
assertEquals(1, tdirEntries.size)
assertEquals(tdirEntries.first(), targetPath / Path("tdir") / Path("tmp3"))
}
@AfterTest
fun tearDown() {
tempDir!!.toFile().deleteRecursively()
}
}