Merge branch 'feature/SNRK-71/document-builder-implementation' into release/SNRK-86/assembly

settings.gradle.kts

@@ -43,4 +43,5 @@ include(
     ":snark-html",
     ":snark-ktor",
     ":snark-storage-driver",
+    ":snark-document-builder",
 )

snark-document-builder/build.gradle.kts

@@ -0,0 +1,17 @@
+plugins {
+    id("space.kscience.gradle.jvm")
+    `maven-publish`
+    id("kotlinx-serialization")
+}
+
+val coroutinesVersion = space.kscience.gradle.KScienceVersions.coroutinesVersion
+val jacksonVersion = "2.14.2"
+
+
+dependencies {
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
+
+    implementation(project(":snark-storage-driver"))
+    implementation("com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion")
+    implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.3")
+}

snark-document-builder/src/main/kotlin/DependencyGraph.kt

@@ -0,0 +1,44 @@
+package documentBuilder
+
+public typealias FileName = String
+
+/**
+ * Node of dependency graph.
+ *
+ * One node represents one file and its dependencies
+ *
+ * @property mdAst - AST tree of current file.
+ * @property dependencies - list of tail end adjacent to this node (dependencies of current file to be resolved).
+ */
+public data class DependencyGraphNode(
+    val mdAst: MdAstRoot,
+    val dependencies: List<DependencyGraphEdge>
+)
+
+/**
+ * Interface of all dependency edges.
+ */
+public sealed interface DependencyGraphEdge {
+}
+
+/**
+ * Include dependency edge.
+ *
+ * @property parentNode - node inside AST tree, that is parent for dependent node.
+ * @property dependentNode - dependent node, i.e. node of part of document with include commands
+ * @property includeList - list of files to be included.
+ */
+public data class IncludeDependency(
+    val parentNode: MdAstParent,
+    val dependentNode: MdAstElement,
+    val includeList: List<FileName>
+) : DependencyGraphEdge
+
+/**
+ * Whole dependency graph.
+ *
+ * @property nodes - map of nodes, where you can find DependencyGraphNode of file by its name.
+ */
+public data class DependencyGraph(
+    val nodes: Map<FileName, DependencyGraphNode>
+)

snark-document-builder/src/main/kotlin/DocumentBuilder.kt

@@ -0,0 +1,50 @@
+package documentBuilder
+
+import com.fasterxml.jackson.core.io.BigDecimalParser
+import space.kscience.snark.storage.*
+import java.nio.file.Path
+import java.nio.file.Paths
+
+private val DEFAULT_DOCUMENT_ROOT = "main.md"
+
+public suspend fun buildDocument(documentDirectory: Directory) {
+    val dependencyGraph = buildDependencyGraph(documentDirectory)
+
+    TODO() /*resolving of dependencies*/
+}
+
+public suspend fun buildDependencyGraph(root: Directory): DependencyGraph {
+    val nodes = HashMap<FileName, DependencyGraphNode>()
+
+    buildNodes(root, nodes)
+
+    return DependencyGraph(nodes)
+}
+
+private suspend fun buildNodes(folder: Directory, nodes: HashMap<FileName, DependencyGraphNode>) {
+    val pathString = folder.path.toString()
+
+    assert(!nodes.containsKey(pathString))
+
+    val rootDcoument = folder.get(DEFAULT_DOCUMENT_ROOT)
+    nodes.put(pathString, buildDependencyGraphNode(rootDcoument.readAll(), folder.path))
+
+    val dependencies = getDependencies(nodes.getValue(pathString))
+
+    for (dependency in dependencies) {
+        if (!nodes.containsKey(dependency))
+            buildNodes(folder.getSubdir(Paths.get(dependency)), nodes)
+    }
+}
+
+public suspend fun getDependencies(node: DependencyGraphNode): Set<FileName> {
+    val dependencies = mutableListOf<FileName>()
+
+    for (dependency in node.dependencies) {
+        when (dependency) {
+            is IncludeDependency -> dependencies.addAll(dependency.includeList)
+        }
+    }
+
+    return dependencies.toSet()
+}

snark-document-builder/src/main/kotlin/MdAstElements.kt

@@ -0,0 +1,84 @@
+package documentBuilder
+
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.SerialName
+import com.fasterxml.jackson.annotation.JsonSubTypes
+import com.fasterxml.jackson.annotation.JsonTypeInfo
+
+@Serializable
+public data class Point(val line: Int, val column: Int, val offset: Int)
+
+@Serializable
+public data class Position(val start: Point, val end: Point)
+
+@JsonTypeInfo(
+    use = JsonTypeInfo.Id.NAME,
+    include = JsonTypeInfo.As.PROPERTY,
+    property = "type"
+)
+
+@JsonSubTypes(
+    JsonSubTypes.Type(value = MdAstRoot::class, name = "root"),
+    JsonSubTypes.Type(value = MdAstParagraph::class, name = "paragraph"),
+    JsonSubTypes.Type(value = MdAstText::class, name = "text"),
+    JsonSubTypes.Type(value = MdAstHeading::class, name = "heading"),
+    JsonSubTypes.Type(value = MdAstCode::class, name = "code"),
+    JsonSubTypes.Type(value = MdAstBlockquote::class, name = "blockquote")
+)
+
+@Serializable
+public sealed interface MdAstElement{
+    public abstract var position: Position
+}
+
+@Serializable
+public sealed interface MdAstParent: MdAstElement{
+    public var children: List<MdAstElement>
+}
+
+@Serializable
+@SerialName("root")
+public data class MdAstRoot(
+    override var children: List<MdAstElement>,
+    override var position: Position
+): MdAstParent
+
+
+@Serializable
+@SerialName("paragraph")
+public data class MdAstParagraph(
+    override var children: List<MdAstElement>,
+    override var position: Position
+): MdAstParent
+
+
+@Serializable
+@SerialName("text")
+public data class MdAstText(
+    val value: String,
+    override var position: Position
+): MdAstElement
+
+@Serializable
+@SerialName("heading")
+public data class MdAstHeading(
+    val depth: Int, 
+    override var children: List<MdAstElement>,
+    override var position: Position
+): MdAstParent
+
+@Serializable
+@SerialName("code")
+public data class MdAstCode(
+    var lang: String? = null,
+    var meta: String? = null,
+    var value: String,
+    override var position: Position,
+) : MdAstElement
+
+@Serializable
+@SerialName("blockquote")
+public data class MdAstBlockquote(
+    override var children: List<MdAstElement>,
+    override var position: Position
+): MdAstParent

snark-document-builder/src/main/kotlin/MdParser.kt

@@ -0,0 +1,57 @@
+package documentBuilder
+
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import com.fasterxml.jackson.module.kotlin.readValue
+import java.nio.file.Path
+
+private val MARKDOWN_PARSER = "../nodejs/MarkdownParser.js"
+private val SNARK_PARSER = "../python/SnarkParse.py"
+
+public suspend fun parseMd(mdFile: ByteArray): MdAstRoot {
+    return jacksonObjectMapper()
+        .readValue<MdAstRoot>(ProcessBuilder("node", MARKDOWN_PARSER, mdFile.toString())
+        .redirectOutput(ProcessBuilder.Redirect.PIPE)
+        .redirectError(ProcessBuilder.Redirect.INHERIT)
+        .start().inputStream.bufferedReader().readText())
+}
+
+public suspend fun buildDependencyGraphNode(mdFile: ByteArray, path: Path): DependencyGraphNode {
+    val treeRoot = parseMd(mdFile)
+    val dependencies = mutableListOf<DependencyGraphEdge>()
+
+    fillDependencies(treeRoot, dependencies, path)
+
+    return DependencyGraphNode(treeRoot, dependencies)
+}
+
+internal suspend fun fillDependencies(
+        currentNode: MdAstElement,
+        dependencies: MutableList<DependencyGraphEdge>,
+        path: Path) {
+    when (currentNode) {
+        is MdAstParent -> {
+            for (child in currentNode.children) {
+                if (child is MdAstText) {
+                    val includeList = getIncludeFiles(child.value).toMutableList()
+
+                    if (includeList.size > 0) {
+                        includeList.replaceAll { path.toString() + "/" + it }
+
+                        dependencies += IncludeDependency(currentNode, child, includeList)
+                    }
+                } else {
+                    fillDependencies(child, dependencies, path)
+                }
+            }
+        }
+        else -> {}
+    }
+}
+
+public suspend fun getIncludeFiles(string: String): List<FileName> {
+    return jacksonObjectMapper()
+        .readValue<List<FileName>>(ProcessBuilder("python3", SNARK_PARSER, string)
+        .redirectOutput(ProcessBuilder.Redirect.PIPE)
+        .redirectError(ProcessBuilder.Redirect.INHERIT)
+        .start().inputStream.bufferedReader().readText())
+}

snark-document-builder/src/main/nodejs/MarkdownParser.js

@@ -0,0 +1,14 @@
+import {fromMarkdown} from 'mdast-util-from-markdown'
+
+main()
+
+function main()
+{
+    if (process.argv.length < 3)
+        throw "No input"
+
+    const markdown_string = process.argv[2] 
+    const mdast = fromMarkdown(markdown_string)
+
+    console.log(JSON.stringify(mdast))
+}

snark-document-builder/src/main/nodejs/package.json

@@ -0,0 +1,30 @@
+{
+
+  "type": "module",
+
+  "dependencies": {
+
+    "fs": "^0.0.1-security",
+
+    "hast-util-to-html": "^8.0.4",
+
+    "mdast-util-to-hast": "^12.3.0",
+
+    "node-fetch": "^3.3.1",
+
+    "remark-html": "^15.0.2",
+
+    "remark-parse": "^10.0.1",
+
+    "require": "^2.4.20",
+
+    "to-vfile": "^7.2.4",
+
+    "unified": "^10.1.2",
+
+    "whatwg-fetch": "^3.6.2"
+
+  }
+
+}
+

snark-document-builder/src/main/python/SnarkParser.py

@@ -0,0 +1,26 @@
+import sys
+import re
+import json
+
+assert(len(sys.argv) >= 2)
+
+string = sys.argv[1]
+
+outputfile = sys.argv[2] if len(sys.argv) >= 3 else None
+
+pattern = r'^([\n|\t| ]*@include\([a-z|0-9|.|_]*.md\)[\n|\t| ]*)*$'
+
+files = []
+
+if re.search("@include", string, re.IGNORECASE):
+    if re.match(pattern, string):
+        matches = re.findall(r'@include\((.*?)\)', string)
+        files.extend(matches)
+    else:
+        sys.exit("Illformed string")
+
+if outputfile is None:
+    print(json.dumps(files))
+else:
+    with open(outputfile, 'w+') as f:
+        json.dump(files, f)

snark-document-builder/src/test/kotlin/DocumentBuilder.kt

@@ -0,0 +1,11 @@
+package documentBuilder
+
+import org.junit.jupiter.api.Test
+import kotlinx.coroutines.runBlocking
+
+class SomeTest {
+    @Test
+    fun justWorks() = runBlocking {
+        // buildDocument(Directory("../example"))
+    }
+}

snark-document-builder/src/test/resources/no_dependencies_example/main.md

@@ -0,0 +1,3 @@
+# Hello
+
+I'm almost empty test document without any dependencies