vviora/varvara_kaplenko
1
0
Fork 0
forked from SPC/spc-site
Code Pull Requests Activity

Spliting site and the snark project

Browse Source
This commit is contained in:
Alexander Nozik 2022-06-30 20:51:57 +03:00
parent 4024eed50f
commit eb81d46238
No known key found for this signature in database
GPG Key ID: F7FCF2DD25C71357
23 changed files with 53 additions and 1057 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
build.gradle.kts
View File

@ -1,16 +1,16 @@
import ru.mipt.npm.gradle.KScienceVersions
import java.time.LocalDateTime
import space.kscience.snark.plugin.JSch
import space.kscience.snark.plugin.execute
import space.kscience.snark.plugin.uploadDirectory
import space.kscience.snark.plugin.useSession
plugins {
id("ru.mipt.npm.gradle.project")
id("ru.mipt.npm.gradle.jvm")
id("space.kscience.snark")
application
}
repositories {
mavenLocal()
}
group = "ru.mipt.npm"
version = "0.1.0"
@ -21,23 +21,14 @@ application {
applicationDefaultJvmArgs = listOf("-Dio.ktor.development=$isDevelopment", "-Xmx200M")
}
val dataforgeVersion by extra("0.6.0-dev-9")
val snarkVersion by extra("0.1.0-dev-1")
val ktorVersion = KScienceVersions.ktorVersion
dependencies {
implementation("io.ktor:ktor-server-core:$ktorVersion")
implementation("org.jetbrains.kotlinx:kotlinx-html:0.7.5")
implementation("io.ktor:ktor-server-html-builder:$ktorVersion")
implementation("org.jetbrains.kotlin-wrappers:kotlin-css")
implementation("io.ktor:ktor-server-host-common:$ktorVersion")
implementation("io.ktor:ktor-server-status-pages:$ktorVersion")
implementation("space.kscience:snark-ktor:$snarkVersion")
implementation("io.ktor:ktor-server-netty:$ktorVersion")
implementation("io.ktor:ktor-server-http-redirect:$ktorVersion")
implementation("ch.qos.logback:logback-classic:1.2.11")
implementation("space.kscience:dataforge-workspace:$dataforgeVersion")
implementation("space.kscience:dataforge-io-yaml:$dataforgeVersion")
implementation("org.jetbrains:markdown:0.3.1")
testImplementation("io.ktor:ktor-server-tests:$ktorVersion")
}
@ -46,6 +37,10 @@ kotlin {
explicitApi = org.jetbrains.kotlin.gradle.dsl.ExplicitApiMode.Disabled
}
apiValidation{
validationDisabled = true
}
tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile> {
kotlinOptions {
freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers"
@ -58,34 +53,6 @@ sourceSets {
}
}
readme {
maturity = ru.mipt.npm.gradle.Maturity.EXPERIMENTAL
feature("data") { "Data-based processing. Instead of traditional layout-based" }
feature("layouts") { "Use custom layouts to represent a data tree" }
feature("parsers") { "Add custom file formats and parsers using DataForge dependency injection" }
feature("preprocessor") { "Preprocessing text files using templates" }
feature("metadata") { "Trademark DataForge metadata layering and transformations" }
feature("dynamic") { "Generating dynamic site using KTor server" }
feature("static") { "Generating static site" }
}
/**
* This task updates the site build date in resource automatically
*/
val writeBuildDate: Task by tasks.creating {
doLast {
val deployDate = LocalDateTime.now()
val file = File(project.buildDir, "resources/main/buildDate")
file.parentFile.mkdirs()
file.writeText(deployDate.toString())
}
outputs.file("resources/main/buildDate")
outputs.upToDateWhen { false }
}
//write build time in build to check outdated external data directory
tasks.getByName("processResources").dependsOn(writeBuildDate)
/* Upload with JSch */
val host = System.getenv("SPC_HOST")

12
buildSrc/build.gradle.kts
View File

@ -1,12 +0,0 @@
plugins{
`kotlin-dsl`
}
repositories{
mavenCentral()
gradlePluginPortal()
}
dependencies{
implementation("com.github.mwiede:jsch:0.2.1")
}

7
buildSrc/settings.gradle.kts
View File

@ -1,7 +0,0 @@
dependencyResolutionManagement {
repositories {
maven("https://repo.kotlin.link")
mavenCentral()
gradlePluginPortal()
}
}

99
buildSrc/src/main/kotlin/uploads.kt
View File

@ -1,99 +0,0 @@
import com.jcraft.jsch.*
import java.io.File
import java.io.FileInputStream
import java.util.*
/**
* https://kodehelp.com/java-program-uploading-folder-content-recursively-from-local-to-sftp-server/
*/
private fun ChannelSftp.recursiveFolderUpload(sourceFile: File, destinationPath: String) {
if (sourceFile.isFile) {
// copy if it is a file
cd(destinationPath)
if (!sourceFile.name.startsWith(".")) put(
FileInputStream(sourceFile),
sourceFile.getName(),
ChannelSftp.OVERWRITE
)
} else {
val files = sourceFile.listFiles()
if (files != null && !sourceFile.getName().startsWith(".")) {
cd(destinationPath)
var attrs: SftpATTRS? = null
// check if the directory is already existing
val directoryPath = destinationPath + "/" + sourceFile.getName()
try {
attrs = stat(directoryPath)
} catch (e: Exception) {
println("$directoryPath does not exist")
}
// else create a directory
if (attrs != null) {
println("Directory $directoryPath exists IsDir=${attrs.isDir()}")
} else {
println("Creating directory $directoryPath")
mkdir(sourceFile.getName())
}
for (f in files) {
recursiveFolderUpload(f, destinationPath + "/" + sourceFile.getName())
}
}
}
}
fun Session.uploadDirectory(
file: File,
targetDirectory: String,
) {
var channel: ChannelSftp? = null
try {
val config = Properties()
config["StrictHostKeyChecking"] = "no"
channel = openChannel("sftp") as ChannelSftp // Open SFTP Channel
channel.connect()
channel.cd(targetDirectory) // Change Directory on SFTP Server
channel.recursiveFolderUpload(file, targetDirectory)
} finally {
channel?.disconnect()
}
}
fun Session.execute(
command: String,
): String {
var channel: ChannelExec? = null
try {
channel = openChannel("exec") as ChannelExec
channel.setCommand(command)
channel.inputStream = null
channel.setErrStream(System.err)
val input = channel.inputStream
channel.connect()
return input.use { it.readAllBytes().decodeToString() }
} finally {
channel?.disconnect()
}
}
inline fun JSch.useSession(
host: String,
user: String,
port: Int = 22,
block: Session.() -> Unit,
) {
var session: Session? = null
try {
session = getSession(user, host, port)
val config = Properties()
config["StrictHostKeyChecking"] = "no"
session.setConfig(config)
session.connect()
session.block()
} finally {
session?.disconnect()
}
}
fun JSch(configuration: JSch.() -> Unit): JSch = JSch().apply(configuration)

8
settings.gradle.kts
View File

@ -8,7 +8,6 @@ pluginManagement {
val toolsVersion: String by extra
repositories {
mavenLocal()
maven("https://repo.kotlin.link")
mavenCentral()
gradlePluginPortal()
@ -19,6 +18,7 @@ pluginManagement {
id("ru.mipt.npm.gradle.mpp") version toolsVersion
id("ru.mipt.npm.gradle.jvm") version toolsVersion
id("ru.mipt.npm.gradle.js") version toolsVersion
id("space.kscience.snark") version "0.1.0-dev-1"
}
}
@ -27,7 +27,6 @@ dependencyResolutionManagement {
val toolsVersion: String by extra
repositories {
mavenLocal()
maven("https://repo.kotlin.link")
mavenCentral()
}
@ -37,4 +36,9 @@ dependencyResolutionManagement {
from("ru.mipt.npm:version-catalog:$toolsVersion")
}
}
}
val snarkProjectDirectory = File("../snark")
if(snarkProjectDirectory.exists()) {
includeBuild("../snark")
}

4
src/main/kotlin/html5up/forty/common.kt
View File

@ -1,7 +1,7 @@
package html5up.forty
import kotlinx.html.*
import space.kscience.snark.PageBuilder
import space.kscience.snark.html.Page
internal fun FlowContent.fortyMenu() {
@ -200,7 +200,7 @@ internal fun FlowContent.fortyFooter() {
}
}
context(PageBuilder) internal fun BODY.fortyScripts() {
context(Page) internal fun BODY.fortyScripts() {
script {
src = resolveRef("assets/js/jquery.min.js")
}

4
src/main/kotlin/html5up/forty/landing.kt
View File

@ -1,9 +1,9 @@
package html5up.forty
import kotlinx.html.*
import space.kscience.snark.PageBuilder
import space.kscience.snark.html.Page
context(PageBuilder) internal fun HTML.landing(){
context(Page) internal fun HTML.landing(){
head {
title {
}

4
src/main/kotlin/html5up/forty/page.kt
View File

@ -1,9 +1,9 @@
package html5up.forty
import kotlinx.html.*
import space.kscience.snark.PageBuilder
import space.kscience.snark.html.Page
context(PageBuilder) internal fun HTML.fortyPage(){
context(Page) internal fun HTML.fortyPage(){
head {
title {
}

4
src/main/kotlin/ru/mipt/spc/Application.kt
View File

@ -7,8 +7,8 @@ import kotlinx.html.CommonAttributeGroupFacade
import kotlinx.html.style
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.fetch
import space.kscience.snark.SnarkPlugin
import space.kscience.snark.snarkSite
import space.kscience.snark.html.SnarkPlugin
import space.kscience.snark.ktor.snarkSite
import java.net.URI
import java.nio.file.FileSystems
import java.nio.file.Files

22
src/main/kotlin/ru/mipt/spc/master.kt
View File

@ -14,6 +14,8 @@ import space.kscience.dataforge.names.asName
import space.kscience.dataforge.names.plus
import space.kscience.dataforge.names.withIndex
import space.kscience.snark.*
import space.kscience.snark.html.*
import space.kscience.snark.html.Page
import java.nio.file.Path
import kotlin.collections.component1
import kotlin.collections.component2
@ -35,14 +37,14 @@ import kotlin.collections.set
private val HtmlData.imagePath: String? get() = meta["image"]?.string ?: meta["image.path"].string
private val HtmlData.name: String get() = meta["name"].string ?: error("Name not found")
context(PageBuilder) class MagProgSection(
context(Page) class MagProgSection(
val id: String,
val title: String,
val style: String,
val content: FlowContent.() -> Unit,
)
context(PageBuilder) private fun wrapSection(
context(Page) private fun wrapSection(
id: String,
title: String,
sectionContent: FlowContent.() -> Unit,
@ -53,7 +55,7 @@ context(PageBuilder) private fun wrapSection(
}
}
context(PageBuilder) private fun wrapSection(
context(Page) private fun wrapSection(
block: HtmlData,
idOverride: String? = null,
): MagProgSection = wrapSection(
@ -71,7 +73,7 @@ private val PROGRAM_PATH: Name = CONTENT_NODE_NAME + "program"
private val RECOMMENDED_COURSES_PATH: Name = CONTENT_NODE_NAME + "recommendedCourses"
private val PARTNERS_PATH: Name = CONTENT_NODE_NAME + "partners"
context(PageBuilder) private fun FlowContent.programSection() {
context(Page) private fun FlowContent.programSection() {
val programBlock = data.resolveHtml(PROGRAM_PATH)!!
val recommendedBlock = data.resolveHtml(RECOMMENDED_COURSES_PATH)!!
div("inner") {
@ -88,7 +90,7 @@ context(PageBuilder) private fun FlowContent.programSection() {
}
}
context(PageBuilder) private fun FlowContent.partners() {
context(Page) private fun FlowContent.partners() {
//val partnersData: Meta = resolve<Any>(PARTNERS_PATH)?.meta ?: Meta.EMPTY
val partnersData: Meta = runBlocking { data.getByType<Meta>(PARTNERS_PATH)?.await() } ?: Meta.EMPTY
div("inner") {
@ -118,7 +120,7 @@ context(PageBuilder) private fun FlowContent.partners() {
// val photo: String? by meta.string()
//}
context(PageBuilder) private fun FlowContent.team() {
context(Page) private fun FlowContent.team() {
val team = data.findByContentType("magprog_team").values.sortedBy { it.order }
div("inner") {
@ -173,7 +175,7 @@ context(PageBuilder) private fun FlowContent.team() {
// }
}
context(PageBuilder) private fun FlowContent.mentors() {
context(Page) private fun FlowContent.mentors() {
val mentors = data.findByContentType("magprog_mentor").entries.sortedBy { it.value.id }
div("inner") {
@ -211,7 +213,7 @@ context(PageBuilder) private fun FlowContent.mentors() {
}
}
context(PageBuilder) internal fun HTML.magProgHead(title: String) {
context(Page) internal fun HTML.magProgHead(title: String) {
head {
this.title = title
meta {
@ -238,7 +240,7 @@ context(PageBuilder) internal fun HTML.magProgHead(title: String) {
}
}
context(PageBuilder) internal fun BODY.magProgFooter() {
context(Page) internal fun BODY.magProgFooter() {
footer("wrapper style1-alt") {
id = "footer"
div("inner") {
@ -277,7 +279,7 @@ context(PageBuilder) internal fun BODY.magProgFooter() {
}
}
private val HtmlData.mentorPageId get() = "mentor-${id}"
context(SnarkContext) private val HtmlData.mentorPageId get() = "mentor-${id}"
internal fun SiteBuilder.spcMaster(dataPath: Path, prefix: Name = "magprog".asName()) {

4
src/main/kotlin/ru/mipt/spc/spcCollection.kt
View File

@ -10,12 +10,12 @@ import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.parseAsName
import space.kscience.dataforge.names.withIndex
import space.kscience.dataforge.values.string
import space.kscience.snark.*
import space.kscience.snark.html.*
import kotlin.collections.component1
import kotlin.collections.component2
import kotlin.collections.set
context(PageBuilder) private fun FlowContent.spcSpotlightContent(
context(Page) private fun FlowContent.spcSpotlightContent(
landing: HtmlData,
content: Map<Name, HtmlData>,
) {

6
src/main/kotlin/ru/mipt/spc/spcHome.kt
View File

@ -10,12 +10,12 @@ import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.asName
import space.kscience.dataforge.names.startsWith
import space.kscience.dataforge.values.string
import space.kscience.snark.*
import space.kscience.snark.html.*
import java.nio.file.Path
import kotlin.reflect.typeOf
context(PageBuilder) internal fun HTML.spcPageContent(
context(Page) internal fun HTML.spcPageContent(
meta: Meta,
title: String = meta["title"].string ?: SPC_TITLE,
fragment: FlowContent.() -> Unit,
@ -65,7 +65,7 @@ internal val FortyDataRenderer: DataRenderer = { name, data ->
}
context(PageBuilder) private fun HTML.spcHome() {
context(Page) private fun HTML.spcHome() {
spcHead()
body("is-preload") {
wrapper {

14
src/main/kotlin/ru/mipt/spc/spcMisc.kt
View File

@ -1,14 +1,14 @@
package ru.mipt.spc
import kotlinx.html.*
import space.kscience.snark.PageBuilder
import space.kscience.snark.homeRef
import space.kscience.snark.resolvePageRef
import space.kscience.snark.html.Page
import space.kscience.snark.html.homeRef
import space.kscience.snark.html.resolvePageRef
internal const val SPC_TITLE = "Scientific Programming Centre"
context(PageBuilder) internal fun HTML.spcHead(title: String = SPC_TITLE) {
context(Page) internal fun HTML.spcHead(title: String = SPC_TITLE) {
head {
title {
+title
@ -27,7 +27,7 @@ context(PageBuilder) internal fun HTML.spcHead(title: String = SPC_TITLE) {
}
}
context(PageBuilder) internal fun FlowContent.spcHomeMenu() {
context(Page) internal fun FlowContent.spcHomeMenu() {
nav {
id = "menu"
ul("links") {
@ -79,7 +79,7 @@ context(PageBuilder) internal fun FlowContent.spcHomeMenu() {
}
}
context(PageBuilder) internal fun FlowContent.spcFooter() {
context(Page) internal fun FlowContent.spcFooter() {
footer {
id = "footer"
div("inner") {
@ -129,7 +129,7 @@ context(PageBuilder) internal fun FlowContent.spcFooter() {
}
}
context(PageBuilder) internal fun FlowContent.wrapper(contentBody: FlowContent.() -> Unit) {
context(Page) internal fun FlowContent.wrapper(contentBody: FlowContent.() -> Unit) {
div {
id = "wrapper"
// Header

6
src/main/kotlin/ru/mipt/spc/staticRender.kt
View File

@ -2,13 +2,13 @@ package ru.mipt.spc
import space.kscience.dataforge.context.Global
import space.kscience.dataforge.context.fetch
import space.kscience.snark.SnarkPlugin
import space.kscience.snark.static
import space.kscience.snark.html.SnarkPlugin
import space.kscience.snark.html.renderStatic
import java.nio.file.Path
import kotlin.io.path.toPath
fun main() {
Global.fetch(SnarkPlugin).static(Path.of("build/out")) {
Global.fetch(SnarkPlugin).renderStatic(Path.of("build/out")) {
spcHome(rootPath = javaClass.getResource("/home")!!.toURI().toPath())
spcMaster(dataPath = javaClass.getResource("/magprog")!!.toURI().toPath())
}

34
src/main/kotlin/space/kscience/snark/HtmlData.kt
View File

@ -1,34 +0,0 @@
package space.kscience.snark
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runBlocking
import kotlinx.html.FlowContent
import kotlinx.html.TagConsumer
import space.kscience.dataforge.data.Data
import space.kscience.dataforge.data.await
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.int
import space.kscience.dataforge.meta.string
//TODO replace by VisionForge type
//typealias HtmlFragment = context(PageBuilder, TagConsumer<*>) () -> Unit
fun interface HtmlFragment {
fun TagConsumer<*>.renderFragment(page: PageBuilder)
//TODO move pageBuilder to a context receiver after KT-52967 is fixed
}
typealias HtmlData = Data<HtmlFragment>
//fun HtmlData(meta: Meta, content: context(PageBuilder) TagConsumer<*>.() -> Unit): HtmlData =
// Data(HtmlFragment(content), meta)
internal val HtmlData.id: String get() = meta["id"]?.string ?: "block[${hashCode()}]"
internal val HtmlData.language: String? get() = meta["language"].string?.lowercase()
internal val HtmlData.order: Int? get() = meta["order"]?.int
context(PageBuilder) fun FlowContent.htmlData(data: HtmlData) = runBlocking(Dispatchers.IO) {
with(data.await()) { consumer.renderFragment(this@PageBuilder) }
}

135
src/main/kotlin/space/kscience/snark/KtorSiteBuilder.kt
View File

@ -1,135 +0,0 @@
package space.kscience.snark
import io.ktor.http.URLBuilder
import io.ktor.http.URLProtocol
import io.ktor.server.application.Application
import io.ktor.server.application.call
import io.ktor.server.html.respondHtml
import io.ktor.server.http.content.*
import io.ktor.server.plugins.origin
import io.ktor.server.request.host
import io.ktor.server.request.port
import io.ktor.server.routing.Route
import io.ktor.server.routing.createRouteFromPath
import io.ktor.server.routing.get
import io.ktor.server.routing.routing
import kotlinx.html.HTML
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.withDefault
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.cutLast
import space.kscience.dataforge.names.endsWith
import java.nio.file.Path
import kotlin.contracts.InvocationKind
import kotlin.contracts.contract
@PublishedApi
internal class KtorSiteBuilder(
override val snark: SnarkPlugin,
override val data: DataTree<*>,
override val meta: Meta,
private val baseUrl: String,
private val ktorRoute: Route,
) : SiteBuilder {
override fun assetFile(remotePath: String, file: Path) {
ktorRoute.file(remotePath, file.toFile())
}
override fun assetDirectory(remotePath: String, directory: Path) {
ktorRoute.static(remotePath) {
files(directory.toFile())
}
}
private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) {
ref
} else if (ref.isEmpty()) {
baseUrl
} else {
"${baseUrl.removeSuffix("/")}/$ref"
}
inner class KtorPageBuilder(
val pageBaseUrl: String,
override val meta: Meta = this@KtorSiteBuilder.meta,
) : PageBuilder {
override val snark: SnarkPlugin get() = this@KtorSiteBuilder.snark
override val data: DataTree<*> get() = this@KtorSiteBuilder.data
override fun resolveRef(ref: String): String = resolveRef(pageBaseUrl, ref)
override fun resolvePageRef(pageName: Name): String = if (pageName.endsWith(SiteBuilder.INDEX_PAGE_TOKEN)) {
resolveRef(pageName.cutLast().toWebPath())
} else {
resolveRef(pageName.toWebPath())
}
}
override fun page(route: Name, content: context(PageBuilder, HTML)() -> Unit) {
ktorRoute.get(route.toWebPath()) {
call.respondHtml {
val request = call.request
//substitute host for url for backwards calls
val url = URLBuilder(baseUrl).apply {
protocol = URLProtocol.createOrDefault(request.origin.scheme)
host = request.host()
port = request.port()
}
val pageBuilder = KtorPageBuilder(url.buildString())
content(pageBuilder, this)
}
}
}
override fun route(
routeName: Name,
dataOverride: DataTree<*>?,
metaOverride: Meta?,
setAsRoot: Boolean,
): SiteBuilder = KtorSiteBuilder(
snark = snark,
data = dataOverride ?: data,
meta = metaOverride?.withDefault(meta) ?: meta,
baseUrl = if (setAsRoot) {
resolveRef(baseUrl, routeName.toWebPath())
} else {
baseUrl
},
ktorRoute = ktorRoute.createRouteFromPath(routeName.toWebPath())
)
override fun assetResourceFile(remotePath: String, resourcesPath: String) {
ktorRoute.resource(resourcesPath, resourcesPath)
}
override fun assetResourceDirectory(resourcesPath: String) {
ktorRoute.resources(resourcesPath)
}
}
inline fun Route.snarkSite(
snark: SnarkPlugin,
data: DataTree<*>,
meta: Meta = data.meta,
block: SiteBuilder.() -> Unit,
) {
contract {
callsInPlace(block, InvocationKind.EXACTLY_ONCE)
}
block(KtorSiteBuilder(snark, data, meta, "", this@snarkSite))
}
fun Application.snarkSite(
snark: SnarkPlugin,
data: DataTree<*> = DataTree.empty(),
meta: Meta = data.meta,
block: SiteBuilder.() -> Unit,
) {
routing {
snarkSite(snark, data, meta, block)
}
}

65
src/main/kotlin/space/kscience/snark/PageBuilder.kt
View File

@ -1,65 +0,0 @@
package space.kscience.snark
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.ContextAware
import space.kscience.dataforge.data.*
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string
import space.kscience.dataforge.names.*
internal fun Name.toWebPath() = tokens.joinToString(separator = "/"){
if (it.hasIndex()) {
"${it.body}[${it.index}]"
} else {
it.body
}
}
interface PageBuilder : ContextAware {
val snark: SnarkPlugin
override val context: Context get() = snark.context
val data: DataTree<*>
val meta: Meta
fun resolveRef(ref: String): String
fun resolvePageRef(pageName: Name): String
}
fun PageBuilder.resolvePageRef(pageName: String) = resolvePageRef(pageName.parseAsName())
val PageBuilder.homeRef get() = resolvePageRef(SiteBuilder.INDEX_PAGE_TOKEN.asName())
/**
* Resolve a Html builder by its full name
*/
fun DataTree<*>.resolveHtml(name: Name): HtmlData? {
val resolved = (getByType<HtmlFragment>(name) ?: getByType<HtmlFragment>(name + SiteBuilder.INDEX_PAGE_TOKEN))
return resolved?.takeIf {
it.published //TODO add language confirmation
}
}
/**
* Find all Html blocks using given name/meta filter
*/
fun DataTree<*>.resolveAllHtml(predicate: (name: Name, meta: Meta) -> Boolean): Map<Name, HtmlData> =
filterByType<HtmlFragment> { name, meta ->
predicate(name, meta)
&& meta["published"].string != "false"
//TODO add language confirmation
}.asSequence().associate { it.name to it.data }
fun DataTree<*>.findByContentType(contentType: String, baseName: Name = Name.EMPTY) = resolveAllHtml { name, meta ->
name.startsWith(baseName) && meta["content_type"].string == contentType
}
internal val Data<*>.published: Boolean get() = meta["published"].string != "false"

97
src/main/kotlin/space/kscience/snark/SiteBuilder.kt
View File

@ -1,97 +0,0 @@
package space.kscience.snark
import kotlinx.html.HTML
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.context.ContextAware
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.data.DataTreeItem
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.NameToken
import space.kscience.dataforge.names.parseAsName
import java.nio.file.Path
import kotlin.reflect.KType
import kotlin.reflect.typeOf
/**
* An abstraction, which is used to render sites to the different rendering engines
*/
interface SiteBuilder : ContextAware {
val data: DataTree<*>
val snark: SnarkPlugin
override val context: Context get() = snark.context
val meta: Meta
fun assetFile(remotePath: String, file: Path)
fun assetDirectory(remotePath: String, directory: Path)
fun assetResourceFile(remotePath: String, resourcesPath: String)
fun assetResourceDirectory(resourcesPath: String)
fun page(route: Name = Name.EMPTY, content: context(PageBuilder, HTML) () -> Unit)
/**
* Create a route with optional data tree override. For example one could use a subtree of the initial tree.
* By default, the same data tree is used for route
*/
fun route(
routeName: Name,
dataOverride: DataTree<*>? = null,
metaOverride: Meta? = null,
setAsRoot: Boolean = false,
): SiteBuilder
companion object {
val INDEX_PAGE_TOKEN: NameToken = NameToken("index")
val UP_PAGE_TOKEN: NameToken = NameToken("..")
}
}
public inline fun SiteBuilder.route(
route: Name,
dataOverride: DataTree<*>? = null,
metaOverride: Meta? = null,
setAsRoot: Boolean = false,
block: SiteBuilder.() -> Unit,
) {
route(route, dataOverride, metaOverride, setAsRoot).apply(block)
}
public inline fun SiteBuilder.route(
route: String,
dataOverride: DataTree<*>? = null,
metaOverride: Meta? = null,
setAsRoot: Boolean = false,
block: SiteBuilder.() -> Unit,
) {
route(route.parseAsName(), dataOverride, metaOverride, setAsRoot).apply(block)
}
///**
// * Create a stand-alone site at a given node
// */
//public fun SiteBuilder.site(route: Name, dataRoot: DataTree<*>, block: SiteBuilder.() -> Unit) {
// val mountedData = data.copy(
// data = dataRoot,
// baseUrlPath = data.resolveRef(route.tokens.joinToString(separator = "/")),
// meta = Laminate(dataRoot.meta, data.meta) //layering dataRoot meta over existing data
// )
// route(route) {
// withData(mountedData).block()
// }
//}
//TODO move to DF
fun DataTree.Companion.empty(meta: Meta = Meta.EMPTY) = object : DataTree<Any> {
override val items: Map<NameToken, DataTreeItem<Any>> get() = emptyMap()
override val dataType: KType get() = typeOf<Any>()
override val meta: Meta get() = meta
}

150
src/main/kotlin/space/kscience/snark/SiteLayout.kt
View File

@ -1,150 +0,0 @@
package space.kscience.snark
import kotlinx.html.body
import kotlinx.html.head
import kotlinx.html.title
import space.kscience.dataforge.data.Data
import space.kscience.dataforge.data.DataTreeItem
import space.kscience.dataforge.data.getItem
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.getIndexed
import space.kscience.dataforge.meta.string
import space.kscience.dataforge.misc.Type
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.NameToken
import space.kscience.dataforge.names.asName
import space.kscience.dataforge.names.parseAsName
import space.kscience.snark.SiteLayout.Companion.ASSETS_KEY
import space.kscience.snark.SiteLayout.Companion.INDEX_PAGE_TOKEN
import space.kscience.snark.SiteLayout.Companion.LAYOUT_KEY
import java.nio.file.Path
import kotlin.reflect.typeOf
internal fun SiteBuilder.assetsFrom(rootMeta: Meta) {
rootMeta.getIndexed("resource".asName()).forEach { (_, meta) ->
val path by meta.string()
val remotePath by meta.string()
path?.let { resourcePath ->
//If remote path provided, use a single resource
remotePath?.let {
assetResourceFile(it, resourcePath)
return@forEach
}
//otherwise use package resources
assetResourceDirectory(resourcePath)
}
}
rootMeta.getIndexed("file".asName()).forEach { (_, meta) ->
val remotePath by meta.string { error("File remote path is not provided") }
val path by meta.string { error("File path is not provided") }
assetFile(remotePath, Path.of(path))
}
rootMeta.getIndexed("directory".asName()).forEach { (_, meta) ->
val path by meta.string { error("Directory path is not provided") }
assetDirectory("", Path.of(path))
}
}
typealias DataRenderer = SiteBuilder.(name: Name, data: Data<Any>) -> Unit
/**
* Recursively renders the data items in [data]. If [LAYOUT_KEY] is defined in an item, use it to load
* layout from the context, otherwise render children nodes as name segments and individual data items using [dataRenderer].
*/
fun SiteBuilder.pages(
data: DataTreeItem<*>,
dataRenderer: DataRenderer = SiteLayout.defaultDataRenderer,
) {
val layoutMeta = data.meta[LAYOUT_KEY]
if (layoutMeta != null) {
//use layout if it is defined
snark.layout(layoutMeta).render(data)
} else {
when (data) {
is DataTreeItem.Node -> {
data.tree.items.forEach { (token, item) ->
//Don't apply index token
if (token == INDEX_PAGE_TOKEN) {
pages(item, dataRenderer)
} else if (item is DataTreeItem.Leaf) {
dataRenderer(this, token.asName(), item.data)
} else {
route(token.asName()) {
pages(item, dataRenderer)
}
}
}
}
is DataTreeItem.Leaf -> {
dataRenderer.invoke(this, Name.EMPTY, data.data)
}
}
data.meta[ASSETS_KEY]?.let {
assetsFrom(it)
}
}
//TODO watch for changes
}
/**
* Render all pages in a node with given name
*/
fun SiteBuilder.pages(
dataPath: Name,
remotePath: Name = dataPath,
dataRenderer: DataRenderer = SiteLayout.defaultDataRenderer,
) {
val item = data.getItem(dataPath) ?: error("No data found by name $dataPath")
route(remotePath) {
pages(item, dataRenderer)
}
}
fun SiteBuilder.pages(
dataPath: String,
remotePath: Name = dataPath.parseAsName(),
dataRenderer: DataRenderer = SiteLayout.defaultDataRenderer,
) {
pages(dataPath.parseAsName(), remotePath, dataRenderer = dataRenderer)
}
@Type(SiteLayout.TYPE)
fun interface SiteLayout {
context(SiteBuilder) fun render(item: DataTreeItem<*>)
companion object {
const val TYPE = "snark.layout"
const val LAYOUT_KEY = "layout"
const val ASSETS_KEY = "assets"
val INDEX_PAGE_TOKEN = NameToken("index")
val defaultDataRenderer: SiteBuilder.(name: Name, data: Data<*>) -> Unit = { name: Name, data: Data<*> ->
if (data.type == typeOf<HtmlData>()) {
page(name) {
head {
title = data.meta["title"].string ?: "Untitled page"
}
body {
@Suppress("UNCHECKED_CAST")
htmlData(data as HtmlData)
}
}
}
}
}
}
object DefaultSiteLayout : SiteLayout {
context(SiteBuilder) override fun render(item: DataTreeItem<*>) {
pages(item)
}
}

146
src/main/kotlin/space/kscience/snark/SnarkPlugin.kt
View File

@ -1,146 +0,0 @@
package space.kscience.snark
import io.ktor.util.extension
import io.ktor.utils.io.core.Input
import io.ktor.utils.io.core.readBytes
import space.kscience.dataforge.context.*
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.io.IOReader
import space.kscience.dataforge.io.JsonMetaFormat
import space.kscience.dataforge.io.asBinary
import space.kscience.dataforge.io.readWith
import space.kscience.dataforge.io.yaml.YamlMetaFormat
import space.kscience.dataforge.io.yaml.YamlPlugin
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import space.kscience.dataforge.meta.string
import space.kscience.dataforge.misc.DFExperimental
import space.kscience.dataforge.misc.Type
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.asName
import space.kscience.dataforge.names.parseAsName
import space.kscience.dataforge.workspace.FileData
import space.kscience.dataforge.workspace.readDataDirectory
import java.nio.file.Path
import kotlin.reflect.KClass
import kotlin.reflect.KType
import kotlin.reflect.typeOf
/**
* A parser of binary content including priority flag and file extensions
*/
@Type(SnarkParser.TYPE)
interface SnarkParser<out R> {
val type: KType
val fileExtensions: Set<String>
val priority: Int get() = DEFAULT_PRIORITY
fun parse(context: Context, meta: Meta, bytes: ByteArray): R
fun reader(context: Context, meta: Meta) = object : IOReader<R> {
override val type: KType get() = this@SnarkParser.type
override fun readObject(input: Input): R = parse(context, meta, input.readBytes())
}
companion object {
const val TYPE = "snark.parser"
const val DEFAULT_PRIORITY = 10
}
}
@PublishedApi
internal class SnarkParserWrapper<R : Any>(
val reader: IOReader<R>,
override val type: KType,
override val fileExtensions: Set<String>,
) : SnarkParser<R> {
override fun parse(context: Context, meta: Meta, bytes: ByteArray): R = bytes.asBinary().readWith(reader)
}
/**
* Create a generic parser from reader
*/
@Suppress("FunctionName")
inline fun <reified R : Any> SnarkParser(
reader: IOReader<R>,
vararg fileExtensions: String,
): SnarkParser<R> = SnarkParserWrapper(reader, typeOf<R>(), fileExtensions.toSet())
@OptIn(DFExperimental::class)
class SnarkPlugin : AbstractPlugin() {
private val yaml by require(YamlPlugin)
val io get() = yaml.io
override val tag: PluginTag get() = Companion.tag
private val parsers: Map<Name, SnarkParser<Any>> by lazy {
context.gather(SnarkParser.TYPE, true)
}
private val layouts: Map<Name, SiteLayout> by lazy {
context.gather(SiteLayout.TYPE, true)
}
private val textTransformations: Map<Name, TextTransformation> by lazy {
context.gather(TextTransformation.TYPE, true)
}
fun readDirectory(path: Path): DataTree<Any> = io.readDataDirectory(path) { dataPath, meta ->
val fileExtension = meta[FileData.META_FILE_EXTENSION_KEY].string ?: dataPath.extension
val parser: SnarkParser<Any> = parsers.values.filter { parser ->
fileExtension in parser.fileExtensions
}.maxByOrNull {
it.priority
} ?: run {
logger.warn { "The parser is not found for file $dataPath with meta $meta" }
byteArraySnarkParser
}
parser.reader(context, meta)
}
internal fun layout(layoutMeta: Meta): SiteLayout {
val layoutName = layoutMeta.string
?: layoutMeta["name"].string ?: error("Layout name not defined in $layoutMeta")
return layouts[layoutName.parseAsName()] ?: error("Layout with name $layoutName not found in $this")
}
internal fun textTransformation(transformationMeta: Meta): TextTransformation {
val transformationName = transformationMeta.string
?: transformationMeta["name"].string ?: error("Transformation name not defined in $transformationMeta")
return textTransformations[transformationName.parseAsName()]
?: error("Text transformation with name $transformationName not found in $this")
}
override fun content(target: String): Map<Name, Any> = when (target) {
SnarkParser.TYPE -> mapOf(
"html".asName() to SnarkHtmlParser,
"markdown".asName() to SnarkMarkdownParser,
"json".asName() to SnarkParser(JsonMetaFormat, "json"),
"yaml".asName() to SnarkParser(YamlMetaFormat, "yaml", "yml"),
"png".asName() to SnarkParser(ImageIOReader, "png"),
"jpg".asName() to SnarkParser(ImageIOReader, "jpg", "jpeg"),
"gif".asName() to SnarkParser(ImageIOReader, "gif"),
)
TextTransformation.TYPE -> mapOf(
"basic".asName() to BasicTextTransformation
)
else -> super.content(target)
}
companion object : PluginFactory<SnarkPlugin> {
override val tag: PluginTag = PluginTag("snark")
override val type: KClass<out SnarkPlugin> = SnarkPlugin::class
override fun build(context: Context, meta: Meta): SnarkPlugin = SnarkPlugin()
private val byteArrayIOReader = IOReader {
readBytes()
}
private val byteArraySnarkParser = SnarkParser(byteArrayIOReader)
}
}

68
src/main/kotlin/space/kscience/snark/SnarkTextParser.kt
View File

@ -1,68 +0,0 @@
package space.kscience.snark
import io.ktor.util.asStream
import io.ktor.utils.io.core.Input
import kotlinx.html.div
import kotlinx.html.unsafe
import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
import org.intellij.markdown.html.HtmlGenerator
import org.intellij.markdown.parser.MarkdownParser
import space.kscience.dataforge.context.Context
import space.kscience.dataforge.io.IOReader
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.get
import java.awt.image.BufferedImage
import javax.imageio.ImageIO
import kotlin.reflect.KType
import kotlin.reflect.typeOf
abstract class SnarkTextParser<R> : SnarkParser<R> {
abstract fun parseText(text: String, meta: Meta): R
override fun parse(context: Context, meta: Meta, bytes: ByteArray): R =
parseText(bytes.decodeToString(), meta)
fun transformText(text: String, meta: Meta, page: PageBuilder): String =
meta[TextTransformation.TEXT_TRANSFORMATION_KEY]?.let {
with(page){ page.snark.textTransformation(it).transform(text)}
} ?: text
}
internal object SnarkHtmlParser : SnarkTextParser<HtmlFragment>() {
override val fileExtensions: Set<String> = setOf("html")
override val type: KType = typeOf<HtmlFragment>()
override fun parseText(text: String, meta: Meta): HtmlFragment = HtmlFragment { page ->
div {
unsafe { +transformText(text, meta, page) }
}
}
}
internal object SnarkMarkdownParser : SnarkTextParser<HtmlFragment>() {
override val fileExtensions: Set<String> = setOf("markdown", "mdown", "mkdn", "mkd", "md")
override val type: KType = typeOf<HtmlFragment>()
private val markdownFlavor = CommonMarkFlavourDescriptor()
private val markdownParser = MarkdownParser(markdownFlavor)
override fun parseText(text: String, meta: Meta): HtmlFragment {
val parsedTree = markdownParser.buildMarkdownTreeFromString(text)
val htmlString = HtmlGenerator(text, parsedTree, markdownFlavor).generateHtml()
return HtmlFragment { page ->
div {
unsafe {
+SnarkHtmlParser.transformText(htmlString, meta, page)
}
}
}
}
}
internal object ImageIOReader : IOReader<BufferedImage> {
override val type: KType get() = typeOf<BufferedImage>()
override fun readObject(input: Input): BufferedImage = ImageIO.read(input.asStream())
}

126
src/main/kotlin/space/kscience/snark/StaticSiteBuilder.kt
View File

@ -1,126 +0,0 @@
package space.kscience.snark
import kotlinx.html.HTML
import kotlinx.html.html
import kotlinx.html.stream.createHTML
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.withDefault
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.isEmpty
import java.nio.file.Files
import java.nio.file.Path
import kotlin.contracts.InvocationKind
import kotlin.contracts.contract
import kotlin.io.path.*
internal class StaticSiteBuilder(
override val snark: SnarkPlugin,
override val data: DataTree<*>,
override val meta: Meta,
private val baseUrl: String,
private val path: Path,
) : SiteBuilder {
private fun Path.copyRecursively(target: Path) {
Files.walk(this).forEach { source: Path ->
val destination: Path = target.resolve(source.relativeTo(this))
if (!destination.isDirectory()) {
//avoid re-creating directories
source.copyTo(destination, true)
}
}
}
override fun assetFile(remotePath: String, file: Path) {
val targetPath = path.resolve(remotePath)
targetPath.parent.createDirectories()
file.copyTo(targetPath, true)
}
override fun assetDirectory(remotePath: String, directory: Path) {
val targetPath = path.resolve(remotePath)
targetPath.parent.createDirectories()
directory.copyRecursively(targetPath)
}
override fun assetResourceFile(remotePath: String, resourcesPath: String) {
val targetPath = path.resolve(remotePath)
targetPath.parent.createDirectories()
javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyTo(targetPath, true)
}
override fun assetResourceDirectory(resourcesPath: String) {
path.parent.createDirectories()
javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyRecursively(path)
}
private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) {
ref
} else if (ref.isEmpty()) {
baseUrl
} else {
"${baseUrl.removeSuffix("/")}/$ref"
}
inner class StaticPageBuilder : PageBuilder {
override val data: DataTree<*> get() = this@StaticSiteBuilder.data
override val meta: Meta get() = this@StaticSiteBuilder.meta
override val snark: SnarkPlugin get() = this@StaticSiteBuilder.snark
override fun resolveRef(ref: String): String = resolveRef(baseUrl, ref)
override fun resolvePageRef(pageName: Name): String = resolveRef(
pageName.toWebPath() + ".html"
)
}
override fun page(route: Name, content: context(PageBuilder, HTML) () -> Unit) {
val htmlBuilder = createHTML()
htmlBuilder.html {
content(StaticPageBuilder(), this)
}
val newPath = if (route.isEmpty()) {
path.resolve("index.html")
} else {
path.resolve(route.toWebPath() + ".html")
}
newPath.parent.createDirectories()
newPath.writeText(htmlBuilder.finalize())
}
override fun route(
routeName: Name,
dataOverride: DataTree<*>?,
metaOverride: Meta?,
setAsRoot: Boolean,
): SiteBuilder = StaticSiteBuilder(
snark = snark,
data = dataOverride ?: data,
meta = metaOverride?.withDefault(meta) ?: meta,
baseUrl = if (setAsRoot) {
resolveRef(baseUrl, routeName.toWebPath())
} else {
baseUrl
},
path = path.resolve(routeName.toWebPath())
)
}
fun SnarkPlugin.static(
outputPath: Path,
data: DataTree<*> = DataTree.empty(),
siteUrl: String = outputPath.absolutePathString().replace("\\", "/"),
block: SiteBuilder.() -> Unit,
) {
contract {
callsInPlace(block, InvocationKind.EXACTLY_ONCE)
}
StaticSiteBuilder(this, data, meta, siteUrl, outputPath).block()
}

38
src/main/kotlin/space/kscience/snark/TextTransformation.kt
View File

@ -1,38 +0,0 @@
package space.kscience.snark
import space.kscience.dataforge.misc.Type
import space.kscience.dataforge.names.NameToken
@Type(TextTransformation.TYPE)
fun interface TextTransformation {
context(PageBuilder) fun transform(text: String): String
companion object {
const val TYPE = "snark.textTransformation"
val TEXT_TRANSFORMATION_KEY = NameToken("transformation")
}
}
object BasicTextTransformation : TextTransformation {
private val regex = "\\\$\\{(\\w*)(?>\\(\"(.*)\"\\))?\\}".toRegex()
context(PageBuilder) override fun transform(text: String): String {
return text.replace(regex) { match ->
when (match.groups[1]!!.value) {
"homeRef" -> homeRef
"resolveRef" -> {
val refString = match.groups[2]?.value ?: error("resolveRef requires a string (quoted) argument")
resolveRef(refString)
}
"resolvePageRef" -> {
val refString = match.groups[2]?.value ?: error("resolvePageRef requires a string (quoted) argument")
resolvePageRef(refString)
}
else -> match.value
}
}
}
}