SPC/snark
5
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

WIP - working on file-less trees

Browse Source
This commit is contained in:
Alexander Nozik 2023-03-13 16:24:50 +03:00
parent f52e1203c3
commit 941da6fab7
13 changed files with 171 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
build.gradle.kts
View File

@ -1,5 +1,3 @@
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
id("space.kscience.gradle.project")
}
@ -8,16 +6,13 @@ allprojects {
group = "space.kscience"
version = "0.1.0-dev-1"
if (name != "snark-gradle-plugin") {
tasks.withType<KotlinCompile> {
kotlinOptions {
freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers"
}
}
repositories {
mavenCentral()
mavenLocal()
}
}
val dataforgeVersion by extra("0.6.0-dev-15")
val dataforgeVersion by extra("0.6.1-dev-4")
ksciencePublish {
github("SciProgCentre", "snark")

2
gradle.properties
View File

@ -1,3 +1,3 @@
kotlin.code.style=official
toolsVersion=0.13.3-kotlin-1.7.20
toolsVersion=0.14.2-kotlin-1.8.10

2
settings.gradle.kts
View File

@ -31,7 +31,7 @@ dependencyResolutionManagement {
}
versionCatalogs {
create("npmlibs") {
create("spclibs") {
from("space.kscience:version-catalog:$toolsVersion")
}
}

13
snark-core/build.gradle.kts
View File

@ -5,12 +5,11 @@ plugins{
val dataforgeVersion: String by rootProject.extra
kotlin{
sourceSets{
commonMain{
dependencies{
api("space.kscience:dataforge-workspace:$dataforgeVersion")
}
}
kscience{
jvm()
js()
dependencies{
api("space.kscience:dataforge-workspace:$dataforgeVersion")
}
useContextReceivers()
}

3
snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkParser.kt
View File

@ -22,9 +22,10 @@ public interface SnarkParser<out R> {
public val priority: Int get() = DEFAULT_PRIORITY
//TODO use Binary instead of ByteArray
public fun parse(context: Context, meta: Meta, bytes: ByteArray): R
public fun reader(context: Context, meta: Meta): IOReader<R> = object : IOReader<R> {
public fun asReader(context: Context, meta: Meta): IOReader<R> = object : IOReader<R> {
override val type: KType get() = this@SnarkParser.type
override fun readObject(input: Input): R = parse(context, meta, input.readBytes())

2
snark-gradle-plugin/build.gradle.kts
View File

@ -10,7 +10,7 @@ repositories{
}
dependencies{
implementation(npmlibs.kotlin.gradle)
implementation(spclibs.kotlin.gradle)
implementation("com.github.mwiede:jsch:0.2.1")
}

6
snark-html/build.gradle.kts
View File

@ -6,6 +6,10 @@ plugins {
val dataforgeVersion: String by rootProject.extra
val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
kscience{
useContextReceivers()
}
dependencies {
api(projects.snarkCore)
@ -15,7 +19,7 @@ dependencies {
api("io.ktor:ktor-utils:$ktorVersion")
api("space.kscience:dataforge-io-yaml:$dataforgeVersion")
api("org.jetbrains:markdown:0.3.5")
api("org.jetbrains:markdown:0.4.0")
}
readme {

59
snark-html/src/main/kotlin/space/kscience/snark/html/SiteBuilder.kt
View File

@ -16,7 +16,6 @@ import space.kscience.dataforge.names.asName
import space.kscience.dataforge.names.parseAsName
import space.kscience.snark.SnarkContext
import space.kscience.snark.html.SiteLayout.Companion.LAYOUT_KEY
import java.nio.file.Path
/**
@ -47,19 +46,24 @@ public interface SiteBuilder : ContextAware, SnarkContext {
public val siteMeta: Meta
/**
* Add a static file or directory to this site/route at [remotePath]
* Serve a static data as a file from [data] with given [dataName] at given [routeName].
*/
public fun file(file: Path, remotePath: String = file.fileName.toString())
/**
* Add a static file (single) from resources
*/
public fun resourceFile(remotePath: String, resourcesPath: String)
/**
* Add a resource directory to route
*/
public fun resourceDirectory(resourcesPath: String)
public fun file(dataName: Name, routeName: Name = dataName)
//
// /**
// * Add a static file or directory to this site/route at [webPath]
// */
// public fun file(file: Path, webPath: String = file.fileName.toString())
//
// /**
// * Add a static file (single) from resources
// */
// public fun resourceFile(resourcesPath: String, webPath: String = resourcesPath)
//
// /**
// * Add a resource directory to route
// */
// public fun resourceDirectory(resourcesPath: String)
/**
* Create a single page at given [route]. If route is empty, create an index page at current route.
@ -153,32 +157,11 @@ public inline fun SiteBuilder.site(
internal fun SiteBuilder.assetsFrom(rootMeta: Meta) {
rootMeta.getIndexed("resource".asName()).forEach { (_, meta) ->
val path by meta.string()
val remotePath by meta.string()
path?.let { resourcePath ->
//If remote path provided, use a single resource
remotePath?.let {
resourceFile(it, resourcePath)
return@forEach
}
//otherwise use package resources
resourceDirectory(resourcePath)
}
}
rootMeta.getIndexed("file".asName()).forEach { (_, meta) ->
val remotePath by meta.string { error("File remote path is not provided") }
val path by meta.string { error("File path is not provided") }
file(Path.of(path), remotePath)
}
rootMeta.getIndexed("directory".asName()).forEach { (_, meta) ->
val path by meta.string { error("Directory path is not provided") }
file(Path.of(path), "")
val webName: String? by meta.string()
val name by meta.string { error("File path is not provided") }
val fileName = name.parseAsName()
file(fileName, webName?.parseAsName() ?: fileName)
}
}

6
snark-html/src/main/kotlin/space/kscience/snark/html/SnarkHtmlPlugin.kt
View File

@ -97,12 +97,12 @@ public fun SnarkEnvironment.buildHtmlPlugin(): SnarkHtmlPlugin {
plugin(it)
}
}
return context.fetch(SnarkHtmlPlugin)
return context.request(SnarkHtmlPlugin)
}
@OptIn(DFExperimental::class)
public fun SnarkHtmlPlugin.readDirectory(path: Path): DataTree<Any> = io.readDataDirectory(path) { dataPath, meta ->
val fileExtension = meta[FileData.META_FILE_EXTENSION_KEY].string ?: dataPath.extension
val fileExtension = meta[FileData.FILE_EXTENSION_KEY].string ?: dataPath.extension
val parser: SnarkParser<Any> = parsers.values.filter { parser ->
fileExtension in parser.fileExtensions
}.maxByOrNull {
@ -112,5 +112,5 @@ public fun SnarkHtmlPlugin.readDirectory(path: Path): DataTree<Any> = io.readDat
SnarkHtmlPlugin.byteArraySnarkParser
}
parser.reader(context, meta)
parser.asReader(context, meta)
}

55
snark-html/src/main/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt
View File

@ -29,6 +29,11 @@ internal class StaticSiteBuilder(
override val route: Name,
private val outputPath: Path,
) : SiteBuilder {
override fun file(dataName: Name, routeName: Name) {
TODO("Not yet implemented")
}
private fun Path.copyRecursively(target: Path) {
Files.walk(this).forEach { source: Path ->
val destination: Path = target.resolve(source.relativeTo(this))
@ -38,30 +43,30 @@ internal class StaticSiteBuilder(
}
}
}
override fun file(file: Path, remotePath: String) {
val targetPath = outputPath.resolve(remotePath)
if (file.isDirectory()) {
targetPath.parent.createDirectories()
file.copyRecursively(targetPath)
} else if (remotePath.isBlank()) {
error("Can't mount file to an empty route")
} else {
targetPath.parent.createDirectories()
file.copyTo(targetPath, true)
}
}
override fun resourceFile(remotePath: String, resourcesPath: String) {
val targetPath = outputPath.resolve(remotePath)
targetPath.parent.createDirectories()
javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyTo(targetPath, true)
}
override fun resourceDirectory(resourcesPath: String) {
outputPath.parent.createDirectories()
javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyRecursively(outputPath)
}
//
// override fun file(file: Path, webPath: String) {
// val targetPath = outputPath.resolve(webPath)
// if (file.isDirectory()) {
// targetPath.parent.createDirectories()
// file.copyRecursively(targetPath)
// } else if (webPath.isBlank()) {
// error("Can't mount file to an empty route")
// } else {
// targetPath.parent.createDirectories()
// file.copyTo(targetPath, true)
// }
// }
//
// override fun resourceFile(resourcesPath: String, webPath: String) {
// val targetPath = outputPath.resolve(webPath)
// targetPath.parent.createDirectories()
// javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyTo(targetPath, true)
// }
//
// override fun resourceDirectory(resourcesPath: String) {
// outputPath.parent.createDirectories()
// javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyRecursively(outputPath)
// }
private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) {
ref
@ -127,7 +132,7 @@ internal class StaticSiteBuilder(
snark = snark,
data = dataOverride ?: data,
siteMeta = Laminate(routeMeta, siteMeta),
baseUrl = resolveRef(baseUrl, routeName.toWebPath()),
baseUrl = if(baseUrl == "") "" else resolveRef(baseUrl, routeName.toWebPath()),
route = Name.EMPTY,
outputPath = outputPath.resolve(routeName.toWebPath())
)

4
snark-ktor/build.gradle.kts
View File

@ -6,6 +6,10 @@ plugins {
val dataforgeVersion: String by rootProject.extra
val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
kscience{
useContextReceivers()
}
dependencies {
api(projects.snarkHtml)

116
snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteBuilder.kt
View File

@ -1,12 +1,17 @@
package space.kscience.snark.ktor
import io.ktor.http.ContentType
import io.ktor.http.URLBuilder
import io.ktor.http.URLProtocol
import io.ktor.http.fromFileExtension
import io.ktor.server.application.Application
import io.ktor.server.application.call
import io.ktor.server.html.respondHtml
import io.ktor.server.http.content.*
import io.ktor.server.http.content.file
import io.ktor.server.http.content.files
import io.ktor.server.http.content.static
import io.ktor.server.plugins.origin
import io.ktor.server.response.respondBytes
import io.ktor.server.routing.Route
import io.ktor.server.routing.createRouteFromPath
import io.ktor.server.routing.get
@ -16,19 +21,23 @@ import kotlinx.html.CommonAttributeGroupFacade
import kotlinx.html.HTML
import kotlinx.html.style
import space.kscience.dataforge.data.DataTree
import space.kscience.dataforge.meta.Laminate
import space.kscience.dataforge.meta.Meta
import space.kscience.dataforge.meta.toMutableMeta
import space.kscience.dataforge.data.DataTreeItem
import space.kscience.dataforge.data.await
import space.kscience.dataforge.data.getItem
import space.kscience.dataforge.io.Binary
import space.kscience.dataforge.io.toByteArray
import space.kscience.dataforge.meta.*
import space.kscience.dataforge.names.Name
import space.kscience.dataforge.names.cutLast
import space.kscience.dataforge.names.endsWith
import space.kscience.dataforge.names.plus
import space.kscience.dataforge.workspace.FileData
import space.kscience.snark.SnarkEnvironment
import space.kscience.snark.html.*
import java.nio.file.Path
import kotlin.contracts.InvocationKind
import kotlin.contracts.contract
import kotlin.io.path.isDirectory
import kotlin.reflect.typeOf
public fun CommonAttributeGroupFacade.css(block: CssBuilder.() -> Unit) {
style = CssBuilder().block().toString()
@ -43,19 +52,78 @@ public class KtorSiteBuilder(
private val ktorRoute: Route,
) : SiteBuilder {
override fun file(file: Path, remotePath: String) {
if (file.isDirectory()) {
ktorRoute.static(remotePath) {
//TODO check non-standard FS and convert
files(file.toFile())
private fun file(item: DataTreeItem<Any>, routeName: Name) {
val extension = item.meta[FileData.FILE_EXTENSION_KEY]?.string?.let { ".$it" } ?: ""
//try using direct file rendering
item.meta[FileData.FILE_PATH_KEY]?.string?.let {
try {
val file = Path.of(it).toFile()
if (file.isDirectory) {
ktorRoute.static(routeName.toWebPath()) {
files(file)
}
} else {
val fileName = routeName.toWebPath() + extension //TODO add extension
ktorRoute.file(fileName, file)
}
//success, don't do anything else
return@file
} catch (ex: Exception) {
//failure,
return@let
}
}
when (item) {
is DataTreeItem.Leaf -> {
val datum = item.data
if (datum.type != typeOf<Binary>()) error("Can't directly serve file of type ${item.data.type}")
ktorRoute.get(routeName.toWebPath() + extension) {
val binary = datum.await() as Binary
val contentType: ContentType = extension
.let(ContentType::fromFileExtension)
.firstOrNull()
?: ContentType.Any
call.respondBytes(contentType = contentType) {
//TODO optimize using streaming
binary.toByteArray()
}
}
}
is DataTreeItem.Node -> {
item.tree.items.forEach { (token, childItem) ->
file(childItem, routeName + token)
}
}
} else if (remotePath.isBlank()) {
error("Can't mount file to an empty route")
} else {
ktorRoute.file(remotePath, file.toFile())
}
}
override fun file(dataName: Name, routeName: Name) {
val item: DataTreeItem<Any> = data.getItem(dataName) ?: error("Data with name is not resolved")
file(item, routeName)
}
//
// override fun file(file: Path, webPath: String) {
// if (file.isDirectory()) {
// ktorRoute.static(webPath) {
// //TODO check non-standard FS and convert
// files(file.toFile())
// }
// } else if (webPath.isBlank()) {
// error("Can't mount file to an empty route")
// } else {
// ktorRoute.file(webPath, file.toFile())
// }
// }
// override fun file(dataName: Name, webPath: String) {
// val fileData = data[dataName]
// if(fileData is FileData){
// ktorRoute.file(webPath)
// }
// }
private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) {
ref
} else if (ref.isEmpty()) {
@ -78,7 +146,7 @@ public class KtorSiteBuilder(
pageName: Name,
relative: Boolean,
): String {
val fullPageName = if(relative) route + pageName else pageName
val fullPageName = if (relative) route + pageName else pageName
return if (fullPageName.endsWith(SiteBuilder.INDEX_PAGE_TOKEN)) {
resolveRef(fullPageName.cutLast().toWebPath())
} else {
@ -94,8 +162,8 @@ public class KtorSiteBuilder(
//substitute host for url for backwards calls
val url = URLBuilder(baseUrl).apply {
protocol = URLProtocol.createOrDefault(request.origin.scheme)
host = request.origin.host
port = request.origin.port
host = request.origin.serverHost
port = request.origin.serverPort
}
val modifiedPageMeta = pageMeta.toMutableMeta().apply {
@ -135,14 +203,14 @@ public class KtorSiteBuilder(
ktorRoute = ktorRoute.createRouteFromPath(routeName.toWebPath())
)
//
// override fun resourceFile(resourcesPath: String, webPath: String) {
// ktorRoute.resource(resourcesPath, resourcesPath)
// }
override fun resourceFile(remotePath: String, resourcesPath: String) {
ktorRoute.resource(resourcesPath, resourcesPath)
}
override fun resourceDirectory(resourcesPath: String) {
ktorRoute.resources(resourcesPath)
}
// override fun resourceDirectory(resourcesPath: String) {
// ktorRoute.resources(resourcesPath)
// }
}
context(Route, SnarkEnvironment)

2
snark-ktor/src/main/kotlin/space/kscience/snark/ktor/extractData.kt
View File

@ -46,7 +46,7 @@ private const val BUILD_DATE_FILE = "/buildDate"
*/
fun Application.prepareSnarkDataCacheDirectory(dataPath: Path) {
// Clear data directory if it is outdated
// Clear data directory if it is outdated
val deployDate = dataPath.resolve(DEPLOY_DATE_FILE).takeIf { it.exists() }
?.readText()?.let { LocalDateTime.parse(it) }
val buildDate = javaClass.getResource(BUILD_DATE_FILE)?.readText()?.let { LocalDateTime.parse(it) }