From 941da6fab766ce410255b6b60119ea2459b6b814 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Mon, 13 Mar 2023 16:24:50 +0300 Subject: [PATCH] WIP - working on file-less trees --- build.gradle.kts | 13 +- gradle.properties | 2 +- settings.gradle.kts | 2 +- snark-core/build.gradle.kts | 13 +- .../space/kscience/snark/SnarkParser.kt | 3 +- snark-gradle-plugin/build.gradle.kts | 2 +- snark-html/build.gradle.kts | 6 +- .../space/kscience/snark/html/SiteBuilder.kt | 59 ++++----- .../kscience/snark/html/SnarkHtmlPlugin.kt | 6 +- .../kscience/snark/html/StaticSiteBuilder.kt | 55 +++++---- snark-ktor/build.gradle.kts | 4 + .../kscience/snark/ktor/KtorSiteBuilder.kt | 116 ++++++++++++++---- .../space/kscience/snark/ktor/extractData.kt | 2 +- 13 files changed, 171 insertions(+), 112 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index a06b422..425d02f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,5 +1,3 @@ -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile - plugins { id("space.kscience.gradle.project") } @@ -8,16 +6,13 @@ allprojects { group = "space.kscience" version = "0.1.0-dev-1" - if (name != "snark-gradle-plugin") { - tasks.withType { - kotlinOptions { - freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers" - } - } + repositories { + mavenCentral() + mavenLocal() } } -val dataforgeVersion by extra("0.6.0-dev-15") +val dataforgeVersion by extra("0.6.1-dev-4") ksciencePublish { github("SciProgCentre", "snark") diff --git a/gradle.properties b/gradle.properties index 24f5db5..bdedf81 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ kotlin.code.style=official -toolsVersion=0.13.3-kotlin-1.7.20 \ No newline at end of file +toolsVersion=0.14.2-kotlin-1.8.10 \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index a45fb4e..5c36ba3 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -31,7 +31,7 @@ dependencyResolutionManagement { } versionCatalogs { - create("npmlibs") { + create("spclibs") { from("space.kscience:version-catalog:$toolsVersion") } } diff --git a/snark-core/build.gradle.kts b/snark-core/build.gradle.kts index f9b15ad..b6e760f 100644 --- a/snark-core/build.gradle.kts +++ b/snark-core/build.gradle.kts @@ -5,12 +5,11 @@ plugins{ val dataforgeVersion: String by rootProject.extra -kotlin{ - sourceSets{ - commonMain{ - dependencies{ - api("space.kscience:dataforge-workspace:$dataforgeVersion") - } - } +kscience{ + jvm() + js() + dependencies{ + api("space.kscience:dataforge-workspace:$dataforgeVersion") } + useContextReceivers() } \ No newline at end of file diff --git a/snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkParser.kt b/snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkParser.kt index c64aae9..d17967f 100644 --- a/snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkParser.kt +++ b/snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkParser.kt @@ -22,9 +22,10 @@ public interface SnarkParser { public val priority: Int get() = DEFAULT_PRIORITY + //TODO use Binary instead of ByteArray public fun parse(context: Context, meta: Meta, bytes: ByteArray): R - public fun reader(context: Context, meta: Meta): IOReader = object : IOReader { + public fun asReader(context: Context, meta: Meta): IOReader = object : IOReader { override val type: KType get() = this@SnarkParser.type override fun readObject(input: Input): R = parse(context, meta, input.readBytes()) diff --git a/snark-gradle-plugin/build.gradle.kts b/snark-gradle-plugin/build.gradle.kts index 23989f0..0216301 100644 --- a/snark-gradle-plugin/build.gradle.kts +++ b/snark-gradle-plugin/build.gradle.kts @@ -10,7 +10,7 @@ repositories{ } dependencies{ - implementation(npmlibs.kotlin.gradle) + implementation(spclibs.kotlin.gradle) implementation("com.github.mwiede:jsch:0.2.1") } diff --git a/snark-html/build.gradle.kts b/snark-html/build.gradle.kts index 46d0fb7..a29eddf 100644 --- a/snark-html/build.gradle.kts +++ b/snark-html/build.gradle.kts @@ -6,6 +6,10 @@ plugins { val dataforgeVersion: String by rootProject.extra val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion +kscience{ + useContextReceivers() +} + dependencies { api(projects.snarkCore) @@ -15,7 +19,7 @@ dependencies { api("io.ktor:ktor-utils:$ktorVersion") api("space.kscience:dataforge-io-yaml:$dataforgeVersion") - api("org.jetbrains:markdown:0.3.5") + api("org.jetbrains:markdown:0.4.0") } readme { diff --git a/snark-html/src/main/kotlin/space/kscience/snark/html/SiteBuilder.kt b/snark-html/src/main/kotlin/space/kscience/snark/html/SiteBuilder.kt index 5164ba7..677bf92 100644 --- a/snark-html/src/main/kotlin/space/kscience/snark/html/SiteBuilder.kt +++ b/snark-html/src/main/kotlin/space/kscience/snark/html/SiteBuilder.kt @@ -16,7 +16,6 @@ import space.kscience.dataforge.names.asName import space.kscience.dataforge.names.parseAsName import space.kscience.snark.SnarkContext import space.kscience.snark.html.SiteLayout.Companion.LAYOUT_KEY -import java.nio.file.Path /** @@ -47,19 +46,24 @@ public interface SiteBuilder : ContextAware, SnarkContext { public val siteMeta: Meta /** - * Add a static file or directory to this site/route at [remotePath] + * Serve a static data as a file from [data] with given [dataName] at given [routeName]. */ - public fun file(file: Path, remotePath: String = file.fileName.toString()) - - /** - * Add a static file (single) from resources - */ - public fun resourceFile(remotePath: String, resourcesPath: String) - - /** - * Add a resource directory to route - */ - public fun resourceDirectory(resourcesPath: String) + public fun file(dataName: Name, routeName: Name = dataName) +// +// /** +// * Add a static file or directory to this site/route at [webPath] +// */ +// public fun file(file: Path, webPath: String = file.fileName.toString()) +// +// /** +// * Add a static file (single) from resources +// */ +// public fun resourceFile(resourcesPath: String, webPath: String = resourcesPath) +// +// /** +// * Add a resource directory to route +// */ +// public fun resourceDirectory(resourcesPath: String) /** * Create a single page at given [route]. If route is empty, create an index page at current route. @@ -153,32 +157,11 @@ public inline fun SiteBuilder.site( internal fun SiteBuilder.assetsFrom(rootMeta: Meta) { - rootMeta.getIndexed("resource".asName()).forEach { (_, meta) -> - - val path by meta.string() - val remotePath by meta.string() - - path?.let { resourcePath -> - //If remote path provided, use a single resource - remotePath?.let { - resourceFile(it, resourcePath) - return@forEach - } - - //otherwise use package resources - resourceDirectory(resourcePath) - } - } - rootMeta.getIndexed("file".asName()).forEach { (_, meta) -> - val remotePath by meta.string { error("File remote path is not provided") } - val path by meta.string { error("File path is not provided") } - file(Path.of(path), remotePath) - } - - rootMeta.getIndexed("directory".asName()).forEach { (_, meta) -> - val path by meta.string { error("Directory path is not provided") } - file(Path.of(path), "") + val webName: String? by meta.string() + val name by meta.string { error("File path is not provided") } + val fileName = name.parseAsName() + file(fileName, webName?.parseAsName() ?: fileName) } } diff --git a/snark-html/src/main/kotlin/space/kscience/snark/html/SnarkHtmlPlugin.kt b/snark-html/src/main/kotlin/space/kscience/snark/html/SnarkHtmlPlugin.kt index ea0413e..a287c6e 100644 --- a/snark-html/src/main/kotlin/space/kscience/snark/html/SnarkHtmlPlugin.kt +++ b/snark-html/src/main/kotlin/space/kscience/snark/html/SnarkHtmlPlugin.kt @@ -97,12 +97,12 @@ public fun SnarkEnvironment.buildHtmlPlugin(): SnarkHtmlPlugin { plugin(it) } } - return context.fetch(SnarkHtmlPlugin) + return context.request(SnarkHtmlPlugin) } @OptIn(DFExperimental::class) public fun SnarkHtmlPlugin.readDirectory(path: Path): DataTree = io.readDataDirectory(path) { dataPath, meta -> - val fileExtension = meta[FileData.META_FILE_EXTENSION_KEY].string ?: dataPath.extension + val fileExtension = meta[FileData.FILE_EXTENSION_KEY].string ?: dataPath.extension val parser: SnarkParser = parsers.values.filter { parser -> fileExtension in parser.fileExtensions }.maxByOrNull { @@ -112,5 +112,5 @@ public fun SnarkHtmlPlugin.readDirectory(path: Path): DataTree = io.readDat SnarkHtmlPlugin.byteArraySnarkParser } - parser.reader(context, meta) + parser.asReader(context, meta) } \ No newline at end of file diff --git a/snark-html/src/main/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt b/snark-html/src/main/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt index fb764ca..fa7c04f 100644 --- a/snark-html/src/main/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt +++ b/snark-html/src/main/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt @@ -29,6 +29,11 @@ internal class StaticSiteBuilder( override val route: Name, private val outputPath: Path, ) : SiteBuilder { + + override fun file(dataName: Name, routeName: Name) { + TODO("Not yet implemented") + } + private fun Path.copyRecursively(target: Path) { Files.walk(this).forEach { source: Path -> val destination: Path = target.resolve(source.relativeTo(this)) @@ -38,30 +43,30 @@ internal class StaticSiteBuilder( } } } - - override fun file(file: Path, remotePath: String) { - val targetPath = outputPath.resolve(remotePath) - if (file.isDirectory()) { - targetPath.parent.createDirectories() - file.copyRecursively(targetPath) - } else if (remotePath.isBlank()) { - error("Can't mount file to an empty route") - } else { - targetPath.parent.createDirectories() - file.copyTo(targetPath, true) - } - } - - override fun resourceFile(remotePath: String, resourcesPath: String) { - val targetPath = outputPath.resolve(remotePath) - targetPath.parent.createDirectories() - javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyTo(targetPath, true) - } - - override fun resourceDirectory(resourcesPath: String) { - outputPath.parent.createDirectories() - javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyRecursively(outputPath) - } +// +// override fun file(file: Path, webPath: String) { +// val targetPath = outputPath.resolve(webPath) +// if (file.isDirectory()) { +// targetPath.parent.createDirectories() +// file.copyRecursively(targetPath) +// } else if (webPath.isBlank()) { +// error("Can't mount file to an empty route") +// } else { +// targetPath.parent.createDirectories() +// file.copyTo(targetPath, true) +// } +// } +// +// override fun resourceFile(resourcesPath: String, webPath: String) { +// val targetPath = outputPath.resolve(webPath) +// targetPath.parent.createDirectories() +// javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyTo(targetPath, true) +// } +// +// override fun resourceDirectory(resourcesPath: String) { +// outputPath.parent.createDirectories() +// javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyRecursively(outputPath) +// } private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) { ref @@ -127,7 +132,7 @@ internal class StaticSiteBuilder( snark = snark, data = dataOverride ?: data, siteMeta = Laminate(routeMeta, siteMeta), - baseUrl = resolveRef(baseUrl, routeName.toWebPath()), + baseUrl = if(baseUrl == "") "" else resolveRef(baseUrl, routeName.toWebPath()), route = Name.EMPTY, outputPath = outputPath.resolve(routeName.toWebPath()) ) diff --git a/snark-ktor/build.gradle.kts b/snark-ktor/build.gradle.kts index cb51636..7969548 100644 --- a/snark-ktor/build.gradle.kts +++ b/snark-ktor/build.gradle.kts @@ -6,6 +6,10 @@ plugins { val dataforgeVersion: String by rootProject.extra val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion +kscience{ + useContextReceivers() +} + dependencies { api(projects.snarkHtml) diff --git a/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteBuilder.kt b/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteBuilder.kt index 86850ed..56548d7 100644 --- a/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteBuilder.kt +++ b/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteBuilder.kt @@ -1,12 +1,17 @@ package space.kscience.snark.ktor +import io.ktor.http.ContentType import io.ktor.http.URLBuilder import io.ktor.http.URLProtocol +import io.ktor.http.fromFileExtension import io.ktor.server.application.Application import io.ktor.server.application.call import io.ktor.server.html.respondHtml -import io.ktor.server.http.content.* +import io.ktor.server.http.content.file +import io.ktor.server.http.content.files +import io.ktor.server.http.content.static import io.ktor.server.plugins.origin +import io.ktor.server.response.respondBytes import io.ktor.server.routing.Route import io.ktor.server.routing.createRouteFromPath import io.ktor.server.routing.get @@ -16,19 +21,23 @@ import kotlinx.html.CommonAttributeGroupFacade import kotlinx.html.HTML import kotlinx.html.style import space.kscience.dataforge.data.DataTree -import space.kscience.dataforge.meta.Laminate -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.meta.toMutableMeta +import space.kscience.dataforge.data.DataTreeItem +import space.kscience.dataforge.data.await +import space.kscience.dataforge.data.getItem +import space.kscience.dataforge.io.Binary +import space.kscience.dataforge.io.toByteArray +import space.kscience.dataforge.meta.* import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.cutLast import space.kscience.dataforge.names.endsWith import space.kscience.dataforge.names.plus +import space.kscience.dataforge.workspace.FileData import space.kscience.snark.SnarkEnvironment import space.kscience.snark.html.* import java.nio.file.Path import kotlin.contracts.InvocationKind import kotlin.contracts.contract -import kotlin.io.path.isDirectory +import kotlin.reflect.typeOf public fun CommonAttributeGroupFacade.css(block: CssBuilder.() -> Unit) { style = CssBuilder().block().toString() @@ -43,19 +52,78 @@ public class KtorSiteBuilder( private val ktorRoute: Route, ) : SiteBuilder { - override fun file(file: Path, remotePath: String) { - if (file.isDirectory()) { - ktorRoute.static(remotePath) { - //TODO check non-standard FS and convert - files(file.toFile()) + private fun file(item: DataTreeItem, routeName: Name) { + val extension = item.meta[FileData.FILE_EXTENSION_KEY]?.string?.let { ".$it" } ?: "" + + //try using direct file rendering + item.meta[FileData.FILE_PATH_KEY]?.string?.let { + try { + val file = Path.of(it).toFile() + if (file.isDirectory) { + ktorRoute.static(routeName.toWebPath()) { + files(file) + } + } else { + val fileName = routeName.toWebPath() + extension //TODO add extension + ktorRoute.file(fileName, file) + } + //success, don't do anything else + return@file + } catch (ex: Exception) { + //failure, + return@let + } + } + when (item) { + is DataTreeItem.Leaf -> { + val datum = item.data + if (datum.type != typeOf()) error("Can't directly serve file of type ${item.data.type}") + ktorRoute.get(routeName.toWebPath() + extension) { + val binary = datum.await() as Binary + val contentType: ContentType = extension + .let(ContentType::fromFileExtension) + .firstOrNull() + ?: ContentType.Any + call.respondBytes(contentType = contentType) { + //TODO optimize using streaming + binary.toByteArray() + } + } + } + + is DataTreeItem.Node -> { + item.tree.items.forEach { (token, childItem) -> + file(childItem, routeName + token) + } } - } else if (remotePath.isBlank()) { - error("Can't mount file to an empty route") - } else { - ktorRoute.file(remotePath, file.toFile()) } } + override fun file(dataName: Name, routeName: Name) { + val item: DataTreeItem = data.getItem(dataName) ?: error("Data with name is not resolved") + file(item, routeName) + } +// +// override fun file(file: Path, webPath: String) { +// if (file.isDirectory()) { +// ktorRoute.static(webPath) { +// //TODO check non-standard FS and convert +// files(file.toFile()) +// } +// } else if (webPath.isBlank()) { +// error("Can't mount file to an empty route") +// } else { +// ktorRoute.file(webPath, file.toFile()) +// } +// } + +// override fun file(dataName: Name, webPath: String) { +// val fileData = data[dataName] +// if(fileData is FileData){ +// ktorRoute.file(webPath) +// } +// } + private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) { ref } else if (ref.isEmpty()) { @@ -78,7 +146,7 @@ public class KtorSiteBuilder( pageName: Name, relative: Boolean, ): String { - val fullPageName = if(relative) route + pageName else pageName + val fullPageName = if (relative) route + pageName else pageName return if (fullPageName.endsWith(SiteBuilder.INDEX_PAGE_TOKEN)) { resolveRef(fullPageName.cutLast().toWebPath()) } else { @@ -94,8 +162,8 @@ public class KtorSiteBuilder( //substitute host for url for backwards calls val url = URLBuilder(baseUrl).apply { protocol = URLProtocol.createOrDefault(request.origin.scheme) - host = request.origin.host - port = request.origin.port + host = request.origin.serverHost + port = request.origin.serverPort } val modifiedPageMeta = pageMeta.toMutableMeta().apply { @@ -135,14 +203,14 @@ public class KtorSiteBuilder( ktorRoute = ktorRoute.createRouteFromPath(routeName.toWebPath()) ) +// +// override fun resourceFile(resourcesPath: String, webPath: String) { +// ktorRoute.resource(resourcesPath, resourcesPath) +// } - override fun resourceFile(remotePath: String, resourcesPath: String) { - ktorRoute.resource(resourcesPath, resourcesPath) - } - - override fun resourceDirectory(resourcesPath: String) { - ktorRoute.resources(resourcesPath) - } +// override fun resourceDirectory(resourcesPath: String) { +// ktorRoute.resources(resourcesPath) +// } } context(Route, SnarkEnvironment) diff --git a/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/extractData.kt b/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/extractData.kt index ec19c4b..fc47faf 100644 --- a/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/extractData.kt +++ b/snark-ktor/src/main/kotlin/space/kscience/snark/ktor/extractData.kt @@ -46,7 +46,7 @@ private const val BUILD_DATE_FILE = "/buildDate" */ fun Application.prepareSnarkDataCacheDirectory(dataPath: Path) { -// Clear data directory if it is outdated + // Clear data directory if it is outdated val deployDate = dataPath.resolve(DEPLOY_DATE_FILE).takeIf { it.exists() } ?.readText()?.let { LocalDateTime.parse(it) } val buildDate = javaClass.getResource(BUILD_DATE_FILE)?.readText()?.let { LocalDateTime.parse(it) }