WIP - working on file-less trees

build.gradle.kts

@@ -1,5 +1,3 @@
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
-
 plugins {
     id("space.kscience.gradle.project")
 }
@@ -8,16 +6,13 @@ allprojects {
     group = "space.kscience"
     version = "0.1.0-dev-1"
 
-    if (name != "snark-gradle-plugin") {
+    repositories {
-        tasks.withType<KotlinCompile> {
+        mavenCentral()
-            kotlinOptions {
+        mavenLocal()
-                freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers"
-            }
-        }
     }
 }
 
-val dataforgeVersion by extra("0.6.0-dev-15")
+val dataforgeVersion by extra("0.6.1-dev-4")
 
 ksciencePublish {
     github("SciProgCentre", "snark")

gradle.properties

@@ -1,3 +1,3 @@
 kotlin.code.style=official
 
-toolsVersion=0.13.3-kotlin-1.7.20
+toolsVersion=0.14.2-kotlin-1.8.10

settings.gradle.kts

@@ -31,7 +31,7 @@ dependencyResolutionManagement {
     }
 
     versionCatalogs {
-        create("npmlibs") {
+        create("spclibs") {
             from("space.kscience:version-catalog:$toolsVersion")
         }
     }

snark-core/build.gradle.kts

@@ -5,12 +5,11 @@ plugins{
 
 val dataforgeVersion: String by rootProject.extra
 
-kotlin{
+kscience{
-    sourceSets{
+    jvm()
-        commonMain{
+    js()
-            dependencies{
+    dependencies{
-                api("space.kscience:dataforge-workspace:$dataforgeVersion")
+        api("space.kscience:dataforge-workspace:$dataforgeVersion")
-            }
-        }
     }
+    useContextReceivers()
 }

snark-core/src/commonMain/kotlin/space/kscience/snark/SnarkParser.kt

@@ -22,9 +22,10 @@ public interface SnarkParser<out R> {
 
     public val priority: Int get() = DEFAULT_PRIORITY
 
+    //TODO use Binary instead of ByteArray
     public fun parse(context: Context, meta: Meta, bytes: ByteArray): R
 
-    public fun reader(context: Context, meta: Meta): IOReader<R> = object : IOReader<R> {
+    public fun asReader(context: Context, meta: Meta): IOReader<R> = object : IOReader<R> {
         override val type: KType get() = this@SnarkParser.type
 
         override fun readObject(input: Input): R = parse(context, meta, input.readBytes())

snark-gradle-plugin/build.gradle.kts

@@ -10,7 +10,7 @@ repositories{
 }
 
 dependencies{
-    implementation(npmlibs.kotlin.gradle)
+    implementation(spclibs.kotlin.gradle)
     implementation("com.github.mwiede:jsch:0.2.1")
 }
 

snark-html/build.gradle.kts

@@ -6,6 +6,10 @@ plugins {
 val dataforgeVersion: String by rootProject.extra
 val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
 
+kscience{
+    useContextReceivers()
+}
+
 dependencies {
     api(projects.snarkCore)
 
@@ -15,7 +19,7 @@ dependencies {
     api("io.ktor:ktor-utils:$ktorVersion")
 
     api("space.kscience:dataforge-io-yaml:$dataforgeVersion")
-    api("org.jetbrains:markdown:0.3.5")
+    api("org.jetbrains:markdown:0.4.0")
 }
 
 readme {

snark-html/src/main/kotlin/space/kscience/snark/html/SiteBuilder.kt

@@ -16,7 +16,6 @@ import space.kscience.dataforge.names.asName
 import space.kscience.dataforge.names.parseAsName
 import space.kscience.snark.SnarkContext
 import space.kscience.snark.html.SiteLayout.Companion.LAYOUT_KEY
-import java.nio.file.Path
 
 
 /**
@@ -47,19 +46,24 @@ public interface SiteBuilder : ContextAware, SnarkContext {
     public val siteMeta: Meta
 
     /**
-     * Add a static file or directory to this site/route at [remotePath]
+     * Serve a static data as a file from [data] with given [dataName] at given [routeName].
      */
-    public fun file(file: Path, remotePath: String = file.fileName.toString())
+    public fun file(dataName: Name, routeName: Name = dataName)
-
+//
-    /**
+//    /**
-     * Add a static file (single) from resources
+//     * Add a static file or directory to this site/route at [webPath]
-     */
+//     */
-    public fun resourceFile(remotePath: String, resourcesPath: String)
+//    public fun file(file: Path, webPath: String = file.fileName.toString())
-
+//
-    /**
+//    /**
-     * Add a resource directory to route
+//     * Add a static file (single) from resources
-     */
+//     */
-    public fun resourceDirectory(resourcesPath: String)
+//    public fun resourceFile(resourcesPath: String, webPath: String = resourcesPath)
+//
+//    /**
+//     * Add a resource directory to route
+//     */
+//    public fun resourceDirectory(resourcesPath: String)
 
     /**
      * Create a single page at given [route]. If route is empty, create an index page at current route.
@@ -153,32 +157,11 @@ public inline fun SiteBuilder.site(
 
 
 internal fun SiteBuilder.assetsFrom(rootMeta: Meta) {
-    rootMeta.getIndexed("resource".asName()).forEach { (_, meta) ->
-
-        val path by meta.string()
-        val remotePath by meta.string()
-
-        path?.let { resourcePath ->
-            //If remote path provided, use a single resource
-            remotePath?.let {
-                resourceFile(it, resourcePath)
-                return@forEach
-            }
-
-            //otherwise use package resources
-            resourceDirectory(resourcePath)
-        }
-    }
-
     rootMeta.getIndexed("file".asName()).forEach { (_, meta) ->
-        val remotePath by meta.string { error("File remote path is not provided") }
+        val webName: String? by meta.string()
-        val path by meta.string { error("File path is not provided") }
+        val name by meta.string { error("File path is not provided") }
-        file(Path.of(path), remotePath)
+        val fileName = name.parseAsName()
-    }
+        file(fileName, webName?.parseAsName() ?: fileName)
-
-    rootMeta.getIndexed("directory".asName()).forEach { (_, meta) ->
-        val path by meta.string { error("Directory path is not provided") }
-        file(Path.of(path), "")
     }
 }
 

snark-html/src/main/kotlin/space/kscience/snark/html/SnarkHtmlPlugin.kt

@@ -97,12 +97,12 @@ public fun SnarkEnvironment.buildHtmlPlugin(): SnarkHtmlPlugin {
             plugin(it)
         }
     }
-    return context.fetch(SnarkHtmlPlugin)
+    return context.request(SnarkHtmlPlugin)
 }
 
 @OptIn(DFExperimental::class)
 public fun SnarkHtmlPlugin.readDirectory(path: Path): DataTree<Any> = io.readDataDirectory(path) { dataPath, meta ->
-    val fileExtension = meta[FileData.META_FILE_EXTENSION_KEY].string ?: dataPath.extension
+    val fileExtension = meta[FileData.FILE_EXTENSION_KEY].string ?: dataPath.extension
     val parser: SnarkParser<Any> = parsers.values.filter { parser ->
         fileExtension in parser.fileExtensions
     }.maxByOrNull {
@@ -112,5 +112,5 @@ public fun SnarkHtmlPlugin.readDirectory(path: Path): DataTree<Any> = io.readDat
         SnarkHtmlPlugin.byteArraySnarkParser
     }
 
-    parser.reader(context, meta)
+    parser.asReader(context, meta)
 }

snark-html/src/main/kotlin/space/kscience/snark/html/StaticSiteBuilder.kt

@@ -29,6 +29,11 @@ internal class StaticSiteBuilder(
     override val route: Name,
     private val outputPath: Path,
 ) : SiteBuilder {
+
+    override fun file(dataName: Name, routeName: Name) {
+        TODO("Not yet implemented")
+    }
+
     private fun Path.copyRecursively(target: Path) {
         Files.walk(this).forEach { source: Path ->
             val destination: Path = target.resolve(source.relativeTo(this))
@@ -38,30 +43,30 @@ internal class StaticSiteBuilder(
             }
         }
     }
-
+//
-    override fun file(file: Path, remotePath: String) {
+//    override fun file(file: Path, webPath: String) {
-        val targetPath = outputPath.resolve(remotePath)
+//        val targetPath = outputPath.resolve(webPath)
-        if (file.isDirectory()) {
+//        if (file.isDirectory()) {
-            targetPath.parent.createDirectories()
+//            targetPath.parent.createDirectories()
-            file.copyRecursively(targetPath)
+//            file.copyRecursively(targetPath)
-        } else if (remotePath.isBlank()) {
+//        } else if (webPath.isBlank()) {
-            error("Can't mount file to an empty route")
+//            error("Can't mount file to an empty route")
-        } else {
+//        } else {
-            targetPath.parent.createDirectories()
+//            targetPath.parent.createDirectories()
-            file.copyTo(targetPath, true)
+//            file.copyTo(targetPath, true)
-        }
+//        }
-    }
+//    }
-
+//
-    override fun resourceFile(remotePath: String, resourcesPath: String) {
+//    override fun resourceFile(resourcesPath: String, webPath: String) {
-        val targetPath = outputPath.resolve(remotePath)
+//        val targetPath = outputPath.resolve(webPath)
-        targetPath.parent.createDirectories()
+//        targetPath.parent.createDirectories()
-        javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyTo(targetPath, true)
+//        javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyTo(targetPath, true)
-    }
+//    }
-
+//
-    override fun resourceDirectory(resourcesPath: String) {
+//    override fun resourceDirectory(resourcesPath: String) {
-        outputPath.parent.createDirectories()
+//        outputPath.parent.createDirectories()
-        javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyRecursively(outputPath)
+//        javaClass.getResource(resourcesPath)?.let { Path.of(it.toURI()) }?.copyRecursively(outputPath)
-    }
+//    }
 
     private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) {
         ref
@@ -127,7 +132,7 @@ internal class StaticSiteBuilder(
         snark = snark,
         data = dataOverride ?: data,
         siteMeta = Laminate(routeMeta, siteMeta),
-        baseUrl = resolveRef(baseUrl, routeName.toWebPath()),
+        baseUrl = if(baseUrl == "") "" else resolveRef(baseUrl, routeName.toWebPath()),
         route = Name.EMPTY,
         outputPath = outputPath.resolve(routeName.toWebPath())
     )

snark-ktor/build.gradle.kts

@@ -6,6 +6,10 @@ plugins {
 val dataforgeVersion: String by rootProject.extra
 val ktorVersion = space.kscience.gradle.KScienceVersions.ktorVersion
 
+kscience{
+    useContextReceivers()
+}
+
 dependencies {
     api(projects.snarkHtml)
 

snark-ktor/src/main/kotlin/space/kscience/snark/ktor/KtorSiteBuilder.kt

@@ -1,12 +1,17 @@
 package space.kscience.snark.ktor
 
+import io.ktor.http.ContentType
 import io.ktor.http.URLBuilder
 import io.ktor.http.URLProtocol
+import io.ktor.http.fromFileExtension
 import io.ktor.server.application.Application
 import io.ktor.server.application.call
 import io.ktor.server.html.respondHtml
-import io.ktor.server.http.content.*
+import io.ktor.server.http.content.file
+import io.ktor.server.http.content.files
+import io.ktor.server.http.content.static
 import io.ktor.server.plugins.origin
+import io.ktor.server.response.respondBytes
 import io.ktor.server.routing.Route
 import io.ktor.server.routing.createRouteFromPath
 import io.ktor.server.routing.get
@@ -16,19 +21,23 @@ import kotlinx.html.CommonAttributeGroupFacade
 import kotlinx.html.HTML
 import kotlinx.html.style
 import space.kscience.dataforge.data.DataTree
-import space.kscience.dataforge.meta.Laminate
+import space.kscience.dataforge.data.DataTreeItem
-import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.data.await
-import space.kscience.dataforge.meta.toMutableMeta
+import space.kscience.dataforge.data.getItem
+import space.kscience.dataforge.io.Binary
+import space.kscience.dataforge.io.toByteArray
+import space.kscience.dataforge.meta.*
 import space.kscience.dataforge.names.Name
 import space.kscience.dataforge.names.cutLast
 import space.kscience.dataforge.names.endsWith
 import space.kscience.dataforge.names.plus
+import space.kscience.dataforge.workspace.FileData
 import space.kscience.snark.SnarkEnvironment
 import space.kscience.snark.html.*
 import java.nio.file.Path
 import kotlin.contracts.InvocationKind
 import kotlin.contracts.contract
-import kotlin.io.path.isDirectory
+import kotlin.reflect.typeOf
 
 public fun CommonAttributeGroupFacade.css(block: CssBuilder.() -> Unit) {
     style = CssBuilder().block().toString()
@@ -43,19 +52,78 @@ public class KtorSiteBuilder(
     private val ktorRoute: Route,
 ) : SiteBuilder {
 
-    override fun file(file: Path, remotePath: String) {
+    private fun file(item: DataTreeItem<Any>, routeName: Name) {
-        if (file.isDirectory()) {
+        val extension = item.meta[FileData.FILE_EXTENSION_KEY]?.string?.let { ".$it" } ?: ""
-            ktorRoute.static(remotePath) {
+
-                //TODO check non-standard FS and convert
+        //try using direct file rendering
-                files(file.toFile())
+        item.meta[FileData.FILE_PATH_KEY]?.string?.let {
+            try {
+                val file = Path.of(it).toFile()
+                if (file.isDirectory) {
+                    ktorRoute.static(routeName.toWebPath()) {
+                        files(file)
+                    }
+                } else {
+                    val fileName = routeName.toWebPath() + extension //TODO add extension
+                    ktorRoute.file(fileName, file)
+                }
+                //success, don't do anything else
+                return@file
+            } catch (ex: Exception) {
+                //failure,
+                return@let
+            }
+        }
+        when (item) {
+            is DataTreeItem.Leaf -> {
+                val datum = item.data
+                if (datum.type != typeOf<Binary>()) error("Can't directly serve file of type ${item.data.type}")
+                ktorRoute.get(routeName.toWebPath() + extension) {
+                    val binary = datum.await() as Binary
+                    val contentType: ContentType = extension
+                        .let(ContentType::fromFileExtension)
+                        .firstOrNull()
+                        ?: ContentType.Any
+                    call.respondBytes(contentType = contentType) {
+                        //TODO optimize using streaming
+                        binary.toByteArray()
+                    }
+                }
+            }
+
+            is DataTreeItem.Node -> {
+                item.tree.items.forEach { (token, childItem) ->
+                    file(childItem, routeName + token)
+                }
             }
-        } else if (remotePath.isBlank()) {
-            error("Can't mount file to an empty route")
-        } else {
-            ktorRoute.file(remotePath, file.toFile())
         }
     }
 
+    override fun file(dataName: Name, routeName: Name) {
+        val item: DataTreeItem<Any> = data.getItem(dataName) ?: error("Data with name is not resolved")
+        file(item, routeName)
+    }
+//
+//    override fun file(file: Path, webPath: String) {
+//        if (file.isDirectory()) {
+//            ktorRoute.static(webPath) {
+//                //TODO check non-standard FS and convert
+//                files(file.toFile())
+//            }
+//        } else if (webPath.isBlank()) {
+//            error("Can't mount file to an empty route")
+//        } else {
+//            ktorRoute.file(webPath, file.toFile())
+//        }
+//    }
+
+//    override fun file(dataName: Name, webPath: String) {
+//        val fileData = data[dataName]
+//        if(fileData is FileData){
+//            ktorRoute.file(webPath)
+//        }
+//    }
+
     private fun resolveRef(baseUrl: String, ref: String) = if (baseUrl.isEmpty()) {
         ref
     } else if (ref.isEmpty()) {
@@ -78,7 +146,7 @@ public class KtorSiteBuilder(
             pageName: Name,
             relative: Boolean,
         ): String {
-            val fullPageName = if(relative) route + pageName else pageName
+            val fullPageName = if (relative) route + pageName else pageName
             return if (fullPageName.endsWith(SiteBuilder.INDEX_PAGE_TOKEN)) {
                 resolveRef(fullPageName.cutLast().toWebPath())
             } else {
@@ -94,8 +162,8 @@ public class KtorSiteBuilder(
                 //substitute host for url for backwards calls
                 val url = URLBuilder(baseUrl).apply {
                     protocol = URLProtocol.createOrDefault(request.origin.scheme)
-                    host = request.origin.host
+                    host = request.origin.serverHost
-                    port = request.origin.port
+                    port = request.origin.serverPort
                 }
 
                 val modifiedPageMeta = pageMeta.toMutableMeta().apply {
@@ -135,14 +203,14 @@ public class KtorSiteBuilder(
         ktorRoute = ktorRoute.createRouteFromPath(routeName.toWebPath())
     )
 
+//
+//    override fun resourceFile(resourcesPath: String, webPath: String) {
+//        ktorRoute.resource(resourcesPath, resourcesPath)
+//    }
 
-    override fun resourceFile(remotePath: String, resourcesPath: String) {
+//    override fun resourceDirectory(resourcesPath: String) {
-        ktorRoute.resource(resourcesPath, resourcesPath)
+//        ktorRoute.resources(resourcesPath)
-    }
+//    }
-
-    override fun resourceDirectory(resourcesPath: String) {
-        ktorRoute.resources(resourcesPath)
-    }
 }
 
 context(Route, SnarkEnvironment)

snark-ktor/src/main/kotlin/space/kscience/snark/ktor/extractData.kt

@@ -46,7 +46,7 @@ private const val BUILD_DATE_FILE = "/buildDate"
  */
 fun Application.prepareSnarkDataCacheDirectory(dataPath: Path) {
 
-// Clear data directory if it is outdated
+    // Clear data directory if it is outdated
     val deployDate = dataPath.resolve(DEPLOY_DATE_FILE).takeIf { it.exists() }
         ?.readText()?.let { LocalDateTime.parse(it) }
     val buildDate = javaClass.getResource(BUILD_DATE_FILE)?.readText()?.let { LocalDateTime.parse(it) }