Add export for all projects

This commit is contained in:
Alexander Nozik 2023-09-03 11:49:47 +03:00
parent ae0240df2b
commit 5f601cbd72
2 changed files with 47 additions and 22 deletions

View File

@ -116,18 +116,32 @@ internal suspend fun SpaceClient.downloadDocument(
when (val body = document.body) { when (val body = document.body) {
is FileDocumentHttpBody -> { is FileDocumentHttpBody -> {
launch(Dispatchers.IO) { launch(Dispatchers.IO) {
extractFile(directory.resolve(document.title), document.id) val filePath = try {
directory.resolve(document.title)
} catch (ex: Exception){
directory.resolve(document.id)
}
extractFile(filePath, document.id)
} }
} }
is TextDocumentHttpBody -> { is TextDocumentHttpBody -> {
val markdownFilePath = directory.resolve(document.title + ".md") val markdownFilePath = try {
directory.resolve(document.title + ".md")
} catch (ex: Exception) {
directory.resolve(document.id + ".md")
}
val content = body.docContent val content = body.docContent
if (content is MdTextDocumentContent) { if (content is MdTextDocumentContent) {
markdownFilePath.writeText(content.markdown, Charsets.UTF_8) markdownFilePath.writeText(content.markdown, Charsets.UTF_8)
} else { } else {
launch(Dispatchers.IO) { launch(Dispatchers.IO) {
extractFile(directory.resolve(document.title), document.id) val filePath = try {
directory.resolve(document.title)
} catch (ex: Exception){
directory.resolve(document.id)
}
extractFile(filePath, document.id)
} }
} }
} }

View File

@ -4,16 +4,15 @@ package center.sciprog.space.documentextractor
import io.ktor.client.engine.cio.CIO import io.ktor.client.engine.cio.CIO
import kotlinx.cli.* import kotlinx.cli.*
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking import kotlinx.coroutines.runBlocking
import space.jetbrains.api.runtime.* import space.jetbrains.api.runtime.*
import space.jetbrains.api.runtime.resources.chats import space.jetbrains.api.runtime.resources.chats
import space.jetbrains.api.runtime.resources.projects import space.jetbrains.api.runtime.resources.projects
import space.jetbrains.api.runtime.resources.teamDirectory import space.jetbrains.api.runtime.resources.teamDirectory
import space.jetbrains.api.runtime.types.ChannelIdentifier import space.jetbrains.api.runtime.types.*
import space.jetbrains.api.runtime.types.FolderIdentifier
import space.jetbrains.api.runtime.types.ProfileIdentifier
import space.jetbrains.api.runtime.types.ProjectIdentifier
import java.nio.file.Files import java.nio.file.Files
import java.nio.file.Path import java.nio.file.Path
import kotlin.io.path.Path import kotlin.io.path.Path
@ -27,7 +26,7 @@ private abstract class ExtractCommand(name: String, description: String) : Subco
description = """ description = """
Root IRL of the space Url like `https://spc.jetbrains.space`. Root IRL of the space Url like `https://spc.jetbrains.space`.
OR OR
Url of a specific conversation like: `https://spc.jetbrains.space/im/user/TestAccount`. Url of a specific page like: `https://spc.jetbrains.space/im/user/TestAccount`.
""".trimIndent() """.trimIndent()
) )
@ -291,16 +290,8 @@ private class ExtractProjectCommand : ExtractCommand("project", "Extract all dat
val spaceUrl = urlMatch.groups["spaceUrl"]?.value ?: error("Space Url token not recognized") val spaceUrl = urlMatch.groups["spaceUrl"]?.value ?: error("Space Url token not recognized")
val project = urlMatch.groups["projectName"]?.value ?: error("Project name token not recognized") val projectKey = urlMatch.groups["projectKey"]?.value
val rootPath = Path(path) / "projects" / project
val documentsPath = rootPath / "documents"
Files.createDirectories(documentsPath)
val repoPath: Path = rootPath / "repositories"
Files.createDirectories(repoPath)
val appInstance = SpaceAppInstance( val appInstance = SpaceAppInstance(
clientId ?: System.getProperty("space.clientId"), clientId ?: System.getProperty("space.clientId"),
@ -314,9 +305,16 @@ private class ExtractProjectCommand : ExtractCommand("project", "Extract all dat
SpaceAuth.ClientCredentials() SpaceAuth.ClientCredentials()
) )
runBlocking { fun CoroutineScope.downloadProject(project: PR_Project) {
val key = ProjectIdentifier.Key(project) val key = ProjectIdentifier.Key(project.key.key)
logger.info("Extracting everything from project \"${spaceClient.projects.getProject(key).name}\"") val projectPath = Path(path) / "projects" / project.name
logger.info("Extracting everything from project \"${project.name}\"")
val documentsPath = projectPath / "documents"
Files.createDirectories(documentsPath)
val repoPath: Path = projectPath / "repositories"
Files.createDirectories(repoPath)
launch { launch {
spaceClient.extractRepos( spaceClient.extractRepos(
repoPath, repoPath,
@ -326,16 +324,29 @@ private class ExtractProjectCommand : ExtractCommand("project", "Extract all dat
launch { launch {
spaceClient.downloadAndProcessDocumentsInProject( spaceClient.downloadAndProcessDocumentsInProject(
documentsPath, documentsPath,
ProjectIdentifier.Key(project), key,
FolderIdentifier.Root FolderIdentifier.Root
) )
} }
} }
runBlocking(Dispatchers.IO) {
if (projectKey == null) {
logger.info("Extracting everything from all available projects")
spaceClient.projects.getAllProjects().data.forEach {
downloadProject(it)
}
} else {
downloadProject(spaceClient.projects.getProject(ProjectIdentifier.Key(projectKey)))
}
}
} }
companion object { companion object {
private val urlRegex = private val urlRegex =
"""(?<spaceUrl>https?:\/\/[^\/]*)\/p\/(?<projectName>[^\/]*)\/?""".toRegex() """(?<spaceUrl>https?:\/\/[^\/]*)(\/p\/(?<projectKey>[^\/]*))?\/?""".toRegex()
} }
} }