SPC/space-export
5
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add export for all projects

Browse Source
This commit is contained in:
Alexander Nozik 2023-09-03 11:49:47 +03:00
parent ae0240df2b
commit 5f601cbd72
2 changed files with 47 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/kotlin/extractDocuments.kt
View File

@ -116,18 +116,32 @@ internal suspend fun SpaceClient.downloadDocument(
when (val body = document.body) {
is FileDocumentHttpBody -> {
launch(Dispatchers.IO) {
extractFile(directory.resolve(document.title), document.id)
val filePath = try {
directory.resolve(document.title)
} catch (ex: Exception){
directory.resolve(document.id)
}
extractFile(filePath, document.id)
}
}
is TextDocumentHttpBody -> {
val markdownFilePath = directory.resolve(document.title + ".md")
val markdownFilePath = try {
directory.resolve(document.title + ".md")
} catch (ex: Exception) {
directory.resolve(document.id + ".md")
}
val content = body.docContent
if (content is MdTextDocumentContent) {
markdownFilePath.writeText(content.markdown, Charsets.UTF_8)
} else {
launch(Dispatchers.IO) {
extractFile(directory.resolve(document.title), document.id)
val filePath = try {
directory.resolve(document.title)
} catch (ex: Exception){
directory.resolve(document.id)
}
extractFile(filePath, document.id)
}
}
}

49
src/main/kotlin/main.kt
View File

@ -4,16 +4,15 @@ package center.sciprog.space.documentextractor
import io.ktor.client.engine.cio.CIO
import kotlinx.cli.*
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import space.jetbrains.api.runtime.*
import space.jetbrains.api.runtime.resources.chats
import space.jetbrains.api.runtime.resources.projects
import space.jetbrains.api.runtime.resources.teamDirectory
import space.jetbrains.api.runtime.types.ChannelIdentifier
import space.jetbrains.api.runtime.types.FolderIdentifier
import space.jetbrains.api.runtime.types.ProfileIdentifier
import space.jetbrains.api.runtime.types.ProjectIdentifier
import space.jetbrains.api.runtime.types.*
import java.nio.file.Files
import java.nio.file.Path
import kotlin.io.path.Path
@ -27,7 +26,7 @@ private abstract class ExtractCommand(name: String, description: String) : Subco
description = """
Root IRL of the space Url like `https://spc.jetbrains.space`.
OR
Url of a specific conversation like: `https://spc.jetbrains.space/im/user/TestAccount`.
Url of a specific page like: `https://spc.jetbrains.space/im/user/TestAccount`.
""".trimIndent()
)
@ -291,16 +290,8 @@ private class ExtractProjectCommand : ExtractCommand("project", "Extract all dat
val spaceUrl = urlMatch.groups["spaceUrl"]?.value ?: error("Space Url token not recognized")
val project = urlMatch.groups["projectName"]?.value ?: error("Project name token not recognized")
val projectKey = urlMatch.groups["projectKey"]?.value
val rootPath = Path(path) / "projects" / project
val documentsPath = rootPath / "documents"
Files.createDirectories(documentsPath)
val repoPath: Path = rootPath / "repositories"
Files.createDirectories(repoPath)
val appInstance = SpaceAppInstance(
clientId ?: System.getProperty("space.clientId"),
@ -314,9 +305,16 @@ private class ExtractProjectCommand : ExtractCommand("project", "Extract all dat
SpaceAuth.ClientCredentials()
)
runBlocking {
val key = ProjectIdentifier.Key(project)
logger.info("Extracting everything from project \"${spaceClient.projects.getProject(key).name}\"")
fun CoroutineScope.downloadProject(project: PR_Project) {
val key = ProjectIdentifier.Key(project.key.key)
val projectPath = Path(path) / "projects" / project.name
logger.info("Extracting everything from project \"${project.name}\"")
val documentsPath = projectPath / "documents"
Files.createDirectories(documentsPath)
val repoPath: Path = projectPath / "repositories"
Files.createDirectories(repoPath)
launch {
spaceClient.extractRepos(
repoPath,
@ -326,16 +324,29 @@ private class ExtractProjectCommand : ExtractCommand("project", "Extract all dat
launch {
spaceClient.downloadAndProcessDocumentsInProject(
documentsPath,
ProjectIdentifier.Key(project),
key,
FolderIdentifier.Root
)
}
}
runBlocking(Dispatchers.IO) {
if (projectKey == null) {
logger.info("Extracting everything from all available projects")
spaceClient.projects.getAllProjects().data.forEach {
downloadProject(it)
}
} else {
downloadProject(spaceClient.projects.getProject(ProjectIdentifier.Key(projectKey)))
}
}
}
companion object {
private val urlRegex =
"""(?<spaceUrl>https?:\/\/[^\/]*)\/p\/(?<projectName>[^\/]*)\/?""".toRegex()
"""(?<spaceUrl>https?:\/\/[^\/]*)(\/p\/(?<projectKey>[^\/]*))?\/?""".toRegex()
}
}