Add export for all projects
This commit is contained in:
parent
ae0240df2b
commit
5f601cbd72
@ -116,18 +116,32 @@ internal suspend fun SpaceClient.downloadDocument(
|
|||||||
when (val body = document.body) {
|
when (val body = document.body) {
|
||||||
is FileDocumentHttpBody -> {
|
is FileDocumentHttpBody -> {
|
||||||
launch(Dispatchers.IO) {
|
launch(Dispatchers.IO) {
|
||||||
extractFile(directory.resolve(document.title), document.id)
|
val filePath = try {
|
||||||
|
directory.resolve(document.title)
|
||||||
|
} catch (ex: Exception){
|
||||||
|
directory.resolve(document.id)
|
||||||
|
}
|
||||||
|
extractFile(filePath, document.id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
is TextDocumentHttpBody -> {
|
is TextDocumentHttpBody -> {
|
||||||
val markdownFilePath = directory.resolve(document.title + ".md")
|
val markdownFilePath = try {
|
||||||
|
directory.resolve(document.title + ".md")
|
||||||
|
} catch (ex: Exception) {
|
||||||
|
directory.resolve(document.id + ".md")
|
||||||
|
}
|
||||||
val content = body.docContent
|
val content = body.docContent
|
||||||
if (content is MdTextDocumentContent) {
|
if (content is MdTextDocumentContent) {
|
||||||
markdownFilePath.writeText(content.markdown, Charsets.UTF_8)
|
markdownFilePath.writeText(content.markdown, Charsets.UTF_8)
|
||||||
} else {
|
} else {
|
||||||
launch(Dispatchers.IO) {
|
launch(Dispatchers.IO) {
|
||||||
extractFile(directory.resolve(document.title), document.id)
|
val filePath = try {
|
||||||
|
directory.resolve(document.title)
|
||||||
|
} catch (ex: Exception){
|
||||||
|
directory.resolve(document.id)
|
||||||
|
}
|
||||||
|
extractFile(filePath, document.id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -4,16 +4,15 @@ package center.sciprog.space.documentextractor
|
|||||||
|
|
||||||
import io.ktor.client.engine.cio.CIO
|
import io.ktor.client.engine.cio.CIO
|
||||||
import kotlinx.cli.*
|
import kotlinx.cli.*
|
||||||
|
import kotlinx.coroutines.CoroutineScope
|
||||||
|
import kotlinx.coroutines.Dispatchers
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
import kotlinx.coroutines.runBlocking
|
import kotlinx.coroutines.runBlocking
|
||||||
import space.jetbrains.api.runtime.*
|
import space.jetbrains.api.runtime.*
|
||||||
import space.jetbrains.api.runtime.resources.chats
|
import space.jetbrains.api.runtime.resources.chats
|
||||||
import space.jetbrains.api.runtime.resources.projects
|
import space.jetbrains.api.runtime.resources.projects
|
||||||
import space.jetbrains.api.runtime.resources.teamDirectory
|
import space.jetbrains.api.runtime.resources.teamDirectory
|
||||||
import space.jetbrains.api.runtime.types.ChannelIdentifier
|
import space.jetbrains.api.runtime.types.*
|
||||||
import space.jetbrains.api.runtime.types.FolderIdentifier
|
|
||||||
import space.jetbrains.api.runtime.types.ProfileIdentifier
|
|
||||||
import space.jetbrains.api.runtime.types.ProjectIdentifier
|
|
||||||
import java.nio.file.Files
|
import java.nio.file.Files
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
import kotlin.io.path.Path
|
import kotlin.io.path.Path
|
||||||
@ -27,7 +26,7 @@ private abstract class ExtractCommand(name: String, description: String) : Subco
|
|||||||
description = """
|
description = """
|
||||||
Root IRL of the space Url like `https://spc.jetbrains.space`.
|
Root IRL of the space Url like `https://spc.jetbrains.space`.
|
||||||
OR
|
OR
|
||||||
Url of a specific conversation like: `https://spc.jetbrains.space/im/user/TestAccount`.
|
Url of a specific page like: `https://spc.jetbrains.space/im/user/TestAccount`.
|
||||||
""".trimIndent()
|
""".trimIndent()
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -291,16 +290,8 @@ private class ExtractProjectCommand : ExtractCommand("project", "Extract all dat
|
|||||||
|
|
||||||
val spaceUrl = urlMatch.groups["spaceUrl"]?.value ?: error("Space Url token not recognized")
|
val spaceUrl = urlMatch.groups["spaceUrl"]?.value ?: error("Space Url token not recognized")
|
||||||
|
|
||||||
val project = urlMatch.groups["projectName"]?.value ?: error("Project name token not recognized")
|
val projectKey = urlMatch.groups["projectKey"]?.value
|
||||||
|
|
||||||
val rootPath = Path(path) / "projects" / project
|
|
||||||
|
|
||||||
val documentsPath = rootPath / "documents"
|
|
||||||
Files.createDirectories(documentsPath)
|
|
||||||
|
|
||||||
val repoPath: Path = rootPath / "repositories"
|
|
||||||
|
|
||||||
Files.createDirectories(repoPath)
|
|
||||||
|
|
||||||
val appInstance = SpaceAppInstance(
|
val appInstance = SpaceAppInstance(
|
||||||
clientId ?: System.getProperty("space.clientId"),
|
clientId ?: System.getProperty("space.clientId"),
|
||||||
@ -314,9 +305,16 @@ private class ExtractProjectCommand : ExtractCommand("project", "Extract all dat
|
|||||||
SpaceAuth.ClientCredentials()
|
SpaceAuth.ClientCredentials()
|
||||||
)
|
)
|
||||||
|
|
||||||
runBlocking {
|
fun CoroutineScope.downloadProject(project: PR_Project) {
|
||||||
val key = ProjectIdentifier.Key(project)
|
val key = ProjectIdentifier.Key(project.key.key)
|
||||||
logger.info("Extracting everything from project \"${spaceClient.projects.getProject(key).name}\"")
|
val projectPath = Path(path) / "projects" / project.name
|
||||||
|
logger.info("Extracting everything from project \"${project.name}\"")
|
||||||
|
val documentsPath = projectPath / "documents"
|
||||||
|
Files.createDirectories(documentsPath)
|
||||||
|
|
||||||
|
val repoPath: Path = projectPath / "repositories"
|
||||||
|
|
||||||
|
Files.createDirectories(repoPath)
|
||||||
launch {
|
launch {
|
||||||
spaceClient.extractRepos(
|
spaceClient.extractRepos(
|
||||||
repoPath,
|
repoPath,
|
||||||
@ -326,16 +324,29 @@ private class ExtractProjectCommand : ExtractCommand("project", "Extract all dat
|
|||||||
launch {
|
launch {
|
||||||
spaceClient.downloadAndProcessDocumentsInProject(
|
spaceClient.downloadAndProcessDocumentsInProject(
|
||||||
documentsPath,
|
documentsPath,
|
||||||
ProjectIdentifier.Key(project),
|
key,
|
||||||
FolderIdentifier.Root
|
FolderIdentifier.Root
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
runBlocking(Dispatchers.IO) {
|
||||||
|
if (projectKey == null) {
|
||||||
|
logger.info("Extracting everything from all available projects")
|
||||||
|
spaceClient.projects.getAllProjects().data.forEach {
|
||||||
|
downloadProject(it)
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
downloadProject(spaceClient.projects.getProject(ProjectIdentifier.Key(projectKey)))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
private val urlRegex =
|
private val urlRegex =
|
||||||
"""(?<spaceUrl>https?:\/\/[^\/]*)\/p\/(?<projectName>[^\/]*)\/?""".toRegex()
|
"""(?<spaceUrl>https?:\/\/[^\/]*)(\/p\/(?<projectKey>[^\/]*))?\/?""".toRegex()
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user