Add direct message export
This commit is contained in:
parent
3eb43d4d19
commit
e2d3c108c2
@ -1,5 +1,4 @@
|
||||
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
|
||||
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
|
||||
|
||||
plugins {
|
||||
kotlin("jvm") version "1.8.22"
|
||||
@ -15,7 +14,7 @@ repositories {
|
||||
maven("https://maven.pkg.jetbrains.space/public/p/space/maven")
|
||||
}
|
||||
|
||||
val ktorVersion = "2.3.3"
|
||||
val ktorVersion = "2.3.4"
|
||||
|
||||
dependencies {
|
||||
implementation("org.jetbrains:space-sdk-jvm:167818-beta")
|
||||
|
@ -1,6 +1,5 @@
|
||||
package center.sciprog.space.documentextractor
|
||||
|
||||
import kotlinx.datetime.Clock
|
||||
import kotlinx.datetime.Instant
|
||||
import space.jetbrains.api.runtime.SpaceClient
|
||||
import space.jetbrains.api.runtime.resources.chats
|
||||
@ -17,13 +16,13 @@ private suspend fun SpaceClient.writeMessages(
|
||||
id: ChannelIdentifier,
|
||||
prefix: String = "",
|
||||
) {
|
||||
var readDateTime: Instant? = Clock.System.now()
|
||||
var readDateTime: Instant? = null
|
||||
var read: Int
|
||||
//reading messages in batches
|
||||
do {
|
||||
val result: GetMessagesResponse = chats.messages.getChannelMessages(
|
||||
channel = id,
|
||||
sorting = MessagesSorting.FromNewestToOldest,
|
||||
sorting = MessagesSorting.FromOldestToNewest,
|
||||
startFromDate = readDateTime,
|
||||
batchSize = 50
|
||||
) {
|
||||
@ -59,7 +58,7 @@ private suspend fun SpaceClient.writeMessages(
|
||||
val name = "${attachment.id}-${attachment.filename}"
|
||||
val file = attachmentsDirectory.resolve(name)
|
||||
extractFile(file, fileId)
|
||||
writer.appendLine("*Attachment*: [name](attachments/$name)\n")
|
||||
writer.appendLine("*Attachment*: [$name](attachments/$name)\n")
|
||||
}
|
||||
|
||||
is ImageAttachment -> {
|
||||
@ -96,17 +95,36 @@ private suspend fun SpaceClient.writeMessages(
|
||||
|
||||
|
||||
suspend fun SpaceClient.extractMessages(
|
||||
chatId: String,
|
||||
id: ChannelIdentifier,
|
||||
parentDirectory: Path,
|
||||
) {
|
||||
val id = ChannelIdentifier.Id(chatId)
|
||||
val channel = chats.channels.getChannel(id)
|
||||
val channel = chats.channels.getChannel(id){
|
||||
content {
|
||||
member {
|
||||
name()
|
||||
username()
|
||||
}
|
||||
}
|
||||
contact{
|
||||
key()
|
||||
}
|
||||
totalMessages()
|
||||
}
|
||||
|
||||
val name = (channel.contact.ext as? M2SharedChannelContent)?.name ?: channel.contact.defaultName
|
||||
if (channel.totalMessages == 0){
|
||||
logger.debug("Channel with {} is empty", id)
|
||||
return
|
||||
}
|
||||
|
||||
val name = when(val ext = channel.content){
|
||||
is M2SharedChannelContent -> ext.name
|
||||
is M2ChannelContentMember -> ext.member.username
|
||||
else -> channel.contact.key
|
||||
}
|
||||
|
||||
val file = parentDirectory.resolve("$name.md")
|
||||
|
||||
logger.info("Extracting messages from channel $chatId to $file")
|
||||
logger.info("Extracting messages from channel $id to $file")
|
||||
|
||||
file.parent.createDirectories()
|
||||
|
||||
|
@ -6,13 +6,13 @@ import io.ktor.client.engine.cio.CIO
|
||||
import kotlinx.cli.*
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import space.jetbrains.api.runtime.SpaceAppInstance
|
||||
import space.jetbrains.api.runtime.SpaceAuth
|
||||
import space.jetbrains.api.runtime.SpaceClient
|
||||
import space.jetbrains.api.runtime.ktorClientForSpace
|
||||
import space.jetbrains.api.runtime.*
|
||||
import space.jetbrains.api.runtime.resources.chats
|
||||
import space.jetbrains.api.runtime.resources.projects
|
||||
import space.jetbrains.api.runtime.resources.teamDirectory
|
||||
import space.jetbrains.api.runtime.types.ChannelIdentifier
|
||||
import space.jetbrains.api.runtime.types.FolderIdentifier
|
||||
import space.jetbrains.api.runtime.types.ProfileIdentifier
|
||||
import space.jetbrains.api.runtime.types.ProjectIdentifier
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
@ -171,19 +171,19 @@ private class ExtractRepositoriesCommand : ExtractCommand("repos", "Extract repo
|
||||
|
||||
}
|
||||
|
||||
private class ExtractMessagesCommand : ExtractCommand("messages", "Extract all messages from a chat") {
|
||||
private class ExtractChannelsCommand : ExtractCommand("channels", "Extract all messages from a channels") {
|
||||
|
||||
val path: String by option(
|
||||
ArgType.String,
|
||||
description = "Target directory. Default is './chats'."
|
||||
).default("./chats")
|
||||
description = "Target directory."
|
||||
).default("./channels")
|
||||
|
||||
override fun execute() {
|
||||
val urlMatch = urlRegex.matchEntire(url) ?: error("Url $url does not match space document url pattern")
|
||||
|
||||
val spaceUrl = urlMatch.groups["spaceUrl"]?.value ?: error("Space Url token not recognized")
|
||||
|
||||
val chatId = urlMatch.groups["chatId"]?.value
|
||||
val channelId = urlMatch.groups["chatId"]?.value
|
||||
|
||||
val appInstance = SpaceAppInstance(
|
||||
clientId ?: System.getProperty("space.clientId"),
|
||||
@ -198,12 +198,12 @@ private class ExtractMessagesCommand : ExtractCommand("messages", "Extract all m
|
||||
)
|
||||
|
||||
runBlocking {
|
||||
if (chatId == null) {
|
||||
if (channelId == null) {
|
||||
spaceClient.chats.channels.listAllChannels(query = "").data.forEach {
|
||||
spaceClient.extractMessages(it.channelId, Path(path))
|
||||
spaceClient.extractMessages(ChannelIdentifier.Id(it.channelId), Path(path))
|
||||
}
|
||||
} else {
|
||||
spaceClient.extractMessages(chatId, Path(path))
|
||||
spaceClient.extractMessages(ChannelIdentifier.Id(channelId), Path(path))
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -214,6 +214,63 @@ private class ExtractMessagesCommand : ExtractCommand("messages", "Extract all m
|
||||
}
|
||||
}
|
||||
|
||||
private class ExtractDirectCommand : Subcommand("direct", "Extract direct messages") {
|
||||
|
||||
val url by argument(
|
||||
ArgType.String,
|
||||
description = "Url of the folder like 'https://spc.jetbrains.space/p/mipt-npm/documents/folders?f=SPC-qn7al1VorKp' or 'https://spc.jetbrains.space/p/mipt-npm/documents/SPC/f/SPC-qn7al1VorKp?f=SPC-qn7al1VorKp'"
|
||||
)
|
||||
|
||||
val token by option(
|
||||
ArgType.String,
|
||||
description = "A permanent token. Must have 'View direct messages', 'View messages' and 'View profile' access."
|
||||
).required()
|
||||
|
||||
val path: String by option(
|
||||
ArgType.String,
|
||||
description = "Target directory."
|
||||
).default("./messages")
|
||||
|
||||
override fun execute() {
|
||||
val urlMatch = urlRegex.matchEntire(url) ?: error("Url $url does not match space document url pattern")
|
||||
|
||||
val spaceUrl = urlMatch.groups["spaceUrl"]?.value ?: error("Space Url token not recognized")
|
||||
|
||||
val profileName = urlMatch.groups["profileName"]?.value
|
||||
|
||||
val spaceClient = SpaceClient(
|
||||
ktorClient = ktorClientForSpace(CIO),
|
||||
serverUrl = spaceUrl,
|
||||
token = token//"eyJhbGciOiJSUzUxMiJ9.eyJzdWIiOiJOcHJJcjFNUU5layIsImF1ZCI6ImNpcmNsZXQtd2ViLXVpIiwib3JnRG9tYWluIjoic3BjIiwibmFtZSI6ImFsdGF2aXIiLCJpc3MiOiJodHRwczpcL1wvc3BjLmpldGJyYWlucy5zcGFjZSIsInBlcm1fdG9rZW4iOiIycDRtSW4zZ281SUciLCJwcmluY2lwYWxfdHlwZSI6IlVTRVIiLCJpYXQiOjE2OTM1Nzg2NDd9.anDeWjBctaC4FWdjDvGS7KqWaScrE1hC2CHzLSd3K_g-xcxtcqMnls8AzjCWSTeyAG5bWsXak73t2JiUqf6LjQnUkXidLhS33Odq5defl-a2QABxWNehCHQJlDQmFX20Hh3WUCqzIxpON_1eAtN5iWXnsxdMryzR7MbwsyLTdhM"
|
||||
)
|
||||
|
||||
runBlocking {
|
||||
if (profileName == null) {
|
||||
spaceClient.teamDirectory.profiles.getAllProfiles(
|
||||
batchInfo = BatchInfo(
|
||||
offset = null,
|
||||
batchSize = 1000
|
||||
)
|
||||
) {
|
||||
id()
|
||||
}.data.forEach {
|
||||
spaceClient.extractMessages(ChannelIdentifier.Profile(ProfileIdentifier.Id(it.id)), Path(path))
|
||||
}
|
||||
} else {
|
||||
spaceClient.extractMessages(
|
||||
ChannelIdentifier.Profile(ProfileIdentifier.Username(profileName)),
|
||||
Path(path)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val urlRegex =
|
||||
"""(?<spaceUrl>https?:\/\/[^\/]*)(\/im\/user\/(?<profileName>.*))?""".toRegex()
|
||||
}
|
||||
}
|
||||
|
||||
private class ExtractProjectCommand : ExtractCommand("project", "Extract all data from a project") {
|
||||
|
||||
val path: String by option(
|
||||
@ -283,7 +340,8 @@ fun main(args: Array<String>) {
|
||||
ExtractDocumentsCommand(),
|
||||
ExtractRepositoriesCommand(),
|
||||
ExtractProjectCommand(),
|
||||
ExtractMessagesCommand()
|
||||
ExtractChannelsCommand(),
|
||||
ExtractDirectCommand()
|
||||
)
|
||||
|
||||
parser.parse(args)
|
||||
|
Loading…
Reference in New Issue
Block a user