Add direct message export

This commit is contained in:
Alexander Nozik 2023-09-01 19:00:31 +03:00
parent 3eb43d4d19
commit e2d3c108c2
3 changed files with 98 additions and 23 deletions

View File

@ -1,5 +1,4 @@
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
kotlin("jvm") version "1.8.22"
@ -15,7 +14,7 @@ repositories {
maven("https://maven.pkg.jetbrains.space/public/p/space/maven")
}
val ktorVersion = "2.3.3"
val ktorVersion = "2.3.4"
dependencies {
implementation("org.jetbrains:space-sdk-jvm:167818-beta")

View File

@ -1,6 +1,5 @@
package center.sciprog.space.documentextractor
import kotlinx.datetime.Clock
import kotlinx.datetime.Instant
import space.jetbrains.api.runtime.SpaceClient
import space.jetbrains.api.runtime.resources.chats
@ -17,13 +16,13 @@ private suspend fun SpaceClient.writeMessages(
id: ChannelIdentifier,
prefix: String = "",
) {
var readDateTime: Instant? = Clock.System.now()
var readDateTime: Instant? = null
var read: Int
//reading messages in batches
do {
val result: GetMessagesResponse = chats.messages.getChannelMessages(
channel = id,
sorting = MessagesSorting.FromNewestToOldest,
sorting = MessagesSorting.FromOldestToNewest,
startFromDate = readDateTime,
batchSize = 50
) {
@ -59,7 +58,7 @@ private suspend fun SpaceClient.writeMessages(
val name = "${attachment.id}-${attachment.filename}"
val file = attachmentsDirectory.resolve(name)
extractFile(file, fileId)
writer.appendLine("*Attachment*: [name](attachments/$name)\n")
writer.appendLine("*Attachment*: [$name](attachments/$name)\n")
}
is ImageAttachment -> {
@ -96,17 +95,36 @@ private suspend fun SpaceClient.writeMessages(
suspend fun SpaceClient.extractMessages(
chatId: String,
id: ChannelIdentifier,
parentDirectory: Path,
) {
val id = ChannelIdentifier.Id(chatId)
val channel = chats.channels.getChannel(id)
val channel = chats.channels.getChannel(id){
content {
member {
name()
username()
}
}
contact{
key()
}
totalMessages()
}
val name = (channel.contact.ext as? M2SharedChannelContent)?.name ?: channel.contact.defaultName
if (channel.totalMessages == 0){
logger.debug("Channel with {} is empty", id)
return
}
val name = when(val ext = channel.content){
is M2SharedChannelContent -> ext.name
is M2ChannelContentMember -> ext.member.username
else -> channel.contact.key
}
val file = parentDirectory.resolve("$name.md")
logger.info("Extracting messages from channel $chatId to $file")
logger.info("Extracting messages from channel $id to $file")
file.parent.createDirectories()

View File

@ -6,13 +6,13 @@ import io.ktor.client.engine.cio.CIO
import kotlinx.cli.*
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import space.jetbrains.api.runtime.SpaceAppInstance
import space.jetbrains.api.runtime.SpaceAuth
import space.jetbrains.api.runtime.SpaceClient
import space.jetbrains.api.runtime.ktorClientForSpace
import space.jetbrains.api.runtime.*
import space.jetbrains.api.runtime.resources.chats
import space.jetbrains.api.runtime.resources.projects
import space.jetbrains.api.runtime.resources.teamDirectory
import space.jetbrains.api.runtime.types.ChannelIdentifier
import space.jetbrains.api.runtime.types.FolderIdentifier
import space.jetbrains.api.runtime.types.ProfileIdentifier
import space.jetbrains.api.runtime.types.ProjectIdentifier
import java.nio.file.Files
import java.nio.file.Path
@ -171,19 +171,19 @@ private class ExtractRepositoriesCommand : ExtractCommand("repos", "Extract repo
}
private class ExtractMessagesCommand : ExtractCommand("messages", "Extract all messages from a chat") {
private class ExtractChannelsCommand : ExtractCommand("channels", "Extract all messages from a channels") {
val path: String by option(
ArgType.String,
description = "Target directory. Default is './chats'."
).default("./chats")
description = "Target directory."
).default("./channels")
override fun execute() {
val urlMatch = urlRegex.matchEntire(url) ?: error("Url $url does not match space document url pattern")
val spaceUrl = urlMatch.groups["spaceUrl"]?.value ?: error("Space Url token not recognized")
val chatId = urlMatch.groups["chatId"]?.value
val channelId = urlMatch.groups["chatId"]?.value
val appInstance = SpaceAppInstance(
clientId ?: System.getProperty("space.clientId"),
@ -198,12 +198,12 @@ private class ExtractMessagesCommand : ExtractCommand("messages", "Extract all m
)
runBlocking {
if (chatId == null) {
if (channelId == null) {
spaceClient.chats.channels.listAllChannels(query = "").data.forEach {
spaceClient.extractMessages(it.channelId, Path(path))
spaceClient.extractMessages(ChannelIdentifier.Id(it.channelId), Path(path))
}
} else {
spaceClient.extractMessages(chatId, Path(path))
spaceClient.extractMessages(ChannelIdentifier.Id(channelId), Path(path))
}
}
}
@ -214,6 +214,63 @@ private class ExtractMessagesCommand : ExtractCommand("messages", "Extract all m
}
}
private class ExtractDirectCommand : Subcommand("direct", "Extract direct messages") {
val url by argument(
ArgType.String,
description = "Url of the folder like 'https://spc.jetbrains.space/p/mipt-npm/documents/folders?f=SPC-qn7al1VorKp' or 'https://spc.jetbrains.space/p/mipt-npm/documents/SPC/f/SPC-qn7al1VorKp?f=SPC-qn7al1VorKp'"
)
val token by option(
ArgType.String,
description = "A permanent token. Must have 'View direct messages', 'View messages' and 'View profile' access."
).required()
val path: String by option(
ArgType.String,
description = "Target directory."
).default("./messages")
override fun execute() {
val urlMatch = urlRegex.matchEntire(url) ?: error("Url $url does not match space document url pattern")
val spaceUrl = urlMatch.groups["spaceUrl"]?.value ?: error("Space Url token not recognized")
val profileName = urlMatch.groups["profileName"]?.value
val spaceClient = SpaceClient(
ktorClient = ktorClientForSpace(CIO),
serverUrl = spaceUrl,
token = token//"eyJhbGciOiJSUzUxMiJ9.eyJzdWIiOiJOcHJJcjFNUU5layIsImF1ZCI6ImNpcmNsZXQtd2ViLXVpIiwib3JnRG9tYWluIjoic3BjIiwibmFtZSI6ImFsdGF2aXIiLCJpc3MiOiJodHRwczpcL1wvc3BjLmpldGJyYWlucy5zcGFjZSIsInBlcm1fdG9rZW4iOiIycDRtSW4zZ281SUciLCJwcmluY2lwYWxfdHlwZSI6IlVTRVIiLCJpYXQiOjE2OTM1Nzg2NDd9.anDeWjBctaC4FWdjDvGS7KqWaScrE1hC2CHzLSd3K_g-xcxtcqMnls8AzjCWSTeyAG5bWsXak73t2JiUqf6LjQnUkXidLhS33Odq5defl-a2QABxWNehCHQJlDQmFX20Hh3WUCqzIxpON_1eAtN5iWXnsxdMryzR7MbwsyLTdhM"
)
runBlocking {
if (profileName == null) {
spaceClient.teamDirectory.profiles.getAllProfiles(
batchInfo = BatchInfo(
offset = null,
batchSize = 1000
)
) {
id()
}.data.forEach {
spaceClient.extractMessages(ChannelIdentifier.Profile(ProfileIdentifier.Id(it.id)), Path(path))
}
} else {
spaceClient.extractMessages(
ChannelIdentifier.Profile(ProfileIdentifier.Username(profileName)),
Path(path)
)
}
}
}
companion object {
private val urlRegex =
"""(?<spaceUrl>https?:\/\/[^\/]*)(\/im\/user\/(?<profileName>.*))?""".toRegex()
}
}
private class ExtractProjectCommand : ExtractCommand("project", "Extract all data from a project") {
val path: String by option(
@ -283,7 +340,8 @@ fun main(args: Array<String>) {
ExtractDocumentsCommand(),
ExtractRepositoriesCommand(),
ExtractProjectCommand(),
ExtractMessagesCommand()
ExtractChannelsCommand(),
ExtractDirectCommand()
)
parser.parse(args)