From e2d3c108c2697f61d51b554f3f922f82f246ac53 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 1 Sep 2023 19:00:31 +0300 Subject: [PATCH] Add direct message export --- build.gradle.kts | 3 +- src/main/kotlin/extractMessages.kt | 36 +++++++++---- src/main/kotlin/main.kt | 82 +++++++++++++++++++++++++----- 3 files changed, 98 insertions(+), 23 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 3a2ad75..3f8d526 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,5 +1,4 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { kotlin("jvm") version "1.8.22" @@ -15,7 +14,7 @@ repositories { maven("https://maven.pkg.jetbrains.space/public/p/space/maven") } -val ktorVersion = "2.3.3" +val ktorVersion = "2.3.4" dependencies { implementation("org.jetbrains:space-sdk-jvm:167818-beta") diff --git a/src/main/kotlin/extractMessages.kt b/src/main/kotlin/extractMessages.kt index 08bce32..e3e97d3 100644 --- a/src/main/kotlin/extractMessages.kt +++ b/src/main/kotlin/extractMessages.kt @@ -1,6 +1,5 @@ package center.sciprog.space.documentextractor -import kotlinx.datetime.Clock import kotlinx.datetime.Instant import space.jetbrains.api.runtime.SpaceClient import space.jetbrains.api.runtime.resources.chats @@ -17,13 +16,13 @@ private suspend fun SpaceClient.writeMessages( id: ChannelIdentifier, prefix: String = "", ) { - var readDateTime: Instant? = Clock.System.now() + var readDateTime: Instant? = null var read: Int //reading messages in batches do { val result: GetMessagesResponse = chats.messages.getChannelMessages( channel = id, - sorting = MessagesSorting.FromNewestToOldest, + sorting = MessagesSorting.FromOldestToNewest, startFromDate = readDateTime, batchSize = 50 ) { @@ -59,7 +58,7 @@ private suspend fun SpaceClient.writeMessages( val name = "${attachment.id}-${attachment.filename}" val file = attachmentsDirectory.resolve(name) extractFile(file, fileId) - writer.appendLine("*Attachment*: [name](attachments/$name)\n") + writer.appendLine("*Attachment*: [$name](attachments/$name)\n") } is ImageAttachment -> { @@ -96,17 +95,36 @@ private suspend fun SpaceClient.writeMessages( suspend fun SpaceClient.extractMessages( - chatId: String, + id: ChannelIdentifier, parentDirectory: Path, ) { - val id = ChannelIdentifier.Id(chatId) - val channel = chats.channels.getChannel(id) + val channel = chats.channels.getChannel(id){ + content { + member { + name() + username() + } + } + contact{ + key() + } + totalMessages() + } - val name = (channel.contact.ext as? M2SharedChannelContent)?.name ?: channel.contact.defaultName + if (channel.totalMessages == 0){ + logger.debug("Channel with {} is empty", id) + return + } + + val name = when(val ext = channel.content){ + is M2SharedChannelContent -> ext.name + is M2ChannelContentMember -> ext.member.username + else -> channel.contact.key + } val file = parentDirectory.resolve("$name.md") - logger.info("Extracting messages from channel $chatId to $file") + logger.info("Extracting messages from channel $id to $file") file.parent.createDirectories() diff --git a/src/main/kotlin/main.kt b/src/main/kotlin/main.kt index 22fd92a..ab32332 100644 --- a/src/main/kotlin/main.kt +++ b/src/main/kotlin/main.kt @@ -6,13 +6,13 @@ import io.ktor.client.engine.cio.CIO import kotlinx.cli.* import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking -import space.jetbrains.api.runtime.SpaceAppInstance -import space.jetbrains.api.runtime.SpaceAuth -import space.jetbrains.api.runtime.SpaceClient -import space.jetbrains.api.runtime.ktorClientForSpace +import space.jetbrains.api.runtime.* import space.jetbrains.api.runtime.resources.chats import space.jetbrains.api.runtime.resources.projects +import space.jetbrains.api.runtime.resources.teamDirectory +import space.jetbrains.api.runtime.types.ChannelIdentifier import space.jetbrains.api.runtime.types.FolderIdentifier +import space.jetbrains.api.runtime.types.ProfileIdentifier import space.jetbrains.api.runtime.types.ProjectIdentifier import java.nio.file.Files import java.nio.file.Path @@ -171,19 +171,19 @@ private class ExtractRepositoriesCommand : ExtractCommand("repos", "Extract repo } -private class ExtractMessagesCommand : ExtractCommand("messages", "Extract all messages from a chat") { +private class ExtractChannelsCommand : ExtractCommand("channels", "Extract all messages from a channels") { val path: String by option( ArgType.String, - description = "Target directory. Default is './chats'." - ).default("./chats") + description = "Target directory." + ).default("./channels") override fun execute() { val urlMatch = urlRegex.matchEntire(url) ?: error("Url $url does not match space document url pattern") val spaceUrl = urlMatch.groups["spaceUrl"]?.value ?: error("Space Url token not recognized") - val chatId = urlMatch.groups["chatId"]?.value + val channelId = urlMatch.groups["chatId"]?.value val appInstance = SpaceAppInstance( clientId ?: System.getProperty("space.clientId"), @@ -198,12 +198,12 @@ private class ExtractMessagesCommand : ExtractCommand("messages", "Extract all m ) runBlocking { - if (chatId == null) { + if (channelId == null) { spaceClient.chats.channels.listAllChannels(query = "").data.forEach { - spaceClient.extractMessages(it.channelId, Path(path)) + spaceClient.extractMessages(ChannelIdentifier.Id(it.channelId), Path(path)) } } else { - spaceClient.extractMessages(chatId, Path(path)) + spaceClient.extractMessages(ChannelIdentifier.Id(channelId), Path(path)) } } } @@ -214,6 +214,63 @@ private class ExtractMessagesCommand : ExtractCommand("messages", "Extract all m } } +private class ExtractDirectCommand : Subcommand("direct", "Extract direct messages") { + + val url by argument( + ArgType.String, + description = "Url of the folder like 'https://spc.jetbrains.space/p/mipt-npm/documents/folders?f=SPC-qn7al1VorKp' or 'https://spc.jetbrains.space/p/mipt-npm/documents/SPC/f/SPC-qn7al1VorKp?f=SPC-qn7al1VorKp'" + ) + + val token by option( + ArgType.String, + description = "A permanent token. Must have 'View direct messages', 'View messages' and 'View profile' access." + ).required() + + val path: String by option( + ArgType.String, + description = "Target directory." + ).default("./messages") + + override fun execute() { + val urlMatch = urlRegex.matchEntire(url) ?: error("Url $url does not match space document url pattern") + + val spaceUrl = urlMatch.groups["spaceUrl"]?.value ?: error("Space Url token not recognized") + + val profileName = urlMatch.groups["profileName"]?.value + + val spaceClient = SpaceClient( + ktorClient = ktorClientForSpace(CIO), + serverUrl = spaceUrl, + token = token//"eyJhbGciOiJSUzUxMiJ9.eyJzdWIiOiJOcHJJcjFNUU5layIsImF1ZCI6ImNpcmNsZXQtd2ViLXVpIiwib3JnRG9tYWluIjoic3BjIiwibmFtZSI6ImFsdGF2aXIiLCJpc3MiOiJodHRwczpcL1wvc3BjLmpldGJyYWlucy5zcGFjZSIsInBlcm1fdG9rZW4iOiIycDRtSW4zZ281SUciLCJwcmluY2lwYWxfdHlwZSI6IlVTRVIiLCJpYXQiOjE2OTM1Nzg2NDd9.anDeWjBctaC4FWdjDvGS7KqWaScrE1hC2CHzLSd3K_g-xcxtcqMnls8AzjCWSTeyAG5bWsXak73t2JiUqf6LjQnUkXidLhS33Odq5defl-a2QABxWNehCHQJlDQmFX20Hh3WUCqzIxpON_1eAtN5iWXnsxdMryzR7MbwsyLTdhM" + ) + + runBlocking { + if (profileName == null) { + spaceClient.teamDirectory.profiles.getAllProfiles( + batchInfo = BatchInfo( + offset = null, + batchSize = 1000 + ) + ) { + id() + }.data.forEach { + spaceClient.extractMessages(ChannelIdentifier.Profile(ProfileIdentifier.Id(it.id)), Path(path)) + } + } else { + spaceClient.extractMessages( + ChannelIdentifier.Profile(ProfileIdentifier.Username(profileName)), + Path(path) + ) + } + } + } + + companion object { + private val urlRegex = + """(?https?:\/\/[^\/]*)(\/im\/user\/(?.*))?""".toRegex() + } +} + private class ExtractProjectCommand : ExtractCommand("project", "Extract all data from a project") { val path: String by option( @@ -283,7 +340,8 @@ fun main(args: Array) { ExtractDocumentsCommand(), ExtractRepositoriesCommand(), ExtractProjectCommand(), - ExtractMessagesCommand() + ExtractChannelsCommand(), + ExtractDirectCommand() ) parser.parse(args)