From 7372bc492727046b6f530ca6a8eb64f5ec120408 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 9 Jul 2023 20:28:20 +0300 Subject: [PATCH] Add basic support for incoming media messages --- ROADMAP.md | 2 +- go.mod | 1 + go.sum | 2 + libgm/client.go | 6 +-- portal.go | 102 ++++++++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 106 insertions(+), 7 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 7620990..eaee429 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -10,7 +10,7 @@ * Google Messages → Matrix * [ ] Message content * [x] Plain text - * [ ] Media/files + * [x] Media/files * [ ] Replies (RCS) * [ ] Reactions * [ ] Typing notifications diff --git a/go.mod b/go.mod index 9e438b0..1c754b4 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( require ( github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/mux v1.8.0 // indirect diff --git a/go.sum b/go.sum index ee5b803..a31e338 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= diff --git a/libgm/client.go b/libgm/client.go index 76f25e4..a218c79 100644 --- a/libgm/client.go +++ b/libgm/client.go @@ -212,7 +212,7 @@ func (c *Client) decryptMedias(messages *binary.FetchMessagesResponse) error { for _, details := range msg.GetMessageInfo() { switch data := details.GetData().(type) { case *binary.MessageInfo_MediaContent: - decryptedMediaData, err := c.decryptMediaData(data.MediaContent.MediaID, data.MediaContent.DecryptionKey) + decryptedMediaData, err := c.DownloadMedia(data.MediaContent.MediaID, data.MediaContent.DecryptionKey) if err != nil { log.Fatal(err) return err @@ -224,11 +224,11 @@ func (c *Client) decryptMedias(messages *binary.FetchMessagesResponse) error { return nil } -func (c *Client) decryptMediaData(mediaId string, key []byte) ([]byte, error) { +func (c *Client) DownloadMedia(mediaID string, key []byte) ([]byte, error) { reqId := util.RandomUUIDv4() download_metadata := &binary.UploadImagePayload{ MetaData: &binary.ImageMetaData{ - ImageID: mediaId, + ImageID: mediaID, Encrypted: true, }, AuthData: &binary.AuthMessage{ diff --git a/portal.go b/portal.go index 78aa44f..bab856c 100644 --- a/portal.go +++ b/portal.go @@ -27,6 +27,7 @@ import ( "sync" "time" + "github.com/gabriel-vasile/mimetype" "github.com/rs/zerolog" log "maunium.net/go/maulogger/v2" @@ -312,6 +313,16 @@ func (portal *Portal) isOutgoingMessage(evt *binary.Message) id.EventID { return "" } +func hasInProgressMedia(msg *binary.Message) bool { + for _, part := range msg.MessageInfo { + media, ok := part.GetData().(*binary.MessageInfo_MediaContent) + if ok && media.MediaContent.GetMediaID() == "" { + return true + } + } + return false +} + func (portal *Portal) handleMessage(source *User, evt *binary.Message) { if len(portal.MXID) == 0 { portal.zlog.Warn().Msg("handleMessage called even though portal.MXID is empty") @@ -322,6 +333,15 @@ func (portal *Portal) handleMessage(source *User, evt *binary.Message) { Str("participant_id", evt.ParticipantID). Str("action", "handleMessage"). Logger() + switch evt.GetMessageStatus().GetStatus() { + case binary.MessageStatusType_INCOMING_AUTO_DOWNLOADING, binary.MessageStatusType_INCOMING_RETRYING_AUTO_DOWNLOAD: + log.Debug().Msg("Not handling incoming message that is auto downloading") + return + } + if hasInProgressMedia(evt) { + log.Debug().Msg("Not handling incoming message that doesn't have full media yet") + return + } if evtID := portal.isOutgoingMessage(evt); evtID != "" { log.Debug().Str("event_id", evtID.String()).Msg("Got echo for outgoing message") return @@ -366,9 +386,15 @@ func (portal *Portal) handleMessage(source *User, evt *binary.Message) { Body: data.MessageContent.GetContent(), } case *binary.MessageInfo_MediaContent: - content = event.MessageEventContent{ - MsgType: event.MsgNotice, - Body: fmt.Sprintf("Attachment %s", data.MediaContent.GetMediaName()), + contentPtr, err := portal.convertGoogleMedia(source, intent, data.MediaContent) + if err != nil { + log.Err(err).Msg("Failed to copy attachment") + content = event.MessageEventContent{ + MsgType: event.MsgNotice, + Body: fmt.Sprintf("Failed to transfer attachment %s", data.MediaContent.GetMediaName()), + } + } else { + content = *contentPtr } } resp, err := portal.sendMessage(intent, event.EventMessage, &content, nil, ts) @@ -384,6 +410,76 @@ func (portal *Portal) handleMessage(source *User, evt *binary.Message) { log.Debug().Interface("event_ids", eventIDs).Msg("Handled message") } +var mediaFormatToMime = map[binary.MediaFormats]string{ + binary.MediaFormats_UNSPECIFIED_TYPE: "", + + binary.MediaFormats_IMAGE_JPEG: "image/jpeg", + binary.MediaFormats_IMAGE_JPG: "image/jpeg", + binary.MediaFormats_IMAGE_PNG: "image/png", + binary.MediaFormats_IMAGE_GIF: "image/gif", + binary.MediaFormats_IMAGE_WBMP: "image/vnd.wap.vbmp", + binary.MediaFormats_IMAGE_X_MS_BMP: "image/bmp", + binary.MediaFormats_IMAGE_UNSPECIFIED: "", + + binary.MediaFormats_VIDEO_MP4: "video/mp4", + binary.MediaFormats_VIDEO_3G2: "video/3gpp2", + binary.MediaFormats_VIDEO_3GPP: "video/3gpp", + binary.MediaFormats_VIDEO_WEBM: "video/webm", + binary.MediaFormats_VIDEO_MKV: "video/x-matroska", + binary.MediaFormats_VIDEO_UNSPECIFIED: "", + + binary.MediaFormats_AUDIO_AAC: "audio/aac", + binary.MediaFormats_AUDIO_AMR: "audio/amr", + binary.MediaFormats_AUDIO_MP3: "audio/mp3", + binary.MediaFormats_AUDIO_MPEG: "audio/mpeg", + binary.MediaFormats_AUDIO_MPG: "audio/mpeg", + binary.MediaFormats_AUDIO_MP4: "audio/mp4", + binary.MediaFormats_AUDIO_MP4_LATM: "audio/mp4a-latm", + binary.MediaFormats_AUDIO_3GPP: "audio/3gpp", + binary.MediaFormats_AUDIO_OGG: "audio/ogg", + binary.MediaFormats_AUDIO_OGG2: "audio/ogg", + binary.MediaFormats_AUDIO_UNSPECIFIED: "", + + binary.MediaFormats_TEXT_VCARD: "text/vcard", + binary.MediaFormats_APP_PDF: "application/pdf", + binary.MediaFormats_APP_TXT: "text/plain", + binary.MediaFormats_APP_HTML: "text/html", + binary.MediaFormats_APP_SMIL: "application/smil", +} + +func (portal *Portal) convertGoogleMedia(source *User, intent *appservice.IntentAPI, msg *binary.MediaContent) (*event.MessageEventContent, error) { + var data []byte + var err error + data, err = source.Client.DownloadMedia(msg.MediaID, msg.DecryptionKey) + if err != nil { + return nil, err + } + mime := mediaFormatToMime[msg.GetFormat()] + if mime == "" { + mime = mimetype.Detect(data).String() + } + msgtype := event.MsgFile + switch strings.Split(mime, "/")[0] { + case "image": + msgtype = event.MsgImage + case "video": + msgtype = event.MsgVideo + // TODO convert weird formats to mp4 + case "audio": + msgtype = event.MsgAudio + // TODO convert everything to ogg and include voice message metadata + } + content := &event.MessageEventContent{ + MsgType: msgtype, + Body: msg.MediaName, + Info: &event.FileInfo{ + MimeType: mime, + Size: len(data), + }, + } + return content, portal.uploadMedia(intent, data, content) +} + func (portal *Portal) isRecentlyHandled(id string) bool { start := portal.recentlyHandledIndex for i := start; i != start; i = (i - 1) % recentlyHandledLength {