seang96/gmessages
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Add basic support for incoming media messages

Browse source
This commit is contained in:
Tulir Asokan 2023-07-09 20:28:20 +03:00
parent 5542492a32
commit 7372bc4927
5 changed files with 106 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
ROADMAP.md
View file

@ -10,7 +10,7 @@
* Google Messages → Matrix
* [ ] Message content
* [x] Plain text
* [ ] Media/files
* [x] Media/files
* [ ] Replies (RCS)
* [ ] Reactions
* [ ] Typing notifications

1
go.mod
View file

@ -13,6 +13,7 @@ require (
require (
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect

2
go.sum
View file

@ -3,6 +3,8 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=

6
libgm/client.go
View file

@ -212,7 +212,7 @@ func (c *Client) decryptMedias(messages *binary.FetchMessagesResponse) error {
for _, details := range msg.GetMessageInfo() {
switch data := details.GetData().(type) {
case *binary.MessageInfo_MediaContent:
decryptedMediaData, err := c.decryptMediaData(data.MediaContent.MediaID, data.MediaContent.DecryptionKey)
decryptedMediaData, err := c.DownloadMedia(data.MediaContent.MediaID, data.MediaContent.DecryptionKey)
if err != nil {
log.Fatal(err)
return err
@ -224,11 +224,11 @@ func (c *Client) decryptMedias(messages *binary.FetchMessagesResponse) error {
return nil
}
func (c *Client) decryptMediaData(mediaId string, key []byte) ([]byte, error) {
func (c *Client) DownloadMedia(mediaID string, key []byte) ([]byte, error) {
reqId := util.RandomUUIDv4()
download_metadata := &binary.UploadImagePayload{
MetaData: &binary.ImageMetaData{
ImageID: mediaId,
ImageID: mediaID,
Encrypted: true,
},
AuthData: &binary.AuthMessage{

102
portal.go
View file

@ -27,6 +27,7 @@ import (
"sync"
"time"
"github.com/gabriel-vasile/mimetype"
"github.com/rs/zerolog"
log "maunium.net/go/maulogger/v2"
@ -312,6 +313,16 @@ func (portal *Portal) isOutgoingMessage(evt *binary.Message) id.EventID {
return ""
}
func hasInProgressMedia(msg *binary.Message) bool {
for _, part := range msg.MessageInfo {
media, ok := part.GetData().(*binary.MessageInfo_MediaContent)
if ok && media.MediaContent.GetMediaID() == "" {
return true
}
}
return false
}
func (portal *Portal) handleMessage(source *User, evt *binary.Message) {
if len(portal.MXID) == 0 {
portal.zlog.Warn().Msg("handleMessage called even though portal.MXID is empty")
@ -322,6 +333,15 @@ func (portal *Portal) handleMessage(source *User, evt *binary.Message) {
Str("participant_id", evt.ParticipantID).
Str("action", "handleMessage").
Logger()
switch evt.GetMessageStatus().GetStatus() {
case binary.MessageStatusType_INCOMING_AUTO_DOWNLOADING, binary.MessageStatusType_INCOMING_RETRYING_AUTO_DOWNLOAD:
log.Debug().Msg("Not handling incoming message that is auto downloading")
return
}
if hasInProgressMedia(evt) {
log.Debug().Msg("Not handling incoming message that doesn't have full media yet")
return
}
if evtID := portal.isOutgoingMessage(evt); evtID != "" {
log.Debug().Str("event_id", evtID.String()).Msg("Got echo for outgoing message")
return
@ -366,9 +386,15 @@ func (portal *Portal) handleMessage(source *User, evt *binary.Message) {
Body: data.MessageContent.GetContent(),
}
case *binary.MessageInfo_MediaContent:
content = event.MessageEventContent{
MsgType: event.MsgNotice,
Body: fmt.Sprintf("Attachment %s", data.MediaContent.GetMediaName()),
contentPtr, err := portal.convertGoogleMedia(source, intent, data.MediaContent)
if err != nil {
log.Err(err).Msg("Failed to copy attachment")
content = event.MessageEventContent{
MsgType: event.MsgNotice,
Body: fmt.Sprintf("Failed to transfer attachment %s", data.MediaContent.GetMediaName()),
}
} else {
content = *contentPtr
}
}
resp, err := portal.sendMessage(intent, event.EventMessage, &content, nil, ts)
@ -384,6 +410,76 @@ func (portal *Portal) handleMessage(source *User, evt *binary.Message) {
log.Debug().Interface("event_ids", eventIDs).Msg("Handled message")
}
var mediaFormatToMime = map[binary.MediaFormats]string{
binary.MediaFormats_UNSPECIFIED_TYPE: "",
binary.MediaFormats_IMAGE_JPEG: "image/jpeg",
binary.MediaFormats_IMAGE_JPG: "image/jpeg",
binary.MediaFormats_IMAGE_PNG: "image/png",
binary.MediaFormats_IMAGE_GIF: "image/gif",
binary.MediaFormats_IMAGE_WBMP: "image/vnd.wap.vbmp",
binary.MediaFormats_IMAGE_X_MS_BMP: "image/bmp",
binary.MediaFormats_IMAGE_UNSPECIFIED: "",
binary.MediaFormats_VIDEO_MP4: "video/mp4",
binary.MediaFormats_VIDEO_3G2: "video/3gpp2",
binary.MediaFormats_VIDEO_3GPP: "video/3gpp",
binary.MediaFormats_VIDEO_WEBM: "video/webm",
binary.MediaFormats_VIDEO_MKV: "video/x-matroska",
binary.MediaFormats_VIDEO_UNSPECIFIED: "",
binary.MediaFormats_AUDIO_AAC: "audio/aac",
binary.MediaFormats_AUDIO_AMR: "audio/amr",
binary.MediaFormats_AUDIO_MP3: "audio/mp3",
binary.MediaFormats_AUDIO_MPEG: "audio/mpeg",
binary.MediaFormats_AUDIO_MPG: "audio/mpeg",
binary.MediaFormats_AUDIO_MP4: "audio/mp4",
binary.MediaFormats_AUDIO_MP4_LATM: "audio/mp4a-latm",
binary.MediaFormats_AUDIO_3GPP: "audio/3gpp",
binary.MediaFormats_AUDIO_OGG: "audio/ogg",
binary.MediaFormats_AUDIO_OGG2: "audio/ogg",
binary.MediaFormats_AUDIO_UNSPECIFIED: "",
binary.MediaFormats_TEXT_VCARD: "text/vcard",
binary.MediaFormats_APP_PDF: "application/pdf",
binary.MediaFormats_APP_TXT: "text/plain",
binary.MediaFormats_APP_HTML: "text/html",
binary.MediaFormats_APP_SMIL: "application/smil",
}
func (portal *Portal) convertGoogleMedia(source *User, intent *appservice.IntentAPI, msg *binary.MediaContent) (*event.MessageEventContent, error) {
var data []byte
var err error
data, err = source.Client.DownloadMedia(msg.MediaID, msg.DecryptionKey)
if err != nil {
return nil, err
}
mime := mediaFormatToMime[msg.GetFormat()]
if mime == "" {
mime = mimetype.Detect(data).String()
}
msgtype := event.MsgFile
switch strings.Split(mime, "/")[0] {
case "image":
msgtype = event.MsgImage
case "video":
msgtype = event.MsgVideo
// TODO convert weird formats to mp4
case "audio":
msgtype = event.MsgAudio
// TODO convert everything to ogg and include voice message metadata
}
content := &event.MessageEventContent{
MsgType: msgtype,
Body: msg.MediaName,
Info: &event.FileInfo{
MimeType: mime,
Size: len(data),
},
}
return content, portal.uploadMedia(intent, data, content)
}
func (portal *Portal) isRecentlyHandled(id string) bool {
start := portal.recentlyHandledIndex
for i := start; i != start; i = (i - 1) % recentlyHandledLength {