From aa89c983538a9565b8beb4e92a48f1a4264e416f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 15 Jul 2023 20:08:11 +0300 Subject: [PATCH] Add support for sending attachments from Matrix --- ROADMAP.md | 6 +- libgm/binary/conversations.pb.go | 204 +++++++++++++++++---------- libgm/binary/raw/conversations.proto | 29 +++- libgm/image_builder.go | 178 ++++++++++------------- libgm/media_processor.go | 19 +-- messagetracking.go | 4 + portal.go | 144 +++++++++---------- 7 files changed, 305 insertions(+), 279 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index c25bd98..5544669 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,15 +1,15 @@ # Features & roadmap * Matrix → Google Messages - * [ ] Message content + * [x] Message content * [x] Plain text - * [ ] Media/files + * [x] Media/files * [x] Replies (RCS) * [x] Reactions (RCS) * [ ] Typing notifications (RCS) * [x] Read receipts (RCS) * [x] Message deletions (own device only) * Google Messages → Matrix - * [ ] Message content + * [x] Message content * [x] Plain text * [x] Media/files * [x] Replies (RCS) diff --git a/libgm/binary/conversations.pb.go b/libgm/binary/conversations.pb.go index 6d46f9b..f6ab3a0 100644 --- a/libgm/binary/conversations.pb.go +++ b/libgm/binary/conversations.pb.go @@ -557,36 +557,50 @@ func (ConvUpdateTypes) EnumDescriptor() ([]byte, []int) { type MediaFormats int32 const ( - MediaFormats_UNSPECIFIED_TYPE MediaFormats = 0 - MediaFormats_IMAGE_JPEG MediaFormats = 1 - MediaFormats_IMAGE_JPG MediaFormats = 2 - MediaFormats_IMAGE_PNG MediaFormats = 3 - MediaFormats_IMAGE_GIF MediaFormats = 4 - MediaFormats_IMAGE_WBMP MediaFormats = 5 - MediaFormats_IMAGE_X_MS_BMP MediaFormats = 6 - MediaFormats_IMAGE_UNSPECIFIED MediaFormats = 7 - MediaFormats_VIDEO_MP4 MediaFormats = 8 - MediaFormats_VIDEO_3G2 MediaFormats = 9 - MediaFormats_VIDEO_3GPP MediaFormats = 10 - MediaFormats_VIDEO_WEBM MediaFormats = 11 - MediaFormats_VIDEO_MKV MediaFormats = 12 - MediaFormats_VIDEO_UNSPECIFIED MediaFormats = 13 - MediaFormats_AUDIO_AAC MediaFormats = 14 - MediaFormats_AUDIO_AMR MediaFormats = 15 - MediaFormats_AUDIO_MP3 MediaFormats = 16 - MediaFormats_AUDIO_MPEG MediaFormats = 17 - MediaFormats_AUDIO_MPG MediaFormats = 18 - MediaFormats_AUDIO_MP4 MediaFormats = 19 - MediaFormats_AUDIO_MP4_LATM MediaFormats = 20 - MediaFormats_AUDIO_3GPP MediaFormats = 21 - MediaFormats_AUDIO_OGG MediaFormats = 22 - MediaFormats_AUDIO_UNSPECIFIED MediaFormats = 23 - MediaFormats_TEXT_VCARD MediaFormats = 24 - MediaFormats_APP_PDF MediaFormats = 28 - MediaFormats_APP_TXT MediaFormats = 29 - MediaFormats_APP_HTML MediaFormats = 30 - MediaFormats_AUDIO_OGG2 MediaFormats = 31 - MediaFormats_APP_SMIL MediaFormats = 32 + MediaFormats_UNSPECIFIED_TYPE MediaFormats = 0 + MediaFormats_IMAGE_JPEG MediaFormats = 1 + MediaFormats_IMAGE_JPG MediaFormats = 2 + MediaFormats_IMAGE_PNG MediaFormats = 3 + MediaFormats_IMAGE_GIF MediaFormats = 4 + MediaFormats_IMAGE_WBMP MediaFormats = 5 + MediaFormats_IMAGE_X_MS_BMP MediaFormats = 6 + MediaFormats_IMAGE_UNSPECIFIED MediaFormats = 7 + MediaFormats_VIDEO_MP4 MediaFormats = 8 + MediaFormats_VIDEO_3G2 MediaFormats = 9 + MediaFormats_VIDEO_3GPP MediaFormats = 10 + MediaFormats_VIDEO_WEBM MediaFormats = 11 + MediaFormats_VIDEO_MKV MediaFormats = 12 + MediaFormats_VIDEO_UNSPECIFIED MediaFormats = 13 + MediaFormats_AUDIO_AAC MediaFormats = 14 + MediaFormats_AUDIO_AMR MediaFormats = 15 + MediaFormats_AUDIO_MP3 MediaFormats = 16 + MediaFormats_AUDIO_MPEG MediaFormats = 17 + MediaFormats_AUDIO_MPG MediaFormats = 18 + MediaFormats_AUDIO_MP4 MediaFormats = 19 + MediaFormats_AUDIO_MP4_LATM MediaFormats = 20 + MediaFormats_AUDIO_3GPP MediaFormats = 21 + MediaFormats_AUDIO_OGG MediaFormats = 22 + MediaFormats_AUDIO_UNSPECIFIED MediaFormats = 23 + MediaFormats_TEXT_VCARD MediaFormats = 24 + MediaFormats_APP_PDF MediaFormats = 25 + MediaFormats_APP_TXT MediaFormats = 26 + MediaFormats_APP_HTML MediaFormats = 27 + MediaFormats_APP_DOC MediaFormats = 28 + MediaFormats_APP_DOCX MediaFormats = 29 + MediaFormats_APP_PPTX MediaFormats = 30 + MediaFormats_APP_PPT MediaFormats = 31 + MediaFormats_APP_XLSX MediaFormats = 32 + MediaFormats_APP_XLS MediaFormats = 33 + MediaFormats_APP_APK MediaFormats = 34 + MediaFormats_APP_ZIP MediaFormats = 35 + MediaFormats_APP_JAR MediaFormats = 36 + MediaFormats_APP_UNSPECIFIED MediaFormats = 37 + MediaFormats_CAL_TEXT_VCALENDAR MediaFormats = 38 + MediaFormats_CAL_TEXT_XVCALENDAR MediaFormats = 39 + MediaFormats_CAL_TEXT_CALENDAR MediaFormats = 40 + MediaFormats_CAL_APPLICATION_VCS MediaFormats = 41 + MediaFormats_CAL_APPLICATION_ICS MediaFormats = 42 + MediaFormats_CAL_APPLICATION_HBSVCS MediaFormats = 43 ) // Enum value maps for MediaFormats. @@ -617,43 +631,71 @@ var ( 22: "AUDIO_OGG", 23: "AUDIO_UNSPECIFIED", 24: "TEXT_VCARD", - 28: "APP_PDF", - 29: "APP_TXT", - 30: "APP_HTML", - 31: "AUDIO_OGG2", - 32: "APP_SMIL", + 25: "APP_PDF", + 26: "APP_TXT", + 27: "APP_HTML", + 28: "APP_DOC", + 29: "APP_DOCX", + 30: "APP_PPTX", + 31: "APP_PPT", + 32: "APP_XLSX", + 33: "APP_XLS", + 34: "APP_APK", + 35: "APP_ZIP", + 36: "APP_JAR", + 37: "APP_UNSPECIFIED", + 38: "CAL_TEXT_VCALENDAR", + 39: "CAL_TEXT_XVCALENDAR", + 40: "CAL_TEXT_CALENDAR", + 41: "CAL_APPLICATION_VCS", + 42: "CAL_APPLICATION_ICS", + 43: "CAL_APPLICATION_HBSVCS", } MediaFormats_value = map[string]int32{ - "UNSPECIFIED_TYPE": 0, - "IMAGE_JPEG": 1, - "IMAGE_JPG": 2, - "IMAGE_PNG": 3, - "IMAGE_GIF": 4, - "IMAGE_WBMP": 5, - "IMAGE_X_MS_BMP": 6, - "IMAGE_UNSPECIFIED": 7, - "VIDEO_MP4": 8, - "VIDEO_3G2": 9, - "VIDEO_3GPP": 10, - "VIDEO_WEBM": 11, - "VIDEO_MKV": 12, - "VIDEO_UNSPECIFIED": 13, - "AUDIO_AAC": 14, - "AUDIO_AMR": 15, - "AUDIO_MP3": 16, - "AUDIO_MPEG": 17, - "AUDIO_MPG": 18, - "AUDIO_MP4": 19, - "AUDIO_MP4_LATM": 20, - "AUDIO_3GPP": 21, - "AUDIO_OGG": 22, - "AUDIO_UNSPECIFIED": 23, - "TEXT_VCARD": 24, - "APP_PDF": 28, - "APP_TXT": 29, - "APP_HTML": 30, - "AUDIO_OGG2": 31, - "APP_SMIL": 32, + "UNSPECIFIED_TYPE": 0, + "IMAGE_JPEG": 1, + "IMAGE_JPG": 2, + "IMAGE_PNG": 3, + "IMAGE_GIF": 4, + "IMAGE_WBMP": 5, + "IMAGE_X_MS_BMP": 6, + "IMAGE_UNSPECIFIED": 7, + "VIDEO_MP4": 8, + "VIDEO_3G2": 9, + "VIDEO_3GPP": 10, + "VIDEO_WEBM": 11, + "VIDEO_MKV": 12, + "VIDEO_UNSPECIFIED": 13, + "AUDIO_AAC": 14, + "AUDIO_AMR": 15, + "AUDIO_MP3": 16, + "AUDIO_MPEG": 17, + "AUDIO_MPG": 18, + "AUDIO_MP4": 19, + "AUDIO_MP4_LATM": 20, + "AUDIO_3GPP": 21, + "AUDIO_OGG": 22, + "AUDIO_UNSPECIFIED": 23, + "TEXT_VCARD": 24, + "APP_PDF": 25, + "APP_TXT": 26, + "APP_HTML": 27, + "APP_DOC": 28, + "APP_DOCX": 29, + "APP_PPTX": 30, + "APP_PPT": 31, + "APP_XLSX": 32, + "APP_XLS": 33, + "APP_APK": 34, + "APP_ZIP": 35, + "APP_JAR": 36, + "APP_UNSPECIFIED": 37, + "CAL_TEXT_VCALENDAR": 38, + "CAL_TEXT_XVCALENDAR": 39, + "CAL_TEXT_CALENDAR": 40, + "CAL_APPLICATION_VCS": 41, + "CAL_APPLICATION_ICS": 42, + "CAL_APPLICATION_HBSVCS": 43, } ) @@ -3529,8 +3571,8 @@ var file_conversations_proto_rawDesc = []byte{ 0x41, 0x52, 0x43, 0x48, 0x49, 0x56, 0x45, 0x44, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x03, 0x12, 0x18, 0x0a, 0x14, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x45, 0x44, 0x5f, 0x41, 0x4e, 0x44, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x10, - 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x45, 0x44, 0x10, 0x06, 0x2a, 0xfb, - 0x03, 0x0a, 0x0c, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x73, 0x12, + 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x45, 0x44, 0x10, 0x06, 0x2a, 0x80, + 0x06, 0x0a, 0x0c, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x73, 0x12, 0x14, 0x0a, 0x10, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x49, 0x4d, 0x41, 0x47, 0x45, 0x5f, 0x4a, 0x50, 0x45, 0x47, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x49, 0x4d, 0x41, 0x47, 0x45, 0x5f, 0x4a, @@ -3558,12 +3600,28 @@ var file_conversations_proto_rawDesc = []byte{ 0x15, 0x0a, 0x11, 0x41, 0x55, 0x44, 0x49, 0x4f, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x17, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x45, 0x58, 0x54, 0x5f, 0x56, 0x43, 0x41, 0x52, 0x44, 0x10, 0x18, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x50, 0x50, 0x5f, 0x50, 0x44, - 0x46, 0x10, 0x1c, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x50, 0x50, 0x5f, 0x54, 0x58, 0x54, 0x10, 0x1d, - 0x12, 0x0c, 0x0a, 0x08, 0x41, 0x50, 0x50, 0x5f, 0x48, 0x54, 0x4d, 0x4c, 0x10, 0x1e, 0x12, 0x0e, - 0x0a, 0x0a, 0x41, 0x55, 0x44, 0x49, 0x4f, 0x5f, 0x4f, 0x47, 0x47, 0x32, 0x10, 0x1f, 0x12, 0x0c, - 0x0a, 0x08, 0x41, 0x50, 0x50, 0x5f, 0x53, 0x4d, 0x49, 0x4c, 0x10, 0x20, 0x42, 0x0e, 0x5a, 0x0c, - 0x2e, 0x2e, 0x2f, 0x2e, 0x2e, 0x2f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x46, 0x10, 0x19, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x50, 0x50, 0x5f, 0x54, 0x58, 0x54, 0x10, 0x1a, + 0x12, 0x0c, 0x0a, 0x08, 0x41, 0x50, 0x50, 0x5f, 0x48, 0x54, 0x4d, 0x4c, 0x10, 0x1b, 0x12, 0x0b, + 0x0a, 0x07, 0x41, 0x50, 0x50, 0x5f, 0x44, 0x4f, 0x43, 0x10, 0x1c, 0x12, 0x0c, 0x0a, 0x08, 0x41, + 0x50, 0x50, 0x5f, 0x44, 0x4f, 0x43, 0x58, 0x10, 0x1d, 0x12, 0x0c, 0x0a, 0x08, 0x41, 0x50, 0x50, + 0x5f, 0x50, 0x50, 0x54, 0x58, 0x10, 0x1e, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x50, 0x50, 0x5f, 0x50, + 0x50, 0x54, 0x10, 0x1f, 0x12, 0x0c, 0x0a, 0x08, 0x41, 0x50, 0x50, 0x5f, 0x58, 0x4c, 0x53, 0x58, + 0x10, 0x20, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x50, 0x50, 0x5f, 0x58, 0x4c, 0x53, 0x10, 0x21, 0x12, + 0x0b, 0x0a, 0x07, 0x41, 0x50, 0x50, 0x5f, 0x41, 0x50, 0x4b, 0x10, 0x22, 0x12, 0x0b, 0x0a, 0x07, + 0x41, 0x50, 0x50, 0x5f, 0x5a, 0x49, 0x50, 0x10, 0x23, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x50, 0x50, + 0x5f, 0x4a, 0x41, 0x52, 0x10, 0x24, 0x12, 0x13, 0x0a, 0x0f, 0x41, 0x50, 0x50, 0x5f, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x25, 0x12, 0x16, 0x0a, 0x12, 0x43, + 0x41, 0x4c, 0x5f, 0x54, 0x45, 0x58, 0x54, 0x5f, 0x56, 0x43, 0x41, 0x4c, 0x45, 0x4e, 0x44, 0x41, + 0x52, 0x10, 0x26, 0x12, 0x17, 0x0a, 0x13, 0x43, 0x41, 0x4c, 0x5f, 0x54, 0x45, 0x58, 0x54, 0x5f, + 0x58, 0x56, 0x43, 0x41, 0x4c, 0x45, 0x4e, 0x44, 0x41, 0x52, 0x10, 0x27, 0x12, 0x15, 0x0a, 0x11, + 0x43, 0x41, 0x4c, 0x5f, 0x54, 0x45, 0x58, 0x54, 0x5f, 0x43, 0x41, 0x4c, 0x45, 0x4e, 0x44, 0x41, + 0x52, 0x10, 0x28, 0x12, 0x17, 0x0a, 0x13, 0x43, 0x41, 0x4c, 0x5f, 0x41, 0x50, 0x50, 0x4c, 0x49, + 0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x56, 0x43, 0x53, 0x10, 0x29, 0x12, 0x17, 0x0a, 0x13, + 0x43, 0x41, 0x4c, 0x5f, 0x41, 0x50, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, + 0x49, 0x43, 0x53, 0x10, 0x2a, 0x12, 0x1a, 0x0a, 0x16, 0x43, 0x41, 0x4c, 0x5f, 0x41, 0x50, 0x50, + 0x4c, 0x49, 0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x48, 0x42, 0x53, 0x56, 0x43, 0x53, 0x10, + 0x2b, 0x42, 0x0e, 0x5a, 0x0c, 0x2e, 0x2e, 0x2f, 0x2e, 0x2e, 0x2f, 0x62, 0x69, 0x6e, 0x61, 0x72, + 0x79, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/libgm/binary/raw/conversations.proto b/libgm/binary/raw/conversations.proto index 94664a6..b439642 100644 --- a/libgm/binary/raw/conversations.proto +++ b/libgm/binary/raw/conversations.proto @@ -359,12 +359,14 @@ enum MediaFormats { IMAGE_WBMP = 5; IMAGE_X_MS_BMP = 6; IMAGE_UNSPECIFIED = 7; + VIDEO_MP4 = 8; VIDEO_3G2 = 9; VIDEO_3GPP = 10; VIDEO_WEBM = 11; VIDEO_MKV = 12; VIDEO_UNSPECIFIED = 13; + AUDIO_AAC = 14; AUDIO_AMR = 15; AUDIO_MP3 = 16; @@ -375,10 +377,27 @@ enum MediaFormats { AUDIO_3GPP = 21; AUDIO_OGG = 22; AUDIO_UNSPECIFIED = 23; + TEXT_VCARD = 24; - APP_PDF = 28; - APP_TXT = 29; - APP_HTML = 30; - AUDIO_OGG2 = 31; - APP_SMIL = 32; + + APP_PDF = 25; + APP_TXT = 26; + APP_HTML = 27; + APP_DOC = 28; + APP_DOCX = 29; + APP_PPTX = 30; + APP_PPT = 31; + APP_XLSX = 32; + APP_XLS = 33; + APP_APK = 34; + APP_ZIP = 35; + APP_JAR = 36; + APP_UNSPECIFIED = 37; + + CAL_TEXT_VCALENDAR = 38; + CAL_TEXT_XVCALENDAR = 39; + CAL_TEXT_CALENDAR = 40; + CAL_APPLICATION_VCS = 41; + CAL_APPLICATION_ICS = 42; + CAL_APPLICATION_HBSVCS = 43; } diff --git a/libgm/image_builder.go b/libgm/image_builder.go index 4a76441..d0affbf 100644 --- a/libgm/image_builder.go +++ b/libgm/image_builder.go @@ -1,116 +1,87 @@ package libgm import ( + "strings" + "go.mau.fi/mautrix-gmessages/libgm/binary" "go.mau.fi/mautrix-gmessages/libgm/crypto" - "go.mau.fi/mautrix-gmessages/libgm/util" ) type MediaType struct { Extension string Format string - Type int64 + Type binary.MediaFormats } -var MediaTypes = map[string]MediaType{ - "image/jpeg": {Extension: "jpeg", Format: "image/jpeg", Type: 1}, - "image/jpg": {Extension: "jpg", Format: "image/jpg", Type: 2}, - "image/png": {Extension: "png", Format: "image/png", Type: 3}, - "image/gif": {Extension: "gif", Format: "image/gif", Type: 4}, - "image/wbmp": {Extension: "wbmp", Format: "image/wbmp", Type: 5}, - "image/bmp": {Extension: "bmp", Format: "image/bmp", Type: 6}, - "image/x-ms-bmp": {Extension: "bmp", Format: "image/x-ms-bmp", Type: 6}, - "image": {Type: 7}, +var MimeToMediaType = map[string]MediaType{ + "image/jpeg": {Extension: "jpeg", Type: binary.MediaFormats_IMAGE_JPEG}, + "image/jpg": {Extension: "jpg", Type: binary.MediaFormats_IMAGE_JPG}, + "image/png": {Extension: "png", Type: binary.MediaFormats_IMAGE_PNG}, + "image/gif": {Extension: "gif", Type: binary.MediaFormats_IMAGE_GIF}, + "image/wbmp": {Extension: "wbmp", Type: binary.MediaFormats_IMAGE_WBMP}, + "image/bmp": {Extension: "bmp", Type: binary.MediaFormats_IMAGE_X_MS_BMP}, + "image/x-ms-bmp": {Extension: "bmp", Type: binary.MediaFormats_IMAGE_X_MS_BMP}, - "video/mp4": {Extension: "mp4", Format: "video/mp4", Type: 8}, - "video/3gpp2": {Extension: "3gpp2", Format: "video/3gpp2", Type: 9}, - "video/3gpp": {Extension: "3gpp", Format: "video/3gpp", Type: 10}, - "video/webm": {Extension: "webm", Format: "video/webm", Type: 11}, - "video/x-matroska": {Extension: "mkv", Format: "video/x-matroska", Type: 12}, - "video": {Type: 13}, + "video/mp4": {Extension: "mp4", Type: binary.MediaFormats_VIDEO_MP4}, + "video/3gpp2": {Extension: "3gpp2", Type: binary.MediaFormats_VIDEO_3G2}, + "video/3gpp": {Extension: "3gpp", Type: binary.MediaFormats_VIDEO_3GPP}, + "video/webm": {Extension: "webm", Type: binary.MediaFormats_VIDEO_WEBM}, + "video/x-matroska": {Extension: "mkv", Type: binary.MediaFormats_VIDEO_MKV}, - "audio/aac": {Extension: "aac", Format: "audio/aac", Type: 14}, - "audio/amr": {Extension: "amr", Format: "audio/amr", Type: 15}, - "audio/mp3": {Extension: "mp3", Format: "audio/mp3", Type: 16}, - "audio/mpeg": {Extension: "mpeg", Format: "audio/mpeg", Type: 17}, - "audio/mpg": {Extension: "mpg", Format: "audio/mpg", Type: 18}, - "audio/mp4": {Extension: "mp4", Format: "audio/mp4", Type: 19}, - "audio/mp4-latm": {Extension: "latm", Format: "audio/mp4-latm", Type: 20}, - "audio/3gpp": {Extension: "3gpp", Format: "audio/3gpp", Type: 21}, - "audio/ogg": {Extension: "ogg", Format: "audio/ogg", Type: 22}, - "auidio": {Type: 23}, + "audio/aac": {Extension: "aac", Type: binary.MediaFormats_AUDIO_AAC}, + "audio/amr": {Extension: "amr", Type: binary.MediaFormats_AUDIO_AMR}, + "audio/mp3": {Extension: "mp3", Type: binary.MediaFormats_AUDIO_MP3}, + "audio/mpeg": {Extension: "mpeg", Type: binary.MediaFormats_AUDIO_MPEG}, + "audio/mpg": {Extension: "mpg", Type: binary.MediaFormats_AUDIO_MPG}, + "audio/mp4": {Extension: "mp4", Type: binary.MediaFormats_AUDIO_MP4}, + "audio/mp4-latm": {Extension: "latm", Type: binary.MediaFormats_AUDIO_MP4_LATM}, + "audio/3gpp": {Extension: "3gpp", Type: binary.MediaFormats_AUDIO_3GPP}, + "audio/ogg": {Extension: "ogg", Type: binary.MediaFormats_AUDIO_OGG}, - "text/vcard": {Extension: "vcard", Format: "text/vcard", Type: 24}, - "text/x-vcard": {Extension: "vcard", Format: "text/x-vcard", Type: 24}, - "application/pdf": {Extension: "pdf", Format: "application/pdf", Type: 25}, - "text/plain": {Extension: "txt", Format: "text/plain", Type: 26}, - "text/html": {Extension: "html", Format: "text/html", Type: 27}, - "application/msword": {Extension: "doc", Format: "application/msword", Type: 28}, - "application/vnd.openxmlformats-officedocument.wordprocessingml.document": {Extension: "docx", Format: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", Type: 29}, - "application/vnd.openxmlformats-officedocument.presentationml.presentation": {Extension: "pptx", Format: "application/vnd.openxmlformats-officedocument.presentationml.presentation", Type: 30}, - "application/vnd.ms-powerpoint": {Extension: "ppt", Format: "application/vnd.ms-powerpoint", Type: 31}, - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": {Extension: "xlsx", Format: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", Type: 32}, - "application/vnd.ms-excel": {Extension: "xls", Format: "application/vnd.ms-excel", Type: 33}, - "application/vnd.android.package-archive": {Extension: "apk", Format: "application/vnd.android.package-archive", Type: 34}, - "application/zip": {Extension: "zip", Format: "application/zip", Type: 35}, - "application/java-archive": {Extension: "jar", Format: "application/java-archive", Type: 36}, - "text/x-vCalendar": {Extension: "vcs", Format: "text/x-vCalendar", Type: 38}, - "text/x-vcalendar": {Extension: "ics", Format: "text/x-vcalendar", Type: 39}, - "text/calendar": {Extension: "ics", Format: "text/calendar", Type: 40}, - "application/vcs": {Extension: "vcs", Format: "application/vcs", Type: 41}, - "application/ics": {Extension: "ics", Format: "application/ics", Type: 42}, - "application/hbs-vcs": {Extension: "vcs", Format: "application/hbs-vcs", Type: 43}, + "text/vcard": {Extension: "vcard", Type: binary.MediaFormats_TEXT_VCARD}, + "application/pdf": {Extension: "pdf", Type: binary.MediaFormats_APP_PDF}, + "text/plain": {Extension: "txt", Type: binary.MediaFormats_APP_TXT}, + "text/html": {Extension: "html", Type: binary.MediaFormats_APP_HTML}, + "application/msword": {Extension: "doc", Type: binary.MediaFormats_APP_DOC}, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": {Extension: "docx", Type: binary.MediaFormats_APP_DOCX}, + "application/vnd.openxmlformats-officedocument.presentationml.presentation": {Extension: "pptx", Type: binary.MediaFormats_APP_PPTX}, + "application/vnd.ms-powerpoint": {Extension: "ppt", Type: binary.MediaFormats_APP_PPT}, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": {Extension: "xlsx", Type: binary.MediaFormats_APP_XLSX}, + "application/vnd.ms-excel": {Extension: "xls", Type: binary.MediaFormats_APP_XLS}, + "application/vnd.android.package-archive": {Extension: "apk", Type: binary.MediaFormats_APP_APK}, + "application/zip": {Extension: "zip", Type: binary.MediaFormats_APP_ZIP}, + "application/java-archive": {Extension: "jar", Type: binary.MediaFormats_APP_JAR}, + "text/x-calendar": {Extension: "vcs", Type: binary.MediaFormats_CAL_TEXT_VCALENDAR}, + "text/calendar": {Extension: "ics", Type: binary.MediaFormats_CAL_TEXT_CALENDAR}, + + "image": {Type: binary.MediaFormats_IMAGE_UNSPECIFIED}, + "video": {Type: binary.MediaFormats_VIDEO_UNSPECIFIED}, + "audio": {Type: binary.MediaFormats_AUDIO_UNSPECIFIED}, + "application": {Type: binary.MediaFormats_APP_UNSPECIFIED}, + "text": {Type: binary.MediaFormats_APP_TXT}, } -type Image struct { - imageCryptor *crypto.ImageCryptor - - imageName string - imageID string - imageType MediaType - imageBytes []byte - imageSize int64 +var FormatToMediaType = map[binary.MediaFormats]MediaType{ + binary.MediaFormats_CAL_TEXT_XVCALENDAR: MimeToMediaType["text/x-calendar"], + binary.MediaFormats_CAL_APPLICATION_VCS: MimeToMediaType["text/x-calendar"], + binary.MediaFormats_CAL_APPLICATION_ICS: MimeToMediaType["text/calendar"], + //binary.MediaFormats_CAL_APPLICATION_HBSVCS: ??? } -func (i *Image) GetEncryptedBytes() ([]byte, error) { - encryptedBytes, encryptErr := i.imageCryptor.EncryptData(i.imageBytes) - if encryptErr != nil { - return nil, encryptErr +func init() { + for key, mediaType := range MimeToMediaType { + if strings.ContainsRune(key, '/') { + mediaType.Format = key + } + FormatToMediaType[mediaType.Type] = mediaType } - return encryptedBytes, nil } -func (i *Image) GetImageCryptor() *crypto.ImageCryptor { - return i.imageCryptor -} - -func (i *Image) GetImageName() string { - return i.imageName -} - -func (i *Image) GetImageBytes() []byte { - return i.imageBytes -} - -func (i *Image) GetImageSize() int64 { - return i.imageSize -} - -func (i *Image) GetImageType() MediaType { - return i.imageType -} - -func (i *Image) GetImageID() string { - return i.imageID -} - -// This is the equivalent of dragging an image into the window on messages web -// -// Keep in mind that adding an image to a MessageBuilder will also upload the image to googles server -func (c *Client) UploadMedia(data []byte, mime string) (*binary.MediaContent, error) { - mediaType := MediaTypes[mime] - mediaID := util.GenerateImageID() - fileName := util.RandStr(8) + "." + mediaType.Extension +func (c *Client) UploadMedia(data []byte, fileName, mime string) (*binary.MediaContent, error) { + mediaType := MimeToMediaType[mime] + if mediaType.Type == 0 { + mediaType = MimeToMediaType[strings.Split(mime, "/")[0]] + } decryptionKey, err := crypto.GenerateKey(32) if err != nil { return nil, err @@ -119,30 +90,23 @@ func (c *Client) UploadMedia(data []byte, mime string) (*binary.MediaContent, er if err != nil { return nil, err } - image := &Image{ - imageCryptor: cryptor, - imageID: mediaID, - imageBytes: data, - imageType: mediaType, - imageSize: int64(len(data)), - imageName: fileName, - } - - startUploadImage, err := c.StartUploadMedia(image) + encryptedBytes, err := cryptor.EncryptData(data) + if err != nil { + return nil, err + } + startUploadImage, err := c.StartUploadMedia(encryptedBytes, mime) if err != nil { return nil, err } - upload, err := c.FinalizeUploadMedia(startUploadImage) if err != nil { return nil, err } - return &binary.MediaContent{ - Format: binary.MediaFormats(image.GetImageType().Type), + Format: mediaType.Type, MediaID: upload.MediaID, - MediaName: image.GetImageName(), - Size: image.GetImageSize(), - DecryptionKey: image.GetImageCryptor().GetKey(), + MediaName: fileName, + Size: int64(len(data)), + DecryptionKey: decryptionKey, }, nil } diff --git a/libgm/media_processor.go b/libgm/media_processor.go index 03878ae..426edb4 100644 --- a/libgm/media_processor.go +++ b/libgm/media_processor.go @@ -19,15 +19,14 @@ type StartGoogleUpload struct { UploadStatus string ChunkGranularity int64 ControlURL string + MimeType string - Image *Image EncryptedMediaBytes []byte } type MediaUpload struct { MediaID string MediaNumber int64 - Image *Image } var ( @@ -36,10 +35,9 @@ var ( ) func (c *Client) FinalizeUploadMedia(upload *StartGoogleUpload) (*MediaUpload, error) { - imageType := upload.Image.GetImageType() encryptedImageSize := strconv.Itoa(len(upload.EncryptedMediaBytes)) - finalizeUploadHeaders := util.NewMediaUploadHeaders(encryptedImageSize, "upload, finalize", "0", imageType.Format, "") + finalizeUploadHeaders := util.NewMediaUploadHeaders(encryptedImageSize, "upload, finalize", "0", upload.MimeType, "") req, reqErr := http.NewRequest("POST", upload.UploadURL, bytes.NewBuffer(upload.EncryptedMediaBytes)) if reqErr != nil { return nil, reqErr @@ -76,20 +74,13 @@ func (c *Client) FinalizeUploadMedia(upload *StartGoogleUpload) (*MediaUpload, e return &MediaUpload{ MediaID: mediaIDs.Media.MediaID, MediaNumber: mediaIDs.Media.MediaNumber, - Image: upload.Image, }, nil } -func (c *Client) StartUploadMedia(image *Image) (*StartGoogleUpload, error) { - imageType := image.GetImageType() - - encryptedImageBytes, encryptErr := image.GetEncryptedBytes() - if encryptErr != nil { - return nil, encryptErr - } +func (c *Client) StartUploadMedia(encryptedImageBytes []byte, mime string) (*StartGoogleUpload, error) { encryptedImageSize := strconv.Itoa(len(encryptedImageBytes)) - startUploadHeaders := util.NewMediaUploadHeaders(encryptedImageSize, "start", "", imageType.Format, "resumable") + startUploadHeaders := util.NewMediaUploadHeaders(encryptedImageSize, "start", "", mime, "resumable") startUploadPayload, buildPayloadErr := c.buildStartUploadPayload() if buildPayloadErr != nil { return nil, buildPayloadErr @@ -125,8 +116,8 @@ func (c *Client) StartUploadMedia(image *Image) (*StartGoogleUpload, error) { UploadStatus: rHeaders.Get("x-goog-upload-status"), ChunkGranularity: int64(chunkGranularity), ControlURL: rHeaders.Get("x-goog-upload-control-url"), + MimeType: mime, - Image: image, EncryptedMediaBytes: encryptedImageBytes, } return uploadResponse, nil diff --git a/messagetracking.go b/messagetracking.go index 28f7bb7..86f7d38 100644 --- a/messagetracking.go +++ b/messagetracking.go @@ -40,6 +40,10 @@ var ( errUnknownMsgType = errors.New("unknown msgtype") errMediaUnsupportedType = errors.New("unsupported media type") errTargetNotFound = errors.New("target event not found") + errMissingMediaURL = errors.New("missing media URL") + errMediaDownloadFailed = errors.New("failed to download media") + errMediaDecryptFailed = errors.New("failed to decrypt media") + errMediaReuploadFailed = errors.New("failed to upload media to google") errMessageTakingLong = errors.New("bridging the message is taking longer than usual") ) diff --git a/portal.go b/portal.go index 9cd3fd3..da02e60 100644 --- a/portal.go +++ b/portal.go @@ -30,6 +30,7 @@ import ( "github.com/gabriel-vasile/mimetype" "github.com/rs/zerolog" "maunium.net/go/maulogger/v2" + mutil "maunium.net/go/mautrix/util" "maunium.net/go/mautrix/util/variationselector" "maunium.net/go/mautrix" @@ -40,6 +41,7 @@ import ( "maunium.net/go/mautrix/id" "go.mau.fi/mautrix-gmessages/database" + "go.mau.fi/mautrix-gmessages/libgm" "go.mau.fi/mautrix-gmessages/libgm/binary" "go.mau.fi/mautrix-gmessages/libgm/util" ) @@ -640,43 +642,6 @@ func (msg *ConvertedMessage) MergeCaption() { msg.Parts = []ConvertedMessagePart{filePart} } -var mediaFormatToMime = map[binary.MediaFormats]string{ - binary.MediaFormats_UNSPECIFIED_TYPE: "", - - binary.MediaFormats_IMAGE_JPEG: "image/jpeg", - binary.MediaFormats_IMAGE_JPG: "image/jpeg", - binary.MediaFormats_IMAGE_PNG: "image/png", - binary.MediaFormats_IMAGE_GIF: "image/gif", - binary.MediaFormats_IMAGE_WBMP: "image/vnd.wap.vbmp", - binary.MediaFormats_IMAGE_X_MS_BMP: "image/bmp", - binary.MediaFormats_IMAGE_UNSPECIFIED: "", - - binary.MediaFormats_VIDEO_MP4: "video/mp4", - binary.MediaFormats_VIDEO_3G2: "video/3gpp2", - binary.MediaFormats_VIDEO_3GPP: "video/3gpp", - binary.MediaFormats_VIDEO_WEBM: "video/webm", - binary.MediaFormats_VIDEO_MKV: "video/x-matroska", - binary.MediaFormats_VIDEO_UNSPECIFIED: "", - - binary.MediaFormats_AUDIO_AAC: "audio/aac", - binary.MediaFormats_AUDIO_AMR: "audio/amr", - binary.MediaFormats_AUDIO_MP3: "audio/mp3", - binary.MediaFormats_AUDIO_MPEG: "audio/mpeg", - binary.MediaFormats_AUDIO_MPG: "audio/mpeg", - binary.MediaFormats_AUDIO_MP4: "audio/mp4", - binary.MediaFormats_AUDIO_MP4_LATM: "audio/mp4a-latm", - binary.MediaFormats_AUDIO_3GPP: "audio/3gpp", - binary.MediaFormats_AUDIO_OGG: "audio/ogg", - binary.MediaFormats_AUDIO_OGG2: "audio/ogg", - binary.MediaFormats_AUDIO_UNSPECIFIED: "", - - binary.MediaFormats_TEXT_VCARD: "text/vcard", - binary.MediaFormats_APP_PDF: "application/pdf", - binary.MediaFormats_APP_TXT: "text/plain", - binary.MediaFormats_APP_HTML: "text/html", - binary.MediaFormats_APP_SMIL: "application/smil", -} - func (portal *Portal) convertGoogleMedia(source *User, intent *appservice.IntentAPI, msg *binary.MediaContent) (*event.MessageEventContent, error) { var data []byte var err error @@ -684,7 +649,7 @@ func (portal *Portal) convertGoogleMedia(source *User, intent *appservice.Intent if err != nil { return nil, err } - mime := mediaFormatToMime[msg.GetFormat()] + mime := libgm.FormatToMediaType[msg.GetFormat()].Format if mime == "" { mime = mimetype.Detect(data).String() } @@ -1207,28 +1172,8 @@ func (portal *Portal) uploadMedia(intent *appservice.IntentAPI, data []byte, con return nil } -func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timings messageTimings) { - ms := metricSender{portal: portal, timings: &timings} - - log := portal.zlog.With(). - Str("event_id", evt.ID.String()). - Str("action", "handle matrix message"). - Logger() - ctx := log.WithContext(context.TODO()) - log.Debug().Dur("age", timings.totalReceive).Msg("Handling Matrix message") - - content, ok := evt.Content.Parsed.(*event.MessageEventContent) - if !ok { - return - } - - txnID := util.GenerateTmpID() - portal.outgoingMessagesLock.Lock() - portal.outgoingMessages[txnID] = &outgoingMessage{Event: evt} - portal.outgoingMessagesLock.Unlock() - if evt.Type == event.EventSticker { - content.MsgType = event.MsgImage - } +func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, content *event.MessageEventContent, txnID string) (*binary.SendMessagePayload, error) { + log := zerolog.Ctx(ctx) req := &binary.SendMessagePayload{ ConversationID: portal.ID, TmpID: txnID, @@ -1265,26 +1210,71 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timing }}, }} case event.MsgImage, event.MsgVideo, event.MsgAudio, event.MsgFile: - //fileName := content.Body - //if content.FileName != "" { - // fileName = content.FileName - //} - //sender.Client.StartUploadMedia() - //req.MessagePayload.MessageInfo = []*binary.MessageInfo{{ - // Data: &binary.MessageInfo_MediaContent{MediaContent: &binary.MediaContent{ - // Format: 0, - // MediaID: mediaID, - // MediaName: fileName, - // Size: int64(len(data)), - // DecryptionKey: decryptionKey, - // }}, - //}} + var url id.ContentURI + if content.File != nil { + url = content.File.URL.ParseOrIgnore() + } else { + url = content.URL.ParseOrIgnore() + } + if url.IsEmpty() { + return nil, errMissingMediaURL + } + data, err := portal.MainIntent().DownloadBytesContext(ctx, url) + if err != nil { + return nil, mutil.NewDualError(errMediaDownloadFailed, err) + } + if content.File != nil { + err = content.File.DecryptInPlace(data) + if err != nil { + return nil, mutil.NewDualError(errMediaDecryptFailed, err) + } + } + if content.Info.MimeType == "" { + content.Info.MimeType = mimetype.Detect(data).String() + } + fileName := content.Body + if content.FileName != "" { + fileName = content.FileName + } + resp, err := sender.Client.UploadMedia(data, fileName, content.Info.MimeType) + if err != nil { + return nil, mutil.NewDualError(errMediaReuploadFailed, err) + } + req.MessagePayload.MessageInfo = []*binary.MessageInfo{{ + Data: &binary.MessageInfo_MediaContent{MediaContent: resp}, + }} default: - go ms.sendMessageMetrics(evt, fmt.Errorf("unsupported msgtype"), "Ignoring", true) + return nil, fmt.Errorf("%w %s", errUnknownMsgType, content.MsgType) + } + return req, nil +} + +func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timings messageTimings) { + ms := metricSender{portal: portal, timings: &timings} + + log := portal.zlog.With(). + Str("event_id", evt.ID.String()). + Str("action", "handle matrix message"). + Logger() + ctx := log.WithContext(context.TODO()) + log.Debug().Dur("age", timings.totalReceive).Msg("Handling Matrix message") + + content, ok := evt.Content.Parsed.(*event.MessageEventContent) + if !ok { return } - _, err := sender.Client.Conversations.SendMessage(req) - if err != nil { + + txnID := util.GenerateTmpID() + portal.outgoingMessagesLock.Lock() + portal.outgoingMessages[txnID] = &outgoingMessage{Event: evt} + portal.outgoingMessagesLock.Unlock() + if evt.Type == event.EventSticker { + content.MsgType = event.MsgImage + } + + if req, err := portal.convertMatrixMessage(ctx, sender, content, txnID); err != nil { + go ms.sendMessageMetrics(evt, err, "Error converting", true) + } else if _, err = sender.Client.Conversations.SendMessage(req); err != nil { go ms.sendMessageMetrics(evt, err, "Error sending", true) } else { go ms.sendMessageMetrics(evt, nil, "", true)