seang96/gmessages
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Convert voice messages from/to m4a

Browse source
This commit is contained in:
Tulir Asokan 2024-03-11 16:55:11 +02:00
parent bfd872b4b8
commit d9364a7b7f
2 changed files with 33 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
messagetracking.go
View file

@ -44,6 +44,7 @@ var (
errMissingMediaURL = errors.New("missing media URL") errMissingMediaURL = errors.New("missing media URL")
errMediaDownloadFailed = errors.New("failed to download media") errMediaDownloadFailed = errors.New("failed to download media")
errMediaDecryptFailed = errors.New("failed to decrypt media") errMediaDecryptFailed = errors.New("failed to decrypt media")
errMediaConvertFailed = errors.New("failed to convert media")
errMediaReuploadFailed = errors.New("failed to upload media to google") errMediaReuploadFailed = errors.New("failed to upload media to google")
errIncorrectUser = errors.New("incorrect user") errIncorrectUser = errors.New("incorrect user")

43
portal.go
View file

@ -34,6 +34,7 @@ import (
"github.com/gabriel-vasile/mimetype" "github.com/gabriel-vasile/mimetype"
"github.com/rs/zerolog" "github.com/rs/zerolog"
"go.mau.fi/util/exerrors" "go.mau.fi/util/exerrors"
"go.mau.fi/util/ffmpeg"
"go.mau.fi/util/random" "go.mau.fi/util/random"
"go.mau.fi/util/variationselector" "go.mau.fi/util/variationselector"
"maunium.net/go/mautrix" "maunium.net/go/mautrix"
@ -1001,6 +1002,7 @@ func (portal *Portal) convertGoogleMessage(ctx context.Context, source *User, ev
cm.MediaStatus = downloadStatus cm.MediaStatus = downloadStatus
for _, part := range evt.MessageInfo { for _, part := range evt.MessageInfo {
var content event.MessageEventContent var content event.MessageEventContent
var extra map[string]any
pendingMedia := false pendingMedia := false
switch data := part.GetData().(type) { switch data := part.GetData().(type) {
case *gmproto.MessageInfo_MessageContent: case *gmproto.MessageInfo_MessageContent:
@ -1023,7 +1025,7 @@ func (portal *Portal) convertGoogleMessage(ctx context.Context, source *User, ev
MsgType: event.MsgNotice, MsgType: event.MsgNotice,
Body: fmt.Sprintf("Waiting for attachment %s", data.MediaContent.GetMediaName()), Body: fmt.Sprintf("Waiting for attachment %s", data.MediaContent.GetMediaName()),
} }
} else if contentPtr, err := portal.convertGoogleMedia(ctx, source, cm.Intent, data.MediaContent); err != nil { } else if contentPtr, extraMap, err := portal.convertGoogleMedia(ctx, source, cm.Intent, data.MediaContent); err != nil {
pendingMedia = true pendingMedia = true
log.Err(err).Msg("Failed to copy attachment") log.Err(err).Msg("Failed to copy attachment")
content = event.MessageEventContent{ content = event.MessageEventContent{
@ -1032,6 +1034,7 @@ func (portal *Portal) convertGoogleMessage(ctx context.Context, source *User, ev
} }
} else { } else {
content = *contentPtr content = *contentPtr
extra = extraMap
} }
default: default:
continue continue
@ -1043,6 +1046,7 @@ func (portal *Portal) convertGoogleMessage(ctx context.Context, source *User, ev
ID: part.GetActionMessageID(), ID: part.GetActionMessageID(),
PendingMedia: pendingMedia, PendingMedia: pendingMedia,
Content: &content, Content: &content,
Extra: extra,
}) })
} }
if downloadStatus != "" { if downloadStatus != "" {
@ -1191,7 +1195,7 @@ func (msg *ConvertedMessage) MergeCaption() {
msg.Parts = []ConvertedMessagePart{filePart} msg.Parts = []ConvertedMessagePart{filePart}
} }
func (portal *Portal) convertGoogleMedia(ctx context.Context, source *User, intent *appservice.IntentAPI, msg *gmproto.MediaContent) (*event.MessageEventContent, error) { func (portal *Portal) convertGoogleMedia(ctx context.Context, source *User, intent *appservice.IntentAPI, msg *gmproto.MediaContent) (*event.MessageEventContent, map[string]any, error) {
var data []byte var data []byte
var err error var err error
if msg.MediaID != "" { if msg.MediaID != "" {
@ -1202,12 +1206,14 @@ func (portal *Portal) convertGoogleMedia(ctx context.Context, source *User, inte
err = fmt.Errorf("no media ID found") err = fmt.Errorf("no media ID found")
} }
if err != nil { if err != nil {
return nil, err return nil, nil, err
} }
mime := libgm.FormatToMediaType[msg.GetFormat()].Format mime := libgm.FormatToMediaType[msg.GetFormat()].Format
if mime == "" { if mime == "" {
mime = mimetype.Detect(data).String() mime = mimetype.Detect(data).String()
} }
fileName := msg.MediaName
extra := make(map[string]any)
msgtype := event.MsgFile msgtype := event.MsgFile
switch strings.Split(mime, "/")[0] { switch strings.Split(mime, "/")[0] {
case "image": case "image":
@ -1217,17 +1223,23 @@ func (portal *Portal) convertGoogleMedia(ctx context.Context, source *User, inte
// TODO convert weird formats to mp4 // TODO convert weird formats to mp4
case "audio": case "audio":
msgtype = event.MsgAudio msgtype = event.MsgAudio
// TODO convert everything to ogg and include voice message metadata data, err = ffmpeg.ConvertBytes(ctx, data, ".ogg", []string{}, []string{"-c:a", "libopus"}, mime)
if err != nil {
return nil, nil, fmt.Errorf("%w (%s to ogg): %w", errMediaConvertFailed, mime, err)
}
extra["org.matrix.msc3245.voice"] = map[string]any{}
fileName += ".ogg"
mime = "audio/ogg"
} }
content := &event.MessageEventContent{ content := &event.MessageEventContent{
MsgType: msgtype, MsgType: msgtype,
Body: msg.MediaName, Body: fileName,
Info: &event.FileInfo{ Info: &event.FileInfo{
MimeType: mime, MimeType: mime,
Size: len(data), Size: len(data),
}, },
} }
return content, portal.uploadMedia(ctx, intent, data, content) return content, extra, portal.uploadMedia(ctx, intent, data, content)
} }
func (portal *Portal) isRecentlyHandled(id string) bool { func (portal *Portal) isRecentlyHandled(id string) bool {
@ -1877,7 +1889,7 @@ func (portal *Portal) uploadMedia(ctx context.Context, intent *appservice.Intent
return nil return nil
} }
func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, content *event.MessageEventContent, txnID string) (*gmproto.SendMessageRequest, error) { func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, content *event.MessageEventContent, raw map[string]any, txnID string) (*gmproto.SendMessageRequest, error) {
log := zerolog.Ctx(ctx) log := zerolog.Ctx(ctx)
req := &gmproto.SendMessageRequest{ req := &gmproto.SendMessageRequest{
ConversationID: portal.ID, ConversationID: portal.ID,
@ -1916,7 +1928,7 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, co
}}, }},
}} }}
case event.MsgImage, event.MsgVideo, event.MsgAudio, event.MsgFile: case event.MsgImage, event.MsgVideo, event.MsgAudio, event.MsgFile:
resp, err := portal.reuploadMedia(ctx, sender, content) resp, err := portal.reuploadMedia(ctx, sender, content, raw)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1925,7 +1937,7 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, co
}} }}
case event.MsgBeeperGallery: case event.MsgBeeperGallery:
for i, part := range content.BeeperGalleryImages { for i, part := range content.BeeperGalleryImages {
convertedPart, err := portal.reuploadMedia(ctx, sender, part) convertedPart, err := portal.reuploadMedia(ctx, sender, part, nil)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to reupload gallery image #%d: %w", i+1, err) return nil, fmt.Errorf("failed to reupload gallery image #%d: %w", i+1, err)
} }
@ -1946,7 +1958,7 @@ func (portal *Portal) convertMatrixMessage(ctx context.Context, sender *User, co
return req, nil return req, nil
} }
func (portal *Portal) reuploadMedia(ctx context.Context, sender *User, content *event.MessageEventContent) (*gmproto.MediaContent, error) { func (portal *Portal) reuploadMedia(ctx context.Context, sender *User, content *event.MessageEventContent, raw map[string]any) (*gmproto.MediaContent, error) {
var url id.ContentURI var url id.ContentURI
if content.File != nil { if content.File != nil {
url = content.File.URL.ParseOrIgnore() url = content.File.URL.ParseOrIgnore()
@ -1973,6 +1985,15 @@ func (portal *Portal) reuploadMedia(ctx context.Context, sender *User, content *
if content.FileName != "" { if content.FileName != "" {
fileName = content.FileName fileName = content.FileName
} }
isVoice, ok := raw["org.matrix.msc3245.voice"].(bool)
if ok && isVoice {
data, err = ffmpeg.ConvertBytes(ctx, data, ".m4a", []string{}, []string{"-c:a", "aac"}, content.Info.MimeType)
if err != nil {
return nil, fmt.Errorf("%w (ogg to m4a): %w", errMediaConvertFailed, err)
}
fileName += ".m4a"
content.Info.MimeType = "audio/mp4"
}
resp, err := sender.Client.UploadMedia(data, fileName, content.Info.MimeType) resp, err := sender.Client.UploadMedia(data, fileName, content.Info.MimeType)
if err != nil { if err != nil {
return nil, exerrors.NewDualError(errMediaReuploadFailed, err) return nil, exerrors.NewDualError(errMediaReuploadFailed, err)
@ -2004,7 +2025,7 @@ func (portal *Portal) HandleMatrixMessage(sender *User, evt *event.Event, timing
} }
start := time.Now() start := time.Now()
req, err := portal.convertMatrixMessage(ctx, sender, content, txnID) req, err := portal.convertMatrixMessage(ctx, sender, content, evt.Content.Raw, txnID)
timings.convert = time.Since(start) timings.convert = time.Since(start)
if err != nil { if err != nil {
go ms.sendMessageMetrics(ctx, sender, evt, err, "Error converting", true) go ms.sendMessageMetrics(ctx, sender, evt, err, "Error converting", true)