diff --git a/pkg/connector/backfill.go b/pkg/connector/backfill.go index 8f006e0b..e2c7d114 100644 --- a/pkg/connector/backfill.go +++ b/pkg/connector/backfill.go @@ -293,7 +293,10 @@ func (wa *WhatsAppClient) handleWAHistorySync( for i, rawMsg := range rawMessages { // Don't store messages that will just be skipped. msgEvt, err := wa.Client.ParseWebMessage(jid, rawMsg.GetMessage()) - if err != nil { + if err != nil && getMessageType(rawMsg.GetMessage().GetMessage()) == "ignore" { + ignoredTypes++ + continue + } else if err != nil { log.Warn().Err(err). Int("msg_index", i). Str("msg_id", rawMsg.GetMessage().GetKey().GetID()). diff --git a/pkg/connector/wadb/upgrades/00-latest-schema.sql b/pkg/connector/wadb/upgrades/00-latest-schema.sql index c4172d09..cd02333f 100644 --- a/pkg/connector/wadb/upgrades/00-latest-schema.sql +++ b/pkg/connector/wadb/upgrades/00-latest-schema.sql @@ -1,4 +1,4 @@ --- v0 -> v11 (compatible with v3+): Latest revision +-- v0 -> v12 (compatible with v3+): Latest revision CREATE TABLE whatsapp_poll_option_id ( bridge_id TEXT NOT NULL, @@ -50,6 +50,8 @@ CREATE TABLE whatsapp_history_sync_message ( REFERENCES whatsapp_history_sync_conversation (bridge_id, user_login_id, chat_jid) ON UPDATE CASCADE ON DELETE CASCADE ); +CREATE INDEX whatsapp_history_sync_message_chat_ts_idx ON whatsapp_history_sync_message (bridge_id, user_login_id, chat_jid, timestamp); + CREATE TABLE whatsapp_media_backfill_request ( bridge_id TEXT NOT NULL, user_login_id TEXT NOT NULL, diff --git a/pkg/connector/wadb/upgrades/12-history-sync-message-timestamp-index.sql b/pkg/connector/wadb/upgrades/12-history-sync-message-timestamp-index.sql new file mode 100644 index 00000000..0d3b910a --- /dev/null +++ b/pkg/connector/wadb/upgrades/12-history-sync-message-timestamp-index.sql @@ -0,0 +1,2 @@ +-- v12 (compatible with v3+): Add index for history sync message timestamp lookups +CREATE INDEX whatsapp_history_sync_message_chat_ts_idx ON whatsapp_history_sync_message (bridge_id, user_login_id, chat_jid, timestamp); diff --git a/pkg/msgconv/from-matrix.go b/pkg/msgconv/from-matrix.go index 47485854..ba2adc23 100644 --- a/pkg/msgconv/from-matrix.go +++ b/pkg/msgconv/from-matrix.go @@ -168,11 +168,6 @@ func (mc *MessageConverter) ToWhatsApp( Comment: &content.Body, ContextInfo: contextInfo, } - if msc3488, ok := evt.Content.Raw["org.matrix.msc3488.location"].(map[string]any); ok { - if description, ok := msc3488["description"].(string); ok && description != "" { - message.LocationMessage.Name = &description - } - } default: return nil, nil, fmt.Errorf("%w %s", bridgev2.ErrUnsupportedMessageType, content.MsgType) } diff --git a/pkg/msgconv/from-whatsapp.go b/pkg/msgconv/from-whatsapp.go index db18f35a..e2ab92cd 100644 --- a/pkg/msgconv/from-whatsapp.go +++ b/pkg/msgconv/from-whatsapp.go @@ -76,8 +76,7 @@ func (mc *MessageConverter) getBasicUserInfo(ctx context.Context, user types.JID var pnJID types.JID if user.Server == types.DefaultUserServer { pnJID = user - } else if user.Server == types.HiddenUserServer { - cli := getClient(ctx) + } else if cli := getClient(ctx); user.Server == types.HiddenUserServer && cli != nil { if user.User == cli.Store.GetLID().User { pnJID = cli.Store.GetJID() } else { @@ -322,6 +321,7 @@ func (mc *MessageConverter) ToMatrix( cm.ThreadRoot = ptr.Ptr(waid.MakeMessageID(chat, pcp, commentTarget.GetID())) } mc.cutOffLongMessage(part.Content) + logOversizedPart(ctx, part) return cm } diff --git a/pkg/msgconv/wa-text.go b/pkg/msgconv/wa-text.go index 52874fd8..7afcc81f 100644 --- a/pkg/msgconv/wa-text.go +++ b/pkg/msgconv/wa-text.go @@ -18,6 +18,7 @@ package msgconv import ( "context" + "encoding/json" "fmt" "html" "regexp" @@ -25,6 +26,7 @@ import ( "strconv" "strings" + "github.com/rs/zerolog" "go.mau.fi/whatsmeow/proto/waE2E" "maunium.net/go/mautrix/bridgev2" "maunium.net/go/mautrix/event" @@ -72,6 +74,36 @@ func (mc *MessageConverter) cutOffLongMessage(content *event.MessageEventContent } } +func logOversizedPart(ctx context.Context, part *bridgev2.ConvertedMessagePart) { + if part == nil || part.Content == nil { + return + } + contentJSON, _ := json.Marshal(part.Content) + extraJSON, _ := json.Marshal(part.Extra) + if len(contentJSON)+len(extraJSON) <= 40000 { + return + } + extraKeys := make([]string, 0, len(part.Extra)) + for key := range part.Extra { + extraKeys = append(extraKeys, key) + } + var mentionCount int + if part.Content.Mentions != nil { + mentionCount = len(part.Content.Mentions.UserIDs) + } + linkPreviewJSON, _ := json.Marshal(part.Content.BeeperLinkPreviews) + zerolog.Ctx(ctx).Warn(). + Str("msg_type", string(part.Content.MsgType)). + Int("content_json_size", len(contentJSON)). + Int("extra_json_size", len(extraJSON)). + Int("body_size", len(part.Content.Body)). + Int("formatted_body_size", len(part.Content.FormattedBody)). + Int("mention_count", mentionCount). + Int("link_preview_json_size", len(linkPreviewJSON)). + Strs("extra_keys", extraKeys). + Msg("Converted message part is larger than the cut-off limit") +} + func (mc *MessageConverter) parseFormatting(content *event.MessageEventContent, allowInlineURL, forceHTML bool) { parsedHTML := parseWAFormattingToHTML(content.Body, allowInlineURL) if forceHTML || parsedHTML != event.TextToHTML(content.Body) {