diff --git a/conversations.php b/conversations.php index 791ca6d..ab2f0d5 100644 --- a/conversations.php +++ b/conversations.php @@ -1886,10 +1886,34 @@ this.conversations = messages.concat(this.conversations); } - // Improved dedupe: use message_id, id, and fallback composite key (direction|type|media|created_sec) + // Improved dedupe: pick the most "readable" message when duplicates exist try { - const seen = new Map(); - const duplicates = []; + const best = new Map(); // key => { index, score } + const toRemove = []; + + const scoreFor = (m) => { + let s = 0; + if (!m) return s; + // Prefer explicit media type messages + if (m.message_type && m.message_type !== 'text') s += 50; + // Prefer messages with thumbnails or local files + if (m.local_thumb || m.local_file || m.media_url_external || m.media_url) s += 30; + // Prefer human text content (avoid raw JSON blobs) + if (m.content) { + const c = String(m.content).trim(); + const looksJson = c.startsWith('{') && c.endsWith('}'); + if (looksJson) { + s -= 20; // penalize raw JSON-looking content + } else { + s += 10; + if (c.length < 200) s += 5; + } + } + if (m.message_id) s += 3; + if (m.reply_to_message_id) s += 2; + return s; + }; + for (let i = this.conversations.length - 1; i >= 0; i--) { const m = this.conversations[i]; if (!m) continue; @@ -1902,15 +1926,32 @@ key = `${m.direction||'?'}|${m.message_type||m.media_type||'text'}|${partMedia}|${ts}`; } - if (seen.has(key)) { - // remove duplicate occurrence (keep the last one) - duplicates.push({index:i, key}); - this.conversations.splice(i, 1); + const s = scoreFor(m); + if (!best.has(key)) { + best.set(key, { index: i, score: s }); } else { - seen.set(key, true); + const prev = best.get(key); + if (s > prev.score) { + // keep current, remove previous + toRemove.push(prev.index); + best.set(key, { index: i, score: s }); + } else { + // remove current + toRemove.push(i); + } + } + } + + if (toRemove.length) { + // dedupe unique indices + const uniq = Array.from(new Set(toRemove)).sort((a,b) => a - b); + // remove from highest index down + for (let j = uniq.length - 1; j >= 0; j--) { + const idx = uniq[j]; + const removed = this.conversations.splice(idx, 1)[0]; + console.debug('Dedupe removed message at index', idx, 'removed=', removed && (removed.message_id || removed.id || removed.content || removed.media_url) ); } } - if (duplicates.length) console.debug('Dedupe removed duplicates (indices):', duplicates); } catch (e) { console.warn('Dedupe failed', e); }