wacrawl-dashboard 0.1.2 → 0.2.0

package/apps/api/dist/routes/stats.js

@@ -1,10 +1,14 @@
 import { Router } from 'express';
 import { getDb } from '../db.js';
-import { cleanDisplayNameSql, contactDisplayNameSql, contactLeftJoins } from '../lib/displayName.js';
-import { bucketDateSql, normalizeBucketDate, parseGranularity, parseLimit, parsePeriod, parseYear, sinceTimestamp, unixSecondsToIso, yearBounds, } from '../lib/query.js';
-import { hourInTimezone, resolveStatsTimeZone } from '../lib/timezone.js';
-import { USEFUL_WORD_STOP_SET } from '../lib/wordCloudUsefulStopWords.js';
-import { categorizeDomain, extractUrls, parseDomain } from '../lib/urlExtract.js';
+import { canonicalJidSql, cleanDisplayNameSql, contactDisplayNameSql, contactLeftJoins, isPersonJidSql } from '../lib/displayName.js';
+import { mean } from '../lib/percentile.js';
+import { etagFor, getCached, statsCacheKey } from '../lib/statsCache.js';
+import { bucketDateSql, normalizeBucketDate, parseGranularity, parseLimit, parsePeriod, parseYear, scanCoverage, sinceTimestamp, unixSecondsToIso, yearBounds, } from '../lib/query.js';
+import { dateInTimezone, hourInTimezone, monthInTimezone, resolveStatsTimeZone, SERVER_TIME_ZONE, tsModifierSql, weekdayHourInTimezone, } from '../lib/timezone.js';
+import { getContactProfile, getGroupDirectory, getGroupLifecycle, getGroupProfile, } from './stats/profiles.js';
+import { getArchivedChats, getDistinctiveWords, getDormancy, getLinkIntelligence, getYearInReview, } from './stats/reports.js';
+import { ChatNotFoundError, EMOJI_REGEX, GHOST_WINDOW_SECONDS, isNoiseToken, NotAGroupError, resolveChatNames, SESSION_GAP_SECONDS, summarizeGaps, WORD_TOKEN_RE, } from './stats/shared.js';
+export { ChatNotFoundError, getArchivedChats, getContactProfile, getDistinctiveWords, getDormancy, getGroupDirectory, getGroupLifecycle, getGroupProfile, getLinkIntelligence, getYearInReview, NotAGroupError, };
 export function getOverviewStats(db = getDb()) {
     const messageRow = db.prepare(`
     SELECT
@@ -86,7 +90,7 @@ export function getTopContacts(params, db = getDb()) {
       SUM(CASE WHEN from_me = 1 THEN 1 ELSE 0 END) AS sentByMe,
       SUM(CASE WHEN from_me = 0 THEN 1 ELSE 0 END) AS sentByThem
     FROM normalized_messages
-    WHERE jid IS NOT NULL AND jid <> ''
+    WHERE ${isPersonJidSql('jid')}
     GROUP BY jid
     ORDER BY messageCount DESC
     LIMIT @limit
@@ -105,7 +109,7 @@ export function getMessageVolume(params, db = getDb()) {
     const fallbackGranularity = period === 'year' || period === 'all' ? 'month' : 'day';
     const granularity = parseGranularity(params.granularity, fallbackGranularity);
     const since = sinceTimestamp(period);
-    const bucketSql = bucketDateSql(granularity);
+    const bucketSql = bucketDateSql(granularity, resolveStatsTimeZone(params.timeZone));
     const rows = db.prepare(`
     SELECT
       ${bucketSql} AS date,
@@ -124,9 +128,10 @@ export function getMessageVolume(params, db = getDb()) {
 }
 export function getActivityHeatmap(params, db = getDb()) {
     const year = parseYear(params.year);
-    const bounds = yearBounds(year);
+    const timeZone = resolveStatsTimeZone(params.timeZone);
+    const bounds = yearBounds(year, timeZone);
     return db.prepare(`
-    SELECT date(ts, 'unixepoch') AS date, COUNT(*) AS count
+    SELECT date(${tsModifierSql(timeZone)}) AS date, COUNT(*) AS count
     FROM messages
     WHERE ts >= @start AND ts < @end
     GROUP BY date
@@ -136,6 +141,15 @@ export function getActivityHeatmap(params, db = getDb()) {
 export function getHourOfDayStats(params, db = getDb()) {
     const since = sinceTimestamp(parsePeriod(params.period));
     const timeZone = resolveStatsTimeZone(params.timeZone);
+    if (timeZone === 'UTC' || timeZone === SERVER_TIME_ZONE) {
+        return db.prepare(`
+      SELECT CAST(strftime('%H', ${tsModifierSql(timeZone)}) AS INTEGER) AS hour, COUNT(*) AS count
+      FROM messages
+      WHERE ts >= @since
+      GROUP BY hour
+      ORDER BY hour ASC
+    `).all({ since });
+    }
     const rows = db.prepare(`
     SELECT ts
     FROM messages
@@ -157,7 +171,7 @@ export function getDayOfWeekStats(params, db = getDb()) {
     const labels = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'];
     const since = sinceTimestamp(parsePeriod(params.period));
     const rows = db.prepare(`
-    SELECT strftime('%w', ts, 'unixepoch') AS day, COUNT(*) AS count
+    SELECT strftime('%w', ${tsModifierSql(resolveStatsTimeZone(params.timeZone))}) AS day, COUNT(*) AS count
     FROM messages
     WHERE ts >= @since
     GROUP BY day
@@ -187,9 +201,9 @@ export function getMediaSenders(params, db = getDb()) {
     const since = sinceTimestamp(parsePeriod(params.period));
     const limit = parseLimit(params.limit, 10, 50);
     const rows = db.prepare(`
-    WITH normalized AS (
+  WITH normalized AS (
       SELECT
-        CASE WHEN from_me = 1 THEN chat_jid ELSE COALESCE(sender_jid, chat_jid) END AS jid,
+        ${canonicalJidSql('contacts', 'CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END')} AS jid,
         ${cleanDisplayNameSql('CASE WHEN from_me = 1 THEN COALESCE(chats.name, chat_name) ELSE sender_name END')} AS message_name,
         ${contactDisplayNameSql('contacts')} AS contact_name,
         media_size
@@ -204,7 +218,7 @@ export function getMediaSenders(params, db = getDb()) {
       COUNT(*) AS mediaCount,
       COALESCE(SUM(media_size), 0) AS totalBytes
     FROM normalized
-    WHERE jid IS NOT NULL AND jid <> ''
+    WHERE ${isPersonJidSql('jid')}
     GROUP BY jid
     ORDER BY mediaCount DESC, name COLLATE NOCASE ASC
     LIMIT @limit
@@ -220,7 +234,7 @@ export function getSentReceivedRatio(params, db = getDb()) {
     const since = sinceTimestamp(parsePeriod(params.period));
     const rows = db.prepare(`
     SELECT
-      strftime('%Y-%m-01', ts, 'unixepoch') AS month,
+      strftime('%Y-%m-01', ${tsModifierSql(resolveStatsTimeZone(params.timeZone))}) AS month,
       SUM(CASE WHEN from_me = 1 THEN 1 ELSE 0 END) AS sent,
       SUM(CASE WHEN from_me = 0 THEN 1 ELSE 0 END) AS received
     FROM messages
@@ -235,43 +249,148 @@ export function getSentReceivedRatio(params, db = getDb()) {
         ratio: row.received ? Number((row.sent / row.received).toFixed(2)) : null,
     }));
 }
-export function getResponseTimes(params, db = getDb()) {
-    const since = sinceTimestamp(parsePeriod(params.period));
-    const limit = parseLimit(params.limit, 10, 50);
+/** Backstop on reply gaps pulled into memory for percentile work (2 narrow columns each). */
+const REPLY_GAP_MAX_ROWS = 400_000;
+const REPLY_GAP_CACHE_TTL_MS = 60_000;
+const replyGapCache = new WeakMap();
+function dbMutationSignature(db) {
+    const changesRow = db.prepare('SELECT total_changes() AS changes').get();
+    const dataVersion = Number(db.pragma('data_version', { simple: true }) ?? 0);
+    return `${changesRow.changes}:${dataVersion}`;
+}
+/**
+ * Every turn-taking gap in the window: one row per message that reversed the
+ * direction of the conversation, carrying who sent it and how long they took.
+ *
+ * Shared by the per-chat leaderboard and the archive-wide latency distribution.
+ */
+function selectReplyGaps(periodKey, since, db) {
+    let perDb = replyGapCache.get(db);
+    if (!perDb) {
+        perDb = new Map();
+        replyGapCache.set(db, perDb);
+    }
+    const now = Date.now();
+    const signature = dbMutationSignature(db);
+    const cached = perDb.get(periodKey);
+    if (cached && cached.signature === signature && cached.expiresAt > now) {
+        return cached.rows;
+    }
     const rows = db.prepare(`
-    WITH ordered AS (
+    SELECT jid, responder_from_me, gap
+    FROM (
       SELECT
-        chat_jid,
-        chat_name,
-        ts,
-        from_me,
-        LAG(ts) OVER (PARTITION BY chat_jid ORDER BY ts, rowid) AS previous_ts,
-        LAG(from_me) OVER (PARTITION BY chat_jid ORDER BY ts, rowid) AS previous_from_me
+        chat_jid AS jid,
+        from_me AS responder_from_me,
+        ts - LAG(ts) OVER (PARTITION BY chat_jid ORDER BY ts, rowid) AS gap,
+        LAG(from_me) OVER (PARTITION BY chat_jid ORDER BY ts, rowid) AS previous_from_me,
+        ts
       FROM messages
       WHERE ts >= @since
     )
-    SELECT
-      chat_jid AS jid,
-      COALESCE(${cleanDisplayNameSql('MAX(chats.name)')}, ${cleanDisplayNameSql('MAX(chat_name)')}, ${contactDisplayNameSql('chat_contacts')}, ${cleanDisplayNameSql('chat_jid')}, 'Unknown') AS name,
-      COUNT(*) AS responseCount,
-      ROUND(AVG(ts - previous_ts)) AS averageSeconds
-    FROM ordered
-    LEFT JOIN chats ON chats.jid = ordered.chat_jid
-    ${contactLeftJoins('chat_contacts', 'ordered.chat_jid')}
-    WHERE previous_ts IS NOT NULL
+    WHERE gap IS NOT NULL
+      AND gap > 0
       AND previous_from_me IS NOT NULL
-      AND previous_from_me <> from_me
-      AND ts > previous_ts
-    GROUP BY chat_jid
-    ORDER BY averageSeconds ASC
-    LIMIT @limit
-  `).all({ since, limit });
-    return rows.map((row) => ({
-        jid: row.jid,
-        name: row.name ?? row.jid,
-        responseCount: row.responseCount,
-        averageSeconds: row.averageSeconds ?? 0,
+      AND previous_from_me <> responder_from_me
+    ORDER BY ts DESC
+    LIMIT ${REPLY_GAP_MAX_ROWS}
+  `).all({ since });
+    perDb.set(periodKey, {
+        signature,
+        expiresAt: now + REPLY_GAP_CACHE_TTL_MS,
+        rows,
+    });
+    return rows;
+}
+/**
+ * Per-chat turnaround, reported as percentiles and split by who was replying.
+ */
+export function getResponseTimes(params, db = getDb()) {
+    const period = parsePeriod(params.period);
+    const since = sinceTimestamp(period);
+    const limit = parseLimit(params.limit, 10, 50);
+    const minResponses = parseLimit(params.minResponses, 5, 1000);
+    const perChat = new Map();
+    for (const row of selectReplyGaps(period, since, db)) {
+        let entry = perChat.get(row.jid);
+        if (!entry) {
+            entry = { mine: [], theirs: [] };
+            perChat.set(row.jid, entry);
+        }
+        (row.responder_from_me ? entry.mine : entry.theirs).push(row.gap);
+    }
+    const ranked = [...perChat.entries()]
+        .map(([jid, { mine, theirs }]) => {
+        const all = [...mine, ...theirs];
+        const overall = summarizeGaps(all);
+        return {
+            jid,
+            responseCount: all.length,
+            averageSeconds: Math.round(mean(all)),
+            medianSeconds: overall.median,
+            p90Seconds: overall.p90,
+            myResponseCount: mine.length,
+            myMedianSeconds: mine.length ? summarizeGaps(mine).median : null,
+            theirResponseCount: theirs.length,
+            theirMedianSeconds: theirs.length ? summarizeGaps(theirs).median : null,
+        };
+    })
+        .filter((stat) => stat.responseCount >= minResponses)
+        .sort((a, b) => a.medianSeconds - b.medianSeconds || b.responseCount - a.responseCount)
+        .slice(0, limit);
+    const names = resolveChatNames(ranked.map((stat) => stat.jid), db);
+    return ranked.map((stat) => ({ ...stat, name: names.get(stat.jid) ?? stat.jid }));
+}
+/** Bucket edges in seconds; the last bucket is open-ended. */
+const REPLY_LATENCY_BUCKETS = [
+    { from: 0, to: 60, label: '< 1 min' },
+    { from: 60, to: 300, label: '1–5 min' },
+    { from: 300, to: 900, label: '5–15 min' },
+    { from: 900, to: 3600, label: '15–60 min' },
+    { from: 3600, to: 4 * 3600, label: '1–4 h' },
+    { from: 4 * 3600, to: 12 * 3600, label: '4–12 h' },
+    { from: 12 * 3600, to: 86_400, label: '12–24 h' },
+    { from: 86_400, to: 3 * 86_400, label: '1–3 d' },
+    { from: 3 * 86_400, to: null, label: '3 d +' },
+];
+/**
+ * Archive-wide shape of how fast replies happen, split by who is replying.
+ */
+export function getReplyLatency(params, db = getDb()) {
+    const period = parsePeriod(params.period);
+    const since = sinceTimestamp(period);
+    const rows = selectReplyGaps(period, since, db);
+    const buckets = REPLY_LATENCY_BUCKETS.map((bucket) => ({
+        fromSeconds: bucket.from,
+        toSeconds: bucket.to,
+        label: bucket.label,
+        mine: 0,
+        theirs: 0,
     }));
+    const mine = [];
+    const theirs = [];
+    for (const row of rows) {
+        const index = REPLY_LATENCY_BUCKETS.findIndex((b) => b.to === null || row.gap < b.to);
+        const bucket = buckets[index === -1 ? buckets.length - 1 : index];
+        if (row.responder_from_me) {
+            bucket.mine += 1;
+            mine.push(row.gap);
+        }
+        else {
+            bucket.theirs += 1;
+            theirs.push(row.gap);
+        }
+    }
+    const side = (gaps) => {
+        const { median, p90 } = summarizeGaps(gaps);
+        return { count: gaps.length, medianSeconds: median, p90Seconds: p90 };
+    };
+    return {
+        buckets,
+        mine: side(mine),
+        theirs: side(theirs),
+        scan: scanCoverage(rows.length, REPLY_GAP_MAX_ROWS),
+    };
 }
 export function getGroupActivity(params, db = getDb()) {
     const since = sinceTimestamp(parsePeriod(params.period));
@@ -298,8 +417,9 @@ export function getGroupActivity(params, db = getDb()) {
 }
 export function getMessageStreaks(params, db = getDb()) {
     const since = sinceTimestamp(parsePeriod(params.period));
+    const timeZone = resolveStatsTimeZone(params.timeZone);
     const rows = db.prepare(`
-    SELECT DISTINCT date(ts, 'unixepoch') AS date
+    SELECT DISTINCT date(${tsModifierSql(timeZone)}) AS date
     FROM messages
     WHERE ts >= @since
     ORDER BY date ASC
@@ -313,8 +433,7 @@ export function getMessageStreaks(params, db = getDb()) {
         longestStreak = Math.max(longestStreak, activeRun);
         previousTime = time;
     }
-    const today = new Date();
-    const todayUtc = Date.UTC(today.getUTCFullYear(), today.getUTCMonth(), today.getUTCDate());
+    const todayUtc = Date.parse(`${dateInTimezone(Math.floor(Date.now() / 1000), timeZone)}T00:00:00.000Z`);
     const lastTime = rows.length ? Date.parse(`${rows[rows.length - 1].date}T00:00:00.000Z`) : null;
     const currentStreak = lastTime != null && todayUtc - lastTime <= 86_400_000 ? activeRun : 0;
     return { currentStreak, longestStreak };
@@ -322,6 +441,8 @@ export function getMessageStreaks(params, db = getDb()) {
 function parseWordCloudFilter(value) {
     return value === 'useful' ? 'useful' : 'all';
 }
+/** Messages scanned for terms; the newest ones, when the archive is larger. */
+const WORD_CLOUD_MAX_ROWS = 30_000;
 export function getWordCloud(params, db = getDb()) {
     const since = sinceTimestamp(parsePeriod(params.period));
     const limit = parseLimit(params.limit, 50, 200);
@@ -331,26 +452,24 @@ export function getWordCloud(params, db = getDb()) {
     FROM messages
     WHERE ts >= @since AND text IS NOT NULL AND TRIM(text) <> ''
     ORDER BY ts DESC
-    LIMIT 30000
+    LIMIT ${WORD_CLOUD_MAX_ROWS}
   `).all({ since });
     const counts = new Map();
     for (const row of rows) {
-        for (const term of (row.text ?? '').toLowerCase().match(/[a-z0-9]{3,}/g) ?? []) {
-            if (filterMode === 'useful') {
-                if (/^\d+$/.test(term))
-                    continue;
-                if (USEFUL_WORD_STOP_SET.has(term))
-                    continue;
-            }
+        for (const term of (row.text ?? '').toLowerCase().match(WORD_TOKEN_RE) ?? []) {
+            if (filterMode === 'useful' && isNoiseToken(term))
+                continue;
             counts.set(term, (counts.get(term) ?? 0) + 1);
         }
     }
-    return [...counts.entries()]
+    const terms = [...counts.entries()]
         .map(([text, value]) => ({ text, value }))
         .sort((a, b) => b.value - a.value || a.text.localeCompare(b.text))
         .slice(0, limit);
+    return { terms, scan: scanCoverage(rows.length, WORD_CLOUD_MAX_ROWS) };
 }
-const EMOJI_REGEX = /[\p{Emoji_Presentation}\p{Extended_Pictographic}]/gu;
+/** Text messages scanned for emoji; the newest ones, when the archive is larger. */
+const EMOJI_MAX_ROWS = 50_000;
 export function getEmojiAnalytics(params, db = getDb()) {
     const since = sinceTimestamp(parsePeriod(params.period));
     const limit = parseLimit(params.limit, 15, 50);
@@ -358,7 +477,7 @@ export function getEmojiAnalytics(params, db = getDb()) {
     SELECT
       messages.text,
       messages.from_me,
-      CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END AS jid,
+      ${canonicalJidSql('contacts', 'CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END')} AS jid,
       ${cleanDisplayNameSql(`CASE WHEN messages.from_me = 1 THEN COALESCE(chats.name, messages.chat_name) ELSE messages.sender_name END`)} AS name,
       ${contactDisplayNameSql('contacts')} AS contact_name
     FROM messages
@@ -366,7 +485,7 @@ export function getEmojiAnalytics(params, db = getDb()) {
     ${contactLeftJoins('contacts', 'CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END')}
     WHERE messages.ts >= @since AND messages.text IS NOT NULL AND messages.text <> ''
     ORDER BY messages.ts DESC
-    LIMIT 50000
+    LIMIT ${EMOJI_MAX_ROWS}
   `).all({ since });
     const allCounts = new Map();
     const sentCounts = new Map();
@@ -379,11 +498,15 @@ export function getEmojiAnalytics(params, db = getDb()) {
             continue;
         totalEmojiCount += emojis.length;
         const jid = row.jid ?? 'unknown';
+        const isPerson = !jid.endsWith('@g.us') && !jid.endsWith('@status') && !jid.endsWith('@broadcast');
         const displayName = row.name ?? row.contact_name ?? jid;
-        if (!userCounts.has(jid)) {
-            userCounts.set(jid, { name: displayName, count: 0, emojis: new Map() });
+        let userEntry;
+        if (isPerson) {
+            if (!userCounts.has(jid)) {
+                userCounts.set(jid, { name: displayName, count: 0, emojis: new Map() });
+            }
+            userEntry = userCounts.get(jid);
         }
-        const userEntry = userCounts.get(jid);
         for (const emoji of emojis) {
             allCounts.set(emoji, (allCounts.get(emoji) ?? 0) + 1);
             if (row.from_me) {
@@ -392,8 +515,10 @@ export function getEmojiAnalytics(params, db = getDb()) {
             else {
                 receivedCounts.set(emoji, (receivedCounts.get(emoji) ?? 0) + 1);
             }
-            userEntry.count += 1;
-            userEntry.emojis.set(emoji, (userEntry.emojis.get(emoji) ?? 0) + 1);
+            if (userEntry) {
+                userEntry.count += 1;
+                userEntry.emojis.set(emoji, (userEntry.emojis.get(emoji) ?? 0) + 1);
+            }
         }
     }
     const sortMap = (map, n) => [...map.entries()]
@@ -414,33 +539,11 @@ export function getEmojiAnalytics(params, db = getDb()) {
         totalEmojiCount,
         uniqueEmojiCount: allCounts.size,
         topEmojiUsers,
+        scan: scanCoverage(rows.length, EMOJI_MAX_ROWS),
     };
 }
-const SESSION_GAP_SECONDS = 4 * 3600;
-const GHOST_WINDOW_SECONDS = 24 * 3600;
-function resolveChatNames(jids, db) {
-    const map = new Map();
-    if (jids.length === 0)
-        return map;
-    const stmt = db.prepare(`
-    SELECT
-      c.jid,
-      COALESCE(
-        ${cleanDisplayNameSql('c.name')},
-        ${contactDisplayNameSql('chat_contacts')},
-        ${cleanDisplayNameSql('c.jid')},
-        'Unknown'
-      ) AS name
-    FROM chats c
-    ${contactLeftJoins('chat_contacts', 'c.jid')}
-    WHERE c.jid = @jid
-  `);
-    for (const jid of jids) {
-        const row = stmt.get({ jid });
-        map.set(jid, row?.name ?? jid);
-    }
-    return map;
-}
+/** Upper bound on rows pulled into memory for session analysis (3 narrow columns each). */
+const CONVERSATION_DYNAMICS_MAX_ROWS = 250_000;
 function computeTrajectoryDirection(trend) {
     if (trend.length < 4)
         return 'stable';
@@ -458,11 +561,17 @@ export function getConversationDynamics(params, db = getDb()) {
     const since = sinceTimestamp(parsePeriod(params.period));
     const limit = parseLimit(params.limit, 10, 50);
     const timeZone = resolveStatsTimeZone(params.timeZone);
+    const ghostCutoff = Math.floor(Date.now() / 1000) - GHOST_WINDOW_SECONDS;
     const rows = db.prepare(`
-    SELECT m.chat_jid, m.ts, m.from_me
-    FROM messages m
-    WHERE m.ts >= @since
-    ORDER BY m.chat_jid, m.ts, m.rowid
+    SELECT chat_jid, ts, from_me
+    FROM (
+      SELECT chat_jid, ts, from_me, rowid
+      FROM messages
+      WHERE ts >= @since
+      ORDER BY ts DESC, rowid DESC
+      LIMIT ${CONVERSATION_DYNAMICS_MAX_ROWS}
+    )
+    ORDER BY chat_jid, ts, rowid
   `).all({ since });
     const chatGroups = new Map();
     for (const row of rows) {
@@ -505,18 +614,17 @@ export function getConversationDynamics(params, db = getDb()) {
         rawDepth.push({ jid, avgDepth: Math.round((messages.length / totalSessions) * 10) / 10, sessions: totalSessions, total: messages.length });
         let ghosted = 0;
         let totalSent = 0;
-        for (let i = 0; i < messages.length; i++) {
-            if (!messages[i].from_me)
+        let nextReceivedTs = null;
+        for (let i = messages.length - 1; i >= 0; i--) {
+            const message = messages[i];
+            if (!message.from_me) {
+                nextReceivedTs = message.ts;
                 continue;
-            totalSent++;
-            let replied = false;
-            for (let j = i + 1; j < messages.length; j++) {
-                if (!messages[j].from_me) {
-                    replied = messages[j].ts - messages[i].ts <= GHOST_WINDOW_SECONDS;
-                    break;
-                }
             }
-            if (!replied)
+            if (message.ts > ghostCutoff)
+                continue;
+            totalSent++;
+            if (nextReceivedTs === null || nextReceivedTs - message.ts > GHOST_WINDOW_SECONDS)
                 ghosted++;
         }
         if (totalSent > 0) {
@@ -537,8 +645,7 @@ export function getConversationDynamics(params, db = getDb()) {
         rawLateNight.push({ jid, lateNight, workHours: workHrs, other, total: messages.length });
         const monthCounts = new Map();
         for (const msg of messages) {
-            const d = new Date(msg.ts * 1000);
-            const month = `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`;
+            const month = monthInTimezone(msg.ts, timeZone);
             monthCounts.set(month, (monthCounts.get(month) ?? 0) + 1);
         }
         rawTrajectory.push({ jid, monthCounts, total: messages.length });
@@ -600,228 +707,71 @@ export function getConversationDynamics(params, db = getDb()) {
             direction: computeTrajectoryDirection(trend),
         };
     });
-    return { initiationRatio, conversationDepth, ghostScore, relationshipTrajectory, lateNightTexters };
+    return {
+        initiationRatio,
+        conversationDepth,
+        ghostScore,
+        relationshipTrajectory,
+        lateNightTexters,
+        scan: scanCoverage(rows.length, CONVERSATION_DYNAMICS_MAX_ROWS),
+    };
 }
-export function getLinkIntelligence(params, db = getDb()) {
+/**
+ * Hour × weekday grid: the week's rhythm as one picture.
+ */
+export function getWeeklyRhythm(params, db = getDb()) {
     const since = sinceTimestamp(parsePeriod(params.period));
-    const limit = parseLimit(params.limit, 15, 50);
-    const rows = db.prepare(`
-    SELECT
-      messages.text,
-      messages.ts,
-      messages.from_me,
-      messages.media_type,
-      CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END AS jid,
-      ${cleanDisplayNameSql(`CASE WHEN messages.from_me = 1 THEN COALESCE(chats.name, messages.chat_name) ELSE messages.sender_name END`)} AS name,
-      ${contactDisplayNameSql('contacts')} AS contact_name
-    FROM messages
-    LEFT JOIN chats ON chats.jid = CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END
-    ${contactLeftJoins('contacts', 'CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END')}
-    WHERE messages.ts >= @since
-    ORDER BY messages.ts ASC
-  `).all({ since });
-    // --- Domain leaderboard + categories ---
-    const domainCounts = new Map();
-    const linksByDate = new Map();
-    const contactLinks = new Map();
-    let totalLinks = 0;
-    for (const row of rows) {
-        const urls = extractUrls(row.text);
-        if (urls.length === 0)
-            continue;
-        totalLinks += urls.length;
-        const date = new Date(row.ts * 1000).toISOString().slice(0, 10);
-        linksByDate.set(date, (linksByDate.get(date) ?? 0) + urls.length);
-        const jid = row.jid ?? 'unknown';
-        const displayName = row.name ?? row.contact_name ?? jid;
-        if (!contactLinks.has(jid)) {
-            contactLinks.set(jid, { name: displayName, sent: 0, received: 0 });
-        }
-        const entry = contactLinks.get(jid);
-        if (row.from_me) {
-            entry.sent += urls.length;
-        }
-        else {
-            entry.received += urls.length;
-        }
-        for (const url of urls) {
-            const domain = parseDomain(url);
-            if (domain) {
-                domainCounts.set(domain, (domainCounts.get(domain) ?? 0) + 1);
-            }
-        }
-    }
-    const domainLeaderboard = [...domainCounts.entries()]
-        .sort((a, b) => b[1] - a[1])
-        .slice(0, limit)
-        .map(([domain, count]) => ({ domain, count, category: categorizeDomain(domain) }));
-    const uniqueDomains = domainCounts.size;
-    // --- Link categories ---
-    const catCounts = new Map();
-    for (const [domain, count] of domainCounts) {
-        const cat = categorizeDomain(domain);
-        if (!catCounts.has(cat)) {
-            catCounts.set(cat, { count: 0, domains: new Map() });
-        }
-        const catEntry = catCounts.get(cat);
-        catEntry.count += count;
-        catEntry.domains.set(domain, (catEntry.domains.get(domain) ?? 0) + count);
-    }
-    const linkCategories = [...catCounts.entries()]
-        .sort((a, b) => b[1].count - a[1].count)
-        .map(([category, { count, domains }]) => {
-        const topDomain = [...domains.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] ?? '';
-        return { category: category, count, topDomain };
-    });
-    // --- Media timeline density ---
-    const timelineBuckets = new Map();
+    const timeZone = resolveStatsTimeZone(params.timeZone);
+    const rows = db.prepare('SELECT ts FROM messages WHERE ts >= @since').all({ since });
+    const counts = new Array(7 * 24).fill(0);
     for (const row of rows) {
-        const date = new Date(row.ts * 1000).toISOString().slice(0, 10);
-        if (!timelineBuckets.has(date)) {
-            timelineBuckets.set(date, { images: 0, videos: 0, audio: 0, links: 0 });
-        }
-        const bucket = timelineBuckets.get(date);
-        const mt = row.media_type?.toLowerCase() ?? '';
-        if (mt === 'image')
-            bucket.images++;
-        else if (mt === 'video')
-            bucket.videos++;
-        else if (mt === 'audio')
-            bucket.audio++;
-        const urls = extractUrls(row.text);
-        bucket.links += urls.length;
+        const { weekday, hour } = weekdayHourInTimezone(row.ts, timeZone);
+        counts[weekday * 24 + hour] += 1;
     }
-    const mediaTimeline = [...timelineBuckets.entries()]
-        .filter(([, b]) => b.images + b.videos + b.audio + b.links > 0)
-        .sort((a, b) => a[0].localeCompare(b[0]))
-        .map(([date, b]) => ({ date, ...b }));
-    // --- Sharing asymmetry ---
-    const contactMedia = new Map();
-    for (const row of rows) {
-        const mt = row.media_type?.toLowerCase() ?? '';
-        if (!mt)
-            continue;
-        const jid = row.jid ?? 'unknown';
-        const displayName = row.name ?? row.contact_name ?? jid;
-        if (!contactMedia.has(jid)) {
-            contactMedia.set(jid, { name: displayName, mediaSent: 0, mediaReceived: 0 });
+    const cells = counts.map((count, index) => ({ day: Math.floor(index / 24), hour: index % 24, count }));
+    const peak = cells.reduce((best, cell) => (cell.count > (best?.count ?? 0) ? cell : best), null);
+    return { cells, peak, total: rows.length };
+}
+export const statsRouter = Router();
+/**
+ * Wrap a stats query in the archive-fingerprinted cache.
+ */
+function cachedStats(route, compute) {
+    return (req, res) => {
+        const key = statsCacheKey(route, req.query);
+        const etag = etagFor(key);
+        res.setHeader('Cache-Control', 'private, no-cache');
+        res.setHeader('ETag', etag);
+        if (req.headers['if-none-match'] === etag) {
+            res.status(304).end();
+            return;
         }
-        const entry = contactMedia.get(jid);
-        if (row.from_me)
-            entry.mediaSent++;
-        else
-            entry.mediaReceived++;
-    }
-    const sharingAsymmetry = [...new Set([...contactMedia.keys(), ...contactLinks.keys()])]
-        .map((jid) => {
-        const media = contactMedia.get(jid) ?? { name: jid, mediaSent: 0, mediaReceived: 0 };
-        const links = contactLinks.get(jid) ?? { name: jid, sent: 0, received: 0 };
-        const name = media.name !== jid ? media.name : links.name;
-        return {
-            jid,
-            name,
-            mediaSent: media.mediaSent,
-            mediaReceived: media.mediaReceived,
-            linksSent: links.sent,
-            linksReceived: links.received,
-        };
-    })
-        .filter((s) => s.mediaSent + s.mediaReceived + s.linksSent + s.linksReceived > 0)
-        .sort((a, b) => (b.mediaSent + b.mediaReceived + b.linksSent + b.linksReceived) - (a.mediaSent + a.mediaReceived + a.linksSent + a.linksReceived))
-        .slice(0, limit);
-    // --- First shared per contact ---
-    const firstSharedRows = db.prepare(`
-    SELECT
-      m.chat_jid,
-      ${cleanDisplayNameSql('COALESCE(chats.name, first_msg.chat_name)')} AS chat_name,
-      ${contactDisplayNameSql('fcontacts')} AS contact_name,
-      first_msg.text AS first_text,
-      first_msg.ts AS first_ts,
-      first_media.media_type AS first_media_type,
-      first_media.ts AS first_media_ts
-    FROM (
-      SELECT chat_jid, MIN(rowid) AS first_rowid
-      FROM messages
-      WHERE ts >= @since
-      GROUP BY chat_jid
-    ) m
-    LEFT JOIN messages first_msg ON first_msg.rowid = m.first_rowid
-    LEFT JOIN chats ON chats.jid = m.chat_jid
-    ${contactLeftJoins('fcontacts', 'm.chat_jid')}
-    LEFT JOIN (
-      SELECT chat_jid, MIN(rowid) AS media_rowid
-      FROM messages
-      WHERE ts >= @since AND media_type IS NOT NULL AND media_type <> ''
-      GROUP BY chat_jid
-    ) fm ON fm.chat_jid = m.chat_jid
-    LEFT JOIN messages first_media ON first_media.rowid = fm.media_rowid
-    ORDER BY first_msg.ts ASC
-    LIMIT @limit
-  `).all({ since, limit });
-    const firstShared = firstSharedRows.map((row) => ({
-        jid: row.chat_jid,
-        name: row.chat_name ?? row.contact_name ?? row.chat_jid,
-        firstMessageText: row.first_text,
-        firstMessageDate: unixSecondsToIso(row.first_ts) ?? new Date(0).toISOString(),
-        firstMediaType: row.first_media_type,
-        firstMediaDate: row.first_media_ts ? unixSecondsToIso(row.first_media_ts) : null,
-    }));
-    return {
-        domainLeaderboard,
-        mediaTimeline,
-        sharingAsymmetry,
-        linkCategories,
-        firstShared,
-        totalLinks,
-        uniqueDomains,
+        res.json(getCached(key, () => compute(req.query)));
     };
 }
-export const statsRouter = Router();
-statsRouter.get('/overview', (_req, res) => {
-    res.json(getOverviewStats());
-});
-statsRouter.get('/top-contacts', (req, res) => {
-    res.json(getTopContacts(req.query));
-});
-statsRouter.get('/message-volume', (req, res) => {
-    res.json(getMessageVolume(req.query));
-});
-statsRouter.get('/activity-heatmap', (req, res) => {
-    res.json(getActivityHeatmap(req.query));
-});
-statsRouter.get('/hour-of-day', (req, res) => {
-    res.json(getHourOfDayStats(req.query));
-});
-statsRouter.get('/day-of-week', (req, res) => {
-    res.json(getDayOfWeekStats(req.query));
-});
-statsRouter.get('/media-breakdown', (req, res) => {
-    res.json(getMediaBreakdown(req.query));
-});
-statsRouter.get('/media-senders', (req, res) => {
-    res.json(getMediaSenders(req.query));
-});
-statsRouter.get('/sent-received-ratio', (req, res) => {
-    res.json(getSentReceivedRatio(req.query));
-});
-statsRouter.get('/response-times', (req, res) => {
-    res.json(getResponseTimes(req.query));
-});
-statsRouter.get('/group-activity', (req, res) => {
-    res.json(getGroupActivity(req.query));
-});
-statsRouter.get('/streaks', (req, res) => {
-    res.json(getMessageStreaks(req.query));
-});
-statsRouter.get('/word-cloud', (req, res) => {
-    res.json(getWordCloud(req.query));
-});
-statsRouter.get('/emoji-analytics', (req, res) => {
-    res.json(getEmojiAnalytics(req.query));
-});
-statsRouter.get('/conversation-dynamics', (req, res) => {
-    res.json(getConversationDynamics(req.query));
-});
-statsRouter.get('/link-intelligence', (req, res) => {
-    res.json(getLinkIntelligence(req.query));
-});
+statsRouter.get('/overview', cachedStats('overview', () => getOverviewStats()));
+statsRouter.get('/top-contacts', cachedStats('top-contacts', (query) => getTopContacts(query)));
+statsRouter.get('/message-volume', cachedStats('message-volume', (query) => getMessageVolume(query)));
+statsRouter.get('/activity-heatmap', cachedStats('activity-heatmap', (query) => getActivityHeatmap(query)));
+statsRouter.get('/hour-of-day', cachedStats('hour-of-day', (query) => getHourOfDayStats(query)));
+statsRouter.get('/day-of-week', cachedStats('day-of-week', (query) => getDayOfWeekStats(query)));
+statsRouter.get('/media-breakdown', cachedStats('media-breakdown', (query) => getMediaBreakdown(query)));
+statsRouter.get('/media-senders', cachedStats('media-senders', (query) => getMediaSenders(query)));
+statsRouter.get('/sent-received-ratio', cachedStats('sent-received-ratio', (query) => getSentReceivedRatio(query)));
+statsRouter.get('/response-times', cachedStats('response-times', (query) => getResponseTimes(query)));
+statsRouter.get('/reply-latency', cachedStats('reply-latency', (query) => getReplyLatency(query)));
+statsRouter.get('/group-activity', cachedStats('group-activity', (query) => getGroupActivity(query)));
+statsRouter.get('/streaks', cachedStats('streaks', (query) => getMessageStreaks(query)));
+statsRouter.get('/word-cloud', cachedStats('word-cloud', (query) => getWordCloud(query)));
+statsRouter.get('/emoji-analytics', cachedStats('emoji-analytics', (query) => getEmojiAnalytics(query)));
+statsRouter.get('/conversation-dynamics', cachedStats('conversation-dynamics', (query) => getConversationDynamics(query)));
+statsRouter.get('/year-in-review', cachedStats('year-in-review', (query) => getYearInReview(query)));
+statsRouter.get('/contact-profile', cachedStats('contact-profile', (query) => getContactProfile(query)));
+statsRouter.get('/dormancy', cachedStats('dormancy', (query) => getDormancy(query)));
+statsRouter.get('/group-profile', cachedStats('group-profile', (query) => getGroupProfile(query)));
+statsRouter.get('/groups', cachedStats('groups', (query) => getGroupDirectory(query)));
+statsRouter.get('/link-intelligence', cachedStats('link-intelligence', (query) => getLinkIntelligence(query)));
+statsRouter.get('/weekly-rhythm', cachedStats('weekly-rhythm', (query) => getWeeklyRhythm(query)));
+statsRouter.get('/distinctive-words', cachedStats('distinctive-words', (query) => getDistinctiveWords(query)));
+statsRouter.get('/group-lifecycle', cachedStats('group-lifecycle', (query) => getGroupLifecycle(query)));
+statsRouter.get('/archived-chats', cachedStats('archived-chats', (query) => getArchivedChats(query)));