wacrawl-dashboard 0.1.1 → 0.1.2

package/apps/api/dist/routes/stats.js

@@ -1,9 +1,10 @@
 import { Router } from 'express';
 import { getDb } from '../db.js';
-import { cleanDisplayNameSql, contactDisplayNameSql, contactMatchSql } from '../lib/displayName.js';
+import { cleanDisplayNameSql, contactDisplayNameSql, contactLeftJoins } from '../lib/displayName.js';
 import { bucketDateSql, normalizeBucketDate, parseGranularity, parseLimit, parsePeriod, parseYear, sinceTimestamp, unixSecondsToIso, yearBounds, } from '../lib/query.js';
 import { hourInTimezone, resolveStatsTimeZone } from '../lib/timezone.js';
 import { USEFUL_WORD_STOP_SET } from '../lib/wordCloudUsefulStopWords.js';
+import { categorizeDomain, extractUrls, parseDomain } from '../lib/urlExtract.js';
 export function getOverviewStats(db = getDb()) {
     const messageRow = db.prepare(`
     SELECT
@@ -32,14 +33,15 @@ export function getTopContacts(params, db = getDb()) {
     WITH normalized_messages AS (
       SELECT
         COALESCE(
-          contacts.jid,
+          contacts_jid.jid,
+          contacts_lid.jid,
           CASE
             WHEN from_me = 1 THEN chat_jid
             ELSE COALESCE(sender_jid, chat_jid)
           END
         ) AS jid,
         ${contactDisplayNameSql('contacts')} AS contact_name,
-        contacts.phone AS contact_phone,
+        COALESCE(contacts_jid.phone, contacts_lid.phone) AS contact_phone,
         ${cleanDisplayNameSql(`
           CASE
             WHEN from_me = 1 THEN messages.chat_name
@@ -64,7 +66,7 @@ export function getTopContacts(params, db = getDb()) {
         WHEN messages.from_me = 1 THEN messages.chat_jid
         ELSE COALESCE(messages.sender_jid, messages.chat_jid)
       END
-      LEFT JOIN contacts ON ${contactMatchSql('contacts', `CASE
+      ${contactLeftJoins('contacts', `CASE
         WHEN messages.from_me = 1 THEN messages.chat_jid
         ELSE COALESCE(messages.sender_jid, messages.chat_jid)
       END`)}
@@ -193,7 +195,7 @@ export function getMediaSenders(params, db = getDb()) {
         media_size
       FROM messages
       LEFT JOIN chats ON chats.jid = CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END
-      LEFT JOIN contacts ON ${contactMatchSql('contacts', 'CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END')}
+      ${contactLeftJoins('contacts', 'CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END')}
       WHERE ts >= @since AND media_type IS NOT NULL AND media_type <> ''
     )
     SELECT
@@ -255,7 +257,7 @@ export function getResponseTimes(params, db = getDb()) {
       ROUND(AVG(ts - previous_ts)) AS averageSeconds
     FROM ordered
     LEFT JOIN chats ON chats.jid = ordered.chat_jid
-    LEFT JOIN contacts AS chat_contacts ON ${contactMatchSql('chat_contacts', 'ordered.chat_jid')}
+    ${contactLeftJoins('chat_contacts', 'ordered.chat_jid')}
     WHERE previous_ts IS NOT NULL
       AND previous_from_me IS NOT NULL
       AND previous_from_me <> from_me
@@ -328,6 +330,8 @@ export function getWordCloud(params, db = getDb()) {
     SELECT text
     FROM messages
     WHERE ts >= @since AND text IS NOT NULL AND TRIM(text) <> ''
+    ORDER BY ts DESC
+    LIMIT 30000
   `).all({ since });
     const counts = new Map();
     for (const row of rows) {
@@ -346,6 +350,432 @@ export function getWordCloud(params, db = getDb()) {
         .sort((a, b) => b.value - a.value || a.text.localeCompare(b.text))
         .slice(0, limit);
 }
+const EMOJI_REGEX = /[\p{Emoji_Presentation}\p{Extended_Pictographic}]/gu;
+export function getEmojiAnalytics(params, db = getDb()) {
+    const since = sinceTimestamp(parsePeriod(params.period));
+    const limit = parseLimit(params.limit, 15, 50);
+    const rows = db.prepare(`
+    SELECT
+      messages.text,
+      messages.from_me,
+      CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END AS jid,
+      ${cleanDisplayNameSql(`CASE WHEN messages.from_me = 1 THEN COALESCE(chats.name, messages.chat_name) ELSE messages.sender_name END`)} AS name,
+      ${contactDisplayNameSql('contacts')} AS contact_name
+    FROM messages
+    LEFT JOIN chats ON chats.jid = CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END
+    ${contactLeftJoins('contacts', 'CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END')}
+    WHERE messages.ts >= @since AND messages.text IS NOT NULL AND messages.text <> ''
+    ORDER BY messages.ts DESC
+    LIMIT 50000
+  `).all({ since });
+    const allCounts = new Map();
+    const sentCounts = new Map();
+    const receivedCounts = new Map();
+    const userCounts = new Map();
+    let totalEmojiCount = 0;
+    for (const row of rows) {
+        const emojis = (row.text ?? '').match(EMOJI_REGEX);
+        if (!emojis || emojis.length === 0)
+            continue;
+        totalEmojiCount += emojis.length;
+        const jid = row.jid ?? 'unknown';
+        const displayName = row.name ?? row.contact_name ?? jid;
+        if (!userCounts.has(jid)) {
+            userCounts.set(jid, { name: displayName, count: 0, emojis: new Map() });
+        }
+        const userEntry = userCounts.get(jid);
+        for (const emoji of emojis) {
+            allCounts.set(emoji, (allCounts.get(emoji) ?? 0) + 1);
+            if (row.from_me) {
+                sentCounts.set(emoji, (sentCounts.get(emoji) ?? 0) + 1);
+            }
+            else {
+                receivedCounts.set(emoji, (receivedCounts.get(emoji) ?? 0) + 1);
+            }
+            userEntry.count += 1;
+            userEntry.emojis.set(emoji, (userEntry.emojis.get(emoji) ?? 0) + 1);
+        }
+    }
+    const sortMap = (map, n) => [...map.entries()]
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, n)
+        .map(([emoji, count]) => ({ emoji, count }));
+    const topEmojiUsers = [...userCounts.entries()]
+        .sort((a, b) => b[1].count - a[1].count)
+        .slice(0, limit)
+        .map(([jid, entry]) => {
+        const topEmoji = [...entry.emojis.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] ?? '';
+        return { jid, name: entry.name, count: entry.count, topEmoji };
+    });
+    return {
+        topEmojis: sortMap(allCounts, limit),
+        topSentEmojis: sortMap(sentCounts, limit),
+        topReceivedEmojis: sortMap(receivedCounts, limit),
+        totalEmojiCount,
+        uniqueEmojiCount: allCounts.size,
+        topEmojiUsers,
+    };
+}
+const SESSION_GAP_SECONDS = 4 * 3600;
+const GHOST_WINDOW_SECONDS = 24 * 3600;
+function resolveChatNames(jids, db) {
+    const map = new Map();
+    if (jids.length === 0)
+        return map;
+    const stmt = db.prepare(`
+    SELECT
+      c.jid,
+      COALESCE(
+        ${cleanDisplayNameSql('c.name')},
+        ${contactDisplayNameSql('chat_contacts')},
+        ${cleanDisplayNameSql('c.jid')},
+        'Unknown'
+      ) AS name
+    FROM chats c
+    ${contactLeftJoins('chat_contacts', 'c.jid')}
+    WHERE c.jid = @jid
+  `);
+    for (const jid of jids) {
+        const row = stmt.get({ jid });
+        map.set(jid, row?.name ?? jid);
+    }
+    return map;
+}
+function computeTrajectoryDirection(trend) {
+    if (trend.length < 4)
+        return 'stable';
+    const half = Math.floor(trend.length / 2);
+    const firstHalfAvg = trend.slice(0, half).reduce((s, p) => s + p.count, 0) / half;
+    const secondHalfAvg = trend.slice(half).reduce((s, p) => s + p.count, 0) / (trend.length - half);
+    const ratio = secondHalfAvg / (firstHalfAvg || 1);
+    if (ratio > 1.25)
+        return 'growing';
+    if (ratio < 0.75)
+        return 'fading';
+    return 'stable';
+}
+export function getConversationDynamics(params, db = getDb()) {
+    const since = sinceTimestamp(parsePeriod(params.period));
+    const limit = parseLimit(params.limit, 10, 50);
+    const timeZone = resolveStatsTimeZone(params.timeZone);
+    const rows = db.prepare(`
+    SELECT m.chat_jid, m.ts, m.from_me
+    FROM messages m
+    WHERE m.ts >= @since
+    ORDER BY m.chat_jid, m.ts, m.rowid
+  `).all({ since });
+    const chatGroups = new Map();
+    for (const row of rows) {
+        let group = chatGroups.get(row.chat_jid);
+        if (!group) {
+            group = [];
+            chatGroups.set(row.chat_jid, group);
+        }
+        group.push(row);
+    }
+    const rawInitiation = [];
+    const rawDepth = [];
+    const rawGhost = [];
+    const rawLateNight = [];
+    const rawTrajectory = [];
+    for (const [jid, messages] of chatGroups) {
+        if (messages.length < 2)
+            continue;
+        let byMe = 0;
+        let byThem = 0;
+        let sessionStart = 0;
+        const sessionSizes = [];
+        if (messages[0].from_me)
+            byMe++;
+        else
+            byThem++;
+        for (let i = 1; i < messages.length; i++) {
+            if (messages[i].ts - messages[i - 1].ts >= SESSION_GAP_SECONDS) {
+                sessionSizes.push(i - sessionStart);
+                sessionStart = i;
+                if (messages[i].from_me)
+                    byMe++;
+                else
+                    byThem++;
+            }
+        }
+        sessionSizes.push(messages.length - sessionStart);
+        const totalSessions = sessionSizes.length;
+        rawInitiation.push({ jid, byMe, byThem, total: byMe + byThem });
+        rawDepth.push({ jid, avgDepth: Math.round((messages.length / totalSessions) * 10) / 10, sessions: totalSessions, total: messages.length });
+        let ghosted = 0;
+        let totalSent = 0;
+        for (let i = 0; i < messages.length; i++) {
+            if (!messages[i].from_me)
+                continue;
+            totalSent++;
+            let replied = false;
+            for (let j = i + 1; j < messages.length; j++) {
+                if (!messages[j].from_me) {
+                    replied = messages[j].ts - messages[i].ts <= GHOST_WINDOW_SECONDS;
+                    break;
+                }
+            }
+            if (!replied)
+                ghosted++;
+        }
+        if (totalSent > 0) {
+            rawGhost.push({ jid, ghosted, totalSent });
+        }
+        let lateNight = 0;
+        let workHrs = 0;
+        let other = 0;
+        for (const msg of messages) {
+            const hour = hourInTimezone(msg.ts, timeZone);
+            if (hour < 5)
+                lateNight++;
+            else if (hour >= 9 && hour < 17)
+                workHrs++;
+            else
+                other++;
+        }
+        rawLateNight.push({ jid, lateNight, workHours: workHrs, other, total: messages.length });
+        const monthCounts = new Map();
+        for (const msg of messages) {
+            const d = new Date(msg.ts * 1000);
+            const month = `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`;
+            monthCounts.set(month, (monthCounts.get(month) ?? 0) + 1);
+        }
+        rawTrajectory.push({ jid, monthCounts, total: messages.length });
+    }
+    const topInitiation = rawInitiation.filter((r) => r.total >= 3).sort((a, b) => b.total - a.total).slice(0, limit);
+    const topDepth = rawDepth.filter((r) => r.sessions >= 3).sort((a, b) => b.avgDepth - a.avgDepth).slice(0, limit);
+    const topGhost = rawGhost.filter((r) => r.totalSent >= 3).sort((a, b) => (b.ghosted / b.totalSent) - (a.ghosted / a.totalSent)).slice(0, limit);
+    const topLateNight = rawLateNight.filter((r) => r.lateNight > 0).sort((a, b) => b.lateNight - a.lateNight).slice(0, limit);
+    const topTrajectory = rawTrajectory.filter((r) => r.monthCounts.size >= 2).sort((a, b) => b.total - a.total).slice(0, limit);
+    const allJids = new Set();
+    for (const r of topInitiation)
+        allJids.add(r.jid);
+    for (const r of topDepth)
+        allJids.add(r.jid);
+    for (const r of topGhost)
+        allJids.add(r.jid);
+    for (const r of topLateNight)
+        allJids.add(r.jid);
+    for (const r of topTrajectory)
+        allJids.add(r.jid);
+    const names = resolveChatNames([...allJids], db);
+    const name = (jid) => names.get(jid) ?? jid;
+    const initiationRatio = topInitiation.map((r) => ({
+        jid: r.jid,
+        name: name(r.jid),
+        initiatedByMe: r.byMe,
+        initiatedByThem: r.byThem,
+        ratio: Math.round((r.byMe / r.total) * 100) / 100,
+    }));
+    const conversationDepth = topDepth.map((r) => ({
+        jid: r.jid,
+        name: name(r.jid),
+        avgMessagesPerSession: r.avgDepth,
+        totalSessions: r.sessions,
+    }));
+    const ghostScore = topGhost.map((r) => ({
+        jid: r.jid,
+        name: name(r.jid),
+        ghostedCount: r.ghosted,
+        totalSent: r.totalSent,
+        ghostRate: Math.round((r.ghosted / r.totalSent) * 100) / 100,
+    }));
+    const lateNightTexters = topLateNight.map((r) => ({
+        jid: r.jid,
+        name: name(r.jid),
+        lateNight: r.lateNight,
+        workHours: r.workHours,
+        otherHours: r.other,
+        totalMessages: r.total,
+        lateNightPct: Math.round((r.lateNight / r.total) * 100),
+    }));
+    const relationshipTrajectory = topTrajectory.map((r) => {
+        const sorted = [...r.monthCounts.entries()].sort((a, b) => a[0].localeCompare(b[0]));
+        const trend = sorted.map(([month, count]) => ({ month, count }));
+        return {
+            jid: r.jid,
+            name: name(r.jid),
+            trend,
+            direction: computeTrajectoryDirection(trend),
+        };
+    });
+    return { initiationRatio, conversationDepth, ghostScore, relationshipTrajectory, lateNightTexters };
+}
+export function getLinkIntelligence(params, db = getDb()) {
+    const since = sinceTimestamp(parsePeriod(params.period));
+    const limit = parseLimit(params.limit, 15, 50);
+    const rows = db.prepare(`
+    SELECT
+      messages.text,
+      messages.ts,
+      messages.from_me,
+      messages.media_type,
+      CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END AS jid,
+      ${cleanDisplayNameSql(`CASE WHEN messages.from_me = 1 THEN COALESCE(chats.name, messages.chat_name) ELSE messages.sender_name END`)} AS name,
+      ${contactDisplayNameSql('contacts')} AS contact_name
+    FROM messages
+    LEFT JOIN chats ON chats.jid = CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END
+    ${contactLeftJoins('contacts', 'CASE WHEN messages.from_me = 1 THEN messages.chat_jid ELSE COALESCE(messages.sender_jid, messages.chat_jid) END')}
+    WHERE messages.ts >= @since
+    ORDER BY messages.ts ASC
+  `).all({ since });
+    // --- Domain leaderboard + categories ---
+    const domainCounts = new Map();
+    const linksByDate = new Map();
+    const contactLinks = new Map();
+    let totalLinks = 0;
+    for (const row of rows) {
+        const urls = extractUrls(row.text);
+        if (urls.length === 0)
+            continue;
+        totalLinks += urls.length;
+        const date = new Date(row.ts * 1000).toISOString().slice(0, 10);
+        linksByDate.set(date, (linksByDate.get(date) ?? 0) + urls.length);
+        const jid = row.jid ?? 'unknown';
+        const displayName = row.name ?? row.contact_name ?? jid;
+        if (!contactLinks.has(jid)) {
+            contactLinks.set(jid, { name: displayName, sent: 0, received: 0 });
+        }
+        const entry = contactLinks.get(jid);
+        if (row.from_me) {
+            entry.sent += urls.length;
+        }
+        else {
+            entry.received += urls.length;
+        }
+        for (const url of urls) {
+            const domain = parseDomain(url);
+            if (domain) {
+                domainCounts.set(domain, (domainCounts.get(domain) ?? 0) + 1);
+            }
+        }
+    }
+    const domainLeaderboard = [...domainCounts.entries()]
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, limit)
+        .map(([domain, count]) => ({ domain, count, category: categorizeDomain(domain) }));
+    const uniqueDomains = domainCounts.size;
+    // --- Link categories ---
+    const catCounts = new Map();
+    for (const [domain, count] of domainCounts) {
+        const cat = categorizeDomain(domain);
+        if (!catCounts.has(cat)) {
+            catCounts.set(cat, { count: 0, domains: new Map() });
+        }
+        const catEntry = catCounts.get(cat);
+        catEntry.count += count;
+        catEntry.domains.set(domain, (catEntry.domains.get(domain) ?? 0) + count);
+    }
+    const linkCategories = [...catCounts.entries()]
+        .sort((a, b) => b[1].count - a[1].count)
+        .map(([category, { count, domains }]) => {
+        const topDomain = [...domains.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] ?? '';
+        return { category: category, count, topDomain };
+    });
+    // --- Media timeline density ---
+    const timelineBuckets = new Map();
+    for (const row of rows) {
+        const date = new Date(row.ts * 1000).toISOString().slice(0, 10);
+        if (!timelineBuckets.has(date)) {
+            timelineBuckets.set(date, { images: 0, videos: 0, audio: 0, links: 0 });
+        }
+        const bucket = timelineBuckets.get(date);
+        const mt = row.media_type?.toLowerCase() ?? '';
+        if (mt === 'image')
+            bucket.images++;
+        else if (mt === 'video')
+            bucket.videos++;
+        else if (mt === 'audio')
+            bucket.audio++;
+        const urls = extractUrls(row.text);
+        bucket.links += urls.length;
+    }
+    const mediaTimeline = [...timelineBuckets.entries()]
+        .filter(([, b]) => b.images + b.videos + b.audio + b.links > 0)
+        .sort((a, b) => a[0].localeCompare(b[0]))
+        .map(([date, b]) => ({ date, ...b }));
+    // --- Sharing asymmetry ---
+    const contactMedia = new Map();
+    for (const row of rows) {
+        const mt = row.media_type?.toLowerCase() ?? '';
+        if (!mt)
+            continue;
+        const jid = row.jid ?? 'unknown';
+        const displayName = row.name ?? row.contact_name ?? jid;
+        if (!contactMedia.has(jid)) {
+            contactMedia.set(jid, { name: displayName, mediaSent: 0, mediaReceived: 0 });
+        }
+        const entry = contactMedia.get(jid);
+        if (row.from_me)
+            entry.mediaSent++;
+        else
+            entry.mediaReceived++;
+    }
+    const sharingAsymmetry = [...new Set([...contactMedia.keys(), ...contactLinks.keys()])]
+        .map((jid) => {
+        const media = contactMedia.get(jid) ?? { name: jid, mediaSent: 0, mediaReceived: 0 };
+        const links = contactLinks.get(jid) ?? { name: jid, sent: 0, received: 0 };
+        const name = media.name !== jid ? media.name : links.name;
+        return {
+            jid,
+            name,
+            mediaSent: media.mediaSent,
+            mediaReceived: media.mediaReceived,
+            linksSent: links.sent,
+            linksReceived: links.received,
+        };
+    })
+        .filter((s) => s.mediaSent + s.mediaReceived + s.linksSent + s.linksReceived > 0)
+        .sort((a, b) => (b.mediaSent + b.mediaReceived + b.linksSent + b.linksReceived) - (a.mediaSent + a.mediaReceived + a.linksSent + a.linksReceived))
+        .slice(0, limit);
+    // --- First shared per contact ---
+    const firstSharedRows = db.prepare(`
+    SELECT
+      m.chat_jid,
+      ${cleanDisplayNameSql('COALESCE(chats.name, first_msg.chat_name)')} AS chat_name,
+      ${contactDisplayNameSql('fcontacts')} AS contact_name,
+      first_msg.text AS first_text,
+      first_msg.ts AS first_ts,
+      first_media.media_type AS first_media_type,
+      first_media.ts AS first_media_ts
+    FROM (
+      SELECT chat_jid, MIN(rowid) AS first_rowid
+      FROM messages
+      WHERE ts >= @since
+      GROUP BY chat_jid
+    ) m
+    LEFT JOIN messages first_msg ON first_msg.rowid = m.first_rowid
+    LEFT JOIN chats ON chats.jid = m.chat_jid
+    ${contactLeftJoins('fcontacts', 'm.chat_jid')}
+    LEFT JOIN (
+      SELECT chat_jid, MIN(rowid) AS media_rowid
+      FROM messages
+      WHERE ts >= @since AND media_type IS NOT NULL AND media_type <> ''
+      GROUP BY chat_jid
+    ) fm ON fm.chat_jid = m.chat_jid
+    LEFT JOIN messages first_media ON first_media.rowid = fm.media_rowid
+    ORDER BY first_msg.ts ASC
+    LIMIT @limit
+  `).all({ since, limit });
+    const firstShared = firstSharedRows.map((row) => ({
+        jid: row.chat_jid,
+        name: row.chat_name ?? row.contact_name ?? row.chat_jid,
+        firstMessageText: row.first_text,
+        firstMessageDate: unixSecondsToIso(row.first_ts) ?? new Date(0).toISOString(),
+        firstMediaType: row.first_media_type,
+        firstMediaDate: row.first_media_ts ? unixSecondsToIso(row.first_media_ts) : null,
+    }));
+    return {
+        domainLeaderboard,
+        mediaTimeline,
+        sharingAsymmetry,
+        linkCategories,
+        firstShared,
+        totalLinks,
+        uniqueDomains,
+    };
+}
 export const statsRouter = Router();
 statsRouter.get('/overview', (_req, res) => {
     res.json(getOverviewStats());
@@ -386,3 +816,12 @@ statsRouter.get('/streaks', (req, res) => {
 statsRouter.get('/word-cloud', (req, res) => {
     res.json(getWordCloud(req.query));
 });
+statsRouter.get('/emoji-analytics', (req, res) => {
+    res.json(getEmojiAnalytics(req.query));
+});
+statsRouter.get('/conversation-dynamics', (req, res) => {
+    res.json(getConversationDynamics(req.query));
+});
+statsRouter.get('/link-intelligence', (req, res) => {
+    res.json(getLinkIntelligence(req.query));
+});