parallelclaw 1.0.0

package/lib/parse-telegram-html.js

@@ -0,0 +1,384 @@
+/**
+ * Telegram Desktop HTML export → Telegram-JSON-shape converter.
+ *
+ * Telegram Desktop offers two export formats:
+ *   - "Machine-readable JSON"  — what memex's importTelegram expects
+ *   - "Human-readable HTML"    — what many users pick by default
+ *
+ * Users frequently export as HTML by accident (often the default in the
+ * Telegram UI), then memex's inbox watcher silently ignores the dropped
+ * directory. This module makes HTML work: parse → emit the same shape
+ * importTelegram already understands.
+ *
+ * Telegram's HTML export is reasonably stable:
+ *
+ *   ChatExport_<chat-title>_<date>/
+ *     ├── messages.html     (or messages.htm — chunked: messages2, messages3, …)
+ *     ├── photos/
+ *     ├── files/
+ *     ├── stickers/
+ *     └── voice_messages/
+ *
+ * Each messages*.html has structure:
+ *
+ *   <div class="message default clearfix" id="message12345">
+ *     <div class="body">
+ *       <div class="from_name"> ↳ Sender Name </div>      (may be absent on "joined" messages)
+ *       <div class="text"> message text </div>
+ *       <div class="pull_right date details" title="2024-01-01 14:23:45 UTC+03:00">14:23</div>
+ *     </div>
+ *   </div>
+ *
+ *   Joined message = same sender as previous, has class "joined", no from_name.
+ *   Service message = class "service" (joined chat, name change, …) — we skip these.
+ *   Forwarded = "forwarded body" wrapping the message body.
+ *   Reply = "reply_to details" sibling.
+ *
+ * We use regex-based parsing (no DOM dependency) because Telegram's class
+ * names are stable and we control which fields we care about. If Telegram
+ * radically changes the schema, parser breaks loudly (returns 0 messages
+ * + clear log) rather than silently corrupting.
+ */
+
+import { readFileSync, existsSync, readdirSync, statSync } from 'node:fs';
+import { join, basename, dirname } from 'node:path';
+
+/**
+ * Detect if a given path is a Telegram HTML export.
+ * Accepts both a directory (most common — ChatExport_xxx/) and a bare
+ * messages.html file (rare — user dropped just the one file).
+ *
+ * Returns { type: 'dir' | 'file' | null, htmlFiles: string[] }
+ *   null type means "not a Telegram HTML export"
+ */
+export function detectTelegramHtml(path) {
+  if (!existsSync(path)) return { type: null, htmlFiles: [] };
+  const stats = statSync(path);
+
+  // Directory case: look for messages*.html inside
+  if (stats.isDirectory()) {
+    let entries = [];
+    try { entries = readdirSync(path); } catch (_) { return { type: null, htmlFiles: [] }; }
+    const htmlFiles = entries
+      .filter((f) => /^messages\d*\.html?$/i.test(f))
+      .map((f) => join(path, f));
+    if (htmlFiles.length === 0) return { type: null, htmlFiles: [] };
+    // Verify the first one contains Telegram-shaped markers
+    const head = safeReadHead(htmlFiles[0]);
+    if (!looksLikeTelegram(head)) return { type: null, htmlFiles: [] };
+    // Sort chunks: messages.html < messages2.html < messages3.html …
+    htmlFiles.sort(numericChunkSort);
+    return { type: 'dir', htmlFiles };
+  }
+
+  // Single file case: must be messages*.html
+  if (stats.isFile() && /\.html?$/i.test(path) && /messages\d*\.html?$/i.test(basename(path))) {
+    const head = safeReadHead(path);
+    if (!looksLikeTelegram(head)) return { type: null, htmlFiles: [] };
+    return { type: 'file', htmlFiles: [path] };
+  }
+
+  return { type: null, htmlFiles: [] };
+}
+
+function safeReadHead(file, bytes = 8192) {
+  try {
+    return readFileSync(file, 'utf-8').slice(0, bytes);
+  } catch (_) {
+    return '';
+  }
+}
+
+function looksLikeTelegram(head) {
+  // Reliable markers in Telegram Desktop HTML exports
+  return /class="page_wrap"/.test(head) ||
+         /class="page_body chat_page"/.test(head) ||
+         (/class="from_name"/.test(head) && /class="text"/.test(head));
+}
+
+function numericChunkSort(a, b) {
+  const numA = parseInt((basename(a).match(/messages(\d*)\.html?/i) || [, '0'])[1] || '0', 10);
+  const numB = parseInt((basename(b).match(/messages(\d*)\.html?/i) || [, '0'])[1] || '0', 10);
+  return numA - numB;
+}
+
+/**
+ * Strip HTML tags and decode common entities → plain text.
+ * Conservative: preserves newlines from <br>, paragraph breaks from </div>.
+ */
+function htmlToText(html) {
+  if (!html) return '';
+  let out = String(html);
+  // Convert breaks to newlines BEFORE stripping tags
+  out = out.replace(/<br\s*\/?>/gi, '\n');
+  out = out.replace(/<\/p>/gi, '\n\n');
+  out = out.replace(/<\/div>/gi, '\n');
+  // Drop all remaining tags
+  out = out.replace(/<[^>]+>/g, '');
+  // Decode common entities
+  out = out
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&apos;/g, "'")
+    .replace(/&#39;/g, "'")
+    .replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(parseInt(hex, 16)))
+    .replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(parseInt(dec, 10)));
+  // Collapse 3+ blank lines, trim
+  out = out.replace(/\n{3,}/g, '\n\n').trim();
+  return out;
+}
+
+/**
+ * Parse a Telegram date title into Unix timestamp.
+ * Telegram emits dates in the user's locale format, e.g.:
+ *   • "2024-01-01 14:23:45 UTC+03:00"   (ISO — English locale)
+ *   • "01.01.2024 14:23:45 UTC+03:00"   (European — Russian / German / etc.)
+ *   • "01/01/2024 14:23:45 UTC+03:00"   (US slash format — less common in exports)
+ * Returns { tsUnix, isoString } or null if unparseable.
+ */
+function parseTelegramDate(title) {
+  if (!title) return null;
+  let y, mo, d, h, mi, s, sign, oh, om;
+  // ISO: YYYY-MM-DD HH:MM:SS [UTC±HH:MM]
+  let m = title.match(/^(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})(?:\s+UTC([+-])(\d{2}):(\d{2}))?$/);
+  if (m) {
+    [, y, mo, d, h, mi, s, sign, oh, om] = m;
+  } else {
+    // European: DD.MM.YYYY HH:MM:SS [UTC±HH:MM]  (also supports "/" or "-" as separator)
+    m = title.match(/^(\d{2})[.\/-](\d{2})[.\/-](\d{4})\s+(\d{2}):(\d{2}):(\d{2})(?:\s+UTC([+-])(\d{2}):(\d{2}))?$/);
+    if (!m) return null;
+    [, d, mo, y, h, mi, s, sign, oh, om] = m;
+  }
+  // Construct an ISO 8601 string with the explicit offset (or UTC if absent)
+  const offset = sign ? `${sign}${oh}:${om}` : 'Z';
+  const iso = `${y}-${mo}-${d}T${h}:${mi}:${s}${offset}`;
+  const date = new Date(iso);
+  if (isNaN(date.getTime())) return null;
+  return {
+    tsUnix: Math.floor(date.getTime() / 1000),
+    isoString: iso.replace(/[+-]\d{2}:\d{2}$/, '').replace('Z', ''),
+  };
+}
+
+/**
+ * Parse a single message div (raw HTML segment).
+ * Returns null for service messages (we skip those) or messages with no text.
+ */
+function parseMessageDiv(messageHtml, lastSender) {
+  // Skip service messages outright
+  if (/class="message service\b/.test(messageHtml)) return null;
+
+  // Extract message id from outer div: id="message12345"
+  const idMatch = messageHtml.match(/id="message(\d+)"/);
+  const msgId = idMatch ? idMatch[1] : null;
+  if (!msgId) return null;
+
+  const isJoined = /class="message [^"]*joined/.test(messageHtml);
+
+  // Forwarded marker
+  const isForwarded = /class="forwarded body"/.test(messageHtml);
+  let forwardedFrom = null;
+  if (isForwarded) {
+    const fwdM = messageHtml.match(/class="forwarded[^"]*"[\s\S]*?<div class="from_name"[^>]*>\s*([\s\S]*?)\s*<\/div>/);
+    if (fwdM) {
+      forwardedFrom = htmlToText(fwdM[1]).replace(/^Forwarded from:?\s*/i, '').trim();
+    }
+  }
+
+  // Sender (from_name) — absent on joined messages
+  let fromName = null;
+  const fromM = messageHtml.match(/<div class="from_name"[^>]*>\s*([\s\S]*?)\s*<\/div>/);
+  if (fromM && !isForwarded) {
+    fromName = htmlToText(fromM[1]).trim();
+  }
+  // If joined, inherit lastSender; otherwise use parsed or fallback
+  if (!fromName && isJoined && lastSender) fromName = lastSender;
+  if (!fromName) fromName = 'Unknown';
+
+  // Date — title attribute on `.date.details`
+  let date = null;
+  const dateM = messageHtml.match(/class="[^"]*\bdate details[^"]*"\s+title="([^"]+)"/);
+  if (dateM) date = parseTelegramDate(dateM[1]);
+
+  // Main text — last `<div class="text">…</div>` inside body (forwards may have one earlier)
+  let text = '';
+  const textMatches = [...messageHtml.matchAll(/<div class="text"[^>]*>([\s\S]*?)<\/div>(?=\s*(?:<div class="(?!text)|<\/div>|<a class="|$))/g)];
+  if (textMatches.length > 0) {
+    // Use last one (the actual message body, after any quoted/forwarded preamble)
+    text = htmlToText(textMatches[textMatches.length - 1][1]);
+  }
+
+  // Reply marker — include as prefix so it's searchable but not lost
+  const replyM = messageHtml.match(/class="reply_to details"[^>]*>([\s\S]*?)<\/div>/);
+  if (replyM) {
+    const replyTxt = htmlToText(replyM[1]).replace(/^In reply to\s+/i, '').trim();
+    if (replyTxt) text = `↩ Reply: ${replyTxt}\n\n${text}`;
+  }
+
+  // Photo / media — if no text, note the media presence so the row isn't lost.
+  // Use word-boundary regexes since class attrs like "photo_wrap clearfix pull_left"
+  // wouldn't match a strict `class="photo_wrap"` pattern.
+  if (!text) {
+    if (/class="[^"]*\bphoto_wrap\b/.test(messageHtml)) text = '[photo]';
+    else if (/class="[^"]*\bmedia_voice_message\b/.test(messageHtml)) text = '[voice message]';
+    else if (/class="[^"]*\bmedia_video_file\b/.test(messageHtml)) text = '[video]';
+    else if (/class="[^"]*\bmedia_audio_file\b/.test(messageHtml)) text = '[audio]';
+    else if (/class="[^"]*\bmedia_file\b/.test(messageHtml)) text = '[file]';
+    else if (/class="[^"]*\bsticker\b/.test(messageHtml)) text = '[sticker]';
+    else return null;  // Truly empty — skip
+  }
+
+  // Build the message object in the shape importTelegram expects
+  // (date and date_unixtime are required by the importer)
+  const isoDate = date ? date.isoString : null;
+  const tsUnix = date ? date.tsUnix : 0;
+
+  return {
+    id: parseInt(msgId, 10),
+    type: 'message',
+    date: isoDate || '1970-01-01T00:00:00',
+    date_unixtime: tsUnix > 0 ? String(tsUnix) : '0',
+    from: fromName,
+    from_id: fromName ? `user_html_${slugify(fromName)}` : 'unknown',
+    text: text,
+    ...(forwardedFrom ? { forwarded_from: forwardedFrom } : {}),
+  };
+}
+
+function slugify(s) {
+  return String(s).toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '').slice(0, 40) || 'anon';
+}
+
+/**
+ * Extract chat title from messages.html (or first chunk).
+ * Falls back to directory name basename, then "Telegram chat".
+ */
+function extractChatTitle(htmlContent, fallbackPath) {
+  // PREFER the page_header — it's the actual chat name as shown in Telegram.
+  // The <title> tag is often the locale-specific "Exported Data" / "Telegram"
+  // boilerplate, which we want to avoid.
+  const headerM = htmlContent.match(/<div class="page_header"[\s\S]*?<div class="text bold"[^>]*>\s*([\s\S]*?)\s*<\/div>/);
+  if (headerM) {
+    const t = htmlToText(headerM[1]).trim();
+    if (t) return t;
+  }
+  // Fallback: <title>...</title> — strip "Chat Export" / "Telegram" / "Exported Data" suffixes
+  const titleM = htmlContent.match(/<title>\s*([^<]+?)\s*<\/title>/i);
+  if (titleM) {
+    let t = titleM[1].trim();
+    t = t.replace(/\s*[—-]\s*(Chat Export|Telegram).*$/i, '').trim();
+    // Skip locale boilerplate that Telegram itself uses as the page <title>
+    if (t && !/^(Telegram|Exported Data|Экспорт(ированные)? данные|Эспортированные данные)$/i.test(t)) {
+      return t;
+    }
+  }
+  // Fallback: dirname of the parent ChatExport_xxx folder
+  if (fallbackPath) {
+    const parent = basename(dirname(fallbackPath));
+    if (parent && parent.startsWith('ChatExport')) {
+      return parent.replace(/^ChatExport_?/, '').replace(/_/g, ' ').trim() || 'Telegram chat';
+    }
+  }
+  return 'Telegram chat';
+}
+
+/**
+ * Main entrypoint. Parse a Telegram HTML export path → return an object
+ * shaped like a Telegram JSON export, ready for importTelegram().
+ *
+ * Returns null if path isn't a valid Telegram HTML export.
+ *
+ * Object shape:
+ *   {
+ *     personal_information: { user_id: "" },
+ *     chats: {
+ *       list: [{
+ *         id: <stable hash of chat title>,
+ *         name: <chat title>,
+ *         type: "personal_chat",
+ *         messages: [{ id, type, date, date_unixtime, from, from_id, text, … }, …]
+ *       }]
+ *     }
+ *   }
+ */
+export function parseTelegramHtmlExport(path, opts = {}) {
+  const detection = detectTelegramHtml(path);
+  if (!detection.type) return null;
+  if (detection.htmlFiles.length === 0) return null;
+
+  let allMessages = [];
+  let chatTitle = null;
+  let lastSender = null;
+
+  for (const htmlPath of detection.htmlFiles) {
+    let content;
+    try { content = readFileSync(htmlPath, 'utf-8'); }
+    catch (_) { continue; }
+
+    if (!chatTitle) chatTitle = extractChatTitle(content, htmlPath);
+
+    // Split into per-message blocks. The reliable boundary is the
+    // opening `<div class="message ` of the next message.
+    // Use a tolerant regex that handles the message default / joined variants.
+    const messageBlocks = [...content.matchAll(/<div class="message [^"]*"[\s\S]*?(?=<div class="message [^"]*"|<div class="page_footer"|<\/body>)/g)];
+
+    for (const blockMatch of messageBlocks) {
+      const msg = parseMessageDiv(blockMatch[0], lastSender);
+      if (msg) {
+        allMessages.push(msg);
+        // Track sender for "joined" continuation messages
+        if (msg.from && msg.from !== 'Unknown') lastSender = msg.from;
+      }
+    }
+  }
+
+  if (allMessages.length === 0) return null;
+
+  // Stable chat id: hash of title + first message ts (good enough for dedup)
+  // We use a simple numeric hash so the synthetic chat_id is stable across re-imports.
+  const chatId = stableChatId(chatTitle || 'Telegram chat', allMessages[0]?.date_unixtime || '0');
+
+  // Detect chat type from sender diversity. A `personal_chat` has at most 2 distinct
+  // senders (you + the other person). 3+ distinct senders → group / supergroup.
+  // We can't distinguish private_group vs public_supergroup from HTML alone, so we
+  // call it `private_group` (matches the JSON export taxonomy).
+  const distinctSenders = new Set();
+  for (const m of allMessages) {
+    if (m.from && m.from !== 'Unknown') distinctSenders.add(m.from);
+    if (distinctSenders.size > 2) break;
+  }
+  const chatType = distinctSenders.size > 2 ? 'private_group' : 'personal_chat';
+
+  return {
+    personal_information: { user_id: '' },
+    chats: {
+      list: [
+        {
+          id: chatId,
+          name: chatTitle || 'Telegram chat',
+          type: chatType,
+          messages: allMessages,
+        },
+      ],
+    },
+    _source: {
+      format: 'telegram-html',
+      original_path: path,
+      chunks: detection.htmlFiles.length,
+      messages_total: allMessages.length,
+    },
+  };
+}
+
+function stableChatId(title, firstTs) {
+  let hash = 0;
+  const key = title + ':' + firstTs;
+  for (let i = 0; i < key.length; i++) {
+    hash = ((hash << 5) - hash) + key.charCodeAt(i);
+    hash |= 0;
+  }
+  return Math.abs(hash);
+}

package/lib/parse.js

@@ -0,0 +1,175 @@
+/**
+ * Shared dialogue-only parser for Claude Code / Cowork JSONL.
+ *
+ * Used by both the MCP server (server.js, importing inbox files) and the
+ * ingest daemon (ingest.js, reading deltas from raw source files).
+ */
+
+/** Skip these top-level event types — they're not dialogue. */
+export const CLAUDE_CODE_SKIP_TYPES = new Set(['queue-operation', 'ai-title', 'summary']);
+
+/** Auto-generated user messages produced by /compact, /resume, and
+ *  continuation flows. They're real messages (we keep them in the
+ *  index), but they're never useful as conversation titles. */
+export const CONTINUATION_PREFIXES = [
+  'This session is being continued',
+  'Continue from where you left off',
+  'Please continue from where you left off',
+];
+
+export function isContinuationBoilerplate(text) {
+  for (const p of CONTINUATION_PREFIXES) if (text.startsWith(p)) return true;
+  // XML/tag-wrapped artefacts (uploaded_files, system-reminder, command-name…)
+  if (text.startsWith('<')) return true;
+  return false;
+}
+
+/** Extract a clean dialogue message from a Claude Code JSONL record.
+ *
+ *  Handles both:
+ *    1. Legacy flat shape (original spec):
+ *       {"role":"user","content":"...","timestamp":"..."}
+ *    2. Real nested shape (current Claude Code / Cowork on disk):
+ *       {"type":"user","message":{"role":"user","content":"..."},"timestamp":"..."}
+ *       {"parentUuid":"...","message":{"role":"assistant","content":[{type:"text",text:"..."},...]}}
+ *
+ *  Filters out everything that isn't human-readable dialogue:
+ *    - queue-operation / ai-title / summary events
+ *    - attachment-only records (deferred_tools_delta, skill_listing, plan_mode)
+ *    - tool_use / tool_result / thinking / redacted_thinking / image content blocks
+ *    - encrypted thinking signatures (multi-kilobyte base64 blobs)
+ *
+ *  Compaction handling:
+ *    Records with isCompactSummary:true (synthetic summary fed back into model
+ *    context by /compact) are returned with role='summary' so the importer
+ *    can route them away from FTS5 indexing — otherwise the summary would
+ *    double-count against the original raw discussion it summarises.
+ *
+ *  Returns null when the record should be skipped, otherwise
+ *  { role, text, id, timestamp, uuid, parentUuid }.
+ */
+export function extractMessageFromRecord(obj) {
+  if (!obj || typeof obj !== 'object') return null;
+
+  // Skip non-dialogue top-level event types
+  if (CLAUDE_CODE_SKIP_TYPES.has(obj.type)) return null;
+
+  // Skip attachment-only records (Claude Code harness bookkeeping)
+  if (obj.attachment && !obj.message) return null;
+
+  // Resolve role/content from either nested or flat shape
+  const nested = obj.message;
+  const fromNested = nested && typeof nested === 'object';
+  let role = fromNested ? nested.role : obj.role;
+  if (!role || typeof role !== 'string') return null;
+
+  let rawContent;
+  if (fromNested) {
+    rawContent = nested.content;
+  } else if (obj.content !== undefined) {
+    rawContent = obj.content;
+  } else {
+    rawContent = obj.text;
+  }
+
+  // Normalise content into dialogue-only text
+  let text = '';
+  if (typeof rawContent === 'string') {
+    text = rawContent;
+  } else if (Array.isArray(rawContent)) {
+    const parts = [];
+    for (const block of rawContent) {
+      if (typeof block === 'string') {
+        parts.push(block);
+        continue;
+      }
+      if (!block || typeof block !== 'object') continue;
+      // Only keep text-bearing blocks. Drop tool_use, tool_result, thinking,
+      // redacted_thinking, image, and any future unknown block types.
+      if (block.type === 'text' && typeof block.text === 'string') {
+        parts.push(block.text);
+      }
+    }
+    text = parts.join('\n');
+  }
+
+  if (!text || !text.trim()) return null;
+
+  // Claude Code marks the synthetic /compact summary message with
+  // isCompactSummary:true (and isVisibleInTranscriptOnly:true). Re-tag
+  // those as role='summary' so the importer can keep them in the messages
+  // table for retrieval but exclude them from FTS5 — otherwise searching
+  // for any topic discussed before a compaction would return both the
+  // original raw turns AND the compressed summary mention, polluting rank.
+  if (
+    role === 'user' &&
+    (obj.isCompactSummary === true || obj.isVisibleInTranscriptOnly === true)
+  ) {
+    role = 'summary';
+  }
+
+  const id = (fromNested && nested.id) || obj.id || null;
+  const timestamp =
+    obj.timestamp || (fromNested && nested.timestamp) || null;
+  const uuid = obj.uuid || null;
+  const parentUuid = obj.parentUuid || null;
+
+  return { role, text, id, timestamp, uuid, parentUuid };
+}
+
+/** Detect a compact_boundary record.
+ *
+ *  Claude Code writes two record types when /compact (or auto-compact) fires:
+ *    1. {type:"system", subtype:"compact_boundary", compactMetadata:{...}, ...}
+ *       — boundary marker. parentUuid is reset to null. compactMetadata
+ *       carries {trigger, preTokens, postTokens, durationMs,
+ *       logicalParentUuid, preCompactDiscoveredTools}.
+ *    2. {type:"user", isCompactSummary:true, message:{...}} — the
+ *       AI-generated summary fed back into model context (handled by
+ *       extractMessageFromRecord via role='summary').
+ *
+ *  We also recognise the daemon's inbox-emitted shape
+ *  {type:"compact-boundary", metadata:{...}, ...} so server.js can import
+ *  either the raw on-disk format or the daemon's snapshot.
+ *
+ *  Returns null when the record isn't a boundary, otherwise
+ *  { timestamp, uuid, parentUuid, logicalParentUuid, metadata, id }.
+ */
+export function extractCompactBoundary(obj) {
+  if (!obj || typeof obj !== 'object') return null;
+
+  let metadata, raw;
+  if (obj.type === 'system' && obj.subtype === 'compact_boundary') {
+    metadata = obj.compactMetadata || {};
+    raw = obj;
+  } else if (obj.type === 'compact-boundary') {
+    metadata = obj.metadata || {};
+    raw = obj;
+  } else {
+    return null;
+  }
+
+  return {
+    timestamp: obj.timestamp || null,
+    uuid: obj.uuid || null,
+    parentUuid: obj.parentUuid || null,
+    logicalParentUuid:
+      obj.logicalParentUuid || (metadata && metadata.logicalParentUuid) || null,
+    metadata,
+    id: obj.id || null,
+    raw,
+  };
+}
+
+/** Pull an ai-title record out of a JSONL line, if present. */
+export function extractAiTitle(obj) {
+  if (
+    obj &&
+    obj.type === 'ai-title' &&
+    typeof obj.aiTitle === 'string' &&
+    obj.aiTitle.trim()
+  ) {
+    return obj.aiTitle.trim();
+  }
+  return null;
+}

package/lib/store-doc/canonicalize.js

@@ -0,0 +1,116 @@
+/**
+ * URL canonicalization for stable deduplication of stored web documents.
+ *
+ * Goal: two URLs that point to "the same document" should map to the same
+ * canonical form, so memex_store_document gives them the same conversation_id
+ * via sha256(canonical).
+ *
+ * What we normalize:
+ *   - Lowercase scheme + host
+ *   - Strip known tracking params (utm_*, fbclid, gclid, ref, mc_*, _ga, …)
+ *   - Drop the fragment (#anchor) — same document
+ *   - Normalize trailing slash on pathname
+ *
+ * What we DON'T normalize:
+ *   - Path case (some servers are case-sensitive)
+ *   - Non-tracking query params (?q= search, ?id= permalinks — meaningful)
+ *   - Port (rare in public URLs)
+ *
+ * If the input isn't a valid URL, we return the input unchanged. Callers
+ * should still hash the result for deduplication.
+ */
+
+// Well-known tracking-param families. Case-insensitive prefix match.
+const TRACKING_PREFIXES = [
+  'utm_',         // Google Analytics
+  'mc_',          // Mailchimp
+];
+const TRACKING_EXACT = new Set([
+  'fbclid',       // Facebook
+  'gclid',        // Google ads
+  'dclid',        // Google DoubleClick
+  'gbraid',       // Google
+  'wbraid',       // Google
+  'yclid',        // Yandex
+  'msclkid',      // Microsoft ads
+  'twclid',       // Twitter
+  'igshid',       // Instagram
+  'ref',          // generic referrer
+  'ref_source',
+  'ref_url',
+  'referrer',
+  'source',       // common referrer flag (NOT always tracking but very often)
+  '_ga',          // Google Analytics
+  '_gl',          // Google Analytics linker
+  'hsCtaTracking',
+  'hsenc',
+  'hsmi',
+  'mkt_tok',
+  'pk_campaign',
+  'pk_source',
+  'pk_medium',
+  'pk_keyword',
+  'pk_content',
+  'vero_id',
+  'vero_conv',
+]);
+
+function isTrackingParam(name) {
+  const lower = name.toLowerCase();
+  if (TRACKING_EXACT.has(lower)) return true;
+  for (const prefix of TRACKING_PREFIXES) {
+    if (lower.startsWith(prefix)) return true;
+  }
+  return false;
+}
+
+/**
+ * @param {string} rawUrl
+ * @returns {string} canonicalized URL (or the input unchanged if unparseable)
+ */
+export function canonicalize(rawUrl) {
+  if (typeof rawUrl !== 'string' || !rawUrl.trim()) return rawUrl;
+
+  let u;
+  try {
+    u = new URL(rawUrl.trim());
+  } catch (_) {
+    return rawUrl.trim();
+  }
+
+  // Lowercase scheme + host (URL parser already does that, but be explicit)
+  u.protocol = u.protocol.toLowerCase();
+  u.hostname = u.hostname.toLowerCase();
+
+  // Drop the fragment
+  u.hash = '';
+
+  // Strip tracking params
+  const cleanParams = new URLSearchParams();
+  for (const [k, v] of u.searchParams) {
+    if (!isTrackingParam(k)) cleanParams.append(k, v);
+  }
+  u.search = cleanParams.toString();
+
+  // Normalize trailing slash: drop trailing slash on non-root paths,
+  // so /foo and /foo/ are treated as the same document
+  if (u.pathname.length > 1 && u.pathname.endsWith('/')) {
+    u.pathname = u.pathname.replace(/\/+$/, '');
+  }
+
+  return u.toString();
+}
+
+/**
+ * Best-effort domain extraction for metadata (e.g. "perplexity.ai").
+ * Returns null for unparseable URLs.
+ */
+export function extractDomain(rawUrl) {
+  if (typeof rawUrl !== 'string') return null;
+  try {
+    const u = new URL(rawUrl);
+    return u.hostname.toLowerCase().replace(/^www\./, '');
+  } catch (_) {
+    return null;
+  }
+}