parallelclaw 1.0.0

package/lib/web/routes/conversations.js

@@ -0,0 +1,180 @@
+/**
+ * GET /conversations — list with optional FTS5 search.
+ * GET /conversations/search — htmx partial (just the list).
+ *
+ * Query params:
+ *   q       — search query (FTS5 MATCH)
+ *   source  — filter by source ("telegram", "claude-code", etc.)
+ *   limit   — page size (default 50, max 200)
+ */
+
+import { renderPage, html, raw, esc, fmtDate, fmtNum } from '../templates.js';
+
+const MAX_LIMIT = 200;
+const DEFAULT_LIMIT = 50;
+
+const SOURCES = ['telegram', 'claude-code', 'claude-cowork', 'cursor', 'obsidian', 'document'];
+
+function clampLimit(raw) {
+  const n = parseInt(raw, 10);
+  if (!Number.isFinite(n) || n <= 0) return DEFAULT_LIMIT;
+  return Math.min(n, MAX_LIMIT);
+}
+
+/**
+ * Build the conv-list HTML for either full page or htmx partial.
+ * Branches on whether `q` is present:
+ *   - With q: FTS5 over messages_fts, group by conversation, rank by recency-boosted BM25
+ *   - Without q: list conversations directly by last_ts DESC
+ */
+function fetchConversations(db, { q, source, limit }) {
+  if (q && q.trim()) {
+    // Sanitise query for FTS5: strip control chars but keep words/quotes/spaces.
+    const safe = q.trim().replace(/[^\p{L}\p{N}\s"'._-]/gu, ' ');
+    const sourceFilter = source ? 'AND c.source = @source' : '';
+    // snippet() can't survive an outer GROUP BY — we just count hits per chat.
+    // The snippet itself is rendered inside /c/:id?q=... via the highlight() helper.
+    const stmt = db.prepare(`
+      SELECT
+        c.conversation_id,
+        c.source,
+        c.title,
+        c.last_ts,
+        c.message_count,
+        COUNT(m.id) AS hit_count,
+        NULL AS snippet
+      FROM messages_fts
+      JOIN messages m ON m.id = messages_fts.rowid
+      JOIN conversations c ON c.conversation_id = m.conversation_id
+      WHERE messages_fts MATCH @q
+        AND c.archived_at IS NULL
+        ${sourceFilter}
+      GROUP BY c.conversation_id
+      ORDER BY c.last_ts DESC
+      LIMIT @limit
+    `);
+    return stmt.all({ q: safe, source, limit });
+  }
+
+  const sourceFilter = source ? 'AND source = @source' : '';
+  return db
+    .prepare(
+      `
+      SELECT conversation_id, source, title, last_ts, message_count,
+             0 AS hit_count, NULL AS snippet
+      FROM conversations
+      WHERE archived_at IS NULL
+        AND message_count > 0
+        ${sourceFilter}
+      ORDER BY last_ts DESC
+      LIMIT @limit
+    `
+    )
+    .all({ source, limit });
+}
+
+function renderList(rows, { q }) {
+  if (rows.length === 0) {
+    return html`
+      <div class="empty">
+        <h3>${q ? 'No matches' : 'No conversations'}</h3>
+        <p>${q ? 'Try a different query or remove filters.' : 'Wait for memex-sync to capture, or import Telegram exports.'}</p>
+      </div>
+    `;
+  }
+
+  return html`
+    <div class="conv-list">
+      ${rows.map(
+        (c) => html`
+          <a class="conv-item" href="/c/${encodeURIComponent(c.conversation_id)}${q ? '?q=' + encodeURIComponent(q) : ''}">
+            <div class="conv-item-top">
+              <span class="conv-title">${c.title || '(untitled)'}</span>
+              <span class="conv-count">
+                ${c.hit_count > 0 ? `${fmtNum(c.hit_count)} hits · ` : ''}${fmtNum(c.message_count)} msgs
+              </span>
+            </div>
+            <div class="conv-meta">
+              <span class="conv-source-tag">${c.source}</span>
+              ${fmtDate(c.last_ts)}
+              ${c.snippet ? raw(` · <span class="search-meta">${c.snippet}</span>`) : null}
+            </div>
+          </a>
+        `
+      )}
+    </div>
+  `;
+}
+
+export async function renderConversations(db, query, status) {
+  const q = query.q || '';
+  const source = query.source && SOURCES.includes(query.source) ? query.source : '';
+  const limit = clampLimit(query.limit);
+
+  let rows;
+  let error = null;
+  try {
+    rows = fetchConversations(db, { q, source, limit });
+  } catch (e) {
+    rows = [];
+    error = e.message;
+  }
+
+  const sourceChips = html`
+    <div class="search-bar" style="gap:6px;">
+      <a class="btn ${source ? '' : 'btn-primary'}" href="/conversations${q ? '?q=' + encodeURIComponent(q) : ''}">all</a>
+      ${SOURCES.map(
+        (s) => html`
+          <a class="btn ${source === s ? 'btn-primary' : ''}" href="/conversations?source=${s}${q ? '&q=' + encodeURIComponent(q) : ''}">${s}</a>
+        `
+      )}
+    </div>
+  `;
+
+  const searchBar = html`
+    <form class="search-bar" hx-get="/conversations/search" hx-target="#conv-list-target" hx-trigger="input changed delay:200ms, search" hx-include="[name=source]">
+      <input
+        class="search-input"
+        type="search"
+        name="q"
+        placeholder="🔍  Search conversations (FTS5)…"
+        value="${esc(q)}"
+        autocomplete="off"
+      />
+      <input type="hidden" name="source" value="${esc(source)}" />
+      <span class="search-meta">${fmtNum(rows.length)} result${rows.length === 1 ? '' : 's'}${source ? ' · ' + source : ''}</span>
+    </form>
+  `;
+
+  const body = html`
+    ${searchBar}
+    ${sourceChips}
+    ${error
+      ? html`<div class="callout" style="border-left-color:var(--red-soft);"><strong>Search error:</strong> ${esc(error)}</div>`
+      : null}
+    <div id="conv-list-target">${renderList(rows, { q })}</div>
+  `;
+
+  return renderPage({
+    title: 'Conversations',
+    active: 'conversations',
+    body,
+    status,
+  });
+}
+
+export async function renderConversationsPartial(db, query) {
+  const q = query.q || '';
+  const source = query.source && SOURCES.includes(query.source) ? query.source : '';
+  const limit = clampLimit(query.limit);
+
+  let rows;
+  try {
+    rows = fetchConversations(db, { q, source, limit });
+  } catch (e) {
+    return `<div class="callout" style="border-left-color:var(--red-soft);"><strong>Search error:</strong> ${esc(e.message)}</div>`;
+  }
+
+  const out = renderList(rows, { q });
+  return out && typeof out === 'object' && out.value ? out.value : String(out);
+}

package/lib/web/routes/dashboard.js

@@ -0,0 +1,175 @@
+/**
+ * GET / — dashboard overview.
+ *
+ * Read-only snapshot of the memex corpus: stats, sources breakdown,
+ * pending Telegram exports, and recent conversations.
+ */
+
+import { renderPage, html, raw, esc, fmtDate, fmtNum } from '../templates.js';
+
+const SOURCE_ICONS = {
+  telegram: '📱',
+  'claude-code': '💬',
+  'claude-cowork': '🤝',
+  cursor: '✏️',
+  obsidian: '📓',
+  document: '📄',
+};
+
+export async function renderDashboard(db, status) {
+  // ----- Core stats -----
+  const totalMessages = db
+    .prepare("SELECT COUNT(*) AS n FROM messages WHERE role NOT IN ('boundary', 'summary')")
+    .get().n;
+  const totalConversations = db
+    .prepare('SELECT COUNT(*) AS n FROM conversations WHERE archived_at IS NULL')
+    .get().n;
+  const totalImports = db.prepare('SELECT COUNT(*) AS n FROM imports').get().n;
+  const dateRange = db
+    .prepare(
+      "SELECT MIN(ts) AS first, MAX(ts) AS last FROM messages WHERE ts IS NOT NULL AND ts > 0 AND role NOT IN ('boundary', 'summary')"
+    )
+    .get();
+
+  // ----- Sources breakdown -----
+  const sources = db
+    .prepare(
+      `
+      SELECT
+        source,
+        COUNT(*) AS msg_count,
+        COUNT(DISTINCT conversation_id) AS conv_count,
+        MIN(ts) AS first_ts,
+        MAX(ts) AS last_ts
+      FROM messages
+      WHERE role NOT IN ('boundary', 'summary')
+      GROUP BY source
+      ORDER BY msg_count DESC
+    `
+    )
+    .all();
+
+  // ----- Pending Telegram exports -----
+  let pendingList = [];
+  try {
+    const { listPending } = await import('../../telegram-pending.js');
+    pendingList = listPending();
+  } catch (_) {
+    /* pending module may be unavailable in stripped builds */
+  }
+
+  // ----- Recent conversations -----
+  const recent = db
+    .prepare(
+      `
+      SELECT conversation_id, source, title, last_ts, message_count
+      FROM conversations
+      WHERE archived_at IS NULL
+        AND message_count > 0
+      ORDER BY last_ts DESC
+      LIMIT 10
+    `
+    )
+    .all();
+
+  // ----- Build HTML -----
+  const statGrid = html`
+    <div class="stat-grid">
+      <div class="stat">
+        <div class="stat-value">${fmtNum(totalMessages)}</div>
+        <div class="stat-label">messages</div>
+      </div>
+      <div class="stat">
+        <div class="stat-value">${fmtNum(totalConversations)}</div>
+        <div class="stat-label">conversations</div>
+      </div>
+      <div class="stat">
+        <div class="stat-value">${fmtNum(sources.length)}</div>
+        <div class="stat-label">sources</div>
+      </div>
+      <div class="stat">
+        <div class="stat-value">${fmtNum(totalImports)}</div>
+        <div class="stat-label">imports</div>
+      </div>
+    </div>
+  `;
+
+  const sourcesCard = html`
+    <section class="card">
+      <div class="card-label">sources</div>
+      <ul class="sources-list">
+        ${sources.map(
+          (s) => html`
+            <li>
+              <span class="src-name">${raw(SOURCE_ICONS[s.source] || '•')} ${s.source}</span>
+              <span class="src-meta">${fmtNum(s.msg_count)} msgs · ${fmtNum(s.conv_count)} chats</span>
+              <span class="src-spacer"></span>
+              <span class="src-meta">${fmtDate(s.first_ts)} → ${fmtDate(s.last_ts)}</span>
+            </li>
+          `
+        )}
+      </ul>
+    </section>
+  `;
+
+  const pendingCallout = pendingList.length
+    ? html`
+        <div class="callout">
+          <strong>📬 ${pendingList.length} Telegram export${pendingList.length === 1 ? '' : 's'} awaiting review</strong>
+          —
+          ${pendingList
+            .slice(0, 3)
+            .map((p) => esc(p.chat_title || p.basename || '?'))
+            .join(', ')}${pendingList.length > 3 ? `, + ${pendingList.length - 3} more` : ''}
+          · <a href="/pending">Review →</a>
+        </div>
+      `
+    : null;
+
+  const recentCard = html`
+    <section class="card">
+      <div class="card-label">recent conversations</div>
+      <div class="conv-list">
+        ${recent.map(
+          (c) => html`
+            <a class="conv-item" href="/c/${encodeURIComponent(c.conversation_id)}">
+              <div class="conv-item-top">
+                <span class="conv-title">${c.title || '(untitled)'}</span>
+                <span class="conv-count">${fmtNum(c.message_count)} msgs</span>
+              </div>
+              <div class="conv-meta">
+                <span class="conv-source-tag">${c.source}</span>
+                ${fmtDate(c.last_ts)}
+              </div>
+            </a>
+          `
+        )}
+        ${recent.length === 0
+          ? html`<div class="empty"><h3>No conversations yet</h3><p>Wait for memex-sync to capture, or import Telegram exports.</p></div>`
+          : null}
+      </div>
+      ${recent.length > 0
+        ? html`<p style="margin-top:14px;text-align:right;"><a href="/conversations">All conversations →</a></p>`
+        : null}
+    </section>
+  `;
+
+  const corpusSpan = dateRange.first
+    ? html`<p class="conv-meta" style="margin-top:6px;">Corpus span: ${fmtDate(dateRange.first)} → ${fmtDate(dateRange.last)}</p>`
+    : null;
+
+  const body = html`
+    ${statGrid}
+    ${corpusSpan}
+    ${pendingCallout}
+    ${sourcesCard}
+    ${recentCard}
+  `;
+
+  return renderPage({
+    title: 'Dashboard',
+    active: 'dashboard',
+    body,
+    status,
+  });
+}

package/lib/web/routes/pending.js

@@ -0,0 +1,277 @@
+/**
+ * GET /pending — Telegram exports awaiting decision.
+ * POST /pending/import — import selected entries (form data, name="index").
+ * POST /pending/skip   — skip selected entries (same form).
+ *
+ * Reuses lib/telegram-pending.js (listPending/removePending) and
+ * lib/telegram-decisions.js (allowChat/skipChat/loadDecisions/saveDecisions).
+ * Imports require a writable DB handle, opened locally and closed on completion.
+ */
+
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+import Database from 'better-sqlite3';
+
+import { renderPage, html, raw, esc, fmtDate, fmtNum, fmtBytes } from '../templates.js';
+
+const HOME = homedir();
+const MEMEX_DIR = process.env.MEMEX_DIR || join(HOME, '.memex');
+const DB_PATH = join(MEMEX_DIR, 'data', 'memex.db');
+
+function openWritableDb() {
+  return new Database(DB_PATH, { fileMustExist: true });
+}
+
+function renderPendingList(pendingList) {
+  if (pendingList.length === 0) {
+    return html`
+      <div class="empty">
+        <h3>Inbox empty</h3>
+        <p>Export a chat from Telegram Desktop (Settings → Advanced → Export) — memex-sync will stage it here for review.</p>
+      </div>
+    `;
+  }
+
+  return html`
+    <form id="pending-form" method="post" hx-target="#pending-target" hx-swap="innerHTML">
+      <div class="pending-list">
+        ${pendingList.map(
+          (p) => html`
+            <div class="pending-item">
+              <div class="pending-item-top">
+                <label style="display:flex;align-items:center;gap:10px;cursor:pointer;flex:1;">
+                  <input type="checkbox" name="index" value="${p.index}" />
+                  <span class="pending-title">📱 ${p.chat_title || p.basename || '(unknown)'}</span>
+                </label>
+                <span class="pending-count">${fmtNum(p.message_count || 0)} msgs</span>
+              </div>
+              <div class="pending-meta">
+                ${p.date_first ? esc(p.date_first.slice(0, 10)) : '?'} → ${p.date_last ? esc(p.date_last.slice(0, 10)) : '?'}
+                · <code>${esc(p.basename)}</code>
+                ${p.size_bytes ? ' · ' + fmtBytes(p.size_bytes) : ''}
+                ${p.senders_sample && p.senders_sample.length
+                  ? raw(' · senders: ' + esc(p.senders_sample.slice(0, 3).join(', ')) + (p.senders_sample.length > 3 ? ', …' : ''))
+                  : null}
+              </div>
+            </div>
+          `
+        )}
+      </div>
+      <div class="pending-actions" style="margin-top:16px;">
+        <button type="submit" class="btn btn-primary" hx-post="/pending/import">Import selected</button>
+        <button type="submit" class="btn btn-danger" hx-post="/pending/skip">Skip selected</button>
+        <span class="search-meta" style="margin-left:auto;align-self:center;">
+          ${fmtNum(pendingList.length)} pending
+        </span>
+      </div>
+    </form>
+  `;
+}
+
+function renderDecisionsSection() {
+  let state;
+  try {
+    // Lazy import — avoids loading at server boot when decisions don't exist yet
+    // (re-require'd via dynamic import below since decisions is ESM).
+    return null;
+  } catch (_) {
+    return null;
+  }
+}
+
+async function loadDecisionsState() {
+  try {
+    const { loadDecisions } = await import('../../telegram-decisions.js');
+    return loadDecisions();
+  } catch (_) {
+    return null;
+  }
+}
+
+export async function renderPending(status) {
+  const { listPending } = await import('../../telegram-pending.js');
+  const pendingList = listPending();
+  const decisions = await loadDecisionsState();
+
+  const allowed = decisions?.allowed ? Object.keys(decisions.allowed) : [];
+  const skipped = decisions?.skipped ? Object.keys(decisions.skipped) : [];
+  const blocked = decisions?.blocked ? Object.keys(decisions.blocked) : [];
+
+  const decisionsCard =
+    allowed.length || skipped.length || blocked.length
+      ? html`
+          <section class="card" style="margin-top:24px;">
+            <div class="card-label">your decisions (history)</div>
+            <ul class="sources-list">
+              <li>
+                <span class="src-name">✅ Allowed</span>
+                <span class="src-meta">${fmtNum(allowed.length)}${allowed.length ? ': ' + esc(allowed.slice(0, 6).join(', ')) + (allowed.length > 6 ? ', …' : '') : ''}</span>
+              </li>
+              <li>
+                <span class="src-name">⏭️ Skipped</span>
+                <span class="src-meta">${fmtNum(skipped.length)}${skipped.length ? ': ' + esc(skipped.slice(0, 6).join(', ')) + (skipped.length > 6 ? ', …' : '') : ''}</span>
+              </li>
+              <li>
+                <span class="src-name">🚫 Blocked patterns</span>
+                <span class="src-meta">${fmtNum(blocked.length)}${blocked.length ? ': ' + esc(blocked.slice(0, 6).join(', ')) + (blocked.length > 6 ? ', …' : '') : ''}</span>
+              </li>
+            </ul>
+          </section>
+        `
+      : null;
+
+  const body = html`
+    <div class="callout">
+      <strong>Privacy-first.</strong> Telegram exports stay on disk until you import them.
+      Importing is a one-way decision — once in memex.db they can be searched and removed via
+      <code>memex telegram remove "&lt;title&gt;"</code>.
+    </div>
+    <div id="pending-target">${renderPendingList(pendingList)}</div>
+    ${decisionsCard}
+  `;
+
+  return renderPage({
+    title: 'Pending',
+    active: 'pending',
+    body,
+    status,
+  });
+}
+
+// ----- POST handlers -----
+
+function parseIndices(body) {
+  const raw = body.index;
+  if (raw == null) return [];
+  const arr = Array.isArray(raw) ? raw : [raw];
+  return arr
+    .map((v) => parseInt(v, 10))
+    .filter((n) => Number.isFinite(n) && n > 0);
+}
+
+function resolveTargets(pendingList, indices) {
+  const byIdx = new Map(pendingList.map((p) => [p.index, p]));
+  const out = [];
+  for (const i of indices) {
+    const p = byIdx.get(i);
+    if (p) out.push(p);
+  }
+  return out;
+}
+
+export async function handleImport(body) {
+  const { listPending, removePending } = await import('../../telegram-pending.js');
+  const decisionsMod = await import('../../telegram-decisions.js');
+  const pendingList = listPending();
+  const indices = parseIndices(body);
+
+  if (indices.length === 0) {
+    return wrapFragment(html`
+      <div class="callout" style="border-left-color:var(--amber);">
+        <strong>Select at least one entry</strong> — tick the checkboxes you want to import.
+      </div>
+      ${renderPendingList(pendingList)}
+    `);
+  }
+
+  const targets = resolveTargets(pendingList, indices);
+  if (targets.length === 0) {
+    return wrapFragment(html`
+      <div class="callout" style="border-left-color:var(--red-soft);">
+        <strong>Stale selection</strong> — the entries you selected are no longer in pending.
+      </div>
+      ${renderPendingList(pendingList)}
+    `);
+  }
+
+  const { importTelegramRaw } = await import('../../import-telegram.js');
+  const { parseTelegramHtmlExport } = await import('../../parse-telegram-html.js');
+  const state = decisionsMod.loadDecisions();
+
+  const results = [];
+  const db = openWritableDb();
+  try {
+    for (const t of targets) {
+      try {
+        let raw;
+        if (t.kind === 'html-dir') {
+          raw = parseTelegramHtmlExport(t.path);
+        } else if (t.kind === 'json-file') {
+          raw = JSON.parse(readFileSync(t.path, 'utf-8'));
+        } else if (t.kind === 'json-in-dir' && t.inner_json_path) {
+          raw = JSON.parse(readFileSync(t.inner_json_path, 'utf-8'));
+        } else {
+          results.push({ title: t.chat_title, error: `unknown kind: ${t.kind}` });
+          continue;
+        }
+        if (!raw) {
+          results.push({ title: t.chat_title, error: 'parse-failed' });
+          continue;
+        }
+        const r = importTelegramRaw(db, raw);
+        const title = raw.chats?.list?.[0]?.name || t.chat_title || 'Telegram chat';
+        decisionsMod.allowChat(state, title);
+        removePending(t.path);
+        results.push({ title, imported: r.totalImported });
+      } catch (e) {
+        results.push({ title: t.chat_title, error: e.message });
+      }
+    }
+    decisionsMod.saveDecisions(state);
+  } finally {
+    db.close();
+  }
+
+  const updated = listPending();
+  const okCount = results.filter((r) => !r.error).length;
+  const errCount = results.filter((r) => r.error).length;
+  const totalMsgs = results.reduce((s, r) => s + (r.imported || 0), 0);
+
+  return wrapFragment(html`
+    <div class="callout">
+      <strong>✓ Imported ${fmtNum(okCount)} chat${okCount === 1 ? '' : 's'}</strong>
+      — ${fmtNum(totalMsgs)} message${totalMsgs === 1 ? '' : 's'} now in memex.db.
+      ${errCount > 0 ? raw(`<br/><span style="color:var(--red-soft);">⚠ ${errCount} failed: ${results.filter(r => r.error).map(r => esc(r.title || '?') + ' (' + esc(r.error) + ')').join(', ')}</span>`) : null}
+    </div>
+    ${renderPendingList(updated)}
+  `);
+}
+
+export async function handleSkip(body) {
+  const { listPending, removePending } = await import('../../telegram-pending.js');
+  const decisionsMod = await import('../../telegram-decisions.js');
+  const pendingList = listPending();
+  const indices = parseIndices(body);
+
+  if (indices.length === 0) {
+    return wrapFragment(html`
+      <div class="callout" style="border-left-color:var(--amber);">
+        <strong>Select at least one entry</strong> — tick the checkboxes you want to skip.
+      </div>
+      ${renderPendingList(pendingList)}
+    `);
+  }
+
+  const targets = resolveTargets(pendingList, indices);
+  const state = decisionsMod.loadDecisions();
+  for (const t of targets) {
+    if (t.chat_title) decisionsMod.skipChat(state, t.chat_title);
+    removePending(t.path);
+  }
+  decisionsMod.saveDecisions(state);
+
+  const updated = listPending();
+  return wrapFragment(html`
+    <div class="callout">
+      <strong>⏭️ Skipped ${fmtNum(targets.length)} entr${targets.length === 1 ? 'y' : 'ies'}.</strong>
+      Future re-exports of these chats will be auto-skipped.
+    </div>
+    ${renderPendingList(updated)}
+  `);
+}
+
+// htmx hx-target="#pending-target" → we return the inner HTML for that div.
+function wrapFragment(node) {
+  return node && typeof node === 'object' && node.value ? node.value : String(node);
+}