parallelclaw 1.0.0

package/lib/telegram-pending.js

@@ -0,0 +1,200 @@
+/**
+ * Manage ~/.memex/pending/ — the staging area where the daemon parks
+ * Telegram exports it found in ~/Downloads/Telegram Desktop/ before the
+ * user explicitly approves them.
+ *
+ * Layout:
+ *   ~/.memex/pending/
+ *     ChatExport_2026-05-15/                   ← exact original name
+ *       messages.html
+ *       photos/ ...
+ *     result-2026-05-12.json
+ *     .meta.json                                ← per-entry metadata cache (preview)
+ *
+ * The .meta.json is recomputed lazily — we cache preview output so listing
+ * 50 pending exports doesn't re-parse 50 HTML trees on every CLI invocation.
+ */
+
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  renameSync,
+  statSync,
+  rmSync,
+  cpSync,
+  writeFileSync,
+  readFileSync,
+} from 'node:fs';
+import { join, basename } from 'node:path';
+import { homedir } from 'node:os';
+import { previewExport } from './telegram-discovery.js';
+
+// Compute lazily so process.env.HOME overrides (used by tests) actually work.
+export function getPendingDir() {
+  return join(homedir(), '.memex', 'pending');
+}
+export function getMetaFile() {
+  return join(getPendingDir(), '.meta.json');
+}
+// Back-compat exports — still used by some callers
+export const PENDING_DIR = getPendingDir();
+export const META_FILE = getMetaFile();
+
+export function ensurePendingDir() {
+  const dir = getPendingDir();
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+}
+
+/**
+ * Move a freshly-detected export from its source location into pending/.
+ * If destination exists (re-export of same name), uses a numeric suffix.
+ *
+ * Returns the absolute destination path that now holds the export.
+ *
+ * `moveOrCopy` — 'move' uses fs.rename (atomic, same FS only); 'copy' uses
+ * cpSync. Default 'move'. If the rename fails (cross-device), automatically
+ * falls back to copy + rmSync.
+ */
+export function stageExport(sourcePath, opts = {}) {
+  ensurePendingDir();
+  const moveOrCopy = opts.moveOrCopy || 'move';
+  const name = basename(sourcePath);
+  const baseDir = getPendingDir();
+  let dest = join(baseDir, name);
+  // Suffix preserves extension so file-type sniffing (e.g. `endsWith('.json')`)
+  // still works on the staged copy: result.json → result__1.json
+  let suffix = 0;
+  while (existsSync(dest)) {
+    suffix += 1;
+    const dot = name.lastIndexOf('.');
+    if (dot > 0 && dot < name.length - 1) {
+      const stem = name.slice(0, dot);
+      const ext = name.slice(dot);
+      dest = join(baseDir, `${stem}__${suffix}${ext}`);
+    } else {
+      dest = join(baseDir, `${name}__${suffix}`);
+    }
+  }
+
+  try {
+    if (moveOrCopy === 'move') {
+      renameSync(sourcePath, dest);
+    } else {
+      cpSync(sourcePath, dest, { recursive: true });
+    }
+  } catch (e) {
+    // EXDEV — cross-device link. Fall back to copy + delete.
+    if (e.code === 'EXDEV') {
+      cpSync(sourcePath, dest, { recursive: true });
+      try { rmSync(sourcePath, { recursive: true, force: true }); } catch (_) { /* ok */ }
+    } else {
+      throw e;
+    }
+  }
+
+  // Invalidate cache — next listPending() will recompute
+  invalidateMeta();
+  return dest;
+}
+
+/**
+ * List everything in pending/, with cached previews.
+ *
+ * Returns an array of objects shaped like:
+ *   {
+ *     index: 1,
+ *     path: '/Users/.../pending/ChatExport_2026-05-15',
+ *     basename: 'ChatExport_2026-05-15',
+ *     kind: 'html-dir'|'json-file',
+ *     chat_title: '...',
+ *     chat_type: 'private_group'|'personal_chat',
+ *     message_count: 492,
+ *     date_first: '2026-03-20T14:08:43',
+ *     date_last:  '2026-05-12T00:40:08',
+ *     senders_sample: ['Oleg', 'Andrey', ...],
+ *     size_bytes: 468468,
+ *     modified_ts: 1773820800
+ *   }
+ *
+ * Indices are 1-based and stable within one process — used by CLI commands
+ * like `memex telegram import 1 3 5`. Order: newest modified first.
+ */
+export function listPending(opts = {}) {
+  const baseDir = getPendingDir();
+  if (!existsSync(baseDir)) return [];
+
+  const cache = loadMeta();
+  const entries = [];
+
+  let names;
+  try { names = readdirSync(baseDir); } catch (_) { return []; }
+  for (const name of names) {
+    if (name.startsWith('.')) continue; // skip .meta.json and other hidden
+    const full = join(baseDir, name);
+    let st;
+    try { st = statSync(full); } catch (_) { continue; }
+    const mtime = Math.floor(st.mtimeMs / 1000);
+
+    let preview = cache[name];
+    if (!preview || preview.modified_ts !== mtime) {
+      preview = previewExport(full);
+      preview.modified_ts = mtime;
+      cache[name] = preview;
+    }
+    entries.push({
+      basename: name,
+      path: full,
+      modified_ts: mtime,
+      ...preview,
+    });
+  }
+
+  saveMeta(cache);
+
+  entries.sort((a, b) => b.modified_ts - a.modified_ts);
+  entries.forEach((e, i) => { e.index = i + 1; });
+  return entries;
+}
+
+/**
+ * Remove an entry from pending/ — used after successful import or after
+ * `memex telegram skip`. Deletes from disk and from the meta cache.
+ */
+export function removePending(absPath) {
+  try { rmSync(absPath, { recursive: true, force: true }); } catch (_) { /* ok */ }
+  const cache = loadMeta();
+  const name = basename(absPath);
+  if (name in cache) {
+    delete cache[name];
+    saveMeta(cache);
+  }
+}
+
+// -------------------- meta cache helpers --------------------
+
+function loadMeta() {
+  const meta = getMetaFile();
+  if (!existsSync(meta)) return {};
+  try { return JSON.parse(readFileSync(meta, 'utf-8')); }
+  catch (_) { return {}; }
+}
+
+function saveMeta(cache) {
+  ensurePendingDir();
+  const meta = getMetaFile();
+  try {
+    const tmp = meta + '.tmp';
+    writeFileSync(tmp, JSON.stringify(cache, null, 2));
+    renameSync(tmp, meta);
+  } catch (_) { /* non-fatal */ }
+}
+
+function invalidateMeta() {
+  const meta = getMetaFile();
+  if (existsSync(meta)) {
+    try { rmSync(meta, { force: true }); } catch (_) { /* ok */ }
+  }
+}

package/lib/web/index.js

@@ -0,0 +1,265 @@
+/**
+ * memex web dashboard — HTTP server.
+ *
+ * Opt-in: invoked via `memex web` CLI command. Binds 127.0.0.1 by default.
+ * Read-only by design (only POST endpoints are pending review actions).
+ *
+ * Stack:
+ *   • Node.js raw http module (no Express, no framework)
+ *   • Tagged template literals for HTML
+ *   • htmx for client-side reactivity (from CDN)
+ *   • Better-sqlite3 read-only DB handle
+ *
+ * Routes:
+ *   GET  /                         → dashboard
+ *   GET  /conversations            → list + search
+ *   GET  /conversations/search     → htmx partial
+ *   GET  /c/:id                    → full transcript
+ *   GET  /pending                  → telegram exports awaiting decision
+ *   POST /pending/import           → import selected
+ *   POST /pending/skip             → skip selected
+ *   GET  /settings                 → daemon status, sources, hooks
+ *   GET  /static/<file>            → static assets (CSS, etc)
+ *   GET  /api/health               → JSON liveness probe
+ */
+
+import { createServer } from 'node:http';
+import { readFileSync, existsSync, statSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { homedir } from 'node:os';
+import { spawn, execSync } from 'node:child_process';
+import Database from 'better-sqlite3';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const STATIC_DIR = join(__dirname, 'static');
+
+const HOME = homedir();
+const MEMEX_DIR = process.env.MEMEX_DIR || join(HOME, '.memex');
+const DB_PATH = join(MEMEX_DIR, 'data', 'memex.db');
+
+// ----- Helpers -----
+
+function send(res, status, body, contentType = 'text/html; charset=utf-8') {
+  res.statusCode = status;
+  res.setHeader('Content-Type', contentType);
+  res.setHeader('X-Content-Type-Options', 'nosniff');
+  res.end(body);
+}
+
+function sendJson(res, status, obj) {
+  send(res, status, JSON.stringify(obj, null, 2), 'application/json; charset=utf-8');
+}
+
+function notFound(res) {
+  send(res, 404, '<h1>404 — not in memex</h1><p><a href="/">back to dashboard</a></p>');
+}
+
+function serverError(res, e) {
+  console.error('[memex web] error:', e);
+  send(res, 500, `<h1>500 — server error</h1><pre>${escapeHtml(e.message || String(e))}</pre>`);
+}
+
+function escapeHtml(s) {
+  return String(s || '').replace(/[&<>"']/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c]));
+}
+
+// Parse the URL into { pathname, query }
+function parseUrl(url) {
+  const i = url.indexOf('?');
+  if (i === -1) return { pathname: url, query: {} };
+  const pathname = url.slice(0, i);
+  const queryStr = url.slice(i + 1);
+  const query = {};
+  for (const pair of queryStr.split('&')) {
+    if (!pair) continue;
+    const [k, v] = pair.split('=');
+    query[decodeURIComponent(k)] = v != null ? decodeURIComponent(v.replace(/\+/g, ' ')) : '';
+  }
+  return { pathname, query };
+}
+
+// ----- Static file serving (CSS, etc) -----
+
+const STATIC_TYPES = {
+  '.css': 'text/css; charset=utf-8',
+  '.js': 'application/javascript; charset=utf-8',
+  '.svg': 'image/svg+xml',
+  '.png': 'image/png',
+  '.ico': 'image/x-icon',
+};
+
+function serveStatic(res, pathname) {
+  const rel = pathname.replace(/^\/static\//, '').replace(/\.\./g, '');
+  const full = join(STATIC_DIR, rel);
+  if (!existsSync(full)) return notFound(res);
+  const ext = full.slice(full.lastIndexOf('.'));
+  const type = STATIC_TYPES[ext] || 'application/octet-stream';
+  const body = readFileSync(full);
+  res.statusCode = 200;
+  res.setHeader('Content-Type', type);
+  res.setHeader('Cache-Control', 'public, max-age=300');
+  res.end(body);
+}
+
+// ----- DB handle (read-only) -----
+
+let _db = null;
+function getDb() {
+  if (_db) return _db;
+  if (!existsSync(DB_PATH)) {
+    throw new Error(`memex.db not found at ${DB_PATH}. Run 'memex-sync install' first.`);
+  }
+  _db = new Database(DB_PATH, { readonly: true, fileMustExist: true });
+  return _db;
+}
+
+// ----- Sync status (daemon health) -----
+
+function getDaemonStatus() {
+  try {
+    // Inline check — read the LaunchAgent plist + try to ps the daemon
+    const plistPath = join(HOME, 'Library/LaunchAgents/com.parallelclaw.memex.sync.plist');
+    const installed = existsSync(plistPath);
+    if (!installed) return { installed: false, running: false, lastCaptureMs: null };
+
+    // Recent activity from ingest.log
+    const logPath = join(MEMEX_DIR, 'data', 'ingest.log');
+    let lastCaptureMs = null;
+    if (existsSync(logPath)) {
+      const ageMs = Date.now() - statSync(logPath).mtimeMs;
+      lastCaptureMs = ageMs;
+    }
+
+    // Process check via launchctl (best effort)
+    let running = false;
+    try {
+      const out = execSync('launchctl list | grep com.parallelclaw.memex.sync', { encoding: 'utf-8', timeout: 1000 });
+      running = !out.match(/^\s*-\s/m); // "-" means not running
+    } catch (_) { /* not running */ }
+
+    return { installed: true, running, lastCaptureMs };
+  } catch (_) {
+    return { installed: false, running: false, lastCaptureMs: null };
+  }
+}
+
+// ----- Read-only auth (optional bearer token) -----
+
+function checkAuth(req, expectedToken) {
+  if (!expectedToken) return true; // no auth required
+  const auth = req.headers.authorization || '';
+  return auth === `Bearer ${expectedToken}`;
+}
+
+// ----- Request handler -----
+
+async function handleRequest(req, res, opts) {
+  const { pathname, query } = parseUrl(req.url);
+
+  // Static files (no auth required)
+  if (pathname.startsWith('/static/')) {
+    return serveStatic(res, pathname);
+  }
+
+  // Health check (no auth required, used by tests)
+  if (pathname === '/api/health') {
+    return sendJson(res, 200, { ok: true, db: existsSync(DB_PATH) });
+  }
+
+  // Auth check
+  if (!checkAuth(req, opts.token)) {
+    return send(res, 401, '<h1>401 — auth required</h1><p>Pass <code>--token</code> when starting <code>memex web</code> and include <code>Authorization: Bearer &lt;token&gt;</code></p>');
+  }
+
+  try {
+    // Route dispatch
+    if (pathname === '/') {
+      const { renderDashboard } = await import('./routes/dashboard.js');
+      return send(res, 200, await renderDashboard(getDb(), getDaemonStatus()));
+    }
+    if (pathname === '/conversations') {
+      const { renderConversations } = await import('./routes/conversations.js');
+      return send(res, 200, await renderConversations(getDb(), query, getDaemonStatus()));
+    }
+    if (pathname === '/conversations/search') {
+      const { renderConversationsPartial } = await import('./routes/conversations.js');
+      return send(res, 200, await renderConversationsPartial(getDb(), query));
+    }
+    if (pathname.startsWith('/c/')) {
+      const { renderConversation } = await import('./routes/conversation.js');
+      const id = decodeURIComponent(pathname.slice(3));
+      return send(res, 200, await renderConversation(getDb(), id, query, getDaemonStatus()));
+    }
+    if (pathname === '/pending') {
+      const { renderPending } = await import('./routes/pending.js');
+      return send(res, 200, await renderPending(getDaemonStatus()));
+    }
+    if (pathname === '/pending/import' && req.method === 'POST') {
+      const { handleImport } = await import('./routes/pending.js');
+      const body = await readBody(req);
+      return send(res, 200, await handleImport(body));
+    }
+    if (pathname === '/pending/skip' && req.method === 'POST') {
+      const { handleSkip } = await import('./routes/pending.js');
+      const body = await readBody(req);
+      return send(res, 200, await handleSkip(body));
+    }
+    if (pathname === '/settings') {
+      const { renderSettings } = await import('./routes/settings.js');
+      return send(res, 200, await renderSettings(getDb(), getDaemonStatus()));
+    }
+    return notFound(res);
+  } catch (e) {
+    return serverError(res, e);
+  }
+}
+
+function readBody(req) {
+  return new Promise((resolve, reject) => {
+    const chunks = [];
+    req.on('data', (c) => chunks.push(c));
+    req.on('end', () => {
+      const raw = Buffer.concat(chunks).toString('utf-8');
+      // Parse form-encoded body
+      const params = {};
+      for (const pair of raw.split('&')) {
+        if (!pair) continue;
+        const [k, v] = pair.split('=');
+        const key = decodeURIComponent(k);
+        const val = v != null ? decodeURIComponent(v.replace(/\+/g, ' ')) : '';
+        // Handle repeated keys (e.g., index=1&index=3 → array)
+        if (key in params) {
+          params[key] = Array.isArray(params[key]) ? [...params[key], val] : [params[key], val];
+        } else {
+          params[key] = val;
+        }
+      }
+      resolve(params);
+    });
+    req.on('error', reject);
+  });
+}
+
+// ----- Public entry point -----
+
+export function startServer({ port = 8765, host = '127.0.0.1', token = null, open = false } = {}) {
+  const server = createServer((req, res) => {
+    handleRequest(req, res, { token }).catch((e) => serverError(res, e));
+  });
+
+  return new Promise((resolve, reject) => {
+    server.on('error', reject);
+    server.listen(port, host, () => {
+      const url = `http://${host === '0.0.0.0' ? 'localhost' : host}:${port}`;
+      console.log(`memex web listening on ${url}`);
+      if (token) console.log(`  auth: bearer token required (Authorization: Bearer ${token})`);
+      if (open) {
+        // Best-effort open browser on macOS / Linux
+        const cmd = process.platform === 'darwin' ? 'open' : 'xdg-open';
+        try { spawn(cmd, [url], { stdio: 'ignore', detached: true }).unref(); } catch (_) {}
+      }
+      resolve({ server, url });
+    });
+  });
+}

package/lib/web/routes/conversation.js

@@ -0,0 +1,193 @@
+/**
+ * GET /c/:id — verbatim transcript of one conversation.
+ *
+ * This is the page that demonstrates memex's verbatim moat: every message
+ * the user and the AI exchanged, in chat-bubble form, never paraphrased.
+ *
+ * Query params:
+ *   q       — optional search term to highlight inside the transcript
+ *   offset  — pagination offset (default 0)
+ *   limit   — page size (default 200, max 1000)
+ */
+
+import { renderPage, html, raw, esc, fmtDate, fmtDateTime, fmtNum } from '../templates.js';
+
+const MAX_LIMIT = 1000;
+const DEFAULT_LIMIT = 200;
+
+function clampLimit(raw) {
+  const n = parseInt(raw, 10);
+  if (!Number.isFinite(n) || n <= 0) return DEFAULT_LIMIT;
+  return Math.min(n, MAX_LIMIT);
+}
+
+/**
+ * Determine which side of the transcript the message should appear on.
+ *
+ * Conventions:
+ *   - role='user' or sender is the user (claude-code/cowork) → right side ("you")
+ *   - role='assistant' or 'model' → left side ("ai")
+ *   - Telegram: sender matches "self_indicator" → right; everyone else → left
+ */
+function bubbleSide(msg) {
+  if (msg.role === 'user') return 'right';
+  if (msg.role === 'assistant' || msg.role === 'model') return 'left';
+  // Telegram heuristic: first sender alphabetically goes "left", rest on alternating sides.
+  // Without a notion of "me", we just put non-user roles on the left.
+  return 'left';
+}
+
+function dayKey(ts) {
+  if (!ts) return null;
+  return new Date(ts * 1000).toISOString().slice(0, 10);
+}
+
+/**
+ * Highlight occurrences of `q` (case-insensitive) inside escaped text.
+ * Receives ALREADY-ESCAPED HTML — we do regex over that string and wrap
+ * matches in <mark>. Safe because we never let user input near a tag opener.
+ */
+function highlight(escapedText, q) {
+  if (!q || !q.trim()) return escapedText;
+  const needle = q.trim().replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  // Avoid matching inside existing tags by requiring no '<' before next '>'
+  const re = new RegExp(`(${needle})`, 'gi');
+  return escapedText.replace(re, '<mark>$1</mark>');
+}
+
+function renderBubble(msg, q) {
+  const side = bubbleSide(msg);
+  const cls = side === 'right' ? 'chat-bubble user' : 'chat-bubble ai';
+  const who = msg.sender || msg.role || (side === 'right' ? 'you' : 'ai');
+  const when = msg.ts ? fmtDateTime(msg.ts) : '';
+  const text = msg.text || '';
+  const highlighted = highlight(esc(text), q);
+  return `
+    <div class="${cls}" id="msg-${msg.id}">
+      <span class="chat-who">${esc(who)}${when ? ' · ' + esc(when) : ''}</span>
+      <p>${highlighted}</p>
+    </div>
+  `;
+}
+
+export async function renderConversation(db, id, query, status) {
+  const q = query.q || '';
+  const offset = Math.max(0, parseInt(query.offset, 10) || 0);
+  const limit = clampLimit(query.limit);
+
+  const conv = db
+    .prepare('SELECT * FROM conversations WHERE conversation_id = ?')
+    .get(id);
+
+  if (!conv) {
+    const body = html`
+      <div class="empty">
+        <h3>Conversation not found</h3>
+        <p>No conversation with id <code>${esc(id)}</code> in this database.</p>
+        <p style="margin-top:12px;"><a class="btn" href="/conversations">← Back to conversations</a></p>
+      </div>
+    `;
+    return renderPage({ title: 'Not found', active: 'conversations', body, status });
+  }
+
+  const messages = db
+    .prepare(
+      `
+      SELECT id, role, sender, text, ts, msg_id
+      FROM messages
+      WHERE conversation_id = ?
+        AND role NOT IN ('boundary', 'summary')
+      ORDER BY COALESCE(ts, 0) ASC, id ASC
+      LIMIT ? OFFSET ?
+    `
+    )
+    .all(id, limit, offset);
+
+  const totalNonMeta = db
+    .prepare(
+      "SELECT COUNT(*) AS n FROM messages WHERE conversation_id = ? AND role NOT IN ('boundary', 'summary')"
+    )
+    .get(id).n;
+
+  // Build day-separated transcript
+  const transcriptParts = [];
+  let lastDay = null;
+  for (const m of messages) {
+    const day = dayKey(m.ts);
+    if (day && day !== lastDay) {
+      transcriptParts.push(`<div class="transcript-day">${esc(day)}</div>`);
+      lastDay = day;
+    }
+    transcriptParts.push(renderBubble(m, q));
+  }
+
+  const transcriptHtml = transcriptParts.join('\n');
+
+  // Pagination
+  const hasPrev = offset > 0;
+  const hasNext = offset + messages.length < totalNonMeta;
+  const baseQs = q ? `?q=${encodeURIComponent(q)}&` : '?';
+  const pagination = (hasPrev || hasNext)
+    ? html`
+        <div class="search-bar" style="justify-content:space-between;margin-top:24px;">
+          ${hasPrev
+            ? html`<a class="btn" href="/c/${encodeURIComponent(id)}${raw(baseQs)}offset=${Math.max(0, offset - limit)}&limit=${limit}">← Previous ${limit}</a>`
+            : html`<span></span>`}
+          <span class="search-meta">
+            Showing ${fmtNum(offset + 1)}–${fmtNum(offset + messages.length)} of ${fmtNum(totalNonMeta)}
+          </span>
+          ${hasNext
+            ? html`<a class="btn" href="/c/${encodeURIComponent(id)}${raw(baseQs)}offset=${offset + limit}&limit=${limit}">Next ${limit} →</a>`
+            : html`<span></span>`}
+        </div>
+      `
+    : null;
+
+  // Search box scoped to this conversation
+  const searchBar = html`
+    <form class="search-bar" method="get" action="/c/${encodeURIComponent(id)}">
+      <input
+        class="search-input"
+        type="search"
+        name="q"
+        placeholder="🔍  Find in this conversation…"
+        value="${esc(q)}"
+        autocomplete="off"
+      />
+      ${q
+        ? html`<a class="btn" href="/c/${encodeURIComponent(id)}">Clear</a>`
+        : null}
+    </form>
+  `;
+
+  const header = html`
+    <p style="margin-bottom:14px;">
+      <a class="btn" href="/conversations">← All conversations</a>
+    </p>
+    <section class="card">
+      <div class="card-label">
+        <span class="conv-source-tag">${conv.source}</span>
+        ${fmtNum(totalNonMeta)} messages
+        · ${fmtDate(conv.first_ts)} → ${fmtDate(conv.last_ts)}
+        ${conv.project_path ? raw(' · <code>' + esc(conv.project_path) + '</code>') : null}
+      </div>
+      <div class="card-body">
+        <h2 style="font-size:20px;font-weight:700;letter-spacing:-0.02em;">${conv.title || '(untitled)'}</h2>
+      </div>
+    </section>
+  `;
+
+  const body = html`
+    ${header}
+    ${searchBar}
+    <div class="transcript">${raw(transcriptHtml)}</div>
+    ${pagination}
+  `;
+
+  return renderPage({
+    title: conv.title || 'Conversation',
+    active: 'conversations',
+    body,
+    status,
+  });
+}