parallelclaw 1.0.0

package/lib/store-doc/detect.js

@@ -0,0 +1,209 @@
+/**
+ * Pattern detection for memex_store_document.
+ *
+ * When the agent passes content to memex_store_document, memex sniffs it
+ * for known failure signatures (Cloudflare challenge, Perplexity-private,
+ * paywalls, …) and returns actionable warnings.
+ *
+ * Each detector returns either null or an object:
+ *   { type, blocking, message }
+ *
+ * `blocking: true` → memex returns stored:false to the agent. Use only for
+ * clear-cut failures where storing the content would pollute the corpus.
+ * `blocking: false` → memex stores the content but appends the warning so
+ * the agent can decide whether to surface it to the user.
+ *
+ * Patterns may grow over time as new failure modes appear in real use.
+ * Single-purpose regexes — order matters (more specific first).
+ */
+
+const CLOUDFLARE_PATTERNS = [
+  /Just a moment\.\.\./i,
+  /Verifying you are human/i,
+  /cf-turnstile/i,
+  /cf_chl_opt/i,
+  /__cf_bm/i,                      // Cloudflare's bot-management cookie name in challenge pages
+  /Checking your browser before accessing/i,
+];
+
+const PERPLEXITY_PRIVATE_PATTERNS = [
+  /This thread is private/i,
+  /Sign in if you are the owner of this thread/i,
+];
+
+const LOGIN_REQUIRED_PATTERNS = [
+  /Sign in to continue/i,
+  /Please log in to continue/i,
+  /Login required/i,
+  /<form[^>]+action=["']\/?(login|signin|sign-in|auth)/i,
+  /You must be logged in/i,
+];
+
+const PAYWALL_PATTERNS = [
+  /Subscribe to (read|continue)/i,
+  /Continue reading with subscription/i,
+  /\bpaywall\b/i,
+  /You've reached your free article limit/i,
+  /metered access/i,
+  /Become a (member|subscriber)/i,
+];
+
+/**
+ * Detect a Cloudflare bot-challenge page. Blocking.
+ */
+function detectCloudflareChallenge(content, _url) {
+  for (const re of CLOUDFLARE_PATTERNS) {
+    if (re.test(content)) {
+      return {
+        type: 'cloudflare-challenge',
+        blocking: true,
+        message:
+          'The content you passed looks like a Cloudflare bot-challenge page, not the actual content. The page is protected by Cloudflare and your direct fetch was blocked.\n\n' +
+          'Retry by prepending https://r.jina.ai/ to the URL:\n\n' +
+          '  WebFetch("https://r.jina.ai/" + originalUrl)\n\n' +
+          'Jina AI Reader runs a real browser server-side and bypasses Cloudflare. Returns clean markdown. Free for personal use, no API key needed.\n\n' +
+          'If Jina ALSO fails or returns "private/login required", the page needs authentication — Jina can\'t bypass that.',
+      };
+    }
+  }
+  return null;
+}
+
+/**
+ * Detect a private Perplexity thread. Blocking — Jina can't help here,
+ * the user needs to make the thread public first.
+ */
+function detectPerplexityPrivate(content, url) {
+  // Only flag if we have a URL hint that it's Perplexity, OR if the message
+  // text is unambiguously Perplexity's phrasing.
+  const isPerplexityUrl =
+    typeof url === 'string' && /perplexity\.ai/i.test(url);
+
+  let matched = false;
+  for (const re of PERPLEXITY_PRIVATE_PATTERNS) {
+    if (re.test(content)) {
+      matched = true;
+      break;
+    }
+  }
+  if (!matched) return null;
+  if (!isPerplexityUrl && !/perplexity/i.test(content)) {
+    // Same phrasing might appear on other sites — only act if we're confident
+    return null;
+  }
+
+  return {
+    type: 'perplexity-private',
+    blocking: true,
+    message:
+      'This Perplexity thread is marked private — even Jina Reader can\'t access it (this is an authentication wall, not Cloudflare bot protection).\n\n' +
+      'Tell the user: "To save this Perplexity thread to memex, you need to make it public first:\n' +
+      '  1. Open the thread in Perplexity\n' +
+      '  2. Click Share (top right)\n' +
+      '  3. Toggle \'Public link\' on\n' +
+      '  4. Copy the new shareable URL Perplexity shows\n' +
+      '  5. Send me THAT URL — it\'ll work"\n\n' +
+      'The URL in the user\'s address bar (perplexity.ai/search/<id>) is the owner\'s private URL, not the shareable one.',
+  };
+}
+
+/**
+ * Suspiciously short content from a URL that should be substantive.
+ * Non-blocking — we store it, but warn.
+ */
+function detectSuspiciouslySmall(content, url) {
+  const trimmed = (content || '').trim();
+  // Threshold: documents shorter than 200 chars are almost certainly noise
+  // (error pages, redirects, JS-only stubs). Pasted snippets can legitimately
+  // be that short, so only flag when we have a URL (suggesting a fetch was
+  // attempted) — pastes get a free pass.
+  if (!url) return null;
+  if (trimmed.length >= 200) return null;
+  return {
+    type: 'suspiciously-small',
+    blocking: false,
+    message:
+      `The content you passed is very short (${trimmed.length} chars). ` +
+      'The page might have been blocked, redirect-failed, or be JS-rendered with no SSR. ' +
+      'Stored as-is — consider verifying with the user that this is what they expected.',
+  };
+}
+
+/**
+ * Login required (form / prompt). Non-blocking but worth flagging.
+ */
+function detectLoginRequired(content, _url) {
+  for (const re of LOGIN_REQUIRED_PATTERNS) {
+    if (re.test(content)) {
+      return {
+        type: 'login-required',
+        blocking: false,
+        message:
+          'The page appears to require login (sign-in prompt / login form detected). ' +
+          'The content you stored may be a login page, not the actual content the user wanted. ' +
+          'Ask the user to paste the content manually if this isn\'t what they expected.',
+      };
+    }
+  }
+  return null;
+}
+
+/**
+ * Paywall / subscription-gated content. Non-blocking.
+ */
+function detectPaywalled(content, _url) {
+  for (const re of PAYWALL_PATTERNS) {
+    if (re.test(content)) {
+      return {
+        type: 'paywalled',
+        blocking: false,
+        message:
+          'The page appears to be paywalled (subscription/payment prompt detected). ' +
+          'The content stored may just be the teaser. ' +
+          'If the user has full access, they can paste the complete article manually.',
+      };
+    }
+  }
+  return null;
+}
+
+/**
+ * Returns array of warnings sorted with blocking warnings first.
+ * If the first warning is blocking, memex should refuse the store
+ * and return that warning to the agent.
+ *
+ * Detectors run in this order (more-specific first):
+ *   1. cloudflare-challenge  (blocking)
+ *   2. perplexity-private    (blocking)
+ *   3. suspiciously-small    (non-blocking)
+ *   4. login-required        (non-blocking)
+ *   5. paywalled             (non-blocking)
+ */
+export function detectIssues(content, url) {
+  const safeContent = typeof content === 'string' ? content : '';
+  const warnings = [];
+
+  // Blocking first — stop on first hit so we surface the most actionable.
+  const blocking =
+    detectCloudflareChallenge(safeContent, url) ||
+    detectPerplexityPrivate(safeContent, url);
+  if (blocking) {
+    warnings.push(blocking);
+    return warnings;
+  }
+
+  // Non-blocking — collect all that match.
+  for (const fn of [detectSuspiciouslySmall, detectLoginRequired, detectPaywalled]) {
+    const w = fn(safeContent, url);
+    if (w) warnings.push(w);
+  }
+
+  return warnings;
+}
+
+/**
+ * Convenience: is any warning blocking?
+ */
+export function isBlocked(warnings) {
+  return Array.isArray(warnings) && warnings.some((w) => w.blocking);
+}

package/lib/store-doc/extract-title.js

@@ -0,0 +1,162 @@
+/**
+ * Extract a title from fetched page content.
+ *
+ * Strategy (first hit wins):
+ *   0. Strip Jina Reader prefix block if present (Jina prepends
+ *      `Title: …\nURL Source: …\nPublished Time: …\nMarkdown Content:\n`
+ *      to its output; the literal "Title:" line is often useless boilerplate
+ *      like "Title: Perplexity" rather than the actual thread title)
+ *   1. Markdown H1 — `# Title text`
+ *   2. Markdown H2 — `## Title text`  (Perplexity threads start with H2)
+ *   3. HTML <title> — `<title>Page Title</title>`
+ *   4. HTML <h1>  — `<h1>Page Title</h1>`
+ *   5. First non-empty line if short enough to look like a title
+ *   6. URL slug fallback — last meaningful path segment, decoded
+ *   7. Domain fallback — just the domain name
+ *   8. "Untitled document"
+ *
+ * Returns a trimmed string up to MAX_LEN characters. Always returns a
+ * non-empty string (worst case "Untitled document").
+ */
+
+const MAX_LEN = 200;
+
+function trimTitle(s) {
+  if (!s) return '';
+  let t = String(s).replace(/\s+/g, ' ').trim();
+  if (t.length > MAX_LEN) t = t.slice(0, MAX_LEN).trim() + '…';
+  return t;
+}
+
+/**
+ * Jina AI Reader (r.jina.ai/<url>) wraps every page in a metadata
+ * prefix:
+ *
+ *   Title: <browser tab title>
+ *
+ *   URL Source: <original URL>
+ *
+ *   Published Time: <date>
+ *
+ *   Markdown Content:
+ *   <actual page markdown follows here>
+ *
+ * The "Title:" line is frequently a generic app shell ("Perplexity",
+ * "Twitter / X", "GitHub") rather than the actual document title — so
+ * we strip the whole prefix and run title extraction against the real
+ * markdown body. The actual H1/H2 inside is what we want.
+ *
+ * Detection is keyed on "URL Source: http" near the top — that line
+ * is unique to Jina's output format. If it's not present, content is
+ * returned unchanged (non-Jina source).
+ */
+function stripJinaPrefix(content) {
+  // Quick gate: look for URL Source line in the first ~500 chars
+  if (!/^URL Source:\s*https?:\/\//m.test(content.slice(0, 500))) {
+    return content;
+  }
+  // Find the "Markdown Content:" delimiter and slice everything after it
+  const m = content.match(/^Markdown Content:\s*\n/m);
+  if (!m) return content;
+  return content.slice(m.index + m[0].length);
+}
+
+function fromMarkdownH1(content) {
+  // Single # at start of line, then space(s), then text.
+  const m = content.match(/^[ \t]*#[ \t]+([^\r\n]+?)[ \t]*$/m);
+  return m ? trimTitle(m[1]) : '';
+}
+
+function fromMarkdownH2(content) {
+  // ## at start of line — used as fallback when H1 absent
+  // (Perplexity, Jina-fetched Twitter threads, many blog "subtopic" layouts).
+  const m = content.match(/^[ \t]*##[ \t]+([^\r\n]+?)[ \t]*$/m);
+  return m ? trimTitle(m[1]) : '';
+}
+
+function fromHtmlTitle(content) {
+  const m = content.match(/<title[^>]*>([^<]+)<\/title>/i);
+  return m ? trimTitle(decodeEntities(m[1])) : '';
+}
+
+function fromHtmlH1(content) {
+  // Inner text only — strip nested tags like <span>...</span>
+  const m = content.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i);
+  if (!m) return '';
+  const inner = m[1].replace(/<[^>]+>/g, '');
+  return trimTitle(decodeEntities(inner));
+}
+
+function fromFirstLine(content) {
+  // First non-empty line, but only if it looks like a heading
+  // (short-ish, no markdown junk).
+  const lines = content.split(/\r?\n/);
+  for (const raw of lines) {
+    const line = raw.trim();
+    if (!line) continue;
+    // Skip leading markdown decorators / metadata
+    if (/^[#\-=*>|`]/.test(line)) continue;
+    if (line.length > 0 && line.length <= 120) {
+      return trimTitle(line);
+    }
+    // First substantive line is too long — give up on this strategy
+    break;
+  }
+  return '';
+}
+
+function fromUrlSlug(rawUrl) {
+  if (!rawUrl) return '';
+  try {
+    const u = new URL(rawUrl);
+    // Last meaningful path segment
+    const segs = u.pathname.split('/').filter(Boolean);
+    if (segs.length) {
+      const slug = decodeURIComponent(segs[segs.length - 1])
+        .replace(/[-_]+/g, ' ')
+        .replace(/\.(html?|md|pdf|txt)$/i, '')
+        .trim();
+      if (slug) return trimTitle(slug);
+    }
+    // No useful path — fall through to domain
+    return trimTitle(u.hostname.replace(/^www\./, ''));
+  } catch (_) {
+    return '';
+  }
+}
+
+// Minimal HTML-entity decode for &amp; &lt; &gt; &quot; &apos; &#39; &#nnn;
+function decodeEntities(s) {
+  if (!s) return s;
+  return String(s)
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&apos;/g, "'")
+    .replace(/&#39;/g, "'")
+    .replace(/&#x([0-9a-f]+);/gi, (_, hex) =>
+      String.fromCodePoint(parseInt(hex, 16))
+    )
+    .replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(parseInt(dec, 10)));
+}
+
+/**
+ * @param {string} content - fetched page content
+ * @param {string|null} url - source URL (used for slug fallback)
+ * @returns {string} a non-empty trimmed title
+ */
+export function extractTitle(content, url) {
+  const safe = typeof content === 'string' ? content : '';
+  const body = stripJinaPrefix(safe);
+
+  return (
+    fromMarkdownH1(body) ||
+    fromMarkdownH2(body) ||
+    fromHtmlTitle(body) ||
+    fromHtmlH1(body) ||
+    fromFirstLine(body) ||
+    fromUrlSlug(url) ||
+    'Untitled document'
+  );
+}

package/lib/sync/auth.js

@@ -0,0 +1,80 @@
+/**
+ * Bearer-token auth for the sync server.
+ *
+ * Every protected endpoint requires:
+ *   Authorization: Bearer <hex-token>
+ *
+ * The token is the same one baked into the pair blob the client received
+ * (256-bit random, hex-encoded). We compare in constant time to avoid
+ * timing oracles.
+ *
+ * On failure: 401 with JSON {error: "unauthorized"} — no detail leaked.
+ *
+ * Why bearer (not OAuth/mTLS):
+ *   - Days vs weeks to ship for the same security model
+ *   - One device pair = one token; rotation by re-pairing
+ *   - mTLS is reasonable to layer on if the user fronts with Caddy
+ */
+
+import { randomBytes, timingSafeEqual } from 'node:crypto';
+
+/**
+ * Generate a 256-bit (32-byte) bearer token, hex-encoded.
+ * Output is 64 chars [0-9a-f].
+ */
+export function generateBearerToken() {
+  return randomBytes(32).toString('hex');
+}
+
+/**
+ * Extract the bearer token from an Authorization header.
+ * Returns the hex string or null if the header is missing/malformed.
+ */
+export function parseAuthHeader(headerValue) {
+  if (!headerValue || typeof headerValue !== 'string') return null;
+  const m = headerValue.match(/^Bearer\s+([0-9a-fA-F]{8,})\s*$/);
+  return m ? m[1].toLowerCase() : null;
+}
+
+/**
+ * Constant-time string comparison. Returns true iff the two strings
+ * are equal as bytes. We hex-decode both to fixed-length buffers so
+ * timingSafeEqual sees same-length inputs (it throws otherwise).
+ *
+ * If either token is malformed hex or wrong length, returns false
+ * without throwing.
+ */
+export function tokensMatch(expected, provided) {
+  if (!expected || !provided) return false;
+  if (expected.length !== provided.length) return false;
+  let bufExpected, bufProvided;
+  try {
+    bufExpected = Buffer.from(expected, 'hex');
+    bufProvided = Buffer.from(provided, 'hex');
+  } catch (_) {
+    return false;
+  }
+  if (bufExpected.length !== bufProvided.length) return false;
+  // timingSafeEqual requires same-length Buffers
+  return timingSafeEqual(bufExpected, bufProvided);
+}
+
+/**
+ * Middleware used by lib/sync/server.js: checks Authorization header
+ * against the server-configured token. Calls `next()` on success,
+ * writes 401 + ends the response on failure.
+ *
+ * Usage:
+ *   if (!requireBearer(req, res, expectedToken)) return;
+ *   // ... handler proceeds
+ */
+export function requireBearer(req, res, expectedToken) {
+  const provided = parseAuthHeader(req.headers.authorization);
+  if (!provided || !tokensMatch(expectedToken, provided)) {
+    res.statusCode = 401;
+    res.setHeader('Content-Type', 'application/json; charset=utf-8');
+    res.end(JSON.stringify({ error: 'unauthorized' }));
+    return false;
+  }
+  return true;
+}

package/lib/sync/cert.js

@@ -0,0 +1,144 @@
+/**
+ * Self-signed TLS cert generation + fingerprint computation for sync server.
+ *
+ * Design choice: every sync server gets its OWN self-signed cert. We don't
+ * use Let's Encrypt because the typical sync deployment has no public DNS
+ * name (it's behind Tailscale, an SSH tunnel, or addressed by raw IP).
+ *
+ * Clients pin the cert fingerprint baked into the pair blob — same trust
+ * model as Plex device pairing or Tailscale's node identity. If the server's
+ * cert ever changes (e.g. you ran `memex sync rotate-cert`), pre-existing
+ * clients refuse to connect until they're re-paired. This is the right
+ * default: silent cert changes are how MITM happens.
+ *
+ * Cert files live in ~/.memex/ alongside the DB:
+ *   sync-cert.pem  — the cert (public)
+ *   sync-key.pem   — the private key (mode 0600)
+ *
+ * Validity period: 10 years. Self-signed; no renewal needed in practice
+ * since clients pin the fingerprint, not the CA chain or expiry.
+ */
+
+import { readFileSync, writeFileSync, existsSync, chmodSync, mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { createHash } from 'node:crypto';
+import selfsigned from 'selfsigned';
+
+/**
+ * Generate a fresh self-signed cert + key, write them to disk (with 0600
+ * on the key file), return { certPath, keyPath, fingerprint }.
+ *
+ * If files already exist, overwrites them — this is the "rotate" path.
+ * Callers who want idempotent "generate-if-missing" should use ensureCert().
+ *
+ * Async because selfsigned ≥5.0 dropped the sync API.
+ */
+export async function generateCert({ certPath, keyPath, commonName = 'memex-sync' }) {
+  if (!certPath || !keyPath) {
+    throw new Error('generateCert: certPath and keyPath are required');
+  }
+  mkdirSync(dirname(certPath), { recursive: true });
+  mkdirSync(dirname(keyPath),  { recursive: true });
+
+  // 10-year validity, 2048-bit RSA. SubjectAltName covers localhost +
+  // the conventional .local mDNS form, so the same cert works for SSH
+  // tunnel localhost connections and LAN-discovery hosts without
+  // extra alt names. We don't bother with public DNS names because
+  // we're pinning by fingerprint anyway.
+  const now = new Date();
+  const tenYears = new Date(now.getTime() + 365 * 10 * 24 * 60 * 60 * 1000);
+  const attrs = [{ name: 'commonName', value: commonName }];
+  const opts = {
+    notBeforeDate: now,
+    notAfterDate:  tenYears,
+    keySize: 2048,
+    algorithm: 'sha256',
+    extensions: [
+      {
+        name: 'subjectAltName',
+        altNames: [
+          { type: 2, value: 'localhost' },
+          { type: 2, value: commonName },
+          { type: 2, value: `${commonName}.local` },
+        ],
+      },
+    ],
+  };
+
+  const pems = await selfsigned.generate(attrs, opts);
+  // pems = { private, public, cert, fingerprint }
+
+  writeFileSync(certPath, pems.cert);
+  writeFileSync(keyPath,  pems.private);
+  // Best-effort 0600 on the key; non-fatal on platforms where chmod doesn't apply.
+  try { chmodSync(keyPath, 0o600); } catch (_) { /* windows etc. */ }
+
+  // Compute our own SHA-256 fingerprint — selfsigned reports SHA-1 historically
+  // and we want consistent sha256:hex for pair-blob pinning.
+  const fingerprint = sha256FingerprintFromPem(pems.cert);
+
+  return { certPath, keyPath, fingerprint, cert: pems.cert, key: pems.private };
+}
+
+/**
+ * Idempotent: if cert+key already exist on disk, returns the existing
+ * fingerprint without regenerating. Otherwise creates new ones.
+ *
+ * Use this on `memex sync server start` to avoid silently rotating
+ * a cert that paired clients depend on.
+ *
+ * Async because generateCert is async.
+ */
+export async function ensureCert({ certPath, keyPath, commonName = 'memex-sync' }) {
+  if (existsSync(certPath) && existsSync(keyPath)) {
+    const fingerprint = sha256FingerprintFromFile(certPath);
+    return { certPath, keyPath, fingerprint, reused: true };
+  }
+  const fresh = await generateCert({ certPath, keyPath, commonName });
+  return { ...fresh, reused: false };
+}
+
+/**
+ * Read a PEM-encoded cert from disk and compute its SHA-256 fingerprint
+ * in the "sha256:AA:BB:CC:..." form that's standard in TLS tooling
+ * (OpenSSL, browsers, Tailscale).
+ */
+export function sha256FingerprintFromFile(certPath) {
+  const pem = readFileSync(certPath, 'utf-8');
+  return sha256FingerprintFromPem(pem);
+}
+
+export function sha256FingerprintFromPem(pem) {
+  // Strip PEM header/footer and decode base64 → raw DER bytes.
+  const body = pem
+    .replace(/-----BEGIN CERTIFICATE-----/g, '')
+    .replace(/-----END CERTIFICATE-----/g, '')
+    .replace(/\s+/g, '');
+  const der = Buffer.from(body, 'base64');
+  const hash = createHash('sha256').update(der).digest('hex').toUpperCase();
+  // Format as "sha256:AA:BB:..." (every two hex chars colon-joined)
+  const colonized = hash.match(/.{2}/g).join(':');
+  return `sha256:${colonized}`;
+}
+
+/**
+ * Compare a (possibly user-supplied) fingerprint against a server's
+ * actual one, tolerant of formatting variations:
+ *   - case-insensitive
+ *   - "sha256:" prefix optional
+ *   - colon-separation optional
+ *
+ * Returns true iff the underlying 32 hex bytes match.
+ */
+export function fingerprintsMatch(a, b) {
+  return normalizeFingerprint(a) === normalizeFingerprint(b);
+}
+
+function normalizeFingerprint(s) {
+  if (!s) return '';
+  let v = String(s).toLowerCase();
+  v = v.replace(/^sha256:/, '');
+  v = v.replace(/:/g, '');
+  v = v.replace(/\s+/g, '');
+  return v;
+}