parallelclaw 1.0.0

package/lib/sync/client.js

@@ -0,0 +1,138 @@
+/**
+ * HTTPS client for talking to a peer's /sync/{health,push,pull} endpoints.
+ *
+ * Tracer-bullet shape: supports plain bearer + --insecure (skip TLS validation).
+ * Cert-fingerprint pinning lands in Day 5; for now we still send the request
+ * over TLS, just don't verify the cert chain.
+ *
+ * The client is intentionally dumb — it doesn't decide WHAT to push or pull,
+ * just shuttles bytes. The replication loop lives in lib/sync/replicate.js.
+ */
+
+import { request as httpsRequest } from 'node:https';
+import { fingerprintsMatch } from './cert.js';
+
+/**
+ * Create a client bound to one remote.
+ *
+ * opts:
+ *   url      — required, e.g. "https://localhost:8765"
+ *   bearer   — required, hex token
+ *   insecure — bool. If true, skip TLS cert validation entirely (tracer-bullet).
+ *   cert_fp  — string. If set, pin the server cert to this SHA-256 fingerprint
+ *              (overrides insecure — pinning is enabled even when insecure=true).
+ *   timeoutMs — request timeout, default 30s
+ */
+export function createSyncClient(opts) {
+  if (!opts || !opts.url) throw new Error('createSyncClient: url required');
+  if (!opts.bearer) throw new Error('createSyncClient: bearer required');
+
+  const url = new URL(opts.url);
+  const insecure = !!opts.insecure;
+  const certFp = opts.cert_fp || null;
+  const timeoutMs = opts.timeoutMs ?? 30_000;
+
+  function makeRequest({ method, path, body }) {
+    return new Promise((resolve, reject) => {
+      const payload = body == null ? null : Buffer.from(JSON.stringify(body));
+      const headers = {
+        Authorization: `Bearer ${opts.bearer}`,
+      };
+      if (payload) {
+        headers['Content-Type'] = 'application/json';
+        headers['Content-Length'] = payload.length;
+      }
+
+      const req = httpsRequest({
+        host: url.hostname,
+        port: url.port || 443,
+        path,
+        method,
+        headers,
+        // When cert pinning is on (cert_fp set), we still set rejectUnauthorized
+        // to false because self-signed certs would otherwise fail the chain check —
+        // we validate the fingerprint manually on 'secureConnect' instead.
+        rejectUnauthorized: (!insecure && !certFp),
+        timeout: timeoutMs,
+        // Disable keep-alive pooling. With pooling, a reused TLS socket
+        // doesn't re-fire 'secureConnect', so our per-request pinning
+        // listener would pile up (MaxListenersExceededWarning seen during
+        // the 326-batch live push). A fresh socket per request keeps the
+        // pinning check correct and leak-free; sync is low-frequency enough
+        // that the extra handshakes are negligible.
+        agent: false,
+      }, (res) => {
+        let chunks = '';
+        res.on('data', (c) => { chunks += c; });
+        res.on('end', () => {
+          let parsed = null;
+          try { parsed = chunks ? JSON.parse(chunks) : null; } catch (_) { parsed = { _raw: chunks }; }
+          resolve({ status: res.statusCode, body: parsed });
+        });
+      });
+
+      // Cert-pinning validation hook. With agent:false each request gets a
+      // fresh socket, so 'socket' and 'secureConnect' each fire exactly once
+      // per request. We use .once() defensively so a listener can never
+      // outlive the handshake it was registered for.
+      if (certFp) {
+        req.once('socket', (socket) => {
+          socket.once('secureConnect', () => {
+            const peerCert = socket.getPeerCertificate(true);
+            // node returns SHA-256 fingerprint as fingerprint256 in "AA:BB:..." form
+            const peerFp = peerCert?.fingerprint256;
+            if (!peerFp || !fingerprintsMatch(certFp, peerFp)) {
+              req.destroy(new Error(
+                `TLS fingerprint mismatch — expected ${certFp}, got ${peerFp || 'none'}`
+              ));
+            }
+          });
+        });
+      }
+
+      req.on('error', reject);
+      req.on('timeout', () => req.destroy(new Error(`request timeout (${timeoutMs}ms)`)));
+
+      if (payload) req.write(payload);
+      req.end();
+    });
+  }
+
+  return {
+    async health() {
+      const r = await makeRequest({ method: 'GET', path: '/sync/health' });
+      if (r.status !== 200) throw new Error(`health failed: ${r.status} ${JSON.stringify(r.body)}`);
+      return r.body;
+    },
+
+    async pull({ since = 0, limit = 500 } = {}) {
+      const r = await makeRequest({
+        method: 'GET',
+        path: `/sync/pull?since=${since}&limit=${limit}`,
+      });
+      if (r.status !== 200) throw new Error(`pull failed: ${r.status} ${JSON.stringify(r.body)}`);
+      return r.body;
+    },
+
+    async push({ rows }) {
+      if (!Array.isArray(rows)) throw new Error('push: rows[] required');
+      const r = await makeRequest({
+        method: 'POST',
+        path: '/sync/push',
+        body: { rows },
+      });
+      if (r.status !== 200) {
+        // Carry the HTTP status on the error so the replication loop can
+        // react — e.g. halve the batch on 413 payload_too_large.
+        const err = new Error(`push failed: ${r.status} ${JSON.stringify(r.body)}`);
+        err.status = r.status;
+        err.body = r.body;
+        throw err;
+      }
+      return r.body;
+    },
+
+    // Direct access for advanced/test callers
+    raw: makeRequest,
+  };
+}

package/lib/sync/config.js

@@ -0,0 +1,130 @@
+/**
+ * Sync section of ~/.memex/config.json — owns its own defaults and helpers,
+ * but persists through the same file as the rest of the config (so a single
+ * config.json round-trips cleanly).
+ *
+ * Schema:
+ *   sync: {
+ *     server: {
+ *       enabled:    false,                // are we running `memex sync server`?
+ *       port:       8765,                 // listen port
+ *       bind:       "0.0.0.0",            // 0.0.0.0 | 127.0.0.1 | tailscale-ip
+ *       bearer:     "<64-hex>",           // 256-bit token (one per server)
+ *       cert_path:  "~/.memex/sync-cert.pem",
+ *       key_path:   "~/.memex/sync-key.pem",
+ *       cert_fp:    "sha256:..."          // computed at server start
+ *     },
+ *     remotes: {
+ *       "<alias>": {
+ *         url:         "https://host:8765",
+ *         bearer:      "<64-hex>",
+ *         cert_fp:     "sha256:...",      // pinned fingerprint (from pair blob)
+ *         pulled_to:   0,                 // last id we pulled FROM this remote
+ *         pushed_to:   0,                 // last OUR id we pushed TO this remote
+ *         last_sync_at: 0,                // unix ms; 0 = never
+ *         last_error:  null               // string | null
+ *       }
+ *     }
+ *   }
+ *
+ * The MEMEX_SYNC_EXPERIMENTAL=1 env var gates all sync activity at the CLI
+ * layer; this module only knows shape, not whether the feature is on.
+ */
+
+import { loadConfig, saveConfig } from '../config.js';
+
+export const DEFAULT_SYNC = Object.freeze({
+  server: {
+    enabled:   false,
+    port:      8765,
+    bind:      '0.0.0.0',
+    bearer:    null,
+    cert_path: null,
+    key_path:  null,
+    cert_fp:   null,
+  },
+  remotes: {},
+});
+
+export function loadSyncConfig() {
+  const cfg = loadConfig();
+  const sync = cfg.sync || {};
+  return {
+    ...sync, // preserve extra keys (e.g. `enabled` set by sync-join) across saves
+    server:  { ...DEFAULT_SYNC.server, ...(sync.server || {}) },
+    remotes: { ...DEFAULT_SYNC.remotes, ...(sync.remotes || {}) },
+  };
+}
+
+export function saveSyncConfig(syncCfg) {
+  const cfg = loadConfig();
+  cfg.sync = syncCfg;
+  saveConfig(cfg);
+}
+
+/**
+ * Merge `patch` into the sync.server section and persist. Other config sections
+ * untouched. Returns the new server config.
+ */
+export function updateSyncServer(patch) {
+  const sync = loadSyncConfig();
+  sync.server = { ...sync.server, ...patch };
+  saveSyncConfig(sync);
+  return sync.server;
+}
+
+/**
+ * Upsert a remote by alias. `patch` shallowly merges into the existing
+ * remote (or seeds a new one with cursor=0).
+ */
+export function upsertSyncRemote(alias, patch) {
+  if (!alias) throw new Error('upsertSyncRemote: alias required');
+  const sync = loadSyncConfig();
+  const existing = sync.remotes[alias] || {
+    url: null, bearer: null, cert_fp: null,
+    pulled_to: 0, pushed_to: 0,
+    last_sync_at: 0, last_error: null,
+  };
+  sync.remotes[alias] = { ...existing, ...patch };
+  saveSyncConfig(sync);
+  return sync.remotes[alias];
+}
+
+export function getSyncRemote(alias) {
+  const sync = loadSyncConfig();
+  return sync.remotes[alias] || null;
+}
+
+export function listSyncRemotes() {
+  return loadSyncConfig().remotes;
+}
+
+export function removeSyncRemote(alias) {
+  const sync = loadSyncConfig();
+  if (!(alias in sync.remotes)) return false;
+  delete sync.remotes[alias];
+  saveSyncConfig(sync);
+  return true;
+}
+
+/**
+ * Returns true if sync is enabled — either via the MEMEX_SYNC_EXPERIMENTAL
+ * env var (the original opt-in) or via `sync.enabled: true` in config.json,
+ * which `sync-join` sets on success. The config path is what makes the
+ * lazy-user flow work: after one successful join, every later sync command
+ * (sync-run, sync-status, the schedule timer) just works in any shell with
+ * no env var to remember.
+ */
+export function syncExperimentEnabled() {
+  const v = process.env.MEMEX_SYNC_EXPERIMENTAL;
+  if (v && (v === '1' || v.toLowerCase() === 'true' || v.toLowerCase() === 'yes')) return true;
+  try { return (loadConfig().sync || {}).enabled === true; }
+  catch (_) { return false; }
+}
+
+/** Persist sync.enabled=true — called by sync-join on success. */
+export function markSyncEnabled() {
+  const sync = loadSyncConfig();
+  sync.enabled = true;
+  saveSyncConfig(sync);
+}

package/lib/sync/pair.js

@@ -0,0 +1,145 @@
+/**
+ * Pair blob — collapse {host, port, cert_fp, bearer} into one copy-pasteable
+ * string so a peer can be added with a single paste instead of three CLI args.
+ *
+ *   memex-pair:<base64url(JSON)>
+ *
+ * The JSON payload:
+ *   { v, host, port, cert_fp, token, exp }
+ *
+ * Design notes:
+ *   • base64url (no +/=) so it survives chat clients, URLs, and shell args
+ *     without escaping.
+ *   • `exp` (unix seconds) — a short TTL (default 10 min). A leaked blob is
+ *     only useful until it expires; after that the client refuses it and the
+ *     operator mints a fresh one. The bearer itself doesn't rotate on expiry
+ *     (it persists server-side); expiry just bounds the pairing window.
+ *   • `v` version gate — a client that doesn't understand the version refuses
+ *     rather than mis-parsing.
+ *   • This is transport-agnostic: `host` is whatever the CLIENT will dial —
+ *     a public IP, a localhost SSH-tunnel port, or a Tailscale MagicDNS name.
+ *     The server can't know that, so the invite step chooses/declares it.
+ *
+ * Security model unchanged from the 3-arg path: cert_fp gives TLS pinning,
+ * token is the 256-bit bearer. The blob just bundles them.
+ */
+
+const PREFIX = 'memex-pair:';
+// Join token (v0.13 lazy-user flow) — same payload as a pair blob PLUS
+// `ssh_target` ("user@host"). The presence of ssh_target tells the client to
+// reach the server through a forward SSH tunnel (-L) instead of dialing
+// host:port directly — the canonical loopback-hub topology where the server
+// never exposes a public port. A distinct prefix so old clients fail with
+// "not a memex-pair token" instead of silently dialing 127.0.0.1.
+const JOIN_PREFIX = 'memex-join:';
+const PAIR_VERSION = 1;
+export const DEFAULT_PAIR_TTL_SEC = 600;  // 10 minutes
+export const DEFAULT_JOIN_TTL_SEC = 1800; // 30 minutes — join is a longer dance
+
+/**
+ * Encode a pair blob. Returns the "memex-pair:..." string.
+ *   host    — required; what the client will connect to
+ *   port    — default 8766
+ *   cert_fp — TLS fingerprint to pin (sha256:AA:BB:...); may be null for
+ *             transport-trusted setups (SSH tunnel / Tailscale)
+ *   token   — required; 256-bit hex bearer
+ *   ttlSec  — seconds until the blob expires (default 10 min)
+ *   now     — injectable clock (ms) for tests
+ */
+export function encodePairBlob({ host, port = 8766, cert_fp = null, token, ttlSec = DEFAULT_PAIR_TTL_SEC, now = Date.now() }) {
+  if (!host) throw new Error('encodePairBlob: host required');
+  if (!token) throw new Error('encodePairBlob: token required');
+  const payload = {
+    v: PAIR_VERSION,
+    host,
+    port: Number(port) || 8766,
+    cert_fp: cert_fp || null,
+    token,
+    exp: Math.floor(now / 1000) + Math.max(1, Math.floor(ttlSec)),
+  };
+  const b64 = Buffer.from(JSON.stringify(payload), 'utf-8').toString('base64url');
+  return PREFIX + b64;
+}
+
+/**
+ * Encode a JOIN token — the lazy-user variant. Same fields as a pair blob,
+ * plus `ssh_target`. `host` is pinned to 127.0.0.1: the server binds loopback
+ * and the client reaches it through its own `-L` tunnel, so loopback is the
+ * only address that's ever dialed.
+ */
+export function encodeJoinBlob({ ssh_target, port = 8766, cert_fp = null, token, ttlSec = DEFAULT_JOIN_TTL_SEC, now = Date.now() }) {
+  if (!ssh_target || !/^[^@\s]+@[^@\s]+$/.test(ssh_target)) {
+    throw new Error('encodeJoinBlob: ssh_target required (user@host)');
+  }
+  if (!token) throw new Error('encodeJoinBlob: token required');
+  const payload = {
+    v: PAIR_VERSION,
+    host: '127.0.0.1',
+    ssh_target,
+    port: Number(port) || 8766,
+    cert_fp: cert_fp || null,
+    token,
+    exp: Math.floor(now / 1000) + Math.max(1, Math.floor(ttlSec)),
+  };
+  return JOIN_PREFIX + Buffer.from(JSON.stringify(payload), 'utf-8').toString('base64url');
+}
+
+/**
+ * Parse + validate a pair OR join blob. Throws a friendly Error on any problem.
+ * Returns { kind, host, port, url, cert_fp, token, exp, ssh_target }.
+ *   kind       — 'pair' | 'join' (by prefix)
+ *   ssh_target — "user@host" for join tokens, null for plain pair blobs
+ *
+ *   now — injectable clock (ms) for tests.
+ */
+export function parsePairBlob(blob, { now = Date.now() } = {}) {
+  if (typeof blob !== 'string' || !blob.trim()) {
+    throw new Error('pair blob must be a non-empty string');
+  }
+  let s = blob.trim();
+  let kind;
+  if (s.startsWith(JOIN_PREFIX)) {
+    kind = 'join';
+    s = s.slice(JOIN_PREFIX.length).trim();
+  } else if (s.startsWith(PREFIX)) {
+    kind = 'pair';
+    s = s.slice(PREFIX.length).trim();
+  } else {
+    throw new Error(`not a memex pair/join token (must start with "${PREFIX}" or "${JOIN_PREFIX}")`);
+  }
+
+  let payload;
+  try {
+    payload = JSON.parse(Buffer.from(s, 'base64url').toString('utf-8'));
+  } catch (_) {
+    throw new Error('pair blob is corrupt (base64/JSON decode failed) — re-copy it whole');
+  }
+
+  if (payload.v !== PAIR_VERSION) {
+    throw new Error(`unsupported pair blob version ${payload.v} — this memex speaks v${PAIR_VERSION}; upgrade the older side`);
+  }
+  if (!payload.host || !payload.token) {
+    throw new Error('pair blob missing host or token');
+  }
+  if (kind === 'join' && !/^[^@\s]+@[^@\s]+$/.test(payload.ssh_target || '')) {
+    throw new Error('join token missing ssh_target (user@host) — re-emit it with `sync-server invite --join`');
+  }
+  if (payload.exp && Math.floor(now / 1000) > payload.exp) {
+    const agoMin = Math.round((Math.floor(now / 1000) - payload.exp) / 60);
+    throw new Error(`pair blob expired ${agoMin}m ago — mint a fresh one with \`memex-sync sync-server invite\``);
+  }
+
+  const port = Number(payload.port) || 8766;
+  return {
+    kind,
+    host: payload.host,
+    port,
+    url: `https://${payload.host}:${port}`,
+    cert_fp: payload.cert_fp || null,
+    token: payload.token,
+    exp: payload.exp || null,
+    ssh_target: kind === 'join' ? payload.ssh_target : null,
+  };
+}
+
+export { PREFIX as PAIR_PREFIX, JOIN_PREFIX, PAIR_VERSION };

package/lib/sync/pull.js

@@ -0,0 +1,158 @@
+/**
+ * GET /sync/pull?since=<int>&limit=<int> — return rows the caller hasn't seen.
+ *
+ * Wire contract (see SYNC.md §Wire protocol):
+ *
+ *   Query:
+ *     since   — caller's last-seen local id from this server (0 = first pull)
+ *     limit   — max rows; default 500, hard cap 1000
+ *
+ *   Response: {
+ *     rows:        [Row, ...],    // ordered by id ASC, id > since
+ *     next_cursor: <int>,         // id of the last row in this batch
+ *                                  // (caller passes this as since= next time)
+ *     has_more:    bool,          // true → more rows wait; caller should
+ *                                  // re-pull immediately with the new cursor
+ *     server_now:  <ms_epoch>     // informational
+ *   }
+ *
+ * Each Row matches the shape POST /sync/push expects, including the embedded
+ * conversation metadata (so the receiver can upsert without a separate
+ * join-and-fetch round trip).
+ *
+ * Conversation embedding is denormalised on read: we LEFT JOIN conversations
+ * and inline the columns. For a typical 500-row pull this is one query.
+ */
+
+const DEFAULT_LIMIT = 500;
+const MAX_LIMIT = 1000;
+
+export function makePullHandler({ db }) {
+  if (!db) throw new Error('makePullHandler: db is required');
+
+  // One query, LEFT JOIN — every message also surfaces its conversation row
+  // for the receiver's upsert. Order is critical: ASC by id ensures the
+  // cursor (max id returned) is monotonic across pulls.
+  const fetchSince = db.prepare(`
+    SELECT
+      m.id                              AS id,
+      m.source                          AS source,
+      m.conversation_id                 AS conversation_id,
+      m.msg_id                          AS msg_id,
+      m.role                            AS role,
+      m.sender                          AS sender,
+      m.text                            AS text,
+      m.ts                              AS ts,
+      m.edited_at                       AS edited_at,
+      m.uuid                            AS uuid,
+      m.channel                         AS channel,
+      m.origin                          AS origin,
+      m.metadata                        AS metadata,
+      c.title                           AS conv_title,
+      c.first_ts                        AS conv_first_ts,
+      c.last_ts                         AS conv_last_ts,
+      c.parent_conversation_id          AS conv_parent,
+      c.project_path                    AS conv_project_path
+    FROM messages m
+    LEFT JOIN conversations c ON c.conversation_id = m.conversation_id
+    WHERE m.id > ?
+    ORDER BY m.id ASC
+    LIMIT ?
+  `);
+
+  return function pullHandler(req, res) {
+    let url;
+    try {
+      url = new URL(req.url, 'https://placeholder.local');
+    } catch (_) {
+      return respondJson(res, 400, { error: 'bad_request', detail: 'malformed URL' });
+    }
+
+    const sinceRaw = url.searchParams.get('since') ?? '0';
+    const limitRaw = url.searchParams.get('limit') ?? String(DEFAULT_LIMIT);
+
+    const since = parseNonNegInt(sinceRaw);
+    if (since == null) {
+      return respondJson(res, 400, { error: 'bad_request', detail: 'since must be a non-negative integer' });
+    }
+    const limit = clampLimit(limitRaw);
+
+    let rows;
+    try {
+      rows = fetchSince.all(since, limit + 1);
+    } catch (err) {
+      return respondJson(res, 500, { error: 'internal', detail: err.message });
+    }
+
+    // If we fetched limit+1, the extra row indicates there's more after this batch.
+    const has_more = rows.length > limit;
+    if (has_more) rows = rows.slice(0, limit);
+
+    const wireRows = rows.map(rowToWire);
+    const next_cursor = wireRows.length
+      ? wireRows[wireRows.length - 1].id_serverside
+      : since;
+
+    // Strip the bookkeeping field — id_serverside was only for next_cursor.
+    for (const r of wireRows) delete r.id_serverside;
+
+    respondJson(res, 200, {
+      rows: wireRows,
+      next_cursor,
+      has_more,
+      server_now: Date.now(),
+    });
+  };
+}
+
+/**
+ * Map a SQLite-row into the wire Row shape per SYNC.md. We tuck the
+ * server-side id into a temporary `id_serverside` field so the caller of
+ * makePullHandler can compute next_cursor without re-iterating. Field is
+ * removed before serialization.
+ */
+function rowToWire(r) {
+  return {
+    source:          r.source,
+    conversation_id: r.conversation_id,
+    msg_id:          r.msg_id,
+    uuid:            r.uuid,
+    role:            r.role,
+    sender:          r.sender,
+    text:            r.text,
+    ts:              r.ts,
+    edited_at:       r.edited_at,
+    channel:         r.channel,
+    origin:          r.origin,
+    metadata:        r.metadata,
+    conversation: {
+      title:                   r.conv_title,
+      first_ts:                r.conv_first_ts,
+      last_ts:                 r.conv_last_ts,
+      parent_conversation_id:  r.conv_parent,
+      project_path:            r.conv_project_path,
+    },
+    id_serverside: r.id, // stripped before response
+  };
+}
+
+function parseNonNegInt(s) {
+  if (s == null) return null;
+  const n = Number(s);
+  if (!Number.isFinite(n)) return null;
+  if (n < 0) return null;
+  return Math.trunc(n);
+}
+
+function clampLimit(s) {
+  const n = parseNonNegInt(s);
+  if (n == null || n === 0) return DEFAULT_LIMIT;
+  return Math.min(n, MAX_LIMIT);
+}
+
+function respondJson(res, status, obj) {
+  if (res.headersSent || res.writableEnded) return;
+  res.statusCode = status;
+  res.setHeader('Content-Type', 'application/json; charset=utf-8');
+  res.end(JSON.stringify(obj));
+}