skalpel 3.4.17 → 3.4.19

package/prosumer-hooks/incremental-ingest.mjs

@@ -0,0 +1,530 @@
+// Durable hosted context-graph updates for Claude Code SessionEnd.
+//
+// The transcript snapshot is written locally before any network call. A stable idempotency key
+// follows that snapshot through POST /ingest and every retry, while the returned job_id is polled
+// directly instead of blindly reposting. This is deliberately a local outbox: a network outage,
+// process kill, or accepted-response loss cannot lose the completed session or duplicate paid work.
+import { createHash } from "node:crypto";
+import {
+  closeSync,
+  existsSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  readdirSync,
+  realpathSync,
+  renameSync,
+  statSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { homedir } from "node:os";
+import { basename, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { gzipSync } from "node:zlib";
+
+import { identity } from "./auth.mjs";
+import { recordInsight } from "./insights.mjs";
+
+const OUTBOX_DIR = "ingest-outbox";
+const LEDGER_FILE = "delivered.json";
+const DRAIN_LOCK = ".drain.lock";
+const ITEM_VERSION = 1;
+const MAX_TRANSCRIPT_BYTES = 90 * 1024 * 1024;
+const MAX_DELIVERED_KEYS = 2048;
+const MAX_TERMINAL_RETRIES = 3;
+const LOCK_STALE_MS = 2 * 60 * 1000;
+const DEFAULT_REQUEST_TIMEOUT_MS = 8_000;
+const DEFAULT_WORK_BUDGET_MS = 45_000;
+const DEFAULT_POLL_INTERVAL_MS = 1_500;
+
+function skalpelDir(home) {
+  return join(home, ".skalpel");
+}
+
+function outboxDir(home) {
+  return join(skalpelDir(home), OUTBOX_DIR);
+}
+
+function itemPath(home, key) {
+  return join(outboxDir(home), `${key}.json`);
+}
+
+function snapshotPath(home, key) {
+  return join(outboxDir(home), `${key}.jsonl`);
+}
+
+function ensureOutbox(home) {
+  mkdirSync(outboxDir(home), { recursive: true, mode: 0o700 });
+}
+
+function readJson(path, fallback = null) {
+  try {
+    return JSON.parse(readFileSync(path, "utf8"));
+  } catch {
+    return fallback;
+  }
+}
+
+function writeAtomic(path, value) {
+  const tmp = `${path}.tmp-${process.pid}-${Date.now()}`;
+  writeFileSync(tmp, JSON.stringify(value), { mode: 0o600 });
+  renameSync(tmp, path);
+}
+
+function writeSnapshotAtomic(path, transcript) {
+  const tmp = `${path}.tmp-${process.pid}-${Date.now()}`;
+  writeFileSync(tmp, transcript, { mode: 0o600 });
+  renameSync(tmp, path);
+}
+
+function remove(path) {
+  try {
+    unlinkSync(path);
+  } catch {
+    // Already absent.
+  }
+}
+
+function clearStaleLock(path, now) {
+  try {
+    if (now() - statSync(path).mtimeMs > LOCK_STALE_MS) unlinkSync(path);
+  } catch {
+    // Missing or concurrently released.
+  }
+}
+
+function acquireLock(path, now) {
+  clearStaleLock(path, now);
+  try {
+    const fd = openSync(path, "wx", 0o600);
+    writeFileSync(fd, String(process.pid));
+    closeSync(fd);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function isWithin(root, candidate) {
+  const rel = relative(resolve(root), resolve(candidate));
+  return rel === "" || (!rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel));
+}
+
+function cleanSessionID(value, transcriptPath) {
+  const fallback = basename(transcriptPath, extname(transcriptPath));
+  const raw = typeof value === "string" && value.trim() ? value.trim() : fallback;
+  return raw.replace(/[\u0000-\u001f/\\]/g, "-").slice(0, 200) || fallback;
+}
+
+function baseKey(sessionID, transcript) {
+  return createHash("sha256")
+    .update("skalpel-session-end-v1\0")
+    .update(sessionID)
+    .update("\0")
+    .update(transcript)
+    .digest("hex");
+}
+
+function idempotencyKey(item) {
+  return `session-end:${item.key}:${item.generation || 0}`;
+}
+
+function readLedger(home) {
+  return readJson(join(outboxDir(home), LEDGER_FILE), { version: 1, keys: {} });
+}
+
+function writeLedger(home, ledger) {
+  const entries = Object.entries(ledger.keys || {})
+    .sort((a, b) => String(b[1]).localeCompare(String(a[1])))
+    .slice(0, MAX_DELIVERED_KEYS);
+  writeAtomic(join(outboxDir(home), LEDGER_FILE), { version: 1, keys: Object.fromEntries(entries) });
+}
+
+function markDelivered(home, item, now) {
+  const ledger = readLedger(home);
+  ledger.keys[`${item.user_id}:${item.key}`] = new Date(now()).toISOString();
+  writeLedger(home, ledger);
+  removeItem(home, item);
+}
+
+function removeItem(home, item) {
+  remove(itemPath(home, item.key));
+  remove(snapshotPath(home, item.key));
+}
+
+function saveItem(home, item) {
+  writeAtomic(itemPath(home, item.key), item);
+}
+
+function notifyOnce(home, item, state, display, insightFn) {
+  if (item.notified_state === state) return;
+  item.notified_state = state;
+  saveItem(home, item);
+  insightFn({ kind: "profile", display });
+}
+
+function listItems(home, preferredKey) {
+  ensureOutbox(home);
+  const items = readdirSync(outboxDir(home))
+    .filter((name) => /^[a-f0-9]{64}\.json$/.test(name))
+    .map((name) => readJson(join(outboxDir(home), name)))
+    .filter((item) => item?.version === ITEM_VERSION && typeof item.key === "string")
+    .sort((a, b) => Date.parse(a.created_at) - Date.parse(b.created_at));
+  if (!preferredKey) return items;
+  return items.sort((a, b) => Number(b.key === preferredKey) - Number(a.key === preferredKey));
+}
+
+function clientAPI(home) {
+  const cfg = readJson(join(skalpelDir(home), "client.json"), {});
+  return process.env.SKALPEL_API || cfg.api || "https://graph.skalpel.ai";
+}
+
+async function sleep(ms) {
+  await new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
+}
+
+async function request(fetchFn, url, init, timeoutMs) {
+  const ctrl = new AbortController();
+  const timer = setTimeout(() => ctrl.abort(), timeoutMs);
+  try {
+    return await fetchFn(url, { ...init, signal: ctrl.signal });
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function retryDelayMs(generation) {
+  return Math.min(60 * 60 * 1000, 5 * 60 * 1000 * 2 ** Math.max(0, generation - 1));
+}
+
+function scheduleTerminalRetry(home, item, status, now, insightFn) {
+  const nextGeneration = (item.generation || 0) + 1;
+  item.last_job_id = item.job_id || null;
+  item.job_id = null;
+  item.server_state = status.state || "error";
+  item.last_error = String(status.error || "graph job failed").slice(0, 240);
+  item.generation = nextGeneration;
+  item.next_poll_at = null;
+  item.updated_at = new Date(now()).toISOString();
+  if (nextGeneration <= MAX_TERMINAL_RETRIES) {
+    item.state = "retry_queued";
+    item.next_attempt_at = now() + retryDelayMs(nextGeneration);
+    saveItem(home, item);
+    notifyOnce(
+      home,
+      item,
+      `retry-${nextGeneration}`,
+      "context graph update hit an error; a safe retry is queued",
+      insightFn,
+    );
+    return;
+  }
+  item.state = "fallback_required";
+  item.next_attempt_at = null;
+  saveItem(home, item);
+  notifyOnce(
+    home,
+    item,
+    "fallback-required",
+    "context graph update needs a retry; reopen Skalpel to run the history catch-up",
+    insightFn,
+  );
+}
+
+async function pollAcceptedJob(home, item, auth, options, deadline) {
+  const {
+    api,
+    fetchFn,
+    insightFn,
+    now,
+    pollIntervalMs,
+    requestTimeoutMs,
+    sleepFn,
+  } = options;
+  const statusURL = `${api}/ingest/status?job_id=${encodeURIComponent(item.job_id)}`;
+  for (;;) {
+    let response;
+    try {
+      response = await request(
+        fetchFn,
+        statusURL,
+        { headers: { authorization: `Bearer ${auth.token}` } },
+        requestTimeoutMs,
+      );
+    } catch {
+      item.state = "status_unreachable";
+      item.updated_at = new Date(now()).toISOString();
+      saveItem(home, item);
+      return "pending";
+    }
+    if (response.status === 401) {
+      item.state = "auth_required";
+      saveItem(home, item);
+      return "pending";
+    }
+    if (response.status === 404) {
+      scheduleTerminalRetry(home, item, { state: "error", error: "job not found" }, now, insightFn);
+      return "retry";
+    }
+    if (!response.ok) {
+      item.state = `status_http_${response.status}`;
+      saveItem(home, item);
+      return "pending";
+    }
+    const status = await response.json().catch(() => ({}));
+    item.state = typeof status.state === "string" ? status.state : "running";
+    item.phase = typeof status.phase === "string" ? status.phase : item.phase || null;
+    item.done = Number.isFinite(status.done) ? status.done : item.done || 0;
+    item.total = Number.isFinite(status.total) ? status.total : item.total || 1;
+    item.errors = Number.isFinite(status.errors) ? status.errors : item.errors || 0;
+    item.heartbeat = status.heartbeat_at || status.updated_at || item.heartbeat || null;
+    item.terminal = status.terminal === true;
+    item.requires_replay = status.requires_replay === true;
+    item.retry_at = typeof status.retry_at === "string" ? status.retry_at : null;
+    if (item.state === "waiting_quota" && !item.terminal && !item.requires_replay) {
+      const retryAt = Date.parse(item.retry_at || "");
+      item.next_poll_at = Number.isFinite(retryAt) ? retryAt : now() + 60_000;
+    } else {
+      item.next_poll_at = null;
+    }
+    item.updated_at = new Date(now()).toISOString();
+    saveItem(home, item);
+
+    if (item.state === "done") {
+      markDelivered(home, item, now);
+      return "done";
+    }
+    if (item.state === "waiting_quota") {
+      // Transitional server: payload is not durable, so this exact job is terminal and the server
+      // explicitly requests a new operation key. Final durable server: same state is nonterminal,
+      // carries retry_at, and resumes the SAME job. Pin both contracts; never infer one from the
+      // state string alone.
+      if (item.terminal && item.requires_replay) {
+        scheduleTerminalRetry(home, item, status, now, insightFn);
+        return "retry";
+      }
+      notifyOnce(
+        home,
+        item,
+        "waiting-quota",
+        "context graph update is safely queued while provider capacity recovers",
+        insightFn,
+      );
+      return "pending";
+    }
+    if (item.terminal && item.requires_replay) {
+      scheduleTerminalRetry(home, item, status, now, insightFn);
+      return "retry";
+    }
+    if (item.state === "error" || item.state === "failed") {
+      scheduleTerminalRetry(home, item, status, now, insightFn);
+      return "retry";
+    }
+    if (now() + pollIntervalMs >= deadline) return "pending";
+    await sleepFn(pollIntervalMs);
+  }
+}
+
+async function submitItem(home, item, auth, options, deadline) {
+  const { api, fetchFn, insightFn, now, requestTimeoutMs } = options;
+  if (!item.job_id) {
+    let transcript;
+    try {
+      transcript = readFileSync(snapshotPath(home, item.key), "utf8");
+    } catch {
+      item.state = "snapshot_missing";
+      saveItem(home, item);
+      notifyOnce(
+        home,
+        item,
+        "snapshot-missing",
+        "context graph update needs a retry; reopen Skalpel to run the history catch-up",
+        insightFn,
+      );
+      return "pending";
+    }
+    const body = JSON.stringify({
+      user_id: auth.uid,
+      raw_transcripts: { [`${item.session_id}.jsonl`]: transcript },
+    });
+    let response;
+    try {
+      response = await request(
+        fetchFn,
+        `${api}/ingest`,
+        {
+          method: "POST",
+          headers: {
+            authorization: `Bearer ${auth.token}`,
+            "content-type": "application/json",
+            "content-encoding": "gzip",
+            "idempotency-key": idempotencyKey(item),
+          },
+          body: gzipSync(body),
+        },
+        requestTimeoutMs,
+      );
+    } catch {
+      item.state = "submit_unreachable";
+      item.updated_at = new Date(now()).toISOString();
+      saveItem(home, item);
+      return "pending";
+    }
+    if (response.status === 401) {
+      item.state = "auth_required";
+      saveItem(home, item);
+      return "pending";
+    }
+    if (response.status === 402 || response.status === 403) {
+      item.state = "membership_required";
+      saveItem(home, item);
+      notifyOnce(
+        home,
+        item,
+        "membership-required",
+        "context graph update is queued until membership access is available",
+        insightFn,
+      );
+      return "pending";
+    }
+    if (!response.ok) {
+      item.state = `submit_http_${response.status}`;
+      saveItem(home, item);
+      return "pending";
+    }
+    const accepted = await response.json().catch(() => ({}));
+    item.user_id = auth.uid;
+    item.replayed = accepted.replayed === true;
+    item.updated_at = new Date(now()).toISOString();
+    if (typeof accepted.n_sessions === "number") {
+      markDelivered(home, item, now);
+      return "done";
+    }
+    if (typeof accepted.job_id !== "string" || !accepted.job_id) {
+      item.state = "invalid_response";
+      saveItem(home, item);
+      return "pending";
+    }
+    item.job_id = accepted.job_id;
+    item.state = "queued";
+    saveItem(home, item);
+  }
+  return pollAcceptedJob(home, item, auth, options, deadline);
+}
+
+// Snapshot exactly the completed Claude transcript. The only accepted source path is Claude's
+// transcript directory; a forged hook payload cannot turn this into an arbitrary local-file upload.
+export function enqueueCompletedSession(payload, options = {}) {
+  const home = options.home || homedir();
+  const now = options.now || Date.now;
+  const transcriptPath = payload?.transcript_path;
+  if (payload?.hook_event_name !== "SessionEnd" || typeof transcriptPath !== "string") return null;
+  if (extname(transcriptPath).toLowerCase() !== ".jsonl") return null;
+  let canonicalPath;
+  try {
+    canonicalPath = realpathSync(transcriptPath);
+    if (!isWithin(realpathSync(join(home, ".claude", "projects")), canonicalPath)) return null;
+  } catch {
+    return null;
+  }
+  let stat;
+  let transcript;
+  try {
+    stat = statSync(canonicalPath);
+    if (!stat.isFile() || stat.size <= 0 || stat.size > MAX_TRANSCRIPT_BYTES) return null;
+    transcript = readFileSync(canonicalPath, "utf8");
+  } catch {
+    return null;
+  }
+  if (!transcript.trim()) return null;
+  const sessionID = cleanSessionID(payload.session_id, canonicalPath);
+  const key = baseKey(sessionID, transcript);
+  ensureOutbox(home);
+  const path = itemPath(home, key);
+  const existing = readJson(path);
+  if (existing?.version === ITEM_VERSION && existsSync(snapshotPath(home, key))) return existing;
+
+  const lockPath = `${path}.lock`;
+  if (!acquireLock(lockPath, now)) return readJson(path);
+  try {
+    const item = {
+      version: ITEM_VERSION,
+      key,
+      generation: 0,
+      session_id: sessionID,
+      source: "claude_code",
+      transcript_path: canonicalPath,
+      transcript_bytes: stat.size,
+      transcript_mtime_ms: stat.mtimeMs,
+      reason: typeof payload.reason === "string" ? payload.reason : "other",
+      state: "queued",
+      job_id: null,
+      user_id: null,
+      created_at: new Date(now()).toISOString(),
+      updated_at: new Date(now()).toISOString(),
+    };
+    writeSnapshotAtomic(snapshotPath(home, key), transcript);
+    writeAtomic(path, item);
+    return item;
+  } finally {
+    remove(lockPath);
+  }
+}
+
+export async function drainIncrementalOutbox(options = {}) {
+  const home = options.home || homedir();
+  const now = options.now || Date.now;
+  const insightFn = options.insightFn || recordInsight;
+  ensureOutbox(home);
+  const lockPath = join(outboxDir(home), DRAIN_LOCK);
+  if (!acquireLock(lockPath, now)) return { locked: true, pending: listItems(home).length };
+  try {
+    const items = listItems(home, options.preferredKey);
+    if (!items.length) return { pending: 0 };
+
+    const identityFn = options.identityFn || identity;
+    const auth = await identityFn();
+    if (!auth?.uid || !auth?.token) return { pending: items.length, authRequired: true };
+    const delivered = readLedger(home);
+    for (const item of items) {
+      if (delivered.keys?.[`${auth.uid}:${item.key}`]) removeItem(home, item);
+    }
+    const pending = listItems(home, options.preferredKey);
+    if (!pending.length) return { pending: 0 };
+    const deadline = now() + (options.workBudgetMs ?? DEFAULT_WORK_BUDGET_MS);
+    const common = {
+      api: options.api || clientAPI(home),
+      fetchFn: options.fetchFn || fetch,
+      insightFn,
+      now,
+      pollIntervalMs: options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS,
+      requestTimeoutMs: options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
+      sleepFn: options.sleepFn || sleep,
+    };
+    let processed = 0;
+    for (const item of pending.slice(0, options.maxItems ?? 3)) {
+      if (item.user_id && item.user_id !== auth.uid) continue;
+      if (item.state === "fallback_required") continue;
+      if (Number.isFinite(item.next_attempt_at) && item.next_attempt_at > now()) continue;
+      if (item.job_id && Number.isFinite(item.next_poll_at) && item.next_poll_at > now()) continue;
+      await submitItem(home, item, auth, common, deadline);
+      processed += 1;
+      if (now() >= deadline) break;
+    }
+    return { processed, pending: listItems(home).length };
+  } finally {
+    remove(lockPath);
+  }
+}
+
+export async function captureAndDrainSession(payload, options = {}) {
+  const item = enqueueCompletedSession(payload, options);
+  if (!item || item.delivered) return { queued: false, delivered: item?.delivered === true };
+  const result = await drainIncrementalOutbox({ ...options, preferredKey: item.key });
+  return { queued: true, key: item.key, ...result };
+}
+
+// Test-only visibility without exporting transcript bodies or mutable paths.
+export function pendingItems(home) {
+  return listItems(home);
+}