skalpel 4.0.28 → 4.0.29

package/catches.mjs

@@ -0,0 +1,413 @@
+#!/usr/bin/env node
+// catches.mjs — the RETROACTIVE first-run "wow": `skalpel catches`.
+//
+// THE PROBLEM (same as verify-shadow): your agent is both the executor AND the reporter of its own
+// work. When it says "done / tests pass / fixed" nothing independent re-checks. verify-shadow closes
+// that loop LIVE, but a brand-new user has no live catches yet — so there is no instant aha on install.
+//
+// THIS closes the aha gap by looking BACKWARD: it scans the user's OWN recent coding transcripts
+// (Claude Code ~/.claude/projects/**/*.jsonl and Codex ~/.codex/{sessions,archived_sessions}) and finds
+// REAL past turns where the agent CLAIMED success while its OWN in-transcript proof — the tool_result of
+// the last test/build/lint it ran before the claim — had ALREADY FAILED at claim time. That is a genuine
+// claim-vs-own-evidence mismatch that is verifiable FROM THE TRANSCRIPT ALONE.
+//
+// HARD RED LINES (why this is safe + instant):
+//   • READ-ONLY. It only READS transcripts already on disk. It NEVER re-runs, spawns, builds, writes,
+//     deploys, or mutates anything. Unlike verify-shadow it does NOT execFile a proof — the pass/fail is
+//     the exit status the transcript ALREADY recorded (Claude's tool_result.is_error / Codex's
+//     "Process exited with code N"). So it is instant and cannot break or slow anything.
+//   • LOCAL ONLY. Imports only node: core + local sibling modules. Opens no socket.
+//   • OPT-IN. User-invoked subcommand; prints and exits. The per-turn hook is untouched (byte-identical).
+//   • NO FABRICATED NUMBERS. Every count is a real tally over real transcripts; every example quotes the
+//     real claim text, the real reconstructed proof command, and the real recorded failure output. If it
+//     finds zero, it SAYS zero honestly — it never invents a catch.
+//
+// REUSE: the detection + classification logic is verify-shadow's, applied to historical turns —
+// detectCompletionClaim (is this a "done" claim?), reconstructProofEntry (the most-recent proof the
+// session actually ran before the claim, + the tool_use id to match its result), and classifyOutcome
+// (PASS / GENUINE_FAIL / HARNESS_ERROR — only a GENUINE_FAIL counts as a lie; a timeout / missing-script
+// / wrong-cwd is harness noise and is NEVER surfaced, so we never cry wolf).
+import { readdirSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { realpathSync } from "node:fs";
+import { tailLines } from "./transcript.mjs";
+import {
+  detectCompletionClaim,
+  reconstructProofEntry,
+  classifyOutcome,
+  extractFailCount,
+} from "./verify-shadow.mjs";
+import { recordInsight } from "./insights.mjs";
+
+const HOME = homedir();
+const CLAUDE_PROJECTS = path.join(HOME, ".claude", "projects");
+const CODEX_DIRS = [
+  path.join(HOME, ".codex", "sessions"),
+  path.join(HOME, ".codex", "archived_sessions"),
+];
+
+// Bounds. We look at RECENT sessions (the aha is about YOUR history, and reading every one of tens of
+// thousands of transcripts would be neither instant nor "recent"). Newest-first by mtime, capped, each
+// read as a bounded tail so a multi-GB transcript can never stall or OOM us.
+const MAX_SESSIONS = 600; // most-recent sessions to scan (across both tools)
+const SESSION_TAIL_BYTES = 4 * 1024 * 1024; // per-file tail — plenty for the last many turns
+const MAX_EXAMPLES = 3; // concrete receipts to print
+
+// ---------- (1) discover recent session files (newest first) ----------
+function walkJsonl(dir, pred, out) {
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return; // dir missing / unreadable — skip (read-only, fail-open)
+  }
+  for (const e of entries) {
+    const p = path.join(dir, e.name);
+    try {
+      if (e.isDirectory()) walkJsonl(p, pred, out);
+      else if (e.isFile() && pred(e.name)) out.push(p);
+    } catch {
+      /* stat/permission hiccup on one entry — skip it */
+    }
+  }
+}
+
+// Return [{ file, tool, mtime }] for the MAX_SESSIONS most-recently-modified transcripts across tools.
+function discoverSessions() {
+  const files = [];
+  const claude = [];
+  walkJsonl(CLAUDE_PROJECTS, (n) => n.endsWith(".jsonl"), claude);
+  for (const f of claude) files.push({ file: f, tool: "claude" });
+  const codex = [];
+  for (const d of CODEX_DIRS) walkJsonl(d, (n) => /^rollout-.*\.jsonl$/.test(n), codex);
+  for (const f of codex) files.push({ file: f, tool: "codex" });
+
+  const withMtime = [];
+  for (const f of files) {
+    try {
+      withMtime.push({ ...f, mtime: statSync(f.file).mtimeMs });
+    } catch {
+      /* vanished between readdir and stat — skip */
+    }
+  }
+  withMtime.sort((a, b) => b.mtime - a.mtime);
+  return withMtime.slice(0, MAX_SESSIONS);
+}
+
+// ---------- (2) load a session as Claude-shaped entries ----------
+// verify-shadow's reconstructProofEntry + claim detection speak the Claude transcript shape
+// (entry.message.content = [{type:'text'|'tool_use'|'tool_result', ...}], entry.cwd). We parse Claude
+// natively and NORMALIZE Codex rollout events into that same shape so ONE code path handles both.
+
+export function loadClaudeEntries(file) {
+  const entries = [];
+  for (const line of tailLines(file, SESSION_TAIL_BYTES)) {
+    try {
+      entries.push(JSON.parse(line));
+    } catch {
+      /* partial/garbage line — skip */
+    }
+  }
+  return entries;
+}
+
+// Codex output wrapper always carries "Process exited with code N" — the recorded exit status, which is
+// exactly the ground truth we need (no re-run). Returns { exitCode:number|null, body:string }.
+function parseCodexOutput(output) {
+  const s = typeof output === "string" ? output : "";
+  const m = s.match(/Process exited with code\s+(-?\d+)/i);
+  const exitCode = m ? parseInt(m[1], 10) : null;
+  // Prefer the text AFTER "Output:" as the evidence body (drops the Command/Chunk/Wall-time header).
+  const oi = s.indexOf("\nOutput:\n");
+  const body = oi >= 0 ? s.slice(oi + "\nOutput:\n".length) : s;
+  return { exitCode, body };
+}
+
+// Normalize one Codex rollout file into Claude-shaped entries. exec_command → a Bash tool_use; its
+// function_call_output → a tool_result carrying the recorded exit code; assistant messages → text.
+export function loadCodexEntries(file) {
+  const entries = [];
+  for (const line of tailLines(file, SESSION_TAIL_BYTES)) {
+    let e;
+    try {
+      e = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (e.type !== "response_item") continue;
+    const p = e.payload || {};
+    if (p.type === "message") {
+      const role = p.role === "assistant" ? "assistant" : "user";
+      const text = Array.isArray(p.content)
+        ? p.content.map((b) => (b && typeof b.text === "string" ? b.text : "")).join("")
+        : "";
+      if (text.trim())
+        entries.push({ type: role, message: { role, content: [{ type: "text", text }] } });
+    } else if (p.type === "function_call" && p.name === "exec_command" && p.call_id) {
+      let cmd = "";
+      let workdir = null;
+      try {
+        const a = JSON.parse(p.arguments || "{}");
+        cmd = typeof a.cmd === "string" ? a.cmd : Array.isArray(a.cmd) ? a.cmd.join(" ") : "";
+        workdir = typeof a.workdir === "string" ? a.workdir : null;
+      } catch {
+        /* unparseable args — leave cmd empty (no proof) */
+      }
+      if (cmd)
+        entries.push({
+          cwd: workdir,
+          message: {
+            role: "assistant",
+            content: [{ type: "tool_use", name: "Bash", id: p.call_id, input: { command: cmd } }],
+          },
+        });
+    } else if (p.type === "function_call_output" && p.call_id) {
+      const { exitCode, body } = parseCodexOutput(p.output);
+      entries.push({
+        message: {
+          role: "user",
+          content: [
+            {
+              type: "tool_result",
+              tool_use_id: p.call_id,
+              is_error: typeof exitCode === "number" ? exitCode !== 0 : false,
+              exit_code: exitCode,
+              content: body,
+            },
+          ],
+        },
+      });
+    }
+  }
+  return entries;
+}
+
+// ---------- (3) index the recorded proof results (no re-run) ----------
+function resultTextOf(block) {
+  const c = block?.content;
+  if (typeof c === "string") return c;
+  if (Array.isArray(c)) return c.map((x) => (typeof x === "string" ? x : x?.text || "")).join(" ");
+  return "";
+}
+
+// tool_use_id -> { isError, exitCode, text } for every tool_result in the session.
+function indexResults(entries) {
+  const idx = new Map();
+  for (const e of entries) {
+    const c = e?.message?.content;
+    if (!Array.isArray(c)) continue;
+    for (const b of c) {
+      if (b && b.type === "tool_result" && b.tool_use_id) {
+        idx.set(b.tool_use_id, {
+          isError: b.is_error === true,
+          exitCode: typeof b.exit_code === "number" ? b.exit_code : null,
+          text: resultTextOf(b),
+        });
+      }
+    }
+  }
+  return idx;
+}
+
+// Reuse verify-shadow's classifyOutcome by synthesizing the `err` object it expects from the RECORDED
+// result (we never ran the process, so there is no live err). This keeps HARNESS_ERROR vs GENUINE_FAIL
+// identical to the live shadow: a timeout / command-not-found / missing-script is harness noise and is
+// NEVER counted as a lie.
+function classifyRecorded({ isError, exitCode, text }) {
+  if (exitCode === 0) return "PASS";
+  if (exitCode == null && !isError) return "PASS"; // no failure signal recorded → treat as pass (never invent)
+  const ev = String(text || "");
+  let err;
+  if (/\btimed?\s*out\b|timeout after|\bSIGKILL\b|\bSIGTERM\b/i.test(ev)) {
+    err = { killed: true, signal: "SIGTERM" }; // classifyOutcome → HARNESS_ERROR
+  } else if (typeof exitCode === "number" && exitCode !== 0) {
+    err = { code: exitCode }; // real recorded non-zero exit (127/126 → HARNESS_ERROR in classifyOutcome)
+  } else {
+    err = { code: 1 }; // Claude is_error with no numeric code → generic non-zero
+  }
+  return classifyOutcome(err, ev);
+}
+
+// ---------- (4) scan one session for genuine claim-vs-proof mismatches ----------
+// For each assistant turn carrying a completion claim, reconstruct the most-recent proof the session ran
+// BEFORE that claim, look up that proof's RECORDED result, and classify it. A GENUINE_FAIL = a caught
+// "done" over its own already-failing test. Dedup by proof tool_use id so repeated "done"s about the
+// same failing run count once.
+function assistantTextOf(entry) {
+  if (entry?.type !== "assistant" && entry?.message?.role !== "assistant") return "";
+  const c = entry?.message?.content;
+  if (typeof c === "string") return c;
+  if (Array.isArray(c))
+    return c
+      .filter((b) => b && b.type === "text")
+      .map((b) => b.text || "")
+      .join(" ")
+      .trim();
+  return "";
+}
+
+export function scanSession(entries, meta = {}) {
+  const catches = [];
+  if (!Array.isArray(entries) || !entries.length) return catches;
+  const results = indexResults(entries);
+  const seenProof = new Set(); // proof tool_use ids already reported in THIS session
+
+  for (let i = 0; i < entries.length; i++) {
+    const text = assistantTextOf(entries[i]);
+    if (!text) continue;
+    const { claim, text: claimText } = detectCompletionClaim(text);
+    if (!claim) continue;
+
+    // The most-recent proof the session actually ran up to and including this claim turn.
+    const found = reconstructProofEntry(entries.slice(0, i + 1), meta.cwd || null);
+    if (!found || !found.toolUseId) continue;
+    if (seenProof.has(found.toolUseId)) continue;
+
+    const res = results.get(found.toolUseId);
+    if (!res) continue; // proof's result not recorded in the scanned window — can't verify, skip honestly
+
+    const outcome = classifyRecorded(res);
+    if (outcome !== "GENUINE_FAIL") continue; // PASS / HARNESS_ERROR are never a caught lie
+
+    seenProof.add(found.toolUseId);
+    const evidence = String(res.text || "")
+      .replace(/\s+$/, "")
+      .slice(-240);
+    catches.push({
+      tool: meta.tool || "claude",
+      session: meta.session || null,
+      claim_text: claimText,
+      proof_command: [found.proof.cmd, ...found.proof.args].join(" ").slice(0, 160),
+      exit_code: res.exitCode,
+      fail_count: extractFailCount(evidence),
+      evidence,
+    });
+  }
+  return catches;
+}
+
+// ---------- (5) run the scan across recent sessions ----------
+export function runScan() {
+  const sessions = discoverSessions();
+  let scanned = 0;
+  const allCatches = [];
+  for (const s of sessions) {
+    let entries;
+    try {
+      entries = s.tool === "codex" ? loadCodexEntries(s.file) : loadClaudeEntries(s.file);
+    } catch {
+      continue; // unreadable file — skip (read-only, fail-open)
+    }
+    if (!entries.length) continue;
+    scanned++;
+    const sessionId = path.basename(s.file, ".jsonl");
+    let found;
+    try {
+      found = scanSession(entries, { tool: s.tool, session: sessionId });
+    } catch {
+      continue; // a single malformed session must never abort the whole scan
+    }
+    for (const c of found) allCatches.push(c);
+  }
+  return { scanned, catches: allCatches };
+}
+
+// ---------- (6) render the honest report ----------
+function clip(s, n) {
+  const t = String(s || "")
+    .replace(/\s+/g, " ")
+    .trim();
+  return t.length > n ? t.slice(0, n - 1) + "…" : t;
+}
+
+export function renderReport({ scanned, catches }) {
+  const N = scanned;
+  const K = catches.length;
+  const L = [];
+  L.push("");
+  L.push("  🔬 skalpel — retroactive claim check (read-only scan of your own coding history)");
+  L.push("  " + "─".repeat(72));
+  if (N === 0) {
+    L.push("");
+    L.push("  No Claude Code or Codex transcripts found on this machine yet.");
+    L.push("  As you code, skalpel watches live — run me again anytime.");
+    L.push("");
+    return L.join("\n");
+  }
+  if (K === 0) {
+    // HONEST zero-state — we do NOT invent a catch.
+    L.push("");
+    L.push(`  skalpel scanned ${N} of your recent sessions and found`);
+    L.push(`  0 times your agent said "done" while its own test/build was already failing.`);
+    L.push("");
+    L.push("  No caught lies in your history — clean. I'll watch live from here:");
+    L.push("  every turn your agent claims done, I check its own proof against the claim.");
+    L.push("");
+    return L.join("\n");
+  }
+
+  const cmds = [...new Set(catches.map((c) => c.proof_command.split(/\s+/).slice(0, 3).join(" ")))];
+  const cmdPhrase = cmds.length === 1 ? `\`${cmds[0]}\`` : "its own test/build";
+  L.push("");
+  L.push(`  skalpel scanned ${N} of your recent sessions and found`);
+  L.push(
+    `  ${K} time${K === 1 ? "" : "s"} your agent said "done" while ${cmdPhrase} was already failing.`,
+  );
+  L.push("");
+  L.push("  Every one is real — pulled straight from your transcript, no re-run needed:");
+  L.push("");
+
+  const top = catches.slice(0, MAX_EXAMPLES);
+  top.forEach((c, i) => {
+    const failN = c.fail_count != null ? ` (${c.fail_count} failing)` : "";
+    L.push(`  ${i + 1}. [${c.tool}]  your agent said:`);
+    L.push(`       "${clip(c.claim_text, 72)}"`);
+    L.push(`     but its own \`${clip(c.proof_command, 64)}\` had just FAILED${failN}:`);
+    L.push(`       ${clip(c.evidence, 72)}`);
+    L.push("");
+  });
+  if (K > top.length) L.push(`  …and ${K - top.length} more.`);
+  L.push("  " + "─".repeat(72));
+  L.push("  That's the gap skalpel closes. From here I check every claim live, as you code.");
+  L.push("");
+  return L.join("\n");
+}
+
+// ---------- CLI ----------
+const isMain = (() => {
+  try {
+    return (
+      Boolean(process.argv[1]) &&
+      realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url))
+    );
+  } catch {
+    return false;
+  }
+})();
+
+if (isMain) {
+  try {
+    const result = runScan();
+    process.stdout.write(renderReport(result) + "\n");
+    // MEASURABLE (local only, no network): record the honest fire-frequency for THIS user — how many
+    // recent sessions were scanned and how many genuine catches were found. Never fabricated.
+    try {
+      recordInsight({
+        kind: "catches_scan",
+        display: `retroactive scan: ${result.catches.length} catch(es) across ${result.scanned} sessions`,
+        catches: result.catches.length,
+        sessions: result.scanned,
+      });
+    } catch {
+      /* insight is a bonus */
+    }
+  } catch {
+    // READ-ONLY + fail-open: never throw at the user.
+    process.stdout.write(
+      "\n  🔬 skalpel — could not read your transcript history on this machine.\n\n",
+    );
+  }
+  process.exit(0);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skalpel",
-  "version": "4.0.28",
+  "version": "4.0.29",
   "description": "Behavioral graph for AI-assisted coding — learns how you work and steers Claude Code + Codex in real time.",
   "type": "module",
   "bin": {

package/skalpel-setup.mjs

@@ -61,6 +61,7 @@ const KNOWN_SUBS = new Set([
   "uninstall",
   "autopsy",
   "xray",
+  "catches",
   "__build",
   "__verify-report",
 ]);
@@ -75,6 +76,7 @@ usage:
   skalpel logout                                clear the saved session
   skalpel autopsy                               local, read-only receipt of your verified patterns
   skalpel xray                                  re-run your last session's own proof — did "done" hold up?
+  skalpel catches                               scan your own history for "done" claims over failing tests
   skalpel uninstall [--purge]                   remove the hooks + local data from this machine
 
 options:
@@ -868,6 +870,14 @@ async function main() {
     spawnSync("node", [join(__dir, "autopsy.mjs"), ...argv.slice(1)], { stdio: "inherit" });
     return;
   }
+  // `skalpel catches` — the RETROACTIVE first-run "wow": a LOCAL, READ-ONLY, ZERO-NETWORK scan of the
+  // user's OWN recent Claude Code + Codex transcripts for real past turns where the agent claimed "done"
+  // while its OWN test/build had already failed in-transcript. Never re-runs anything (the pass/fail is
+  // the exit status already recorded), so it is instant + safe; prints an honest summary (0 if none).
+  if (sub === "catches") {
+    spawnSync("node", [join(__dir, "catches.mjs"), ...argv.slice(1)], { stdio: "inherit" });
+    return;
+  }
   // `skalpel __verify-report` — the claim-verification SHADOW instrument: reads the local
   // verify-shadow.log and prints the three numbers (claim fire-rate, proof-reconstruction rate, and the
   // headline MISMATCH rate = how often the agent claimed done but its OWN proof, re-run, failed).

package/verify-shadow.mjs

@@ -425,11 +425,13 @@ function deniedToolUseIds(entries) {
   return denied;
 }
 
-// reconstructProof(entries, cwd) -> { cmd, args, cwd } | null
-// The MOST RECENT test/build/typecheck/lint command the session actually ran. Scans newest→oldest; the
+// reconstructProofEntry(entries, cwd) -> { proof:{cmd,args,cwd}, toolUseId, command } | null
+// The MOST RECENT test/build/typecheck/lint command the session actually ran, WITH the id of the
+// tool_use it came from (so a caller can match it back to its recorded tool_result — the retroactive
+// `skalpel catches` scanner reads that stored result INSTEAD of re-running). Scans newest→oldest; the
 // entry's own recorded cwd is the base (a `cd` inside the command still overrides it). SHARP-3: a
 // command the user DENIED (never ran) is skipped — we only re-run what the session actually executed.
-export function reconstructProof(entries, cwd) {
+export function reconstructProofEntry(entries, cwd) {
   const list = Array.isArray(entries) ? entries : [];
   const denied = deniedToolUseIds(list);
   for (let i = list.length - 1; i >= 0; i--) {
@@ -442,30 +444,41 @@ export function reconstructProof(entries, cwd) {
       const id = cmds[k].id;
       if (id ? denied.has(id) : denied.size > 0) continue;
       const proof = parseCommandForProof(cmds[k].command, cmds[k].cwd || cwd);
-      if (proof) return proof;
+      if (proof) return { proof, toolUseId: id || null, command: cmds[k].command };
     }
   }
   return null;
 }
 
+// reconstructProof(entries, cwd) -> { cmd, args, cwd } | null — thin wrapper over reconstructProofEntry
+// (the live shadow re-runner only needs the proof itself, not the originating tool_use id).
+export function reconstructProof(entries, cwd) {
+  const found = reconstructProofEntry(entries, cwd);
+  return found ? found.proof : null;
+}
+
 // Classify a re-run into PASS / GENUINE_FAIL / HARNESS_ERROR. This is what keeps the money metric HONEST:
 // only a GENUINE_FAIL (the test/build itself ran and reported failure) may count as "the agent lied".
 // A HARNESS_ERROR (timeout, missing binary/script, wrong cwd — the PROOF machinery broke, not the code)
 // must NEVER be shown as a lie. Real-data measurement showed 6/6 raw "mismatches" were harness noise
 // (a 3-min lint timeout, a missing lint script, a wrong-cwd eslint) — counting those would cry wolf and
 // burn the whole point of an honesty product.
-function classifyOutcome(err, evidence) {
+export function classifyOutcome(err, evidence) {
   if (!err) return "PASS";
   if (err.killed || err.signal === "SIGTERM") return "HARNESS_ERROR"; // hit the timeout
   if (err.code === "ENOENT") return "HARNESS_ERROR"; // binary/cwd does not exist
   const exit = typeof err.code === "number" ? err.code : null;
   if (exit === 127 || exit === 126) return "HARNESS_ERROR"; // command not found / not executable
+  // The proof harness/environment is broken, not the work: a missing script, a `command not found`, or a
+  // tool binary/runtime that simply isn't installed (a Playwright browser not downloaded — "Executable
+  // doesn't exist" / "download new browsers"). Surfacing any of these would cry wolf on setup state, so
+  // they are HARNESS_ERROR, never counted as a lie.
   if (
-    /\bno such file or directory\b|\bENOENT\b|\bcommand not found\b|missing script|no [a-z:-]{1,24} script|npm err!.*missing script/i.test(
+    /\bno such file or directory\b|\bENOENT\b|\bcommand not found\b|missing script|no [a-z:-]{1,24} script|npm err!.*missing script|executable doesn'?t exist|download new browsers/i.test(
       String(evidence || ""),
     )
   )
-    return "HARNESS_ERROR"; // the proof harness itself is broken, not the work
+    return "HARNESS_ERROR";
   return "GENUINE_FAIL"; // a real non-zero exit from the test/build/lint itself
 }