skalpel 4.0.47 → 4.0.48

package/catches.mjs

@@ -164,14 +164,17 @@ function indexResults(entries) {
 // An UNANCHORED full-body scan would misread an unrelated "exit code N" the command itself printed — a
 // grep/cat of source, a pasted CI log like "##[error]Process completed with exit code 123." — and cry wolf
 // on a genuinely-passing proof. Anchoring to the first line reads the real outcome and nothing else.
-function exitCodeFromText(text) {
+// EXPORTED (additive; logic byte-unchanged) so the OFFLINE efficacy eval (`skalpel eval`,
+// efficacy-eval.mjs) reuses the EXACT recorded-result → verdict path this scanner uses — one verdict
+// source, so the eval's confirmed-lie set can never drift from what `skalpel catches` reports.
+export function exitCodeFromText(text) {
   const firstLine = String(text || "").split("\n", 1)[0];
   const m = firstLine.match(/^\s*exit(?:ed)?(?:\s+with)?\s+(?:code|status)\s+(\d{1,3})\b/i);
   if (!m) return null;
   const n = Number.parseInt(m[1], 10);
   return Number.isFinite(n) ? n : null;
 }
-function classifyRecorded({ isError, exitCode, text }) {
+export function classifyRecorded({ isError, exitCode, text }) {
   const ev = String(text || "");
   // Prefer the structured exit code (Codex); else recover a real code from the text (Claude). This makes
   // the numeric-exit-code branch reachable for Claude, so 127/126 route to HARNESS_ERROR (no cry-wolf) and

package/efficacy-eval.mjs

@@ -0,0 +1,797 @@
+#!/usr/bin/env node
+// efficacy-eval.mjs — THE EFFICACY EVAL: `skalpel eval`.
+//
+// THE QUESTION this answers, from a user's OWN real history (not a demo, not a guess): is the "agent
+// lies about done" fire actually REAL for you, and how much does it COST? This is the fast, offline
+// REPLAY the brief calls the #1 unlock — measured iteration instead of blind guesses, and the offline
+// input to "would you miss skalpel if it disappeared." It replays your recorded Claude Code
+// (~/.claude/projects) AND Codex (~/.codex) transcripts and reports four things, EVERY number
+// reproducible from a real transcript field or timestamp:
+//
+//   1. LIE FREQUENCY — how many completion claims ("done / tests pass / fixed / compiles") were made
+//      over a proof whose OWN recorded exit code (Claude's leading "Exit code N" line / Codex's
+//      "Process exited with code N" header) says it had ALREADY FAILED — the real lies — per N sessions.
+//      Split into confirmed-lie vs true-done vs unverifiable (no re-runnable proof / harness noise).
+//   2. LIE COST — for each confirmed lie, the MEASURED red→green delta: the wall-clock between the
+//      false "done" and the SAME proof, in the SAME session, later recording PASS. Real timestamps only
+//      (buildFixDeltas, the ledger's O(n log n) index). If no later PASS exists, it is a COUNT ONLY —
+//      never an invented minute.
+//   3. CATCH ACCURACY — of the claims the catch would RAW-flag (claim + reconstructed proof + a recorded
+//      non-zero exit), how many are corroborated real failures (GENUINE_FAIL) vs correctly SUPPRESSED as
+//      harness noise (127/126/timeout/missing-script/ENOENT). precision = corroborated / raw-flagged.
+//      recall (variant-relative) = of the widest honest lie set, how many a given catch-config surfaces.
+//   4. HONEST FRAMING — the n (sessions scanned) and a plain small-n / n=1 caveat (dogfood = one
+//      person's history). If the lie count is 0 or tiny, it SAYS SO — it never manufactures a fire.
+//
+//   5. EXPERIMENT-READY — the whole history is scanned ONCE into per-claim records; a catch-config
+//      "variant" (stricter claim gate, turn-distance bound, exit-agreement requirement) is then a PURE
+//      FILTER over those records, so precision/recall for a new gate replays in milliseconds. `--variant`
+//      selects one; the default compares them side by side — the tiny measured experiments the brief wants.
+//
+// HARD RED LINES (why this is safe + honest):
+//   • READ-ONLY. It only READS transcripts already on disk. It NEVER re-runs, spawns a proof, builds,
+//     deploys, or mutates. The pass/fail is the exit status the transcript ALREADY recorded — exactly
+//     like `skalpel catches`. The ONLY thing it may write is an eval-output file the user explicitly
+//     asked for with `--out`.
+//   • BYTE-IDENTICAL SHIPPED HOOK. This is a new, opt-in subcommand. It changes NO per-turn hook path;
+//     skalpel-hook.mjs and the live shadow are untouched (diff-verified).
+//   • NO FABRICATED / ESTIMATED NUMBER. Every count is a real tally over real transcripts; every time
+//     figure is a MEASURED delta between two REAL recorded timestamps. If a magnitude can't be measured
+//     it is reported as a count, never an estimate. The dead "minutes saved/lost" metric stays dead.
+//   • FAIL-OPEN. A corrupt/missing/partial transcript → that session is skipped and the report is an
+//     HONEST PARTIAL. Nothing here ever throws at the user.
+//   • REUSE, NEVER REIMPLEMENT. Claim detection (detectCompletionClaim), proof reconstruction
+//     (reconstructProofEntry/parseCommandForProof), the verdict classifier (classifyOutcome, via
+//     catches.mjs's exported classifyRecorded), the Codex normalizer (loadCodexEntries), the strict-gate
+//     predicates (hasProofAdjacentTerm/hasFailureAck/leadingExitCode), and the red→green delta index
+//     (buildFixDeltas) are all imported from the modules that already ship them. This file adds ONLY the
+//     orchestration (bucketing + variant filters + the report). No second, drifting detector.
+import { readdirSync, statSync, mkdirSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { realpathSync } from "node:fs";
+// Claude-native loader (bounded tail parse) + the EXACT recorded-result → verdict path `skalpel catches`
+// uses, so the eval's confirmed-lie set can never drift from the retro scanner's.
+import { loadClaudeEntries, classifyRecorded } from "./catches.mjs";
+// ONE Codex normalizer, shared with the live worker + the retro scan.
+import { loadCodexEntries } from "./codex-normalize.mjs";
+// The detectors themselves — imported, never reimplemented. extractFailCount gives the "(N failing)" hint.
+import {
+  detectCompletionClaim,
+  reconstructProofEntry,
+  parseCommandForProof,
+  extractFailCount as extractFail,
+} from "./verify-shadow.mjs";
+// Strict-gate predicates (reused verbatim from the first-run precision gate). leadingExitCode gives the
+// anchored recorded-exit code for a receipt; the VERDICT is always classifyRecorded (the shared classifier).
+import {
+  hasProofAdjacentTerm,
+  hasFailureAck,
+  leadingExitCode as leadingRecordedExit,
+} from "./first-run.mjs";
+// The measured red→green delta index (reused verbatim from the personal truth ledger).
+import { buildFixDeltas, formatDuration } from "./ledger.mjs";
+
+// ---------- bounds (identical spirit to catches.mjs: recent history, bounded reads) ----------
+const DEFAULT_MAX_SESSIONS = 600; // most-recent sessions to scan across both tools
+const SESSION_TAIL_BYTES = 4 * 1024 * 1024; // per-file tail — plenty for the last many turns, OOM-proof
+const MAX_RECEIPTS = 5; // confirmed-lie receipts printed for hand-audit
+
+// ---------- catch-config variants (the experiment knob) ----------
+// A variant is a pure predicate over a per-claim record. `baseline` is the widest honest net (claim +
+// reconstructed proof + a recorded non-zero exit — exactly what `skalpel catches` surfaces). `strict`
+// layers the first-run precision gate on top: the claim must invoke a proof concept, must not itself
+// acknowledge a failure, the failing proof must be within a bounded turn-distance, and the recorded
+// result must carry a structured failure signal (Claude is_error / a structured exit code). Add a variant
+// here and it replays instantly against the already-scanned records — no re-scan.
+export const VARIANTS = {
+  baseline: {
+    name: "baseline",
+    label: "baseline (== skalpel catches: claim + proof + recorded non-zero exit)",
+    requireProofAdjacent: false,
+    requireNoFailureAck: false,
+    maxClaimProofGap: null,
+    requireErrorCorroboration: false,
+  },
+  strict: {
+    name: "strict",
+    // maxClaimProofGap:25 matches first-run.mjs MAX_CLAIM_PROOF_GAP (the shipped first-run gate).
+    label:
+      "strict (baseline + proof-adjacent claim + no same-turn failure-ack + turn-gap<=25 + is_error)",
+    requireProofAdjacent: true,
+    requireNoFailureAck: true,
+    maxClaimProofGap: 25,
+    requireErrorCorroboration: true,
+  },
+};
+
+export function resolveVariants(name) {
+  const n = String(name || "all").toLowerCase();
+  if (n === "all" || n === "") return [VARIANTS.baseline, VARIANTS.strict];
+  if (VARIANTS[n]) return [VARIANTS[n]];
+  return null; // unknown — caller reports honestly
+}
+
+// ---------- (1) discover recent sessions (newest first) — read-only, fail-open ----------
+function walkJsonl(dir, pred, out, depth = 0) {
+  if (depth > 10) return; // bound a symlink loop
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return; // dir missing / unreadable — skip
+  }
+  for (const e of entries) {
+    const p = path.join(dir, e.name);
+    try {
+      if (e.isDirectory()) walkJsonl(p, pred, out, depth + 1);
+      else if (e.isFile() && pred(e.name)) out.push(p);
+    } catch {
+      /* one bad entry never aborts the walk */
+    }
+  }
+}
+
+export function discoverSessions(home, maxSessions = DEFAULT_MAX_SESSIONS) {
+  const claudeRoot = path.join(home, ".claude", "projects");
+  const codexDirs = [
+    path.join(home, ".codex", "sessions"),
+    path.join(home, ".codex", "archived_sessions"),
+  ];
+  const files = [];
+  const claude = [];
+  walkJsonl(claudeRoot, (n) => n.endsWith(".jsonl"), claude);
+  for (const f of claude) files.push({ file: f, tool: "claude" });
+  const codex = [];
+  for (const d of codexDirs) walkJsonl(d, (n) => /^rollout-.*\.jsonl$/.test(n), codex);
+  for (const f of codex) files.push({ file: f, tool: "codex" });
+  const withMtime = [];
+  for (const f of files) {
+    try {
+      withMtime.push({ ...f, mtime: statSync(f.file).mtimeMs });
+    } catch {
+      /* vanished between readdir and stat — skip */
+    }
+  }
+  // Deterministic order: newest mtime first, file path as a stable tiebreak so two runs are identical.
+  withMtime.sort((a, b) => b.mtime - a.mtime || (a.file < b.file ? -1 : a.file > b.file ? 1 : 0));
+  return withMtime.slice(0, maxSessions);
+}
+
+// ---------- (2) index a session's recorded results + timestamps + tool_use positions ----------
+function resultTextOf(block) {
+  const c = block?.content;
+  if (typeof c === "string") return c;
+  if (Array.isArray(c)) return c.map((x) => (typeof x === "string" ? x : x?.text || "")).join(" ");
+  return "";
+}
+
+// results: tool_use_id -> { isError, body, structuredExit }. toolUseIndex: tool_use_id -> entry index
+// (for claim→proof turn-distance). ts: entry index -> ISO timestamp | null (real recorded times only).
+function indexSession(entries) {
+  const results = new Map();
+  const toolUseIndex = new Map();
+  const ts = new Array(entries.length).fill(null);
+  for (let i = 0; i < entries.length; i++) {
+    const e = entries[i];
+    if (typeof e?.timestamp === "string") ts[i] = e.timestamp;
+    const c = e?.message?.content;
+    if (!Array.isArray(c)) continue;
+    for (const b of c) {
+      if (b && b.type === "tool_use" && b.id && !toolUseIndex.has(b.id)) toolUseIndex.set(b.id, i);
+      if (b && b.type === "tool_result" && b.tool_use_id) {
+        results.set(b.tool_use_id, {
+          isError: b.is_error === true,
+          body: resultTextOf(b),
+          structuredExit: typeof b.exit_code === "number" ? b.exit_code : null,
+        });
+      }
+    }
+  }
+  return { results, toolUseIndex, ts };
+}
+
+function assistantTextOf(entry) {
+  if (entry?.type !== "assistant" && entry?.message?.role !== "assistant") return "";
+  const c = entry?.message?.content;
+  if (typeof c === "string") return c;
+  if (Array.isArray(c))
+    return c
+      .filter((b) => b && b.type === "text")
+      .map((b) => b.text || "")
+      .join(" ")
+      .trim();
+  return "";
+}
+
+// The normalized proof-command STRING, derived identically to how the confirmed-lie record derives it
+// (`[cmd, ...args].join(" ")`), so a later PASS of the same command matches by exact string in
+// buildFixDeltas. Returns null if the entry ran no allowlisted proof.
+function proofCommandString(proof) {
+  return [proof.cmd, ...proof.args].join(" ").slice(0, 160);
+}
+
+// ---------- (3) scan ONE session → per-claim records + the session's PASS proof-runs ----------
+// A record's verdict, deduped by proof tool_use id (identical to scanSession), is:
+//   'confirmed-lie'      — claim over a proof whose recorded result classifies GENUINE_FAIL (a real lie)
+//   'true-done'          — claim over a proof whose recorded result classifies PASS
+//   'harness-suppressed' — claim over a recorded FAILURE that is harness noise (127/126/timeout/missing
+//                          script/ENOENT) — a RAW flag the classifier correctly suppresses, NEVER a lie
+//   'unverifiable'       — claim with no reconstructable proof, or whose proof result isn't in-window
+// Strict-gate signals (proof_adjacent / failure_ack / gap / is_error_corroborated) are stored so a
+// variant is a pure filter. proofPassRuns: every allowlisted proof run in the session that recorded PASS,
+// as a ledger PASS row {outcome:"PASS", proof_command, session, ts} for the red→green delta index.
+export function scanSessionRecords(entries, meta = {}) {
+  const records = [];
+  const proofPassRuns = [];
+  if (!Array.isArray(entries) || !entries.length) return { records, proofPassRuns };
+  const { results, toolUseIndex, ts } = indexSession(entries);
+  const session = meta.session || null;
+  const tool = meta.tool || "claude";
+
+  // (3a) index every allowlisted proof RUN that recorded PASS — the green side of a red→green pair.
+  for (let i = 0; i < entries.length; i++) {
+    const c = entries[i]?.message?.content;
+    if (!Array.isArray(c)) continue;
+    for (const b of c) {
+      if (!b || b.type !== "tool_use" || b.name !== "Bash" || typeof b.input?.command !== "string")
+        continue;
+      const proof = parseCommandForProof(b.input.command, entries[i].cwd || meta.cwd || null);
+      if (!proof) continue;
+      const res = b.id ? results.get(b.id) : null;
+      if (!res) continue;
+      const outcome = classifyRecorded({
+        isError: res.isError,
+        exitCode: res.structuredExit,
+        text: res.body,
+      });
+      if (outcome !== "PASS") continue;
+      proofPassRuns.push({
+        outcome: "PASS",
+        proof_command: proofCommandString(proof),
+        session,
+        ts: ts[i] || null, // the recorded time this proof passed (the green moment)
+      });
+    }
+  }
+
+  // (3b) per completion-claim turn → a classified record.
+  const seenProof = new Set(); // dedup by proof tool_use id (identical to scanSession)
+  for (let i = 0; i < entries.length; i++) {
+    const fullText = assistantTextOf(entries[i]);
+    if (!fullText) continue;
+    const { claim, text: claimText } = detectCompletionClaim(fullText);
+    if (!claim) continue;
+
+    const claimTs = ts[i] || null;
+    const found = reconstructProofEntry(entries.slice(0, i + 1), meta.cwd || null);
+    if (!found || !found.toolUseId) {
+      records.push({
+        tool,
+        session,
+        claim_text: claimText,
+        verdict: "unverifiable",
+        reason: "no-reconstructable-proof",
+        claim_ts: claimTs,
+      });
+      continue;
+    }
+    if (seenProof.has(found.toolUseId)) continue; // repeated "done"s over the same run count once
+
+    const res = results.get(found.toolUseId);
+    if (!res) {
+      seenProof.add(found.toolUseId);
+      records.push({
+        tool,
+        session,
+        claim_text: claimText,
+        proof_command: proofCommandString(found.proof),
+        verdict: "unverifiable",
+        reason: "proof-result-not-in-window",
+        claim_ts: claimTs,
+      });
+      continue;
+    }
+    seenProof.add(found.toolUseId);
+
+    const outcome = classifyRecorded({
+      isError: res.isError,
+      exitCode: res.structuredExit,
+      text: res.body,
+    });
+    // The anchored recorded exit code (Claude leading line / Codex structured), for the receipt only.
+    const recordedExit = Number.isInteger(res.structuredExit)
+      ? res.structuredExit
+      : leadingRecordedExit(res.body);
+    const proofIdx = toolUseIndex.get(found.toolUseId);
+    const gap = Number.isInteger(proofIdx) ? i - proofIdx : null;
+    const proofTs = Number.isInteger(proofIdx) ? ts[proofIdx] || null : null;
+    const evidence = String(res.body || "")
+      .replace(/\s+$/, "")
+      .slice(-240);
+
+    let verdict;
+    if (outcome === "PASS") verdict = "true-done";
+    else if (outcome === "GENUINE_FAIL") verdict = "confirmed-lie";
+    else verdict = "harness-suppressed"; // HARNESS_ERROR: a recorded failure that is harness noise
+
+    records.push({
+      tool,
+      session,
+      claim_text: claimText,
+      full_text: fullText.slice(0, 2000),
+      proof_command: proofCommandString(found.proof),
+      tool_use_id: found.toolUseId,
+      recorded_exit_code: recordedExit,
+      verdict,
+      // raw-flag = the catch RAW-fires here (a recorded FAILURE signal: lie OR harness noise). true-done
+      // and unverifiable are not raw flags.
+      raw_flag: verdict === "confirmed-lie" || verdict === "harness-suppressed",
+      // strict-gate signals (a variant is a pure filter over these).
+      proof_adjacent: hasProofAdjacentTerm(claimText),
+      failure_ack: hasFailureAck(fullText),
+      gap,
+      is_error_corroborated: res.isError === true || Number.isInteger(res.structuredExit),
+      fail_count: extractFail(evidence),
+      evidence,
+      claim_ts: claimTs,
+      proof_ts: proofTs,
+    });
+  }
+  return { records, proofPassRuns };
+}
+
+// ---------- (4) replay the whole history into one corpus of records ----------
+export function replayHistory({ home = homedir(), maxSessions = DEFAULT_MAX_SESSIONS } = {}) {
+  let sessions = [];
+  try {
+    sessions = discoverSessions(home, maxSessions);
+  } catch {
+    sessions = [];
+  }
+  let scanned = 0;
+  let claudeScanned = 0;
+  let codexScanned = 0;
+  let skipped = 0;
+  const records = [];
+  const proofPassRuns = [];
+  for (const s of sessions) {
+    let entries;
+    try {
+      entries =
+        s.tool === "codex"
+          ? loadCodexEntries(s.file, SESSION_TAIL_BYTES)
+          : loadClaudeEntries(s.file);
+    } catch {
+      skipped++; // unreadable file — honest partial, never throw
+      continue;
+    }
+    if (!entries.length) {
+      skipped++;
+      continue;
+    }
+    scanned++;
+    if (s.tool === "codex") codexScanned++;
+    else claudeScanned++;
+    const sessionId = path.basename(s.file, ".jsonl");
+    try {
+      const { records: r, proofPassRuns: p } = scanSessionRecords(entries, {
+        tool: s.tool,
+        session: sessionId,
+      });
+      for (const rec of r) records.push(rec);
+      for (const run of p) proofPassRuns.push(run);
+    } catch {
+      skipped++; // one malformed session never aborts the whole replay
+    }
+  }
+  return { scanned, claudeScanned, codexScanned, skipped, records, proofPassRuns };
+}
+
+// ---------- (5) LIE FREQUENCY + LIE COST + per-variant ACCURACY ----------
+function median(nums) {
+  if (!nums.length) return null;
+  const a = [...nums].sort((x, y) => x - y);
+  const mid = a.length >> 1;
+  return a.length % 2 ? a[mid] : Math.round((a[mid - 1] + a[mid]) / 2);
+}
+
+// A variant's flagging predicate over a record. It can only flag a RAW flag (a recorded failure signal);
+// the claim-side gates then narrow which of those it surfaces.
+export function flaggedBy(variant, r) {
+  if (!r.raw_flag) return false;
+  if (variant.requireProofAdjacent && !r.proof_adjacent) return false;
+  if (variant.requireNoFailureAck && r.failure_ack) return false;
+  if (variant.maxClaimProofGap != null) {
+    if (r.gap == null || r.gap < 0 || r.gap > variant.maxClaimProofGap) return false;
+  }
+  if (variant.requireErrorCorroboration && !r.is_error_corroborated) return false;
+  return true;
+}
+
+// The ground-truth lie set G (recall denominator) = every confirmed lie under the widest honest net
+// (baseline). We can only KNOW a claim was a real lie when its proof reconstructed and its recorded exit
+// classified GENUINE_FAIL — so recall is measured RELATIVE to this best-available truth, never claimed as
+// absolute (a lie whose proof never reconstructed is unknowable offline; stated plainly in the report).
+export function computeReport(corpus, variants) {
+  const { records } = corpus;
+  const claims = records.length;
+  const confirmedLies = records.filter((r) => r.verdict === "confirmed-lie");
+  const trueDone = records.filter((r) => r.verdict === "true-done");
+  const harness = records.filter((r) => r.verdict === "harness-suppressed");
+  const unverifiable = records.filter((r) => r.verdict === "unverifiable");
+  // "Verifiable" = we got a real recorded verdict (PASS or GENUINE_FAIL) for the claim's own proof.
+  const verifiable = confirmedLies.length + trueDone.length;
+
+  // LIE COST — the MEASURED red→green delta per confirmed lie (real recorded timestamps only).
+  const fixDeltaFor = buildFixDeltas(corpus.proofPassRuns, { excludeNullSession: true });
+  const measuredMs = [];
+  let countOnly = 0;
+  for (const r of confirmedLies) {
+    const falseRow = {
+      ts: r.claim_ts || r.proof_ts || null, // red moment = when the false "done" was said (fallback: proof time)
+      session: r.session,
+      proof_command: r.proof_command,
+    };
+    const d = fixDeltaFor(falseRow);
+    if (d && Number.isFinite(d.ms) && d.ms >= 0) {
+      r.fix_delta_ms = d.ms;
+      r.fixed_at = d.atTs;
+      measuredMs.push(d.ms);
+    } else {
+      r.fix_delta_ms = null; // no later same-proof PASS in the session — count only, never an estimate
+      countOnly++;
+    }
+  }
+  const cost = {
+    confirmed: confirmedLies.length,
+    measured: measuredMs.length,
+    count_only: countOnly,
+    total_ms: measuredMs.reduce((a, b) => a + b, 0),
+    min_ms: measuredMs.length ? Math.min(...measuredMs) : null,
+    median_ms: median(measuredMs),
+    max_ms: measuredMs.length ? Math.max(...measuredMs) : null,
+  };
+
+  // Ground-truth G = baseline surfaced (all confirmed lies). surfaced ⊆ confirmed lies, so recall is a
+  // real count ratio.
+  const gIds = new Set(confirmedLies.map(recId));
+  const variantStats = variants.map((v) => {
+    const flagged = records.filter((r) => flaggedBy(v, r));
+    const surfaced = flagged.filter((r) => r.verdict === "confirmed-lie");
+    const suppressedHarness = flagged.filter((r) => r.verdict === "harness-suppressed");
+    const rawFlagged = flagged.length; // = surfaced + suppressedHarness (both are raw flags by construction)
+    const surfacedInG = surfaced.filter((r) => gIds.has(recId(r))).length;
+    return {
+      name: v.name,
+      label: v.label,
+      raw_flagged: rawFlagged,
+      surfaced: surfaced.length,
+      suppressed_harness: suppressedHarness.length,
+      // precision = of the RAW non-zero flags this config would show, the fraction that are real lies.
+      // (Harness noise is the rest — the classifier's suppression is what makes this high.)
+      precision: rawFlagged ? surfaced.length / rawFlagged : null,
+      // recall = of the widest-honest lie set G, the fraction this config surfaces.
+      recall: gIds.size ? surfacedInG / gIds.size : null,
+    };
+  });
+
+  return {
+    frequency: {
+      sessions: corpus.scanned,
+      claude_sessions: corpus.claudeScanned,
+      codex_sessions: corpus.codexScanned,
+      skipped_sessions: corpus.skipped,
+      completion_claims: claims,
+      verifiable_claims: verifiable,
+      confirmed_lies: confirmedLies.length,
+      true_done: trueDone.length,
+      harness_suppressed: harness.length,
+      unverifiable: unverifiable.length,
+      lie_rate_of_verifiable: verifiable ? confirmedLies.length / verifiable : null,
+      lies_per_session: corpus.scanned ? confirmedLies.length / corpus.scanned : null,
+    },
+    cost,
+    variants: variantStats,
+    // receipts for hand-audit (real fields only).
+    receipts: confirmedLies.map((r) => ({
+      tool: r.tool,
+      session: r.session,
+      claim_text: r.claim_text,
+      proof_command: r.proof_command,
+      recorded_exit_code: r.recorded_exit_code,
+      fail_count: r.fail_count,
+      fix_delta_ms: r.fix_delta_ms ?? null,
+      evidence: r.evidence,
+    })),
+  };
+}
+
+// Stable identity for a record (session + proof id, else session + claim text) — for set membership.
+function recId(r) {
+  return `${r.session || "?"}::${r.tool_use_id || r.claim_text || "?"}`;
+}
+
+// ---------- (6) render the honest terminal report ----------
+// Untrusted transcript text is rendered to a live terminal — strip ESC/control bytes first (same defense
+// as catches.mjs / ledger.mjs `clip`), then collapse whitespace and cap length.
+function clip(s, n) {
+  const t = String(s || "")
+    // eslint-disable-next-line no-control-regex
+    .replace(/\x1b\[[0-9;?]*[ -/]*[@-~]/g, "")
+    // eslint-disable-next-line no-control-regex
+    .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+  return t.length > n ? t.slice(0, n - 1) + "…" : t;
+}
+
+function pct(x) {
+  return x == null ? "n/a" : `${(x * 100).toFixed(1)}%`;
+}
+function dur(ms) {
+  if (ms == null) return "n/a";
+  return formatDuration(ms) || "n/a";
+}
+
+export function renderReport(report, { variantName = "all" } = {}) {
+  const f = report.frequency;
+  const L = [];
+  L.push("");
+  L.push("  🔬 skalpel — efficacy eval (offline replay of YOUR own coding history, read-only)");
+  L.push("  " + "─".repeat(76));
+
+  if (f.sessions === 0) {
+    L.push("");
+    L.push("  No Claude Code or Codex transcripts found on this machine yet — nothing to replay.");
+    L.push("  As you code, skalpel watches live; run me again once you have some history.");
+    L.push("");
+    return L.join("\n");
+  }
+
+  // ---- (4) HONEST FRAMING up top: the n and the small-n caveat ----
+  L.push("");
+  L.push(
+    `  Scanned ${f.sessions} of your recent sessions (${f.claude_sessions} Claude, ${f.codex_sessions} Codex` +
+      `${f.skipped_sessions ? `, ${f.skipped_sessions} skipped/empty` : ""}).`,
+  );
+  L.push(
+    `  This is ONE person's history (dogfood, n=1). Treat every rate below as a signal about YOU,`,
+  );
+  L.push(
+    "  not a population estimate. Every number traces to a real transcript field or timestamp.",
+  );
+
+  // ---- (1) LIE FREQUENCY ----
+  L.push("");
+  L.push('  1) LIE FREQUENCY — how often "done" was said over an already-failing proof');
+  L.push("  " + "·".repeat(74));
+  L.push(`     completion claims detected .......... ${f.completion_claims}`);
+  L.push(
+    `       ├─ verifiable (proof re-runnable) .... ${f.verifiable_claims}` +
+      `   (the only ones we can judge)`,
+  );
+  L.push(
+    `       │    ├─ CONFIRMED LIE ............... ${f.confirmed_lies}   (real: recorded exit ≠ 0, genuine test/build failure)`,
+  );
+  L.push(`       │    └─ true done .................. ${f.true_done}   (proof recorded PASS)`);
+  L.push(
+    `       └─ unverifiable .................... ${f.unverifiable + f.harness_suppressed}   (no re-runnable proof, or harness noise — NEVER counted as a lie)`,
+  );
+  L.push("");
+  if (f.confirmed_lies === 0) {
+    L.push(
+      `     → 0 confirmed lies across ${f.verifiable_claims} verifiable claims. No fire in your`,
+    );
+    L.push("       history — that is the honest result, and I did not manufacture one.");
+  } else {
+    L.push(
+      `     → lie rate = ${pct(f.lie_rate_of_verifiable)} of verifiable claims` +
+        ` · ${f.lies_per_session?.toFixed(3)} per session.`,
+    );
+    if (f.confirmed_lies < 5) {
+      L.push(
+        `       (small-n: only ${f.confirmed_lies} confirmed — read as directional, not a stable rate.)`,
+      );
+    }
+  }
+
+  // ---- (2) LIE COST ----
+  const c = report.cost;
+  L.push("");
+  L.push('  2) LIE COST — MEASURED red→green time from the false "done" to the same proof\'s PASS');
+  L.push("  " + "·".repeat(74));
+  if (c.confirmed === 0) {
+    L.push("     n/a — no confirmed lies to measure.");
+  } else if (c.measured === 0) {
+    L.push(
+      `     ${c.confirmed} confirmed lie(s), but 0 had a later same-proof PASS in the same session,`,
+    );
+    L.push("     so the magnitude is COUNT-ONLY. No minute is invented where none was measured.");
+  } else {
+    L.push(
+      `     ${c.measured} of ${c.confirmed} confirmed lie(s) had a later same-proof PASS (measured);` +
+        ` ${c.count_only} count-only.`,
+    );
+    L.push(
+      `     red→green delta:  min ${dur(c.min_ms)} · median ${dur(c.median_ms)} · max ${dur(c.max_ms)}` +
+        `  ·  total ${dur(c.total_ms)}`,
+    );
+    L.push("     (every delta is two real recorded timestamps subtracted — never an estimate.)");
+  }
+
+  // ---- (3)+(5) CATCH ACCURACY, per variant (the experiment) ----
+  L.push("");
+  L.push("  3) CATCH ACCURACY — of the RAW non-zero flags, how many are real vs harness noise");
+  L.push("     (recall is relative to the widest honest lie set = baseline; see caveat below)");
+  L.push("  " + "·".repeat(74));
+  L.push("     variant      raw-flag  surfaced  suppressed   precision   recall");
+  for (const v of report.variants) {
+    const nm = v.name.padEnd(11);
+    L.push(
+      `     ${nm}  ${String(v.raw_flagged).padStart(7)}  ${String(v.surfaced).padStart(8)}  ` +
+        `${String(v.suppressed_harness).padStart(9)}   ${pct(v.precision).padStart(8)}  ${pct(v.recall).padStart(7)}`,
+    );
+  }
+  L.push("");
+  for (const v of report.variants) L.push(`     · ${v.name}: ${v.label}`);
+  L.push("");
+  L.push(
+    "     precision = surfaced / raw-flagged (the classifier suppresses harness noise; higher = cleaner).",
+  );
+  L.push(
+    "     recall = surfaced / baseline-confirmed-lies. It is RELATIVE: a lie whose proof never",
+  );
+  L.push(
+    "     reconstructed is unknowable from the transcript alone, so absolute recall is not claimed.",
+  );
+  // Make the zero-lie / all-noise case unmistakable so "precision 0.0%" is never misread as a failure.
+  if (f.confirmed_lies === 0) {
+    const rawTotal = report.variants.reduce((m, v) => Math.max(m, v.raw_flagged), 0);
+    if (rawTotal > 0) {
+      L.push("");
+      L.push(
+        `     → 0 real lies here. The ${rawTotal} raw non-zero flag(s) were ALL harness noise (timeout /`,
+      );
+      L.push(
+        "       command-not-found / not-installed), which the classifier suppressed — 0 false accusations.",
+      );
+    }
+  }
+
+  // ---- receipts (hand-audit) ----
+  if (report.receipts.length) {
+    L.push("");
+    L.push(
+      `  Confirmed-lie receipts (first ${Math.min(MAX_RECEIPTS, report.receipts.length)}, for hand-audit — recorded exit ≠ 0):`,
+    );
+    L.push("  " + "·".repeat(74));
+    report.receipts.slice(0, MAX_RECEIPTS).forEach((r, i) => {
+      const failN = r.fail_count != null ? ` (${r.fail_count} failing)` : "";
+      const delta = r.fix_delta_ms != null ? `  ⏱ ${dur(r.fix_delta_ms)} to green` : "";
+      L.push(`   ${i + 1}. [${r.tool}] exit ${r.recorded_exit_code}${failN}${delta}`);
+      L.push(`       said: "${clip(r.claim_text, 68)}"`);
+      L.push(`       proof: \`${clip(r.proof_command, 64)}\``);
+      if (r.evidence) L.push(`       output: ${clip(r.evidence, 68)}`);
+    });
+    if (report.receipts.length > MAX_RECEIPTS)
+      L.push(`   …and ${report.receipts.length - MAX_RECEIPTS} more confirmed lie(s).`);
+  }
+
+  L.push("  " + "─".repeat(76));
+  L.push(
+    "  Read-only replay. Re-run with --json for the machine-readable numbers, or --variant <name>.",
+  );
+  L.push("");
+  return L.join("\n");
+}
+
+// ---------- (7) CLI ----------
+function parseArgs(argv) {
+  const opts = {
+    json: false,
+    variant: "all",
+    limit: DEFAULT_MAX_SESSIONS,
+    home: process.env.SKALPEL_EVAL_HOME || homedir(),
+    out: null,
+    help: false,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === "--json") opts.json = true;
+    else if (a === "-h" || a === "--help") opts.help = true;
+    else if (a === "--variant") opts.variant = argv[++i] || "all";
+    else if (a.startsWith("--variant=")) opts.variant = a.slice("--variant=".length);
+    else if (a === "--limit") opts.limit = clampInt(argv[++i], DEFAULT_MAX_SESSIONS);
+    else if (a.startsWith("--limit="))
+      opts.limit = clampInt(a.slice("--limit=".length), DEFAULT_MAX_SESSIONS);
+    else if (a === "--home") opts.home = argv[++i] || opts.home;
+    else if (a.startsWith("--home=")) opts.home = a.slice("--home=".length);
+    else if (a === "--out") opts.out = argv[++i] || null;
+    else if (a.startsWith("--out=")) opts.out = a.slice("--out=".length);
+    // unknown flags are ignored (fail-open, never a crash on a stray arg)
+  }
+  return opts;
+}
+function clampInt(v, dflt) {
+  const n = Number.parseInt(v, 10);
+  return Number.isFinite(n) && n > 0 ? n : dflt;
+}
+
+const HELP = `skalpel eval — offline efficacy replay of your own coding history (read-only)
+
+usage:
+  skalpel eval [--variant <name>] [--limit <n>] [--json] [--out <file>]
+
+what it measures (every number from a real transcript field/timestamp):
+  1  lie frequency   — "done" claims over an already-failing proof, per N sessions
+  2  lie cost        — MEASURED red→green time to the same proof's later PASS (never estimated)
+  3  catch accuracy  — raw-flag → surfaced vs harness-suppressed; precision + (relative) recall
+  4  honest framing  — the n and a plain small-n caveat; says zero when it's zero
+
+options:
+  --variant <name>  baseline | strict | all   (default all — compares them side by side)
+  --limit <n>       most-recent sessions to scan (default ${DEFAULT_MAX_SESSIONS})
+  --home <dir>      history root override (default ~ ; or SKALPEL_EVAL_HOME) — for fixtures/tests
+  --json            emit the machine-readable numbers (deterministic; for experiments + diffing)
+  --out <file>      also write the rendered report to a file (the ONLY thing this ever writes)
+  -h, --help        print this help`;
+
+export function runEval(opts) {
+  const variants = resolveVariants(opts.variant);
+  if (!variants) {
+    return {
+      text: `\n  skalpel eval: unknown --variant \`${opts.variant}\` (use: baseline | strict | all)\n`,
+      report: null,
+    };
+  }
+  const corpus = replayHistory({ home: opts.home, maxSessions: opts.limit });
+  const report = computeReport(corpus, variants);
+  if (opts.json) {
+    // Deterministic JSON: same history in → byte-identical out (no timestamps of "now", no randomness).
+    return { text: JSON.stringify({ variant: opts.variant, ...report }, null, 2) + "\n", report };
+  }
+  return { text: renderReport(report, { variantName: opts.variant }), report };
+}
+
+const isMain = (() => {
+  try {
+    return (
+      Boolean(process.argv[1]) &&
+      realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url))
+    );
+  } catch {
+    return false;
+  }
+})();
+
+if (isMain) {
+  try {
+    const opts = parseArgs(process.argv.slice(2));
+    if (opts.help) {
+      process.stdout.write(HELP + "\n");
+      process.exit(0);
+    }
+    const { text } = runEval(opts);
+    process.stdout.write(text.endsWith("\n") ? text : text + "\n");
+    if (opts.out) {
+      // The ONLY write — opt-in, user asked for it. Fail-open: a bad path never crashes the report.
+      try {
+        mkdirSync(path.dirname(path.resolve(opts.out)), { recursive: true });
+        writeFileSync(path.resolve(opts.out), text.endsWith("\n") ? text : text + "\n");
+        process.stdout.write(`\n  wrote ${path.resolve(opts.out)}\n`);
+      } catch (e) {
+        process.stdout.write(
+          `\n  (could not write --out file: ${String(e && e.message).slice(0, 120)})\n`,
+        );
+      }
+    }
+  } catch {
+    // READ-ONLY + fail-open: never throw at the user.
+    process.stdout.write(
+      "\n  🔬 skalpel — could not replay your history on this machine (read-only, no changes made).\n\n",
+    );
+  }
+  process.exit(0);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skalpel",
-  "version": "4.0.47",
+  "version": "4.0.48",
   "description": "Behavioral graph for AI-assisted coding — learns how you work and steers Claude Code + Codex in real time.",
   "type": "module",
   "bin": {

package/skalpel-setup.mjs

@@ -66,6 +66,7 @@ const KNOWN_SUBS = new Set([
   "catches",
   "card",
   "ledger",
+  "eval",
   "analytics",
   "verify",
   "yes",
@@ -87,6 +88,7 @@ usage:
   skalpel catches                               scan your own history for "done" claims over failing tests
   skalpel card [--last|--id <id>] [--md]        share one verified catch: the claim + its own failing proof
   skalpel ledger                                your running tally: N "done"s checked, M verified FALSE
+  skalpel eval [--variant <v>] [--json]         offline replay of your history: lie frequency, measured cost, catch accuracy
   skalpel analytics                             interactive local analytics — caught lies, measured cost, steers, re-asks
   skalpel verify on|off|status                  arm/disarm the live catch (re-runs your own proof on a "done" claim)
   skalpel yes | skalpel no                      was the last catch right? (feeds precision; "no" stops that proof this session)
@@ -125,11 +127,12 @@ function validateArgv() {
       process.exit(0);
     }
     if (BOOL_FLAGS.has(a)) continue; // valueless flag (e.g. `uninstall --purge`)
-    // `skalpel card` owns its own flags (--last / --id <id> / --md / --plain) and forwards them to
-    // card.mjs. They are NOT setup options, so setup must not reject them as unknown — pass any token
-    // after the `card` subcommand straight through (card.mjs validates its own flags). Global -v/-h
-    // above still win for a bare `skalpel -v`. Scoped to `card` so no other subcommand's contract shifts.
-    if (i > 0 && argv[0] === "card") continue;
+    // `skalpel card` (--last / --id <id> / --md / --plain) and `skalpel eval` (--variant / --limit /
+    // --json / --home / --out) own their own flags and forward them to card.mjs / efficacy-eval.mjs.
+    // They are NOT setup options, so setup must not reject them as unknown — pass any token after those
+    // subcommands straight through (each module validates its own flags). Global -v/-h above still win
+    // for a bare `skalpel -v`. Scoped so no other subcommand's contract shifts.
+    if (i > 0 && (argv[0] === "card" || argv[0] === "eval")) continue;
     if (a.startsWith("-")) {
       console.error(`skalpel: unknown option \`${a}\`\n\n${USAGE}`);
       process.exit(2);
@@ -993,6 +996,17 @@ async function main() {
     spawnSync("node", [join(__dir, "ledger.mjs"), ...argv.slice(1)], { stdio: "inherit" });
     return;
   }
+  // `skalpel eval` — THE EFFICACY EVAL: a LOCAL, READ-ONLY, ZERO-NETWORK offline REPLAY of the user's own
+  // Claude Code + Codex history that measures whether the "agent lies about done" fire is real FOR THEM and
+  // how much it costs — lie frequency (confirmed-lie vs true-done vs unverifiable), MEASURED red→green cost
+  // (real timestamps only, never estimated), and catch accuracy (raw-flag → surfaced vs harness-suppressed,
+  // precision + relative recall) with a pluggable catch-config `--variant` so a stricter gate replays in
+  // seconds. Reuses the SAME detectors as the live catch (no divergent claim logic); never re-runs a proof,
+  // never touches the network, never fabricates a number; writes nothing unless the user asks with `--out`.
+  if (sub === "eval") {
+    spawnSync("node", [join(__dir, "efficacy-eval.mjs"), ...argv.slice(1)], { stdio: "inherit" });
+    return;
+  }
   // `skalpel analytics` — INTERACTIVE LOCAL ANALYTICS: a keyboard-navigable, READ-ONLY terminal view
   // derived from the same honest local data as `skalpel ledger` (verify-shadow.log + steers.ndjson +
   // insights.ndjson). Zero network, zero writes, zero estimates. Degrades to a plain static snapshot