skalpel 4.0.48 → 4.0.49

package/efficacy-eval.mjs

@@ -72,12 +72,28 @@ import {
 } from "./first-run.mjs";
 // The measured red→green delta index (reused verbatim from the personal truth ledger).
 import { buildFixDeltas, formatDuration } from "./ledger.mjs";
+// The ONE canonical definition of "a genuine human user turn" (mirrors the server's
+// engine._user_turn_count) — reused so the re-ask analysis can never drift from how sessions are counted.
+import { isGenuineUserTurn } from "./session-turns.mjs";
 
 // ---------- bounds (identical spirit to catches.mjs: recent history, bounded reads) ----------
 const DEFAULT_MAX_SESSIONS = 600; // most-recent sessions to scan across both tools
 const SESSION_TAIL_BYTES = 4 * 1024 * 1024; // per-file tail — plenty for the last many turns, OOM-proof
 const MAX_RECEIPTS = 5; // confirmed-lie receipts printed for hand-audit
 
+// ---------- RE-ASK / CORRECTION FIRE (the semantic-lie proxy the test-catch cannot see) ----------
+// The test-catch measures ONE thing: "done" over a proof whose OWN recorded exit says it failed. Over
+// 26k real sessions that found ~0 confirmed lies — the agent rarely lies about a RE-RUNNABLE test. The
+// fire a dev actually burns time on is SEMANTIC: the agent says "done / handles X", the tests pass, but
+// the code is wrong — invisible to any re-run of the same proof. The only offline PROXY for that fire is
+// the dev catching it themselves: right after a "done" claim, does the NEXT human turn push back
+// ("no / still broken / that doesn't work / try again / also handle X")? That is this section. It is a
+// KEYWORD HEURISTIC over real turns, NOT ground truth — every limit is stated in the report.
+const REASK_MAX_ASSISTANT_TURNS = 6; // window: the correcting user turn must be within this many assistant
+// natural-language turns of the claim. Beyond it, the user is reacting to a much-evolved state, not this
+// claim — so attribution is too weak to count (conservative: precision over recall).
+const MAX_REASK_SAMPLES = 12; // flagged corrections printed for hand-audit
+
 // ---------- catch-config variants (the experiment knob) ----------
 // A variant is a pure predicate over a per-claim record. `baseline` is the widest honest net (claim +
 // reconstructed proof + a recorded non-zero exit — exactly what `skalpel catches` surfaces). `strict`
@@ -212,6 +228,168 @@ function proofCommandString(proof) {
   return [proof.cmd, ...proof.args].join(" ").slice(0, 160);
 }
 
+// ---------- RE-ASK: genuine-human-turn text + the conservative correction classifier ----------
+// genuineUserTurnText(entry) → the CLEANED natural-language a human actually typed, or null when the
+// entry is not a real human turn we can read a correction from. isGenuineUserTurn (the shared session
+// definition) already rejects tool_result carriers + non-human promptSources; on top of that we drop the
+// system/command WRAPPERS Claude Code logs as role:user rows (slash commands, local-command stdout,
+// hook/task callbacks, system reminders, skalpel's own injected block) and strip interrupt/image/paste
+// placeholders that carry no words. An image-only or wrapper-only turn → null (we cannot classify it, so
+// it is honestly excluded, never guessed).
+const USER_WRAPPER_OPEN_RE =
+  /^\s*(?:)?<(?:command-name|command-message|command-args|local-command-stdout|local-command-caveat|task-notification|system-reminder|user-prompt-submit-hook|environment_context|user_instructions|editor_context)\b/i;
+// MACHINE-INJECTED turns that Claude Code logs as role:user but are NOT the human: automated background
+// events + agent-to-agent orchestration ("coordinator") messages. Verified against real history — without
+// this, an autonomous-loop machine like this one reads its OWN orchestration as "the dev re-asking" and
+// the re-ask rate is pure fiction. Rejected outright (never a correction).
+const MACHINE_INJECT_RE =
+  /^\s*(?:\[SYSTEM NOTIFICATION|the coordinator sent a message while you were working:)|NOT USER INPUT|automated background-task event/i;
+// Claude Code SURFACES a genuine human message the user queued mid-turn wrapped in boilerplate:
+//   "The user sent a (new )message while you were working: <PAYLOAD> This is how Claude Code surfaces …"
+// The PAYLOAD is real human text; the boilerplate is not. Unwrap to the payload before classifying so a
+// queued "no, still broken" counts and the boilerplate ("…alongside the next tool result…") does not.
+const QUEUED_HUMAN_WRAP_RE =
+  /^\s*The user sent a (?:new )?message while you were working:\s*([\s\S]*?)\s*This is how Claude Code surfaces messages the user (?:sends mid-turn|queued)[\s\S]*$/i;
+// INJECTED_TEMPLATE_RE — Claude-Code / SKILL-injected role:user boilerplate that is NOT a turn a human
+// typed (each class verified against real history, isMeta:true unless noted). Before this filter, an
+// autonomous machine read its OWN injected templates as "the dev re-asking": a rendered slash-command /
+// skill body ("# /loop — schedule …" appeared 16× in one session, 11 scored STRONG; "# Schedule Cloud
+// Agents"), a design-skill preamble ("Approach this as the design lead …", 3×, all STRONG), Stop-hook
+// feedback, a compaction/continuation summary re-injected at session start (isMeta:FALSE — so it needs
+// this content rule, not the isMeta gate below), a skill-output injection ("The user just ran /insights
+// …"), and a plan/collaboration-mode wrapper. They trip the correction keywords on incidental words
+// (wrong / revert / undo / broken / fail) and are pure machine text. Rejected outright — never a human
+// correction. (The genuine "The user sent a message while you were working:" QUEUED wrap is NOT here: it
+// is unwrapped to its real human payload below.)
+const INJECTED_TEMPLATE_RE = new RegExp(
+  [
+    "^#{1,6}\\s*\\/[a-z][\\w:-]*\\s+[\\u2014\\u2013-]\\s", // "# /loop — schedule …" rendered command/skill body
+    "^Approach this as the design lead\\b", // design-skill preamble (logged role:user, not typed)
+    "^Stop hook feedback:", // Stop-hook feedback wrapper
+    "^This session is being continued from a previous conversation", // compaction/continuation summary (isMeta:false)
+    "^The user just ran \\/[a-z][\\w:-]*", // skill-output injection ("… ran /insights …")
+    "^# Schedule Cloud Agents\\b", // /schedule skill body
+    "^<collaboration_mode>", // plan/collaboration-mode wrapper (Codex)
+  ].join("|"),
+  "i",
+);
+function cleanUserText(raw, { isMeta = false } = {}) {
+  let s = String(raw || "").trim();
+  if (!s) return null;
+  if (USER_WRAPPER_OPEN_RE.test(s)) return null; // a system/command wrapper, not a human message
+  if (/^\s*\[skalpel\b/i.test(s)) return null; // skalpel's own injected per-turn context block
+  if (MACHINE_INJECT_RE.test(s)) return null; // machine orchestration / automated event, not the human
+  if (INJECTED_TEMPLATE_RE.test(s)) return null; // named skill / hook / continuation injected boilerplate
+  const q = s.match(QUEUED_HUMAN_WRAP_RE);
+  if (q) {
+    s = q[1] || ""; // a GENUINE human message queued mid-turn — keep the real payload, drop the boilerplate
+  } else if (isMeta) {
+    // isMeta:true AND not a surfaced queued human message = a Claude-Code-injected META turn (a skill
+    // body, agent orchestration, a local-command caveat, an /insights dump, …), never a typed human
+    // correction. Being strict about these promptSource:null meta rows — a genuine typed turn vs an
+    // injected template — is the whole fix. Rejecting a rare genuine one only lowers the STRONG rate (a
+    // floor); it can NEVER manufacture a false correction, which is the red line.
+    return null;
+  }
+  s = s
+    .replace(/<system-reminder>[\s\S]*?<\/system-reminder>/gi, " ")
+    .replace(/\[Request interrupted by user[^\]]*\]/gi, " ")
+    .replace(/\[Image[^\]]*\]/gi, " ")
+    .replace(/\[Pasted text[^\]]*\]/gi, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+  return s || null;
+}
+function genuineUserTurnText(entry) {
+  if (!isGenuineUserTurn(entry)) return null;
+  const c = entry?.message?.content;
+  let raw = "";
+  if (typeof c === "string") raw = c;
+  else if (Array.isArray(c)) {
+    for (const b of c) {
+      if (b && b.type === "tool_result") return null; // belt-and-suspenders (isGenuineUserTurn covers it)
+      if (b && b.type === "text" && typeof b.text === "string") raw += " " + b.text;
+    }
+  } else return null;
+  // isMeta is Claude Code's own marker for an INJECTED (non-typed) role:user row — the strict signal for
+  // the promptSource:null fallback. Passed through so an injected meta turn that isn't a queued human
+  // message is rejected rather than mistaken for the dev re-asking.
+  return cleanUserText(raw, { isMeta: entry?.isMeta === true });
+}
+
+// classifyReask(text) → "strong" | "soft" | "none". CONSERVATIVE (precision over recall), two tiers so
+// the honest split is visible in the report:
+//   strong — a NEGATION / still-broken / redo / wrong signal: the "done" did not hold ("no", "still
+//            failing", "that doesn't work", "try again", "you broke it", "undo/revert"). High-confidence
+//            that the previous claim was insufficient.
+//   soft   — a CONTINUATION / scope-add / hedge ("also", "but", "actually", "one more", "what about").
+//            The brief lists these as re-ask words, but they are frequently a NEW ask layered on a
+//            genuinely-correct "done" (scope-add, not a lie) — so they are reported SEPARATELY, never
+//            folded into the strong rate. This is the key honesty caveat, made structural.
+// This is a NEW signal (there is no prior user-correction detector to reuse); the completion-claim side is
+// still the shipped detectCompletionClaim, so the "done" definition matches the catch exactly.
+const STRONG_CORRECTION_RE = new RegExp(
+  [
+    "^\\s*(no|nope|nah)\\b(?!\\s+(?:problem|worries|worry|thanks|thank|rush|need|prob|biggie|issues?|stress))",
+    "\\bstill\\s+(?:broken|failing|fail(?:s|ed)?|not\\b|does\\s?n'?t|did\\s?n'?t|do\\s?n'?t|red|the\\s+same|happening|there|erroring|crashing|wrong|breaks?|hangs?)",
+    "\\bstill\\s+(?:a\\s+)?(?:bug|error|issue|problem|failing|broken)\\b",
+    "\\bdoes\\s?n'?t\\s+work\\b",
+    "\\bdo\\s?n'?t\\s+work\\b",
+    "\\bdid\\s?n'?t\\s+work\\b",
+    "\\bnot\\s+work(?:ing)?\\b",
+    "\\bnot\\s+fixed\\b",
+    "\\bnot\\s+(?:yet\\s+)?(?:done|working|passing|resolved|right|correct)\\b",
+    "\\bnot\\s+quite\\b",
+    "\\bthat'?s\\s+(?:wrong|not\\s+right|incorrect|broken|not\\s+it)\\b",
+    "\\bthis\\s+is\\s+(?:wrong|broken|not\\s+right)\\b",
+    "\\bit'?s\\s+(?:still\\s+)?(?:broken|failing|wrong|not\\s+working)\\b",
+    "\\b(?:that|this|it)\\s+broke\\b",
+    "\\byou\\s+(?:did\\s?n'?t|missed|forgot|broke|removed|deleted|didnt)\\b",
+    "\\bsame\\s+(?:error|issue|problem|thing|result|bug|failure)\\b",
+    "\\btry\\s+again\\b",
+    "\\b(?:re-?do|redo|undo|revert|roll\\s?back|re-?open|reopen)\\b",
+    "\\bregress(?:ed|ion|es)?\\b",
+    "\\b(?:broken|broke)\\b",
+    "\\bfail(?:s|ing|ed|ure)?\\b",
+    "\\bincorrect\\b",
+    "\\bwrong\\b",
+    // AFFECTIVE / INDIRECT dissatisfaction — added CONSERVATIVELY (each observed in real corrections this
+    // keyword net was missing). It is still only a rough floor: a reliable semantic-fire number needs an
+    // LLM classifier (a future unit), NOT more keywords. Do not overfit past these.
+    "\\bvibe[\\s-]?coded\\b", // "looks vibe coded"
+    "\\bai\\s+slop\\b",
+    "\\bslop\\b", // derogatory about the output ("this is such slop")
+    "\\bi\\s+hate\\b", // "i hate the black", "i hate this"
+    "\\bnot\\s+what\\s+i\\s+(?:meant|wanted|asked|said|had\\s+in\\s+mind)\\b",
+    "\\b(?:not|no)\\s+chang(?:ed|e)\\b", // "not changed", "no change"
+    "\\bnothing\\s+chang(?:ed|es)\\b",
+    "\\bdid\\s?n'?t\\s+chang(?:e|ed)\\b",
+    "\\bi\\s+(?:do\\s?n'?t|don'?t)\\s+see\\b", // "i don't see it", "i don't see any difference"
+  ].join("|"),
+  "i",
+);
+const SOFT_CONTINUATION_RE = new RegExp(
+  [
+    "\\balso\\b",
+    "\\band\\s+(?:also|now|can|could|one|then)\\b",
+    "\\bone\\s+more\\b",
+    "\\banother\\b",
+    "\\baddition(?:al|ally)\\b",
+    "\\bwhat\\s+about\\b",
+    "\\bcan\\s+you\\s+(?:also|now)\\b",
+    "\\b(?:but|however|actually|wait)\\b",
+    "\\bnow\\s+(?:add|also|do|make|can|let'?s|fix|the)\\b",
+  ].join("|"),
+  "i",
+);
+export function classifyReask(text) {
+  const s = String(text || "");
+  if (!s.trim()) return "none";
+  if (STRONG_CORRECTION_RE.test(s)) return "strong";
+  if (SOFT_CONTINUATION_RE.test(s)) return "soft";
+  return "none";
+}
+
 // ---------- (3) scan ONE session → per-claim records + the session's PASS proof-runs ----------
 // A record's verdict, deduped by proof tool_use id (identical to scanSession), is:
 //   'confirmed-lie'      — claim over a proof whose recorded result classifies GENUINE_FAIL (a real lie)
@@ -256,6 +434,55 @@ export function scanSessionRecords(entries, meta = {}) {
     }
   }
 
+  // (3b-pre) RE-ASK index — three ascending entry-index lists, all deterministic from the transcript:
+  //   userTurns    : {idx, text} of every GENUINE human turn (the correction candidates)
+  //   claimIdxs    : entry indices where a completion claim was detected (to detect a superseding claim)
+  //   asstTurnIdxs : entry indices carrying assistant natural-language text (the turns-to-correction window)
+  const userTurns = [];
+  const claimIdxs = [];
+  const asstTurnIdxs = [];
+  for (let i = 0; i < entries.length; i++) {
+    const at = assistantTextOf(entries[i]);
+    if (at) {
+      asstTurnIdxs.push(i);
+      if (detectCompletionClaim(at).claim) claimIdxs.push(i);
+    }
+    const ut = genuineUserTurnText(entries[i]);
+    if (ut) userTurns.push({ idx: i, text: ut });
+  }
+  // reaskForClaim(i) — pair a claim at entry i with the NEXT human turn and classify the push-back:
+  //   no_followup — no human turn after it in-window (session ended on the claim, or the reply is beyond
+  //                 REASK_MAX_ASSISTANT_TURNS assistant turns → too weakly attributable to this claim)
+  //   superseded  — a LATER "done" claim landed before the human replied → that later claim owns the reply
+  //   evaluated   — the human's next turn is this claim's response; classifyReask labels it strong/soft/none
+  const reaskForClaim = (i) => {
+    let u = null;
+    for (const t of userTurns) {
+      if (t.idx > i) {
+        u = t;
+        break;
+      }
+    }
+    if (!u) return { reask_status: "no_followup", reask_reason: "session-ended-on-claim" };
+    let asstTurns = 0;
+    for (const a of asstTurnIdxs) {
+      if (a >= u.idx) break;
+      if (a >= i) asstTurns++;
+    }
+    if (asstTurns > REASK_MAX_ASSISTANT_TURNS)
+      return { reask_status: "no_followup", reask_reason: "out-of-window" };
+    for (const k of claimIdxs) {
+      if (k >= u.idx) break;
+      if (k > i) return { reask_status: "superseded" };
+    }
+    return {
+      reask_status: "evaluated",
+      reask_class: classifyReask(u.text),
+      turns_to_correction: asstTurns,
+      next_user_text: u.text.slice(0, 300),
+    };
+  };
+
   // (3b) per completion-claim turn → a classified record.
   const seenProof = new Set(); // dedup by proof tool_use id (identical to scanSession)
   for (let i = 0; i < entries.length; i++) {
@@ -265,6 +492,7 @@ export function scanSessionRecords(entries, meta = {}) {
     if (!claim) continue;
 
     const claimTs = ts[i] || null;
+    const reask = reaskForClaim(i); // the human's next-turn push-back (attached to every record below)
     const found = reconstructProofEntry(entries.slice(0, i + 1), meta.cwd || null);
     if (!found || !found.toolUseId) {
       records.push({
@@ -274,6 +502,7 @@ export function scanSessionRecords(entries, meta = {}) {
         verdict: "unverifiable",
         reason: "no-reconstructable-proof",
         claim_ts: claimTs,
+        ...reask,
       });
       continue;
     }
@@ -290,6 +519,7 @@ export function scanSessionRecords(entries, meta = {}) {
         verdict: "unverifiable",
         reason: "proof-result-not-in-window",
         claim_ts: claimTs,
+        ...reask,
       });
       continue;
     }
@@ -337,6 +567,7 @@ export function scanSessionRecords(entries, meta = {}) {
       evidence,
       claim_ts: claimTs,
       proof_ts: proofTs,
+      ...reask,
     });
   }
   return { records, proofPassRuns };
@@ -513,6 +744,103 @@ function recId(r) {
   return `${r.session || "?"}::${r.tool_use_id || r.claim_text || "?"}`;
 }
 
+// ---------- RE-ASK / CORRECTION report (the semantic-lie proxy the test-catch misses) ----------
+// Every count here is a real tally over real (claim → next-human-turn) pairs; no number is invented. The
+// denominator for the rate is EVALUATED claims only (a human actually replied within-window) — a claim
+// the session ended on, or that a later claim superseded, cannot be a re-ask and is excluded (and
+// reported). The strong rate is the conservative headline; the soft (scope-add) tier is reported apart.
+export function computeReaskReport(corpus) {
+  const recs = corpus.records || [];
+  const totalDone = recs.length;
+  const evaluated = recs.filter((r) => r.reask_status === "evaluated");
+  const superseded = recs.filter((r) => r.reask_status === "superseded").length;
+  const sessionEnded = recs.filter(
+    (r) => r.reask_status === "no_followup" && r.reask_reason === "session-ended-on-claim",
+  ).length;
+  const outOfWindow = recs.filter(
+    (r) => r.reask_status === "no_followup" && r.reask_reason === "out-of-window",
+  ).length;
+  const noReask = totalDone - evaluated.length - superseded; // = session-ended + out-of-window
+
+  const strong = evaluated.filter((r) => r.reask_class === "strong");
+  const soft = evaluated.filter((r) => r.reask_class === "soft");
+  const nEval = evaluated.length;
+
+  // turns-to-correction distribution over the STRONG corrections (the conservative signal).
+  const ttcBuckets = { 1: 0, 2: 0, 3: 0, 4: 0, "5+": 0 };
+  const ttcValues = [];
+  for (const r of strong) {
+    const t = Number.isInteger(r.turns_to_correction) ? r.turns_to_correction : null;
+    if (t == null) continue;
+    ttcValues.push(t);
+    if (t <= 1) ttcBuckets["1"]++;
+    else if (t === 2) ttcBuckets["2"]++;
+    else if (t === 3) ttcBuckets["3"]++;
+    else if (t === 4) ttcBuckets["4"]++;
+    else ttcBuckets["5+"]++;
+  }
+
+  // Cross-tab by the claim's OWN proof verdict. The headline semantic-lie signal is a passing-proof
+  // ("true-done") claim that STILL drew a strong correction: the test was green, the human still had to
+  // re-ask — exactly the lie the re-runnable-test catch is blind to.
+  const VERDICTS = ["true-done", "confirmed-lie", "harness-suppressed", "unverifiable"];
+  const byVerdict = {};
+  for (const v of VERDICTS) {
+    const ev = evaluated.filter((r) => r.verdict === v);
+    const s = ev.filter((r) => r.reask_class === "strong").length;
+    const so = ev.filter((r) => r.reask_class === "soft").length;
+    byVerdict[v] = {
+      evaluated: ev.length,
+      strong: s,
+      soft: so,
+      reask_rate_strong: ev.length ? s / ev.length : null,
+      reask_rate_any: ev.length ? (s + so) / ev.length : null,
+    };
+  }
+  const passingProof = byVerdict["true-done"]; // the semantic-lie signal
+
+  // Hand-audit samples: strong corrections first (most load-bearing), then soft. Deterministic order
+  // (records are produced in the deterministic session-scan order), capped, real fields only.
+  const sampleOf = (r) => ({
+    tool: r.tool,
+    session: r.session,
+    verdict: r.verdict,
+    tier: r.reask_class,
+    turns_to_correction: r.turns_to_correction ?? null,
+    recorded_exit_code: r.recorded_exit_code ?? null,
+    claim_text: r.claim_text,
+    next_user_text: r.next_user_text,
+  });
+  const samples = [...strong, ...soft].slice(0, MAX_REASK_SAMPLES).map(sampleOf);
+
+  return {
+    window_assistant_turns: REASK_MAX_ASSISTANT_TURNS,
+    total_done_claims: totalDone,
+    evaluated: nEval,
+    excluded_no_followup: noReask,
+    excluded_session_ended: sessionEnded,
+    excluded_out_of_window: outOfWindow,
+    excluded_superseded: superseded,
+    corrections_strong: strong.length,
+    corrections_soft: soft.length,
+    reask_rate_strong: nEval ? strong.length / nEval : null,
+    reask_rate_any: nEval ? (strong.length + soft.length) / nEval : null,
+    turns_to_correction: {
+      buckets: ttcBuckets,
+      median: median(ttcValues),
+      max: ttcValues.length ? Math.max(...ttcValues) : null,
+    },
+    // THE semantic-lie signal — passing proof, human still re-asked.
+    passing_proof_evaluated: passingProof.evaluated,
+    passing_proof_corrected_strong: passingProof.strong,
+    passing_proof_corrected_any: passingProof.strong + passingProof.soft,
+    passing_proof_reask_rate_strong: passingProof.reask_rate_strong,
+    passing_proof_reask_rate_any: passingProof.reask_rate_any,
+    by_verdict: byVerdict,
+    samples,
+  };
+}
+
 // ---------- (6) render the honest terminal report ----------
 // Untrusted transcript text is rendered to a live terminal — strip ESC/control bytes first (same defense
 // as catches.mjs / ledger.mjs `clip`), then collapse whitespace and cap length.
@@ -687,6 +1015,213 @@ export function renderReport(report, { variantName = "all" } = {}) {
   return L.join("\n");
 }
 
+// ---------- (6b) render the RE-ASK / CORRECTION report (the semantic-lie proxy) ----------
+export function renderReaskReport(reask, frequency = {}) {
+  const R = reask;
+  const L = [];
+  L.push("");
+  L.push("  🔬 skalpel — efficacy eval · RE-ASK FIRE (the semantic lie the test-catch can't see)");
+  L.push("  " + "─".repeat(76));
+  L.push("");
+  L.push(
+    "  WHY this metric: the re-runnable-test catch finds ~0 lies in real history — an agent rarely",
+  );
+  L.push(
+    '  lies about a TEST it just ran. The lie a dev burns time on is SEMANTIC: "done / handles X",',
+  );
+  L.push(
+    "  the tests pass, but the code is wrong — invisible to any re-run of the same proof. The only",
+  );
+  L.push(
+    '  offline PROXY is the dev catching it themselves: right after a "done" claim, does the NEXT',
+  );
+  L.push("  human turn push back? This measures that, honestly, from your real transcripts.");
+
+  if (frequency.sessions != null) {
+    L.push("");
+    L.push(
+      `  Scanned ${frequency.sessions} recent sessions (${frequency.claude_sessions} Claude, ${frequency.codex_sessions} Codex). ONE person's`,
+    );
+    L.push(
+      "  history (dogfood, n=1) — read every rate as a signal about YOU, not a population estimate.",
+    );
+  }
+
+  if (R.total_done_claims === 0) {
+    L.push("");
+    L.push("  No completion claims found to evaluate — nothing to measure.");
+    L.push("");
+    return L.join("\n");
+  }
+
+  // ---- the funnel ----
+  L.push("");
+  L.push('  1) THE FUNNEL — from every "done" claim to the ones a human actually answered');
+  L.push("  " + "·".repeat(74));
+  L.push(`     "done"-type claims detected ............ ${R.total_done_claims}`);
+  L.push(
+    `       ├─ EVALUATED (human replied in-window) .. ${R.evaluated}   (the only ones a re-ask can be measured on)`,
+  );
+  L.push(
+    `       ├─ session ended on the claim ........... ${R.excluded_session_ended}   (no next human turn — excluded)`,
+  );
+  L.push(
+    `       ├─ reply beyond ${String(R.window_assistant_turns).padStart(2)}-assistant-turn window . ${R.excluded_out_of_window}   (too weakly attributable — excluded)`,
+  );
+  L.push(
+    `       └─ superseded by a later "done" ......... ${R.excluded_superseded}   (a later claim owns that reply — excluded)`,
+  );
+
+  // ---- the rate ----
+  L.push("");
+  L.push("  2) RE-ASK RATE — of the claims a human answered, how many drew a correction");
+  L.push("  " + "·".repeat(74));
+  if (R.evaluated === 0) {
+    L.push(
+      "     n/a — no claim had an in-window human reply to measure (fully-autonomous history?).",
+    );
+  } else {
+    L.push(
+      `     STRONG re-ask (no / still broken / doesn't work / try again / undo) : ${R.corrections_strong}/${R.evaluated}` +
+        `  =  ${pct(R.reask_rate_strong)}`,
+    );
+    L.push(
+      `     + SOFT continuation (also / but / actually — MAY be scope-add) ...... ${R.corrections_soft} more` +
+        `  →  ${pct(R.reask_rate_any)} combined`,
+    );
+    if (R.evaluated < 20) {
+      L.push(
+        `     (small-n: only ${R.evaluated} evaluated claim(s) — directional, not a stable rate.)`,
+      );
+    }
+    // turns-to-correction distribution (strong corrections).
+    const b = R.turns_to_correction.buckets;
+    L.push("");
+    L.push(
+      `     turns-to-correction (strong): 1:${b["1"]}  2:${b["2"]}  3:${b["3"]}  4:${b["4"]}  5+:${b["5+"]}` +
+        `   (median ${R.turns_to_correction.median ?? "n/a"})`,
+    );
+  }
+
+  // ---- the semantic-lie signal ----
+  L.push("");
+  L.push("  3) SEMANTIC-LIE SIGNAL — claims whose OWN proof PASSED that STILL drew a correction");
+  L.push("     (green test + human still had to re-ask = the exact lie a re-run cannot catch)");
+  L.push("  " + "·".repeat(74));
+  if (R.passing_proof_evaluated === 0) {
+    L.push("     n/a — no passing-proof claim had an in-window human reply.");
+  } else {
+    L.push(`     passing-proof claims answered by a human ... ${R.passing_proof_evaluated}`);
+    L.push(
+      `       └─ STILL got a STRONG correction ......... ${R.passing_proof_corrected_strong}` +
+        `  =  ${pct(R.passing_proof_reask_rate_strong)}   (+${R.passing_proof_corrected_any - R.passing_proof_corrected_strong} soft → ${pct(R.passing_proof_reask_rate_any)})`,
+    );
+  }
+  // per-verdict breakdown
+  L.push("");
+  L.push("     by proof verdict:   verdict          evaluated  strong  soft   strong-rate");
+  for (const v of ["true-done", "confirmed-lie", "harness-suppressed", "unverifiable"]) {
+    const s = R.by_verdict[v];
+    if (!s) continue;
+    L.push(
+      `                         ${v.padEnd(18)} ${String(s.evaluated).padStart(8)}  ${String(s.strong).padStart(6)}  ${String(s.soft).padStart(4)}   ${pct(s.reask_rate_strong).padStart(8)}`,
+    );
+  }
+
+  // ---- honest limits ----
+  L.push("");
+  L.push("  4) HEURISTIC LIMITS (read before trusting any number above)");
+  L.push("  " + "·".repeat(74));
+  L.push(
+    "     • This is a KEYWORD heuristic over the next human turn, NOT ground truth. A flagged",
+  );
+  L.push(
+    "       correction may be a SCOPE-ADD, a next-task ('now do Z'), or pasted content (a tweet /",
+  );
+  L.push(
+    "       spec) that trips a keyword — so PRECISION is imperfect. Hand-audit the samples & discount:",
+  );
+  L.push(
+    "       on the dogfood corpus the STRONG tier hand-audited ~50-67% genuine (roughly 6-8 of 12),",
+  );
+  L.push("       NOT the ~100% a tiny first-window slice suggested. Treat the rate accordingly.");
+  L.push(
+    "     • MACHINE-INJECTED role:user turns are REJECTED, never counted as a correction: skill",
+  );
+  L.push(
+    "       bodies (e.g. '# /loop — …', design-lead preamble), Stop-hook feedback, compaction /",
+  );
+  L.push(
+    "       continuation summaries, agent-orchestration + isMeta meta rows. (Before this filter one",
+  );
+  L.push(
+    "       session's 16 injected /loop templates alone scored 11 false STRONG 'corrections'.)",
+  );
+  L.push(
+    "     • UNDER-COUNT: affective / indirect corrections ('not what I meant', 'i don't see it',",
+  );
+  L.push(
+    "       'looks vibe coded', 'i hate the X') are only partly keyword-covered — so the STRONG",
+  );
+  L.push(
+    "       rate is a FLOOR (lower bound). A reliable semantic-fire number needs an LLM classifier.",
+  );
+  L.push(
+    "     • That is WHY strong (negation/redo) and soft (also/but) are reported apart — trust the",
+  );
+  L.push(
+    "       strong rate as a floor; treat the soft tier as an upper bound that includes scope-adds.",
+  );
+  L.push(
+    "     • It measures a PROXY (the dev pushing back), not the code being wrong — a real re-ask",
+  );
+  L.push(
+    "       may reflect the dev changing their mind, and a silent 'done' may still hide a bug.",
+  );
+  L.push(
+    "     • Denominator = claims a human answered in-window; fully-autonomous claims are excluded.",
+  );
+  L.push(
+    "       Repeated 'done's over the same proof are collapsed (same dedup as the lie metric).",
+  );
+  L.push(
+    `     • WINDOW: this run scanned ${frequency.sessions ?? "N"} session(s). The DEFAULT --limit ` +
+      `(${DEFAULT_MAX_SESSIONS}) is`,
+  );
+  L.push(
+    "       UNREPRESENTATIVE: the strong-flag mix differs materially across the full corpus — the",
+  );
+  L.push(
+    "       older sessions hold both the machine-contamination and many affective corrections a small",
+  );
+  L.push(
+    "       recent window misses. Re-run with a large --limit for the whole history before trusting a rate.",
+  );
+  L.push(
+    `     • n = ${R.evaluated} evaluated claim(s).${R.evaluated < 20 ? " SMALL — directional only." : ""}`,
+  );
+
+  // ---- samples for hand-audit ----
+  if (R.samples.length) {
+    L.push("");
+    L.push(
+      `  Flagged corrections for HAND-AUDIT (first ${Math.min(MAX_REASK_SAMPLES, R.samples.length)} — confirm each is a real correction, not a scope-add):`,
+    );
+    L.push("  " + "·".repeat(74));
+    R.samples.forEach((s, i) => {
+      const green = s.verdict === "true-done" ? "  [proof PASSED]" : "";
+      L.push(`   ${i + 1}. [${s.tool}] ${s.tier} · ${s.verdict}${green}`);
+      L.push(`       said:  "${clip(s.claim_text, 66)}"`);
+      L.push(`       reply: "${clip(s.next_user_text, 66)}"`);
+    });
+  }
+
+  L.push("  " + "─".repeat(76));
+  L.push("  Read-only replay. --reask --json for the machine-readable re-ask numbers.");
+  L.push("");
+  return L.join("\n");
+}
+
 // ---------- (7) CLI ----------
 function parseArgs(argv) {
   const opts = {
@@ -695,11 +1230,13 @@ function parseArgs(argv) {
     limit: DEFAULT_MAX_SESSIONS,
     home: process.env.SKALPEL_EVAL_HOME || homedir(),
     out: null,
+    reask: false,
     help: false,
   };
   for (let i = 0; i < argv.length; i++) {
     const a = argv[i];
     if (a === "--json") opts.json = true;
+    else if (a === "--reask") opts.reask = true;
     else if (a === "-h" || a === "--help") opts.help = true;
     else if (a === "--variant") opts.variant = argv[++i] || "all";
     else if (a.startsWith("--variant=")) opts.variant = a.slice("--variant=".length);
@@ -729,8 +1266,12 @@ what it measures (every number from a real transcript field/timestamp):
   2  lie cost        — MEASURED red→green time to the same proof's later PASS (never estimated)
   3  catch accuracy  — raw-flag → surfaced vs harness-suppressed; precision + (relative) recall
   4  honest framing  — the n and a plain small-n caveat; says zero when it's zero
+  +  --reask         — the SEMANTIC-lie proxy the test-catch misses: after a "done" claim, did the
+                       NEXT human turn push back ("no / still broken / try again / also X")? Reports the
+                       RE-ASK RATE, turns-to-correction, and the passing-proof-still-corrected signal.
 
 options:
+  --reask           the re-ask / correction report (semantic-lie proxy); combine with --json
   --variant <name>  baseline | strict | all   (default all — compares them side by side)
   --limit <n>       most-recent sessions to scan (default ${DEFAULT_MAX_SESSIONS})
   --home <dir>      history root override (default ~ ; or SKALPEL_EVAL_HOME) — for fixtures/tests
@@ -748,11 +1289,20 @@ export function runEval(opts) {
   }
   const corpus = replayHistory({ home: opts.home, maxSessions: opts.limit });
   const report = computeReport(corpus, variants);
+  const reask = computeReaskReport(corpus);
   if (opts.json) {
     // Deterministic JSON: same history in → byte-identical out (no timestamps of "now", no randomness).
-    return { text: JSON.stringify({ variant: opts.variant, ...report }, null, 2) + "\n", report };
+    // The reask block always rides along so `--json` is the complete machine-readable picture.
+    return {
+      text: JSON.stringify({ variant: opts.variant, ...report, reask }, null, 2) + "\n",
+      report,
+      reask,
+    };
+  }
+  if (opts.reask) {
+    return { text: renderReaskReport(reask, report.frequency), report, reask };
   }
-  return { text: renderReport(report, { variantName: opts.variant }), report };
+  return { text: renderReport(report, { variantName: opts.variant }), report, reask };
 }
 
 const isMain = (() => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skalpel",
-  "version": "4.0.48",
+  "version": "4.0.49",
   "description": "Behavioral graph for AI-assisted coding — learns how you work and steers Claude Code + Codex in real time.",
   "type": "module",
   "bin": {