skalpel 4.0.24 → 4.0.26

package/eval-harness.mjs

@@ -0,0 +1,1125 @@
+#!/usr/bin/env node
+// eval-harness.mjs — Skalpel intervention eval harness (pure Node, local-only, NO network).
+//
+// TRUE NORTH: KNOW whether a steering intervention actually helps, MEASURED from real
+// Claude Code transcripts — before building or scaling it. This harness FILTERS + RANKS
+// candidate intervention variants fast on real data. It does NOT prove live lift.
+//
+// ┌───────────────────────────────────────────────────────────────────────────────────┐
+// │ HONESTY BANNER (printed on every report, and true of every number this emits):      │
+// │   • TRIGGER metrics + COUNTERFACTUAL PROXY are OFFLINE FILTER/RANK signals.          │
+// │   • They are NOT lift. Firing the intervention changes the future; the recorded      │
+// │     transcript is off-policy. True lift REQUIRES a live A/B.  needs_live_ab = true.   │
+// │   • Auto-labels are NOT ground truth. Without a human gold set the report is flagged  │
+// │     UNCALIBRATED and downstream metrics are marked untrustworthy.                     │
+// │   • Every number below traces to a real transcript event. No fabricated/estimated    │
+// │     values. Proxy results are labelled "proxy (needs live A/B)".                      │
+// └───────────────────────────────────────────────────────────────────────────────────┘
+//
+// Exports:
+//   labelThrashEpisodes(entries)                       -> { prompts, episodes, transitions }
+//   scoreVariant({trigger, mechanism}, transcriptsDir) -> aggregate scores {needs_live_ab:true, ...}
+//   loadSession(filePath) / listSessions(dir)          -> helpers
+// CLI:
+//   node eval-harness.mjs <transcriptsDir> [--max-sessions N] [--gold gold.json]
+
+import fs from "node:fs";
+import path from "node:path";
+
+// ───────────────────────────── tunables (UNCALIBRATED defaults) ─────────────────────────────
+// These thresholds are NOT tuned against a human gold set unless one is supplied via --gold.
+// Reported alongside every run so the reader knows what produced the labels.
+export const DEFAULTS = {
+  tau_sim: 0.5, // TF-IDF cosine similarity threshold for "same target"
+  tau_tgt: 0.3, // Jaccard threshold on target-tuple (paths ∪ rare identifiers)
+  tau_dup: 0.8, // near-duplicate restatement threshold
+  kappa_min: 0.6, // below this vs gold => UNCALIBRATED + suppress
+  winsor_gap_s: 15 * 60, // cap inter-turn gaps at 15 min so overnight AFK can't pollute durations
+  bootstrap_B: 2000,
+  proxy_cover: 0.5, // IDF-weighted coverage of resolving-delta R required for a supply-match
+  proxy_min_rare: 2, // require >= this many rare (IDF>median) tokens matched
+};
+
+const NOISE_PREFIXES = [
+  "<local-command",
+  "<command-name>",
+  "<command-message>",
+  "<command-args>",
+  "<local-command-stdout>",
+  "<task-notification>",
+  "<system-reminder",
+  "[skalpel", // the injected behavioral-graph hook block — NEVER a genuine user re-ask
+  "[Image:", // pasted-image placeholder, not a typed prompt
+  "[Request interrupted",
+  "caveat:",
+];
+const DROP_TYPES = new Set([
+  "last-prompt",
+  "mode",
+  "permission-mode",
+  "attachment",
+  "file-history-snapshot",
+  "ai-title",
+  "queue-operation",
+  "system",
+]);
+
+// dissatisfaction / corrective / repeat cues (secondary signal — semantic sim is primary)
+const CORRECTIVE_CUES = [
+  /\bno\b/i,
+  /\bstill\b/i,
+  /\bagain\b/i,
+  /\bwrong\b/i,
+  /\bdidn'?t work\b/i,
+  /\bdoesn'?t work\b/i,
+  /\bnot working\b/i,
+  /\bsame\b/i,
+  /\bas i (said|asked)\b/i,
+  /\brevert\b/i,
+  /\bundo\b/i,
+  /\bnot what i (asked|wanted)\b/i,
+  /\bverify\b/i,
+  /\bbroke(n)?\b/i,
+  /\bfails?\b/i,
+  /\berror\b/i,
+  /\bthat'?s not\b/i,
+  /\bfix (it|this|that)\b/i,
+];
+// acceptance / expansion => collaboration, NOT thrash (excluded)
+const ACCEPT_CUES = [
+  /\bthanks\b/i,
+  /\bthank you\b/i,
+  /\bgreat\b/i,
+  /\bperfect\b/i,
+  /\bnice\b/i,
+  /\blooks good\b/i,
+  /\blgtm\b/i,
+  /\bnow also\b/i,
+  /\bnext(,| step)\b/i,
+  /\bnow let'?s\b/i,
+  /\balso add\b/i,
+];
+const ITERATE_CUES = [/\bkeep (iterating|going)\b/i, /\bcontinue\b/i, /\bkeep at it\b/i];
+
+const STOPWORDS = new Set(
+  (
+    "the a an and or but if then else for to of in on at by with without from into over under this that these " +
+    "those it its is are was were be been being do does did done have has had you your i me my we our they them " +
+    "as so not no yes can could would should will just now also please make sure use using need want get got let " +
+    "up down out about all any some more most less can't cant dont don't didnt it's im i'm ok okay thanks"
+  )
+    .split(/\s+/)
+    .filter(Boolean),
+);
+
+// ───────────────────────────── parsing ─────────────────────────────
+export function loadSession(filePath) {
+  const raw = fs.readFileSync(filePath, "utf8");
+  const entries = [];
+  for (const line of raw.split("\n")) {
+    if (!line) continue;
+    try {
+      entries.push(JSON.parse(line));
+    } catch {
+      /* skip malformed line */
+    }
+  }
+  return entries;
+}
+
+export function listSessions(dir) {
+  const out = [];
+  const walk = (d) => {
+    let items;
+    try {
+      items = fs.readdirSync(d, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const it of items) {
+      const p = path.join(d, it.name);
+      if (it.isDirectory()) walk(p);
+      else if (it.isFile() && it.name.endsWith(".jsonl")) out.push(p);
+    }
+  };
+  walk(dir);
+  return out;
+}
+
+function contentToText(content) {
+  if (typeof content === "string") return content;
+  if (Array.isArray(content)) {
+    return content
+      .filter((b) => b && b.type === "text" && typeof b.text === "string")
+      .map((b) => b.text)
+      .join("\n");
+  }
+  return "";
+}
+
+function assistantActivity(entry) {
+  // returns {tool:bool, text:string} describing what the assistant did in this record
+  if (entry.type !== "assistant") return null;
+  const c = entry.message && entry.message.content;
+  if (!Array.isArray(c)) return { tool: false, text: contentToText(c) };
+  const tool = c.some((b) => b && b.type === "tool_use");
+  const text = c
+    .filter((b) => b && b.type === "text")
+    .map((b) => b.text)
+    .join("\n");
+  return { tool, text };
+}
+
+// ───────────────────────────── STEP 1: real user-prompt stream + interleaved attempts ─────
+export function extractStream(entries) {
+  const prompts = [];
+  let pendingAttempt = { tool: false, text: "" }; // assistant activity since previous kept prompt
+  for (let idx = 0; idx < entries.length; idx++) {
+    const e = entries[idx];
+    if (!e || !e.type) continue;
+    if (e.type === "assistant") {
+      const a = assistantActivity(e);
+      if (a) {
+        if (a.tool) pendingAttempt.tool = true;
+        if (a.text) pendingAttempt.text += (pendingAttempt.text ? "\n" : "") + a.text;
+      }
+      continue;
+    }
+    if (e.type !== "user") continue;
+    if (DROP_TYPES.has(e.type)) continue;
+    if (e.isMeta) continue;
+    const c = e.message && e.message.content;
+    if (typeof c !== "string") continue; // DROP array content (tool_result) — never a typed re-ask
+    const text = c.trim();
+    if (!text) continue;
+    const trimmedStart = c.trimStart();
+    if (NOISE_PREFIXES.some((n) => trimmedStart.startsWith(n))) continue;
+    prompts.push({
+      index: prompts.length,
+      entryIndex: idx,
+      ts: e.timestamp ? Date.parse(e.timestamp) : null,
+      text,
+      uuid: e.uuid,
+      parentUuid: e.parentUuid,
+      // attempt that the assistant made BEFORE this prompt (since the previous kept prompt)
+      attemptBefore: { tool: pendingAttempt.tool, hadText: pendingAttempt.text.length > 0 },
+    });
+    pendingAttempt = { tool: false, text: "" };
+  }
+  return prompts;
+}
+
+// ───────────────────────────── STEP 2: target signature + TF-IDF ─────────────────────────
+const PATH_RE =
+  /(?:[\w.-]*\/)+[\w.-]+|\b[\w-]+\.(?:mjs|js|ts|tsx|rs|go|py|json|toml|sql|md|sh|yml|yaml|css|html)\b/g;
+const IDENT_RE = /\b[A-Za-z_][A-Za-z0-9_]{2,}\b/g;
+
+function tokenize(text) {
+  const toks = [];
+  const lower = text.toLowerCase();
+  for (const m of lower.matchAll(IDENT_RE)) {
+    const t = m[0];
+    if (!STOPWORDS.has(t)) toks.push(t);
+  }
+  return toks;
+}
+
+function extractPaths(text) {
+  const set = new Set();
+  for (const m of text.matchAll(PATH_RE)) set.add(m[0].toLowerCase());
+  return set;
+}
+
+function buildIdf(prompts) {
+  // document frequency across the prompts in THIS session (episodes are within-session)
+  const df = new Map();
+  const N = prompts.length || 1;
+  for (const p of prompts) {
+    const seen = new Set(tokenize(p.text));
+    for (const t of seen) df.set(t, (df.get(t) || 0) + 1);
+  }
+  const idf = new Map();
+  for (const [t, d] of df) idf.set(t, Math.log((N + 1) / (d + 1)) + 1);
+  return { idf, N };
+}
+
+function tfidfVec(text, idf) {
+  const tf = new Map();
+  const toks = tokenize(text);
+  for (const t of toks) tf.set(t, (tf.get(t) || 0) + 1);
+  const vec = new Map();
+  let norm = 0;
+  for (const [t, f] of tf) {
+    const w = (f / toks.length) * (idf.get(t) || Math.log(2) + 1);
+    vec.set(t, w);
+    norm += w * w;
+  }
+  norm = Math.sqrt(norm) || 1;
+  for (const [t, w] of vec) vec.set(t, w / norm);
+  return vec;
+}
+
+function cosine(a, b) {
+  let dot = 0;
+  const [small, big] = a.size < b.size ? [a, b] : [b, a];
+  for (const [t, w] of small) {
+    const w2 = big.get(t);
+    if (w2) dot += w * w2;
+  }
+  return dot;
+}
+
+function targetTuple(text, idf) {
+  // paths ∪ rare (high-IDF) identifier tokens
+  const set = new Set(extractPaths(text));
+  const scored = tokenize(text)
+    .map((t) => [t, idf.get(t) || 0])
+    .sort((x, y) => y[1] - x[1])
+    .slice(0, 8);
+  for (const [t] of scored) set.add(t);
+  return set;
+}
+
+function jaccard(a, b) {
+  if (!a.size && !b.size) return 0;
+  let inter = 0;
+  const [small, big] = a.size < b.size ? [a, b] : [b, a];
+  for (const t of small) if (big.has(t)) inter++;
+  return inter / (a.size + b.size - inter);
+}
+
+const hasCue = (text, cues) => cues.some((re) => re.test(text));
+
+// ───────────────────────────── STEP 3+4: episodes ─────────────────────────
+export function labelThrashEpisodes(entries, opts = {}) {
+  const cfg = { ...DEFAULTS, ...opts };
+  const prompts = extractStream(entries);
+  const { idf } = buildIdf(prompts);
+  // precompute vectors + target tuples (cache => seconds target)
+  const vecs = prompts.map((p) => tfidfVec(p.text, idf));
+  const tgts = prompts.map((p) => targetTuple(p.text, idf));
+
+  const transitions = []; // per adjacent pair (i,i+1)
+  for (let i = 0; i + 1 < prompts.length; i++) {
+    const a = prompts[i];
+    const b = prompts[i + 1];
+    const attempted = b.attemptBefore.tool || b.attemptBefore.hadText;
+    const sim = cosine(vecs[i], vecs[i + 1]);
+    const jac = jaccard(tgts[i], tgts[i + 1]);
+    const accept = hasCue(b.text, ACCEPT_CUES) && !hasCue(b.text, CORRECTIVE_CUES);
+    const iterate = hasCue(a.text, ITERATE_CUES) || hasCue(b.text, ITERATE_CUES);
+    const sameTarget = sim >= cfg.tau_sim || jac >= cfg.tau_tgt;
+    const corrective = hasCue(b.text, CORRECTIVE_CUES) || sim >= cfg.tau_dup || jac >= cfg.tau_tgt;
+    const reengaged = attempted && sameTarget && corrective && !accept && !iterate;
+    transitions.push({ i, attempted, sim, jac, sameTarget, corrective, accept, reengaged });
+  }
+
+  const episodes = [];
+  let start = -1;
+  for (let i = 0; i < transitions.length; i++) {
+    if (transitions[i].reengaged) {
+      if (start === -1) start = i;
+    } else if (start !== -1) {
+      episodes.push(buildEpisode(start, i, prompts, tgts, idf, cfg));
+      start = -1;
+    }
+  }
+  if (start !== -1) episodes.push(buildEpisode(start, transitions.length, prompts, tgts, idf, cfg));
+
+  return { prompts, vecs, tgts, idf, transitions, episodes };
+}
+
+function winsorGap(a, b, cap) {
+  if (a == null || b == null) return 0;
+  return Math.min(Math.max(0, b - a), cap * 1000);
+}
+
+function buildEpisode(startTrans, endTrans, prompts, tgts, idf, cfg) {
+  // members are prompt indices [startTrans .. endTrans]  (endTrans transition was the last reengaged)
+  const t0 = startTrans;
+  const tEnd = endTrans; // last member prompt index
+  const members = [];
+  for (let k = t0; k <= tEnd; k++) members.push(k);
+  // duration winsorized over member span
+  let dur = 0;
+  for (let k = t0; k < tEnd; k++)
+    dur += winsorGap(prompts[k].ts, prompts[k + 1].ts, cfg.winsor_gap_s);
+  // target signature = union of member target tuples
+  const target = new Set();
+  for (const k of members) for (const t of tgts[k]) target.add(t);
+  // terminal state: if a prompt follows the last member (session continued past loop) => RESOLVED
+  const hasNext = tEnd + 1 < prompts.length;
+  return {
+    onset_index: t0,
+    onset_entryIndex: prompts[t0].entryIndex,
+    onset_ts: prompts[t0].ts,
+    member_indices: members,
+    n_reengagements: members.length - 1,
+    duration_s: Math.round(dur / 1000),
+    terminal_state: hasNext ? "RESOLVED" : "ABANDONED",
+    esc_index: hasNext ? tEnd + 1 : null, // escape turn = first prompt AFTER the loop (the switch/resolution)
+    target_signature: [...target].slice(0, 20),
+    _targetSet: target,
+  };
+}
+
+// ───────────────────────────── trigger evaluation (strict causal truncation) ─────────────
+// A decision point f = "after prompt index f, before prompt f+1". The trigger sees ONLY
+// prompts[0..f] (ts <= that point). It NEVER sees prompt f+1 — enforced here by slicing.
+function evalTriggerOnSession(variant, labeled) {
+  const { prompts, vecs, tgts, transitions, episodes } = labeled;
+  const fires = [];
+  for (let f = 0; f < prompts.length - 1; f++) {
+    // causal prefix: prompts up to and including f, with their vecs/tgts/transitions[< f]
+    const ctx = {
+      prompts: prompts.slice(0, f + 1),
+      vecs: vecs.slice(0, f + 1),
+      tgts: tgts.slice(0, f + 1),
+      transitions: transitions.slice(0, f), // transition j links j->j+1; j<f are fully in-prefix
+      f,
+    };
+    let fire = false;
+    try {
+      fire = !!variant.trigger(ctx);
+    } catch {
+      fire = false;
+    }
+    if (fire) fires.push(f);
+  }
+
+  // classify fires against episodes
+  const episodeSpans = episodes.map((ep) => ({
+    t0: ep.onset_index,
+    tEnd: ep.member_indices[ep.member_indices.length - 1],
+    ep,
+  }));
+  let firesInside = 0;
+  const perEpisode = episodeSpans.map(() => ({ onset: false, any: false, firstFire: null }));
+  const nonThrashPoints = prompts.length - 1; // total decision points
+  let insideAnyEpisodePoints = 0;
+  const pointInEpisode = new Array(prompts.length - 1).fill(false);
+  episodeSpans.forEach((s) => {
+    for (let f = s.t0; f <= s.tEnd && f < pointInEpisode.length; f++) pointInEpisode[f] = true;
+  });
+  for (let f = 0; f < pointInEpisode.length; f++) if (pointInEpisode[f]) insideAnyEpisodePoints++;
+
+  for (const f of fires) {
+    let inside = false;
+    episodeSpans.forEach((s, ei) => {
+      if (f >= s.t0 && f <= s.tEnd) {
+        inside = true;
+        perEpisode[ei].any = true;
+        if (perEpisode[ei].firstFire == null) perEpisode[ei].firstFire = f;
+        if (f <= s.t0 + 1) perEpisode[ei].onset = true;
+      }
+    });
+    if (inside) firesInside++;
+  }
+
+  const falseFires = fires.length - firesInside;
+  const nonEpisodePoints = nonThrashPoints - insideAnyEpisodePoints;
+
+  // fires-by-offset-from-onset histogram
+  const hist = {};
+  episodeSpans.forEach((s, ei) => {
+    if (perEpisode[ei].firstFire != null) {
+      const off = perEpisode[ei].firstFire - s.t0;
+      const key = off <= -1 ? "<0" : off >= 5 ? ">=5" : String(off);
+      hist[key] = (hist[key] || 0) + 1;
+    }
+  });
+
+  return {
+    total_fires: fires.length,
+    fires_inside: firesInside,
+    false_fires: falseFires,
+    non_episode_points: nonEpisodePoints,
+    episodes: episodeSpans.length,
+    perEpisode,
+    hist,
+    fires,
+  };
+}
+
+// ───────────────────────────── counterfactual proxy (would-have-short-circuited) ─────────
+// PROXY — RANKING signal, explicitly NOT lift. Only RESOLVED episodes (known escape).
+function proxyOnSession(variant, labeled, cfg) {
+  const { prompts, tgts, idf, episodes } = labeled;
+  const results = [];
+  for (const ep of episodes) {
+    if (ep.terminal_state !== "RESOLVED" || ep.esc_index == null) continue;
+    const escIdx = ep.esc_index;
+    if (escIdx >= prompts.length) continue;
+    const members = ep.member_indices;
+    const earlier = members.filter((m) => m < escIdx); // all members (escIdx is the post-loop turn)
+    if (!earlier.length) continue;
+
+    // RESOLVING DELTA R = high-IDF tokens in the ESCAPE turn (the prompt where the loop resolved /
+    // the user switched) that are ABSENT from the thrash members. The honesty firewall: the mechanism
+    // is drawn ONLY from pre-f prompts, so a token can match R only if it demonstrably existed earlier.
+    const escTokens = new Map();
+    for (const t of tokenize(prompts[escIdx].text)) escTokens.set(t, idf.get(t) || 0);
+    const earlierTokens = new Set();
+    for (const m of earlier) for (const t of tokenize(prompts[m].text)) earlierTokens.add(t);
+    const idfVals = [...escTokens.values()].sort((a, b) => a - b);
+    const medIdf = idfVals.length ? idfVals[Math.floor(idfVals.length / 2)] : 0;
+    const R = new Map(); // token -> idf, the delta
+    for (const [t, w] of escTokens) if (!earlierTokens.has(t)) R.set(t, w);
+    if (!R.size) continue;
+
+    // where would the variant have fired inside this episode?
+    const trig = evalTriggerOnSession(variant, labeled);
+    const spanFires = trig.fires.filter((f) => f >= ep.onset_index && f < escIdx);
+    if (!spanFires.length) continue;
+    const f = spanFires[0];
+
+    // EXECUTE mechanism against context truncated at f (pre-f state ONLY). Never hand-supplied.
+    const preCtx = {
+      prompts: prompts.slice(0, f + 1),
+      tgts: tgts.slice(0, f + 1),
+      idf,
+      target: ep._targetSet,
+    };
+    let injected = new Set();
+    try {
+      injected = variant.mechanism ? variant.mechanism(preCtx) : new Set();
+    } catch {
+      injected = new Set();
+    }
+
+    // SUPPLY match: injected (drawn from pre-f) covers R by IDF-weighted mass, with >= min rare tokens.
+    let coveredMass = 0;
+    let totalMass = 0;
+    let rareHits = 0;
+    for (const [t, w] of R) {
+      totalMass += w;
+      if (injected.has(t)) {
+        coveredMass += w;
+        if (w >= medIdf) rareHits++;
+      }
+    }
+    const coverage = totalMass ? coveredMass / totalMass : 0;
+    const supplyMatch = coverage >= cfg.proxy_cover && rareHits >= cfg.proxy_min_rare;
+
+    results.push({
+      onset: ep.onset_index,
+      fire: f,
+      esc: escIdx,
+      turns_short_circuited: escIdx - f,
+      coverage: +coverage.toFixed(3),
+      rareHits,
+      supplyMatch,
+      removeMatch: false, // these example mechanisms are supply-type
+    });
+  }
+  return results;
+}
+
+// ───────────────────────────── recurrence baseline (steered vs silent analog) ────────────
+// Reproduce the cycle-12 reference (~50% steered vs ~45% silent) on THIS folder, SAME labeler.
+// steered-analog = session contained a [skalpel injection block; silent-analog = it did not.
+function sessionHadInjection(entries) {
+  for (const e of entries) {
+    if (e && e.type === "user" && typeof (e.message && e.message.content) === "string") {
+      if (e.message.content.trimStart().startsWith("[skalpel")) return true;
+    }
+  }
+  return false;
+}
+
+function recurrenceRate(labeled) {
+  // fraction of eligible transitions (assistant attempted between) that are same-target re-engagements
+  let eligible = 0;
+  let reeng = 0;
+  for (const t of labeled.transitions) {
+    if (t.attempted) {
+      eligible++;
+      if (t.reengaged) reeng++;
+    }
+  }
+  return { eligible, reeng };
+}
+
+// ───────────────────────────── bootstrap CI ─────────────────────────
+function bootstrapCI(values, B, agg) {
+  if (!values.length) return [0, 0];
+  const n = values.length;
+  const stats = [];
+  for (let b = 0; b < B; b++) {
+    const sample = new Array(n);
+    for (let i = 0; i < n; i++) sample[i] = values[(Math.random() * n) | 0];
+    stats.push(agg(sample));
+  }
+  stats.sort((a, b) => a - b);
+  return [stats[Math.floor(0.025 * B)], stats[Math.floor(0.975 * B)]];
+}
+
+// ───────────────────────────── corpus loading (one disk pass, cached) ─────────────────────
+// Load + label every session ONCE. Variants are then scored against this cache so a full
+// multi-variant CLI run is a single pass over disk (hits the seconds target).
+export function labelCorpus(transcriptsDir, opts = {}) {
+  const cfg = { ...DEFAULTS, ...opts };
+  const files = listSessions(transcriptsDir);
+  const cap = opts.maxSessions || files.length;
+  const chosen = files.slice(0, cap);
+  const sessions = [];
+  for (const f of chosen) {
+    let entries;
+    try {
+      entries = loadSession(f);
+    } catch {
+      continue;
+    }
+    if (!entries.length) continue;
+    const labeled = labelThrashEpisodes(entries, cfg);
+    if (labeled.prompts.length < 2) continue;
+    sessions.push({
+      file: f,
+      labeled,
+      steered: sessionHadInjection(entries),
+      recurrence: recurrenceRate(labeled),
+    });
+  }
+  return sessions;
+}
+
+// ───────────────────────────── public: scoreVariant ─────────────────────────
+export function scoreVariant(variant, transcriptsDir, opts = {}) {
+  const corpus = labelCorpus(transcriptsDir, opts);
+  return scoreVariantCached(variant, corpus, opts);
+}
+
+export function scoreVariantCached(variant, corpus, opts = {}) {
+  const cfg = { ...DEFAULTS, ...opts };
+
+  let n_sessions = 0,
+    n_prompts = 0,
+    n_episodes = 0,
+    n_resolved = 0,
+    n_abandoned = 0;
+  let total_fires = 0,
+    fires_inside = 0,
+    false_fires = 0,
+    non_episode_points = 0;
+  let ep_with_onset = 0,
+    ep_with_any = 0;
+  const leadTurns = [];
+  const leadSecs = [];
+  const proxyPerEp = []; // 1/0 supplyMatch over resolved
+  const proxyTurns = [];
+  let proxySupply = 0,
+    proxyRemove = 0,
+    proxyResolvedScored = 0;
+  const hist = {};
+  const episodeOnsetFlags = []; // for bootstrap of onset-recall
+  // recurrence arms
+  const arm = {
+    steered: { elig: 0, re: 0, perSession: [] },
+    silent: { elig: 0, re: 0, perSession: [] },
+  };
+
+  for (const sess of corpus) {
+    const labeled = sess.labeled;
+    n_sessions++;
+    n_prompts += labeled.prompts.length;
+    n_episodes += labeled.episodes.length;
+    for (const ep of labeled.episodes) {
+      if (ep.terminal_state === "RESOLVED") n_resolved++;
+      else n_abandoned++;
+    }
+
+    // recurrence arm
+    const rr = sess.recurrence;
+    const steered = sess.steered;
+    const a = steered ? arm.steered : arm.silent;
+    a.elig += rr.eligible;
+    a.re += rr.reeng;
+    if (rr.eligible > 0) a.perSession.push(rr.reeng / rr.eligible);
+
+    // trigger
+    const t = evalTriggerOnSession(variant, labeled);
+    total_fires += t.total_fires;
+    fires_inside += t.fires_inside;
+    false_fires += t.false_fires;
+    non_episode_points += t.non_episode_points;
+    t.perEpisode.forEach((pe, ei) => {
+      episodeOnsetFlags.push(pe.onset ? 1 : 0);
+      if (pe.onset) ep_with_onset++;
+      if (pe.any) ep_with_any++;
+      if (pe.firstFire != null) {
+        const ep = labeled.episodes[ei];
+        const tEnd = ep.member_indices[ep.member_indices.length - 1];
+        const lt = tEnd - pe.firstFire; // remaining member turns after fire
+        leadTurns.push(lt);
+        const ftS = labeled.prompts[pe.firstFire].ts;
+        const teS = labeled.prompts[tEnd].ts;
+        if (ftS != null && teS != null)
+          leadSecs.push(Math.min(teS - ftS, cfg.winsor_gap_s * 1000 * 4) / 1000);
+      }
+    });
+    for (const [k, v] of Object.entries(t.hist)) hist[k] = (hist[k] || 0) + v;
+
+    // proxy
+    const pr = proxyOnSession(variant, labeled, cfg);
+    for (const r of pr) {
+      proxyResolvedScored++;
+      proxyPerEp.push(r.supplyMatch || r.removeMatch ? 1 : 0);
+      if (r.supplyMatch || r.removeMatch) {
+        proxyTurns.push(r.turns_short_circuited);
+        if (r.supplyMatch) proxySupply++;
+        if (r.removeMatch) proxyRemove++;
+      }
+    }
+  }
+
+  const median = (arr) => {
+    if (!arr.length) return null;
+    const s = [...arr].sort((a, b) => a - b);
+    return s[Math.floor(s.length / 2)];
+  };
+  const rate = (a, b) => (b ? a / b : 0);
+
+  const onsetRecall = rate(ep_with_onset, n_episodes);
+  const anyRecall = rate(ep_with_any, n_episodes);
+  const precision = rate(fires_inside, total_fires);
+  const falseFireRate = rate(false_fires, non_episode_points);
+
+  return {
+    needs_live_ab: true,
+    thresholds: { tau_sim: cfg.tau_sim, tau_tgt: cfg.tau_tgt, tau_dup: cfg.tau_dup },
+    corpus: {
+      n_sessions,
+      n_real_user_turns: n_prompts,
+      n_thrash_episodes: n_episodes,
+      n_resolved,
+      n_abandoned,
+    },
+    trigger: {
+      total_fires,
+      trigger_precision: +precision.toFixed(4),
+      trigger_recall_any: +anyRecall.toFixed(4),
+      timing_at_onset_rate: +onsetRecall.toFixed(4), // HEADLINE
+      median_lead_turns: median(leadTurns),
+      median_lead_seconds: median(leadSecs) == null ? null : Math.round(median(leadSecs)),
+      false_fire_rate: +falseFireRate.toFixed(5),
+      fires_by_offset_from_onset: hist,
+      onset_recall_ci95: bootstrapCI(
+        episodeOnsetFlags,
+        cfg.bootstrap_B,
+        (s) => s.reduce((x, y) => x + y, 0) / s.length,
+      ).map((x) => +x.toFixed(4)),
+    },
+    counterfactual_proxy: {
+      _label: "proxy (needs live A/B) — RANKING signal, NOT lift",
+      resolved_episodes_scored: proxyResolvedScored,
+      would_short_circuit_rate_over_resolved: +rate(
+        proxySupply + proxyRemove,
+        proxyResolvedScored,
+      ).toFixed(4),
+      would_short_circuit_rate_over_all_episodes: +rate(
+        proxySupply + proxyRemove,
+        n_episodes,
+      ).toFixed(4),
+      median_turns_short_circuited: median(proxyTurns),
+      supply_matches: proxySupply,
+      remove_matches: proxyRemove,
+      abandoned_episodes_unscorable: n_abandoned,
+      proxy_ci95_over_resolved: bootstrapCI(proxyPerEp, cfg.bootstrap_B, (s) =>
+        s.length ? s.reduce((x, y) => x + y, 0) / s.length : 0,
+      ).map((x) => +x.toFixed(4)),
+    },
+    recurrence_baseline: buildRecurrence(arm, cfg),
+  };
+}
+
+function buildRecurrence(arm, cfg) {
+  const rate = (a, b) => (b ? a / b : 0);
+  const steeredN = arm.steered.perSession.length;
+  const silentN = arm.silent.perSession.length;
+  const splitOk = steeredN > 0 && silentN > 0;
+  return {
+    _label:
+      "reference to beat — SAME labeler, THIS folder (cycle-12 ref was ~50% steered vs ~45% silent)",
+    _caveat:
+      "steered-analog = session carried a [skalpel injection; silent-analog = none. Pooled recurrence, NOT a controlled A/B.",
+    _reproducible_split: splitOk,
+    _reconciliation: splitOk
+      ? "both arms present — compare recurrence_rate across arms (still NOT a controlled A/B)."
+      : "SPLIT NOT REPRODUCIBLE on this folder: one arm has 0 sessions (no genuine hook-injected transcripts " +
+        "detected). FLAGGING per anti-misleading policy — a rosier baseline is NOT substituted. The single-arm " +
+        "recurrence below is the honest number-to-beat under THIS (stricter, per-transition, conservative) " +
+        "labeler; it is NOT expected to equal the cycle-12 ~45% figure, which used a different definition.",
+    steered_analog: {
+      sessions: arm.steered.perSession.length,
+      eligible_transitions: arm.steered.elig,
+      recurrence_rate: +rate(arm.steered.re, arm.steered.elig).toFixed(4),
+      ci95: bootstrapCI(
+        arm.steered.perSession,
+        cfg.bootstrap_B,
+        (s) => s.reduce((x, y) => x + y, 0) / s.length,
+      ).map((x) => +x.toFixed(4)),
+    },
+    silent_analog: {
+      sessions: arm.silent.perSession.length,
+      eligible_transitions: arm.silent.elig,
+      recurrence_rate: +rate(arm.silent.re, arm.silent.elig).toFixed(4),
+      ci95: bootstrapCI(
+        arm.silent.perSession,
+        cfg.bootstrap_B,
+        (s) => s.reduce((x, y) => x + y, 0) / s.length,
+      ).map((x) => +x.toFixed(4)),
+    },
+  };
+}
+
+// ───────────────────────────── calibration vs human gold ─────────────────────────
+// gold.json format: [{ file: "<basename>.jsonl", onset_indices: [<prompt-index>, ...] }, ...]
+function calibrate(goldPath, transcriptsDir, cfg) {
+  let gold;
+  try {
+    gold = JSON.parse(fs.readFileSync(goldPath, "utf8"));
+  } catch {
+    return null;
+  }
+  const files = listSessions(transcriptsDir);
+  const byName = new Map(files.map((f) => [path.basename(f), f]));
+  let tp = 0,
+    fp = 0,
+    fn = 0,
+    goldN = 0;
+  // for kappa we treat each prompt as a unit: is it a thrash onset?
+  let a00 = 0,
+    a01 = 0,
+    a10 = 0,
+    a11 = 0;
+  for (const g of gold) {
+    const f = byName.get(g.file);
+    if (!f) continue;
+    const labeled = labelThrashEpisodes(loadSession(f), cfg);
+    const autoOnsets = new Set(labeled.episodes.map((e) => e.onset_index));
+    const goldOnsets = new Set(g.onset_indices || []);
+    goldN += goldOnsets.size;
+    const nP = labeled.prompts.length;
+    for (let i = 0; i < nP; i++) {
+      const auto = autoOnsets.has(i);
+      const human = goldOnsets.has(i);
+      if (auto && human) {
+        tp++;
+        a11++;
+      } else if (auto && !human) {
+        fp++;
+        a10++;
+      } else if (!auto && human) {
+        fn++;
+        a01++;
+      } else a00++;
+    }
+  }
+  const precision = tp + fp ? tp / (tp + fp) : 0;
+  const recall = tp + fn ? tp / (tp + fn) : 0;
+  const f1 = precision + recall ? (2 * precision * recall) / (precision + recall) : 0;
+  const total = a00 + a01 + a10 + a11 || 1;
+  const po = (a00 + a11) / total;
+  const pAuto = (a10 + a11) / total,
+    pHuman = (a01 + a11) / total;
+  const pe = pAuto * pHuman + (1 - pAuto) * (1 - pHuman);
+  const kappa = pe < 1 ? (po - pe) / (1 - pe) : 1;
+  return {
+    gold_n: goldN,
+    precision: +precision.toFixed(3),
+    recall: +recall.toFixed(3),
+    f1: +f1.toFixed(3),
+    cohens_kappa: +kappa.toFixed(3),
+  };
+}
+
+// ───────────────────────────── example variants ─────────────────────────
+// each mechanism is EXECUTED against pre-f context; it may ONLY return tokens retrievable
+// from the truncated transcript state — it can never hand-supply the resolving delta.
+
+// V1: fire at thrash ONSET — the moment the last two prompts form a same-target re-engagement.
+const V1_onset = {
+  name: "V1 fire-at-onset (last-2 re-engagement)",
+  trigger: (ctx) => {
+    const n = ctx.prompts.length;
+    if (n < 2) return false;
+    const a = ctx.vecs[n - 2],
+      b = ctx.vecs[n - 1];
+    const ta = ctx.tgts[n - 2],
+      tb = ctx.tgts[n - 1];
+    const sim = cosine(a, b);
+    const jac = jaccard(ta, tb);
+    const attempted =
+      ctx.prompts[n - 1].attemptBefore.tool || ctx.prompts[n - 1].attemptBefore.hadText;
+    const corrective =
+      hasCue(ctx.prompts[n - 1].text, CORRECTIVE_CUES) ||
+      sim >= DEFAULTS.tau_dup ||
+      jac >= DEFAULTS.tau_tgt;
+    const accept =
+      hasCue(ctx.prompts[n - 1].text, ACCEPT_CUES) &&
+      !hasCue(ctx.prompts[n - 1].text, CORRECTIVE_CUES);
+    return (
+      attempted && (sim >= DEFAULTS.tau_sim || jac >= DEFAULTS.tau_tgt) && corrective && !accept
+    );
+  },
+  // mechanism: re-surface the high-IDF context of the ORIGINAL ask + all prior on-target messages
+  mechanism: (ctx) => {
+    const out = new Set();
+    for (const p of ctx.prompts) {
+      const tg = targetTuple(p.text, ctx.idf);
+      if (jaccard(tg, ctx.target) > 0 || cosineTargets(tg, ctx.target)) {
+        for (const t of tokenize(p.text)) out.add(t);
+      }
+    }
+    return out;
+  },
+};
+function cosineTargets(a, b) {
+  for (const t of a) if (b.has(t)) return true;
+  return false;
+}
+
+// V2: fire on ANY dissatisfaction cue in the latest prompt (looser — more false fires expected).
+const V2_cue = {
+  name: "V2 fire-on-any-dissatisfaction-cue",
+  trigger: (ctx) => {
+    const n = ctx.prompts.length;
+    if (n < 1) return false;
+    return hasCue(ctx.prompts[n - 1].text, CORRECTIVE_CUES);
+  },
+  mechanism: V1_onset.mechanism,
+};
+
+// V3: fire on repeated target regardless of cue (target Jaccard high vs previous).
+const V3_repeat = {
+  name: "V3 fire-on-repeated-target (no cue required)",
+  trigger: (ctx) => {
+    const n = ctx.prompts.length;
+    if (n < 2) return false;
+    return jaccard(ctx.tgts[n - 2], ctx.tgts[n - 1]) >= DEFAULTS.tau_tgt;
+  },
+  mechanism: V1_onset.mechanism,
+};
+
+// reference posts
+const FIRE_ALWAYS = { name: "REF fire-always", trigger: () => true, mechanism: V1_onset.mechanism };
+const FIRE_NEVER = { name: "REF fire-never", trigger: () => false, mechanism: () => new Set() };
+
+// ───────────────────────────── self-tests (synthetic, prove the machinery) ───────────────
+function mkUser(text, ts) {
+  return {
+    type: "user",
+    message: { content: text },
+    timestamp: new Date(ts).toISOString(),
+    uuid: "u" + ts,
+  };
+}
+function mkTool(ts) {
+  return {
+    type: "assistant",
+    message: { content: [{ type: "tool_use", name: "Bash", input: {} }] },
+    timestamp: new Date(ts).toISOString(),
+  };
+}
+function mkToolResult(ts) {
+  return {
+    type: "user",
+    message: { content: [{ type: "tool_result", content: "ok" }] },
+    timestamp: new Date(ts).toISOString(),
+  };
+}
+
+export function runSelfTests() {
+  const assert = (cond, msg) => {
+    if (!cond) throw new Error("SELFTEST FAIL: " + msg);
+    console.log("  ok:", msg);
+  };
+  let t = Date.now();
+  const step = () => (t += 60000);
+
+  // TEST 1 — noise filter: tool_result, slash-command, task-notification, skalpel block all dropped.
+  const noisy = [
+    mkUser("<local-command-stdout>build output</local-command-stdout>", step()),
+    mkToolResult(step()),
+    mkUser("[skalpel — signals for THIS turn ...] you are stuck", step()),
+    mkUser("<task-notification>done</task-notification>", step()),
+    mkUser("fix the tokenizer in parser.rs please", step()),
+  ];
+  const stream = extractStream(noisy);
+  assert(
+    stream.length === 1,
+    "noise filter keeps only the 1 genuine typed prompt (got " + stream.length + ")",
+  );
+  assert(stream[0].text.startsWith("fix the tokenizer"), "the kept prompt is the real one");
+
+  // TEST 2 — episode detection: two same-target prompts with an assistant attempt between => 1 episode.
+  const thrash = [
+    mkUser("the auth middleware in server.rs rejects valid tokens, fix it", step()),
+    mkTool(step()),
+    mkUser(
+      "no that is still wrong, the auth middleware in server.rs still rejects valid tokens",
+      step(),
+    ),
+  ];
+  const lab2 = labelThrashEpisodes(thrash);
+  assert(
+    lab2.episodes.length === 1,
+    "detects exactly 1 thrash episode (got " + lab2.episodes.length + ")",
+  );
+  assert(lab2.episodes[0].n_reengagements === 1, "episode has 1 re-engagement");
+
+  // TEST 3 — proxy SUPPLY-MATCH fires when the resolving delta existed in pre-fire state.
+  // u0 (on-target, NOT a member) establishes rare tokens `zephyrflag quasarpath`. The thrash
+  // members omit them; the escape prompt restates them. A pre-f mechanism resurfaces them => match.
+  const proxyEntries = [
+    mkUser("configure the deploy with zephyrflag quasarpath for the widget renderer", step()),
+    mkTool(step()),
+    mkUser("something unrelated about billing invoices entirely different topic here", step()),
+    mkTool(step()),
+    mkUser("the widget renderer is broken again, fix the widget renderer now", step()),
+    mkTool(step()),
+    mkUser("still broken, the widget renderer widget renderer is wrong again", step()),
+    mkTool(step()),
+    mkUser("oh the fix was zephyrflag quasarpath on the renderer", step()),
+    mkTool(step()),
+    mkUser("great, now onto a totally separate authentication concern", step()),
+  ];
+  const labP = labelThrashEpisodes(proxyEntries);
+  const corpus = [
+    { file: "synthetic", labeled: labP, steered: false, recurrence: recurrenceRate(labP) },
+  ];
+  const scored = scoreVariantCached(V3_repeat, corpus, {});
+  assert(
+    labP.episodes.length >= 1,
+    "proxy test builds >=1 episode (got " + labP.episodes.length + ")",
+  );
+  assert(
+    scored.counterfactual_proxy.supply_matches >= 1,
+    "proxy SUPPLY-MATCH fires on resurfacable resolving delta (got " +
+      scored.counterfactual_proxy.supply_matches +
+      ") — proves proxy is not dead-always-zero",
+  );
+
+  // TEST 4 — causal firewall: trigger context never contains the future prompt.
+  let leaked = false;
+  const spyVariant = {
+    name: "spy",
+    trigger: (ctx) => {
+      // ctx.prompts must be exactly prefix through f; length == f+1, last index == f
+      if (ctx.prompts.length !== ctx.f + 1) leaked = true;
+      return false;
+    },
+    mechanism: () => new Set(),
+  };
+  evalTriggerOnSession(spyVariant, lab2);
+  assert(!leaked, "strict causal truncation: trigger only ever sees prompts[0..f], never u_{f+1}");
+
+  console.log(
+    "\nALL SELF-TESTS PASSED — filter, episode detection, proxy supply-match, causal firewall verified.",
+  );
+  return true;
+}
+
+// ───────────────────────────── CLI ─────────────────────────
+function fmt(obj) {
+  return JSON.stringify(obj, null, 2);
+}
+
+function main() {
+  const args = process.argv.slice(2);
+  if (!args.length) {
+    console.error(
+      "usage: node eval-harness.mjs <transcriptsDir> [--max-sessions N] [--gold gold.json]",
+    );
+    console.error("       node eval-harness.mjs --selftest   (run synthetic machinery checks)");
+    process.exit(1);
+  }
+  if (args[0] === "--selftest") {
+    runSelfTests();
+    return;
+  }
+  const dir = args[0];
+  const maxIdx = args.indexOf("--max-sessions");
+  const goldIdx = args.indexOf("--gold");
+  const maxSessions = maxIdx >= 0 ? parseInt(args[maxIdx + 1], 10) : undefined;
+  const goldPath = goldIdx >= 0 ? args[goldIdx + 1] : null;
+  const cfg = { ...DEFAULTS, maxSessions };
+
+  const t0 = Date.now();
+  console.log("═".repeat(90));
+  console.log("SKALPEL INTERVENTION EVAL HARNESS — offline FILTER/RANK, NOT lift");
+  console.log("═".repeat(90));
+  console.log(
+    "HONESTY BANNER: TRIGGER metrics + COUNTERFACTUAL PROXY are OFFLINE filter/rank signals.\n" +
+      "  They are NOT lift — firing changes the future; the transcript is off-policy. needs_live_ab = TRUE.\n" +
+      "  Every number below traces to a real transcript event. Proxy is labelled 'proxy (needs live A/B)'.",
+  );
+  console.log(
+    "Transcripts dir:",
+    dir,
+    maxSessions ? `(capped to ${maxSessions} sessions)` : "(all sessions)",
+  );
+
+  // calibration
+  console.log("\n" + "─".repeat(90));
+  console.log("AUTO-LABELER CALIBRATION vs HUMAN GOLD");
+  console.log("─".repeat(90));
+  let cal = null;
+  if (goldPath) cal = calibrate(goldPath, dir, cfg);
+  if (!cal) {
+    console.log(
+      "UNCALIBRATED: no human gold set supplied (--gold). Auto-labels are NOT ground truth.\n" +
+        "  Downstream trigger/proxy numbers are UNTRUSTWORTHY as absolutes — use for RELATIVE variant\n" +
+        "  ranking only, and calibrate before any decision. (gold format: [{file, onset_indices:[...]}, ...])",
+    );
+  } else {
+    console.log(fmt(cal));
+    if (cal.cohens_kappa < DEFAULTS.kappa_min)
+      console.log(
+        `\n*** UNCALIBRATED: kappa ${cal.cohens_kappa} < ${DEFAULTS.kappa_min} — labels untrustworthy. ***`,
+      );
+  }
+
+  // ONE disk pass: load + label the whole corpus, then score every variant from cache.
+  const corpus = labelCorpus(dir, cfg);
+
+  // baseline labeling + reference recurrence
+  console.log("\n" + "─".repeat(90));
+  console.log(
+    "BASELINE: thrash episodes labelled across the REAL transcripts + reference recurrence",
+  );
+  console.log("─".repeat(90));
+  const baseline = scoreVariantCached(FIRE_NEVER, corpus, cfg);
+  console.log("Corpus counts:", fmt(baseline.corpus));
+  console.log("Reference recurrence (number to beat):", fmt(baseline.recurrence_baseline));
+  console.log("Labeler thresholds (UNCALIBRATED defaults):", fmt(baseline.thresholds));
+
+  // variants
+  const variants = [V1_onset, V2_cue, V3_repeat, FIRE_ALWAYS, FIRE_NEVER];
+  console.log("\n" + "─".repeat(90));
+  console.log(
+    "VARIANT SCORES (relative ranking — reference posts fire-always/fire-never included)",
+  );
+  console.log("─".repeat(90));
+  for (const v of variants) {
+    const s = scoreVariantCached(v, corpus, cfg);
+    console.log("\n### " + v.name);
+    console.log("  trigger:", fmt(s.trigger).replace(/\n/g, "\n  "));
+    console.log("  counterfactual_proxy:", fmt(s.counterfactual_proxy).replace(/\n/g, "\n  "));
+  }
+
+  console.log("\n" + "═".repeat(90));
+  console.log(`Runtime: ${((Date.now() - t0) / 1000).toFixed(1)}s`);
+  console.log(
+    "VERDICT POLICY: no 'winner' is declared here. A proxy/trigger lead only FILTERS+RANKS variants;\n" +
+      "  a variant must clear a LIVE A/B before any lift claim. needs_live_ab = TRUE for every variant.",
+  );
+  console.log("═".repeat(90));
+}
+
+const isMain =
+  process.argv[1] &&
+  path.resolve(process.argv[1]) === path.resolve(new URL(import.meta.url).pathname);
+if (isMain) main();
+
+export { V1_onset, V2_cue, V3_repeat, FIRE_ALWAYS, FIRE_NEVER, calibrate };