skalpel 4.0.50 → 4.0.51

package/efficacy-eval.mjs

@@ -390,6 +390,276 @@ export function classifyReask(text) {
   return "none";
 }
 
+// ---------- DRIFT-LOOP detector (a repeated SAME-target correction chain — the loop a dev FEELS) ----------
+// PRECISION IS THE POINT: this must NEVER falsely accuse. Three stacked false-positive guards below —
+// (1) the strong-tier + SOFT-subtraction + anaphora ANCHOR gate, (2) a concrete-noun target the USER
+// themselves named AND that appears in the claim, (3) a per-session verdict gate that SUPPRESSES a run
+// where every member was a genuine true-done (that is scope-growth / flakiness, not drift). Pure + offline.
+//
+// ANCHOR_RE — an ANAPHORA / PRIOR-CLAIM reference. A correction that points BACK at the last "done"
+// ("still", "again", "you didn't…", "that's wrong", "same error", "doesn't work", "nothing changed",
+// "i don't see…"). A bare domain-vocab turn ("handle the login-fail case in login.ts") has NONE of these
+// and is therefore NOT drift-eligible — it is a fresh ask, not a push-back on a prior claim.
+const ANCHOR_RE = new RegExp(
+  [
+    "\\bstill\\b",
+    "\\bagain\\b",
+    "\\b(?:re-?do|redo|undo|revert|roll\\s?back|rollback|re-?open|reopen)\\b",
+    "\\byou\\s+(?:did\\s?n'?t|didnt|did\\s+not|broke|missed|removed|deleted|forgot)\\b",
+    "\\b(?:that|this|it)(?:'?s|\\s+is|\\s+isn'?t|\\s+still|\\s+broke|\\s+breaks)\\b",
+    "\\bsame\\s+(?:error|issue|problem|thing|result|bug|failure)\\b",
+    "\\bnot\\s+(?:fixed|working|done|right|correct|quite|what)\\b",
+    "\\bdoes\\s?n'?t\\s+work\\b",
+    "\\bdo\\s?n'?t\\s+work\\b",
+    "\\bdid\\s?n'?t\\s+work\\b",
+    "\\bnot\\s+work(?:ing)?\\b",
+    "\\bthat'?s\\s+(?:wrong|not\\s+right|incorrect|not\\s+it)\\b",
+    "\\bthis\\s+is\\s+(?:wrong|broken|not\\s+right)\\b",
+    "\\bit'?s\\s+(?:still\\s+)?(?:broken|failing|wrong|not\\s+working)\\b",
+    "\\bnot\\s+what\\s+i\\s+(?:meant|wanted|asked|said|had\\s+in\\s+mind)\\b",
+    "\\bi\\s+hate\\b",
+    "\\bvibe[\\s-]?coded\\b",
+    "\\b(?:ai\\s+slop|slop)\\b",
+    "\\bnothing\\s+chang(?:ed|es)\\b",
+    "\\bdid\\s?n'?t\\s+chang(?:e|ed)\\b",
+    "\\bno\\s+chang(?:e|ed)\\b",
+    "\\bi\\s+(?:do\\s?n'?t|don'?t)\\s+see\\b",
+  ].join("|"),
+  "i",
+);
+
+// isDriftEligibleStrong(text) — the single-sourced precision gate. TRUE iff ALL hold:
+//   (a) classifyReask(text) === 'strong'                 (necessary condition — reuse the offline detector)
+//   (b) SOFT_CONTINUATION_RE.test(text) === false         (SOFT-SUBTRACTION: classifyReask tests STRONG
+//       before SOFT, so a scope-add that also carries a strong keyword — "also, the old test was wrong,
+//       add another" — leaks in as 'strong'. If SOFT also matches, it is a scope-add → EXCLUDE.)
+//   (c) an ANAPHORA / PRIOR-CLAIM anchor is present (ANCHOR_RE).
+export function isDriftEligibleStrong(text) {
+  const s = String(text || "");
+  if (!s.trim()) return false;
+  if (classifyReask(s) !== "strong") return false; // (a)
+  if (SOFT_CONTINUATION_RE.test(s)) return false; // (b) SOFT-subtraction
+  if (!ANCHOR_RE.test(s)) return false; // (c) anaphora / prior-claim anchor
+  return true;
+}
+
+// Generic project-ubiquitous tokens that are NEVER a distinguishing target (case-insensitive drop).
+const DRIFT_GENERIC_DROP = new Set([
+  "npm",
+  "run",
+  "src",
+  "index",
+  "main",
+  "app",
+  "test",
+  "tests",
+  "build",
+  "node",
+  "pnpm",
+  "yarn",
+  "dist",
+  "lib",
+  "the",
+  "fix",
+  "add",
+  "make",
+  "get",
+  "set",
+  "use",
+  "now",
+  "also",
+]);
+const DRIFT_CODE_EXT_RE =
+  /\.(?:ts|tsx|js|jsx|mjs|cjs|py|go|rs|css|scss|json|md|html|sql|sh|yml|yaml|toml)$/i;
+const DRIFT_IDENT_CHARSET_RE = /^[A-Za-z0-9_.-]+$/;
+
+// concreteNouns(text) → unique concrete targets, ORIGINAL CASE preserved (compared case-insensitively).
+// A surviving token MUST be one of: a file path (contains "/" OR a code extension), a URL, a "#<digits>"
+// PR/issue ref, a backtick-/quote-delimited identifier, or a camelCase/snake_case/kebab identifier of
+// length>=3 — never a bare generic lowercase English word. Generic project tokens are then dropped.
+function concreteNouns(text) {
+  const s = String(text || "");
+  const out = [];
+  const seen = new Set(); // lowercase dedup keys
+  const consider = (raw, delimited = false) => {
+    const tok = String(raw || "").trim();
+    if (!tok) return;
+    const key = tok.toLowerCase();
+    if (DRIFT_GENERIC_DROP.has(key)) return;
+    if (seen.has(key)) return;
+    const isUrl = /^https?:\/\//i.test(tok);
+    const isRef = /^#\d+$/.test(tok);
+    const isPath = tok.includes("/") || DRIFT_CODE_EXT_RE.test(tok);
+    const isIdent =
+      tok.length >= 3 &&
+      DRIFT_IDENT_CHARSET_RE.test(tok) &&
+      (/[a-z][A-Z]/.test(tok) || tok.includes("_") || tok.includes("-"));
+    // A delimited (backtick/quote) single token is an explicit user reference — it qualifies even as a
+    // plain lowercase word, but must still be an identifier-ish charset with a letter (never pure prose).
+    const isDelimited = delimited && /^[A-Za-z0-9_./#-]{2,}$/.test(tok) && /[A-Za-z]/.test(tok);
+    if (!(isUrl || isRef || isPath || isIdent || isDelimited)) return;
+    seen.add(key);
+    out.push(tok);
+  };
+  // (1) backtick / single- / double-quote delimited SINGLE tokens (multi-word spans are ignored — an
+  //     identifier has no internal whitespace, and this avoids apostrophe-span false tokens).
+  const delimRe = /`([^`]+)`|'([^']+)'|"([^"]+)"/g;
+  let m;
+  while ((m = delimRe.exec(s)) !== null) {
+    const inner = (m[1] ?? m[2] ?? m[3] ?? "").trim();
+    if (!inner || /\s/.test(inner)) continue;
+    consider(inner, true);
+  }
+  // (2) general whitespace scan — strip surrounding punctuation (keep a leading #/ and inner .-_).
+  const stripEdges = (w) =>
+    String(w || "")
+      .replace(/^[^A-Za-z0-9#/]+/, "")
+      .replace(/[^A-Za-z0-9/]+$/, "");
+  for (const w0 of s.split(/\s+/)) {
+    const w = stripEdges(w0);
+    if (w) consider(w, false);
+  }
+  return out;
+}
+
+// targetTokens(record) — the concrete things the USER named that ALSO appear in the claim (case-insensitive
+// intersection). Display value is taken VERBATIM from the user's turn (next_user_text). If the user named no
+// surviving concrete target, the record cannot anchor a loop → []. This guarantees every emitted target is a
+// concrete noun the user themselves typed, never one invented from the agent's own claim text.
+function targetTokens(record) {
+  const r = record || {};
+  const userNouns = concreteNouns(r.next_user_text);
+  if (!userNouns.length) return [];
+  const claimNouns = concreteNouns((r.claim_text || "") + " " + (r.proof_command || ""));
+  const claimKeys = new Set(claimNouns.map((t) => t.toLowerCase()));
+  const out = [];
+  for (const t of userNouns) {
+    if (claimKeys.has(t.toLowerCase())) out.push(t); // userNouns is already unique + user-case
+  }
+  return out;
+}
+
+// detectDriftLoop(records, opts) — per-session, pure, no network. Walks the drift-eligible corrections in
+// each session and links CONSECUTIVE ones that keep re-naming an overlapping concrete target within a time
+// span, then SUPPRESSES any run whose every member was a genuine true-done. Returns loop objects.
+export function detectDriftLoop(records, opts = {}) {
+  const minRepeat = Number.isFinite(opts.minRepeat) ? opts.minRepeat : 2;
+  const maxSpanMs = Number.isFinite(opts.maxSpanMs) ? opts.maxSpanMs : 30 * 60 * 1000;
+  const IDLE_MS = 10 * 60 * 1000;
+  if (!Array.isArray(records) || !records.length) return [];
+
+  // group by .session, stable input order (= session-transcript order)
+  const order = [];
+  const groups = new Map();
+  for (const r of records) {
+    const key = r && r.session != null ? String(r.session) : "?";
+    if (!groups.has(key)) {
+      groups.set(key, []);
+      order.push(key);
+    }
+    groups.get(key).push(r);
+  }
+
+  const toMap = (tokens) => {
+    const mp = new Map();
+    for (const t of tokens) {
+      const k = t.toLowerCase();
+      if (!mp.has(k)) mp.set(k, t); // keep first (earliest user-turn) display case
+    }
+    return mp;
+  };
+  const intersect = (runMap, tokens) => {
+    const keys = new Set(tokens.map((t) => t.toLowerCase()));
+    const mp = new Map();
+    for (const [k, disp] of runMap) if (keys.has(k)) mp.set(k, disp);
+    return mp;
+  };
+
+  const loops = [];
+  for (const key of order) {
+    // (i) eligible list, in order: evaluated + drift-eligible-strong + has a user-named concrete target
+    const eligible = [];
+    for (const r of groups.get(key)) {
+      if (r && r.reask_status === "evaluated" && isDriftEligibleStrong(r.next_user_text)) {
+        const tokens = targetTokens(r);
+        if (tokens.length > 0) eligible.push({ r, tokens });
+      }
+    }
+    if (!eligible.length) continue;
+
+    // (ii) build runs — extend IFF target overlap AND (span + per-step gap) within window (time checks
+    // skipped only when a ts is missing; a run with any missing ts yields elapsed_ms=null downstream).
+    const runs = [];
+    let run = null;
+    for (const e of eligible) {
+      if (!run) {
+        run = { members: [e], targets: toMap(e.tokens) };
+        continue;
+      }
+      const inter = intersect(run.targets, e.tokens);
+      let extend = inter.size > 0;
+      if (extend) {
+        const tFirst = Date.parse(run.members[0].r.claim_ts);
+        const tPrev = Date.parse(run.members[run.members.length - 1].r.claim_ts);
+        const tNext = Date.parse(e.r.claim_ts);
+        if (Number.isFinite(tFirst) && Number.isFinite(tPrev) && Number.isFinite(tNext)) {
+          if (!(tNext - tFirst <= maxSpanMs && tNext - tPrev <= maxSpanMs)) extend = false;
+        }
+      }
+      if (extend) {
+        run.members.push(e);
+        run.targets = inter;
+      } else {
+        runs.push(run);
+        run = { members: [e], targets: toMap(e.tokens) };
+      }
+    }
+    if (run) runs.push(run);
+
+    // (iii) verdict gate + emit
+    for (const rn of runs) {
+      const repeat = rn.members.length;
+      if (repeat < minRepeat) continue;
+      const verdicts = rn.members.map((mm) => mm.r.verdict);
+      if (verdicts.every((v) => v === "true-done")) continue; // SUPPRESS: scope-growth/flakiness, not drift
+
+      const first_ts = rn.members[0].r.claim_ts ?? null;
+      const last_ts = rn.members[repeat - 1].r.claim_ts ?? null;
+      const fp = Date.parse(first_ts);
+      const lp = Date.parse(last_ts);
+      const elapsed_ms = Number.isFinite(fp) && Number.isFinite(lp) ? lp - fp : null;
+
+      // largest single inter-member claim_ts gap (only over pairs with both ts present)
+      let maxGap = -1;
+      for (let j = 1; j < repeat; j++) {
+        const a = Date.parse(rn.members[j - 1].r.claim_ts);
+        const b = Date.parse(rn.members[j].r.claim_ts);
+        if (Number.isFinite(a) && Number.isFinite(b)) maxGap = Math.max(maxGap, b - a);
+      }
+      const idle_suspect = maxGap > maxSpanMs || maxGap > IDLE_MS;
+
+      // stable target: longest, then lexicographically-first (verbatim user-turn case)
+      const targets = [...rn.targets.values()].sort(
+        (a, b) => b.length - a.length || (a < b ? -1 : a > b ? 1 : 0),
+      );
+
+      loops.push({
+        session: key === "?" ? null : key,
+        target: targets[0],
+        repeat,
+        first_ts,
+        last_ts,
+        elapsed_ms,
+        idle_suspect,
+        members_verdicts: verdicts,
+        next_user_texts: rn.members.map((mm) => String(mm.r.next_user_text || "").slice(0, 300)),
+        turn_targets: rn.members.map((mm) => mm.tokens.slice()),
+      });
+    }
+  }
+  return loops;
+}
+
 // ---------- (3) scan ONE session → per-claim records + the session's PASS proof-runs ----------
 // A record's verdict, deduped by proof tool_use id (identical to scanSession), is:
 //   'confirmed-lie'      — claim over a proof whose recorded result classifies GENUINE_FAIL (a real lie)
@@ -751,6 +1021,8 @@ function recId(r) {
 // reported). The strong rate is the conservative headline; the soft (scope-add) tier is reported apart.
 export function computeReaskReport(corpus) {
   const recs = corpus.records || [];
+  // DRIFT LOOPS — repeated same-target STRONG corrections in a session (pure, from the same records).
+  const driftLoops = detectDriftLoop(recs);
   const totalDone = recs.length;
   const evaluated = recs.filter((r) => r.reask_status === "evaluated");
   const superseded = recs.filter((r) => r.reask_status === "superseded").length;
@@ -837,6 +1109,9 @@ export function computeReaskReport(corpus) {
     passing_proof_reask_rate_strong: passingProof.reask_rate_strong,
     passing_proof_reask_rate_any: passingProof.reask_rate_any,
     by_verdict: byVerdict,
+    // DRIFT LOOPS — repeated same-target correction chains (deterministic; identical to detectDriftLoop(recs)).
+    drift_loops: driftLoops,
+    drift_loop_count: driftLoops.length,
     samples,
   };
 }
@@ -1216,6 +1491,35 @@ export function renderReaskReport(reask, frequency = {}) {
     });
   }
 
+  // ---- DRIFT LOOPS (repeated same-target corrections — the loop you FEEL) ----
+  L.push("");
+  L.push("  5) DRIFT LOOPS — repeated STRONG corrections about the SAME concrete target you named");
+  L.push(
+    "     (a same-file/-symbol re-ask chain; every noun below is re-derived from YOUR own turns)",
+  );
+  L.push("  " + "·".repeat(74));
+  const loops = Array.isArray(R.drift_loops) ? R.drift_loops : [];
+  if ((R.drift_loop_count || 0) === 0) {
+    L.push("     no repeated same-target correction loops found.");
+  } else {
+    L.push(`     ${R.drift_loop_count} drift loop(s) found (showing up to 5):`);
+    loops.slice(0, 5).forEach((lp, i) => {
+      // Neutral wall-clock duration ONLY when we have two real timestamps, the run is not idle-suspect,
+      // and the span is at least a minute. NEVER "wasted/burned" — it is a plain elapsed delta.
+      let dstr = "";
+      if (lp.elapsed_ms != null && !lp.idle_suspect && lp.elapsed_ms >= 60000) {
+        const d = formatDuration(lp.elapsed_ms);
+        if (d) dstr = ` · ${d} wall-clock between first and last correction`;
+      }
+      L.push(
+        `   ${i + 1}. \`${clip(lp.target, 48)}\` — corrected ${lp.repeat}× in this session${dstr}`,
+      );
+      (lp.next_user_texts || []).forEach((t) => {
+        L.push(`       you: "${clip(t, 66)}"`);
+      });
+    });
+  }
+
   L.push("  " + "─".repeat(76));
   L.push("  Read-only replay. --reask --json for the machine-readable re-ask numbers.");
   L.push("");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skalpel",
-  "version": "4.0.50",
+  "version": "4.0.51",
   "description": "Behavioral graph for AI-assisted coding — learns how you work and steers Claude Code + Codex in real time.",
   "type": "module",
   "bin": {