skalpel 4.0.38 → 4.0.39

package/analytics.mjs

@@ -13,7 +13,7 @@
 //   • NO FABRICATED / ESTIMATED NUMBERS. Every figure reproduces from a real logged row:
 //       - caught lies   = ledger FALSE rows (mismatch:true) in verify-shadow.log
 //       - measured cost = Σ of (later same-session same-proof PASS ts − caught-FALSE ts) — real
-//         timestamps via measureFixDelta; a catch with no later PASS contributes NOTHING
+//         timestamps via buildFixDeltas; a catch with no later PASS contributes NOTHING
 //       - steers        = count of valid JSON lines in ~/.skalpel/steers.ndjson (the append-only
 //         STEERS SOURCE CONTRACT log; one atomic line per APPLIED steer) — a COUNT only; the banned
 //         "minutes saved" metric never appears
@@ -38,7 +38,7 @@ import { emitKeypressEvents } from "node:readline";
 // READ-ONLY reuse of the honest engine. ledger.mjs derives everything from verify-shadow.log;
 // verify-shadow.mjs single-sources the log path + the fail-count extractor. Importing either runs
 // nothing (their CLI blocks are isMain-guarded).
-import { readLedgerRows, buildLedger, measureFixDelta, formatDuration } from "./ledger.mjs";
+import { readLedgerRows, buildLedger, formatDuration } from "./ledger.mjs";
 import { VERIFY_LOG_PATH, extractFailCount } from "./verify-shadow.mjs";
 
 // ---------- palette (the same vocabulary as ledger.mjs / card.mjs / the statusline) ----------
@@ -177,15 +177,21 @@ export function collectData({
     ledger = null;
   }
   // Caught lies, newest first (the navigable list). Sanitize once here — content is untrusted.
+  // fix_delta_ms/fixed_at are REUSED from buildLedger (which already measured them via buildFixDeltas in
+  // one indexed pass) — no per-entry re-scan. The old `?? measureFixDelta(...)` fallbacks were dead weight:
+  // buildLedger already sets both to a number-or-null, so the fallback only ever fired on a null (a catch
+  // with no later PASS) and recomputed the very same null at O(rows) cost. `ts` is sanitized here too
+  // (Finding 1): it is the only ledger-row field rendered raw in the detail view, so it goes through the
+  // same clean() as claim_text/proof_command/evidence. A normal ISO ts is unchanged by clean().
   const catches = (ledger ? [...ledger.falseEntries] : []).reverse().map((e) => ({
-    ts: e.ts,
+    ts: clean(e.ts, 40),
     session: e.session,
     claim_text: clean(e.claim_text, 2000),
     proof_command: clean(e.proof_command, 400),
     evidence: clean(e.evidence, 2000),
     fail_count: e.fail_count,
-    fix_delta_ms: e.fix_delta_ms ?? measureFixDelta(e._row, rows || [])?.ms ?? null,
-    fixed_at: e.fixed_at ?? measureFixDelta(e._row, rows || [])?.atTs ?? null,
+    fix_delta_ms: e.fix_delta_ms ?? null,
+    fixed_at: e.fixed_at ?? null,
   }));
   // Measured cost: Σ of REAL red→green deltas. A catch with no later same-session same-proof PASS
   // contributes NOTHING (never estimated).
@@ -279,9 +285,32 @@ export function renderStatic(data) {
 }
 
 // ---------- (4) interactive rendering helpers ----------
+// East-Asian Wide / Fullwidth BMP code points occupy 2 terminal cells. The old clipAnsi budgeted these
+// (CJK/Hangul/Kana/fullwidth, all ≤ 0xFFFF) as width 1 and so under-clipped — e.g. a wall of "中" ran
+// twice as wide as the terminal. This is the standard compact wcwidth "wide" range set (no dependency):
+// Hangul Jamo, CJK radicals→symbols, Hiragana→CJK-compat, CJK Ext-A, CJK Unified, Yi, Hangul syllables,
+// CJK-compat ideographs, vertical/compat/small forms, and the Fullwidth Forms block.
+function isWideBMP(cp) {
+  return (
+    (cp >= 0x1100 && cp <= 0x115f) || // Hangul Jamo
+    (cp >= 0x2e80 && cp <= 0x303e) || // CJK Radicals … CJK Symbols/Punctuation (wide part)
+    (cp >= 0x3041 && cp <= 0x33ff) || // Hiragana, Katakana, Bopomofo, Hangul Compat Jamo … CJK Compat
+    (cp >= 0x3400 && cp <= 0x4dbf) || // CJK Unified Ideographs Extension A
+    (cp >= 0x4e00 && cp <= 0x9fff) || // CJK Unified Ideographs
+    (cp >= 0xa000 && cp <= 0xa4cf) || // Yi Syllables + Radicals
+    (cp >= 0xac00 && cp <= 0xd7a3) || // Hangul Syllables
+    (cp >= 0xf900 && cp <= 0xfaff) || // CJK Compatibility Ideographs
+    (cp >= 0xfe10 && cp <= 0xfe19) || // Vertical Forms
+    (cp >= 0xfe30 && cp <= 0xfe6f) || // CJK Compatibility Forms + Small Form Variants
+    (cp >= 0xff00 && cp <= 0xff60) || // Fullwidth Forms (fullwidth ASCII, punctuation)
+    (cp >= 0xffe0 && cp <= 0xffe6) //   Fullwidth signs (¢ £ ¥ ₩ …)
+  );
+}
+
 // Visible-width-aware clip that PRESERVES our own CSI sequences (content is already sanitized, so
-// any escape in the string is ours). Astral code points (e.g. 🔬) budget 2 cells. Always ends with
-// RESET when styling was present so a clipped line can never bleed style into the next.
+// any escape in the string is ours). Astral code points (e.g. 🔬) budget 2 cells, as do East-Asian
+// Wide/Fullwidth BMP glyphs (isWideBMP). Always ends with RESET when styling was present so a clipped
+// line can never bleed style into the next.
 export function clipAnsi(line, max) {
   const s = String(line);
   let out = "";
@@ -302,7 +331,7 @@ export function clipAnsi(line, max) {
     }
     const cp = s.codePointAt(i);
     const ch = String.fromCodePoint(cp);
-    const w = cp > 0xffff ? 2 : 1;
+    const w = cp > 0xffff || isWideBMP(cp) ? 2 : 1;
     if (vis + w > max) break;
     out += ch;
     vis += w;

package/ledger.mjs

@@ -24,9 +24,15 @@
 //     logged PASS. If no such later PASS exists, we show the count ONLY — never an estimate, never a
 //     "minutes saved / lost" tally (that metric stays DEAD).
 //   • FAIL-OPEN. Any read/parse error yields an honest zero-state, never a throw at the user.
-import { readFileSync } from "node:fs";
+import { openSync, closeSync } from "node:fs";
 import { fileURLToPath } from "node:url";
 import { realpathSync } from "node:fs";
+// Bounded tail read (same helper the per-turn hook + statusline use) so a pathological/tampered/
+// writer-regressed log can NEVER freeze this reader. transcript.mjs imports only node:fs (no heavy graph).
+import { tailLines } from "./transcript.mjs";
+// cleanText strips terminal-control chars (C0 incl. ESC, DEL, C1). Used to sanitize the untrusted `ts`
+// field at the source (Finding 1 defense-in-depth); it passes a normal ISO string through unchanged.
+import { cleanText } from "./insights.mjs";
 // READ-ONLY reuse of the shadow's single-sourced log path + its honest fail-count extractor. Importing
 // verify-shadow.mjs runs no side effects (its CLI block is guarded on isMain) and modifies nothing there.
 import { VERIFY_LOG_PATH, extractFailCount } from "./verify-shadow.mjs";
@@ -35,16 +41,28 @@ import { VERIFY_LOG_PATH, extractFailCount } from "./verify-shadow.mjs";
 // One NDJSON row per logged claim:
 //   { ts, session, claim_text, proof_command|null, pass_fail: "PASS"|"FAIL"|"HARNESS_ERROR"|null,
 //     outcome: "PASS"|"GENUINE_FAIL"|"HARNESS_ERROR", mismatch: boolean, evidence }
-// The file is size-capped by the shadow (newest rows kept), so a plain read is safe.
+// The shadow self-trims the log to ~1MB/500 lines, so the retained window is small. But this reader must
+// never TRUST that: a pathological, tampered, or writer-regressed log could grow without bound and a full
+// readFileSync would freeze the reader (and, via the statusline path, the always-on bar). So we read only
+// the last LEDGER_READ_BYTES via the shared bounded tail read. 2MB is >2× the writer's 1MB cap, so
+// legitimate data is never dropped — the cap only bounds pathological growth. The "cumulative caught"
+// total is therefore over the RETAINED log window (already true — the log rotates); the cap only adds a
+// hard ceiling so growth beyond the writer's contract can never stall a read.
+export const LEDGER_READ_BYTES = 2 * 1024 * 1024;
 export function readLedgerRows(path = VERIFY_LOG_PATH) {
-  let raw;
+  // Distinguish "no shadow log yet" (→ null, honest zero-state) from "log exists but empty" (→ []).
+  // tailLines returns [] for BOTH, so probe first — exactly as origin/main's readFileSync did: a missing
+  // file (ENOENT) OR an existing-but-unreadable file (EACCES) both throw here → null; a readable file
+  // (empty or not) proceeds to the bounded read (empty → []).
   try {
-    raw = readFileSync(path, "utf8");
+    closeSync(openSync(path, "r"));
   } catch {
-    return null; // no shadow log yet → honest zero-state (distinct from "log exists but empty")
+    return null;
   }
   const rows = [];
-  for (const line of raw.split("\n")) {
+  // Bounded: even a 30MB log reads only the last 2MB, so the reader can never freeze. tailLines drops the
+  // partial first line and never throws (any read error → [], i.e. a fail-open empty ledger).
+  for (const line of tailLines(path, LEDGER_READ_BYTES)) {
     const l = line.trim();
     if (!l) continue;
     try {
@@ -65,27 +83,66 @@ function tsMs(ts) {
   return Number.isFinite(n) ? n : null;
 }
 
-// measureFixDelta(falseRow, rows) → { ms, atTs } for the EARLIEST later row that is, in the SAME session,
-// the SAME proof_command, and a genuine PASS whose timestamp is strictly after the caught-false claim.
-// That is the real moment the false "done" became a true "done": the exact proof that failed later passed.
-// Returns null when no such row exists (then the ledger shows the count only — never an invented minute).
-export function measureFixDelta(falseRow, rows) {
-  const t0 = tsMs(falseRow.ts);
-  if (t0 == null) return null;
-  const session = falseRow.session ?? null;
-  const proof = falseRow.proof_command || null;
-  if (!proof) return null; // can't tie a fix to a claim with no proof — count only
-  let best = null;
-  for (const r of rows) {
-    if (r === falseRow) continue;
-    if ((r.session ?? null) !== session) continue;
-    if ((r.proof_command || null) !== proof) continue;
-    if (r.outcome !== "PASS") continue; // the SAME proof genuinely passing = the real done
-    const t1 = tsMs(r.ts);
-    if (t1 == null || t1 <= t0) continue; // must be strictly LATER than the false claim
-    if (best == null || t1 < best) best = t1;
+// Binary search: the smallest element of an ascending array STRICTLY greater than x, or null. (upper_bound)
+function firstGreater(sorted, x) {
+  let lo = 0;
+  let hi = sorted.length;
+  while (lo < hi) {
+    const mid = (lo + hi) >> 1;
+    if (sorted[mid] > x) hi = mid;
+    else lo = mid + 1;
   }
-  return best == null ? null : { ms: best - t0, atTs: new Date(best).toISOString() };
+  return lo < sorted.length ? sorted[lo] : null;
+}
+
+// buildFixDeltas(rows, opts) — the O(n log n) replacement for the old O(falseCount × rows) per-row scan.
+// ONE pass indexes every genuine PASS by (session → proof_command) into an ascending timestamp array;
+// the returned closure answers each caught-false row by binary-searching the EARLIEST PASS ts strictly
+// later than that row's ts. Byte-for-byte the same result as the old per-row measureFixDelta:
+//   • earliest later, SAME session, SAME proof_command, outcome === "PASS", ts strictly > the false ts
+//   • no proof on the false row → null (can't tie a fix to a proofless claim)
+//   • unparseable false-row ts → null
+// The old loop skipped `r === falseRow`, but a false row is a GENUINE_FAIL (never outcome:"PASS"), and
+// even in a tampered log its own ts equals t0, which the strict `> t0` search already excludes — so the
+// self-skip was a no-op and omitting it changes nothing (proven by the equivalence fixtures).
+// opts.excludeNullSession — when true, a false row with a null session yields null (the statusline's
+// Finding-3 guard). ledger.mjs's own view keeps the historical behavior (null-session rows CAN pair with
+// other null-session rows) by leaving it false, so its results stay byte-identical to the old code.
+export function buildFixDeltas(rows, { excludeNullSession = false } = {}) {
+  const index = new Map(); // (session ?? null) → Map<proof_command, ascending ts[]>
+  for (const r of rows || []) {
+    if (r.outcome !== "PASS") continue; // only a genuine PASS can be the "became true done" moment
+    const proof = r.proof_command || null;
+    if (!proof) continue; // a false row with a proof can never pair with a proofless PASS row
+    const t = tsMs(r.ts);
+    if (t == null) continue; // matches the old loop's `t1 == null` skip
+    const session = r.session ?? null;
+    let byProof = index.get(session);
+    if (!byProof) {
+      byProof = new Map();
+      index.set(session, byProof);
+    }
+    const arr = byProof.get(proof);
+    if (arr) arr.push(t);
+    else byProof.set(proof, [t]);
+  }
+  for (const byProof of index.values())
+    for (const arr of byProof.values()) arr.sort((a, b) => a - b);
+
+  // { ms, atTs } for the earliest later same-session same-proof genuine PASS, or null. Same contract as
+  // the old measureFixDelta — returns the count-only signal (null) when no real later PASS exists.
+  return function fixDeltaFor(falseRow) {
+    const t0 = tsMs(falseRow.ts);
+    if (t0 == null) return null;
+    const session = falseRow.session ?? null;
+    if (excludeNullSession && session == null) return null;
+    const proof = falseRow.proof_command || null;
+    if (!proof) return null; // can't tie a fix to a claim with no proof — count only
+    const arr = index.get(session)?.get(proof);
+    if (!arr) return null;
+    const best = firstGreater(arr, t0); // earliest PASS strictly LATER than the false claim
+    return best == null ? null : { ms: best - t0, atTs: new Date(best).toISOString() };
+  };
 }
 
 // ---------- (3) build the ledger view from the rows ----------
@@ -100,7 +157,11 @@ export function buildLedger(rows) {
     else if (r.pass_fail === "PASS") verdict = "verified";
     else verdict = "unverifiable";
     return {
-      ts: r.ts || null,
+      // Finding 1 (defense-in-depth): `ts` is an UNTRUSTED log field that flows to a live terminal (the
+      // analytics detail view). Sanitize it at the source through the same control-char stripper as the
+      // other rendered fields so a tampered ts can't inject ANSI/CSI escapes. A normal ISO ts is
+      // unaffected — cleanText strips only C0/C1/DEL, which an ISO string contains none of.
+      ts: cleanText(r.ts) || null,
       session: r.session ?? null,
       claim_text: r.claim_text || "",
       // Ship-status rows (category:"ship") record the check under probe_command, not proof_command —
@@ -117,9 +178,11 @@ export function buildLedger(rows) {
   // rows (no proof / harness noise) are logged but NOT counted as checked — we can't stand behind them.
   const checked = entries.filter((e) => e.verdict === "FALSE" || e.verdict === "verified").length;
   const falseEntries = entries.filter((e) => e.verdict === "FALSE");
-  // Attach the MEASURED fix delta where — and only where — a real later same-proof PASS exists.
+  // Attach the MEASURED fix delta where — and only where — a real later same-proof PASS exists. ONE index
+  // pass (buildFixDeltas) then a binary-search per false row: O(n log n), not the old O(falseCount × rows).
+  const fixDeltaFor = buildFixDeltas(rows || []);
   for (const e of falseEntries) {
-    const d = measureFixDelta(e._row, rows || []);
+    const d = fixDeltaFor(e._row);
     e.fix_delta_ms = d ? d.ms : null;
     e.fixed_at = d ? d.atTs : null;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skalpel",
-  "version": "4.0.38",
+  "version": "4.0.39",
   "description": "Behavioral graph for AI-assisted coding — learns how you work and steers Claude Code + Codex in real time.",
   "type": "module",
   "bin": {

package/skalpel-statusline.mjs

@@ -15,7 +15,7 @@
 //     TIME clause is chosen by BASELINE_TIME_MODE (Ryan flips one constant):
 //       · <Xm> caught — the CUMULATIVE LIFETIME sum of every real MEASURED red→green delta (each caught
 //         false "done" and the moment that SAME proof, that SAME session, was later logged PASS — real
-//         timestamps; reuses ledger.mjs's measureFixDelta logic). ALWAYS-ON once real catches exist (it is
+//         timestamps; reuses ledger.mjs's buildFixDeltas logic). ALWAYS-ON once real catches exist (it is
 //         skalpel's terminal presence — never gated/hidden). A measured total, NEVER an estimate; a catch
 //         with no later PASS adds 0. Cumulative framing (not present-tense) keeps it honest for old data.
 //       · ~Xm saved — Raahil's #609 estimate: (applied steers) × (the user's OWN median loop cost from
@@ -190,6 +190,13 @@ function verdictNote() {
 // garbage/partial lines are skipped; the sum only ever adds ≥0 deltas, so it is honest and never negative.
 function readState() {
   try {
+    // Read the WHOLE file — the count MUST stay exact. Unlike verify-shadow.log (writer-trimmed to
+    // ~1MB/500 lines), steers.ndjson is APPEND-ONLY with NO writer-side rotation (see the STEERS SOURCE
+    // CONTRACT in skalpel-hook.mjs). So a reader byte-cap has no safety margin: past the cap it would
+    // FREEZE the "N steers" count while the true count keeps growing — a false, dishonest number on the
+    // always-on bar. An exact O(n) line count is cheap (KB in practice; even a 20MB file reads in ~135ms,
+    // no freeze), and the O(n²) hot-path cost that actually mattered lives in the verify-shadow.log delta
+    // scan, already fixed by buildFixDeltas. So: exact whole-file count here, matching origin/main.
     const raw = readFileSync(join(homedir(), ".skalpel", "steers.ndjson"), "utf8");
     let steers = 0;
     let savedMin = 0;
@@ -227,7 +234,7 @@ function savedPhrase(min) {
 }
 
 // ── MEASURED red→green delta (real timestamps only — the honest replacement for the banned "~Xh saved").
-// tsMs + measureFixDelta are the EXACT logic from ledger.mjs, replicated here (not imported) so the hot
+// tsMs + buildFixDeltas are the EXACT logic from ledger.mjs, replicated here (not imported) so the hot
 // statusline path never pulls verify-shadow.mjs's heavy graph. Parse an ISO ts to epoch ms, or null.
 function tsMs(ts) {
   if (typeof ts !== "string") return null;
@@ -235,31 +242,62 @@ function tsMs(ts) {
   return Number.isFinite(n) ? n : null;
 }
 
-// measureFixDelta(falseRow, rows) → { ms, atTs } for the EARLIEST later row that is, in the SAME session,
-// the SAME proof_command, and a genuine PASS whose timestamp is strictly after the caught-false claim.
-// That is the real moment the false "done" became a true "done": the exact proof that failed later passed.
-// Returns null when no such row exists (then we show NO time — never an invented minute). [ledger.mjs]
-function measureFixDelta(falseRow, rows) {
-  const t0 = tsMs(falseRow.ts);
-  if (t0 == null) return null;
-  const session = falseRow.session ?? null;
-  // Finding 3: a REAL session id is required to tie a fix to a claim. With a null session, the
-  // `(r.session ?? null) !== session` guard below would treat every OTHER null-session row as
-  // "same session" and cross-attribute a bogus delta — so refuse to measure when session is null.
-  if (session == null) return null;
-  const proof = falseRow.proof_command || null;
-  if (!proof) return null; // can't tie a fix to a claim with no proof — count only
-  let best = null;
+// Binary search: the smallest element of an ascending array STRICTLY greater than x, or null. (upper_bound)
+function firstGreater(sorted, x) {
+  let lo = 0;
+  let hi = sorted.length;
+  while (lo < hi) {
+    const mid = (lo + hi) >> 1;
+    if (sorted[mid] > x) hi = mid;
+    else lo = mid + 1;
+  }
+  return lo < sorted.length ? sorted[lo] : null;
+}
+
+// buildFixDeltas(rows) — the O(n log n) replacement for the old O(falseCount × rows) per-row scan, which
+// ran on EVERY bar render (the always-on hot path). ONE pass indexes every genuine PASS by
+// (session → proof_command) into an ascending timestamp array; the returned closure answers each
+// caught-false row by binary-searching the EARLIEST PASS ts strictly later than that row's ts. Byte-for-
+// byte the same result as the old per-row measureFixDelta (earliest later, SAME session, SAME
+// proof_command, outcome === "PASS", ts strictly >; no proof → null; unparseable ts → null).
+// Finding 3 (statusline-only): a REAL session id is required — a null-session false row yields null, so a
+// caught false with no session can't cross-attribute a bogus delta against other null-session rows. This
+// mirrors ledger.mjs buildFixDeltas({ excludeNullSession: true }); replicated inline to keep the hot path
+// import-free. The old loop's `r === falseRow` self-skip was a no-op (a false row is a GENUINE_FAIL, never
+// outcome:"PASS", and its own ts equals t0, which the strict `> t0` search already excludes). [ledger.mjs]
+function buildFixDeltas(rows) {
+  const index = new Map(); // (session ?? null) → Map<proof_command, ascending ts[]>
   for (const r of rows) {
-    if (r === falseRow) continue;
-    if ((r.session ?? null) !== session) continue;
-    if ((r.proof_command || null) !== proof) continue;
-    if (r.outcome !== "PASS") continue; // the SAME proof genuinely passing = the real done
-    const t1 = tsMs(r.ts);
-    if (t1 == null || t1 <= t0) continue; // must be strictly LATER than the false claim
-    if (best == null || t1 < best) best = t1;
+    if (r.outcome !== "PASS") continue;
+    const proof = r.proof_command || null;
+    if (!proof) continue;
+    const t = tsMs(r.ts);
+    if (t == null) continue;
+    const session = r.session ?? null;
+    let byProof = index.get(session);
+    if (!byProof) {
+      byProof = new Map();
+      index.set(session, byProof);
+    }
+    const arr = byProof.get(proof);
+    if (arr) arr.push(t);
+    else byProof.set(proof, [t]);
   }
-  return best == null ? null : { ms: best - t0, atTs: new Date(best).toISOString() };
+  for (const byProof of index.values())
+    for (const arr of byProof.values()) arr.sort((a, b) => a - b);
+
+  return function fixDeltaFor(falseRow) {
+    const t0 = tsMs(falseRow.ts);
+    if (t0 == null) return null;
+    const session = falseRow.session ?? null;
+    if (session == null) return null; // Finding 3: no real session id → refuse to measure (count only)
+    const proof = falseRow.proof_command || null;
+    if (!proof) return null; // can't tie a fix to a claim with no proof — count only
+    const arr = index.get(session)?.get(proof);
+    if (!arr) return null;
+    const best = firstGreater(arr, t0); // earliest PASS strictly LATER than the false claim
+    return best == null ? null : { ms: best - t0, atTs: new Date(best).toISOString() };
+  };
 }
 
 // Compact one-line duration for the bar. Input is a real (t_pass − t_false) MEASURED delta; this only
@@ -300,10 +338,13 @@ function measuredCaughtTotalMs() {
         /* partial / garbage line — skip it, keep the rest (fail-open) */
       }
     }
+    // ONE index pass over all rows, then a binary-search per caught-false row — O(n log n), not the old
+    // O(falseCount × rows) that re-scanned every row for each catch on EVERY bar render.
+    const fixDeltaFor = buildFixDeltas(rows);
     let total = 0;
     for (const r of rows) {
       if (r.mismatch !== true) continue; // only a caught false "done" (verify-shadow's money metric)
-      const d = measureFixDelta(r, rows); // exact ledger.mjs logic; null-session rows excluded (Finding 3)
+      const d = fixDeltaFor(r); // exact ledger.mjs logic; null-session rows excluded (Finding 3)
       if (d && Number.isFinite(d.ms) && d.ms > 0) total += d.ms; // real measured delta only; else +0
     }
     return total;