skalpel 4.0.45 → 4.0.47

package/codex-normalize.mjs

@@ -0,0 +1,216 @@
+// codex-normalize.mjs — the ONE Codex-rollout → Claude-transcript normalizer, shared by the retroactive
+// scanner (catches.mjs) AND the LIVE claim-verification worker (verify-shadow.mjs). Codex records a session
+// as a JSONL "rollout" of `response_item` events; Claude records `message.content[]` blocks. verify-shadow's
+// whole engine — claim detection, proof reconstruction, allowlist, classifyOutcome, flake-confirm — speaks
+// the Claude shape. Rather than teach every one of those functions a second dialect, we translate the
+// rollout INTO the Claude shape ONCE, here, and everything downstream is unchanged. A single normalizer
+// means the live catch and the retroactive catch can never drift.
+//
+// RED LINES (inherited, enforced by construction):
+//   • READ-ONLY + fail-open. This only PARSES bytes already on disk. Unparseable args / unknown schema /
+//     missing fields → the record is dropped (no proof), never a throw. A crafted rollout can at worst make
+//     us find NOTHING to verify — never make us execute something we shouldn't (the allowlist + execFile
+//     arg-array in verify-shadow still gate every re-run).
+//   • The recorded exit code Codex already wrote ("Process exited with code N") is carried onto the
+//     normalized tool_result (exit_code / is_error). The live worker uses it as CORROBORATING evidence:
+//     a live re-run FAIL is only ever surfaced when Codex's OWN recorded run of that same command also
+//     failed (recorded-exit-code agreement) — so a Codex catch is strictly MORE precision-gated than a
+//     Claude one, and an environment/flake divergence can never cry wolf.
+import { readdirSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import path from "node:path";
+import { tailLines } from "./transcript.mjs";
+
+// Default per-file tail. Codex rollouts can be multi-GB; a bounded tail keeps normalization constant-time
+// and OOM-proof. Callers pass their own budget (the retroactive scan is generous, the hot live path tighter).
+const DEFAULT_TAIL_BYTES = 4 * 1024 * 1024;
+
+// ---------- detection ----------
+// A Codex rollout is identified by WHERE it lives (under ~/.codex — or $CODEX_HOME) OR by its canonical
+// filename (`rollout-<iso>-<uuid>.jsonl`). Path-based so we never have to read a byte to route. Fail-open:
+// anything ambiguous returns false and the caller treats it as a Claude transcript (the safe default).
+export function isCodexRollout(p) {
+  if (!p || typeof p !== "string") return false;
+  try {
+    const base = p.split(/[/\\]/).pop() || "";
+    if (/^rollout-.*\.jsonl$/.test(base)) return true;
+    const norm = p.replace(/\\/g, "/");
+    if (/(^|\/)\.codex(\/|$)/.test(norm)) return true;
+    const codexHome = process.env.CODEX_HOME;
+    if (codexHome && norm.startsWith(codexHome.replace(/\\/g, "/"))) return true;
+    return false;
+  } catch {
+    return false;
+  }
+}
+
+// The Codex session roots to scan when we must resolve a session ourselves (the hook did not hand us a
+// usable rollout path — see verify-shadow #6). $CODEX_HOME wins; otherwise ~/.codex.
+function codexRoots() {
+  const home = process.env.CODEX_HOME || path.join(homedir(), ".codex");
+  return [path.join(home, "sessions"), path.join(home, "archived_sessions")];
+}
+
+// Bounded newest-first walk for `rollout-*.jsonl`. Used only as the LAST resort when no usable rollout path
+// was provided for a Codex turn. Read-only, fail-open. depth-bounded so a symlink loop can't spin forever.
+function walkRollouts(dir, out, depth = 0) {
+  if (depth > 8) return; // Codex nests sessions/<yyyy>/<mm>/<dd>/ — 8 is plenty and bounds a symlink loop
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  for (const e of entries) {
+    const p = path.join(dir, e.name);
+    try {
+      if (e.isDirectory()) walkRollouts(p, out, depth + 1);
+      else if (e.isFile() && /^rollout-.*\.jsonl$/.test(e.name)) out.push(p);
+    } catch {
+      /* one bad entry never aborts the walk */
+    }
+  }
+}
+
+// newestCodexRollout() → absolute path of the most-recently-modified rollout, or null. Fail-open.
+export function newestCodexRollout() {
+  try {
+    const files = [];
+    for (const d of codexRoots()) walkRollouts(d, files);
+    let best = null;
+    let bestMtime = -Infinity;
+    for (const f of files) {
+      try {
+        const m = statSync(f).mtimeMs;
+        if (m > bestMtime) {
+          bestMtime = m;
+          best = f;
+        }
+      } catch {
+        /* vanished mid-scan — skip */
+      }
+    }
+    return best;
+  } catch {
+    return null;
+  }
+}
+
+// ---------- output parsing ----------
+// A Codex exec output wrapper always carries "Process exited with code N" — the recorded exit status, which
+// is exactly the ground truth we need (no re-run to learn it). Returns { exitCode:number|null, body:string }.
+// A sandbox/harness failure ("exec_command failed: CreateProcess …") has no such line → exitCode null →
+// treated downstream as "no failure signal recorded" (never invented as a failure).
+export function parseCodexOutput(output) {
+  const s = typeof output === "string" ? output : "";
+  // Codex records its exit status as a HEADER line, BEFORE the "\nOutput:\n" body (verified against real
+  // ~/.codex rollouts — the status line always precedes the Output body and reads):
+  //   [Command: …\n]Chunk ID: …\nWall time: … seconds\nProcess exited with code N\nOriginal token count: …\nOutput:\n<body>
+  // The command's OWN stdout/stderr (the body, AFTER "Output:") — OR the echoed "Command:" line — can also
+  // contain a "Process exited with code M" substring (a nested runner/CI-supervisor logging a child's exit, a
+  // grep of this very phrase). Neither is Codex's status. Reading the FIRST match can grab the Command echo;
+  // reading the LAST over the WHOLE string can grab the body. So we bind to Codex's OWN status line: scan the
+  // HEADER region only (everything before "\nOutput:\n") and take the LAST match there — the real status line
+  // always follows the Command echo, and the body is excluded outright. This is the recorded exit code the
+  // agreement gate corroborates against; it can no longer be spoofed by command output — closing the
+  // false-catch path. A thinner wrapper with no "\nOutput:\n" degrades to a last-match over the whole string;
+  // no match at all → null (suppressed downstream, never invented as a failure).
+  const oi = s.indexOf("\nOutput:\n");
+  const header = oi >= 0 ? s.slice(0, oi) : s;
+  let exitCode = null;
+  const re = /Process exited with code\s+(-?\d+)/gi;
+  let m;
+  while ((m = re.exec(header)) !== null) exitCode = parseInt(m[1], 10);
+  // Prefer the text AFTER "Output:" as the evidence body (drops the Command/Chunk/Wall-time header).
+  const body = oi >= 0 ? s.slice(oi + "\nOutput:\n".length) : s;
+  return { exitCode, body };
+}
+
+// ---------- normalization ----------
+// Normalize one Codex rollout file into Claude-shaped entries. exec_command → a Bash tool_use; its
+// function_call_output → a tool_result carrying the recorded exit code; assistant/user messages → text.
+// maxBytes bounds the tail read (see DEFAULT_TAIL_BYTES). Never throws.
+export function loadCodexEntries(file, maxBytes = DEFAULT_TAIL_BYTES) {
+  const entries = [];
+  for (const line of tailLines(file, maxBytes)) {
+    let e;
+    try {
+      e = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (e.type !== "response_item") continue;
+    const p = e.payload || {};
+    if (p.type === "message") {
+      const role = p.role === "assistant" ? "assistant" : "user";
+      const text = Array.isArray(p.content)
+        ? p.content.map((b) => (b && typeof b.text === "string" ? b.text : "")).join("")
+        : "";
+      if (text.trim())
+        entries.push({ type: role, message: { role, content: [{ type: "text", text }] } });
+    } else if (p.type === "function_call" && p.name === "exec_command" && p.call_id) {
+      let cmd = "";
+      let workdir = null;
+      try {
+        const a = JSON.parse(p.arguments || "{}");
+        // A STRING cmd is a shell command verify-shadow's tokenizer parses safely. An ARRAY cmd is argv;
+        // joining it back into a string is LOSSY (quoting is lost — `["sh","-c","a b"]` → `sh -c a b`),
+        // so we DROP it (leave cmd empty → no proof) rather than reconstruct a command that differs from
+        // what actually ran. Fail-open on argv-join quoting doubt (RED LINE #4).
+        cmd = typeof a.cmd === "string" ? a.cmd : "";
+        workdir = typeof a.workdir === "string" ? a.workdir : null;
+      } catch {
+        /* unparseable args — leave cmd empty (no proof) */
+      }
+      if (cmd)
+        entries.push({
+          cwd: workdir,
+          message: {
+            role: "assistant",
+            content: [{ type: "tool_use", name: "Bash", id: p.call_id, input: { command: cmd } }],
+          },
+        });
+    } else if (p.type === "function_call_output" && p.call_id) {
+      const { exitCode, body } = parseCodexOutput(p.output);
+      entries.push({
+        message: {
+          role: "user",
+          content: [
+            {
+              type: "tool_result",
+              tool_use_id: p.call_id,
+              is_error: typeof exitCode === "number" ? exitCode !== 0 : false,
+              exit_code: exitCode,
+              content: body,
+            },
+          ],
+        },
+      });
+    }
+  }
+  return entries;
+}
+
+// ---------- recorded-exit lookup (the Codex precision lever) ----------
+// The recorded exit code Codex wrote for the tool_use `id` (call_id) of a reconstructed proof. This is the
+// CORROBORATING evidence the live worker demands before it surfaces a Codex catch: a live re-run FAIL is
+// only revealed when Codex's OWN recorded run of that exact command ALSO failed. Returns
+// { exitCode:number|null, isError:bool, found:bool }. Scans newest→oldest so the most recent result wins.
+export function recordedExitFor(entries, toolUseId) {
+  if (!toolUseId || !Array.isArray(entries))
+    return { exitCode: null, isError: false, found: false };
+  for (let i = entries.length - 1; i >= 0; i--) {
+    const c = entries[i]?.message?.content;
+    if (!Array.isArray(c)) continue;
+    for (const b of c) {
+      if (b && b.type === "tool_result" && b.tool_use_id === toolUseId) {
+        return {
+          exitCode: typeof b.exit_code === "number" ? b.exit_code : null,
+          isError: b.is_error === true,
+          found: true,
+        };
+      }
+    }
+  }
+  return { exitCode: null, isError: false, found: false };
+}