skalpel 4.0.46 → 4.0.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/catches.mjs +12 -71
- package/codex-normalize.mjs +216 -0
- package/eval-harness.mjs +316 -3
- package/first-run.mjs +5 -2
- package/install.mjs +1 -0
- package/package.json +1 -1
- package/verify-shadow.mjs +263 -1
package/catches.mjs
CHANGED
|
@@ -39,6 +39,10 @@ import {
|
|
|
39
39
|
classifyOutcome,
|
|
40
40
|
extractFailCount,
|
|
41
41
|
} from "./verify-shadow.mjs";
|
|
42
|
+
// ONE normalizer, never two: the Codex rollout → Claude-shape translation lives in codex-normalize.mjs and
|
|
43
|
+
// is shared verbatim with the LIVE verify-shadow worker, so the retroactive scan and the live catch can
|
|
44
|
+
// never drift. (This module keeps only the Claude-native loader below.)
|
|
45
|
+
import { loadCodexEntries } from "./codex-normalize.mjs";
|
|
42
46
|
import { recordInsight } from "./insights.mjs";
|
|
43
47
|
|
|
44
48
|
const HOME = homedir();
|
|
@@ -113,76 +117,10 @@ export function loadClaudeEntries(file) {
|
|
|
113
117
|
return entries;
|
|
114
118
|
}
|
|
115
119
|
|
|
116
|
-
// Codex
|
|
117
|
-
//
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
const m = s.match(/Process exited with code\s+(-?\d+)/i);
|
|
121
|
-
const exitCode = m ? parseInt(m[1], 10) : null;
|
|
122
|
-
// Prefer the text AFTER "Output:" as the evidence body (drops the Command/Chunk/Wall-time header).
|
|
123
|
-
const oi = s.indexOf("\nOutput:\n");
|
|
124
|
-
const body = oi >= 0 ? s.slice(oi + "\nOutput:\n".length) : s;
|
|
125
|
-
return { exitCode, body };
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
// Normalize one Codex rollout file into Claude-shaped entries. exec_command → a Bash tool_use; its
|
|
129
|
-
// function_call_output → a tool_result carrying the recorded exit code; assistant messages → text.
|
|
130
|
-
export function loadCodexEntries(file) {
|
|
131
|
-
const entries = [];
|
|
132
|
-
for (const line of tailLines(file, SESSION_TAIL_BYTES)) {
|
|
133
|
-
let e;
|
|
134
|
-
try {
|
|
135
|
-
e = JSON.parse(line);
|
|
136
|
-
} catch {
|
|
137
|
-
continue;
|
|
138
|
-
}
|
|
139
|
-
if (e.type !== "response_item") continue;
|
|
140
|
-
const p = e.payload || {};
|
|
141
|
-
if (p.type === "message") {
|
|
142
|
-
const role = p.role === "assistant" ? "assistant" : "user";
|
|
143
|
-
const text = Array.isArray(p.content)
|
|
144
|
-
? p.content.map((b) => (b && typeof b.text === "string" ? b.text : "")).join("")
|
|
145
|
-
: "";
|
|
146
|
-
if (text.trim())
|
|
147
|
-
entries.push({ type: role, message: { role, content: [{ type: "text", text }] } });
|
|
148
|
-
} else if (p.type === "function_call" && p.name === "exec_command" && p.call_id) {
|
|
149
|
-
let cmd = "";
|
|
150
|
-
let workdir = null;
|
|
151
|
-
try {
|
|
152
|
-
const a = JSON.parse(p.arguments || "{}");
|
|
153
|
-
cmd = typeof a.cmd === "string" ? a.cmd : Array.isArray(a.cmd) ? a.cmd.join(" ") : "";
|
|
154
|
-
workdir = typeof a.workdir === "string" ? a.workdir : null;
|
|
155
|
-
} catch {
|
|
156
|
-
/* unparseable args — leave cmd empty (no proof) */
|
|
157
|
-
}
|
|
158
|
-
if (cmd)
|
|
159
|
-
entries.push({
|
|
160
|
-
cwd: workdir,
|
|
161
|
-
message: {
|
|
162
|
-
role: "assistant",
|
|
163
|
-
content: [{ type: "tool_use", name: "Bash", id: p.call_id, input: { command: cmd } }],
|
|
164
|
-
},
|
|
165
|
-
});
|
|
166
|
-
} else if (p.type === "function_call_output" && p.call_id) {
|
|
167
|
-
const { exitCode, body } = parseCodexOutput(p.output);
|
|
168
|
-
entries.push({
|
|
169
|
-
message: {
|
|
170
|
-
role: "user",
|
|
171
|
-
content: [
|
|
172
|
-
{
|
|
173
|
-
type: "tool_result",
|
|
174
|
-
tool_use_id: p.call_id,
|
|
175
|
-
is_error: typeof exitCode === "number" ? exitCode !== 0 : false,
|
|
176
|
-
exit_code: exitCode,
|
|
177
|
-
content: body,
|
|
178
|
-
},
|
|
179
|
-
],
|
|
180
|
-
},
|
|
181
|
-
});
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
return entries;
|
|
185
|
-
}
|
|
120
|
+
// Codex rollout normalization (loadCodexEntries / parseCodexOutput) now lives in ./codex-normalize.mjs and
|
|
121
|
+
// is imported above — ONE normalizer shared with the live verify-shadow worker (deleted here to avoid a
|
|
122
|
+
// second, drifting copy). The retroactive scan still reads the RECORDED exit code those entries carry
|
|
123
|
+
// (exit_code / is_error) instead of re-running.
|
|
186
124
|
|
|
187
125
|
// ---------- (3) index the recorded proof results (no re-run) ----------
|
|
188
126
|
function resultTextOf(block) {
|
|
@@ -318,7 +256,10 @@ export function runScan() {
|
|
|
318
256
|
for (const s of sessions) {
|
|
319
257
|
let entries;
|
|
320
258
|
try {
|
|
321
|
-
entries =
|
|
259
|
+
entries =
|
|
260
|
+
s.tool === "codex"
|
|
261
|
+
? loadCodexEntries(s.file, SESSION_TAIL_BYTES)
|
|
262
|
+
: loadClaudeEntries(s.file);
|
|
322
263
|
} catch {
|
|
323
264
|
continue; // unreadable file — skip (read-only, fail-open)
|
|
324
265
|
}
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
// codex-normalize.mjs — the ONE Codex-rollout → Claude-transcript normalizer, shared by the retroactive
|
|
2
|
+
// scanner (catches.mjs) AND the LIVE claim-verification worker (verify-shadow.mjs). Codex records a session
|
|
3
|
+
// as a JSONL "rollout" of `response_item` events; Claude records `message.content[]` blocks. verify-shadow's
|
|
4
|
+
// whole engine — claim detection, proof reconstruction, allowlist, classifyOutcome, flake-confirm — speaks
|
|
5
|
+
// the Claude shape. Rather than teach every one of those functions a second dialect, we translate the
|
|
6
|
+
// rollout INTO the Claude shape ONCE, here, and everything downstream is unchanged. A single normalizer
|
|
7
|
+
// means the live catch and the retroactive catch can never drift.
|
|
8
|
+
//
|
|
9
|
+
// RED LINES (inherited, enforced by construction):
|
|
10
|
+
// • READ-ONLY + fail-open. This only PARSES bytes already on disk. Unparseable args / unknown schema /
|
|
11
|
+
// missing fields → the record is dropped (no proof), never a throw. A crafted rollout can at worst make
|
|
12
|
+
// us find NOTHING to verify — never make us execute something we shouldn't (the allowlist + execFile
|
|
13
|
+
// arg-array in verify-shadow still gate every re-run).
|
|
14
|
+
// • The recorded exit code Codex already wrote ("Process exited with code N") is carried onto the
|
|
15
|
+
// normalized tool_result (exit_code / is_error). The live worker uses it as CORROBORATING evidence:
|
|
16
|
+
// a live re-run FAIL is only ever surfaced when Codex's OWN recorded run of that same command also
|
|
17
|
+
// failed (recorded-exit-code agreement) — so a Codex catch is strictly MORE precision-gated than a
|
|
18
|
+
// Claude one, and an environment/flake divergence can never cry wolf.
|
|
19
|
+
import { readdirSync, statSync } from "node:fs";
|
|
20
|
+
import { homedir } from "node:os";
|
|
21
|
+
import path from "node:path";
|
|
22
|
+
import { tailLines } from "./transcript.mjs";
|
|
23
|
+
|
|
24
|
+
// Default per-file tail. Codex rollouts can be multi-GB; a bounded tail keeps normalization constant-time
|
|
25
|
+
// and OOM-proof. Callers pass their own budget (the retroactive scan is generous, the hot live path tighter).
|
|
26
|
+
const DEFAULT_TAIL_BYTES = 4 * 1024 * 1024;
|
|
27
|
+
|
|
28
|
+
// ---------- detection ----------
|
|
29
|
+
// A Codex rollout is identified by WHERE it lives (under ~/.codex — or $CODEX_HOME) OR by its canonical
|
|
30
|
+
// filename (`rollout-<iso>-<uuid>.jsonl`). Path-based so we never have to read a byte to route. Fail-open:
|
|
31
|
+
// anything ambiguous returns false and the caller treats it as a Claude transcript (the safe default).
|
|
32
|
+
export function isCodexRollout(p) {
|
|
33
|
+
if (!p || typeof p !== "string") return false;
|
|
34
|
+
try {
|
|
35
|
+
const base = p.split(/[/\\]/).pop() || "";
|
|
36
|
+
if (/^rollout-.*\.jsonl$/.test(base)) return true;
|
|
37
|
+
const norm = p.replace(/\\/g, "/");
|
|
38
|
+
if (/(^|\/)\.codex(\/|$)/.test(norm)) return true;
|
|
39
|
+
const codexHome = process.env.CODEX_HOME;
|
|
40
|
+
if (codexHome && norm.startsWith(codexHome.replace(/\\/g, "/"))) return true;
|
|
41
|
+
return false;
|
|
42
|
+
} catch {
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// The Codex session roots to scan when we must resolve a session ourselves (the hook did not hand us a
|
|
48
|
+
// usable rollout path — see verify-shadow #6). $CODEX_HOME wins; otherwise ~/.codex.
|
|
49
|
+
function codexRoots() {
|
|
50
|
+
const home = process.env.CODEX_HOME || path.join(homedir(), ".codex");
|
|
51
|
+
return [path.join(home, "sessions"), path.join(home, "archived_sessions")];
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Bounded newest-first walk for `rollout-*.jsonl`. Used only as the LAST resort when no usable rollout path
|
|
55
|
+
// was provided for a Codex turn. Read-only, fail-open. depth-bounded so a symlink loop can't spin forever.
|
|
56
|
+
function walkRollouts(dir, out, depth = 0) {
|
|
57
|
+
if (depth > 8) return; // Codex nests sessions/<yyyy>/<mm>/<dd>/ — 8 is plenty and bounds a symlink loop
|
|
58
|
+
let entries;
|
|
59
|
+
try {
|
|
60
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
61
|
+
} catch {
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
for (const e of entries) {
|
|
65
|
+
const p = path.join(dir, e.name);
|
|
66
|
+
try {
|
|
67
|
+
if (e.isDirectory()) walkRollouts(p, out, depth + 1);
|
|
68
|
+
else if (e.isFile() && /^rollout-.*\.jsonl$/.test(e.name)) out.push(p);
|
|
69
|
+
} catch {
|
|
70
|
+
/* one bad entry never aborts the walk */
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// newestCodexRollout() → absolute path of the most-recently-modified rollout, or null. Fail-open.
|
|
76
|
+
export function newestCodexRollout() {
|
|
77
|
+
try {
|
|
78
|
+
const files = [];
|
|
79
|
+
for (const d of codexRoots()) walkRollouts(d, files);
|
|
80
|
+
let best = null;
|
|
81
|
+
let bestMtime = -Infinity;
|
|
82
|
+
for (const f of files) {
|
|
83
|
+
try {
|
|
84
|
+
const m = statSync(f).mtimeMs;
|
|
85
|
+
if (m > bestMtime) {
|
|
86
|
+
bestMtime = m;
|
|
87
|
+
best = f;
|
|
88
|
+
}
|
|
89
|
+
} catch {
|
|
90
|
+
/* vanished mid-scan — skip */
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
return best;
|
|
94
|
+
} catch {
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// ---------- output parsing ----------
|
|
100
|
+
// A Codex exec output wrapper always carries "Process exited with code N" — the recorded exit status, which
|
|
101
|
+
// is exactly the ground truth we need (no re-run to learn it). Returns { exitCode:number|null, body:string }.
|
|
102
|
+
// A sandbox/harness failure ("exec_command failed: CreateProcess …") has no such line → exitCode null →
|
|
103
|
+
// treated downstream as "no failure signal recorded" (never invented as a failure).
|
|
104
|
+
export function parseCodexOutput(output) {
|
|
105
|
+
const s = typeof output === "string" ? output : "";
|
|
106
|
+
// Codex records its exit status as a HEADER line, BEFORE the "\nOutput:\n" body (verified against real
|
|
107
|
+
// ~/.codex rollouts — the status line always precedes the Output body and reads):
|
|
108
|
+
// [Command: …\n]Chunk ID: …\nWall time: … seconds\nProcess exited with code N\nOriginal token count: …\nOutput:\n<body>
|
|
109
|
+
// The command's OWN stdout/stderr (the body, AFTER "Output:") — OR the echoed "Command:" line — can also
|
|
110
|
+
// contain a "Process exited with code M" substring (a nested runner/CI-supervisor logging a child's exit, a
|
|
111
|
+
// grep of this very phrase). Neither is Codex's status. Reading the FIRST match can grab the Command echo;
|
|
112
|
+
// reading the LAST over the WHOLE string can grab the body. So we bind to Codex's OWN status line: scan the
|
|
113
|
+
// HEADER region only (everything before "\nOutput:\n") and take the LAST match there — the real status line
|
|
114
|
+
// always follows the Command echo, and the body is excluded outright. This is the recorded exit code the
|
|
115
|
+
// agreement gate corroborates against; it can no longer be spoofed by command output — closing the
|
|
116
|
+
// false-catch path. A thinner wrapper with no "\nOutput:\n" degrades to a last-match over the whole string;
|
|
117
|
+
// no match at all → null (suppressed downstream, never invented as a failure).
|
|
118
|
+
const oi = s.indexOf("\nOutput:\n");
|
|
119
|
+
const header = oi >= 0 ? s.slice(0, oi) : s;
|
|
120
|
+
let exitCode = null;
|
|
121
|
+
const re = /Process exited with code\s+(-?\d+)/gi;
|
|
122
|
+
let m;
|
|
123
|
+
while ((m = re.exec(header)) !== null) exitCode = parseInt(m[1], 10);
|
|
124
|
+
// Prefer the text AFTER "Output:" as the evidence body (drops the Command/Chunk/Wall-time header).
|
|
125
|
+
const body = oi >= 0 ? s.slice(oi + "\nOutput:\n".length) : s;
|
|
126
|
+
return { exitCode, body };
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// ---------- normalization ----------
|
|
130
|
+
// Normalize one Codex rollout file into Claude-shaped entries. exec_command → a Bash tool_use; its
|
|
131
|
+
// function_call_output → a tool_result carrying the recorded exit code; assistant/user messages → text.
|
|
132
|
+
// maxBytes bounds the tail read (see DEFAULT_TAIL_BYTES). Never throws.
|
|
133
|
+
export function loadCodexEntries(file, maxBytes = DEFAULT_TAIL_BYTES) {
|
|
134
|
+
const entries = [];
|
|
135
|
+
for (const line of tailLines(file, maxBytes)) {
|
|
136
|
+
let e;
|
|
137
|
+
try {
|
|
138
|
+
e = JSON.parse(line);
|
|
139
|
+
} catch {
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
if (e.type !== "response_item") continue;
|
|
143
|
+
const p = e.payload || {};
|
|
144
|
+
if (p.type === "message") {
|
|
145
|
+
const role = p.role === "assistant" ? "assistant" : "user";
|
|
146
|
+
const text = Array.isArray(p.content)
|
|
147
|
+
? p.content.map((b) => (b && typeof b.text === "string" ? b.text : "")).join("")
|
|
148
|
+
: "";
|
|
149
|
+
if (text.trim())
|
|
150
|
+
entries.push({ type: role, message: { role, content: [{ type: "text", text }] } });
|
|
151
|
+
} else if (p.type === "function_call" && p.name === "exec_command" && p.call_id) {
|
|
152
|
+
let cmd = "";
|
|
153
|
+
let workdir = null;
|
|
154
|
+
try {
|
|
155
|
+
const a = JSON.parse(p.arguments || "{}");
|
|
156
|
+
// A STRING cmd is a shell command verify-shadow's tokenizer parses safely. An ARRAY cmd is argv;
|
|
157
|
+
// joining it back into a string is LOSSY (quoting is lost — `["sh","-c","a b"]` → `sh -c a b`),
|
|
158
|
+
// so we DROP it (leave cmd empty → no proof) rather than reconstruct a command that differs from
|
|
159
|
+
// what actually ran. Fail-open on argv-join quoting doubt (RED LINE #4).
|
|
160
|
+
cmd = typeof a.cmd === "string" ? a.cmd : "";
|
|
161
|
+
workdir = typeof a.workdir === "string" ? a.workdir : null;
|
|
162
|
+
} catch {
|
|
163
|
+
/* unparseable args — leave cmd empty (no proof) */
|
|
164
|
+
}
|
|
165
|
+
if (cmd)
|
|
166
|
+
entries.push({
|
|
167
|
+
cwd: workdir,
|
|
168
|
+
message: {
|
|
169
|
+
role: "assistant",
|
|
170
|
+
content: [{ type: "tool_use", name: "Bash", id: p.call_id, input: { command: cmd } }],
|
|
171
|
+
},
|
|
172
|
+
});
|
|
173
|
+
} else if (p.type === "function_call_output" && p.call_id) {
|
|
174
|
+
const { exitCode, body } = parseCodexOutput(p.output);
|
|
175
|
+
entries.push({
|
|
176
|
+
message: {
|
|
177
|
+
role: "user",
|
|
178
|
+
content: [
|
|
179
|
+
{
|
|
180
|
+
type: "tool_result",
|
|
181
|
+
tool_use_id: p.call_id,
|
|
182
|
+
is_error: typeof exitCode === "number" ? exitCode !== 0 : false,
|
|
183
|
+
exit_code: exitCode,
|
|
184
|
+
content: body,
|
|
185
|
+
},
|
|
186
|
+
],
|
|
187
|
+
},
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
return entries;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// ---------- recorded-exit lookup (the Codex precision lever) ----------
|
|
195
|
+
// The recorded exit code Codex wrote for the tool_use `id` (call_id) of a reconstructed proof. This is the
|
|
196
|
+
// CORROBORATING evidence the live worker demands before it surfaces a Codex catch: a live re-run FAIL is
|
|
197
|
+
// only revealed when Codex's OWN recorded run of that exact command ALSO failed. Returns
|
|
198
|
+
// { exitCode:number|null, isError:bool, found:bool }. Scans newest→oldest so the most recent result wins.
|
|
199
|
+
export function recordedExitFor(entries, toolUseId) {
|
|
200
|
+
if (!toolUseId || !Array.isArray(entries))
|
|
201
|
+
return { exitCode: null, isError: false, found: false };
|
|
202
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
203
|
+
const c = entries[i]?.message?.content;
|
|
204
|
+
if (!Array.isArray(c)) continue;
|
|
205
|
+
for (const b of c) {
|
|
206
|
+
if (b && b.type === "tool_result" && b.tool_use_id === toolUseId) {
|
|
207
|
+
return {
|
|
208
|
+
exitCode: typeof b.exit_code === "number" ? b.exit_code : null,
|
|
209
|
+
isError: b.is_error === true,
|
|
210
|
+
found: true,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
return { exitCode: null, isError: false, found: false };
|
|
216
|
+
}
|
package/eval-harness.mjs
CHANGED
|
@@ -31,6 +31,11 @@ import { fileURLToPath } from "node:url";
|
|
|
31
31
|
// via an INJECTED exec so every branch runs hermetically (no network, no gh/curl/git/npm needed).
|
|
32
32
|
import { detectShipClaim, resolveShipStatus } from "./verify-shadow.mjs";
|
|
33
33
|
import { reconstructShipTargetV1, shipProbeFamily, SHIP_KIND_PROBE } from "./anchor-shadow.mjs";
|
|
34
|
+
// CODEX LIVE-CATCH PARITY corpus + real-history audit — drives the REAL Codex catch decision
|
|
35
|
+
// (evaluateCodexProof: claim → reconstruct proof → re-run → 2/2 flake-confirm + recorded-exit agreement)
|
|
36
|
+
// over synthetic fixtures (hermetic, injected re-run) AND this machine's real ~/.codex rollouts.
|
|
37
|
+
import { evaluateCodexProof, rerunProof } from "./verify-shadow.mjs";
|
|
38
|
+
import { loadCodexEntries } from "./codex-normalize.mjs";
|
|
34
39
|
|
|
35
40
|
// ───────────────────────────── tunables (UNCALIBRATED defaults) ─────────────────────────────
|
|
36
41
|
// These thresholds are NOT tuned against a human gold set unless one is supplied via --gold.
|
|
@@ -1511,6 +1516,298 @@ function main() {
|
|
|
1511
1516
|
console.log("═".repeat(90));
|
|
1512
1517
|
}
|
|
1513
1518
|
|
|
1519
|
+
// ═════════════════════ CODEX LIVE-CATCH FIXTURE CORPUS (the publish gate) ═════════════════════
|
|
1520
|
+
// Synthetic Codex rollouts exercised END-TO-END against the REAL Codex catch decision
|
|
1521
|
+
// (loadCodexEntries → detectCompletionClaim → reconstructProofEntry → recorded-exit lookup →
|
|
1522
|
+
// assembleCodexVerdict). Only the LIVE re-run is injected (a stub — hermetic, no process spawned), so the
|
|
1523
|
+
// fixtures pin the exact (claim → recorded-exit → live-outcome → confirmed?) contract. The recorded-exit
|
|
1524
|
+
// agreement makes a Codex catch strictly more precision-gated than a Claude one; a false Codex red is the
|
|
1525
|
+
// #1 death vector, so this corpus is the gate to enabling the reveal.
|
|
1526
|
+
async function runCodexSelfTests() {
|
|
1527
|
+
const assert = (cond, msg) => {
|
|
1528
|
+
if (!cond) throw new Error("CODEX-SELFTEST FAIL: " + msg);
|
|
1529
|
+
};
|
|
1530
|
+
console.log("\n" + "═".repeat(96));
|
|
1531
|
+
console.log("CODEX LIVE-CATCH FIXTURE CORPUS — the publish gate (recorded-exit-code agreement)");
|
|
1532
|
+
console.log("═".repeat(96));
|
|
1533
|
+
|
|
1534
|
+
const os = await import("node:os");
|
|
1535
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "skalpel-codex-selftest-"));
|
|
1536
|
+
|
|
1537
|
+
// ── rollout event builders (the real Codex `response_item` shapes) ──────────────────────────────────
|
|
1538
|
+
const meta = { timestamp: "t", type: "session_meta", payload: { id: "selftest" } };
|
|
1539
|
+
const execCall = (call_id, cmd) => ({
|
|
1540
|
+
timestamp: "t",
|
|
1541
|
+
type: "response_item",
|
|
1542
|
+
payload: {
|
|
1543
|
+
type: "function_call",
|
|
1544
|
+
name: "exec_command",
|
|
1545
|
+
arguments: JSON.stringify({ cmd, workdir: tmp }),
|
|
1546
|
+
call_id,
|
|
1547
|
+
},
|
|
1548
|
+
});
|
|
1549
|
+
const execCallRaw = (call_id, rawArgs) => ({
|
|
1550
|
+
timestamp: "t",
|
|
1551
|
+
type: "response_item",
|
|
1552
|
+
payload: { type: "function_call", name: "exec_command", arguments: rawArgs, call_id },
|
|
1553
|
+
});
|
|
1554
|
+
const execOut = (call_id, code, body = "") => ({
|
|
1555
|
+
timestamp: "t",
|
|
1556
|
+
type: "response_item",
|
|
1557
|
+
payload: {
|
|
1558
|
+
type: "function_call_output",
|
|
1559
|
+
call_id,
|
|
1560
|
+
output: `Chunk ID: x\nWall time: 0.01 seconds\nProcess exited with code ${code}\nOriginal token count: 1\nOutput:\n${body}`,
|
|
1561
|
+
},
|
|
1562
|
+
});
|
|
1563
|
+
const execOutNoCode = (call_id) => ({
|
|
1564
|
+
timestamp: "t",
|
|
1565
|
+
type: "response_item",
|
|
1566
|
+
payload: {
|
|
1567
|
+
type: "function_call_output",
|
|
1568
|
+
call_id,
|
|
1569
|
+
output: `exec_command failed: CreateProcess { message: "Rejected(\\"No such file\\")" }`,
|
|
1570
|
+
},
|
|
1571
|
+
});
|
|
1572
|
+
const assistant = (text) => ({
|
|
1573
|
+
timestamp: "t",
|
|
1574
|
+
type: "response_item",
|
|
1575
|
+
payload: { type: "message", role: "assistant", content: [{ type: "output_text", text }] },
|
|
1576
|
+
});
|
|
1577
|
+
const writeRollout = (name, lines) => {
|
|
1578
|
+
const f = path.join(tmp, `rollout-2026-01-01T00-00-00-${name}.jsonl`);
|
|
1579
|
+
fs.writeFileSync(f, lines.map((l) => JSON.stringify(l)).join("\n") + "\n");
|
|
1580
|
+
return f;
|
|
1581
|
+
};
|
|
1582
|
+
// Injected LIVE re-run: returns a scripted sequence of outcomes (rerunProof's result shape). No process.
|
|
1583
|
+
const stubRerun = (seq) => {
|
|
1584
|
+
let i = 0;
|
|
1585
|
+
return async () => {
|
|
1586
|
+
const o = seq[Math.min(i, seq.length - 1)];
|
|
1587
|
+
i++;
|
|
1588
|
+
if (o === "FAIL") return { pass: false, outcome: "GENUINE_FAIL", evidence: "2 failed" };
|
|
1589
|
+
if (o === "HARNESS") return { pass: false, outcome: "HARNESS_ERROR", evidence: "boom" };
|
|
1590
|
+
return { pass: true, outcome: "PASS", evidence: "ok" };
|
|
1591
|
+
};
|
|
1592
|
+
};
|
|
1593
|
+
|
|
1594
|
+
let pass = 0;
|
|
1595
|
+
let total = 0;
|
|
1596
|
+
const check = async (label, file, seq, expect) => {
|
|
1597
|
+
total++;
|
|
1598
|
+
const entries = loadCodexEntries(file);
|
|
1599
|
+
const res = await evaluateCodexProof(entries, tmp, stubRerun(seq));
|
|
1600
|
+
if ("claim" in expect)
|
|
1601
|
+
assert(res.claim === expect.claim, `${label}: claim=${res.claim} expected ${expect.claim}`);
|
|
1602
|
+
if (expect.proofNull) assert(res.proof == null, `${label}: expected NO reconstructed proof`);
|
|
1603
|
+
if ("confirmed" in expect)
|
|
1604
|
+
// Compare by TRUTHINESS: a no-proof turn returns `confirmed` absent (undefined) — still a no-catch,
|
|
1605
|
+
// which must satisfy an `expect.confirmed === false` (never a surfaced red).
|
|
1606
|
+
assert(
|
|
1607
|
+
!!res.confirmed === !!expect.confirmed,
|
|
1608
|
+
`${label}: confirmed=${res.confirmed} expected ${expect.confirmed}`,
|
|
1609
|
+
);
|
|
1610
|
+
if ("agreement" in expect)
|
|
1611
|
+
assert(
|
|
1612
|
+
res.agreement === expect.agreement,
|
|
1613
|
+
`${label}: agreement=${res.agreement} expected ${expect.agreement}`,
|
|
1614
|
+
);
|
|
1615
|
+
console.log(
|
|
1616
|
+
` ✓ ${label} (claim=${res.claim} proof=${res.proof ? "yes" : "no"} outcome=${res.outcome ?? "-"} recExit=${res.recorded_exit_code ?? "-"} agree=${res.agreement ?? "-"} confirmed=${!!res.confirmed})`,
|
|
1617
|
+
);
|
|
1618
|
+
pass++;
|
|
1619
|
+
};
|
|
1620
|
+
|
|
1621
|
+
// a) done-claim over a RECORDED FAIL, live 2/2 fail → CAUGHT (agreement)
|
|
1622
|
+
await check(
|
|
1623
|
+
"a done-over-recorded-FAIL, live 2/2 fail → CAUGHT",
|
|
1624
|
+
writeRollout("a", [
|
|
1625
|
+
meta,
|
|
1626
|
+
execCall("c1", "pytest"),
|
|
1627
|
+
execOut("c1", 1, "2 failed"),
|
|
1628
|
+
assistant("all tests pass — done"),
|
|
1629
|
+
]),
|
|
1630
|
+
["FAIL", "FAIL"],
|
|
1631
|
+
{ claim: true, confirmed: true, agreement: "agree" },
|
|
1632
|
+
);
|
|
1633
|
+
// b) recorded PASS but the re-run fails now → SUPPRESSED (recorded-exit disagreement = flake/drift)
|
|
1634
|
+
await check(
|
|
1635
|
+
"b recorded-PASS but re-run fails → SUPPRESSED (disagree)",
|
|
1636
|
+
writeRollout("b", [
|
|
1637
|
+
meta,
|
|
1638
|
+
execCall("c2", "pytest"),
|
|
1639
|
+
execOut("c2", 0, "all passed"),
|
|
1640
|
+
assistant("all tests pass now"),
|
|
1641
|
+
]),
|
|
1642
|
+
["FAIL", "FAIL"],
|
|
1643
|
+
{ claim: true, confirmed: false, agreement: "disagree" },
|
|
1644
|
+
);
|
|
1645
|
+
// c) unparseable exec arguments → no proof reconstructed → no catch, NO THROW
|
|
1646
|
+
await check(
|
|
1647
|
+
"c unparseable exec args → no proof, no catch, no throw",
|
|
1648
|
+
writeRollout("c", [
|
|
1649
|
+
meta,
|
|
1650
|
+
execCallRaw("c3", "{not valid json"),
|
|
1651
|
+
execOut("c3", 1, "x"),
|
|
1652
|
+
assistant("done, fixed it"),
|
|
1653
|
+
]),
|
|
1654
|
+
["FAIL", "FAIL"],
|
|
1655
|
+
{ claim: true, proofNull: true, confirmed: false },
|
|
1656
|
+
);
|
|
1657
|
+
// d) live first FAIL then PASS (a flake) → SUPPRESSED (2/2 not met)
|
|
1658
|
+
await check(
|
|
1659
|
+
"d live fail then pass (flake) → SUPPRESSED",
|
|
1660
|
+
writeRollout("d", [
|
|
1661
|
+
meta,
|
|
1662
|
+
execCall("c4", "pytest"),
|
|
1663
|
+
execOut("c4", 1, "2 failed"),
|
|
1664
|
+
assistant("done"),
|
|
1665
|
+
]),
|
|
1666
|
+
["FAIL", "PASS"],
|
|
1667
|
+
{ claim: true, confirmed: false, agreement: "n/a" },
|
|
1668
|
+
);
|
|
1669
|
+
// e) recorded exit UNKNOWN (sandbox CreateProcess failure, no code) + live 2/2 fail → SUPPRESSED (unknown)
|
|
1670
|
+
await check(
|
|
1671
|
+
"e recorded exit unknown + live 2/2 fail → SUPPRESSED (unknown)",
|
|
1672
|
+
writeRollout("e", [
|
|
1673
|
+
meta,
|
|
1674
|
+
execCall("c5", "pytest"),
|
|
1675
|
+
execOutNoCode("c5"),
|
|
1676
|
+
assistant("all green"),
|
|
1677
|
+
]),
|
|
1678
|
+
["FAIL", "FAIL"],
|
|
1679
|
+
{ claim: true, confirmed: false, agreement: "unknown" },
|
|
1680
|
+
);
|
|
1681
|
+
// f) recorded FAIL but the re-run PASSES now → not a catch (the agent actually fixed it)
|
|
1682
|
+
await check(
|
|
1683
|
+
"f recorded-FAIL but re-run passes → not a catch",
|
|
1684
|
+
writeRollout("f", [
|
|
1685
|
+
meta,
|
|
1686
|
+
execCall("c6", "pytest"),
|
|
1687
|
+
execOut("c6", 1, "2 failed"),
|
|
1688
|
+
assistant("fixed, all pass"),
|
|
1689
|
+
]),
|
|
1690
|
+
["PASS"],
|
|
1691
|
+
{ claim: true, confirmed: false },
|
|
1692
|
+
);
|
|
1693
|
+
// g) no completion claim in the last turn → nothing to verify
|
|
1694
|
+
await check(
|
|
1695
|
+
"g no completion claim → nothing to verify",
|
|
1696
|
+
writeRollout("g", [
|
|
1697
|
+
meta,
|
|
1698
|
+
execCall("c7", "pytest"),
|
|
1699
|
+
execOut("c7", 1, "2 failed"),
|
|
1700
|
+
assistant("here is what I found so far"),
|
|
1701
|
+
]),
|
|
1702
|
+
["FAIL", "FAIL"],
|
|
1703
|
+
{ claim: false },
|
|
1704
|
+
);
|
|
1705
|
+
|
|
1706
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
1707
|
+
console.log("─".repeat(96));
|
|
1708
|
+
console.log(`CODEX-SELFTEST: ${pass}/${total} pass`);
|
|
1709
|
+
return pass === total;
|
|
1710
|
+
}
|
|
1711
|
+
|
|
1712
|
+
// ═════════════════════ CODEX real-history REPLAY audit ═════════════════════
|
|
1713
|
+
// Replays the REAL live Codex worker decision (evaluateCodexProof with the REAL rerunProof) over every
|
|
1714
|
+
// rollout under a dir (default ~/.codex). Prints one row per completion-claim turn + a summary, and lists
|
|
1715
|
+
// every CONFIRMED catch for hand-audit — the gate is ZERO false catches. Read-only except rerunProof, which
|
|
1716
|
+
// only ever re-runs the session's OWN allowlisted, non-mutating test/build/lint command (60s cap each).
|
|
1717
|
+
async function runCodexReplay(dir) {
|
|
1718
|
+
const os = await import("node:os");
|
|
1719
|
+
const root = dir || path.join(os.homedir(), ".codex");
|
|
1720
|
+
const files = [];
|
|
1721
|
+
const walk = (d, depth = 0) => {
|
|
1722
|
+
if (depth > 8) return;
|
|
1723
|
+
let ents;
|
|
1724
|
+
try {
|
|
1725
|
+
ents = fs.readdirSync(d, { withFileTypes: true });
|
|
1726
|
+
} catch {
|
|
1727
|
+
return;
|
|
1728
|
+
}
|
|
1729
|
+
for (const e of ents) {
|
|
1730
|
+
const p = path.join(d, e.name);
|
|
1731
|
+
try {
|
|
1732
|
+
if (e.isDirectory()) walk(p, depth + 1);
|
|
1733
|
+
else if (e.isFile() && /^rollout-.*\.jsonl$/.test(e.name)) files.push(p);
|
|
1734
|
+
} catch {
|
|
1735
|
+
/* skip one bad entry */
|
|
1736
|
+
}
|
|
1737
|
+
}
|
|
1738
|
+
};
|
|
1739
|
+
walk(root);
|
|
1740
|
+
console.log("\n" + "═".repeat(96));
|
|
1741
|
+
console.log(`CODEX REAL-HISTORY REPLAY — ${files.length} rollout(s) under ${root}`);
|
|
1742
|
+
console.log(
|
|
1743
|
+
" (live worker decision over your OWN Codex history; every CONFIRMED catch is hand-audited)",
|
|
1744
|
+
);
|
|
1745
|
+
console.log("═".repeat(96));
|
|
1746
|
+
|
|
1747
|
+
let scanned = 0;
|
|
1748
|
+
let claims = 0;
|
|
1749
|
+
let proofs = 0;
|
|
1750
|
+
let liveFail = 0;
|
|
1751
|
+
let harness = 0;
|
|
1752
|
+
let disagree = 0;
|
|
1753
|
+
let unknown = 0;
|
|
1754
|
+
let confirmed = 0;
|
|
1755
|
+
const catches = [];
|
|
1756
|
+
for (const f of files) {
|
|
1757
|
+
scanned++;
|
|
1758
|
+
let entries;
|
|
1759
|
+
try {
|
|
1760
|
+
entries = loadCodexEntries(f);
|
|
1761
|
+
} catch {
|
|
1762
|
+
continue;
|
|
1763
|
+
}
|
|
1764
|
+
if (!entries.length) continue;
|
|
1765
|
+
let res;
|
|
1766
|
+
try {
|
|
1767
|
+
res = await evaluateCodexProof(entries, null, rerunProof);
|
|
1768
|
+
} catch (e) {
|
|
1769
|
+
console.log(
|
|
1770
|
+
` ! ${path.basename(f)} evaluate threw (fail-open): ${String(e && e.message).slice(0, 60)}`,
|
|
1771
|
+
);
|
|
1772
|
+
continue;
|
|
1773
|
+
}
|
|
1774
|
+
if (!res.claim) continue;
|
|
1775
|
+
claims++;
|
|
1776
|
+
if (!res.proof) {
|
|
1777
|
+
console.log(` · ${path.basename(f).slice(0, 52)} claim ✓ no re-runnable proof`);
|
|
1778
|
+
continue;
|
|
1779
|
+
}
|
|
1780
|
+
proofs++;
|
|
1781
|
+
if (res.outcome === "GENUINE_FAIL") liveFail++;
|
|
1782
|
+
if (res.outcome === "HARNESS_ERROR" || res.outcome === "REFUSED") harness++;
|
|
1783
|
+
if (res.confirmed) {
|
|
1784
|
+
confirmed++;
|
|
1785
|
+
catches.push({ f, res });
|
|
1786
|
+
} else if (res.agreement === "disagree") disagree++;
|
|
1787
|
+
else if (res.agreement === "unknown") unknown++;
|
|
1788
|
+
console.log(
|
|
1789
|
+
` ${res.confirmed ? "🔴" : "·"} ${path.basename(f).slice(0, 44)} claim ✓ outcome=${res.outcome} recExit=${res.recorded_exit_code ?? "-"} agree=${res.agreement} confirmed=${res.confirmed}`,
|
|
1790
|
+
);
|
|
1791
|
+
}
|
|
1792
|
+
console.log("─".repeat(96));
|
|
1793
|
+
console.log(
|
|
1794
|
+
`scanned ${scanned} · claims ${claims} · with-proof ${proofs} · live-fail ${liveFail} · harness ${harness} · disagree ${disagree} · unknown ${unknown}`,
|
|
1795
|
+
);
|
|
1796
|
+
console.log(`CONFIRMED CODEX CATCHES (must hand-audit → gate = ZERO false): ${confirmed}`);
|
|
1797
|
+
for (const c of catches) {
|
|
1798
|
+
console.log(" ── catch ──");
|
|
1799
|
+
console.log(` file: ${c.f}`);
|
|
1800
|
+
console.log(` claim: ${String(c.res.claim_text || "").slice(0, 88)}`);
|
|
1801
|
+
console.log(` proof: ${c.res.proof_command}`);
|
|
1802
|
+
console.log(
|
|
1803
|
+
` recorded exit: ${c.res.recorded_exit_code} · live outcome: ${c.res.outcome} · agree: ${c.res.agreement}`,
|
|
1804
|
+
);
|
|
1805
|
+
console.log(` evidence: ${String(c.res.evidence || "").slice(0, 120)}`);
|
|
1806
|
+
}
|
|
1807
|
+
console.log("═".repeat(96));
|
|
1808
|
+
return { scanned, claims, proofs, liveFail, harness, disagree, unknown, confirmed, catches };
|
|
1809
|
+
}
|
|
1810
|
+
|
|
1514
1811
|
// Robust main detection (mirrors verify-shadow.mjs / skalpel-setup.mjs): compare REAL paths via
|
|
1515
1812
|
// realpathSync, not a hand-built path.resolve(url.pathname). On Windows, `new URL(import.meta.url)
|
|
1516
1813
|
// .pathname` yields "/C:/Users/..." (a leading slash BEFORE the drive letter); path.resolve() on that
|
|
@@ -1537,16 +1834,32 @@ if (isMain) {
|
|
|
1537
1834
|
console.error(String((e && e.stack) || e));
|
|
1538
1835
|
process.exit(1);
|
|
1539
1836
|
});
|
|
1837
|
+
} else if (a0 === "--codex-selftest") {
|
|
1838
|
+
// CODEX LIVE-CATCH publish gate — the hermetic recorded-exit-agreement fixture corpus.
|
|
1839
|
+
runCodexSelfTests()
|
|
1840
|
+
.then((ok) => process.exit(ok ? 0 : 1))
|
|
1841
|
+
.catch((e) => {
|
|
1842
|
+
console.error(String((e && e.stack) || e));
|
|
1843
|
+
process.exit(1);
|
|
1844
|
+
});
|
|
1845
|
+
} else if (a0 === "--codex-replay") {
|
|
1846
|
+
// CODEX real-history audit — replay the live worker over ~/.codex (or a given dir). Hand-audit output.
|
|
1847
|
+
runCodexReplay(process.argv[3] || null)
|
|
1848
|
+
.then(() => process.exit(0))
|
|
1849
|
+
.catch((e) => {
|
|
1850
|
+
console.error(String((e && e.stack) || e));
|
|
1851
|
+
process.exit(1);
|
|
1852
|
+
});
|
|
1540
1853
|
} else if (a0 === "--selftest") {
|
|
1541
|
-
// `--selftest` now covers
|
|
1854
|
+
// `--selftest` now covers the intervention machinery, the ship-status v1 corpus, AND the Codex corpus.
|
|
1542
1855
|
try {
|
|
1543
1856
|
runSelfTests();
|
|
1544
1857
|
} catch (e) {
|
|
1545
1858
|
console.error(String((e && e.stack) || e));
|
|
1546
1859
|
process.exit(1);
|
|
1547
1860
|
}
|
|
1548
|
-
runShipSelfTests()
|
|
1549
|
-
.then((
|
|
1861
|
+
Promise.all([runShipSelfTests(), runCodexSelfTests()])
|
|
1862
|
+
.then((oks) => process.exit(oks.every(Boolean) ? 0 : 1))
|
|
1550
1863
|
.catch((e) => {
|
|
1551
1864
|
console.error(String((e && e.stack) || e));
|
|
1552
1865
|
process.exit(1);
|
package/first-run.mjs
CHANGED
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
// recorded output tail, the recorded non-zero exit code, real timestamps). No invented minutes.
|
|
21
21
|
//
|
|
22
22
|
// REUSE: the detection + classification ENGINE is verify-shadow's / catches.mjs's, imported unedited —
|
|
23
|
-
// loadClaudeEntries / loadCodexEntries (
|
|
23
|
+
// loadClaudeEntries (catches.mjs) / loadCodexEntries (codex-normalize.mjs) normalize both transcript shapes; detectCompletionClaim,
|
|
24
24
|
// reconstructProofEntry, classifyOutcome, extractFailCount (verify-shadow.mjs) do claim + proof + verdict.
|
|
25
25
|
// We run a FIRST-RUN-specific loop over those primitives (rather than calling runScan) because the first-run
|
|
26
26
|
// profile (newest ~150 sessions, HARD ~3s budget, a live progress line) and the strict precision gate need
|
|
@@ -40,7 +40,10 @@ import { spawnSync } from "node:child_process";
|
|
|
40
40
|
import path from "node:path";
|
|
41
41
|
import { fileURLToPath } from "node:url";
|
|
42
42
|
import { realpathSync } from "node:fs";
|
|
43
|
-
import { loadClaudeEntries
|
|
43
|
+
import { loadClaudeEntries } from "./catches.mjs";
|
|
44
|
+
// ONE normalizer, never two: loadCodexEntries now lives in codex-normalize.mjs (shared with the live
|
|
45
|
+
// verify-shadow worker + the retroactive catches scan). Imported from the canonical source here.
|
|
46
|
+
import { loadCodexEntries } from "./codex-normalize.mjs";
|
|
44
47
|
import {
|
|
45
48
|
detectCompletionClaim,
|
|
46
49
|
reconstructProofEntry,
|
package/install.mjs
CHANGED
|
@@ -55,6 +55,7 @@ const RUNTIME_FILES = [
|
|
|
55
55
|
"insights.mjs", // hooks + statusline + reveal + verify-shadow import ./insights.mjs
|
|
56
56
|
"reveal.mjs", // both prompt/session hooks import ./reveal.mjs
|
|
57
57
|
"transcript.mjs", // skalpel-hook + skalpel-statusline + verify-shadow import ./transcript.mjs (tailLines)
|
|
58
|
+
"codex-normalize.mjs", // verify-shadow.mjs (staged worker) imports ./codex-normalize.mjs — Codex rollout → Claude-shape normalizer (imports ./transcript.mjs)
|
|
58
59
|
"skalpel-hook.mjs", // UserPromptSubmit entrypoint (wired)
|
|
59
60
|
"skalpel-hook-session.mjs", // SessionStart entrypoint (wired)
|
|
60
61
|
"incremental-ingest.mjs", // skalpel-hook-session-end.mjs imports ./incremental-ingest.mjs
|
package/package.json
CHANGED
package/verify-shadow.mjs
CHANGED
|
@@ -98,6 +98,17 @@ import {
|
|
|
98
98
|
shipProbeFamily,
|
|
99
99
|
SHIP_KIND_PROBE,
|
|
100
100
|
} from "./anchor-shadow.mjs";
|
|
101
|
+
// CODEX LIVE-CATCH PARITY: the ONE shared Codex-rollout normalizer (also used by the retroactive
|
|
102
|
+
// `skalpel catches` scanner). It translates a Codex rollout into the SAME Claude-shaped entries the whole
|
|
103
|
+
// engine below already speaks, and carries Codex's OWN recorded exit code onto each tool_result so the live
|
|
104
|
+
// worker can corroborate a re-run against it (the Codex precision lever). Staged next to this worker by
|
|
105
|
+
// install.mjs so the detached HOOKS_DIR copy can import it.
|
|
106
|
+
import {
|
|
107
|
+
isCodexRollout,
|
|
108
|
+
loadCodexEntries,
|
|
109
|
+
newestCodexRollout,
|
|
110
|
+
recordedExitFor,
|
|
111
|
+
} from "./codex-normalize.mjs";
|
|
101
112
|
|
|
102
113
|
const DIR = join(homedir(), ".skalpel");
|
|
103
114
|
export const VERIFY_LOG_PATH = join(DIR, "verify-shadow.log");
|
|
@@ -121,6 +132,15 @@ const CLASSIFY_TAIL_BYTES = 4000;
|
|
|
121
132
|
const TRANSCRIPT_TAIL_BYTES = 2 * 1024 * 1024;
|
|
122
133
|
const LOCK_STALE_MS = 90_000;
|
|
123
134
|
|
|
135
|
+
// CODEX REVEAL GATE (dark-ship kill switch). A Codex catch is ALWAYS logged under the shadow, but the
|
|
136
|
+
// user-visible RED reveal for a Codex session is additionally gated here. It was flipped ON only AFTER the
|
|
137
|
+
// real-history audit (eval-harness `--codex-replay` over this machine's ~/.codex history, every catch
|
|
138
|
+
// hand-audited → ZERO false catches; `--codex-selftest` 3/3). The recorded-exit-code agreement below makes
|
|
139
|
+
// a Codex catch strictly more precision-gated than a Claude one; this constant is the belt-and-suspenders
|
|
140
|
+
// switch on top of that + the arming resolver (revealEnabled), so with the arming flag OFF (every real
|
|
141
|
+
// install) the whole feature stays byte-identical-dark regardless of this value.
|
|
142
|
+
const CODEX_REVEAL_ENABLED = true;
|
|
143
|
+
|
|
124
144
|
// ======================= (1) completion-claim detection =======================
|
|
125
145
|
// A deliberately broad lexicon: this is an INSTRUMENT, and the MISMATCH metric is computed only among
|
|
126
146
|
// turns where a proof actually re-ran, so a loose claim match cannot fabricate a "lie" — it can only
|
|
@@ -627,6 +647,42 @@ export function classifyConfirm(firstOutcome, confirmOutcome) {
|
|
|
627
647
|
return { needsConfirm, confirmedFail, unconfirmedFail };
|
|
628
648
|
}
|
|
629
649
|
|
|
650
|
+
// ======================= (2c) Codex precision lever (recorded-exit-code agreement) =======================
|
|
651
|
+
// A Codex rollout records the REAL exit code of every command Codex ran. So on top of the SAME 2/2 flake-
|
|
652
|
+
// confirm every catch already clears, a Codex catch demands one more corroboration: the LIVE re-run's
|
|
653
|
+
// confirmed fail must AGREE with Codex's OWN recorded run of that exact command (it too exited non-zero).
|
|
654
|
+
// • "agree" — recorded exit is a real non-zero → Codex's own run failed too → the "done" was a lie.
|
|
655
|
+
// • "disagree" — recorded exit 0 → Codex's run PASSED; our re-run fails only now → environment/flake drift,
|
|
656
|
+
// NOT a claim-time lie → SUPPRESS.
|
|
657
|
+
// • "unknown" — no recorded exit (sandbox/harness "CreateProcess failed", missing wrapper) → can't
|
|
658
|
+
// corroborate → SUPPRESS.
|
|
659
|
+
// • "n/a" — the live re-run was not a confirmed 2/2 fail, so agreement is moot.
|
|
660
|
+
// Only "agree" is ever surfaced → a Codex catch is strictly MORE precision-gated than a Claude one. Pure.
|
|
661
|
+
export function codexAgreement(confirmedFail, recordedExitCode) {
|
|
662
|
+
if (!confirmedFail) return "n/a";
|
|
663
|
+
if (typeof recordedExitCode === "number" && recordedExitCode !== 0) return "agree";
|
|
664
|
+
if (recordedExitCode === 0) return "disagree";
|
|
665
|
+
return "unknown";
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
// assembleCodexVerdict({outcome, confirmOutcome, recorded}) → the ONE place the Codex catch decision lives
|
|
669
|
+
// (shared by the live worker AND the audit harness so they can never drift). Combines the 2/2 flake-confirm
|
|
670
|
+
// with the recorded-exit agreement. `confirmed` is the only state that may surface a red; every other fail
|
|
671
|
+
// bucket (un-reproduced flake, recorded disagreement, recorded unknown) is logged-not-shown. Pure + total.
|
|
672
|
+
export function assembleCodexVerdict({ outcome, confirmOutcome, recorded }) {
|
|
673
|
+
const { confirmedFail, unconfirmedFail } = classifyConfirm(outcome, confirmOutcome);
|
|
674
|
+
const recordedExit = recorded && typeof recorded.exitCode === "number" ? recorded.exitCode : null;
|
|
675
|
+
const agreement = codexAgreement(confirmedFail, recordedExit);
|
|
676
|
+
const confirmed = confirmedFail && agreement === "agree";
|
|
677
|
+
return {
|
|
678
|
+
confirmedFail,
|
|
679
|
+
agreement,
|
|
680
|
+
confirmed,
|
|
681
|
+
// recorded a suppressed fail (flake OR recorded-exit disagreement/unknown) for the instrument — never shown.
|
|
682
|
+
unconfirmed_fail: unconfirmedFail || (confirmedFail && !confirmed),
|
|
683
|
+
};
|
|
684
|
+
}
|
|
685
|
+
|
|
630
686
|
// ======================= (3) re-run the proof =======================
|
|
631
687
|
// rerunProof({cmd,args,cwd}) -> Promise<{ pass, outcome, evidence, refused? }>
|
|
632
688
|
// outcome ∈ {PASS, GENUINE_FAIL, HARNESS_ERROR, REFUSED}. execFile with an ARGUMENT ARRAY and NO shell.
|
|
@@ -701,8 +757,15 @@ export function rerunProof({ cmd, args, cwd }) {
|
|
|
701
757
|
}
|
|
702
758
|
|
|
703
759
|
// ======================= (4) wire it: shadow-log a claim turn =======================
|
|
760
|
+
// readEntries(transcriptPath) → the session's entries in the CLAUDE shape. A Codex rollout (path under
|
|
761
|
+
// ~/.codex or a `rollout-*.jsonl` file) is routed through the shared normalizer so it comes back Claude-
|
|
762
|
+
// native — every downstream function (claim detection, proof reconstruction, allowlist, classifyOutcome,
|
|
763
|
+
// strict kind↔probe) is then unchanged. A Claude transcript takes the original tail-parse path, byte for
|
|
764
|
+
// byte. Fail-open on any error → [].
|
|
704
765
|
function readEntries(transcriptPath) {
|
|
705
766
|
try {
|
|
767
|
+
if (isCodexRollout(transcriptPath))
|
|
768
|
+
return loadCodexEntries(transcriptPath, TRANSCRIPT_TAIL_BYTES);
|
|
706
769
|
const lines = tailLines(transcriptPath, TRANSCRIPT_TAIL_BYTES);
|
|
707
770
|
const entries = [];
|
|
708
771
|
for (const line of lines) {
|
|
@@ -1397,11 +1460,166 @@ async function maybeShipStatusCatch({ text, session, cwd, transcriptPath }) {
|
|
|
1397
1460
|
}
|
|
1398
1461
|
}
|
|
1399
1462
|
|
|
1463
|
+
// resolveCodexRollout(transcriptPath) → the ACTUAL Codex rollout file to read for THIS turn, or null if
|
|
1464
|
+
// this is not a Codex session. The trigger is a PER-SESSION signal from the transcript PATH itself
|
|
1465
|
+
// (isCodexRollout: a `rollout-*.jsonl` name, or a path under ~/.codex / $CODEX_HOME) — NEVER the mere
|
|
1466
|
+
// presence of $CODEX_HOME in the environment, which would wrongly hijack a CLAUDE session for any user who
|
|
1467
|
+
// exports CODEX_HOME globally. If the path is a usable rollout FILE we read it directly; (#6) if it's a
|
|
1468
|
+
// Codex context but not a readable rollout (a dir, a session-id path, a non-rollout .codex file) we resolve
|
|
1469
|
+
// the NEWEST rollout ourselves — entirely inside this worker, so skalpel-hook.mjs stays untouched.
|
|
1470
|
+
// Fail-open → null (→ the Claude path below runs, byte-for-byte unchanged).
|
|
1471
|
+
function resolveCodexRollout(transcriptPath) {
|
|
1472
|
+
try {
|
|
1473
|
+
if (!isCodexRollout(transcriptPath)) return null; // not a Codex session → Claude path, unchanged
|
|
1474
|
+
const base = String(transcriptPath).split(/[/\\]/).pop() || "";
|
|
1475
|
+
if (/^rollout-.*\.jsonl$/.test(base)) {
|
|
1476
|
+
try {
|
|
1477
|
+
if (statSync(transcriptPath).isFile()) return transcriptPath; // the handed rollout file, directly
|
|
1478
|
+
} catch {
|
|
1479
|
+
/* named like a rollout but not a readable file — fall through to the newest-rollout resolve */
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
return newestCodexRollout(); // #6: a Codex context but no directly-usable rollout → newest rollout
|
|
1483
|
+
} catch {
|
|
1484
|
+
return null;
|
|
1485
|
+
}
|
|
1486
|
+
}
|
|
1487
|
+
|
|
1488
|
+
// evaluateCodexProof(entries, cwd, rerun=rerunProof) → the PURE Codex catch decision: claim → reconstruct
|
|
1489
|
+
// the session's OWN proof → re-run (2/2 flake-confirm) → corroborate against Codex's recorded exit code.
|
|
1490
|
+
// No lock, no logging, no reveal — `rerun` is injected so the audit harness can drive every branch
|
|
1491
|
+
// hermetically (and the real-history replay uses the real rerunProof). Never throws (rerun is fail-open).
|
|
1492
|
+
export async function evaluateCodexProof(entries, cwd, rerun = rerunProof) {
|
|
1493
|
+
const assistantText = lastAssistantText(entries);
|
|
1494
|
+
const { claim, text } = detectCompletionClaim(assistantText);
|
|
1495
|
+
if (!claim) return { claim: false };
|
|
1496
|
+
const found = reconstructProofEntry(entries, cwd);
|
|
1497
|
+
const proof = found ? found.proof : null;
|
|
1498
|
+
if (!proof) return { claim: true, claim_text: text, proof: null };
|
|
1499
|
+
const result = await rerun(proof);
|
|
1500
|
+
const outcome = result.outcome || (result.pass ? "PASS" : "GENUINE_FAIL");
|
|
1501
|
+
// 2/2 flake-confirm: only re-run a SECOND time on a first GENUINE_FAIL (mirrors the Claude live path).
|
|
1502
|
+
const confirm = outcome === "GENUINE_FAIL" ? await rerun(proof) : null;
|
|
1503
|
+
const confirmOutcome = confirm
|
|
1504
|
+
? confirm.outcome || (confirm.pass ? "PASS" : "GENUINE_FAIL")
|
|
1505
|
+
: null;
|
|
1506
|
+
const recorded = recordedExitFor(entries, found.toolUseId);
|
|
1507
|
+
const v = assembleCodexVerdict({ outcome, confirmOutcome, recorded });
|
|
1508
|
+
return {
|
|
1509
|
+
claim: true,
|
|
1510
|
+
claim_text: text,
|
|
1511
|
+
proof,
|
|
1512
|
+
proof_command: [proof.cmd, ...proof.args].join(" ").slice(0, 200),
|
|
1513
|
+
toolUseId: found.toolUseId,
|
|
1514
|
+
outcome,
|
|
1515
|
+
confirm_outcome: confirmOutcome,
|
|
1516
|
+
recorded_exit_code: recorded.exitCode,
|
|
1517
|
+
agreement: v.agreement,
|
|
1518
|
+
confirmed: v.confirmed,
|
|
1519
|
+
unconfirmed_fail: v.unconfirmed_fail,
|
|
1520
|
+
evidence: String(result.evidence || "").slice(0, 200),
|
|
1521
|
+
};
|
|
1522
|
+
}
|
|
1523
|
+
|
|
1524
|
+
// recordVerifyShadowCodex({rolloutPath, session, cwd}) — the Codex LIVE test-proof catch. Mirrors the Claude
|
|
1525
|
+
// test path (claim → reconstruct proof → re-run → 2/2 flake-confirm) and ADDS the recorded-exit agreement
|
|
1526
|
+
// gate. Logs one `harness:"codex"` shadow row; surfaces a red ONLY on a confirmed+agreeing catch (and only
|
|
1527
|
+
// when the Codex reveal gate + arming resolver are on). Ship-status is intentionally NOT run for Codex here:
|
|
1528
|
+
// the recorded-exit lever has no probe analogue and routing raw rollout text through the URL/PR miner is
|
|
1529
|
+
// unproven — honest silence beats a possible false ship-red. LOG ONLY otherwise; never injects, never throws.
|
|
1530
|
+
async function recordVerifyShadowCodex({ rolloutPath, session, cwd }) {
|
|
1531
|
+
try {
|
|
1532
|
+
const entries = readEntries(rolloutPath); // normalized Claude shape (Codex detected inside readEntries)
|
|
1533
|
+
if (!entries.length) return;
|
|
1534
|
+
const { claim, text } = detectCompletionClaim(lastAssistantText(entries));
|
|
1535
|
+
if (!claim) return; // no claim this turn → nothing to verify
|
|
1536
|
+
const sig = `codex|${session || ""}|${djb2(text)}`;
|
|
1537
|
+
if (alreadyVerified(sig)) return;
|
|
1538
|
+
const found = reconstructProofEntry(entries, cwd);
|
|
1539
|
+
const proof = found ? found.proof : null;
|
|
1540
|
+
if (!proof) {
|
|
1541
|
+
markVerified(sig);
|
|
1542
|
+
appendShadowLog({
|
|
1543
|
+
ts: new Date().toISOString(),
|
|
1544
|
+
session: session || null,
|
|
1545
|
+
harness: "codex",
|
|
1546
|
+
claim_text: text,
|
|
1547
|
+
proof_command: null,
|
|
1548
|
+
pass_fail: null,
|
|
1549
|
+
mismatch: false,
|
|
1550
|
+
evidence: "",
|
|
1551
|
+
});
|
|
1552
|
+
return;
|
|
1553
|
+
}
|
|
1554
|
+
// A proof exists → re-run it (single-flight, shared with the Claude/ship paths). touchLock refreshes our
|
|
1555
|
+
// own lock mtime between the two 2/2 re-runs so it isn't read as stale mid-confirm.
|
|
1556
|
+
if (!acquireLock()) return;
|
|
1557
|
+
let result;
|
|
1558
|
+
let confirm = null;
|
|
1559
|
+
try {
|
|
1560
|
+
result = await rerunProof(proof);
|
|
1561
|
+
if ((result.outcome || (result.pass ? "PASS" : "GENUINE_FAIL")) === "GENUINE_FAIL") {
|
|
1562
|
+
touchLock();
|
|
1563
|
+
confirm = await rerunProof(proof);
|
|
1564
|
+
}
|
|
1565
|
+
} finally {
|
|
1566
|
+
releaseLock();
|
|
1567
|
+
}
|
|
1568
|
+
markVerified(sig);
|
|
1569
|
+
const outcome = result.outcome || (result.pass ? "PASS" : "GENUINE_FAIL");
|
|
1570
|
+
const confirmOutcome = confirm
|
|
1571
|
+
? confirm.outcome || (confirm.pass ? "PASS" : "GENUINE_FAIL")
|
|
1572
|
+
: null;
|
|
1573
|
+
const recorded = recordedExitFor(entries, found.toolUseId);
|
|
1574
|
+
const v = assembleCodexVerdict({ outcome, confirmOutcome, recorded });
|
|
1575
|
+
const pass_fail =
|
|
1576
|
+
outcome === "PASS" ? "PASS" : outcome === "GENUINE_FAIL" ? "FAIL" : "HARNESS_ERROR";
|
|
1577
|
+
const row = {
|
|
1578
|
+
ts: new Date().toISOString(),
|
|
1579
|
+
session: session || null,
|
|
1580
|
+
harness: "codex",
|
|
1581
|
+
claim_text: text,
|
|
1582
|
+
proof_command: [proof.cmd, ...proof.args].join(" ").slice(0, 200),
|
|
1583
|
+
pass_fail,
|
|
1584
|
+
outcome,
|
|
1585
|
+
confirm_outcome: confirmOutcome,
|
|
1586
|
+
// Codex's OWN recorded exit for this exact command — the corroboration the reveal requires.
|
|
1587
|
+
recorded_exit_code: recorded.exitCode,
|
|
1588
|
+
codex_agreement: v.agreement,
|
|
1589
|
+
// THE money metric for Codex: 2/2 live GENUINE_FAIL AND recorded-exit agreement.
|
|
1590
|
+
mismatch: v.confirmed,
|
|
1591
|
+
// logged-not-shown: a flake OR a recorded-exit disagreement/unknown.
|
|
1592
|
+
unconfirmed_fail: v.unconfirmed_fail,
|
|
1593
|
+
evidence: String(result.evidence || "").slice(0, 200),
|
|
1594
|
+
};
|
|
1595
|
+
appendShadowLog(row);
|
|
1596
|
+
// Surface a red ONLY on a confirmed+agreeing Codex catch, and only when the Codex reveal gate is on.
|
|
1597
|
+
// maybeSurfaceReveal still enforces the arming resolver + suppression + auto-darken, so this stays
|
|
1598
|
+
// byte-identical-dark by default. A corroborated PASS retracts any stale Codex red (no green verdict in
|
|
1599
|
+
// v1 — the earned-green surface stays Claude-only).
|
|
1600
|
+
if (v.confirmed && CODEX_REVEAL_ENABLED) {
|
|
1601
|
+
maybeSurfaceReveal(row);
|
|
1602
|
+
} else if (outcome === "PASS" && recorded.exitCode === 0 && revealEnabled(process.env)) {
|
|
1603
|
+
clearReveal(session || null);
|
|
1604
|
+
}
|
|
1605
|
+
} catch {
|
|
1606
|
+
/* SHADOW: any failure is swallowed — this must never affect the hook or the turn */
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1400
1610
|
// recordVerifyShadow({transcriptPath, session, cwd}) — the wired entry. On (claim + reconstructable
|
|
1401
1611
|
// proof) it re-runs the proof and appends one shadow row. LOG ONLY. Never injects, never throws.
|
|
1402
1612
|
export async function recordVerifyShadow({ transcriptPath, session, cwd }) {
|
|
1403
1613
|
try {
|
|
1404
1614
|
if (!transcriptPath) return;
|
|
1615
|
+
// CODEX LIVE-CATCH PARITY: a Codex session takes its own precision-gated path (recorded-exit agreement)
|
|
1616
|
+
// and returns; the Claude path below is unchanged. resolveCodexRollout also covers the #6 fallback
|
|
1617
|
+
// (resolve the newest ~/.codex rollout when the handed path isn't a usable rollout) — hook untouched.
|
|
1618
|
+
const codexRollout = resolveCodexRollout(transcriptPath);
|
|
1619
|
+
if (codexRollout) {
|
|
1620
|
+
await recordVerifyShadowCodex({ rolloutPath: codexRollout, session, cwd });
|
|
1621
|
+
return;
|
|
1622
|
+
}
|
|
1405
1623
|
const entries = readEntries(transcriptPath);
|
|
1406
1624
|
if (!entries.length) return;
|
|
1407
1625
|
const assistantText = lastAssistantText(entries);
|
|
@@ -1515,8 +1733,11 @@ export function renderVerifyReport() {
|
|
|
1515
1733
|
// The test-proof instrument counts ONLY test rows (no `category`). Ship-status rows (category:"ship")
|
|
1516
1734
|
// are a separate proof-category with their own probe verbs, reported in their own section below — so
|
|
1517
1735
|
// adding them never perturbs the pre-existing test numbers on old logs.
|
|
1518
|
-
|
|
1736
|
+
// Codex catch rows carry `harness:"codex"` and get their OWN section below — excluded here so they never
|
|
1737
|
+
// perturb the Claude test numbers (a pure-Claude log has no such rows → this filter is a no-op there).
|
|
1738
|
+
const testRows = rows.filter((r) => r.category !== "ship" && r.harness !== "codex");
|
|
1519
1739
|
const shipRows = rows.filter((r) => r.category === "ship");
|
|
1740
|
+
const codexRows = rows.filter((r) => r.harness === "codex");
|
|
1520
1741
|
|
|
1521
1742
|
const claims = testRows.length;
|
|
1522
1743
|
const withProof = testRows.filter((r) => r.proof_command).length;
|
|
@@ -1651,6 +1872,47 @@ export function renderVerifyReport() {
|
|
|
1651
1872
|
}
|
|
1652
1873
|
}
|
|
1653
1874
|
|
|
1875
|
+
// CODEX LIVE-CATCH PARITY — the test-proof catch over a Codex session. Same 2/2 flake-confirm as Claude
|
|
1876
|
+
// PLUS a recorded-exit-code agreement gate (a live fail is a catch ONLY when Codex's OWN recorded run of
|
|
1877
|
+
// that command also failed), so a Codex catch is strictly more precision-gated. Its own section so it
|
|
1878
|
+
// never perturbs the Claude test numbers above.
|
|
1879
|
+
if (codexRows.length) {
|
|
1880
|
+
const cReran = codexRows.filter((r) => r.pass_fail === "PASS" || r.pass_fail === "FAIL").length;
|
|
1881
|
+
const cWithProof = codexRows.filter((r) => r.proof_command).length;
|
|
1882
|
+
const cCaught = codexRows.filter((r) => r.mismatch === true).length;
|
|
1883
|
+
const cSuppressed = codexRows.filter((r) => r.unconfirmed_fail === true).length;
|
|
1884
|
+
const cDisagree = codexRows.filter((r) => r.codex_agreement === "disagree").length;
|
|
1885
|
+
const cUnknown = codexRows.filter((r) => r.codex_agreement === "unknown").length;
|
|
1886
|
+
L.push("");
|
|
1887
|
+
L.push("─".repeat(72));
|
|
1888
|
+
L.push(" CODEX catch (Codex session · 2/2 flake-confirm + recorded-exit-code agreement)");
|
|
1889
|
+
L.push(` codex claims logged ${codexRows.length}`);
|
|
1890
|
+
L.push(
|
|
1891
|
+
` (C1) proof-reconstruction rate ${pct(cWithProof, codexRows.length)} ${cWithProof}/${codexRows.length} claims had a re-runnable proof`,
|
|
1892
|
+
);
|
|
1893
|
+
L.push(
|
|
1894
|
+
` (C2) CODEX-MISMATCH rate ${pct(cCaught, cReran)} ${cCaught}/${cReran} re-run proofs failed 2/2 AND agreed with Codex's recorded exit`,
|
|
1895
|
+
);
|
|
1896
|
+
L.push(
|
|
1897
|
+
` (suppressed: ${cSuppressed} fail(s) never shown — ${cDisagree} recorded-exit disagreement + ${cUnknown} unknown + flakes)`,
|
|
1898
|
+
);
|
|
1899
|
+
const recentCodex = codexRows
|
|
1900
|
+
.filter((r) => r.mismatch)
|
|
1901
|
+
.slice(-3)
|
|
1902
|
+
.reverse();
|
|
1903
|
+
if (recentCodex.length) {
|
|
1904
|
+
L.push("");
|
|
1905
|
+
L.push(" recent codex catches:");
|
|
1906
|
+
for (const r of recentCodex) {
|
|
1907
|
+
L.push(` • claim: ${String(r.claim_text || "").slice(0, 68)}`);
|
|
1908
|
+
L.push(
|
|
1909
|
+
` proof: ${String(r.proof_command || "").slice(0, 68)} → FAIL (recorded exit ${r.recorded_exit_code})`,
|
|
1910
|
+
);
|
|
1911
|
+
if (r.evidence) L.push(` ${String(r.evidence).slice(0, 68)}`);
|
|
1912
|
+
}
|
|
1913
|
+
}
|
|
1914
|
+
}
|
|
1915
|
+
|
|
1654
1916
|
for (const l of fieldProofLines(mismatches)) L.push(l);
|
|
1655
1917
|
|
|
1656
1918
|
L.push("");
|