skalpel 4.0.27 → 4.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/catches.mjs +413 -0
- package/insights.mjs +6 -0
- package/package.json +1 -1
- package/skalpel-setup.mjs +28 -0
- package/skalpel-statusline.mjs +39 -1
- package/verify-shadow.mjs +196 -8
package/catches.mjs
ADDED
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// catches.mjs — the RETROACTIVE first-run "wow": `skalpel catches`.
|
|
3
|
+
//
|
|
4
|
+
// THE PROBLEM (same as verify-shadow): your agent is both the executor AND the reporter of its own
|
|
5
|
+
// work. When it says "done / tests pass / fixed" nothing independent re-checks. verify-shadow closes
|
|
6
|
+
// that loop LIVE, but a brand-new user has no live catches yet — so there is no instant aha on install.
|
|
7
|
+
//
|
|
8
|
+
// THIS closes the aha gap by looking BACKWARD: it scans the user's OWN recent coding transcripts
|
|
9
|
+
// (Claude Code ~/.claude/projects/**/*.jsonl and Codex ~/.codex/{sessions,archived_sessions}) and finds
|
|
10
|
+
// REAL past turns where the agent CLAIMED success while its OWN in-transcript proof — the tool_result of
|
|
11
|
+
// the last test/build/lint it ran before the claim — had ALREADY FAILED at claim time. That is a genuine
|
|
12
|
+
// claim-vs-own-evidence mismatch that is verifiable FROM THE TRANSCRIPT ALONE.
|
|
13
|
+
//
|
|
14
|
+
// HARD RED LINES (why this is safe + instant):
|
|
15
|
+
// • READ-ONLY. It only READS transcripts already on disk. It NEVER re-runs, spawns, builds, writes,
|
|
16
|
+
// deploys, or mutates anything. Unlike verify-shadow it does NOT execFile a proof — the pass/fail is
|
|
17
|
+
// the exit status the transcript ALREADY recorded (Claude's tool_result.is_error / Codex's
|
|
18
|
+
// "Process exited with code N"). So it is instant and cannot break or slow anything.
|
|
19
|
+
// • LOCAL ONLY. Imports only node: core + local sibling modules. Opens no socket.
|
|
20
|
+
// • OPT-IN. User-invoked subcommand; prints and exits. The per-turn hook is untouched (byte-identical).
|
|
21
|
+
// • NO FABRICATED NUMBERS. Every count is a real tally over real transcripts; every example quotes the
|
|
22
|
+
// real claim text, the real reconstructed proof command, and the real recorded failure output. If it
|
|
23
|
+
// finds zero, it SAYS zero honestly — it never invents a catch.
|
|
24
|
+
//
|
|
25
|
+
// REUSE: the detection + classification logic is verify-shadow's, applied to historical turns —
|
|
26
|
+
// detectCompletionClaim (is this a "done" claim?), reconstructProofEntry (the most-recent proof the
|
|
27
|
+
// session actually ran before the claim, + the tool_use id to match its result), and classifyOutcome
|
|
28
|
+
// (PASS / GENUINE_FAIL / HARNESS_ERROR — only a GENUINE_FAIL counts as a lie; a timeout / missing-script
|
|
29
|
+
// / wrong-cwd is harness noise and is NEVER surfaced, so we never cry wolf).
|
|
30
|
+
import { readdirSync, statSync } from "node:fs";
|
|
31
|
+
import { homedir } from "node:os";
|
|
32
|
+
import path from "node:path";
|
|
33
|
+
import { fileURLToPath } from "node:url";
|
|
34
|
+
import { realpathSync } from "node:fs";
|
|
35
|
+
import { tailLines } from "./transcript.mjs";
|
|
36
|
+
import {
|
|
37
|
+
detectCompletionClaim,
|
|
38
|
+
reconstructProofEntry,
|
|
39
|
+
classifyOutcome,
|
|
40
|
+
extractFailCount,
|
|
41
|
+
} from "./verify-shadow.mjs";
|
|
42
|
+
import { recordInsight } from "./insights.mjs";
|
|
43
|
+
|
|
44
|
+
const HOME = homedir();
|
|
45
|
+
const CLAUDE_PROJECTS = path.join(HOME, ".claude", "projects");
|
|
46
|
+
const CODEX_DIRS = [
|
|
47
|
+
path.join(HOME, ".codex", "sessions"),
|
|
48
|
+
path.join(HOME, ".codex", "archived_sessions"),
|
|
49
|
+
];
|
|
50
|
+
|
|
51
|
+
// Bounds. We look at RECENT sessions (the aha is about YOUR history, and reading every one of tens of
|
|
52
|
+
// thousands of transcripts would be neither instant nor "recent"). Newest-first by mtime, capped, each
|
|
53
|
+
// read as a bounded tail so a multi-GB transcript can never stall or OOM us.
|
|
54
|
+
const MAX_SESSIONS = 600; // most-recent sessions to scan (across both tools)
|
|
55
|
+
const SESSION_TAIL_BYTES = 4 * 1024 * 1024; // per-file tail — plenty for the last many turns
|
|
56
|
+
const MAX_EXAMPLES = 3; // concrete receipts to print
|
|
57
|
+
|
|
58
|
+
// ---------- (1) discover recent session files (newest first) ----------
|
|
59
|
+
function walkJsonl(dir, pred, out) {
|
|
60
|
+
let entries;
|
|
61
|
+
try {
|
|
62
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
63
|
+
} catch {
|
|
64
|
+
return; // dir missing / unreadable — skip (read-only, fail-open)
|
|
65
|
+
}
|
|
66
|
+
for (const e of entries) {
|
|
67
|
+
const p = path.join(dir, e.name);
|
|
68
|
+
try {
|
|
69
|
+
if (e.isDirectory()) walkJsonl(p, pred, out);
|
|
70
|
+
else if (e.isFile() && pred(e.name)) out.push(p);
|
|
71
|
+
} catch {
|
|
72
|
+
/* stat/permission hiccup on one entry — skip it */
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Return [{ file, tool, mtime }] for the MAX_SESSIONS most-recently-modified transcripts across tools.
|
|
78
|
+
function discoverSessions() {
|
|
79
|
+
const files = [];
|
|
80
|
+
const claude = [];
|
|
81
|
+
walkJsonl(CLAUDE_PROJECTS, (n) => n.endsWith(".jsonl"), claude);
|
|
82
|
+
for (const f of claude) files.push({ file: f, tool: "claude" });
|
|
83
|
+
const codex = [];
|
|
84
|
+
for (const d of CODEX_DIRS) walkJsonl(d, (n) => /^rollout-.*\.jsonl$/.test(n), codex);
|
|
85
|
+
for (const f of codex) files.push({ file: f, tool: "codex" });
|
|
86
|
+
|
|
87
|
+
const withMtime = [];
|
|
88
|
+
for (const f of files) {
|
|
89
|
+
try {
|
|
90
|
+
withMtime.push({ ...f, mtime: statSync(f.file).mtimeMs });
|
|
91
|
+
} catch {
|
|
92
|
+
/* vanished between readdir and stat — skip */
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
withMtime.sort((a, b) => b.mtime - a.mtime);
|
|
96
|
+
return withMtime.slice(0, MAX_SESSIONS);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// ---------- (2) load a session as Claude-shaped entries ----------
|
|
100
|
+
// verify-shadow's reconstructProofEntry + claim detection speak the Claude transcript shape
|
|
101
|
+
// (entry.message.content = [{type:'text'|'tool_use'|'tool_result', ...}], entry.cwd). We parse Claude
|
|
102
|
+
// natively and NORMALIZE Codex rollout events into that same shape so ONE code path handles both.
|
|
103
|
+
|
|
104
|
+
export function loadClaudeEntries(file) {
|
|
105
|
+
const entries = [];
|
|
106
|
+
for (const line of tailLines(file, SESSION_TAIL_BYTES)) {
|
|
107
|
+
try {
|
|
108
|
+
entries.push(JSON.parse(line));
|
|
109
|
+
} catch {
|
|
110
|
+
/* partial/garbage line — skip */
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return entries;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// Codex output wrapper always carries "Process exited with code N" — the recorded exit status, which is
|
|
117
|
+
// exactly the ground truth we need (no re-run). Returns { exitCode:number|null, body:string }.
|
|
118
|
+
function parseCodexOutput(output) {
|
|
119
|
+
const s = typeof output === "string" ? output : "";
|
|
120
|
+
const m = s.match(/Process exited with code\s+(-?\d+)/i);
|
|
121
|
+
const exitCode = m ? parseInt(m[1], 10) : null;
|
|
122
|
+
// Prefer the text AFTER "Output:" as the evidence body (drops the Command/Chunk/Wall-time header).
|
|
123
|
+
const oi = s.indexOf("\nOutput:\n");
|
|
124
|
+
const body = oi >= 0 ? s.slice(oi + "\nOutput:\n".length) : s;
|
|
125
|
+
return { exitCode, body };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Normalize one Codex rollout file into Claude-shaped entries. exec_command → a Bash tool_use; its
|
|
129
|
+
// function_call_output → a tool_result carrying the recorded exit code; assistant messages → text.
|
|
130
|
+
export function loadCodexEntries(file) {
|
|
131
|
+
const entries = [];
|
|
132
|
+
for (const line of tailLines(file, SESSION_TAIL_BYTES)) {
|
|
133
|
+
let e;
|
|
134
|
+
try {
|
|
135
|
+
e = JSON.parse(line);
|
|
136
|
+
} catch {
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
if (e.type !== "response_item") continue;
|
|
140
|
+
const p = e.payload || {};
|
|
141
|
+
if (p.type === "message") {
|
|
142
|
+
const role = p.role === "assistant" ? "assistant" : "user";
|
|
143
|
+
const text = Array.isArray(p.content)
|
|
144
|
+
? p.content.map((b) => (b && typeof b.text === "string" ? b.text : "")).join("")
|
|
145
|
+
: "";
|
|
146
|
+
if (text.trim())
|
|
147
|
+
entries.push({ type: role, message: { role, content: [{ type: "text", text }] } });
|
|
148
|
+
} else if (p.type === "function_call" && p.name === "exec_command" && p.call_id) {
|
|
149
|
+
let cmd = "";
|
|
150
|
+
let workdir = null;
|
|
151
|
+
try {
|
|
152
|
+
const a = JSON.parse(p.arguments || "{}");
|
|
153
|
+
cmd = typeof a.cmd === "string" ? a.cmd : Array.isArray(a.cmd) ? a.cmd.join(" ") : "";
|
|
154
|
+
workdir = typeof a.workdir === "string" ? a.workdir : null;
|
|
155
|
+
} catch {
|
|
156
|
+
/* unparseable args — leave cmd empty (no proof) */
|
|
157
|
+
}
|
|
158
|
+
if (cmd)
|
|
159
|
+
entries.push({
|
|
160
|
+
cwd: workdir,
|
|
161
|
+
message: {
|
|
162
|
+
role: "assistant",
|
|
163
|
+
content: [{ type: "tool_use", name: "Bash", id: p.call_id, input: { command: cmd } }],
|
|
164
|
+
},
|
|
165
|
+
});
|
|
166
|
+
} else if (p.type === "function_call_output" && p.call_id) {
|
|
167
|
+
const { exitCode, body } = parseCodexOutput(p.output);
|
|
168
|
+
entries.push({
|
|
169
|
+
message: {
|
|
170
|
+
role: "user",
|
|
171
|
+
content: [
|
|
172
|
+
{
|
|
173
|
+
type: "tool_result",
|
|
174
|
+
tool_use_id: p.call_id,
|
|
175
|
+
is_error: typeof exitCode === "number" ? exitCode !== 0 : false,
|
|
176
|
+
exit_code: exitCode,
|
|
177
|
+
content: body,
|
|
178
|
+
},
|
|
179
|
+
],
|
|
180
|
+
},
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
return entries;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// ---------- (3) index the recorded proof results (no re-run) ----------
|
|
188
|
+
function resultTextOf(block) {
|
|
189
|
+
const c = block?.content;
|
|
190
|
+
if (typeof c === "string") return c;
|
|
191
|
+
if (Array.isArray(c)) return c.map((x) => (typeof x === "string" ? x : x?.text || "")).join(" ");
|
|
192
|
+
return "";
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// tool_use_id -> { isError, exitCode, text } for every tool_result in the session.
|
|
196
|
+
function indexResults(entries) {
|
|
197
|
+
const idx = new Map();
|
|
198
|
+
for (const e of entries) {
|
|
199
|
+
const c = e?.message?.content;
|
|
200
|
+
if (!Array.isArray(c)) continue;
|
|
201
|
+
for (const b of c) {
|
|
202
|
+
if (b && b.type === "tool_result" && b.tool_use_id) {
|
|
203
|
+
idx.set(b.tool_use_id, {
|
|
204
|
+
isError: b.is_error === true,
|
|
205
|
+
exitCode: typeof b.exit_code === "number" ? b.exit_code : null,
|
|
206
|
+
text: resultTextOf(b),
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return idx;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// Reuse verify-shadow's classifyOutcome by synthesizing the `err` object it expects from the RECORDED
|
|
215
|
+
// result (we never ran the process, so there is no live err). This keeps HARNESS_ERROR vs GENUINE_FAIL
|
|
216
|
+
// identical to the live shadow: a timeout / command-not-found / missing-script is harness noise and is
|
|
217
|
+
// NEVER counted as a lie.
|
|
218
|
+
function classifyRecorded({ isError, exitCode, text }) {
|
|
219
|
+
if (exitCode === 0) return "PASS";
|
|
220
|
+
if (exitCode == null && !isError) return "PASS"; // no failure signal recorded → treat as pass (never invent)
|
|
221
|
+
const ev = String(text || "");
|
|
222
|
+
let err;
|
|
223
|
+
if (/\btimed?\s*out\b|timeout after|\bSIGKILL\b|\bSIGTERM\b/i.test(ev)) {
|
|
224
|
+
err = { killed: true, signal: "SIGTERM" }; // classifyOutcome → HARNESS_ERROR
|
|
225
|
+
} else if (typeof exitCode === "number" && exitCode !== 0) {
|
|
226
|
+
err = { code: exitCode }; // real recorded non-zero exit (127/126 → HARNESS_ERROR in classifyOutcome)
|
|
227
|
+
} else {
|
|
228
|
+
err = { code: 1 }; // Claude is_error with no numeric code → generic non-zero
|
|
229
|
+
}
|
|
230
|
+
return classifyOutcome(err, ev);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// ---------- (4) scan one session for genuine claim-vs-proof mismatches ----------
|
|
234
|
+
// For each assistant turn carrying a completion claim, reconstruct the most-recent proof the session ran
|
|
235
|
+
// BEFORE that claim, look up that proof's RECORDED result, and classify it. A GENUINE_FAIL = a caught
|
|
236
|
+
// "done" over its own already-failing test. Dedup by proof tool_use id so repeated "done"s about the
|
|
237
|
+
// same failing run count once.
|
|
238
|
+
function assistantTextOf(entry) {
|
|
239
|
+
if (entry?.type !== "assistant" && entry?.message?.role !== "assistant") return "";
|
|
240
|
+
const c = entry?.message?.content;
|
|
241
|
+
if (typeof c === "string") return c;
|
|
242
|
+
if (Array.isArray(c))
|
|
243
|
+
return c
|
|
244
|
+
.filter((b) => b && b.type === "text")
|
|
245
|
+
.map((b) => b.text || "")
|
|
246
|
+
.join(" ")
|
|
247
|
+
.trim();
|
|
248
|
+
return "";
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
export function scanSession(entries, meta = {}) {
|
|
252
|
+
const catches = [];
|
|
253
|
+
if (!Array.isArray(entries) || !entries.length) return catches;
|
|
254
|
+
const results = indexResults(entries);
|
|
255
|
+
const seenProof = new Set(); // proof tool_use ids already reported in THIS session
|
|
256
|
+
|
|
257
|
+
for (let i = 0; i < entries.length; i++) {
|
|
258
|
+
const text = assistantTextOf(entries[i]);
|
|
259
|
+
if (!text) continue;
|
|
260
|
+
const { claim, text: claimText } = detectCompletionClaim(text);
|
|
261
|
+
if (!claim) continue;
|
|
262
|
+
|
|
263
|
+
// The most-recent proof the session actually ran up to and including this claim turn.
|
|
264
|
+
const found = reconstructProofEntry(entries.slice(0, i + 1), meta.cwd || null);
|
|
265
|
+
if (!found || !found.toolUseId) continue;
|
|
266
|
+
if (seenProof.has(found.toolUseId)) continue;
|
|
267
|
+
|
|
268
|
+
const res = results.get(found.toolUseId);
|
|
269
|
+
if (!res) continue; // proof's result not recorded in the scanned window — can't verify, skip honestly
|
|
270
|
+
|
|
271
|
+
const outcome = classifyRecorded(res);
|
|
272
|
+
if (outcome !== "GENUINE_FAIL") continue; // PASS / HARNESS_ERROR are never a caught lie
|
|
273
|
+
|
|
274
|
+
seenProof.add(found.toolUseId);
|
|
275
|
+
const evidence = String(res.text || "")
|
|
276
|
+
.replace(/\s+$/, "")
|
|
277
|
+
.slice(-240);
|
|
278
|
+
catches.push({
|
|
279
|
+
tool: meta.tool || "claude",
|
|
280
|
+
session: meta.session || null,
|
|
281
|
+
claim_text: claimText,
|
|
282
|
+
proof_command: [found.proof.cmd, ...found.proof.args].join(" ").slice(0, 160),
|
|
283
|
+
exit_code: res.exitCode,
|
|
284
|
+
fail_count: extractFailCount(evidence),
|
|
285
|
+
evidence,
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
return catches;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// ---------- (5) run the scan across recent sessions ----------
|
|
292
|
+
export function runScan() {
|
|
293
|
+
const sessions = discoverSessions();
|
|
294
|
+
let scanned = 0;
|
|
295
|
+
const allCatches = [];
|
|
296
|
+
for (const s of sessions) {
|
|
297
|
+
let entries;
|
|
298
|
+
try {
|
|
299
|
+
entries = s.tool === "codex" ? loadCodexEntries(s.file) : loadClaudeEntries(s.file);
|
|
300
|
+
} catch {
|
|
301
|
+
continue; // unreadable file — skip (read-only, fail-open)
|
|
302
|
+
}
|
|
303
|
+
if (!entries.length) continue;
|
|
304
|
+
scanned++;
|
|
305
|
+
const sessionId = path.basename(s.file, ".jsonl");
|
|
306
|
+
let found;
|
|
307
|
+
try {
|
|
308
|
+
found = scanSession(entries, { tool: s.tool, session: sessionId });
|
|
309
|
+
} catch {
|
|
310
|
+
continue; // a single malformed session must never abort the whole scan
|
|
311
|
+
}
|
|
312
|
+
for (const c of found) allCatches.push(c);
|
|
313
|
+
}
|
|
314
|
+
return { scanned, catches: allCatches };
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// ---------- (6) render the honest report ----------
|
|
318
|
+
function clip(s, n) {
|
|
319
|
+
const t = String(s || "")
|
|
320
|
+
.replace(/\s+/g, " ")
|
|
321
|
+
.trim();
|
|
322
|
+
return t.length > n ? t.slice(0, n - 1) + "…" : t;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
export function renderReport({ scanned, catches }) {
|
|
326
|
+
const N = scanned;
|
|
327
|
+
const K = catches.length;
|
|
328
|
+
const L = [];
|
|
329
|
+
L.push("");
|
|
330
|
+
L.push(" 🔬 skalpel — retroactive claim check (read-only scan of your own coding history)");
|
|
331
|
+
L.push(" " + "─".repeat(72));
|
|
332
|
+
if (N === 0) {
|
|
333
|
+
L.push("");
|
|
334
|
+
L.push(" No Claude Code or Codex transcripts found on this machine yet.");
|
|
335
|
+
L.push(" As you code, skalpel watches live — run me again anytime.");
|
|
336
|
+
L.push("");
|
|
337
|
+
return L.join("\n");
|
|
338
|
+
}
|
|
339
|
+
if (K === 0) {
|
|
340
|
+
// HONEST zero-state — we do NOT invent a catch.
|
|
341
|
+
L.push("");
|
|
342
|
+
L.push(` skalpel scanned ${N} of your recent sessions and found`);
|
|
343
|
+
L.push(` 0 times your agent said "done" while its own test/build was already failing.`);
|
|
344
|
+
L.push("");
|
|
345
|
+
L.push(" No caught lies in your history — clean. I'll watch live from here:");
|
|
346
|
+
L.push(" every turn your agent claims done, I check its own proof against the claim.");
|
|
347
|
+
L.push("");
|
|
348
|
+
return L.join("\n");
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
const cmds = [...new Set(catches.map((c) => c.proof_command.split(/\s+/).slice(0, 3).join(" ")))];
|
|
352
|
+
const cmdPhrase = cmds.length === 1 ? `\`${cmds[0]}\`` : "its own test/build";
|
|
353
|
+
L.push("");
|
|
354
|
+
L.push(` skalpel scanned ${N} of your recent sessions and found`);
|
|
355
|
+
L.push(
|
|
356
|
+
` ${K} time${K === 1 ? "" : "s"} your agent said "done" while ${cmdPhrase} was already failing.`,
|
|
357
|
+
);
|
|
358
|
+
L.push("");
|
|
359
|
+
L.push(" Every one is real — pulled straight from your transcript, no re-run needed:");
|
|
360
|
+
L.push("");
|
|
361
|
+
|
|
362
|
+
const top = catches.slice(0, MAX_EXAMPLES);
|
|
363
|
+
top.forEach((c, i) => {
|
|
364
|
+
const failN = c.fail_count != null ? ` (${c.fail_count} failing)` : "";
|
|
365
|
+
L.push(` ${i + 1}. [${c.tool}] your agent said:`);
|
|
366
|
+
L.push(` "${clip(c.claim_text, 72)}"`);
|
|
367
|
+
L.push(` but its own \`${clip(c.proof_command, 64)}\` had just FAILED${failN}:`);
|
|
368
|
+
L.push(` ${clip(c.evidence, 72)}`);
|
|
369
|
+
L.push("");
|
|
370
|
+
});
|
|
371
|
+
if (K > top.length) L.push(` …and ${K - top.length} more.`);
|
|
372
|
+
L.push(" " + "─".repeat(72));
|
|
373
|
+
L.push(" That's the gap skalpel closes. From here I check every claim live, as you code.");
|
|
374
|
+
L.push("");
|
|
375
|
+
return L.join("\n");
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
// ---------- CLI ----------
|
|
379
|
+
const isMain = (() => {
|
|
380
|
+
try {
|
|
381
|
+
return (
|
|
382
|
+
Boolean(process.argv[1]) &&
|
|
383
|
+
realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url))
|
|
384
|
+
);
|
|
385
|
+
} catch {
|
|
386
|
+
return false;
|
|
387
|
+
}
|
|
388
|
+
})();
|
|
389
|
+
|
|
390
|
+
if (isMain) {
|
|
391
|
+
try {
|
|
392
|
+
const result = runScan();
|
|
393
|
+
process.stdout.write(renderReport(result) + "\n");
|
|
394
|
+
// MEASURABLE (local only, no network): record the honest fire-frequency for THIS user — how many
|
|
395
|
+
// recent sessions were scanned and how many genuine catches were found. Never fabricated.
|
|
396
|
+
try {
|
|
397
|
+
recordInsight({
|
|
398
|
+
kind: "catches_scan",
|
|
399
|
+
display: `retroactive scan: ${result.catches.length} catch(es) across ${result.scanned} sessions`,
|
|
400
|
+
catches: result.catches.length,
|
|
401
|
+
sessions: result.scanned,
|
|
402
|
+
});
|
|
403
|
+
} catch {
|
|
404
|
+
/* insight is a bonus */
|
|
405
|
+
}
|
|
406
|
+
} catch {
|
|
407
|
+
// READ-ONLY + fail-open: never throw at the user.
|
|
408
|
+
process.stdout.write(
|
|
409
|
+
"\n 🔬 skalpel — could not read your transcript history on this machine.\n\n",
|
|
410
|
+
);
|
|
411
|
+
}
|
|
412
|
+
process.exit(0);
|
|
413
|
+
}
|
package/insights.mjs
CHANGED
|
@@ -24,6 +24,12 @@ export const INSIGHTS_PATH = join(DIR, "insights.ndjson");
|
|
|
24
24
|
// user-visible bar). Both the flag semantics and the file path live HERE — the one light module both the
|
|
25
25
|
// writer and the reader (and the hook's spawn gate) already import — so they can never drift apart.
|
|
26
26
|
export const REVEAL_PATH = join(DIR, "verify-reveal.json");
|
|
27
|
+
// verify-verdict: the POSITIVE half of the same surface — the quiet green "✓ verified" stamp written on
|
|
28
|
+
// a PASS (the agent claimed done and its OWN proof, re-run, really passed). Same writer (verify-shadow.mjs)
|
|
29
|
+
// and reader (skalpel-statusline.mjs), same opt-in flag as the red reveal. Ambient greens are what let the
|
|
30
|
+
// rare red catch land like a thunderclap — without a stream of honest verified stamps a lone red reads as
|
|
31
|
+
// noise, not a verdict. Path single-sourced HERE alongside REVEAL_PATH so writer + reader never drift.
|
|
32
|
+
export const VERDICT_PATH = join(DIR, "verify-verdict.json");
|
|
27
33
|
// SKALPEL_VERIFY_REVEAL is OPT-IN and OFF BY DEFAULT. Unset / "" / "0" / "off" / "false" / "no" → DARK
|
|
28
34
|
// (byte-identical to the shadow-only behavior). Only an explicit truthy value arms the visible reveal.
|
|
29
35
|
export function revealEnabled(env) {
|
package/package.json
CHANGED
package/skalpel-setup.mjs
CHANGED
|
@@ -60,6 +60,8 @@ const KNOWN_SUBS = new Set([
|
|
|
60
60
|
"logout",
|
|
61
61
|
"uninstall",
|
|
62
62
|
"autopsy",
|
|
63
|
+
"xray",
|
|
64
|
+
"catches",
|
|
63
65
|
"__build",
|
|
64
66
|
"__verify-report",
|
|
65
67
|
]);
|
|
@@ -73,6 +75,8 @@ usage:
|
|
|
73
75
|
skalpel login (re-)run the Google sign-in
|
|
74
76
|
skalpel logout clear the saved session
|
|
75
77
|
skalpel autopsy local, read-only receipt of your verified patterns
|
|
78
|
+
skalpel xray re-run your last session's own proof — did "done" hold up?
|
|
79
|
+
skalpel catches scan your own history for "done" claims over failing tests
|
|
76
80
|
skalpel uninstall [--purge] remove the hooks + local data from this machine
|
|
77
81
|
|
|
78
82
|
options:
|
|
@@ -432,6 +436,13 @@ function promptLaunch(targets) {
|
|
|
432
436
|
|
|
433
437
|
async function reportLiveAndLaunch(message) {
|
|
434
438
|
console.log(message);
|
|
439
|
+
// FIRST-RUN X-RAY pointer: offer the user their own first catch — an independent re-run of the proof
|
|
440
|
+
// from the session they just had. A pointer, not an auto-run (an interactive install must never spawn a
|
|
441
|
+
// 60s test suite unasked); the command itself does the honest work when they choose to run it.
|
|
442
|
+
console.log(
|
|
443
|
+
` ${D}Want to check the session you just had? Run ${X}${B}skalpel xray${X}${D} — I'll re-run your` +
|
|
444
|
+
` agent's own proof and tell you if "done" really held up.${X}\n`,
|
|
445
|
+
);
|
|
435
446
|
await promptLaunch(detectLaunchers());
|
|
436
447
|
}
|
|
437
448
|
|
|
@@ -859,6 +870,14 @@ async function main() {
|
|
|
859
870
|
spawnSync("node", [join(__dir, "autopsy.mjs"), ...argv.slice(1)], { stdio: "inherit" });
|
|
860
871
|
return;
|
|
861
872
|
}
|
|
873
|
+
// `skalpel catches` — the RETROACTIVE first-run "wow": a LOCAL, READ-ONLY, ZERO-NETWORK scan of the
|
|
874
|
+
// user's OWN recent Claude Code + Codex transcripts for real past turns where the agent claimed "done"
|
|
875
|
+
// while its OWN test/build had already failed in-transcript. Never re-runs anything (the pass/fail is
|
|
876
|
+
// the exit status already recorded), so it is instant + safe; prints an honest summary (0 if none).
|
|
877
|
+
if (sub === "catches") {
|
|
878
|
+
spawnSync("node", [join(__dir, "catches.mjs"), ...argv.slice(1)], { stdio: "inherit" });
|
|
879
|
+
return;
|
|
880
|
+
}
|
|
862
881
|
// `skalpel __verify-report` — the claim-verification SHADOW instrument: reads the local
|
|
863
882
|
// verify-shadow.log and prints the three numbers (claim fire-rate, proof-reconstruction rate, and the
|
|
864
883
|
// headline MISMATCH rate = how often the agent claimed done but its OWN proof, re-run, failed).
|
|
@@ -867,6 +886,15 @@ async function main() {
|
|
|
867
886
|
spawnSync("node", [join(__dir, "verify-shadow.mjs"), "--report"], { stdio: "inherit" });
|
|
868
887
|
return;
|
|
869
888
|
}
|
|
889
|
+
// `skalpel xray` — the FIRST-RUN X-RAY: independently re-run the session you JUST had. Finds your most
|
|
890
|
+
// recent Claude Code transcript, reconstructs your agent's OWN last proof (test/build/typecheck/lint),
|
|
891
|
+
// re-runs it once, and stamps it verified or caught. Local, read-only against ~/.claude/projects; the
|
|
892
|
+
// only thing it executes is your session's own allowlisted proof. Reports a real verdict from a real
|
|
893
|
+
// re-run — never a minutes-lost tally, never a fabricated number.
|
|
894
|
+
if (sub === "xray") {
|
|
895
|
+
spawnSync("node", [join(__dir, "verify-shadow.mjs"), "--xray"], { stdio: "inherit" });
|
|
896
|
+
return;
|
|
897
|
+
}
|
|
870
898
|
// Persist --api/--user to ~/.skalpel/client.json so the hooks can read them without shell-env
|
|
871
899
|
// plumbing (env vars SKALPEL_API / SKALPEL_USER still win at hook runtime).
|
|
872
900
|
if (apiFlag || userFlag) {
|
package/skalpel-statusline.mjs
CHANGED
|
@@ -11,9 +11,10 @@ import { readFileSync, rmSync } from "node:fs";
|
|
|
11
11
|
import { homedir } from "node:os";
|
|
12
12
|
import { join } from "node:path";
|
|
13
13
|
import { tailLines } from "./transcript.mjs";
|
|
14
|
-
import { cleanText, REVEAL_PATH, revealEnabled } from "./insights.mjs";
|
|
14
|
+
import { cleanText, REVEAL_PATH, VERDICT_PATH, revealEnabled } from "./insights.mjs";
|
|
15
15
|
|
|
16
16
|
const CYAN = "\x1b[36m";
|
|
17
|
+
const GREEN = "\x1b[32m";
|
|
17
18
|
const BOLD = "\x1b[1m";
|
|
18
19
|
const DIM = "\x1b[2m";
|
|
19
20
|
const RESET = "\x1b[0m";
|
|
@@ -22,6 +23,11 @@ const RESET = "\x1b[0m";
|
|
|
22
23
|
// wallpapers). verify-shadow.mjs writes the reveal the moment a GENUINE_FAIL mismatch is re-run; it also
|
|
23
24
|
// retracts it if a later proof re-run PASSES. This TTL covers the abandoned case (user moved on).
|
|
24
25
|
const REVEAL_TTL_MS = 90_000;
|
|
26
|
+
// LIVE VERDICT green stamp — the earned "✓ verified" written on a clean PASS. DELIBERATELY short-lived:
|
|
27
|
+
// it flashes right after the claim, then recedes so a stream of greens stays ambient (quiet trust) and
|
|
28
|
+
// never competes with the loud red catch. Short TTL is the whole point — a green that lingers is clutter;
|
|
29
|
+
// a green that blinks by, turn after turn, is what makes the rare red land like a thunderclap.
|
|
30
|
+
const VERDICT_TTL_MS = 20_000;
|
|
25
31
|
|
|
26
32
|
// Claude Code pipes a JSON status payload on stdin (transcript_path, session_id, …). Read it ONLY when
|
|
27
33
|
// stdin is actually piped — never block on a TTY (manual invocation) where reading fd 0 would hang.
|
|
@@ -116,6 +122,30 @@ function revealNote() {
|
|
|
116
122
|
}
|
|
117
123
|
}
|
|
118
124
|
|
|
125
|
+
// The LIVE VERDICT green stamp: read the verdict state verify-shadow.mjs wrote on a clean PASS. Returns
|
|
126
|
+
// the plain "✓ verified" line (fresh + honest — the statusline never invents it, it only renders what the
|
|
127
|
+
// re-run recorded) or null. Auto-clears past its short TTL so a stale green never lingers. GATED entirely
|
|
128
|
+
// on SKALPEL_VERIFY_REVEAL: when off, this is never called and the bar is byte-identical to the original.
|
|
129
|
+
function verdictNote() {
|
|
130
|
+
try {
|
|
131
|
+
const r = JSON.parse(readFileSync(VERDICT_PATH, "utf8"));
|
|
132
|
+
if (!r || typeof r.line !== "string" || !r.line.trim()) return null;
|
|
133
|
+
const age = Date.now() - Date.parse(r.ts);
|
|
134
|
+
if (!Number.isFinite(age) || age > VERDICT_TTL_MS) {
|
|
135
|
+
try {
|
|
136
|
+
rmSync(VERDICT_PATH, { force: true }); // stale → retract so it never lingers
|
|
137
|
+
} catch {
|
|
138
|
+
/* best-effort; a leftover verdict still self-expires by TTL on the next read */
|
|
139
|
+
}
|
|
140
|
+
return null;
|
|
141
|
+
}
|
|
142
|
+
const clean = cleanText(r.line).trim(); // last-line defense: this goes RAW to the live terminal
|
|
143
|
+
return clean || null;
|
|
144
|
+
} catch {
|
|
145
|
+
return null; // no verdict pending, or unreadable → print nothing (fail-open)
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
119
149
|
async function main() {
|
|
120
150
|
const payload = readPayload();
|
|
121
151
|
|
|
@@ -128,6 +158,14 @@ async function main() {
|
|
|
128
158
|
process.stdout.write(`${BOLD}${reveal}${RESET}`);
|
|
129
159
|
return;
|
|
130
160
|
}
|
|
161
|
+
// No red catch pending → the quiet green half. A fresh PASS stamp shows dim (ambient, not shouting)
|
|
162
|
+
// for a short beat. Lower priority than the red catch, higher than the re-ask count: it's the most
|
|
163
|
+
// recent verdict on the last claim, and it's what earns the trust the red catch spends.
|
|
164
|
+
const verdict = verdictNote();
|
|
165
|
+
if (verdict) {
|
|
166
|
+
process.stdout.write(`${DIM}${GREEN}${verdict}${RESET}`);
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
131
169
|
}
|
|
132
170
|
|
|
133
171
|
// Acute + reproducible: the user has re-asked / interrupted repeatedly this stretch. Surface the count
|
package/verify-shadow.mjs
CHANGED
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
import {
|
|
28
28
|
appendFileSync,
|
|
29
29
|
mkdirSync,
|
|
30
|
+
readdirSync,
|
|
30
31
|
readFileSync,
|
|
31
32
|
writeFileSync,
|
|
32
33
|
renameSync,
|
|
@@ -42,7 +43,7 @@ import { realpathSync } from "node:fs";
|
|
|
42
43
|
import { tailLines } from "./transcript.mjs";
|
|
43
44
|
// Local sibling modules only (no sockets — RED LINE #4 holds): recordInsight logs the reveal fire for
|
|
44
45
|
// measurement; REVEAL_PATH/revealEnabled are the single-sourced state-file path + opt-in flag.
|
|
45
|
-
import { recordInsight, REVEAL_PATH, revealEnabled } from "./insights.mjs";
|
|
46
|
+
import { recordInsight, REVEAL_PATH, VERDICT_PATH, revealEnabled } from "./insights.mjs";
|
|
46
47
|
|
|
47
48
|
const DIR = join(homedir(), ".skalpel");
|
|
48
49
|
export const VERIFY_LOG_PATH = join(DIR, "verify-shadow.log");
|
|
@@ -424,11 +425,13 @@ function deniedToolUseIds(entries) {
|
|
|
424
425
|
return denied;
|
|
425
426
|
}
|
|
426
427
|
|
|
427
|
-
//
|
|
428
|
-
// The MOST RECENT test/build/typecheck/lint command the session actually ran
|
|
428
|
+
// reconstructProofEntry(entries, cwd) -> { proof:{cmd,args,cwd}, toolUseId, command } | null
|
|
429
|
+
// The MOST RECENT test/build/typecheck/lint command the session actually ran, WITH the id of the
|
|
430
|
+
// tool_use it came from (so a caller can match it back to its recorded tool_result — the retroactive
|
|
431
|
+
// `skalpel catches` scanner reads that stored result INSTEAD of re-running). Scans newest→oldest; the
|
|
429
432
|
// entry's own recorded cwd is the base (a `cd` inside the command still overrides it). SHARP-3: a
|
|
430
433
|
// command the user DENIED (never ran) is skipped — we only re-run what the session actually executed.
|
|
431
|
-
export function
|
|
434
|
+
export function reconstructProofEntry(entries, cwd) {
|
|
432
435
|
const list = Array.isArray(entries) ? entries : [];
|
|
433
436
|
const denied = deniedToolUseIds(list);
|
|
434
437
|
for (let i = list.length - 1; i >= 0; i--) {
|
|
@@ -441,30 +444,41 @@ export function reconstructProof(entries, cwd) {
|
|
|
441
444
|
const id = cmds[k].id;
|
|
442
445
|
if (id ? denied.has(id) : denied.size > 0) continue;
|
|
443
446
|
const proof = parseCommandForProof(cmds[k].command, cmds[k].cwd || cwd);
|
|
444
|
-
if (proof) return proof;
|
|
447
|
+
if (proof) return { proof, toolUseId: id || null, command: cmds[k].command };
|
|
445
448
|
}
|
|
446
449
|
}
|
|
447
450
|
return null;
|
|
448
451
|
}
|
|
449
452
|
|
|
453
|
+
// reconstructProof(entries, cwd) -> { cmd, args, cwd } | null — thin wrapper over reconstructProofEntry
|
|
454
|
+
// (the live shadow re-runner only needs the proof itself, not the originating tool_use id).
|
|
455
|
+
export function reconstructProof(entries, cwd) {
|
|
456
|
+
const found = reconstructProofEntry(entries, cwd);
|
|
457
|
+
return found ? found.proof : null;
|
|
458
|
+
}
|
|
459
|
+
|
|
450
460
|
// Classify a re-run into PASS / GENUINE_FAIL / HARNESS_ERROR. This is what keeps the money metric HONEST:
|
|
451
461
|
// only a GENUINE_FAIL (the test/build itself ran and reported failure) may count as "the agent lied".
|
|
452
462
|
// A HARNESS_ERROR (timeout, missing binary/script, wrong cwd — the PROOF machinery broke, not the code)
|
|
453
463
|
// must NEVER be shown as a lie. Real-data measurement showed 6/6 raw "mismatches" were harness noise
|
|
454
464
|
// (a 3-min lint timeout, a missing lint script, a wrong-cwd eslint) — counting those would cry wolf and
|
|
455
465
|
// burn the whole point of an honesty product.
|
|
456
|
-
function classifyOutcome(err, evidence) {
|
|
466
|
+
export function classifyOutcome(err, evidence) {
|
|
457
467
|
if (!err) return "PASS";
|
|
458
468
|
if (err.killed || err.signal === "SIGTERM") return "HARNESS_ERROR"; // hit the timeout
|
|
459
469
|
if (err.code === "ENOENT") return "HARNESS_ERROR"; // binary/cwd does not exist
|
|
460
470
|
const exit = typeof err.code === "number" ? err.code : null;
|
|
461
471
|
if (exit === 127 || exit === 126) return "HARNESS_ERROR"; // command not found / not executable
|
|
472
|
+
// The proof harness/environment is broken, not the work: a missing script, a `command not found`, or a
|
|
473
|
+
// tool binary/runtime that simply isn't installed (a Playwright browser not downloaded — "Executable
|
|
474
|
+
// doesn't exist" / "download new browsers"). Surfacing any of these would cry wolf on setup state, so
|
|
475
|
+
// they are HARNESS_ERROR, never counted as a lie.
|
|
462
476
|
if (
|
|
463
|
-
/\bno such file or directory\b|\bENOENT\b|\bcommand not found\b|missing script|no [a-z:-]{1,24} script|npm err!.*missing script/i.test(
|
|
477
|
+
/\bno such file or directory\b|\bENOENT\b|\bcommand not found\b|missing script|no [a-z:-]{1,24} script|npm err!.*missing script|executable doesn'?t exist|download new browsers/i.test(
|
|
464
478
|
String(evidence || ""),
|
|
465
479
|
)
|
|
466
480
|
)
|
|
467
|
-
return "HARNESS_ERROR";
|
|
481
|
+
return "HARNESS_ERROR";
|
|
468
482
|
return "GENUINE_FAIL"; // a real non-zero exit from the test/build/lint itself
|
|
469
483
|
}
|
|
470
484
|
|
|
@@ -710,12 +724,52 @@ function clearReveal() {
|
|
|
710
724
|
}
|
|
711
725
|
}
|
|
712
726
|
|
|
727
|
+
// ---- the POSITIVE half: the quiet green VERDICT STAMP (LIVE VERDICT). On a PASS (claim + the session's
|
|
728
|
+
// OWN proof re-run really exited 0) we write a dim-green "✓ verified" stamp the statusline renders for a
|
|
729
|
+
// short beat. HONESTY: every token traces to the real run — the real reconstructed proof command and the
|
|
730
|
+
// literal fact that its process exited 0. NO number is shown here (a pass has no count to invent). The
|
|
731
|
+
// ambient stream of these earned greens is what makes the rare red catch feel like a verdict, not noise.
|
|
732
|
+
|
|
733
|
+
// buildVerdictLine(row) → the plain-text green stamp (no ANSI — the statusline styles + re-sanitizes).
|
|
734
|
+
export function buildVerdictLine(row) {
|
|
735
|
+
const proof = clipText(row.proof_command, 48) || "its own proof";
|
|
736
|
+
return `✓ skalpel verified — re-ran \`${proof}\`, exit 0. It's real.`;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
// writeVerdict(row) — atomic write of the green-stamp state file the statusline reads. Best-effort.
|
|
740
|
+
function writeVerdict(row) {
|
|
741
|
+
try {
|
|
742
|
+
mkdirSync(DIR, { recursive: true });
|
|
743
|
+
const rec = {
|
|
744
|
+
ts: row.ts,
|
|
745
|
+
session: row.session || null,
|
|
746
|
+
proof_command: row.proof_command,
|
|
747
|
+
line: buildVerdictLine(row),
|
|
748
|
+
};
|
|
749
|
+
const tmp = `${VERDICT_PATH}.tmp`;
|
|
750
|
+
writeFileSync(tmp, JSON.stringify(rec));
|
|
751
|
+
renameSync(tmp, VERDICT_PATH); // atomic swap — a concurrent statusline never sees a half write
|
|
752
|
+
} catch {
|
|
753
|
+
/* the verdict stamp is a bonus; a write failure must never affect the shadow */
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
// clearVerdict() — a fresh RED catch supersedes any lingering green (never show both at once).
|
|
758
|
+
function clearVerdict() {
|
|
759
|
+
try {
|
|
760
|
+
rmSync(VERDICT_PATH, { force: true });
|
|
761
|
+
} catch {
|
|
762
|
+
/* stale verdict self-expires via the statusline's short TTL anyway */
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
|
|
713
766
|
// maybeSurfaceReveal(row) — the single opt-in branch. OFF by default → does nothing (fully dark).
|
|
714
767
|
function maybeSurfaceReveal(row) {
|
|
715
768
|
try {
|
|
716
769
|
if (!revealEnabled(process.env)) return; // SKALPEL_VERIFY_REVEAL unset/off → dark, no-op
|
|
717
770
|
if (row.outcome === "GENUINE_FAIL" && row.mismatch === true) {
|
|
718
771
|
writeReveal(row);
|
|
772
|
+
clearVerdict(); // a real catch overrides any stale green — never show a ✓ next to a ✗
|
|
719
773
|
// MEASURABLE: one reveal_shown insight per fire → fire-rate + (later) retention, no fabricated data.
|
|
720
774
|
recordInsight({
|
|
721
775
|
kind: "reveal_shown",
|
|
@@ -725,7 +779,15 @@ function maybeSurfaceReveal(row) {
|
|
|
725
779
|
});
|
|
726
780
|
} else if (row.outcome === "PASS") {
|
|
727
781
|
clearReveal(); // resolved — don't wallpaper a catch the agent already fixed
|
|
782
|
+
writeVerdict(row); // …and stamp the earned green so the ambient trust-line keeps building
|
|
783
|
+
recordInsight({
|
|
784
|
+
kind: "verdict_shown",
|
|
785
|
+
display: buildVerdictLine(row),
|
|
786
|
+
proof: row.proof_command,
|
|
787
|
+
session: row.session || null,
|
|
788
|
+
});
|
|
728
789
|
}
|
|
790
|
+
// HARNESS_ERROR / REFUSED: neither a catch nor a clean pass — surface nothing, touch no state file.
|
|
729
791
|
} catch {
|
|
730
792
|
/* reveal surfacing is a bonus; never affect the shadow path */
|
|
731
793
|
}
|
|
@@ -883,8 +945,130 @@ export function renderVerifyReport() {
|
|
|
883
945
|
return L.join("\n");
|
|
884
946
|
}
|
|
885
947
|
|
|
948
|
+
// ======================= FIRST-RUN X-RAY: `skalpel xray` =======================
|
|
949
|
+
// A one-shot, user-INVOKED replay over the session you JUST had: find your most recent Claude Code
|
|
950
|
+
// transcript, read the agent's last completion claim, reconstruct the session's OWN proof, re-run it
|
|
951
|
+
// once, and stamp it verified or caught. Same engine as the live shadow — the only new thing is finding
|
|
952
|
+
// the newest transcript and printing a human verdict. Explicit invocation IS the opt-in; it re-runs only
|
|
953
|
+
// the session's own allowlisted proof (never anything from prompt/claim CONTENT). It reports ONLY a real
|
|
954
|
+
// verdict from a real re-run — NEVER a minutes-lost tally, never a fabricated number (the banned
|
|
955
|
+
// confession). Local, read-only against ~/.claude/projects; opens no socket. Never throws.
|
|
956
|
+
const CLAUDE_PROJECTS_DIR = join(homedir(), ".claude", "projects");
|
|
957
|
+
|
|
958
|
+
// newestTranscript() → absolute path of the most-recently-modified *.jsonl under ~/.claude/projects, or
|
|
959
|
+
// null. Bounded: one level of project dirs (Claude's layout), newest mtime wins. Fail-open on any error.
|
|
960
|
+
export function newestTranscript(root = CLAUDE_PROJECTS_DIR) {
|
|
961
|
+
let best = null;
|
|
962
|
+
let bestMtime = -Infinity;
|
|
963
|
+
let dirs;
|
|
964
|
+
try {
|
|
965
|
+
dirs = readdirSync(root, { withFileTypes: true });
|
|
966
|
+
} catch {
|
|
967
|
+
return null; // no ~/.claude/projects → nothing to x-ray
|
|
968
|
+
}
|
|
969
|
+
for (const d of dirs) {
|
|
970
|
+
if (!d.isDirectory()) continue;
|
|
971
|
+
const sub = join(root, d.name);
|
|
972
|
+
let files;
|
|
973
|
+
try {
|
|
974
|
+
files = readdirSync(sub);
|
|
975
|
+
} catch {
|
|
976
|
+
continue;
|
|
977
|
+
}
|
|
978
|
+
for (const f of files) {
|
|
979
|
+
if (!f.endsWith(".jsonl")) continue;
|
|
980
|
+
const p = join(sub, f);
|
|
981
|
+
try {
|
|
982
|
+
const m = statSync(p).mtimeMs;
|
|
983
|
+
if (m > bestMtime) {
|
|
984
|
+
bestMtime = m;
|
|
985
|
+
best = p;
|
|
986
|
+
}
|
|
987
|
+
} catch {
|
|
988
|
+
/* vanished mid-scan — skip */
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
return best;
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
// xrayLastSession() → runs the replay and PRINTS the verdict. Returns a small result object (also used by
|
|
996
|
+
// tests). Ansi via the same palette the statusline uses; honest by construction — no derived numbers.
|
|
997
|
+
export async function xrayLastSession({ transcriptPath } = {}) {
|
|
998
|
+
const RED = "\x1b[38;2;233;38;31m";
|
|
999
|
+
const GREEN = "\x1b[38;2;124;184;108m";
|
|
1000
|
+
const DIM = "\x1b[2m";
|
|
1001
|
+
const BOLD = "\x1b[1m";
|
|
1002
|
+
const X = "\x1b[0m";
|
|
1003
|
+
const say = (s) => process.stdout.write(s + "\n");
|
|
1004
|
+
|
|
1005
|
+
const tp = transcriptPath || newestTranscript();
|
|
1006
|
+
if (!tp) {
|
|
1007
|
+
say(
|
|
1008
|
+
`\n ${DIM}No recent Claude Code session found to check (looked in ~/.claude/projects).${X}\n`,
|
|
1009
|
+
);
|
|
1010
|
+
return { status: "no_session" };
|
|
1011
|
+
}
|
|
1012
|
+
const entries = readEntries(tp);
|
|
1013
|
+
if (!entries.length) {
|
|
1014
|
+
say(`\n ${DIM}Your most recent session had nothing readable to check.${X}\n`);
|
|
1015
|
+
return { status: "empty" };
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
const { claim, text } = detectCompletionClaim(lastAssistantText(entries));
|
|
1019
|
+
say(`\n ${BOLD}skalpel x-ray${X} ${DIM}· replaying the session you just had${X}`);
|
|
1020
|
+
if (!claim) {
|
|
1021
|
+
// No completion claim in the last turn — nothing was asserted done, so there's nothing to catch.
|
|
1022
|
+
say(
|
|
1023
|
+
`\n ${DIM}Your agent's last turn didn't claim it was done — nothing to independently verify.${X}\n`,
|
|
1024
|
+
);
|
|
1025
|
+
return { status: "no_claim" };
|
|
1026
|
+
}
|
|
1027
|
+
say(` ${DIM}claim:${X} "${clipText(text, 72)}"`);
|
|
1028
|
+
|
|
1029
|
+
const proof = reconstructProof(entries, null);
|
|
1030
|
+
if (!proof) {
|
|
1031
|
+
// Honest limit: it claimed done but ran no test/build/typecheck we could re-run out-of-band.
|
|
1032
|
+
say(
|
|
1033
|
+
`\n ${DIM}It said done, but ran no test/build/typecheck this session — nothing skalpel can` +
|
|
1034
|
+
` independently re-run. No verdict (and no guess).${X}\n`,
|
|
1035
|
+
);
|
|
1036
|
+
return { status: "no_proof" };
|
|
1037
|
+
}
|
|
1038
|
+
const proofStr = [proof.cmd, ...proof.args].join(" ").slice(0, 120);
|
|
1039
|
+
say(` ${DIM}re-running its own proof:${X} ${proofStr}${DIM} …${X}`);
|
|
1040
|
+
|
|
1041
|
+
const result = await rerunProof(proof);
|
|
1042
|
+
const outcome = result.outcome || (result.pass ? "PASS" : "GENUINE_FAIL");
|
|
1043
|
+
if (outcome === "PASS") {
|
|
1044
|
+
say(
|
|
1045
|
+
`\n ${GREEN}✓ verified${X} — re-ran \`${proofStr}\`, exit 0. Your agent's "done" holds up.\n`,
|
|
1046
|
+
);
|
|
1047
|
+
return { status: "pass", proof: proofStr };
|
|
1048
|
+
}
|
|
1049
|
+
if (outcome === "GENUINE_FAIL") {
|
|
1050
|
+
const failN = extractFailCount(result.evidence);
|
|
1051
|
+
const count = failN != null ? ` (${failN} failing)` : "";
|
|
1052
|
+
say(
|
|
1053
|
+
`\n ${RED}${BOLD}✗ caught${X} — your agent said "${clipText(text, 60)}" but \`${proofStr}\`` +
|
|
1054
|
+
` just failed${count}. It's not done.`,
|
|
1055
|
+
);
|
|
1056
|
+
if (result.evidence) say(` ${DIM}real output:${X} ${clipText(result.evidence, 180)}`);
|
|
1057
|
+
say("");
|
|
1058
|
+
return { status: "caught", proof: proofStr, evidence: result.evidence };
|
|
1059
|
+
}
|
|
1060
|
+
// HARNESS_ERROR / REFUSED — the proof machinery broke, NOT the agent's work. Never render this as a
|
|
1061
|
+
// catch (crying wolf on a timeout / missing script / wrong cwd is exactly what kills the trust).
|
|
1062
|
+
say(
|
|
1063
|
+
`\n ${DIM}Couldn't independently verify — the proof \`${proofStr}\` errored` +
|
|
1064
|
+
`${result.evidence ? ` (${clipText(result.evidence, 120)})` : ""}. Not a catch.${X}\n`,
|
|
1065
|
+
);
|
|
1066
|
+
return { status: "harness_error", proof: proofStr };
|
|
1067
|
+
}
|
|
1068
|
+
|
|
886
1069
|
// CLI: `node verify-shadow.mjs --run` → detached worker launched by the per-turn hook.
|
|
887
1070
|
// `node verify-shadow.mjs --report`→ print the instrument (also reachable via `skalpel __verify-report`).
|
|
1071
|
+
// `node verify-shadow.mjs --xray` → the one-shot FIRST-RUN X-RAY (reachable via `skalpel xray`).
|
|
888
1072
|
// Robust main detection (mirrors skalpel-setup.mjs): compare REAL paths, not a hand-built file:// URL —
|
|
889
1073
|
// the staged path lives under the home dir, which on macOS/Windows can contain a space that would break
|
|
890
1074
|
// `new URL("file://"+path)` and wrongly disable the CLI. Importing this module (tests) never trips it.
|
|
@@ -903,6 +1087,10 @@ if (isMain) {
|
|
|
903
1087
|
if (arg === "--report") {
|
|
904
1088
|
process.stdout.write(renderVerifyReport() + "\n");
|
|
905
1089
|
process.exit(0);
|
|
1090
|
+
} else if (arg === "--xray") {
|
|
1091
|
+
xrayLastSession({ transcriptPath: process.env.SKALPEL_VERIFY_TRANSCRIPT || null })
|
|
1092
|
+
.then(() => process.exit(0))
|
|
1093
|
+
.catch(() => process.exit(0));
|
|
906
1094
|
} else if (arg === "--run") {
|
|
907
1095
|
recordVerifyShadow({
|
|
908
1096
|
transcriptPath: process.env.SKALPEL_VERIFY_TRANSCRIPT || null,
|