skalpel 4.0.43 → 4.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/card.mjs +64 -12
- package/first-run.mjs +722 -0
- package/metrics.mjs +35 -1
- package/package.json +1 -1
- package/skalpel-hook.mjs +165 -28
- package/skalpel-setup.mjs +8 -0
- package/verify-races.test.mjs +182 -0
- package/verify-shadow.mjs +60 -10
package/first-run.mjs
ADDED
|
@@ -0,0 +1,722 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// first-run.mjs — THE FIRST CATCH: first-run magic for skalpel.
|
|
3
|
+
//
|
|
4
|
+
// THE MOMENT: the instant a user installs + consents (the #631 consent-arm flow), we scan their OWN
|
|
5
|
+
// coding history and reveal ONE real catch from it — a past turn where their agent said "done" while its
|
|
6
|
+
// own recorded test/build had ALREADY failed at claim time. Spotify-Wrapped-for-forensics: it collapses
|
|
7
|
+
// time-to-first-aha from "keep coding until the live shadow catches something" to seconds, using history
|
|
8
|
+
// the user already has on disk.
|
|
9
|
+
//
|
|
10
|
+
// HARD RED LINES (why this is safe + honest):
|
|
11
|
+
// • READ-ONLY over transcripts already on disk. It NEVER re-runs, spawns a proof, builds, or mutates.
|
|
12
|
+
// The pass/fail is the exit status the transcript ALREADY recorded — exactly like `skalpel catches`.
|
|
13
|
+
// • FAIL-OPEN. Any scan/render/IO error → skip SILENTLY to the normal setup ending. Never throws.
|
|
14
|
+
// • BYTE-IDENTICAL-OFF. Non-TTY / CI / declined-consent → returns before ANY transcript read or file
|
|
15
|
+
// write. The per-turn hook path is not touched at all.
|
|
16
|
+
// • PRECISION OVER RECALL. A false catch at the moment of maximum scrutiny is the worst failure this
|
|
17
|
+
// feature can have (magic → betrayal), so the first-run gate is STRICTER than `skalpel catches`:
|
|
18
|
+
// see passesFirstRunGate. If nothing qualifies we show an HONEST ZERO STATE — we never manufacture.
|
|
19
|
+
// • NO FABRICATED VALUES. Every rendered value traces to a real transcript field (verbatim claim, the
|
|
20
|
+
// recorded output tail, the recorded non-zero exit code, real timestamps). No invented minutes.
|
|
21
|
+
//
|
|
22
|
+
// REUSE: the detection + classification ENGINE is verify-shadow's / catches.mjs's, imported unedited —
|
|
23
|
+
// loadClaudeEntries / loadCodexEntries (catches.mjs) normalize both transcript shapes; detectCompletionClaim,
|
|
24
|
+
// reconstructProofEntry, classifyOutcome, extractFailCount (verify-shadow.mjs) do claim + proof + verdict.
|
|
25
|
+
// We run a FIRST-RUN-specific loop over those primitives (rather than calling runScan) because the first-run
|
|
26
|
+
// profile (newest ~150 sessions, HARD ~3s budget, a live progress line) and the strict precision gate need
|
|
27
|
+
// per-catch metadata that runScan/scanSession deliberately drop: the ANCHORED first-line exit code (their
|
|
28
|
+
// evidence is a clipped tail — the leading "Exit code N" line is gone), the claim↔proof turn-distance, and
|
|
29
|
+
// the real per-entry timestamps. catches.mjs is imported, never edited (PR #634 owns its classifier).
|
|
30
|
+
import {
|
|
31
|
+
appendFileSync,
|
|
32
|
+
mkdirSync,
|
|
33
|
+
readdirSync,
|
|
34
|
+
readFileSync,
|
|
35
|
+
statSync,
|
|
36
|
+
writeFileSync,
|
|
37
|
+
} from "node:fs";
|
|
38
|
+
import { homedir } from "node:os";
|
|
39
|
+
import { spawnSync } from "node:child_process";
|
|
40
|
+
import path from "node:path";
|
|
41
|
+
import { fileURLToPath } from "node:url";
|
|
42
|
+
import { realpathSync } from "node:fs";
|
|
43
|
+
import { loadClaudeEntries, loadCodexEntries } from "./catches.mjs";
|
|
44
|
+
import {
|
|
45
|
+
detectCompletionClaim,
|
|
46
|
+
reconstructProofEntry,
|
|
47
|
+
classifyOutcome,
|
|
48
|
+
extractFailCount,
|
|
49
|
+
VERIFY_LOG_PATH,
|
|
50
|
+
} from "./verify-shadow.mjs";
|
|
51
|
+
import { recordInsight, revealEnabled } from "./insights.mjs";
|
|
52
|
+
import { renderMarkdownCard } from "./card.mjs";
|
|
53
|
+
|
|
54
|
+
// ---------- first-run profile (the knobs that make this instant + safe) ----------
|
|
55
|
+
const MAX_SESSIONS = 150; // newest sessions to consider (across Claude + Codex)
|
|
56
|
+
const BUDGET_MS = 3000; // HARD wall-clock budget for the whole scan
|
|
57
|
+
const MAX_CLAIM_PROOF_GAP = 25; // strict gate (c): max transcript-entry distance claim→failing proof
|
|
58
|
+
const EVIDENCE_TAIL = 200; // verbatim recorded-output tail we surface / store
|
|
59
|
+
const WITHIN_30D_MS = 30 * 24 * 60 * 60 * 1000;
|
|
60
|
+
|
|
61
|
+
// ---------- (0) sanitize (transcript-derived text prints to a terminal — strip escape/control bytes) ----------
|
|
62
|
+
function clean(s, n) {
|
|
63
|
+
const t = String(s == null ? "" : s)
|
|
64
|
+
// eslint-disable-next-line no-control-regex
|
|
65
|
+
.replace(/\x1b\[[0-9;?]*[ -/]*[@-~]/g, "") // ANSI CSI
|
|
66
|
+
// eslint-disable-next-line no-control-regex
|
|
67
|
+
.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, " ") // other C0 controls → space
|
|
68
|
+
.replace(/\s+/g, " ")
|
|
69
|
+
.trim();
|
|
70
|
+
if (n && t.length > n) return t.slice(0, n - 1) + "…";
|
|
71
|
+
return t;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// ---------- (1) strict-gate vocab ----------
|
|
75
|
+
// Gate (a): the claim must invoke a PROOF concept (test/build/typecheck/fix/done-over-a-proof), NOT a
|
|
76
|
+
// vibe-only "lgtm / good to go / works now / all set / ready" (those match verify-shadow's CLAIM_RE but
|
|
77
|
+
// are not a claim OVER a proof, so at first impression we do not indict on them).
|
|
78
|
+
const PROOF_ADJACENT_RE =
|
|
79
|
+
/\b(tests?|passe?s|passing|passed|green|builds?|built|compiles?|compiled|compilation|fix(?:ed|es)?|suite|lint|typechecks?|done|complete[ds]?|finished|implemented|resolved|merged)\b/i;
|
|
80
|
+
|
|
81
|
+
// Gate (b): a SAME-TURN acknowledgment that something is still failing/not passing. detectCompletionClaim
|
|
82
|
+
// already rejects a negated claim LINE; this is the stricter full-message guard so a message that carries
|
|
83
|
+
// one clean positive line AND "…but the suite still fails" is never surfaced.
|
|
84
|
+
const FAILURE_ACK_RE = new RegExp(
|
|
85
|
+
[
|
|
86
|
+
"\\bstill\\s+(fail(s|ing)?|red|broken|not\\b|failing)",
|
|
87
|
+
"\\btests?\\s+(are\\s+)?(still\\s+)?(fail(s|ing)?|not\\s+pass|red|broken)\\b",
|
|
88
|
+
"\\bfail(s|ing|ed)?\\s+(still|again)\\b",
|
|
89
|
+
"\\bnot\\s+(yet\\s+)?pass(ing|es)?\\b",
|
|
90
|
+
"\\bdoes(n'?t| not)\\s+pass\\b",
|
|
91
|
+
"\\bdid(n'?t| not)\\s+pass\\b",
|
|
92
|
+
"\\bwon'?t\\s+pass\\b",
|
|
93
|
+
"\\bremaining\\s+failures?\\b",
|
|
94
|
+
"\\bthere\\s+(are|were)\\s+(still\\s+)?(\\d+\\s+)?(test\\s+)?failures?\\b",
|
|
95
|
+
"\\b(some|a\\s+few|several)\\s+(tests?\\s+)?(are\\s+)?fail",
|
|
96
|
+
"\\bknown\\s+(failure|issue|bug)\\b",
|
|
97
|
+
].join("|"),
|
|
98
|
+
"i",
|
|
99
|
+
);
|
|
100
|
+
|
|
101
|
+
export function hasProofAdjacentTerm(claimText) {
|
|
102
|
+
return PROOF_ADJACENT_RE.test(String(claimText || ""));
|
|
103
|
+
}
|
|
104
|
+
export function hasFailureAck(messageText) {
|
|
105
|
+
return FAILURE_ACK_RE.test(String(messageText || ""));
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// leadingExitCode(body) — the ANCHORED recorded exit code. Real Bash tool_results declare the genuine
|
|
109
|
+
// outcome on the LEADING line ("Exit code 1\n…"); Codex on "Process exited with code N". We read ONLY the
|
|
110
|
+
// first non-empty line so an unrelated "exit code 123" buried in a passing proof's body can NEVER be read
|
|
111
|
+
// as a failure. Returns a finite integer or null.
|
|
112
|
+
export function leadingExitCode(body) {
|
|
113
|
+
const s = String(body || "");
|
|
114
|
+
let first = "";
|
|
115
|
+
for (const line of s.split("\n")) {
|
|
116
|
+
if (line.trim()) {
|
|
117
|
+
first = line.trim();
|
|
118
|
+
break;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
const m = first.match(/^(?:Exit code|Process exited with code)\s+(-?\d+)\b/i);
|
|
122
|
+
if (!m) return null;
|
|
123
|
+
const n = Number.parseInt(m[1], 10);
|
|
124
|
+
return Number.isInteger(n) ? n : null;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// ---------- (2) index a session's recorded results (full body + anchored exit + entry indices + ts) ----------
|
|
128
|
+
function resultTextOf(block) {
|
|
129
|
+
const c = block?.content;
|
|
130
|
+
if (typeof c === "string") return c;
|
|
131
|
+
if (Array.isArray(c)) return c.map((x) => (typeof x === "string" ? x : x?.text || "")).join(" ");
|
|
132
|
+
return "";
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function indexSession(entries) {
|
|
136
|
+
const results = new Map(); // tool_use_id -> { isError, body, structuredExit }
|
|
137
|
+
const toolUseIndex = new Map(); // tool_use_id -> entry index (for turn-distance)
|
|
138
|
+
const ts = new Array(entries.length).fill(null); // entry index -> ISO timestamp | null
|
|
139
|
+
for (let i = 0; i < entries.length; i++) {
|
|
140
|
+
const e = entries[i];
|
|
141
|
+
if (typeof e?.timestamp === "string") ts[i] = e.timestamp;
|
|
142
|
+
const c = e?.message?.content;
|
|
143
|
+
if (!Array.isArray(c)) continue;
|
|
144
|
+
for (const b of c) {
|
|
145
|
+
if (b && b.type === "tool_use" && b.id && !toolUseIndex.has(b.id)) toolUseIndex.set(b.id, i);
|
|
146
|
+
if (b && b.type === "tool_result" && b.tool_use_id) {
|
|
147
|
+
results.set(b.tool_use_id, {
|
|
148
|
+
isError: b.is_error === true,
|
|
149
|
+
body: resultTextOf(b),
|
|
150
|
+
// Codex-normalized results carry a STRUCTURED numeric exit_code (parsed from Codex's own
|
|
151
|
+
// wrapper by loadCodexEntries) — a trustworthy, non-free-text signal.
|
|
152
|
+
structuredExit: typeof b.exit_code === "number" ? b.exit_code : null,
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return { results, toolUseIndex, ts };
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function assistantTextOf(entry) {
|
|
161
|
+
if (entry?.type !== "assistant" && entry?.message?.role !== "assistant") return "";
|
|
162
|
+
const c = entry?.message?.content;
|
|
163
|
+
if (typeof c === "string") return c;
|
|
164
|
+
if (Array.isArray(c))
|
|
165
|
+
return c
|
|
166
|
+
.filter((b) => b && b.type === "text")
|
|
167
|
+
.map((b) => b.text || "")
|
|
168
|
+
.join(" ")
|
|
169
|
+
.trim();
|
|
170
|
+
return "";
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// ---------- (3) scan ONE session for strict first-run catches ----------
|
|
174
|
+
// Returns [{ claim_text, proof_command, recorded_exit_code, fail_count, evidence_tail, claim_ts,
|
|
175
|
+
// proof_ts, seconds_before }]. Uses the SAME claim/proof primitives as scanSession, then
|
|
176
|
+
// applies the strict gate (a)+(b)+(c)+(d) with an INDEPENDENT anchored exit-code check.
|
|
177
|
+
export function scanEntriesForCatches(entries, meta = {}) {
|
|
178
|
+
const out = [];
|
|
179
|
+
if (!Array.isArray(entries) || !entries.length) return out;
|
|
180
|
+
const { results, toolUseIndex, ts } = indexSession(entries);
|
|
181
|
+
const seenProof = new Set();
|
|
182
|
+
|
|
183
|
+
for (let i = 0; i < entries.length; i++) {
|
|
184
|
+
const fullText = assistantTextOf(entries[i]);
|
|
185
|
+
if (!fullText) continue;
|
|
186
|
+
|
|
187
|
+
const { claim, text: claimText } = detectCompletionClaim(fullText);
|
|
188
|
+
if (!claim) continue;
|
|
189
|
+
// Gate (a): proof-adjacent claim, and (b): no same-turn failure acknowledgment.
|
|
190
|
+
if (!hasProofAdjacentTerm(claimText)) continue;
|
|
191
|
+
if (hasFailureAck(fullText)) continue;
|
|
192
|
+
|
|
193
|
+
// The most-recent proof the session actually ran up to and including this claim.
|
|
194
|
+
const found = reconstructProofEntry(entries.slice(0, i + 1), meta.cwd || null);
|
|
195
|
+
if (!found || !found.toolUseId) continue;
|
|
196
|
+
if (seenProof.has(found.toolUseId)) continue;
|
|
197
|
+
|
|
198
|
+
// Gate (c): the failing proof must be within a bounded turn-distance of the claim.
|
|
199
|
+
const proofIdx = toolUseIndex.get(found.toolUseId);
|
|
200
|
+
if (!Number.isInteger(proofIdx)) continue;
|
|
201
|
+
const gap = i - proofIdx;
|
|
202
|
+
if (gap < 0 || gap > MAX_CLAIM_PROOF_GAP) continue;
|
|
203
|
+
|
|
204
|
+
const res = results.get(found.toolUseId);
|
|
205
|
+
if (!res) continue;
|
|
206
|
+
|
|
207
|
+
// Gate (d): the recorded exit code is a LITERAL non-zero integer, read ANCHORED (never a body scan).
|
|
208
|
+
// Claude: leading "Exit code N" line + is_error corroboration. Codex: the structured wrapper code.
|
|
209
|
+
let exitCode = leadingExitCode(res.body);
|
|
210
|
+
if (exitCode == null && Number.isInteger(res.structuredExit)) exitCode = res.structuredExit;
|
|
211
|
+
if (!Number.isInteger(exitCode) || exitCode === 0) continue;
|
|
212
|
+
// Claude corroboration: a real command failure sets is_error. If a numeric exit came only from the
|
|
213
|
+
// body (no is_error and no structured code), refuse it — precision over recall.
|
|
214
|
+
if (!res.isError && !Number.isInteger(res.structuredExit)) continue;
|
|
215
|
+
|
|
216
|
+
// Reuse the verdict classifier so timeouts / 127 / 126 / missing-script noise are NEVER surfaced.
|
|
217
|
+
if (classifyOutcome({ code: exitCode }, res.body) !== "GENUINE_FAIL") continue;
|
|
218
|
+
|
|
219
|
+
seenProof.add(found.toolUseId);
|
|
220
|
+
const claimTs = ts[i] || null;
|
|
221
|
+
const proofTs = ts[proofIdx] || null;
|
|
222
|
+
let secondsBefore = null;
|
|
223
|
+
if (claimTs && proofTs) {
|
|
224
|
+
const d = (Date.parse(claimTs) - Date.parse(proofTs)) / 1000;
|
|
225
|
+
if (Number.isFinite(d) && d >= 0) secondsBefore = Math.round(d);
|
|
226
|
+
}
|
|
227
|
+
const evidenceTail = String(res.body || "")
|
|
228
|
+
.replace(/\s+$/, "")
|
|
229
|
+
.slice(-EVIDENCE_TAIL);
|
|
230
|
+
out.push({
|
|
231
|
+
tool: meta.tool || "claude",
|
|
232
|
+
session: meta.session || null,
|
|
233
|
+
claim_text: claimText,
|
|
234
|
+
proof_command: [found.proof.cmd, ...found.proof.args].join(" ").slice(0, 160),
|
|
235
|
+
recorded_exit_code: exitCode,
|
|
236
|
+
fail_count: extractFailCount(evidenceTail),
|
|
237
|
+
evidence_tail: evidenceTail,
|
|
238
|
+
claim_ts: claimTs,
|
|
239
|
+
proof_ts: proofTs,
|
|
240
|
+
seconds_before: secondsBefore,
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
return out;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// ---------- (4) discover newest sessions (read-only) ----------
|
|
247
|
+
function walkJsonl(dir, pred, out) {
|
|
248
|
+
let entries;
|
|
249
|
+
try {
|
|
250
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
251
|
+
} catch {
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
for (const e of entries) {
|
|
255
|
+
const p = path.join(dir, e.name);
|
|
256
|
+
try {
|
|
257
|
+
if (e.isDirectory()) walkJsonl(p, pred, out);
|
|
258
|
+
else if (e.isFile() && pred(e.name)) out.push(p);
|
|
259
|
+
} catch {
|
|
260
|
+
/* one bad entry — skip */
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
function discoverSessions(home, maxSessions) {
|
|
266
|
+
const claudeRoot = path.join(home, ".claude", "projects");
|
|
267
|
+
const codexDirs = [
|
|
268
|
+
path.join(home, ".codex", "sessions"),
|
|
269
|
+
path.join(home, ".codex", "archived_sessions"),
|
|
270
|
+
];
|
|
271
|
+
const files = [];
|
|
272
|
+
const claude = [];
|
|
273
|
+
walkJsonl(claudeRoot, (n) => n.endsWith(".jsonl"), claude);
|
|
274
|
+
for (const f of claude) files.push({ file: f, tool: "claude" });
|
|
275
|
+
const codex = [];
|
|
276
|
+
for (const d of codexDirs) walkJsonl(d, (n) => /^rollout-.*\.jsonl$/.test(n), codex);
|
|
277
|
+
for (const f of codex) files.push({ file: f, tool: "codex" });
|
|
278
|
+
const withMtime = [];
|
|
279
|
+
for (const f of files) {
|
|
280
|
+
try {
|
|
281
|
+
withMtime.push({ ...f, mtime: statSync(f.file).mtimeMs });
|
|
282
|
+
} catch {
|
|
283
|
+
/* vanished between readdir + stat — skip */
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
withMtime.sort((a, b) => b.mtime - a.mtime);
|
|
287
|
+
return withMtime.slice(0, maxSessions);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// ---------- (5) the first-run scan (budgeted, progress-reporting) ----------
|
|
291
|
+
// Returns { scanned, candidates, best, moreWithin30d }. Fail-open: any error → whatever we had so far.
|
|
292
|
+
export function firstRunScan({
|
|
293
|
+
home = homedir(),
|
|
294
|
+
maxSessions = MAX_SESSIONS,
|
|
295
|
+
budgetMs = BUDGET_MS,
|
|
296
|
+
onProgress = null,
|
|
297
|
+
now = Date.now(),
|
|
298
|
+
} = {}) {
|
|
299
|
+
const started = Date.now();
|
|
300
|
+
let scanned = 0;
|
|
301
|
+
const candidates = [];
|
|
302
|
+
let sessions = [];
|
|
303
|
+
try {
|
|
304
|
+
sessions = discoverSessions(home, maxSessions);
|
|
305
|
+
} catch {
|
|
306
|
+
sessions = [];
|
|
307
|
+
}
|
|
308
|
+
for (const s of sessions) {
|
|
309
|
+
if (Date.now() - started > budgetMs) break; // HARD budget — stop cleanly (precision unaffected)
|
|
310
|
+
let entries;
|
|
311
|
+
try {
|
|
312
|
+
entries = s.tool === "codex" ? loadCodexEntries(s.file) : loadClaudeEntries(s.file);
|
|
313
|
+
} catch {
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
316
|
+
if (!entries.length) continue;
|
|
317
|
+
scanned++;
|
|
318
|
+
if (typeof onProgress === "function") {
|
|
319
|
+
try {
|
|
320
|
+
onProgress(scanned);
|
|
321
|
+
} catch {
|
|
322
|
+
/* progress is cosmetic */
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
const sessionId = path.basename(s.file, ".jsonl");
|
|
326
|
+
let found = [];
|
|
327
|
+
try {
|
|
328
|
+
found = scanEntriesForCatches(entries, { tool: s.tool, session: sessionId });
|
|
329
|
+
} catch {
|
|
330
|
+
continue; // one malformed session must never abort the scan
|
|
331
|
+
}
|
|
332
|
+
for (const c of found) {
|
|
333
|
+
c.session_mtime = s.mtime;
|
|
334
|
+
candidates.push(c);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
// Rank: most recent first (real claim ts, else session mtime), then by extractable fail count.
|
|
339
|
+
const timeOf = (c) => (c.claim_ts ? Date.parse(c.claim_ts) : c.session_mtime) || 0;
|
|
340
|
+
candidates.sort((a, b) => timeOf(b) - timeOf(a) || (b.fail_count || 0) - (a.fail_count || 0));
|
|
341
|
+
const best = candidates[0] || null;
|
|
342
|
+
const moreWithin30d = best
|
|
343
|
+
? candidates.filter((c) => c !== best && now - timeOf(c) <= WITHIN_30D_MS).length
|
|
344
|
+
: 0;
|
|
345
|
+
return { scanned, candidates, best, moreWithin30d };
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
// ---------- (6) the retro CARD row (append to the shared catch log the card reads) ----------
|
|
349
|
+
// A retro row is a RECORDED catch (never a re-run). It carries recorded_exit_code (required by card.mjs's
|
|
350
|
+
// isGenuineCatch for retro) and provenance markers. pass_fail:"RETRO" + mismatch:false keep it OUT of the
|
|
351
|
+
// live test-proof instrument's re-run / mismatch counters (renderVerifyReport), so first-run never inflates
|
|
352
|
+
// the live numbers; outcome:"GENUINE_FAIL" is what lets the card accept + render it.
|
|
353
|
+
export function buildRetroRow(best) {
|
|
354
|
+
return {
|
|
355
|
+
ts: best.claim_ts || new Date().toISOString(),
|
|
356
|
+
session: best.session || null,
|
|
357
|
+
tool: best.tool || "claude",
|
|
358
|
+
category: "retro",
|
|
359
|
+
claim_text: best.claim_text,
|
|
360
|
+
proof_command: best.proof_command,
|
|
361
|
+
pass_fail: "RETRO",
|
|
362
|
+
outcome: "GENUINE_FAIL",
|
|
363
|
+
mismatch: false,
|
|
364
|
+
recorded_exit_code: best.recorded_exit_code,
|
|
365
|
+
recorded: true,
|
|
366
|
+
provenance: "recorded-in-transcript-at-claim-time",
|
|
367
|
+
fail_count: best.fail_count,
|
|
368
|
+
evidence: best.evidence_tail,
|
|
369
|
+
};
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
function appendRetroRow(row) {
|
|
373
|
+
try {
|
|
374
|
+
mkdirSync(path.dirname(VERIFY_LOG_PATH), { recursive: true });
|
|
375
|
+
appendFileSync(VERIFY_LOG_PATH, JSON.stringify(row) + "\n");
|
|
376
|
+
return true;
|
|
377
|
+
} catch {
|
|
378
|
+
return false;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// ---------- (7) the staged reveal ----------
|
|
383
|
+
const R = "\x1b[38;2;233;38;31m";
|
|
384
|
+
const G = "\x1b[38;2;124;184;108m";
|
|
385
|
+
const D = "\x1b[2m";
|
|
386
|
+
const B = "\x1b[1m";
|
|
387
|
+
const X = "\x1b[0m";
|
|
388
|
+
|
|
389
|
+
// Real day + time from a REAL ISO timestamp (or a real session mtime fallback). Never fabricated.
|
|
390
|
+
function whenPhrase(best) {
|
|
391
|
+
const ms = best.claim_ts ? Date.parse(best.claim_ts) : best.session_mtime;
|
|
392
|
+
if (!Number.isFinite(ms)) return null;
|
|
393
|
+
const d = new Date(ms);
|
|
394
|
+
try {
|
|
395
|
+
const day = d.toLocaleDateString("en-US", { weekday: "long", month: "short", day: "numeric" });
|
|
396
|
+
const time = d.toLocaleTimeString("en-US", { hour: "numeric", minute: "2-digit" });
|
|
397
|
+
return `${day} at ${time}`;
|
|
398
|
+
} catch {
|
|
399
|
+
return d.toISOString();
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
export function revealLines(best, moreWithin30d, color = true) {
|
|
404
|
+
const c = (code, s) => (color ? `${code}${s}${X}` : s);
|
|
405
|
+
const when = whenPhrase(best);
|
|
406
|
+
const claim = clean(best.claim_text, 160) || "it's done";
|
|
407
|
+
const evidence = clean(best.evidence_tail, 200);
|
|
408
|
+
const failN = best.fail_count;
|
|
409
|
+
const secs = best.seconds_before;
|
|
410
|
+
const L = [];
|
|
411
|
+
L.push("");
|
|
412
|
+
L.push(
|
|
413
|
+
` ${c(B, "🔬 skalpel — your first catch")} ${c(D, "· from your own history, read-only")}`,
|
|
414
|
+
);
|
|
415
|
+
L.push(` ${c(D, "─".repeat(70))}`);
|
|
416
|
+
L.push("");
|
|
417
|
+
L.push(
|
|
418
|
+
when
|
|
419
|
+
? ` On ${c(B, when)}, your agent told you:`
|
|
420
|
+
: ` Earlier in your history, your agent told you:`,
|
|
421
|
+
);
|
|
422
|
+
L.push(` ${c(B, `"${claim}"`)}`);
|
|
423
|
+
L.push("");
|
|
424
|
+
// Line 3 — every clause here is a real recorded field; anything absent is OMITTED, never invented.
|
|
425
|
+
const earlier = secs != null ? `, ${secs}s earlier,` : "";
|
|
426
|
+
const failsClause =
|
|
427
|
+
failN != null ? `recorded ${failN} failure${failN === 1 ? "" : "s"}` : "had already failed";
|
|
428
|
+
L.push(` Its own test run${earlier} ${failsClause}. Here's the line:`);
|
|
429
|
+
if (evidence) L.push(` ${c(R, evidence)}`);
|
|
430
|
+
L.push("");
|
|
431
|
+
const more = moreWithin30d > 0 ? `…and ${moreWithin30d} more in your last 30 days. ` : "";
|
|
432
|
+
L.push(` ${c(D, `${more}From now on, skalpel catches these live.`)}`);
|
|
433
|
+
L.push("");
|
|
434
|
+
return L.join("\n");
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
function zeroStateLines(scanned, color = true) {
|
|
438
|
+
const c = (code, s) => (color ? `${code}${s}${X}` : s);
|
|
439
|
+
const L = [];
|
|
440
|
+
L.push("");
|
|
441
|
+
L.push(` ${c(B, "🔬 skalpel — first-run scan")} ${c(D, "· your own history, read-only")}`);
|
|
442
|
+
L.push(` ${c(D, "─".repeat(70))}`);
|
|
443
|
+
L.push("");
|
|
444
|
+
L.push(` Scanned ${c(B, String(scanned))} of your recent sessions. Zero times your agent`);
|
|
445
|
+
L.push(` said "done" over a failing test. ${c(G, "Clean")} — I'll watch from here.`);
|
|
446
|
+
L.push("");
|
|
447
|
+
return L.join("\n");
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
// copy the pre-rendered markdown card to the OS clipboard. Best-effort; returns true on success.
|
|
451
|
+
// SKALPEL_NO_CLIPBOARD=1 disables the OS write (CI / tests must not touch the developer's clipboard).
|
|
452
|
+
function copyToClipboard(text) {
|
|
453
|
+
if (String(process.env.SKALPEL_NO_CLIPBOARD || "") === "1") return false;
|
|
454
|
+
const tries =
|
|
455
|
+
process.platform === "darwin"
|
|
456
|
+
? [["pbcopy", []]]
|
|
457
|
+
: process.platform === "win32"
|
|
458
|
+
? [["clip", []]]
|
|
459
|
+
: [
|
|
460
|
+
["wl-copy", []],
|
|
461
|
+
["xclip", ["-selection", "clipboard"]],
|
|
462
|
+
["xsel", ["--clipboard", "--input"]],
|
|
463
|
+
];
|
|
464
|
+
for (const [cmd, args] of tries) {
|
|
465
|
+
try {
|
|
466
|
+
const r = spawnSync(cmd, args, { input: text, stdio: ["pipe", "ignore", "ignore"] });
|
|
467
|
+
if (r && r.status === 0) return true;
|
|
468
|
+
} catch {
|
|
469
|
+
/* try the next clipboard tool */
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
return false;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
// Interactive exit keys: [c] copy the (pre-rendered) markdown card · [x] not right? · [enter] done.
|
|
476
|
+
// Fail-open: no raw TTY → print the hint, do not block. `answer` forces a key for tests.
|
|
477
|
+
function promptRevealExit({ out, stdin, interactive, answer, markdown, best, env }) {
|
|
478
|
+
const done = (key) => {
|
|
479
|
+
if (key === "c") {
|
|
480
|
+
const ok = copyToClipboard(markdown);
|
|
481
|
+
out.write(
|
|
482
|
+
ok
|
|
483
|
+
? ` ${G}✓${X} ${D}card copied — paste it in Slack / a PR / X.${X}\n\n`
|
|
484
|
+
: ` ${D}(clipboard unavailable — run ${X}skalpel card --md${D} to copy it.)${X}\n\n`,
|
|
485
|
+
);
|
|
486
|
+
recordInsight({
|
|
487
|
+
kind: "first_run_card_copied",
|
|
488
|
+
display: "first-run: card copied to clipboard",
|
|
489
|
+
copied: ok,
|
|
490
|
+
session: best.session || null,
|
|
491
|
+
});
|
|
492
|
+
} else if (key === "x") {
|
|
493
|
+
out.write(` ${D}Got it — flagged as disputed. I won't count this one.${X}\n\n`);
|
|
494
|
+
recordInsight({
|
|
495
|
+
kind: "first_run_disputed",
|
|
496
|
+
display: "first-run: user disputed the catch",
|
|
497
|
+
session: best.session || null,
|
|
498
|
+
});
|
|
499
|
+
} else {
|
|
500
|
+
out.write("\n");
|
|
501
|
+
}
|
|
502
|
+
};
|
|
503
|
+
|
|
504
|
+
const hint = ` ${D}[c]${X} copy card ${D}[x]${X} not right? ${D}[enter]${X} done `;
|
|
505
|
+
// Non-interactive (tests / piped): honor a forced answer, else just print the hint and continue.
|
|
506
|
+
const canRaw =
|
|
507
|
+
interactive && stdin && stdin.isTTY && typeof stdin.setRawMode === "function" && !env.CI;
|
|
508
|
+
if (!canRaw) {
|
|
509
|
+
out.write(hint + "\n");
|
|
510
|
+
if (answer) done(String(answer).toLowerCase());
|
|
511
|
+
return Promise.resolve();
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
out.write(hint);
|
|
515
|
+
return new Promise((resolve) => {
|
|
516
|
+
let settled = false;
|
|
517
|
+
const finish = (key) => {
|
|
518
|
+
if (settled) return;
|
|
519
|
+
settled = true;
|
|
520
|
+
try {
|
|
521
|
+
stdin.setRawMode(false);
|
|
522
|
+
stdin.pause();
|
|
523
|
+
stdin.removeListener("data", onData);
|
|
524
|
+
} catch {
|
|
525
|
+
/* teardown best-effort */
|
|
526
|
+
}
|
|
527
|
+
out.write("\n");
|
|
528
|
+
try {
|
|
529
|
+
done(key);
|
|
530
|
+
} catch {
|
|
531
|
+
/* never throw at the user */
|
|
532
|
+
}
|
|
533
|
+
resolve();
|
|
534
|
+
};
|
|
535
|
+
const onData = (buf) => {
|
|
536
|
+
const k = buf.toString("utf8").toLowerCase();
|
|
537
|
+
if (k === "") return finish("enter"); // Ctrl-C → just finish (don't kill setup)
|
|
538
|
+
if (k === "c") return finish("c");
|
|
539
|
+
if (k === "x") return finish("x");
|
|
540
|
+
return finish("enter"); // Enter / any other key → done
|
|
541
|
+
};
|
|
542
|
+
try {
|
|
543
|
+
stdin.setRawMode(true);
|
|
544
|
+
stdin.resume();
|
|
545
|
+
stdin.on("data", onData);
|
|
546
|
+
} catch {
|
|
547
|
+
finish("enter");
|
|
548
|
+
}
|
|
549
|
+
});
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
// ---------- (8) coin-flip variant + first-run state ----------
|
|
553
|
+
function firstRunPath(home) {
|
|
554
|
+
return path.join(home, ".skalpel", "first-run.json");
|
|
555
|
+
}
|
|
556
|
+
function readState(home) {
|
|
557
|
+
try {
|
|
558
|
+
return JSON.parse(readFileSync(firstRunPath(home), "utf8")) || {};
|
|
559
|
+
} catch {
|
|
560
|
+
return {};
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
function writeState(home, state) {
|
|
564
|
+
try {
|
|
565
|
+
mkdirSync(path.join(home, ".skalpel"), { recursive: true });
|
|
566
|
+
writeFileSync(firstRunPath(home), JSON.stringify(state, null, 2));
|
|
567
|
+
return true;
|
|
568
|
+
} catch {
|
|
569
|
+
return false;
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
// assignVariant(home, env) — persist a coin-flip variant ONCE. Dev/test override: SKALPEL_FIRST_RUN_VARIANT.
|
|
574
|
+
export function assignVariant(home, env = process.env) {
|
|
575
|
+
const state = readState(home);
|
|
576
|
+
const forced = String(env.SKALPEL_FIRST_RUN_VARIANT || "").toLowerCase();
|
|
577
|
+
if (forced === "reveal" || forced === "holdout") {
|
|
578
|
+
if (state.variant !== forced) {
|
|
579
|
+
state.variant = forced;
|
|
580
|
+
state.assigned_at = state.assigned_at || new Date().toISOString();
|
|
581
|
+
writeState(home, state);
|
|
582
|
+
}
|
|
583
|
+
return forced;
|
|
584
|
+
}
|
|
585
|
+
if (state.variant === "reveal" || state.variant === "holdout") return state.variant;
|
|
586
|
+
const variant = Math.random() < 0.5 ? "reveal" : "holdout";
|
|
587
|
+
state.variant = variant;
|
|
588
|
+
state.assigned_at = new Date().toISOString();
|
|
589
|
+
writeState(home, state);
|
|
590
|
+
return variant;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
// ---------- (9) THE orchestrator (called from skalpel-setup.mjs right after consent) ----------
|
|
594
|
+
// Contract: never throws, never blocks setup. Returns a small summary object for tests.
|
|
595
|
+
export async function maybeFirstRunCatch({
|
|
596
|
+
isTTY = process.stdout.isTTY && !process.env.CI,
|
|
597
|
+
env = process.env,
|
|
598
|
+
home = homedir(),
|
|
599
|
+
out = process.stdout,
|
|
600
|
+
stdin = process.stdin,
|
|
601
|
+
interactive = true,
|
|
602
|
+
answer = null,
|
|
603
|
+
now = Date.now(),
|
|
604
|
+
} = {}) {
|
|
605
|
+
try {
|
|
606
|
+
// BYTE-IDENTICAL-OFF: no TTY, or consent not accepted → return before ANY transcript read or write.
|
|
607
|
+
if (!isTTY) return { ran: false, reason: "no_tty" };
|
|
608
|
+
if (!revealEnabled(env)) return { ran: false, reason: "not_consented" };
|
|
609
|
+
|
|
610
|
+
// Once only (unless forced) — the first catch is a first-run moment, not a per-setup nag.
|
|
611
|
+
const state0 = readState(home);
|
|
612
|
+
if (state0.ran && env.SKALPEL_FIRST_RUN_FORCE !== "1") {
|
|
613
|
+
return { ran: false, reason: "already_ran", variant: state0.variant || null };
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
const variant = assignVariant(home, env);
|
|
617
|
+
|
|
618
|
+
// HOLDOUT arm = current flow, no scan, no reveal. Record the assignment for the D7 A/B and return.
|
|
619
|
+
if (variant === "holdout") {
|
|
620
|
+
recordInsight({
|
|
621
|
+
kind: "first_run",
|
|
622
|
+
display: "first-run holdout (no reveal)",
|
|
623
|
+
variant,
|
|
624
|
+
shown: false,
|
|
625
|
+
});
|
|
626
|
+
const st = readState(home);
|
|
627
|
+
writeState(home, { ...st, variant, ran: true, ran_at: new Date().toISOString() });
|
|
628
|
+
return { ran: true, variant, shown: false };
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
// REVEAL arm — scan with a live progress line, then reveal the single best catch (or the honest zero
|
|
632
|
+
// state). Progress writes to the same line; cleared before the reveal renders.
|
|
633
|
+
let lastLen = 0;
|
|
634
|
+
const onProgress = (nSessions) => {
|
|
635
|
+
const line = ` ${D}scanning your own history — read-only — ${nSessions} sessions…${X}`;
|
|
636
|
+
out.write("\r" + line);
|
|
637
|
+
lastLen = line.length;
|
|
638
|
+
};
|
|
639
|
+
const result = firstRunScan({ home, onProgress, now });
|
|
640
|
+
if (lastLen) out.write("\r" + " ".repeat(lastLen) + "\r"); // wipe the progress line
|
|
641
|
+
|
|
642
|
+
const shown = Boolean(result.best);
|
|
643
|
+
recordInsight({
|
|
644
|
+
kind: "first_run",
|
|
645
|
+
display: shown
|
|
646
|
+
? `first-run reveal: caught 1 of ${result.candidates.length} across ${result.scanned} sessions`
|
|
647
|
+
: `first-run zero-state: 0 catches across ${result.scanned} sessions`,
|
|
648
|
+
variant,
|
|
649
|
+
sessions: result.scanned,
|
|
650
|
+
gate_catches: result.candidates.length,
|
|
651
|
+
shown,
|
|
652
|
+
session: result.best ? result.best.session : null,
|
|
653
|
+
});
|
|
654
|
+
|
|
655
|
+
if (!shown) {
|
|
656
|
+
out.write(zeroStateLines(result.scanned, Boolean(out.isTTY)) + "\n");
|
|
657
|
+
const st = readState(home);
|
|
658
|
+
writeState(home, {
|
|
659
|
+
...st,
|
|
660
|
+
variant,
|
|
661
|
+
ran: true,
|
|
662
|
+
ran_at: new Date().toISOString(),
|
|
663
|
+
shown: false,
|
|
664
|
+
});
|
|
665
|
+
return { ran: true, variant, shown: false, scanned: result.scanned };
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
const best = result.best;
|
|
669
|
+
out.write(revealLines(best, result.moreWithin30d, Boolean(out.isTTY)) + "\n");
|
|
670
|
+
|
|
671
|
+
// Append the retro row so `skalpel card` can render it later, and pre-render the markdown for [c].
|
|
672
|
+
const row = buildRetroRow(best);
|
|
673
|
+
appendRetroRow(row);
|
|
674
|
+
const markdown = (() => {
|
|
675
|
+
try {
|
|
676
|
+
return renderMarkdownCard(row);
|
|
677
|
+
} catch {
|
|
678
|
+
return "";
|
|
679
|
+
}
|
|
680
|
+
})();
|
|
681
|
+
|
|
682
|
+
await promptRevealExit({ out, stdin, interactive, answer, markdown, best, env });
|
|
683
|
+
|
|
684
|
+
const st = readState(home);
|
|
685
|
+
writeState(home, {
|
|
686
|
+
...st,
|
|
687
|
+
variant,
|
|
688
|
+
ran: true,
|
|
689
|
+
ran_at: new Date().toISOString(),
|
|
690
|
+
shown: true,
|
|
691
|
+
catch_session: best.session || null,
|
|
692
|
+
});
|
|
693
|
+
return { ran: true, variant, shown: true, scanned: result.scanned, best };
|
|
694
|
+
} catch {
|
|
695
|
+
// FAIL-OPEN: any failure → silently skip to the normal setup ending.
|
|
696
|
+
return { ran: false, reason: "error" };
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
// ---------- CLI: read-only dry run (prints what WOULD be revealed; writes nothing) ----------
|
|
701
|
+
const isMain = (() => {
|
|
702
|
+
try {
|
|
703
|
+
return (
|
|
704
|
+
Boolean(process.argv[1]) &&
|
|
705
|
+
realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url))
|
|
706
|
+
);
|
|
707
|
+
} catch {
|
|
708
|
+
return false;
|
|
709
|
+
}
|
|
710
|
+
})();
|
|
711
|
+
|
|
712
|
+
if (isMain) {
|
|
713
|
+
try {
|
|
714
|
+
const result = firstRunScan({});
|
|
715
|
+
if (result.best)
|
|
716
|
+
process.stdout.write(revealLines(result.best, result.moreWithin30d, true) + "\n");
|
|
717
|
+
else process.stdout.write(zeroStateLines(result.scanned, true) + "\n");
|
|
718
|
+
} catch {
|
|
719
|
+
process.stdout.write("\n 🔬 skalpel — first-run dry scan unavailable.\n\n");
|
|
720
|
+
}
|
|
721
|
+
process.exit(0);
|
|
722
|
+
}
|