skalpel 4.0.29 → 4.0.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/anchor-shadow.mjs +627 -0
- package/package.json +1 -1
- package/verify-shadow.mjs +326 -34
|
@@ -0,0 +1,627 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// anchor-shadow.mjs — ship-status "Auto-Run-the-Anchor", SHADOW mode, READ-ONLY.
|
|
3
|
+
//
|
|
4
|
+
// WHAT: when the user falls into the cycle-12 signature (>=2 consecutive terse re-verify/doubt turns
|
|
5
|
+
// about the SAME claim — "did it merge?", "you sure it's live?", "did that actually push?") AND that
|
|
6
|
+
// doubt is about SHIP STATUS (merged / pushed / live / deployed / price / trial / CI / passing /
|
|
7
|
+
// released), reconstruct the concrete anchor from recent session + repo context (a PR number, a URL,
|
|
8
|
+
// a commit sha, a Stripe price), run ONE read-only allowlisted probe to resolve the doubt, and LOG
|
|
9
|
+
// what we WOULD have injected. It NEVER injects into the live model and NEVER mutates anything.
|
|
10
|
+
//
|
|
11
|
+
// RED LINES honored by construction:
|
|
12
|
+
// - LOCAL-ONLY: the only side effect is appending to ~/.skalpel/anchor-shadow.log. No network exfil.
|
|
13
|
+
// - FAIL-OPEN: the per-turn wiring only spawns a DETACHED, unref'd child to run the probe, so the
|
|
14
|
+
// hook returns immediately — a probe can never block or slow a turn. Every probe has a hard 5s cap.
|
|
15
|
+
// - REVERSIBLE / SHADOW: log only, never inject, never throw out to the caller.
|
|
16
|
+
// - PRECISION OVER RECALL: fire only on a tight, consecutive doubt streak about a ship-status claim.
|
|
17
|
+
// - NO FABRICATED NUMBERS: probe evidence is the REAL stdout of a read-only command (execFile arg
|
|
18
|
+
// arrays, never a shell string), or an honest "unknown" on error/timeout.
|
|
19
|
+
//
|
|
20
|
+
// INJECTION SAFETY: every probe is `execFile(bin, [args...])` — arguments are an array, so there is NO
|
|
21
|
+
// shell and NO string interpolation. On top of that (defense in depth) every reconstructed value is
|
|
22
|
+
// re-validated against a strict whitelist before it is ever passed as an argument: PR = integer, sha =
|
|
23
|
+
// /^[0-9a-f]{7,40}$/, url = parseable http(s) URL with no whitespace/control chars, repo dir = an
|
|
24
|
+
// existing absolute directory. A crafted target such as url = "; rm -rf x" fails `new URL()` and is
|
|
25
|
+
// rejected; even if it weren't, execFile would hand it to curl as one literal argv, not to a shell.
|
|
26
|
+
|
|
27
|
+
import {
|
|
28
|
+
appendFileSync,
|
|
29
|
+
mkdirSync,
|
|
30
|
+
existsSync,
|
|
31
|
+
statSync,
|
|
32
|
+
writeFileSync,
|
|
33
|
+
readFileSync,
|
|
34
|
+
realpathSync,
|
|
35
|
+
} from "node:fs";
|
|
36
|
+
import { homedir } from "node:os";
|
|
37
|
+
import { join, dirname, isAbsolute } from "node:path";
|
|
38
|
+
import { fileURLToPath } from "node:url";
|
|
39
|
+
import { execFile, spawn } from "node:child_process";
|
|
40
|
+
import { tailLines } from "./transcript.mjs";
|
|
41
|
+
|
|
42
|
+
const SKALPEL_DIR = join(homedir(), ".skalpel");
|
|
43
|
+
const LOG_PATH = join(SKALPEL_DIR, "anchor-shadow.log");
|
|
44
|
+
const PENDING_PATH = join(SKALPEL_DIR, "anchor-pending.json");
|
|
45
|
+
const TRANSCRIPT_TAIL_BYTES = 256 * 1024;
|
|
46
|
+
const PROBE_TIMEOUT_MS = 5000;
|
|
47
|
+
|
|
48
|
+
// The claim classifier: the doubt only counts as SHIP-STATUS if one of these tokens is present.
|
|
49
|
+
// Word-boundary matched, case-insensitive. Kept tight (precision over recall).
|
|
50
|
+
const SHIP_LEXICON =
|
|
51
|
+
/\b(merged?|push(?:ed)?|live|deployed?|deploy|prod(?:uction)?|price|pricing|trial|ci|passing|pass(?:ed)?|released?|release|shipp?ed|ship)\b/i;
|
|
52
|
+
|
|
53
|
+
// The re-verify / doubt signature: a TERSE turn that questions a prior claim without taking a new
|
|
54
|
+
// action. Either it hits one of these doubt phrases, or it is a short bare question ("live?", "did it?").
|
|
55
|
+
const DOUBT_LEXICON =
|
|
56
|
+
/\b(did (?:it|that|you|they|we)|are you sure|you sure|is it (?:really|actually|live|up|merged|deployed|passing)|are they|has it|have you|really|actually|for real|for sure|double[- ]?check|recheck|re-?verify|verify|confirm|check again|wait\b|hmm+|sure\?)\b/i;
|
|
57
|
+
|
|
58
|
+
// A NEW ACTION (imperative work) disqualifies a turn from being a pure re-verify. If a "doubt" turn
|
|
59
|
+
// also issues fresh work, it isn't the passive cycle-12 loop we target.
|
|
60
|
+
const NEW_ACTION =
|
|
61
|
+
/\b(add|fix|change|write|implement|refactor|make|create|update|remove|delete|build|run|try|use|set|rename)\b/i;
|
|
62
|
+
|
|
63
|
+
// ---------------------------------------------------------------------------------------------------
|
|
64
|
+
// Transcript helpers (bounded tail read — reuses the same helper the hook uses, so no unbounded slurp).
|
|
65
|
+
// ---------------------------------------------------------------------------------------------------
|
|
66
|
+
function entryText(e) {
|
|
67
|
+
const raw = e?.content ?? e?.message?.content;
|
|
68
|
+
if (typeof raw === "string") return raw;
|
|
69
|
+
if (Array.isArray(raw))
|
|
70
|
+
return raw
|
|
71
|
+
.map((b) => {
|
|
72
|
+
if (typeof b === "string") return b;
|
|
73
|
+
if (b?.type === "text") return b.text || "";
|
|
74
|
+
// tool_result content can be a string or nested blocks — flatten best-effort for anchor mining.
|
|
75
|
+
if (b?.type === "tool_result") {
|
|
76
|
+
const c = b.content;
|
|
77
|
+
if (typeof c === "string") return c;
|
|
78
|
+
if (Array.isArray(c)) return c.map((x) => x?.text || "").join(" ");
|
|
79
|
+
}
|
|
80
|
+
return "";
|
|
81
|
+
})
|
|
82
|
+
.join(" ");
|
|
83
|
+
return "";
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// The last `n` USER turns, newest first, as trimmed strings. Bounded tail read; never throws.
|
|
87
|
+
export function recentUserTurns(payload, n = 5) {
|
|
88
|
+
const out = [];
|
|
89
|
+
try {
|
|
90
|
+
if (payload?.prompt) out.push(String(payload.prompt).trim());
|
|
91
|
+
if (!payload?.transcript_path) return out.slice(0, n);
|
|
92
|
+
const lines = tailLines(payload.transcript_path, TRANSCRIPT_TAIL_BYTES);
|
|
93
|
+
for (let i = lines.length - 1; i >= 0 && out.length < n; i--) {
|
|
94
|
+
let e;
|
|
95
|
+
try {
|
|
96
|
+
e = JSON.parse(lines[i]);
|
|
97
|
+
} catch {
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
if (e.type === "user" || e.role === "user") {
|
|
101
|
+
const t = entryText(e).trim();
|
|
102
|
+
if (t) out.push(t);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
} catch {
|
|
106
|
+
/* fail-open */
|
|
107
|
+
}
|
|
108
|
+
return out.slice(0, n);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// A broad recent-context string (user + assistant + tool text) for anchor mining — this is where a PR
|
|
112
|
+
// number, URL, or sha most often lives (in the assistant's prior `gh`/`git` output, not the user turn).
|
|
113
|
+
export function recentContextText(payload, entries = 60) {
|
|
114
|
+
try {
|
|
115
|
+
if (!payload?.transcript_path) return String(payload?.prompt || "");
|
|
116
|
+
const lines = tailLines(payload.transcript_path, TRANSCRIPT_TAIL_BYTES);
|
|
117
|
+
const slice = lines.slice(-entries);
|
|
118
|
+
const parts = [];
|
|
119
|
+
for (const ln of slice) {
|
|
120
|
+
try {
|
|
121
|
+
parts.push(entryText(JSON.parse(ln)));
|
|
122
|
+
} catch {
|
|
123
|
+
/* skip */
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
if (payload?.prompt) parts.push(String(payload.prompt));
|
|
127
|
+
return parts.join("\n");
|
|
128
|
+
} catch {
|
|
129
|
+
return "";
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// ---------------------------------------------------------------------------------------------------
|
|
134
|
+
// (1) detectShipStatusSpiral — the cycle-12 re-verify signature + ship-status classification.
|
|
135
|
+
// `turns` = user turns newest-first (as returned by recentUserTurns). Returns {fire, claim}.
|
|
136
|
+
// ---------------------------------------------------------------------------------------------------
|
|
137
|
+
function isTerse(t) {
|
|
138
|
+
return t.split(/\s+/).filter(Boolean).length <= 10;
|
|
139
|
+
}
|
|
140
|
+
function isDoubtTurn(t) {
|
|
141
|
+
if (!isTerse(t)) return false;
|
|
142
|
+
if (NEW_ACTION.test(t)) return false; // a fresh instruction is not a passive re-verify
|
|
143
|
+
return DOUBT_LEXICON.test(t) || /\?\s*$/.test(t); // doubt phrase OR a short bare question
|
|
144
|
+
}
|
|
145
|
+
function shipToken(t) {
|
|
146
|
+
const m = SHIP_LEXICON.exec(t);
|
|
147
|
+
return m ? m[1].toLowerCase() : null;
|
|
148
|
+
}
|
|
149
|
+
// Normalize surface variants to a canonical claim keyword for stable grouping/probing.
|
|
150
|
+
function canonicalClaim(tok) {
|
|
151
|
+
if (!tok) return null;
|
|
152
|
+
if (/^merg/.test(tok)) return "merged";
|
|
153
|
+
if (/^push/.test(tok)) return "pushed";
|
|
154
|
+
if (/^deploy/.test(tok)) return "deployed";
|
|
155
|
+
if (/^prod/.test(tok)) return "deployed";
|
|
156
|
+
if (tok === "live") return "live";
|
|
157
|
+
if (/^releas/.test(tok)) return "released";
|
|
158
|
+
if (/^ship/.test(tok)) return "released";
|
|
159
|
+
if (/^pric/.test(tok)) return "price";
|
|
160
|
+
if (tok === "trial") return "trial";
|
|
161
|
+
if (tok === "ci" || /^pass/.test(tok)) return "ci";
|
|
162
|
+
return tok;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export function detectShipStatusSpiral(turns) {
|
|
166
|
+
const none = { fire: false, claim: null };
|
|
167
|
+
if (!Array.isArray(turns) || turns.length < 2) return none;
|
|
168
|
+
// Walk consecutive doubt turns from the newest. Stop at the first non-doubt turn.
|
|
169
|
+
const streak = [];
|
|
170
|
+
for (const t of turns) {
|
|
171
|
+
if (isDoubtTurn(t)) streak.push(t);
|
|
172
|
+
else break;
|
|
173
|
+
}
|
|
174
|
+
if (streak.length < 2) return none; // need >=2 consecutive terse re-verify/doubt turns
|
|
175
|
+
// The doubt must classify as SHIP-STATUS: a ship-status token present across the streak. "SAME claim"
|
|
176
|
+
// = the streak shares one canonical ship claim (no conflicting ship subject dominates the streak).
|
|
177
|
+
const claims = streak.map((t) => canonicalClaim(shipToken(t))).filter(Boolean);
|
|
178
|
+
if (claims.length === 0) return none; // doubt, but not about ship status → do not fire
|
|
179
|
+
// pick the most-recent ship claim; require the streak not to mix two different ship subjects.
|
|
180
|
+
const claim = claims[0];
|
|
181
|
+
const distinct = new Set(claims);
|
|
182
|
+
if (distinct.size > 1) return none; // ambiguous — different claims → precision guard, do not fire
|
|
183
|
+
return { fire: true, claim };
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// ---------------------------------------------------------------------------------------------------
|
|
187
|
+
// (2) reconstructTarget — pull the concrete anchor from recent turns + repo context. Returns a STRUCTURED
|
|
188
|
+
// target or null (null = "not confidently reconstructable" → the caller logs a miss and does NOT fire).
|
|
189
|
+
// ---------------------------------------------------------------------------------------------------
|
|
190
|
+
function findPr(text) {
|
|
191
|
+
// "#123", "PR 123", "PR#123", "pull/123", "pull request 123"
|
|
192
|
+
const m = /(?:\bpr\s*#?\s*|\bpull(?:\s*request)?\s*#?\s*|\/pull\/|#)(\d{1,6})\b/i.exec(text);
|
|
193
|
+
if (!m) return null;
|
|
194
|
+
const n = Number(m[1]);
|
|
195
|
+
return Number.isInteger(n) && n > 0 && n < 1e7 ? n : null;
|
|
196
|
+
}
|
|
197
|
+
function findUrl(text) {
|
|
198
|
+
const m = /\bhttps?:\/\/[^\s'")<>`]+/i.exec(text);
|
|
199
|
+
if (!m) return null;
|
|
200
|
+
let url = m[0].replace(/[.,);]+$/, ""); // trim trailing punctuation from prose
|
|
201
|
+
return isSafeUrl(url) ? url : null;
|
|
202
|
+
}
|
|
203
|
+
function findSha(text) {
|
|
204
|
+
// Prefer a sha that appears near commit-ish words; otherwise the longest lone hex token in [7,40].
|
|
205
|
+
const near = /\b(?:commit|sha|rev(?:ision)?|HEAD|at)\s+([0-9a-f]{7,40})\b/i.exec(text);
|
|
206
|
+
if (near && /^[0-9a-f]{7,40}$/.test(near[1])) return near[1].toLowerCase();
|
|
207
|
+
let best = null;
|
|
208
|
+
const re = /\b([0-9a-f]{7,40})\b/gi;
|
|
209
|
+
let m;
|
|
210
|
+
while ((m = re.exec(text))) {
|
|
211
|
+
const tok = m[1].toLowerCase();
|
|
212
|
+
if (/^\d+$/.test(tok)) continue; // pure decimal → not a sha (avoids matching long numbers)
|
|
213
|
+
if (!best || tok.length > best.length) best = tok;
|
|
214
|
+
}
|
|
215
|
+
return best;
|
|
216
|
+
}
|
|
217
|
+
function repoDirOf(payload) {
|
|
218
|
+
const cwd = payload?.cwd && isAbsolute(payload.cwd) ? payload.cwd : process.cwd();
|
|
219
|
+
try {
|
|
220
|
+
return existsSync(cwd) && statSync(cwd).isDirectory() ? cwd : null;
|
|
221
|
+
} catch {
|
|
222
|
+
return null;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
export function reconstructTarget(claim, contextText, payload) {
|
|
227
|
+
const text = String(contextText || "");
|
|
228
|
+
const repoDir = repoDirOf(payload);
|
|
229
|
+
const pr = findPr(text);
|
|
230
|
+
const url = findUrl(text);
|
|
231
|
+
const sha = findSha(text);
|
|
232
|
+
|
|
233
|
+
// Claim-driven preference, each with fallbacks. Return the first that reconstructs; null otherwise.
|
|
234
|
+
const tryPr = () => (pr != null ? { kind: "pr", n: pr, repoDir } : null);
|
|
235
|
+
const tryUrl = () => (url ? { kind: "url", url } : null);
|
|
236
|
+
const tryGit = () => (sha && repoDir ? { kind: "git", sha, repoDir } : null);
|
|
237
|
+
const tryStripe = () => ({ kind: "stripe-price" });
|
|
238
|
+
|
|
239
|
+
let order;
|
|
240
|
+
switch (claim) {
|
|
241
|
+
case "merged":
|
|
242
|
+
case "pushed":
|
|
243
|
+
case "ci":
|
|
244
|
+
case "released":
|
|
245
|
+
order = [tryPr, tryGit, tryUrl];
|
|
246
|
+
break;
|
|
247
|
+
case "live":
|
|
248
|
+
case "deployed":
|
|
249
|
+
order = [tryUrl, tryPr, tryGit];
|
|
250
|
+
break;
|
|
251
|
+
case "price":
|
|
252
|
+
case "trial":
|
|
253
|
+
// Only claim a stripe-price target if the context actually talks price/product — otherwise a
|
|
254
|
+
// bare "stripe prices list" is not a confident reconstruction of THIS doubt.
|
|
255
|
+
order = /\b(price|pricing|trial|checkout|stripe|plan|subscription|\$\d)/i.test(text)
|
|
256
|
+
? [tryStripe, tryUrl]
|
|
257
|
+
: [tryUrl];
|
|
258
|
+
break;
|
|
259
|
+
default:
|
|
260
|
+
order = [tryPr, tryUrl, tryGit];
|
|
261
|
+
}
|
|
262
|
+
for (const f of order) {
|
|
263
|
+
const t = f();
|
|
264
|
+
if (t) return t;
|
|
265
|
+
}
|
|
266
|
+
return null;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
// ---------------------------------------------------------------------------------------------------
|
|
270
|
+
// Strict validators (defense in depth — execFile already means no shell).
|
|
271
|
+
// ---------------------------------------------------------------------------------------------------
|
|
272
|
+
function isSafeUrl(u) {
|
|
273
|
+
if (typeof u !== "string" || u.length === 0 || u.length > 2048) return false;
|
|
274
|
+
if (/[\s-<>`"'\\]/.test(u)) return false; // no whitespace/control/quote/backtick/backslash
|
|
275
|
+
try {
|
|
276
|
+
const p = new URL(u);
|
|
277
|
+
return p.protocol === "http:" || p.protocol === "https:";
|
|
278
|
+
} catch {
|
|
279
|
+
return false;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
function validateTarget(t) {
|
|
283
|
+
if (!t || typeof t !== "object") return null;
|
|
284
|
+
switch (t.kind) {
|
|
285
|
+
case "pr": {
|
|
286
|
+
const n = Number(t.n);
|
|
287
|
+
if (!Number.isInteger(n) || n <= 0 || n >= 1e7) return null;
|
|
288
|
+
const repoDir = validDir(t.repoDir);
|
|
289
|
+
return { kind: "pr", n, repoDir };
|
|
290
|
+
}
|
|
291
|
+
case "url":
|
|
292
|
+
return isSafeUrl(t.url) ? { kind: "url", url: t.url } : null;
|
|
293
|
+
case "git": {
|
|
294
|
+
if (!/^[0-9a-f]{7,40}$/.test(String(t.sha || ""))) return null;
|
|
295
|
+
const repoDir = validDir(t.repoDir);
|
|
296
|
+
if (!repoDir) return null;
|
|
297
|
+
return { kind: "git", sha: String(t.sha).toLowerCase(), repoDir };
|
|
298
|
+
}
|
|
299
|
+
case "stripe-price":
|
|
300
|
+
return { kind: "stripe-price" };
|
|
301
|
+
default:
|
|
302
|
+
return null;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
function validDir(d) {
|
|
306
|
+
try {
|
|
307
|
+
return d && isAbsolute(d) && existsSync(d) && statSync(d).isDirectory() ? d : null;
|
|
308
|
+
} catch {
|
|
309
|
+
return null;
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
// ---------------------------------------------------------------------------------------------------
|
|
314
|
+
// (3) runProbe — ONE read-only allowlisted command via execFile (arg ARRAYS, never a shell string),
|
|
315
|
+
// hard 5s timeout, catch-all. Returns {pass, evidence}. pass ∈ {true,false,null(unknown)}.
|
|
316
|
+
// ---------------------------------------------------------------------------------------------------
|
|
317
|
+
function run(bin, args, opts = {}) {
|
|
318
|
+
return new Promise((resolve) => {
|
|
319
|
+
let done = false;
|
|
320
|
+
const finish = (v) => {
|
|
321
|
+
if (!done) {
|
|
322
|
+
done = true;
|
|
323
|
+
resolve(v);
|
|
324
|
+
}
|
|
325
|
+
};
|
|
326
|
+
try {
|
|
327
|
+
const child = execFile(
|
|
328
|
+
bin,
|
|
329
|
+
args,
|
|
330
|
+
{ timeout: PROBE_TIMEOUT_MS, killSignal: "SIGKILL", maxBuffer: 1 << 20, ...opts },
|
|
331
|
+
(err, stdout, stderr) => {
|
|
332
|
+
finish({ err, stdout: String(stdout || ""), stderr: String(stderr || "") });
|
|
333
|
+
},
|
|
334
|
+
);
|
|
335
|
+
child.on("error", (err) => finish({ err, stdout: "", stderr: String(err) }));
|
|
336
|
+
} catch (err) {
|
|
337
|
+
finish({ err, stdout: "", stderr: String(err) });
|
|
338
|
+
}
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
const clip = (s) =>
|
|
342
|
+
String(s || "")
|
|
343
|
+
.replace(/\s+/g, " ")
|
|
344
|
+
.trim()
|
|
345
|
+
.slice(0, 200);
|
|
346
|
+
|
|
347
|
+
export async function runProbe(rawTarget) {
|
|
348
|
+
const t = validateTarget(rawTarget);
|
|
349
|
+
if (!t) return { pass: null, evidence: "rejected: target failed validation" };
|
|
350
|
+
try {
|
|
351
|
+
if (t.kind === "pr") {
|
|
352
|
+
// gh pr view <n> --json state,mergedAt (READ-ONLY)
|
|
353
|
+
const opts = t.repoDir ? { cwd: t.repoDir } : {};
|
|
354
|
+
const { err, stdout } = await run(
|
|
355
|
+
"gh",
|
|
356
|
+
["pr", "view", String(t.n), "--json", "state,mergedAt"],
|
|
357
|
+
opts,
|
|
358
|
+
);
|
|
359
|
+
if (err)
|
|
360
|
+
return { pass: null, evidence: clip(`gh error: ${err.killed ? "timeout" : err.message}`) };
|
|
361
|
+
let j;
|
|
362
|
+
try {
|
|
363
|
+
j = JSON.parse(stdout);
|
|
364
|
+
} catch {
|
|
365
|
+
return { pass: null, evidence: clip(`gh non-json: ${stdout}`) };
|
|
366
|
+
}
|
|
367
|
+
const merged = j.state === "MERGED" || !!j.mergedAt;
|
|
368
|
+
return { pass: merged, evidence: clip(`state=${j.state} mergedAt=${j.mergedAt || "null"}`) };
|
|
369
|
+
}
|
|
370
|
+
if (t.kind === "url") {
|
|
371
|
+
// curl -sS -m 5 -o /dev/null -w "%{http_code}" <url> (READ-ONLY reachability)
|
|
372
|
+
const { err, stdout } = await run("curl", [
|
|
373
|
+
"-sS",
|
|
374
|
+
"-m",
|
|
375
|
+
"5",
|
|
376
|
+
"-o",
|
|
377
|
+
"/dev/null",
|
|
378
|
+
"-w",
|
|
379
|
+
"%{http_code}",
|
|
380
|
+
t.url,
|
|
381
|
+
]);
|
|
382
|
+
if (err)
|
|
383
|
+
return {
|
|
384
|
+
pass: null,
|
|
385
|
+
evidence: clip(`curl error: ${err.killed ? "timeout" : err.message}`),
|
|
386
|
+
};
|
|
387
|
+
const code = parseInt(stdout, 10);
|
|
388
|
+
if (!Number.isFinite(code) || code === 0)
|
|
389
|
+
return { pass: null, evidence: clip(`no http code: ${stdout}`) };
|
|
390
|
+
return { pass: code >= 200 && code < 400, evidence: `http_code=${code}` };
|
|
391
|
+
}
|
|
392
|
+
if (t.kind === "git") {
|
|
393
|
+
// git -C <repo> branch --contains <sha> -a (READ-ONLY: is the commit on main?)
|
|
394
|
+
const { err, stdout } = await run("git", [
|
|
395
|
+
"-C",
|
|
396
|
+
t.repoDir,
|
|
397
|
+
"branch",
|
|
398
|
+
"--contains",
|
|
399
|
+
t.sha,
|
|
400
|
+
"-a",
|
|
401
|
+
"--format=%(refname:short)",
|
|
402
|
+
]);
|
|
403
|
+
if (err)
|
|
404
|
+
return { pass: null, evidence: clip(`git error: ${err.killed ? "timeout" : err.message}`) };
|
|
405
|
+
const branches = stdout
|
|
406
|
+
.split("\n")
|
|
407
|
+
.map((s) => s.trim())
|
|
408
|
+
.filter(Boolean);
|
|
409
|
+
const onMain = branches.some((b) => /(^|\/)main$|(^|\/)master$/.test(b));
|
|
410
|
+
return { pass: onMain, evidence: clip(`contains: ${branches.join(",") || "(none)"}`) };
|
|
411
|
+
}
|
|
412
|
+
if (t.kind === "stripe-price") {
|
|
413
|
+
// stripe prices list --limit 3 (READ-ONLY)
|
|
414
|
+
const { err, stdout } = await run("stripe", ["prices", "list", "--limit", "3"]);
|
|
415
|
+
if (err)
|
|
416
|
+
return {
|
|
417
|
+
pass: null,
|
|
418
|
+
evidence: clip(`stripe error: ${err.killed ? "timeout" : err.message}`),
|
|
419
|
+
};
|
|
420
|
+
const has = /"id"\s*:/.test(stdout) || /price_/.test(stdout);
|
|
421
|
+
return { pass: has, evidence: clip(has ? "prices listed" : `no prices: ${stdout}`) };
|
|
422
|
+
}
|
|
423
|
+
} catch (e) {
|
|
424
|
+
return { pass: null, evidence: clip(`probe threw: ${e}`) };
|
|
425
|
+
}
|
|
426
|
+
return { pass: null, evidence: "unknown target kind" };
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
// ---------------------------------------------------------------------------------------------------
|
|
430
|
+
// Logging — the ONLY side effect. Append-only NDJSON at ~/.skalpel/anchor-shadow.log. Never throws.
|
|
431
|
+
// ---------------------------------------------------------------------------------------------------
|
|
432
|
+
function logRow(row) {
|
|
433
|
+
try {
|
|
434
|
+
mkdirSync(SKALPEL_DIR, { recursive: true });
|
|
435
|
+
appendFileSync(LOG_PATH, JSON.stringify(row) + "\n");
|
|
436
|
+
} catch {
|
|
437
|
+
/* logging is best-effort; never block or throw */
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
function readPending() {
|
|
441
|
+
try {
|
|
442
|
+
return JSON.parse(readFileSync(PENDING_PATH, "utf8"));
|
|
443
|
+
} catch {
|
|
444
|
+
return null;
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
function writePending(obj) {
|
|
448
|
+
try {
|
|
449
|
+
mkdirSync(SKALPEL_DIR, { recursive: true });
|
|
450
|
+
writeFileSync(PENDING_PATH, JSON.stringify(obj || {}));
|
|
451
|
+
} catch {
|
|
452
|
+
/* best-effort */
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
// ---------------------------------------------------------------------------------------------------
|
|
457
|
+
// (4) recordAnchorShadow — the per-turn wiring. SHADOW: log only, never inject, never throw. Cheap on
|
|
458
|
+
// the common path (no spiral → returns after a bounded tail read). On a fire with a reconstructable
|
|
459
|
+
// target it spawns a DETACHED, unref'd child to run the probe so the hook is NEVER slowed by it.
|
|
460
|
+
// ---------------------------------------------------------------------------------------------------
|
|
461
|
+
export function recordAnchorShadow(payload) {
|
|
462
|
+
try {
|
|
463
|
+
const session = payload?.session_id ?? null;
|
|
464
|
+
const turns = recentUserTurns(payload, 5);
|
|
465
|
+
|
|
466
|
+
// n+1 instrument (3): resolve the PREVIOUS fire against THIS turn — did the spiral end? (the user's
|
|
467
|
+
// newest turn no longer doubts the same claim). Race-free: uses only doubt-cessation, not the probe.
|
|
468
|
+
const pending = readPending();
|
|
469
|
+
if (pending && pending.claim) {
|
|
470
|
+
const newest = turns[0] || "";
|
|
471
|
+
const stillDoubting =
|
|
472
|
+
isDoubtTurn(newest) && canonicalClaim(shipToken(newest)) === pending.claim;
|
|
473
|
+
logRow({
|
|
474
|
+
ts: new Date().toISOString(),
|
|
475
|
+
session,
|
|
476
|
+
event: "resolve",
|
|
477
|
+
ref_ts: pending.ts,
|
|
478
|
+
claim: pending.claim,
|
|
479
|
+
spiral_ended: !stillDoubting,
|
|
480
|
+
});
|
|
481
|
+
writePending({});
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
const { fire, claim } = detectShipStatusSpiral(turns);
|
|
485
|
+
if (!fire) return;
|
|
486
|
+
|
|
487
|
+
const target = reconstructTarget(claim, recentContextText(payload), payload);
|
|
488
|
+
if (!target) {
|
|
489
|
+
// Fire but no concrete anchor → DO NOT fire the probe; log a miss (instrument 2 denominator).
|
|
490
|
+
logRow({ ts: new Date().toISOString(), session, event: "miss", claim, reason: "no-target" });
|
|
491
|
+
return;
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
// Arm the n+1 resolution for next turn, then hand the read-only probe to a detached child.
|
|
495
|
+
writePending({ ts: new Date().toISOString(), claim });
|
|
496
|
+
spawnProbeChild({ session, claim, target });
|
|
497
|
+
} catch {
|
|
498
|
+
/* SHADOW: swallow everything — this can NEVER affect the hook or its injected output. */
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
// Spawn a detached, unref'd child that runs ONE read-only probe and appends the fire row, then exits.
|
|
503
|
+
// The parent returns immediately → zero added per-turn latency. Job passed via env as JSON (no shell,
|
|
504
|
+
// no argv interpolation); the child re-validates every field before executing anything.
|
|
505
|
+
function spawnProbeChild(job) {
|
|
506
|
+
try {
|
|
507
|
+
const self = fileURLToPath(import.meta.url);
|
|
508
|
+
const child = spawn(process.execPath, [self, "--run-probe"], {
|
|
509
|
+
detached: true,
|
|
510
|
+
stdio: "ignore",
|
|
511
|
+
env: { ...process.env, SKALPEL_ANCHOR_JOB: JSON.stringify(job) },
|
|
512
|
+
});
|
|
513
|
+
child.on("error", () => {});
|
|
514
|
+
child.unref();
|
|
515
|
+
} catch {
|
|
516
|
+
/* if we can't spawn, log a miss so the fire is still accounted for */
|
|
517
|
+
logRow({
|
|
518
|
+
ts: new Date().toISOString(),
|
|
519
|
+
session: job?.session ?? null,
|
|
520
|
+
event: "miss",
|
|
521
|
+
claim: job?.claim ?? null,
|
|
522
|
+
reason: "spawn-failed",
|
|
523
|
+
});
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
async function runDetachedProbe() {
|
|
528
|
+
let job;
|
|
529
|
+
try {
|
|
530
|
+
job = JSON.parse(process.env.SKALPEL_ANCHOR_JOB || "{}");
|
|
531
|
+
} catch {
|
|
532
|
+
return;
|
|
533
|
+
}
|
|
534
|
+
const { pass, evidence } = await runProbe(job.target);
|
|
535
|
+
logRow({
|
|
536
|
+
ts: new Date().toISOString(),
|
|
537
|
+
session: job.session ?? null,
|
|
538
|
+
event: "probe",
|
|
539
|
+
claim: job.claim ?? null,
|
|
540
|
+
target: safeTarget(job.target),
|
|
541
|
+
would_inject_pass_fail: pass === null ? "UNKNOWN" : pass ? "PASS" : "FAIL",
|
|
542
|
+
evidence: clip(evidence),
|
|
543
|
+
});
|
|
544
|
+
}
|
|
545
|
+
// A compact, non-sensitive view of the target for the log.
|
|
546
|
+
function safeTarget(t) {
|
|
547
|
+
if (!t || typeof t !== "object") return null;
|
|
548
|
+
if (t.kind === "pr") return { kind: "pr", n: t.n };
|
|
549
|
+
if (t.kind === "url") return { kind: "url", url: clip(t.url) };
|
|
550
|
+
if (t.kind === "git") return { kind: "git", sha: t.sha };
|
|
551
|
+
if (t.kind === "stripe-price") return { kind: "stripe-price" };
|
|
552
|
+
return null;
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
// ---------------------------------------------------------------------------------------------------
|
|
556
|
+
// The tiny analyzer — `skalpel __anchor-report`. Prints the three instrument numbers from the log.
|
|
557
|
+
// ---------------------------------------------------------------------------------------------------
|
|
558
|
+
export function report() {
|
|
559
|
+
let lines = [];
|
|
560
|
+
try {
|
|
561
|
+
lines = readFileSync(LOG_PATH, "utf8").trim().split("\n").filter(Boolean);
|
|
562
|
+
} catch {
|
|
563
|
+
console.log(`no anchor-shadow log yet (${LOG_PATH})`);
|
|
564
|
+
return;
|
|
565
|
+
}
|
|
566
|
+
const rows = [];
|
|
567
|
+
for (const ln of lines) {
|
|
568
|
+
try {
|
|
569
|
+
rows.push(JSON.parse(ln));
|
|
570
|
+
} catch {
|
|
571
|
+
/* skip */
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
const fires = rows.filter((r) => r.event === "probe" || r.event === "miss");
|
|
575
|
+
const probes = rows.filter((r) => r.event === "probe");
|
|
576
|
+
const resolves = rows.filter((r) => r.event === "resolve");
|
|
577
|
+
|
|
578
|
+
const withTarget = probes.length;
|
|
579
|
+
const decisive = probes.filter((r) => r.would_inject_pass_fail !== "UNKNOWN").length;
|
|
580
|
+
const spiralEnded = resolves.filter((r) => r.spiral_ended).length;
|
|
581
|
+
|
|
582
|
+
const pct = (a, b) => (b > 0 ? `${((100 * a) / b).toFixed(0)}%` : "n/a");
|
|
583
|
+
|
|
584
|
+
console.log(`\nskalpel anchor-shadow report — ${rows.length} rows — ${LOG_PATH}\n`);
|
|
585
|
+
console.log(` 1. fires (detected ship-status spirals): ${fires.length}`);
|
|
586
|
+
console.log(
|
|
587
|
+
` 2. target-reconstruction success rate: ${pct(withTarget, fires.length)} (${withTarget}/${fires.length})`,
|
|
588
|
+
);
|
|
589
|
+
console.log(
|
|
590
|
+
` 3. probe would-resolve the doubt (decisive): ${pct(decisive, withTarget)} (${decisive}/${withTarget})`,
|
|
591
|
+
);
|
|
592
|
+
console.log(
|
|
593
|
+
` └ spiral ended on the next human turn: ${pct(spiralEnded, resolves.length)} (${spiralEnded}/${resolves.length})`,
|
|
594
|
+
);
|
|
595
|
+
// Pass/fail split of what we WOULD have surfaced — real command output, never fabricated.
|
|
596
|
+
const p = probes.filter((r) => r.would_inject_pass_fail === "PASS").length;
|
|
597
|
+
const f = probes.filter((r) => r.would_inject_pass_fail === "FAIL").length;
|
|
598
|
+
const u = probes.filter((r) => r.would_inject_pass_fail === "UNKNOWN").length;
|
|
599
|
+
console.log(`\n would-inject verdicts: PASS ${p} · FAIL ${f} · UNKNOWN ${u}\n`);
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
// ---------------------------------------------------------------------------------------------------
|
|
603
|
+
// CLI entry (only when run directly). `--run-probe` = the detached child; `--report` = the analyzer.
|
|
604
|
+
// When imported (from the hook or tests) none of this runs — only the exports above are used.
|
|
605
|
+
// ---------------------------------------------------------------------------------------------------
|
|
606
|
+
const isMain = (() => {
|
|
607
|
+
try {
|
|
608
|
+
if (!process.argv[1]) return false;
|
|
609
|
+
const self = fileURLToPath(import.meta.url);
|
|
610
|
+
if (self === process.argv[1]) return true;
|
|
611
|
+
// Tolerate symlinked staging paths (~/.skalpel/hooks may be reached via a symlink).
|
|
612
|
+
return realpathSync(self) === realpathSync(process.argv[1]);
|
|
613
|
+
} catch {
|
|
614
|
+
return false;
|
|
615
|
+
}
|
|
616
|
+
})();
|
|
617
|
+
if (isMain) {
|
|
618
|
+
if (process.argv.includes("--run-probe")) {
|
|
619
|
+
runDetachedProbe().finally(() => process.exit(0));
|
|
620
|
+
} else if (process.argv.includes("--report")) {
|
|
621
|
+
report();
|
|
622
|
+
process.exit(0);
|
|
623
|
+
} else {
|
|
624
|
+
console.log("anchor-shadow.mjs — internal skalpel module. Use `skalpel __anchor-report`.");
|
|
625
|
+
process.exit(0);
|
|
626
|
+
}
|
|
627
|
+
}
|
package/package.json
CHANGED
package/verify-shadow.mjs
CHANGED
|
@@ -8,22 +8,40 @@
|
|
|
8
8
|
// invented), RE-RUN it out-of-band, and record the real PASS/FAIL the agent cannot forge. The money
|
|
9
9
|
// metric is the MISMATCH rate: how often the agent claimed success but its own proof, re-run, FAILS.
|
|
10
10
|
//
|
|
11
|
+
// SHIP-STATUS catch (v0, same DARK gates): the SAME "the agent is its own reporter" problem extends past
|
|
12
|
+
// tests to the DEPLOY/GTM claim — "it's live / deployed / merged / pushed / returns 200". When the last
|
|
13
|
+
// turn ASSERTS a ship status, we reconstruct the concrete target from the session (a URL, a PR number, a
|
|
14
|
+
// commit sha) and run ONE READ-ONLY probe to check it — curl -sS -o /dev/null -w %{http_code} for a URL,
|
|
15
|
+
// gh pr view <n> --json state,mergedAt for a PR, git branch --contains <sha> for a push. On a decisive
|
|
16
|
+
// negative (claimed live but 404, claimed merged but OPEN) we surface the SAME honest reveal the test
|
|
17
|
+
// catch uses — from REAL probe output, gated on the SAME opt-in flag. The read-only probe primitives
|
|
18
|
+
// (execFile arg-arrays, strict allowlist, validated args, 5s cap) are REUSED verbatim from the
|
|
19
|
+
// security-reviewed anchor-shadow.mjs (PR #586) — never re-implemented here.
|
|
20
|
+
//
|
|
11
21
|
// SHADOW: this LOGS ONLY. It never injects into the model, never blocks or slows a turn, never writes
|
|
12
|
-
// or deploys anything. The per-turn hook launches it DETACHED
|
|
13
|
-
// is never delayed by the (up to 60s) re-run.
|
|
22
|
+
// or deploys anything (every probe is READ-ONLY — no mutations). The per-turn hook launches it DETACHED
|
|
23
|
+
// (see skalpel-hook.mjs) so the user's turn is never delayed by the (up to 60s) re-run / (5s) probe.
|
|
14
24
|
//
|
|
15
25
|
// HARD RED LINES (enforced by construction below):
|
|
16
26
|
// 1. ALLOWLIST ONLY. Only test/build/typecheck/lint binaries re-run (npm/pnpm/yarn test|build|run,
|
|
17
27
|
// npx tsc|jest|vitest|eslint|biome, pytest, cargo test|build|check|clippy, go test|build|vet,
|
|
18
28
|
// tsc, jest, vitest, eslint, biome, ruff, mypy, pyright, python -m <allowlisted>). Anything else
|
|
19
29
|
// is REFUSED. The command never comes from prompt/claim CONTENT — only from the session's own
|
|
20
|
-
// prior Bash tool_use commands.
|
|
30
|
+
// prior Bash tool_use commands. The SHIP-STATUS probe is likewise allowlisted to a fixed set of
|
|
31
|
+
// READ-ONLY reachability/status commands (gh pr view, curl reachability, git branch/log, stripe
|
|
32
|
+
// prices list) — see anchor-shadow.mjs runProbe/validateTarget.
|
|
21
33
|
// 2. execFile, NEVER a shell. Args are passed as a literal ARGUMENT ARRAY; there is no shell to
|
|
22
34
|
// interpret metacharacters, so a crafted string can never be executed as a command. shell:false.
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
//
|
|
35
|
+
// The ship probe validates every reconstructed value (pr=digits, url=well-formed http(s), sha=hex)
|
|
36
|
+
// before it is ever passed as an argv element.
|
|
37
|
+
// 3. HARD TIMEOUT (~60s re-run / ~5s probe), bounded output, single-flight lock. FAIL-OPEN: any error →
|
|
38
|
+
// log-or-skip, never throw, never affect the hook. An UNKNOWN probe (timeout, missing gh/curl, DNS)
|
|
39
|
+
// NEVER surfaces a reveal — only a decisive negative does — so we never cry wolf.
|
|
40
|
+
// 4. LOCAL ONLY. The proof re-run opens no socket. The ship probe makes a single READ-ONLY, user-driven
|
|
41
|
+
// reachability/status check to a target the SESSION itself named (never exfil) and logs to
|
|
42
|
+
// ~/.skalpel/verify-shadow.log.
|
|
43
|
+
// 5. NO FABRICATED NUMBERS. pass_fail is the real process exit code / the real HTTP code / the real PR
|
|
44
|
+
// state; evidence is the real tail output. A reveal never shows a number the probe did not print.
|
|
27
45
|
import {
|
|
28
46
|
appendFileSync,
|
|
29
47
|
mkdirSync,
|
|
@@ -44,10 +62,18 @@ import { tailLines } from "./transcript.mjs";
|
|
|
44
62
|
// Local sibling modules only (no sockets — RED LINE #4 holds): recordInsight logs the reveal fire for
|
|
45
63
|
// measurement; REVEAL_PATH/revealEnabled are the single-sourced state-file path + opt-in flag.
|
|
46
64
|
import { recordInsight, REVEAL_PATH, VERDICT_PATH, revealEnabled } from "./insights.mjs";
|
|
65
|
+
// SHIP-STATUS catch: REUSE the security-reviewed, READ-ONLY probe primitives from anchor-shadow.mjs
|
|
66
|
+
// (PR #586) verbatim — never re-implemented. runProbe(target) runs ONE allowlisted read-only command via
|
|
67
|
+
// execFile arg-arrays with a 5s cap and re-validates the target; reconstructTarget mines a PR/URL/sha from
|
|
68
|
+
// session context; recentContextText is the bounded-tail context reader those two use. Importing (not
|
|
69
|
+
// copying) keeps a SINGLE copy of the safety-critical logic under one review.
|
|
70
|
+
import { runProbe, reconstructTarget, recentContextText } from "./anchor-shadow.mjs";
|
|
47
71
|
|
|
48
72
|
const DIR = join(homedir(), ".skalpel");
|
|
49
73
|
export const VERIFY_LOG_PATH = join(DIR, "verify-shadow.log");
|
|
50
74
|
const LAST_PATH = join(DIR, "verify-last.json");
|
|
75
|
+
// Ship-status catch keeps its OWN dedup cursor so it never collides with the test-proof dedup (LAST_PATH).
|
|
76
|
+
const SHIP_LAST_PATH = join(DIR, "verify-ship-last.json");
|
|
51
77
|
const LOCK_PATH = join(DIR, "verify-shadow.lock");
|
|
52
78
|
|
|
53
79
|
// Re-run budget + read caps. The re-run is out-of-band (detached), so it is NOT on the hook deadline —
|
|
@@ -590,19 +616,21 @@ function djb2(s) {
|
|
|
590
616
|
|
|
591
617
|
// Dedup: don't re-run the same (session, claim) twice — guards double hook fires and repeated prompts
|
|
592
618
|
// that still see the same prior assistant turn. Best-effort; a miss just means one extra honest re-run.
|
|
593
|
-
|
|
619
|
+
// `cursorPath` selects which dedup cursor to use (the test-proof path and the ship-status path each keep
|
|
620
|
+
// their own so a claim of one kind never suppresses the other in the same turn).
|
|
621
|
+
function alreadyVerified(sig, cursorPath = LAST_PATH) {
|
|
594
622
|
try {
|
|
595
|
-
return JSON.parse(readFileSync(
|
|
623
|
+
return JSON.parse(readFileSync(cursorPath, "utf8"))?.sig === sig;
|
|
596
624
|
} catch {
|
|
597
625
|
return false;
|
|
598
626
|
}
|
|
599
627
|
}
|
|
600
|
-
function markVerified(sig) {
|
|
628
|
+
function markVerified(sig, cursorPath = LAST_PATH) {
|
|
601
629
|
try {
|
|
602
630
|
mkdirSync(DIR, { recursive: true });
|
|
603
|
-
const tmp = `${
|
|
631
|
+
const tmp = `${cursorPath}.tmp`;
|
|
604
632
|
writeFileSync(tmp, JSON.stringify({ sig, ts: Date.now() }));
|
|
605
|
-
renameSync(tmp,
|
|
633
|
+
renameSync(tmp, cursorPath);
|
|
606
634
|
} catch {
|
|
607
635
|
/* dedup is a bonus */
|
|
608
636
|
}
|
|
@@ -694,6 +722,126 @@ export function buildRevealLine(row) {
|
|
|
694
722
|
return `🔬 skalpel caught it — your agent said "${claim}" but \`${proof}\` just failed${count}. It's not done.`;
|
|
695
723
|
}
|
|
696
724
|
|
|
725
|
+
// ===================== SHIP-STATUS claim detection + reveal (new proof-category) =====================
|
|
726
|
+
// Distinct from a test claim: the proof is a READ-ONLY probe (is the URL live? is the PR merged? is the
|
|
727
|
+
// sha on main?), not a re-run. We look at the agent's LAST assistant prose (same source as the test
|
|
728
|
+
// claim) and fire ONLY on an ASSERTION of ship status — never a question, never a stated intention.
|
|
729
|
+
//
|
|
730
|
+
// Each pattern maps to a canonical claim keyword that anchor-shadow.mjs `reconstructTarget` understands
|
|
731
|
+
// (live | deployed | merged | pushed | released). A "returns 200 / is up" claim maps to `live` (a URL
|
|
732
|
+
// reachability probe). Precision over recall: a missed claim just means no catch, but a false claim
|
|
733
|
+
// would probe a target the agent never actually shipped.
|
|
734
|
+
const SHIP_CLAIM_PATTERNS = [
|
|
735
|
+
// deployed / deploy complete / pushed the deploy
|
|
736
|
+
[
|
|
737
|
+
"deployed",
|
|
738
|
+
/\bdeployed\b|\bdeploy(?:ment)?\s+(?:is\s+)?(?:complete|done|succeeded|finished|live|up)\b|\bpushed\s+(?:the\s+|a\s+)?deploy(?:ment)?\b/i,
|
|
739
|
+
],
|
|
740
|
+
// it's live / now live / went live / live at <url>
|
|
741
|
+
[
|
|
742
|
+
"live",
|
|
743
|
+
/\bit'?s\s+live\b|\bnow\s+live\b|\bis\s+(?:now\s+)?live\b|\b(?:went|gone)\s+live\b|\blive\s+(?:at|on|now)\b/i,
|
|
744
|
+
],
|
|
745
|
+
// returns 200 / 200 OK / responds with 200 / is up / reachable / responding
|
|
746
|
+
[
|
|
747
|
+
"live",
|
|
748
|
+
/\breturns?\s+(?:a\s+)?(?:http\s+)?200\b|\b200\s*ok\b|\bresponds?\s+with\s+(?:a\s+)?(?:http\s+)?200\b|\b(?:site|url|page|endpoint)\s+is\s+(?:up|reachable|responding)\b/i,
|
|
749
|
+
],
|
|
750
|
+
// merged / merged the PR / merged to main
|
|
751
|
+
[
|
|
752
|
+
"merged",
|
|
753
|
+
/\b(?:it'?s\s+|now\s+|has\s+been\s+|is\s+)?merged\b|\bmerged\s+(?:it|the\s+pr|to\s+main|into\s+main|it\s+in)\b/i,
|
|
754
|
+
],
|
|
755
|
+
// pushed / pushed to main|origin|remote
|
|
756
|
+
[
|
|
757
|
+
"pushed",
|
|
758
|
+
/\b(?:it'?s\s+|now\s+|has\s+been\s+|is\s+)?pushed\b|\bpushed\s+(?:it|the\s+(?:commit|branch|changes?)|to\s+(?:main|origin|remote))\b/i,
|
|
759
|
+
],
|
|
760
|
+
// released / shipped / shipped to prod
|
|
761
|
+
[
|
|
762
|
+
"released",
|
|
763
|
+
/\b(?:it'?s\s+|now\s+|has\s+been\s+)?released\b|\bshipp?ed\s+(?:it|to\s+prod(?:uction)?)\b/i,
|
|
764
|
+
],
|
|
765
|
+
];
|
|
766
|
+
|
|
767
|
+
// A stated INTENTION or a QUESTION is not a completed ship claim. If a line hits one of these, skip it.
|
|
768
|
+
const SHIP_FUTURE_OR_QUESTION =
|
|
769
|
+
/\b(?:will|i'?ll|we'?ll|going\s+to|gonna|about\s+to|let\s+me|planning\s+to|need\s+to|shall\s+i|should\s+i|can\s+i|once\s+|after\s+i|before\s+i|ready\s+to)\b/i;
|
|
770
|
+
|
|
771
|
+
// detectShipClaim(text) → { claim:bool, kind, text:<the claim line, <=120c> }. Scans lines; returns the
|
|
772
|
+
// first ASSERTIVE ship-status line. `kind` is the canonical claim keyword reconstructTarget consumes.
|
|
773
|
+
export function detectShipClaim(text) {
|
|
774
|
+
const s = typeof text === "string" ? text : "";
|
|
775
|
+
if (!s.trim()) return { claim: false, kind: null, text: "" };
|
|
776
|
+
for (const line of s.split(/\r?\n/)) {
|
|
777
|
+
const l = line.trim();
|
|
778
|
+
if (!l) continue;
|
|
779
|
+
if (/\?\s*$/.test(l)) continue; // a question is not a claim
|
|
780
|
+
if (SHIP_FUTURE_OR_QUESTION.test(l)) continue; // intention, not "it's done"
|
|
781
|
+
for (const [kind, re] of SHIP_CLAIM_PATTERNS) {
|
|
782
|
+
if (re.test(l)) {
|
|
783
|
+
return { claim: true, kind, text: l.replace(/\s+/g, " ").trim().slice(0, 120) };
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
return { claim: false, kind: null, text: "" };
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
// A compact, non-sensitive view of the reconstructed target, kept on the log row AND used to render the
|
|
791
|
+
// reveal. Retains only the identifier the reveal names (url / pr number / sha) — no repo paths.
|
|
792
|
+
function safeShipTarget(t) {
|
|
793
|
+
if (!t || typeof t !== "object") return null;
|
|
794
|
+
if (t.kind === "url") return { kind: "url", url: clipText(t.url, 160) };
|
|
795
|
+
if (t.kind === "pr") return { kind: "pr", n: t.n };
|
|
796
|
+
if (t.kind === "git") return { kind: "git", sha: String(t.sha || "").toLowerCase() };
|
|
797
|
+
if (t.kind === "stripe-price") return { kind: "stripe-price" };
|
|
798
|
+
return null;
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
// probeCommandString(t) — a human receipt of the exact READ-ONLY command that ran (for the log + report).
|
|
802
|
+
// It mirrors anchor-shadow.mjs runProbe's argv EXACTLY; it is never executed (execFile ran the arg-array).
|
|
803
|
+
function probeCommandString(t) {
|
|
804
|
+
if (!t || typeof t !== "object") return null;
|
|
805
|
+
if (t.kind === "url") return `curl -sS -m 5 -o /dev/null -w %{http_code} ${clipText(t.url, 120)}`;
|
|
806
|
+
if (t.kind === "pr") return `gh pr view ${t.n} --json state,mergedAt`;
|
|
807
|
+
if (t.kind === "git") return `git branch --contains ${String(t.sha || "").slice(0, 12)}`;
|
|
808
|
+
if (t.kind === "stripe-price") return `stripe prices list --limit 3`;
|
|
809
|
+
return null;
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
// buildShipRevealLine(row) → the plain-text ship reveal. Every number/state comes from the REAL probe
|
|
813
|
+
// evidence anchor-shadow.mjs recorded (`http_code=<n>`, `state=<S> mergedAt=…`, `contains: …`) — nothing
|
|
814
|
+
// is fabricated. Falls back to a generic negative phrasing if the structured field isn't present.
|
|
815
|
+
export function buildShipRevealLine(row) {
|
|
816
|
+
const claim = clipText(row.claim_text, 72) || "it's shipped";
|
|
817
|
+
const t = row.target || {};
|
|
818
|
+
const ev = String(row.evidence || "");
|
|
819
|
+
if (t.kind === "url") {
|
|
820
|
+
const m = ev.match(/http_code=(\d+)/);
|
|
821
|
+
const tail = m ? `returns HTTP ${m[1]}` : "is not reachable";
|
|
822
|
+
return `🔬 skalpel caught it — your agent said "${claim}" but \`${clipText(t.url, 48)}\` ${tail}. It's not live.`;
|
|
823
|
+
}
|
|
824
|
+
if (t.kind === "pr") {
|
|
825
|
+
const m = ev.match(/state=([A-Za-z]+)/);
|
|
826
|
+
const st = m ? m[1].toUpperCase() : "OPEN";
|
|
827
|
+
return `🔬 skalpel caught it — your agent said "${claim}" but PR #${t.n} is ${st}, not merged. It's not done.`;
|
|
828
|
+
}
|
|
829
|
+
if (t.kind === "git") {
|
|
830
|
+
const sha = String(t.sha || "").slice(0, 7);
|
|
831
|
+
return `🔬 skalpel caught it — your agent said "${claim}" but commit ${sha} isn't on main yet. It's not pushed.`;
|
|
832
|
+
}
|
|
833
|
+
if (t.kind === "stripe-price") {
|
|
834
|
+
return `🔬 skalpel caught it — your agent said "${claim}" but Stripe returned no matching price. It's not live.`;
|
|
835
|
+
}
|
|
836
|
+
return `🔬 skalpel caught it — your agent said "${claim}" but the ship-status probe came back negative.`;
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
// The single reveal-line dispatcher: ship rows render the ship line, everything else the test line. Keeps
|
|
840
|
+
// the test path byte-for-byte identical (a row with no `category` is a test row).
|
|
841
|
+
function revealLineFor(row) {
|
|
842
|
+
return row && row.category === "ship" ? buildShipRevealLine(row) : buildRevealLine(row);
|
|
843
|
+
}
|
|
844
|
+
|
|
697
845
|
// writeReveal(row) — atomic write of the reveal state file the statusline reads. Best-effort; never throws.
|
|
698
846
|
function writeReveal(row) {
|
|
699
847
|
try {
|
|
@@ -702,10 +850,12 @@ function writeReveal(row) {
|
|
|
702
850
|
ts: row.ts,
|
|
703
851
|
session: row.session || null,
|
|
704
852
|
claim_text: row.claim_text,
|
|
705
|
-
|
|
706
|
-
|
|
853
|
+
// A ship row's "proof" is the read-only probe command; a test row's is the re-run command.
|
|
854
|
+
proof_command: row.category === "ship" ? row.probe_command : row.proof_command,
|
|
855
|
+
// fail_count is a test-runner failing-test count; a ship probe has none (HTTP code/PR state ≠ count).
|
|
856
|
+
fail_count: row.category === "ship" ? null : extractFailCount(row.evidence),
|
|
707
857
|
evidence: row.evidence,
|
|
708
|
-
line:
|
|
858
|
+
line: revealLineFor(row),
|
|
709
859
|
};
|
|
710
860
|
const tmp = `${REVEAL_PATH}.tmp`;
|
|
711
861
|
writeFileSync(tmp, JSON.stringify(rec));
|
|
@@ -763,29 +913,40 @@ function clearVerdict() {
|
|
|
763
913
|
}
|
|
764
914
|
}
|
|
765
915
|
|
|
766
|
-
// maybeSurfaceReveal(row) — the single opt-in branch. OFF by default → does nothing (fully dark).
|
|
916
|
+
// maybeSurfaceReveal(row) — the single opt-in branch. OFF by default → does nothing (fully dark). It
|
|
917
|
+
// surfaces on a decisive negative (a test GENUINE_FAIL, or a ship SHIP_FAIL: claimed live but 404 /
|
|
918
|
+
// claimed merged but OPEN) and retracts on a decisive positive (test PASS, or ship SHIP_OK). An
|
|
919
|
+
// UNKNOWN/HARNESS_ERROR/ship SHIP_UNKNOWN NEVER surfaces — we never cry wolf on a broken probe.
|
|
767
920
|
function maybeSurfaceReveal(row) {
|
|
768
921
|
try {
|
|
769
922
|
if (!revealEnabled(process.env)) return; // SKALPEL_VERIFY_REVEAL unset/off → dark, no-op
|
|
770
|
-
|
|
923
|
+
const isFail =
|
|
924
|
+
(row.outcome === "GENUINE_FAIL" || row.outcome === "SHIP_FAIL") && row.mismatch === true;
|
|
925
|
+
const isPass = row.outcome === "PASS" || row.outcome === "SHIP_OK";
|
|
926
|
+
if (isFail) {
|
|
771
927
|
writeReveal(row);
|
|
772
928
|
clearVerdict(); // a real catch overrides any stale green — never show a ✓ next to a ✗
|
|
773
929
|
// MEASURABLE: one reveal_shown insight per fire → fire-rate + (later) retention, no fabricated data.
|
|
774
930
|
recordInsight({
|
|
775
931
|
kind: "reveal_shown",
|
|
776
|
-
display:
|
|
777
|
-
proof: row.proof_command,
|
|
932
|
+
display: revealLineFor(row),
|
|
933
|
+
proof: row.category === "ship" ? row.probe_command : row.proof_command,
|
|
778
934
|
session: row.session || null,
|
|
779
935
|
});
|
|
780
|
-
} else if (
|
|
936
|
+
} else if (isPass) {
|
|
781
937
|
clearReveal(); // resolved — don't wallpaper a catch the agent already fixed
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
938
|
+
// The green VERDICT stamp is #612's TEST-pass path only: its line ("re-ran `<proof>`, exit 0") is a
|
|
939
|
+
// real test re-run receipt. A ship SHIP_OK (read-only probe came back positive) has no re-run and no
|
|
940
|
+
// proof_command, so it must NOT borrow that wording — it only clears any prior catch (as #611 did).
|
|
941
|
+
if (row.outcome === "PASS") {
|
|
942
|
+
writeVerdict(row); // …and stamp the earned green so the ambient trust-line keeps building
|
|
943
|
+
recordInsight({
|
|
944
|
+
kind: "verdict_shown",
|
|
945
|
+
display: buildVerdictLine(row),
|
|
946
|
+
proof: row.proof_command,
|
|
947
|
+
session: row.session || null,
|
|
948
|
+
});
|
|
949
|
+
}
|
|
789
950
|
}
|
|
790
951
|
// HARNESS_ERROR / REFUSED: neither a catch nor a clean pass — surface nothing, touch no state file.
|
|
791
952
|
} catch {
|
|
@@ -793,6 +954,82 @@ function maybeSurfaceReveal(row) {
|
|
|
793
954
|
}
|
|
794
955
|
}
|
|
795
956
|
|
|
957
|
+
// maybeShipStatusCatch(...) — the SHIP-STATUS proof-category. When the agent's last turn ASSERTS a ship
|
|
958
|
+
// status, reconstruct the concrete target from session context and run ONE READ-ONLY allowlisted probe
|
|
959
|
+
// (reused verbatim from anchor-shadow.mjs). Appends one `category:"ship"` shadow row and, on a decisive
|
|
960
|
+
// negative, surfaces the SAME reveal (gated on SKALPEL_VERIFY_REVEAL). Returns { surfaced } — surfaced is
|
|
961
|
+
// true ONLY when a decisive ship FAIL was revealed, so the caller can skip the test path (the concrete
|
|
962
|
+
// ship catch is the stronger signal). LOG ONLY otherwise; never injects, never throws.
|
|
963
|
+
async function maybeShipStatusCatch({ text, session, cwd, transcriptPath }) {
|
|
964
|
+
try {
|
|
965
|
+
const { claim, kind, text: claimText } = detectShipClaim(text);
|
|
966
|
+
if (!claim) return { surfaced: false };
|
|
967
|
+
|
|
968
|
+
const sig = `ship|${session || ""}|${djb2(claimText)}`;
|
|
969
|
+
if (alreadyVerified(sig, SHIP_LAST_PATH)) return { surfaced: false };
|
|
970
|
+
|
|
971
|
+
// Reconstruct the target with the SAME primitives anchor-shadow.mjs uses. The context is the bounded
|
|
972
|
+
// recent transcript tail (where the agent's own `gh`/`git`/deploy output printed the PR/URL/sha); the
|
|
973
|
+
// claim line is appended so a target named only in the claim ("deployed to https://x") is seen too.
|
|
974
|
+
const payload = {
|
|
975
|
+
cwd: cwd || process.cwd(),
|
|
976
|
+
transcript_path: transcriptPath,
|
|
977
|
+
prompt: claimText,
|
|
978
|
+
};
|
|
979
|
+
const target = reconstructTarget(kind, recentContextText(payload), payload);
|
|
980
|
+
if (!target) {
|
|
981
|
+
markVerified(sig, SHIP_LAST_PATH);
|
|
982
|
+
appendShadowLog({
|
|
983
|
+
ts: new Date().toISOString(),
|
|
984
|
+
session: session || null,
|
|
985
|
+
category: "ship",
|
|
986
|
+
claim_text: claimText,
|
|
987
|
+
ship_claim: kind,
|
|
988
|
+
target: null,
|
|
989
|
+
probe_command: null,
|
|
990
|
+
pass_fail: null,
|
|
991
|
+
outcome: "NO_TARGET",
|
|
992
|
+
mismatch: false,
|
|
993
|
+
evidence: "",
|
|
994
|
+
});
|
|
995
|
+
return { surfaced: false };
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
// One read-only probe, single-flight (shares the lock with the test re-run so a burst never piles up).
|
|
999
|
+
if (!acquireLock()) return { surfaced: false };
|
|
1000
|
+
let res;
|
|
1001
|
+
try {
|
|
1002
|
+
res = await runProbe(target); // execFile arg-array, 5s cap, re-validates target — no mutations
|
|
1003
|
+
} finally {
|
|
1004
|
+
releaseLock();
|
|
1005
|
+
}
|
|
1006
|
+
markVerified(sig, SHIP_LAST_PATH);
|
|
1007
|
+
// pass ∈ { true (reachable/merged/on-main), false (decisive negative), null (unknown — never a lie) }.
|
|
1008
|
+
const pass = res && res.pass;
|
|
1009
|
+
const outcome = pass === true ? "SHIP_OK" : pass === false ? "SHIP_FAIL" : "SHIP_UNKNOWN";
|
|
1010
|
+
const pass_fail = pass === true ? "PASS" : pass === false ? "FAIL" : "UNKNOWN";
|
|
1011
|
+
const compact = safeShipTarget(target);
|
|
1012
|
+
const row = {
|
|
1013
|
+
ts: new Date().toISOString(),
|
|
1014
|
+
session: session || null,
|
|
1015
|
+
category: "ship",
|
|
1016
|
+
claim_text: claimText,
|
|
1017
|
+
ship_claim: kind,
|
|
1018
|
+
target: compact,
|
|
1019
|
+
probe_command: probeCommandString(target),
|
|
1020
|
+
pass_fail,
|
|
1021
|
+
outcome,
|
|
1022
|
+
mismatch: pass === false, // claimed shipped, the read-only probe says it is NOT
|
|
1023
|
+
evidence: String((res && res.evidence) || "").slice(0, 200),
|
|
1024
|
+
};
|
|
1025
|
+
appendShadowLog(row);
|
|
1026
|
+
maybeSurfaceReveal(row); // gated on SKALPEL_VERIFY_REVEAL — dark by default
|
|
1027
|
+
return { surfaced: pass === false && revealEnabled(process.env) };
|
|
1028
|
+
} catch {
|
|
1029
|
+
return { surfaced: false }; // SHADOW: swallow everything — never affect the hook or the turn
|
|
1030
|
+
}
|
|
1031
|
+
}
|
|
1032
|
+
|
|
796
1033
|
// recordVerifyShadow({transcriptPath, session, cwd}) — the wired entry. On (claim + reconstructable
|
|
797
1034
|
// proof) it re-runs the proof and appends one shadow row. LOG ONLY. Never injects, never throws.
|
|
798
1035
|
export async function recordVerifyShadow({ transcriptPath, session, cwd }) {
|
|
@@ -800,7 +1037,16 @@ export async function recordVerifyShadow({ transcriptPath, session, cwd }) {
|
|
|
800
1037
|
if (!transcriptPath) return;
|
|
801
1038
|
const entries = readEntries(transcriptPath);
|
|
802
1039
|
if (!entries.length) return;
|
|
803
|
-
const
|
|
1040
|
+
const assistantText = lastAssistantText(entries);
|
|
1041
|
+
|
|
1042
|
+
// NEW proof-category: the SHIP-STATUS catch runs first. A decisive ship FAIL (claimed live but 404,
|
|
1043
|
+
// claimed merged but OPEN) is the most concrete catch, so when it surfaces a reveal we stop here and
|
|
1044
|
+
// don't also run the test path (which could clobber the ship reveal). On no ship claim / PASS /
|
|
1045
|
+
// UNKNOWN it falls through to the unchanged test-proof path below.
|
|
1046
|
+
const ship = await maybeShipStatusCatch({ text: assistantText, session, cwd, transcriptPath });
|
|
1047
|
+
if (ship && ship.surfaced) return;
|
|
1048
|
+
|
|
1049
|
+
const { claim, text } = detectCompletionClaim(assistantText);
|
|
804
1050
|
if (!claim) return; // no claim this turn → nothing to verify (keeps the log claim-only)
|
|
805
1051
|
|
|
806
1052
|
const sig = `${session || ""}|${djb2(text)}`;
|
|
@@ -877,15 +1123,21 @@ export function renderVerifyReport() {
|
|
|
877
1123
|
return `no claim-verification shadow data yet (${VERIFY_LOG_PATH})\n\nEnable the shadow with SKALPEL_VERIFY_SHADOW=1, then keep coding — every turn your agent claims "done / tests pass" gets its own proof re-run and logged here.`;
|
|
878
1124
|
}
|
|
879
1125
|
|
|
880
|
-
|
|
881
|
-
|
|
1126
|
+
// The test-proof instrument counts ONLY test rows (no `category`). Ship-status rows (category:"ship")
|
|
1127
|
+
// are a separate proof-category with their own probe verbs, reported in their own section below — so
|
|
1128
|
+
// adding them never perturbs the pre-existing test numbers on old logs.
|
|
1129
|
+
const testRows = rows.filter((r) => r.category !== "ship");
|
|
1130
|
+
const shipRows = rows.filter((r) => r.category === "ship");
|
|
1131
|
+
|
|
1132
|
+
const claims = testRows.length;
|
|
1133
|
+
const withProof = testRows.filter((r) => r.proof_command).length;
|
|
882
1134
|
// Genuine re-runs only (PASS or a real test FAIL). HARNESS_ERROR rows (timeout, missing script, wrong
|
|
883
1135
|
// cwd) are EXCLUDED from the mismatch denominator — the proof machinery broke, not the agent's work.
|
|
884
|
-
const reran =
|
|
885
|
-
const harnessErrors =
|
|
1136
|
+
const reran = testRows.filter((r) => r.pass_fail === "PASS" || r.pass_fail === "FAIL").length;
|
|
1137
|
+
const harnessErrors = testRows.filter(
|
|
886
1138
|
(r) => r.pass_fail === "HARNESS_ERROR" || r.outcome === "HARNESS_ERROR",
|
|
887
1139
|
).length;
|
|
888
|
-
const mismatches =
|
|
1140
|
+
const mismatches = testRows.filter((r) => r.mismatch === true).length;
|
|
889
1141
|
|
|
890
1142
|
// Denominator for claim fire-rate: total prompt turns the hook saw (metrics.ndjson prompt events).
|
|
891
1143
|
let promptTurns = 0;
|
|
@@ -924,7 +1176,7 @@ export function renderVerifyReport() {
|
|
|
924
1176
|
` ${mismatches} time(s) the agent claimed done but its own proof, re-run, did NOT pass.`,
|
|
925
1177
|
);
|
|
926
1178
|
// Show the most recent few mismatches concretely (the receipt).
|
|
927
|
-
const recentMismatch =
|
|
1179
|
+
const recentMismatch = testRows
|
|
928
1180
|
.filter((r) => r.mismatch)
|
|
929
1181
|
.slice(-3)
|
|
930
1182
|
.reverse();
|
|
@@ -937,6 +1189,46 @@ export function renderVerifyReport() {
|
|
|
937
1189
|
if (r.evidence) L.push(` ${String(r.evidence).slice(0, 68)}`);
|
|
938
1190
|
}
|
|
939
1191
|
}
|
|
1192
|
+
|
|
1193
|
+
// SHIP-STATUS proof-category — the deploy/GTM claim ("it's live / merged / pushed / returns 200")
|
|
1194
|
+
// checked by a READ-ONLY probe. Its own instrument: how many ship claims were probed, how many were
|
|
1195
|
+
// decisive, and how many were caught (claimed shipped, the probe said NO). Every number is real probe
|
|
1196
|
+
// output — an UNKNOWN probe (timeout/missing gh|curl/DNS) is excluded, never counted as a lie.
|
|
1197
|
+
if (shipRows.length) {
|
|
1198
|
+
const shipProbed = shipRows.filter((r) => r.target).length;
|
|
1199
|
+
const shipDecisive = shipRows.filter(
|
|
1200
|
+
(r) => r.pass_fail === "PASS" || r.pass_fail === "FAIL",
|
|
1201
|
+
).length;
|
|
1202
|
+
const shipCaught = shipRows.filter((r) => r.mismatch === true).length;
|
|
1203
|
+
const shipUnknown = shipRows.filter((r) => r.outcome === "SHIP_UNKNOWN").length;
|
|
1204
|
+
L.push("");
|
|
1205
|
+
L.push("─".repeat(72));
|
|
1206
|
+
L.push(" SHIP-STATUS catch (claimed live/merged/pushed → checked by a READ-ONLY probe)");
|
|
1207
|
+
L.push(` ship claims logged ${shipRows.length}`);
|
|
1208
|
+
L.push(
|
|
1209
|
+
` (S1) target-reconstruction rate ${pct(shipProbed, shipRows.length)} ${shipProbed}/${shipRows.length} ship claims yielded a probeable target`,
|
|
1210
|
+
);
|
|
1211
|
+
L.push(
|
|
1212
|
+
` (S2) SHIP-MISMATCH rate ${pct(shipCaught, shipDecisive)} ${shipCaught}/${shipDecisive} decisive probes said NOT shipped after a ship claim`,
|
|
1213
|
+
);
|
|
1214
|
+
L.push(
|
|
1215
|
+
` (excluded: ${shipUnknown} unknown probe(s) — timeout/missing gh|curl/DNS, not counted as a lie)`,
|
|
1216
|
+
);
|
|
1217
|
+
const recentShip = shipRows
|
|
1218
|
+
.filter((r) => r.mismatch)
|
|
1219
|
+
.slice(-3)
|
|
1220
|
+
.reverse();
|
|
1221
|
+
if (recentShip.length) {
|
|
1222
|
+
L.push("");
|
|
1223
|
+
L.push(" recent ship-status catches:");
|
|
1224
|
+
for (const r of recentShip) {
|
|
1225
|
+
L.push(` • claim: ${String(r.claim_text || "").slice(0, 68)}`);
|
|
1226
|
+
L.push(` probe: ${String(r.probe_command || "").slice(0, 68)} → NOT SHIPPED`);
|
|
1227
|
+
if (r.evidence) L.push(` ${String(r.evidence).slice(0, 68)}`);
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
|
|
940
1232
|
L.push("");
|
|
941
1233
|
L.push(
|
|
942
1234
|
" SHADOW mode: logged locally only — never injected into the model, never blocks a turn.",
|