skalpel 4.0.42 → 4.0.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/anchor-shadow.mjs +20 -8
- package/autopsy.mjs +38 -8
- package/catches.mjs +40 -6
- package/incremental-ingest.mjs +14 -2
- package/ledger.mjs +16 -3
- package/login.mjs +52 -12
- package/metrics.mjs +35 -1
- package/package.json +1 -1
- package/skalpel-hook.mjs +165 -28
- package/verify-races.test.mjs +182 -0
- package/verify-shadow.mjs +60 -10
package/anchor-shadow.mjs
CHANGED
|
@@ -187,18 +187,30 @@ export function detectShipStatusSpiral(turns) {
|
|
|
187
187
|
// (2) reconstructTarget — pull the concrete anchor from recent turns + repo context. Returns a STRUCTURED
|
|
188
188
|
// target or null (null = "not confidently reconstructable" → the caller logs a miss and does NOT fire).
|
|
189
189
|
// ---------------------------------------------------------------------------------------------------
|
|
190
|
+
// The context passed in (recentContextText) is joined OLDEST→NEWEST, so a plain non-global .exec() would
|
|
191
|
+
// return the STALEST match. A ship-status probe must target the claim the user is doubting NOW — so scan
|
|
192
|
+
// ALL matches globally and keep the LAST (most-recent) valid one, not the first (a stale PR/URL from many
|
|
193
|
+
// turns ago).
|
|
190
194
|
function findPr(text) {
|
|
191
195
|
// "#123", "PR 123", "PR#123", "pull/123", "pull request 123"
|
|
192
|
-
const
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
+
const re = /(?:\bpr\s*#?\s*|\bpull(?:\s*request)?\s*#?\s*|\/pull\/|#)(\d{1,6})\b/gi;
|
|
197
|
+
let last = null;
|
|
198
|
+
let m;
|
|
199
|
+
while ((m = re.exec(text))) {
|
|
200
|
+
const n = Number(m[1]);
|
|
201
|
+
if (Number.isInteger(n) && n > 0 && n < 1e7) last = n;
|
|
202
|
+
}
|
|
203
|
+
return last;
|
|
196
204
|
}
|
|
197
205
|
function findUrl(text) {
|
|
198
|
-
const
|
|
199
|
-
|
|
200
|
-
let
|
|
201
|
-
|
|
206
|
+
const re = /\bhttps?:\/\/[^\s'")<>`]+/gi;
|
|
207
|
+
let last = null;
|
|
208
|
+
let m;
|
|
209
|
+
while ((m = re.exec(text))) {
|
|
210
|
+
const url = m[0].replace(/[.,);]+$/, ""); // trim trailing punctuation from prose
|
|
211
|
+
if (isSafeUrl(url)) last = url;
|
|
212
|
+
}
|
|
213
|
+
return last;
|
|
202
214
|
}
|
|
203
215
|
function findSha(text) {
|
|
204
216
|
// Prefer a sha that appears near commit-ish words; otherwise the longest lone hex token in [7,40].
|
package/autopsy.mjs
CHANGED
|
@@ -49,6 +49,7 @@ import fs from "node:fs";
|
|
|
49
49
|
import path from "node:path";
|
|
50
50
|
import os from "node:os";
|
|
51
51
|
import readline from "node:readline";
|
|
52
|
+
import { fileURLToPath } from "node:url";
|
|
52
53
|
|
|
53
54
|
const HOME = os.homedir();
|
|
54
55
|
const PROJECTS_DIR = path.join(HOME, ".claude", "projects");
|
|
@@ -212,10 +213,23 @@ function extractAnchors(text) {
|
|
|
212
213
|
return out;
|
|
213
214
|
}
|
|
214
215
|
|
|
215
|
-
|
|
216
|
+
// Captured transcript prose + tool output become quoted/shareable snippets via span() and vb(). Strip ESC
|
|
217
|
+
// + other C0 control chars so a hostile log line can't inject ANSI/CSI/OSC escapes (color, cursor moves,
|
|
218
|
+
// title/clipboard spoofing) into the receipt or a paste of it. redact() still runs on top for PII on the
|
|
219
|
+
// shareable surfaces; this is the terminal-safety layer beneath it (same defense as card.mjs `clean`).
|
|
220
|
+
export function stripCtrl(s) {
|
|
221
|
+
return (
|
|
222
|
+
String(s == null ? "" : s)
|
|
223
|
+
// eslint-disable-next-line no-control-regex
|
|
224
|
+
.replace(/\x1b\[[0-9;?]*[ -/]*[@-~]/g, "") // ANSI CSI sequences
|
|
225
|
+
// eslint-disable-next-line no-control-regex
|
|
226
|
+
.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, " ")
|
|
227
|
+
); // other control chars (incl. ESC/BEL) → space
|
|
228
|
+
}
|
|
229
|
+
export function span(text, at, len) {
|
|
216
230
|
const a = Math.max(0, at - SPAN),
|
|
217
231
|
b = Math.min(text.length, at + len + SPAN);
|
|
218
|
-
return text.slice(a, b).replace(/\s+/g, " ").trim().slice(0, 200);
|
|
232
|
+
return stripCtrl(text.slice(a, b)).replace(/\s+/g, " ").trim().slice(0, 200);
|
|
219
233
|
}
|
|
220
234
|
|
|
221
235
|
// ~/.claude/projects encodes cwd as a dash-flattened path. Every vendor's project root is normalized to
|
|
@@ -1282,7 +1296,7 @@ async function main() {
|
|
|
1282
1296
|
const full = t.text || t.toolErrText || "";
|
|
1283
1297
|
const pos = full.toLowerCase().indexOf(rec.anchor.toLowerCase());
|
|
1284
1298
|
return pos < 0
|
|
1285
|
-
? full.replace(/\s+/g, " ").trim().slice(0, 200)
|
|
1299
|
+
? stripCtrl(full).replace(/\s+/g, " ").trim().slice(0, 200)
|
|
1286
1300
|
: span(full, pos, rec.anchor.length);
|
|
1287
1301
|
};
|
|
1288
1302
|
cellC.push({
|
|
@@ -1618,8 +1632,24 @@ function renderFooter(P, presentVendors) {
|
|
|
1618
1632
|
P();
|
|
1619
1633
|
}
|
|
1620
1634
|
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1635
|
+
// CLI guard: run the autopsy only when executed directly (`node autopsy.mjs`), never on import — so tests
|
|
1636
|
+
// and siblings can import helpers (span/stripCtrl) without triggering a full disk scan. autopsy is always
|
|
1637
|
+
// launched as a subprocess (`spawnSync node autopsy.mjs`), so direct-run behavior is unchanged. Mirrors the
|
|
1638
|
+
// isMain idiom in verify-shadow.mjs / ledger.mjs.
|
|
1639
|
+
const isMain = (() => {
|
|
1640
|
+
try {
|
|
1641
|
+
return (
|
|
1642
|
+
Boolean(process.argv[1]) &&
|
|
1643
|
+
fs.realpathSync(process.argv[1]) === fs.realpathSync(fileURLToPath(import.meta.url))
|
|
1644
|
+
);
|
|
1645
|
+
} catch {
|
|
1646
|
+
return false;
|
|
1647
|
+
}
|
|
1648
|
+
})();
|
|
1649
|
+
if (isMain) {
|
|
1650
|
+
main().catch((e) => {
|
|
1651
|
+
// fail closed and quiet: never crash a user's terminal, never invent output.
|
|
1652
|
+
process.stderr.write(`skalpel autopsy: ${e && e.message ? e.message : e}\n`);
|
|
1653
|
+
process.exit(0);
|
|
1654
|
+
});
|
|
1655
|
+
}
|
package/catches.mjs
CHANGED
|
@@ -215,17 +215,39 @@ function indexResults(entries) {
|
|
|
215
215
|
// result (we never ran the process, so there is no live err). This keeps HARNESS_ERROR vs GENUINE_FAIL
|
|
216
216
|
// identical to the live shadow: a timeout / command-not-found / missing-script is harness noise and is
|
|
217
217
|
// NEVER counted as a lie.
|
|
218
|
+
// A real Claude/Anthropic tool_result block is {tool_use_id, content, is_error} — it carries NO structured
|
|
219
|
+
// exit_code field (only Codex rollouts, normalized in loadCodexEntries, synthesize one). So indexResults
|
|
220
|
+
// hands us exitCode=null for EVERY Claude result, which made the numeric-exit-code branch below dead for
|
|
221
|
+
// Claude: a 127/126 harness failure fell through to `{code:1}` and cried wolf as a GENUINE_FAIL, and a real
|
|
222
|
+
// non-zero exit Claude didn't flag via is_error was silently dropped as PASS. For a Claude proof the code
|
|
223
|
+
// lives in the recorded text — but read it ONLY from the LEADING line. Claude Code prepends the failing
|
|
224
|
+
// process's outcome as the first line of the Bash tool_result ("Exit code 1\n…", verified across real
|
|
225
|
+
// transcripts: 142/142 non-zero-exit results start with "Exit code N"; a PASSING command has no such line).
|
|
226
|
+
// An UNANCHORED full-body scan would misread an unrelated "exit code N" the command itself printed — a
|
|
227
|
+
// grep/cat of source, a pasted CI log like "##[error]Process completed with exit code 123." — and cry wolf
|
|
228
|
+
// on a genuinely-passing proof. Anchoring to the first line reads the real outcome and nothing else.
|
|
229
|
+
function exitCodeFromText(text) {
|
|
230
|
+
const firstLine = String(text || "").split("\n", 1)[0];
|
|
231
|
+
const m = firstLine.match(/^\s*exit(?:ed)?(?:\s+with)?\s+(?:code|status)\s+(\d{1,3})\b/i);
|
|
232
|
+
if (!m) return null;
|
|
233
|
+
const n = Number.parseInt(m[1], 10);
|
|
234
|
+
return Number.isFinite(n) ? n : null;
|
|
235
|
+
}
|
|
218
236
|
function classifyRecorded({ isError, exitCode, text }) {
|
|
219
|
-
if (exitCode === 0) return "PASS";
|
|
220
|
-
if (exitCode == null && !isError) return "PASS"; // no failure signal recorded → treat as pass (never invent)
|
|
221
237
|
const ev = String(text || "");
|
|
238
|
+
// Prefer the structured exit code (Codex); else recover a real code from the text (Claude). This makes
|
|
239
|
+
// the numeric-exit-code branch reachable for Claude, so 127/126 route to HARNESS_ERROR (no cry-wolf) and
|
|
240
|
+
// a textual non-zero exit is caught even when is_error wasn't set.
|
|
241
|
+
const code = typeof exitCode === "number" ? exitCode : exitCodeFromText(ev);
|
|
242
|
+
if (code === 0) return "PASS";
|
|
243
|
+
if (code == null && !isError) return "PASS"; // no failure signal at all → never invent a lie
|
|
222
244
|
let err;
|
|
223
245
|
if (/\btimed?\s*out\b|timeout after|\bSIGKILL\b|\bSIGTERM\b/i.test(ev)) {
|
|
224
246
|
err = { killed: true, signal: "SIGTERM" }; // classifyOutcome → HARNESS_ERROR
|
|
225
|
-
} else if (typeof
|
|
226
|
-
err = { code
|
|
247
|
+
} else if (typeof code === "number" && code !== 0) {
|
|
248
|
+
err = { code }; // real recorded non-zero exit (127/126 → HARNESS_ERROR in classifyOutcome)
|
|
227
249
|
} else {
|
|
228
|
-
err = { code: 1 }; //
|
|
250
|
+
err = { code: 1 }; // is_error flagged but no numeric code recorded → generic non-zero
|
|
229
251
|
}
|
|
230
252
|
return classifyOutcome(err, ev);
|
|
231
253
|
}
|
|
@@ -315,8 +337,16 @@ export function runScan() {
|
|
|
315
337
|
}
|
|
316
338
|
|
|
317
339
|
// ---------- (6) render the honest report ----------
|
|
340
|
+
// claim_text / proof_command / evidence are UNTRUSTED captured strings (agent transcript + test output)
|
|
341
|
+
// rendered straight to a live terminal. Strip ESC + other C0 control chars FIRST (same defense as
|
|
342
|
+
// card.mjs / analytics.mjs `clean`) so a hostile transcript row can't inject ANSI/CSI/OSC escapes
|
|
343
|
+
// (color, cursor moves, title/clipboard spoofing) — then collapse whitespace and cap length.
|
|
318
344
|
function clip(s, n) {
|
|
319
345
|
const t = String(s || "")
|
|
346
|
+
// eslint-disable-next-line no-control-regex
|
|
347
|
+
.replace(/\x1b\[[0-9;?]*[ -/]*[@-~]/g, "") // ANSI CSI sequences
|
|
348
|
+
// eslint-disable-next-line no-control-regex
|
|
349
|
+
.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, " ") // other control chars (incl. ESC/BEL) → space
|
|
320
350
|
.replace(/\s+/g, " ")
|
|
321
351
|
.trim();
|
|
322
352
|
return t.length > n ? t.slice(0, n - 1) + "…" : t;
|
|
@@ -348,7 +378,11 @@ export function renderReport({ scanned, catches }) {
|
|
|
348
378
|
return L.join("\n");
|
|
349
379
|
}
|
|
350
380
|
|
|
351
|
-
|
|
381
|
+
// clip() here too: the headline command phrase is built from the untrusted proof_command and rendered
|
|
382
|
+
// to the terminal, so it needs the same ESC/control-char stripping as the receipt bodies below.
|
|
383
|
+
const cmds = [
|
|
384
|
+
...new Set(catches.map((c) => clip(c.proof_command, 64).split(/\s+/).slice(0, 3).join(" "))),
|
|
385
|
+
];
|
|
352
386
|
const cmdPhrase = cmds.length === 1 ? `\`${cmds[0]}\`` : "its own test/build";
|
|
353
387
|
L.push("");
|
|
354
388
|
L.push(` skalpel scanned ${N} of your recent sessions and found`);
|
package/incremental-ingest.mjs
CHANGED
|
@@ -215,11 +215,23 @@ async function sleep(ms) {
|
|
|
215
215
|
await new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
|
|
216
216
|
}
|
|
217
217
|
|
|
218
|
-
|
|
218
|
+
// fetch() resolves as soon as the response HEADERS arrive; the BODY then streams separately. Clearing the
|
|
219
|
+
// abort timer the instant fetch() resolves (the old `finally`) left every body read — `res.json()` at the
|
|
220
|
+
// call sites — with NO deadline, so a slow or hung response body could stall the drain loop forever. Keep
|
|
221
|
+
// the timer armed and read the FULL body HERE, under the same timeout; hand callers a tiny response-like
|
|
222
|
+
// shim exposing only what they use (status, ok, json()). A stalled body now aborts and rejects instead of
|
|
223
|
+
// hanging. Callers only ever read status/ok and `await response.json().catch(...)`, so the shim suffices.
|
|
224
|
+
export async function request(fetchFn, url, init, timeoutMs) {
|
|
219
225
|
const ctrl = new AbortController();
|
|
220
226
|
const timer = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
221
227
|
try {
|
|
222
|
-
|
|
228
|
+
const res = await fetchFn(url, { ...init, signal: ctrl.signal });
|
|
229
|
+
const body = await res.text(); // fully consume the body while the timeout is still armed
|
|
230
|
+
return {
|
|
231
|
+
status: res.status,
|
|
232
|
+
ok: res.ok,
|
|
233
|
+
json: async () => JSON.parse(body), // async so callers' `.json().catch(...)` catches parse errors
|
|
234
|
+
};
|
|
223
235
|
} finally {
|
|
224
236
|
clearTimeout(timer);
|
|
225
237
|
}
|
package/ledger.mjs
CHANGED
|
@@ -106,8 +106,10 @@ function firstGreater(sorted, x) {
|
|
|
106
106
|
// even in a tampered log its own ts equals t0, which the strict `> t0` search already excludes — so the
|
|
107
107
|
// self-skip was a no-op and omitting it changes nothing (proven by the equivalence fixtures).
|
|
108
108
|
// opts.excludeNullSession — when true, a false row with a null session yields null (the statusline's
|
|
109
|
-
// Finding-3 guard). ledger.mjs
|
|
110
|
-
//
|
|
109
|
+
// Finding-3 guard). Both live views (ledger.mjs AND the statusline) now pass it true: a NULL session
|
|
110
|
+
// carries no real identity, so `(r.session ?? null) === (falseRow.session ?? null)` would read null===null
|
|
111
|
+
// and pair a caught-false row with an UNRELATED null-session PASS of the same proof — a bogus, cross-
|
|
112
|
+
// attributed fix delta. Requiring a real session id to measure a fix is the honest floor.
|
|
111
113
|
export function buildFixDeltas(rows, { excludeNullSession = false } = {}) {
|
|
112
114
|
const index = new Map(); // (session ?? null) → Map<proof_command, ascending ts[]>
|
|
113
115
|
for (const r of rows || []) {
|
|
@@ -180,7 +182,10 @@ export function buildLedger(rows) {
|
|
|
180
182
|
const falseEntries = entries.filter((e) => e.verdict === "FALSE");
|
|
181
183
|
// Attach the MEASURED fix delta where — and only where — a real later same-proof PASS exists. ONE index
|
|
182
184
|
// pass (buildFixDeltas) then a binary-search per false row: O(n log n), not the old O(falseCount × rows).
|
|
183
|
-
|
|
185
|
+
// excludeNullSession: a caught-false row with a NULL session can't be tied to a fix (null carries no
|
|
186
|
+
// identity), so it never cross-attributes another null-session run's PASS — the same guard the statusline
|
|
187
|
+
// sibling applies. Count-only for such rows, never a fabricated delta.
|
|
188
|
+
const fixDeltaFor = buildFixDeltas(rows || [], { excludeNullSession: true });
|
|
184
189
|
for (const e of falseEntries) {
|
|
185
190
|
const d = fixDeltaFor(e._row);
|
|
186
191
|
e.fix_delta_ms = d ? d.ms : null;
|
|
@@ -198,8 +203,16 @@ export function buildLedger(rows) {
|
|
|
198
203
|
}
|
|
199
204
|
|
|
200
205
|
// ---------- (4) render ----------
|
|
206
|
+
// claim_text / proof_command / evidence are UNTRUSTED captured strings (agent transcript + test output)
|
|
207
|
+
// rendered straight to a live terminal. Strip ESC + other C0 control chars FIRST (same defense as
|
|
208
|
+
// card.mjs / analytics.mjs `clean`) so a tampered/hostile log row can't inject ANSI/CSI/OSC escapes
|
|
209
|
+
// (color, cursor moves, title/clipboard spoofing) — then collapse whitespace and cap length.
|
|
201
210
|
function clip(s, n) {
|
|
202
211
|
const t = String(s || "")
|
|
212
|
+
// eslint-disable-next-line no-control-regex
|
|
213
|
+
.replace(/\x1b\[[0-9;?]*[ -/]*[@-~]/g, "") // ANSI CSI sequences
|
|
214
|
+
// eslint-disable-next-line no-control-regex
|
|
215
|
+
.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, " ") // other control chars (incl. ESC/BEL) → space
|
|
203
216
|
.replace(/\s+/g, " ")
|
|
204
217
|
.trim();
|
|
205
218
|
return t.length > n ? t.slice(0, n - 1) + "…" : t;
|
package/login.mjs
CHANGED
|
@@ -4,11 +4,12 @@
|
|
|
4
4
|
// the token → save it. The server holds the client secret and does the code exchange; we only ever
|
|
5
5
|
// see the finished Google id_token. Branded onboarding reproduces the original prosumer `setup.mjs`
|
|
6
6
|
// red block wordmark 1:1. Run: `node skalpel-login.mjs`.
|
|
7
|
-
import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
|
|
7
|
+
import { readFileSync, writeFileSync, mkdirSync, realpathSync } from "node:fs";
|
|
8
8
|
import { homedir } from "node:os";
|
|
9
9
|
import { join, dirname } from "node:path";
|
|
10
10
|
import { randomBytes } from "node:crypto";
|
|
11
11
|
import { spawn } from "node:child_process";
|
|
12
|
+
import { fileURLToPath } from "node:url";
|
|
12
13
|
|
|
13
14
|
const API = process.env.SKALPEL_API || "https://graph.skalpel.ai";
|
|
14
15
|
const CLIENT_ID =
|
|
@@ -51,15 +52,38 @@ function authPath() {
|
|
|
51
52
|
return join(process.env.XDG_CONFIG_HOME || join(homedir(), ".config"), "skalpel", "auth.json");
|
|
52
53
|
}
|
|
53
54
|
|
|
55
|
+
// Build the spawn recipe to open a URL in the default browser, safely, per-platform. Windows is the sharp
|
|
56
|
+
// edge: `start` is a cmd builtin (so it needs cmd.exe), and the OAuth URL is full of `&` param separators.
|
|
57
|
+
// The old path — spawn("start", [url], { shell: true }) — let cmd SPLIT the URL on its unquoted `&`, so
|
|
58
|
+
// sign-in silently broke (URL truncated at the first `&`) and any `&<token>` ran as a separate command
|
|
59
|
+
// (injection surface). Here we spawn cmd.exe with NO shell and windowsVerbatimArguments, quoting the URL
|
|
60
|
+
// ourselves so cmd treats the whole thing as one literal token. `'""'` is start's window title — it must be
|
|
61
|
+
// a LITERAL empty quote-pair (a zero-length JS string would contribute 0 chars to the verbatim command
|
|
62
|
+
// line, so start would consume the URL itself as the title and never open the browser). Any stray
|
|
63
|
+
// double-quote in the URL is stripped so it can't break out of its quotes (URLSearchParams never emits one
|
|
64
|
+
// anyway). macOS/Linux never use a shell, so their URL already rides as a single argv entry — unchanged.
|
|
65
|
+
export function browserOpenCommand(url, platform = process.platform) {
|
|
66
|
+
if (platform === "win32") {
|
|
67
|
+
const safe = String(url).replace(/"/g, "");
|
|
68
|
+
return {
|
|
69
|
+
cmd: "cmd.exe",
|
|
70
|
+
args: ["/c", "start", '""', `"${safe}"`],
|
|
71
|
+
opts: {
|
|
72
|
+
stdio: "ignore",
|
|
73
|
+
detached: true,
|
|
74
|
+
windowsHide: true,
|
|
75
|
+
windowsVerbatimArguments: true,
|
|
76
|
+
},
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
const cmd = platform === "darwin" ? "open" : "xdg-open";
|
|
80
|
+
return { cmd, args: [url], opts: { stdio: "ignore", detached: true } };
|
|
81
|
+
}
|
|
82
|
+
|
|
54
83
|
function openBrowser(url) {
|
|
55
|
-
const cmd =
|
|
56
|
-
process.platform === "darwin" ? "open" : process.platform === "win32" ? "start" : "xdg-open";
|
|
57
84
|
try {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
detached: true,
|
|
61
|
-
shell: process.platform === "win32",
|
|
62
|
-
}).unref();
|
|
85
|
+
const { cmd, args, opts } = browserOpenCommand(url);
|
|
86
|
+
spawn(cmd, args, opts).unref();
|
|
63
87
|
} catch {
|
|
64
88
|
/* manual-paste fallback below */
|
|
65
89
|
}
|
|
@@ -158,7 +182,23 @@ async function main() {
|
|
|
158
182
|
process.exit(0);
|
|
159
183
|
}
|
|
160
184
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
185
|
+
// CLI guard: run the sign-in flow only when executed directly (`node login.mjs`, how skalpel-setup.mjs
|
|
186
|
+
// always spawns it), never on import — so browserOpenCommand can be imported/tested without launching a
|
|
187
|
+
// real OAuth flow. Mirrors the isMain idiom in verify-shadow.mjs / ledger.mjs.
|
|
188
|
+
const isMain = (() => {
|
|
189
|
+
try {
|
|
190
|
+
return (
|
|
191
|
+
Boolean(process.argv[1]) &&
|
|
192
|
+
realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url))
|
|
193
|
+
);
|
|
194
|
+
} catch {
|
|
195
|
+
return false;
|
|
196
|
+
}
|
|
197
|
+
})();
|
|
198
|
+
|
|
199
|
+
if (isMain) {
|
|
200
|
+
main().catch((e) => {
|
|
201
|
+
console.log(` ${R}✗${X} login error: ${D}${e?.message || String(e)}${X}\x1b[?25h`);
|
|
202
|
+
process.exit(1);
|
|
203
|
+
});
|
|
204
|
+
}
|
package/metrics.mjs
CHANGED
|
@@ -3,20 +3,54 @@
|
|
|
3
3
|
// the file (a live-mutated counter file WOULD lose increments under that race). record() never throws
|
|
4
4
|
// and never blocks — telemetry must not be able to break a fail-open hook.
|
|
5
5
|
//
|
|
6
|
+
// BOUNDED: this file was previously append-only with no cap, so it grew without limit — one row every
|
|
7
|
+
// turn, forever, for every user. record() now trims the file back to the last LOG_KEEP_LINES rows once
|
|
8
|
+
// it crosses LOG_MAX_BYTES (same rotate-in-place discipline as verify-shadow.log's appendShadowLog),
|
|
9
|
+
// so on-disk growth is bounded. stats.mjs folds whatever rows remain, so a trim only drops the oldest
|
|
10
|
+
// telemetry — never breaks the reader.
|
|
11
|
+
//
|
|
6
12
|
// Read it back with: node ~/.skalpel/hooks/stats.mjs
|
|
7
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
appendFileSync,
|
|
15
|
+
mkdirSync,
|
|
16
|
+
statSync,
|
|
17
|
+
readFileSync,
|
|
18
|
+
writeFileSync,
|
|
19
|
+
renameSync,
|
|
20
|
+
} from "node:fs";
|
|
8
21
|
import { homedir } from "node:os";
|
|
9
22
|
import { join } from "node:path";
|
|
10
23
|
|
|
11
24
|
const DIR = join(homedir(), ".skalpel");
|
|
12
25
|
export const METRICS_PATH = join(DIR, "metrics.ndjson");
|
|
13
26
|
|
|
27
|
+
// Cap the telemetry file: trim to the last LOG_KEEP_LINES rows once it exceeds LOG_MAX_BYTES.
|
|
28
|
+
const LOG_MAX_BYTES = 1024 * 1024; // 1 MiB
|
|
29
|
+
const LOG_KEEP_LINES = 5000;
|
|
30
|
+
|
|
14
31
|
// outcome ∈ inject | silent | server_error | abort | fetch_error | skipped | budget_spent
|
|
15
32
|
// `abort` == the fetch hit our deadline == a graph cold-start hit (the whole diagnosis).
|
|
16
33
|
// ms == wall-clock of the network call (null for skipped / pre-network exits).
|
|
17
34
|
export function record(event, outcome, ms) {
|
|
18
35
|
try {
|
|
19
36
|
mkdirSync(DIR, { recursive: true });
|
|
37
|
+
// Bound the file BEFORE appending: once it crosses the cap, keep only the newest rows. The trim is
|
|
38
|
+
// atomic (tmp + rename) and wrapped so a rotation failure still falls through to the append. A
|
|
39
|
+
// per-pid tmp keeps concurrent trims from colliding on one temp path.
|
|
40
|
+
try {
|
|
41
|
+
if (statSync(METRICS_PATH).size > LOG_MAX_BYTES) {
|
|
42
|
+
const kept = readFileSync(METRICS_PATH, "utf8")
|
|
43
|
+
.trim()
|
|
44
|
+
.split("\n")
|
|
45
|
+
.filter(Boolean)
|
|
46
|
+
.slice(-LOG_KEEP_LINES);
|
|
47
|
+
const tmp = `${METRICS_PATH}.${process.pid}.tmp`;
|
|
48
|
+
writeFileSync(tmp, kept.join("\n") + "\n");
|
|
49
|
+
renameSync(tmp, METRICS_PATH);
|
|
50
|
+
}
|
|
51
|
+
} catch {
|
|
52
|
+
/* no file yet, or a trim race — still append below */
|
|
53
|
+
}
|
|
20
54
|
appendFileSync(
|
|
21
55
|
METRICS_PATH,
|
|
22
56
|
JSON.stringify({
|
package/package.json
CHANGED
package/skalpel-hook.mjs
CHANGED
|
@@ -8,9 +8,22 @@
|
|
|
8
8
|
// {"hooks":{"UserPromptSubmit":[{"type":"command","command":"node /path/to/skalpel-hook.mjs"}]}}
|
|
9
9
|
// Config: SKALPEL_API (default https://graph.skalpel.ai). Identity comes from the Google sign-in
|
|
10
10
|
// saved by `skalpel-prosumer login` (~/.skalpel/auth.json); SKALPEL_USER overrides for dev.
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
readFileSync,
|
|
13
|
+
writeFileSync,
|
|
14
|
+
renameSync,
|
|
15
|
+
appendFileSync,
|
|
16
|
+
mkdirSync,
|
|
17
|
+
openSync,
|
|
18
|
+
writeSync,
|
|
19
|
+
closeSync,
|
|
20
|
+
statSync,
|
|
21
|
+
rmSync,
|
|
22
|
+
realpathSync,
|
|
23
|
+
} from "node:fs";
|
|
12
24
|
import { homedir } from "node:os";
|
|
13
25
|
import { join } from "node:path";
|
|
26
|
+
import { fileURLToPath } from "node:url";
|
|
14
27
|
import { identity } from "./auth.mjs";
|
|
15
28
|
import { record } from "./metrics.mjs";
|
|
16
29
|
import { recordInsight, cleanText, verifySpawnArmed } from "./insights.mjs";
|
|
@@ -230,8 +243,8 @@ function writeSteer(label, type, wt, id) {
|
|
|
230
243
|
// suppress[key] : true — "fewer like this" -> stop firing this steer type/trap for me
|
|
231
244
|
// verbose[key] : true — "tell me more…" -> expand this trap's steer when it fires
|
|
232
245
|
// score[type] : {up,down} — implicit + explicit usefulness, for future gating
|
|
233
|
-
const PREFS_PATH = join(homedir(), ".skalpel", "prefs.json");
|
|
234
|
-
function readPrefs() {
|
|
246
|
+
export const PREFS_PATH = join(homedir(), ".skalpel", "prefs.json");
|
|
247
|
+
export function readPrefs() {
|
|
235
248
|
const base = { muted_session: false, suppress: {}, verbose: {}, score: {}, stats: {} };
|
|
236
249
|
try {
|
|
237
250
|
return { ...base, ...JSON.parse(readFileSync(PREFS_PATH, "utf8")) };
|
|
@@ -264,10 +277,11 @@ function ignoredByAdherence(prefs, type) {
|
|
|
264
277
|
const st = prefs.stats && prefs.stats[type];
|
|
265
278
|
return !!(st && st.shown >= 6 && st.acted / st.shown < 0.2);
|
|
266
279
|
}
|
|
267
|
-
function writePrefs(p) {
|
|
280
|
+
export function writePrefs(p) {
|
|
268
281
|
try {
|
|
269
282
|
mkdirSync(join(homedir(), ".skalpel"), { recursive: true });
|
|
270
|
-
|
|
283
|
+
// Per-pid tmp so two concurrent writers can't tear each other's temp file before the rename commit.
|
|
284
|
+
const tmp = `${PREFS_PATH}.${process.pid}.tmp`;
|
|
271
285
|
writeFileSync(tmp, JSON.stringify(p));
|
|
272
286
|
renameSync(tmp, PREFS_PATH);
|
|
273
287
|
} catch {
|
|
@@ -275,6 +289,89 @@ function writePrefs(p) {
|
|
|
275
289
|
}
|
|
276
290
|
}
|
|
277
291
|
|
|
292
|
+
// ---- LOST-UPDATE FIX (prefs.json) --------------------------------------------------------------------
|
|
293
|
+
// Every hook used to readPrefs() at the TOP of the turn, then await identity()+/retrieve (seconds), then
|
|
294
|
+
// write that SAME in-memory object back at the end. During that long window a concurrent hook could mute
|
|
295
|
+
// / suppress / record feedback — and this hook's end-of-turn write, carrying its stale snapshot, silently
|
|
296
|
+
// CLOBBERED it (a user's "mute" or "fewer like this" just vanished). The fix is a race-safe read-modify-
|
|
297
|
+
// write: serialize writers on a tiny cross-process lock, RE-READ prefs fresh immediately before writing,
|
|
298
|
+
// apply only the intended field mutation to THAT fresh copy, then atomic-write. So a concurrent update
|
|
299
|
+
// that landed during our await window is merged in, never overwritten. FAIL-OPEN throughout: prefs are a
|
|
300
|
+
// bonus — a missed lock or a bad mutator degrades to a best-effort write and NEVER breaks the hook.
|
|
301
|
+
const PREFS_LOCK_PATH = `${PREFS_PATH}.lock`;
|
|
302
|
+
const PREFS_LOCK_STALE_MS = 10_000; // a hook that died holding the lock is reclaimed after this
|
|
303
|
+
const PREFS_LOCK_SPIN_MS = 250; // bounded wait; real contention (a double-fire) clears in sub-ms
|
|
304
|
+
|
|
305
|
+
// Synchronous sleep with no busy-spin (Atomics.wait on a throwaway buffer). Keeps the RMW critical
|
|
306
|
+
// section short without pulling the hot path into async just for a lock.
|
|
307
|
+
function sleepMs(ms) {
|
|
308
|
+
try {
|
|
309
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
310
|
+
} catch {
|
|
311
|
+
/* SharedArrayBuffer unavailable — fall through; the spin just polls a touch faster */
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// Acquire the prefs lock via an atomic exclusive create (O_EXCL "wx"): exactly one racer wins, the rest
|
|
316
|
+
// back off and retry within a bounded budget; a stale lock (dead holder) is reclaimed by mtime. Returns
|
|
317
|
+
// true if held (caller must releasePrefsLock), false if it gave up (caller still does a best-effort write).
|
|
318
|
+
function acquirePrefsLock() {
|
|
319
|
+
const deadline = Date.now() + PREFS_LOCK_SPIN_MS;
|
|
320
|
+
for (;;) {
|
|
321
|
+
try {
|
|
322
|
+
const fd = openSync(PREFS_LOCK_PATH, "wx");
|
|
323
|
+
try {
|
|
324
|
+
writeSync(fd, String(process.pid));
|
|
325
|
+
} finally {
|
|
326
|
+
closeSync(fd);
|
|
327
|
+
}
|
|
328
|
+
return true;
|
|
329
|
+
} catch (e) {
|
|
330
|
+
if (e && e.code === "EEXIST") {
|
|
331
|
+
try {
|
|
332
|
+
const st = statSync(PREFS_LOCK_PATH);
|
|
333
|
+
if (Date.now() - st.mtimeMs >= PREFS_LOCK_STALE_MS) {
|
|
334
|
+
rmSync(PREFS_LOCK_PATH, { force: true }); // stale holder — reclaim, then retry the create
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
} catch {
|
|
338
|
+
continue; // lock vanished (just released) — retry immediately
|
|
339
|
+
}
|
|
340
|
+
} else {
|
|
341
|
+
return false; // unexpected error → don't spin; caller falls back to an unlocked write
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
if (Date.now() >= deadline) return false;
|
|
345
|
+
sleepMs(2);
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
function releasePrefsLock() {
|
|
349
|
+
try {
|
|
350
|
+
rmSync(PREFS_LOCK_PATH, { force: true });
|
|
351
|
+
} catch {
|
|
352
|
+
/* best-effort — a stale lock self-expires after PREFS_LOCK_STALE_MS */
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// Race-safe prefs mutation. Under the lock: re-read fresh, apply `mutate(fresh)`, atomic-write. `mutate`
|
|
357
|
+
// receives the on-disk state so its change is merged onto whatever concurrent hooks have persisted — the
|
|
358
|
+
// caller must NOT rely on a snapshot read earlier in the turn. Returns the written object. Never throws.
|
|
359
|
+
export function updatePrefs(mutate) {
|
|
360
|
+
const locked = acquirePrefsLock();
|
|
361
|
+
try {
|
|
362
|
+
const fresh = readPrefs();
|
|
363
|
+
try {
|
|
364
|
+
mutate(fresh);
|
|
365
|
+
} catch {
|
|
366
|
+
/* a mutator bug must never break the hook */
|
|
367
|
+
}
|
|
368
|
+
writePrefs(fresh);
|
|
369
|
+
return fresh;
|
|
370
|
+
} finally {
|
|
371
|
+
if (locked) releasePrefsLock();
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
|
|
278
375
|
// Parse a control PHRASE aimed at skalpel out of the user's prompt — the explicit feedback channel,
|
|
279
376
|
// captured natively through the prompt (no button UI needed; this IS the harness wrapping). Returns
|
|
280
377
|
// {action} or null. Deliberately narrow so normal prose never trips it: the word "skalpel", or a
|
|
@@ -353,51 +450,64 @@ function logSessionEvent(ev) {
|
|
|
353
450
|
function applyControl(action, prefs) {
|
|
354
451
|
const last = readSteer();
|
|
355
452
|
const key = last.type ? (last.wt ? `${last.type}:${last.wt}` : last.type) : null;
|
|
356
|
-
|
|
453
|
+
// A mutation is applied to a prefs object `P`; the score bump reads+writes P (not the outer snapshot).
|
|
454
|
+
const bump = (P, dir) => {
|
|
357
455
|
if (!last.type) return;
|
|
358
|
-
const sc =
|
|
456
|
+
const sc = P.score[last.type] || { up: 0, down: 0 };
|
|
359
457
|
sc[dir] += 1;
|
|
360
|
-
|
|
458
|
+
P.score[last.type] = sc;
|
|
459
|
+
};
|
|
460
|
+
// Apply the SAME field mutation to (a) the in-memory `prefs` so THIS turn's downstream gating sees it,
|
|
461
|
+
// and (b) the on-disk prefs via the race-safe read-modify-write so a concurrent hook's update isn't
|
|
462
|
+
// clobbered. The disk copy is re-read fresh inside updatePrefs, so each object is mutated once from its
|
|
463
|
+
// own base (set-once flags stay idempotent; a score bump lands exactly once on disk).
|
|
464
|
+
const persist = (m) => {
|
|
465
|
+
m(prefs);
|
|
466
|
+
updatePrefs(m);
|
|
361
467
|
};
|
|
362
468
|
if (action === "mute") {
|
|
363
|
-
|
|
364
|
-
|
|
469
|
+
persist((P) => {
|
|
470
|
+
P.muted_session = true;
|
|
471
|
+
});
|
|
365
472
|
return (
|
|
366
473
|
"[skalpel — the user asked to MUTE skalpel for this session. Acknowledge in ONE short line " +
|
|
367
474
|
"(“🔬 skalpel · muted for this session — I'll stay quiet”) and surface NO steers the rest of this session.]"
|
|
368
475
|
);
|
|
369
476
|
}
|
|
370
477
|
if (action === "unmute") {
|
|
371
|
-
|
|
372
|
-
|
|
478
|
+
persist((P) => {
|
|
479
|
+
P.muted_session = false;
|
|
480
|
+
});
|
|
373
481
|
return "[skalpel — the user re-enabled skalpel. Acknowledge in one short line and continue.]";
|
|
374
482
|
}
|
|
375
483
|
if (action === "fewer") {
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
484
|
+
persist((P) => {
|
|
485
|
+
if (key) P.suppress[key] = true;
|
|
486
|
+
bump(P, "down");
|
|
487
|
+
});
|
|
379
488
|
return (
|
|
380
489
|
`[skalpel — the user wants FEWER steers like the last one${last.wt ? ` (${last.wt})` : ""}. ` +
|
|
381
490
|
"Acknowledge in one line (“🔬 skalpel · got it — easing off those”); skalpel will stop firing that trap for them.]"
|
|
382
491
|
);
|
|
383
492
|
}
|
|
384
493
|
if (action === "more") {
|
|
385
|
-
|
|
386
|
-
|
|
494
|
+
persist((P) => {
|
|
495
|
+
if (key) P.verbose[key] = true;
|
|
496
|
+
});
|
|
387
497
|
return (
|
|
388
498
|
`[skalpel — the user wants MORE detail whenever they hit this${last.wt ? ` ${last.wt}` : ""} trap. ` +
|
|
389
499
|
"Acknowledge briefly, and from now on EXPAND that steer: name the shape, the count, and the way out.]"
|
|
390
500
|
);
|
|
391
501
|
}
|
|
392
502
|
if (action === "up") {
|
|
393
|
-
bump("up");
|
|
394
|
-
writePrefs(prefs);
|
|
503
|
+
persist((P) => bump(P, "up"));
|
|
395
504
|
return null; // positive signal — just record it; no need to make the model announce anything
|
|
396
505
|
}
|
|
397
506
|
if (action === "down") {
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
507
|
+
persist((P) => {
|
|
508
|
+
if (key) P.suppress[key] = true;
|
|
509
|
+
bump(P, "down");
|
|
510
|
+
});
|
|
401
511
|
return "[skalpel — the user found the last steer unhelpful. Acknowledge briefly; skalpel will ease off that one.]";
|
|
402
512
|
}
|
|
403
513
|
return null;
|
|
@@ -438,8 +548,13 @@ function pushTraj(cls) {
|
|
|
438
548
|
}
|
|
439
549
|
|
|
440
550
|
// local debug trail — proves the hook RAN (in any session), and what it decided. Fail-safe.
|
|
551
|
+
// OPT-IN ONLY. The old guard (`=== "0"` → skip) meant the log was written on EVERY hook invocation for
|
|
552
|
+
// EVERY user unless they explicitly set SKALPEL_DEBUG=0 — an unbounded ~/.skalpel/hook.log growing one
|
|
553
|
+
// entry per turn, forever, that nobody asked for. dbg now writes ONLY when SKALPEL_DEBUG is explicitly
|
|
554
|
+
// enabled (any value other than the off sentinels), so the default install produces no hook.log at all.
|
|
441
555
|
function dbg(msg) {
|
|
442
|
-
|
|
556
|
+
const flag = process.env.SKALPEL_DEBUG;
|
|
557
|
+
if (!flag || flag === "0" || flag === "false" || flag === "off") return; // default OFF — no log leak
|
|
443
558
|
try {
|
|
444
559
|
const p = join(homedir(), ".skalpel");
|
|
445
560
|
mkdirSync(p, { recursive: true });
|
|
@@ -690,8 +805,14 @@ async function main() {
|
|
|
690
805
|
// off (a break-out this turn). Accretes per steer-type; the gate above reads it next time. This is
|
|
691
806
|
// categorical revealed-preference data, not a number shown to the user.
|
|
692
807
|
const paid = brokeOut;
|
|
693
|
-
|
|
694
|
-
|
|
808
|
+
// Race-safe write: this is the end-of-turn write that follows the long identity()+/retrieve await,
|
|
809
|
+
// so it must NOT persist the stale `prefs` snapshot read at the top of the turn (that clobbered a
|
|
810
|
+
// concurrent hook's mute/suppress). Re-read fresh inside updatePrefs and apply the adherence
|
|
811
|
+
// increment to THAT, merging in whatever landed during our await window.
|
|
812
|
+
let adh = null;
|
|
813
|
+
updatePrefs((fresh) => {
|
|
814
|
+
adh = recordAdherence(fresh, prevSteer, steerWt, paid);
|
|
815
|
+
});
|
|
695
816
|
// ADHERENCE INSIGHT ROW (data, not display — no `display` field): the same judgment, exported
|
|
696
817
|
// for the TUI's rail, joined to the prior steer by its minted id. followed = the n+1 credit
|
|
697
818
|
// landed; ignored = you stayed on the very work-type the steer pushed you off AND no credit;
|
|
@@ -788,6 +909,22 @@ async function main() {
|
|
|
788
909
|
}
|
|
789
910
|
}
|
|
790
911
|
|
|
791
|
-
main()
|
|
792
|
-
|
|
793
|
-
|
|
912
|
+
// Run main() only when invoked as the hook (node skalpel-hook.mjs), never on import. Importing the module
|
|
913
|
+
// (the prefs concurrency test) must not read fd 0 or emit hook output. Robust main detection mirrors
|
|
914
|
+
// verify-shadow.mjs / skalpel-setup.mjs: compare REAL paths, not a hand-built file:// URL (a home dir with
|
|
915
|
+
// a space breaks `new URL("file://"+path)`). Any error → treat as not-main (safe: a test import stays inert).
|
|
916
|
+
const isMain = (() => {
|
|
917
|
+
try {
|
|
918
|
+
return (
|
|
919
|
+
Boolean(process.argv[1]) &&
|
|
920
|
+
realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url))
|
|
921
|
+
);
|
|
922
|
+
} catch {
|
|
923
|
+
return false;
|
|
924
|
+
}
|
|
925
|
+
})();
|
|
926
|
+
if (isMain) {
|
|
927
|
+
main()
|
|
928
|
+
.then(() => process.exit(0))
|
|
929
|
+
.catch(() => process.exit(0));
|
|
930
|
+
}
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
// verify-races.test.mjs — regression tests for the core-engine race/growth fixes (fix/core-engine-races):
|
|
2
|
+
// • classifyOutcome no longer misclassifies a GENUINE fail whose OWN output contains a harness phrase
|
|
3
|
+
// • acquireLock is single-flight under a CONCURRENT burst (O_EXCL, not statSync→writeFileSync TOCTOU)
|
|
4
|
+
// • updatePrefs is a race-safe read-modify-write (a concurrent mute/suppress is merged, never clobbered)
|
|
5
|
+
// • metrics.ndjson is bounded (rotate-in-place), not unbounded
|
|
6
|
+
// HOME-isolated (import "./_test-home.mjs" FIRST) so nothing here reads/clobbers a real ~/.skalpel.
|
|
7
|
+
import "./_test-home.mjs";
|
|
8
|
+
import { test } from "node:test";
|
|
9
|
+
import assert from "node:assert/strict";
|
|
10
|
+
import { spawn } from "node:child_process";
|
|
11
|
+
import {
|
|
12
|
+
mkdtempSync,
|
|
13
|
+
mkdirSync,
|
|
14
|
+
writeFileSync,
|
|
15
|
+
readFileSync,
|
|
16
|
+
rmSync,
|
|
17
|
+
statSync,
|
|
18
|
+
utimesSync,
|
|
19
|
+
} from "node:fs";
|
|
20
|
+
import { tmpdir, homedir } from "node:os";
|
|
21
|
+
import { join, dirname } from "node:path";
|
|
22
|
+
import { fileURLToPath } from "node:url";
|
|
23
|
+
import { classifyOutcome, acquireLock } from "./verify-shadow.mjs";
|
|
24
|
+
import { updatePrefs, readPrefs, writePrefs } from "./skalpel-hook.mjs";
|
|
25
|
+
import { record, METRICS_PATH } from "./metrics.mjs";
|
|
26
|
+
|
|
27
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
28
|
+
|
|
29
|
+
// ---- Bug 4: classifyOutcome — a genuine fail's OWN output containing a harness phrase is NOT quarantined
|
|
30
|
+
test("classifyOutcome — a GENUINE fail that MENTIONS 'command not found' stays GENUINE_FAIL", () => {
|
|
31
|
+
const genuine =
|
|
32
|
+
`FAIL src/cli.test.js\n` +
|
|
33
|
+
` ● prints a helpful error for an unknown subcommand\n` +
|
|
34
|
+
` Expected substring: "command not found"\n` +
|
|
35
|
+
` Received string: "unknown option"\n` +
|
|
36
|
+
` 1 failed`;
|
|
37
|
+
assert.equal(classifyOutcome({ code: 1 }, genuine), "GENUINE_FAIL");
|
|
38
|
+
const genuine2 = `AssertionError: expected error to include "no such file or directory"\n 1 failing`;
|
|
39
|
+
assert.equal(classifyOutcome({ code: 1 }, genuine2), "GENUINE_FAIL");
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test("classifyOutcome — a REAL harness break is STILL HARNESS_ERROR (true positives preserved)", () => {
|
|
43
|
+
assert.equal(classifyOutcome({ code: 1 }, "bash: pytest: command not found"), "HARNESS_ERROR");
|
|
44
|
+
assert.equal(classifyOutcome({ code: 1 }, "sh: 1: eslint: not found"), "HARNESS_ERROR");
|
|
45
|
+
assert.equal(classifyOutcome({ code: 1 }, 'npm error Missing script: "test"'), "HARNESS_ERROR");
|
|
46
|
+
assert.equal(
|
|
47
|
+
classifyOutcome({ code: 1 }, "Error: ENOENT: no such file or directory, open 'x'"),
|
|
48
|
+
"HARNESS_ERROR",
|
|
49
|
+
);
|
|
50
|
+
assert.equal(
|
|
51
|
+
classifyOutcome({ code: 1 }, "browserType.launch: Executable doesn't exist at /ms-playwright"),
|
|
52
|
+
"HARNESS_ERROR",
|
|
53
|
+
);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
// ---- Bug 3: acquireLock — exactly ONE holder under a concurrent burst (the TOCTOU is closed) ----
|
|
57
|
+
test("acquireLock — a concurrent burst yields EXACTLY ONE holder (no double-acquire)", () => {
|
|
58
|
+
const home = mkdtempSync(join(tmpdir(), "skalpel-lockrace-"));
|
|
59
|
+
mkdirSync(join(home, ".skalpel"), { recursive: true });
|
|
60
|
+
const worker = join(home, "w.mjs");
|
|
61
|
+
writeFileSync(
|
|
62
|
+
worker,
|
|
63
|
+
`import { acquireLock } from ${JSON.stringify(join(HERE, "verify-shadow.mjs"))};\n` +
|
|
64
|
+
`const START = Number(process.env.START_MS);\n` +
|
|
65
|
+
`while (Date.now() < START) {}\n` +
|
|
66
|
+
`process.stdout.write(acquireLock() ? "ACQUIRED" : "denied");\n`,
|
|
67
|
+
);
|
|
68
|
+
const N = 40;
|
|
69
|
+
const START_MS = Date.now() + 400;
|
|
70
|
+
const kids = [];
|
|
71
|
+
for (let i = 0; i < N; i++) {
|
|
72
|
+
kids.push(
|
|
73
|
+
spawn("node", [worker], {
|
|
74
|
+
env: { ...process.env, HOME: home, USERPROFILE: home, START_MS: String(START_MS) },
|
|
75
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
76
|
+
}),
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
const outs = kids.map(
|
|
80
|
+
(c) =>
|
|
81
|
+
new Promise((res) => {
|
|
82
|
+
let o = "";
|
|
83
|
+
c.stdout.on("data", (d) => (o += d));
|
|
84
|
+
c.on("exit", () => res(o));
|
|
85
|
+
}),
|
|
86
|
+
);
|
|
87
|
+
return Promise.all(outs).then((results) => {
|
|
88
|
+
const acquired = results.filter((r) => r.includes("ACQUIRED")).length;
|
|
89
|
+
rmSync(home, { recursive: true, force: true });
|
|
90
|
+
assert.equal(acquired, 1, `expected exactly 1 holder, got ${acquired}`);
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("acquireLock — respects a fresh lock, reclaims a stale one", () => {
|
|
95
|
+
const dir = join(homedir(), ".skalpel");
|
|
96
|
+
rmSync(dir, { recursive: true, force: true });
|
|
97
|
+
mkdirSync(dir, { recursive: true });
|
|
98
|
+
const LOCK = join(dir, "verify-shadow.lock");
|
|
99
|
+
assert.equal(acquireLock(), true, "first acquire creates the lock");
|
|
100
|
+
assert.equal(acquireLock(), false, "a fresh lock is respected (no re-acquire)");
|
|
101
|
+
// Backdate the lock past the 90s stale window → a reclaim is allowed.
|
|
102
|
+
const old = Date.now() / 1000 - 200;
|
|
103
|
+
utimesSync(LOCK, old, old);
|
|
104
|
+
assert.equal(acquireLock(), true, "a stale lock is reclaimed");
|
|
105
|
+
rmSync(dir, { recursive: true, force: true });
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
// ---- Bug 1: updatePrefs — re-read + merge, so a concurrent update is never clobbered ----
|
|
109
|
+
test("updatePrefs — merges a concurrent mute instead of clobbering it with a stale snapshot", () => {
|
|
110
|
+
rmSync(join(homedir(), ".skalpel"), { recursive: true, force: true });
|
|
111
|
+
mkdirSync(join(homedir(), ".skalpel"), { recursive: true });
|
|
112
|
+
writePrefs({ muted_session: false, suppress: {}, verbose: {}, score: {}, stats: {} });
|
|
113
|
+
// The hook read a stale snapshot at the top of its turn:
|
|
114
|
+
const stale = readPrefs();
|
|
115
|
+
assert.equal(stale.muted_session, false);
|
|
116
|
+
// …then a CONCURRENT hook muted the session (persisted to disk) during the await window:
|
|
117
|
+
writePrefs({ ...stale, muted_session: true });
|
|
118
|
+
// …now the first hook's end-of-turn write goes through updatePrefs, which re-reads fresh + merges:
|
|
119
|
+
updatePrefs((f) => {
|
|
120
|
+
f.suppress["SINK:refine"] = true;
|
|
121
|
+
});
|
|
122
|
+
const after = readPrefs();
|
|
123
|
+
assert.equal(after.muted_session, true, "the concurrent mute survived (not clobbered)");
|
|
124
|
+
assert.equal(after.suppress["SINK:refine"], true, "our own suppress landed too");
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test("updatePrefs — 50 concurrent distinct-key writers lose NONE", () => {
|
|
128
|
+
const home = mkdtempSync(join(tmpdir(), "skalpel-keys-"));
|
|
129
|
+
mkdirSync(join(home, ".skalpel"), { recursive: true });
|
|
130
|
+
writeFileSync(
|
|
131
|
+
join(home, ".skalpel", "prefs.json"),
|
|
132
|
+
JSON.stringify({ muted_session: false, suppress: {}, verbose: {}, score: {}, stats: {} }),
|
|
133
|
+
);
|
|
134
|
+
const worker = join(home, "kw.mjs");
|
|
135
|
+
writeFileSync(
|
|
136
|
+
worker,
|
|
137
|
+
`import { updatePrefs } from ${JSON.stringify(join(HERE, "skalpel-hook.mjs"))};\n` +
|
|
138
|
+
`const i = Number(process.env.KEY_I);\n` +
|
|
139
|
+
`const START = Number(process.env.START_MS);\n` +
|
|
140
|
+
`while (Date.now() < START) {}\n` +
|
|
141
|
+
`updatePrefs((P) => { P.suppress["k" + i] = true; });\n`,
|
|
142
|
+
);
|
|
143
|
+
const N = 50;
|
|
144
|
+
const START_MS = Date.now() + 500;
|
|
145
|
+
const kids = [];
|
|
146
|
+
for (let i = 0; i < N; i++) {
|
|
147
|
+
kids.push(
|
|
148
|
+
new Promise((res) => {
|
|
149
|
+
const c = spawn("node", [worker], {
|
|
150
|
+
env: {
|
|
151
|
+
...process.env,
|
|
152
|
+
HOME: home,
|
|
153
|
+
USERPROFILE: home,
|
|
154
|
+
KEY_I: String(i),
|
|
155
|
+
START_MS: String(START_MS),
|
|
156
|
+
},
|
|
157
|
+
stdio: "ignore",
|
|
158
|
+
});
|
|
159
|
+
c.on("exit", res);
|
|
160
|
+
}),
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
return Promise.all(kids).then(() => {
|
|
164
|
+
const suppress =
|
|
165
|
+
JSON.parse(readFileSync(join(home, ".skalpel", "prefs.json"), "utf8")).suppress || {};
|
|
166
|
+
let survived = 0;
|
|
167
|
+
for (let i = 0; i < N; i++) if (suppress["k" + i] === true) survived++;
|
|
168
|
+
rmSync(home, { recursive: true, force: true });
|
|
169
|
+
assert.equal(survived, N, `expected all ${N} keys, got ${survived}`);
|
|
170
|
+
});
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
// ---- Bug 5: metrics.ndjson is bounded (rotate-in-place), not unbounded ----
|
|
174
|
+
test("record — metrics.ndjson stays bounded under a heavy write load", () => {
|
|
175
|
+
rmSync(METRICS_PATH, { force: true });
|
|
176
|
+
for (let i = 0; i < 30000; i++) record("prompt", i % 2 ? "inject" : "silent", 1800);
|
|
177
|
+
const size = statSync(METRICS_PATH).size;
|
|
178
|
+
assert.ok(size <= 1024 * 1024, `metrics.ndjson should be <= 1 MiB, was ${size} bytes`);
|
|
179
|
+
// …and it's still valid NDJSON the reader can fold (last line parses).
|
|
180
|
+
const lines = readFileSync(METRICS_PATH, "utf8").trim().split("\n").filter(Boolean);
|
|
181
|
+
assert.doesNotThrow(() => JSON.parse(lines[lines.length - 1]));
|
|
182
|
+
});
|
package/verify-shadow.mjs
CHANGED
|
@@ -54,6 +54,9 @@ import {
|
|
|
54
54
|
renameSync,
|
|
55
55
|
statSync,
|
|
56
56
|
rmSync,
|
|
57
|
+
openSync,
|
|
58
|
+
writeSync,
|
|
59
|
+
closeSync,
|
|
57
60
|
} from "node:fs";
|
|
58
61
|
import { homedir } from "node:os";
|
|
59
62
|
import { join } from "node:path";
|
|
@@ -575,10 +578,25 @@ export function classifyOutcome(err, evidence) {
|
|
|
575
578
|
// tool binary/runtime that simply isn't installed (a Playwright browser not downloaded — "Executable
|
|
576
579
|
// doesn't exist" / "download new browsers"). Surfacing any of these would cry wolf on setup state, so
|
|
577
580
|
// they are HARNESS_ERROR, never counted as a lie.
|
|
581
|
+
//
|
|
582
|
+
// ANCHORED, not a bare substring scan. The old test was `/…command not found…/.test(evidence)` over the
|
|
583
|
+
// WHOLE tail, so a GENUINE test failure whose OWN output merely CONTAINS one of these phrases (a test
|
|
584
|
+
// asserting on a shell error message, e.g. Expected substring: "command not found") was misread as
|
|
585
|
+
// HARNESS_ERROR and its real catch silently DROPPED — the money case, and an easy way for an adversarial
|
|
586
|
+
// agent to dodge us by printing the phrase. Each signature now anchors to the LINE SHAPE a broken proof
|
|
587
|
+
// actually emits (a tool/errno prefix + line boundaries), so an embedded/quoted mention no longer trips.
|
|
588
|
+
const ev = String(evidence || "");
|
|
578
589
|
if (
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
)
|
|
590
|
+
// shell "command not found" / "not found": the phrase ENDS a line whose leading `<cmd>: ` is the
|
|
591
|
+
// shell's own prefix — `bash: pytest: command not found`, `sh: 1: eslint: not found`.
|
|
592
|
+
/(^|\n)[^\n]*: (?:command not found|not found)[ \t]*(\n|$)/i.test(ev) ||
|
|
593
|
+
// errno string in its real `<path>: No such file or directory` shape, or Node's ENOENT code token.
|
|
594
|
+
/(^|\n)[^\n]*: no such file or directory[ \t]*(\n|$)/i.test(ev) ||
|
|
595
|
+
/\bENOENT\b/.test(ev) ||
|
|
596
|
+
// npm / pnpm / yarn missing-script — their exact `Missing script:` wording (with the colon).
|
|
597
|
+
/\bmissing script:/i.test(ev) ||
|
|
598
|
+
// Playwright browser binary not installed — a setup-state error, never the agent's code failing.
|
|
599
|
+
/executable doesn'?t exist|download new browsers/i.test(ev)
|
|
582
600
|
)
|
|
583
601
|
return "HARNESS_ERROR";
|
|
584
602
|
return "GENUINE_FAIL"; // a real non-zero exit from the test/build/lint itself
|
|
@@ -739,19 +757,51 @@ function markVerified(sig, cursorPath = LAST_PATH) {
|
|
|
739
757
|
}
|
|
740
758
|
|
|
741
759
|
// Single-flight: only one shadow re-run at a time (a burst of prompts must not spawn parallel test runs).
|
|
742
|
-
|
|
760
|
+
// The old body did `statSync(LOCK_PATH)` then an UNCONDITIONAL `writeFileSync` — a TOCTOU hole: when no
|
|
761
|
+
// lock file existed (the common case), N concurrent shadows all saw "no lock" and all wrote it, so ALL
|
|
762
|
+
// "acquired" and ran parallel test re-runs. The atomic primitive is an EXCLUSIVE create (O_EXCL via the
|
|
763
|
+
// "wx" flag): exactly one racer's create can win; every other gets EEXIST and backs off. Stale-lock
|
|
764
|
+
// takeover (a shadow that died holding it) is preserved, but done through the same exclusive create so
|
|
765
|
+
// two reclaimers can't both win. FAIL-OPEN: any unexpected error resolves to "not acquired" (no re-run).
|
|
766
|
+
// Exported for the concurrency test (proves exactly-one-holder under a burst). Never throws.
|
|
767
|
+
export function acquireLock() {
|
|
743
768
|
try {
|
|
744
769
|
mkdirSync(DIR, { recursive: true });
|
|
770
|
+
} catch {
|
|
771
|
+
return false;
|
|
772
|
+
}
|
|
773
|
+
// Atomic exclusive create — the win/lose decision is the create itself, not a prior stat.
|
|
774
|
+
try {
|
|
775
|
+
const fd = openSync(LOCK_PATH, "wx");
|
|
745
776
|
try {
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
777
|
+
writeSync(fd, String(process.pid));
|
|
778
|
+
} finally {
|
|
779
|
+
closeSync(fd);
|
|
780
|
+
}
|
|
781
|
+
return true;
|
|
782
|
+
} catch (e) {
|
|
783
|
+
if (!e || e.code !== "EEXIST") return false; // not a contention error → fail-open, no lock
|
|
784
|
+
}
|
|
785
|
+
// A lock already exists. Take it over ONLY if it is STALE (holder died/hung > LOCK_STALE_MS ago); an
|
|
786
|
+
// actively-held fresh lock is respected (return false). Reclaim through the SAME exclusive create so
|
|
787
|
+
// only one reclaimer wins even if several fire at once.
|
|
788
|
+
try {
|
|
789
|
+
const st = statSync(LOCK_PATH);
|
|
790
|
+
if (Date.now() - st.mtimeMs < LOCK_STALE_MS) return false; // fresh — respect the active holder
|
|
791
|
+
} catch {
|
|
792
|
+
/* vanished between EEXIST and stat (just released) — fall through and try to claim it */
|
|
793
|
+
}
|
|
794
|
+
try {
|
|
795
|
+
rmSync(LOCK_PATH, { force: true });
|
|
796
|
+
const fd = openSync(LOCK_PATH, "wx");
|
|
797
|
+
try {
|
|
798
|
+
writeSync(fd, String(process.pid));
|
|
799
|
+
} finally {
|
|
800
|
+
closeSync(fd);
|
|
750
801
|
}
|
|
751
|
-
writeFileSync(LOCK_PATH, String(process.pid));
|
|
752
802
|
return true;
|
|
753
803
|
} catch {
|
|
754
|
-
return false;
|
|
804
|
+
return false; // another reclaimer re-created it first — they hold it
|
|
755
805
|
}
|
|
756
806
|
}
|
|
757
807
|
function releaseLock() {
|