skalpel 4.0.43 → 4.0.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/metrics.mjs +35 -1
- package/package.json +1 -1
- package/skalpel-hook.mjs +165 -28
- package/verify-races.test.mjs +182 -0
- package/verify-shadow.mjs +60 -10
package/metrics.mjs
CHANGED
|
@@ -3,20 +3,54 @@
|
|
|
3
3
|
// the file (a live-mutated counter file WOULD lose increments under that race). record() never throws
|
|
4
4
|
// and never blocks — telemetry must not be able to break a fail-open hook.
|
|
5
5
|
//
|
|
6
|
+
// BOUNDED: this file was previously append-only with no cap, so it grew without limit — one row every
|
|
7
|
+
// turn, forever, for every user. record() now trims the file back to the last LOG_KEEP_LINES rows once
|
|
8
|
+
// it crosses LOG_MAX_BYTES (same rotate-in-place discipline as verify-shadow.log's appendShadowLog),
|
|
9
|
+
// so on-disk growth is bounded. stats.mjs folds whatever rows remain, so a trim only drops the oldest
|
|
10
|
+
// telemetry — never breaks the reader.
|
|
11
|
+
//
|
|
6
12
|
// Read it back with: node ~/.skalpel/hooks/stats.mjs
|
|
7
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
appendFileSync,
|
|
15
|
+
mkdirSync,
|
|
16
|
+
statSync,
|
|
17
|
+
readFileSync,
|
|
18
|
+
writeFileSync,
|
|
19
|
+
renameSync,
|
|
20
|
+
} from "node:fs";
|
|
8
21
|
import { homedir } from "node:os";
|
|
9
22
|
import { join } from "node:path";
|
|
10
23
|
|
|
11
24
|
const DIR = join(homedir(), ".skalpel");
|
|
12
25
|
export const METRICS_PATH = join(DIR, "metrics.ndjson");
|
|
13
26
|
|
|
27
|
+
// Cap the telemetry file: trim to the last LOG_KEEP_LINES rows once it exceeds LOG_MAX_BYTES.
|
|
28
|
+
const LOG_MAX_BYTES = 1024 * 1024; // 1 MiB
|
|
29
|
+
const LOG_KEEP_LINES = 5000;
|
|
30
|
+
|
|
14
31
|
// outcome ∈ inject | silent | server_error | abort | fetch_error | skipped | budget_spent
|
|
15
32
|
// `abort` == the fetch hit our deadline == a graph cold-start hit (the whole diagnosis).
|
|
16
33
|
// ms == wall-clock of the network call (null for skipped / pre-network exits).
|
|
17
34
|
export function record(event, outcome, ms) {
|
|
18
35
|
try {
|
|
19
36
|
mkdirSync(DIR, { recursive: true });
|
|
37
|
+
// Bound the file BEFORE appending: once it crosses the cap, keep only the newest rows. The trim is
|
|
38
|
+
// atomic (tmp + rename) and wrapped so a rotation failure still falls through to the append. A
|
|
39
|
+
// per-pid tmp keeps concurrent trims from colliding on one temp path.
|
|
40
|
+
try {
|
|
41
|
+
if (statSync(METRICS_PATH).size > LOG_MAX_BYTES) {
|
|
42
|
+
const kept = readFileSync(METRICS_PATH, "utf8")
|
|
43
|
+
.trim()
|
|
44
|
+
.split("\n")
|
|
45
|
+
.filter(Boolean)
|
|
46
|
+
.slice(-LOG_KEEP_LINES);
|
|
47
|
+
const tmp = `${METRICS_PATH}.${process.pid}.tmp`;
|
|
48
|
+
writeFileSync(tmp, kept.join("\n") + "\n");
|
|
49
|
+
renameSync(tmp, METRICS_PATH);
|
|
50
|
+
}
|
|
51
|
+
} catch {
|
|
52
|
+
/* no file yet, or a trim race — still append below */
|
|
53
|
+
}
|
|
20
54
|
appendFileSync(
|
|
21
55
|
METRICS_PATH,
|
|
22
56
|
JSON.stringify({
|
package/package.json
CHANGED
package/skalpel-hook.mjs
CHANGED
|
@@ -8,9 +8,22 @@
|
|
|
8
8
|
// {"hooks":{"UserPromptSubmit":[{"type":"command","command":"node /path/to/skalpel-hook.mjs"}]}}
|
|
9
9
|
// Config: SKALPEL_API (default https://graph.skalpel.ai). Identity comes from the Google sign-in
|
|
10
10
|
// saved by `skalpel-prosumer login` (~/.skalpel/auth.json); SKALPEL_USER overrides for dev.
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
readFileSync,
|
|
13
|
+
writeFileSync,
|
|
14
|
+
renameSync,
|
|
15
|
+
appendFileSync,
|
|
16
|
+
mkdirSync,
|
|
17
|
+
openSync,
|
|
18
|
+
writeSync,
|
|
19
|
+
closeSync,
|
|
20
|
+
statSync,
|
|
21
|
+
rmSync,
|
|
22
|
+
realpathSync,
|
|
23
|
+
} from "node:fs";
|
|
12
24
|
import { homedir } from "node:os";
|
|
13
25
|
import { join } from "node:path";
|
|
26
|
+
import { fileURLToPath } from "node:url";
|
|
14
27
|
import { identity } from "./auth.mjs";
|
|
15
28
|
import { record } from "./metrics.mjs";
|
|
16
29
|
import { recordInsight, cleanText, verifySpawnArmed } from "./insights.mjs";
|
|
@@ -230,8 +243,8 @@ function writeSteer(label, type, wt, id) {
|
|
|
230
243
|
// suppress[key] : true — "fewer like this" -> stop firing this steer type/trap for me
|
|
231
244
|
// verbose[key] : true — "tell me more…" -> expand this trap's steer when it fires
|
|
232
245
|
// score[type] : {up,down} — implicit + explicit usefulness, for future gating
|
|
233
|
-
const PREFS_PATH = join(homedir(), ".skalpel", "prefs.json");
|
|
234
|
-
function readPrefs() {
|
|
246
|
+
export const PREFS_PATH = join(homedir(), ".skalpel", "prefs.json");
|
|
247
|
+
export function readPrefs() {
|
|
235
248
|
const base = { muted_session: false, suppress: {}, verbose: {}, score: {}, stats: {} };
|
|
236
249
|
try {
|
|
237
250
|
return { ...base, ...JSON.parse(readFileSync(PREFS_PATH, "utf8")) };
|
|
@@ -264,10 +277,11 @@ function ignoredByAdherence(prefs, type) {
|
|
|
264
277
|
const st = prefs.stats && prefs.stats[type];
|
|
265
278
|
return !!(st && st.shown >= 6 && st.acted / st.shown < 0.2);
|
|
266
279
|
}
|
|
267
|
-
function writePrefs(p) {
|
|
280
|
+
export function writePrefs(p) {
|
|
268
281
|
try {
|
|
269
282
|
mkdirSync(join(homedir(), ".skalpel"), { recursive: true });
|
|
270
|
-
|
|
283
|
+
// Per-pid tmp so two concurrent writers can't tear each other's temp file before the rename commit.
|
|
284
|
+
const tmp = `${PREFS_PATH}.${process.pid}.tmp`;
|
|
271
285
|
writeFileSync(tmp, JSON.stringify(p));
|
|
272
286
|
renameSync(tmp, PREFS_PATH);
|
|
273
287
|
} catch {
|
|
@@ -275,6 +289,89 @@ function writePrefs(p) {
|
|
|
275
289
|
}
|
|
276
290
|
}
|
|
277
291
|
|
|
292
|
+
// ---- LOST-UPDATE FIX (prefs.json) --------------------------------------------------------------------
|
|
293
|
+
// Every hook used to readPrefs() at the TOP of the turn, then await identity()+/retrieve (seconds), then
|
|
294
|
+
// write that SAME in-memory object back at the end. During that long window a concurrent hook could mute
|
|
295
|
+
// / suppress / record feedback — and this hook's end-of-turn write, carrying its stale snapshot, silently
|
|
296
|
+
// CLOBBERED it (a user's "mute" or "fewer like this" just vanished). The fix is a race-safe read-modify-
|
|
297
|
+
// write: serialize writers on a tiny cross-process lock, RE-READ prefs fresh immediately before writing,
|
|
298
|
+
// apply only the intended field mutation to THAT fresh copy, then atomic-write. So a concurrent update
|
|
299
|
+
// that landed during our await window is merged in, never overwritten. FAIL-OPEN throughout: prefs are a
|
|
300
|
+
// bonus — a missed lock or a bad mutator degrades to a best-effort write and NEVER breaks the hook.
|
|
301
|
+
const PREFS_LOCK_PATH = `${PREFS_PATH}.lock`;
|
|
302
|
+
const PREFS_LOCK_STALE_MS = 10_000; // a hook that died holding the lock is reclaimed after this
|
|
303
|
+
const PREFS_LOCK_SPIN_MS = 250; // bounded wait; real contention (a double-fire) clears in sub-ms
|
|
304
|
+
|
|
305
|
+
// Synchronous sleep with no busy-spin (Atomics.wait on a throwaway buffer). Keeps the RMW critical
|
|
306
|
+
// section short without pulling the hot path into async just for a lock.
|
|
307
|
+
function sleepMs(ms) {
|
|
308
|
+
try {
|
|
309
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
310
|
+
} catch {
|
|
311
|
+
/* SharedArrayBuffer unavailable — fall through; the spin just polls a touch faster */
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// Acquire the prefs lock via an atomic exclusive create (O_EXCL "wx"): exactly one racer wins, the rest
|
|
316
|
+
// back off and retry within a bounded budget; a stale lock (dead holder) is reclaimed by mtime. Returns
|
|
317
|
+
// true if held (caller must releasePrefsLock), false if it gave up (caller still does a best-effort write).
|
|
318
|
+
function acquirePrefsLock() {
|
|
319
|
+
const deadline = Date.now() + PREFS_LOCK_SPIN_MS;
|
|
320
|
+
for (;;) {
|
|
321
|
+
try {
|
|
322
|
+
const fd = openSync(PREFS_LOCK_PATH, "wx");
|
|
323
|
+
try {
|
|
324
|
+
writeSync(fd, String(process.pid));
|
|
325
|
+
} finally {
|
|
326
|
+
closeSync(fd);
|
|
327
|
+
}
|
|
328
|
+
return true;
|
|
329
|
+
} catch (e) {
|
|
330
|
+
if (e && e.code === "EEXIST") {
|
|
331
|
+
try {
|
|
332
|
+
const st = statSync(PREFS_LOCK_PATH);
|
|
333
|
+
if (Date.now() - st.mtimeMs >= PREFS_LOCK_STALE_MS) {
|
|
334
|
+
rmSync(PREFS_LOCK_PATH, { force: true }); // stale holder — reclaim, then retry the create
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
} catch {
|
|
338
|
+
continue; // lock vanished (just released) — retry immediately
|
|
339
|
+
}
|
|
340
|
+
} else {
|
|
341
|
+
return false; // unexpected error → don't spin; caller falls back to an unlocked write
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
if (Date.now() >= deadline) return false;
|
|
345
|
+
sleepMs(2);
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
function releasePrefsLock() {
|
|
349
|
+
try {
|
|
350
|
+
rmSync(PREFS_LOCK_PATH, { force: true });
|
|
351
|
+
} catch {
|
|
352
|
+
/* best-effort — a stale lock self-expires after PREFS_LOCK_STALE_MS */
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// Race-safe prefs mutation. Under the lock: re-read fresh, apply `mutate(fresh)`, atomic-write. `mutate`
|
|
357
|
+
// receives the on-disk state so its change is merged onto whatever concurrent hooks have persisted — the
|
|
358
|
+
// caller must NOT rely on a snapshot read earlier in the turn. Returns the written object. Never throws.
|
|
359
|
+
export function updatePrefs(mutate) {
|
|
360
|
+
const locked = acquirePrefsLock();
|
|
361
|
+
try {
|
|
362
|
+
const fresh = readPrefs();
|
|
363
|
+
try {
|
|
364
|
+
mutate(fresh);
|
|
365
|
+
} catch {
|
|
366
|
+
/* a mutator bug must never break the hook */
|
|
367
|
+
}
|
|
368
|
+
writePrefs(fresh);
|
|
369
|
+
return fresh;
|
|
370
|
+
} finally {
|
|
371
|
+
if (locked) releasePrefsLock();
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
|
|
278
375
|
// Parse a control PHRASE aimed at skalpel out of the user's prompt — the explicit feedback channel,
|
|
279
376
|
// captured natively through the prompt (no button UI needed; this IS the harness wrapping). Returns
|
|
280
377
|
// {action} or null. Deliberately narrow so normal prose never trips it: the word "skalpel", or a
|
|
@@ -353,51 +450,64 @@ function logSessionEvent(ev) {
|
|
|
353
450
|
function applyControl(action, prefs) {
|
|
354
451
|
const last = readSteer();
|
|
355
452
|
const key = last.type ? (last.wt ? `${last.type}:${last.wt}` : last.type) : null;
|
|
356
|
-
|
|
453
|
+
// A mutation is applied to a prefs object `P`; the score bump reads+writes P (not the outer snapshot).
|
|
454
|
+
const bump = (P, dir) => {
|
|
357
455
|
if (!last.type) return;
|
|
358
|
-
const sc =
|
|
456
|
+
const sc = P.score[last.type] || { up: 0, down: 0 };
|
|
359
457
|
sc[dir] += 1;
|
|
360
|
-
|
|
458
|
+
P.score[last.type] = sc;
|
|
459
|
+
};
|
|
460
|
+
// Apply the SAME field mutation to (a) the in-memory `prefs` so THIS turn's downstream gating sees it,
|
|
461
|
+
// and (b) the on-disk prefs via the race-safe read-modify-write so a concurrent hook's update isn't
|
|
462
|
+
// clobbered. The disk copy is re-read fresh inside updatePrefs, so each object is mutated once from its
|
|
463
|
+
// own base (set-once flags stay idempotent; a score bump lands exactly once on disk).
|
|
464
|
+
const persist = (m) => {
|
|
465
|
+
m(prefs);
|
|
466
|
+
updatePrefs(m);
|
|
361
467
|
};
|
|
362
468
|
if (action === "mute") {
|
|
363
|
-
|
|
364
|
-
|
|
469
|
+
persist((P) => {
|
|
470
|
+
P.muted_session = true;
|
|
471
|
+
});
|
|
365
472
|
return (
|
|
366
473
|
"[skalpel — the user asked to MUTE skalpel for this session. Acknowledge in ONE short line " +
|
|
367
474
|
"(“🔬 skalpel · muted for this session — I'll stay quiet”) and surface NO steers the rest of this session.]"
|
|
368
475
|
);
|
|
369
476
|
}
|
|
370
477
|
if (action === "unmute") {
|
|
371
|
-
|
|
372
|
-
|
|
478
|
+
persist((P) => {
|
|
479
|
+
P.muted_session = false;
|
|
480
|
+
});
|
|
373
481
|
return "[skalpel — the user re-enabled skalpel. Acknowledge in one short line and continue.]";
|
|
374
482
|
}
|
|
375
483
|
if (action === "fewer") {
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
484
|
+
persist((P) => {
|
|
485
|
+
if (key) P.suppress[key] = true;
|
|
486
|
+
bump(P, "down");
|
|
487
|
+
});
|
|
379
488
|
return (
|
|
380
489
|
`[skalpel — the user wants FEWER steers like the last one${last.wt ? ` (${last.wt})` : ""}. ` +
|
|
381
490
|
"Acknowledge in one line (“🔬 skalpel · got it — easing off those”); skalpel will stop firing that trap for them.]"
|
|
382
491
|
);
|
|
383
492
|
}
|
|
384
493
|
if (action === "more") {
|
|
385
|
-
|
|
386
|
-
|
|
494
|
+
persist((P) => {
|
|
495
|
+
if (key) P.verbose[key] = true;
|
|
496
|
+
});
|
|
387
497
|
return (
|
|
388
498
|
`[skalpel — the user wants MORE detail whenever they hit this${last.wt ? ` ${last.wt}` : ""} trap. ` +
|
|
389
499
|
"Acknowledge briefly, and from now on EXPAND that steer: name the shape, the count, and the way out.]"
|
|
390
500
|
);
|
|
391
501
|
}
|
|
392
502
|
if (action === "up") {
|
|
393
|
-
bump("up");
|
|
394
|
-
writePrefs(prefs);
|
|
503
|
+
persist((P) => bump(P, "up"));
|
|
395
504
|
return null; // positive signal — just record it; no need to make the model announce anything
|
|
396
505
|
}
|
|
397
506
|
if (action === "down") {
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
507
|
+
persist((P) => {
|
|
508
|
+
if (key) P.suppress[key] = true;
|
|
509
|
+
bump(P, "down");
|
|
510
|
+
});
|
|
401
511
|
return "[skalpel — the user found the last steer unhelpful. Acknowledge briefly; skalpel will ease off that one.]";
|
|
402
512
|
}
|
|
403
513
|
return null;
|
|
@@ -438,8 +548,13 @@ function pushTraj(cls) {
|
|
|
438
548
|
}
|
|
439
549
|
|
|
440
550
|
// local debug trail — proves the hook RAN (in any session), and what it decided. Fail-safe.
|
|
551
|
+
// OPT-IN ONLY. The old guard (`=== "0"` → skip) meant the log was written on EVERY hook invocation for
|
|
552
|
+
// EVERY user unless they explicitly set SKALPEL_DEBUG=0 — an unbounded ~/.skalpel/hook.log growing one
|
|
553
|
+
// entry per turn, forever, that nobody asked for. dbg now writes ONLY when SKALPEL_DEBUG is explicitly
|
|
554
|
+
// enabled (any value other than the off sentinels), so the default install produces no hook.log at all.
|
|
441
555
|
function dbg(msg) {
|
|
442
|
-
|
|
556
|
+
const flag = process.env.SKALPEL_DEBUG;
|
|
557
|
+
if (!flag || flag === "0" || flag === "false" || flag === "off") return; // default OFF — no log leak
|
|
443
558
|
try {
|
|
444
559
|
const p = join(homedir(), ".skalpel");
|
|
445
560
|
mkdirSync(p, { recursive: true });
|
|
@@ -690,8 +805,14 @@ async function main() {
|
|
|
690
805
|
// off (a break-out this turn). Accretes per steer-type; the gate above reads it next time. This is
|
|
691
806
|
// categorical revealed-preference data, not a number shown to the user.
|
|
692
807
|
const paid = brokeOut;
|
|
693
|
-
|
|
694
|
-
|
|
808
|
+
// Race-safe write: this is the end-of-turn write that follows the long identity()+/retrieve await,
|
|
809
|
+
// so it must NOT persist the stale `prefs` snapshot read at the top of the turn (that clobbered a
|
|
810
|
+
// concurrent hook's mute/suppress). Re-read fresh inside updatePrefs and apply the adherence
|
|
811
|
+
// increment to THAT, merging in whatever landed during our await window.
|
|
812
|
+
let adh = null;
|
|
813
|
+
updatePrefs((fresh) => {
|
|
814
|
+
adh = recordAdherence(fresh, prevSteer, steerWt, paid);
|
|
815
|
+
});
|
|
695
816
|
// ADHERENCE INSIGHT ROW (data, not display — no `display` field): the same judgment, exported
|
|
696
817
|
// for the TUI's rail, joined to the prior steer by its minted id. followed = the n+1 credit
|
|
697
818
|
// landed; ignored = you stayed on the very work-type the steer pushed you off AND no credit;
|
|
@@ -788,6 +909,22 @@ async function main() {
|
|
|
788
909
|
}
|
|
789
910
|
}
|
|
790
911
|
|
|
791
|
-
main()
|
|
792
|
-
|
|
793
|
-
|
|
912
|
+
// Run main() only when invoked as the hook (node skalpel-hook.mjs), never on import. Importing the module
|
|
913
|
+
// (the prefs concurrency test) must not read fd 0 or emit hook output. Robust main detection mirrors
|
|
914
|
+
// verify-shadow.mjs / skalpel-setup.mjs: compare REAL paths, not a hand-built file:// URL (a home dir with
|
|
915
|
+
// a space breaks `new URL("file://"+path)`). Any error → treat as not-main (safe: a test import stays inert).
|
|
916
|
+
const isMain = (() => {
|
|
917
|
+
try {
|
|
918
|
+
return (
|
|
919
|
+
Boolean(process.argv[1]) &&
|
|
920
|
+
realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url))
|
|
921
|
+
);
|
|
922
|
+
} catch {
|
|
923
|
+
return false;
|
|
924
|
+
}
|
|
925
|
+
})();
|
|
926
|
+
if (isMain) {
|
|
927
|
+
main()
|
|
928
|
+
.then(() => process.exit(0))
|
|
929
|
+
.catch(() => process.exit(0));
|
|
930
|
+
}
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
// verify-races.test.mjs — regression tests for the core-engine race/growth fixes (fix/core-engine-races):
|
|
2
|
+
// • classifyOutcome no longer misclassifies a GENUINE fail whose OWN output contains a harness phrase
|
|
3
|
+
// • acquireLock is single-flight under a CONCURRENT burst (O_EXCL, not statSync→writeFileSync TOCTOU)
|
|
4
|
+
// • updatePrefs is a race-safe read-modify-write (a concurrent mute/suppress is merged, never clobbered)
|
|
5
|
+
// • metrics.ndjson is bounded (rotate-in-place), not unbounded
|
|
6
|
+
// HOME-isolated (import "./_test-home.mjs" FIRST) so nothing here reads/clobbers a real ~/.skalpel.
|
|
7
|
+
import "./_test-home.mjs";
|
|
8
|
+
import { test } from "node:test";
|
|
9
|
+
import assert from "node:assert/strict";
|
|
10
|
+
import { spawn } from "node:child_process";
|
|
11
|
+
import {
|
|
12
|
+
mkdtempSync,
|
|
13
|
+
mkdirSync,
|
|
14
|
+
writeFileSync,
|
|
15
|
+
readFileSync,
|
|
16
|
+
rmSync,
|
|
17
|
+
statSync,
|
|
18
|
+
utimesSync,
|
|
19
|
+
} from "node:fs";
|
|
20
|
+
import { tmpdir, homedir } from "node:os";
|
|
21
|
+
import { join, dirname } from "node:path";
|
|
22
|
+
import { fileURLToPath } from "node:url";
|
|
23
|
+
import { classifyOutcome, acquireLock } from "./verify-shadow.mjs";
|
|
24
|
+
import { updatePrefs, readPrefs, writePrefs } from "./skalpel-hook.mjs";
|
|
25
|
+
import { record, METRICS_PATH } from "./metrics.mjs";
|
|
26
|
+
|
|
27
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
28
|
+
|
|
29
|
+
// ---- Bug 4: classifyOutcome — a genuine fail's OWN output containing a harness phrase is NOT quarantined
|
|
30
|
+
test("classifyOutcome — a GENUINE fail that MENTIONS 'command not found' stays GENUINE_FAIL", () => {
|
|
31
|
+
const genuine =
|
|
32
|
+
`FAIL src/cli.test.js\n` +
|
|
33
|
+
` ● prints a helpful error for an unknown subcommand\n` +
|
|
34
|
+
` Expected substring: "command not found"\n` +
|
|
35
|
+
` Received string: "unknown option"\n` +
|
|
36
|
+
` 1 failed`;
|
|
37
|
+
assert.equal(classifyOutcome({ code: 1 }, genuine), "GENUINE_FAIL");
|
|
38
|
+
const genuine2 = `AssertionError: expected error to include "no such file or directory"\n 1 failing`;
|
|
39
|
+
assert.equal(classifyOutcome({ code: 1 }, genuine2), "GENUINE_FAIL");
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test("classifyOutcome — a REAL harness break is STILL HARNESS_ERROR (true positives preserved)", () => {
|
|
43
|
+
assert.equal(classifyOutcome({ code: 1 }, "bash: pytest: command not found"), "HARNESS_ERROR");
|
|
44
|
+
assert.equal(classifyOutcome({ code: 1 }, "sh: 1: eslint: not found"), "HARNESS_ERROR");
|
|
45
|
+
assert.equal(classifyOutcome({ code: 1 }, 'npm error Missing script: "test"'), "HARNESS_ERROR");
|
|
46
|
+
assert.equal(
|
|
47
|
+
classifyOutcome({ code: 1 }, "Error: ENOENT: no such file or directory, open 'x'"),
|
|
48
|
+
"HARNESS_ERROR",
|
|
49
|
+
);
|
|
50
|
+
assert.equal(
|
|
51
|
+
classifyOutcome({ code: 1 }, "browserType.launch: Executable doesn't exist at /ms-playwright"),
|
|
52
|
+
"HARNESS_ERROR",
|
|
53
|
+
);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
// ---- Bug 3: acquireLock — exactly ONE holder under a concurrent burst (the TOCTOU is closed) ----
|
|
57
|
+
test("acquireLock — a concurrent burst yields EXACTLY ONE holder (no double-acquire)", () => {
|
|
58
|
+
const home = mkdtempSync(join(tmpdir(), "skalpel-lockrace-"));
|
|
59
|
+
mkdirSync(join(home, ".skalpel"), { recursive: true });
|
|
60
|
+
const worker = join(home, "w.mjs");
|
|
61
|
+
writeFileSync(
|
|
62
|
+
worker,
|
|
63
|
+
`import { acquireLock } from ${JSON.stringify(join(HERE, "verify-shadow.mjs"))};\n` +
|
|
64
|
+
`const START = Number(process.env.START_MS);\n` +
|
|
65
|
+
`while (Date.now() < START) {}\n` +
|
|
66
|
+
`process.stdout.write(acquireLock() ? "ACQUIRED" : "denied");\n`,
|
|
67
|
+
);
|
|
68
|
+
const N = 40;
|
|
69
|
+
const START_MS = Date.now() + 400;
|
|
70
|
+
const kids = [];
|
|
71
|
+
for (let i = 0; i < N; i++) {
|
|
72
|
+
kids.push(
|
|
73
|
+
spawn("node", [worker], {
|
|
74
|
+
env: { ...process.env, HOME: home, USERPROFILE: home, START_MS: String(START_MS) },
|
|
75
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
76
|
+
}),
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
const outs = kids.map(
|
|
80
|
+
(c) =>
|
|
81
|
+
new Promise((res) => {
|
|
82
|
+
let o = "";
|
|
83
|
+
c.stdout.on("data", (d) => (o += d));
|
|
84
|
+
c.on("exit", () => res(o));
|
|
85
|
+
}),
|
|
86
|
+
);
|
|
87
|
+
return Promise.all(outs).then((results) => {
|
|
88
|
+
const acquired = results.filter((r) => r.includes("ACQUIRED")).length;
|
|
89
|
+
rmSync(home, { recursive: true, force: true });
|
|
90
|
+
assert.equal(acquired, 1, `expected exactly 1 holder, got ${acquired}`);
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("acquireLock — respects a fresh lock, reclaims a stale one", () => {
|
|
95
|
+
const dir = join(homedir(), ".skalpel");
|
|
96
|
+
rmSync(dir, { recursive: true, force: true });
|
|
97
|
+
mkdirSync(dir, { recursive: true });
|
|
98
|
+
const LOCK = join(dir, "verify-shadow.lock");
|
|
99
|
+
assert.equal(acquireLock(), true, "first acquire creates the lock");
|
|
100
|
+
assert.equal(acquireLock(), false, "a fresh lock is respected (no re-acquire)");
|
|
101
|
+
// Backdate the lock past the 90s stale window → a reclaim is allowed.
|
|
102
|
+
const old = Date.now() / 1000 - 200;
|
|
103
|
+
utimesSync(LOCK, old, old);
|
|
104
|
+
assert.equal(acquireLock(), true, "a stale lock is reclaimed");
|
|
105
|
+
rmSync(dir, { recursive: true, force: true });
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
// ---- Bug 1: updatePrefs — re-read + merge, so a concurrent update is never clobbered ----
|
|
109
|
+
test("updatePrefs — merges a concurrent mute instead of clobbering it with a stale snapshot", () => {
|
|
110
|
+
rmSync(join(homedir(), ".skalpel"), { recursive: true, force: true });
|
|
111
|
+
mkdirSync(join(homedir(), ".skalpel"), { recursive: true });
|
|
112
|
+
writePrefs({ muted_session: false, suppress: {}, verbose: {}, score: {}, stats: {} });
|
|
113
|
+
// The hook read a stale snapshot at the top of its turn:
|
|
114
|
+
const stale = readPrefs();
|
|
115
|
+
assert.equal(stale.muted_session, false);
|
|
116
|
+
// …then a CONCURRENT hook muted the session (persisted to disk) during the await window:
|
|
117
|
+
writePrefs({ ...stale, muted_session: true });
|
|
118
|
+
// …now the first hook's end-of-turn write goes through updatePrefs, which re-reads fresh + merges:
|
|
119
|
+
updatePrefs((f) => {
|
|
120
|
+
f.suppress["SINK:refine"] = true;
|
|
121
|
+
});
|
|
122
|
+
const after = readPrefs();
|
|
123
|
+
assert.equal(after.muted_session, true, "the concurrent mute survived (not clobbered)");
|
|
124
|
+
assert.equal(after.suppress["SINK:refine"], true, "our own suppress landed too");
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test("updatePrefs — 50 concurrent distinct-key writers lose NONE", () => {
|
|
128
|
+
const home = mkdtempSync(join(tmpdir(), "skalpel-keys-"));
|
|
129
|
+
mkdirSync(join(home, ".skalpel"), { recursive: true });
|
|
130
|
+
writeFileSync(
|
|
131
|
+
join(home, ".skalpel", "prefs.json"),
|
|
132
|
+
JSON.stringify({ muted_session: false, suppress: {}, verbose: {}, score: {}, stats: {} }),
|
|
133
|
+
);
|
|
134
|
+
const worker = join(home, "kw.mjs");
|
|
135
|
+
writeFileSync(
|
|
136
|
+
worker,
|
|
137
|
+
`import { updatePrefs } from ${JSON.stringify(join(HERE, "skalpel-hook.mjs"))};\n` +
|
|
138
|
+
`const i = Number(process.env.KEY_I);\n` +
|
|
139
|
+
`const START = Number(process.env.START_MS);\n` +
|
|
140
|
+
`while (Date.now() < START) {}\n` +
|
|
141
|
+
`updatePrefs((P) => { P.suppress["k" + i] = true; });\n`,
|
|
142
|
+
);
|
|
143
|
+
const N = 50;
|
|
144
|
+
const START_MS = Date.now() + 500;
|
|
145
|
+
const kids = [];
|
|
146
|
+
for (let i = 0; i < N; i++) {
|
|
147
|
+
kids.push(
|
|
148
|
+
new Promise((res) => {
|
|
149
|
+
const c = spawn("node", [worker], {
|
|
150
|
+
env: {
|
|
151
|
+
...process.env,
|
|
152
|
+
HOME: home,
|
|
153
|
+
USERPROFILE: home,
|
|
154
|
+
KEY_I: String(i),
|
|
155
|
+
START_MS: String(START_MS),
|
|
156
|
+
},
|
|
157
|
+
stdio: "ignore",
|
|
158
|
+
});
|
|
159
|
+
c.on("exit", res);
|
|
160
|
+
}),
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
return Promise.all(kids).then(() => {
|
|
164
|
+
const suppress =
|
|
165
|
+
JSON.parse(readFileSync(join(home, ".skalpel", "prefs.json"), "utf8")).suppress || {};
|
|
166
|
+
let survived = 0;
|
|
167
|
+
for (let i = 0; i < N; i++) if (suppress["k" + i] === true) survived++;
|
|
168
|
+
rmSync(home, { recursive: true, force: true });
|
|
169
|
+
assert.equal(survived, N, `expected all ${N} keys, got ${survived}`);
|
|
170
|
+
});
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
// ---- Bug 5: metrics.ndjson is bounded (rotate-in-place), not unbounded ----
|
|
174
|
+
test("record — metrics.ndjson stays bounded under a heavy write load", () => {
|
|
175
|
+
rmSync(METRICS_PATH, { force: true });
|
|
176
|
+
for (let i = 0; i < 30000; i++) record("prompt", i % 2 ? "inject" : "silent", 1800);
|
|
177
|
+
const size = statSync(METRICS_PATH).size;
|
|
178
|
+
assert.ok(size <= 1024 * 1024, `metrics.ndjson should be <= 1 MiB, was ${size} bytes`);
|
|
179
|
+
// …and it's still valid NDJSON the reader can fold (last line parses).
|
|
180
|
+
const lines = readFileSync(METRICS_PATH, "utf8").trim().split("\n").filter(Boolean);
|
|
181
|
+
assert.doesNotThrow(() => JSON.parse(lines[lines.length - 1]));
|
|
182
|
+
});
|
package/verify-shadow.mjs
CHANGED
|
@@ -54,6 +54,9 @@ import {
|
|
|
54
54
|
renameSync,
|
|
55
55
|
statSync,
|
|
56
56
|
rmSync,
|
|
57
|
+
openSync,
|
|
58
|
+
writeSync,
|
|
59
|
+
closeSync,
|
|
57
60
|
} from "node:fs";
|
|
58
61
|
import { homedir } from "node:os";
|
|
59
62
|
import { join } from "node:path";
|
|
@@ -575,10 +578,25 @@ export function classifyOutcome(err, evidence) {
|
|
|
575
578
|
// tool binary/runtime that simply isn't installed (a Playwright browser not downloaded — "Executable
|
|
576
579
|
// doesn't exist" / "download new browsers"). Surfacing any of these would cry wolf on setup state, so
|
|
577
580
|
// they are HARNESS_ERROR, never counted as a lie.
|
|
581
|
+
//
|
|
582
|
+
// ANCHORED, not a bare substring scan. The old test was `/…command not found…/.test(evidence)` over the
|
|
583
|
+
// WHOLE tail, so a GENUINE test failure whose OWN output merely CONTAINS one of these phrases (a test
|
|
584
|
+
// asserting on a shell error message, e.g. Expected substring: "command not found") was misread as
|
|
585
|
+
// HARNESS_ERROR and its real catch silently DROPPED — the money case, and an easy way for an adversarial
|
|
586
|
+
// agent to dodge us by printing the phrase. Each signature now anchors to the LINE SHAPE a broken proof
|
|
587
|
+
// actually emits (a tool/errno prefix + line boundaries), so an embedded/quoted mention no longer trips.
|
|
588
|
+
const ev = String(evidence || "");
|
|
578
589
|
if (
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
)
|
|
590
|
+
// shell "command not found" / "not found": the phrase ENDS a line whose leading `<cmd>: ` is the
|
|
591
|
+
// shell's own prefix — `bash: pytest: command not found`, `sh: 1: eslint: not found`.
|
|
592
|
+
/(^|\n)[^\n]*: (?:command not found|not found)[ \t]*(\n|$)/i.test(ev) ||
|
|
593
|
+
// errno string in its real `<path>: No such file or directory` shape, or Node's ENOENT code token.
|
|
594
|
+
/(^|\n)[^\n]*: no such file or directory[ \t]*(\n|$)/i.test(ev) ||
|
|
595
|
+
/\bENOENT\b/.test(ev) ||
|
|
596
|
+
// npm / pnpm / yarn missing-script — their exact `Missing script:` wording (with the colon).
|
|
597
|
+
/\bmissing script:/i.test(ev) ||
|
|
598
|
+
// Playwright browser binary not installed — a setup-state error, never the agent's code failing.
|
|
599
|
+
/executable doesn'?t exist|download new browsers/i.test(ev)
|
|
582
600
|
)
|
|
583
601
|
return "HARNESS_ERROR";
|
|
584
602
|
return "GENUINE_FAIL"; // a real non-zero exit from the test/build/lint itself
|
|
@@ -739,19 +757,51 @@ function markVerified(sig, cursorPath = LAST_PATH) {
|
|
|
739
757
|
}
|
|
740
758
|
|
|
741
759
|
// Single-flight: only one shadow re-run at a time (a burst of prompts must not spawn parallel test runs).
|
|
742
|
-
|
|
760
|
+
// The old body did `statSync(LOCK_PATH)` then an UNCONDITIONAL `writeFileSync` — a TOCTOU hole: when no
|
|
761
|
+
// lock file existed (the common case), N concurrent shadows all saw "no lock" and all wrote it, so ALL
|
|
762
|
+
// "acquired" and ran parallel test re-runs. The atomic primitive is an EXCLUSIVE create (O_EXCL via the
|
|
763
|
+
// "wx" flag): exactly one racer's create can win; every other gets EEXIST and backs off. Stale-lock
|
|
764
|
+
// takeover (a shadow that died holding it) is preserved, but done through the same exclusive create so
|
|
765
|
+
// two reclaimers can't both win. FAIL-OPEN: any unexpected error resolves to "not acquired" (no re-run).
|
|
766
|
+
// Exported for the concurrency test (proves exactly-one-holder under a burst). Never throws.
|
|
767
|
+
export function acquireLock() {
|
|
743
768
|
try {
|
|
744
769
|
mkdirSync(DIR, { recursive: true });
|
|
770
|
+
} catch {
|
|
771
|
+
return false;
|
|
772
|
+
}
|
|
773
|
+
// Atomic exclusive create — the win/lose decision is the create itself, not a prior stat.
|
|
774
|
+
try {
|
|
775
|
+
const fd = openSync(LOCK_PATH, "wx");
|
|
745
776
|
try {
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
777
|
+
writeSync(fd, String(process.pid));
|
|
778
|
+
} finally {
|
|
779
|
+
closeSync(fd);
|
|
780
|
+
}
|
|
781
|
+
return true;
|
|
782
|
+
} catch (e) {
|
|
783
|
+
if (!e || e.code !== "EEXIST") return false; // not a contention error → fail-open, no lock
|
|
784
|
+
}
|
|
785
|
+
// A lock already exists. Take it over ONLY if it is STALE (holder died/hung > LOCK_STALE_MS ago); an
|
|
786
|
+
// actively-held fresh lock is respected (return false). Reclaim through the SAME exclusive create so
|
|
787
|
+
// only one reclaimer wins even if several fire at once.
|
|
788
|
+
try {
|
|
789
|
+
const st = statSync(LOCK_PATH);
|
|
790
|
+
if (Date.now() - st.mtimeMs < LOCK_STALE_MS) return false; // fresh — respect the active holder
|
|
791
|
+
} catch {
|
|
792
|
+
/* vanished between EEXIST and stat (just released) — fall through and try to claim it */
|
|
793
|
+
}
|
|
794
|
+
try {
|
|
795
|
+
rmSync(LOCK_PATH, { force: true });
|
|
796
|
+
const fd = openSync(LOCK_PATH, "wx");
|
|
797
|
+
try {
|
|
798
|
+
writeSync(fd, String(process.pid));
|
|
799
|
+
} finally {
|
|
800
|
+
closeSync(fd);
|
|
750
801
|
}
|
|
751
|
-
writeFileSync(LOCK_PATH, String(process.pid));
|
|
752
802
|
return true;
|
|
753
803
|
} catch {
|
|
754
|
-
return false;
|
|
804
|
+
return false; // another reclaimer re-created it first — they hold it
|
|
755
805
|
}
|
|
756
806
|
}
|
|
757
807
|
function releaseLock() {
|