skalpel 4.0.41 → 4.0.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/_test-home.mjs +18 -0
- package/insights.mjs +184 -3
- package/mutation-check.mjs +158 -0
- package/package.json +1 -1
- package/skalpel-hook.mjs +6 -8
- package/skalpel-setup.mjs +148 -0
- package/skalpel-statusline.mjs +16 -2
- package/verify-core.test.mjs +186 -0
- package/verify-integration.test.mjs +194 -0
- package/verify-safeguards.test.mjs +178 -0
- package/verify-shadow.mjs +287 -16
package/_test-home.mjs
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// _test-home.mjs — TEST-ONLY isolation shim. Imported FIRST (before any client module) by every test
|
|
2
|
+
// that touches ~/.skalpel, so os.homedir() resolves to a throwaway temp dir and the suite can never read
|
|
3
|
+
// or clobber the developer's real ~/.skalpel. ESM evaluates imports depth-first in source order, so a test
|
|
4
|
+
// that does `import "./_test-home.mjs"` before `import "./insights.mjs"` guarantees HOME is redirected
|
|
5
|
+
// BEFORE insights.mjs binds its DIR = join(homedir(), ".skalpel") at module load. Not a test file itself.
|
|
6
|
+
import { mkdtempSync } from "node:fs";
|
|
7
|
+
import { tmpdir } from "node:os";
|
|
8
|
+
import { join } from "node:path";
|
|
9
|
+
|
|
10
|
+
const home = mkdtempSync(join(tmpdir(), "skalpel-verify-test-"));
|
|
11
|
+
process.env.HOME = home;
|
|
12
|
+
process.env.USERPROFILE = home; // Windows
|
|
13
|
+
// Never let a stray dev env var arm/disarm the resolver mid-test — each test sets these explicitly.
|
|
14
|
+
delete process.env.SKALPEL_VERIFY_REVEAL;
|
|
15
|
+
delete process.env.SKALPEL_VERIFY_SHADOW;
|
|
16
|
+
|
|
17
|
+
export const TEST_HOME = home;
|
|
18
|
+
export const SKALPEL_DIR = join(home, ".skalpel");
|
package/insights.mjs
CHANGED
|
@@ -17,6 +17,9 @@ import { join } from "node:path";
|
|
|
17
17
|
|
|
18
18
|
const DIR = join(homedir(), ".skalpel");
|
|
19
19
|
export const INSIGHTS_PATH = join(DIR, "insights.ndjson");
|
|
20
|
+
// The persisted client config `skalpel setup` writes (api/user/verify). Read HERE too so the ONE arming
|
|
21
|
+
// resolver can consult the consented `verify` key without any writer/reader plumbing.
|
|
22
|
+
export const CLIENT_JSON_PATH = join(DIR, "client.json");
|
|
20
23
|
|
|
21
24
|
// ---- verify-reveal: the opt-in "aha catch" surface — single source for the flag + the state-file path.
|
|
22
25
|
// The reveal is WRITTEN by verify-shadow.mjs (the detached proof re-runner, when the agent claimed
|
|
@@ -30,10 +33,188 @@ export const REVEAL_PATH = join(DIR, "verify-reveal.json");
|
|
|
30
33
|
// rare red catch land like a thunderclap — without a stream of honest verified stamps a lone red reads as
|
|
31
34
|
// noise, not a verdict. Path single-sourced HERE alongside REVEAL_PATH so writer + reader never drift.
|
|
32
35
|
export const VERDICT_PATH = join(DIR, "verify-verdict.json");
|
|
33
|
-
//
|
|
34
|
-
//
|
|
36
|
+
// ---- THE ONE ARMING RESOLVER (single source of truth) --------------------------------------------
|
|
37
|
+
// The "aha catch" is OPT-IN and OFF BY DEFAULT. Arming is resolved by ONE function everyone reads —
|
|
38
|
+
// the hook spawn gate, the statusline reveal, verify-shadow's surfacing, and reveal.mjs's agent note —
|
|
39
|
+
// so writer, reader, and gate can never drift. PRECEDENCE:
|
|
40
|
+
// 1. ENV OVERRIDE (dev/CI): SKALPEL_VERIFY_REVEAL, explicit on OR off, always wins.
|
|
41
|
+
// 2. CONSENTED CONFIG: ~/.skalpel/client.json "verify" key ("on"/"off"), written by `skalpel setup`.
|
|
42
|
+
// 3. DEFAULT OFF — an absent config + unset env is DARK, byte-identical to the shadow-only baseline.
|
|
43
|
+
// Only an explicit consented "on" (or a dev env override) arms it; existing installs stay dark.
|
|
44
|
+
const ARM_TRUTHY = /^(1|true|on|yes)$/i;
|
|
45
|
+
const ARM_FALSY = /^(0|false|off|no)$/i;
|
|
46
|
+
// parseArmToken(v) → true (armed) | false (disarmed) | null (no opinion — unset/blank/unrecognized).
|
|
47
|
+
export function parseArmToken(v) {
|
|
48
|
+
const s = String(v ?? "").trim();
|
|
49
|
+
if (ARM_TRUTHY.test(s)) return true;
|
|
50
|
+
if (ARM_FALSY.test(s)) return false;
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
// verifyConfigArmed() → the persisted consent from client.json's "verify" key, or null if unset/unreadable.
|
|
54
|
+
// Fail-open: any read/parse error → null (no opinion → falls through to DEFAULT OFF). Never throws.
|
|
55
|
+
export function verifyConfigArmed() {
|
|
56
|
+
try {
|
|
57
|
+
return parseArmToken(JSON.parse(readFileSync(CLIENT_JSON_PATH, "utf8"))?.verify);
|
|
58
|
+
} catch {
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
// revealEnabled(env) — is the USER-VISIBLE reveal armed? env override > consented config > default off.
|
|
63
|
+
// SKALPEL_VERIFY_SHADOW is deliberately NOT consulted here: it arms only the DARK log-only shadow (no
|
|
64
|
+
// reveal); the spawn gate ORs it in separately (verifySpawnArmed). Wrapped so it can never throw.
|
|
35
65
|
export function revealEnabled(env) {
|
|
36
|
-
|
|
66
|
+
try {
|
|
67
|
+
const envOverride = parseArmToken((env || {}).SKALPEL_VERIFY_REVEAL);
|
|
68
|
+
if (envOverride !== null) return envOverride; // 1) dev/CI env override wins
|
|
69
|
+
const cfg = verifyConfigArmed();
|
|
70
|
+
if (cfg !== null) return cfg; // 2) consented config
|
|
71
|
+
} catch {
|
|
72
|
+
/* fall through to default */
|
|
73
|
+
}
|
|
74
|
+
return false; // 3) DEFAULT OFF
|
|
75
|
+
}
|
|
76
|
+
// verifySpawnArmed(env) — should the per-turn hook SPAWN the detached shadow worker at all? True when the
|
|
77
|
+
// reveal is armed OR the dev log-only shadow (SKALPEL_VERIFY_SHADOW=1) is set. This is the single gate the
|
|
78
|
+
// hook reads; it stays byte-identical to the prior inline `SHADOW==="1" || revealEnabled(env)` when the
|
|
79
|
+
// config is absent, so no existing install starts spawning.
|
|
80
|
+
export function verifySpawnArmed(env) {
|
|
81
|
+
try {
|
|
82
|
+
if (String((env || {}).SKALPEL_VERIFY_SHADOW || "").trim() === "1") return true;
|
|
83
|
+
return revealEnabled(env);
|
|
84
|
+
} catch {
|
|
85
|
+
return false;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// ---- FALSE-ACCUSATION SAFEGUARDS: adjudication ledger + derived auto-darken -----------------------
|
|
90
|
+
// A "was this right?" flight recorder. `skalpel yes|no` appends ONE row here keyed to the most-recent
|
|
91
|
+
// reveal; precision = right/(right+wrong) is derived from these rows, never a stored number that can lie.
|
|
92
|
+
export const ADJUDICATIONS_PATH = join(DIR, "verify-adjudications.ndjson");
|
|
93
|
+
// Every reveal actually SURFACED to the user is logged here (append-only) so `skalpel yes|no` can key an
|
|
94
|
+
// adjudication to the real reveal it saw, and the field-proof report can count real fires.
|
|
95
|
+
export const REVEALS_LOG_PATH = join(DIR, "verify-reveals.ndjson");
|
|
96
|
+
// Session-scoped suppression: a `skalpel no` writes the (session, proof) here so that proof never
|
|
97
|
+
// re-accuses for the rest of THAT session (matched by session id → a new session is never suppressed).
|
|
98
|
+
export const SUPPRESS_PATH = join(DIR, "verify-suppressed.ndjson");
|
|
99
|
+
|
|
100
|
+
// Auto-darken thresholds. If adjudicated precision < 90% over >= 10 adjudications, the LIVE red reveal
|
|
101
|
+
// self-darkens (derived from these rows every read — NOT a stored flag) and falls back to `skalpel catches`.
|
|
102
|
+
export const AUTO_DARKEN_MIN_ADJ = 10;
|
|
103
|
+
export const AUTO_DARKEN_MIN_PRECISION = 0.9;
|
|
104
|
+
|
|
105
|
+
// proofSig(s) — normalize a proof command for stable suppression matching (collapse whitespace, bound).
|
|
106
|
+
export function proofSig(s) {
|
|
107
|
+
return String(s ?? "")
|
|
108
|
+
.replace(/\s+/g, " ")
|
|
109
|
+
.trim()
|
|
110
|
+
.slice(0, 200);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function readNdjson(path, cap = 5000) {
|
|
114
|
+
try {
|
|
115
|
+
const rows = readFileSync(path, "utf8").trim().split("\n").filter(Boolean);
|
|
116
|
+
const out = [];
|
|
117
|
+
for (const l of rows.slice(-cap)) {
|
|
118
|
+
try {
|
|
119
|
+
out.push(JSON.parse(l));
|
|
120
|
+
} catch {
|
|
121
|
+
/* skip a partial/garbage line */
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return out;
|
|
125
|
+
} catch {
|
|
126
|
+
return [];
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// adjudicationStats() → { right, wrong, total, precision } from the REAL adjudication rows. precision is
|
|
131
|
+
// right/(right+wrong) or null when there are none. Only "yes"/"no" verdicts count. Never throws.
|
|
132
|
+
export function adjudicationStats() {
|
|
133
|
+
let right = 0;
|
|
134
|
+
let wrong = 0;
|
|
135
|
+
for (const r of readNdjson(ADJUDICATIONS_PATH)) {
|
|
136
|
+
if (r && r.verdict === "yes") right++;
|
|
137
|
+
else if (r && r.verdict === "no") wrong++;
|
|
138
|
+
}
|
|
139
|
+
const total = right + wrong;
|
|
140
|
+
return { right, wrong, total, precision: total > 0 ? right / total : null };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// autoDarkened() → is the live red reveal auto-paused? DERIVED from the adjudication rows every call (never
|
|
144
|
+
// a stored boolean): true only once there are >= AUTO_DARKEN_MIN_ADJ adjudications AND precision < 90%.
|
|
145
|
+
// Fail-open: any error → false (stay live — a broken read must never silently darken the catch).
|
|
146
|
+
export function autoDarkened() {
|
|
147
|
+
try {
|
|
148
|
+
const { total, precision } = adjudicationStats();
|
|
149
|
+
return (
|
|
150
|
+
total >= AUTO_DARKEN_MIN_ADJ && precision !== null && precision < AUTO_DARKEN_MIN_PRECISION
|
|
151
|
+
);
|
|
152
|
+
} catch {
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function appendNdjson(path, row) {
|
|
158
|
+
try {
|
|
159
|
+
mkdirSync(DIR, { recursive: true });
|
|
160
|
+
appendFileSync(path, JSON.stringify(row) + "\n");
|
|
161
|
+
return true;
|
|
162
|
+
} catch {
|
|
163
|
+
return false;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// appendAdjudication(row) — ONE flight-recorder row (verdict yes/no, keyed to a reveal). Never throws.
|
|
168
|
+
export function appendAdjudication(row) {
|
|
169
|
+
return appendNdjson(ADJUDICATIONS_PATH, { ts: new Date().toISOString(), ...row });
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// appendRevealLog(row) — record a reveal that was actually SHOWN to the user (append-only, bounded).
|
|
173
|
+
export function appendRevealLog(row) {
|
|
174
|
+
return appendNdjson(REVEALS_LOG_PATH, { ts: new Date().toISOString(), ...row });
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// lastReveal() — the most-recent reveal actually shown (the row `skalpel yes|no` adjudicates). Null if none.
|
|
178
|
+
export function lastReveal() {
|
|
179
|
+
const rows = readNdjson(REVEALS_LOG_PATH);
|
|
180
|
+
return rows.length ? rows[rows.length - 1] : null;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// suppressProof(session, proof) — after a `skalpel no`, stop this proof re-accusing for the rest of the
|
|
184
|
+
// session. Requires a real session id (a null-session suppression would over-suppress). Never throws.
|
|
185
|
+
export function suppressProof(session, proof) {
|
|
186
|
+
const sid = session ? String(session) : "";
|
|
187
|
+
const sig = proofSig(proof);
|
|
188
|
+
if (!sid || !sig) return false;
|
|
189
|
+
return appendNdjson(SUPPRESS_PATH, { ts: new Date().toISOString(), session: sid, proof: sig });
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// isProofSuppressed(session, proof) — has this (session, proof) been marked "no" this session? Both sides
|
|
193
|
+
// must have a real session id and they must match, plus the normalized proof must match. Fail-open → false.
|
|
194
|
+
export function isProofSuppressed(session, proof) {
|
|
195
|
+
try {
|
|
196
|
+
const sid = session ? String(session) : "";
|
|
197
|
+
const sig = proofSig(proof);
|
|
198
|
+
if (!sid || !sig) return false;
|
|
199
|
+
for (const r of readNdjson(SUPPRESS_PATH)) {
|
|
200
|
+
if (r && String(r.session || "") === sid && proofSig(r.proof) === sig) return true;
|
|
201
|
+
}
|
|
202
|
+
return false;
|
|
203
|
+
} catch {
|
|
204
|
+
return false;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// verifyArmedAt() — the real ISO timestamp `verify` was last turned ON (recorded in client.json at consent
|
|
209
|
+
// time), as a Date, or null. Used only for the honest "armed N days" line — a recorded fact, not a metric.
|
|
210
|
+
export function verifyArmedAt() {
|
|
211
|
+
try {
|
|
212
|
+
const t = JSON.parse(readFileSync(CLIENT_JSON_PATH, "utf8"))?.verify_armed_at;
|
|
213
|
+
const d = t ? new Date(t) : null;
|
|
214
|
+
return d && Number.isFinite(d.getTime()) ? d : null;
|
|
215
|
+
} catch {
|
|
216
|
+
return null;
|
|
217
|
+
}
|
|
37
218
|
}
|
|
38
219
|
|
|
39
220
|
// Unlike metrics.ndjson (write-only telemetry, uncapped), this file is READ by a TUI every render —
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// mutation-check.mjs — mutation testing for the CRITICAL LOGIC: detectCompletionClaim (incl. the negation
|
|
3
|
+
// guard), classifyOutcome, classifyConfirm, plus the false-accusation safeguards shipUrlPass (URL
|
|
4
|
+
// reachability) and hasMutatingFlag (non-idempotent re-run guard). For each mutation we flip ONE key
|
|
5
|
+
// predicate/branch in verify-shadow.mjs and re-run the pure-function suite (verify-core.test.mjs) against
|
|
6
|
+
// the mutated copy. A mutation the suite still PASSES is a SURVIVOR (a test gap). Gate: survivors must be
|
|
7
|
+
// 0. Run: `node mutation-check.mjs`.
|
|
8
|
+
import { cpSync, mkdtempSync, readFileSync, writeFileSync, rmSync } from "node:fs";
|
|
9
|
+
import { tmpdir } from "node:os";
|
|
10
|
+
import { join, dirname } from "node:path";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
import { spawnSync } from "node:child_process";
|
|
13
|
+
|
|
14
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
15
|
+
const TARGET = "verify-shadow.mjs";
|
|
16
|
+
const SUITE = "verify-core.test.mjs";
|
|
17
|
+
|
|
18
|
+
// Each mutation: a UNIQUE substring in verify-shadow.mjs and the flipped version. Every one MUST be caught
|
|
19
|
+
// by at least one assertion in verify-core.test.mjs.
|
|
20
|
+
const MUTATIONS = [
|
|
21
|
+
// ---- detectCompletionClaim (incl. the negation/hedge guard, Fix 2) ----
|
|
22
|
+
{
|
|
23
|
+
fn: "detectCompletionClaim",
|
|
24
|
+
find: 'if (!CLAIM_RE.test(s)) return { claim: false, text: "" };',
|
|
25
|
+
replace: 'if (CLAIM_RE.test(s)) return { claim: false, text: "" };',
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
fn: "detectCompletionClaim",
|
|
29
|
+
find: "if (!l || !CLAIM_RE.test(l)) continue;",
|
|
30
|
+
replace: "if (!l || CLAIM_RE.test(l)) continue;",
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
fn: "detectCompletionClaim",
|
|
34
|
+
find: "if (NEGATION_RE.test(l)) continue; // negated/hedged → not a completion claim",
|
|
35
|
+
replace: "if (!NEGATION_RE.test(l)) continue; // negated/hedged → not a completion claim",
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
fn: "detectCompletionClaim",
|
|
39
|
+
find: 'if (NEGATION_RE.test(s)) return { claim: false, text: "" };',
|
|
40
|
+
replace: 'if (!NEGATION_RE.test(s)) return { claim: false, text: "" };',
|
|
41
|
+
},
|
|
42
|
+
// ---- shipUrlPass (Fix 1: URL false-red safeguard) ----
|
|
43
|
+
{
|
|
44
|
+
fn: "shipUrlPass",
|
|
45
|
+
find: "if (urlResponded(ev1)) return true;",
|
|
46
|
+
replace: "if (urlResponded(ev1)) return false;",
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
fn: "shipUrlPass",
|
|
50
|
+
find: " if (urlResponded(ev2)) return true;\n return false;",
|
|
51
|
+
replace: " if (urlResponded(ev2)) return true;\n return true;",
|
|
52
|
+
},
|
|
53
|
+
// ---- hasMutatingFlag (Fix 3: non-idempotent re-run guard) ----
|
|
54
|
+
{
|
|
55
|
+
fn: "hasMutatingFlag",
|
|
56
|
+
find: "if (a.some((x) => MUTATING_FLAG.test(String(x)))) return true;",
|
|
57
|
+
replace: "if (a.some((x) => MUTATING_FLAG.test(String(x)))) return false;",
|
|
58
|
+
},
|
|
59
|
+
// ---- classifyOutcome ----
|
|
60
|
+
{
|
|
61
|
+
fn: "classifyOutcome",
|
|
62
|
+
find: 'if (!err) return "PASS";',
|
|
63
|
+
replace: 'if (err) return "PASS";',
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
fn: "classifyOutcome",
|
|
67
|
+
find: 'if (err.killed || err.signal === "SIGTERM") return "HARNESS_ERROR"; // hit the timeout',
|
|
68
|
+
replace: 'if (err.killed && err.signal === "SIGTERM") return "PASS"; // hit the timeout',
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
fn: "classifyOutcome",
|
|
72
|
+
find: 'if (exit === 127 || exit === 126) return "HARNESS_ERROR"; // command not found / not executable',
|
|
73
|
+
replace:
|
|
74
|
+
'if (exit === 127 && exit === 126) return "HARNESS_ERROR"; // command not found / not executable',
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
fn: "classifyOutcome",
|
|
78
|
+
find: 'return "GENUINE_FAIL"; // a real non-zero exit from the test/build/lint itself',
|
|
79
|
+
replace: 'return "HARNESS_ERROR"; // a real non-zero exit from the test/build/lint itself',
|
|
80
|
+
},
|
|
81
|
+
// ---- classifyConfirm (the flake-confirm state machine) ----
|
|
82
|
+
{
|
|
83
|
+
fn: "classifyConfirm",
|
|
84
|
+
find: 'const needsConfirm = firstOutcome === "GENUINE_FAIL";',
|
|
85
|
+
replace: 'const needsConfirm = firstOutcome !== "GENUINE_FAIL";',
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
fn: "classifyConfirm",
|
|
89
|
+
find: 'const confirmedFail = needsConfirm && confirmOutcome === "GENUINE_FAIL";',
|
|
90
|
+
replace: 'const confirmedFail = needsConfirm || confirmOutcome === "GENUINE_FAIL";',
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
fn: "classifyConfirm",
|
|
94
|
+
find: 'const confirmedFail = needsConfirm && confirmOutcome === "GENUINE_FAIL";',
|
|
95
|
+
replace: 'const confirmedFail = needsConfirm && confirmOutcome !== "GENUINE_FAIL";',
|
|
96
|
+
},
|
|
97
|
+
{
|
|
98
|
+
fn: "classifyConfirm",
|
|
99
|
+
find: "const unconfirmedFail = needsConfirm && !confirmedFail;",
|
|
100
|
+
replace: "const unconfirmedFail = needsConfirm && confirmedFail;",
|
|
101
|
+
},
|
|
102
|
+
];
|
|
103
|
+
|
|
104
|
+
const workRoot = mkdtempSync(join(tmpdir(), "skalpel-mutants-"));
|
|
105
|
+
cpSync(HERE, join(workRoot, "src"), {
|
|
106
|
+
recursive: true,
|
|
107
|
+
filter: (s) => !s.includes("node_modules"),
|
|
108
|
+
});
|
|
109
|
+
const dir = join(workRoot, "src");
|
|
110
|
+
const targetPath = join(dir, TARGET);
|
|
111
|
+
const pristine = readFileSync(targetPath, "utf8");
|
|
112
|
+
|
|
113
|
+
function runSuite() {
|
|
114
|
+
const r = spawnSync("node", ["--test", SUITE], { cwd: dir, encoding: "utf8" });
|
|
115
|
+
return r.status; // 0 = suite passed (mutant SURVIVED); non-zero = suite failed (mutant KILLED)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Baseline: the pristine copy must pass, else the harness itself is broken.
|
|
119
|
+
if (runSuite() !== 0) {
|
|
120
|
+
console.error(
|
|
121
|
+
"BASELINE FAIL: verify-core.test.mjs does not pass on the pristine copy — aborting.",
|
|
122
|
+
);
|
|
123
|
+
process.exit(2);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
let survivors = 0;
|
|
127
|
+
let missing = 0;
|
|
128
|
+
const results = [];
|
|
129
|
+
for (const m of MUTATIONS) {
|
|
130
|
+
const count = pristine.split(m.find).length - 1;
|
|
131
|
+
if (count !== 1) {
|
|
132
|
+
missing++;
|
|
133
|
+
results.push(` ✗ MUTATION TARGET NOT UNIQUE (${count}×) [${m.fn}] ${m.find.slice(0, 50)}…`);
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
writeFileSync(targetPath, pristine.replace(m.find, m.replace));
|
|
137
|
+
const killed = runSuite() !== 0;
|
|
138
|
+
writeFileSync(targetPath, pristine); // restore
|
|
139
|
+
if (killed) {
|
|
140
|
+
results.push(` ✓ killed [${m.fn}] ${m.find.slice(0, 52)}…`);
|
|
141
|
+
} else {
|
|
142
|
+
survivors++;
|
|
143
|
+
results.push(` ☠ SURVIVED [${m.fn}] ${m.find.slice(0, 52)}…`);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
rmSync(workRoot, { recursive: true, force: true });
|
|
148
|
+
console.log(results.join("\n"));
|
|
149
|
+
console.log(
|
|
150
|
+
`\nmutants: ${MUTATIONS.length} killed: ${MUTATIONS.length - survivors - missing} survivors: ${survivors} target-errors: ${missing}`,
|
|
151
|
+
);
|
|
152
|
+
if (survivors === 0 && missing === 0) {
|
|
153
|
+
console.log(
|
|
154
|
+
"MUTATION GATE: survivors = 0 on the critical + false-accusation-safeguard functions ✓",
|
|
155
|
+
);
|
|
156
|
+
process.exit(0);
|
|
157
|
+
}
|
|
158
|
+
process.exit(1);
|
package/package.json
CHANGED
package/skalpel-hook.mjs
CHANGED
|
@@ -13,7 +13,7 @@ import { homedir } from "node:os";
|
|
|
13
13
|
import { join } from "node:path";
|
|
14
14
|
import { identity } from "./auth.mjs";
|
|
15
15
|
import { record } from "./metrics.mjs";
|
|
16
|
-
import { recordInsight, cleanText,
|
|
16
|
+
import { recordInsight, cleanText, verifySpawnArmed } from "./insights.mjs";
|
|
17
17
|
import { graphReadyNote, verifyCatchNote } from "./reveal.mjs";
|
|
18
18
|
import { tailLines } from "./transcript.mjs";
|
|
19
19
|
|
|
@@ -510,13 +510,11 @@ async function main() {
|
|
|
510
510
|
// recent test/build/typecheck/lint command and RE-RUNs it out-of-band to log the real PASS/FAIL the
|
|
511
511
|
// agent can't forge. LOG ONLY — it writes nothing to this hook's stdout and never blocks the turn (the
|
|
512
512
|
// re-run, up to 60s, lives in a separate unref'd process). Wrapped so it can never affect the injection.
|
|
513
|
-
//
|
|
514
|
-
//
|
|
515
|
-
// the
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
payload.transcript_path
|
|
519
|
-
) {
|
|
513
|
+
// ARMING is resolved by the ONE shared resolver (verifySpawnArmed): the consented `verify:"on"` in
|
|
514
|
+
// client.json OR a dev env override (SKALPEL_VERIFY_SHADOW=1 / SKALPEL_VERIFY_REVEAL). When arming
|
|
515
|
+
// resolves OFF (the default for every existing install — env unset + config off/absent) this branch is
|
|
516
|
+
// skipped and the hook's stdout is byte-identical to before; the spawn is detached with stdio ignored.
|
|
517
|
+
if (verifySpawnArmed(process.env) && payload.transcript_path) {
|
|
520
518
|
try {
|
|
521
519
|
const { spawn } = await import("node:child_process");
|
|
522
520
|
const { fileURLToPath } = await import("node:url");
|
package/skalpel-setup.mjs
CHANGED
|
@@ -22,6 +22,7 @@ import { fileURLToPath } from "node:url";
|
|
|
22
22
|
import { spawn, spawnSync } from "node:child_process";
|
|
23
23
|
import { identity as authIdentity, cliAuthPath } from "./auth.mjs";
|
|
24
24
|
import { isGenuineUserTurn } from "./session-turns.mjs";
|
|
25
|
+
import { appendAdjudication, suppressProof, lastReveal, parseArmToken } from "./insights.mjs";
|
|
25
26
|
|
|
26
27
|
// CLI (the `skalpel` bin): `skalpel setup [--api https://graph.skalpel.ai]
|
|
27
28
|
// [--user <id>]`, plus `skalpel uninstall` and `skalpel login`. A bare invocation
|
|
@@ -65,6 +66,9 @@ const KNOWN_SUBS = new Set([
|
|
|
65
66
|
"card",
|
|
66
67
|
"ledger",
|
|
67
68
|
"analytics",
|
|
69
|
+
"verify",
|
|
70
|
+
"yes",
|
|
71
|
+
"no",
|
|
68
72
|
"__build",
|
|
69
73
|
"__verify-report",
|
|
70
74
|
]);
|
|
@@ -83,6 +87,8 @@ usage:
|
|
|
83
87
|
skalpel card [--last|--id <id>] [--md] share one verified catch: the claim + its own failing proof
|
|
84
88
|
skalpel ledger your running tally: N "done"s checked, M verified FALSE
|
|
85
89
|
skalpel analytics interactive local analytics — caught lies, measured cost, steers, re-asks
|
|
90
|
+
skalpel verify on|off|status arm/disarm the live catch (re-runs your own proof on a "done" claim)
|
|
91
|
+
skalpel yes | skalpel no was the last catch right? (feeds precision; "no" stops that proof this session)
|
|
86
92
|
skalpel uninstall [--purge] remove the hooks + local data from this machine
|
|
87
93
|
|
|
88
94
|
options:
|
|
@@ -157,6 +163,84 @@ function fileCfg() {
|
|
|
157
163
|
}
|
|
158
164
|
const API = process.env.SKALPEL_API || apiFlag || fileCfg().api || DEFAULT_API;
|
|
159
165
|
|
|
166
|
+
// writeVerifyConfig(state) — persist the CONSENTED arming for the live catch to ~/.skalpel/client.json.
|
|
167
|
+
// state ∈ { "on", "off" }. On "on" we stamp verify_armed_at (a REAL timestamp) only if not already armed,
|
|
168
|
+
// so re-confirming never resets the honest "armed N days" clock; on "off" we clear it. This is the ONLY
|
|
169
|
+
// writer of the `verify` key; insights.mjs's resolver is the only reader — one key, no drift. Never throws.
|
|
170
|
+
function writeVerifyConfig(state) {
|
|
171
|
+
const cfgDir = join(homedir(), ".skalpel");
|
|
172
|
+
const cfgPath = join(cfgDir, "client.json");
|
|
173
|
+
let cfg = {};
|
|
174
|
+
try {
|
|
175
|
+
cfg = JSON.parse(readFileSync(cfgPath, "utf8")) || {};
|
|
176
|
+
} catch {
|
|
177
|
+
/* fresh config */
|
|
178
|
+
}
|
|
179
|
+
const wasOn = parseArmToken(cfg.verify) === true;
|
|
180
|
+
cfg.verify = state === "on" ? "on" : "off";
|
|
181
|
+
if (state === "on") {
|
|
182
|
+
if (!wasOn || !cfg.verify_armed_at) cfg.verify_armed_at = new Date().toISOString();
|
|
183
|
+
} else {
|
|
184
|
+
delete cfg.verify_armed_at;
|
|
185
|
+
}
|
|
186
|
+
mkdirSync(cfgDir, { recursive: true });
|
|
187
|
+
writeFileSync(cfgPath, JSON.stringify(cfg, null, 2));
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// promptVerifyConsent() — the ONE honest consent line at setup. The prompt DEFAULTS TO THE EXISTING
|
|
191
|
+
// SETTING so a setup re-run never silently flips a deliberate choice: a user who ran `skalpel verify off`
|
|
192
|
+
// and re-runs setup + Enter STAYS OFF; unset/on defaults to on (the activation intent). Non-interactive /
|
|
193
|
+
// CI must NOT hang: no TTY → leave the config untouched and return. The copy is accurate, not hype: the
|
|
194
|
+
// re-run is only the session's OWN already-executed command (reconstructed from its Bash tool calls) —
|
|
195
|
+
// never a new class of side effect. Best-effort: never breaks setup.
|
|
196
|
+
// resolveConsentChoice(current, answerRaw) → "on" | "off". The prompt DEFAULTS TO THE EXISTING SETTING so
|
|
197
|
+
// an empty Enter never silently flips a deliberate choice: current "off" → stays off unless explicitly
|
|
198
|
+
// re-enabled (y/yes/on); current "on"/unset → the activation default, on unless declined (n/no/off).
|
|
199
|
+
export function resolveConsentChoice(current, answerRaw) {
|
|
200
|
+
const defaultOn = parseArmToken(current) !== false; // off → default off; on/unset → default on
|
|
201
|
+
const t = String(answerRaw || "")
|
|
202
|
+
.trim()
|
|
203
|
+
.toLowerCase();
|
|
204
|
+
if (defaultOn) return t === "n" || t === "no" || t === "off" ? "off" : "on";
|
|
205
|
+
return t === "y" || t === "yes" || t === "on" ? "on" : "off"; // stays off unless explicitly turned on
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
async function promptVerifyConsent() {
|
|
209
|
+
try {
|
|
210
|
+
const interactive = process.stdin.isTTY && process.stdout.isTTY && !process.env.CI;
|
|
211
|
+
if (!interactive || process.env.SKALPEL_DEMO || process.env.SKALPEL_SKIP_VERIFY_CONSENT === "1")
|
|
212
|
+
return; // CI / piped / demo → leave config untouched, never block
|
|
213
|
+
// Default the prompt to the EXISTING setting: off → Enter keeps off; on/unset → Enter accepts.
|
|
214
|
+
const current = fileCfg().verify; // "on" | "off" | undefined
|
|
215
|
+
const defaultOn = parseArmToken(current) !== false;
|
|
216
|
+
console.log(
|
|
217
|
+
` ${B}The live catch${X}\n` +
|
|
218
|
+
` ${D}When your agent says "done / tests pass", skalpel re-runs the SAME test/build command\n` +
|
|
219
|
+
` it just ran, in the same directory, and shows you the real result. It only ever re-runs\n` +
|
|
220
|
+
` your session's own command — never anything new.${X}`,
|
|
221
|
+
);
|
|
222
|
+
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
223
|
+
const promptLine = defaultOn ? `[Enter = on · n = off]` : `[Enter = keep off · y = on]`;
|
|
224
|
+
const ans = await new Promise((res) =>
|
|
225
|
+
rl.question(` ${B}${promptLine}${X} `, (a) => {
|
|
226
|
+
rl.close();
|
|
227
|
+
res(a);
|
|
228
|
+
}),
|
|
229
|
+
);
|
|
230
|
+
// Resolve against the existing setting so an empty Enter preserves the current choice.
|
|
231
|
+
const choice = resolveConsentChoice(current, ans);
|
|
232
|
+
const on = choice === "on";
|
|
233
|
+
writeVerifyConfig(choice);
|
|
234
|
+
console.log(
|
|
235
|
+
on
|
|
236
|
+
? ` ${G}✓${X} ${D}Live catch on — it'll re-run your own proof and flag a false "done". ${X}skalpel verify off${D} anytime.${X}\n`
|
|
237
|
+
: ` ${D}Live catch off. Turn it on anytime with ${X}skalpel verify on${D}.${X}\n`,
|
|
238
|
+
);
|
|
239
|
+
} catch {
|
|
240
|
+
/* consent is best-effort — never break setup */
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
160
244
|
// who's signed in — the session saved by the login flow (~/.skalpel/auth.json). Its `uid` keys the
|
|
161
245
|
// hosted graph; `access_token` authenticates the upload. SKALPEL_USER overrides for dev (no token).
|
|
162
246
|
// Uses auth.mjs's REFRESH-CAPABLE identity() so a token that expired since login (Supabase tokens
|
|
@@ -933,6 +1017,66 @@ async function main() {
|
|
|
933
1017
|
spawnSync("node", [join(__dir, "verify-shadow.mjs"), "--xray"], { stdio: "inherit" });
|
|
934
1018
|
return;
|
|
935
1019
|
}
|
|
1020
|
+
// `skalpel verify on|off|status` — the one-command reverse for the live catch. Local, no network, no
|
|
1021
|
+
// sign-in. `on`/`off` persist the consented arming to client.json (the single writer); `status` (and a
|
|
1022
|
+
// bare `skalpel verify`) render the arming + FIELD-PROOF (real fires, adjudicated precision, armed-days,
|
|
1023
|
+
// auto-darken state) — all from real local rows.
|
|
1024
|
+
if (sub === "verify") {
|
|
1025
|
+
const action = (argv[1] || "status").toLowerCase();
|
|
1026
|
+
if (action === "on" || action === "off") {
|
|
1027
|
+
writeVerifyConfig(action);
|
|
1028
|
+
if (action === "on") {
|
|
1029
|
+
console.log(
|
|
1030
|
+
`\n ${G}✓${X} Live catch ${B}armed${X}. On a "done" claim, skalpel re-runs your session's own\n` +
|
|
1031
|
+
` test/build once (twice to confirm before any red) and shows the real result.\n` +
|
|
1032
|
+
` ${D}Reverse anytime: ${X}skalpel verify off${D} · see the field proof: ${X}skalpel verify status\n`,
|
|
1033
|
+
);
|
|
1034
|
+
} else {
|
|
1035
|
+
console.log(
|
|
1036
|
+
`\n ${G}✓${X} Live catch ${B}off${X}. ${D}Re-arm anytime: ${X}skalpel verify on\n`,
|
|
1037
|
+
);
|
|
1038
|
+
}
|
|
1039
|
+
} else if (action === "status") {
|
|
1040
|
+
spawnSync("node", [join(__dir, "verify-shadow.mjs"), "--status"], { stdio: "inherit" });
|
|
1041
|
+
} else {
|
|
1042
|
+
console.error(`skalpel verify: unknown action \`${argv[1]}\` (use: on | off | status)`);
|
|
1043
|
+
process.exitCode = 2;
|
|
1044
|
+
}
|
|
1045
|
+
return;
|
|
1046
|
+
}
|
|
1047
|
+
// `skalpel yes` / `skalpel no` — ONE-KEY ADJUDICATION of the most-recent catch (the flight recorder).
|
|
1048
|
+
// Local, no network, no sign-in. Appends one NDJSON row keyed to the reveal you just saw; a "no" also
|
|
1049
|
+
// suppresses that exact proof for the rest of THAT session and counts against precision (feeding the
|
|
1050
|
+
// auto-darken kill-switch). Fail-open: nothing to adjudicate → an honest note, never an error.
|
|
1051
|
+
if (sub === "yes" || sub === "no") {
|
|
1052
|
+
const r = lastReveal();
|
|
1053
|
+
if (!r || !r.proof_command) {
|
|
1054
|
+
console.log(
|
|
1055
|
+
`\n ${D}No recent skalpel catch to adjudicate — nothing has surfaced yet.${X}\n` +
|
|
1056
|
+
` ${D}When skalpel flags a false "done", it'll say: was this right? ${X}skalpel yes / skalpel no\n`,
|
|
1057
|
+
);
|
|
1058
|
+
return;
|
|
1059
|
+
}
|
|
1060
|
+
const proofShort = String(r.proof_command).slice(0, 60);
|
|
1061
|
+
appendAdjudication({
|
|
1062
|
+
verdict: sub, // "yes" | "no"
|
|
1063
|
+
reveal_ts: r.ts || null,
|
|
1064
|
+
session: r.session || null,
|
|
1065
|
+
proof: r.proof_command,
|
|
1066
|
+
});
|
|
1067
|
+
if (sub === "no") {
|
|
1068
|
+
suppressProof(r.session || null, r.proof_command);
|
|
1069
|
+
console.log(
|
|
1070
|
+
`\n ${G}✓${X} Marked ${B}not a real catch${X}. ${D}skalpel won't re-accuse ${X}\`${proofShort}\`${D}\n` +
|
|
1071
|
+
` for the rest of this session, and it counts against precision.${X}\n`,
|
|
1072
|
+
);
|
|
1073
|
+
} else {
|
|
1074
|
+
console.log(
|
|
1075
|
+
`\n ${G}✓${X} Marked ${B}a real catch${X}. ${D}Thanks — this counts toward precision.${X}\n`,
|
|
1076
|
+
);
|
|
1077
|
+
}
|
|
1078
|
+
return;
|
|
1079
|
+
}
|
|
936
1080
|
// Persist --api/--user to ~/.skalpel/client.json so the hooks can read them without shell-env
|
|
937
1081
|
// plumbing (env vars SKALPEL_API / SKALPEL_USER still win at hook runtime).
|
|
938
1082
|
if (apiFlag || userFlag) {
|
|
@@ -971,6 +1115,10 @@ async function main() {
|
|
|
971
1115
|
` ${G}✓${X} Signed in as ${D}${id.name ? `${id.name} · ${id.email}` : id.email || id.uid}${X}\n`,
|
|
972
1116
|
);
|
|
973
1117
|
|
|
1118
|
+
// CONSENT for the live catch (default-accept). One honest line; persisted to client.json. Non-TTY/CI
|
|
1119
|
+
// never blocks (stays default OFF). Reversible anytime with `skalpel verify off`.
|
|
1120
|
+
await promptVerifyConsent();
|
|
1121
|
+
|
|
974
1122
|
// 2) find the real coding sessions in the last 30 days (the client reads; the server builds)
|
|
975
1123
|
const s1 = spinner(`Looking through your last ${LOOKBACK_DAYS} days of Claude Code + Codex…`);
|
|
976
1124
|
await sleep(500);
|
package/skalpel-statusline.mjs
CHANGED
|
@@ -30,7 +30,7 @@ import { readFileSync, rmSync } from "node:fs";
|
|
|
30
30
|
import { homedir } from "node:os";
|
|
31
31
|
import { join } from "node:path";
|
|
32
32
|
import { tailLines } from "./transcript.mjs";
|
|
33
|
-
import { cleanText, REVEAL_PATH, VERDICT_PATH, revealEnabled } from "./insights.mjs";
|
|
33
|
+
import { cleanText, REVEAL_PATH, VERDICT_PATH, revealEnabled, autoDarkened } from "./insights.mjs";
|
|
34
34
|
|
|
35
35
|
const CYAN = "\x1b[36m";
|
|
36
36
|
const GREEN = "\x1b[32m";
|
|
@@ -368,9 +368,23 @@ function main() {
|
|
|
368
368
|
// SKALPEL_VERIFY_REVEAL — when off (the default) this branch never runs and the bar is byte-identical to
|
|
369
369
|
// the always-on baseline below.
|
|
370
370
|
if (revealEnabled(process.env)) {
|
|
371
|
+
// AUTO-DARKEN kill-switch: once adjudicated precision drops below 90% over >= 10 `skalpel yes|no`
|
|
372
|
+
// rows, the LIVE red reveal self-pauses and the bar points at the retro `skalpel catches`
|
|
373
|
+
// (transcript-grounded — it can't flake) until the user re-arms. autoDarkened() is DERIVED from the
|
|
374
|
+
// real adjudication rows on every render (never a stored flag), so writer + reader can't drift.
|
|
375
|
+
if (autoDarkened()) {
|
|
376
|
+
process.stdout.write(
|
|
377
|
+
`${BOLD}skalpel${RESET} · ${DIM}live catch paused (precision <90%) · run: skalpel catches${RESET}`,
|
|
378
|
+
);
|
|
379
|
+
return;
|
|
380
|
+
}
|
|
371
381
|
const reveal = revealNote(payload.session_id);
|
|
372
382
|
if (reveal) {
|
|
373
|
-
|
|
383
|
+
// The red carries its one-key adjudication pointer — the flight recorder that feeds precision +
|
|
384
|
+
// the auto-darken kill-switch. A `skalpel no` also suppresses this exact proof for the session.
|
|
385
|
+
process.stdout.write(
|
|
386
|
+
`${BOLD}${reveal}${RESET} ${DIM}· was this right? skalpel yes / skalpel no${RESET}`,
|
|
387
|
+
);
|
|
374
388
|
return;
|
|
375
389
|
}
|
|
376
390
|
const verdict = verdictNote();
|