vigiles 15.0.0 → 15.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude-code/run-scripts.d.ts +55 -5
- package/dist/adapters/claude-code/run-scripts.js +105 -24
- package/dist/audit-score.js +1 -1
- package/dist/check-count.d.ts +40 -0
- package/dist/check-count.js +136 -0
- package/dist/cli.js +7 -4
- package/dist/core/hook-matcher.d.ts +71 -29
- package/dist/core/hook-matcher.js +245 -131
- package/dist/core/lethal-trifecta.d.ts +31 -2
- package/dist/core/lethal-trifecta.js +32 -7
- package/dist/core/rule-meta.js +1 -1
- package/dist/core/skill-resources.js +41 -9
- package/dist/core/types.d.ts +10 -6
- package/dist/core/validate.js +3 -2
- package/dist/eval.js +6 -0
- package/dist/harness-assert.js +14 -4
- package/dist/harness-test.js +4 -0
- package/dist/run-script.js +6 -0
- package/dist/scan-core.js +56 -2
- package/dist/scan.d.ts +4 -3
- package/dist/scan.js +1 -1
- package/dist/score-core.js +1 -1
- package/dist/testing.d.ts +1 -0
- package/dist/testing.js +9 -1
- package/package.json +1 -1
|
@@ -1,8 +1,21 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* The outcome of running one script.
|
|
3
|
+
*
|
|
4
|
+
* `"vacuous"` — the script exited 0 and reported that it made ZERO checks. It
|
|
5
|
+
* neither passed nor failed: nothing was verified. See {@link statusFor}.
|
|
6
|
+
*/
|
|
7
|
+
export type ScriptStatus = "pass" | "skip" | "fail" | "vacuous";
|
|
2
8
|
export interface ScriptRunResult {
|
|
3
9
|
readonly file: string;
|
|
4
10
|
readonly code: number;
|
|
5
11
|
readonly status: ScriptStatus;
|
|
12
|
+
/**
|
|
13
|
+
* How many checks the script reported making, or `undefined` when it reported
|
|
14
|
+
* nothing at all — a script that never imports `vigiles/testing` has no way to
|
|
15
|
+
* report, and that silence is NOT a claim about it. `0` is a claim: the script
|
|
16
|
+
* loaded the library and used none of it.
|
|
17
|
+
*/
|
|
18
|
+
readonly checks?: number;
|
|
6
19
|
}
|
|
7
20
|
/**
|
|
8
21
|
* Exit code a harness/eval script uses to report itself SKIPPED (e.g. the
|
|
@@ -11,6 +24,32 @@ export interface ScriptRunResult {
|
|
|
11
24
|
* skip never fails the run. Scripts call `skip()` (vigiles/testing) to emit it.
|
|
12
25
|
*/
|
|
13
26
|
export declare const SKIP_EXIT_CODE = 77;
|
|
27
|
+
/**
|
|
28
|
+
* Classify one script's run from its exit code and its reported check count.
|
|
29
|
+
*
|
|
30
|
+
* 🔴 THE FOURTH STATE, AND WHY. Exit codes answer "did it fail?", never "did it
|
|
31
|
+
* do anything?". Measured 2026-08-08: a file whose whole body is
|
|
32
|
+
* `export default { "never runs": () => assert.equal(1, 2) }` imports fine,
|
|
33
|
+
* exits 0, and printed `✓ … 1 passed` — a false assertion, never called,
|
|
34
|
+
* reported as a pass. A consumer repo hit exactly that and now hand-copies a
|
|
35
|
+
* warning into every new harness header, because the runner could not enforce
|
|
36
|
+
* it: eight harnesses resting on a comment.
|
|
37
|
+
*
|
|
38
|
+
* So a run that ends clean having recorded ZERO checks is `"vacuous"` — its own
|
|
39
|
+
* visible state, not folded into `passed`, the same way a skip is not.
|
|
40
|
+
*
|
|
41
|
+
* NOT A FAILURE, deliberately. Harnesses in the wild predate the counter, and a
|
|
42
|
+
* tool that turned CI red on the release that taught it a new word would be
|
|
43
|
+
* punishing people for upgrading. It is loud and it is not fatal.
|
|
44
|
+
*
|
|
45
|
+
* AND SILENCE IS NOT ZERO. `checks === undefined` means the script never
|
|
46
|
+
* reported — it may not import `vigiles/testing` at all — so it stays a plain
|
|
47
|
+
* `pass`, exactly as before. Only a script that loaded the library and used
|
|
48
|
+
* none of it says zero. This is the `undefined`-vs-`[]` distinction
|
|
49
|
+
* `assertNoWrite` already draws: "nobody looked" must not read as "nothing
|
|
50
|
+
* happened".
|
|
51
|
+
*/
|
|
52
|
+
export declare function statusFor(code: number, checks: number | undefined): ScriptStatus;
|
|
14
53
|
/** Filename extensions accepted for harness/eval scripts (JS and TS). */
|
|
15
54
|
export declare const SCRIPT_EXTS: readonly ["mjs", "cjs", "js", "mts", "cts", "ts"];
|
|
16
55
|
/** Glob suffix matching every accepted script extension, e.g. `harness`. */
|
|
@@ -41,10 +80,20 @@ export declare function discoverScripts(patterns: readonly string[], defaultGlob
|
|
|
41
80
|
/**
|
|
42
81
|
* Run each script as `node <file>`, inheriting stdio so the script's own report
|
|
43
82
|
* streams to the console. `env` is merged over `process.env` for every child
|
|
44
|
-
* (e.g. `VIGILES_TRIALS`). Returns the per-file exit codes.
|
|
83
|
+
* (e.g. `VIGILES_TRIALS`). Returns the per-file exit codes + check counts.
|
|
84
|
+
*
|
|
85
|
+
* Each child is handed its OWN scratch path in `VIGILES_CHECK_COUNT_ENV`, which
|
|
86
|
+
* `vigiles/testing` writes its check count to on exit — the channel that makes
|
|
87
|
+
* "ran nothing" distinguishable from "ran and passed" (see check-count.ts). It
|
|
88
|
+
* has to be a file: stdio is inherited so the script's report streams live,
|
|
89
|
+
* which leaves no stream to parse.
|
|
45
90
|
*/
|
|
46
91
|
export declare function runScripts(files: readonly string[], cwd: string, env?: NodeJS.ProcessEnv): ScriptRunResult[];
|
|
47
|
-
/**
|
|
92
|
+
/**
|
|
93
|
+
* Whether any script FAILED. Neither a skip nor a vacuous run counts: the first
|
|
94
|
+
* declined to run, the second ran and verified nothing, and neither is evidence
|
|
95
|
+
* that anything is broken. Both are visible in the summary instead.
|
|
96
|
+
*/
|
|
48
97
|
export declare function anyFailed(results: readonly ScriptRunResult[]): boolean;
|
|
49
98
|
/**
|
|
50
99
|
* What a `test`/`eval` invocation should do about actually RUNNING the discovered
|
|
@@ -87,7 +136,8 @@ export interface RunScriptsEnv {
|
|
|
87
136
|
* CLI.
|
|
88
137
|
*/
|
|
89
138
|
export declare function decideRunScripts(o: RunScriptsEnv): RunScriptsDecision;
|
|
90
|
-
/** One line per file + an explicit pass/skip/fail tally. Skips
|
|
91
|
-
* folded into "passed" — a `⊘ SKIPPED` is loud,
|
|
139
|
+
/** One line per file + an explicit pass/skip/vacuous/fail tally. Skips and
|
|
140
|
+
* vacuous runs are SHOWN, never folded into "passed" — a `⊘ SKIPPED` is loud,
|
|
141
|
+
* not a silent green, and so is a file that verified nothing. */
|
|
92
142
|
export declare function formatScriptSummary(results: readonly ScriptRunResult[]): string;
|
|
93
143
|
//# sourceMappingURL=run-scripts.d.ts.map
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.SCRIPT_EXTS = exports.SKIP_EXIT_CODE = void 0;
|
|
4
|
+
exports.statusFor = statusFor;
|
|
4
5
|
exports.scriptGlob = scriptGlob;
|
|
5
6
|
exports.interpreterArgs = interpreterArgs;
|
|
6
7
|
exports.detectNodeCaps = detectNodeCaps;
|
|
@@ -24,7 +25,9 @@ exports.formatScriptSummary = formatScriptSummary;
|
|
|
24
25
|
const node_child_process_1 = require("node:child_process");
|
|
25
26
|
const node_path_1 = require("node:path");
|
|
26
27
|
const node_fs_1 = require("node:fs");
|
|
28
|
+
const node_os_1 = require("node:os");
|
|
27
29
|
const glob_1 = require("glob");
|
|
30
|
+
const check_count_js_1 = require("../../check-count.js");
|
|
28
31
|
/**
|
|
29
32
|
* Exit code a harness/eval script uses to report itself SKIPPED (e.g. the
|
|
30
33
|
* deterministic tier when `claude` isn't installed) — the autotools convention.
|
|
@@ -32,12 +35,37 @@ const glob_1 = require("glob");
|
|
|
32
35
|
* skip never fails the run. Scripts call `skip()` (vigiles/testing) to emit it.
|
|
33
36
|
*/
|
|
34
37
|
exports.SKIP_EXIT_CODE = 77;
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
+
/**
|
|
39
|
+
* Classify one script's run from its exit code and its reported check count.
|
|
40
|
+
*
|
|
41
|
+
* 🔴 THE FOURTH STATE, AND WHY. Exit codes answer "did it fail?", never "did it
|
|
42
|
+
* do anything?". Measured 2026-08-08: a file whose whole body is
|
|
43
|
+
* `export default { "never runs": () => assert.equal(1, 2) }` imports fine,
|
|
44
|
+
* exits 0, and printed `✓ … 1 passed` — a false assertion, never called,
|
|
45
|
+
* reported as a pass. A consumer repo hit exactly that and now hand-copies a
|
|
46
|
+
* warning into every new harness header, because the runner could not enforce
|
|
47
|
+
* it: eight harnesses resting on a comment.
|
|
48
|
+
*
|
|
49
|
+
* So a run that ends clean having recorded ZERO checks is `"vacuous"` — its own
|
|
50
|
+
* visible state, not folded into `passed`, the same way a skip is not.
|
|
51
|
+
*
|
|
52
|
+
* NOT A FAILURE, deliberately. Harnesses in the wild predate the counter, and a
|
|
53
|
+
* tool that turned CI red on the release that taught it a new word would be
|
|
54
|
+
* punishing people for upgrading. It is loud and it is not fatal.
|
|
55
|
+
*
|
|
56
|
+
* AND SILENCE IS NOT ZERO. `checks === undefined` means the script never
|
|
57
|
+
* reported — it may not import `vigiles/testing` at all — so it stays a plain
|
|
58
|
+
* `pass`, exactly as before. Only a script that loaded the library and used
|
|
59
|
+
* none of it says zero. This is the `undefined`-vs-`[]` distinction
|
|
60
|
+
* `assertNoWrite` already draws: "nobody looked" must not read as "nothing
|
|
61
|
+
* happened".
|
|
62
|
+
*/
|
|
63
|
+
function statusFor(code, checks) {
|
|
38
64
|
if (code === exports.SKIP_EXIT_CODE)
|
|
39
65
|
return "skip";
|
|
40
|
-
|
|
66
|
+
if (code !== 0)
|
|
67
|
+
return "fail";
|
|
68
|
+
return checks === 0 ? "vacuous" : "pass";
|
|
41
69
|
}
|
|
42
70
|
/** Filename extensions accepted for harness/eval scripts (JS and TS). */
|
|
43
71
|
exports.SCRIPT_EXTS = ["mjs", "cjs", "js", "mts", "cts", "ts"];
|
|
@@ -104,35 +132,73 @@ function discoverScripts(patterns, defaultGlob, cwd) {
|
|
|
104
132
|
}
|
|
105
133
|
return [...found].sort();
|
|
106
134
|
}
|
|
135
|
+
/**
|
|
136
|
+
* The count a script left behind, or `undefined` if it left none (it never
|
|
137
|
+
* imported `vigiles/testing`, or died before its exit handler). Anything that
|
|
138
|
+
* isn't a non-negative integer is treated as no report — a corrupt scratch file
|
|
139
|
+
* must not invent a verdict.
|
|
140
|
+
*/
|
|
141
|
+
function readCheckCount(path) {
|
|
142
|
+
if (!(0, node_fs_1.existsSync)(path))
|
|
143
|
+
return undefined;
|
|
144
|
+
let raw;
|
|
145
|
+
try {
|
|
146
|
+
raw = (0, node_fs_1.readFileSync)(path, "utf8").trim();
|
|
147
|
+
}
|
|
148
|
+
catch {
|
|
149
|
+
return undefined;
|
|
150
|
+
}
|
|
151
|
+
if (!/^\d+$/.test(raw))
|
|
152
|
+
return undefined;
|
|
153
|
+
return Number(raw);
|
|
154
|
+
}
|
|
107
155
|
/**
|
|
108
156
|
* Run each script as `node <file>`, inheriting stdio so the script's own report
|
|
109
157
|
* streams to the console. `env` is merged over `process.env` for every child
|
|
110
|
-
* (e.g. `VIGILES_TRIALS`). Returns the per-file exit codes.
|
|
158
|
+
* (e.g. `VIGILES_TRIALS`). Returns the per-file exit codes + check counts.
|
|
159
|
+
*
|
|
160
|
+
* Each child is handed its OWN scratch path in `VIGILES_CHECK_COUNT_ENV`, which
|
|
161
|
+
* `vigiles/testing` writes its check count to on exit — the channel that makes
|
|
162
|
+
* "ran nothing" distinguishable from "ran and passed" (see check-count.ts). It
|
|
163
|
+
* has to be a file: stdio is inherited so the script's report streams live,
|
|
164
|
+
* which leaves no stream to parse.
|
|
111
165
|
*/
|
|
112
166
|
function runScripts(files, cwd, env = {}) {
|
|
113
167
|
const caps = detectNodeCaps(cwd);
|
|
114
168
|
const results = [];
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
argv
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
169
|
+
const countDir = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-checks-"));
|
|
170
|
+
try {
|
|
171
|
+
files.forEach((file, i) => {
|
|
172
|
+
let argv;
|
|
173
|
+
try {
|
|
174
|
+
argv = interpreterArgs(file, caps);
|
|
175
|
+
}
|
|
176
|
+
catch (e) {
|
|
177
|
+
console.error(`✗ ${file}: ${e.message}`);
|
|
178
|
+
results.push({ file, code: 1, status: "fail" });
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
const countFile = (0, node_path_1.join)(countDir, `${String(i)}.count`);
|
|
182
|
+
const res = (0, node_child_process_1.spawnSync)("node", argv, {
|
|
183
|
+
cwd,
|
|
184
|
+
stdio: "inherit",
|
|
185
|
+
env: { ...process.env, ...env, [check_count_js_1.CHECK_COUNT_ENV]: countFile },
|
|
186
|
+
});
|
|
187
|
+
const code = res.status ?? 1;
|
|
188
|
+
const checks = readCheckCount(countFile);
|
|
189
|
+
results.push({ file, code, status: statusFor(code, checks), checks });
|
|
129
190
|
});
|
|
130
|
-
|
|
131
|
-
|
|
191
|
+
}
|
|
192
|
+
finally {
|
|
193
|
+
(0, node_fs_1.rmSync)(countDir, { recursive: true, force: true });
|
|
132
194
|
}
|
|
133
195
|
return results;
|
|
134
196
|
}
|
|
135
|
-
/**
|
|
197
|
+
/**
|
|
198
|
+
* Whether any script FAILED. Neither a skip nor a vacuous run counts: the first
|
|
199
|
+
* declined to run, the second ran and verified nothing, and neither is evidence
|
|
200
|
+
* that anything is broken. Both are visible in the summary instead.
|
|
201
|
+
*/
|
|
136
202
|
function anyFailed(results) {
|
|
137
203
|
return results.some((r) => r.status === "fail");
|
|
138
204
|
}
|
|
@@ -165,24 +231,39 @@ const MARK = {
|
|
|
165
231
|
pass: "✓",
|
|
166
232
|
skip: "⊘",
|
|
167
233
|
fail: "✗",
|
|
234
|
+
vacuous: "∅",
|
|
168
235
|
};
|
|
169
|
-
/** One line per file + an explicit pass/skip/fail tally. Skips
|
|
170
|
-
* folded into "passed" — a `⊘ SKIPPED` is loud,
|
|
236
|
+
/** One line per file + an explicit pass/skip/vacuous/fail tally. Skips and
|
|
237
|
+
* vacuous runs are SHOWN, never folded into "passed" — a `⊘ SKIPPED` is loud,
|
|
238
|
+
* not a silent green, and so is a file that verified nothing. */
|
|
171
239
|
function formatScriptSummary(results) {
|
|
172
240
|
const lines = results.map((r) => {
|
|
173
241
|
if (r.status === "skip")
|
|
174
242
|
return ` ⊘ ${r.file} — SKIPPED`;
|
|
175
243
|
if (r.status === "fail")
|
|
176
244
|
return ` ✗ ${r.file} (exit ${String(r.code)})`;
|
|
245
|
+
if (r.status === "vacuous") {
|
|
246
|
+
return ` ${MARK.vacuous} ${r.file} — 0 CHECKS (it ran clean and verified nothing)`;
|
|
247
|
+
}
|
|
177
248
|
return ` ${MARK.pass} ${r.file}`;
|
|
178
249
|
});
|
|
179
250
|
const n = (s) => results.filter((r) => r.status === s).length;
|
|
180
251
|
const parts = [`${String(n("pass"))} passed`];
|
|
181
252
|
if (n("skip") > 0)
|
|
182
253
|
parts.push(`${String(n("skip"))} skipped`);
|
|
254
|
+
if (n("vacuous") > 0)
|
|
255
|
+
parts.push(`${String(n("vacuous"))} with 0 checks`);
|
|
183
256
|
if (n("fail") > 0)
|
|
184
257
|
parts.push(`${String(n("fail"))} failed`);
|
|
185
258
|
lines.push(`\n${parts.join(", ")}.`);
|
|
259
|
+
// Name the remedy where it's read, once — the usual cause is a file that
|
|
260
|
+
// DEFINES tests and never calls them, and the usual second cause is a harness
|
|
261
|
+
// asserting some other way, which the runner cannot see.
|
|
262
|
+
if (n("vacuous") > 0) {
|
|
263
|
+
lines.push(` ∅ = the file loaded vigiles/testing and used none of it. Either nothing ran ` +
|
|
264
|
+
`(an exported test object nobody calls), or it asserts another way — in which ` +
|
|
265
|
+
`case call recordCheck() from vigiles/testing so those count.`);
|
|
266
|
+
}
|
|
186
267
|
return lines.join("\n");
|
|
187
268
|
}
|
|
188
269
|
//# sourceMappingURL=run-scripts.js.map
|
package/dist/audit-score.js
CHANGED
|
@@ -176,7 +176,7 @@ function structure(r) {
|
|
|
176
176
|
{
|
|
177
177
|
n: r.hookMatcherFindings.length,
|
|
178
178
|
weight: score_core_js_1.W_MISSING_HOOK,
|
|
179
|
-
label: "hook matcher(s) that
|
|
179
|
+
label: "hook matcher(s) that don't fire as written (dead, or too narrow for real MCP names)",
|
|
180
180
|
},
|
|
181
181
|
]);
|
|
182
182
|
// inherit-all (no `tools:` line) is ADVISORY, not graded: it's surfaced as a
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Env var naming the file a script writes its check count to. Set per-script by
|
|
3
|
+
* the runner (`runScripts`), read once here at import.
|
|
4
|
+
*/
|
|
5
|
+
export declare const CHECK_COUNT_ENV = "VIGILES_CHECK_COUNT_FILE";
|
|
6
|
+
/**
|
|
7
|
+
* Record `n` checks against this script's run.
|
|
8
|
+
*
|
|
9
|
+
* The tiers call it themselves, so an ordinary harness never needs to. Call it
|
|
10
|
+
* directly when you assert some OTHER way — `node:assert`, vitest's `expect`, a
|
|
11
|
+
* hand-rolled comparison — and want those visible to `vigiles test` instead of
|
|
12
|
+
* leaving it to conclude the file did nothing.
|
|
13
|
+
*/
|
|
14
|
+
export declare function recordCheck(n?: number): void;
|
|
15
|
+
/** How many checks this process has recorded so far. */
|
|
16
|
+
export declare function checksRecorded(): number;
|
|
17
|
+
/**
|
|
18
|
+
* Reset the counter AND the armed flag. For vigiles's own tests, which drive
|
|
19
|
+
* {@link armCheckReport} with fakes several times in one process; a harness
|
|
20
|
+
* script has no use for it.
|
|
21
|
+
*/
|
|
22
|
+
export declare function resetCheckCount(): void;
|
|
23
|
+
/** Injection seam for {@link armCheckReport} — the process bits it needs. */
|
|
24
|
+
export interface CheckReportEnv {
|
|
25
|
+
readonly env: NodeJS.ProcessEnv;
|
|
26
|
+
readonly onExit: (fn: () => void) => void;
|
|
27
|
+
readonly write: (path: string, contents: string) => void;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Arm the exit-time report, returning whether it armed (i.e. whether the runner
|
|
31
|
+
* asked for a count). Pure except for the two effects it is handed.
|
|
32
|
+
*
|
|
33
|
+
* It DELETES the env var after reading it. A harness spawns child processes —
|
|
34
|
+
* that is its whole job — and a child inheriting the path would write ITS count
|
|
35
|
+
* over the parent's on exit, reporting a sub-process's activity as the file's.
|
|
36
|
+
* Reading the variable once and dropping it makes that unrepresentable rather
|
|
37
|
+
* than merely unlikely.
|
|
38
|
+
*/
|
|
39
|
+
export declare function armCheckReport(deps: CheckReportEnv): boolean;
|
|
40
|
+
//# sourceMappingURL=check-count.d.ts.map
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CHECK_COUNT_ENV = void 0;
|
|
4
|
+
exports.recordCheck = recordCheck;
|
|
5
|
+
exports.checksRecorded = checksRecorded;
|
|
6
|
+
exports.resetCheckCount = resetCheckCount;
|
|
7
|
+
exports.armCheckReport = armCheckReport;
|
|
8
|
+
/**
|
|
9
|
+
* vigiles — the CHECK COUNTER: the channel a `*.harness.*` / `*.eval.*` script
|
|
10
|
+
* uses to tell `vigiles test` / `vigiles eval` how much it actually did.
|
|
11
|
+
*
|
|
12
|
+
* 🔴 WHY THIS EXISTS. The runner knew exit codes and nothing else, so a script
|
|
13
|
+
* that ran NOTHING was indistinguishable from one that ran and passed. Measured
|
|
14
|
+
* 2026-08-08 — a file whose entire content is
|
|
15
|
+
*
|
|
16
|
+
* export default { "never runs": () => assert.equal(1, 2) };
|
|
17
|
+
*
|
|
18
|
+
* imports cleanly, exits 0, and the runner printed `✓ … 1 passed`. The assertion
|
|
19
|
+
* inside it is false; it is never called, because nothing calls an exported
|
|
20
|
+
* object. Not hypothetical: a consumer repo hit it, and now hand-copies a warning
|
|
21
|
+
* into the header of every new harness file — "An earlier version exported a
|
|
22
|
+
* `tests` object; nothing ran and the runner printed ✓" — because the runner
|
|
23
|
+
* could not enforce it. Eight harnesses resting on a comment.
|
|
24
|
+
*
|
|
25
|
+
* This is the same distinction `assertNoWrite` already draws between `undefined`
|
|
26
|
+
* and `[]`: "nobody looked" and "nothing happened" must not print the same.
|
|
27
|
+
*
|
|
28
|
+
* HOW IT REPORTS. `vigiles test` puts a scratch path in `VIGILES_CHECK_COUNT_FILE`
|
|
29
|
+
* before spawning each script; on exit, a script that loaded this module writes
|
|
30
|
+
* its count there. A FILE, not stdout, because the runner inherits stdio so the
|
|
31
|
+
* script's own report streams live — there is no stream left to parse. Not an
|
|
32
|
+
* exit code either: those already carry pass/skip/fail, and overloading one would
|
|
33
|
+
* make a real failure ambiguous.
|
|
34
|
+
*
|
|
35
|
+
* WHAT COUNTS AS A CHECK: an observation vigiles can see the script make — a run
|
|
36
|
+
* through one of the tiers (`runHook`, `runHarnessTest`, `runEval`, …), an
|
|
37
|
+
* in-process compiled-hook decision asserted, or an explicit {@link recordCheck}
|
|
38
|
+
* from a harness that asserts some other way (`node:assert`, a runner's
|
|
39
|
+
* `expect`). Deliberately generous: the question this answers is "did this file
|
|
40
|
+
* exercise the harness at all", and a false "0 checks" against a script that
|
|
41
|
+
* genuinely tested something would be exactly the crying wolf the rest of the
|
|
42
|
+
* tool avoids.
|
|
43
|
+
*
|
|
44
|
+
* WHAT A MISSING COUNT MEANS: nothing at all. A script that never imports
|
|
45
|
+
* `vigiles/testing` cannot report, so the runner sees no file and treats it
|
|
46
|
+
* exactly as before — a plain pass. Silence is the legacy branch, never a
|
|
47
|
+
* verdict; only a count of literally zero is a finding. (The alternative —
|
|
48
|
+
* force-loading this module into every child with `node --import` so silence
|
|
49
|
+
* became impossible — was rejected: it would report `0` for a hand-rolled
|
|
50
|
+
* harness that spawns and asserts entirely on its own, which is a real and
|
|
51
|
+
* blameless way to write one.)
|
|
52
|
+
*/
|
|
53
|
+
const node_fs_1 = require("node:fs");
|
|
54
|
+
/**
|
|
55
|
+
* Env var naming the file a script writes its check count to. Set per-script by
|
|
56
|
+
* the runner (`runScripts`), read once here at import.
|
|
57
|
+
*/
|
|
58
|
+
exports.CHECK_COUNT_ENV = "VIGILES_CHECK_COUNT_FILE";
|
|
59
|
+
/**
|
|
60
|
+
* The counter lives on the global registry, not in module scope, so two copies
|
|
61
|
+
* of vigiles loaded in one child (a global CLI plus a local dependency, say)
|
|
62
|
+
* share ONE count instead of one copy counting while the other reports zero.
|
|
63
|
+
* The cheap version of that bug is a false "this file verified nothing".
|
|
64
|
+
*/
|
|
65
|
+
const STATE = Symbol.for("vigiles.check-count");
|
|
66
|
+
function state() {
|
|
67
|
+
const g = globalThis;
|
|
68
|
+
return (g[STATE] ??= { count: 0, armed: false });
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Record `n` checks against this script's run.
|
|
72
|
+
*
|
|
73
|
+
* The tiers call it themselves, so an ordinary harness never needs to. Call it
|
|
74
|
+
* directly when you assert some OTHER way — `node:assert`, vitest's `expect`, a
|
|
75
|
+
* hand-rolled comparison — and want those visible to `vigiles test` instead of
|
|
76
|
+
* leaving it to conclude the file did nothing.
|
|
77
|
+
*/
|
|
78
|
+
function recordCheck(n = 1) {
|
|
79
|
+
state().count += n;
|
|
80
|
+
}
|
|
81
|
+
/** How many checks this process has recorded so far. */
|
|
82
|
+
function checksRecorded() {
|
|
83
|
+
return state().count;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Reset the counter AND the armed flag. For vigiles's own tests, which drive
|
|
87
|
+
* {@link armCheckReport} with fakes several times in one process; a harness
|
|
88
|
+
* script has no use for it.
|
|
89
|
+
*/
|
|
90
|
+
function resetCheckCount() {
|
|
91
|
+
const s = state();
|
|
92
|
+
s.count = 0;
|
|
93
|
+
s.armed = false;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Arm the exit-time report, returning whether it armed (i.e. whether the runner
|
|
97
|
+
* asked for a count). Pure except for the two effects it is handed.
|
|
98
|
+
*
|
|
99
|
+
* It DELETES the env var after reading it. A harness spawns child processes —
|
|
100
|
+
* that is its whole job — and a child inheriting the path would write ITS count
|
|
101
|
+
* over the parent's on exit, reporting a sub-process's activity as the file's.
|
|
102
|
+
* Reading the variable once and dropping it makes that unrepresentable rather
|
|
103
|
+
* than merely unlikely.
|
|
104
|
+
*/
|
|
105
|
+
function armCheckReport(deps) {
|
|
106
|
+
const s = state();
|
|
107
|
+
if (s.armed)
|
|
108
|
+
return false;
|
|
109
|
+
const file = deps.env[exports.CHECK_COUNT_ENV];
|
|
110
|
+
if (file === undefined || file === "")
|
|
111
|
+
return false;
|
|
112
|
+
// `Reflect.deleteProperty`, not `delete env[KEY]`: the key is a const, which
|
|
113
|
+
// the lint rules count as a dynamic delete.
|
|
114
|
+
Reflect.deleteProperty(deps.env, exports.CHECK_COUNT_ENV);
|
|
115
|
+
s.armed = true;
|
|
116
|
+
deps.onExit(() => {
|
|
117
|
+
try {
|
|
118
|
+
deps.write(file, String(s.count));
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
// An unwritable scratch path must never turn a passing harness into a
|
|
122
|
+
// crash on the way out. No count reported = the legacy branch = a pass.
|
|
123
|
+
}
|
|
124
|
+
});
|
|
125
|
+
return true;
|
|
126
|
+
}
|
|
127
|
+
armCheckReport({
|
|
128
|
+
env: process.env,
|
|
129
|
+
onExit: (fn) => {
|
|
130
|
+
process.on("exit", fn);
|
|
131
|
+
},
|
|
132
|
+
write: (path, contents) => {
|
|
133
|
+
(0, node_fs_1.writeFileSync)(path, contents);
|
|
134
|
+
},
|
|
135
|
+
});
|
|
136
|
+
//# sourceMappingURL=check-count.js.map
|
package/dist/cli.js
CHANGED
|
@@ -1127,8 +1127,9 @@ async function runLint(restArgs, flags, config) {
|
|
|
1127
1127
|
// doesn't (block decision on a non-blocking event, or the legacy `decision`
|
|
1128
1128
|
// field on a permission-gated event). The #1 verified hook pain (#19009).
|
|
1129
1129
|
const hookBlock = checkHookBlockIneffective(config, silent, adapter, scanRoot);
|
|
1130
|
-
// 7u. Hook-matcher — a hook `matcher` that
|
|
1131
|
-
//
|
|
1130
|
+
// 7u. Hook-matcher — a hook `matcher` that doesn't fire as written (tool-name
|
|
1131
|
+
// typo, an uncompilable or unreachable MCP pattern, one too narrow for real
|
|
1132
|
+
// server naming, or an undeclared MCP server).
|
|
1132
1133
|
const hookMatcher = checkHookMatcher(config, silent, adapter, scanRoot);
|
|
1133
1134
|
// 8. Validate vigiles builder calls inside markdown code blocks. Default
|
|
1134
1135
|
// is to validate every ref; illustrative blocks opt out via
|
|
@@ -3445,8 +3446,10 @@ function checkHookBlockIneffective(config, silent, adapter, scanRoot) {
|
|
|
3445
3446
|
return { issues: found.length, errors: sev === "error" ? found.length : 0 };
|
|
3446
3447
|
}
|
|
3447
3448
|
/**
|
|
3448
|
-
* Apply the `hook-matcher` rule: a hook `matcher` string that
|
|
3449
|
-
*
|
|
3449
|
+
* Apply the `hook-matcher` rule: a hook `matcher` string that doesn't fire as
|
|
3450
|
+
* written — a tool-name typo (`bash`→`Bash`), a matcher that doesn't compile, an
|
|
3451
|
+
* MCP pattern that reaches no tool name or is too narrow for real server naming,
|
|
3452
|
+
* or an undeclared MCP server.
|
|
3450
3453
|
* Reuses `scanPlugin`'s `hookMatcherFindings` (one detector, no drift). Warning
|
|
3451
3454
|
* by default; "error" gates CI.
|
|
3452
3455
|
*/
|
|
@@ -1,42 +1,81 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Hook-matcher verification — the cross-referencing moat applied to the MATCHER
|
|
3
|
-
* string inside a hook registration. A
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* (research/hook-pain-points.md).
|
|
3
|
+
* string inside a hook registration. A hook fires only when its `matcher` selects
|
|
4
|
+
* the tool name the harness emits; a typo or an unmatchable pattern silently
|
|
5
|
+
* prevents the hook from ever running — exactly the FALSE CONFIDENCE failure the
|
|
6
|
+
* compiled-hooks design exists to eliminate (research/hook-pain-points.md).
|
|
8
7
|
*
|
|
9
|
-
*
|
|
8
|
+
* ## The matching semantics this detector models (MEASURED, not assumed)
|
|
10
9
|
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* `tool-contract.ts` — same edit-distance logic, same ≤ 2 confidence bound.
|
|
10
|
+
* A matcher is NOT a literal tool name — it is a pattern. Measured against the
|
|
11
|
+
* real `claude` CLI (2.1.226) with the scripted mock model, one hook per run,
|
|
12
|
+
* marker file as the oracle:
|
|
15
13
|
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
14
|
+
* | matcher | tool called | fired |
|
|
15
|
+
* | -------------------- | ----------------------------------------------- | ----- |
|
|
16
|
+
* | `Write` | `Write` | yes |
|
|
17
|
+
* | `Writ` / `rit` | `Write` | NO |
|
|
18
|
+
* | `rit.` | `Write` | yes |
|
|
19
|
+
* | `W(rit)e` | `Write` | yes |
|
|
20
|
+
* | `mcp__.*` | `mcp__some_server__list_events` | yes |
|
|
21
|
+
* | `mcp__.*__.*` | `mcp__some_server__list_events` | yes |
|
|
22
|
+
* | `mcp__[^_]+__[^_]+` | `mcp__some_server__list_events` | NO |
|
|
23
|
+
* | `mcp__[^_]+__[^_]+` | `mcp__4f54037d-…-6130f3da1ef8__list_events` | yes |
|
|
24
|
+
* | `mcp__\w+__\w+` | `mcp__some_server__list_events` | yes |
|
|
25
|
+
* | `mcp__\w+__\w+` | `mcp__4f54037d-…-6130f3da1ef8__list_events` | NO |
|
|
20
26
|
*
|
|
21
|
-
*
|
|
22
|
-
* is NOT in the plugin's declared MCP servers. Gated EXACTLY like
|
|
23
|
-
* `mcp-tool-resolves`: (a) no declared set → skip (reaches global/project
|
|
24
|
-
* servers); (b) built-ins allowlisted via `dialect.knownMcpServers`; (c) the
|
|
25
|
-
* plugin-namespaced `mcp__plugin_…__…` form is skipped. Reuses `mcpToolServer`
|
|
26
|
-
* from `mcp-tool.ts` for the extraction — one parser, no drift.
|
|
27
|
+
* Two facts follow, and the detector encodes exactly these:
|
|
27
28
|
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
29
|
+
* 1. A matcher with NO regex metacharacter is matched by STRING EQUALITY
|
|
30
|
+
* (`rit` does not fire on `Write`, though it is a substring).
|
|
31
|
+
* 2. A matcher WITH metacharacters is matched as an UNANCHORED regex
|
|
32
|
+
* (`rit.` fires on `Write`; `mcp__[^_]+__[^_]+` fires on the hyphenated
|
|
33
|
+
* server because `[^_]+` only has to reach *into* the tool segment).
|
|
34
|
+
*
|
|
35
|
+
* ## What is flagged (five kinds)
|
|
36
|
+
*
|
|
37
|
+
* 1. **tool-typo** — a LITERAL bare token that is a close typo (edit distance ≤ 2)
|
|
38
|
+
* of a real built-in tool (`bash` → `Bash`). Reuses `closestTool`.
|
|
39
|
+
* 2. **invalid-regex** — the matcher does not COMPILE. A dead hook no other check
|
|
40
|
+
* catches, and its own finding rather than a silent skip.
|
|
41
|
+
* 3. **mcp-form** — an MCP-ish matcher that can match NO MCP tool name at all:
|
|
42
|
+
* a literal that isn't the `mcp__<server>__<tool>` shape (`mcp_memory_search`),
|
|
43
|
+
* or a pattern that matches none of the synthetic probes (`mcp_memory_*`).
|
|
44
|
+
* 4. **mcp-narrow** — an MCP-ish pattern that DOES fire, but not on the server
|
|
45
|
+
* naming that occurs in the wild. `mcp__[^_]+__[^_]+` cannot cross the `_` in
|
|
46
|
+
* `Google_Calendar`; `mcp__\w+__\w+` cannot cross the `-` in the uuid form —
|
|
47
|
+
* and the SAME server appears both ways in different sessions. This is a real
|
|
48
|
+
* gap but it is NOT "never fires", and the message says so.
|
|
49
|
+
* 5. **mcp-undeclared** — a matcher pinning a literal `mcp__<server>__…` the
|
|
50
|
+
* plugin doesn't declare. Gated exactly like `mcp-tool-resolves`.
|
|
51
|
+
*
|
|
52
|
+
* ## Why patterns are validated by PROBING, not by shape
|
|
53
|
+
*
|
|
54
|
+
* The server segment is not stable: the same Google Calendar server is
|
|
55
|
+
* `mcp__Google_Calendar__list_events` in one session and
|
|
56
|
+
* `mcp__4f54037d-…__list_events` in another, so a hook keyed to one literal id
|
|
57
|
+
* dies silently when the id changes — patterns are the CORRECT authoring form.
|
|
58
|
+
* Validating a pattern against the literal shape therefore inverts the verdict:
|
|
59
|
+
* it rejected `mcp__.*` (fires on everything) and accepted `mcp__[^_]+__[^_]+`
|
|
60
|
+
* (fires on nothing with an underscored server). So a pattern is instead COMPILED
|
|
61
|
+
* and run against synthetic probes — including a probe whose server segment holds
|
|
62
|
+
* an underscore and one whose server segment holds hyphens, because both occur.
|
|
63
|
+
* Probes are also DERIVED from the matcher's own literal segments, so a correctly
|
|
64
|
+
* server-scoped `mcp__memory__.*` is never called unreachable (#131).
|
|
65
|
+
*
|
|
66
|
+
* FP-SAFE, unchanged in spirit: a match-all (`*`, `.*`, `**`, empty) or an
|
|
67
|
+
* ALTERNATION (`Edit|Write`) is skipped — each arm of an alternation would have
|
|
68
|
+
* to be judged separately, and a mixed arm set is legitimate. A non-MCP token
|
|
69
|
+
* carrying regex/glob syntax is skipped too (it is a pattern over built-in tool
|
|
70
|
+
* names, not a name).
|
|
32
71
|
*
|
|
33
72
|
* Pure + ONE detector reused by `scan` + the `hook-matcher` lint rule
|
|
34
73
|
* (one-detector-no-drift). The dialect is injected (core ⊄ adapter).
|
|
35
74
|
*/
|
|
36
75
|
import type { HarnessDialect } from "./dialect.js";
|
|
37
76
|
/** Which matching failure was detected in the hook matcher string. */
|
|
38
|
-
export type HookMatcherKind = "tool-typo" | "mcp-form" | "mcp-undeclared";
|
|
39
|
-
/** One finding for a hook matcher that
|
|
77
|
+
export type HookMatcherKind = "tool-typo" | "invalid-regex" | "mcp-form" | "mcp-narrow" | "mcp-undeclared";
|
|
78
|
+
/** One finding for a hook matcher that doesn't fire the way it reads. */
|
|
40
79
|
export interface HookMatcherFinding {
|
|
41
80
|
/** The matcher string exactly as written. */
|
|
42
81
|
readonly matcher: string;
|
|
@@ -46,6 +85,9 @@ export interface HookMatcherFinding {
|
|
|
46
85
|
* The corrected matcher when the intent is recoverable (e.g. `Bash` for
|
|
47
86
|
* `bash`, `mcp__memory__.*` for `mcp_memory_*`). Absent when the server
|
|
48
87
|
* segment can't be recovered from a malformed MCP form.
|
|
88
|
+
*
|
|
89
|
+
* INVARIANT (property-tested): a suggestion, fed back through this detector,
|
|
90
|
+
* produces no finding — the advice converges in one step.
|
|
49
91
|
*/
|
|
50
92
|
readonly suggestion?: string;
|
|
51
93
|
/** A ready-to-show, actionable message. */
|
|
@@ -57,10 +99,10 @@ export interface HookMatcherEntry {
|
|
|
57
99
|
readonly matcher: string;
|
|
58
100
|
}
|
|
59
101
|
/**
|
|
60
|
-
* Verify hook-matcher strings for the
|
|
102
|
+
* Verify hook-matcher strings for the ways a matcher fails to fire as written.
|
|
61
103
|
* Returns one {@link HookMatcherFinding} per offending entry. De-duplicates
|
|
62
|
-
* repeated matchers. Returns `[]` when
|
|
63
|
-
*
|
|
104
|
+
* repeated matchers. Returns `[]` when every matcher is FP-safe to skip or is
|
|
105
|
+
* correct.
|
|
64
106
|
*/
|
|
65
107
|
export declare function hookMatcherIssues(entries: readonly HookMatcherEntry[], declaredServers: readonly string[], dialect: HarnessDialect): HookMatcherFinding[];
|
|
66
108
|
//# sourceMappingURL=hook-matcher.d.ts.map
|