vigiles 15.0.2 → 15.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude-code/layout.js +3 -0
- package/dist/adapters/claude-code/run-scripts.d.ts +21 -9
- package/dist/adapters/claude-code/run-scripts.js +33 -22
- package/dist/adapters/codex/driver.js +3 -1
- package/dist/adapters/codex/eval.js +5 -0
- package/dist/audit-score.js +32 -6
- package/dist/check-count.d.ts +62 -3
- package/dist/check-count.js +123 -5
- package/dist/check.d.ts +55 -0
- package/dist/check.js +91 -0
- package/dist/cli.js +456 -54
- package/dist/core/bash-effects.d.ts +41 -0
- package/dist/core/bash-effects.js +278 -0
- package/dist/core/foreign-runner.d.ts +185 -0
- package/dist/core/foreign-runner.js +228 -0
- package/dist/core/harness-driver.d.ts +18 -1
- package/dist/core/hook-program.d.ts +319 -20
- package/dist/core/hook-program.js +743 -49
- package/dist/core/layout.d.ts +13 -0
- package/dist/core/lethal-trifecta.d.ts +93 -0
- package/dist/core/lethal-trifecta.js +409 -65
- package/dist/core/markdown.d.ts +32 -0
- package/dist/core/markdown.js +36 -0
- package/dist/core/merge-conflict.d.ts +50 -0
- package/dist/core/merge-conflict.js +84 -0
- package/dist/core/test-file-ext.d.ts +80 -0
- package/dist/core/test-file-ext.js +90 -0
- package/dist/core/types.d.ts +11 -0
- package/dist/coverage-artifact.d.ts +383 -0
- package/dist/coverage-artifact.js +586 -0
- package/dist/coverage-evidence.d.ts +87 -79
- package/dist/coverage-evidence.js +212 -211
- package/dist/coverage-probe.d.ts +125 -0
- package/dist/coverage-probe.js +700 -0
- package/dist/doc-commands.d.ts +119 -0
- package/dist/doc-commands.js +158 -0
- package/dist/eval-lock.d.ts +9 -0
- package/dist/eval-lock.js +14 -2
- package/dist/eval.js +28 -0
- package/dist/fs-walk.d.ts +80 -0
- package/dist/fs-walk.js +148 -0
- package/dist/harness-assert.js +42 -3
- package/dist/harness-test.js +31 -2
- package/dist/judge.js +4 -0
- package/dist/leaderboard.d.ts +16 -1
- package/dist/leaderboard.js +19 -2
- package/dist/load-hook.js +6 -1
- package/dist/mock-model.d.ts +58 -4
- package/dist/mock-model.js +133 -5
- package/dist/observe.d.ts +1 -1
- package/dist/observe.js +44 -1
- package/dist/plugin-loader.js +52 -13
- package/dist/run-hook.d.ts +58 -0
- package/dist/run-hook.js +70 -0
- package/dist/run-script.js +75 -0
- package/dist/scaffold-test.d.ts +3 -0
- package/dist/scaffold-test.js +10 -5
- package/dist/scan-behavioral.js +4 -0
- package/dist/scan-core.d.ts +22 -0
- package/dist/scan-core.js +46 -14
- package/dist/scan-files.js +19 -2
- package/dist/scan.d.ts +22 -5
- package/dist/scan.js +67 -13
- package/dist/skill-contract.d.ts +46 -0
- package/dist/skill-contract.js +201 -0
- package/dist/skill-refs.d.ts +67 -0
- package/dist/skill-refs.js +117 -0
- package/dist/test-coverage-files.d.ts +7 -1
- package/dist/test-coverage-files.js +66 -55
- package/dist/test-coverage.d.ts +155 -19
- package/dist/test-coverage.js +430 -91
- package/dist/testing.d.ts +8 -2
- package/dist/testing.js +19 -1
- package/dist/trigger-containment.d.ts +85 -0
- package/dist/trigger-containment.js +125 -0
- package/dist/ts-runner-caps.d.ts +16 -0
- package/dist/ts-runner-caps.js +43 -0
- package/dist/unit.d.ts +2 -2
- package/dist/unit.js +2 -1
- package/hooks/eval-lock-nudge.sh +14 -6
- package/package.json +2 -2
- package/skills/test-harness/SKILL.md +68 -2
|
@@ -18,6 +18,9 @@ exports.claudeCodeLayout = {
|
|
|
18
18
|
commandDir: "commands",
|
|
19
19
|
materializeRoot: ".claude",
|
|
20
20
|
pluginRootToken: "${CLAUDE_PLUGIN_ROOT}",
|
|
21
|
+
// Both names Claude Code uses for the project root (mirrors the
|
|
22
|
+
// `NON_PLUGIN_VARS` set in plugin-loader.ts / scan-files.ts).
|
|
23
|
+
projectRootTokens: ["${CLAUDE_PROJECT_DIR}", "${CLAUDE_PROJECT}"],
|
|
21
24
|
mcpConfigFile: ".mcp.json",
|
|
22
25
|
mcpManifestKey: "mcpServers",
|
|
23
26
|
intraRefDirs: ["hooks", "skills", "agents", "commands"],
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type SurfaceProbe } from "../../check-count.js";
|
|
1
2
|
/**
|
|
2
3
|
* The outcome of running one script.
|
|
3
4
|
*
|
|
@@ -16,6 +17,17 @@ export interface ScriptRunResult {
|
|
|
16
17
|
* loaded the library and used none of it.
|
|
17
18
|
*/
|
|
18
19
|
readonly checks?: number;
|
|
20
|
+
/**
|
|
21
|
+
* The surfaces this script was seen to exercise — derived by the tiers from the
|
|
22
|
+
* command they ran and the transcripts they got back, never declared by the
|
|
23
|
+
* author (see `coverage-probe.ts`). Feeds `.vigiles/coverage.json`, which lets
|
|
24
|
+
* coverage answer "tested?" by EXECUTION instead of by file name.
|
|
25
|
+
*
|
|
26
|
+
* Absent for a script that reported nothing, and empty for one that reported a
|
|
27
|
+
* count but exercised no identifiable surface — a unit test of a pure helper,
|
|
28
|
+
* say. Neither is a finding.
|
|
29
|
+
*/
|
|
30
|
+
readonly surfaces?: readonly SurfaceProbe[];
|
|
19
31
|
}
|
|
20
32
|
/**
|
|
21
33
|
* Exit code a harness/eval script uses to report itself SKIPPED (e.g. the
|
|
@@ -54,22 +66,18 @@ export declare function statusFor(code: number, checks: number | undefined): Scr
|
|
|
54
66
|
export declare const SCRIPT_EXTS: readonly ["mjs", "cjs", "js", "mts", "cts", "ts"];
|
|
55
67
|
/** Glob suffix matching every accepted script extension, e.g. `harness`. */
|
|
56
68
|
export declare function scriptGlob(kind: "harness" | "eval"): string;
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
/** `tsx` is installed locally (the preferred, version-agnostic TS loader). */
|
|
60
|
-
readonly tsx: boolean;
|
|
61
|
-
/** Node supports `--experimental-strip-types` (>= 22.6). */
|
|
62
|
-
readonly stripTypes: boolean;
|
|
63
|
-
}
|
|
69
|
+
export { detectNodeCaps, canRunTypeScript, type NodeCaps, } from "../../ts-runner-caps.js";
|
|
70
|
+
import type { NodeCaps } from "../../ts-runner-caps.js";
|
|
64
71
|
/**
|
|
65
72
|
* The `node` argv (after the binary) to run a single script. Plain JS runs
|
|
66
73
|
* directly; a TypeScript script picks `tsx` when available, else Node's native
|
|
67
74
|
* type stripping. Throws a clear, actionable error when neither is available.
|
|
68
75
|
* Pure — exported for testing.
|
|
76
|
+
*
|
|
77
|
+
* 🔴 The disjunction below is `canRunTypeScript` — keep them together. When they
|
|
78
|
+
* drifted, the tool recommended a `.ts` file and then refused to run it.
|
|
69
79
|
*/
|
|
70
80
|
export declare function interpreterArgs(file: string, caps: NodeCaps): string[];
|
|
71
|
-
/** Detect TS-running capabilities for a project root. */
|
|
72
|
-
export declare function detectNodeCaps(cwd: string): NodeCaps;
|
|
73
81
|
/**
|
|
74
82
|
* Expand the given path/glob patterns into concrete script files. A pattern
|
|
75
83
|
* that is an existing file passes through unchanged; anything else is treated
|
|
@@ -87,6 +95,10 @@ export declare function discoverScripts(patterns: readonly string[], defaultGlob
|
|
|
87
95
|
* "ran nothing" distinguishable from "ran and passed" (see check-count.ts). It
|
|
88
96
|
* has to be a file: stdio is inherited so the script's report streams live,
|
|
89
97
|
* which leaves no stream to parse.
|
|
98
|
+
*
|
|
99
|
+
* The same channel carries WHICH SURFACES the script exercised, so the caller can
|
|
100
|
+
* record them (`.vigiles/coverage.json`) and coverage can answer "tested?" from
|
|
101
|
+
* execution rather than from a matching file name.
|
|
90
102
|
*/
|
|
91
103
|
export declare function runScripts(files: readonly string[], cwd: string, env?: NodeJS.ProcessEnv): ScriptRunResult[];
|
|
92
104
|
/**
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.SCRIPT_EXTS = exports.SKIP_EXIT_CODE = void 0;
|
|
3
|
+
exports.canRunTypeScript = exports.detectNodeCaps = exports.SCRIPT_EXTS = exports.SKIP_EXIT_CODE = void 0;
|
|
4
4
|
exports.statusFor = statusFor;
|
|
5
5
|
exports.scriptGlob = scriptGlob;
|
|
6
6
|
exports.interpreterArgs = interpreterArgs;
|
|
7
|
-
exports.detectNodeCaps = detectNodeCaps;
|
|
8
7
|
exports.discoverScripts = discoverScripts;
|
|
9
8
|
exports.runScripts = runScripts;
|
|
10
9
|
exports.anyFailed = anyFailed;
|
|
@@ -74,11 +73,23 @@ function scriptGlob(kind) {
|
|
|
74
73
|
return `**/*.${kind}.{${exports.SCRIPT_EXTS.join(",")}}`;
|
|
75
74
|
}
|
|
76
75
|
const TS_EXT = /\.(?:m|c)?ts$/;
|
|
76
|
+
// The capability probe lives in `src/ts-runner-caps.ts` so the SUGGESTER
|
|
77
|
+
// (`testFileExt`, harness-agnostic) can ask the same question this runner
|
|
78
|
+
// answers. It used to recommend `.ts` from a `tsconfig.json` alone, on a Node 20
|
|
79
|
+
// box with no `tsx` — a file `interpreterArgs` then refused to run. Re-exported
|
|
80
|
+
// here so every existing importer of `run-scripts.js` is unchanged.
|
|
81
|
+
var ts_runner_caps_js_1 = require("../../ts-runner-caps.js");
|
|
82
|
+
Object.defineProperty(exports, "detectNodeCaps", { enumerable: true, get: function () { return ts_runner_caps_js_1.detectNodeCaps; } });
|
|
83
|
+
Object.defineProperty(exports, "canRunTypeScript", { enumerable: true, get: function () { return ts_runner_caps_js_1.canRunTypeScript; } });
|
|
84
|
+
const ts_runner_caps_js_2 = require("../../ts-runner-caps.js");
|
|
77
85
|
/**
|
|
78
86
|
* The `node` argv (after the binary) to run a single script. Plain JS runs
|
|
79
87
|
* directly; a TypeScript script picks `tsx` when available, else Node's native
|
|
80
88
|
* type stripping. Throws a clear, actionable error when neither is available.
|
|
81
89
|
* Pure — exported for testing.
|
|
90
|
+
*
|
|
91
|
+
* 🔴 The disjunction below is `canRunTypeScript` — keep them together. When they
|
|
92
|
+
* drifted, the tool recommended a `.ts` file and then refused to run it.
|
|
82
93
|
*/
|
|
83
94
|
function interpreterArgs(file, caps) {
|
|
84
95
|
if (!TS_EXT.test(file))
|
|
@@ -90,13 +101,6 @@ function interpreterArgs(file, caps) {
|
|
|
90
101
|
throw new Error(`Cannot run TypeScript test script "${file}": install tsx ` +
|
|
91
102
|
`(npm i -D tsx) or use Node >= 22.6, or author it as a .mjs file.`);
|
|
92
103
|
}
|
|
93
|
-
/** Detect TS-running capabilities for a project root. */
|
|
94
|
-
function detectNodeCaps(cwd) {
|
|
95
|
-
const tsx = (0, node_fs_1.existsSync)((0, node_path_1.resolve)(cwd, "node_modules/tsx/package.json")) ||
|
|
96
|
-
(0, node_fs_1.existsSync)((0, node_path_1.resolve)(cwd, "node_modules/.bin/tsx"));
|
|
97
|
-
const stripTypes = process.allowedNodeEnvironmentFlags.has("--experimental-strip-types");
|
|
98
|
-
return { tsx, stripTypes };
|
|
99
|
-
}
|
|
100
104
|
/**
|
|
101
105
|
* Expand the given path/glob patterns into concrete script files. A pattern
|
|
102
106
|
* that is an existing file passes through unchanged; anything else is treated
|
|
@@ -133,24 +137,21 @@ function discoverScripts(patterns, defaultGlob, cwd) {
|
|
|
133
137
|
return [...found].sort();
|
|
134
138
|
}
|
|
135
139
|
/**
|
|
136
|
-
*
|
|
137
|
-
* imported `vigiles/testing`, or died before its exit handler).
|
|
138
|
-
*
|
|
139
|
-
* must not invent a
|
|
140
|
+
* What a script left behind, or `undefined` if it left nothing (it never
|
|
141
|
+
* imported `vigiles/testing`, or died before its exit handler). The parse itself
|
|
142
|
+
* lives beside the writer in `check-count.ts` so the two cannot drift; anything
|
|
143
|
+
* malformed is treated as no report — a corrupt scratch file must not invent a
|
|
144
|
+
* verdict.
|
|
140
145
|
*/
|
|
141
|
-
function
|
|
146
|
+
function readCheckReport(path) {
|
|
142
147
|
if (!(0, node_fs_1.existsSync)(path))
|
|
143
148
|
return undefined;
|
|
144
|
-
let raw;
|
|
145
149
|
try {
|
|
146
|
-
|
|
150
|
+
return (0, check_count_js_1.parseCheckReport)((0, node_fs_1.readFileSync)(path, "utf8"));
|
|
147
151
|
}
|
|
148
152
|
catch {
|
|
149
153
|
return undefined;
|
|
150
154
|
}
|
|
151
|
-
if (!/^\d+$/.test(raw))
|
|
152
|
-
return undefined;
|
|
153
|
-
return Number(raw);
|
|
154
155
|
}
|
|
155
156
|
/**
|
|
156
157
|
* Run each script as `node <file>`, inheriting stdio so the script's own report
|
|
@@ -162,9 +163,13 @@ function readCheckCount(path) {
|
|
|
162
163
|
* "ran nothing" distinguishable from "ran and passed" (see check-count.ts). It
|
|
163
164
|
* has to be a file: stdio is inherited so the script's report streams live,
|
|
164
165
|
* which leaves no stream to parse.
|
|
166
|
+
*
|
|
167
|
+
* The same channel carries WHICH SURFACES the script exercised, so the caller can
|
|
168
|
+
* record them (`.vigiles/coverage.json`) and coverage can answer "tested?" from
|
|
169
|
+
* execution rather than from a matching file name.
|
|
165
170
|
*/
|
|
166
171
|
function runScripts(files, cwd, env = {}) {
|
|
167
|
-
const caps = detectNodeCaps(cwd);
|
|
172
|
+
const caps = (0, ts_runner_caps_js_2.detectNodeCaps)(cwd);
|
|
168
173
|
const results = [];
|
|
169
174
|
const countDir = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-checks-"));
|
|
170
175
|
try {
|
|
@@ -185,8 +190,14 @@ function runScripts(files, cwd, env = {}) {
|
|
|
185
190
|
env: { ...process.env, ...env, [check_count_js_1.CHECK_COUNT_ENV]: countFile },
|
|
186
191
|
});
|
|
187
192
|
const code = res.status ?? 1;
|
|
188
|
-
const
|
|
189
|
-
results.push({
|
|
193
|
+
const report = readCheckReport(countFile);
|
|
194
|
+
results.push({
|
|
195
|
+
file,
|
|
196
|
+
code,
|
|
197
|
+
status: statusFor(code, report?.checks),
|
|
198
|
+
checks: report?.checks,
|
|
199
|
+
...(report ? { surfaces: report.surfaces } : {}),
|
|
200
|
+
});
|
|
190
201
|
});
|
|
191
202
|
}
|
|
192
203
|
finally {
|
|
@@ -53,7 +53,9 @@ function parseCodexRun(stdout) {
|
|
|
53
53
|
codex-backed suite, the pure helpers above carry the testable logic. */
|
|
54
54
|
function codexAvailable() {
|
|
55
55
|
try {
|
|
56
|
-
return (
|
|
56
|
+
return (
|
|
57
|
+
// vigiles:free-tier — availability probe; `--version` prints and exits.
|
|
58
|
+
(0, node_child_process_1.spawnSync)(runtime_js_1.codexRuntime.agentBinary, ["--version"], {
|
|
57
59
|
stdio: "ignore",
|
|
58
60
|
}).status === 0);
|
|
59
61
|
}
|
|
@@ -35,6 +35,7 @@ exports.installCodexSkills = installCodexSkills;
|
|
|
35
35
|
exports.codexEvalAgentRunner = codexEvalAgentRunner;
|
|
36
36
|
exports.codexEvalRunner = codexEvalRunner;
|
|
37
37
|
const node_child_process_1 = require("node:child_process");
|
|
38
|
+
const foreign_runner_js_1 = require("../../core/foreign-runner.js");
|
|
38
39
|
const node_fs_1 = require("node:fs");
|
|
39
40
|
const node_path_1 = require("node:path");
|
|
40
41
|
const str = (v) => (typeof v === "string" ? v : "");
|
|
@@ -225,6 +226,10 @@ exports.codexEvalDriver = {
|
|
|
225
226
|
* network egress to the model backend.
|
|
226
227
|
*/
|
|
227
228
|
function codexEvalRunner(args) {
|
|
229
|
+
// The DOCUMENTED alternative to the default Claude runner, and the one the
|
|
230
|
+
// untested-skill nudge now recommends — so it is exactly as able to be
|
|
231
|
+
// collected by a stray `npx vitest run`, and exactly as expensive.
|
|
232
|
+
(0, foreign_runner_js_1.refuseUnderForeignRunner)("driving `codex exec`");
|
|
228
233
|
const r = (0, node_child_process_1.spawnSync)("codex", [
|
|
229
234
|
"exec",
|
|
230
235
|
"--json",
|
package/dist/audit-score.js
CHANGED
|
@@ -82,6 +82,15 @@ function truthfulness(r) {
|
|
|
82
82
|
weight: score_core_js_1.W_DANGLING_REF,
|
|
83
83
|
label: "broken intra-plugin reference(s)",
|
|
84
84
|
},
|
|
85
|
+
{
|
|
86
|
+
// Same weight and same category as the path-shaped one above: both are a
|
|
87
|
+
// reference that resolves to nothing. Counted separately only because the
|
|
88
|
+
// labels must read differently — "a path that isn't there" and "a skill
|
|
89
|
+
// that isn't there" send the reader to different fixes.
|
|
90
|
+
n: r.skillRefIssues?.length ?? 0,
|
|
91
|
+
weight: score_core_js_1.W_DANGLING_REF,
|
|
92
|
+
label: "skill(s) naming a sibling skill that does not exist",
|
|
93
|
+
},
|
|
85
94
|
{
|
|
86
95
|
n: missingHooks,
|
|
87
96
|
weight: score_core_js_1.W_MISSING_HOOK,
|
|
@@ -198,10 +207,12 @@ function structure(r) {
|
|
|
198
207
|
}
|
|
199
208
|
/**
|
|
200
209
|
* SAFETY — fed by the STATIC lethal-trifecta check (`report.trifectaFindings`), a
|
|
201
|
-
* GRADED ring. EVERY unit holding all three capability legs counts, whether
|
|
202
|
-
*
|
|
203
|
-
* `
|
|
204
|
-
*
|
|
210
|
+
* GRADED ring. EVERY unit holding all three capability legs counts, whether a
|
|
211
|
+
* subagent's `tools:` NAMED them (`"hard"`) or the unit INHERITED them
|
|
212
|
+
* (`"advisory"` — a subagent with no `tools:` line, or a skill with no
|
|
213
|
+
* `disallowed-tools:` fence, which is every skill by default: `allowed-tools:` is
|
|
214
|
+
* a pre-approval and restricts nothing). Grading only the explicit case made the
|
|
215
|
+
* ring non-monotone:
|
|
205
216
|
* declaring a tool contract — a genuine risk REDUCTION — could only ever lower the
|
|
206
217
|
* score. The penalty is the shared {@link trifectaExposure} (capped against the
|
|
207
218
|
* share of the surface exposed), the SAME number `reportDeductions` sums into the
|
|
@@ -230,10 +241,24 @@ function safety(r) {
|
|
|
230
241
|
}
|
|
231
242
|
// NAME the inherited ones: they're both the worst (every capability, not just
|
|
232
243
|
// the three legs) and the cheapest to fix — declare a contract that drops a leg.
|
|
244
|
+
//
|
|
245
|
+
// EXCEPT the unfenced skills, which collapse into ONE line. A skill's
|
|
246
|
+
// `allowed-tools:` pre-approves rather than restricts (measured 2026-08-11), so
|
|
247
|
+
// EVERY skill without a `disallowed-tools:` line is in this state — naming them
|
|
248
|
+
// one at a time would bury the subagent findings under a list as long as the
|
|
249
|
+
// skill corpus. Same reasoning as the report section; see `trifectaLines`.
|
|
250
|
+
const unfenced = r.trifectaFindings.filter((f) => f.kind === "skill" && f.finding.fence === "none");
|
|
233
251
|
for (const f of r.trifectaFindings) {
|
|
234
252
|
if (f.finding.severity !== "advisory")
|
|
235
253
|
continue;
|
|
236
|
-
|
|
254
|
+
if (unfenced.includes(f))
|
|
255
|
+
continue;
|
|
256
|
+
findings.push(f.kind === "skill"
|
|
257
|
+
? `${f.name}: ${f.finding.message}`
|
|
258
|
+
: `${f.name} inherits all tools — the "lethal trifecta" (reads data, reaches the web, runs commands) plus every other capability, so a prompt injection could exfiltrate secrets`);
|
|
259
|
+
}
|
|
260
|
+
if (unfenced.length > 0) {
|
|
261
|
+
findings.push(`${String(unfenced.length)} skill(s) declare no \`disallowed-tools:\` fence, so each inherits every tool the session grants — reads data, reaches the web, runs commands. \`allowed-tools:\` pre-approves, it does not restrict, so narrowing it does not reduce this; one \`disallowed-tools:\` line per skill drops a leg.`);
|
|
237
262
|
}
|
|
238
263
|
return {
|
|
239
264
|
key: "Safety",
|
|
@@ -243,7 +268,8 @@ function safety(r) {
|
|
|
243
268
|
};
|
|
244
269
|
}
|
|
245
270
|
/**
|
|
246
|
-
* TESTED — DETERMINISTIC harness coverage only (`*.harness
|
|
271
|
+
* TESTED — DETERMINISTIC harness coverage only (`*.harness.*`, plus any custom
|
|
272
|
+
* `testGlobs`):
|
|
247
273
|
* free, milliseconds, every push. It answers "does this gate still catch what it
|
|
248
274
|
* claims?" The real-model tier is a SEPARATE ring ({@link evaluated}) because it
|
|
249
275
|
* differs on cost, on cadence AND on the question it answers — folding both into
|
package/dist/check-count.d.ts
CHANGED
|
@@ -3,6 +3,31 @@
|
|
|
3
3
|
* the runner (`runScripts`), read once here at import.
|
|
4
4
|
*/
|
|
5
5
|
export declare const CHECK_COUNT_ENV = "VIGILES_CHECK_COUNT_FILE";
|
|
6
|
+
/**
|
|
7
|
+
* How a tier came to name a surface. Not a strength ranking — a statement of
|
|
8
|
+
* what the machinery went by, so a reader can tell an inference from a sighting:
|
|
9
|
+
*
|
|
10
|
+
* - `command` — the executed command line named a program file (`runScript` /
|
|
11
|
+
* `runHook`). The strongest kind: that exact path was handed to a process.
|
|
12
|
+
* - `fired` — the surface appears in the RUN'S TRANSCRIPT as having activated (a
|
|
13
|
+
* `Skill` tool call that resolved, a hook that reported). For a skill this is
|
|
14
|
+
* the only honest attribution: what was INSTALLED is a set, what RAN is one.
|
|
15
|
+
*
|
|
16
|
+
* There is deliberately NO author-declared origin. A `surface:` field on a spec
|
|
17
|
+
* would be `vigiles:covers` with extra steps — a claim about a test written by
|
|
18
|
+
* whoever wrote the test — and that tier was removed after its first real use
|
|
19
|
+
* declared a conformance lint over 21 skills as coverage of all 21.
|
|
20
|
+
*/
|
|
21
|
+
export type ProbeOrigin = "command" | "fired" | "dispatched";
|
|
22
|
+
/**
|
|
23
|
+
* One surface a run went by. `ref` is whatever the tier saw — a script path, a
|
|
24
|
+
* namespaced skill id (`plugin:skill`), a hook name — NOT a resolved repo path.
|
|
25
|
+
* Resolution happens in the runner, where discovery lives.
|
|
26
|
+
*/
|
|
27
|
+
export interface SurfaceProbe {
|
|
28
|
+
readonly how: ProbeOrigin;
|
|
29
|
+
readonly ref: string;
|
|
30
|
+
}
|
|
6
31
|
/**
|
|
7
32
|
* Record `n` checks against this script's run.
|
|
8
33
|
*
|
|
@@ -15,11 +40,45 @@ export declare function recordCheck(n?: number): void;
|
|
|
15
40
|
/** How many checks this process has recorded so far. */
|
|
16
41
|
export declare function checksRecorded(): number;
|
|
17
42
|
/**
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
43
|
+
* Record that this run exercised the surface `ref` names. Called by the TIERS
|
|
44
|
+
* from what they were pointed at — never by a harness author, because a field an
|
|
45
|
+
* author fills in is a claim, not a measurement (that was `vigiles:covers`, and
|
|
46
|
+
* it was removed for exactly this reason).
|
|
47
|
+
*
|
|
48
|
+
* Deduped: a harness that fires the same hook forty times names it once.
|
|
49
|
+
* Empty/blank refs are dropped rather than stored as a surface called "".
|
|
50
|
+
*/
|
|
51
|
+
export declare function recordSurfaceProbe(how: ProbeOrigin, ref: string): void;
|
|
52
|
+
/** The surfaces this process has been seen to exercise, in first-seen order. */
|
|
53
|
+
export declare function surfacesRecorded(): readonly SurfaceProbe[];
|
|
54
|
+
/**
|
|
55
|
+
* Reset the counter, the probes AND the armed flag. For vigiles's own tests,
|
|
56
|
+
* which drive {@link armCheckReport} with fakes several times in one process; a
|
|
57
|
+
* harness script has no use for it.
|
|
21
58
|
*/
|
|
22
59
|
export declare function resetCheckCount(): void;
|
|
60
|
+
/** What a run reported: how much it did, and what it did it against. */
|
|
61
|
+
export interface CheckReport {
|
|
62
|
+
readonly checks: number;
|
|
63
|
+
readonly surfaces: readonly SurfaceProbe[];
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Serialize a report for the scratch file. A run with no probes writes the BARE
|
|
67
|
+
* NUMBER it has always written — byte for byte, so the legacy reader and the
|
|
68
|
+
* legacy tests are describing the same thing they always were. Only a run with
|
|
69
|
+
* an attribution to make spends a JSON object on it.
|
|
70
|
+
*/
|
|
71
|
+
export declare function formatCheckReport(report: CheckReport): string;
|
|
72
|
+
/**
|
|
73
|
+
* Parse a scratch file's contents, or `undefined` for anything that is not a
|
|
74
|
+
* report. Lives beside {@link formatCheckReport} so the two cannot drift.
|
|
75
|
+
*
|
|
76
|
+
* Tolerant in exactly one direction: a bare integer is the legacy form and is
|
|
77
|
+
* read as a report with no attribution. Anything malformed — a torn write, a
|
|
78
|
+
* negative, a JSON object with the wrong shape — is NOT a report, because
|
|
79
|
+
* "corrupt" must never be turned into a verdict about someone's tests.
|
|
80
|
+
*/
|
|
81
|
+
export declare function parseCheckReport(raw: string): CheckReport | undefined;
|
|
23
82
|
/** Injection seam for {@link armCheckReport} — the process bits it needs. */
|
|
24
83
|
export interface CheckReportEnv {
|
|
25
84
|
readonly env: NodeJS.ProcessEnv;
|
package/dist/check-count.js
CHANGED
|
@@ -3,7 +3,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.CHECK_COUNT_ENV = void 0;
|
|
4
4
|
exports.recordCheck = recordCheck;
|
|
5
5
|
exports.checksRecorded = checksRecorded;
|
|
6
|
+
exports.recordSurfaceProbe = recordSurfaceProbe;
|
|
7
|
+
exports.surfacesRecorded = surfacesRecorded;
|
|
6
8
|
exports.resetCheckCount = resetCheckCount;
|
|
9
|
+
exports.formatCheckReport = formatCheckReport;
|
|
10
|
+
exports.parseCheckReport = parseCheckReport;
|
|
7
11
|
exports.armCheckReport = armCheckReport;
|
|
8
12
|
/**
|
|
9
13
|
* vigiles — the CHECK COUNTER: the channel a `*.harness.*` / `*.eval.*` script
|
|
@@ -49,6 +53,34 @@ exports.armCheckReport = armCheckReport;
|
|
|
49
53
|
* became impossible — was rejected: it would report `0` for a hand-rolled
|
|
50
54
|
* harness that spawns and asserts entirely on its own, which is a real and
|
|
51
55
|
* blameless way to write one.)
|
|
56
|
+
*
|
|
57
|
+
* ## 2026-08-11 — the channel also carries WHAT was exercised
|
|
58
|
+
*
|
|
59
|
+
* 🔴 THE DEFECT. Coverage answered "is surface X tested?" from a FILE NAME —
|
|
60
|
+
* colocation. Measured on a real repo, `.claude/skills/paper-pipeline/` held six
|
|
61
|
+
* `*.eval.mjs`, exactly one of them about that skill, and the orchestrator scored
|
|
62
|
+
* as covered with no test of its own; an EMPTY `foo.eval.mjs` counts just the
|
|
63
|
+
* same. Every mature ecosystem (`go test -cover`, coverage.py, nyc, tarpaulin)
|
|
64
|
+
* answers that question from EXECUTION and uses the name only to find the file.
|
|
65
|
+
*
|
|
66
|
+
* So the tiers now also record WHICH SURFACE a run went by — a {@link SurfaceProbe}
|
|
67
|
+
* — and this channel carries it out alongside the count. Two properties matter:
|
|
68
|
+
*
|
|
69
|
+
* - **Derived, never declared.** The probe comes from what the tier already knew
|
|
70
|
+
* it was pointed at: the command string `runScript`/`runHook` executed, the
|
|
71
|
+
* Skill/hook events in a run's transcript. A field an AUTHOR fills in would be
|
|
72
|
+
* the retired `vigiles:covers` marker with extra steps — a claim about a test,
|
|
73
|
+
* made by the person who wrote the test.
|
|
74
|
+
* - **A probe is a REFERENCE, not a verdict.** Resolving `hooks/pre-edit.sh` or
|
|
75
|
+
* `plugin:argument-arc` to a repo surface needs discovery, which this module
|
|
76
|
+
* has no business doing inside a user's test process. The runner resolves it
|
|
77
|
+
* (`coverage-artifact.ts`); anything unresolvable is dropped, never guessed.
|
|
78
|
+
*
|
|
79
|
+
* WIRE FORMAT, and why it is two shapes. A run with no probes writes the bare
|
|
80
|
+
* number it always wrote, byte for byte; only a run with something to say writes
|
|
81
|
+
* JSON (`{"checks":N,"surfaces":[…]}`). Readers accept both — an older runner
|
|
82
|
+
* meeting the JSON form reads "no report", i.e. the legacy branch, which is a
|
|
83
|
+
* plain pass rather than a wrong verdict. Silence stays silence.
|
|
52
84
|
*/
|
|
53
85
|
const node_fs_1 = require("node:fs");
|
|
54
86
|
/**
|
|
@@ -65,7 +97,12 @@ exports.CHECK_COUNT_ENV = "VIGILES_CHECK_COUNT_FILE";
|
|
|
65
97
|
const STATE = Symbol.for("vigiles.check-count");
|
|
66
98
|
function state() {
|
|
67
99
|
const g = globalThis;
|
|
68
|
-
|
|
100
|
+
const s = (g[STATE] ??= { count: 0, armed: false, surfaces: new Map() });
|
|
101
|
+
// Two copies of vigiles can share one process (a global CLI plus a local
|
|
102
|
+
// dependency), and the OLDER copy's state object has no `surfaces` at all.
|
|
103
|
+
// Without this, the newer copy's first probe throws inside a user's test run.
|
|
104
|
+
s.surfaces ??= new Map();
|
|
105
|
+
return s;
|
|
69
106
|
}
|
|
70
107
|
/**
|
|
71
108
|
* Record `n` checks against this script's run.
|
|
@@ -83,14 +120,92 @@ function checksRecorded() {
|
|
|
83
120
|
return state().count;
|
|
84
121
|
}
|
|
85
122
|
/**
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
123
|
+
* Record that this run exercised the surface `ref` names. Called by the TIERS
|
|
124
|
+
* from what they were pointed at — never by a harness author, because a field an
|
|
125
|
+
* author fills in is a claim, not a measurement (that was `vigiles:covers`, and
|
|
126
|
+
* it was removed for exactly this reason).
|
|
127
|
+
*
|
|
128
|
+
* Deduped: a harness that fires the same hook forty times names it once.
|
|
129
|
+
* Empty/blank refs are dropped rather than stored as a surface called "".
|
|
130
|
+
*/
|
|
131
|
+
function recordSurfaceProbe(how, ref) {
|
|
132
|
+
const trimmed = ref.trim();
|
|
133
|
+
if (!trimmed)
|
|
134
|
+
return;
|
|
135
|
+
const s = state();
|
|
136
|
+
const key = `${how}\u0000${trimmed}`;
|
|
137
|
+
if (!s.surfaces.has(key))
|
|
138
|
+
s.surfaces.set(key, { how, ref: trimmed });
|
|
139
|
+
}
|
|
140
|
+
/** The surfaces this process has been seen to exercise, in first-seen order. */
|
|
141
|
+
function surfacesRecorded() {
|
|
142
|
+
return [...state().surfaces.values()];
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Reset the counter, the probes AND the armed flag. For vigiles's own tests,
|
|
146
|
+
* which drive {@link armCheckReport} with fakes several times in one process; a
|
|
147
|
+
* harness script has no use for it.
|
|
89
148
|
*/
|
|
90
149
|
function resetCheckCount() {
|
|
91
150
|
const s = state();
|
|
92
151
|
s.count = 0;
|
|
93
152
|
s.armed = false;
|
|
153
|
+
s.surfaces = new Map();
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Serialize a report for the scratch file. A run with no probes writes the BARE
|
|
157
|
+
* NUMBER it has always written — byte for byte, so the legacy reader and the
|
|
158
|
+
* legacy tests are describing the same thing they always were. Only a run with
|
|
159
|
+
* an attribution to make spends a JSON object on it.
|
|
160
|
+
*/
|
|
161
|
+
function formatCheckReport(report) {
|
|
162
|
+
if (report.surfaces.length === 0)
|
|
163
|
+
return String(report.checks);
|
|
164
|
+
return JSON.stringify({ checks: report.checks, surfaces: report.surfaces });
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Parse a scratch file's contents, or `undefined` for anything that is not a
|
|
168
|
+
* report. Lives beside {@link formatCheckReport} so the two cannot drift.
|
|
169
|
+
*
|
|
170
|
+
* Tolerant in exactly one direction: a bare integer is the legacy form and is
|
|
171
|
+
* read as a report with no attribution. Anything malformed — a torn write, a
|
|
172
|
+
* negative, a JSON object with the wrong shape — is NOT a report, because
|
|
173
|
+
* "corrupt" must never be turned into a verdict about someone's tests.
|
|
174
|
+
*/
|
|
175
|
+
function parseCheckReport(raw) {
|
|
176
|
+
const trimmed = raw.trim();
|
|
177
|
+
if (/^\d+$/.test(trimmed))
|
|
178
|
+
return { checks: Number(trimmed), surfaces: [] };
|
|
179
|
+
if (!trimmed.startsWith("{"))
|
|
180
|
+
return undefined;
|
|
181
|
+
let value;
|
|
182
|
+
try {
|
|
183
|
+
value = JSON.parse(trimmed);
|
|
184
|
+
}
|
|
185
|
+
catch {
|
|
186
|
+
return undefined;
|
|
187
|
+
}
|
|
188
|
+
if (!value || typeof value !== "object")
|
|
189
|
+
return undefined;
|
|
190
|
+
const obj = value;
|
|
191
|
+
const checks = obj.checks;
|
|
192
|
+
if (typeof checks !== "number" || !Number.isInteger(checks) || checks < 0)
|
|
193
|
+
return undefined;
|
|
194
|
+
const surfaces = Array.isArray(obj.surfaces)
|
|
195
|
+
? obj.surfaces.map(toProbe).filter((p) => p !== null)
|
|
196
|
+
: [];
|
|
197
|
+
return { checks, surfaces };
|
|
198
|
+
}
|
|
199
|
+
/** One serialized probe, or null for anything that isn't one. */
|
|
200
|
+
function toProbe(entry) {
|
|
201
|
+
if (!entry || typeof entry !== "object")
|
|
202
|
+
return null;
|
|
203
|
+
const { how, ref } = entry;
|
|
204
|
+
if (how !== "command" && how !== "fired" && how !== "dispatched")
|
|
205
|
+
return null;
|
|
206
|
+
if (typeof ref !== "string" || !ref.trim())
|
|
207
|
+
return null;
|
|
208
|
+
return { how, ref };
|
|
94
209
|
}
|
|
95
210
|
/**
|
|
96
211
|
* Arm the exit-time report, returning whether it armed (i.e. whether the runner
|
|
@@ -115,7 +230,10 @@ function armCheckReport(deps) {
|
|
|
115
230
|
s.armed = true;
|
|
116
231
|
deps.onExit(() => {
|
|
117
232
|
try {
|
|
118
|
-
deps.write(file,
|
|
233
|
+
deps.write(file, formatCheckReport({
|
|
234
|
+
checks: s.count,
|
|
235
|
+
surfaces: [...s.surfaces.values()],
|
|
236
|
+
}));
|
|
119
237
|
}
|
|
120
238
|
catch {
|
|
121
239
|
// An unwritable scratch path must never turn a passing harness into a
|
package/dist/check.d.ts
CHANGED
|
@@ -78,6 +78,61 @@ export declare function toolWith(name: string, args: ArgMatcher): Check<Trace>;
|
|
|
78
78
|
* not what it correctly refrained from doing.
|
|
79
79
|
*/
|
|
80
80
|
export declare function notTool(name: string, args?: ArgMatcher): Check<Trace>;
|
|
81
|
+
/**
|
|
82
|
+
* The agent used **only** tools drawn from `allowed` — the white-list, and the
|
|
83
|
+
* missing half of a symmetry the file surface already has:
|
|
84
|
+
*
|
|
85
|
+
* | | "not this one" | "nothing but these" |
|
|
86
|
+
* | ----- | --------------- | ---------------------- |
|
|
87
|
+
* | files | `assertNoWrite` | `assertWroteOnly` |
|
|
88
|
+
* | tools | {@link notTool} | **`onlyTools`** (this) |
|
|
89
|
+
*
|
|
90
|
+
* Why the asymmetry mattered: `notTool` can only forbid the calls the test
|
|
91
|
+
* author thought of, and the set of undeclared tools is unbounded — so "this
|
|
92
|
+
* skill stayed inside the tools it declares" was not expressible, which is
|
|
93
|
+
* exactly the claim a `SKILL.md` frontmatter makes in prose. See
|
|
94
|
+
* `skillContract`, which builds this check from that declaration.
|
|
95
|
+
*
|
|
96
|
+
* **Fails closed on an empty trace.** A `Trace` records tool calls only when the
|
|
97
|
+
* run captured the stream (`transcript: true` on the harness tier; always on the
|
|
98
|
+
* eval tier), so an uncaptured run is indistinguishable from a tool-free one —
|
|
99
|
+
* and passing it would assert nothing. Same discipline as `assertWroteOnly`
|
|
100
|
+
* refusing a result that never recorded writes: "we didn't look" is not "it was
|
|
101
|
+
* clean".
|
|
102
|
+
*
|
|
103
|
+
* 🔴 A `Tool(restriction)` DECLARATION IS MATCHED BY ITS BASE NAME. `allowed-tools:
|
|
104
|
+
* Bash(git:*)` is the ordinary way to write a narrow grant, and `skillContract`
|
|
105
|
+
* hands that literal string here — while a trace only ever carries the base name
|
|
106
|
+
* `Bash`, because that is the tool the harness reports. Set membership therefore
|
|
107
|
+
* missed, and every legitimate `Bash` call was reported as outside the declared
|
|
108
|
+
* surface, against a declaration that literally lists Bash (measured 2026-08-11:
|
|
109
|
+
* `agent used tool(s) outside its declared set: Bash (declared: Bash(git:*),
|
|
110
|
+
* Read, Skill)`). A check that fires on its own happy path gets deleted, not
|
|
111
|
+
* debugged — and it would fire on exactly the authors who narrowed their grant.
|
|
112
|
+
*
|
|
113
|
+
* ⚠️ AND THE RESTRICTION ITSELF IS NOT VERIFIED HERE, deliberately. A trace names
|
|
114
|
+
* the tool, not the grant it was matched against, so `Bash(git:*)` can be held to
|
|
115
|
+
* "no tool outside the declared set" and no further; whether the command really
|
|
116
|
+
* was a `git` one is a claim this layer has no evidence for. Narrowing beyond the
|
|
117
|
+
* base name is `disallowed-tools:` + `lethal-trifecta`'s job (see
|
|
118
|
+
* `skill-contract.ts`), which reads the grant instead of the run. The same
|
|
119
|
+
* `split("(")` normalization the rest of the codebase already applies to tool
|
|
120
|
+
* declarations — `core/tool-contract.ts`, `core/lethal-trifecta.ts`,
|
|
121
|
+
* `core/delegation-trifecta.ts`.
|
|
122
|
+
*
|
|
123
|
+
* ⚠️ SPANS THE WHOLE TRACE, deliberately. It answers "did this RUN stay inside
|
|
124
|
+
* these tools?", which is the right question for a harness that drives one thing.
|
|
125
|
+
* It is the wrong question when the trace has setup calls before the surface
|
|
126
|
+
* under test activated, drives several skills, or dispatches a subagent whose
|
|
127
|
+
* nested calls the parser folds into the same flat list — and reporting those as
|
|
128
|
+
* a violation is an accusation about the wrong code.
|
|
129
|
+
*
|
|
130
|
+
* `skillContract` therefore does NOT use this directly: it wraps it in a window
|
|
131
|
+
* bounded by its own `Skill` activation (see `duringSkill` in
|
|
132
|
+
* `src/skill-contract.ts`). If you are checking one skill, prefer the contract;
|
|
133
|
+
* reach for this when the claim really is about the whole run.
|
|
134
|
+
*/
|
|
135
|
+
export declare function onlyTools(allowed: readonly string[]): Check<Trace>;
|
|
81
136
|
/** A skill resolved to this id (`<plugin>:<skill>`) without erroring. */
|
|
82
137
|
export declare function skill(id: string): Check<Trace>;
|
|
83
138
|
/** The agent's final output contains a substring / matches a RegExp. */
|