vigiles 15.0.2 → 15.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/dist/adapters/claude-code/layout.js +3 -0
  2. package/dist/adapters/claude-code/run-scripts.d.ts +21 -9
  3. package/dist/adapters/claude-code/run-scripts.js +33 -22
  4. package/dist/adapters/codex/driver.js +3 -1
  5. package/dist/adapters/codex/eval.js +5 -0
  6. package/dist/audit-score.js +32 -6
  7. package/dist/check-count.d.ts +62 -3
  8. package/dist/check-count.js +123 -5
  9. package/dist/check.d.ts +55 -0
  10. package/dist/check.js +91 -0
  11. package/dist/cli.js +563 -63
  12. package/dist/core/bash-effects.d.ts +41 -0
  13. package/dist/core/bash-effects.js +278 -0
  14. package/dist/core/foreign-runner.d.ts +185 -0
  15. package/dist/core/foreign-runner.js +228 -0
  16. package/dist/core/harness-driver.d.ts +18 -1
  17. package/dist/core/hook-program.d.ts +456 -43
  18. package/dist/core/hook-program.js +900 -76
  19. package/dist/core/hook-providers.d.ts +22 -5
  20. package/dist/core/hook-providers.js +13 -1
  21. package/dist/core/hook-state.d.ts +307 -0
  22. package/dist/core/hook-state.js +349 -0
  23. package/dist/core/layout.d.ts +13 -0
  24. package/dist/core/lethal-trifecta.d.ts +93 -0
  25. package/dist/core/lethal-trifecta.js +409 -65
  26. package/dist/core/markdown.d.ts +32 -0
  27. package/dist/core/markdown.js +36 -0
  28. package/dist/core/merge-conflict.d.ts +50 -0
  29. package/dist/core/merge-conflict.js +84 -0
  30. package/dist/core/test-file-ext.d.ts +80 -0
  31. package/dist/core/test-file-ext.js +90 -0
  32. package/dist/core/types.d.ts +11 -0
  33. package/dist/coverage-artifact.d.ts +383 -0
  34. package/dist/coverage-artifact.js +586 -0
  35. package/dist/coverage-evidence.d.ts +87 -79
  36. package/dist/coverage-evidence.js +212 -211
  37. package/dist/coverage-probe.d.ts +125 -0
  38. package/dist/coverage-probe.js +700 -0
  39. package/dist/doc-commands.d.ts +119 -0
  40. package/dist/doc-commands.js +158 -0
  41. package/dist/eval-lock.d.ts +9 -0
  42. package/dist/eval-lock.js +14 -2
  43. package/dist/eval.js +28 -0
  44. package/dist/fs-walk.d.ts +80 -0
  45. package/dist/fs-walk.js +148 -0
  46. package/dist/harness-assert.js +42 -3
  47. package/dist/harness-test.js +31 -2
  48. package/dist/hook.d.ts +3 -1
  49. package/dist/hook.js +18 -1
  50. package/dist/judge.js +4 -0
  51. package/dist/leaderboard.d.ts +16 -1
  52. package/dist/leaderboard.js +19 -2
  53. package/dist/load-hook.js +6 -1
  54. package/dist/mock-model.d.ts +58 -4
  55. package/dist/mock-model.js +133 -5
  56. package/dist/observe.d.ts +1 -1
  57. package/dist/observe.js +44 -1
  58. package/dist/plugin-loader.js +52 -13
  59. package/dist/run-hook.d.ts +58 -0
  60. package/dist/run-hook.js +70 -0
  61. package/dist/run-script.js +75 -0
  62. package/dist/scaffold-test.d.ts +3 -0
  63. package/dist/scaffold-test.js +10 -5
  64. package/dist/scan-behavioral.js +4 -0
  65. package/dist/scan-core.d.ts +22 -0
  66. package/dist/scan-core.js +46 -14
  67. package/dist/scan-files.js +19 -2
  68. package/dist/scan.d.ts +22 -5
  69. package/dist/scan.js +67 -13
  70. package/dist/skill-contract.d.ts +46 -0
  71. package/dist/skill-contract.js +201 -0
  72. package/dist/skill-refs.d.ts +67 -0
  73. package/dist/skill-refs.js +117 -0
  74. package/dist/test-coverage-files.d.ts +7 -1
  75. package/dist/test-coverage-files.js +66 -55
  76. package/dist/test-coverage.d.ts +155 -19
  77. package/dist/test-coverage.js +430 -91
  78. package/dist/testing.d.ts +8 -2
  79. package/dist/testing.js +19 -1
  80. package/dist/trigger-containment.d.ts +85 -0
  81. package/dist/trigger-containment.js +125 -0
  82. package/dist/ts-runner-caps.d.ts +16 -0
  83. package/dist/ts-runner-caps.js +43 -0
  84. package/dist/unit.d.ts +2 -2
  85. package/dist/unit.js +2 -1
  86. package/hooks/eval-lock-nudge.sh +14 -6
  87. package/package.json +2 -2
  88. package/skills/test-harness/SKILL.md +68 -2
@@ -18,6 +18,9 @@ exports.claudeCodeLayout = {
18
18
  commandDir: "commands",
19
19
  materializeRoot: ".claude",
20
20
  pluginRootToken: "${CLAUDE_PLUGIN_ROOT}",
21
+ // Both names Claude Code uses for the project root (mirrors the
22
+ // `NON_PLUGIN_VARS` set in plugin-loader.ts / scan-files.ts).
23
+ projectRootTokens: ["${CLAUDE_PROJECT_DIR}", "${CLAUDE_PROJECT}"],
21
24
  mcpConfigFile: ".mcp.json",
22
25
  mcpManifestKey: "mcpServers",
23
26
  intraRefDirs: ["hooks", "skills", "agents", "commands"],
@@ -1,3 +1,4 @@
1
+ import { type SurfaceProbe } from "../../check-count.js";
1
2
  /**
2
3
  * The outcome of running one script.
3
4
  *
@@ -16,6 +17,17 @@ export interface ScriptRunResult {
16
17
  * loaded the library and used none of it.
17
18
  */
18
19
  readonly checks?: number;
20
+ /**
21
+ * The surfaces this script was seen to exercise — derived by the tiers from the
22
+ * command they ran and the transcripts they got back, never declared by the
23
+ * author (see `coverage-probe.ts`). Feeds `.vigiles/coverage.json`, which lets
24
+ * coverage answer "tested?" by EXECUTION instead of by file name.
25
+ *
26
+ * Absent for a script that reported nothing, and empty for one that reported a
27
+ * count but exercised no identifiable surface — a unit test of a pure helper,
28
+ * say. Neither is a finding.
29
+ */
30
+ readonly surfaces?: readonly SurfaceProbe[];
19
31
  }
20
32
  /**
21
33
  * Exit code a harness/eval script uses to report itself SKIPPED (e.g. the
@@ -54,22 +66,18 @@ export declare function statusFor(code: number, checks: number | undefined): Scr
54
66
  export declare const SCRIPT_EXTS: readonly ["mjs", "cjs", "js", "mts", "cts", "ts"];
55
67
  /** Glob suffix matching every accepted script extension, e.g. `harness`. */
56
68
  export declare function scriptGlob(kind: "harness" | "eval"): string;
57
- /** Node runtime capabilities that decide how a TypeScript script is run. */
58
- export interface NodeCaps {
59
- /** `tsx` is installed locally (the preferred, version-agnostic TS loader). */
60
- readonly tsx: boolean;
61
- /** Node supports `--experimental-strip-types` (>= 22.6). */
62
- readonly stripTypes: boolean;
63
- }
69
+ export { detectNodeCaps, canRunTypeScript, type NodeCaps, } from "../../ts-runner-caps.js";
70
+ import type { NodeCaps } from "../../ts-runner-caps.js";
64
71
  /**
65
72
  * The `node` argv (after the binary) to run a single script. Plain JS runs
66
73
  * directly; a TypeScript script picks `tsx` when available, else Node's native
67
74
  * type stripping. Throws a clear, actionable error when neither is available.
68
75
  * Pure — exported for testing.
76
+ *
77
+ * 🔴 The disjunction below is `canRunTypeScript` — keep them together. When they
78
+ * drifted, the tool recommended a `.ts` file and then refused to run it.
69
79
  */
70
80
  export declare function interpreterArgs(file: string, caps: NodeCaps): string[];
71
- /** Detect TS-running capabilities for a project root. */
72
- export declare function detectNodeCaps(cwd: string): NodeCaps;
73
81
  /**
74
82
  * Expand the given path/glob patterns into concrete script files. A pattern
75
83
  * that is an existing file passes through unchanged; anything else is treated
@@ -87,6 +95,10 @@ export declare function discoverScripts(patterns: readonly string[], defaultGlob
87
95
  * "ran nothing" distinguishable from "ran and passed" (see check-count.ts). It
88
96
  * has to be a file: stdio is inherited so the script's report streams live,
89
97
  * which leaves no stream to parse.
98
+ *
99
+ * The same channel carries WHICH SURFACES the script exercised, so the caller can
100
+ * record them (`.vigiles/coverage.json`) and coverage can answer "tested?" from
101
+ * execution rather than from a matching file name.
90
102
  */
91
103
  export declare function runScripts(files: readonly string[], cwd: string, env?: NodeJS.ProcessEnv): ScriptRunResult[];
92
104
  /**
@@ -1,10 +1,9 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.SCRIPT_EXTS = exports.SKIP_EXIT_CODE = void 0;
3
+ exports.canRunTypeScript = exports.detectNodeCaps = exports.SCRIPT_EXTS = exports.SKIP_EXIT_CODE = void 0;
4
4
  exports.statusFor = statusFor;
5
5
  exports.scriptGlob = scriptGlob;
6
6
  exports.interpreterArgs = interpreterArgs;
7
- exports.detectNodeCaps = detectNodeCaps;
8
7
  exports.discoverScripts = discoverScripts;
9
8
  exports.runScripts = runScripts;
10
9
  exports.anyFailed = anyFailed;
@@ -74,11 +73,23 @@ function scriptGlob(kind) {
74
73
  return `**/*.${kind}.{${exports.SCRIPT_EXTS.join(",")}}`;
75
74
  }
76
75
  const TS_EXT = /\.(?:m|c)?ts$/;
76
+ // The capability probe lives in `src/ts-runner-caps.ts` so the SUGGESTER
77
+ // (`testFileExt`, harness-agnostic) can ask the same question this runner
78
+ // answers. It used to recommend `.ts` from a `tsconfig.json` alone, on a Node 20
79
+ // box with no `tsx` — a file `interpreterArgs` then refused to run. Re-exported
80
+ // here so every existing importer of `run-scripts.js` is unchanged.
81
+ var ts_runner_caps_js_1 = require("../../ts-runner-caps.js");
82
+ Object.defineProperty(exports, "detectNodeCaps", { enumerable: true, get: function () { return ts_runner_caps_js_1.detectNodeCaps; } });
83
+ Object.defineProperty(exports, "canRunTypeScript", { enumerable: true, get: function () { return ts_runner_caps_js_1.canRunTypeScript; } });
84
+ const ts_runner_caps_js_2 = require("../../ts-runner-caps.js");
77
85
  /**
78
86
  * The `node` argv (after the binary) to run a single script. Plain JS runs
79
87
  * directly; a TypeScript script picks `tsx` when available, else Node's native
80
88
  * type stripping. Throws a clear, actionable error when neither is available.
81
89
  * Pure — exported for testing.
90
+ *
91
+ * 🔴 The disjunction below is `canRunTypeScript` — keep them together. When they
92
+ * drifted, the tool recommended a `.ts` file and then refused to run it.
82
93
  */
83
94
  function interpreterArgs(file, caps) {
84
95
  if (!TS_EXT.test(file))
@@ -90,13 +101,6 @@ function interpreterArgs(file, caps) {
90
101
  throw new Error(`Cannot run TypeScript test script "${file}": install tsx ` +
91
102
  `(npm i -D tsx) or use Node >= 22.6, or author it as a .mjs file.`);
92
103
  }
93
- /** Detect TS-running capabilities for a project root. */
94
- function detectNodeCaps(cwd) {
95
- const tsx = (0, node_fs_1.existsSync)((0, node_path_1.resolve)(cwd, "node_modules/tsx/package.json")) ||
96
- (0, node_fs_1.existsSync)((0, node_path_1.resolve)(cwd, "node_modules/.bin/tsx"));
97
- const stripTypes = process.allowedNodeEnvironmentFlags.has("--experimental-strip-types");
98
- return { tsx, stripTypes };
99
- }
100
104
  /**
101
105
  * Expand the given path/glob patterns into concrete script files. A pattern
102
106
  * that is an existing file passes through unchanged; anything else is treated
@@ -133,24 +137,21 @@ function discoverScripts(patterns, defaultGlob, cwd) {
133
137
  return [...found].sort();
134
138
  }
135
139
  /**
136
- * The count a script left behind, or `undefined` if it left none (it never
137
- * imported `vigiles/testing`, or died before its exit handler). Anything that
138
- * isn't a non-negative integer is treated as no report a corrupt scratch file
139
- * must not invent a verdict.
140
+ * What a script left behind, or `undefined` if it left nothing (it never
141
+ * imported `vigiles/testing`, or died before its exit handler). The parse itself
142
+ * lives beside the writer in `check-count.ts` so the two cannot drift; anything
143
+ * malformed is treated as no report — a corrupt scratch file must not invent a
144
+ * verdict.
140
145
  */
141
- function readCheckCount(path) {
146
+ function readCheckReport(path) {
142
147
  if (!(0, node_fs_1.existsSync)(path))
143
148
  return undefined;
144
- let raw;
145
149
  try {
146
- raw = (0, node_fs_1.readFileSync)(path, "utf8").trim();
150
+ return (0, check_count_js_1.parseCheckReport)((0, node_fs_1.readFileSync)(path, "utf8"));
147
151
  }
148
152
  catch {
149
153
  return undefined;
150
154
  }
151
- if (!/^\d+$/.test(raw))
152
- return undefined;
153
- return Number(raw);
154
155
  }
155
156
  /**
156
157
  * Run each script as `node <file>`, inheriting stdio so the script's own report
@@ -162,9 +163,13 @@ function readCheckCount(path) {
162
163
  * "ran nothing" distinguishable from "ran and passed" (see check-count.ts). It
163
164
  * has to be a file: stdio is inherited so the script's report streams live,
164
165
  * which leaves no stream to parse.
166
+ *
167
+ * The same channel carries WHICH SURFACES the script exercised, so the caller can
168
+ * record them (`.vigiles/coverage.json`) and coverage can answer "tested?" from
169
+ * execution rather than from a matching file name.
165
170
  */
166
171
  function runScripts(files, cwd, env = {}) {
167
- const caps = detectNodeCaps(cwd);
172
+ const caps = (0, ts_runner_caps_js_2.detectNodeCaps)(cwd);
168
173
  const results = [];
169
174
  const countDir = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-checks-"));
170
175
  try {
@@ -185,8 +190,14 @@ function runScripts(files, cwd, env = {}) {
185
190
  env: { ...process.env, ...env, [check_count_js_1.CHECK_COUNT_ENV]: countFile },
186
191
  });
187
192
  const code = res.status ?? 1;
188
- const checks = readCheckCount(countFile);
189
- results.push({ file, code, status: statusFor(code, checks), checks });
193
+ const report = readCheckReport(countFile);
194
+ results.push({
195
+ file,
196
+ code,
197
+ status: statusFor(code, report?.checks),
198
+ checks: report?.checks,
199
+ ...(report ? { surfaces: report.surfaces } : {}),
200
+ });
190
201
  });
191
202
  }
192
203
  finally {
@@ -53,7 +53,9 @@ function parseCodexRun(stdout) {
53
53
  codex-backed suite, the pure helpers above carry the testable logic. */
54
54
  function codexAvailable() {
55
55
  try {
56
- return ((0, node_child_process_1.spawnSync)(runtime_js_1.codexRuntime.agentBinary, ["--version"], {
56
+ return (
57
+ // vigiles:free-tier — availability probe; `--version` prints and exits.
58
+ (0, node_child_process_1.spawnSync)(runtime_js_1.codexRuntime.agentBinary, ["--version"], {
57
59
  stdio: "ignore",
58
60
  }).status === 0);
59
61
  }
@@ -35,6 +35,7 @@ exports.installCodexSkills = installCodexSkills;
35
35
  exports.codexEvalAgentRunner = codexEvalAgentRunner;
36
36
  exports.codexEvalRunner = codexEvalRunner;
37
37
  const node_child_process_1 = require("node:child_process");
38
+ const foreign_runner_js_1 = require("../../core/foreign-runner.js");
38
39
  const node_fs_1 = require("node:fs");
39
40
  const node_path_1 = require("node:path");
40
41
  const str = (v) => (typeof v === "string" ? v : "");
@@ -225,6 +226,10 @@ exports.codexEvalDriver = {
225
226
  * network egress to the model backend.
226
227
  */
227
228
  function codexEvalRunner(args) {
229
+ // The DOCUMENTED alternative to the default Claude runner, and the one the
230
+ // untested-skill nudge now recommends — so it is exactly as able to be
231
+ // collected by a stray `npx vitest run`, and exactly as expensive.
232
+ (0, foreign_runner_js_1.refuseUnderForeignRunner)("driving `codex exec`");
228
233
  const r = (0, node_child_process_1.spawnSync)("codex", [
229
234
  "exec",
230
235
  "--json",
@@ -82,6 +82,15 @@ function truthfulness(r) {
82
82
  weight: score_core_js_1.W_DANGLING_REF,
83
83
  label: "broken intra-plugin reference(s)",
84
84
  },
85
+ {
86
+ // Same weight and same category as the path-shaped one above: both are a
87
+ // reference that resolves to nothing. Counted separately only because the
88
+ // labels must read differently — "a path that isn't there" and "a skill
89
+ // that isn't there" send the reader to different fixes.
90
+ n: r.skillRefIssues?.length ?? 0,
91
+ weight: score_core_js_1.W_DANGLING_REF,
92
+ label: "skill(s) naming a sibling skill that does not exist",
93
+ },
85
94
  {
86
95
  n: missingHooks,
87
96
  weight: score_core_js_1.W_MISSING_HOOK,
@@ -198,10 +207,12 @@ function structure(r) {
198
207
  }
199
208
  /**
200
209
  * SAFETY — fed by the STATIC lethal-trifecta check (`report.trifectaFindings`), a
201
- * GRADED ring. EVERY unit holding all three capability legs counts, whether the
202
- * contract NAMED them (`"hard"`) or INHERITED them (`"advisory"` — no `tools:` /
203
- * `allowed-tools:` line, so it holds all three legs implicitly AND every other
204
- * capability besides). Grading only the explicit case made the ring non-monotone:
210
+ * GRADED ring. EVERY unit holding all three capability legs counts, whether a
211
+ * subagent's `tools:` NAMED them (`"hard"`) or the unit INHERITED them
212
+ * (`"advisory"` a subagent with no `tools:` line, or a skill with no
213
+ * `disallowed-tools:` fence, which is every skill by default: `allowed-tools:` is
214
+ * a pre-approval and restricts nothing). Grading only the explicit case made the
215
+ * ring non-monotone:
205
216
  * declaring a tool contract — a genuine risk REDUCTION — could only ever lower the
206
217
  * score. The penalty is the shared {@link trifectaExposure} (capped against the
207
218
  * share of the surface exposed), the SAME number `reportDeductions` sums into the
@@ -230,10 +241,24 @@ function safety(r) {
230
241
  }
231
242
  // NAME the inherited ones: they're both the worst (every capability, not just
232
243
  // the three legs) and the cheapest to fix — declare a contract that drops a leg.
244
+ //
245
+ // EXCEPT the unfenced skills, which collapse into ONE line. A skill's
246
+ // `allowed-tools:` pre-approves rather than restricts (measured 2026-08-11), so
247
+ // EVERY skill without a `disallowed-tools:` line is in this state — naming them
248
+ // one at a time would bury the subagent findings under a list as long as the
249
+ // skill corpus. Same reasoning as the report section; see `trifectaLines`.
250
+ const unfenced = r.trifectaFindings.filter((f) => f.kind === "skill" && f.finding.fence === "none");
233
251
  for (const f of r.trifectaFindings) {
234
252
  if (f.finding.severity !== "advisory")
235
253
  continue;
236
- findings.push(`${f.name} inherits all tools — the "lethal trifecta" (reads data, reaches the web, runs commands) plus every other capability, so a prompt injection could exfiltrate secrets`);
254
+ if (unfenced.includes(f))
255
+ continue;
256
+ findings.push(f.kind === "skill"
257
+ ? `${f.name}: ${f.finding.message}`
258
+ : `${f.name} inherits all tools — the "lethal trifecta" (reads data, reaches the web, runs commands) plus every other capability, so a prompt injection could exfiltrate secrets`);
259
+ }
260
+ if (unfenced.length > 0) {
261
+ findings.push(`${String(unfenced.length)} skill(s) declare no \`disallowed-tools:\` fence, so each inherits every tool the session grants — reads data, reaches the web, runs commands. \`allowed-tools:\` pre-approves, it does not restrict, so narrowing it does not reduce this; one \`disallowed-tools:\` line per skill drops a leg.`);
237
262
  }
238
263
  return {
239
264
  key: "Safety",
@@ -243,7 +268,8 @@ function safety(r) {
243
268
  };
244
269
  }
245
270
  /**
246
- * TESTED — DETERMINISTIC harness coverage only (`*.harness.mjs`, `*.test.*`):
271
+ * TESTED — DETERMINISTIC harness coverage only (`*.harness.*`, plus any custom
272
+ * `testGlobs`):
247
273
  * free, milliseconds, every push. It answers "does this gate still catch what it
248
274
  * claims?" The real-model tier is a SEPARATE ring ({@link evaluated}) because it
249
275
  * differs on cost, on cadence AND on the question it answers — folding both into
@@ -3,6 +3,31 @@
3
3
  * the runner (`runScripts`), read once here at import.
4
4
  */
5
5
  export declare const CHECK_COUNT_ENV = "VIGILES_CHECK_COUNT_FILE";
6
+ /**
7
+ * How a tier came to name a surface. Not a strength ranking — a statement of
8
+ * what the machinery went by, so a reader can tell an inference from a sighting:
9
+ *
10
+ * - `command` — the executed command line named a program file (`runScript` /
11
+ * `runHook`). The strongest kind: that exact path was handed to a process.
12
+ * - `fired` — the surface appears in the RUN'S TRANSCRIPT as having activated (a
13
+ * `Skill` tool call that resolved, a hook that reported). For a skill this is
14
+ * the only honest attribution: what was INSTALLED is a set, what RAN is one.
15
+ *
16
+ * There is deliberately NO author-declared origin. A `surface:` field on a spec
17
+ * would be `vigiles:covers` with extra steps — a claim about a test written by
18
+ * whoever wrote the test — and that tier was removed after its first real use
19
+ * declared a conformance lint over 21 skills as coverage of all 21.
20
+ */
21
+ export type ProbeOrigin = "command" | "fired" | "dispatched";
22
+ /**
23
+ * One surface a run went by. `ref` is whatever the tier saw — a script path, a
24
+ * namespaced skill id (`plugin:skill`), a hook name — NOT a resolved repo path.
25
+ * Resolution happens in the runner, where discovery lives.
26
+ */
27
+ export interface SurfaceProbe {
28
+ readonly how: ProbeOrigin;
29
+ readonly ref: string;
30
+ }
6
31
  /**
7
32
  * Record `n` checks against this script's run.
8
33
  *
@@ -15,11 +40,45 @@ export declare function recordCheck(n?: number): void;
15
40
  /** How many checks this process has recorded so far. */
16
41
  export declare function checksRecorded(): number;
17
42
  /**
18
- * Reset the counter AND the armed flag. For vigiles's own tests, which drive
19
- * {@link armCheckReport} with fakes several times in one process; a harness
20
- * script has no use for it.
43
+ * Record that this run exercised the surface `ref` names. Called by the TIERS
44
+ * from what they were pointed at never by a harness author, because a field an
45
+ * author fills in is a claim, not a measurement (that was `vigiles:covers`, and
46
+ * it was removed for exactly this reason).
47
+ *
48
+ * Deduped: a harness that fires the same hook forty times names it once.
49
+ * Empty/blank refs are dropped rather than stored as a surface called "".
50
+ */
51
+ export declare function recordSurfaceProbe(how: ProbeOrigin, ref: string): void;
52
+ /** The surfaces this process has been seen to exercise, in first-seen order. */
53
+ export declare function surfacesRecorded(): readonly SurfaceProbe[];
54
+ /**
55
+ * Reset the counter, the probes AND the armed flag. For vigiles's own tests,
56
+ * which drive {@link armCheckReport} with fakes several times in one process; a
57
+ * harness script has no use for it.
21
58
  */
22
59
  export declare function resetCheckCount(): void;
60
+ /** What a run reported: how much it did, and what it did it against. */
61
+ export interface CheckReport {
62
+ readonly checks: number;
63
+ readonly surfaces: readonly SurfaceProbe[];
64
+ }
65
+ /**
66
+ * Serialize a report for the scratch file. A run with no probes writes the BARE
67
+ * NUMBER it has always written — byte for byte, so the legacy reader and the
68
+ * legacy tests are describing the same thing they always were. Only a run with
69
+ * an attribution to make spends a JSON object on it.
70
+ */
71
+ export declare function formatCheckReport(report: CheckReport): string;
72
+ /**
73
+ * Parse a scratch file's contents, or `undefined` for anything that is not a
74
+ * report. Lives beside {@link formatCheckReport} so the two cannot drift.
75
+ *
76
+ * Tolerant in exactly one direction: a bare integer is the legacy form and is
77
+ * read as a report with no attribution. Anything malformed — a torn write, a
78
+ * negative, a JSON object with the wrong shape — is NOT a report, because
79
+ * "corrupt" must never be turned into a verdict about someone's tests.
80
+ */
81
+ export declare function parseCheckReport(raw: string): CheckReport | undefined;
23
82
  /** Injection seam for {@link armCheckReport} — the process bits it needs. */
24
83
  export interface CheckReportEnv {
25
84
  readonly env: NodeJS.ProcessEnv;
@@ -3,7 +3,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.CHECK_COUNT_ENV = void 0;
4
4
  exports.recordCheck = recordCheck;
5
5
  exports.checksRecorded = checksRecorded;
6
+ exports.recordSurfaceProbe = recordSurfaceProbe;
7
+ exports.surfacesRecorded = surfacesRecorded;
6
8
  exports.resetCheckCount = resetCheckCount;
9
+ exports.formatCheckReport = formatCheckReport;
10
+ exports.parseCheckReport = parseCheckReport;
7
11
  exports.armCheckReport = armCheckReport;
8
12
  /**
9
13
  * vigiles — the CHECK COUNTER: the channel a `*.harness.*` / `*.eval.*` script
@@ -49,6 +53,34 @@ exports.armCheckReport = armCheckReport;
49
53
  * became impossible — was rejected: it would report `0` for a hand-rolled
50
54
  * harness that spawns and asserts entirely on its own, which is a real and
51
55
  * blameless way to write one.)
56
+ *
57
+ * ## 2026-08-11 — the channel also carries WHAT was exercised
58
+ *
59
+ * 🔴 THE DEFECT. Coverage answered "is surface X tested?" from a FILE NAME —
60
+ * colocation. Measured on a real repo, `.claude/skills/paper-pipeline/` held six
61
+ * `*.eval.mjs`, exactly one of them about that skill, and the orchestrator scored
62
+ * as covered with no test of its own; an EMPTY `foo.eval.mjs` counts just the
63
+ * same. Every mature ecosystem (`go test -cover`, coverage.py, nyc, tarpaulin)
64
+ * answers that question from EXECUTION and uses the name only to find the file.
65
+ *
66
+ * So the tiers now also record WHICH SURFACE a run went by — a {@link SurfaceProbe}
67
+ * — and this channel carries it out alongside the count. Two properties matter:
68
+ *
69
+ * - **Derived, never declared.** The probe comes from what the tier already knew
70
+ * it was pointed at: the command string `runScript`/`runHook` executed, the
71
+ * Skill/hook events in a run's transcript. A field an AUTHOR fills in would be
72
+ * the retired `vigiles:covers` marker with extra steps — a claim about a test,
73
+ * made by the person who wrote the test.
74
+ * - **A probe is a REFERENCE, not a verdict.** Resolving `hooks/pre-edit.sh` or
75
+ * `plugin:argument-arc` to a repo surface needs discovery, which this module
76
+ * has no business doing inside a user's test process. The runner resolves it
77
+ * (`coverage-artifact.ts`); anything unresolvable is dropped, never guessed.
78
+ *
79
+ * WIRE FORMAT, and why it is two shapes. A run with no probes writes the bare
80
+ * number it always wrote, byte for byte; only a run with something to say writes
81
+ * JSON (`{"checks":N,"surfaces":[…]}`). Readers accept both — an older runner
82
+ * meeting the JSON form reads "no report", i.e. the legacy branch, which is a
83
+ * plain pass rather than a wrong verdict. Silence stays silence.
52
84
  */
53
85
  const node_fs_1 = require("node:fs");
54
86
  /**
@@ -65,7 +97,12 @@ exports.CHECK_COUNT_ENV = "VIGILES_CHECK_COUNT_FILE";
65
97
  const STATE = Symbol.for("vigiles.check-count");
66
98
  function state() {
67
99
  const g = globalThis;
68
- return (g[STATE] ??= { count: 0, armed: false });
100
+ const s = (g[STATE] ??= { count: 0, armed: false, surfaces: new Map() });
101
+ // Two copies of vigiles can share one process (a global CLI plus a local
102
+ // dependency), and the OLDER copy's state object has no `surfaces` at all.
103
+ // Without this, the newer copy's first probe throws inside a user's test run.
104
+ s.surfaces ??= new Map();
105
+ return s;
69
106
  }
70
107
  /**
71
108
  * Record `n` checks against this script's run.
@@ -83,14 +120,92 @@ function checksRecorded() {
83
120
  return state().count;
84
121
  }
85
122
  /**
86
- * Reset the counter AND the armed flag. For vigiles's own tests, which drive
87
- * {@link armCheckReport} with fakes several times in one process; a harness
88
- * script has no use for it.
123
+ * Record that this run exercised the surface `ref` names. Called by the TIERS
124
+ * from what they were pointed at never by a harness author, because a field an
125
+ * author fills in is a claim, not a measurement (that was `vigiles:covers`, and
126
+ * it was removed for exactly this reason).
127
+ *
128
+ * Deduped: a harness that fires the same hook forty times names it once.
129
+ * Empty/blank refs are dropped rather than stored as a surface called "".
130
+ */
131
+ function recordSurfaceProbe(how, ref) {
132
+ const trimmed = ref.trim();
133
+ if (!trimmed)
134
+ return;
135
+ const s = state();
136
+ const key = `${how}\u0000${trimmed}`;
137
+ if (!s.surfaces.has(key))
138
+ s.surfaces.set(key, { how, ref: trimmed });
139
+ }
140
+ /** The surfaces this process has been seen to exercise, in first-seen order. */
141
+ function surfacesRecorded() {
142
+ return [...state().surfaces.values()];
143
+ }
144
+ /**
145
+ * Reset the counter, the probes AND the armed flag. For vigiles's own tests,
146
+ * which drive {@link armCheckReport} with fakes several times in one process; a
147
+ * harness script has no use for it.
89
148
  */
90
149
  function resetCheckCount() {
91
150
  const s = state();
92
151
  s.count = 0;
93
152
  s.armed = false;
153
+ s.surfaces = new Map();
154
+ }
155
+ /**
156
+ * Serialize a report for the scratch file. A run with no probes writes the BARE
157
+ * NUMBER it has always written — byte for byte, so the legacy reader and the
158
+ * legacy tests are describing the same thing they always were. Only a run with
159
+ * an attribution to make spends a JSON object on it.
160
+ */
161
+ function formatCheckReport(report) {
162
+ if (report.surfaces.length === 0)
163
+ return String(report.checks);
164
+ return JSON.stringify({ checks: report.checks, surfaces: report.surfaces });
165
+ }
166
+ /**
167
+ * Parse a scratch file's contents, or `undefined` for anything that is not a
168
+ * report. Lives beside {@link formatCheckReport} so the two cannot drift.
169
+ *
170
+ * Tolerant in exactly one direction: a bare integer is the legacy form and is
171
+ * read as a report with no attribution. Anything malformed — a torn write, a
172
+ * negative, a JSON object with the wrong shape — is NOT a report, because
173
+ * "corrupt" must never be turned into a verdict about someone's tests.
174
+ */
175
+ function parseCheckReport(raw) {
176
+ const trimmed = raw.trim();
177
+ if (/^\d+$/.test(trimmed))
178
+ return { checks: Number(trimmed), surfaces: [] };
179
+ if (!trimmed.startsWith("{"))
180
+ return undefined;
181
+ let value;
182
+ try {
183
+ value = JSON.parse(trimmed);
184
+ }
185
+ catch {
186
+ return undefined;
187
+ }
188
+ if (!value || typeof value !== "object")
189
+ return undefined;
190
+ const obj = value;
191
+ const checks = obj.checks;
192
+ if (typeof checks !== "number" || !Number.isInteger(checks) || checks < 0)
193
+ return undefined;
194
+ const surfaces = Array.isArray(obj.surfaces)
195
+ ? obj.surfaces.map(toProbe).filter((p) => p !== null)
196
+ : [];
197
+ return { checks, surfaces };
198
+ }
199
+ /** One serialized probe, or null for anything that isn't one. */
200
+ function toProbe(entry) {
201
+ if (!entry || typeof entry !== "object")
202
+ return null;
203
+ const { how, ref } = entry;
204
+ if (how !== "command" && how !== "fired" && how !== "dispatched")
205
+ return null;
206
+ if (typeof ref !== "string" || !ref.trim())
207
+ return null;
208
+ return { how, ref };
94
209
  }
95
210
  /**
96
211
  * Arm the exit-time report, returning whether it armed (i.e. whether the runner
@@ -115,7 +230,10 @@ function armCheckReport(deps) {
115
230
  s.armed = true;
116
231
  deps.onExit(() => {
117
232
  try {
118
- deps.write(file, String(s.count));
233
+ deps.write(file, formatCheckReport({
234
+ checks: s.count,
235
+ surfaces: [...s.surfaces.values()],
236
+ }));
119
237
  }
120
238
  catch {
121
239
  // An unwritable scratch path must never turn a passing harness into a
package/dist/check.d.ts CHANGED
@@ -78,6 +78,61 @@ export declare function toolWith(name: string, args: ArgMatcher): Check<Trace>;
78
78
  * not what it correctly refrained from doing.
79
79
  */
80
80
  export declare function notTool(name: string, args?: ArgMatcher): Check<Trace>;
81
+ /**
82
+ * The agent used **only** tools drawn from `allowed` — the white-list, and the
83
+ * missing half of a symmetry the file surface already has:
84
+ *
85
+ * | | "not this one" | "nothing but these" |
86
+ * | ----- | --------------- | ---------------------- |
87
+ * | files | `assertNoWrite` | `assertWroteOnly` |
88
+ * | tools | {@link notTool} | **`onlyTools`** (this) |
89
+ *
90
+ * Why the asymmetry mattered: `notTool` can only forbid the calls the test
91
+ * author thought of, and the set of undeclared tools is unbounded — so "this
92
+ * skill stayed inside the tools it declares" was not expressible, which is
93
+ * exactly the claim a `SKILL.md` frontmatter makes in prose. See
94
+ * `skillContract`, which builds this check from that declaration.
95
+ *
96
+ * **Fails closed on an empty trace.** A `Trace` records tool calls only when the
97
+ * run captured the stream (`transcript: true` on the harness tier; always on the
98
+ * eval tier), so an uncaptured run is indistinguishable from a tool-free one —
99
+ * and passing it would assert nothing. Same discipline as `assertWroteOnly`
100
+ * refusing a result that never recorded writes: "we didn't look" is not "it was
101
+ * clean".
102
+ *
103
+ * 🔴 A `Tool(restriction)` DECLARATION IS MATCHED BY ITS BASE NAME. `allowed-tools:
104
+ * Bash(git:*)` is the ordinary way to write a narrow grant, and `skillContract`
105
+ * hands that literal string here — while a trace only ever carries the base name
106
+ * `Bash`, because that is the tool the harness reports. Set membership therefore
107
+ * missed, and every legitimate `Bash` call was reported as outside the declared
108
+ * surface, against a declaration that literally lists Bash (measured 2026-08-11:
109
+ * `agent used tool(s) outside its declared set: Bash (declared: Bash(git:*),
110
+ * Read, Skill)`). A check that fires on its own happy path gets deleted, not
111
+ * debugged — and it would fire on exactly the authors who narrowed their grant.
112
+ *
113
+ * ⚠️ AND THE RESTRICTION ITSELF IS NOT VERIFIED HERE, deliberately. A trace names
114
+ * the tool, not the grant it was matched against, so `Bash(git:*)` can be held to
115
+ * "no tool outside the declared set" and no further; whether the command really
116
+ * was a `git` one is a claim this layer has no evidence for. Narrowing beyond the
117
+ * base name is `disallowed-tools:` + `lethal-trifecta`'s job (see
118
+ * `skill-contract.ts`), which reads the grant instead of the run. The same
119
+ * `split("(")` normalization the rest of the codebase already applies to tool
120
+ * declarations — `core/tool-contract.ts`, `core/lethal-trifecta.ts`,
121
+ * `core/delegation-trifecta.ts`.
122
+ *
123
+ * ⚠️ SPANS THE WHOLE TRACE, deliberately. It answers "did this RUN stay inside
124
+ * these tools?", which is the right question for a harness that drives one thing.
125
+ * It is the wrong question when the trace has setup calls before the surface
126
+ * under test activated, drives several skills, or dispatches a subagent whose
127
+ * nested calls the parser folds into the same flat list — and reporting those as
128
+ * a violation is an accusation about the wrong code.
129
+ *
130
+ * `skillContract` therefore does NOT use this directly: it wraps it in a window
131
+ * bounded by its own `Skill` activation (see `duringSkill` in
132
+ * `src/skill-contract.ts`). If you are checking one skill, prefer the contract;
133
+ * reach for this when the claim really is about the whole run.
134
+ */
135
+ export declare function onlyTools(allowed: readonly string[]): Check<Trace>;
81
136
  /** A skill resolved to this id (`<plugin>:<skill>`) without erroring. */
82
137
  export declare function skill(id: string): Check<Trace>;
83
138
  /** The agent's final output contains a substring / matches a RegExp. */