vigiles 6.0.0 → 7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/README.md +78 -68
  2. package/dist/action-gate.js +1 -1
  3. package/dist/adapters/claude-code/agent-runtime.d.ts +46 -11
  4. package/dist/adapters/claude-code/agent-runtime.js +95 -24
  5. package/dist/adapters/claude-code/effect-region.js +1 -1
  6. package/dist/adapters/claude-code/skill-runtime.d.ts +1 -1
  7. package/dist/adapters/claude-code/skill-runtime.js +1 -1
  8. package/dist/adapters/codex/hook-protocol.js +3 -0
  9. package/dist/adapters/codex/mock-model.js +1 -1
  10. package/dist/cli-commands.d.ts +19 -0
  11. package/dist/cli-commands.js +51 -0
  12. package/dist/cli.js +599 -86
  13. package/dist/core/bash-effects.d.ts +12 -0
  14. package/dist/core/bash-effects.js +31 -0
  15. package/dist/core/capability-diff.d.ts +46 -0
  16. package/dist/core/capability-diff.js +97 -0
  17. package/dist/core/guards.d.ts +126 -0
  18. package/dist/core/guards.js +309 -0
  19. package/dist/core/harness-driver.d.ts +1 -1
  20. package/dist/core/hook-program.d.ts +459 -0
  21. package/dist/core/hook-program.js +468 -0
  22. package/dist/core/hook-protocol.d.ts +7 -0
  23. package/dist/core/hook-providers.d.ts +138 -0
  24. package/dist/core/hook-providers.js +155 -0
  25. package/dist/core/hook-spec.d.ts +74 -0
  26. package/dist/core/hook-spec.js +130 -0
  27. package/dist/core/inline.js +1 -1
  28. package/dist/core/mcp-tool.d.ts +12 -0
  29. package/dist/core/mcp-tool.js +20 -0
  30. package/dist/core/mcp.d.ts +13 -0
  31. package/dist/core/mcp.js +67 -0
  32. package/dist/core/types.d.ts +8 -0
  33. package/dist/dialect-drift.d.ts +65 -0
  34. package/dist/dialect-drift.js +216 -0
  35. package/dist/eval.d.ts +40 -5
  36. package/dist/eval.js +59 -5
  37. package/dist/guardrail-check.d.ts +85 -0
  38. package/dist/guardrail-check.js +152 -0
  39. package/dist/harness-assert.d.ts +10 -0
  40. package/dist/harness-assert.js +30 -0
  41. package/dist/hook-install.d.ts +43 -0
  42. package/dist/hook-install.js +91 -0
  43. package/dist/hook.d.ts +52 -0
  44. package/dist/hook.js +98 -0
  45. package/dist/leaderboard.d.ts +6 -0
  46. package/dist/leaderboard.js +43 -1
  47. package/dist/linting.d.ts +9 -5
  48. package/dist/linting.js +17 -5
  49. package/dist/optimize.js +1 -1
  50. package/dist/scaffold-test.js +21 -7
  51. package/dist/scan-behavioral.d.ts +60 -0
  52. package/dist/scan-behavioral.js +239 -1
  53. package/dist/scan.d.ts +14 -0
  54. package/dist/scan.js +33 -1
  55. package/dist/score-explainer.js +1 -1
  56. package/dist/self-command-refs.d.ts +21 -0
  57. package/dist/self-command-refs.js +125 -0
  58. package/dist/testing.d.ts +5 -3
  59. package/dist/testing.js +37 -23
  60. package/dist/tool-intercept.d.ts +4 -4
  61. package/dist/tool-intercept.js +5 -5
  62. package/dist/unit.d.ts +2 -0
  63. package/dist/unit.js +8 -1
  64. package/hooks/refs-nudge.sh +1 -1
  65. package/package.json +5 -3
@@ -17,12 +17,17 @@ Object.defineProperty(exports, "__esModule", { value: true });
17
17
  exports.probePluginTriggersWith = probePluginTriggersWith;
18
18
  exports.probePluginTriggers = probePluginTriggers;
19
19
  exports.formatBehavioralReport = formatBehavioralReport;
20
+ exports.buildSelectionReport = buildSelectionReport;
21
+ exports.measurePluginSelectionWith = measurePluginSelectionWith;
22
+ exports.measurePluginSelection = measurePluginSelection;
23
+ exports.formatSelectionReport = formatSelectionReport;
20
24
  const node_fs_1 = require("node:fs");
21
25
  const node_path_1 = require("node:path");
22
26
  const scan_js_1 = require("./scan.js");
23
27
  const eval_js_1 = require("./eval.js");
24
28
  const harness_assert_js_1 = require("./harness-assert.js");
25
29
  const harness_test_js_1 = require("./harness-test.js");
30
+ const plugin_loader_js_1 = require("./adapters/claude-code/plugin-loader.js");
26
31
  const eval_js_2 = require("./adapters/codex/eval.js");
27
32
  const driver_js_1 = require("./adapters/codex/driver.js");
28
33
  function buildProbe(dir, harness) {
@@ -56,6 +61,39 @@ function pluginName(dir) {
56
61
  return null;
57
62
  }
58
63
  }
64
+ /**
65
+ * Does the plugin declare a SessionStart hook? The STUBBED measurement path rebuilds
66
+ * the plugin to skills-only (`packageSkillsDir`), DROPPING `hooks/` — so a SessionStart
67
+ * hook that primes skill selection (e.g. superpowers' `using-superpowers` gateway
68
+ * injection) is silently lost, and a recall collapse to 0 under stubbing is then a
69
+ * measurement ARTIFACT, not a real miss. Detect it to LABEL honestly (Layer 1) rather
70
+ * than report a misleading 0%. See `research/plugin-selection-collision.md`.
71
+ */
72
+ function hasSessionStartHook(dir) {
73
+ try {
74
+ const hooks = (0, plugin_loader_js_1.loadPlugin)(dir).settings.hooks;
75
+ return hooks !== undefined && Object.keys(hooks).includes("SessionStart");
76
+ }
77
+ catch {
78
+ return false;
79
+ }
80
+ }
81
+ const HOOK_PRIMED_NOTE = "hook-primed — the stubbed run dropped the plugin's SessionStart hook (which can " +
82
+ "prime skill selection), so 0% recall is likely a measurement artifact; re-run " +
83
+ "against the full plugin install to measure faithfully";
84
+ /**
85
+ * Layer-1 honesty: a STUBBED run on a SessionStart-hooked plugin where EVERY measured
86
+ * skill sits at recall 0 is the dropped-hook artifact — not a real result. The
87
+ * all-zero gate keeps a genuine single-skill miss reported as real (if siblings fired,
88
+ * the hook ran or wasn't needed). Applied to both the trigger column and the matrix.
89
+ */
90
+ function isStubbedHookArtifact(dir, stub, recalls) {
91
+ if (!stub || recalls.length === 0)
92
+ return false;
93
+ if (!recalls.every((r) => r === 0))
94
+ return false;
95
+ return hasSessionStartHook(dir);
96
+ }
59
97
  /** Probe one skill via the harness probe's eval driver → result, never throwing. */
60
98
  async function probeSkill(ctx, name, ps) {
61
99
  try {
@@ -108,7 +146,19 @@ async function probePluginTriggersWith(dir, promptSet, probe, opts = {}) {
108
146
  }
109
147
  results.push(await probeSkill(ctx, s.name, ps));
110
148
  }
111
- return { available: true, results };
149
+ return {
150
+ available: true,
151
+ results: relabelTriggerArtifact(dir, probe, results),
152
+ };
153
+ }
154
+ /** Relabel an all-zero-recall stubbed run on a hooked plugin as unmeasured (Layer 1). */
155
+ function relabelTriggerArtifact(dir, probe, results) {
156
+ const recalls = results.filter((r) => r.measured).map((r) => r.recall ?? 0);
157
+ if (!isStubbedHookArtifact(dir, probe.stub, recalls))
158
+ return [...results];
159
+ return results.map((r) => r.measured && (r.recall ?? 0) === 0
160
+ ? { skill: r.skill, measured: false, note: HOOK_PRIMED_NOTE }
161
+ : r);
112
162
  }
113
163
  /**
114
164
  * Probe a plugin's skills against the real harness (default Claude Code; Codex via
@@ -147,4 +197,192 @@ function formatBehavioralReport(b) {
147
197
  }
148
198
  return lines.join("\n");
149
199
  }
200
+ /** Map a namespaced skill id (`ns:name`) to its bare name. */
201
+ function bareSkillName(id) {
202
+ const i = id.lastIndexOf(":");
203
+ return i >= 0 ? id.slice(i + 1) : id;
204
+ }
205
+ function emptySelection(skills) {
206
+ return {
207
+ available: true,
208
+ skills,
209
+ matrix: skills.map(() => []),
210
+ perSkill: [],
211
+ collisionRate: 0,
212
+ n: 0,
213
+ };
214
+ }
215
+ function buildSkillStat(a) {
216
+ const { skill, i, skills, row, n, collisions } = a;
217
+ const collidesWith = skills
218
+ .map((s, j) => ({ skill: s, rate: n > 0 ? row[j] / n : 0 }))
219
+ .filter((c) => c.skill !== skill && c.rate > 0)
220
+ .sort((x, y) => y.rate - x.rate);
221
+ return {
222
+ skill,
223
+ recall: n > 0 ? row[i] / n : 0,
224
+ collisionRate: n > 0 ? collisions / n : 0,
225
+ n,
226
+ collidesWith,
227
+ };
228
+ }
229
+ /**
230
+ * Pure aggregation: fold per-run fired-skill sets into the N×N selection matrix +
231
+ * per-skill recall/collision + the plugin-level collision rate. Separated from the
232
+ * model-driving so it's unit-testable with synthetic runs (no model).
233
+ */
234
+ function buildSelectionReport(skills, runs) {
235
+ const index = new Map(skills.map((s, i) => [s, i]));
236
+ const n = skills.length;
237
+ const matrix = skills.map(() => new Array(n).fill(0));
238
+ const nBy = new Array(n).fill(0);
239
+ const collisionBy = new Array(n).fill(0);
240
+ let collisionTotal = 0;
241
+ for (const run of runs) {
242
+ const i = index.get(run.intended);
243
+ if (i === undefined)
244
+ continue;
245
+ nBy[i] += 1;
246
+ let collided = false;
247
+ for (const fb of run.firedBare) {
248
+ const j = index.get(fb);
249
+ if (j === undefined)
250
+ continue;
251
+ matrix[i][j] += 1;
252
+ if (j !== i)
253
+ collided = true;
254
+ }
255
+ if (collided) {
256
+ collisionBy[i] += 1;
257
+ collisionTotal += 1;
258
+ }
259
+ }
260
+ const total = nBy.reduce((a, b) => a + b, 0);
261
+ return {
262
+ available: true,
263
+ skills,
264
+ matrix,
265
+ perSkill: skills.map((s, i) => buildSkillStat({
266
+ skill: s,
267
+ i,
268
+ skills,
269
+ row: matrix[i],
270
+ n: nBy[i],
271
+ collisions: collisionBy[i],
272
+ })),
273
+ collisionRate: total > 0 ? collisionTotal / total : 0,
274
+ n: total,
275
+ };
276
+ }
277
+ /** Build the prompts × trials work list across every skill that has prompts. */
278
+ function selectionJobs(candidates, promptSet, trials) {
279
+ return candidates.flatMap((c) => {
280
+ const ps = promptSet[c.name];
281
+ if (!ps || ps.prompts.length === 0)
282
+ return [];
283
+ return ps.prompts.flatMap((prompt) => Array.from({ length: trials }, () => ({ intended: c.name, prompt })));
284
+ });
285
+ }
286
+ /** The injectable core (for tests): drive the matrix via a fake/real probe. */
287
+ async function measurePluginSelectionWith(dir, promptSet, probe, opts = {}) {
288
+ const candidates = (0, scan_js_1.scanPlugin)(dir).skills.filter((s) => !s.userInvoked && s.hasDescription);
289
+ const skills = candidates.map((c) => c.name);
290
+ if (skills.length < 2) {
291
+ return {
292
+ ...emptySelection(skills),
293
+ note: "needs ≥2 model-invocable skills to measure cross-skill collision",
294
+ };
295
+ }
296
+ const own = new Set(skills);
297
+ const jobs = selectionJobs(candidates, promptSet, Math.max(1, opts.trials ?? 1));
298
+ if (jobs.length === 0) {
299
+ return {
300
+ ...emptySelection(skills),
301
+ note: "no prompts supplied for any skill",
302
+ };
303
+ }
304
+ // Selection is decided at the frontmatter (the selector picks BEFORE the body
305
+ // loads), so stub each body to a no-op: the run stops AT selection instead of
306
+ // executing the whole workflow — the same affordability trick trigger-rate uses.
307
+ const pluginDir = probe.stub ? (0, eval_js_1.stubbedPluginDir)(dir) : dir;
308
+ try {
309
+ const d = probe.evalDriver;
310
+ const outcomes = await (0, eval_js_1.runPool)(jobs, Math.max(1, opts.concurrency ?? 1), (job) => (0, eval_js_1.runSkillSelectionTrial)({
311
+ prompt: job.prompt,
312
+ pluginDir,
313
+ runner: d.runner,
314
+ parse: d.parse,
315
+ runError: d.runError,
316
+ model: opts.model ?? "sonnet",
317
+ }));
318
+ const runs = [];
319
+ jobs.forEach((job, k) => {
320
+ if (outcomes[k].errored)
321
+ return;
322
+ runs.push({
323
+ intended: job.intended,
324
+ firedBare: outcomes[k].fired
325
+ .map(bareSkillName)
326
+ .filter((b) => own.has(b)),
327
+ });
328
+ });
329
+ const report = buildSelectionReport(skills, runs);
330
+ // Layer-1 honesty: an all-zero-recall stubbed run on a hooked plugin is the
331
+ // dropped-hook artifact — flag it instead of presenting a 0%-collision result
332
+ // computed from a plugin that never fired.
333
+ const recalls = report.perSkill.filter((s) => s.n > 0).map((s) => s.recall);
334
+ return isStubbedHookArtifact(dir, probe.stub, recalls)
335
+ ? { ...report, note: HOOK_PRIMED_NOTE }
336
+ : report;
337
+ }
338
+ finally {
339
+ if (probe.stub)
340
+ (0, node_fs_1.rmSync)(pluginDir, { recursive: true, force: true });
341
+ }
342
+ }
343
+ /**
344
+ * Measure a plugin's cross-skill selection-collision matrix against the real
345
+ * harness (Claude Code only — Codex has no skill-selection event). Needs the
346
+ * `claude` CLI + model auth; degrades to `available: false` otherwise.
347
+ */
348
+ async function measurePluginSelection(dir, promptSet, opts = {}) {
349
+ const harness = opts.harness ?? "claude-code";
350
+ if (harness !== "claude-code") {
351
+ return {
352
+ ...emptySelection([]),
353
+ available: false,
354
+ note: `selection-collision is Claude Code only (no skill-selection event on ${harness})`,
355
+ };
356
+ }
357
+ const probe = buildProbe(dir, harness);
358
+ if (!probe.available()) {
359
+ return {
360
+ ...emptySelection([]),
361
+ available: false,
362
+ note: "needs the claude CLI + model auth",
363
+ };
364
+ }
365
+ return measurePluginSelectionWith(dir, promptSet, probe, opts);
366
+ }
367
+ /** Format the selection-collision matrix as a scan-report section. */
368
+ function formatSelectionReport(r) {
369
+ if (!r.available)
370
+ return `Selection-collision: unavailable — ${r.note ?? "n/a"}`;
371
+ if (r.n === 0)
372
+ return `Selection-collision: ${r.note ?? "not measured"}`;
373
+ const lines = [
374
+ `Selection-collision: ${pct(r.collisionRate)} of ${String(r.n)} runs hit a sibling skill`,
375
+ ];
376
+ if (r.note)
377
+ lines.push(` ⓘ ${r.note}`);
378
+ for (const s of r.perSkill) {
379
+ if (s.n === 0)
380
+ continue;
381
+ const mark = s.collisionRate === 0 ? "✓" : "⚠";
382
+ const top = s.collidesWith[0];
383
+ const tail = top ? ` — top collider: ${top.skill} ${pct(top.rate)}` : "";
384
+ lines.push(` ${mark} ${s.skill} — recall ${pct(s.recall)}, collision ${pct(s.collisionRate)}${tail}`);
385
+ }
386
+ return lines.join("\n");
387
+ }
150
388
  //# sourceMappingURL=scan-behavioral.js.map
package/dist/scan.d.ts CHANGED
@@ -18,6 +18,7 @@ import { type HookEventIssue } from "./core/hook-events.js";
18
18
  import { type McpIssue } from "./core/mcp-config.js";
19
19
  import { type DescriptionOverlap } from "./core/description-overlap.js";
20
20
  import { type McpToolIssue } from "./core/mcp-tool.js";
21
+ import { type McpContractToolError } from "./core/mcp.js";
21
22
  import { type McpHookIssue } from "./core/mcp-hook.js";
22
23
  import { type PurityLevel, type EffectSurface } from "./core/effects.js";
23
24
  /** A named writing system. The label `unexpectedScript` reports + the config's expectation parse into this. */
@@ -176,6 +177,19 @@ export interface SurfaceClassifier {
176
177
  export declare function unexpectedScript(text: string, expected?: Script): Script | null;
177
178
  /** Scan a plugin/repo directory and report its surfaces + structural issues. */
178
179
  export declare function scanPlugin(dir: string, layout?: PluginLayout, dialect?: HarnessDialect): ScanReport;
180
+ /**
181
+ * LIVE MCP tool resolution for a scanned plugin — the opt-in (`scan --verify-mcp`)
182
+ * dynamic check no static linter can do: it STARTS each declared MCP server and
183
+ * checks every `mcp__server__tool` the plugin's agents reference actually exists on
184
+ * it (catching rename/removal rot, e.g. `create_issue`→`issue_write`). Reuses the
185
+ * already-computed `report` (its agents' tool lists) + the declared server configs;
186
+ * returns `[]` when the plugin declares no MCP servers (nothing to start). Async +
187
+ * side-effecting (spawns servers) — which is exactly why it's opt-in, not a default
188
+ * lint rule. See `verifyMcpContractTools` (core/mcp.ts).
189
+ */
190
+ export declare function verifyLiveMcpTools(report: ScanReport, layout: PluginLayout, dialect: HarnessDialect, timeoutMs?: number): Promise<McpContractToolError[]>;
191
+ /** Render the live MCP tool-check result (human-readable). */
192
+ export declare function formatMcpContractReport(errors: readonly McpContractToolError[]): string;
179
193
  /**
180
194
  * A plugin MARKETPLACE (`.claude-plugin/marketplace.json`) decomposed into its
181
195
  * members. A marketplace either VENDORS its plugins in-tree (string `source`
package/dist/scan.js CHANGED
@@ -15,6 +15,8 @@
15
15
  Object.defineProperty(exports, "__esModule", { value: true });
16
16
  exports.unexpectedScript = unexpectedScript;
17
17
  exports.scanPlugin = scanPlugin;
18
+ exports.verifyLiveMcpTools = verifyLiveMcpTools;
19
+ exports.formatMcpContractReport = formatMcpContractReport;
18
20
  exports.inspectMarketplace = inspectMarketplace;
19
21
  exports.expandMarketplace = expandMarketplace;
20
22
  exports.formatScanReport = formatScanReport;
@@ -31,6 +33,7 @@ const linters_js_1 = require("./core/linters.js");
31
33
  const frontmatter_read_js_1 = require("./core/frontmatter-read.js");
32
34
  const description_overlap_js_1 = require("./core/description-overlap.js");
33
35
  const mcp_tool_js_1 = require("./core/mcp-tool.js");
36
+ const mcp_js_1 = require("./core/mcp.js");
34
37
  const mcp_hook_js_1 = require("./core/mcp-hook.js");
35
38
  const agent_runtime_js_1 = require("./adapters/claude-code/agent-runtime.js");
36
39
  const test_coverage_js_1 = require("./test-coverage.js");
@@ -535,6 +538,35 @@ function scanPlugin(dir, layout, dialect = dialect_js_1.claudeCodeDialect) {
535
538
  puritySummary,
536
539
  };
537
540
  }
541
+ /**
542
+ * LIVE MCP tool resolution for a scanned plugin — the opt-in (`scan --verify-mcp`)
543
+ * dynamic check no static linter can do: it STARTS each declared MCP server and
544
+ * checks every `mcp__server__tool` the plugin's agents reference actually exists on
545
+ * it (catching rename/removal rot, e.g. `create_issue`→`issue_write`). Reuses the
546
+ * already-computed `report` (its agents' tool lists) + the declared server configs;
547
+ * returns `[]` when the plugin declares no MCP servers (nothing to start). Async +
548
+ * side-effecting (spawns servers) — which is exactly why it's opt-in, not a default
549
+ * lint rule. See `verifyMcpContractTools` (core/mcp.ts).
550
+ */
551
+ async function verifyLiveMcpTools(report, layout, dialect, timeoutMs = 10000) {
552
+ // collectMcpServers yields the raw JSON server entries; a malformed one (no
553
+ // command) just fails to start → server-unreachable (handled), so the cast is safe.
554
+ const servers = collectMcpServers((0, node_path_1.resolve)(report.dir), layout);
555
+ if (Object.keys(servers).length === 0)
556
+ return [];
557
+ const tools = report.agents.flatMap((a) => a.tools ?? []);
558
+ return (0, mcp_js_1.verifyMcpContractTools)(tools, servers, dialect, timeoutMs);
559
+ }
560
+ /** Render the live MCP tool-check result (human-readable). */
561
+ function formatMcpContractReport(errors) {
562
+ if (errors.length === 0) {
563
+ return "Live MCP tool check: every referenced mcp__server__tool resolves ✓";
564
+ }
565
+ const lines = [`Live MCP tool check — ${String(errors.length)} issue(s):`];
566
+ for (const e of errors)
567
+ lines.push(" ✗ " + (0, mcp_js_1.mcpContractToolMessage)(e));
568
+ return lines.join("\n");
569
+ }
538
570
  /**
539
571
  * Read a `marketplace.json` beside the layout's plugin manifest and classify its
540
572
  * members into on-disk vs external. Returns `null` when `dir` is not a
@@ -710,7 +742,7 @@ function formatScanReport(r) {
710
742
  // verdict — it points at the behavioral column, it doesn't fail the scan.
711
743
  const mismatched = r.skills.filter((s) => s.descriptionScript);
712
744
  if (mismatched.length > 0) {
713
- out.push(`⚠ ${String(mismatched.length)} skill(s) have descriptions in an unexpected script (cross-language trigger risk) — measure with \`scan --trigger\``, "");
745
+ out.push(`⚠ ${String(mismatched.length)} skill(s) have descriptions in an unexpected script (cross-language trigger risk) — measure with \`vigiles measure\``, "");
714
746
  }
715
747
  // Skill-metadata is a RECOMMENDATION, not a structural defect (the skill loads
716
748
  // via fallbacks) — reported as a soft note, never counted in the verdict.
@@ -155,7 +155,7 @@ function explainSurface(report, surface) {
155
155
  /** Render explanations for a CLI/report — grouped under the symptom, fix called out. */
156
156
  function formatExplanations(exps) {
157
157
  if (exps.length === 0) {
158
- return "No deterministic cause found — the cause is likely behavioral (measure with `scan --trigger` / an eval).";
158
+ return "No deterministic cause found — the cause is likely behavioral (measure with `vigiles measure` / an eval).";
159
159
  }
160
160
  const lines = [];
161
161
  for (const e of exps) {
@@ -0,0 +1,21 @@
1
+ export interface CommandRefIssue {
2
+ readonly file: string;
3
+ readonly line: number;
4
+ /** The offending invocation, e.g. `vigiles compile-hook`. */
5
+ readonly ref: string;
6
+ readonly reason: string;
7
+ }
8
+ export interface KnownCommands {
9
+ readonly verbs: readonly string[];
10
+ readonly kinds: readonly string[];
11
+ }
12
+ /**
13
+ * Find stale/unknown vigiles command references across the given files. Pure —
14
+ * the caller supplies file contents (so it works over the repo in a test, or any
15
+ * file set).
16
+ */
17
+ export declare function findStaleCommandRefs(files: readonly {
18
+ readonly path: string;
19
+ readonly content: string;
20
+ }[], known?: KnownCommands): CommandRefIssue[];
21
+ //# sourceMappingURL=self-command-refs.d.ts.map
@@ -0,0 +1,125 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.findStaleCommandRefs = findStaleCommandRefs;
4
+ /**
5
+ * Self-command-reference verification — the cross-reference moat applied to
6
+ * vigiles's OWN docs. vigiles verifies that a CLAUDE.md's commands resolve; its
7
+ * own docs cite ITS OWN commands, and nothing checked those until a rename left
8
+ * `vigiles compile-hook` / `vigiles run-skill` rotting in the docs. This is the
9
+ * deterministic gate: every `vigiles <cmd>` reference in a doc/comment must
10
+ * resolve to a real command (a VERB or a `hook-runtime <kind>`).
11
+ *
12
+ * HIGH-PRECISION by construction (don't-cry-wolf). A reference is inspected only
13
+ * when it's an unambiguous COMMAND — i.e. it sits inside an inline `` `span` ``,
14
+ * inside a ```shell fence```, or is prefixed `npx vigiles` / `Usage: vigiles` /
15
+ * is a `cli.js` or `${CLI}` invocation (the harness-test convention `node ${CLI}
16
+ * <cmd>`). Prose ("vigiles compiles the spec", "adds vigiles to devDependencies")
17
+ * and non-shell fences (```text agent prompts, ```ts) are never matched. A bare unknown VERB is flagged only when hyphenated (every
18
+ * renamed vigiles command is — `compile-hook`, `run-skill`) or in an explicit
19
+ * invocation; `hook-runtime <kind>` is checked in any command context. Source of
20
+ * truth: {@link VERBS} / {@link HOOK_RUNTIME_KINDS}.
21
+ *
22
+ * KNOWN LIMITATION (measured, deliberately not "fixed"): this catches the
23
+ * `vigiles <cmd>` / `cli.js <cmd>` INVOCATION form. It does NOT flag a BARE
24
+ * command name (`agent-hook` without the `vigiles ` prefix) nor a bare file path
25
+ * (`src/foo.ts`), because both collide with legitimate non-references: bare
26
+ * command names double as CONCEPT names (`the refs-hook nudge` appears as a
27
+ * feature name in ~10 files; `agent-hook` as a rail name + in test labels +
28
+ * `makeTmpDir("agent-hook-cli")`), and bare paths collide with ILLUSTRATIVE
29
+ * examples (the README's `src/auth/login.ts`, `docs/foo.md`) and test fixtures
30
+ * (`session.test.ts` builds fake `src/compile.ts` diffs). A denylist over either
31
+ * would cry wolf on dozens of valid usages — so terminology/path-accuracy stays
32
+ * the JUDGMENT half the `doc-consistency` rule assigns to discipline, not a check.
33
+ */
34
+ const cli_commands_js_1 = require("./cli-commands.js");
35
+ const DEFAULT_KNOWN = {
36
+ verbs: cli_commands_js_1.VERBS,
37
+ kinds: cli_commands_js_1.HOOK_RUNTIME_KINDS,
38
+ };
39
+ const TOKEN = "[a-z][a-z0-9-]*";
40
+ // `[npx ][vigiles|cli.js|${CLI}] <verb> [<kind>]` — capture the optional prefix +
41
+ // literal. `${CLI}` is the harness-test convention (`const CLI = new
42
+ // URL("…/dist/cli.js")` → `node ${CLI} <cmd>`); without it a stale runtime ref in
43
+ // a `.mjs`/`.test.ts` invocation slips through (it did: refs-nudge.harness.mjs kept
44
+ // calling `refs-hook` after the hook-runtime rename). `\b` can't anchor `${CLI}`
45
+ // (it starts with `$`, a non-word char), so each word literal carries its own `\b`.
46
+ const INVOKE = new RegExp(String.raw `(npx |Usage: )?(\bvigiles|\bcli\.js|\$\{CLI\})\s+(${TOKEN})(?:\s+(${TOKEN}))?`, "g");
47
+ // A bare `hook-runtime <kind>` (e.g. `node ${CLI} hook-runtime agent` in a test).
48
+ const KIND = new RegExp(String.raw `hook-runtime\s+(${TOKEN})`, "g");
49
+ const SHELL_FENCE = /^\s*```(?:bash|sh|shell|zsh|console|shell-session)\s*$/;
50
+ const FENCE = /^\s*```/;
51
+ /** Inclusive index ranges of inline `code spans` in a line. */
52
+ function codeSpans(line) {
53
+ const ranges = [];
54
+ for (const m of line.matchAll(/`[^`\n]+`/g))
55
+ ranges.push([m.index, m.index + m[0].length]);
56
+ return ranges;
57
+ }
58
+ const inSpan = (i, spans) => spans.some(([a, b]) => i >= a && i < b);
59
+ function scanLine(line, inShellFence, known) {
60
+ const spans = inShellFence ? null : codeSpans(line);
61
+ const ctx = (i) => inShellFence || inSpan(i, spans ?? []);
62
+ const out = [];
63
+ const seen = new Set();
64
+ const push = (ref, reason) => {
65
+ const key = ref + "\0" + reason;
66
+ if (!seen.has(key)) {
67
+ seen.add(key);
68
+ out.push({ ref, reason });
69
+ }
70
+ };
71
+ for (const m of line.matchAll(INVOKE)) {
72
+ const explicit = inShellFence || Boolean(m[1]) || m[2] === "cli.js" || m[2] === "${CLI}";
73
+ if (!explicit && !ctx(m.index))
74
+ continue;
75
+ const verb = m[3];
76
+ if (verb === "hook-runtime") {
77
+ if (m[4] !== undefined && !known.kinds.includes(m[4]))
78
+ push(`hook-runtime ${m[4]}`, `unknown hook-runtime kind "${m[4]}"`);
79
+ }
80
+ else if (!known.verbs.includes(verb) &&
81
+ (verb.includes("-") || explicit)) {
82
+ push(`${m[2]} ${verb}`, `unknown/removed command "${verb}"`);
83
+ }
84
+ }
85
+ // Bare `hook-runtime <kind>` (no vigiles/cli.js literal) inside a command context.
86
+ for (const m of line.matchAll(KIND)) {
87
+ if (!ctx(m.index))
88
+ continue;
89
+ if (!known.kinds.includes(m[1]))
90
+ push(`hook-runtime ${m[1]}`, `unknown hook-runtime kind "${m[1]}"`);
91
+ }
92
+ return out;
93
+ }
94
+ /**
95
+ * Find stale/unknown vigiles command references across the given files. Pure —
96
+ * the caller supplies file contents (so it works over the repo in a test, or any
97
+ * file set).
98
+ */
99
+ function findStaleCommandRefs(files, known = DEFAULT_KNOWN) {
100
+ const issues = [];
101
+ for (const { path, content } of files) {
102
+ let inAnyFence = false;
103
+ let inShellFence = false;
104
+ const lines = content.split("\n");
105
+ for (let i = 0; i < lines.length; i++) {
106
+ const line = lines[i];
107
+ if (FENCE.test(line)) {
108
+ if (inAnyFence) {
109
+ inAnyFence = inShellFence = false;
110
+ }
111
+ else {
112
+ inAnyFence = true;
113
+ inShellFence = SHELL_FENCE.test(line);
114
+ }
115
+ continue;
116
+ }
117
+ if (line.includes("vigiles:ignore-cmd"))
118
+ continue;
119
+ for (const f of scanLine(line, inShellFence, known))
120
+ issues.push({ file: path, line: i + 1, ref: f.ref, reason: f.reason });
121
+ }
122
+ }
123
+ return issues;
124
+ }
125
+ //# sourceMappingURL=self-command-refs.js.map
package/dist/testing.d.ts CHANGED
@@ -10,13 +10,15 @@
10
10
  * boundary forbids importing `src/adapters/*` from here. See
11
11
  * `research/adapter-api-design.md`.
12
12
  */
13
- export * from "./run-hook.js";
14
- export * from "./eval.js";
13
+ export { runHook, propertyHook } from "./run-hook.js";
14
+ export type { HookRunResult, RunHookOptions, HookInput, HookOutput, HookPropertyResult, } from "./run-hook.js";
15
+ export { runEval, measure, measureArms, measureTriggerRate, assertRates, assertPromptDiversity, checkPromptDiversity, checkReportToJUnit, formatCheckReport, formatEvalReport, formatTriggerRateReport, claudeEvalDriver, parseClaudeRun, stubSkillBody, } from "./eval.js";
16
+ export type { EvalArm, EvalDriver, EvalSpec, EvalReport, EvalUsage, MeasureSpec, ArmsMeasureSpec, ArmReport, ArmUsage, ArmsCheckReport, CheckRate, CheckReport, MetricStat, Metrics, ModelOutputParser, ParsedModelRun, PromptDiversityIssue, PromptTriggerStat, RunContext, RunOut, SelectionTrialResult, TriggerRateReport, TriggerRateSpec, AgentRunArgs, AgentRunner, } from "./eval.js";
15
17
  export * from "./harness-assert.js";
16
18
  export * from "./check.js";
17
19
  export { hookFired } from "./check.js";
18
20
  export { judge } from "./judge.js";
19
21
  export * from "./tool-stub.js";
20
- export { runHarnessTest, runHarness, parseToolCalls, parseSubagents, parseResultEvent, parseOutput, parseHooks, decideSandbox, specTrusted, sandboxAvailable, } from "./harness-test.js";
22
+ export { runHarnessTest, runHarness } from "./harness-test.js";
21
23
  export type { HarnessTestSpec, Trace, SubagentTrace, HarnessTestResult, RunHarnessTestOptions, ModelTurn, ModelRequest, ToolCall, HookFire, HarnessTestDriver, SandboxMode, } from "./harness-test.js";
22
24
  //# sourceMappingURL=testing.d.ts.map
package/dist/testing.js CHANGED
@@ -1,4 +1,20 @@
1
1
  "use strict";
2
+ /**
3
+ * `vigiles/testing` — Pillar 2 entry point: the **harness-testing** API. Re-exports
4
+ * the three tiers — `runHook` (unit), `runHarnessTest` (deterministic), `runEval`
5
+ * (eval) — plus the runner-agnostic predicates/assertions. Kept deliberately
6
+ * separate from `vigiles/claude-code` so this surface can stay harness-agnostic as
7
+ * more harnesses are added. Granular paths (`vigiles/run-hook`, etc.) still work.
8
+ *
9
+ * It re-exports the composition-root runner modules (which do the Claude-Code
10
+ * default-wiring), never an adapter directly — the `agnostic-surface` eslint
11
+ * boundary forbids importing `src/adapters/*` from here. See
12
+ * `research/adapter-api-design.md`.
13
+ */
14
+ // Curated public re-exports (named, not `export *`) so the internal seams —
15
+ // the injectable `*With` runners, low-level `parse*`, pool/aggregate/model-tier
16
+ // helpers — stay out of the public surface, the api reports, and the docs site.
17
+ // (vigiles's own tests import those from the source modules directly.)
2
18
  var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
19
  if (k2 === undefined) k2 = k;
4
20
  var desc = Object.getOwnPropertyDescriptor(m, k);
@@ -14,21 +30,27 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
30
  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
31
  };
16
32
  Object.defineProperty(exports, "__esModule", { value: true });
17
- exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.parseHooks = exports.parseOutput = exports.parseResultEvent = exports.parseSubagents = exports.parseToolCalls = exports.runHarness = exports.runHarnessTest = exports.judge = exports.hookFired = void 0;
18
- /**
19
- * `vigiles/testing` — Pillar 2 entry point: the **harness-testing** API. Re-exports
20
- * the three tiers `runHook` (unit), `runHarnessTest` (deterministic), `runEval`
21
- * (eval) plus the runner-agnostic predicates/assertions. Kept deliberately
22
- * separate from `vigiles/claude-code` so this surface can stay harness-agnostic as
23
- * more harnesses are added. Granular paths (`vigiles/run-hook`, etc.) still work.
24
- *
25
- * It re-exports the composition-root runner modules (which do the Claude-Code
26
- * default-wiring), never an adapter directly the `agnostic-surface` eslint
27
- * boundary forbids importing `src/adapters/*` from here. See
28
- * `research/adapter-api-design.md`.
29
- */
30
- __exportStar(require("./run-hook.js"), exports);
31
- __exportStar(require("./eval.js"), exports);
33
+ exports.runHarness = exports.runHarnessTest = exports.judge = exports.hookFired = exports.stubSkillBody = exports.parseClaudeRun = exports.claudeEvalDriver = exports.formatTriggerRateReport = exports.formatEvalReport = exports.formatCheckReport = exports.checkReportToJUnit = exports.checkPromptDiversity = exports.assertPromptDiversity = exports.assertRates = exports.measureTriggerRate = exports.measureArms = exports.measure = exports.runEval = exports.propertyHook = exports.runHook = void 0;
34
+ // --- unit tier: runHook ---
35
+ var run_hook_js_1 = require("./run-hook.js");
36
+ Object.defineProperty(exports, "runHook", { enumerable: true, get: function () { return run_hook_js_1.runHook; } });
37
+ Object.defineProperty(exports, "propertyHook", { enumerable: true, get: function () { return run_hook_js_1.propertyHook; } });
38
+ // --- eval tier: runEval / measure / trigger-rate ---
39
+ var eval_js_1 = require("./eval.js");
40
+ Object.defineProperty(exports, "runEval", { enumerable: true, get: function () { return eval_js_1.runEval; } });
41
+ Object.defineProperty(exports, "measure", { enumerable: true, get: function () { return eval_js_1.measure; } });
42
+ Object.defineProperty(exports, "measureArms", { enumerable: true, get: function () { return eval_js_1.measureArms; } });
43
+ Object.defineProperty(exports, "measureTriggerRate", { enumerable: true, get: function () { return eval_js_1.measureTriggerRate; } });
44
+ Object.defineProperty(exports, "assertRates", { enumerable: true, get: function () { return eval_js_1.assertRates; } });
45
+ Object.defineProperty(exports, "assertPromptDiversity", { enumerable: true, get: function () { return eval_js_1.assertPromptDiversity; } });
46
+ Object.defineProperty(exports, "checkPromptDiversity", { enumerable: true, get: function () { return eval_js_1.checkPromptDiversity; } });
47
+ Object.defineProperty(exports, "checkReportToJUnit", { enumerable: true, get: function () { return eval_js_1.checkReportToJUnit; } });
48
+ Object.defineProperty(exports, "formatCheckReport", { enumerable: true, get: function () { return eval_js_1.formatCheckReport; } });
49
+ Object.defineProperty(exports, "formatEvalReport", { enumerable: true, get: function () { return eval_js_1.formatEvalReport; } });
50
+ Object.defineProperty(exports, "formatTriggerRateReport", { enumerable: true, get: function () { return eval_js_1.formatTriggerRateReport; } });
51
+ Object.defineProperty(exports, "claudeEvalDriver", { enumerable: true, get: function () { return eval_js_1.claudeEvalDriver; } });
52
+ Object.defineProperty(exports, "parseClaudeRun", { enumerable: true, get: function () { return eval_js_1.parseClaudeRun; } });
53
+ Object.defineProperty(exports, "stubSkillBody", { enumerable: true, get: function () { return eval_js_1.stubSkillBody; } });
32
54
  __exportStar(require("./harness-assert.js"), exports);
33
55
  // The declarative check vocabulary is now first-class at the front door. Its
34
56
  // `hookFired` (a `Check<Trace>`) supersedes the legacy boolean predicate of the
@@ -51,12 +73,4 @@ __exportStar(require("./tool-stub.js"), exports);
51
73
  var harness_test_js_1 = require("./harness-test.js");
52
74
  Object.defineProperty(exports, "runHarnessTest", { enumerable: true, get: function () { return harness_test_js_1.runHarnessTest; } });
53
75
  Object.defineProperty(exports, "runHarness", { enumerable: true, get: function () { return harness_test_js_1.runHarness; } });
54
- Object.defineProperty(exports, "parseToolCalls", { enumerable: true, get: function () { return harness_test_js_1.parseToolCalls; } });
55
- Object.defineProperty(exports, "parseSubagents", { enumerable: true, get: function () { return harness_test_js_1.parseSubagents; } });
56
- Object.defineProperty(exports, "parseResultEvent", { enumerable: true, get: function () { return harness_test_js_1.parseResultEvent; } });
57
- Object.defineProperty(exports, "parseOutput", { enumerable: true, get: function () { return harness_test_js_1.parseOutput; } });
58
- Object.defineProperty(exports, "parseHooks", { enumerable: true, get: function () { return harness_test_js_1.parseHooks; } });
59
- Object.defineProperty(exports, "decideSandbox", { enumerable: true, get: function () { return harness_test_js_1.decideSandbox; } });
60
- Object.defineProperty(exports, "specTrusted", { enumerable: true, get: function () { return harness_test_js_1.specTrusted; } });
61
- Object.defineProperty(exports, "sandboxAvailable", { enumerable: true, get: function () { return harness_test_js_1.sandboxAvailable; } });
62
76
  //# sourceMappingURL=testing.js.map
@@ -15,7 +15,7 @@
15
15
  *
16
16
  * - `decideIntercept` — does this call get intercepted, and with what deny reason;
17
17
  * - `buildInterceptSettings` — the PreToolUse hook fragment routing matched tools
18
- * through `vigiles intercept-tool-hook`;
18
+ * through `vigiles hook-runtime intercept-tool`;
19
19
  * - `serializeIntercepts` / `parseIntercepts` — the env round-trip (incl. RegExp
20
20
  * matchers) the hook subprocess reads back.
21
21
  *
@@ -32,7 +32,7 @@
32
32
  * tools, so deny+reason is the closest available — with that ceiling.
33
33
  */
34
34
  import { type ArgMatcher } from "./arg-match.js";
35
- /** Env var the spawned `vigiles intercept-tool-hook` reads its intercept list from. */
35
+ /** Env var the spawned `vigiles hook-runtime intercept-tool` reads its intercept list from. */
36
36
  export declare const INTERCEPT_TOOLS_ENV = "VIGILES_INTERCEPT_TOOLS";
37
37
  /** Declare a tool to intercept: deny its real execution with a block message. */
38
38
  export interface ToolIntercept {
@@ -63,7 +63,7 @@ export type InterceptDecision = {
63
63
  /**
64
64
  * Decide whether a tool call should be intercepted. Returns the first matching
65
65
  * intercept's denial reason (preventing real execution), or `{ intercept: false }`
66
- * to let the call run for real. Pure — the same logic `vigiles intercept-tool-hook`
66
+ * to let the call run for real. Pure — the same logic `vigiles hook-runtime intercept-tool`
67
67
  * runs.
68
68
  */
69
69
  export declare function decideIntercept(toolName: string, input: unknown, intercepts: readonly ToolIntercept[]): InterceptDecision;
@@ -81,7 +81,7 @@ export interface InterceptSettings {
81
81
  }
82
82
  /**
83
83
  * Build the PreToolUse hook fragment that routes every intercepted tool through
84
- * `vigiles intercept-tool-hook`. The `matcher` is a CC tool-name regex over the
84
+ * `vigiles hook-runtime intercept-tool`. The `matcher` is a CC tool-name regex over the
85
85
  * union of intercepted tool names (each escaped), so unrelated tools are never
86
86
  * intercepted. Merge the result into an arm's `settings`; the intercept list
87
87
  * itself travels in the {@link INTERCEPT_TOOLS_ENV} env var (see