vigiles 5.1.0 → 5.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +2 -2
  2. package/dist/adapters/claude-code/adapter.js +1 -0
  3. package/dist/adapters/claude-code/agent-runtime.d.ts +20 -6
  4. package/dist/adapters/claude-code/agent-runtime.js +51 -8
  5. package/dist/adapters/claude-code/dialect.js +19 -0
  6. package/dist/adapters/claude-code/effect-region.d.ts +9 -0
  7. package/dist/adapters/claude-code/effect-region.js +45 -0
  8. package/dist/adapters/claude-code/layout.js +3 -0
  9. package/dist/adapters/claude-code/skill-runtime.d.ts +25 -0
  10. package/dist/adapters/claude-code/skill-runtime.js +48 -0
  11. package/dist/adapters/codex/adapter.js +3 -0
  12. package/dist/adapters/codex/layout.js +3 -0
  13. package/dist/adapters/opencode/adapter.js +1 -0
  14. package/dist/adapters/opencode/layout.js +3 -0
  15. package/dist/check.d.ts +8 -0
  16. package/dist/check.js +27 -3
  17. package/dist/cli.js +323 -88
  18. package/dist/core/adapter.d.ts +10 -0
  19. package/dist/core/bash-effects.d.ts +41 -0
  20. package/dist/core/bash-effects.js +405 -0
  21. package/dist/core/compile.d.ts +3 -1
  22. package/dist/core/compile.js +162 -39
  23. package/dist/core/dialect.d.ts +10 -0
  24. package/dist/core/effects.d.ts +172 -0
  25. package/dist/core/effects.js +245 -0
  26. package/dist/core/layout.d.ts +6 -0
  27. package/dist/core/mcp-tool.d.ts +1 -1
  28. package/dist/core/orphans.js +21 -0
  29. package/dist/core/spec.d.ts +142 -3
  30. package/dist/core/spec.js +48 -0
  31. package/dist/core/tool-contract.d.ts +1 -1
  32. package/dist/core/types.d.ts +6 -6
  33. package/dist/core/validate.js +4 -4
  34. package/dist/harness-test.d.ts +7 -0
  35. package/dist/harness-test.js +19 -7
  36. package/dist/leaderboard.d.ts +2 -0
  37. package/dist/leaderboard.js +2 -0
  38. package/dist/optimize.d.ts +74 -0
  39. package/dist/optimize.js +94 -0
  40. package/dist/scaffold-test.d.ts +30 -0
  41. package/dist/scaffold-test.js +158 -0
  42. package/dist/scan.d.ts +40 -0
  43. package/dist/scan.js +91 -43
  44. package/dist/score-explainer.d.ts +69 -0
  45. package/dist/score-explainer.js +169 -0
  46. package/dist/test-coverage.d.ts +7 -0
  47. package/dist/test-coverage.js +39 -24
  48. package/package.json +2 -1
  49. package/skills/{migrate-to-spec → adopt-spec}/SKILL.md +4 -4
  50. package/skills/edit-spec/SKILL.md +1 -1
@@ -43,15 +43,15 @@ const DEFAULT_RULES = {
43
43
  coverage: false,
44
44
  // Per-kind surface-coverage: a skill/agent/hook must ship with a test or eval.
45
45
  "untested-skill": "warn",
46
- "untested-agent": "warn",
46
+ "untested-subagent": "warn",
47
47
  "untested-hook": "warn",
48
48
  "unmarked-refs": "warn",
49
49
  // High-precision (never-available + close typos only), so on by default at warn.
50
- "agent-tool-contract": "warn",
50
+ "subagent-tool-contract": "warn",
51
51
  // High-precision (close typos only), on by default at warn.
52
52
  "hook-events": "warn",
53
53
  // Missing required frontmatter (name/description) — on by default at warn.
54
- "agent-frontmatter": "warn",
54
+ "subagent-frontmatter": "warn",
55
55
  // A declared MCP server with no command/url can't start — on by default at warn.
56
56
  "mcp-config": "warn",
57
57
  // Best-practice nudge (skills load without frontmatter) — warn, not error.
@@ -60,7 +60,7 @@ const DEFAULT_RULES = {
60
60
  "mcp-tool-resolves": "warn",
61
61
  // A hook script referenced but missing never runs — on by default at warn.
62
62
  "hook-script-exists": "warn",
63
- // High-precision (close-typo only) deny-list mirror of agent-tool-contract.
63
+ // High-precision (close-typo only) deny-list mirror of subagent-tool-contract.
64
64
  "disallowed-tools-contract": "warn",
65
65
  // Deterministic NCD precision proxy (near-identical skill descriptions) — warn.
66
66
  "description-overlap": "warn",
@@ -110,6 +110,13 @@ export interface SubagentTrace {
110
110
  readonly name: string;
111
111
  /** The tools the subagent invoked (events tagged with the Task's id). */
112
112
  readonly toolCalls: readonly ToolCall[];
113
+ /**
114
+ * The subagent's RETURNED text — the dispatch tool_result the orchestrator
115
+ * receives back. This is where a `result()` contract's `vigiles:ok`/`vigiles:err`
116
+ * block lands, so `subagent(name, [output(/vigiles:ok/)])` can assert the typed
117
+ * outcome. "" if not captured.
118
+ */
119
+ readonly output: string;
113
120
  }
114
121
  export interface HarnessTestResult extends Trace {
115
122
  readonly exitCode: number;
@@ -130,6 +130,7 @@ function parseToolCalls(streamJson) {
130
130
  */
131
131
  function parseSubagents(streamJson) {
132
132
  const tasks = new Map(); // dispatch id → subagent name
133
+ const dispatchOutput = new Map(); // dispatch id → returned text
133
134
  const byParent = new Map();
134
135
  const groupFor = (parent) => {
135
136
  let g = byParent.get(parent);
@@ -162,7 +163,10 @@ function parseSubagents(streamJson) {
162
163
  // A subagent dispatch is any top-level tool_use whose input carries a
163
164
  // `subagent_type` — the dispatch tool is named "Agent" on the live CLI
164
165
  // (older docs say "Task"), so match the input field, NOT the tool name,
165
- // to survive the rename. Confirmed against real claude output.
166
+ // to survive the rename. Confirmed against real claude output. CC NOTE:
167
+ // under `--plugin-dir` the value is NAMESPACED `plugin:agent` (captured
168
+ // "reviewer-spec:code-reviewer"); the bare agent name is matched in the
169
+ // `subagent()` check (src/check.ts), so the full id is preserved here.
166
170
  const sub = b.input?.subagent_type;
167
171
  if (typeof sub === "string")
168
172
  tasks.set(id, sub);
@@ -170,12 +174,20 @@ function parseSubagents(streamJson) {
170
174
  if (parent)
171
175
  groupFor(parent).uses.push({ id, name: b.name, input: b.input });
172
176
  }
173
- else if (b.type === "tool_result" && parent) {
177
+ else if (b.type === "tool_result") {
174
178
  const id = typeof b.tool_use_id === "string" ? b.tool_use_id : "";
175
- groupFor(parent).results.set(id, {
176
- text: contentText(b.content),
177
- isError: b.is_error === true,
178
- });
179
+ if (parent) {
180
+ groupFor(parent).results.set(id, {
181
+ text: contentText(b.content),
182
+ isError: b.is_error === true,
183
+ });
184
+ }
185
+ else if (id) {
186
+ // A top-level tool_result whose id is a subagent dispatch is the SUB's
187
+ // RETURN to the orchestrator (where a result() vigiles:ok/err block
188
+ // lands). Record it; matched to its dispatch by id below.
189
+ dispatchOutput.set(id, contentText(b.content));
190
+ }
179
191
  }
180
192
  }
181
193
  }
@@ -188,7 +200,7 @@ function parseSubagents(streamJson) {
188
200
  resultText: g?.results.get(u.id)?.text ?? "",
189
201
  isError: g?.results.get(u.id)?.isError ?? false,
190
202
  }));
191
- out.push({ name, toolCalls });
203
+ out.push({ name, toolCalls, output: dispatchOutput.get(taskId) ?? "" });
192
204
  }
193
205
  return out;
194
206
  }
@@ -21,6 +21,8 @@ export interface PluginScore {
21
21
  readonly issues: readonly string[];
22
22
  readonly report: ScanReport;
23
23
  }
24
+ /** Map a 0–100 structural-health score to its letter grade (A ≥90 … F <60). */
25
+ export declare function gradeFor(score: number): PluginScore["grade"];
24
26
  /** Deterministic structural-health score for one scanned plugin. */
25
27
  export declare function scoreReport(r: ScanReport): {
26
28
  score: number;
@@ -12,6 +12,7 @@
12
12
  * model and stack on top later; this part runs anywhere in CI for free.
13
13
  */
14
14
  Object.defineProperty(exports, "__esModule", { value: true });
15
+ exports.gradeFor = gradeFor;
15
16
  exports.scoreReport = scoreReport;
16
17
  exports.rankPlugins = rankPlugins;
17
18
  exports.formatLeaderboard = formatLeaderboard;
@@ -23,6 +24,7 @@ const W_NO_DESCRIPTION = 10; // a skill with no usable description → can't tri
23
24
  const W_DANGLING_REF = 8; // a referenced intra-plugin file that's missing → broken path
24
25
  const W_NO_CONTRACT = 5; // an agent with no `tools:` line → inherits everything
25
26
  const W_UNTESTED = 3; // a surface with no test/eval → warning-tier
27
+ /** Map a 0–100 structural-health score to its letter grade (A ≥90 … F <60). */
26
28
  function gradeFor(score) {
27
29
  if (score >= 90)
28
30
  return "A";
@@ -0,0 +1,74 @@
1
+ /**
2
+ * The per-repo harness optimizer's DETERMINISTIC spine — shipped as the
3
+ * `vigiles scan --fix-plan` lens (NOT its own `optimize` verb: until the measured
4
+ * A/B half lands, an "optimizer" that only re-prints scan's findings doesn't earn
5
+ * a separate command, so it's folded into scan as one more view on the same
6
+ * report; see research/roadmap.md §P2 "reconsider an `optimize` verb").
7
+ *
8
+ * A2 in the measurement-authority pivot is the ADOPTION product: measure a user's
9
+ * own skills/model/rules on their tasks and recommend add/drop/swap with a MEASURED
10
+ * delta. The measured delta is the real-model layer (gated on the Pro/Max
11
+ * subscription, costs tokens); this v0 ships the deterministic HALF — the free
12
+ * pre-filter that runs on every commit with no model.
13
+ *
14
+ * It answers "what should I fix in my harness, and why" using only the cross-ref
15
+ * findings the linter already computes: it reuses `scoreReport` for the headline
16
+ * structural-health score and `explainScore` for the per-surface cause + one-line
17
+ * fix (one-detector-no-drift — it never re-detects). The result is a prioritized,
18
+ * typed action list — the spine the measured A/B (the behavioral delta) stacks on.
19
+ *
20
+ * This is the "linting as a free pre-filter to measurement" thesis made a command:
21
+ * clear the structural dead-ends a model can't help with FIRST (free, certain),
22
+ * THEN spend tokens measuring whether the structurally-clean skills earn their keep.
23
+ *
24
+ * Distinct from `vigiles explain` (which diagnoses ONE underperforming surface a
25
+ * measurement flagged): `optimize` is the whole-repo adoption view — health score +
26
+ * the ranked fix list + the hand-off to the measured layer. Same findings, the
27
+ * optimization framing. See research/measurement-authority.md (A2) + roadmap §P1.
28
+ */
29
+ import { type PluginScore } from "./leaderboard.js";
30
+ import { type ExplanationConfidence } from "./score-explainer.js";
31
+ import type { ScanReport } from "./scan.js";
32
+ /**
33
+ * The action a recommendation asks for. The deterministic detectors yield two:
34
+ * `"differentiate"` (a description-overlap PAIR — make the two distinct so the
35
+ * selector can disambiguate) and `"fix"` (every other structural dead-end —
36
+ * correct/add/remove the offending bit). The richer add/drop/swap vocabulary
37
+ * belongs to the MEASURED layer, once a behavioral delta ranks the alternatives.
38
+ */
39
+ export type OptimizeAction = "fix" | "differentiate";
40
+ export interface Recommendation {
41
+ /** The affected surface (a skill/agent/hook name or path, or an `a ↔ b` pair). */
42
+ readonly surface: string;
43
+ readonly action: OptimizeAction;
44
+ /** The deterministic cause (the detector's own message — no drift). */
45
+ readonly rationale: string;
46
+ /** A single, actionable fix. */
47
+ readonly fix: string;
48
+ /** The lint rule that found it (open `docs/rules/<detector>.md`). */
49
+ readonly detector: string;
50
+ readonly confidence: ExplanationConfidence;
51
+ }
52
+ export interface OptimizeReport {
53
+ readonly dir: string;
54
+ /** Structural-health score 0–100 (the same `scoreReport` the leaderboard uses). */
55
+ readonly score: number;
56
+ readonly grade: PluginScore["grade"];
57
+ /** The free deterministic fixes, `likely` dead-ends first (explainScore's order). */
58
+ readonly recommendations: readonly Recommendation[];
59
+ /**
60
+ * No loadable surface at all (not a plugin, or a broken load) — distinct from a
61
+ * clean-and-loaded harness with zero findings, so the formatter doesn't call an
62
+ * EMPTY machine "clean". Mirrors `scoreReport`'s empty-machine case.
63
+ */
64
+ readonly empty: boolean;
65
+ }
66
+ /**
67
+ * Turn a scan report into a prioritized optimization plan: the structural-health
68
+ * score + a typed recommendation per deterministic finding (`likely` dead-ends
69
+ * before `possible` proxies, via explainScore's own ordering). Pure over the report.
70
+ */
71
+ export declare function optimize(report: ScanReport): OptimizeReport;
72
+ /** Render an optimization plan for the CLI. */
73
+ export declare function formatOptimize(rep: OptimizeReport): string;
74
+ //# sourceMappingURL=optimize.d.ts.map
@@ -0,0 +1,94 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.optimize = optimize;
4
+ exports.formatOptimize = formatOptimize;
5
+ /**
6
+ * The per-repo harness optimizer's DETERMINISTIC spine — shipped as the
7
+ * `vigiles scan --fix-plan` lens (NOT its own `optimize` verb: until the measured
8
+ * A/B half lands, an "optimizer" that only re-prints scan's findings doesn't earn
9
+ * a separate command, so it's folded into scan as one more view on the same
10
+ * report; see research/roadmap.md §P2 "reconsider an `optimize` verb").
11
+ *
12
+ * A2 in the measurement-authority pivot is the ADOPTION product: measure a user's
13
+ * own skills/model/rules on their tasks and recommend add/drop/swap with a MEASURED
14
+ * delta. The measured delta is the real-model layer (gated on the Pro/Max
15
+ * subscription, costs tokens); this v0 ships the deterministic HALF — the free
16
+ * pre-filter that runs on every commit with no model.
17
+ *
18
+ * It answers "what should I fix in my harness, and why" using only the cross-ref
19
+ * findings the linter already computes: it reuses `scoreReport` for the headline
20
+ * structural-health score and `explainScore` for the per-surface cause + one-line
21
+ * fix (one-detector-no-drift — it never re-detects). The result is a prioritized,
22
+ * typed action list — the spine the measured A/B (the behavioral delta) stacks on.
23
+ *
24
+ * This is the "linting as a free pre-filter to measurement" thesis made a command:
25
+ * clear the structural dead-ends a model can't help with FIRST (free, certain),
26
+ * THEN spend tokens measuring whether the structurally-clean skills earn their keep.
27
+ *
28
+ * Distinct from `vigiles explain` (which diagnoses ONE underperforming surface a
29
+ * measurement flagged): `optimize` is the whole-repo adoption view — health score +
30
+ * the ranked fix list + the hand-off to the measured layer. Same findings, the
31
+ * optimization framing. See research/measurement-authority.md (A2) + roadmap §P1.
32
+ */
33
+ const leaderboard_js_1 = require("./leaderboard.js");
34
+ const score_explainer_js_1 = require("./score-explainer.js");
35
+ function actionFor(e) {
36
+ return e.symptom === "wrong-skill-fires" ? "differentiate" : "fix";
37
+ }
38
+ function isEmptyMachine(r) {
39
+ const surfaces = r.skills.length + r.agents.length + r.hooks.length + r.commands;
40
+ return surfaces === 0 && !r.mcp;
41
+ }
42
+ /**
43
+ * Turn a scan report into a prioritized optimization plan: the structural-health
44
+ * score + a typed recommendation per deterministic finding (`likely` dead-ends
45
+ * before `possible` proxies, via explainScore's own ordering). Pure over the report.
46
+ */
47
+ function optimize(report) {
48
+ const { score } = (0, leaderboard_js_1.scoreReport)(report);
49
+ const recommendations = (0, score_explainer_js_1.explainScore)(report).map((e) => ({
50
+ surface: e.surface,
51
+ action: actionFor(e),
52
+ rationale: e.cause,
53
+ fix: e.fix,
54
+ detector: e.detector,
55
+ confidence: e.confidence,
56
+ }));
57
+ return {
58
+ dir: report.dir,
59
+ score,
60
+ grade: (0, leaderboard_js_1.gradeFor)(score),
61
+ recommendations,
62
+ empty: isEmptyMachine(report),
63
+ };
64
+ }
65
+ const ACTION_LABEL = {
66
+ fix: "FIX",
67
+ differentiate: "DIFFERENTIATE",
68
+ };
69
+ const measureHint = (dir) => `\`vigiles scan ${dir} --trigger\` — real-model, runs on your subscription`;
70
+ /** Render an optimization plan for the CLI. */
71
+ function formatOptimize(rep) {
72
+ const head = `Harness health: ${String(rep.score)}/100 (${rep.grade}) — ${rep.dir}`;
73
+ if (rep.empty) {
74
+ return `${head}\n\nNothing loaded — this isn't a plugin/harness, or the load failed. Point optimize at a dir with a CLAUDE.md/AGENTS.md, skills, agents, or hooks.`;
75
+ }
76
+ if (rep.recommendations.length === 0) {
77
+ return `${head}\n\nNo deterministic fixes found — the structure is clean. Whether your skills actually help is a BEHAVIORAL question; measure it with ${measureHint(rep.dir)}.`;
78
+ }
79
+ const lines = [
80
+ head,
81
+ "",
82
+ `${String(rep.recommendations.length)} deterministic fix(es) — free, no model. Apply these before measuring:`,
83
+ "",
84
+ ];
85
+ for (const r of rep.recommendations) {
86
+ const mark = r.confidence === "likely" ? "✗" : "⚠";
87
+ lines.push(`${mark} [${ACTION_LABEL[r.action]}] ${r.surface}`);
88
+ lines.push(` why: ${r.rationale} [${r.detector}]`);
89
+ lines.push(` → ${r.fix}`);
90
+ }
91
+ lines.push("", `Then measure the behavioral delta of what's left (does each skill earn its keep?) with ${measureHint(rep.dir)}.`);
92
+ return lines.join("\n");
93
+ }
94
+ //# sourceMappingURL=optimize.js.map
@@ -0,0 +1,30 @@
1
+ export type SurfaceKind = "skill" | "agent" | "hook";
2
+ /** The cheapest meaningful tier for a surface kind (mirrors the test-harness skill). */
3
+ export type TestTier = "unit" | "harness" | "eval";
4
+ /** What the generator needs to know about a surface to scaffold its test. */
5
+ export interface ScaffoldInput {
6
+ readonly kind: SurfaceKind;
7
+ /** Skill dir name / agent name / hook-script basename. */
8
+ readonly name: string;
9
+ /** Repo-relative path to the surface (SKILL.md / agent .md / hook script). */
10
+ readonly path: string;
11
+ /** Plugin name for the namespaced skill id; a placeholder TODO when unknown. */
12
+ readonly pluginName?: string;
13
+ /** A user-invoked skill — trigger-rate is for model-invocable skills, so note it. */
14
+ readonly userInvoked?: boolean;
15
+ /** A subagent's declared tool contract (drives the assertion hint); null = inherits all. */
16
+ readonly tools?: readonly string[] | null;
17
+ /** How the CLI invokes the hook (e.g. `bash hooks/pre-edit.sh`); a TODO when unknown. */
18
+ readonly hookCommand?: string;
19
+ }
20
+ export interface Scaffold {
21
+ readonly path: string;
22
+ readonly content: string;
23
+ readonly kind: SurfaceKind;
24
+ readonly tier: TestTier;
25
+ }
26
+ /** Scaffold a starter test for one surface. Pure: path + content, no I/O. */
27
+ export declare function scaffoldTest(input: ScaffoldInput): Scaffold;
28
+ /** Render a set of scaffolds for the CLI (what was generated, where, which tier). */
29
+ export declare function formatScaffolds(scaffolds: readonly Scaffold[]): string;
30
+ //# sourceMappingURL=scaffold-test.d.ts.map
@@ -0,0 +1,158 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.scaffoldTest = scaffoldTest;
4
+ exports.formatScaffolds = formatScaffolds;
5
+ /**
6
+ * `vigiles scaffold-test` — the deterministic test-gen engine (B1 v0).
7
+ *
8
+ * Free-form in, a RUNNABLE starter test out. Given an existing hand-written
9
+ * skill / subagent / hook, emit a scaffolded `*.harness.mjs` / `*.eval.mjs` at the
10
+ * surface's suggested test path — the deterministic counterpart to the
11
+ * `test-harness` SKILL (which picks the tier with a model). The scaffold picks the
12
+ * cheapest meaningful tier for the kind, wires the real public API + the surface's
13
+ * own metadata (name, namespaced id, declared tools), and leaves TODOs only where a
14
+ * human/model must supply judgement (the prompts, the event input, the assertion).
15
+ *
16
+ * Pure + model-free: hand it a `ScaffoldInput`, get back a `{ path, content }`. The
17
+ * CLI resolves a surface path / plugin dir into inputs (reusing the scan + untested
18
+ * detectors) and writes the files; this module owns only the templating.
19
+ */
20
+ const node_path_1 = require("node:path");
21
+ const PLUGIN_TODO = "<plugin>";
22
+ /**
23
+ * The suggested test path for a surface — mirrors `suggestedTestPath` in
24
+ * `test-coverage.ts` (a skill gets an `.eval.mjs`, agent/hook a `.harness.mjs`),
25
+ * so a generated file is colocated where the untested-surface detector looks for it
26
+ * and the surface stops being reported untested.
27
+ */
28
+ function suggestedPath(input) {
29
+ const dir = (0, node_path_1.dirname)(input.path);
30
+ const ext = input.kind === "skill" ? "eval.mjs" : "harness.mjs";
31
+ return `${dir}/${input.name}.${ext}`;
32
+ }
33
+ function header(title, run) {
34
+ return [
35
+ "/**",
36
+ ` * ${title}`,
37
+ " *",
38
+ " * Generated by `vigiles scaffold-test` — a STARTER, not a finished test. Fill in",
39
+ " * the TODOs (they're where a human/model must supply judgement), then run:",
40
+ ` * ${run}`,
41
+ " */",
42
+ ].join("\n");
43
+ }
44
+ /** A hook → the unit tier (`runHook`): free, no model, reaches every event. */
45
+ function hookScaffold(input) {
46
+ const cmd = input.hookCommand ?? `bash ${input.path}`;
47
+ return `${header(`Starter unit test for the \`${input.name}\` hook.`, `npx vigiles test ${suggestedPath(input)}`)}
48
+ import { runHook, assertHookAllowed } from "vigiles/unit";
49
+
50
+ // TODO: set the event + input your hook actually inspects (PreToolUse/Bash shown).
51
+ const event = {
52
+ hook_event_name: "PreToolUse",
53
+ tool_name: "Bash",
54
+ tool_input: { command: "echo hello" },
55
+ };
56
+
57
+ const r = runHook(${JSON.stringify(cmd)}, event);
58
+
59
+ // TODO: assert the decision you expect. Use assertHookBlocked(r) for the deny case
60
+ // (and a second runHook with the input that SHOULD be blocked).
61
+ assertHookAllowed(r);
62
+ console.log("✓ ${input.name}: hook allowed the benign event");
63
+ `;
64
+ }
65
+ /** A skill → the eval tier (`measureTriggerRate`): does its description FIRE? */
66
+ function skillScaffold(input) {
67
+ const id = `${input.pluginName ?? PLUGIN_TODO}:${input.name}`;
68
+ const note = input.userInvoked
69
+ ? "\n// NOTE: this skill is user-invoked (disableModelInvocation). Trigger-rate\n// measures MODEL-invocable skills; either make it model-invocable or test its\n// slash-command invocation with runHarnessTest instead.\n"
70
+ : "";
71
+ return `${header(`Starter trigger-rate eval for the \`${id}\` skill (recall + precision).`, `npx vigiles eval ${suggestedPath(input)} # real model, on your subscription`)}
72
+ import {
73
+ measureTriggerRate,
74
+ formatTriggerRateReport,
75
+ assertTriggerRate,
76
+ skillResolved,
77
+ } from "vigiles/testing";
78
+ import { fileURLToPath } from "node:url";
79
+ ${note}
80
+ // TODO: point at the plugin root (the dir holding .claude-plugin/ or skills/).
81
+ const pluginDir = fileURLToPath(new URL("../../", import.meta.url));
82
+ const skill = ${JSON.stringify(id)};
83
+
84
+ const report = await measureTriggerRate({
85
+ pluginDir,
86
+ stubSkillBodies: true, // firing is a frontmatter property — stub bodies, pay less
87
+ prompts: [
88
+ // TODO: >=5 varied prompts that SHOULD fire ${input.name} (recall).
89
+ "TODO: a realistic task that should trigger ${input.name}",
90
+ ],
91
+ irrelevantPrompts: [
92
+ // TODO: >=5 unrelated prompts that should NOT fire it (precision).
93
+ "TODO: an unrelated coding task",
94
+ ],
95
+ fired: (t) => skillResolved(t, skill),
96
+ trials: Number(process.env.VIGILES_TRIALS || 1),
97
+ });
98
+
99
+ console.log(formatTriggerRateReport(report));
100
+ assertTriggerRate(report, { min: 0.8, maxFalsePositive: 0.3 });
101
+ `;
102
+ }
103
+ /** A subagent → the deterministic harness tier; point at the railway/Result path. */
104
+ function agentScaffold(input) {
105
+ const toolHint = input.tools && input.tools.length > 0
106
+ ? `assertToolUsed(r, ${JSON.stringify(input.tools[0])}); // its declared contract: ${input.tools.join(", ")}`
107
+ : `assertToolUsed(r, "Task"); // TODO: assert what the subagent should do`;
108
+ return `${header(`Starter harness test for the \`${input.name}\` subagent.`, `npx vigiles test ${suggestedPath(input)}`)}
109
+ import { runHarnessTest, assertToolUsed } from "vigiles/testing";
110
+
111
+ // A subagent's OUTCOME is best asserted via a result() contract — deterministic,
112
+ // no LLM judge. If ${input.name} has one, use assertAgentOk(r.output, contract)
113
+ // instead; see the railway-result example in the vigiles docs.
114
+
115
+ const r = await runHarnessTest({
116
+ plugin: ".", // TODO: the plugin dir holding this subagent
117
+ // TODO: a prompt that dispatches ${input.name} (via the Task tool).
118
+ prompt: "TODO: a task that should dispatch ${input.name}",
119
+ transcript: true,
120
+ model: [{ text: "on it" }],
121
+ });
122
+
123
+ ${toolHint}
124
+ console.log("✓ ${input.name}: subagent test ran");
125
+ `;
126
+ }
127
+ const TIER = {
128
+ hook: "unit",
129
+ skill: "eval",
130
+ agent: "harness",
131
+ };
132
+ const BUILDER = {
133
+ hook: hookScaffold,
134
+ skill: skillScaffold,
135
+ agent: agentScaffold,
136
+ };
137
+ /** Scaffold a starter test for one surface. Pure: path + content, no I/O. */
138
+ function scaffoldTest(input) {
139
+ return {
140
+ path: suggestedPath(input),
141
+ content: BUILDER[input.kind](input),
142
+ kind: input.kind,
143
+ tier: TIER[input.kind],
144
+ };
145
+ }
146
+ /** Render a set of scaffolds for the CLI (what was generated, where, which tier). */
147
+ function formatScaffolds(scaffolds) {
148
+ if (scaffolds.length === 0) {
149
+ return "Nothing to scaffold — every surface already has a test, or none was found.";
150
+ }
151
+ const lines = [`Scaffolded ${String(scaffolds.length)} starter test(s):`, ""];
152
+ for (const s of scaffolds) {
153
+ lines.push(` ${s.path} [${s.kind} → ${s.tier} tier]`);
154
+ }
155
+ lines.push("", "These are STARTERS — fill in the TODOs (prompts / event / assertions), then run", "them with `npx vigiles test` (deterministic) or `npx vigiles eval` (real model).");
156
+ return lines.join("\n");
157
+ }
158
+ //# sourceMappingURL=scaffold-test.js.map
package/dist/scan.d.ts CHANGED
@@ -19,6 +19,7 @@ import { type McpIssue } from "./core/mcp-config.js";
19
19
  import { type DescriptionOverlap } from "./core/description-overlap.js";
20
20
  import { type McpToolIssue } from "./core/mcp-tool.js";
21
21
  import { type McpHookIssue } from "./core/mcp-hook.js";
22
+ import { type PurityLevel, type EffectSurface } from "./core/effects.js";
22
23
  /** A named writing system. The label `unexpectedScript` reports + the config's expectation parse into this. */
23
24
  export type Script = "Latin" | "Cyrillic" | "Han" | "Japanese" | "Korean" | "Arabic" | "Hebrew" | "Greek" | "Devanagari" | "Thai";
24
25
  export interface ScanSkill {
@@ -47,6 +48,19 @@ export interface ScanAgent {
47
48
  readonly mcpToolIssues: readonly McpToolIssue[];
48
49
  /** `disallowedTools:` block-list entries that are typos of a real tool (block nothing). */
49
50
  readonly disallowedToolIssues: readonly ToolIssue[];
51
+ /**
52
+ * Static effect-surface purity of the agent's declared tool contract.
53
+ * - `"pure"` — no side-effecting tools (read-only, deterministically testable).
54
+ * - `"bounded"` — has side-effecting tools (Edit/Write/…) but no Bash or unknown.
55
+ * - `"unrestricted"` — has Bash, any MCP/unknown tool, or inherits-all (no contract).
56
+ * Computed by `effectSurface()` from `src/core/effects.ts` — one detector, no drift.
57
+ */
58
+ readonly purity: PurityLevel;
59
+ /**
60
+ * The three tool buckets from `effectSurface()`: read-only, side-effecting, and
61
+ * unknown-effect (MCP or unrecognized) tool names in the declared contract.
62
+ */
63
+ readonly effectBuckets: Pick<EffectSurface, "readOnly" | "sideEffecting" | "unknown">;
50
64
  }
51
65
  /** A skill/agent whose frontmatter is missing a required field (name / description). */
52
66
  export interface FrontmatterIssue {
@@ -121,6 +135,32 @@ export interface ScanReport {
121
135
  readonly malformedFrontmatter: readonly FrontmatterParseIssue[];
122
136
  readonly warnings: readonly string[];
123
137
  readonly untested: number;
138
+ /**
139
+ * Harness-level purity summary: how many scanned agents fall into each purity
140
+ * rung. A high `pure` count means more of the harness is statically testable
141
+ * (deterministic, no mocks); `unrestricted` is the blind-spot count.
142
+ * Computed by `effectSurface()` (one detector, no drift).
143
+ */
144
+ readonly puritySummary: {
145
+ pure: number;
146
+ bounded: number;
147
+ unrestricted: number;
148
+ };
149
+ }
150
+ /**
151
+ * Per-kind surface classifiers, built from the harness `PluginLayout`'s
152
+ * `skillDir`/`agentDir`/`commandDir` — so adding a harness whose subagents live
153
+ * somewhere other than `agents/` (OpenCode's `.opencode/agent`) needs no change
154
+ * here. Each anchors on a real path boundary (start-of-path or a `/`), so a
155
+ * directory whose NAME merely ends in the keyword isn't misclassified — e.g. the
156
+ * skill `skills/dispatching-parallel-agents/SKILL.md` must NOT register as an
157
+ * agent named "SKILL" (the `-agents/` substring), which real plugins like
158
+ * obra/superpowers ship. See scan.test.ts for the regression cases.
159
+ */
160
+ export interface SurfaceClassifier {
161
+ readonly isSkill: (f: string) => boolean;
162
+ readonly isAgent: (f: string) => boolean;
163
+ readonly isCommand: (f: string) => boolean;
124
164
  }
125
165
  /**
126
166
  * The description's dominant alphabetic script when it DIFFERS from `expected`