vigiles 5.1.0 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/adapters/claude-code/adapter.js +1 -0
- package/dist/adapters/claude-code/agent-runtime.d.ts +20 -6
- package/dist/adapters/claude-code/agent-runtime.js +51 -8
- package/dist/adapters/claude-code/dialect.js +19 -0
- package/dist/adapters/claude-code/effect-region.d.ts +9 -0
- package/dist/adapters/claude-code/effect-region.js +45 -0
- package/dist/adapters/claude-code/layout.js +3 -0
- package/dist/adapters/claude-code/skill-runtime.d.ts +25 -0
- package/dist/adapters/claude-code/skill-runtime.js +48 -0
- package/dist/adapters/codex/adapter.js +3 -0
- package/dist/adapters/codex/layout.js +3 -0
- package/dist/adapters/opencode/adapter.js +1 -0
- package/dist/adapters/opencode/layout.js +3 -0
- package/dist/check.d.ts +8 -0
- package/dist/check.js +27 -3
- package/dist/cli.js +323 -88
- package/dist/core/adapter.d.ts +10 -0
- package/dist/core/bash-effects.d.ts +41 -0
- package/dist/core/bash-effects.js +405 -0
- package/dist/core/compile.d.ts +3 -1
- package/dist/core/compile.js +162 -39
- package/dist/core/dialect.d.ts +10 -0
- package/dist/core/effects.d.ts +172 -0
- package/dist/core/effects.js +245 -0
- package/dist/core/layout.d.ts +6 -0
- package/dist/core/mcp-tool.d.ts +1 -1
- package/dist/core/orphans.js +21 -0
- package/dist/core/spec.d.ts +142 -3
- package/dist/core/spec.js +48 -0
- package/dist/core/tool-contract.d.ts +1 -1
- package/dist/core/types.d.ts +6 -6
- package/dist/core/validate.js +4 -4
- package/dist/harness-test.d.ts +7 -0
- package/dist/harness-test.js +19 -7
- package/dist/leaderboard.d.ts +2 -0
- package/dist/leaderboard.js +2 -0
- package/dist/optimize.d.ts +74 -0
- package/dist/optimize.js +94 -0
- package/dist/scaffold-test.d.ts +30 -0
- package/dist/scaffold-test.js +158 -0
- package/dist/scan.d.ts +40 -0
- package/dist/scan.js +91 -43
- package/dist/score-explainer.d.ts +69 -0
- package/dist/score-explainer.js +169 -0
- package/dist/test-coverage.d.ts +7 -0
- package/dist/test-coverage.js +39 -24
- package/package.json +2 -1
- package/skills/{migrate-to-spec → adopt-spec}/SKILL.md +4 -4
- package/skills/edit-spec/SKILL.md +1 -1
package/dist/core/validate.js
CHANGED
|
@@ -43,15 +43,15 @@ const DEFAULT_RULES = {
|
|
|
43
43
|
coverage: false,
|
|
44
44
|
// Per-kind surface-coverage: a skill/agent/hook must ship with a test or eval.
|
|
45
45
|
"untested-skill": "warn",
|
|
46
|
-
"untested-
|
|
46
|
+
"untested-subagent": "warn",
|
|
47
47
|
"untested-hook": "warn",
|
|
48
48
|
"unmarked-refs": "warn",
|
|
49
49
|
// High-precision (never-available + close typos only), so on by default at warn.
|
|
50
|
-
"
|
|
50
|
+
"subagent-tool-contract": "warn",
|
|
51
51
|
// High-precision (close typos only), on by default at warn.
|
|
52
52
|
"hook-events": "warn",
|
|
53
53
|
// Missing required frontmatter (name/description) — on by default at warn.
|
|
54
|
-
"
|
|
54
|
+
"subagent-frontmatter": "warn",
|
|
55
55
|
// A declared MCP server with no command/url can't start — on by default at warn.
|
|
56
56
|
"mcp-config": "warn",
|
|
57
57
|
// Best-practice nudge (skills load without frontmatter) — warn, not error.
|
|
@@ -60,7 +60,7 @@ const DEFAULT_RULES = {
|
|
|
60
60
|
"mcp-tool-resolves": "warn",
|
|
61
61
|
// A hook script referenced but missing never runs — on by default at warn.
|
|
62
62
|
"hook-script-exists": "warn",
|
|
63
|
-
// High-precision (close-typo only) deny-list mirror of
|
|
63
|
+
// High-precision (close-typo only) deny-list mirror of subagent-tool-contract.
|
|
64
64
|
"disallowed-tools-contract": "warn",
|
|
65
65
|
// Deterministic NCD precision proxy (near-identical skill descriptions) — warn.
|
|
66
66
|
"description-overlap": "warn",
|
package/dist/harness-test.d.ts
CHANGED
|
@@ -110,6 +110,13 @@ export interface SubagentTrace {
|
|
|
110
110
|
readonly name: string;
|
|
111
111
|
/** The tools the subagent invoked (events tagged with the Task's id). */
|
|
112
112
|
readonly toolCalls: readonly ToolCall[];
|
|
113
|
+
/**
|
|
114
|
+
* The subagent's RETURNED text — the dispatch tool_result the orchestrator
|
|
115
|
+
* receives back. This is where a `result()` contract's `vigiles:ok`/`vigiles:err`
|
|
116
|
+
* block lands, so `subagent(name, [output(/vigiles:ok/)])` can assert the typed
|
|
117
|
+
* outcome. "" if not captured.
|
|
118
|
+
*/
|
|
119
|
+
readonly output: string;
|
|
113
120
|
}
|
|
114
121
|
export interface HarnessTestResult extends Trace {
|
|
115
122
|
readonly exitCode: number;
|
package/dist/harness-test.js
CHANGED
|
@@ -130,6 +130,7 @@ function parseToolCalls(streamJson) {
|
|
|
130
130
|
*/
|
|
131
131
|
function parseSubagents(streamJson) {
|
|
132
132
|
const tasks = new Map(); // dispatch id → subagent name
|
|
133
|
+
const dispatchOutput = new Map(); // dispatch id → returned text
|
|
133
134
|
const byParent = new Map();
|
|
134
135
|
const groupFor = (parent) => {
|
|
135
136
|
let g = byParent.get(parent);
|
|
@@ -162,7 +163,10 @@ function parseSubagents(streamJson) {
|
|
|
162
163
|
// A subagent dispatch is any top-level tool_use whose input carries a
|
|
163
164
|
// `subagent_type` — the dispatch tool is named "Agent" on the live CLI
|
|
164
165
|
// (older docs say "Task"), so match the input field, NOT the tool name,
|
|
165
|
-
// to survive the rename. Confirmed against real claude output.
|
|
166
|
+
// to survive the rename. Confirmed against real claude output. CC NOTE:
|
|
167
|
+
// under `--plugin-dir` the value is NAMESPACED `plugin:agent` (captured
|
|
168
|
+
// "reviewer-spec:code-reviewer"); the bare agent name is matched in the
|
|
169
|
+
// `subagent()` check (src/check.ts), so the full id is preserved here.
|
|
166
170
|
const sub = b.input?.subagent_type;
|
|
167
171
|
if (typeof sub === "string")
|
|
168
172
|
tasks.set(id, sub);
|
|
@@ -170,12 +174,20 @@ function parseSubagents(streamJson) {
|
|
|
170
174
|
if (parent)
|
|
171
175
|
groupFor(parent).uses.push({ id, name: b.name, input: b.input });
|
|
172
176
|
}
|
|
173
|
-
else if (b.type === "tool_result"
|
|
177
|
+
else if (b.type === "tool_result") {
|
|
174
178
|
const id = typeof b.tool_use_id === "string" ? b.tool_use_id : "";
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
+
if (parent) {
|
|
180
|
+
groupFor(parent).results.set(id, {
|
|
181
|
+
text: contentText(b.content),
|
|
182
|
+
isError: b.is_error === true,
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
else if (id) {
|
|
186
|
+
// A top-level tool_result whose id is a subagent dispatch is the SUB's
|
|
187
|
+
// RETURN to the orchestrator (where a result() vigiles:ok/err block
|
|
188
|
+
// lands). Record it; matched to its dispatch by id below.
|
|
189
|
+
dispatchOutput.set(id, contentText(b.content));
|
|
190
|
+
}
|
|
179
191
|
}
|
|
180
192
|
}
|
|
181
193
|
}
|
|
@@ -188,7 +200,7 @@ function parseSubagents(streamJson) {
|
|
|
188
200
|
resultText: g?.results.get(u.id)?.text ?? "",
|
|
189
201
|
isError: g?.results.get(u.id)?.isError ?? false,
|
|
190
202
|
}));
|
|
191
|
-
out.push({ name, toolCalls });
|
|
203
|
+
out.push({ name, toolCalls, output: dispatchOutput.get(taskId) ?? "" });
|
|
192
204
|
}
|
|
193
205
|
return out;
|
|
194
206
|
}
|
package/dist/leaderboard.d.ts
CHANGED
|
@@ -21,6 +21,8 @@ export interface PluginScore {
|
|
|
21
21
|
readonly issues: readonly string[];
|
|
22
22
|
readonly report: ScanReport;
|
|
23
23
|
}
|
|
24
|
+
/** Map a 0–100 structural-health score to its letter grade (A ≥90 … F <60). */
|
|
25
|
+
export declare function gradeFor(score: number): PluginScore["grade"];
|
|
24
26
|
/** Deterministic structural-health score for one scanned plugin. */
|
|
25
27
|
export declare function scoreReport(r: ScanReport): {
|
|
26
28
|
score: number;
|
package/dist/leaderboard.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* model and stack on top later; this part runs anywhere in CI for free.
|
|
13
13
|
*/
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.gradeFor = gradeFor;
|
|
15
16
|
exports.scoreReport = scoreReport;
|
|
16
17
|
exports.rankPlugins = rankPlugins;
|
|
17
18
|
exports.formatLeaderboard = formatLeaderboard;
|
|
@@ -23,6 +24,7 @@ const W_NO_DESCRIPTION = 10; // a skill with no usable description → can't tri
|
|
|
23
24
|
const W_DANGLING_REF = 8; // a referenced intra-plugin file that's missing → broken path
|
|
24
25
|
const W_NO_CONTRACT = 5; // an agent with no `tools:` line → inherits everything
|
|
25
26
|
const W_UNTESTED = 3; // a surface with no test/eval → warning-tier
|
|
27
|
+
/** Map a 0–100 structural-health score to its letter grade (A ≥90 … F <60). */
|
|
26
28
|
function gradeFor(score) {
|
|
27
29
|
if (score >= 90)
|
|
28
30
|
return "A";
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The per-repo harness optimizer's DETERMINISTIC spine — shipped as the
|
|
3
|
+
* `vigiles scan --fix-plan` lens (NOT its own `optimize` verb: until the measured
|
|
4
|
+
* A/B half lands, an "optimizer" that only re-prints scan's findings doesn't earn
|
|
5
|
+
* a separate command, so it's folded into scan as one more view on the same
|
|
6
|
+
* report; see research/roadmap.md §P2 "reconsider an `optimize` verb").
|
|
7
|
+
*
|
|
8
|
+
* A2 in the measurement-authority pivot is the ADOPTION product: measure a user's
|
|
9
|
+
* own skills/model/rules on their tasks and recommend add/drop/swap with a MEASURED
|
|
10
|
+
* delta. The measured delta is the real-model layer (gated on the Pro/Max
|
|
11
|
+
* subscription, costs tokens); this v0 ships the deterministic HALF — the free
|
|
12
|
+
* pre-filter that runs on every commit with no model.
|
|
13
|
+
*
|
|
14
|
+
* It answers "what should I fix in my harness, and why" using only the cross-ref
|
|
15
|
+
* findings the linter already computes: it reuses `scoreReport` for the headline
|
|
16
|
+
* structural-health score and `explainScore` for the per-surface cause + one-line
|
|
17
|
+
* fix (one-detector-no-drift — it never re-detects). The result is a prioritized,
|
|
18
|
+
* typed action list — the spine the measured A/B (the behavioral delta) stacks on.
|
|
19
|
+
*
|
|
20
|
+
* This is the "linting as a free pre-filter to measurement" thesis made a command:
|
|
21
|
+
* clear the structural dead-ends a model can't help with FIRST (free, certain),
|
|
22
|
+
* THEN spend tokens measuring whether the structurally-clean skills earn their keep.
|
|
23
|
+
*
|
|
24
|
+
* Distinct from `vigiles explain` (which diagnoses ONE underperforming surface a
|
|
25
|
+
* measurement flagged): `optimize` is the whole-repo adoption view — health score +
|
|
26
|
+
* the ranked fix list + the hand-off to the measured layer. Same findings, the
|
|
27
|
+
* optimization framing. See research/measurement-authority.md (A2) + roadmap §P1.
|
|
28
|
+
*/
|
|
29
|
+
import { type PluginScore } from "./leaderboard.js";
|
|
30
|
+
import { type ExplanationConfidence } from "./score-explainer.js";
|
|
31
|
+
import type { ScanReport } from "./scan.js";
|
|
32
|
+
/**
|
|
33
|
+
* The action a recommendation asks for. The deterministic detectors yield two:
|
|
34
|
+
* `"differentiate"` (a description-overlap PAIR — make the two distinct so the
|
|
35
|
+
* selector can disambiguate) and `"fix"` (every other structural dead-end —
|
|
36
|
+
* correct/add/remove the offending bit). The richer add/drop/swap vocabulary
|
|
37
|
+
* belongs to the MEASURED layer, once a behavioral delta ranks the alternatives.
|
|
38
|
+
*/
|
|
39
|
+
export type OptimizeAction = "fix" | "differentiate";
|
|
40
|
+
export interface Recommendation {
|
|
41
|
+
/** The affected surface (a skill/agent/hook name or path, or an `a ↔ b` pair). */
|
|
42
|
+
readonly surface: string;
|
|
43
|
+
readonly action: OptimizeAction;
|
|
44
|
+
/** The deterministic cause (the detector's own message — no drift). */
|
|
45
|
+
readonly rationale: string;
|
|
46
|
+
/** A single, actionable fix. */
|
|
47
|
+
readonly fix: string;
|
|
48
|
+
/** The lint rule that found it (open `docs/rules/<detector>.md`). */
|
|
49
|
+
readonly detector: string;
|
|
50
|
+
readonly confidence: ExplanationConfidence;
|
|
51
|
+
}
|
|
52
|
+
export interface OptimizeReport {
|
|
53
|
+
readonly dir: string;
|
|
54
|
+
/** Structural-health score 0–100 (the same `scoreReport` the leaderboard uses). */
|
|
55
|
+
readonly score: number;
|
|
56
|
+
readonly grade: PluginScore["grade"];
|
|
57
|
+
/** The free deterministic fixes, `likely` dead-ends first (explainScore's order). */
|
|
58
|
+
readonly recommendations: readonly Recommendation[];
|
|
59
|
+
/**
|
|
60
|
+
* No loadable surface at all (not a plugin, or a broken load) — distinct from a
|
|
61
|
+
* clean-and-loaded harness with zero findings, so the formatter doesn't call an
|
|
62
|
+
* EMPTY machine "clean". Mirrors `scoreReport`'s empty-machine case.
|
|
63
|
+
*/
|
|
64
|
+
readonly empty: boolean;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Turn a scan report into a prioritized optimization plan: the structural-health
|
|
68
|
+
* score + a typed recommendation per deterministic finding (`likely` dead-ends
|
|
69
|
+
* before `possible` proxies, via explainScore's own ordering). Pure over the report.
|
|
70
|
+
*/
|
|
71
|
+
export declare function optimize(report: ScanReport): OptimizeReport;
|
|
72
|
+
/** Render an optimization plan for the CLI. */
|
|
73
|
+
export declare function formatOptimize(rep: OptimizeReport): string;
|
|
74
|
+
//# sourceMappingURL=optimize.d.ts.map
|
package/dist/optimize.js
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.optimize = optimize;
|
|
4
|
+
exports.formatOptimize = formatOptimize;
|
|
5
|
+
/**
|
|
6
|
+
* The per-repo harness optimizer's DETERMINISTIC spine — shipped as the
|
|
7
|
+
* `vigiles scan --fix-plan` lens (NOT its own `optimize` verb: until the measured
|
|
8
|
+
* A/B half lands, an "optimizer" that only re-prints scan's findings doesn't earn
|
|
9
|
+
* a separate command, so it's folded into scan as one more view on the same
|
|
10
|
+
* report; see research/roadmap.md §P2 "reconsider an `optimize` verb").
|
|
11
|
+
*
|
|
12
|
+
* A2 in the measurement-authority pivot is the ADOPTION product: measure a user's
|
|
13
|
+
* own skills/model/rules on their tasks and recommend add/drop/swap with a MEASURED
|
|
14
|
+
* delta. The measured delta is the real-model layer (gated on the Pro/Max
|
|
15
|
+
* subscription, costs tokens); this v0 ships the deterministic HALF — the free
|
|
16
|
+
* pre-filter that runs on every commit with no model.
|
|
17
|
+
*
|
|
18
|
+
* It answers "what should I fix in my harness, and why" using only the cross-ref
|
|
19
|
+
* findings the linter already computes: it reuses `scoreReport` for the headline
|
|
20
|
+
* structural-health score and `explainScore` for the per-surface cause + one-line
|
|
21
|
+
* fix (one-detector-no-drift — it never re-detects). The result is a prioritized,
|
|
22
|
+
* typed action list — the spine the measured A/B (the behavioral delta) stacks on.
|
|
23
|
+
*
|
|
24
|
+
* This is the "linting as a free pre-filter to measurement" thesis made a command:
|
|
25
|
+
* clear the structural dead-ends a model can't help with FIRST (free, certain),
|
|
26
|
+
* THEN spend tokens measuring whether the structurally-clean skills earn their keep.
|
|
27
|
+
*
|
|
28
|
+
* Distinct from `vigiles explain` (which diagnoses ONE underperforming surface a
|
|
29
|
+
* measurement flagged): `optimize` is the whole-repo adoption view — health score +
|
|
30
|
+
* the ranked fix list + the hand-off to the measured layer. Same findings, the
|
|
31
|
+
* optimization framing. See research/measurement-authority.md (A2) + roadmap §P1.
|
|
32
|
+
*/
|
|
33
|
+
const leaderboard_js_1 = require("./leaderboard.js");
|
|
34
|
+
const score_explainer_js_1 = require("./score-explainer.js");
|
|
35
|
+
function actionFor(e) {
|
|
36
|
+
return e.symptom === "wrong-skill-fires" ? "differentiate" : "fix";
|
|
37
|
+
}
|
|
38
|
+
function isEmptyMachine(r) {
|
|
39
|
+
const surfaces = r.skills.length + r.agents.length + r.hooks.length + r.commands;
|
|
40
|
+
return surfaces === 0 && !r.mcp;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Turn a scan report into a prioritized optimization plan: the structural-health
|
|
44
|
+
* score + a typed recommendation per deterministic finding (`likely` dead-ends
|
|
45
|
+
* before `possible` proxies, via explainScore's own ordering). Pure over the report.
|
|
46
|
+
*/
|
|
47
|
+
function optimize(report) {
|
|
48
|
+
const { score } = (0, leaderboard_js_1.scoreReport)(report);
|
|
49
|
+
const recommendations = (0, score_explainer_js_1.explainScore)(report).map((e) => ({
|
|
50
|
+
surface: e.surface,
|
|
51
|
+
action: actionFor(e),
|
|
52
|
+
rationale: e.cause,
|
|
53
|
+
fix: e.fix,
|
|
54
|
+
detector: e.detector,
|
|
55
|
+
confidence: e.confidence,
|
|
56
|
+
}));
|
|
57
|
+
return {
|
|
58
|
+
dir: report.dir,
|
|
59
|
+
score,
|
|
60
|
+
grade: (0, leaderboard_js_1.gradeFor)(score),
|
|
61
|
+
recommendations,
|
|
62
|
+
empty: isEmptyMachine(report),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
const ACTION_LABEL = {
|
|
66
|
+
fix: "FIX",
|
|
67
|
+
differentiate: "DIFFERENTIATE",
|
|
68
|
+
};
|
|
69
|
+
const measureHint = (dir) => `\`vigiles scan ${dir} --trigger\` — real-model, runs on your subscription`;
|
|
70
|
+
/** Render an optimization plan for the CLI. */
|
|
71
|
+
function formatOptimize(rep) {
|
|
72
|
+
const head = `Harness health: ${String(rep.score)}/100 (${rep.grade}) — ${rep.dir}`;
|
|
73
|
+
if (rep.empty) {
|
|
74
|
+
return `${head}\n\nNothing loaded — this isn't a plugin/harness, or the load failed. Point optimize at a dir with a CLAUDE.md/AGENTS.md, skills, agents, or hooks.`;
|
|
75
|
+
}
|
|
76
|
+
if (rep.recommendations.length === 0) {
|
|
77
|
+
return `${head}\n\nNo deterministic fixes found — the structure is clean. Whether your skills actually help is a BEHAVIORAL question; measure it with ${measureHint(rep.dir)}.`;
|
|
78
|
+
}
|
|
79
|
+
const lines = [
|
|
80
|
+
head,
|
|
81
|
+
"",
|
|
82
|
+
`${String(rep.recommendations.length)} deterministic fix(es) — free, no model. Apply these before measuring:`,
|
|
83
|
+
"",
|
|
84
|
+
];
|
|
85
|
+
for (const r of rep.recommendations) {
|
|
86
|
+
const mark = r.confidence === "likely" ? "✗" : "⚠";
|
|
87
|
+
lines.push(`${mark} [${ACTION_LABEL[r.action]}] ${r.surface}`);
|
|
88
|
+
lines.push(` why: ${r.rationale} [${r.detector}]`);
|
|
89
|
+
lines.push(` → ${r.fix}`);
|
|
90
|
+
}
|
|
91
|
+
lines.push("", `Then measure the behavioral delta of what's left (does each skill earn its keep?) with ${measureHint(rep.dir)}.`);
|
|
92
|
+
return lines.join("\n");
|
|
93
|
+
}
|
|
94
|
+
//# sourceMappingURL=optimize.js.map
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export type SurfaceKind = "skill" | "agent" | "hook";
|
|
2
|
+
/** The cheapest meaningful tier for a surface kind (mirrors the test-harness skill). */
|
|
3
|
+
export type TestTier = "unit" | "harness" | "eval";
|
|
4
|
+
/** What the generator needs to know about a surface to scaffold its test. */
|
|
5
|
+
export interface ScaffoldInput {
|
|
6
|
+
readonly kind: SurfaceKind;
|
|
7
|
+
/** Skill dir name / agent name / hook-script basename. */
|
|
8
|
+
readonly name: string;
|
|
9
|
+
/** Repo-relative path to the surface (SKILL.md / agent .md / hook script). */
|
|
10
|
+
readonly path: string;
|
|
11
|
+
/** Plugin name for the namespaced skill id; a placeholder TODO when unknown. */
|
|
12
|
+
readonly pluginName?: string;
|
|
13
|
+
/** A user-invoked skill — trigger-rate is for model-invocable skills, so note it. */
|
|
14
|
+
readonly userInvoked?: boolean;
|
|
15
|
+
/** A subagent's declared tool contract (drives the assertion hint); null = inherits all. */
|
|
16
|
+
readonly tools?: readonly string[] | null;
|
|
17
|
+
/** How the CLI invokes the hook (e.g. `bash hooks/pre-edit.sh`); a TODO when unknown. */
|
|
18
|
+
readonly hookCommand?: string;
|
|
19
|
+
}
|
|
20
|
+
export interface Scaffold {
|
|
21
|
+
readonly path: string;
|
|
22
|
+
readonly content: string;
|
|
23
|
+
readonly kind: SurfaceKind;
|
|
24
|
+
readonly tier: TestTier;
|
|
25
|
+
}
|
|
26
|
+
/** Scaffold a starter test for one surface. Pure: path + content, no I/O. */
|
|
27
|
+
export declare function scaffoldTest(input: ScaffoldInput): Scaffold;
|
|
28
|
+
/** Render a set of scaffolds for the CLI (what was generated, where, which tier). */
|
|
29
|
+
export declare function formatScaffolds(scaffolds: readonly Scaffold[]): string;
|
|
30
|
+
//# sourceMappingURL=scaffold-test.d.ts.map
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.scaffoldTest = scaffoldTest;
|
|
4
|
+
exports.formatScaffolds = formatScaffolds;
|
|
5
|
+
/**
|
|
6
|
+
* `vigiles scaffold-test` — the deterministic test-gen engine (B1 v0).
|
|
7
|
+
*
|
|
8
|
+
* Free-form in, a RUNNABLE starter test out. Given an existing hand-written
|
|
9
|
+
* skill / subagent / hook, emit a scaffolded `*.harness.mjs` / `*.eval.mjs` at the
|
|
10
|
+
* surface's suggested test path — the deterministic counterpart to the
|
|
11
|
+
* `test-harness` SKILL (which picks the tier with a model). The scaffold picks the
|
|
12
|
+
* cheapest meaningful tier for the kind, wires the real public API + the surface's
|
|
13
|
+
* own metadata (name, namespaced id, declared tools), and leaves TODOs only where a
|
|
14
|
+
* human/model must supply judgement (the prompts, the event input, the assertion).
|
|
15
|
+
*
|
|
16
|
+
* Pure + model-free: hand it a `ScaffoldInput`, get back a `{ path, content }`. The
|
|
17
|
+
* CLI resolves a surface path / plugin dir into inputs (reusing the scan + untested
|
|
18
|
+
* detectors) and writes the files; this module owns only the templating.
|
|
19
|
+
*/
|
|
20
|
+
const node_path_1 = require("node:path");
|
|
21
|
+
const PLUGIN_TODO = "<plugin>";
|
|
22
|
+
/**
|
|
23
|
+
* The suggested test path for a surface — mirrors `suggestedTestPath` in
|
|
24
|
+
* `test-coverage.ts` (a skill gets an `.eval.mjs`, agent/hook a `.harness.mjs`),
|
|
25
|
+
* so a generated file is colocated where the untested-surface detector looks for it
|
|
26
|
+
* and the surface stops being reported untested.
|
|
27
|
+
*/
|
|
28
|
+
function suggestedPath(input) {
|
|
29
|
+
const dir = (0, node_path_1.dirname)(input.path);
|
|
30
|
+
const ext = input.kind === "skill" ? "eval.mjs" : "harness.mjs";
|
|
31
|
+
return `${dir}/${input.name}.${ext}`;
|
|
32
|
+
}
|
|
33
|
+
function header(title, run) {
|
|
34
|
+
return [
|
|
35
|
+
"/**",
|
|
36
|
+
` * ${title}`,
|
|
37
|
+
" *",
|
|
38
|
+
" * Generated by `vigiles scaffold-test` — a STARTER, not a finished test. Fill in",
|
|
39
|
+
" * the TODOs (they're where a human/model must supply judgement), then run:",
|
|
40
|
+
` * ${run}`,
|
|
41
|
+
" */",
|
|
42
|
+
].join("\n");
|
|
43
|
+
}
|
|
44
|
+
/** A hook → the unit tier (`runHook`): free, no model, reaches every event. */
|
|
45
|
+
function hookScaffold(input) {
|
|
46
|
+
const cmd = input.hookCommand ?? `bash ${input.path}`;
|
|
47
|
+
return `${header(`Starter unit test for the \`${input.name}\` hook.`, `npx vigiles test ${suggestedPath(input)}`)}
|
|
48
|
+
import { runHook, assertHookAllowed } from "vigiles/unit";
|
|
49
|
+
|
|
50
|
+
// TODO: set the event + input your hook actually inspects (PreToolUse/Bash shown).
|
|
51
|
+
const event = {
|
|
52
|
+
hook_event_name: "PreToolUse",
|
|
53
|
+
tool_name: "Bash",
|
|
54
|
+
tool_input: { command: "echo hello" },
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
const r = runHook(${JSON.stringify(cmd)}, event);
|
|
58
|
+
|
|
59
|
+
// TODO: assert the decision you expect. Use assertHookBlocked(r) for the deny case
|
|
60
|
+
// (and a second runHook with the input that SHOULD be blocked).
|
|
61
|
+
assertHookAllowed(r);
|
|
62
|
+
console.log("✓ ${input.name}: hook allowed the benign event");
|
|
63
|
+
`;
|
|
64
|
+
}
|
|
65
|
+
/** A skill → the eval tier (`measureTriggerRate`): does its description FIRE? */
|
|
66
|
+
function skillScaffold(input) {
|
|
67
|
+
const id = `${input.pluginName ?? PLUGIN_TODO}:${input.name}`;
|
|
68
|
+
const note = input.userInvoked
|
|
69
|
+
? "\n// NOTE: this skill is user-invoked (disableModelInvocation). Trigger-rate\n// measures MODEL-invocable skills; either make it model-invocable or test its\n// slash-command invocation with runHarnessTest instead.\n"
|
|
70
|
+
: "";
|
|
71
|
+
return `${header(`Starter trigger-rate eval for the \`${id}\` skill (recall + precision).`, `npx vigiles eval ${suggestedPath(input)} # real model, on your subscription`)}
|
|
72
|
+
import {
|
|
73
|
+
measureTriggerRate,
|
|
74
|
+
formatTriggerRateReport,
|
|
75
|
+
assertTriggerRate,
|
|
76
|
+
skillResolved,
|
|
77
|
+
} from "vigiles/testing";
|
|
78
|
+
import { fileURLToPath } from "node:url";
|
|
79
|
+
${note}
|
|
80
|
+
// TODO: point at the plugin root (the dir holding .claude-plugin/ or skills/).
|
|
81
|
+
const pluginDir = fileURLToPath(new URL("../../", import.meta.url));
|
|
82
|
+
const skill = ${JSON.stringify(id)};
|
|
83
|
+
|
|
84
|
+
const report = await measureTriggerRate({
|
|
85
|
+
pluginDir,
|
|
86
|
+
stubSkillBodies: true, // firing is a frontmatter property — stub bodies, pay less
|
|
87
|
+
prompts: [
|
|
88
|
+
// TODO: >=5 varied prompts that SHOULD fire ${input.name} (recall).
|
|
89
|
+
"TODO: a realistic task that should trigger ${input.name}",
|
|
90
|
+
],
|
|
91
|
+
irrelevantPrompts: [
|
|
92
|
+
// TODO: >=5 unrelated prompts that should NOT fire it (precision).
|
|
93
|
+
"TODO: an unrelated coding task",
|
|
94
|
+
],
|
|
95
|
+
fired: (t) => skillResolved(t, skill),
|
|
96
|
+
trials: Number(process.env.VIGILES_TRIALS || 1),
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
console.log(formatTriggerRateReport(report));
|
|
100
|
+
assertTriggerRate(report, { min: 0.8, maxFalsePositive: 0.3 });
|
|
101
|
+
`;
|
|
102
|
+
}
|
|
103
|
+
/** A subagent → the deterministic harness tier; point at the railway/Result path. */
|
|
104
|
+
function agentScaffold(input) {
|
|
105
|
+
const toolHint = input.tools && input.tools.length > 0
|
|
106
|
+
? `assertToolUsed(r, ${JSON.stringify(input.tools[0])}); // its declared contract: ${input.tools.join(", ")}`
|
|
107
|
+
: `assertToolUsed(r, "Task"); // TODO: assert what the subagent should do`;
|
|
108
|
+
return `${header(`Starter harness test for the \`${input.name}\` subagent.`, `npx vigiles test ${suggestedPath(input)}`)}
|
|
109
|
+
import { runHarnessTest, assertToolUsed } from "vigiles/testing";
|
|
110
|
+
|
|
111
|
+
// A subagent's OUTCOME is best asserted via a result() contract — deterministic,
|
|
112
|
+
// no LLM judge. If ${input.name} has one, use assertAgentOk(r.output, contract)
|
|
113
|
+
// instead; see the railway-result example in the vigiles docs.
|
|
114
|
+
|
|
115
|
+
const r = await runHarnessTest({
|
|
116
|
+
plugin: ".", // TODO: the plugin dir holding this subagent
|
|
117
|
+
// TODO: a prompt that dispatches ${input.name} (via the Task tool).
|
|
118
|
+
prompt: "TODO: a task that should dispatch ${input.name}",
|
|
119
|
+
transcript: true,
|
|
120
|
+
model: [{ text: "on it" }],
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
${toolHint}
|
|
124
|
+
console.log("✓ ${input.name}: subagent test ran");
|
|
125
|
+
`;
|
|
126
|
+
}
|
|
127
|
+
const TIER = {
|
|
128
|
+
hook: "unit",
|
|
129
|
+
skill: "eval",
|
|
130
|
+
agent: "harness",
|
|
131
|
+
};
|
|
132
|
+
const BUILDER = {
|
|
133
|
+
hook: hookScaffold,
|
|
134
|
+
skill: skillScaffold,
|
|
135
|
+
agent: agentScaffold,
|
|
136
|
+
};
|
|
137
|
+
/** Scaffold a starter test for one surface. Pure: path + content, no I/O. */
|
|
138
|
+
function scaffoldTest(input) {
|
|
139
|
+
return {
|
|
140
|
+
path: suggestedPath(input),
|
|
141
|
+
content: BUILDER[input.kind](input),
|
|
142
|
+
kind: input.kind,
|
|
143
|
+
tier: TIER[input.kind],
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
/** Render a set of scaffolds for the CLI (what was generated, where, which tier). */
|
|
147
|
+
function formatScaffolds(scaffolds) {
|
|
148
|
+
if (scaffolds.length === 0) {
|
|
149
|
+
return "Nothing to scaffold — every surface already has a test, or none was found.";
|
|
150
|
+
}
|
|
151
|
+
const lines = [`Scaffolded ${String(scaffolds.length)} starter test(s):`, ""];
|
|
152
|
+
for (const s of scaffolds) {
|
|
153
|
+
lines.push(` ${s.path} [${s.kind} → ${s.tier} tier]`);
|
|
154
|
+
}
|
|
155
|
+
lines.push("", "These are STARTERS — fill in the TODOs (prompts / event / assertions), then run", "them with `npx vigiles test` (deterministic) or `npx vigiles eval` (real model).");
|
|
156
|
+
return lines.join("\n");
|
|
157
|
+
}
|
|
158
|
+
//# sourceMappingURL=scaffold-test.js.map
|
package/dist/scan.d.ts
CHANGED
|
@@ -19,6 +19,7 @@ import { type McpIssue } from "./core/mcp-config.js";
|
|
|
19
19
|
import { type DescriptionOverlap } from "./core/description-overlap.js";
|
|
20
20
|
import { type McpToolIssue } from "./core/mcp-tool.js";
|
|
21
21
|
import { type McpHookIssue } from "./core/mcp-hook.js";
|
|
22
|
+
import { type PurityLevel, type EffectSurface } from "./core/effects.js";
|
|
22
23
|
/** A named writing system. The label `unexpectedScript` reports + the config's expectation parse into this. */
|
|
23
24
|
export type Script = "Latin" | "Cyrillic" | "Han" | "Japanese" | "Korean" | "Arabic" | "Hebrew" | "Greek" | "Devanagari" | "Thai";
|
|
24
25
|
export interface ScanSkill {
|
|
@@ -47,6 +48,19 @@ export interface ScanAgent {
|
|
|
47
48
|
readonly mcpToolIssues: readonly McpToolIssue[];
|
|
48
49
|
/** `disallowedTools:` block-list entries that are typos of a real tool (block nothing). */
|
|
49
50
|
readonly disallowedToolIssues: readonly ToolIssue[];
|
|
51
|
+
/**
|
|
52
|
+
* Static effect-surface purity of the agent's declared tool contract.
|
|
53
|
+
* - `"pure"` — no side-effecting tools (read-only, deterministically testable).
|
|
54
|
+
* - `"bounded"` — has side-effecting tools (Edit/Write/…) but no Bash or unknown.
|
|
55
|
+
* - `"unrestricted"` — has Bash, any MCP/unknown tool, or inherits-all (no contract).
|
|
56
|
+
* Computed by `effectSurface()` from `src/core/effects.ts` — one detector, no drift.
|
|
57
|
+
*/
|
|
58
|
+
readonly purity: PurityLevel;
|
|
59
|
+
/**
|
|
60
|
+
* The three tool buckets from `effectSurface()`: read-only, side-effecting, and
|
|
61
|
+
* unknown-effect (MCP or unrecognized) tool names in the declared contract.
|
|
62
|
+
*/
|
|
63
|
+
readonly effectBuckets: Pick<EffectSurface, "readOnly" | "sideEffecting" | "unknown">;
|
|
50
64
|
}
|
|
51
65
|
/** A skill/agent whose frontmatter is missing a required field (name / description). */
|
|
52
66
|
export interface FrontmatterIssue {
|
|
@@ -121,6 +135,32 @@ export interface ScanReport {
|
|
|
121
135
|
readonly malformedFrontmatter: readonly FrontmatterParseIssue[];
|
|
122
136
|
readonly warnings: readonly string[];
|
|
123
137
|
readonly untested: number;
|
|
138
|
+
/**
|
|
139
|
+
* Harness-level purity summary: how many scanned agents fall into each purity
|
|
140
|
+
* rung. A high `pure` count means more of the harness is statically testable
|
|
141
|
+
* (deterministic, no mocks); `unrestricted` is the blind-spot count.
|
|
142
|
+
* Computed by `effectSurface()` (one detector, no drift).
|
|
143
|
+
*/
|
|
144
|
+
readonly puritySummary: {
|
|
145
|
+
pure: number;
|
|
146
|
+
bounded: number;
|
|
147
|
+
unrestricted: number;
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Per-kind surface classifiers, built from the harness `PluginLayout`'s
|
|
152
|
+
* `skillDir`/`agentDir`/`commandDir` — so adding a harness whose subagents live
|
|
153
|
+
* somewhere other than `agents/` (OpenCode's `.opencode/agent`) needs no change
|
|
154
|
+
* here. Each anchors on a real path boundary (start-of-path or a `/`), so a
|
|
155
|
+
* directory whose NAME merely ends in the keyword isn't misclassified — e.g. the
|
|
156
|
+
* skill `skills/dispatching-parallel-agents/SKILL.md` must NOT register as an
|
|
157
|
+
* agent named "SKILL" (the `-agents/` substring), which real plugins like
|
|
158
|
+
* obra/superpowers ship. See scan.test.ts for the regression cases.
|
|
159
|
+
*/
|
|
160
|
+
export interface SurfaceClassifier {
|
|
161
|
+
readonly isSkill: (f: string) => boolean;
|
|
162
|
+
readonly isAgent: (f: string) => boolean;
|
|
163
|
+
readonly isCommand: (f: string) => boolean;
|
|
124
164
|
}
|
|
125
165
|
/**
|
|
126
166
|
* The description's dominant alphabetic script when it DIFFERS from `expected`
|