vigiles 21.0.0 → 21.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/audit-score.js +51 -2
- package/dist/core/description-overlap.js +4 -1
- package/dist/core/hook-program.d.ts +5 -3
- package/dist/core/hook-program.js +5 -3
- package/dist/guardrail-check.d.ts +3 -1
- package/dist/guardrail-check.js +3 -1
- package/dist/hook.d.ts +7 -5
- package/dist/hook.js +7 -5
- package/dist/score-core.d.ts +48 -0
- package/dist/score-core.js +91 -2
- package/package.json +1 -1
- package/skills/test-harness/SKILL.md +16 -228
- package/skills/test-harness/references/cost-and-expectations.md +53 -0
- package/skills/test-harness/references/observing-a-run.md +51 -0
- package/skills/test-harness/references/writing-tests.md +143 -0
package/README.md
CHANGED
|
@@ -175,6 +175,8 @@ Every path, script, symbol, and rule verified against reality — plus tool cont
|
|
|
175
175
|
A hook that blocks nothing, a skill that hijacks unrelated prompts, context that never reaches the model — each passes a naive "did it run?" check. That gap is **false confidence**: a guard that looks like it works and silently doesn't. vigiles tests the real thing — hooks block, skills fire, subagents finish what they promised, a stray `git push` is caught before it happens. It drives a scripted stand-in for the model, not a live call, so it needs no key and runs on every commit.
|
|
176
176
|
**[How testing works →](docs/harness-testing.md)**
|
|
177
177
|
|
|
178
|
+
**Nothing you scan leaves your machine.** `lint`, `audit` and the deterministic test tiers make no network call at all — no telemetry, no analytics, no HTTP client in the package. Evals drive your own `claude` CLI on your own subscription, so no third party is introduced. **[What is and isn't transmitted →](docs/safety.md#does-vigiles-send-my-code-anywhere)**
|
|
179
|
+
|
|
178
180
|
### 📊 Eval — the only way to put a real number on cost
|
|
179
181
|
|
|
180
182
|
_"Caveman Mode cuts 65% of your tokens." Says who?_ vigiles A/Bs the claim on real coding tasks and hands you three numbers: the **token bill**, whether it hit its **target**, and whether your code still **works**.
|
package/dist/audit-score.js
CHANGED
|
@@ -198,9 +198,24 @@ function structure(r) {
|
|
|
198
198
|
`${String(noContract)} agent(s) inherit all tools (no contract) (advisory)`,
|
|
199
199
|
]
|
|
200
200
|
: [];
|
|
201
|
+
// A confident breakage caps this ring too, so the breakdown can't render a
|
|
202
|
+
// healthy `●` over a dead surface — the defect this fixes was measured as
|
|
203
|
+
// "Structure 92 ●" while an agent named a never-available tool. Only the
|
|
204
|
+
// rows that mean a surface is DEAD count; the typo'd `disallowedTools` and
|
|
205
|
+
// invalid model/color rows are footguns, not breakage, so they stay out.
|
|
206
|
+
const breakage = deadTools +
|
|
207
|
+
deadMcpTools +
|
|
208
|
+
r.hookEventIssues.length +
|
|
209
|
+
r.mcpIssues.length +
|
|
210
|
+
r.mcpHookIssues.length +
|
|
211
|
+
r.frontmatterIssues.length +
|
|
212
|
+
r.pluginLayoutIssues.length +
|
|
213
|
+
r.skillFenceIssues.length +
|
|
214
|
+
r.hookBlockFindings.length +
|
|
215
|
+
r.hookMatcherFindings.length;
|
|
201
216
|
return {
|
|
202
217
|
key: "Structure",
|
|
203
|
-
score,
|
|
218
|
+
score: breakage > 0 ? Math.min(score, score_core_js_1.CONFIDENT_BREAKAGE_CAP) : score,
|
|
204
219
|
weight: 1,
|
|
205
220
|
findings: [...findings, ...advisory],
|
|
206
221
|
};
|
|
@@ -248,15 +263,39 @@ function safety(r) {
|
|
|
248
263
|
// one at a time would bury the subagent findings under a list as long as the
|
|
249
264
|
// skill corpus. Same reasoning as the report section; see `trifectaLines`.
|
|
250
265
|
const unfenced = r.trifectaFindings.filter((f) => f.kind === "skill" && f.finding.fence === "none");
|
|
266
|
+
// An INEFFECTIVE fence (`disallowed-tools:` that closes no leg) keeps its own
|
|
267
|
+
// line while there are FEW of them — that is the documented intent in
|
|
268
|
+
// core/lethal-trifecta.ts: it is a genuine mistake, the author believed they had
|
|
269
|
+
// fenced, so naming the skill is what a reader acts on.
|
|
270
|
+
//
|
|
271
|
+
// MEASURED 2026-08-28 that the premise behind that shape — "Rare" — does not
|
|
272
|
+
// hold: a fixture where every skill carried a naive `disallowed-tools: WebFetch`
|
|
273
|
+
// put ALL of them in this state, and the ring printed the same ~450-character
|
|
274
|
+
// paragraph ten times, ~4,500 characters into the terminal report. Past the
|
|
275
|
+
// threshold it stops being N facts about N skills and becomes one fact about the
|
|
276
|
+
// harness — exactly the reasoning the `fence: "none"` aggregate already uses.
|
|
277
|
+
// Names are kept as detail, so nothing is lost, only repetition.
|
|
278
|
+
const ineffective = r.trifectaFindings.filter((f) => f.finding.severity === "advisory" &&
|
|
279
|
+
!unfenced.includes(f) &&
|
|
280
|
+
f.kind === "skill" &&
|
|
281
|
+
f.finding.fence === "ineffective");
|
|
282
|
+
const collapseIneffective = ineffective.length > MAX_NAMED_INEFFECTIVE_FENCES;
|
|
251
283
|
for (const f of r.trifectaFindings) {
|
|
252
284
|
if (f.finding.severity !== "advisory")
|
|
253
285
|
continue;
|
|
254
286
|
if (unfenced.includes(f))
|
|
255
287
|
continue;
|
|
288
|
+
if (collapseIneffective && ineffective.includes(f))
|
|
289
|
+
continue;
|
|
256
290
|
findings.push(f.kind === "skill"
|
|
257
291
|
? `${f.name}: ${f.finding.message}`
|
|
258
292
|
: `${f.name} inherits all tools — the "lethal trifecta" (reads data, reaches the web, runs commands) plus every other capability, so a prompt injection could exfiltrate secrets`);
|
|
259
293
|
}
|
|
294
|
+
if (collapseIneffective) {
|
|
295
|
+
findings.push(`${String(ineffective.length)} skill(s) declare a \`disallowed-tools:\` that closes no lethal-trifecta leg — a leg is closed only when EVERY built-in supplying it is denied: ` +
|
|
296
|
+
`${ineffective.map((f) => f.name).join(", ")}. ` +
|
|
297
|
+
`Name every supplier of the leg you mean to close — private-data read = Read, Grep, Glob, Bash; untrusted intake = WebFetch, WebSearch, Bash; exfiltration = WebFetch, WebSearch, Bash.`);
|
|
298
|
+
}
|
|
260
299
|
if (unfenced.length > 0) {
|
|
261
300
|
findings.push(`${String(unfenced.length)} skill(s) declare no \`disallowed-tools:\` fence, so each inherits every tool the session grants — reads data, reaches the web, runs commands. \`allowed-tools:\` pre-approves, it does not restrict, so narrowing it does not reduce this; one \`disallowed-tools:\` line per skill drops a leg.`);
|
|
262
301
|
}
|
|
@@ -456,9 +495,19 @@ function auditScore(report, opts = {}) {
|
|
|
456
495
|
// of the rings — averaging would let a real problem in one category be diluted
|
|
457
496
|
// by clean siblings. The rings above stay a diagnostic breakdown; Tested and
|
|
458
497
|
// Evaluated (both advisory) are never summed in (neither drags the grade).
|
|
459
|
-
const { score:
|
|
498
|
+
const { score: summed } = (0, score_core_js_1.computeIntegrityScore)((0, score_core_js_1.reportDeductions)(report));
|
|
499
|
+
// A confident breakage caps the headline too, so it can never read `A` while a
|
|
500
|
+
// surface is definitively dead (score-core.ts::applyBreakageCap).
|
|
501
|
+
const overall = (0, score_core_js_1.applyBreakageCap)(summed, report);
|
|
460
502
|
return { overall, grade: (0, score_core_js_1.gradeFor)(overall), categories, empty: false };
|
|
461
503
|
}
|
|
504
|
+
/**
|
|
505
|
+
* How many INEFFECTIVE `disallowed-tools:` fences are named one at a time before
|
|
506
|
+
* the Safety ring collapses them into a single line. Past this it is one fact
|
|
507
|
+
* about the harness, not N facts about N skills — see the measurement in
|
|
508
|
+
* {@link safety}.
|
|
509
|
+
*/
|
|
510
|
+
const MAX_NAMED_INEFFECTIVE_FENCES = 3;
|
|
462
511
|
// A 22-cell bar gauge ("ring" in the terminal; the real rings are the HTML).
|
|
463
512
|
const BAR_CELLS = 22;
|
|
464
513
|
/** A glyph that signals the band at a glance (green/amber/red, no ANSI needed).
|
|
@@ -8,7 +8,10 @@ exports.findDescriptionOverlaps = findDescriptionOverlaps;
|
|
|
8
8
|
* selector, so the wrong one fires (a precision collision). This catches a
|
|
9
9
|
* `--trigger`-class problem with NO model, reusing the NCD engine in proofs.ts
|
|
10
10
|
* (the same one `findSimilarRules` uses) — the bridge between the deterministic
|
|
11
|
-
* and behavioral columns
|
|
11
|
+
* and behavioral columns. The CHECK is not unique — cisco-ai-defense/skill-scanner
|
|
12
|
+
* ships `--check-overlap` under a security framing (skill impersonation). What is
|
|
13
|
+
* ours: the cutoff calibrated against a real corpus, and the precision-collision
|
|
14
|
+
* framing (verified 2026-08-28).
|
|
12
15
|
*
|
|
13
16
|
* Calibrated HIGH-PRECISION against the mid-2026 sweep: across 4678 within-plugin
|
|
14
17
|
* skill-description pairs, the MOST-similar legitimately-distinct pair
|
|
@@ -24,9 +24,11 @@
|
|
|
24
24
|
* OpenCode via the HookProtocol port later — OpenCode hooks ARE in-process TS).
|
|
25
25
|
*
|
|
26
26
|
* Pure core, harness-neutral. HONEST SCOPE (kept in every doc): compile/verify fix
|
|
27
|
-
* the hook's AUTHORING + LOGIC, not
|
|
28
|
-
*
|
|
29
|
-
*
|
|
27
|
+
* the hook's AUTHORING + LOGIC, not the harness's DELIVERY. #34692 (a subagent's
|
|
28
|
+
* calls never reaching PreToolUse) is FIXED as of CC 2.1.241 — measured on a stock
|
|
29
|
+
* install, pinned by src/subagent-delivery.test.ts. A gate is STILL a strong default
|
|
30
|
+
* rather than an unbypassable wall, because a model can route around a tool
|
|
31
|
+
* entirely (#45427 / #32376). Limits (buy-in, node-startup latency) +
|
|
30
32
|
* full record in research/hook-pain-points.md.
|
|
31
33
|
*/
|
|
32
34
|
import { type BashEffect } from "./bash-effects.js";
|
|
@@ -64,9 +64,11 @@ exports.isLoadPathRepairEvent = isLoadPathRepairEvent;
|
|
|
64
64
|
* OpenCode via the HookProtocol port later — OpenCode hooks ARE in-process TS).
|
|
65
65
|
*
|
|
66
66
|
* Pure core, harness-neutral. HONEST SCOPE (kept in every doc): compile/verify fix
|
|
67
|
-
* the hook's AUTHORING + LOGIC, not
|
|
68
|
-
*
|
|
69
|
-
*
|
|
67
|
+
* the hook's AUTHORING + LOGIC, not the harness's DELIVERY. #34692 (a subagent's
|
|
68
|
+
* calls never reaching PreToolUse) is FIXED as of CC 2.1.241 — measured on a stock
|
|
69
|
+
* install, pinned by src/subagent-delivery.test.ts. A gate is STILL a strong default
|
|
70
|
+
* rather than an unbypassable wall, because a model can route around a tool
|
|
71
|
+
* entirely (#45427 / #32376). Limits (buy-in, node-startup latency) +
|
|
70
72
|
* full record in research/hook-pain-points.md.
|
|
71
73
|
*/
|
|
72
74
|
const bash_effects_js_1 = require("./bash-effects.js");
|
|
@@ -14,7 +14,9 @@
|
|
|
14
14
|
* the hook via {@link runHook} and check the normalized decision is BLOCK. No model,
|
|
15
15
|
* no API key, runs in CI, works on a hand-written hook with NO vigiles spec — it
|
|
16
16
|
* verifies the hook's decision LOGIC, so it sidesteps CC's runtime delivery bugs
|
|
17
|
-
* (
|
|
17
|
+
* (the model routing around a tool entirely, #45427 / #32376) which it deliberately
|
|
18
|
+
* does NOT claim to fix. (#34692, the old subagent-delivery gap, is fixed as of CC
|
|
19
|
+
* 2.1.241 — see src/subagent-delivery.test.ts.)
|
|
18
20
|
*
|
|
19
21
|
* Pure-ish (wraps the existing runHook tier). The catalog is harness-neutral data;
|
|
20
22
|
* the scaffold-test generator emits a test that calls these, and the same engine
|
package/dist/guardrail-check.js
CHANGED
|
@@ -21,7 +21,9 @@ exports.formatGuardrailReport = formatGuardrailReport;
|
|
|
21
21
|
* the hook via {@link runHook} and check the normalized decision is BLOCK. No model,
|
|
22
22
|
* no API key, runs in CI, works on a hand-written hook with NO vigiles spec — it
|
|
23
23
|
* verifies the hook's decision LOGIC, so it sidesteps CC's runtime delivery bugs
|
|
24
|
-
* (
|
|
24
|
+
* (the model routing around a tool entirely, #45427 / #32376) which it deliberately
|
|
25
|
+
* does NOT claim to fix. (#34692, the old subagent-delivery gap, is fixed as of CC
|
|
26
|
+
* 2.1.241 — see src/subagent-delivery.test.ts.)
|
|
25
27
|
*
|
|
26
28
|
* Pure-ish (wraps the existing runHook tier). The catalog is harness-neutral data;
|
|
27
29
|
* the scaffold-test generator emits a test that calls these, and the same engine
|
package/dist/hook.d.ts
CHANGED
|
@@ -39,11 +39,13 @@
|
|
|
39
39
|
* inline `provide(name, cmd)` (read-only) or `dangerously(name, cmd)` (the loud
|
|
40
40
|
* escape) right in `needs`. See `research/hook-context-providers.md`.
|
|
41
41
|
*
|
|
42
|
-
* ⚠️ Honest scope: compile/verify fix the hook's AUTHORING + LOGIC
|
|
43
|
-
*
|
|
44
|
-
* PreToolUse
|
|
45
|
-
*
|
|
46
|
-
*
|
|
42
|
+
* ⚠️ Honest scope: compile/verify fix the hook's AUTHORING + LOGIC, not the
|
|
43
|
+
* harness's DELIVERY. The delivery floor MOVED — #34692 (a subagent's tool calls
|
|
44
|
+
* never reaching PreToolUse) is FIXED as of Claude Code 2.1.241, measured against
|
|
45
|
+
* a stock registry install and pinned by src/subagent-delivery.test.ts, which goes
|
|
46
|
+
* red if it regresses. What has NOT changed: a model can still route around a tool
|
|
47
|
+
* entirely (#45427 / #32376 — a Bash heredoc instead of `Write`), so a gate is a
|
|
48
|
+
* strong default and is NEVER an unbypassable wall. See `docs/compiled-hooks.md`.
|
|
47
49
|
*/
|
|
48
50
|
export { experimental_defineHook, experimental_defineFileGate, experimental_definePromptGate, experimental_defineStopGate, tool, tools, allow, deny, ask, commandView, pathView, gateAction, hookMode, experimental_defineInject, inject, experimental_defineReact, run, notice, nothing, responseView, decideProgram, decideFileGate, decidePromptGate, decideStopGate, runInject, runReact, runHookProgram, decisionExitCode, dispatchKind, hookRouting, hookNeeds, injectionOf, outcomeWrites, matchesTool, invalidToolPatterns, compileHookProgram, checkHookImports, stampHook, verifyHookStamp, HookCompileError, } from "./core/hook-program.js";
|
|
49
51
|
export type { Decision, HookMode, GateAction, CommandView, PathView, ResponseView, BashToolEvent, FileToolEvent, PromptEvent, StopEvent, ReactEvent, SessionEvent, HookProgram, FileGateHook, PromptGateHook, StopGateHook, InjectHook, ReactHook, AnyHook, DispatchKind, Injection, Reaction, RunReaction, CompiledHookProgram, CompileHookOptions, RawHookEvent, HookProgramOutcome, } from "./core/hook-program.js";
|
package/dist/hook.js
CHANGED
|
@@ -43,11 +43,13 @@ exports.leafCommandsNormalized = exports.HookStateError = exports.durationSecond
|
|
|
43
43
|
* inline `provide(name, cmd)` (read-only) or `dangerously(name, cmd)` (the loud
|
|
44
44
|
* escape) right in `needs`. See `research/hook-context-providers.md`.
|
|
45
45
|
*
|
|
46
|
-
* ⚠️ Honest scope: compile/verify fix the hook's AUTHORING + LOGIC
|
|
47
|
-
*
|
|
48
|
-
* PreToolUse
|
|
49
|
-
*
|
|
50
|
-
*
|
|
46
|
+
* ⚠️ Honest scope: compile/verify fix the hook's AUTHORING + LOGIC, not the
|
|
47
|
+
* harness's DELIVERY. The delivery floor MOVED — #34692 (a subagent's tool calls
|
|
48
|
+
* never reaching PreToolUse) is FIXED as of Claude Code 2.1.241, measured against
|
|
49
|
+
* a stock registry install and pinned by src/subagent-delivery.test.ts, which goes
|
|
50
|
+
* red if it regresses. What has NOT changed: a model can still route around a tool
|
|
51
|
+
* entirely (#45427 / #32376 — a Bash heredoc instead of `Write`), so a gate is a
|
|
52
|
+
* strong default and is NEVER an unbypassable wall. See `docs/compiled-hooks.md`.
|
|
51
53
|
*/
|
|
52
54
|
var hook_program_js_1 = require("./core/hook-program.js");
|
|
53
55
|
// ── the six ENTRY POINTS carry the experimental marking ─────────────────────
|
package/dist/score-core.d.ts
CHANGED
|
@@ -57,6 +57,54 @@ export declare const W_TRIFECTA_MAX = 30;
|
|
|
57
57
|
export declare const TRIFECTA_LABEL = "unit(s) can read data, reach the web, and run commands \u2014 the \"lethal trifecta\", so a prompt injection could exfiltrate secrets";
|
|
58
58
|
/** Map a 0–100 structural-health score to its letter grade (A ≥90 … F <60). */
|
|
59
59
|
export declare function gradeFor(score: number): PluginScore["grade"];
|
|
60
|
+
/**
|
|
61
|
+
* The health score a report may NOT exceed while it carries a CONFIDENT BREAKAGE
|
|
62
|
+
* — one point below the healthy band, so a definitively-broken harness can never
|
|
63
|
+
* render as `●` / grade `A`.
|
|
64
|
+
*
|
|
65
|
+
* WHY a cap and not a heavier weight: the score is a SUM of deductions, so one
|
|
66
|
+
* real breakage among many clean surfaces is diluted by its siblings. MEASURED
|
|
67
|
+
* 2026-08-28 on a fixture (10 distinct-description skills + one agent naming a
|
|
68
|
+
* never-available tool): Structure scored **92 with a `●` green dot** while
|
|
69
|
+
* carrying a dead tool contract, and the grade was driven instead by the
|
|
70
|
+
* lethal-trifecta pattern that {@link W_TRIFECTA} deliberately calls a ding and
|
|
71
|
+
* not a fail. A weight big enough to fix that would cry wolf on a large clean
|
|
72
|
+
* plugin; a cap fixes the reading without touching the arithmetic.
|
|
73
|
+
*
|
|
74
|
+
* The same defect, with higher stakes, is documented in an external 517-skill
|
|
75
|
+
* catalog run of a different scanner: its weighted aggregate returned MEDIUM for
|
|
76
|
+
* a skill carrying THREE HIGH findings, so gating on the aggregate would have
|
|
77
|
+
* admitted exactly the skill the gate existed to stop. That team's conclusion —
|
|
78
|
+
* judge by the worst finding, not the aggregate — is what this encodes.
|
|
79
|
+
*/
|
|
80
|
+
export declare const CONFIDENT_BREAKAGE_CAP = 89;
|
|
81
|
+
/**
|
|
82
|
+
* The findings that CAP the score — each one means a surface is definitively
|
|
83
|
+
* dead, not merely risky: it will silently not run, not resolve, or not register.
|
|
84
|
+
*
|
|
85
|
+
* ENUMERATED on purpose, never derived from "severity". Two properties decide
|
|
86
|
+
* membership, and both must hold: the finding is DECIDABLE from the artifact plus
|
|
87
|
+
* the world (the `structural-closed` / `external-decidable` buckets of the
|
|
88
|
+
* lint-rule-calibration rule), and its consequence is BREAKAGE rather than
|
|
89
|
+
* exposure. So the heuristic rings stay out by construction — a lethal-trifecta
|
|
90
|
+
* unit and a description overlap are real signals but are a capability PATTERN
|
|
91
|
+
* and a calibrated PROXY, and capping on either is how a gate earns the reputation
|
|
92
|
+
* that gets it switched off.
|
|
93
|
+
*
|
|
94
|
+
* Adding a row here is a deliberate act: it makes a finding grade-capping for
|
|
95
|
+
* every consumer, so it belongs only to a check whose false-positive rate is
|
|
96
|
+
* already known to be ~0.
|
|
97
|
+
*/
|
|
98
|
+
export declare function confidentBreakages(r: ScanReport): {
|
|
99
|
+
readonly n: number;
|
|
100
|
+
readonly label: string;
|
|
101
|
+
}[];
|
|
102
|
+
/**
|
|
103
|
+
* Cap a health score at {@link CONFIDENT_BREAKAGE_CAP} when the report carries any
|
|
104
|
+
* {@link confidentBreakages} finding. Applied to the OVERALL and to the ring that
|
|
105
|
+
* owns the finding, so the headline and the breakdown cannot disagree.
|
|
106
|
+
*/
|
|
107
|
+
export declare function applyBreakageCap(score: number, r: ScanReport): number;
|
|
60
108
|
/** One deduction: a count, its per-item weight, and the label if non-zero. */
|
|
61
109
|
export interface Deduction {
|
|
62
110
|
readonly n: number;
|
package/dist/score-core.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.TRIFECTA_LABEL = exports.W_TRIFECTA_MAX = exports.W_TRIFECTA = exports.W_NO_CONTRACT = exports.W_OVERLAP = exports.W_DANGLING_REF = exports.W_NO_DESCRIPTION = exports.W_MISSING_HOOK = void 0;
|
|
3
|
+
exports.CONFIDENT_BREAKAGE_CAP = exports.TRIFECTA_LABEL = exports.W_TRIFECTA_MAX = exports.W_TRIFECTA = exports.W_NO_CONTRACT = exports.W_OVERLAP = exports.W_DANGLING_REF = exports.W_NO_DESCRIPTION = exports.W_MISSING_HOOK = void 0;
|
|
4
4
|
exports.gradeFor = gradeFor;
|
|
5
|
+
exports.confidentBreakages = confidentBreakages;
|
|
6
|
+
exports.applyBreakageCap = applyBreakageCap;
|
|
5
7
|
exports.trifectaExposure = trifectaExposure;
|
|
6
8
|
exports.reportDeductions = reportDeductions;
|
|
7
9
|
exports.isEmptyMachine = isEmptyMachine;
|
|
@@ -52,6 +54,90 @@ function gradeFor(score) {
|
|
|
52
54
|
return "D";
|
|
53
55
|
return "F";
|
|
54
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* The health score a report may NOT exceed while it carries a CONFIDENT BREAKAGE
|
|
59
|
+
* — one point below the healthy band, so a definitively-broken harness can never
|
|
60
|
+
* render as `●` / grade `A`.
|
|
61
|
+
*
|
|
62
|
+
* WHY a cap and not a heavier weight: the score is a SUM of deductions, so one
|
|
63
|
+
* real breakage among many clean surfaces is diluted by its siblings. MEASURED
|
|
64
|
+
* 2026-08-28 on a fixture (10 distinct-description skills + one agent naming a
|
|
65
|
+
* never-available tool): Structure scored **92 with a `●` green dot** while
|
|
66
|
+
* carrying a dead tool contract, and the grade was driven instead by the
|
|
67
|
+
* lethal-trifecta pattern that {@link W_TRIFECTA} deliberately calls a ding and
|
|
68
|
+
* not a fail. A weight big enough to fix that would cry wolf on a large clean
|
|
69
|
+
* plugin; a cap fixes the reading without touching the arithmetic.
|
|
70
|
+
*
|
|
71
|
+
* The same defect, with higher stakes, is documented in an external 517-skill
|
|
72
|
+
* catalog run of a different scanner: its weighted aggregate returned MEDIUM for
|
|
73
|
+
* a skill carrying THREE HIGH findings, so gating on the aggregate would have
|
|
74
|
+
* admitted exactly the skill the gate existed to stop. That team's conclusion —
|
|
75
|
+
* judge by the worst finding, not the aggregate — is what this encodes.
|
|
76
|
+
*/
|
|
77
|
+
exports.CONFIDENT_BREAKAGE_CAP = 89;
|
|
78
|
+
/**
|
|
79
|
+
* The findings that CAP the score — each one means a surface is definitively
|
|
80
|
+
* dead, not merely risky: it will silently not run, not resolve, or not register.
|
|
81
|
+
*
|
|
82
|
+
* ENUMERATED on purpose, never derived from "severity". Two properties decide
|
|
83
|
+
* membership, and both must hold: the finding is DECIDABLE from the artifact plus
|
|
84
|
+
* the world (the `structural-closed` / `external-decidable` buckets of the
|
|
85
|
+
* lint-rule-calibration rule), and its consequence is BREAKAGE rather than
|
|
86
|
+
* exposure. So the heuristic rings stay out by construction — a lethal-trifecta
|
|
87
|
+
* unit and a description overlap are real signals but are a capability PATTERN
|
|
88
|
+
* and a calibrated PROXY, and capping on either is how a gate earns the reputation
|
|
89
|
+
* that gets it switched off.
|
|
90
|
+
*
|
|
91
|
+
* Adding a row here is a deliberate act: it makes a finding grade-capping for
|
|
92
|
+
* every consumer, so it belongs only to a check whose false-positive rate is
|
|
93
|
+
* already known to be ~0.
|
|
94
|
+
*/
|
|
95
|
+
function confidentBreakages(r) {
|
|
96
|
+
const rows = [
|
|
97
|
+
{
|
|
98
|
+
n: r.hooks.filter((h) => h.status === "missing").length,
|
|
99
|
+
label: "hook script missing",
|
|
100
|
+
},
|
|
101
|
+
{ n: r.hookEventIssues.length, label: "hook on an unknown event" },
|
|
102
|
+
{ n: r.danglingRefs.length, label: "broken intra-plugin reference" },
|
|
103
|
+
{
|
|
104
|
+
n: r.agents.reduce((n, a) => n + a.toolIssues.length, 0),
|
|
105
|
+
label: "unavailable agent tool",
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
n: r.agents.reduce((n, a) => n + a.mcpToolIssues.length, 0),
|
|
109
|
+
label: "agent MCP tool whose server isn't declared",
|
|
110
|
+
},
|
|
111
|
+
{ n: r.mcpIssues.length, label: "MCP server that can't start" },
|
|
112
|
+
{
|
|
113
|
+
n: r.mcpHookIssues.length,
|
|
114
|
+
label: "mcp_tool hook incomplete / undeclared server",
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
n: r.skillResourceIssues.length,
|
|
118
|
+
label: "skill bundled-resource ref that doesn't resolve",
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
n: r.skillFenceIssues.length,
|
|
122
|
+
label: "invisible skill (no opening `---` fence)",
|
|
123
|
+
},
|
|
124
|
+
{
|
|
125
|
+
n: r.frontmatterIssues.length,
|
|
126
|
+
label: "surface missing required frontmatter",
|
|
127
|
+
},
|
|
128
|
+
];
|
|
129
|
+
return rows.filter((row) => row.n > 0);
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Cap a health score at {@link CONFIDENT_BREAKAGE_CAP} when the report carries any
|
|
133
|
+
* {@link confidentBreakages} finding. Applied to the OVERALL and to the ring that
|
|
134
|
+
* owns the finding, so the headline and the breakdown cannot disagree.
|
|
135
|
+
*/
|
|
136
|
+
function applyBreakageCap(score, r) {
|
|
137
|
+
return confidentBreakages(r).length > 0
|
|
138
|
+
? Math.min(score, exports.CONFIDENT_BREAKAGE_CAP)
|
|
139
|
+
: score;
|
|
140
|
+
}
|
|
55
141
|
/**
|
|
56
142
|
* The lethal-trifecta exposure a report incurs — the ONE number the Safety ring
|
|
57
143
|
* and the overall grade both read.
|
|
@@ -263,7 +349,10 @@ function scoreReport(r) {
|
|
|
263
349
|
return { score: 0, issues: ["no loadable plugin surface"] };
|
|
264
350
|
}
|
|
265
351
|
const deductions = reportDeductions(r);
|
|
266
|
-
const { score } = computeIntegrityScore(deductions);
|
|
352
|
+
const { score: summed } = computeIntegrityScore(deductions);
|
|
353
|
+
// A confident breakage caps the score — a summed model dilutes one real
|
|
354
|
+
// breakage among clean siblings. See applyBreakageCap for the measurement.
|
|
355
|
+
const score = applyBreakageCap(summed, r);
|
|
267
356
|
const issues = [];
|
|
268
357
|
for (const d of deductions) {
|
|
269
358
|
if (d.n === 0)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "vigiles",
|
|
3
|
-
"version": "21.0.
|
|
3
|
+
"version": "21.0.2",
|
|
4
4
|
"description": "Audit, test and measure the harness your AI agent runs on — grade your CLAUDE.md / AGENTS.md, skills, subagents and hooks, run them against a scripted model, and measure whether they actually fire.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"claude-code",
|
|
@@ -49,83 +49,18 @@ not established, which is why the floor stays.
|
|
|
49
49
|
If the unit and deterministic tiers can both answer it, **prefer unit**: it's
|
|
50
50
|
faster and reaches events the deterministic mock can't drive.
|
|
51
51
|
|
|
52
|
-
## Step 0.4 — Observing a run
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
questions are keyed on the **observation** instead — "what did this skill
|
|
56
|
-
actually do?" — and they have answers already. Reach for these before building
|
|
57
|
-
anything; every one of them ships today.
|
|
58
|
-
|
|
59
|
-
| The question you're actually asking | Use |
|
|
60
|
-
| ----------------------------------------------------------------------- | -------------------------------------------------------------------------------------- |
|
|
61
|
-
| Which tools did it call, and with what arguments? | `trace.toolCalls` · `tool` / `toolWith` checks · `parseToolCalls` (`vigiles`) |
|
|
62
|
-
| Did it call a tool it must not? | `notTool(name)` |
|
|
63
|
-
| Did it call **only** tools from a known set? | `onlyTools([...])` — the white-list, symmetric to `assertWroteOnly` |
|
|
64
|
-
| Did it stay inside the `allowed-tools` its own frontmatter declares? | `skillContract(dir).surface` — builds that check FROM the declaration |
|
|
65
|
-
| What files did the run write? | `filesWritten` · `wrote(path)` / `didNotWrite(path)` · `r.file(path)` |
|
|
66
|
-
| Did it write **only** where it was supposed to? | `assertWroteOnly([...])` / `assertNoWrite()` — needs `{ sandbox: "auto" }` |
|
|
67
|
-
| Run a tool call but **don't let it execute** — capture the args instead | the `interceptTools` option on `measure` / `runEval` (a `ToolIntercept[]`) |
|
|
68
|
-
| Did a subagent do it, and which one? | `subagent(name, [...])` · `SubagentTrace` |
|
|
69
|
-
| Was it an MCP tool? | `mcp(server, toolName)` |
|
|
70
|
-
| Assert the whole effect boundary deterministically | `assertChecks` + the checks above (see `examples/harness/effect-boundary.harness.mjs`) |
|
|
71
|
-
|
|
72
|
-
`interceptTools` is the one worth knowing about, because it is not obvious it
|
|
73
|
-
exists: it denies a tool its **real execution** via an auto-wired `PreToolUse`
|
|
74
|
-
hook while still recording the call and its arguments into the trace. That is
|
|
75
|
-
how you test a skill that would otherwise mutate a real external service — a
|
|
76
|
-
calendar, an upload — without mocking anything yourself.
|
|
77
|
-
|
|
78
|
-
**Verify a skill against its own declaration** with `skillContract` — it reads
|
|
79
|
-
the `allowed-tools:` the skill already claims and hands back ready checks, so
|
|
80
|
-
the claim is verified instead of restated:
|
|
81
|
-
|
|
82
|
-
```ts
|
|
83
|
-
import { skillContract, assertChecks } from "vigiles";
|
|
84
|
-
|
|
85
|
-
const c = skillContract(".claude/skills/my-skill");
|
|
86
|
-
assertChecks(trace, [c.activation, ...c.surface]);
|
|
87
|
-
```
|
|
52
|
+
## Step 0.4 — Observing a run, and what it costs
|
|
53
|
+
|
|
54
|
+
Two questions have their own references — open the one you need, don't guess:
|
|
88
55
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
assertion is total.
|
|
98
|
-
|
|
99
|
-
## Step 0.5 — Set honest expectations (what's testable, and at what cost)
|
|
100
|
-
|
|
101
|
-
Be explicit with the user about which bucket each surface falls into — never let
|
|
102
|
-
"we'll test it" hide whether that's free, sub-priced, or needs a container. Every
|
|
103
|
-
surface sorts into one of three buckets:
|
|
104
|
-
|
|
105
|
-
- **A — Free & deterministic** (no model, runs in CI on every commit): a hook's
|
|
106
|
-
block/allow decision (`runHook`), a tool-contract / "did NOT call the forbidden
|
|
107
|
-
tool" check, structural facts (`vigiles audit`), and **record-replay** of any tool
|
|
108
|
-
a skill shells out to (record the real result once, replay it via a PATH stub).
|
|
109
|
-
- **B — Model-gated, on your subscription** (real model, **no metered API**): does a
|
|
110
|
-
skill's description **fire** (`measureTriggerRate`, recall + precision) **and**
|
|
111
|
-
does its guidance actually **produce good output** (score it directly:
|
|
112
|
-
`measure({ checks: [judged(rubric)] })` + `assertRates` — the absolute oracle;
|
|
113
|
-
use a `runEval` A/B on-vs-off only when you need the _relative_ lift). This is
|
|
114
|
-
the half a **prose / guidance skill** lives in —
|
|
115
|
-
its worth is behavioral, so only a model can judge it. That is **not** "uncovered"
|
|
116
|
-
and **not** free: it's fully testable on the sub. State it that way.
|
|
117
|
-
- **C — Needs a real service** (a real browser / DB / redis / a11y runtime): vigiles
|
|
118
|
-
**composes with a container** here; it does not fake real semantics. Name the
|
|
119
|
-
service and hand off — don't pretend a cheap tier substitutes for it.
|
|
120
|
-
|
|
121
|
-
So a prose-skill library is roughly **~100% testable (some free, most on your sub),
|
|
122
|
-
~0% needs-a-container** — not "poorly covered." An accessibility/browser plugin is
|
|
123
|
-
the worst case, with a large bucket C. When you report coverage, give **two
|
|
124
|
-
numbers**: "% testable at all (free + sub)" vs "% that needs a container", and say
|
|
125
|
-
which surfaces are free vs sub-priced. The model-gated half is the **point** of the
|
|
126
|
-
eval pillar (affordable on the sub), not a gap — and testing a prose skill's
|
|
127
|
-
_behavior_ requires a real model for **everyone** (promptfoo, the SDKs, all of it);
|
|
128
|
-
vigiles just does it on your subscription instead of metered API.
|
|
56
|
+
- **"What did the run actually DO?"** — which tools it called, whether it stayed
|
|
57
|
+
inside its declared `allowed-tools`, what it wrote, how to record a call without
|
|
58
|
+
executing it → [`references/observing-a-run.md`](references/observing-a-run.md)
|
|
59
|
+
- **"Is this free, sub-priced, or does it need a container?"** — the three buckets,
|
|
60
|
+
and what to tell the user after a paid run →
|
|
61
|
+
[`references/cost-and-expectations.md`](references/cost-and-expectations.md)
|
|
62
|
+
|
|
63
|
+
Never say "we'll test it" without settling the second one first.
|
|
129
64
|
|
|
130
65
|
## Step 1 — Ensure vigiles is installed
|
|
131
66
|
|
|
@@ -154,142 +89,12 @@ Pick one concrete thing to pin down — a specific `PreToolUse` hook, a specific
|
|
|
154
89
|
|
|
155
90
|
## Step 3 — Write the test for the chosen tier
|
|
156
91
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
import { runHook, assertHookBlocked } from "vigiles";
|
|
161
|
-
|
|
162
|
-
const r = runHook(hookCommand, {
|
|
163
|
-
hook_event_name: "PreToolUse",
|
|
164
|
-
tool_name: "Bash",
|
|
165
|
-
tool_input: { command: "git commit --no-verify" },
|
|
166
|
-
});
|
|
167
|
-
assertHookBlocked(r); // exit 2 / decision:"block" / permissionDecision:"deny"
|
|
168
|
-
```
|
|
169
|
-
|
|
170
|
-
Testing a hook you didn't write (a vendored third-party script)? Mark it
|
|
171
|
-
`{ trusted: false }` and it runs confined under bubblewrap by default (read-only
|
|
172
|
-
host, cleared env, no network egress). Add `{ recordEgress: true }` to also
|
|
173
|
-
**record** what it tries to reach — `r.egress` plus `assertNoEgress(r)` /
|
|
174
|
-
`assertEgressOnly(r, [...])` — the supply-chain check for "what does this skill
|
|
175
|
-
phone home to / install from?". When the hook's setup needs a _real_ install,
|
|
176
|
-
`{ egress: { allow: ["registry.npmjs.org"] } }` lets it reach only that
|
|
177
|
-
allowlist (a packet-layer `nft` wall, so a raw socket off-list is dropped too) →
|
|
178
|
-
`r.egress` (allowed hosts) + `r.egressDropped`. Be precise about the boundaries:
|
|
179
|
-
see
|
|
180
|
-
[`docs/sandboxing.md`](../../docs/sandboxing.md) (it blocks destruction and
|
|
181
|
-
egress, but does NOT isolate reads of host files, and only under bwrap).
|
|
182
|
-
|
|
183
|
-
**Deterministic (`runHarnessTest`)** — load the real plugin, drive a scripted
|
|
184
|
-
mock model, assert the hook fired (or the context landed):
|
|
185
|
-
|
|
186
|
-
```ts
|
|
187
|
-
import {
|
|
188
|
-
runHarnessTest,
|
|
189
|
-
assertHookFired,
|
|
190
|
-
assertRequestContains,
|
|
191
|
-
} from "vigiles";
|
|
192
|
-
// `scriptModel` is the Claude-Code TRANSPORT, deliberately not re-exported from
|
|
193
|
-
// the harness-agnostic root surface — import it from the harness package:
|
|
194
|
-
import { scriptModel } from "vigiles/claude-code";
|
|
195
|
-
|
|
196
|
-
const r = await runHarnessTest({
|
|
197
|
-
pluginDir: "./", // or { settings: { hooks: {...} } }
|
|
198
|
-
transcript: true,
|
|
199
|
-
model: scriptModel([{ text: "ok" }]),
|
|
200
|
-
});
|
|
201
|
-
assertHookFired(r, "SessionStart");
|
|
202
|
-
assertRequestContains(r, "expected injected text"); // did it actually land?
|
|
203
|
-
```
|
|
204
|
-
|
|
205
|
-
**Eval — absolute (`paid_measure` + `paid_judged`)** — testing _one_ skill, the usual case:
|
|
206
|
-
score its output directly against a rubric. No on/off baseline — this is the
|
|
207
|
-
"is it any good?" oracle (what promptfoo/DeepEval lead with), and the right
|
|
208
|
-
default when there's nothing to compare against:
|
|
209
|
-
|
|
210
|
-
An eval file **describes** its eval — it must never run one at the top level,
|
|
211
|
-
because importing such a file spends real money. Write `<name>.eval.mjs`:
|
|
212
|
-
|
|
213
|
-
```ts
|
|
214
|
-
import { defineEval, skill, assertRates } from "vigiles";
|
|
215
|
-
import { paid_judged } from "vigiles/eval"; // a Check whose default judge bills
|
|
216
|
-
|
|
217
|
-
export default defineEval({
|
|
218
|
-
measure: {
|
|
219
|
-
pluginDir: "./",
|
|
220
|
-
task: "…a task the skill should handle…",
|
|
221
|
-
checks: [
|
|
222
|
-
skill("my-plugin:my-skill"), // it fired
|
|
223
|
-
paid_judged("the answer correctly does X and avoids Y"), // …and the output is good
|
|
224
|
-
],
|
|
225
|
-
trials: 6,
|
|
226
|
-
},
|
|
227
|
-
assert: (report) => assertRates(report, { min: 0.8 }), // each check ≥ 80% of trials
|
|
228
|
-
});
|
|
229
|
-
```
|
|
230
|
-
|
|
231
|
-
Run it with `npx vigiles eval <file>` — never `node <file>`, which refuses.
|
|
232
|
-
|
|
233
|
-
**Eval — relative (`paid_runEval` + `assertSignificant`)** — when the question is
|
|
234
|
-
_lift over no-skill_ (regression, or proving a change isn't noise): A/B the
|
|
235
|
-
change on vs off and gate on significance, not eyeballing:
|
|
236
|
-
|
|
237
|
-
```ts
|
|
238
|
-
import { defineEval, assertSignificant } from "vigiles";
|
|
239
|
-
|
|
240
|
-
export default defineEval({
|
|
241
|
-
runEval: {
|
|
242
|
-
arms: { off: {}, on: { pluginDir: "./" } },
|
|
243
|
-
task: "…a task the harness change should affect…",
|
|
244
|
-
measure: (ctx) => ({ ok: /* a bare predicate over the trace */ true }),
|
|
245
|
-
trials: 6,
|
|
246
|
-
cache: "readwrite",
|
|
247
|
-
},
|
|
248
|
-
assert: (report) =>
|
|
249
|
-
assertSignificant(report, { baseline: "off", arm: "on", metric: "ok" }),
|
|
250
|
-
});
|
|
251
|
-
```
|
|
252
|
-
|
|
253
|
-
### Never hand-roll the runner — it silently eats stderr
|
|
92
|
+
Per-tier skeletons, and the one mistake that silently swallows failures (a
|
|
93
|
+
hand-rolled runner eats stderr) →
|
|
94
|
+
[`references/writing-tests.md`](references/writing-tests.md)
|
|
254
95
|
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
success, while advisory output — including vigiles's own compiled-hook
|
|
258
|
-
`notice()` — is written to **stderr**. A hand-rolled runner therefore reports a
|
|
259
|
-
perfectly healthy react hook as **dead**, and an assertion about a warning can
|
|
260
|
-
never pass. (Observed three times in one repo, twice after the first fix.)
|
|
261
|
-
|
|
262
|
-
Every vigiles result already carries **both streams**, so the bug is
|
|
263
|
-
unrepresentable:
|
|
264
|
-
|
|
265
|
-
| Runner | Result | Carries |
|
|
266
|
-
| ---------------- | ------------------- | --------------------------------------------------- |
|
|
267
|
-
| `runScript` | `ScriptRunResult` | `exitCode`, `stdout`, `stderr`, `filesWritten?` |
|
|
268
|
-
| `runHook` | `HookRunResult` | all of the above, **plus** `blocked` / `decision` |
|
|
269
|
-
| `runHarnessTest` | `HarnessTestResult` | `exitCode`, `stdout`, `stderr`, `cwd` + the `Trace` |
|
|
270
|
-
|
|
271
|
-
**Testing a plain helper script** (a bash/node/python program that isn't a hook)?
|
|
272
|
-
Use **`runScript`** — it runs any command and reports what it did:
|
|
273
|
-
|
|
274
|
-
```ts
|
|
275
|
-
import { runScript } from "vigiles";
|
|
276
|
-
|
|
277
|
-
const r = runScript("bash scripts/check-links.sh", { cwd: repoDir });
|
|
278
|
-
assert.equal(r.exitCode, 0);
|
|
279
|
-
assert.match(r.stderr, /0 broken links/); // advisory output lives HERE
|
|
280
|
-
```
|
|
281
|
-
|
|
282
|
-
`runHook` is exactly `runScript` plus the hook protocol (event → stdin, exit code
|
|
283
|
-
→ allow/deny). Pick by the question you're asking: a **hook** has a _decision_, a
|
|
284
|
-
**script** has _effects_. That's why `ScriptRunResult` has no `decision` field —
|
|
285
|
-
a field that is always meaningless is worse than no field.
|
|
286
|
-
|
|
287
|
-
⚠️ **Asserting what a script wrote requires confinement.** `filesWritten` is
|
|
288
|
-
recorded by diffing the work dir, which only a confined run does — so it is
|
|
289
|
-
`undefined` after a plain run. That is deliberately _not_ the same as `[]`
|
|
290
|
-
("recorded, wrote nothing"): `assertNoWrite` / `assertWroteOnly` **throw** on an
|
|
291
|
-
unrecorded result rather than pass having inspected nothing. Pass
|
|
292
|
-
`{ sandbox: "auto" }` (Linux + bubblewrap) to actually record writes.
|
|
96
|
+
Read it before writing the file — the skeleton differs per tier, and the runner
|
|
97
|
+
warning has cost real debugging time.
|
|
293
98
|
|
|
294
99
|
## Step 4 — Run it
|
|
295
100
|
|
|
@@ -309,23 +114,6 @@ job that asserts the capability is present, run **`vigiles test --no-skip`** so
|
|
|
309
114
|
skipped tier fails — a green-with-skips is untested surface. Keep unit +
|
|
310
115
|
deterministic tests in CI (free); run evals locally or on a schedule with auth.
|
|
311
116
|
|
|
312
|
-
### After a real-model run: TELL THE USER WHAT IT SPENT
|
|
313
|
-
|
|
314
|
-
Whenever you run a real-model eval (`runEval` / `measureArms` / `measureTriggerRate`
|
|
315
|
-
/ `measure`), **surface the spend to the user in your reply** — don't let a paid run
|
|
316
|
-
be silent. `runEval` prints a cost block to stderr and every report carries `usage`
|
|
317
|
-
(`report.arms[*].usage`: `totalCostUsd` + token counts). Relay, in plain words:
|
|
318
|
-
|
|
319
|
-
- **tokens spent** and the **API-equivalent `$`** (`total_cost_usd` — what it _would_
|
|
320
|
-
cost at metered API rates);
|
|
321
|
-
- **how it was billed** — "on your Claude subscription (**$0 metered**)" if you're
|
|
322
|
-
logged in, or a **⚠ warning** if `ANTHROPIC_API_KEY` is set (that run was billed
|
|
323
|
-
**per token** — tell them to unset it and `claude login` to run free).
|
|
324
|
-
|
|
325
|
-
We do **not** show "% of your subscription" — Anthropic doesn't expose a plan's
|
|
326
|
-
quota, so any percentage would be invented. Tokens + API-equivalent `$` + the
|
|
327
|
-
billed-to line is the honest, complete picture. Keep the user's cost visible, always.
|
|
328
|
-
|
|
329
117
|
## CI — don't hand-write the steps
|
|
330
118
|
|
|
331
119
|
These tiers belong in CI, and there is a published Action for it. Run `vigiles init`: it
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# Honest expectations and cost
|
|
2
|
+
|
|
3
|
+
Read this before telling a user "we'll test it" — it fixes which bucket a surface
|
|
4
|
+
falls into (free / on your subscription / needs a container) and what to report
|
|
5
|
+
after a paid run.
|
|
6
|
+
|
|
7
|
+
## Set honest expectations (what's testable, and at what cost)
|
|
8
|
+
|
|
9
|
+
Be explicit with the user about which bucket each surface falls into — never let
|
|
10
|
+
"we'll test it" hide whether that's free, sub-priced, or needs a container. Every
|
|
11
|
+
surface sorts into one of three buckets:
|
|
12
|
+
|
|
13
|
+
- **A — Free & deterministic** (no model, runs in CI on every commit): a hook's
|
|
14
|
+
block/allow decision (`runHook`), a tool-contract / "did NOT call the forbidden
|
|
15
|
+
tool" check, structural facts (`vigiles audit`), and **record-replay** of any tool
|
|
16
|
+
a skill shells out to (record the real result once, replay it via a PATH stub).
|
|
17
|
+
- **B — Model-gated, on your subscription** (real model, **no metered API**): does a
|
|
18
|
+
skill's description **fire** (`measureTriggerRate`, recall + precision) **and**
|
|
19
|
+
does its guidance actually **produce good output** (score it directly:
|
|
20
|
+
`measure({ checks: [judged(rubric)] })` + `assertRates` — the absolute oracle;
|
|
21
|
+
use a `runEval` A/B on-vs-off only when you need the _relative_ lift). This is
|
|
22
|
+
the half a **prose / guidance skill** lives in —
|
|
23
|
+
its worth is behavioral, so only a model can judge it. That is **not** "uncovered"
|
|
24
|
+
and **not** free: it's fully testable on the sub. State it that way.
|
|
25
|
+
- **C — Needs a real service** (a real browser / DB / redis / a11y runtime): vigiles
|
|
26
|
+
**composes with a container** here; it does not fake real semantics. Name the
|
|
27
|
+
service and hand off — don't pretend a cheap tier substitutes for it.
|
|
28
|
+
|
|
29
|
+
So a prose-skill library is roughly **~100% testable (some free, most on your sub),
|
|
30
|
+
~0% needs-a-container** — not "poorly covered." An accessibility/browser plugin is
|
|
31
|
+
the worst case, with a large bucket C. When you report coverage, give **two
|
|
32
|
+
numbers**: "% testable at all (free + sub)" vs "% that needs a container", and say
|
|
33
|
+
which surfaces are free vs sub-priced. The model-gated half is the **point** of the
|
|
34
|
+
eval pillar (affordable on the sub), not a gap — and testing a prose skill's
|
|
35
|
+
_behavior_ requires a real model for **everyone** (promptfoo, the SDKs, all of it);
|
|
36
|
+
vigiles just does it on your subscription instead of metered API.
|
|
37
|
+
|
|
38
|
+
### After a real-model run: TELL THE USER WHAT IT SPENT
|
|
39
|
+
|
|
40
|
+
Whenever you run a real-model eval (`runEval` / `measureArms` / `measureTriggerRate`
|
|
41
|
+
/ `measure`), **surface the spend to the user in your reply** — don't let a paid run
|
|
42
|
+
be silent. `runEval` prints a cost block to stderr and every report carries `usage`
|
|
43
|
+
(`report.arms[*].usage`: `totalCostUsd` + token counts). Relay, in plain words:
|
|
44
|
+
|
|
45
|
+
- **tokens spent** and the **API-equivalent `$`** (`total_cost_usd` — what it _would_
|
|
46
|
+
cost at metered API rates);
|
|
47
|
+
- **how it was billed** — "on your Claude subscription (**$0 metered**)" if you're
|
|
48
|
+
logged in, or a **⚠ warning** if `ANTHROPIC_API_KEY` is set (that run was billed
|
|
49
|
+
**per token** — tell them to unset it and `claude login` to run free).
|
|
50
|
+
|
|
51
|
+
We do **not** show "% of your subscription" — Anthropic doesn't expose a plan's
|
|
52
|
+
quota, so any percentage would be invented. Tokens + API-equivalent `$` + the
|
|
53
|
+
billed-to line is the honest, complete picture. Keep the user's cost visible, always.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# Observing a run — what it CALLED, WROTE, and TOUCHED
|
|
2
|
+
|
|
3
|
+
Read this when the question is about **what a run did**, not about which tier to
|
|
4
|
+
pick. Every predicate here ships today.
|
|
5
|
+
|
|
6
|
+
## Observing a run (what it CALLED, WROTE, and TOUCHED)
|
|
7
|
+
|
|
8
|
+
The table above is keyed on the harness _surface_ under test. Half the real
|
|
9
|
+
questions are keyed on the **observation** instead — "what did this skill
|
|
10
|
+
actually do?" — and they have answers already. Reach for these before building
|
|
11
|
+
anything; every one of them ships today.
|
|
12
|
+
|
|
13
|
+
| The question you're actually asking | Use |
|
|
14
|
+
| ----------------------------------------------------------------------- | -------------------------------------------------------------------------------------- |
|
|
15
|
+
| Which tools did it call, and with what arguments? | `trace.toolCalls` · `tool` / `toolWith` checks · `parseToolCalls` (`vigiles`) |
|
|
16
|
+
| Did it call a tool it must not? | `notTool(name)` |
|
|
17
|
+
| Did it call **only** tools from a known set? | `onlyTools([...])` — the white-list, symmetric to `assertWroteOnly` |
|
|
18
|
+
| Did it stay inside the `allowed-tools` its own frontmatter declares? | `skillContract(dir).surface` — builds that check FROM the declaration |
|
|
19
|
+
| What files did the run write? | `filesWritten` · `wrote(path)` / `didNotWrite(path)` · `r.file(path)` |
|
|
20
|
+
| Did it write **only** where it was supposed to? | `assertWroteOnly([...])` / `assertNoWrite()` — needs `{ sandbox: "auto" }` |
|
|
21
|
+
| Run a tool call but **don't let it execute** — capture the args instead | the `interceptTools` option on `measure` / `runEval` (a `ToolIntercept[]`) |
|
|
22
|
+
| Did a subagent do it, and which one? | `subagent(name, [...])` · `SubagentTrace` |
|
|
23
|
+
| Was it an MCP tool? | `mcp(server, toolName)` |
|
|
24
|
+
| Assert the whole effect boundary deterministically | `assertChecks` + the checks above (see `examples/harness/effect-boundary.harness.mjs`) |
|
|
25
|
+
|
|
26
|
+
`interceptTools` is the one worth knowing about, because it is not obvious it
|
|
27
|
+
exists: it denies a tool its **real execution** via an auto-wired `PreToolUse`
|
|
28
|
+
hook while still recording the call and its arguments into the trace. That is
|
|
29
|
+
how you test a skill that would otherwise mutate a real external service — a
|
|
30
|
+
calendar, an upload — without mocking anything yourself.
|
|
31
|
+
|
|
32
|
+
**Verify a skill against its own declaration** with `skillContract` — it reads
|
|
33
|
+
the `allowed-tools:` the skill already claims and hands back ready checks, so
|
|
34
|
+
the claim is verified instead of restated:
|
|
35
|
+
|
|
36
|
+
```ts
|
|
37
|
+
import { skillContract, assertChecks } from "vigiles";
|
|
38
|
+
|
|
39
|
+
const c = skillContract(".claude/skills/my-skill");
|
|
40
|
+
assertChecks(trace, [c.activation, ...c.surface]);
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Two of its states are **findings**, not clean bills, and their `surface` check
|
|
44
|
+
fails rather than passing on nothing: `undeclared` (no `allowed-tools:` line, so
|
|
45
|
+
the skill inherits _every_ tool) and `malformed` (frontmatter that isn't valid
|
|
46
|
+
YAML, so a strict loader reads no contract at all — one unquoted `: ` does it).
|
|
47
|
+
|
|
48
|
+
⚠️ **What is still NOT checked.** `onlyTools` compares tool _names_, so a narrow
|
|
49
|
+
allowlist entry like `Bash(node scripts/x.mjs:*)` is satisfied by any `Bash` call
|
|
50
|
+
at all. Scope inside a tool is unverified — say so rather than implying the
|
|
51
|
+
assertion is total.
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# Writing the test for the chosen tier
|
|
2
|
+
|
|
3
|
+
Read this once the tier is picked. Covers the per-tier skeleton and the one
|
|
4
|
+
mistake that silently swallows failures.
|
|
5
|
+
|
|
6
|
+
## Write the test for the chosen tier
|
|
7
|
+
|
|
8
|
+
**Unit (`runHook`)** — hand a hook a synthesized event, assert the decision:
|
|
9
|
+
|
|
10
|
+
```ts
|
|
11
|
+
import { runHook, assertHookBlocked } from "vigiles";
|
|
12
|
+
|
|
13
|
+
const r = runHook(hookCommand, {
|
|
14
|
+
hook_event_name: "PreToolUse",
|
|
15
|
+
tool_name: "Bash",
|
|
16
|
+
tool_input: { command: "git commit --no-verify" },
|
|
17
|
+
});
|
|
18
|
+
assertHookBlocked(r); // exit 2 / decision:"block" / permissionDecision:"deny"
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Testing a hook you didn't write (a vendored third-party script)? Mark it
|
|
22
|
+
`{ trusted: false }` and it runs confined under bubblewrap by default (read-only
|
|
23
|
+
host, cleared env, no network egress). Add `{ recordEgress: true }` to also
|
|
24
|
+
**record** what it tries to reach — `r.egress` plus `assertNoEgress(r)` /
|
|
25
|
+
`assertEgressOnly(r, [...])` — the supply-chain check for "what does this skill
|
|
26
|
+
phone home to / install from?". When the hook's setup needs a _real_ install,
|
|
27
|
+
`{ egress: { allow: ["registry.npmjs.org"] } }` lets it reach only that
|
|
28
|
+
allowlist (a packet-layer `nft` wall, so a raw socket off-list is dropped too) →
|
|
29
|
+
`r.egress` (allowed hosts) + `r.egressDropped`. Be precise about the boundaries:
|
|
30
|
+
see
|
|
31
|
+
[`docs/sandboxing.md`](../../docs/sandboxing.md) (it blocks destruction and
|
|
32
|
+
egress, but does NOT isolate reads of host files, and only under bwrap).
|
|
33
|
+
|
|
34
|
+
**Deterministic (`runHarnessTest`)** — load the real plugin, drive a scripted
|
|
35
|
+
mock model, assert the hook fired (or the context landed):
|
|
36
|
+
|
|
37
|
+
```ts
|
|
38
|
+
import {
|
|
39
|
+
runHarnessTest,
|
|
40
|
+
assertHookFired,
|
|
41
|
+
assertRequestContains,
|
|
42
|
+
} from "vigiles";
|
|
43
|
+
// `scriptModel` is the Claude-Code TRANSPORT, deliberately not re-exported from
|
|
44
|
+
// the harness-agnostic root surface — import it from the harness package:
|
|
45
|
+
import { scriptModel } from "vigiles/claude-code";
|
|
46
|
+
|
|
47
|
+
const r = await runHarnessTest({
|
|
48
|
+
pluginDir: "./", // or { settings: { hooks: {...} } }
|
|
49
|
+
transcript: true,
|
|
50
|
+
model: scriptModel([{ text: "ok" }]),
|
|
51
|
+
});
|
|
52
|
+
assertHookFired(r, "SessionStart");
|
|
53
|
+
assertRequestContains(r, "expected injected text"); // did it actually land?
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
**Eval — absolute (`paid_measure` + `paid_judged`)** — testing _one_ skill, the usual case:
|
|
57
|
+
score its output directly against a rubric. No on/off baseline — this is the
|
|
58
|
+
"is it any good?" oracle (what promptfoo/DeepEval lead with), and the right
|
|
59
|
+
default when there's nothing to compare against:
|
|
60
|
+
|
|
61
|
+
An eval file **describes** its eval — it must never run one at the top level,
|
|
62
|
+
because importing such a file spends real money. Write `<name>.eval.mjs`:
|
|
63
|
+
|
|
64
|
+
```ts
|
|
65
|
+
import { defineEval, skill, assertRates } from "vigiles";
|
|
66
|
+
import { paid_judged } from "vigiles/eval"; // a Check whose default judge bills
|
|
67
|
+
|
|
68
|
+
export default defineEval({
|
|
69
|
+
measure: {
|
|
70
|
+
pluginDir: "./",
|
|
71
|
+
task: "…a task the skill should handle…",
|
|
72
|
+
checks: [
|
|
73
|
+
skill("my-plugin:my-skill"), // it fired
|
|
74
|
+
paid_judged("the answer correctly does X and avoids Y"), // …and the output is good
|
|
75
|
+
],
|
|
76
|
+
trials: 6,
|
|
77
|
+
},
|
|
78
|
+
assert: (report) => assertRates(report, { min: 0.8 }), // each check ≥ 80% of trials
|
|
79
|
+
});
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Run it with `npx vigiles eval <file>` — never `node <file>`, which refuses.
|
|
83
|
+
|
|
84
|
+
**Eval — relative (`paid_runEval` + `assertSignificant`)** — when the question is
|
|
85
|
+
_lift over no-skill_ (regression, or proving a change isn't noise): A/B the
|
|
86
|
+
change on vs off and gate on significance, not eyeballing:
|
|
87
|
+
|
|
88
|
+
```ts
|
|
89
|
+
import { defineEval, assertSignificant } from "vigiles";
|
|
90
|
+
|
|
91
|
+
export default defineEval({
|
|
92
|
+
runEval: {
|
|
93
|
+
arms: { off: {}, on: { pluginDir: "./" } },
|
|
94
|
+
task: "…a task the harness change should affect…",
|
|
95
|
+
measure: (ctx) => ({ ok: /* a bare predicate over the trace */ true }),
|
|
96
|
+
trials: 6,
|
|
97
|
+
cache: "readwrite",
|
|
98
|
+
},
|
|
99
|
+
assert: (report) =>
|
|
100
|
+
assertSignificant(report, { baseline: "off", arm: "on", metric: "ok" }),
|
|
101
|
+
});
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
### Never hand-roll the runner — it silently eats stderr
|
|
105
|
+
|
|
106
|
+
Do **not** reach for `execFileSync` / `spawnSync` to drive the thing under test.
|
|
107
|
+
The failure is quiet and repeats: `execFileSync` returns **stdout only** on
|
|
108
|
+
success, while advisory output — including vigiles's own compiled-hook
|
|
109
|
+
`notice()` — is written to **stderr**. A hand-rolled runner therefore reports a
|
|
110
|
+
perfectly healthy react hook as **dead**, and an assertion about a warning can
|
|
111
|
+
never pass. (Observed three times in one repo, twice after the first fix.)
|
|
112
|
+
|
|
113
|
+
Every vigiles result already carries **both streams**, so the bug is
|
|
114
|
+
unrepresentable:
|
|
115
|
+
|
|
116
|
+
| Runner | Result | Carries |
|
|
117
|
+
| ---------------- | ------------------- | --------------------------------------------------- |
|
|
118
|
+
| `runScript` | `ScriptRunResult` | `exitCode`, `stdout`, `stderr`, `filesWritten?` |
|
|
119
|
+
| `runHook` | `HookRunResult` | all of the above, **plus** `blocked` / `decision` |
|
|
120
|
+
| `runHarnessTest` | `HarnessTestResult` | `exitCode`, `stdout`, `stderr`, `cwd` + the `Trace` |
|
|
121
|
+
|
|
122
|
+
**Testing a plain helper script** (a bash/node/python program that isn't a hook)?
|
|
123
|
+
Use **`runScript`** — it runs any command and reports what it did:
|
|
124
|
+
|
|
125
|
+
```ts
|
|
126
|
+
import { runScript } from "vigiles";
|
|
127
|
+
|
|
128
|
+
const r = runScript("bash scripts/check-links.sh", { cwd: repoDir });
|
|
129
|
+
assert.equal(r.exitCode, 0);
|
|
130
|
+
assert.match(r.stderr, /0 broken links/); // advisory output lives HERE
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
`runHook` is exactly `runScript` plus the hook protocol (event → stdin, exit code
|
|
134
|
+
→ allow/deny). Pick by the question you're asking: a **hook** has a _decision_, a
|
|
135
|
+
**script** has _effects_. That's why `ScriptRunResult` has no `decision` field —
|
|
136
|
+
a field that is always meaningless is worse than no field.
|
|
137
|
+
|
|
138
|
+
⚠️ **Asserting what a script wrote requires confinement.** `filesWritten` is
|
|
139
|
+
recorded by diffing the work dir, which only a confined run does — so it is
|
|
140
|
+
`undefined` after a plain run. That is deliberately _not_ the same as `[]`
|
|
141
|
+
("recorded, wrote nothing"): `assertNoWrite` / `assertWroteOnly` **throw** on an
|
|
142
|
+
unrecorded result rather than pass having inspected nothing. Pass
|
|
143
|
+
`{ sandbox: "auto" }` (Linux + bubblewrap) to actually record writes.
|