vigiles 22.0.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/adapters/claude-code/hook-protocol.js +3 -0
- package/dist/adapters/claude-code/layout.js +2 -0
- package/dist/adapters/claude-code/run-scripts.js +26 -2
- package/dist/cli-flag-check.js +2 -1
- package/dist/cli.d.ts +121 -0
- package/dist/cli.js +375 -32
- package/dist/core/hook-block-ineffective.js +24 -1
- package/dist/core/hook-protocol.d.ts +14 -0
- package/dist/core/layout.d.ts +15 -0
- package/dist/core/rule-meta.d.ts +1 -1
- package/dist/core/rule-meta.js +16 -0
- package/dist/core/types.d.ts +44 -0
- package/dist/core/validate.js +18 -0
- package/dist/coverage-evidence.d.ts +48 -2
- package/dist/coverage-evidence.js +96 -3
- package/dist/eval.d.ts +10 -0
- package/dist/eval.js +14 -3
- package/dist/harness-resolve-hooks.d.mts +13 -0
- package/dist/harness-resolve-hooks.mjs +50 -0
- package/dist/plugin-loader.js +35 -0
- package/dist/run-hook.d.ts +23 -2
- package/dist/run-hook.js +10 -3
- package/dist/scan-core.d.ts +5 -0
- package/dist/scan-core.js +10 -1
- package/dist/test-coverage-files.js +17 -8
- package/dist/test-coverage.js +29 -16
- package/package.json +1 -1
package/dist/core/layout.d.ts
CHANGED
|
@@ -51,6 +51,21 @@ export interface PluginLayout {
|
|
|
51
51
|
readonly agentDir: string;
|
|
52
52
|
/** Slash-commands dir, holding flat `<dir>/<name>.md`, e.g. `commands`. */
|
|
53
53
|
readonly commandDir: string;
|
|
54
|
+
/**
|
|
55
|
+
* Path-scoped RULES dir, holding flat `<dir>/<name>.md`, e.g. `rules`
|
|
56
|
+
* (`""` or absent = this harness has no such layer).
|
|
57
|
+
*
|
|
58
|
+
* Claude Code loads `.claude/rules/*.md` as project instructions, scoped by a
|
|
59
|
+
* `paths:` frontmatter key. It is an INSTRUCTION surface — often where a
|
|
60
|
+
* team's hardest policies actually live — and until now no layout named it, so
|
|
61
|
+
* `frontmatter-valid` and the rule map simply never saw those files. An
|
|
62
|
+
* adopter reported five such files arriving in a session labelled "project
|
|
63
|
+
* instructions" while `lint` did not mention them at all (#175.3).
|
|
64
|
+
*
|
|
65
|
+
* Optional and additive: a layout that omits it behaves exactly as before, so
|
|
66
|
+
* this adds a directory to the existing checks rather than a new check.
|
|
67
|
+
*/
|
|
68
|
+
readonly rulesDir?: string;
|
|
54
69
|
/** Dir the surfaces are materialized under, e.g. `.claude`. */
|
|
55
70
|
readonly materializeRoot: string;
|
|
56
71
|
/** Env token expanded to the plugin's absolute root in hook commands. */
|
package/dist/core/rule-meta.d.ts
CHANGED
|
@@ -41,7 +41,7 @@ export type RuleSurface = "instruction" | "skill" | "subagent" | "hook" | "mcp"
|
|
|
41
41
|
/** Where a rule sits by default — `"off"` is the normalized form of `false`. */
|
|
42
42
|
export type RuleDefaultSeverity = "error" | "warn" | "off";
|
|
43
43
|
/** Every named rule: the `RulesConfig` keys plus the built-in `orphan-docs`. */
|
|
44
|
-
export type RuleName = keyof RulesConfig
|
|
44
|
+
export type RuleName = keyof RulesConfig;
|
|
45
45
|
/**
|
|
46
46
|
* The declared shape of one rule — co-located metadata in the ESLint `meta`
|
|
47
47
|
* sense, gathered into one registry because vigiles shares detectors.
|
package/dist/core/rule-meta.js
CHANGED
|
@@ -262,6 +262,22 @@ exports.RULE_META = {
|
|
|
262
262
|
detector: "delegationTrifectaIssues",
|
|
263
263
|
},
|
|
264
264
|
// --- Docs hygiene ---------------------------------------------------------
|
|
265
|
+
"duplicate-rules": {
|
|
266
|
+
id: "duplicate-rules",
|
|
267
|
+
bucket: "heuristic-behavioral",
|
|
268
|
+
surface: ["instruction"],
|
|
269
|
+
defaultSeverity: "warn",
|
|
270
|
+
summary: "Near-duplicate rules within one spec (NCD similarity) — two rules saying the same thing.",
|
|
271
|
+
detector: "findDuplicateRules",
|
|
272
|
+
},
|
|
273
|
+
"spec-refs": {
|
|
274
|
+
id: "spec-refs",
|
|
275
|
+
bucket: "external-decidable",
|
|
276
|
+
surface: ["instruction"],
|
|
277
|
+
defaultSeverity: "error",
|
|
278
|
+
summary: "A compiled instruction file whose spec references a file/script that no longer exists.",
|
|
279
|
+
detector: "compileClaude",
|
|
280
|
+
},
|
|
265
281
|
"orphan-docs": {
|
|
266
282
|
id: "orphan-docs",
|
|
267
283
|
bucket: "heuristic-behavioral",
|
package/dist/core/types.d.ts
CHANGED
|
@@ -99,6 +99,36 @@ export interface TestCoverageConfig {
|
|
|
99
99
|
testExtension?: string;
|
|
100
100
|
}
|
|
101
101
|
export interface RulesConfig {
|
|
102
|
+
/**
|
|
103
|
+
* Opt-in: a doc in a configured dir (default `docs/`) that no other `.md`
|
|
104
|
+
* references. The `orphans` block turns the SCAN on; this turns the FINDING
|
|
105
|
+
* into a warning or an error.
|
|
106
|
+
*
|
|
107
|
+
* It lived outside this interface until 2026-09 and therefore could not be
|
|
108
|
+
* set at all: `"warn"` and `"off"` were both ignored and the check always
|
|
109
|
+
* exited 1, so a repo's only choice was an always-blocking check or deleting
|
|
110
|
+
* the `orphans` block (#181). Default: "error", matching the old behaviour.
|
|
111
|
+
*/
|
|
112
|
+
"orphan-docs"?: RuleSeverity;
|
|
113
|
+
/**
|
|
114
|
+
* Re-derive a compiled instruction file's references from its `.spec.ts` and
|
|
115
|
+
* report the dead ones.
|
|
116
|
+
*
|
|
117
|
+
* The integrity hash answers "is this file still what the spec compiled to";
|
|
118
|
+
* it says nothing about whether the paths and scripts it NAMES still exist. So
|
|
119
|
+
* an artifact committed while its refs were live stayed green forever after
|
|
120
|
+
* the target was deleted — `compile` errored, `lint` said "hash valid" and
|
|
121
|
+
* exited 0 (#173). Default: "error", matching `compile`.
|
|
122
|
+
*/
|
|
123
|
+
"spec-refs"?: RuleSeverity;
|
|
124
|
+
/**
|
|
125
|
+
* Near-duplicate rules WITHIN one spec, by NCD similarity — spec bloat, two
|
|
126
|
+
* rules saying the same thing in different words.
|
|
127
|
+
*
|
|
128
|
+
* Also previously untierable, and worse: it had no rule id at all, so there
|
|
129
|
+
* was no name a config could even mention (#181). Default: "error".
|
|
130
|
+
*/
|
|
131
|
+
"duplicate-rules"?: RuleSeverity;
|
|
102
132
|
/**
|
|
103
133
|
* Require a `.spec.ts` behind each instruction file (CLAUDE.md / AGENTS.md) —
|
|
104
134
|
* the file must be compiled from a typed spec, not hand-written. NARROW: only a
|
|
@@ -363,6 +393,20 @@ export interface VigilesConfig {
|
|
|
363
393
|
rulesDir?: string | string[];
|
|
364
394
|
}>;
|
|
365
395
|
/** Orphan-docs check configuration. Include/exclude globs, tsconfig-style. */
|
|
396
|
+
/**
|
|
397
|
+
* Which bundles `lint` scores: `"root"` (default) or `"all"`.
|
|
398
|
+
*
|
|
399
|
+
* A monorepo holding `skills/` plus `plugins/ * /skills/` had its nested skills
|
|
400
|
+
* silently uncounted — the counters looked complete while whole surfaces were
|
|
401
|
+
* never read (#185). `"all"` scores every discovered bundle in one pass, so a
|
|
402
|
+
* CI gate keeps ONE exit code over the whole repo.
|
|
403
|
+
*
|
|
404
|
+
* Root-only remains the default because descending unconditionally would score
|
|
405
|
+
* vendored third-party plugins (a repo may keep a pinned corpus on disk) as if
|
|
406
|
+
* they were the project's own. The default no longer hides the skip: `lint`
|
|
407
|
+
* names the bundles it did not score.
|
|
408
|
+
*/
|
|
409
|
+
bundles?: "root" | "all";
|
|
366
410
|
orphans?: OrphansConfig;
|
|
367
411
|
/**
|
|
368
412
|
* Glob patterns of instruction/skill files to EXCLUDE from `lint` discovery
|
package/dist/core/validate.js
CHANGED
|
@@ -32,6 +32,24 @@ const INSTRUCTION_FILES = ["CLAUDE.md", "AGENTS.md"];
|
|
|
32
32
|
// The default instruction file to validate when no config names one.
|
|
33
33
|
const DEFAULT_FILES = [INSTRUCTION_FILES[0]];
|
|
34
34
|
exports.DEFAULT_RULES = {
|
|
35
|
+
// 🔴 BOTH DROP TO "warn", and that is a deliberate behaviour change.
|
|
36
|
+
//
|
|
37
|
+
// They used to feed the exit code directly and could not be tiered at all
|
|
38
|
+
// (#181), so a single unreferenced doc turned a PR red with no way to say
|
|
39
|
+
// "report it, do not block". Naming them as rules made the contradiction
|
|
40
|
+
// visible: both are HEURISTIC-BEHAVIORAL (an NCD similarity proxy, an
|
|
41
|
+
// "unreferenced" guess that an OSS sweep measured at ~100% false positives on
|
|
42
|
+
// nav-managed doc sites), and this repo's own calibration rule is that a
|
|
43
|
+
// heuristic never defaults to `error` because it cries wolf. `orphan-docs`
|
|
44
|
+
// already DECLARED `warn` in its meta while behaving as `error` — the gate
|
|
45
|
+
// caught that disagreement the moment the rule was registered properly.
|
|
46
|
+
//
|
|
47
|
+
// Set either to `"error"` to keep the old blocking behaviour.
|
|
48
|
+
// Hard error, like `compile` itself: a dead reference is decidable from the
|
|
49
|
+
// filesystem, not a proxy — the calibration rule's `external-decidable` tier.
|
|
50
|
+
"spec-refs": "error",
|
|
51
|
+
"orphan-docs": "warn",
|
|
52
|
+
"duplicate-rules": "warn",
|
|
35
53
|
"require-instructions-spec": "warn",
|
|
36
54
|
// Default OFF — the consistent `require-<surface>-spec` parallel. Skills are
|
|
37
55
|
// legitimately hand-written, so requiring a .spec.ts per SKILL.md is the wrong
|
|
@@ -8,7 +8,9 @@
|
|
|
8
8
|
* conventions: one is about a process that ran, the other about a directory
|
|
9
9
|
* listing. See the module header.
|
|
10
10
|
*/
|
|
11
|
-
export type CoverageEvidence = "executed" | "colocated";
|
|
11
|
+
export type CoverageEvidence = "executed" | "colocated" | "configured";
|
|
12
|
+
/** Is `a` stronger evidence than `b`? */
|
|
13
|
+
export declare function strongerEvidence(a: CoverageEvidence, b: CoverageEvidence): boolean;
|
|
12
14
|
/** The minimum a surface must expose to be matched — structural, no import cycle. */
|
|
13
15
|
export interface CoverableSurface {
|
|
14
16
|
/** Repo-relative path of the surface file (SKILL.md / agent .md / hook script). */
|
|
@@ -100,12 +102,56 @@ export declare function hookScriptRefs(manifestText: string | undefined, layout:
|
|
|
100
102
|
* `colocated` is passed in because placement is a path question the two twins
|
|
101
103
|
* answer with their own (disk vs POSIX-string) path helpers.
|
|
102
104
|
*/
|
|
103
|
-
export declare function evidenceFor(_surface: CoverableSurface, _test: PreparedTest, colocated: boolean): CoverageEvidence | null;
|
|
105
|
+
export declare function evidenceFor(_surface: CoverableSurface, _test: PreparedTest, colocated: boolean, configured?: boolean): CoverageEvidence | null;
|
|
106
|
+
/**
|
|
107
|
+
* The `{surface}` placeholder in a user's `testGlobs` — the ONE thing that makes
|
|
108
|
+
* a centralized test layout expressible without weakening what coverage MEANS.
|
|
109
|
+
*
|
|
110
|
+
* The retired `declared` and `name-mentioned` tiers died because they could
|
|
111
|
+
* credit a surface no test touched: a mention is a substring, and a substring
|
|
112
|
+
* matched this file's own fixtures. `{surface}` cannot do that. The user writes
|
|
113
|
+
* `tests/{surface}/evals/promptfooconfig*.yaml`, and the placeholder is replaced
|
|
114
|
+
* with the surface's NAME before matching — so the binding between test and
|
|
115
|
+
* surface is still the name, exactly as under colocation. Only the PLACE moves.
|
|
116
|
+
*
|
|
117
|
+
* What it costs, stated plainly because it is the argument colocation was chosen
|
|
118
|
+
* on: `ls` beside the skill no longer answers "is this tested?" — you have to
|
|
119
|
+
* know where the project keeps its tests. That is a real loss, and it is why
|
|
120
|
+
* this is opt-in per repo rather than a second default. A project that has
|
|
121
|
+
* already centralized its suites has paid that cost anyway.
|
|
122
|
+
*/
|
|
123
|
+
export declare const SURFACE_TOKEN = "{surface}";
|
|
124
|
+
/** Does this glob delegate its surface binding to the placeholder? */
|
|
125
|
+
export declare function hasSurfaceToken(glob: string): boolean;
|
|
126
|
+
/**
|
|
127
|
+
* The pattern to DISCOVER files with: the placeholder widened to `*` so one
|
|
128
|
+
* glob pass finds every candidate. Narrowing back to the right surface happens
|
|
129
|
+
* at match time — discovery must stay surface-agnostic or it would be one glob
|
|
130
|
+
* pass per surface.
|
|
131
|
+
*/
|
|
132
|
+
export declare function discoveryGlob(glob: string): string;
|
|
133
|
+
/**
|
|
134
|
+
* Does this test file sit at a `{surface}` path configured FOR THIS SURFACE?
|
|
135
|
+
*
|
|
136
|
+
* The placeholder is replaced with the surface's own name, so
|
|
137
|
+
* `tests/{surface}/evals/*.yaml` credits `tests/mysql-designer/evals/x.yaml` to
|
|
138
|
+
* `mysql-designer` and to nothing else. A glob WITHOUT the placeholder returns
|
|
139
|
+
* false here on purpose: a plain custom glob widens what counts as a test file,
|
|
140
|
+
* which it always did, but it says nothing about WHICH surface the file is for
|
|
141
|
+
* — and inferring that from a substring is exactly the retired `name-mentioned`
|
|
142
|
+
* tier that credited surfaces nothing had touched.
|
|
143
|
+
*
|
|
144
|
+
* `minimatch` (already a direct dependency, pure JS) so the browser twin can
|
|
145
|
+
* share this instead of growing a second matcher that disagrees.
|
|
146
|
+
*/
|
|
147
|
+
export declare function matchesSurfaceGlob(surface: Pick<CoverableSurface, "name">, testPath: string, globs: readonly string[]): boolean;
|
|
104
148
|
/** Per-evidence tallies — the provenance summary the report prints. */
|
|
105
149
|
export interface EvidenceCounts {
|
|
106
150
|
/** Decided by a recorded run against this version of the surface. */
|
|
107
151
|
readonly executed: number;
|
|
108
152
|
readonly colocated: number;
|
|
153
|
+
/** Decided by a `{surface}` testGlob — the name still binds, the place moved. */
|
|
154
|
+
readonly configured: number;
|
|
109
155
|
}
|
|
110
156
|
/** Tally a list of decisions by evidence kind. */
|
|
111
157
|
export declare function countEvidence(decisions: readonly {
|
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SURFACE_TOKEN = void 0;
|
|
4
|
+
exports.strongerEvidence = strongerEvidence;
|
|
3
5
|
exports.isColocatedTest = isColocatedTest;
|
|
4
6
|
exports.prepareTest = prepareTest;
|
|
5
7
|
exports.isEvalScript = isEvalScript;
|
|
6
8
|
exports.hookScriptRefs = hookScriptRefs;
|
|
7
9
|
exports.evidenceFor = evidenceFor;
|
|
10
|
+
exports.hasSurfaceToken = hasSurfaceToken;
|
|
11
|
+
exports.discoveryGlob = discoveryGlob;
|
|
12
|
+
exports.matchesSurfaceGlob = matchesSurfaceGlob;
|
|
8
13
|
exports.countEvidence = countEvidence;
|
|
9
14
|
exports.formatEvidence = formatEvidence;
|
|
10
15
|
exports.declaredSurfaceName = declaredSurfaceName;
|
|
@@ -92,8 +97,31 @@ exports.declaredSurfaceName = declaredSurfaceName;
|
|
|
92
97
|
* impossible there and its count is 0 — see `test-coverage-files.ts`.
|
|
93
98
|
*/
|
|
94
99
|
const frontmatter_read_js_1 = require("./core/frontmatter-read.js");
|
|
100
|
+
const minimatch_1 = require("minimatch");
|
|
95
101
|
const posix_path_js_1 = require("./posix-path.js");
|
|
96
102
|
const source_refs_js_1 = require("./core/source-refs.js");
|
|
103
|
+
/**
|
|
104
|
+
* Rank, strongest first — what "the STRONGEST evidence" is measured against.
|
|
105
|
+
*
|
|
106
|
+
* It exists because `coverageOf` promised strongest-not-first-found while
|
|
107
|
+
* actually keeping the first match, which was harmless only while `colocated`
|
|
108
|
+
* was the sole surviving tier: one tier cannot be out-ranked. Adding
|
|
109
|
+
* `configured` made the promise load-bearing again — a surface with both a
|
|
110
|
+
* colocated harness and a configured suite would otherwise be reported as
|
|
111
|
+
* whichever the glob list happened to yield first, so the provenance summary
|
|
112
|
+
* would depend on the ORDER of a config array. Colocation ranks higher because
|
|
113
|
+
* it is the stronger statement: the filesystem enforces the binding rather than
|
|
114
|
+
* a pattern asserting it.
|
|
115
|
+
*/
|
|
116
|
+
const EVIDENCE_RANK = {
|
|
117
|
+
executed: 0,
|
|
118
|
+
colocated: 1,
|
|
119
|
+
configured: 2,
|
|
120
|
+
};
|
|
121
|
+
/** Is `a` stronger evidence than `b`? */
|
|
122
|
+
function strongerEvidence(a, b) {
|
|
123
|
+
return EVIDENCE_RANK[a] < EVIDENCE_RANK[b];
|
|
124
|
+
}
|
|
97
125
|
/**
|
|
98
126
|
* Is `testPath` the colocated test OF this surface — NAMED after it, SITTING
|
|
99
127
|
* BESIDE it?
|
|
@@ -252,20 +280,80 @@ function hookScriptRefs(manifestText, layout, exists) {
|
|
|
252
280
|
* `colocated` is passed in because placement is a path question the two twins
|
|
253
281
|
* answer with their own (disk vs POSIX-string) path helpers.
|
|
254
282
|
*/
|
|
255
|
-
function evidenceFor(_surface, _test, colocated) {
|
|
256
|
-
|
|
283
|
+
function evidenceFor(_surface, _test, colocated, configured = false) {
|
|
284
|
+
if (colocated)
|
|
285
|
+
return "colocated";
|
|
286
|
+
return configured ? "configured" : null;
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* The `{surface}` placeholder in a user's `testGlobs` — the ONE thing that makes
|
|
290
|
+
* a centralized test layout expressible without weakening what coverage MEANS.
|
|
291
|
+
*
|
|
292
|
+
* The retired `declared` and `name-mentioned` tiers died because they could
|
|
293
|
+
* credit a surface no test touched: a mention is a substring, and a substring
|
|
294
|
+
* matched this file's own fixtures. `{surface}` cannot do that. The user writes
|
|
295
|
+
* `tests/{surface}/evals/promptfooconfig*.yaml`, and the placeholder is replaced
|
|
296
|
+
* with the surface's NAME before matching — so the binding between test and
|
|
297
|
+
* surface is still the name, exactly as under colocation. Only the PLACE moves.
|
|
298
|
+
*
|
|
299
|
+
* What it costs, stated plainly because it is the argument colocation was chosen
|
|
300
|
+
* on: `ls` beside the skill no longer answers "is this tested?" — you have to
|
|
301
|
+
* know where the project keeps its tests. That is a real loss, and it is why
|
|
302
|
+
* this is opt-in per repo rather than a second default. A project that has
|
|
303
|
+
* already centralized its suites has paid that cost anyway.
|
|
304
|
+
*/
|
|
305
|
+
exports.SURFACE_TOKEN = "{surface}";
|
|
306
|
+
/** Does this glob delegate its surface binding to the placeholder? */
|
|
307
|
+
function hasSurfaceToken(glob) {
|
|
308
|
+
return glob.includes(exports.SURFACE_TOKEN);
|
|
309
|
+
}
|
|
310
|
+
/**
|
|
311
|
+
* The pattern to DISCOVER files with: the placeholder widened to `*` so one
|
|
312
|
+
* glob pass finds every candidate. Narrowing back to the right surface happens
|
|
313
|
+
* at match time — discovery must stay surface-agnostic or it would be one glob
|
|
314
|
+
* pass per surface.
|
|
315
|
+
*/
|
|
316
|
+
function discoveryGlob(glob) {
|
|
317
|
+
return glob.split(exports.SURFACE_TOKEN).join("*");
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* Does this test file sit at a `{surface}` path configured FOR THIS SURFACE?
|
|
321
|
+
*
|
|
322
|
+
* The placeholder is replaced with the surface's own name, so
|
|
323
|
+
* `tests/{surface}/evals/*.yaml` credits `tests/mysql-designer/evals/x.yaml` to
|
|
324
|
+
* `mysql-designer` and to nothing else. A glob WITHOUT the placeholder returns
|
|
325
|
+
* false here on purpose: a plain custom glob widens what counts as a test file,
|
|
326
|
+
* which it always did, but it says nothing about WHICH surface the file is for
|
|
327
|
+
* — and inferring that from a substring is exactly the retired `name-mentioned`
|
|
328
|
+
* tier that credited surfaces nothing had touched.
|
|
329
|
+
*
|
|
330
|
+
* `minimatch` (already a direct dependency, pure JS) so the browser twin can
|
|
331
|
+
* share this instead of growing a second matcher that disagrees.
|
|
332
|
+
*/
|
|
333
|
+
function matchesSurfaceGlob(surface, testPath, globs) {
|
|
334
|
+
const test = posixly(testPath);
|
|
335
|
+
return globs.some((g) => {
|
|
336
|
+
if (!hasSurfaceToken(g))
|
|
337
|
+
return false;
|
|
338
|
+
return (0, minimatch_1.minimatch)(test, g.split(exports.SURFACE_TOKEN).join(surface.name), {
|
|
339
|
+
dot: true,
|
|
340
|
+
});
|
|
341
|
+
});
|
|
257
342
|
}
|
|
258
343
|
/** Tally a list of decisions by evidence kind. */
|
|
259
344
|
function countEvidence(decisions) {
|
|
260
345
|
let executed = 0;
|
|
261
346
|
let colocated = 0;
|
|
347
|
+
let configured = 0;
|
|
262
348
|
for (const d of decisions) {
|
|
263
349
|
if (d.evidence === "executed")
|
|
264
350
|
executed += 1;
|
|
351
|
+
else if (d.evidence === "configured")
|
|
352
|
+
configured += 1;
|
|
265
353
|
else
|
|
266
354
|
colocated += 1;
|
|
267
355
|
}
|
|
268
|
-
return { executed, colocated };
|
|
356
|
+
return { executed, colocated, configured };
|
|
269
357
|
}
|
|
270
358
|
/**
|
|
271
359
|
* One line naming how the coverage was established. Printed wherever a coverage
|
|
@@ -287,6 +375,11 @@ function formatEvidence(counts) {
|
|
|
287
375
|
parts.push(`${String(counts.colocated)} colocated — a test NAMED after the surface, ` +
|
|
288
376
|
`in the surface's own place. This says the file EXISTS, not that it ran`);
|
|
289
377
|
}
|
|
378
|
+
if (counts.configured > 0) {
|
|
379
|
+
parts.push(`${String(counts.configured)} configured — a test NAMED after the surface ` +
|
|
380
|
+
`at a \`{surface}\` path you configured. Same name binding as colocation, ` +
|
|
381
|
+
`different place; still only says the file EXISTS`);
|
|
382
|
+
}
|
|
290
383
|
if (parts.length === 0)
|
|
291
384
|
return "";
|
|
292
385
|
return `How coverage was decided: ${parts.join("; ")}.`;
|
package/dist/eval.d.ts
CHANGED
|
@@ -724,6 +724,16 @@ export interface TriggerRateReport {
|
|
|
724
724
|
* description). A non-zero count is the whole-harness measurement.
|
|
725
725
|
*/
|
|
726
726
|
readonly competitors: number;
|
|
727
|
+
/**
|
|
728
|
+
* The plugin namespace the skills actually installed under — the `<plugin>`
|
|
729
|
+
* half of the `<plugin>:<skill>` id `skillResolved` matches.
|
|
730
|
+
*
|
|
731
|
+
* Reported because with `skillsDir` the name is chosen by the packager, not by
|
|
732
|
+
* the caller, so the single most common cause of a 0% run was a value the
|
|
733
|
+
* caller had no way to know. Optional so a report recorded before this field
|
|
734
|
+
* still parses.
|
|
735
|
+
*/
|
|
736
|
+
readonly namespace?: string;
|
|
727
737
|
/**
|
|
728
738
|
* Runs EXCLUDED because the turn errored / was rate-limited (detected by the
|
|
729
739
|
* driver's `runError`), present only when > 0. These are NOT counted in `n` or
|
package/dist/eval.js
CHANGED
|
@@ -1577,6 +1577,7 @@ function resolveTriggerPluginDir(spec) {
|
|
|
1577
1577
|
pluginDir,
|
|
1578
1578
|
packaged,
|
|
1579
1579
|
competitors: Math.max(0, countSkills(pluginDir) - 1),
|
|
1580
|
+
namespace: underTestSource(spec).name,
|
|
1580
1581
|
};
|
|
1581
1582
|
}
|
|
1582
1583
|
/** Run one prompt set × trials through `runner`, aggregating fired counts. */
|
|
@@ -1683,7 +1684,7 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
|
|
|
1683
1684
|
throw new Error(`measureTriggerRate: model "${model}" is below the minimum "${minModel}" — ` +
|
|
1684
1685
|
"trigger-rate under-measures selection on a weaker model " +
|
|
1685
1686
|
"(raise the model, or lower `minModel` for a deliberately cheap run).");
|
|
1686
|
-
const { pluginDir, packaged, competitors } = resolveTriggerPluginDir(spec);
|
|
1687
|
+
const { pluginDir, packaged, competitors, namespace } = resolveTriggerPluginDir(spec);
|
|
1687
1688
|
const cfg = {
|
|
1688
1689
|
trials: spec.trials ?? 1,
|
|
1689
1690
|
// Sonnet, not haiku: trigger-rate is a selection measurement and haiku
|
|
@@ -1729,6 +1730,7 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
|
|
|
1729
1730
|
n: relevant.n,
|
|
1730
1731
|
perPrompt: relevant.perPrompt,
|
|
1731
1732
|
competitors,
|
|
1733
|
+
namespace,
|
|
1732
1734
|
errored: positiveOrUndefined(relevant.errored),
|
|
1733
1735
|
usage: aggregateUsage(relevant.usages),
|
|
1734
1736
|
};
|
|
@@ -1833,8 +1835,17 @@ function formatTriggerRateReport(report) {
|
|
|
1833
1835
|
// must not be second-guessed; a checker that hedges on good data gets ignored.
|
|
1834
1836
|
if (report.n > 0 && report.rate === 0)
|
|
1835
1837
|
lines.push("⚠ nothing fired on ANY prompt. That is usually SETUP, not the description — check, in order:\n" +
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
+
// The runtime RESOLVED the namespace before spending a token, so it
|
|
1839
|
+
// prints the id that should have matched instead of telling the reader
|
|
1840
|
+
// to go work it out. With `skillsDir` the name is not even the user's
|
|
1841
|
+
// choice — the packager picks it — so "check the id" was advice about a
|
|
1842
|
+
// value they had never seen.
|
|
1843
|
+
(report.namespace !== undefined
|
|
1844
|
+
? " 1. the id in `fired` — your skills installed under " +
|
|
1845
|
+
`\`${report.namespace}\`, so \`skillResolved\` matches ` +
|
|
1846
|
+
`\`${report.namespace}:<skill>\`; a bare name silently never matches;\n`
|
|
1847
|
+
: " 1. the id in `fired` — `skillResolved` matches the NAMESPACED id " +
|
|
1848
|
+
"(`<plugin>:<skill>`); a bare name silently never matches;\n") +
|
|
1838
1849
|
" 2. the install field — a loose `.claude/skills` dir needs `skillsDir`, " +
|
|
1839
1850
|
"not `pluginDir` (which wants a full plugin manifest);\n" +
|
|
1840
1851
|
" 3. the `fixture` — a run starts in an EMPTY cwd, so a prompt about a " +
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
type ResolveContext = {
|
|
2
|
+
parentURL?: string;
|
|
3
|
+
conditions: string[];
|
|
4
|
+
};
|
|
5
|
+
type Resolved = {
|
|
6
|
+
url: string;
|
|
7
|
+
format?: string | null;
|
|
8
|
+
shortCircuit?: boolean;
|
|
9
|
+
};
|
|
10
|
+
type NextResolve = (specifier: string, context: ResolveContext) => Resolved | Promise<Resolved>;
|
|
11
|
+
export declare function resolve(specifier: string, context: ResolveContext, nextResolve: NextResolve): Promise<Resolved>;
|
|
12
|
+
export {};
|
|
13
|
+
//# sourceMappingURL=harness-resolve-hooks.d.mts.map
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Module-resolution hook for HARNESS scripts: make a bare `vigiles` import
|
|
3
|
+
* resolve to the CLI's OWN installation.
|
|
4
|
+
*
|
|
5
|
+
* 🔴 WHY. A harness file does `import { runHook } from "vigiles"`, so the
|
|
6
|
+
* package has to sit in a `node_modules` Node can reach from that file. In a
|
|
7
|
+
* repo that already has a `package.json`, the obvious way to put it there is
|
|
8
|
+
* `npm install` in the root — which installs the whole dependency tree. Measured
|
|
9
|
+
* by an adopter (#184): **840 packages in 2 minutes** where vigiles alone is 42
|
|
10
|
+
* and about 90 MB; the other 798 were a model-eval framework and an agent SDK
|
|
11
|
+
* that the gate — `lint` and `test`, both deterministic reads — never touches.
|
|
12
|
+
* One run sat 11 minutes in that step before being cancelled. Their workaround
|
|
13
|
+
* was installing into a directory outside the workspace and symlinking the tree
|
|
14
|
+
* back in, which works and is not something every adopter should reinvent.
|
|
15
|
+
*
|
|
16
|
+
* ⚠️ `NODE_PATH` does NOT solve this, and that was measured rather than assumed:
|
|
17
|
+
* Node ignores it for ESM resolution, and a harness is ESM. So the only ways to
|
|
18
|
+
* resolve a bare specifier from elsewhere are a real `node_modules` entry (the
|
|
19
|
+
* symlink) or a resolver hook. This is the hook.
|
|
20
|
+
*
|
|
21
|
+
* Scope is deliberately narrow: ONLY the `vigiles` specifier and its subpaths,
|
|
22
|
+
* and only when the normal resolution fails. A harness that has vigiles
|
|
23
|
+
* installed locally keeps resolving to the local copy, so nothing changes for a
|
|
24
|
+
* repo that already worked — this only fills the hole where resolution would
|
|
25
|
+
* otherwise throw.
|
|
26
|
+
*/
|
|
27
|
+
import { createRequire } from "node:module";
|
|
28
|
+
import { pathToFileURL } from "node:url";
|
|
29
|
+
/** The CLI's own package root, handed in by the parent process. */
|
|
30
|
+
const SELF = process.env.VIGILES_SELF_ROOT ?? "";
|
|
31
|
+
export async function resolve(specifier, context, nextResolve) {
|
|
32
|
+
try {
|
|
33
|
+
return await nextResolve(specifier, context);
|
|
34
|
+
}
|
|
35
|
+
catch (err) {
|
|
36
|
+
// Only rescue OUR specifier, and only after normal resolution failed, so a
|
|
37
|
+
// locally installed vigiles always wins and no other package is affected.
|
|
38
|
+
if (!SELF)
|
|
39
|
+
throw err;
|
|
40
|
+
if (specifier !== "vigiles" && !specifier.startsWith("vigiles/"))
|
|
41
|
+
throw err;
|
|
42
|
+
const require = createRequire(pathToFileURL(`${SELF}/package.json`));
|
|
43
|
+
// Resolve through the package's own `exports` map rather than guessing a
|
|
44
|
+
// file path, so a subpath like `vigiles/eval` obeys the same contract it
|
|
45
|
+
// would from a normal install.
|
|
46
|
+
const target = require.resolve(specifier, { paths: [SELF] });
|
|
47
|
+
return { url: pathToFileURL(target).href, shortCircuit: true };
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=harness-resolve-hooks.mjs.map
|
package/dist/plugin-loader.js
CHANGED
|
@@ -254,6 +254,40 @@ function materializeSurfaces(root, layout, files, sources) {
|
|
|
254
254
|
counts[surface] = (counts[surface] ?? 0) + Object.keys(tree).length;
|
|
255
255
|
}
|
|
256
256
|
};
|
|
257
|
+
/**
|
|
258
|
+
* Read the path-scoped RULES dir (`.claude/rules/*.md` on Claude Code).
|
|
259
|
+
*
|
|
260
|
+
* DELIBERATELY NOT a `surfaceDirs` entry, and the distinction is the whole
|
|
261
|
+
* design: `surfaceDirs` decides whether a directory counts as a LOADABLE
|
|
262
|
+
* MACHINE, and rules are instructions, not an invocable surface — folding them
|
|
263
|
+
* in would silently change what "an empty machine" means for every harness.
|
|
264
|
+
* So they are read here, added to `files` for the checks that read text
|
|
265
|
+
* (frontmatter-valid, the rule map), and left out of `counts` and
|
|
266
|
+
* `hasLoadable`. A layout with no `rulesDir` reads nothing and behaves exactly
|
|
267
|
+
* as before. Closes #175.3: a whole instruction layer the audit could not see.
|
|
268
|
+
*/
|
|
269
|
+
const materializeRules = () => {
|
|
270
|
+
const dir = layout.rulesDir;
|
|
271
|
+
if (!dir)
|
|
272
|
+
return;
|
|
273
|
+
// Read BOTH candidate bases, and NOT the resolved `scopes`. Rules are not
|
|
274
|
+
// tied to where the invocable surfaces live: a repo can keep its skills at
|
|
275
|
+
// the root (a published plugin) while its rules sit under `.claude/`, and
|
|
276
|
+
// keying off scopes then read the wrong directory and found nothing —
|
|
277
|
+
// measured on exactly that shape while building this.
|
|
278
|
+
//
|
|
279
|
+
// Each base keys at its own real path (`rules/…` and `.claude/rules/…`), so
|
|
280
|
+
// a repo with both loses neither and nothing collides.
|
|
281
|
+
const bases = [
|
|
282
|
+
"",
|
|
283
|
+
...(layout.userSurfaceRoot !== undefined ? [layout.userSurfaceRoot] : []),
|
|
284
|
+
];
|
|
285
|
+
for (const base of bases) {
|
|
286
|
+
const tree = surfaceTree((0, node_path_1.join)(root, base, dir));
|
|
287
|
+
for (const [rel, content] of Object.entries(tree))
|
|
288
|
+
add((0, node_path_1.join)(base, dir, rel), content, (0, node_path_1.join)(root, base, dir, rel));
|
|
289
|
+
}
|
|
290
|
+
};
|
|
257
291
|
const source = (0, surface_scopes_js_1.surfaceSource)(layout, {
|
|
258
292
|
hasRootSkillFile: (0, node_fs_1.existsSync)((0, node_path_1.join)(root, "SKILL.md")),
|
|
259
293
|
skillName: (0, node_path_1.basename)(root),
|
|
@@ -280,6 +314,7 @@ function materializeSurfaces(root, layout, files, sources) {
|
|
|
280
314
|
(0, surface_scopes_js_1.assertDistinctScopeKeys)(source.scopes, layout.name);
|
|
281
315
|
for (const scope of source.scopes)
|
|
282
316
|
materializeScope(scope, scope.base === "" ? rootTrees : userTrees);
|
|
317
|
+
materializeRules();
|
|
283
318
|
return { counts, scopes: source.scopes };
|
|
284
319
|
}
|
|
285
320
|
/* v8 ignore next 2 -- exhaustiveness guard, unreachable given SurfaceSource */
|
package/dist/run-hook.d.ts
CHANGED
|
@@ -163,10 +163,30 @@ export interface HookRunResult extends ScriptRunResult {
|
|
|
163
163
|
/** Parsed stdout JSON if the hook emitted a JSON decision, else null. */
|
|
164
164
|
readonly json: HookOutput | null;
|
|
165
165
|
/**
|
|
166
|
-
* Normalized decision:
|
|
167
|
-
* `permissionDecision:"deny"
|
|
166
|
+
* Normalized decision: the dangerous call did NOT go through. Set by exit 2,
|
|
167
|
+
* `decision:"block"`, `permissionDecision:"deny"`, or the harness's
|
|
168
|
+
* halt-the-turn field (Claude Code `{"continue": false}` — see
|
|
169
|
+
* {@link HookRunResult.haltsTurn}).
|
|
170
|
+
*
|
|
171
|
+
* The halt case was missing until 2026-08-31 (#174), and the shape of that bug
|
|
172
|
+
* is worth keeping written down: `verifyGuardrail` reads this field, so a real
|
|
173
|
+
* `PreToolUse` guard that stopped every one of the disaster battery's commands
|
|
174
|
+
* was reported by `assertBlocksDisasters` as blocking NONE of them. A tool
|
|
175
|
+
* whose stated job is catching a guard that looks fine and silently does
|
|
176
|
+
* nothing said the opposite about a working guard — the same false-confidence
|
|
177
|
+
* failure, with the sign flipped.
|
|
168
178
|
*/
|
|
169
179
|
readonly blocked: boolean;
|
|
180
|
+
/**
|
|
181
|
+
* The hook halted the WHOLE TURN rather than denying one call — the harness's
|
|
182
|
+
* `haltsTurnField` came back `false`.
|
|
183
|
+
*
|
|
184
|
+
* Reported separately because it is strictly stronger than a deny and the two
|
|
185
|
+
* are worth telling apart when a test asks WHICH mechanism fired. `blocked`
|
|
186
|
+
* stays the question nearly every caller means ("did the action happen?"), so
|
|
187
|
+
* a halt sets both.
|
|
188
|
+
*/
|
|
189
|
+
readonly haltsTurn: boolean;
|
|
170
190
|
/**
|
|
171
191
|
* The decision the hook expressed, preferring the structured
|
|
172
192
|
* `permissionDecision` ("allow"|"deny"|"ask") then legacy `decision`
|
|
@@ -183,6 +203,7 @@ export declare function parseHookOutput(stdout: string): HookOutput | null;
|
|
|
183
203
|
export declare function decideHook(exitCode: number, json: HookOutput | null, protocol?: HookProtocol): {
|
|
184
204
|
blocked: boolean;
|
|
185
205
|
decision: HookRunResult["decision"];
|
|
206
|
+
haltsTurn: boolean;
|
|
186
207
|
};
|
|
187
208
|
/**
|
|
188
209
|
* The hook layer over {@link runScriptWith}: serialize the event to stdin, run
|
package/dist/run-hook.js
CHANGED
|
@@ -130,9 +130,16 @@ function parseHookOutput(stdout) {
|
|
|
130
130
|
function decideHook(exitCode, json, protocol = hook_protocol_js_1.claudeCodeHookProtocol) {
|
|
131
131
|
const permission = json?.hookSpecificOutput?.permissionDecision;
|
|
132
132
|
const decision = permission ?? json?.decision;
|
|
133
|
+
// The halt field is read from the PORT, never hard-coded: `"continue"` is a
|
|
134
|
+
// documented Claude Code fact and an unverified one for Codex, so the harness
|
|
135
|
+
// that has it declares it (core ⊄ adapter). `=== false` and not falsy —
|
|
136
|
+
// an absent field must not read as a halt.
|
|
137
|
+
const haltField = protocol.haltsTurnField;
|
|
138
|
+
const haltsTurn = haltField !== undefined && json?.[haltField] === false;
|
|
133
139
|
const blocked = exitCode === protocol.blockExitCode ||
|
|
140
|
+
haltsTurn ||
|
|
134
141
|
(decision !== undefined && protocol.denyDecisionValues.includes(decision));
|
|
135
|
-
return { blocked, decision };
|
|
142
|
+
return { blocked, decision, haltsTurn };
|
|
136
143
|
}
|
|
137
144
|
/**
|
|
138
145
|
* The hook layer over {@link runScriptWith}: serialize the event to stdin, run
|
|
@@ -143,8 +150,8 @@ function decideHook(exitCode, json, protocol = hook_protocol_js_1.claudeCodeHook
|
|
|
143
150
|
function runHookWith(command, input, opts, deps) {
|
|
144
151
|
const res = (0, run_script_js_1.runScriptWith)(command, JSON.stringify(input), opts, deps);
|
|
145
152
|
const json = parseHookOutput(res.stdout);
|
|
146
|
-
const { blocked, decision } = decideHook(res.exitCode, json);
|
|
147
|
-
return { ...res, json, blocked, decision };
|
|
153
|
+
const { blocked, decision, haltsTurn } = decideHook(res.exitCode, json);
|
|
154
|
+
return { ...res, json, blocked, decision, haltsTurn };
|
|
148
155
|
}
|
|
149
156
|
/**
|
|
150
157
|
* Run a hook command, piping `input` as JSON to its stdin, and report the exit
|
package/dist/scan-core.d.ts
CHANGED
|
@@ -30,6 +30,11 @@ export interface SurfaceClassifier {
|
|
|
30
30
|
* one. Null for a path this classifier does not call an agent.
|
|
31
31
|
*/
|
|
32
32
|
readonly agentName: (f: string) => string | null;
|
|
33
|
+
/**
|
|
34
|
+
* A path-scoped RULES file (`<rulesDir>/<name>.md`) — an INSTRUCTION surface,
|
|
35
|
+
* not an invocable one. Always false for a layout with no `rulesDir`.
|
|
36
|
+
*/
|
|
37
|
+
readonly isRule: (f: string) => boolean;
|
|
33
38
|
}
|
|
34
39
|
export declare function makeClassifier(layout: PluginLayout): SurfaceClassifier;
|
|
35
40
|
/** The plugin-root + materialize-root + dialect context skill scanning needs. */
|
package/dist/scan-core.js
CHANGED
|
@@ -146,9 +146,13 @@ function makeClassifier(layout) {
|
|
|
146
146
|
const skill = at(layout.skillDir);
|
|
147
147
|
const agent = at(layout.agentDir);
|
|
148
148
|
const command = at(layout.commandDir);
|
|
149
|
+
const rules = at(layout.rulesDir ?? "");
|
|
149
150
|
const skillRe = skill ? new RegExp(`${skill}[^/]+/SKILL\\.md$`) : null;
|
|
150
151
|
const agentRe = agent ? new RegExp(`${agent}${layout_js_1.AGENT_FILE_LEAF_RE}$`) : null;
|
|
151
152
|
const commandRe = command ? new RegExp(`${command}.+\\.md$`) : null;
|
|
153
|
+
// Flat `<rulesDir>/<name>.md`, like commands. A layout without a rules dir
|
|
154
|
+
// yields null and every path below answers false — the additive default.
|
|
155
|
+
const ruleRe = rules ? new RegExp(`${rules}[^/]+\\.md$`) : null;
|
|
152
156
|
// A subagent lives under the plugin's `agents/` dir AT ANY DEPTH (the harness
|
|
153
157
|
// reads it recursively — see AGENT_FILE_LEAF_RE for the vendor's wording and
|
|
154
158
|
// the measurement), but never under ANOTHER surface dir. Two real-world
|
|
@@ -191,6 +195,7 @@ function makeClassifier(layout) {
|
|
|
191
195
|
isSkill,
|
|
192
196
|
isAgent,
|
|
193
197
|
isCommand: (f) => commandRe?.test(f) ?? false,
|
|
198
|
+
isRule: (f) => ruleRe?.test(f) ?? false,
|
|
194
199
|
agentName: (f) => isAgent(f) ? (0, layout_js_1.agentSurfaceName)(f, layout.agentDir) : null,
|
|
195
200
|
};
|
|
196
201
|
}
|
|
@@ -746,7 +751,11 @@ function frontmatterValueIssuesFor(files, cls) {
|
|
|
746
751
|
function malformedFrontmatterFor(files, cls) {
|
|
747
752
|
const out = [];
|
|
748
753
|
for (const [path, md] of Object.entries(files)) {
|
|
749
|
-
|
|
754
|
+
// Rules join skills + agents here: `.claude/rules/*.md` carries a `paths:`
|
|
755
|
+
// frontmatter key that SCOPES the instruction, so unparseable YAML there
|
|
756
|
+
// silently changes which files the rule applies to — the same defect this
|
|
757
|
+
// check exists for, on a surface no layout named until now (#175.3).
|
|
758
|
+
if (!cls.isSkill(path) && !cls.isAgent(path) && !cls.isRule(path))
|
|
750
759
|
continue;
|
|
751
760
|
if (!(0, frontmatter_read_js_1.readFrontmatter)(md).malformed)
|
|
752
761
|
continue;
|