vigiles 21.0.2 → 23.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/adapters/claude-code/hook-protocol.js +3 -0
- package/dist/adapters/claude-code/layout.js +2 -0
- package/dist/cli-flag-check.js +1 -0
- package/dist/cli.d.ts +2 -1
- package/dist/cli.js +229 -45
- package/dist/core/hook-block-ineffective.js +24 -1
- package/dist/core/hook-protocol.d.ts +14 -0
- package/dist/core/layout.d.ts +15 -0
- package/dist/coverage-evidence.d.ts +48 -2
- package/dist/coverage-evidence.js +96 -3
- package/dist/eval.d.ts +10 -0
- package/dist/eval.js +14 -3
- package/dist/plugin-loader.js +35 -0
- package/dist/run-hook.d.ts +23 -2
- package/dist/run-hook.js +10 -3
- package/dist/scan-core.d.ts +5 -0
- package/dist/scan-core.js +10 -1
- package/dist/spec-hooks.d.mts +35 -0
- package/dist/spec-hooks.mjs +68 -0
- package/dist/spec-host.d.mts +2 -0
- package/dist/spec-host.mjs +79 -0
- package/dist/test-coverage-files.js +17 -8
- package/dist/test-coverage.js +29 -16
- package/package.json +1 -1
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SURFACE_TOKEN = void 0;
|
|
4
|
+
exports.strongerEvidence = strongerEvidence;
|
|
3
5
|
exports.isColocatedTest = isColocatedTest;
|
|
4
6
|
exports.prepareTest = prepareTest;
|
|
5
7
|
exports.isEvalScript = isEvalScript;
|
|
6
8
|
exports.hookScriptRefs = hookScriptRefs;
|
|
7
9
|
exports.evidenceFor = evidenceFor;
|
|
10
|
+
exports.hasSurfaceToken = hasSurfaceToken;
|
|
11
|
+
exports.discoveryGlob = discoveryGlob;
|
|
12
|
+
exports.matchesSurfaceGlob = matchesSurfaceGlob;
|
|
8
13
|
exports.countEvidence = countEvidence;
|
|
9
14
|
exports.formatEvidence = formatEvidence;
|
|
10
15
|
exports.declaredSurfaceName = declaredSurfaceName;
|
|
@@ -92,8 +97,31 @@ exports.declaredSurfaceName = declaredSurfaceName;
|
|
|
92
97
|
* impossible there and its count is 0 — see `test-coverage-files.ts`.
|
|
93
98
|
*/
|
|
94
99
|
const frontmatter_read_js_1 = require("./core/frontmatter-read.js");
|
|
100
|
+
const minimatch_1 = require("minimatch");
|
|
95
101
|
const posix_path_js_1 = require("./posix-path.js");
|
|
96
102
|
const source_refs_js_1 = require("./core/source-refs.js");
|
|
103
|
+
/**
|
|
104
|
+
* Rank, strongest first — what "the STRONGEST evidence" is measured against.
|
|
105
|
+
*
|
|
106
|
+
* It exists because `coverageOf` promised strongest-not-first-found while
|
|
107
|
+
* actually keeping the first match, which was harmless only while `colocated`
|
|
108
|
+
* was the sole surviving tier: one tier cannot be out-ranked. Adding
|
|
109
|
+
* `configured` made the promise load-bearing again — a surface with both a
|
|
110
|
+
* colocated harness and a configured suite would otherwise be reported as
|
|
111
|
+
* whichever the glob list happened to yield first, so the provenance summary
|
|
112
|
+
* would depend on the ORDER of a config array. Colocation ranks higher because
|
|
113
|
+
* it is the stronger statement: the filesystem enforces the binding rather than
|
|
114
|
+
* a pattern asserting it.
|
|
115
|
+
*/
|
|
116
|
+
const EVIDENCE_RANK = {
|
|
117
|
+
executed: 0,
|
|
118
|
+
colocated: 1,
|
|
119
|
+
configured: 2,
|
|
120
|
+
};
|
|
121
|
+
/** Is `a` stronger evidence than `b`? */
|
|
122
|
+
function strongerEvidence(a, b) {
|
|
123
|
+
return EVIDENCE_RANK[a] < EVIDENCE_RANK[b];
|
|
124
|
+
}
|
|
97
125
|
/**
|
|
98
126
|
* Is `testPath` the colocated test OF this surface — NAMED after it, SITTING
|
|
99
127
|
* BESIDE it?
|
|
@@ -252,20 +280,80 @@ function hookScriptRefs(manifestText, layout, exists) {
|
|
|
252
280
|
* `colocated` is passed in because placement is a path question the two twins
|
|
253
281
|
* answer with their own (disk vs POSIX-string) path helpers.
|
|
254
282
|
*/
|
|
255
|
-
function evidenceFor(_surface, _test, colocated) {
|
|
256
|
-
|
|
283
|
+
function evidenceFor(_surface, _test, colocated, configured = false) {
|
|
284
|
+
if (colocated)
|
|
285
|
+
return "colocated";
|
|
286
|
+
return configured ? "configured" : null;
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* The `{surface}` placeholder in a user's `testGlobs` — the ONE thing that makes
|
|
290
|
+
* a centralized test layout expressible without weakening what coverage MEANS.
|
|
291
|
+
*
|
|
292
|
+
* The retired `declared` and `name-mentioned` tiers died because they could
|
|
293
|
+
* credit a surface no test touched: a mention is a substring, and a substring
|
|
294
|
+
* matched this file's own fixtures. `{surface}` cannot do that. The user writes
|
|
295
|
+
* `tests/{surface}/evals/promptfooconfig*.yaml`, and the placeholder is replaced
|
|
296
|
+
* with the surface's NAME before matching — so the binding between test and
|
|
297
|
+
* surface is still the name, exactly as under colocation. Only the PLACE moves.
|
|
298
|
+
*
|
|
299
|
+
* What it costs, stated plainly because it is the argument colocation was chosen
|
|
300
|
+
* on: `ls` beside the skill no longer answers "is this tested?" — you have to
|
|
301
|
+
* know where the project keeps its tests. That is a real loss, and it is why
|
|
302
|
+
* this is opt-in per repo rather than a second default. A project that has
|
|
303
|
+
* already centralized its suites has paid that cost anyway.
|
|
304
|
+
*/
|
|
305
|
+
exports.SURFACE_TOKEN = "{surface}";
|
|
306
|
+
/** Does this glob delegate its surface binding to the placeholder? */
|
|
307
|
+
function hasSurfaceToken(glob) {
|
|
308
|
+
return glob.includes(exports.SURFACE_TOKEN);
|
|
309
|
+
}
|
|
310
|
+
/**
|
|
311
|
+
* The pattern to DISCOVER files with: the placeholder widened to `*` so one
|
|
312
|
+
* glob pass finds every candidate. Narrowing back to the right surface happens
|
|
313
|
+
* at match time — discovery must stay surface-agnostic or it would be one glob
|
|
314
|
+
* pass per surface.
|
|
315
|
+
*/
|
|
316
|
+
function discoveryGlob(glob) {
|
|
317
|
+
return glob.split(exports.SURFACE_TOKEN).join("*");
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* Does this test file sit at a `{surface}` path configured FOR THIS SURFACE?
|
|
321
|
+
*
|
|
322
|
+
* The placeholder is replaced with the surface's own name, so
|
|
323
|
+
* `tests/{surface}/evals/*.yaml` credits `tests/mysql-designer/evals/x.yaml` to
|
|
324
|
+
* `mysql-designer` and to nothing else. A glob WITHOUT the placeholder returns
|
|
325
|
+
* false here on purpose: a plain custom glob widens what counts as a test file,
|
|
326
|
+
* which it always did, but it says nothing about WHICH surface the file is for
|
|
327
|
+
* — and inferring that from a substring is exactly the retired `name-mentioned`
|
|
328
|
+
* tier that credited surfaces nothing had touched.
|
|
329
|
+
*
|
|
330
|
+
* `minimatch` (already a direct dependency, pure JS) so the browser twin can
|
|
331
|
+
* share this instead of growing a second matcher that disagrees.
|
|
332
|
+
*/
|
|
333
|
+
function matchesSurfaceGlob(surface, testPath, globs) {
|
|
334
|
+
const test = posixly(testPath);
|
|
335
|
+
return globs.some((g) => {
|
|
336
|
+
if (!hasSurfaceToken(g))
|
|
337
|
+
return false;
|
|
338
|
+
return (0, minimatch_1.minimatch)(test, g.split(exports.SURFACE_TOKEN).join(surface.name), {
|
|
339
|
+
dot: true,
|
|
340
|
+
});
|
|
341
|
+
});
|
|
257
342
|
}
|
|
258
343
|
/** Tally a list of decisions by evidence kind. */
|
|
259
344
|
function countEvidence(decisions) {
|
|
260
345
|
let executed = 0;
|
|
261
346
|
let colocated = 0;
|
|
347
|
+
let configured = 0;
|
|
262
348
|
for (const d of decisions) {
|
|
263
349
|
if (d.evidence === "executed")
|
|
264
350
|
executed += 1;
|
|
351
|
+
else if (d.evidence === "configured")
|
|
352
|
+
configured += 1;
|
|
265
353
|
else
|
|
266
354
|
colocated += 1;
|
|
267
355
|
}
|
|
268
|
-
return { executed, colocated };
|
|
356
|
+
return { executed, colocated, configured };
|
|
269
357
|
}
|
|
270
358
|
/**
|
|
271
359
|
* One line naming how the coverage was established. Printed wherever a coverage
|
|
@@ -287,6 +375,11 @@ function formatEvidence(counts) {
|
|
|
287
375
|
parts.push(`${String(counts.colocated)} colocated — a test NAMED after the surface, ` +
|
|
288
376
|
`in the surface's own place. This says the file EXISTS, not that it ran`);
|
|
289
377
|
}
|
|
378
|
+
if (counts.configured > 0) {
|
|
379
|
+
parts.push(`${String(counts.configured)} configured — a test NAMED after the surface ` +
|
|
380
|
+
`at a \`{surface}\` path you configured. Same name binding as colocation, ` +
|
|
381
|
+
`different place; still only says the file EXISTS`);
|
|
382
|
+
}
|
|
290
383
|
if (parts.length === 0)
|
|
291
384
|
return "";
|
|
292
385
|
return `How coverage was decided: ${parts.join("; ")}.`;
|
package/dist/eval.d.ts
CHANGED
|
@@ -724,6 +724,16 @@ export interface TriggerRateReport {
|
|
|
724
724
|
* description). A non-zero count is the whole-harness measurement.
|
|
725
725
|
*/
|
|
726
726
|
readonly competitors: number;
|
|
727
|
+
/**
|
|
728
|
+
* The plugin namespace the skills actually installed under — the `<plugin>`
|
|
729
|
+
* half of the `<plugin>:<skill>` id `skillResolved` matches.
|
|
730
|
+
*
|
|
731
|
+
* Reported because with `skillsDir` the name is chosen by the packager, not by
|
|
732
|
+
* the caller, so the single most common cause of a 0% run was a value the
|
|
733
|
+
* caller had no way to know. Optional so a report recorded before this field
|
|
734
|
+
* still parses.
|
|
735
|
+
*/
|
|
736
|
+
readonly namespace?: string;
|
|
727
737
|
/**
|
|
728
738
|
* Runs EXCLUDED because the turn errored / was rate-limited (detected by the
|
|
729
739
|
* driver's `runError`), present only when > 0. These are NOT counted in `n` or
|
package/dist/eval.js
CHANGED
|
@@ -1577,6 +1577,7 @@ function resolveTriggerPluginDir(spec) {
|
|
|
1577
1577
|
pluginDir,
|
|
1578
1578
|
packaged,
|
|
1579
1579
|
competitors: Math.max(0, countSkills(pluginDir) - 1),
|
|
1580
|
+
namespace: underTestSource(spec).name,
|
|
1580
1581
|
};
|
|
1581
1582
|
}
|
|
1582
1583
|
/** Run one prompt set × trials through `runner`, aggregating fired counts. */
|
|
@@ -1683,7 +1684,7 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
|
|
|
1683
1684
|
throw new Error(`measureTriggerRate: model "${model}" is below the minimum "${minModel}" — ` +
|
|
1684
1685
|
"trigger-rate under-measures selection on a weaker model " +
|
|
1685
1686
|
"(raise the model, or lower `minModel` for a deliberately cheap run).");
|
|
1686
|
-
const { pluginDir, packaged, competitors } = resolveTriggerPluginDir(spec);
|
|
1687
|
+
const { pluginDir, packaged, competitors, namespace } = resolveTriggerPluginDir(spec);
|
|
1687
1688
|
const cfg = {
|
|
1688
1689
|
trials: spec.trials ?? 1,
|
|
1689
1690
|
// Sonnet, not haiku: trigger-rate is a selection measurement and haiku
|
|
@@ -1729,6 +1730,7 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
|
|
|
1729
1730
|
n: relevant.n,
|
|
1730
1731
|
perPrompt: relevant.perPrompt,
|
|
1731
1732
|
competitors,
|
|
1733
|
+
namespace,
|
|
1732
1734
|
errored: positiveOrUndefined(relevant.errored),
|
|
1733
1735
|
usage: aggregateUsage(relevant.usages),
|
|
1734
1736
|
};
|
|
@@ -1833,8 +1835,17 @@ function formatTriggerRateReport(report) {
|
|
|
1833
1835
|
// must not be second-guessed; a checker that hedges on good data gets ignored.
|
|
1834
1836
|
if (report.n > 0 && report.rate === 0)
|
|
1835
1837
|
lines.push("⚠ nothing fired on ANY prompt. That is usually SETUP, not the description — check, in order:\n" +
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
+
// The runtime RESOLVED the namespace before spending a token, so it
|
|
1839
|
+
// prints the id that should have matched instead of telling the reader
|
|
1840
|
+
// to go work it out. With `skillsDir` the name is not even the user's
|
|
1841
|
+
// choice — the packager picks it — so "check the id" was advice about a
|
|
1842
|
+
// value they had never seen.
|
|
1843
|
+
(report.namespace !== undefined
|
|
1844
|
+
? " 1. the id in `fired` — your skills installed under " +
|
|
1845
|
+
`\`${report.namespace}\`, so \`skillResolved\` matches ` +
|
|
1846
|
+
`\`${report.namespace}:<skill>\`; a bare name silently never matches;\n`
|
|
1847
|
+
: " 1. the id in `fired` — `skillResolved` matches the NAMESPACED id " +
|
|
1848
|
+
"(`<plugin>:<skill>`); a bare name silently never matches;\n") +
|
|
1838
1849
|
" 2. the install field — a loose `.claude/skills` dir needs `skillsDir`, " +
|
|
1839
1850
|
"not `pluginDir` (which wants a full plugin manifest);\n" +
|
|
1840
1851
|
" 3. the `fixture` — a run starts in an EMPTY cwd, so a prompt about a " +
|
package/dist/plugin-loader.js
CHANGED
|
@@ -254,6 +254,40 @@ function materializeSurfaces(root, layout, files, sources) {
|
|
|
254
254
|
counts[surface] = (counts[surface] ?? 0) + Object.keys(tree).length;
|
|
255
255
|
}
|
|
256
256
|
};
|
|
257
|
+
/**
|
|
258
|
+
* Read the path-scoped RULES dir (`.claude/rules/*.md` on Claude Code).
|
|
259
|
+
*
|
|
260
|
+
* DELIBERATELY NOT a `surfaceDirs` entry, and the distinction is the whole
|
|
261
|
+
* design: `surfaceDirs` decides whether a directory counts as a LOADABLE
|
|
262
|
+
* MACHINE, and rules are instructions, not an invocable surface — folding them
|
|
263
|
+
* in would silently change what "an empty machine" means for every harness.
|
|
264
|
+
* So they are read here, added to `files` for the checks that read text
|
|
265
|
+
* (frontmatter-valid, the rule map), and left out of `counts` and
|
|
266
|
+
* `hasLoadable`. A layout with no `rulesDir` reads nothing and behaves exactly
|
|
267
|
+
* as before. Closes #175.3: a whole instruction layer the audit could not see.
|
|
268
|
+
*/
|
|
269
|
+
const materializeRules = () => {
|
|
270
|
+
const dir = layout.rulesDir;
|
|
271
|
+
if (!dir)
|
|
272
|
+
return;
|
|
273
|
+
// Read BOTH candidate bases, and NOT the resolved `scopes`. Rules are not
|
|
274
|
+
// tied to where the invocable surfaces live: a repo can keep its skills at
|
|
275
|
+
// the root (a published plugin) while its rules sit under `.claude/`, and
|
|
276
|
+
// keying off scopes then read the wrong directory and found nothing —
|
|
277
|
+
// measured on exactly that shape while building this.
|
|
278
|
+
//
|
|
279
|
+
// Each base keys at its own real path (`rules/…` and `.claude/rules/…`), so
|
|
280
|
+
// a repo with both loses neither and nothing collides.
|
|
281
|
+
const bases = [
|
|
282
|
+
"",
|
|
283
|
+
...(layout.userSurfaceRoot !== undefined ? [layout.userSurfaceRoot] : []),
|
|
284
|
+
];
|
|
285
|
+
for (const base of bases) {
|
|
286
|
+
const tree = surfaceTree((0, node_path_1.join)(root, base, dir));
|
|
287
|
+
for (const [rel, content] of Object.entries(tree))
|
|
288
|
+
add((0, node_path_1.join)(base, dir, rel), content, (0, node_path_1.join)(root, base, dir, rel));
|
|
289
|
+
}
|
|
290
|
+
};
|
|
257
291
|
const source = (0, surface_scopes_js_1.surfaceSource)(layout, {
|
|
258
292
|
hasRootSkillFile: (0, node_fs_1.existsSync)((0, node_path_1.join)(root, "SKILL.md")),
|
|
259
293
|
skillName: (0, node_path_1.basename)(root),
|
|
@@ -280,6 +314,7 @@ function materializeSurfaces(root, layout, files, sources) {
|
|
|
280
314
|
(0, surface_scopes_js_1.assertDistinctScopeKeys)(source.scopes, layout.name);
|
|
281
315
|
for (const scope of source.scopes)
|
|
282
316
|
materializeScope(scope, scope.base === "" ? rootTrees : userTrees);
|
|
317
|
+
materializeRules();
|
|
283
318
|
return { counts, scopes: source.scopes };
|
|
284
319
|
}
|
|
285
320
|
/* v8 ignore next 2 -- exhaustiveness guard, unreachable given SurfaceSource */
|
package/dist/run-hook.d.ts
CHANGED
|
@@ -163,10 +163,30 @@ export interface HookRunResult extends ScriptRunResult {
|
|
|
163
163
|
/** Parsed stdout JSON if the hook emitted a JSON decision, else null. */
|
|
164
164
|
readonly json: HookOutput | null;
|
|
165
165
|
/**
|
|
166
|
-
* Normalized decision:
|
|
167
|
-
* `permissionDecision:"deny"
|
|
166
|
+
* Normalized decision: the dangerous call did NOT go through. Set by exit 2,
|
|
167
|
+
* `decision:"block"`, `permissionDecision:"deny"`, or the harness's
|
|
168
|
+
* halt-the-turn field (Claude Code `{"continue": false}` — see
|
|
169
|
+
* {@link HookRunResult.haltsTurn}).
|
|
170
|
+
*
|
|
171
|
+
* The halt case was missing until 2026-08-31 (#174), and the shape of that bug
|
|
172
|
+
* is worth keeping written down: `verifyGuardrail` reads this field, so a real
|
|
173
|
+
* `PreToolUse` guard that stopped every one of the disaster battery's commands
|
|
174
|
+
* was reported by `assertBlocksDisasters` as blocking NONE of them. A tool
|
|
175
|
+
* whose stated job is catching a guard that looks fine and silently does
|
|
176
|
+
* nothing said the opposite about a working guard — the same false-confidence
|
|
177
|
+
* failure, with the sign flipped.
|
|
168
178
|
*/
|
|
169
179
|
readonly blocked: boolean;
|
|
180
|
+
/**
|
|
181
|
+
* The hook halted the WHOLE TURN rather than denying one call — the harness's
|
|
182
|
+
* `haltsTurnField` came back `false`.
|
|
183
|
+
*
|
|
184
|
+
* Reported separately because it is strictly stronger than a deny and the two
|
|
185
|
+
* are worth telling apart when a test asks WHICH mechanism fired. `blocked`
|
|
186
|
+
* stays the question nearly every caller means ("did the action happen?"), so
|
|
187
|
+
* a halt sets both.
|
|
188
|
+
*/
|
|
189
|
+
readonly haltsTurn: boolean;
|
|
170
190
|
/**
|
|
171
191
|
* The decision the hook expressed, preferring the structured
|
|
172
192
|
* `permissionDecision` ("allow"|"deny"|"ask") then legacy `decision`
|
|
@@ -183,6 +203,7 @@ export declare function parseHookOutput(stdout: string): HookOutput | null;
|
|
|
183
203
|
export declare function decideHook(exitCode: number, json: HookOutput | null, protocol?: HookProtocol): {
|
|
184
204
|
blocked: boolean;
|
|
185
205
|
decision: HookRunResult["decision"];
|
|
206
|
+
haltsTurn: boolean;
|
|
186
207
|
};
|
|
187
208
|
/**
|
|
188
209
|
* The hook layer over {@link runScriptWith}: serialize the event to stdin, run
|
package/dist/run-hook.js
CHANGED
|
@@ -130,9 +130,16 @@ function parseHookOutput(stdout) {
|
|
|
130
130
|
function decideHook(exitCode, json, protocol = hook_protocol_js_1.claudeCodeHookProtocol) {
|
|
131
131
|
const permission = json?.hookSpecificOutput?.permissionDecision;
|
|
132
132
|
const decision = permission ?? json?.decision;
|
|
133
|
+
// The halt field is read from the PORT, never hard-coded: `"continue"` is a
|
|
134
|
+
// documented Claude Code fact and an unverified one for Codex, so the harness
|
|
135
|
+
// that has it declares it (core ⊄ adapter). `=== false` and not falsy —
|
|
136
|
+
// an absent field must not read as a halt.
|
|
137
|
+
const haltField = protocol.haltsTurnField;
|
|
138
|
+
const haltsTurn = haltField !== undefined && json?.[haltField] === false;
|
|
133
139
|
const blocked = exitCode === protocol.blockExitCode ||
|
|
140
|
+
haltsTurn ||
|
|
134
141
|
(decision !== undefined && protocol.denyDecisionValues.includes(decision));
|
|
135
|
-
return { blocked, decision };
|
|
142
|
+
return { blocked, decision, haltsTurn };
|
|
136
143
|
}
|
|
137
144
|
/**
|
|
138
145
|
* The hook layer over {@link runScriptWith}: serialize the event to stdin, run
|
|
@@ -143,8 +150,8 @@ function decideHook(exitCode, json, protocol = hook_protocol_js_1.claudeCodeHook
|
|
|
143
150
|
function runHookWith(command, input, opts, deps) {
|
|
144
151
|
const res = (0, run_script_js_1.runScriptWith)(command, JSON.stringify(input), opts, deps);
|
|
145
152
|
const json = parseHookOutput(res.stdout);
|
|
146
|
-
const { blocked, decision } = decideHook(res.exitCode, json);
|
|
147
|
-
return { ...res, json, blocked, decision };
|
|
153
|
+
const { blocked, decision, haltsTurn } = decideHook(res.exitCode, json);
|
|
154
|
+
return { ...res, json, blocked, decision, haltsTurn };
|
|
148
155
|
}
|
|
149
156
|
/**
|
|
150
157
|
* Run a hook command, piping `input` as JSON to its stdin, and report the exit
|
package/dist/scan-core.d.ts
CHANGED
|
@@ -30,6 +30,11 @@ export interface SurfaceClassifier {
|
|
|
30
30
|
* one. Null for a path this classifier does not call an agent.
|
|
31
31
|
*/
|
|
32
32
|
readonly agentName: (f: string) => string | null;
|
|
33
|
+
/**
|
|
34
|
+
* A path-scoped RULES file (`<rulesDir>/<name>.md`) — an INSTRUCTION surface,
|
|
35
|
+
* not an invocable one. Always false for a layout with no `rulesDir`.
|
|
36
|
+
*/
|
|
37
|
+
readonly isRule: (f: string) => boolean;
|
|
33
38
|
}
|
|
34
39
|
export declare function makeClassifier(layout: PluginLayout): SurfaceClassifier;
|
|
35
40
|
/** The plugin-root + materialize-root + dialect context skill scanning needs. */
|
package/dist/scan-core.js
CHANGED
|
@@ -146,9 +146,13 @@ function makeClassifier(layout) {
|
|
|
146
146
|
const skill = at(layout.skillDir);
|
|
147
147
|
const agent = at(layout.agentDir);
|
|
148
148
|
const command = at(layout.commandDir);
|
|
149
|
+
const rules = at(layout.rulesDir ?? "");
|
|
149
150
|
const skillRe = skill ? new RegExp(`${skill}[^/]+/SKILL\\.md$`) : null;
|
|
150
151
|
const agentRe = agent ? new RegExp(`${agent}${layout_js_1.AGENT_FILE_LEAF_RE}$`) : null;
|
|
151
152
|
const commandRe = command ? new RegExp(`${command}.+\\.md$`) : null;
|
|
153
|
+
// Flat `<rulesDir>/<name>.md`, like commands. A layout without a rules dir
|
|
154
|
+
// yields null and every path below answers false — the additive default.
|
|
155
|
+
const ruleRe = rules ? new RegExp(`${rules}[^/]+\\.md$`) : null;
|
|
152
156
|
// A subagent lives under the plugin's `agents/` dir AT ANY DEPTH (the harness
|
|
153
157
|
// reads it recursively — see AGENT_FILE_LEAF_RE for the vendor's wording and
|
|
154
158
|
// the measurement), but never under ANOTHER surface dir. Two real-world
|
|
@@ -191,6 +195,7 @@ function makeClassifier(layout) {
|
|
|
191
195
|
isSkill,
|
|
192
196
|
isAgent,
|
|
193
197
|
isCommand: (f) => commandRe?.test(f) ?? false,
|
|
198
|
+
isRule: (f) => ruleRe?.test(f) ?? false,
|
|
194
199
|
agentName: (f) => isAgent(f) ? (0, layout_js_1.agentSurfaceName)(f, layout.agentDir) : null,
|
|
195
200
|
};
|
|
196
201
|
}
|
|
@@ -746,7 +751,11 @@ function frontmatterValueIssuesFor(files, cls) {
|
|
|
746
751
|
function malformedFrontmatterFor(files, cls) {
|
|
747
752
|
const out = [];
|
|
748
753
|
for (const [path, md] of Object.entries(files)) {
|
|
749
|
-
|
|
754
|
+
// Rules join skills + agents here: `.claude/rules/*.md` carries a `paths:`
|
|
755
|
+
// frontmatter key that SCOPES the instruction, so unparseable YAML there
|
|
756
|
+
// silently changes which files the rule applies to — the same defect this
|
|
757
|
+
// check exists for, on a surface no layout named until now (#175.3).
|
|
758
|
+
if (!cls.isSkill(path) && !cls.isAgent(path) && !cls.isRule(path))
|
|
750
759
|
continue;
|
|
751
760
|
if (!(0, frontmatter_read_js_1.readFrontmatter)(md).malformed)
|
|
752
761
|
continue;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
type ResolveContext = {
|
|
2
|
+
parentURL?: string;
|
|
3
|
+
conditions: string[];
|
|
4
|
+
};
|
|
5
|
+
type Resolved = {
|
|
6
|
+
url: string;
|
|
7
|
+
format?: string | null;
|
|
8
|
+
shortCircuit?: boolean;
|
|
9
|
+
};
|
|
10
|
+
type NextResolve = (specifier: string, context: ResolveContext) => Resolved | Promise<Resolved>;
|
|
11
|
+
type LoadContext = {
|
|
12
|
+
format?: string | null;
|
|
13
|
+
conditions: string[];
|
|
14
|
+
};
|
|
15
|
+
type Loaded = {
|
|
16
|
+
format: string;
|
|
17
|
+
source?: string | ArrayBuffer;
|
|
18
|
+
shortCircuit?: boolean;
|
|
19
|
+
};
|
|
20
|
+
type NextLoad = (url: string, context: LoadContext) => Loaded | Promise<Loaded>;
|
|
21
|
+
/**
|
|
22
|
+
* `./x.js` → `./x.ts` when the sibling exists.
|
|
23
|
+
*
|
|
24
|
+
* This is the TypeScript ESM convention (`tsc` under `nodenext` requires the
|
|
25
|
+
* `.js` extension in the source), which `tsx` implements and native Node does
|
|
26
|
+
* not. It is the ONE divergence that matters in practice: this repository's own
|
|
27
|
+
* dogfood specs import `src/core/spec.js`, a file that does not exist on disk.
|
|
28
|
+
* Attempted only AFTER normal resolution fails, so it can never shadow a real
|
|
29
|
+
* `.js` file.
|
|
30
|
+
*/
|
|
31
|
+
export declare function resolve(specifier: string, context: ResolveContext, nextResolve: NextResolve): Promise<Resolved>;
|
|
32
|
+
/** Transpile `.ts`/`.mts` with the TypeScript this package already ships. */
|
|
33
|
+
export declare function load(url: string, context: LoadContext, nextLoad: NextLoad): Promise<Loaded>;
|
|
34
|
+
export {};
|
|
35
|
+
//# sourceMappingURL=spec-hooks.d.mts.map
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Module customization hooks for the spec host — vigiles' OWN loader for `.ts`
|
|
3
|
+
* specs, replacing "whichever loader happens to be installed".
|
|
4
|
+
*
|
|
5
|
+
* Why vigiles owns this rather than shelling to `tsx`:
|
|
6
|
+
*
|
|
7
|
+
* - **No install, no network.** `typescript` is already a runtime dependency
|
|
8
|
+
* of this package (`dependencies`, and `core/compile-generator.ts` uses it),
|
|
9
|
+
* so `ts.transpileModule` costs nothing extra. The bug that started this
|
|
10
|
+
* work was a consuming repo without `tsx`, where `npx tsx` went to the
|
|
11
|
+
* registry and every one of 50 specs blew a 15s budget.
|
|
12
|
+
* - **One resolution contract.** Before this, a spec's module resolution
|
|
13
|
+
* depended on the user's Node version and on which loader won — so a spec
|
|
14
|
+
* could load locally and fail in CI under different rules. A tool that
|
|
15
|
+
* audits other tools for that kind of quiet divergence should not have it.
|
|
16
|
+
*
|
|
17
|
+
* Scope is deliberately small and documented as such: `.ts`/`.mts` sources, the
|
|
18
|
+
* `./x.js` → `./x.ts` specifier rewrite, and bare specifiers. NOT tsconfig
|
|
19
|
+
* `paths`, JSX, or decorator configuration — specs are configuration modules,
|
|
20
|
+
* not applications.
|
|
21
|
+
*/
|
|
22
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
23
|
+
import { fileURLToPath } from "node:url";
|
|
24
|
+
import ts from "typescript";
|
|
25
|
+
const TS_SOURCE = /\.m?ts$/;
|
|
26
|
+
/**
|
|
27
|
+
* `./x.js` → `./x.ts` when the sibling exists.
|
|
28
|
+
*
|
|
29
|
+
* This is the TypeScript ESM convention (`tsc` under `nodenext` requires the
|
|
30
|
+
* `.js` extension in the source), which `tsx` implements and native Node does
|
|
31
|
+
* not. It is the ONE divergence that matters in practice: this repository's own
|
|
32
|
+
* dogfood specs import `src/core/spec.js`, a file that does not exist on disk.
|
|
33
|
+
* Attempted only AFTER normal resolution fails, so it can never shadow a real
|
|
34
|
+
* `.js` file.
|
|
35
|
+
*/
|
|
36
|
+
export async function resolve(specifier, context, nextResolve) {
|
|
37
|
+
try {
|
|
38
|
+
return await nextResolve(specifier, context);
|
|
39
|
+
}
|
|
40
|
+
catch (err) {
|
|
41
|
+
if (specifier.endsWith(".js") && context.parentURL) {
|
|
42
|
+
const candidate = new URL(specifier.slice(0, -3) + ".ts", context.parentURL);
|
|
43
|
+
if (existsSync(fileURLToPath(candidate))) {
|
|
44
|
+
return { url: candidate.href, format: "module", shortCircuit: true };
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
throw err;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
/** Transpile `.ts`/`.mts` with the TypeScript this package already ships. */
|
|
51
|
+
export async function load(url, context, nextLoad) {
|
|
52
|
+
if (!TS_SOURCE.test(new URL(url).pathname))
|
|
53
|
+
return nextLoad(url, context);
|
|
54
|
+
const fileName = fileURLToPath(url);
|
|
55
|
+
const { outputText } = ts.transpileModule(readFileSync(fileName, "utf-8"), {
|
|
56
|
+
fileName,
|
|
57
|
+
compilerOptions: {
|
|
58
|
+
module: ts.ModuleKind.ESNext,
|
|
59
|
+
target: ts.ScriptTarget.ES2022,
|
|
60
|
+
// Erasing types is the whole job; anything that changes SEMANTICS is not
|
|
61
|
+
// ours to decide for a spec.
|
|
62
|
+
verbatimModuleSyntax: false,
|
|
63
|
+
isolatedModules: true,
|
|
64
|
+
},
|
|
65
|
+
});
|
|
66
|
+
return { format: "module", source: outputText, shortCircuit: true };
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=spec-hooks.mjs.map
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The spec host — a child process that loads specs and streams results as NDJSON.
|
|
3
|
+
*
|
|
4
|
+
* ONE host per CLI command, not one per spec: Node startup and the TypeScript
|
|
5
|
+
* load are paid once, then each spec costs a transpile.
|
|
6
|
+
*
|
|
7
|
+
* 🔴 **Why a child process at all, when `import()` works in-process.** Because a
|
|
8
|
+
* module evaluation cannot be cancelled once started. `Promise.race` returns
|
|
9
|
+
* control to the caller but the evaluation keeps running and holds the event
|
|
10
|
+
* loop, so a spec that stalls at top level hangs `compile`, `test` and `audit`
|
|
11
|
+
* with no bound. A child can be killed. That is the entire argument, and it is
|
|
12
|
+
* why the in-process loader this replaced could not be repaired: it also had to
|
|
13
|
+
* answer "did the module body already run?" to know whether re-running was
|
|
14
|
+
* safe, and Node does not expose that bit — `ERR_MODULE_NOT_FOUND` and
|
|
15
|
+
* `SyntaxError` each occur both before and during evaluation.
|
|
16
|
+
*
|
|
17
|
+
* Protocol, one JSON object per line each way:
|
|
18
|
+
* in {"path":"<abs path to spec>"}
|
|
19
|
+
* out {"path":"…","phase":"start"} — emitted BEFORE evaluation
|
|
20
|
+
* out {"path":"…","ok":true,"value":{…}}
|
|
21
|
+
* out {"path":"…","ok":false,"error":"…"}
|
|
22
|
+
*
|
|
23
|
+
* The `start` line is what makes a hang diagnosable: when the parent's deadline
|
|
24
|
+
* fires, the last `start` without a result NAMES the spec that stalled. Before
|
|
25
|
+
* this, a stalled load produced N identical failures and no culprit.
|
|
26
|
+
*
|
|
27
|
+
* Values cross as JSON, which is not a new constraint — the previous `npx tsx`
|
|
28
|
+
* path already did `JSON.stringify` in the child and `JSON.parse` in the parent,
|
|
29
|
+
* so every spec that has ever loaded survived this round trip. Spec types carry
|
|
30
|
+
* no functions; TypeScript is the authoring layer, the value is data.
|
|
31
|
+
*/
|
|
32
|
+
import { register } from "node:module";
|
|
33
|
+
import { pathToFileURL } from "node:url";
|
|
34
|
+
register(new URL("./spec-hooks.mjs", import.meta.url));
|
|
35
|
+
function say(line) {
|
|
36
|
+
process.stdout.write(JSON.stringify(line) + "\n");
|
|
37
|
+
}
|
|
38
|
+
async function loadOne(path) {
|
|
39
|
+
say({ path, phase: "start" });
|
|
40
|
+
try {
|
|
41
|
+
const mod = (await import(pathToFileURL(path).href));
|
|
42
|
+
// CJS interop can nest the default one level deeper.
|
|
43
|
+
const raw = mod.default;
|
|
44
|
+
const value = raw && typeof raw === "object" && "default" in raw ? raw.default : raw;
|
|
45
|
+
if (value === undefined) {
|
|
46
|
+
say({ path, ok: false, error: "the spec has no default export." });
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
say({ path, ok: true, value });
|
|
50
|
+
}
|
|
51
|
+
catch (err) {
|
|
52
|
+
say({
|
|
53
|
+
path,
|
|
54
|
+
ok: false,
|
|
55
|
+
error: err instanceof Error ? (err.stack ?? err.message) : String(err),
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
// Requests are serialised: a spec may depend on module state a previous one set
|
|
60
|
+
// up, and interleaving would make a hang impossible to attribute.
|
|
61
|
+
let queue = Promise.resolve();
|
|
62
|
+
let buffered = "";
|
|
63
|
+
process.stdin.setEncoding("utf-8");
|
|
64
|
+
process.stdin.on("data", (chunk) => {
|
|
65
|
+
buffered += chunk;
|
|
66
|
+
let nl;
|
|
67
|
+
while ((nl = buffered.indexOf("\n")) >= 0) {
|
|
68
|
+
const line = buffered.slice(0, nl).trim();
|
|
69
|
+
buffered = buffered.slice(nl + 1);
|
|
70
|
+
if (!line)
|
|
71
|
+
continue;
|
|
72
|
+
const { path } = JSON.parse(line);
|
|
73
|
+
queue = queue.then(() => loadOne(path));
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
process.stdin.on("end", () => {
|
|
77
|
+
queue.then(() => process.exit(0));
|
|
78
|
+
});
|
|
79
|
+
//# sourceMappingURL=spec-host.mjs.map
|
|
@@ -202,26 +202,28 @@ function isColocated(surface, testPath) {
|
|
|
202
202
|
return (0, coverage_evidence_js_1.isColocatedTest)(surface, testPath);
|
|
203
203
|
}
|
|
204
204
|
/** Mirror of test-coverage.ts `coverageOf` — strongest evidence across tests. */
|
|
205
|
-
function coverageOf(surface, tests) {
|
|
205
|
+
function coverageOf(surface, tests, globs) {
|
|
206
206
|
let best = null;
|
|
207
207
|
for (const t of tests) {
|
|
208
208
|
if (t.path === surface.path)
|
|
209
209
|
continue;
|
|
210
|
-
const ev = (0, coverage_evidence_js_1.evidenceFor)(surface, t, isColocated(surface, t.path));
|
|
210
|
+
const ev = (0, coverage_evidence_js_1.evidenceFor)(surface, t, isColocated(surface, t.path), (0, coverage_evidence_js_1.matchesSurfaceGlob)(surface, t.path, globs));
|
|
211
211
|
if (!ev)
|
|
212
212
|
continue;
|
|
213
|
-
|
|
213
|
+
// Rank, do not first-win: with a colocated harness AND a configured
|
|
214
|
+
// suite the reported provenance must not depend on glob order.
|
|
215
|
+
if (!best || (0, coverage_evidence_js_1.strongerEvidence)(ev, best.evidence))
|
|
214
216
|
best = { surface, evidence: ev, by: t.path };
|
|
215
217
|
}
|
|
216
218
|
return best;
|
|
217
219
|
}
|
|
218
220
|
/** Mirror of test-coverage.ts `tierOf` — one tier's covered/untested split. */
|
|
219
|
-
function tierOf(considered, tests) {
|
|
221
|
+
function tierOf(considered, tests, globs) {
|
|
220
222
|
const covered = [];
|
|
221
223
|
const untested = [];
|
|
222
224
|
const decisions = [];
|
|
223
225
|
for (const s of considered) {
|
|
224
|
-
const decision = coverageOf(s, tests);
|
|
226
|
+
const decision = coverageOf(s, tests, globs);
|
|
225
227
|
if (decision) {
|
|
226
228
|
covered.push(s);
|
|
227
229
|
decisions.push(decision);
|
|
@@ -252,12 +254,19 @@ repoName) {
|
|
|
252
254
|
];
|
|
253
255
|
const considered = surfaces.filter((s) => !s.ignored);
|
|
254
256
|
const tests = discoverTests(files);
|
|
255
|
-
|
|
257
|
+
// NO configured `{surface}` globs here, and that is a property of this twin
|
|
258
|
+
// rather than a gap: the browser engine reads a file MAP, not a repo, so there
|
|
259
|
+
// is no `.vigilesrc.json` for a user to have configured. Passing an empty list
|
|
260
|
+
// makes `matchesSurfaceGlob` false for everything, so this path behaves exactly
|
|
261
|
+
// as it did before the tier existed. Said out loud because a silent asymmetry
|
|
262
|
+
// between the twins is how they drift.
|
|
263
|
+
const noConfiguredGlobs = [];
|
|
264
|
+
const union = tierOf(considered, tests, noConfiguredGlobs);
|
|
256
265
|
return {
|
|
257
266
|
untested: [...union.untested],
|
|
258
267
|
decisions: union.decisions,
|
|
259
|
-
harness: tierOf(considered, tests.filter((t) => !(0, coverage_evidence_js_1.isEvalScript)((0, posix_path_js_1.basename)(t.path)))),
|
|
260
|
-
evals: tierOf(considered, tests.filter((t) => (0, coverage_evidence_js_1.isEvalScript)((0, posix_path_js_1.basename)(t.path)))),
|
|
268
|
+
harness: tierOf(considered, tests.filter((t) => !(0, coverage_evidence_js_1.isEvalScript)((0, posix_path_js_1.basename)(t.path))), noConfiguredGlobs),
|
|
269
|
+
evals: tierOf(considered, tests.filter((t) => (0, coverage_evidence_js_1.isEvalScript)((0, posix_path_js_1.basename)(t.path))), noConfiguredGlobs),
|
|
261
270
|
};
|
|
262
271
|
}
|
|
263
272
|
//# sourceMappingURL=test-coverage-files.js.map
|