vigiles 5.1.0 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/adapters/claude-code/adapter.js +1 -0
- package/dist/adapters/claude-code/agent-runtime.d.ts +20 -6
- package/dist/adapters/claude-code/agent-runtime.js +51 -8
- package/dist/adapters/claude-code/dialect.js +19 -0
- package/dist/adapters/claude-code/effect-region.d.ts +9 -0
- package/dist/adapters/claude-code/effect-region.js +45 -0
- package/dist/adapters/claude-code/layout.js +3 -0
- package/dist/adapters/claude-code/skill-runtime.d.ts +25 -0
- package/dist/adapters/claude-code/skill-runtime.js +48 -0
- package/dist/adapters/codex/adapter.js +3 -0
- package/dist/adapters/codex/layout.js +3 -0
- package/dist/adapters/opencode/adapter.js +1 -0
- package/dist/adapters/opencode/layout.js +3 -0
- package/dist/check.d.ts +8 -0
- package/dist/check.js +27 -3
- package/dist/cli.js +323 -88
- package/dist/core/adapter.d.ts +10 -0
- package/dist/core/bash-effects.d.ts +41 -0
- package/dist/core/bash-effects.js +405 -0
- package/dist/core/compile.d.ts +3 -1
- package/dist/core/compile.js +162 -39
- package/dist/core/dialect.d.ts +10 -0
- package/dist/core/effects.d.ts +172 -0
- package/dist/core/effects.js +245 -0
- package/dist/core/layout.d.ts +6 -0
- package/dist/core/mcp-tool.d.ts +1 -1
- package/dist/core/orphans.js +21 -0
- package/dist/core/spec.d.ts +142 -3
- package/dist/core/spec.js +48 -0
- package/dist/core/tool-contract.d.ts +1 -1
- package/dist/core/types.d.ts +6 -6
- package/dist/core/validate.js +4 -4
- package/dist/harness-test.d.ts +7 -0
- package/dist/harness-test.js +19 -7
- package/dist/leaderboard.d.ts +2 -0
- package/dist/leaderboard.js +2 -0
- package/dist/optimize.d.ts +74 -0
- package/dist/optimize.js +94 -0
- package/dist/scaffold-test.d.ts +30 -0
- package/dist/scaffold-test.js +158 -0
- package/dist/scan.d.ts +40 -0
- package/dist/scan.js +91 -43
- package/dist/score-explainer.d.ts +69 -0
- package/dist/score-explainer.js +169 -0
- package/dist/test-coverage.d.ts +7 -0
- package/dist/test-coverage.js +39 -24
- package/package.json +2 -1
- package/skills/{migrate-to-spec → adopt-spec}/SKILL.md +4 -4
- package/skills/edit-spec/SKILL.md +1 -1
package/dist/scan.js
CHANGED
|
@@ -34,6 +34,7 @@ const mcp_tool_js_1 = require("./core/mcp-tool.js");
|
|
|
34
34
|
const mcp_hook_js_1 = require("./core/mcp-hook.js");
|
|
35
35
|
const agent_runtime_js_1 = require("./adapters/claude-code/agent-runtime.js");
|
|
36
36
|
const test_coverage_js_1 = require("./test-coverage.js");
|
|
37
|
+
const effects_js_1 = require("./core/effects.js");
|
|
37
38
|
// ---------------------------------------------------------------------------
|
|
38
39
|
// Internals
|
|
39
40
|
// ---------------------------------------------------------------------------
|
|
@@ -51,14 +52,24 @@ function frontmatter(md) {
|
|
|
51
52
|
color: (0, frontmatter_read_js_1.frontmatterScalar)(fm, "color"),
|
|
52
53
|
};
|
|
53
54
|
}
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
//
|
|
59
|
-
const
|
|
60
|
-
const
|
|
61
|
-
const
|
|
55
|
+
function escapeRe(s) {
|
|
56
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
57
|
+
}
|
|
58
|
+
function makeClassifier(layout) {
|
|
59
|
+
// An empty dir means "this harness has no such surface" → never matches.
|
|
60
|
+
const at = (dir) => dir ? `(?:^|/)${escapeRe(dir)}/` : null;
|
|
61
|
+
const skill = at(layout.skillDir);
|
|
62
|
+
const agent = at(layout.agentDir);
|
|
63
|
+
const command = at(layout.commandDir);
|
|
64
|
+
const skillRe = skill ? new RegExp(`${skill}[^/]+/SKILL\\.md$`) : null;
|
|
65
|
+
const agentRe = agent ? new RegExp(`${agent}[^/]+\\.md$`) : null;
|
|
66
|
+
const commandRe = command ? new RegExp(`${command}.+\\.md$`) : null;
|
|
67
|
+
return {
|
|
68
|
+
isSkill: (f) => skillRe?.test(f) ?? false,
|
|
69
|
+
isAgent: (f) => (agentRe?.test(f) ?? false) && !f.endsWith(".spec.ts"),
|
|
70
|
+
isCommand: (f) => commandRe?.test(f) ?? false,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
62
73
|
function skillName(path) {
|
|
63
74
|
return (path
|
|
64
75
|
.replace(/\/SKILL\.md$/, "")
|
|
@@ -137,10 +148,10 @@ function firstBodyParagraph(md) {
|
|
|
137
148
|
}
|
|
138
149
|
return para.join(" ").trim() || undefined;
|
|
139
150
|
}
|
|
140
|
-
function scanSkills(files) {
|
|
151
|
+
function scanSkills(files, cls) {
|
|
141
152
|
const out = [];
|
|
142
153
|
for (const [path, md] of Object.entries(files)) {
|
|
143
|
-
if (!isSkill(path))
|
|
154
|
+
if (!cls.isSkill(path))
|
|
144
155
|
continue;
|
|
145
156
|
const fm = frontmatter(md);
|
|
146
157
|
// A skill's trigger surface is its frontmatter `description` OR — when that's
|
|
@@ -165,10 +176,10 @@ function scanSkills(files) {
|
|
|
165
176
|
* logic as `scanSkills` (frontmatter `description` ← first body paragraph), then
|
|
166
177
|
* the NCD precision-proxy. See description-overlap.ts.
|
|
167
178
|
*/
|
|
168
|
-
function descriptionOverlapsFor(files) {
|
|
179
|
+
function descriptionOverlapsFor(files, cls) {
|
|
169
180
|
const surfaces = [];
|
|
170
181
|
for (const [path, md] of Object.entries(files)) {
|
|
171
|
-
if (!isSkill(path))
|
|
182
|
+
if (!cls.isSkill(path))
|
|
172
183
|
continue;
|
|
173
184
|
if (/^\s*disable-model-invocation:\s*true\s*$/m.test(md))
|
|
174
185
|
continue;
|
|
@@ -180,12 +191,16 @@ function descriptionOverlapsFor(files) {
|
|
|
180
191
|
}
|
|
181
192
|
return (0, description_overlap_js_1.findDescriptionOverlaps)(surfaces);
|
|
182
193
|
}
|
|
183
|
-
function scanAgents(files, dialect, declaredServers) {
|
|
194
|
+
function scanAgents(files, dialect, declaredServers, cls) {
|
|
184
195
|
const out = [];
|
|
185
196
|
for (const [path, md] of Object.entries(files)) {
|
|
186
|
-
if (!isAgent(path))
|
|
197
|
+
if (!cls.isAgent(path))
|
|
187
198
|
continue;
|
|
188
199
|
const tools = (0, agent_runtime_js_1.parseAgentTools)(md);
|
|
200
|
+
// An inherits-all agent (no `tools:` line) grants access to every tool
|
|
201
|
+
// including every side-effecting one — pass the wildcard sentinel so
|
|
202
|
+
// effectSurface correctly classifies it as `"unrestricted"`.
|
|
203
|
+
const surface = (0, effects_js_1.effectSurface)(tools ?? ["*"], dialect);
|
|
189
204
|
out.push({
|
|
190
205
|
name: (0, node_path_1.basename)(path, ".md"),
|
|
191
206
|
path,
|
|
@@ -206,21 +221,31 @@ function scanAgents(files, dialect, declaredServers) {
|
|
|
206
221
|
// The block-list mirror: a `disallowedTools:` entry that's a typo of a real
|
|
207
222
|
// tool blocks nothing (close-typo only — high-precision). See tool-contract.ts.
|
|
208
223
|
disallowedToolIssues: (0, tool_contract_js_1.disallowedToolIssues)((0, agent_runtime_js_1.parseAgentToolList)(md, "disallowedTools") ?? [], dialect),
|
|
224
|
+
purity: surface.purity,
|
|
225
|
+
effectBuckets: {
|
|
226
|
+
readOnly: surface.readOnly,
|
|
227
|
+
sideEffecting: surface.sideEffecting,
|
|
228
|
+
unknown: surface.unknown,
|
|
229
|
+
},
|
|
209
230
|
});
|
|
210
231
|
}
|
|
211
232
|
return out.sort((a, b) => a.name.localeCompare(b.name));
|
|
212
233
|
}
|
|
213
234
|
/**
|
|
214
235
|
* Resolve a hook script token to a checkable path. `loadPlugin` expands the
|
|
215
|
-
* braced `${CLAUDE_PLUGIN_ROOT}
|
|
216
|
-
* survives, so resolve
|
|
217
|
-
*
|
|
236
|
+
* braced plugin-root token (`${CLAUDE_PLUGIN_ROOT}`, Codex `${PLUGIN_ROOT}`, …);
|
|
237
|
+
* the unbraced shell form survives, so resolve BOTH forms of the HARNESS's token
|
|
238
|
+
* (from the layout, not hard-coded) against the plugin root and strip shell
|
|
239
|
+
* quotes. A token that still carries any `$VAR` after that is genuinely
|
|
240
|
+
* uncheckable.
|
|
218
241
|
*/
|
|
219
|
-
function resolveScript(token, root) {
|
|
242
|
+
function resolveScript(token, root, pluginRootToken) {
|
|
243
|
+
// "${CLAUDE_PLUGIN_ROOT}" → unbraced "$CLAUDE_PLUGIN_ROOT".
|
|
244
|
+
const unbraced = pluginRootToken.replace(/^\$\{(.+)\}$/, "$$$1");
|
|
220
245
|
const cleaned = token
|
|
221
246
|
.replace(/["']/g, "")
|
|
222
|
-
.replaceAll(
|
|
223
|
-
.replaceAll(
|
|
247
|
+
.replaceAll(pluginRootToken, root)
|
|
248
|
+
.replaceAll(unbraced, root);
|
|
224
249
|
if (cleaned.includes("$"))
|
|
225
250
|
return { script: token, status: "unresolved" };
|
|
226
251
|
// A relative hook path (`./hooks/x.sh`, `scripts/x.py`) is the plugin's own —
|
|
@@ -238,7 +263,7 @@ function resolveScript(token, root) {
|
|
|
238
263
|
// command as MISSING (a false positive caught on gmickel/flow-next's ralph-guard).
|
|
239
264
|
const EXISTENCE_GUARD = /(?:\[\[?\s*!?\s*-[efsx]\s)|(?:\btest\s+!?\s*-[efsx]\s)/;
|
|
240
265
|
/** Pull script-file hook commands out of the resolved settings; count inline ones. */
|
|
241
|
-
function scanHooks(settings, root) {
|
|
266
|
+
function scanHooks(settings, root, pluginRootToken) {
|
|
242
267
|
const text = JSON.stringify(settings.hooks ?? {});
|
|
243
268
|
const commands = [...text.matchAll(/"command":\s*"((?:[^"\\]|\\.)*)"/g)].map((m) => m[1]);
|
|
244
269
|
const byScript = new Map();
|
|
@@ -257,7 +282,7 @@ function scanHooks(settings, root) {
|
|
|
257
282
|
continue;
|
|
258
283
|
}
|
|
259
284
|
for (const tok of found) {
|
|
260
|
-
const hook = resolveScript(tok, root);
|
|
285
|
+
const hook = resolveScript(tok, root, pluginRootToken);
|
|
261
286
|
byScript.set(hook.script, hook);
|
|
262
287
|
}
|
|
263
288
|
}
|
|
@@ -276,10 +301,10 @@ function scanHooks(settings, root) {
|
|
|
276
301
|
* is a separate, behavioral concern). See https://code.claude.com/docs/en/skills
|
|
277
302
|
* and …/sub-agents.
|
|
278
303
|
*/
|
|
279
|
-
function frontmatterIssuesFor(files) {
|
|
304
|
+
function frontmatterIssuesFor(files, cls) {
|
|
280
305
|
const out = [];
|
|
281
306
|
for (const [path, md] of Object.entries(files)) {
|
|
282
|
-
if (!isAgent(path))
|
|
307
|
+
if (!cls.isAgent(path))
|
|
283
308
|
continue; // skills require no frontmatter (dir/body fallbacks)
|
|
284
309
|
const fm = frontmatter(md);
|
|
285
310
|
const missing = [];
|
|
@@ -337,12 +362,12 @@ function closeCandidate(value, candidates) {
|
|
|
337
362
|
* Agent frontmatter VALUE validity — a `model:` or `color:` that's a close typo
|
|
338
363
|
* of a real one. A bad `model:` silently falls back; a bad `color:` is ignored.
|
|
339
364
|
* High-precision (close-typo only); a full/dated model id is left alone. Folded
|
|
340
|
-
* into the `
|
|
365
|
+
* into the `subagent-frontmatter` rule. Agents only (skills have no model/color).
|
|
341
366
|
*/
|
|
342
|
-
function frontmatterValueIssuesFor(files) {
|
|
367
|
+
function frontmatterValueIssuesFor(files, cls) {
|
|
343
368
|
const out = [];
|
|
344
369
|
for (const [path, md] of Object.entries(files)) {
|
|
345
|
-
if (!isAgent(path))
|
|
370
|
+
if (!cls.isAgent(path))
|
|
346
371
|
continue;
|
|
347
372
|
const fm = frontmatter(md);
|
|
348
373
|
// A model id with a digit/hyphen is an explicit form, not an alias typo.
|
|
@@ -382,10 +407,10 @@ function frontmatterValueIssuesFor(files) {
|
|
|
382
407
|
* as an informational note (NOT a structural defect) and the lint rule is a
|
|
383
408
|
* warn, not an error. The file's other fields are still salvaged.
|
|
384
409
|
*/
|
|
385
|
-
function malformedFrontmatterFor(files) {
|
|
410
|
+
function malformedFrontmatterFor(files, cls) {
|
|
386
411
|
const out = [];
|
|
387
412
|
for (const [path, md] of Object.entries(files)) {
|
|
388
|
-
if (!isSkill(path) && !isAgent(path))
|
|
413
|
+
if (!cls.isSkill(path) && !cls.isAgent(path))
|
|
389
414
|
continue;
|
|
390
415
|
if (!(0, frontmatter_read_js_1.readFrontmatter)(md).malformed)
|
|
391
416
|
continue;
|
|
@@ -405,10 +430,10 @@ function malformedFrontmatterFor(files) {
|
|
|
405
430
|
* missing either; surfaced as a soft note in scan (NOT a structural defect, NOT
|
|
406
431
|
* scored) and gated by the `skill-frontmatter` lint rule (warn by default).
|
|
407
432
|
*/
|
|
408
|
-
function skillMetaIssuesFor(files) {
|
|
433
|
+
function skillMetaIssuesFor(files, cls) {
|
|
409
434
|
const out = [];
|
|
410
435
|
for (const [path, md] of Object.entries(files)) {
|
|
411
|
-
if (!isSkill(path))
|
|
436
|
+
if (!cls.isSkill(path))
|
|
412
437
|
continue;
|
|
413
438
|
const fm = frontmatter(md);
|
|
414
439
|
const missing = [];
|
|
@@ -457,8 +482,9 @@ function collectMcpServers(root, layout) {
|
|
|
457
482
|
/** Scan a plugin/repo directory and report its surfaces + structural issues. */
|
|
458
483
|
function scanPlugin(dir, layout, dialect = dialect_js_1.claudeCodeDialect) {
|
|
459
484
|
const lay = layout ?? layout_js_1.claudeCodeLayout;
|
|
485
|
+
const cls = makeClassifier(lay);
|
|
460
486
|
const loaded = (0, plugin_loader_js_1.loadPlugin)(dir, lay);
|
|
461
|
-
const { hooks, inline } = scanHooks(loaded.settings, (0, node_path_1.resolve)(dir));
|
|
487
|
+
const { hooks, inline } = scanHooks(loaded.settings, (0, node_path_1.resolve)(dir), lay.pluginRootToken);
|
|
462
488
|
// Hook-event keys are a CLOSED platform set — an unrecognized one is a dead
|
|
463
489
|
// registration (the hook never fires), so flag every unknown (not just typos).
|
|
464
490
|
// ONLY for the canonical object-keyed-by-event shape: a plugin shipping a
|
|
@@ -480,26 +506,33 @@ function scanPlugin(dir, layout, dialect = dialect_js_1.claudeCodeDialect) {
|
|
|
480
506
|
: null;
|
|
481
507
|
const mcpServers = collectMcpServers((0, node_path_1.resolve)(dir), lay);
|
|
482
508
|
const declaredServers = Object.keys(mcpServers);
|
|
509
|
+
const agents = scanAgents(loaded.files, dialect, declaredServers, cls);
|
|
510
|
+
const puritySummary = agents.reduce((acc, a) => {
|
|
511
|
+
acc[a.purity]++;
|
|
512
|
+
return acc;
|
|
513
|
+
}, { pure: 0, bounded: 0, unrestricted: 0 });
|
|
483
514
|
return {
|
|
484
515
|
dir,
|
|
485
516
|
instructions,
|
|
486
|
-
skills: scanSkills(loaded.files),
|
|
487
|
-
agents
|
|
517
|
+
skills: scanSkills(loaded.files, cls),
|
|
518
|
+
agents,
|
|
488
519
|
hooks,
|
|
489
520
|
inlineHooks: inline,
|
|
490
|
-
commands: Object.keys(loaded.files).filter(isCommand).length,
|
|
521
|
+
commands: Object.keys(loaded.files).filter(cls.isCommand).length,
|
|
491
522
|
mcp: loaded.warnings.some((w) => w.includes("MCP server")),
|
|
492
523
|
danglingRefs: (0, plugin_loader_js_2.danglingRefs)((0, node_path_1.resolve)(dir), lay),
|
|
493
524
|
hookEventIssues,
|
|
494
|
-
frontmatterIssues: frontmatterIssuesFor(loaded.files),
|
|
495
|
-
frontmatterValueIssues: frontmatterValueIssuesFor(loaded.files),
|
|
496
|
-
skillMetaIssues: skillMetaIssuesFor(loaded.files),
|
|
525
|
+
frontmatterIssues: frontmatterIssuesFor(loaded.files, cls),
|
|
526
|
+
frontmatterValueIssues: frontmatterValueIssuesFor(loaded.files, cls),
|
|
527
|
+
skillMetaIssues: skillMetaIssuesFor(loaded.files, cls),
|
|
497
528
|
mcpIssues: (0, mcp_config_js_1.verifyMcpServers)(mcpServers),
|
|
498
529
|
mcpHookIssues: (0, mcp_hook_js_1.verifyMcpHookTargets)(loaded.settings.hooks, declaredServers, dialect),
|
|
499
|
-
descriptionOverlaps: descriptionOverlapsFor(loaded.files),
|
|
500
|
-
malformedFrontmatter: malformedFrontmatterFor(loaded.files),
|
|
530
|
+
descriptionOverlaps: descriptionOverlapsFor(loaded.files, cls),
|
|
531
|
+
malformedFrontmatter: malformedFrontmatterFor(loaded.files, cls),
|
|
501
532
|
warnings: loaded.warnings,
|
|
502
|
-
untested: (0, test_coverage_js_1.findUntestedSurfaces)({ basePath: dir }).untested
|
|
533
|
+
untested: (0, test_coverage_js_1.findUntestedSurfaces)({ basePath: dir, layout: lay }).untested
|
|
534
|
+
.length,
|
|
535
|
+
puritySummary,
|
|
503
536
|
};
|
|
504
537
|
}
|
|
505
538
|
/**
|
|
@@ -588,7 +621,7 @@ function skillLine(s) {
|
|
|
588
621
|
const mark = s.descriptionScript ? "⚠" : "✓";
|
|
589
622
|
return ` ${mark} ${s.name}${notes.length ? ` (${notes.join("; ")})` : ""}`;
|
|
590
623
|
}
|
|
591
|
-
/** One agent's report block: ✗ (broken contract) / ⚠ (inherits all) / ✓ + issues. */
|
|
624
|
+
/** One agent's report block: ✗ (broken contract) / ⚠ (inherits all) / ✓ + issues + purity. */
|
|
592
625
|
function agentLines(a) {
|
|
593
626
|
const tools = a.tools === null
|
|
594
627
|
? "tools: (inherits all — no contract)"
|
|
@@ -601,7 +634,15 @@ function agentLines(a) {
|
|
|
601
634
|
mark = "✗";
|
|
602
635
|
else if (a.tools === null)
|
|
603
636
|
mark = "⚠";
|
|
604
|
-
|
|
637
|
+
// Purity is an informational health signal (not a structural defect); mark it
|
|
638
|
+
// clearly so a reader knows which rung this agent is on.
|
|
639
|
+
const PURITY_TAGS = {
|
|
640
|
+
pure: "pure",
|
|
641
|
+
bounded: "bounded",
|
|
642
|
+
unrestricted: "unrestricted",
|
|
643
|
+
};
|
|
644
|
+
const purityTag = PURITY_TAGS[a.purity] ?? "unrestricted";
|
|
645
|
+
const lines = [` ${mark} ${a.name} — ${tools} [${purityTag}]`];
|
|
605
646
|
for (const issue of a.toolIssues)
|
|
606
647
|
lines.push(` ✗ ${issue.message}`);
|
|
607
648
|
for (const issue of a.mcpToolIssues)
|
|
@@ -650,6 +691,13 @@ function formatScanReport(r) {
|
|
|
650
691
|
facts.push(`Commands: ${String(r.commands)}`);
|
|
651
692
|
facts.push(`MCP servers: ${r.mcp ? "yes" : "no"}`);
|
|
652
693
|
facts.push(`Untested surfaces: ${String(r.untested)}`);
|
|
694
|
+
// Effect surface: harness-level purity summary across all scanned agents.
|
|
695
|
+
// Informational (higher pure% = more constrained, cheaper to test); shown
|
|
696
|
+
// only when there are agents to summarize (no agents → no summary line).
|
|
697
|
+
if (r.agents.length > 0) {
|
|
698
|
+
const { pure, bounded, unrestricted } = r.puritySummary;
|
|
699
|
+
facts.push(`Effect surface: ${String(pure)} pure · ${String(bounded)} bounded · ${String(unrestricted)} unrestricted`);
|
|
700
|
+
}
|
|
653
701
|
out.push(...facts, "");
|
|
654
702
|
// The dangling-ref warning is now shown as a first-class ✗ section above, so
|
|
655
703
|
// drop it from the free-text list to avoid saying the same thing twice.
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Score-explainer — the deterministic WHY behind a low measured score (C4 of the
|
|
3
|
+
* measurement-authority pivot; the strongest pairing in it).
|
|
4
|
+
*
|
|
5
|
+
* The MEASUREMENT layer finds a behavioral SYMPTOM: "this skill underperforms",
|
|
6
|
+
* "the wrong skill fires", "this agent fails its task". A behavioral symptom alone
|
|
7
|
+
* is not actionable — you can drop the skill, but you don't know WHY it lost. The
|
|
8
|
+
* cross-reference engine (the linting layer) already detects the deterministic
|
|
9
|
+
* CAUSES: two skills with near-identical descriptions can't be told apart by the
|
|
10
|
+
* selector; a never-available / typo'd tool is silently dropped from a subagent; a
|
|
11
|
+
* hook on a misspelled event never fires. This module BRIDGES the two: given the
|
|
12
|
+
* `ScanReport` the linter already computes, it surfaces — per affected surface —
|
|
13
|
+
* the deterministic cause of a behavioral symptom and the one-line fix.
|
|
14
|
+
*
|
|
15
|
+
* Measurement says "caveman underperforms"
|
|
16
|
+
* the explainer says "...BECAUSE its description overlaps `compress` (0.86) —
|
|
17
|
+
* differentiate them so the selector can disambiguate."
|
|
18
|
+
*
|
|
19
|
+
* It REUSES the scan findings (one-detector-no-drift) — it never re-detects. So
|
|
20
|
+
* it's pure over a `ScanReport`, free, model-less, and consistent with `vigiles
|
|
21
|
+
* lint`/`scan`. It's the diagnostic the per-repo optimizer (`vigiles optimize`,
|
|
22
|
+
* A2) prints next to each "drop / swap" recommendation. See
|
|
23
|
+
* `research/measurement-authority.md` ("what becomes of the linting", role 2).
|
|
24
|
+
*/
|
|
25
|
+
import type { ScanReport } from "./scan.js";
|
|
26
|
+
/**
|
|
27
|
+
* The behavioral failure a MEASUREMENT would observe — the symptom an explanation
|
|
28
|
+
* accounts for. Discriminated so a consumer can group/filter by symptom and so an
|
|
29
|
+
* explanation can never carry a symptom it has no cause for.
|
|
30
|
+
*/
|
|
31
|
+
export type BehavioralSymptom = "wrong-skill-fires" | "skill-never-fires" | "agent-underperforms" | "hook-never-runs" | "subagent-never-dispatches";
|
|
32
|
+
/**
|
|
33
|
+
* How firmly the deterministic finding EXPLAINS the symptom:
|
|
34
|
+
* - `"likely"` — a hard structural dead-end (a missing script can't run, a
|
|
35
|
+
* never-available tool can't be called); the cause is near-certain.
|
|
36
|
+
* - `"possible"` — a high-precision PROXY for a behavioral risk (a description
|
|
37
|
+
* overlap / a foreign-script description); deterministic to detect, but whether
|
|
38
|
+
* it actually moved behaviour is confirmed by `scan --trigger`.
|
|
39
|
+
*/
|
|
40
|
+
export type ExplanationConfidence = "likely" | "possible";
|
|
41
|
+
export interface ScoreExplanation {
|
|
42
|
+
/** The affected surface (a skill/agent/hook name or path) the symptom attaches to. */
|
|
43
|
+
readonly surface: string;
|
|
44
|
+
/** What a measurement would SEE. */
|
|
45
|
+
readonly symptom: BehavioralSymptom;
|
|
46
|
+
/** The deterministic finding (the scan/lint detector's own message — no drift). */
|
|
47
|
+
readonly cause: string;
|
|
48
|
+
/** The lint rule that found it, so a reader can open `docs/rules/<detector>.md`. */
|
|
49
|
+
readonly detector: string;
|
|
50
|
+
/** A single, actionable fix. */
|
|
51
|
+
readonly fix: string;
|
|
52
|
+
readonly confidence: ExplanationConfidence;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Explain every behavioral symptom the report's deterministic findings account
|
|
56
|
+
* for. Returns one `ScoreExplanation` per finding, `"likely"` causes first (a hard
|
|
57
|
+
* dead-end is more certain than a proxy). Pure over the report.
|
|
58
|
+
*/
|
|
59
|
+
export declare function explainScore(report: ScanReport): ScoreExplanation[];
|
|
60
|
+
/**
|
|
61
|
+
* The explanations that attach to ONE underperforming surface — the call the
|
|
62
|
+
* benchmark/optimizer makes when a measurement flags a single skill/agent. Matches
|
|
63
|
+
* a surface name case-insensitively, including the `"a ↔ b"` overlap pairs (so
|
|
64
|
+
* explaining "caveman" surfaces an overlap with another skill).
|
|
65
|
+
*/
|
|
66
|
+
export declare function explainSurface(report: ScanReport, surface: string): ScoreExplanation[];
|
|
67
|
+
/** Render explanations for a CLI/report — grouped under the symptom, fix called out. */
|
|
68
|
+
export declare function formatExplanations(exps: readonly ScoreExplanation[]): string;
|
|
69
|
+
//# sourceMappingURL=score-explainer.d.ts.map
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Score-explainer — the deterministic WHY behind a low measured score (C4 of the
|
|
4
|
+
* measurement-authority pivot; the strongest pairing in it).
|
|
5
|
+
*
|
|
6
|
+
* The MEASUREMENT layer finds a behavioral SYMPTOM: "this skill underperforms",
|
|
7
|
+
* "the wrong skill fires", "this agent fails its task". A behavioral symptom alone
|
|
8
|
+
* is not actionable — you can drop the skill, but you don't know WHY it lost. The
|
|
9
|
+
* cross-reference engine (the linting layer) already detects the deterministic
|
|
10
|
+
* CAUSES: two skills with near-identical descriptions can't be told apart by the
|
|
11
|
+
* selector; a never-available / typo'd tool is silently dropped from a subagent; a
|
|
12
|
+
* hook on a misspelled event never fires. This module BRIDGES the two: given the
|
|
13
|
+
* `ScanReport` the linter already computes, it surfaces — per affected surface —
|
|
14
|
+
* the deterministic cause of a behavioral symptom and the one-line fix.
|
|
15
|
+
*
|
|
16
|
+
* Measurement says "caveman underperforms"
|
|
17
|
+
* the explainer says "...BECAUSE its description overlaps `compress` (0.86) —
|
|
18
|
+
* differentiate them so the selector can disambiguate."
|
|
19
|
+
*
|
|
20
|
+
* It REUSES the scan findings (one-detector-no-drift) — it never re-detects. So
|
|
21
|
+
* it's pure over a `ScanReport`, free, model-less, and consistent with `vigiles
|
|
22
|
+
* lint`/`scan`. It's the diagnostic the per-repo optimizer (`vigiles optimize`,
|
|
23
|
+
* A2) prints next to each "drop / swap" recommendation. See
|
|
24
|
+
* `research/measurement-authority.md` ("what becomes of the linting", role 2).
|
|
25
|
+
*/
|
|
26
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
27
|
+
exports.explainScore = explainScore;
|
|
28
|
+
exports.explainSurface = explainSurface;
|
|
29
|
+
exports.formatExplanations = formatExplanations;
|
|
30
|
+
const SYMPTOM_LABEL = {
|
|
31
|
+
"wrong-skill-fires": "the selector fires the wrong skill",
|
|
32
|
+
"skill-never-fires": "the skill never fires",
|
|
33
|
+
"agent-underperforms": "the subagent loses a declared tool",
|
|
34
|
+
"hook-never-runs": "the hook never runs",
|
|
35
|
+
"subagent-never-dispatches": "the subagent won't register",
|
|
36
|
+
};
|
|
37
|
+
// 1. Description overlap → the selector can't disambiguate (precision collision).
|
|
38
|
+
function overlapExplanations(report) {
|
|
39
|
+
return report.descriptionOverlaps.map((o) => ({
|
|
40
|
+
surface: `${o.a} ↔ ${o.b}`,
|
|
41
|
+
symptom: "wrong-skill-fires",
|
|
42
|
+
cause: o.message,
|
|
43
|
+
detector: "description-overlap",
|
|
44
|
+
fix: `Differentiate the descriptions of "${o.a}" and "${o.b}" (${o.similarity} similar) — the selector picks by description, so near-identical text makes it fire the wrong one.`,
|
|
45
|
+
confidence: "possible",
|
|
46
|
+
}));
|
|
47
|
+
}
|
|
48
|
+
// 2. Skill with no usable description → nothing for the selector to match on.
|
|
49
|
+
function skillExplanations(report) {
|
|
50
|
+
return report.skills
|
|
51
|
+
.filter((s) => !s.hasDescription)
|
|
52
|
+
.map((s) => ({
|
|
53
|
+
surface: s.name,
|
|
54
|
+
symptom: "skill-never-fires",
|
|
55
|
+
cause: `"${s.name}" has no usable description`,
|
|
56
|
+
detector: "skill-frontmatter",
|
|
57
|
+
fix: `Add a "description:" to "${s.name}" — the selector matches on it; without one the skill has no trigger surface.`,
|
|
58
|
+
confidence: "likely",
|
|
59
|
+
}));
|
|
60
|
+
}
|
|
61
|
+
// 3. Subagent tool-contract dead entries → the tool is silently dropped.
|
|
62
|
+
function agentExplanations(report) {
|
|
63
|
+
const out = [];
|
|
64
|
+
for (const a of report.agents) {
|
|
65
|
+
for (const t of a.toolIssues) {
|
|
66
|
+
out.push({
|
|
67
|
+
surface: a.name,
|
|
68
|
+
symptom: "agent-underperforms",
|
|
69
|
+
cause: t.message,
|
|
70
|
+
detector: "subagent-tool-contract",
|
|
71
|
+
fix: t.suggestion
|
|
72
|
+
? `In "${a.name}", change the tool "${t.tool}" to "${t.suggestion}" — as written it isn't a real tool, so it's dropped and the agent can't use it.`
|
|
73
|
+
: `In "${a.name}", remove or correct the tool "${t.tool}" — it isn't an available tool, so it's silently dropped from the contract.`,
|
|
74
|
+
confidence: "likely",
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
for (const m of a.mcpToolIssues) {
|
|
78
|
+
out.push({
|
|
79
|
+
surface: a.name,
|
|
80
|
+
symptom: "agent-underperforms",
|
|
81
|
+
cause: m.message,
|
|
82
|
+
detector: "mcp-tool-resolves",
|
|
83
|
+
fix: `"${a.name}" lists the MCP tool "${m.tool}" but its server "${m.server}" isn't declared in the plugin's mcpServers — declare the server or drop the tool, else the call can't resolve.`,
|
|
84
|
+
confidence: "likely",
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return out;
|
|
89
|
+
}
|
|
90
|
+
// 4. Hook on an unknown event, or 5. a missing hook script → the hook never runs.
|
|
91
|
+
function hookExplanations(report) {
|
|
92
|
+
const out = report.hookEventIssues.map((h) => ({
|
|
93
|
+
surface: h.event,
|
|
94
|
+
symptom: "hook-never-runs",
|
|
95
|
+
cause: h.message,
|
|
96
|
+
detector: "hook-events",
|
|
97
|
+
fix: h.suggestion
|
|
98
|
+
? `Change the hook event "${h.event}" to "${h.suggestion}" — the harness doesn't define "${h.event}", so the hook never fires.`
|
|
99
|
+
: `Fix the hook event "${h.event}" — the harness doesn't define it, so the hook never fires.`,
|
|
100
|
+
confidence: "likely",
|
|
101
|
+
}));
|
|
102
|
+
for (const h of report.hooks) {
|
|
103
|
+
if (h.status === "missing") {
|
|
104
|
+
out.push({
|
|
105
|
+
surface: h.script,
|
|
106
|
+
symptom: "hook-never-runs",
|
|
107
|
+
cause: `hook script "${h.script}" does not exist on disk`,
|
|
108
|
+
detector: "hook-script-exists",
|
|
109
|
+
fix: `Create "${h.script}" or fix its path — the hook references a script that isn't on disk, so it silently never runs.`,
|
|
110
|
+
confidence: "likely",
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
return out;
|
|
115
|
+
}
|
|
116
|
+
// 6. Subagent frontmatter missing a required field → it won't register at all.
|
|
117
|
+
function frontmatterExplanations(report) {
|
|
118
|
+
return report.frontmatterIssues
|
|
119
|
+
.filter((f) => f.kind === "agent")
|
|
120
|
+
.map((f) => ({
|
|
121
|
+
surface: f.path,
|
|
122
|
+
symptom: "subagent-never-dispatches",
|
|
123
|
+
cause: f.message,
|
|
124
|
+
detector: "subagent-frontmatter",
|
|
125
|
+
fix: `Add the missing ${f.missing.join(" + ")} to "${f.path}" — a subagent without it won't register, so it can never be dispatched.`,
|
|
126
|
+
confidence: "likely",
|
|
127
|
+
}));
|
|
128
|
+
}
|
|
129
|
+
const confidenceRank = (c) => c === "likely" ? 0 : 1;
|
|
130
|
+
/**
|
|
131
|
+
* Explain every behavioral symptom the report's deterministic findings account
|
|
132
|
+
* for. Returns one `ScoreExplanation` per finding, `"likely"` causes first (a hard
|
|
133
|
+
* dead-end is more certain than a proxy). Pure over the report.
|
|
134
|
+
*/
|
|
135
|
+
function explainScore(report) {
|
|
136
|
+
return [
|
|
137
|
+
...overlapExplanations(report),
|
|
138
|
+
...skillExplanations(report),
|
|
139
|
+
...agentExplanations(report),
|
|
140
|
+
...hookExplanations(report),
|
|
141
|
+
...frontmatterExplanations(report),
|
|
142
|
+
// `likely` before `possible` — surface the certain dead-ends first.
|
|
143
|
+
].sort((x, y) => confidenceRank(x.confidence) - confidenceRank(y.confidence));
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* The explanations that attach to ONE underperforming surface — the call the
|
|
147
|
+
* benchmark/optimizer makes when a measurement flags a single skill/agent. Matches
|
|
148
|
+
* a surface name case-insensitively, including the `"a ↔ b"` overlap pairs (so
|
|
149
|
+
* explaining "caveman" surfaces an overlap with another skill).
|
|
150
|
+
*/
|
|
151
|
+
function explainSurface(report, surface) {
|
|
152
|
+
const needle = surface.toLowerCase();
|
|
153
|
+
return explainScore(report).filter((e) => e.surface.toLowerCase().includes(needle));
|
|
154
|
+
}
|
|
155
|
+
/** Render explanations for a CLI/report — grouped under the symptom, fix called out. */
|
|
156
|
+
function formatExplanations(exps) {
|
|
157
|
+
if (exps.length === 0) {
|
|
158
|
+
return "No deterministic cause found — the cause is likely behavioral (measure with `scan --trigger` / an eval).";
|
|
159
|
+
}
|
|
160
|
+
const lines = [];
|
|
161
|
+
for (const e of exps) {
|
|
162
|
+
const mark = e.confidence === "likely" ? "✗" : "⚠";
|
|
163
|
+
lines.push(`${mark} ${e.surface} — ${SYMPTOM_LABEL[e.symptom]}`);
|
|
164
|
+
lines.push(` cause: ${e.cause} [${e.detector}]`);
|
|
165
|
+
lines.push(` fix: ${e.fix}`);
|
|
166
|
+
}
|
|
167
|
+
return lines.join("\n");
|
|
168
|
+
}
|
|
169
|
+
//# sourceMappingURL=score-explainer.js.map
|
package/dist/test-coverage.d.ts
CHANGED
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
* The only opt-out is explicit: a `vigiles:ignore-test` marker in the surface
|
|
22
22
|
* file, which is reported as `exempt` so the skip is visible, never silent.
|
|
23
23
|
*/
|
|
24
|
+
import type { PluginLayout } from "./core/layout.js";
|
|
24
25
|
export type SurfaceKind = "skill" | "agent" | "hook";
|
|
25
26
|
export interface Surface {
|
|
26
27
|
readonly kind: SurfaceKind;
|
|
@@ -54,6 +55,12 @@ export interface TestCoverageOptions {
|
|
|
54
55
|
readonly testGlobs?: readonly string[];
|
|
55
56
|
/** Extra ignore globs (added to node_modules/dist/.git/.vigiles). */
|
|
56
57
|
readonly exclude?: readonly string[];
|
|
58
|
+
/**
|
|
59
|
+
* Harness layout — where skills/agents live, the plugin-root token, the
|
|
60
|
+
* manifest/settings paths. Defaults to Claude Code; a non-CC adapter passes its
|
|
61
|
+
* own so the surface globs and hook-token expansion aren't hard-coded.
|
|
62
|
+
*/
|
|
63
|
+
readonly layout?: PluginLayout;
|
|
57
64
|
}
|
|
58
65
|
/**
|
|
59
66
|
* Find harness surfaces (skills / agents / hooks) that no test or eval covers.
|