vigiles 9.0.0 → 9.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +147 -209
- package/dist/audit-report.d.ts +37 -0
- package/dist/audit-report.js +21 -0
- package/dist/audit-report.template.html +37 -22
- package/dist/audit-score.d.ts +27 -7
- package/dist/audit-score.js +48 -45
- package/dist/cli.js +136 -5
- package/dist/core/adopt.d.ts +28 -0
- package/dist/core/adopt.js +203 -0
- package/dist/leaderboard.d.ts +32 -0
- package/dist/leaderboard.js +92 -44
- package/dist/scan.js +13 -1
- package/package.json +1 -1
package/dist/core/adopt.js
CHANGED
|
@@ -22,7 +22,11 @@
|
|
|
22
22
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
23
|
exports.adoptToSpec = adoptToSpec;
|
|
24
24
|
exports.adoptMarkdown = adoptMarkdown;
|
|
25
|
+
exports.adoptSkill = adoptSkill;
|
|
26
|
+
exports.adoptAgent = adoptAgent;
|
|
25
27
|
const integrity_js_1 = require("./integrity.js");
|
|
28
|
+
const frontmatter_read_js_1 = require("./frontmatter-read.js");
|
|
29
|
+
const spec_js_1 = require("./spec.js");
|
|
26
30
|
// A top-level heading is `#` or `##` (the levels the compiler reserves for
|
|
27
31
|
// document/section structure). `###`+ stay inside a section body.
|
|
28
32
|
const HEADING_RE = /^ {0,3}(#{1,2})\s+(.*)$/;
|
|
@@ -196,4 +200,203 @@ function adoptMarkdown(markdown, target) {
|
|
|
196
200
|
sectionCount: Object.keys(spec.sections).length,
|
|
197
201
|
};
|
|
198
202
|
}
|
|
203
|
+
// Consumes the WHOLE leading frontmatter block (through its closing `---` and the
|
|
204
|
+
// trailing newline) so the remainder is the verbatim body — mirrors BLOCK_RE in
|
|
205
|
+
// frontmatter-read.ts but matches past the closing fence.
|
|
206
|
+
const FRONTMATTER_CONSUME_RE = /^\uFEFF?(?:<!--[\s\S]*?-->\s*)?---\r?\n[\s\S]*?\r?\n---[ \t]*\r?\n?/;
|
|
207
|
+
function splitFrontmatterBody(markdown) {
|
|
208
|
+
const fm = (0, frontmatter_read_js_1.readFrontmatter)(markdown);
|
|
209
|
+
if (fm.block === null)
|
|
210
|
+
return { fm, body: markdown.replace(/^\uFEFF/, "") };
|
|
211
|
+
const m = FRONTMATTER_CONSUME_RE.exec(markdown);
|
|
212
|
+
return { fm, body: m ? markdown.slice(m[0].length) : markdown };
|
|
213
|
+
}
|
|
214
|
+
/** The first non-empty, non-heading paragraph — the CC fallback for a skill's
|
|
215
|
+
* description when its frontmatter omits one (name←dir, description←first ¶). */
|
|
216
|
+
function firstParagraph(body) {
|
|
217
|
+
const para = [];
|
|
218
|
+
let started = false;
|
|
219
|
+
for (const line of body.split("\n")) {
|
|
220
|
+
const t = line.trim();
|
|
221
|
+
if (!started) {
|
|
222
|
+
if (t === "" || /^#{1,6}\s/.test(t))
|
|
223
|
+
continue;
|
|
224
|
+
started = true;
|
|
225
|
+
para.push(t);
|
|
226
|
+
}
|
|
227
|
+
else {
|
|
228
|
+
if (t === "")
|
|
229
|
+
break;
|
|
230
|
+
para.push(t);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
return para.join(" ").trim();
|
|
234
|
+
}
|
|
235
|
+
/** Keys the typed spec models — everything else is reported as unmapped. */
|
|
236
|
+
const SKILL_KNOWN_KEYS = new Set([
|
|
237
|
+
"name",
|
|
238
|
+
"description",
|
|
239
|
+
"allowed-tools",
|
|
240
|
+
"tools",
|
|
241
|
+
"disable-model-invocation",
|
|
242
|
+
"argument-hint",
|
|
243
|
+
"context",
|
|
244
|
+
]);
|
|
245
|
+
const AGENT_KNOWN_KEYS = new Set([
|
|
246
|
+
"name",
|
|
247
|
+
"description",
|
|
248
|
+
"model",
|
|
249
|
+
"color",
|
|
250
|
+
"tools",
|
|
251
|
+
"disallowedTools",
|
|
252
|
+
"disallowed-tools",
|
|
253
|
+
]);
|
|
254
|
+
function unmappedFrontmatterKeys(fm, known) {
|
|
255
|
+
if (!fm.data)
|
|
256
|
+
return [];
|
|
257
|
+
return Object.keys(fm.data).filter((k) => !known.has(k));
|
|
258
|
+
}
|
|
259
|
+
/** A `// NOTE:` banner naming any frontmatter keys we couldn't represent. */
|
|
260
|
+
function unmappedNote(kind, keys) {
|
|
261
|
+
if (keys.length === 0)
|
|
262
|
+
return "";
|
|
263
|
+
return (`// NOTE: these frontmatter keys had no ${kind}() field and were left out —\n` +
|
|
264
|
+
`// re-add them by hand if they matter: ${keys.join(", ")}\n`);
|
|
265
|
+
}
|
|
266
|
+
const SURFACE_HEADER = (from) => `// Adopted from ${from} by \`vigiles init\` — body verbatim, standard\n` +
|
|
267
|
+
`// frontmatter mapped; no rules inferred. Review the diff, then \`compile\`.\n`;
|
|
268
|
+
/**
|
|
269
|
+
* Adopt an existing SKILL.md into a `skill()` spec. The body is carried verbatim
|
|
270
|
+
* (skills are freeform markdown — `##` headings stay in the body), so a clean
|
|
271
|
+
* skill round-trips below the integrity header.
|
|
272
|
+
*
|
|
273
|
+
* @param markdown the SKILL.md content
|
|
274
|
+
* @param dirName the skill's directory name — the CC fallback for `name` when
|
|
275
|
+
* frontmatter omits it
|
|
276
|
+
*/
|
|
277
|
+
function adoptSkill(markdown, dirName) {
|
|
278
|
+
const { fm, body } = splitFrontmatterBody(markdown);
|
|
279
|
+
const name = (0, frontmatter_read_js_1.frontmatterScalar)(fm, "name") ?? dirName;
|
|
280
|
+
const description = (0, frontmatter_read_js_1.frontmatterScalar)(fm, "description") ?? firstParagraph(body) ?? name;
|
|
281
|
+
const tools = (0, frontmatter_read_js_1.frontmatterList)(fm, "allowed-tools") ?? (0, frontmatter_read_js_1.frontmatterList)(fm, "tools");
|
|
282
|
+
const argumentHint = (0, frontmatter_read_js_1.frontmatterScalar)(fm, "argument-hint");
|
|
283
|
+
const disableModelInvocation = (0, frontmatter_read_js_1.frontmatterScalar)(fm, "disable-model-invocation") === "true";
|
|
284
|
+
const unmappedKeys = unmappedFrontmatterKeys(fm, SKILL_KNOWN_KEYS);
|
|
285
|
+
const lines = [
|
|
286
|
+
` name: ${JSON.stringify(name)},`,
|
|
287
|
+
` description: ${JSON.stringify(description)},`,
|
|
288
|
+
];
|
|
289
|
+
if (argumentHint)
|
|
290
|
+
lines.push(` argumentHint: ${JSON.stringify(argumentHint)},`);
|
|
291
|
+
if (disableModelInvocation)
|
|
292
|
+
lines.push(` disableModelInvocation: true,`);
|
|
293
|
+
if (tools && tools.length > 0)
|
|
294
|
+
lines.push(` tools: ${JSON.stringify(tools)},`);
|
|
295
|
+
const trimmedBody = body.trim();
|
|
296
|
+
if (trimmedBody)
|
|
297
|
+
lines.push(` body: ${tsTemplate(trimmedBody)},`);
|
|
298
|
+
const spec = (0, spec_js_1.skill)({
|
|
299
|
+
name,
|
|
300
|
+
description,
|
|
301
|
+
...(argumentHint ? { argumentHint } : {}),
|
|
302
|
+
...(disableModelInvocation ? { disableModelInvocation: true } : {}),
|
|
303
|
+
...(tools && tools.length > 0 ? { tools } : {}),
|
|
304
|
+
...(trimmedBody ? { body: trimmedBody } : {}),
|
|
305
|
+
});
|
|
306
|
+
const source = SURFACE_HEADER(`${dirName}/SKILL.md`) +
|
|
307
|
+
unmappedNote("skill", unmappedKeys) +
|
|
308
|
+
`import { skill } from "vigiles/spec";\n\n` +
|
|
309
|
+
`export default skill({\n${lines.join("\n")}\n});\n`;
|
|
310
|
+
return { source, kind: "skill", spec, unmappedKeys };
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Adopt an existing subagent (`agents/<name>.md`) into an `agent()` spec. Unlike
|
|
314
|
+
* a skill, an agent's `sections` reject `##` headers, so the body is split: the
|
|
315
|
+
* lead preamble becomes `body` and each `##`/`#` heading becomes a named section
|
|
316
|
+
* (reusing the instruction-file splitter). The tool contract is carried as-is —
|
|
317
|
+
* if the source lists a never-available tool, the generated spec surfaces it on
|
|
318
|
+
* `compile` (which is the point).
|
|
319
|
+
*
|
|
320
|
+
* @param markdown the subagent file content
|
|
321
|
+
* @param fileBase the file's base name (sans `.md`) — the fallback for `name`
|
|
322
|
+
*/
|
|
323
|
+
/** Split a subagent system prompt: preamble → `body`, each `#`/`##` heading →
|
|
324
|
+
* a named section (agent `sections` reject `##` in the body, so they're hoisted). */
|
|
325
|
+
function splitAgentBody(body) {
|
|
326
|
+
const blocks = splitIntoBlocks(body);
|
|
327
|
+
let lead = "";
|
|
328
|
+
const used = new Set();
|
|
329
|
+
const sectionEntries = [];
|
|
330
|
+
for (const block of blocks) {
|
|
331
|
+
if (block.level === null) {
|
|
332
|
+
lead = block.lines.join("\n").trim();
|
|
333
|
+
}
|
|
334
|
+
else {
|
|
335
|
+
sectionEntries.push({
|
|
336
|
+
key: allocKey(safeKey(block.heading ?? ""), used),
|
|
337
|
+
content: block.lines.join("\n").trim(),
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
return { lead, sectionEntries };
|
|
342
|
+
}
|
|
343
|
+
/** Render the `agent({…})` source lines from the extracted fields. */
|
|
344
|
+
function buildAgentLines(f) {
|
|
345
|
+
const lines = [
|
|
346
|
+
` name: ${JSON.stringify(f.name)},`,
|
|
347
|
+
` description: ${JSON.stringify(f.description)},`,
|
|
348
|
+
];
|
|
349
|
+
if (f.model)
|
|
350
|
+
lines.push(` model: ${JSON.stringify(f.model)},`);
|
|
351
|
+
if (f.color)
|
|
352
|
+
lines.push(` color: ${JSON.stringify(f.color)},`);
|
|
353
|
+
if (f.tools && f.tools.length > 0)
|
|
354
|
+
lines.push(` tools: ${JSON.stringify(f.tools)},`);
|
|
355
|
+
if (f.disallowedTools && f.disallowedTools.length > 0)
|
|
356
|
+
lines.push(` disallowedTools: ${JSON.stringify(f.disallowedTools)},`);
|
|
357
|
+
if (f.lead)
|
|
358
|
+
lines.push(` body: ${tsTemplate(f.lead)},`);
|
|
359
|
+
if (f.sectionEntries.length > 0) {
|
|
360
|
+
const entries = f.sectionEntries
|
|
361
|
+
.map(({ key, content }) => ` ${JSON.stringify(key)}: ${tsTemplate(content)},`)
|
|
362
|
+
.join("\n");
|
|
363
|
+
lines.push(` sections: {\n${entries}\n },`);
|
|
364
|
+
}
|
|
365
|
+
return lines;
|
|
366
|
+
}
|
|
367
|
+
function adoptAgent(markdown, fileBase) {
|
|
368
|
+
const { fm, body } = splitFrontmatterBody(markdown);
|
|
369
|
+
const name = (0, frontmatter_read_js_1.frontmatterScalar)(fm, "name") ?? fileBase;
|
|
370
|
+
const f = {
|
|
371
|
+
name,
|
|
372
|
+
description: (0, frontmatter_read_js_1.frontmatterScalar)(fm, "description") ?? name,
|
|
373
|
+
model: (0, frontmatter_read_js_1.frontmatterScalar)(fm, "model"),
|
|
374
|
+
color: (0, frontmatter_read_js_1.frontmatterScalar)(fm, "color"),
|
|
375
|
+
tools: (0, frontmatter_read_js_1.frontmatterList)(fm, "tools"),
|
|
376
|
+
disallowedTools: (0, frontmatter_read_js_1.frontmatterList)(fm, "disallowedTools") ??
|
|
377
|
+
(0, frontmatter_read_js_1.frontmatterList)(fm, "disallowed-tools"),
|
|
378
|
+
...splitAgentBody(body),
|
|
379
|
+
};
|
|
380
|
+
const unmappedKeys = unmappedFrontmatterKeys(fm, AGENT_KNOWN_KEYS);
|
|
381
|
+
const sections = {};
|
|
382
|
+
for (const { key, content } of f.sectionEntries)
|
|
383
|
+
sections[key] = content;
|
|
384
|
+
const spec = (0, spec_js_1.agent)({
|
|
385
|
+
name: f.name,
|
|
386
|
+
description: f.description,
|
|
387
|
+
...(f.model ? { model: f.model } : {}),
|
|
388
|
+
...(f.color ? { color: f.color } : {}),
|
|
389
|
+
...(f.tools && f.tools.length > 0 ? { tools: f.tools } : {}),
|
|
390
|
+
...(f.disallowedTools && f.disallowedTools.length > 0
|
|
391
|
+
? { disallowedTools: f.disallowedTools }
|
|
392
|
+
: {}),
|
|
393
|
+
...(f.lead ? { body: f.lead } : {}),
|
|
394
|
+
...(f.sectionEntries.length > 0 ? { sections } : {}),
|
|
395
|
+
});
|
|
396
|
+
const source = SURFACE_HEADER(`${fileBase}.md`) +
|
|
397
|
+
unmappedNote("agent", unmappedKeys) +
|
|
398
|
+
`import { agent } from "vigiles/spec";\n\n` +
|
|
399
|
+
`export default agent({\n${buildAgentLines(f).join("\n")}\n});\n`;
|
|
400
|
+
return { source, kind: "agent", spec, unmappedKeys };
|
|
401
|
+
}
|
|
199
402
|
//# sourceMappingURL=adopt.js.map
|
package/dist/leaderboard.d.ts
CHANGED
|
@@ -21,8 +21,40 @@ export interface PluginScore {
|
|
|
21
21
|
readonly issues: readonly string[];
|
|
22
22
|
readonly report: ScanReport;
|
|
23
23
|
}
|
|
24
|
+
export declare const W_MISSING_HOOK = 15;
|
|
25
|
+
export declare const W_NO_DESCRIPTION = 10;
|
|
26
|
+
export declare const W_DANGLING_REF = 8;
|
|
27
|
+
export declare const W_OVERLAP = 8;
|
|
28
|
+
export declare const W_NO_CONTRACT = 5;
|
|
24
29
|
/** Map a 0–100 structural-health score to its letter grade (A ≥90 … F <60). */
|
|
25
30
|
export declare function gradeFor(score: number): PluginScore["grade"];
|
|
31
|
+
/** One deduction: a count, its per-item weight, and the label if non-zero. */
|
|
32
|
+
export interface Deduction {
|
|
33
|
+
readonly n: number;
|
|
34
|
+
readonly weight: number;
|
|
35
|
+
readonly label: string;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The COMPLETE graded-penalty list a report incurs — the single source of truth
|
|
39
|
+
* BOTH the leaderboard's single health number and the audit's category rings
|
|
40
|
+
* read, so the overall can never drift between the two surfaces. Each entry is a
|
|
41
|
+
* graded penalty; untested surfaces are deliberately ABSENT (they're advisory,
|
|
42
|
+
* surfaced separately, never scored).
|
|
43
|
+
*/
|
|
44
|
+
export declare function reportDeductions(r: ScanReport): Deduction[];
|
|
45
|
+
/** True when a report has no loadable plugin surface at all (the empty machine). */
|
|
46
|
+
export declare function isEmptyMachine(r: ScanReport): boolean;
|
|
47
|
+
/**
|
|
48
|
+
* THE shared integrity score — `100 − Σ(all graded penalties)`, clamped to
|
|
49
|
+
* [0,100]. Both the leaderboard's single health number AND the audit's headline
|
|
50
|
+
* overall read this, so the two can never disagree (the summed model is the
|
|
51
|
+
* honest one — averaging rings would dilute a real problem). Returns the score
|
|
52
|
+
* plus the per-item deductions so callers render their own issue/finding lists.
|
|
53
|
+
*/
|
|
54
|
+
export declare function computeIntegrityScore(deductions: readonly Deduction[]): {
|
|
55
|
+
score: number;
|
|
56
|
+
penalty: number;
|
|
57
|
+
};
|
|
26
58
|
/** Deterministic structural-health score for one scanned plugin. */
|
|
27
59
|
export declare function scoreReport(r: ScanReport): {
|
|
28
60
|
score: number;
|
package/dist/leaderboard.js
CHANGED
|
@@ -12,7 +12,11 @@
|
|
|
12
12
|
* model and stack on top later; this part runs anywhere in CI for free.
|
|
13
13
|
*/
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.W_NO_CONTRACT = exports.W_OVERLAP = exports.W_DANGLING_REF = exports.W_NO_DESCRIPTION = exports.W_MISSING_HOOK = void 0;
|
|
15
16
|
exports.gradeFor = gradeFor;
|
|
17
|
+
exports.reportDeductions = reportDeductions;
|
|
18
|
+
exports.isEmptyMachine = isEmptyMachine;
|
|
19
|
+
exports.computeIntegrityScore = computeIntegrityScore;
|
|
16
20
|
exports.scoreReport = scoreReport;
|
|
17
21
|
exports.rankPlugins = rankPlugins;
|
|
18
22
|
exports.formatLeaderboard = formatLeaderboard;
|
|
@@ -38,11 +42,14 @@ function pluginLabel(dir) {
|
|
|
38
42
|
return (0, node_path_1.basename)(dir) || dir;
|
|
39
43
|
}
|
|
40
44
|
// Penalty weights — broken-at-runtime costs most, footguns less, nudges least.
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
45
|
+
// Exported so the category view (audit-score.ts) reuses the SAME weights and the
|
|
46
|
+
// two surfaces can never drift on a per-item cost.
|
|
47
|
+
exports.W_MISSING_HOOK = 15; // a hook script that doesn't exist → never runs
|
|
48
|
+
exports.W_NO_DESCRIPTION = 10; // a skill with no usable description → can't trigger
|
|
49
|
+
exports.W_DANGLING_REF = 8; // a referenced intra-plugin file that's missing → broken path
|
|
50
|
+
exports.W_OVERLAP = 8; // a description collision → the wrong skill fires
|
|
51
|
+
exports.W_NO_CONTRACT = 5; // an agent with no `tools:` line → inherits everything
|
|
52
|
+
// (untested surfaces are advisory, not a penalty — see scoreReport)
|
|
46
53
|
/** Map a 0–100 structural-health score to its letter grade (A ≥90 … F <60). */
|
|
47
54
|
function gradeFor(score) {
|
|
48
55
|
if (score >= 90)
|
|
@@ -55,101 +62,142 @@ function gradeFor(score) {
|
|
|
55
62
|
return "D";
|
|
56
63
|
return "F";
|
|
57
64
|
}
|
|
58
|
-
/**
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
return { score: 0, issues: ["no loadable plugin surface"] };
|
|
67
|
-
}
|
|
65
|
+
/**
|
|
66
|
+
* The COMPLETE graded-penalty list a report incurs — the single source of truth
|
|
67
|
+
* BOTH the leaderboard's single health number and the audit's category rings
|
|
68
|
+
* read, so the overall can never drift between the two surfaces. Each entry is a
|
|
69
|
+
* graded penalty; untested surfaces are deliberately ABSENT (they're advisory,
|
|
70
|
+
* surfaced separately, never scored).
|
|
71
|
+
*/
|
|
72
|
+
function reportDeductions(r) {
|
|
68
73
|
const missingHooks = r.hooks.filter((h) => h.status === "missing").length;
|
|
69
74
|
const noDesc = r.skills.filter((s) => !s.hasDescription).length;
|
|
70
75
|
const noContract = r.agents.filter((a) => a.tools === null).length;
|
|
71
76
|
const deadTools = r.agents.reduce((n, a) => n + a.toolIssues.length, 0);
|
|
72
77
|
const deadMcpTools = r.agents.reduce((n, a) => n + a.mcpToolIssues.length, 0);
|
|
73
78
|
const deadDisallowed = r.agents.reduce((n, a) => n + a.disallowedToolIssues.length, 0);
|
|
74
|
-
|
|
75
|
-
const badFrontmatter = r.frontmatterIssues.length;
|
|
76
|
-
const badFrontmatterValues = r.frontmatterValueIssues.length;
|
|
77
|
-
const badMcp = r.mcpIssues.length;
|
|
78
|
-
const badMcpHooks = r.mcpHookIssues.length;
|
|
79
|
-
const deductions = [
|
|
79
|
+
return [
|
|
80
80
|
{
|
|
81
81
|
n: missingHooks,
|
|
82
|
-
weight: W_MISSING_HOOK,
|
|
82
|
+
weight: exports.W_MISSING_HOOK,
|
|
83
83
|
label: "hook script(s) MISSING",
|
|
84
84
|
},
|
|
85
85
|
{
|
|
86
|
-
n:
|
|
87
|
-
weight: W_MISSING_HOOK,
|
|
86
|
+
n: r.hookEventIssues.length,
|
|
87
|
+
weight: exports.W_MISSING_HOOK,
|
|
88
88
|
label: "hook(s) on an unknown event (never fire)",
|
|
89
89
|
},
|
|
90
90
|
{
|
|
91
91
|
n: noDesc,
|
|
92
|
-
weight: W_NO_DESCRIPTION,
|
|
92
|
+
weight: exports.W_NO_DESCRIPTION,
|
|
93
93
|
label: "skill(s) with no usable description",
|
|
94
94
|
},
|
|
95
|
+
{
|
|
96
|
+
n: r.descriptionOverlaps.length,
|
|
97
|
+
weight: exports.W_OVERLAP,
|
|
98
|
+
label: "near-identical skill description(s) (wrong one fires)",
|
|
99
|
+
},
|
|
95
100
|
{
|
|
96
101
|
n: r.danglingRefs.length,
|
|
97
|
-
weight: W_DANGLING_REF,
|
|
102
|
+
weight: exports.W_DANGLING_REF,
|
|
98
103
|
label: "broken intra-plugin reference(s)",
|
|
99
104
|
},
|
|
100
105
|
{
|
|
101
106
|
n: deadTools,
|
|
102
|
-
weight: W_DANGLING_REF,
|
|
107
|
+
weight: exports.W_DANGLING_REF,
|
|
103
108
|
label: "agent tool(s) that don't exist (typo / never-available)",
|
|
104
109
|
},
|
|
105
110
|
{
|
|
106
111
|
n: deadMcpTools,
|
|
107
|
-
weight: W_DANGLING_REF,
|
|
112
|
+
weight: exports.W_DANGLING_REF,
|
|
108
113
|
label: "agent MCP tool(s) whose server isn't declared (can't resolve)",
|
|
109
114
|
},
|
|
110
115
|
{
|
|
111
116
|
n: deadDisallowed,
|
|
112
|
-
weight: W_NO_CONTRACT,
|
|
117
|
+
weight: exports.W_NO_CONTRACT,
|
|
113
118
|
label: "agent disallowedTools typo(s) that block nothing",
|
|
114
119
|
},
|
|
115
120
|
{
|
|
116
121
|
n: noContract,
|
|
117
|
-
weight: W_NO_CONTRACT,
|
|
122
|
+
weight: exports.W_NO_CONTRACT,
|
|
118
123
|
label: "agent(s) inherit all tools (no contract)",
|
|
119
124
|
},
|
|
120
125
|
{
|
|
121
|
-
n:
|
|
122
|
-
weight: W_NO_DESCRIPTION,
|
|
126
|
+
n: r.frontmatterIssues.length,
|
|
127
|
+
weight: exports.W_NO_DESCRIPTION,
|
|
123
128
|
label: "surface(s) missing required frontmatter (name/description)",
|
|
124
129
|
},
|
|
125
130
|
{
|
|
126
|
-
n:
|
|
127
|
-
weight: W_NO_CONTRACT,
|
|
131
|
+
n: r.frontmatterValueIssues.length,
|
|
132
|
+
weight: exports.W_NO_CONTRACT,
|
|
128
133
|
label: "agent(s) with an invalid model/color (typo → silent fallback)",
|
|
129
134
|
},
|
|
130
135
|
{
|
|
131
|
-
n:
|
|
132
|
-
weight: W_DANGLING_REF,
|
|
136
|
+
n: r.mcpIssues.length,
|
|
137
|
+
weight: exports.W_DANGLING_REF,
|
|
133
138
|
label: "MCP server(s) that can't start (no command/url)",
|
|
134
139
|
},
|
|
135
140
|
{
|
|
136
|
-
n:
|
|
137
|
-
weight: W_DANGLING_REF,
|
|
141
|
+
n: r.mcpHookIssues.length,
|
|
142
|
+
weight: exports.W_DANGLING_REF,
|
|
138
143
|
label: "mcp_tool hook(s) incomplete / targeting an undeclared server",
|
|
139
144
|
},
|
|
140
|
-
|
|
145
|
+
// NB: untested surfaces are NOT a penalty — an untested surface is a hardening
|
|
146
|
+
// gap, not breakage, so it never drags the health score (it's appended as an
|
|
147
|
+
// advisory note below). The score ranks what's BROKEN.
|
|
141
148
|
];
|
|
149
|
+
}
|
|
150
|
+
/** True when a report has no loadable plugin surface at all (the empty machine). */
|
|
151
|
+
function isEmptyMachine(r) {
|
|
152
|
+
const surfaces = r.skills.length +
|
|
153
|
+
r.agents.length +
|
|
154
|
+
r.hooks.length +
|
|
155
|
+
r.inlineHooks +
|
|
156
|
+
r.commands;
|
|
157
|
+
return surfaces === 0 && !r.mcp;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* THE shared integrity score — `100 − Σ(all graded penalties)`, clamped to
|
|
161
|
+
* [0,100]. Both the leaderboard's single health number AND the audit's headline
|
|
162
|
+
* overall read this, so the two can never disagree (the summed model is the
|
|
163
|
+
* honest one — averaging rings would dilute a real problem). Returns the score
|
|
164
|
+
* plus the per-item deductions so callers render their own issue/finding lists.
|
|
165
|
+
*/
|
|
166
|
+
function computeIntegrityScore(deductions) {
|
|
142
167
|
let penalty = 0;
|
|
168
|
+
for (const d of deductions) {
|
|
169
|
+
if (d.n <= 0)
|
|
170
|
+
continue;
|
|
171
|
+
penalty += d.n * d.weight;
|
|
172
|
+
}
|
|
173
|
+
return { score: Math.max(0, 100 - penalty), penalty };
|
|
174
|
+
}
|
|
175
|
+
/** Deterministic structural-health score for one scanned plugin. */
|
|
176
|
+
function scoreReport(r) {
|
|
177
|
+
// An empty/unloadable machine isn't healthy — it's a non-plugin or a broken
|
|
178
|
+
// load. A command-only or MCP-only plugin (commands/*.md or .mcp.json with no
|
|
179
|
+
// skills/agents/hooks) IS a legitimate plugin, though — Anthropic ships
|
|
180
|
+
// command-only plugins in its own marketplace — so it must NOT score 0.
|
|
181
|
+
const surfaces = r.skills.length + r.agents.length + r.hooks.length + r.commands;
|
|
182
|
+
if (surfaces === 0 && !r.mcp) {
|
|
183
|
+
return { score: 0, issues: ["no loadable plugin surface"] };
|
|
184
|
+
}
|
|
185
|
+
const deductions = reportDeductions(r);
|
|
186
|
+
const { score } = computeIntegrityScore(deductions);
|
|
143
187
|
const issues = [];
|
|
144
188
|
for (const d of deductions) {
|
|
145
189
|
if (d.n === 0)
|
|
146
190
|
continue;
|
|
147
|
-
penalty += d.n * d.weight;
|
|
148
191
|
issues.push(`${String(d.n)} ${d.label}`);
|
|
149
192
|
}
|
|
150
193
|
// Sort issues by cost (worst first) so the report leads with what matters.
|
|
151
194
|
issues.sort((a, b) => Number(b.split(" ")[0]) - Number(a.split(" ")[0]));
|
|
152
|
-
|
|
195
|
+
// Untested surfaces are advisory — surfaced for visibility, but they don't
|
|
196
|
+
// affect the score, so they come AFTER the real (score-affecting) issues.
|
|
197
|
+
if (r.untested > 0) {
|
|
198
|
+
issues.push(`${String(r.untested)} untested surface(s) (advisory)`);
|
|
199
|
+
}
|
|
200
|
+
return { score, issues };
|
|
153
201
|
}
|
|
154
202
|
/** Scan + score each directory, ranked best-first (ties broken by name). */
|
|
155
203
|
function rankPlugins(dirs) {
|
|
@@ -180,13 +228,13 @@ function formatLeaderboard(scores) {
|
|
|
180
228
|
const issue = s.issues.length > 0 ? ` — ${s.issues.join("; ")}` : "";
|
|
181
229
|
out.push(` ${rank} ${score} ${s.grade} ${s.name}${issue}`);
|
|
182
230
|
});
|
|
183
|
-
out.push("", "Structural health only (no model). Weights: missing hook -15, no-description", "skill -10, broken intra-plugin ref -8, agent-without-tool-contract -5
|
|
231
|
+
out.push("", "Structural health only (no model). Weights: missing hook -15, no-description", "skill -10, broken intra-plugin ref -8, agent-without-tool-contract -5.", "Untested surfaces are advisory — shown, but they don't affect the score.");
|
|
184
232
|
return out.join("\n");
|
|
185
233
|
}
|
|
186
234
|
const LEADERBOARD_METHOD = "_Structural health only (deterministic, no model): missing hook −15, " +
|
|
187
235
|
"no-description skill −10, broken intra-plugin ref −8, " +
|
|
188
|
-
"agent-without-tool-contract −5
|
|
189
|
-
"Behavioural columns (trigger-rate, collisions, egress) stack on top._";
|
|
236
|
+
"agent-without-tool-contract −5. Untested surfaces are advisory (shown, not " +
|
|
237
|
+
"scored). Behavioural columns (trigger-rate, collisions, egress) stack on top._";
|
|
190
238
|
/**
|
|
191
239
|
* Format a ranked leaderboard as a Markdown table — the PUBLISHABLE form (a README,
|
|
192
240
|
* a gist, the leaderboard site). Shows the top 2 deductions per plugin; the full
|
package/dist/scan.js
CHANGED
|
@@ -69,9 +69,21 @@ function makeClassifier(layout) {
|
|
|
69
69
|
const skillRe = skill ? new RegExp(`${skill}[^/]+/SKILL\\.md$`) : null;
|
|
70
70
|
const agentRe = agent ? new RegExp(`${agent}[^/]+\\.md$`) : null;
|
|
71
71
|
const commandRe = command ? new RegExp(`${command}.+\\.md$`) : null;
|
|
72
|
+
// A subagent lives at the plugin's TOP-LEVEL `agents/` dir, never recursively
|
|
73
|
+
// under a skill (`skills/<x>/agents/`). Those nested files are skill-internal
|
|
74
|
+
// worker docs (e.g. Anthropic's own skill-creator), NOT dispatchable Claude
|
|
75
|
+
// Code subagents — flagging them is a false positive. The agent dir nested
|
|
76
|
+
// under the skill dir is excluded; a genuine top-level `agents/foo.md` still
|
|
77
|
+
// matches. See scan.test.ts for the regression.
|
|
78
|
+
const nestedAgentRe = layout.skillDir && layout.agentDir
|
|
79
|
+
? new RegExp(`(?:^|/)${escapeRe(layout.skillDir)}/.+/${escapeRe(layout.agentDir)}/`)
|
|
80
|
+
: null;
|
|
81
|
+
const isAgent = (f) => (agentRe?.test(f) ?? false) &&
|
|
82
|
+
!f.endsWith(".spec.ts") &&
|
|
83
|
+
!(nestedAgentRe?.test(f) ?? false);
|
|
72
84
|
return {
|
|
73
85
|
isSkill: (f) => skillRe?.test(f) ?? false,
|
|
74
|
-
isAgent
|
|
86
|
+
isAgent,
|
|
75
87
|
isCommand: (f) => commandRe?.test(f) ?? false,
|
|
76
88
|
};
|
|
77
89
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "vigiles",
|
|
3
|
-
"version": "9.
|
|
3
|
+
"version": "9.1.0",
|
|
4
4
|
"description": "Lint & test the harness your AI agent runs on — verify the references in your CLAUDE.md / AGENTS.md and test that your hooks and skills actually work.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"claude-code",
|