vigiles 9.0.0 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +144 -222
- package/dist/audit-html.d.ts +15 -4
- package/dist/audit-html.js +15 -6
- package/dist/audit-report.d.ts +94 -1
- package/dist/audit-report.js +50 -0
- package/dist/audit-report.template.html +47 -22
- package/dist/audit-score.d.ts +27 -7
- package/dist/audit-score.js +63 -50
- package/dist/audit-serve.d.ts +109 -0
- package/dist/audit-serve.js +257 -0
- package/dist/cli.js +322 -25
- package/dist/core/adopt.d.ts +28 -0
- package/dist/core/adopt.js +203 -0
- package/dist/core/compile.d.ts +5 -1
- package/dist/core/compile.js +19 -10
- package/dist/leaderboard.d.ts +32 -0
- package/dist/leaderboard.js +109 -50
- package/dist/scan-behavioral.d.ts +85 -0
- package/dist/scan-behavioral.js +225 -0
- package/dist/scan.js +23 -1
- package/package.json +1 -1
package/dist/core/adopt.js
CHANGED
|
@@ -22,7 +22,11 @@
|
|
|
22
22
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
23
|
exports.adoptToSpec = adoptToSpec;
|
|
24
24
|
exports.adoptMarkdown = adoptMarkdown;
|
|
25
|
+
exports.adoptSkill = adoptSkill;
|
|
26
|
+
exports.adoptAgent = adoptAgent;
|
|
25
27
|
const integrity_js_1 = require("./integrity.js");
|
|
28
|
+
const frontmatter_read_js_1 = require("./frontmatter-read.js");
|
|
29
|
+
const spec_js_1 = require("./spec.js");
|
|
26
30
|
// A top-level heading is `#` or `##` (the levels the compiler reserves for
|
|
27
31
|
// document/section structure). `###`+ stay inside a section body.
|
|
28
32
|
const HEADING_RE = /^ {0,3}(#{1,2})\s+(.*)$/;
|
|
@@ -196,4 +200,203 @@ function adoptMarkdown(markdown, target) {
|
|
|
196
200
|
sectionCount: Object.keys(spec.sections).length,
|
|
197
201
|
};
|
|
198
202
|
}
|
|
203
|
+
// Consumes the WHOLE leading frontmatter block (through its closing `---` and the
|
|
204
|
+
// trailing newline) so the remainder is the verbatim body — mirrors BLOCK_RE in
|
|
205
|
+
// frontmatter-read.ts but matches past the closing fence.
|
|
206
|
+
const FRONTMATTER_CONSUME_RE = /^\uFEFF?(?:<!--[\s\S]*?-->\s*)?---\r?\n[\s\S]*?\r?\n---[ \t]*\r?\n?/;
|
|
207
|
+
function splitFrontmatterBody(markdown) {
|
|
208
|
+
const fm = (0, frontmatter_read_js_1.readFrontmatter)(markdown);
|
|
209
|
+
if (fm.block === null)
|
|
210
|
+
return { fm, body: markdown.replace(/^\uFEFF/, "") };
|
|
211
|
+
const m = FRONTMATTER_CONSUME_RE.exec(markdown);
|
|
212
|
+
return { fm, body: m ? markdown.slice(m[0].length) : markdown };
|
|
213
|
+
}
|
|
214
|
+
/** The first non-empty, non-heading paragraph — the CC fallback for a skill's
|
|
215
|
+
* description when its frontmatter omits one (name←dir, description←first ¶). */
|
|
216
|
+
function firstParagraph(body) {
|
|
217
|
+
const para = [];
|
|
218
|
+
let started = false;
|
|
219
|
+
for (const line of body.split("\n")) {
|
|
220
|
+
const t = line.trim();
|
|
221
|
+
if (!started) {
|
|
222
|
+
if (t === "" || /^#{1,6}\s/.test(t))
|
|
223
|
+
continue;
|
|
224
|
+
started = true;
|
|
225
|
+
para.push(t);
|
|
226
|
+
}
|
|
227
|
+
else {
|
|
228
|
+
if (t === "")
|
|
229
|
+
break;
|
|
230
|
+
para.push(t);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
return para.join(" ").trim();
|
|
234
|
+
}
|
|
235
|
+
/** Keys the typed spec models — everything else is reported as unmapped. */
|
|
236
|
+
const SKILL_KNOWN_KEYS = new Set([
|
|
237
|
+
"name",
|
|
238
|
+
"description",
|
|
239
|
+
"allowed-tools",
|
|
240
|
+
"tools",
|
|
241
|
+
"disable-model-invocation",
|
|
242
|
+
"argument-hint",
|
|
243
|
+
"context",
|
|
244
|
+
]);
|
|
245
|
+
const AGENT_KNOWN_KEYS = new Set([
|
|
246
|
+
"name",
|
|
247
|
+
"description",
|
|
248
|
+
"model",
|
|
249
|
+
"color",
|
|
250
|
+
"tools",
|
|
251
|
+
"disallowedTools",
|
|
252
|
+
"disallowed-tools",
|
|
253
|
+
]);
|
|
254
|
+
function unmappedFrontmatterKeys(fm, known) {
|
|
255
|
+
if (!fm.data)
|
|
256
|
+
return [];
|
|
257
|
+
return Object.keys(fm.data).filter((k) => !known.has(k));
|
|
258
|
+
}
|
|
259
|
+
/** A `// NOTE:` banner naming any frontmatter keys we couldn't represent. */
|
|
260
|
+
function unmappedNote(kind, keys) {
|
|
261
|
+
if (keys.length === 0)
|
|
262
|
+
return "";
|
|
263
|
+
return (`// NOTE: these frontmatter keys had no ${kind}() field and were left out —\n` +
|
|
264
|
+
`// re-add them by hand if they matter: ${keys.join(", ")}\n`);
|
|
265
|
+
}
|
|
266
|
+
const SURFACE_HEADER = (from) => `// Adopted from ${from} by \`vigiles init\` — body verbatim, standard\n` +
|
|
267
|
+
`// frontmatter mapped; no rules inferred. Review the diff, then \`compile\`.\n`;
|
|
268
|
+
/**
|
|
269
|
+
* Adopt an existing SKILL.md into a `skill()` spec. The body is carried verbatim
|
|
270
|
+
* (skills are freeform markdown — `##` headings stay in the body), so a clean
|
|
271
|
+
* skill round-trips below the integrity header.
|
|
272
|
+
*
|
|
273
|
+
* @param markdown the SKILL.md content
|
|
274
|
+
* @param dirName the skill's directory name — the CC fallback for `name` when
|
|
275
|
+
* frontmatter omits it
|
|
276
|
+
*/
|
|
277
|
+
function adoptSkill(markdown, dirName) {
|
|
278
|
+
const { fm, body } = splitFrontmatterBody(markdown);
|
|
279
|
+
const name = (0, frontmatter_read_js_1.frontmatterScalar)(fm, "name") ?? dirName;
|
|
280
|
+
const description = (0, frontmatter_read_js_1.frontmatterScalar)(fm, "description") ?? firstParagraph(body) ?? name;
|
|
281
|
+
const tools = (0, frontmatter_read_js_1.frontmatterList)(fm, "allowed-tools") ?? (0, frontmatter_read_js_1.frontmatterList)(fm, "tools");
|
|
282
|
+
const argumentHint = (0, frontmatter_read_js_1.frontmatterScalar)(fm, "argument-hint");
|
|
283
|
+
const disableModelInvocation = (0, frontmatter_read_js_1.frontmatterScalar)(fm, "disable-model-invocation") === "true";
|
|
284
|
+
const unmappedKeys = unmappedFrontmatterKeys(fm, SKILL_KNOWN_KEYS);
|
|
285
|
+
const lines = [
|
|
286
|
+
` name: ${JSON.stringify(name)},`,
|
|
287
|
+
` description: ${JSON.stringify(description)},`,
|
|
288
|
+
];
|
|
289
|
+
if (argumentHint)
|
|
290
|
+
lines.push(` argumentHint: ${JSON.stringify(argumentHint)},`);
|
|
291
|
+
if (disableModelInvocation)
|
|
292
|
+
lines.push(` disableModelInvocation: true,`);
|
|
293
|
+
if (tools && tools.length > 0)
|
|
294
|
+
lines.push(` tools: ${JSON.stringify(tools)},`);
|
|
295
|
+
const trimmedBody = body.trim();
|
|
296
|
+
if (trimmedBody)
|
|
297
|
+
lines.push(` body: ${tsTemplate(trimmedBody)},`);
|
|
298
|
+
const spec = (0, spec_js_1.skill)({
|
|
299
|
+
name,
|
|
300
|
+
description,
|
|
301
|
+
...(argumentHint ? { argumentHint } : {}),
|
|
302
|
+
...(disableModelInvocation ? { disableModelInvocation: true } : {}),
|
|
303
|
+
...(tools && tools.length > 0 ? { tools } : {}),
|
|
304
|
+
...(trimmedBody ? { body: trimmedBody } : {}),
|
|
305
|
+
});
|
|
306
|
+
const source = SURFACE_HEADER(`${dirName}/SKILL.md`) +
|
|
307
|
+
unmappedNote("skill", unmappedKeys) +
|
|
308
|
+
`import { skill } from "vigiles/spec";\n\n` +
|
|
309
|
+
`export default skill({\n${lines.join("\n")}\n});\n`;
|
|
310
|
+
return { source, kind: "skill", spec, unmappedKeys };
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Adopt an existing subagent (`agents/<name>.md`) into an `agent()` spec. Unlike
|
|
314
|
+
* a skill, an agent's `sections` reject `##` headers, so the body is split: the
|
|
315
|
+
* lead preamble becomes `body` and each `##`/`#` heading becomes a named section
|
|
316
|
+
* (reusing the instruction-file splitter). The tool contract is carried as-is —
|
|
317
|
+
* if the source lists a never-available tool, the generated spec surfaces it on
|
|
318
|
+
* `compile` (which is the point).
|
|
319
|
+
*
|
|
320
|
+
* @param markdown the subagent file content
|
|
321
|
+
* @param fileBase the file's base name (sans `.md`) — the fallback for `name`
|
|
322
|
+
*/
|
|
323
|
+
/** Split a subagent system prompt: preamble → `body`, each `#`/`##` heading →
|
|
324
|
+
* a named section (agent `sections` reject `##` in the body, so they're hoisted). */
|
|
325
|
+
function splitAgentBody(body) {
|
|
326
|
+
const blocks = splitIntoBlocks(body);
|
|
327
|
+
let lead = "";
|
|
328
|
+
const used = new Set();
|
|
329
|
+
const sectionEntries = [];
|
|
330
|
+
for (const block of blocks) {
|
|
331
|
+
if (block.level === null) {
|
|
332
|
+
lead = block.lines.join("\n").trim();
|
|
333
|
+
}
|
|
334
|
+
else {
|
|
335
|
+
sectionEntries.push({
|
|
336
|
+
key: allocKey(safeKey(block.heading ?? ""), used),
|
|
337
|
+
content: block.lines.join("\n").trim(),
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
return { lead, sectionEntries };
|
|
342
|
+
}
|
|
343
|
+
/** Render the `agent({…})` source lines from the extracted fields. */
|
|
344
|
+
function buildAgentLines(f) {
|
|
345
|
+
const lines = [
|
|
346
|
+
` name: ${JSON.stringify(f.name)},`,
|
|
347
|
+
` description: ${JSON.stringify(f.description)},`,
|
|
348
|
+
];
|
|
349
|
+
if (f.model)
|
|
350
|
+
lines.push(` model: ${JSON.stringify(f.model)},`);
|
|
351
|
+
if (f.color)
|
|
352
|
+
lines.push(` color: ${JSON.stringify(f.color)},`);
|
|
353
|
+
if (f.tools && f.tools.length > 0)
|
|
354
|
+
lines.push(` tools: ${JSON.stringify(f.tools)},`);
|
|
355
|
+
if (f.disallowedTools && f.disallowedTools.length > 0)
|
|
356
|
+
lines.push(` disallowedTools: ${JSON.stringify(f.disallowedTools)},`);
|
|
357
|
+
if (f.lead)
|
|
358
|
+
lines.push(` body: ${tsTemplate(f.lead)},`);
|
|
359
|
+
if (f.sectionEntries.length > 0) {
|
|
360
|
+
const entries = f.sectionEntries
|
|
361
|
+
.map(({ key, content }) => ` ${JSON.stringify(key)}: ${tsTemplate(content)},`)
|
|
362
|
+
.join("\n");
|
|
363
|
+
lines.push(` sections: {\n${entries}\n },`);
|
|
364
|
+
}
|
|
365
|
+
return lines;
|
|
366
|
+
}
|
|
367
|
+
function adoptAgent(markdown, fileBase) {
|
|
368
|
+
const { fm, body } = splitFrontmatterBody(markdown);
|
|
369
|
+
const name = (0, frontmatter_read_js_1.frontmatterScalar)(fm, "name") ?? fileBase;
|
|
370
|
+
const f = {
|
|
371
|
+
name,
|
|
372
|
+
description: (0, frontmatter_read_js_1.frontmatterScalar)(fm, "description") ?? name,
|
|
373
|
+
model: (0, frontmatter_read_js_1.frontmatterScalar)(fm, "model"),
|
|
374
|
+
color: (0, frontmatter_read_js_1.frontmatterScalar)(fm, "color"),
|
|
375
|
+
tools: (0, frontmatter_read_js_1.frontmatterList)(fm, "tools"),
|
|
376
|
+
disallowedTools: (0, frontmatter_read_js_1.frontmatterList)(fm, "disallowedTools") ??
|
|
377
|
+
(0, frontmatter_read_js_1.frontmatterList)(fm, "disallowed-tools"),
|
|
378
|
+
...splitAgentBody(body),
|
|
379
|
+
};
|
|
380
|
+
const unmappedKeys = unmappedFrontmatterKeys(fm, AGENT_KNOWN_KEYS);
|
|
381
|
+
const sections = {};
|
|
382
|
+
for (const { key, content } of f.sectionEntries)
|
|
383
|
+
sections[key] = content;
|
|
384
|
+
const spec = (0, spec_js_1.agent)({
|
|
385
|
+
name: f.name,
|
|
386
|
+
description: f.description,
|
|
387
|
+
...(f.model ? { model: f.model } : {}),
|
|
388
|
+
...(f.color ? { color: f.color } : {}),
|
|
389
|
+
...(f.tools && f.tools.length > 0 ? { tools: f.tools } : {}),
|
|
390
|
+
...(f.disallowedTools && f.disallowedTools.length > 0
|
|
391
|
+
? { disallowedTools: f.disallowedTools }
|
|
392
|
+
: {}),
|
|
393
|
+
...(f.lead ? { body: f.lead } : {}),
|
|
394
|
+
...(f.sectionEntries.length > 0 ? { sections } : {}),
|
|
395
|
+
});
|
|
396
|
+
const source = SURFACE_HEADER(`${fileBase}.md`) +
|
|
397
|
+
unmappedNote("agent", unmappedKeys) +
|
|
398
|
+
`import { agent } from "vigiles/spec";\n\n` +
|
|
399
|
+
`export default agent({\n${buildAgentLines(f).join("\n")}\n});\n`;
|
|
400
|
+
return { source, kind: "agent", spec, unmappedKeys };
|
|
401
|
+
}
|
|
199
402
|
//# sourceMappingURL=adopt.js.map
|
package/dist/core/compile.d.ts
CHANGED
|
@@ -25,7 +25,7 @@ export declare function verifyHash(content: string): {
|
|
|
25
25
|
*/
|
|
26
26
|
/** @internal */ export declare function estimateTokens(text: string): number;
|
|
27
27
|
export interface CompileError {
|
|
28
|
-
type: "stale-file" | "stale-command" | "stale-ref" | "invalid-rule" | "budget-exceeded" | "section-too-long" | "section-has-header" | "reserved-section-key" | "spec-name-mismatch" | "unknown-tool" | "invalid-railway" | "purity-violation" | "output-without-fork" | "effect-in-skill";
|
|
28
|
+
type: "stale-file" | "stale-command" | "stale-ref" | "invalid-rule" | "budget-exceeded" | "section-too-long" | "section-has-header" | "reserved-section-key" | "spec-name-mismatch" | "unknown-tool" | "invalid-railway" | "purity-violation" | "output-without-fork" | "effect-in-skill" | "inline-code-too-long";
|
|
29
29
|
message: string;
|
|
30
30
|
path?: string;
|
|
31
31
|
}
|
|
@@ -76,6 +76,8 @@ export declare function compileClaude(spec: ClaudeSpec, options?: CompileClaudeO
|
|
|
76
76
|
export interface CompileSkillResult {
|
|
77
77
|
markdown: string;
|
|
78
78
|
errors: CompileError[];
|
|
79
|
+
/** Non-blocking advisories (e.g. an over-long inline code block). */
|
|
80
|
+
warnings: CompileError[];
|
|
79
81
|
}
|
|
80
82
|
/**
|
|
81
83
|
* Compile a SkillSpec into SKILL.md markdown with YAML frontmatter.
|
|
@@ -90,6 +92,8 @@ export declare function compileSkill(spec: SkillSpec, options?: {
|
|
|
90
92
|
export interface CompileAgentResult {
|
|
91
93
|
markdown: string;
|
|
92
94
|
errors: CompileError[];
|
|
95
|
+
/** Non-blocking advisories (e.g. an over-long inline code block). */
|
|
96
|
+
warnings: CompileError[];
|
|
93
97
|
}
|
|
94
98
|
/**
|
|
95
99
|
* Compile an AgentSpec into a subagent markdown file with YAML frontmatter.
|
package/dist/core/compile.js
CHANGED
|
@@ -664,11 +664,17 @@ function renderSkillSections(spec) {
|
|
|
664
664
|
return sections.join("\n\n");
|
|
665
665
|
}
|
|
666
666
|
const DEFAULT_MAX_INLINE_CODE_LINES = 20;
|
|
667
|
-
/**
|
|
667
|
+
/**
|
|
668
|
+
* Flag inline fenced code blocks longer than `max` lines (0 = disabled). These
|
|
669
|
+
* are WARNINGS, not errors: a big inline code block is an authoring smell worth
|
|
670
|
+
* surfacing ("extract it to a file"), but it never breaks the harness — and a
|
|
671
|
+
* faithful adoption of an existing skill/subagent (`init`) must still compile.
|
|
672
|
+
* Callers route the result into a result's `warnings` channel, never `errors`.
|
|
673
|
+
*/
|
|
668
674
|
function checkInlineCode(markdown, max) {
|
|
669
675
|
if (max <= 0)
|
|
670
676
|
return [];
|
|
671
|
-
const
|
|
677
|
+
const warns = [];
|
|
672
678
|
const lines = markdown.split("\n");
|
|
673
679
|
let start = -1;
|
|
674
680
|
let lang = "";
|
|
@@ -683,16 +689,16 @@ function checkInlineCode(markdown, max) {
|
|
|
683
689
|
else {
|
|
684
690
|
const len = i - start - 1;
|
|
685
691
|
if (len > max) {
|
|
686
|
-
|
|
687
|
-
type: "
|
|
688
|
-
message: `Inline ${lang || "code"} block is ${String(len)} lines (max ${String(max)});
|
|
692
|
+
warns.push({
|
|
693
|
+
type: "inline-code-too-long",
|
|
694
|
+
message: `Inline ${lang || "code"} block is ${String(len)} lines (max ${String(max)}); consider extracting it to a file and referencing it with file().`,
|
|
689
695
|
});
|
|
690
696
|
}
|
|
691
697
|
start = -1;
|
|
692
698
|
lang = "";
|
|
693
699
|
}
|
|
694
700
|
}
|
|
695
|
-
return
|
|
701
|
+
return warns;
|
|
696
702
|
}
|
|
697
703
|
/**
|
|
698
704
|
* Compile a SkillSpec into SKILL.md markdown with YAML frontmatter.
|
|
@@ -756,14 +762,16 @@ function compileSkill(spec, options = {}) {
|
|
|
756
762
|
}
|
|
757
763
|
}
|
|
758
764
|
const sections = renderSkillSections(spec);
|
|
759
|
-
|
|
765
|
+
// Over-long inline code blocks are WARNINGS, not errors — they don't block
|
|
766
|
+
// compilation (so adoption always compiles), just nudge toward file().
|
|
767
|
+
const warnings = checkInlineCode(sections, spec.maxInlineCodeLines ?? DEFAULT_MAX_INLINE_CODE_LINES);
|
|
760
768
|
const marker = purityMarker(spec.purity);
|
|
761
769
|
const content = renderSkillFrontmatter(spec, profile) +
|
|
762
770
|
"\n\n" +
|
|
763
771
|
(marker ? marker + "\n\n" : "") +
|
|
764
772
|
sections.trim() +
|
|
765
773
|
"\n";
|
|
766
|
-
return { markdown: addHash(content, specFile), errors };
|
|
774
|
+
return { markdown: addHash(content, specFile), errors, warnings };
|
|
767
775
|
}
|
|
768
776
|
// ---------------------------------------------------------------------------
|
|
769
777
|
// Compile a subagent spec → agents/<name>.md
|
|
@@ -944,14 +952,15 @@ function compileAgent(spec, options) {
|
|
|
944
952
|
if (spec.output)
|
|
945
953
|
sections.push(renderOutputContract(spec.output));
|
|
946
954
|
const body = sections.join("\n\n");
|
|
947
|
-
errors
|
|
955
|
+
// Over-long inline code blocks are WARNINGS, not errors (see checkInlineCode).
|
|
956
|
+
const warnings = checkInlineCode(body, DEFAULT_MAX_INLINE_CODE_LINES);
|
|
948
957
|
const marker = purityMarker(spec.purity);
|
|
949
958
|
const content = renderAgentFrontmatter(spec) +
|
|
950
959
|
"\n\n" +
|
|
951
960
|
(marker ? marker + "\n\n" : "") +
|
|
952
961
|
body.trim() +
|
|
953
962
|
"\n";
|
|
954
|
-
return { markdown: addHash(content, specFile), errors };
|
|
963
|
+
return { markdown: addHash(content, specFile), errors, warnings };
|
|
955
964
|
}
|
|
956
965
|
/** Verify a railway: non-empty, bounded recovery, every delegate target real. */
|
|
957
966
|
function validateRailway(rw, knownAgents) {
|
package/dist/leaderboard.d.ts
CHANGED
|
@@ -21,8 +21,40 @@ export interface PluginScore {
|
|
|
21
21
|
readonly issues: readonly string[];
|
|
22
22
|
readonly report: ScanReport;
|
|
23
23
|
}
|
|
24
|
+
export declare const W_MISSING_HOOK = 15;
|
|
25
|
+
export declare const W_NO_DESCRIPTION = 10;
|
|
26
|
+
export declare const W_DANGLING_REF = 8;
|
|
27
|
+
export declare const W_OVERLAP = 8;
|
|
28
|
+
export declare const W_NO_CONTRACT = 5;
|
|
24
29
|
/** Map a 0–100 structural-health score to its letter grade (A ≥90 … F <60). */
|
|
25
30
|
export declare function gradeFor(score: number): PluginScore["grade"];
|
|
31
|
+
/** One deduction: a count, its per-item weight, and the label if non-zero. */
|
|
32
|
+
export interface Deduction {
|
|
33
|
+
readonly n: number;
|
|
34
|
+
readonly weight: number;
|
|
35
|
+
readonly label: string;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The COMPLETE graded-penalty list a report incurs — the single source of truth
|
|
39
|
+
* BOTH the leaderboard's single health number and the audit's category rings
|
|
40
|
+
* read, so the overall can never drift between the two surfaces. Each entry is a
|
|
41
|
+
* graded penalty; untested surfaces are deliberately ABSENT (they're advisory,
|
|
42
|
+
* surfaced separately, never scored).
|
|
43
|
+
*/
|
|
44
|
+
export declare function reportDeductions(r: ScanReport): Deduction[];
|
|
45
|
+
/** True when a report has no loadable plugin surface at all (the empty machine). */
|
|
46
|
+
export declare function isEmptyMachine(r: ScanReport): boolean;
|
|
47
|
+
/**
|
|
48
|
+
* THE shared integrity score — `100 − Σ(all graded penalties)`, clamped to
|
|
49
|
+
* [0,100]. Both the leaderboard's single health number AND the audit's headline
|
|
50
|
+
* overall read this, so the two can never disagree (the summed model is the
|
|
51
|
+
* honest one — averaging rings would dilute a real problem). Returns the score
|
|
52
|
+
* plus the per-item deductions so callers render their own issue/finding lists.
|
|
53
|
+
*/
|
|
54
|
+
export declare function computeIntegrityScore(deductions: readonly Deduction[]): {
|
|
55
|
+
score: number;
|
|
56
|
+
penalty: number;
|
|
57
|
+
};
|
|
26
58
|
/** Deterministic structural-health score for one scanned plugin. */
|
|
27
59
|
export declare function scoreReport(r: ScanReport): {
|
|
28
60
|
score: number;
|
package/dist/leaderboard.js
CHANGED
|
@@ -12,7 +12,11 @@
|
|
|
12
12
|
* model and stack on top later; this part runs anywhere in CI for free.
|
|
13
13
|
*/
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.W_NO_CONTRACT = exports.W_OVERLAP = exports.W_DANGLING_REF = exports.W_NO_DESCRIPTION = exports.W_MISSING_HOOK = void 0;
|
|
15
16
|
exports.gradeFor = gradeFor;
|
|
17
|
+
exports.reportDeductions = reportDeductions;
|
|
18
|
+
exports.isEmptyMachine = isEmptyMachine;
|
|
19
|
+
exports.computeIntegrityScore = computeIntegrityScore;
|
|
16
20
|
exports.scoreReport = scoreReport;
|
|
17
21
|
exports.rankPlugins = rankPlugins;
|
|
18
22
|
exports.formatLeaderboard = formatLeaderboard;
|
|
@@ -38,11 +42,16 @@ function pluginLabel(dir) {
|
|
|
38
42
|
return (0, node_path_1.basename)(dir) || dir;
|
|
39
43
|
}
|
|
40
44
|
// Penalty weights — broken-at-runtime costs most, footguns less, nudges least.
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
45
|
+
// Exported so the category view (audit-score.ts) reuses the SAME weights and the
|
|
46
|
+
// two surfaces can never drift on a per-item cost.
|
|
47
|
+
exports.W_MISSING_HOOK = 15; // a hook script that doesn't exist → never runs
|
|
48
|
+
exports.W_NO_DESCRIPTION = 10; // a skill with no usable description → can't trigger
|
|
49
|
+
exports.W_DANGLING_REF = 8; // a referenced intra-plugin file that's missing → broken path
|
|
50
|
+
exports.W_OVERLAP = 8; // a description collision → the wrong skill fires
|
|
51
|
+
exports.W_NO_CONTRACT = 5; // generic small-footgun weight (disallowedTools typo, invalid model/color)
|
|
52
|
+
// Two things are advisory, NOT graded penalties (shown, never scored — see scoreReport):
|
|
53
|
+
// - untested surfaces — a hardening gap, not breakage.
|
|
54
|
+
// - an agent that inherits all tools (no `tools:` line) — see reportDeductions for why.
|
|
46
55
|
/** Map a 0–100 structural-health score to its letter grade (A ≥90 … F <60). */
|
|
47
56
|
function gradeFor(score) {
|
|
48
57
|
if (score >= 90)
|
|
@@ -55,101 +64,151 @@ function gradeFor(score) {
|
|
|
55
64
|
return "D";
|
|
56
65
|
return "F";
|
|
57
66
|
}
|
|
58
|
-
/**
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
return { score: 0, issues: ["no loadable plugin surface"] };
|
|
67
|
-
}
|
|
67
|
+
/**
|
|
68
|
+
* The COMPLETE graded-penalty list a report incurs — the single source of truth
|
|
69
|
+
* BOTH the leaderboard's single health number and the audit's category rings
|
|
70
|
+
* read, so the overall can never drift between the two surfaces. Each entry is a
|
|
71
|
+
* graded penalty; untested surfaces are deliberately ABSENT (they're advisory,
|
|
72
|
+
* surfaced separately, never scored).
|
|
73
|
+
*/
|
|
74
|
+
function reportDeductions(r) {
|
|
68
75
|
const missingHooks = r.hooks.filter((h) => h.status === "missing").length;
|
|
69
76
|
const noDesc = r.skills.filter((s) => !s.hasDescription).length;
|
|
70
|
-
const noContract = r.agents.filter((a) => a.tools === null).length;
|
|
71
77
|
const deadTools = r.agents.reduce((n, a) => n + a.toolIssues.length, 0);
|
|
72
78
|
const deadMcpTools = r.agents.reduce((n, a) => n + a.mcpToolIssues.length, 0);
|
|
73
79
|
const deadDisallowed = r.agents.reduce((n, a) => n + a.disallowedToolIssues.length, 0);
|
|
74
|
-
|
|
75
|
-
const badFrontmatter = r.frontmatterIssues.length;
|
|
76
|
-
const badFrontmatterValues = r.frontmatterValueIssues.length;
|
|
77
|
-
const badMcp = r.mcpIssues.length;
|
|
78
|
-
const badMcpHooks = r.mcpHookIssues.length;
|
|
79
|
-
const deductions = [
|
|
80
|
+
return [
|
|
80
81
|
{
|
|
81
82
|
n: missingHooks,
|
|
82
|
-
weight: W_MISSING_HOOK,
|
|
83
|
+
weight: exports.W_MISSING_HOOK,
|
|
83
84
|
label: "hook script(s) MISSING",
|
|
84
85
|
},
|
|
85
86
|
{
|
|
86
|
-
n:
|
|
87
|
-
weight: W_MISSING_HOOK,
|
|
87
|
+
n: r.hookEventIssues.length,
|
|
88
|
+
weight: exports.W_MISSING_HOOK,
|
|
88
89
|
label: "hook(s) on an unknown event (never fire)",
|
|
89
90
|
},
|
|
90
91
|
{
|
|
91
92
|
n: noDesc,
|
|
92
|
-
weight: W_NO_DESCRIPTION,
|
|
93
|
+
weight: exports.W_NO_DESCRIPTION,
|
|
93
94
|
label: "skill(s) with no usable description",
|
|
94
95
|
},
|
|
96
|
+
{
|
|
97
|
+
n: r.descriptionOverlaps.length,
|
|
98
|
+
weight: exports.W_OVERLAP,
|
|
99
|
+
label: "near-identical skill description(s) (wrong one fires)",
|
|
100
|
+
},
|
|
95
101
|
{
|
|
96
102
|
n: r.danglingRefs.length,
|
|
97
|
-
weight: W_DANGLING_REF,
|
|
103
|
+
weight: exports.W_DANGLING_REF,
|
|
98
104
|
label: "broken intra-plugin reference(s)",
|
|
99
105
|
},
|
|
100
106
|
{
|
|
101
107
|
n: deadTools,
|
|
102
|
-
weight: W_DANGLING_REF,
|
|
108
|
+
weight: exports.W_DANGLING_REF,
|
|
103
109
|
label: "agent tool(s) that don't exist (typo / never-available)",
|
|
104
110
|
},
|
|
105
111
|
{
|
|
106
112
|
n: deadMcpTools,
|
|
107
|
-
weight: W_DANGLING_REF,
|
|
113
|
+
weight: exports.W_DANGLING_REF,
|
|
108
114
|
label: "agent MCP tool(s) whose server isn't declared (can't resolve)",
|
|
109
115
|
},
|
|
110
116
|
{
|
|
111
117
|
n: deadDisallowed,
|
|
112
|
-
weight: W_NO_CONTRACT,
|
|
118
|
+
weight: exports.W_NO_CONTRACT,
|
|
113
119
|
label: "agent disallowedTools typo(s) that block nothing",
|
|
114
120
|
},
|
|
121
|
+
// NB: an agent that inherits all tools (no `tools:` line) is ADVISORY, not a
|
|
122
|
+
// graded penalty — it's surfaced by scoreReport / the Structure ring but never
|
|
123
|
+
// drags the score. WHY: omitting the `tools:` line is a near-universal,
|
|
124
|
+
// legitimate authoring style (a measured OSS sweep of 122 real plugins found
|
|
125
|
+
// 109 whose ONLY finding was this), so penalizing it makes the grade cry wolf
|
|
126
|
+
// on idiomatic subagents. A health score should mean "something is BROKEN", and
|
|
127
|
+
// a broad-by-default tool surface is a hardening/least-privilege NUDGE, not
|
|
128
|
+
// breakage. The count is re-derived where the advisory note is built.
|
|
115
129
|
{
|
|
116
|
-
n:
|
|
117
|
-
weight:
|
|
118
|
-
label: "agent(s) inherit all tools (no contract)",
|
|
119
|
-
},
|
|
120
|
-
{
|
|
121
|
-
n: badFrontmatter,
|
|
122
|
-
weight: W_NO_DESCRIPTION,
|
|
130
|
+
n: r.frontmatterIssues.length,
|
|
131
|
+
weight: exports.W_NO_DESCRIPTION,
|
|
123
132
|
label: "surface(s) missing required frontmatter (name/description)",
|
|
124
133
|
},
|
|
125
134
|
{
|
|
126
|
-
n:
|
|
127
|
-
weight: W_NO_CONTRACT,
|
|
135
|
+
n: r.frontmatterValueIssues.length,
|
|
136
|
+
weight: exports.W_NO_CONTRACT,
|
|
128
137
|
label: "agent(s) with an invalid model/color (typo → silent fallback)",
|
|
129
138
|
},
|
|
130
139
|
{
|
|
131
|
-
n:
|
|
132
|
-
weight: W_DANGLING_REF,
|
|
140
|
+
n: r.mcpIssues.length,
|
|
141
|
+
weight: exports.W_DANGLING_REF,
|
|
133
142
|
label: "MCP server(s) that can't start (no command/url)",
|
|
134
143
|
},
|
|
135
144
|
{
|
|
136
|
-
n:
|
|
137
|
-
weight: W_DANGLING_REF,
|
|
145
|
+
n: r.mcpHookIssues.length,
|
|
146
|
+
weight: exports.W_DANGLING_REF,
|
|
138
147
|
label: "mcp_tool hook(s) incomplete / targeting an undeclared server",
|
|
139
148
|
},
|
|
140
|
-
|
|
149
|
+
// NB: untested surfaces are NOT a penalty — an untested surface is a hardening
|
|
150
|
+
// gap, not breakage, so it never drags the health score (it's appended as an
|
|
151
|
+
// advisory note below). The score ranks what's BROKEN.
|
|
141
152
|
];
|
|
153
|
+
}
|
|
154
|
+
/** True when a report has no loadable plugin surface at all (the empty machine). */
|
|
155
|
+
function isEmptyMachine(r) {
|
|
156
|
+
const surfaces = r.skills.length +
|
|
157
|
+
r.agents.length +
|
|
158
|
+
r.hooks.length +
|
|
159
|
+
r.inlineHooks +
|
|
160
|
+
r.commands;
|
|
161
|
+
return surfaces === 0 && !r.mcp;
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* THE shared integrity score — `100 − Σ(all graded penalties)`, clamped to
|
|
165
|
+
* [0,100]. Both the leaderboard's single health number AND the audit's headline
|
|
166
|
+
* overall read this, so the two can never disagree (the summed model is the
|
|
167
|
+
* honest one — averaging rings would dilute a real problem). Returns the score
|
|
168
|
+
* plus the per-item deductions so callers render their own issue/finding lists.
|
|
169
|
+
*/
|
|
170
|
+
function computeIntegrityScore(deductions) {
|
|
142
171
|
let penalty = 0;
|
|
172
|
+
for (const d of deductions) {
|
|
173
|
+
if (d.n <= 0)
|
|
174
|
+
continue;
|
|
175
|
+
penalty += d.n * d.weight;
|
|
176
|
+
}
|
|
177
|
+
return { score: Math.max(0, 100 - penalty), penalty };
|
|
178
|
+
}
|
|
179
|
+
/** Deterministic structural-health score for one scanned plugin. */
|
|
180
|
+
function scoreReport(r) {
|
|
181
|
+
// An empty/unloadable machine isn't healthy — it's a non-plugin or a broken
|
|
182
|
+
// load. A command-only or MCP-only plugin (commands/*.md or .mcp.json with no
|
|
183
|
+
// skills/agents/hooks) IS a legitimate plugin, though — Anthropic ships
|
|
184
|
+
// command-only plugins in its own marketplace — so it must NOT score 0.
|
|
185
|
+
const surfaces = r.skills.length + r.agents.length + r.hooks.length + r.commands;
|
|
186
|
+
if (surfaces === 0 && !r.mcp) {
|
|
187
|
+
return { score: 0, issues: ["no loadable plugin surface"] };
|
|
188
|
+
}
|
|
189
|
+
const deductions = reportDeductions(r);
|
|
190
|
+
const { score } = computeIntegrityScore(deductions);
|
|
143
191
|
const issues = [];
|
|
144
192
|
for (const d of deductions) {
|
|
145
193
|
if (d.n === 0)
|
|
146
194
|
continue;
|
|
147
|
-
penalty += d.n * d.weight;
|
|
148
195
|
issues.push(`${String(d.n)} ${d.label}`);
|
|
149
196
|
}
|
|
150
197
|
// Sort issues by cost (worst first) so the report leads with what matters.
|
|
151
198
|
issues.sort((a, b) => Number(b.split(" ")[0]) - Number(a.split(" ")[0]));
|
|
152
|
-
|
|
199
|
+
// Advisory notes are surfaced for visibility but DON'T affect the score, so they
|
|
200
|
+
// come AFTER the real (score-affecting) issues:
|
|
201
|
+
// - inherit-all (no tool contract): a least-privilege NUDGE, not breakage —
|
|
202
|
+
// see reportDeductions for the full rationale.
|
|
203
|
+
// - untested surfaces: a hardening gap, not breakage.
|
|
204
|
+
const noContract = r.agents.filter((a) => a.tools === null).length;
|
|
205
|
+
if (noContract > 0) {
|
|
206
|
+
issues.push(`${String(noContract)} agent(s) inherit all tools (no contract) (advisory)`);
|
|
207
|
+
}
|
|
208
|
+
if (r.untested > 0) {
|
|
209
|
+
issues.push(`${String(r.untested)} untested surface(s) (advisory)`);
|
|
210
|
+
}
|
|
211
|
+
return { score, issues };
|
|
153
212
|
}
|
|
154
213
|
/** Scan + score each directory, ranked best-first (ties broken by name). */
|
|
155
214
|
function rankPlugins(dirs) {
|
|
@@ -180,13 +239,13 @@ function formatLeaderboard(scores) {
|
|
|
180
239
|
const issue = s.issues.length > 0 ? ` — ${s.issues.join("; ")}` : "";
|
|
181
240
|
out.push(` ${rank} ${score} ${s.grade} ${s.name}${issue}`);
|
|
182
241
|
});
|
|
183
|
-
out.push("", "Structural health only (no model). Weights: missing hook -15, no-description", "skill -10, broken intra-plugin ref -8,
|
|
242
|
+
out.push("", "Structural health only (no model). Weights: missing hook -15, no-description", "skill -10, broken intra-plugin ref -8, dead tool/MCP ref -8.", "Inherit-all subagents and untested surfaces are advisory — shown, not scored.");
|
|
184
243
|
return out.join("\n");
|
|
185
244
|
}
|
|
186
245
|
const LEADERBOARD_METHOD = "_Structural health only (deterministic, no model): missing hook −15, " +
|
|
187
|
-
"no-description skill −10, broken intra-plugin ref −8
|
|
188
|
-
"
|
|
189
|
-
"Behavioural columns (trigger-rate, collisions, egress) stack on top._";
|
|
246
|
+
"no-description skill −10, broken intra-plugin / dead-tool ref −8. " +
|
|
247
|
+
"Inherit-all subagents and untested surfaces are advisory (shown, not " +
|
|
248
|
+
"scored). Behavioural columns (trigger-rate, collisions, egress) stack on top._";
|
|
190
249
|
/**
|
|
191
250
|
* Format a ranked leaderboard as a Markdown table — the PUBLISHABLE form (a README,
|
|
192
251
|
* a gist, the leaderboard site). Shows the top 2 deductions per plugin; the full
|