vigiles 8.0.0 → 9.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +152 -186
- package/dist/adoptability.d.ts +55 -0
- package/dist/adoptability.js +196 -0
- package/dist/audit-html.d.ts +20 -0
- package/dist/audit-html.js +61 -0
- package/dist/audit-prompts.d.ts +46 -0
- package/dist/audit-prompts.js +90 -0
- package/dist/audit-report.d.ts +107 -0
- package/dist/audit-report.js +72 -0
- package/dist/audit-report.template.html +125 -0
- package/dist/audit-score.d.ts +64 -0
- package/dist/audit-score.js +224 -0
- package/dist/cli-commands.d.ts +1 -1
- package/dist/cli-commands.js +1 -1
- package/dist/cli.js +476 -116
- package/dist/core/adopt.d.ts +28 -0
- package/dist/core/adopt.js +203 -0
- package/dist/core/inline.js +11 -1
- package/dist/core/types.d.ts +12 -0
- package/dist/dialect-drift.js +1 -1
- package/dist/eval.d.ts +1 -1
- package/dist/eval.js +1 -1
- package/dist/leaderboard.d.ts +32 -0
- package/dist/leaderboard.js +92 -44
- package/dist/optimize.d.ts +12 -5
- package/dist/optimize.js +27 -5
- package/dist/scan-behavioral.d.ts +8 -2
- package/dist/scan-behavioral.js +6 -4
- package/dist/scan-trigger-suggest.d.ts +68 -31
- package/dist/scan-trigger-suggest.js +66 -33
- package/dist/scan.d.ts +36 -11
- package/dist/scan.js +73 -15
- package/dist/score-explainer.d.ts +1 -1
- package/package.json +4 -2
- package/skills/test-harness/SKILL.md +1 -1
package/dist/core/adopt.d.ts
CHANGED
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
* vs `enforce()` is deliberately NOT guessed here — that cross-referencing is
|
|
19
19
|
* `strengthen`'s separate, later job; adoption is lossless transcription.
|
|
20
20
|
*/
|
|
21
|
+
import { type SkillSpec, type AgentSpec } from "./spec.js";
|
|
21
22
|
export type AdoptTier = "structured" | "raw";
|
|
22
23
|
export interface AdoptResult {
|
|
23
24
|
/** Generated `.spec.ts` source (compiles back to ~the original file). */
|
|
@@ -62,4 +63,31 @@ export declare function adoptToSpec(markdown: string, target: string): AdoptedSp
|
|
|
62
63
|
* (the deliverable `init` writes).
|
|
63
64
|
*/
|
|
64
65
|
export declare function adoptMarkdown(markdown: string, target: string): AdoptResult;
|
|
66
|
+
export interface AdoptSurfaceResult {
|
|
67
|
+
/** Generated `.spec.ts` source. */
|
|
68
|
+
source: string;
|
|
69
|
+
kind: "skill" | "agent";
|
|
70
|
+
/**
|
|
71
|
+
* The parsed spec object the source builds — exposed so a round-trip test can
|
|
72
|
+
* feed it straight to `compileSkill`/`compileAgent` without evaluating the
|
|
73
|
+
* generated TS (the same split as `adoptToSpec`/`renderSpecSource`).
|
|
74
|
+
*/
|
|
75
|
+
spec: SkillSpec | AgentSpec;
|
|
76
|
+
/**
|
|
77
|
+
* Frontmatter keys present in the source that the typed spec has no field for
|
|
78
|
+
* — emitted as a `// NOTE:` comment in the source so nothing is lost silently.
|
|
79
|
+
*/
|
|
80
|
+
unmappedKeys: string[];
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Adopt an existing SKILL.md into a `skill()` spec. The body is carried verbatim
|
|
84
|
+
* (skills are freeform markdown — `##` headings stay in the body), so a clean
|
|
85
|
+
* skill round-trips below the integrity header.
|
|
86
|
+
*
|
|
87
|
+
* @param markdown the SKILL.md content
|
|
88
|
+
* @param dirName the skill's directory name — the CC fallback for `name` when
|
|
89
|
+
* frontmatter omits it
|
|
90
|
+
*/
|
|
91
|
+
export declare function adoptSkill(markdown: string, dirName: string): AdoptSurfaceResult;
|
|
92
|
+
export declare function adoptAgent(markdown: string, fileBase: string): AdoptSurfaceResult;
|
|
65
93
|
//# sourceMappingURL=adopt.d.ts.map
|
package/dist/core/adopt.js
CHANGED
|
@@ -22,7 +22,11 @@
|
|
|
22
22
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
23
|
exports.adoptToSpec = adoptToSpec;
|
|
24
24
|
exports.adoptMarkdown = adoptMarkdown;
|
|
25
|
+
exports.adoptSkill = adoptSkill;
|
|
26
|
+
exports.adoptAgent = adoptAgent;
|
|
25
27
|
const integrity_js_1 = require("./integrity.js");
|
|
28
|
+
const frontmatter_read_js_1 = require("./frontmatter-read.js");
|
|
29
|
+
const spec_js_1 = require("./spec.js");
|
|
26
30
|
// A top-level heading is `#` or `##` (the levels the compiler reserves for
|
|
27
31
|
// document/section structure). `###`+ stay inside a section body.
|
|
28
32
|
const HEADING_RE = /^ {0,3}(#{1,2})\s+(.*)$/;
|
|
@@ -196,4 +200,203 @@ function adoptMarkdown(markdown, target) {
|
|
|
196
200
|
sectionCount: Object.keys(spec.sections).length,
|
|
197
201
|
};
|
|
198
202
|
}
|
|
203
|
+
// Consumes the WHOLE leading frontmatter block (through its closing `---` and the
|
|
204
|
+
// trailing newline) so the remainder is the verbatim body — mirrors BLOCK_RE in
|
|
205
|
+
// frontmatter-read.ts but matches past the closing fence.
|
|
206
|
+
const FRONTMATTER_CONSUME_RE = /^\uFEFF?(?:<!--[\s\S]*?-->\s*)?---\r?\n[\s\S]*?\r?\n---[ \t]*\r?\n?/;
|
|
207
|
+
function splitFrontmatterBody(markdown) {
|
|
208
|
+
const fm = (0, frontmatter_read_js_1.readFrontmatter)(markdown);
|
|
209
|
+
if (fm.block === null)
|
|
210
|
+
return { fm, body: markdown.replace(/^\uFEFF/, "") };
|
|
211
|
+
const m = FRONTMATTER_CONSUME_RE.exec(markdown);
|
|
212
|
+
return { fm, body: m ? markdown.slice(m[0].length) : markdown };
|
|
213
|
+
}
|
|
214
|
+
/** The first non-empty, non-heading paragraph — the CC fallback for a skill's
|
|
215
|
+
* description when its frontmatter omits one (name←dir, description←first ¶). */
|
|
216
|
+
function firstParagraph(body) {
|
|
217
|
+
const para = [];
|
|
218
|
+
let started = false;
|
|
219
|
+
for (const line of body.split("\n")) {
|
|
220
|
+
const t = line.trim();
|
|
221
|
+
if (!started) {
|
|
222
|
+
if (t === "" || /^#{1,6}\s/.test(t))
|
|
223
|
+
continue;
|
|
224
|
+
started = true;
|
|
225
|
+
para.push(t);
|
|
226
|
+
}
|
|
227
|
+
else {
|
|
228
|
+
if (t === "")
|
|
229
|
+
break;
|
|
230
|
+
para.push(t);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
return para.join(" ").trim();
|
|
234
|
+
}
|
|
235
|
+
/** Keys the typed spec models — everything else is reported as unmapped. */
|
|
236
|
+
const SKILL_KNOWN_KEYS = new Set([
|
|
237
|
+
"name",
|
|
238
|
+
"description",
|
|
239
|
+
"allowed-tools",
|
|
240
|
+
"tools",
|
|
241
|
+
"disable-model-invocation",
|
|
242
|
+
"argument-hint",
|
|
243
|
+
"context",
|
|
244
|
+
]);
|
|
245
|
+
const AGENT_KNOWN_KEYS = new Set([
|
|
246
|
+
"name",
|
|
247
|
+
"description",
|
|
248
|
+
"model",
|
|
249
|
+
"color",
|
|
250
|
+
"tools",
|
|
251
|
+
"disallowedTools",
|
|
252
|
+
"disallowed-tools",
|
|
253
|
+
]);
|
|
254
|
+
function unmappedFrontmatterKeys(fm, known) {
|
|
255
|
+
if (!fm.data)
|
|
256
|
+
return [];
|
|
257
|
+
return Object.keys(fm.data).filter((k) => !known.has(k));
|
|
258
|
+
}
|
|
259
|
+
/** A `// NOTE:` banner naming any frontmatter keys we couldn't represent. */
|
|
260
|
+
function unmappedNote(kind, keys) {
|
|
261
|
+
if (keys.length === 0)
|
|
262
|
+
return "";
|
|
263
|
+
return (`// NOTE: these frontmatter keys had no ${kind}() field and were left out —\n` +
|
|
264
|
+
`// re-add them by hand if they matter: ${keys.join(", ")}\n`);
|
|
265
|
+
}
|
|
266
|
+
const SURFACE_HEADER = (from) => `// Adopted from ${from} by \`vigiles init\` — body verbatim, standard\n` +
|
|
267
|
+
`// frontmatter mapped; no rules inferred. Review the diff, then \`compile\`.\n`;
|
|
268
|
+
/**
|
|
269
|
+
* Adopt an existing SKILL.md into a `skill()` spec. The body is carried verbatim
|
|
270
|
+
* (skills are freeform markdown — `##` headings stay in the body), so a clean
|
|
271
|
+
* skill round-trips below the integrity header.
|
|
272
|
+
*
|
|
273
|
+
* @param markdown the SKILL.md content
|
|
274
|
+
* @param dirName the skill's directory name — the CC fallback for `name` when
|
|
275
|
+
* frontmatter omits it
|
|
276
|
+
*/
|
|
277
|
+
function adoptSkill(markdown, dirName) {
|
|
278
|
+
const { fm, body } = splitFrontmatterBody(markdown);
|
|
279
|
+
const name = (0, frontmatter_read_js_1.frontmatterScalar)(fm, "name") ?? dirName;
|
|
280
|
+
const description = (0, frontmatter_read_js_1.frontmatterScalar)(fm, "description") ?? firstParagraph(body) ?? name;
|
|
281
|
+
const tools = (0, frontmatter_read_js_1.frontmatterList)(fm, "allowed-tools") ?? (0, frontmatter_read_js_1.frontmatterList)(fm, "tools");
|
|
282
|
+
const argumentHint = (0, frontmatter_read_js_1.frontmatterScalar)(fm, "argument-hint");
|
|
283
|
+
const disableModelInvocation = (0, frontmatter_read_js_1.frontmatterScalar)(fm, "disable-model-invocation") === "true";
|
|
284
|
+
const unmappedKeys = unmappedFrontmatterKeys(fm, SKILL_KNOWN_KEYS);
|
|
285
|
+
const lines = [
|
|
286
|
+
` name: ${JSON.stringify(name)},`,
|
|
287
|
+
` description: ${JSON.stringify(description)},`,
|
|
288
|
+
];
|
|
289
|
+
if (argumentHint)
|
|
290
|
+
lines.push(` argumentHint: ${JSON.stringify(argumentHint)},`);
|
|
291
|
+
if (disableModelInvocation)
|
|
292
|
+
lines.push(` disableModelInvocation: true,`);
|
|
293
|
+
if (tools && tools.length > 0)
|
|
294
|
+
lines.push(` tools: ${JSON.stringify(tools)},`);
|
|
295
|
+
const trimmedBody = body.trim();
|
|
296
|
+
if (trimmedBody)
|
|
297
|
+
lines.push(` body: ${tsTemplate(trimmedBody)},`);
|
|
298
|
+
const spec = (0, spec_js_1.skill)({
|
|
299
|
+
name,
|
|
300
|
+
description,
|
|
301
|
+
...(argumentHint ? { argumentHint } : {}),
|
|
302
|
+
...(disableModelInvocation ? { disableModelInvocation: true } : {}),
|
|
303
|
+
...(tools && tools.length > 0 ? { tools } : {}),
|
|
304
|
+
...(trimmedBody ? { body: trimmedBody } : {}),
|
|
305
|
+
});
|
|
306
|
+
const source = SURFACE_HEADER(`${dirName}/SKILL.md`) +
|
|
307
|
+
unmappedNote("skill", unmappedKeys) +
|
|
308
|
+
`import { skill } from "vigiles/spec";\n\n` +
|
|
309
|
+
`export default skill({\n${lines.join("\n")}\n});\n`;
|
|
310
|
+
return { source, kind: "skill", spec, unmappedKeys };
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Adopt an existing subagent (`agents/<name>.md`) into an `agent()` spec. Unlike
|
|
314
|
+
* a skill, an agent's `sections` reject `##` headers, so the body is split: the
|
|
315
|
+
* lead preamble becomes `body` and each `##`/`#` heading becomes a named section
|
|
316
|
+
* (reusing the instruction-file splitter). The tool contract is carried as-is —
|
|
317
|
+
* if the source lists a never-available tool, the generated spec surfaces it on
|
|
318
|
+
* `compile` (which is the point).
|
|
319
|
+
*
|
|
320
|
+
* @param markdown the subagent file content
|
|
321
|
+
* @param fileBase the file's base name (sans `.md`) — the fallback for `name`
|
|
322
|
+
*/
|
|
323
|
+
/** Split a subagent system prompt: preamble → `body`, each `#`/`##` heading →
|
|
324
|
+
* a named section (agent `sections` reject `##` in the body, so they're hoisted). */
|
|
325
|
+
function splitAgentBody(body) {
|
|
326
|
+
const blocks = splitIntoBlocks(body);
|
|
327
|
+
let lead = "";
|
|
328
|
+
const used = new Set();
|
|
329
|
+
const sectionEntries = [];
|
|
330
|
+
for (const block of blocks) {
|
|
331
|
+
if (block.level === null) {
|
|
332
|
+
lead = block.lines.join("\n").trim();
|
|
333
|
+
}
|
|
334
|
+
else {
|
|
335
|
+
sectionEntries.push({
|
|
336
|
+
key: allocKey(safeKey(block.heading ?? ""), used),
|
|
337
|
+
content: block.lines.join("\n").trim(),
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
return { lead, sectionEntries };
|
|
342
|
+
}
|
|
343
|
+
/** Render the `agent({…})` source lines from the extracted fields. */
|
|
344
|
+
function buildAgentLines(f) {
|
|
345
|
+
const lines = [
|
|
346
|
+
` name: ${JSON.stringify(f.name)},`,
|
|
347
|
+
` description: ${JSON.stringify(f.description)},`,
|
|
348
|
+
];
|
|
349
|
+
if (f.model)
|
|
350
|
+
lines.push(` model: ${JSON.stringify(f.model)},`);
|
|
351
|
+
if (f.color)
|
|
352
|
+
lines.push(` color: ${JSON.stringify(f.color)},`);
|
|
353
|
+
if (f.tools && f.tools.length > 0)
|
|
354
|
+
lines.push(` tools: ${JSON.stringify(f.tools)},`);
|
|
355
|
+
if (f.disallowedTools && f.disallowedTools.length > 0)
|
|
356
|
+
lines.push(` disallowedTools: ${JSON.stringify(f.disallowedTools)},`);
|
|
357
|
+
if (f.lead)
|
|
358
|
+
lines.push(` body: ${tsTemplate(f.lead)},`);
|
|
359
|
+
if (f.sectionEntries.length > 0) {
|
|
360
|
+
const entries = f.sectionEntries
|
|
361
|
+
.map(({ key, content }) => ` ${JSON.stringify(key)}: ${tsTemplate(content)},`)
|
|
362
|
+
.join("\n");
|
|
363
|
+
lines.push(` sections: {\n${entries}\n },`);
|
|
364
|
+
}
|
|
365
|
+
return lines;
|
|
366
|
+
}
|
|
367
|
+
function adoptAgent(markdown, fileBase) {
|
|
368
|
+
const { fm, body } = splitFrontmatterBody(markdown);
|
|
369
|
+
const name = (0, frontmatter_read_js_1.frontmatterScalar)(fm, "name") ?? fileBase;
|
|
370
|
+
const f = {
|
|
371
|
+
name,
|
|
372
|
+
description: (0, frontmatter_read_js_1.frontmatterScalar)(fm, "description") ?? name,
|
|
373
|
+
model: (0, frontmatter_read_js_1.frontmatterScalar)(fm, "model"),
|
|
374
|
+
color: (0, frontmatter_read_js_1.frontmatterScalar)(fm, "color"),
|
|
375
|
+
tools: (0, frontmatter_read_js_1.frontmatterList)(fm, "tools"),
|
|
376
|
+
disallowedTools: (0, frontmatter_read_js_1.frontmatterList)(fm, "disallowedTools") ??
|
|
377
|
+
(0, frontmatter_read_js_1.frontmatterList)(fm, "disallowed-tools"),
|
|
378
|
+
...splitAgentBody(body),
|
|
379
|
+
};
|
|
380
|
+
const unmappedKeys = unmappedFrontmatterKeys(fm, AGENT_KNOWN_KEYS);
|
|
381
|
+
const sections = {};
|
|
382
|
+
for (const { key, content } of f.sectionEntries)
|
|
383
|
+
sections[key] = content;
|
|
384
|
+
const spec = (0, spec_js_1.agent)({
|
|
385
|
+
name: f.name,
|
|
386
|
+
description: f.description,
|
|
387
|
+
...(f.model ? { model: f.model } : {}),
|
|
388
|
+
...(f.color ? { color: f.color } : {}),
|
|
389
|
+
...(f.tools && f.tools.length > 0 ? { tools: f.tools } : {}),
|
|
390
|
+
...(f.disallowedTools && f.disallowedTools.length > 0
|
|
391
|
+
? { disallowedTools: f.disallowedTools }
|
|
392
|
+
: {}),
|
|
393
|
+
...(f.lead ? { body: f.lead } : {}),
|
|
394
|
+
...(f.sectionEntries.length > 0 ? { sections } : {}),
|
|
395
|
+
});
|
|
396
|
+
const source = SURFACE_HEADER(`${fileBase}.md`) +
|
|
397
|
+
unmappedNote("agent", unmappedKeys) +
|
|
398
|
+
`import { agent } from "vigiles/spec";\n\n` +
|
|
399
|
+
`export default agent({\n${buildAgentLines(f).join("\n")}\n});\n`;
|
|
400
|
+
return { source, kind: "agent", spec, unmappedKeys };
|
|
401
|
+
}
|
|
199
402
|
//# sourceMappingURL=adopt.js.map
|
package/dist/core/inline.js
CHANGED
|
@@ -35,6 +35,16 @@ const ENFORCE_RE = /<!--\s*vigiles:enforce\s+([@A-Za-z0-9_/:.-]+)\s+"([^"\n]*)"\
|
|
|
35
35
|
* ambiguous against the closing `-->`.
|
|
36
36
|
*/
|
|
37
37
|
const FILE_RE = /<!--\s*vigiles:file\s+(\S+)\s*-->/;
|
|
38
|
+
/**
|
|
39
|
+
* Strip a single surrounding pair of matching quotes from a captured token.
|
|
40
|
+
* The documented `vigiles:file` syntax is an unquoted path, but users
|
|
41
|
+
* reasonably quote it for symmetry with `vigiles:cmd "..."` — without this,
|
|
42
|
+
* the quotes become part of the path, so it never resolves and the error
|
|
43
|
+
* reads `File not found: ""path""`. Normalize at the parse boundary.
|
|
44
|
+
*/
|
|
45
|
+
function unquote(token) {
|
|
46
|
+
return /^"[^]*"$|^'[^]*'$/.test(token) ? token.slice(1, -1) : token;
|
|
47
|
+
}
|
|
38
48
|
/**
|
|
39
49
|
* Match `<!-- vigiles:cmd "<command>" -->`. The command is quoted because
|
|
40
50
|
* commands contain spaces (e.g. `npm run build`).
|
|
@@ -122,7 +132,7 @@ function parseInlineRules(content) {
|
|
|
122
132
|
}
|
|
123
133
|
const fileMatch = FILE_RE.exec(scannable);
|
|
124
134
|
if (fileMatch) {
|
|
125
|
-
files.push({ path: fileMatch[1], line: i + 1 });
|
|
135
|
+
files.push({ path: unquote(fileMatch[1]), line: i + 1 });
|
|
126
136
|
continue;
|
|
127
137
|
}
|
|
128
138
|
const cmdMatch = CMD_RE.exec(scannable);
|
package/dist/core/types.d.ts
CHANGED
|
@@ -263,6 +263,18 @@ export interface VigilesConfig {
|
|
|
263
263
|
* `"claude-code"`. See research/multi-harness-compile.md.
|
|
264
264
|
*/
|
|
265
265
|
harness?: string | string[];
|
|
266
|
+
/**
|
|
267
|
+
* `vigiles audit` preferences. `measure` is the sticky remembered answer to the
|
|
268
|
+
* "run the executing checks against your harness?" prompt — at a TTY `audit`
|
|
269
|
+
* asks once, then records the choice here so it never asks again. `true` runs
|
|
270
|
+
* the executing checks (safety battery · live MCP · skill firing) on every
|
|
271
|
+
* interactive run, `false` keeps them off (edit this key to change). Written by
|
|
272
|
+
* the audit consent prompt, not `init`. Headless runs never execute regardless
|
|
273
|
+
* (audit is a local report, not a CI step — there is no execution flag).
|
|
274
|
+
*/
|
|
275
|
+
audit?: {
|
|
276
|
+
measure?: boolean;
|
|
277
|
+
};
|
|
266
278
|
}
|
|
267
279
|
/** Valid marker types for rule detection. */
|
|
268
280
|
export type MarkerType = "headings" | "checkboxes";
|
package/dist/dialect-drift.js
CHANGED
|
@@ -40,7 +40,7 @@ exports.formatDialectDrift = formatDialectDrift;
|
|
|
40
40
|
*
|
|
41
41
|
* Pure parsers (testable with fixtures) + a local-install locator. TWO consumers:
|
|
42
42
|
* the gated CI test in `dialect-drift.test.ts` (fails loud on tool/event drift), and
|
|
43
|
-
* `vigiles
|
|
43
|
+
* `vigiles audit` at runtime via `checkDialectDrift`/`formatDialectDrift` (a best-effort,
|
|
44
44
|
* read-local freshness WARN when the installed CC's tool surface drifts from ours).
|
|
45
45
|
*/
|
|
46
46
|
const node_fs_1 = require("node:fs");
|
package/dist/eval.d.ts
CHANGED
|
@@ -251,7 +251,7 @@ export type AgentRunner = (args: AgentRunArgs) => Promise<RunOut>;
|
|
|
251
251
|
*/
|
|
252
252
|
export declare function resolveSpawnEnv(a: Pick<AgentRunArgs, "env" | "replaceEnv">, base?: NodeJS.ProcessEnv): NodeJS.ProcessEnv;
|
|
253
253
|
/** The real `claude`-spawning runner (composition root). Exported so other
|
|
254
|
-
* real-model entries (e.g. `
|
|
254
|
+
* real-model entries (e.g. the `audit` trigger tier) bind the same runner. */
|
|
255
255
|
export declare function spawnAgent(a: AgentRunArgs): Promise<RunOut>;
|
|
256
256
|
/**
|
|
257
257
|
* Run the eval: every arm × every trial against the real `claude` CLI, with the
|
package/dist/eval.js
CHANGED
|
@@ -96,7 +96,7 @@ function resolveSpawnEnv(a, base = process.env) {
|
|
|
96
96
|
}
|
|
97
97
|
/* v8 ignore start -- real claude subprocess; exercised by bench/, not the unit gate */
|
|
98
98
|
/** The real `claude`-spawning runner (composition root). Exported so other
|
|
99
|
-
* real-model entries (e.g. `
|
|
99
|
+
* real-model entries (e.g. the `audit` trigger tier) bind the same runner. */
|
|
100
100
|
function spawnAgent(a) {
|
|
101
101
|
return new Promise((resolvePromise) => {
|
|
102
102
|
const args = [
|
package/dist/leaderboard.d.ts
CHANGED
|
@@ -21,8 +21,40 @@ export interface PluginScore {
|
|
|
21
21
|
readonly issues: readonly string[];
|
|
22
22
|
readonly report: ScanReport;
|
|
23
23
|
}
|
|
24
|
+
export declare const W_MISSING_HOOK = 15;
|
|
25
|
+
export declare const W_NO_DESCRIPTION = 10;
|
|
26
|
+
export declare const W_DANGLING_REF = 8;
|
|
27
|
+
export declare const W_OVERLAP = 8;
|
|
28
|
+
export declare const W_NO_CONTRACT = 5;
|
|
24
29
|
/** Map a 0–100 structural-health score to its letter grade (A ≥90 … F <60). */
|
|
25
30
|
export declare function gradeFor(score: number): PluginScore["grade"];
|
|
31
|
+
/** One deduction: a count, its per-item weight, and the label if non-zero. */
|
|
32
|
+
export interface Deduction {
|
|
33
|
+
readonly n: number;
|
|
34
|
+
readonly weight: number;
|
|
35
|
+
readonly label: string;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The COMPLETE graded-penalty list a report incurs — the single source of truth
|
|
39
|
+
* BOTH the leaderboard's single health number and the audit's category rings
|
|
40
|
+
* read, so the overall can never drift between the two surfaces. Each entry is a
|
|
41
|
+
* graded penalty; untested surfaces are deliberately ABSENT (they're advisory,
|
|
42
|
+
* surfaced separately, never scored).
|
|
43
|
+
*/
|
|
44
|
+
export declare function reportDeductions(r: ScanReport): Deduction[];
|
|
45
|
+
/** True when a report has no loadable plugin surface at all (the empty machine). */
|
|
46
|
+
export declare function isEmptyMachine(r: ScanReport): boolean;
|
|
47
|
+
/**
|
|
48
|
+
* THE shared integrity score — `100 − Σ(all graded penalties)`, clamped to
|
|
49
|
+
* [0,100]. Both the leaderboard's single health number AND the audit's headline
|
|
50
|
+
* overall read this, so the two can never disagree (the summed model is the
|
|
51
|
+
* honest one — averaging rings would dilute a real problem). Returns the score
|
|
52
|
+
* plus the per-item deductions so callers render their own issue/finding lists.
|
|
53
|
+
*/
|
|
54
|
+
export declare function computeIntegrityScore(deductions: readonly Deduction[]): {
|
|
55
|
+
score: number;
|
|
56
|
+
penalty: number;
|
|
57
|
+
};
|
|
26
58
|
/** Deterministic structural-health score for one scanned plugin. */
|
|
27
59
|
export declare function scoreReport(r: ScanReport): {
|
|
28
60
|
score: number;
|
package/dist/leaderboard.js
CHANGED
|
@@ -12,7 +12,11 @@
|
|
|
12
12
|
* model and stack on top later; this part runs anywhere in CI for free.
|
|
13
13
|
*/
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.W_NO_CONTRACT = exports.W_OVERLAP = exports.W_DANGLING_REF = exports.W_NO_DESCRIPTION = exports.W_MISSING_HOOK = void 0;
|
|
15
16
|
exports.gradeFor = gradeFor;
|
|
17
|
+
exports.reportDeductions = reportDeductions;
|
|
18
|
+
exports.isEmptyMachine = isEmptyMachine;
|
|
19
|
+
exports.computeIntegrityScore = computeIntegrityScore;
|
|
16
20
|
exports.scoreReport = scoreReport;
|
|
17
21
|
exports.rankPlugins = rankPlugins;
|
|
18
22
|
exports.formatLeaderboard = formatLeaderboard;
|
|
@@ -38,11 +42,14 @@ function pluginLabel(dir) {
|
|
|
38
42
|
return (0, node_path_1.basename)(dir) || dir;
|
|
39
43
|
}
|
|
40
44
|
// Penalty weights — broken-at-runtime costs most, footguns less, nudges least.
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
45
|
+
// Exported so the category view (audit-score.ts) reuses the SAME weights and the
|
|
46
|
+
// two surfaces can never drift on a per-item cost.
|
|
47
|
+
exports.W_MISSING_HOOK = 15; // a hook script that doesn't exist → never runs
|
|
48
|
+
exports.W_NO_DESCRIPTION = 10; // a skill with no usable description → can't trigger
|
|
49
|
+
exports.W_DANGLING_REF = 8; // a referenced intra-plugin file that's missing → broken path
|
|
50
|
+
exports.W_OVERLAP = 8; // a description collision → the wrong skill fires
|
|
51
|
+
exports.W_NO_CONTRACT = 5; // an agent with no `tools:` line → inherits everything
|
|
52
|
+
// (untested surfaces are advisory, not a penalty — see scoreReport)
|
|
46
53
|
/** Map a 0–100 structural-health score to its letter grade (A ≥90 … F <60). */
|
|
47
54
|
function gradeFor(score) {
|
|
48
55
|
if (score >= 90)
|
|
@@ -55,101 +62,142 @@ function gradeFor(score) {
|
|
|
55
62
|
return "D";
|
|
56
63
|
return "F";
|
|
57
64
|
}
|
|
58
|
-
/**
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
return { score: 0, issues: ["no loadable plugin surface"] };
|
|
67
|
-
}
|
|
65
|
+
/**
|
|
66
|
+
* The COMPLETE graded-penalty list a report incurs — the single source of truth
|
|
67
|
+
* BOTH the leaderboard's single health number and the audit's category rings
|
|
68
|
+
* read, so the overall can never drift between the two surfaces. Each entry is a
|
|
69
|
+
* graded penalty; untested surfaces are deliberately ABSENT (they're advisory,
|
|
70
|
+
* surfaced separately, never scored).
|
|
71
|
+
*/
|
|
72
|
+
function reportDeductions(r) {
|
|
68
73
|
const missingHooks = r.hooks.filter((h) => h.status === "missing").length;
|
|
69
74
|
const noDesc = r.skills.filter((s) => !s.hasDescription).length;
|
|
70
75
|
const noContract = r.agents.filter((a) => a.tools === null).length;
|
|
71
76
|
const deadTools = r.agents.reduce((n, a) => n + a.toolIssues.length, 0);
|
|
72
77
|
const deadMcpTools = r.agents.reduce((n, a) => n + a.mcpToolIssues.length, 0);
|
|
73
78
|
const deadDisallowed = r.agents.reduce((n, a) => n + a.disallowedToolIssues.length, 0);
|
|
74
|
-
|
|
75
|
-
const badFrontmatter = r.frontmatterIssues.length;
|
|
76
|
-
const badFrontmatterValues = r.frontmatterValueIssues.length;
|
|
77
|
-
const badMcp = r.mcpIssues.length;
|
|
78
|
-
const badMcpHooks = r.mcpHookIssues.length;
|
|
79
|
-
const deductions = [
|
|
79
|
+
return [
|
|
80
80
|
{
|
|
81
81
|
n: missingHooks,
|
|
82
|
-
weight: W_MISSING_HOOK,
|
|
82
|
+
weight: exports.W_MISSING_HOOK,
|
|
83
83
|
label: "hook script(s) MISSING",
|
|
84
84
|
},
|
|
85
85
|
{
|
|
86
|
-
n:
|
|
87
|
-
weight: W_MISSING_HOOK,
|
|
86
|
+
n: r.hookEventIssues.length,
|
|
87
|
+
weight: exports.W_MISSING_HOOK,
|
|
88
88
|
label: "hook(s) on an unknown event (never fire)",
|
|
89
89
|
},
|
|
90
90
|
{
|
|
91
91
|
n: noDesc,
|
|
92
|
-
weight: W_NO_DESCRIPTION,
|
|
92
|
+
weight: exports.W_NO_DESCRIPTION,
|
|
93
93
|
label: "skill(s) with no usable description",
|
|
94
94
|
},
|
|
95
|
+
{
|
|
96
|
+
n: r.descriptionOverlaps.length,
|
|
97
|
+
weight: exports.W_OVERLAP,
|
|
98
|
+
label: "near-identical skill description(s) (wrong one fires)",
|
|
99
|
+
},
|
|
95
100
|
{
|
|
96
101
|
n: r.danglingRefs.length,
|
|
97
|
-
weight: W_DANGLING_REF,
|
|
102
|
+
weight: exports.W_DANGLING_REF,
|
|
98
103
|
label: "broken intra-plugin reference(s)",
|
|
99
104
|
},
|
|
100
105
|
{
|
|
101
106
|
n: deadTools,
|
|
102
|
-
weight: W_DANGLING_REF,
|
|
107
|
+
weight: exports.W_DANGLING_REF,
|
|
103
108
|
label: "agent tool(s) that don't exist (typo / never-available)",
|
|
104
109
|
},
|
|
105
110
|
{
|
|
106
111
|
n: deadMcpTools,
|
|
107
|
-
weight: W_DANGLING_REF,
|
|
112
|
+
weight: exports.W_DANGLING_REF,
|
|
108
113
|
label: "agent MCP tool(s) whose server isn't declared (can't resolve)",
|
|
109
114
|
},
|
|
110
115
|
{
|
|
111
116
|
n: deadDisallowed,
|
|
112
|
-
weight: W_NO_CONTRACT,
|
|
117
|
+
weight: exports.W_NO_CONTRACT,
|
|
113
118
|
label: "agent disallowedTools typo(s) that block nothing",
|
|
114
119
|
},
|
|
115
120
|
{
|
|
116
121
|
n: noContract,
|
|
117
|
-
weight: W_NO_CONTRACT,
|
|
122
|
+
weight: exports.W_NO_CONTRACT,
|
|
118
123
|
label: "agent(s) inherit all tools (no contract)",
|
|
119
124
|
},
|
|
120
125
|
{
|
|
121
|
-
n:
|
|
122
|
-
weight: W_NO_DESCRIPTION,
|
|
126
|
+
n: r.frontmatterIssues.length,
|
|
127
|
+
weight: exports.W_NO_DESCRIPTION,
|
|
123
128
|
label: "surface(s) missing required frontmatter (name/description)",
|
|
124
129
|
},
|
|
125
130
|
{
|
|
126
|
-
n:
|
|
127
|
-
weight: W_NO_CONTRACT,
|
|
131
|
+
n: r.frontmatterValueIssues.length,
|
|
132
|
+
weight: exports.W_NO_CONTRACT,
|
|
128
133
|
label: "agent(s) with an invalid model/color (typo → silent fallback)",
|
|
129
134
|
},
|
|
130
135
|
{
|
|
131
|
-
n:
|
|
132
|
-
weight: W_DANGLING_REF,
|
|
136
|
+
n: r.mcpIssues.length,
|
|
137
|
+
weight: exports.W_DANGLING_REF,
|
|
133
138
|
label: "MCP server(s) that can't start (no command/url)",
|
|
134
139
|
},
|
|
135
140
|
{
|
|
136
|
-
n:
|
|
137
|
-
weight: W_DANGLING_REF,
|
|
141
|
+
n: r.mcpHookIssues.length,
|
|
142
|
+
weight: exports.W_DANGLING_REF,
|
|
138
143
|
label: "mcp_tool hook(s) incomplete / targeting an undeclared server",
|
|
139
144
|
},
|
|
140
|
-
|
|
145
|
+
// NB: untested surfaces are NOT a penalty — an untested surface is a hardening
|
|
146
|
+
// gap, not breakage, so it never drags the health score (it's appended as an
|
|
147
|
+
// advisory note below). The score ranks what's BROKEN.
|
|
141
148
|
];
|
|
149
|
+
}
|
|
150
|
+
/** True when a report has no loadable plugin surface at all (the empty machine). */
|
|
151
|
+
function isEmptyMachine(r) {
|
|
152
|
+
const surfaces = r.skills.length +
|
|
153
|
+
r.agents.length +
|
|
154
|
+
r.hooks.length +
|
|
155
|
+
r.inlineHooks +
|
|
156
|
+
r.commands;
|
|
157
|
+
return surfaces === 0 && !r.mcp;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* THE shared integrity score — `100 − Σ(all graded penalties)`, clamped to
|
|
161
|
+
* [0,100]. Both the leaderboard's single health number AND the audit's headline
|
|
162
|
+
* overall read this, so the two can never disagree (the summed model is the
|
|
163
|
+
* honest one — averaging rings would dilute a real problem). Returns the score
|
|
164
|
+
* plus the per-item deductions so callers render their own issue/finding lists.
|
|
165
|
+
*/
|
|
166
|
+
function computeIntegrityScore(deductions) {
|
|
142
167
|
let penalty = 0;
|
|
168
|
+
for (const d of deductions) {
|
|
169
|
+
if (d.n <= 0)
|
|
170
|
+
continue;
|
|
171
|
+
penalty += d.n * d.weight;
|
|
172
|
+
}
|
|
173
|
+
return { score: Math.max(0, 100 - penalty), penalty };
|
|
174
|
+
}
|
|
175
|
+
/** Deterministic structural-health score for one scanned plugin. */
|
|
176
|
+
function scoreReport(r) {
|
|
177
|
+
// An empty/unloadable machine isn't healthy — it's a non-plugin or a broken
|
|
178
|
+
// load. A command-only or MCP-only plugin (commands/*.md or .mcp.json with no
|
|
179
|
+
// skills/agents/hooks) IS a legitimate plugin, though — Anthropic ships
|
|
180
|
+
// command-only plugins in its own marketplace — so it must NOT score 0.
|
|
181
|
+
const surfaces = r.skills.length + r.agents.length + r.hooks.length + r.commands;
|
|
182
|
+
if (surfaces === 0 && !r.mcp) {
|
|
183
|
+
return { score: 0, issues: ["no loadable plugin surface"] };
|
|
184
|
+
}
|
|
185
|
+
const deductions = reportDeductions(r);
|
|
186
|
+
const { score } = computeIntegrityScore(deductions);
|
|
143
187
|
const issues = [];
|
|
144
188
|
for (const d of deductions) {
|
|
145
189
|
if (d.n === 0)
|
|
146
190
|
continue;
|
|
147
|
-
penalty += d.n * d.weight;
|
|
148
191
|
issues.push(`${String(d.n)} ${d.label}`);
|
|
149
192
|
}
|
|
150
193
|
// Sort issues by cost (worst first) so the report leads with what matters.
|
|
151
194
|
issues.sort((a, b) => Number(b.split(" ")[0]) - Number(a.split(" ")[0]));
|
|
152
|
-
|
|
195
|
+
// Untested surfaces are advisory — surfaced for visibility, but they don't
|
|
196
|
+
// affect the score, so they come AFTER the real (score-affecting) issues.
|
|
197
|
+
if (r.untested > 0) {
|
|
198
|
+
issues.push(`${String(r.untested)} untested surface(s) (advisory)`);
|
|
199
|
+
}
|
|
200
|
+
return { score, issues };
|
|
153
201
|
}
|
|
154
202
|
/** Scan + score each directory, ranked best-first (ties broken by name). */
|
|
155
203
|
function rankPlugins(dirs) {
|
|
@@ -180,13 +228,13 @@ function formatLeaderboard(scores) {
|
|
|
180
228
|
const issue = s.issues.length > 0 ? ` — ${s.issues.join("; ")}` : "";
|
|
181
229
|
out.push(` ${rank} ${score} ${s.grade} ${s.name}${issue}`);
|
|
182
230
|
});
|
|
183
|
-
out.push("", "Structural health only (no model). Weights: missing hook -15, no-description", "skill -10, broken intra-plugin ref -8, agent-without-tool-contract -5
|
|
231
|
+
out.push("", "Structural health only (no model). Weights: missing hook -15, no-description", "skill -10, broken intra-plugin ref -8, agent-without-tool-contract -5.", "Untested surfaces are advisory — shown, but they don't affect the score.");
|
|
184
232
|
return out.join("\n");
|
|
185
233
|
}
|
|
186
234
|
const LEADERBOARD_METHOD = "_Structural health only (deterministic, no model): missing hook −15, " +
|
|
187
235
|
"no-description skill −10, broken intra-plugin ref −8, " +
|
|
188
|
-
"agent-without-tool-contract −5
|
|
189
|
-
"Behavioural columns (trigger-rate, collisions, egress) stack on top._";
|
|
236
|
+
"agent-without-tool-contract −5. Untested surfaces are advisory (shown, not " +
|
|
237
|
+
"scored). Behavioural columns (trigger-rate, collisions, egress) stack on top._";
|
|
190
238
|
/**
|
|
191
239
|
* Format a ranked leaderboard as a Markdown table — the PUBLISHABLE form (a README,
|
|
192
240
|
* a gist, the leaderboard site). Shows the top 2 deductions per plugin; the full
|
package/dist/optimize.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The per-repo harness optimizer's DETERMINISTIC spine —
|
|
3
|
-
* `vigiles
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
2
|
+
* The per-repo harness optimizer's DETERMINISTIC spine — folded INLINE into the
|
|
3
|
+
* default `vigiles audit` report (each finding carries its fix; NOT its own
|
|
4
|
+
* `optimize` verb, and no longer a `--fix-plan` flag: until the measured A/B half
|
|
5
|
+
* lands, an "optimizer" that only re-prints audit's findings doesn't earn a
|
|
6
|
+
* separate surface; see research/roadmap.md §P2 "reconsider an `optimize` verb").
|
|
7
7
|
*
|
|
8
8
|
* A2 in the measurement-authority pivot is the ADOPTION product: measure a user's
|
|
9
9
|
* own skills/model/rules on their tasks and recommend add/drop/swap with a MEASURED
|
|
@@ -69,6 +69,13 @@ export interface OptimizeReport {
|
|
|
69
69
|
* before `possible` proxies, via explainScore's own ordering). Pure over the report.
|
|
70
70
|
*/
|
|
71
71
|
export declare function optimize(report: ScanReport): OptimizeReport;
|
|
72
|
+
/**
|
|
73
|
+
* Just the deterministic fix list (no score header) — folded into the default
|
|
74
|
+
* `vigiles audit` report so every finding carries its fix inline (replaces the
|
|
75
|
+
* former `--fix-plan`/`--explain` flags). Empty string when there's nothing to
|
|
76
|
+
* fix (or no loadable surface), so the caller can skip the section entirely.
|
|
77
|
+
*/
|
|
78
|
+
export declare function formatRecommendations(rep: OptimizeReport): string;
|
|
72
79
|
/** Render an optimization plan for the CLI. */
|
|
73
80
|
export declare function formatOptimize(rep: OptimizeReport): string;
|
|
74
81
|
//# sourceMappingURL=optimize.d.ts.map
|