vigiles 12.0.0 → 12.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/claude-code.d.ts +2 -0
- package/dist/claude-code.js +9 -1
- package/dist/cli.js +34 -0
- package/dist/core/rule-meta.js +8 -0
- package/dist/core/skill-description-budget.d.ts +42 -0
- package/dist/core/skill-description-budget.js +47 -0
- package/dist/core/types.d.ts +9 -0
- package/dist/core/validate.js +3 -0
- package/dist/doc-command-coverage.d.ts +20 -0
- package/dist/doc-command-coverage.js +60 -0
- package/dist/eval-cost.d.ts +75 -0
- package/dist/eval-cost.js +134 -0
- package/dist/eval.d.ts +4 -0
- package/dist/eval.js +29 -6
- package/dist/research-index.d.ts +31 -0
- package/dist/research-index.js +48 -0
- package/dist/scan-behavioral.d.ts +42 -0
- package/dist/scan-behavioral.js +67 -0
- package/dist/scan.d.ts +3 -23
- package/dist/scan.js +18 -69
- package/dist/setup-plan.d.ts +1 -1
- package/dist/setup-plan.js +1 -0
- package/package.json +1 -1
- package/skills/adopt-spec/SKILL.md +10 -1
- package/skills/edit-spec/SKILL.md +1 -0
- package/skills/strengthen/SKILL.md +4 -0
- package/skills/test-harness/SKILL.md +17 -0
package/README.md
CHANGED
|
@@ -254,7 +254,7 @@ JS **or** TS (`*.harness.{mjs,ts}`) — run with `npx vigiles test`.
|
|
|
254
254
|
## FAQ
|
|
255
255
|
|
|
256
256
|
- **Isn't this just a markdown linter?** No — it checks whether your instruction file is _true_ (every path/script/symbol/rule exists and is enabled), then tests and measures your harness. A style linter can't do any of that.
|
|
257
|
-
- **Do I have to write TypeScript?** No — your agent writes the spec (`init` adopts your CLAUDE.md into one), or plain markdown lints with zero new files. Compiler-grade guarantees are opt-in, like TS's `strict
|
|
257
|
+
- **Do I have to write TypeScript?** No — your agent writes the spec (`init` adopts your CLAUDE.md into one), or plain markdown lints with zero new files. Compiler-grade guarantees are opt-in, like TS's `strict` ([why?](docs/faq.md#why-are-the-strongest-guarantees-opt-in-not-the-default)).
|
|
258
258
|
- **Non-JS repo?** `npx vigiles lint` verifies your CLAUDE.md with no install (Ruff/Clippy/Pylint/… too).
|
|
259
259
|
|
|
260
260
|
**[Full FAQ →](docs/faq.md)**
|
package/dist/claude-code.d.ts
CHANGED
|
@@ -10,6 +10,8 @@ export * from "./mock-model.js";
|
|
|
10
10
|
export { claudeCodeDriver, buildClaudeArgs, parseClaudeRun, claudeAvailable, } from "./harness-test.js";
|
|
11
11
|
export * from "./adapters/claude-code/dialect.js";
|
|
12
12
|
export { agent, skill, type ClaudeCodeToolVocabulary, } from "./adapters/claude-code/typed-spec.js";
|
|
13
|
+
export { measureSelectionMatrix, assertNoCollision, formatSelectionReport, } from "./scan-behavioral.js";
|
|
14
|
+
export type { SelectionReport, SkillSelectionStat, SelectionOptions, SelectionMatrixOptions, } from "./scan-behavioral.js";
|
|
13
15
|
export * from "./adapters/claude-code/layout.js";
|
|
14
16
|
export * from "./adapters/claude-code/runtime.js";
|
|
15
17
|
export * from "./adapters/claude-code/hook-protocol.js";
|
package/dist/claude-code.js
CHANGED
|
@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.skill = exports.agent = exports.claudeAvailable = exports.parseClaudeRun = exports.buildClaudeArgs = exports.claudeCodeDriver = void 0;
|
|
17
|
+
exports.formatSelectionReport = exports.assertNoCollision = exports.measureSelectionMatrix = exports.skill = exports.agent = exports.claudeAvailable = exports.parseClaudeRun = exports.buildClaudeArgs = exports.claudeCodeDriver = void 0;
|
|
18
18
|
/**
|
|
19
19
|
* `vigiles/claude-code` — the Claude Code-specific harness pieces a *different*
|
|
20
20
|
* harness would swap out: the plugin/repo loader (reads real Claude Code plugin
|
|
@@ -42,6 +42,14 @@ __exportStar(require("./adapters/claude-code/dialect.js"), exports);
|
|
|
42
42
|
var typed_spec_js_1 = require("./adapters/claude-code/typed-spec.js");
|
|
43
43
|
Object.defineProperty(exports, "agent", { enumerable: true, get: function () { return typed_spec_js_1.agent; } });
|
|
44
44
|
Object.defineProperty(exports, "skill", { enumerable: true, get: function () { return typed_spec_js_1.skill; } });
|
|
45
|
+
// Selection-collision — a Claude-Code-ONLY behavioral measurement (Codex has no
|
|
46
|
+
// skill-selection event to read), so it lives on this surface, not the agnostic
|
|
47
|
+
// `vigiles/testing`. `measureSelectionMatrix` builds the N×N "which skill fired?"
|
|
48
|
+
// matrix (diagonal = recall, off-diagonal = collision); `assertNoCollision` gates it.
|
|
49
|
+
var scan_behavioral_js_1 = require("./scan-behavioral.js");
|
|
50
|
+
Object.defineProperty(exports, "measureSelectionMatrix", { enumerable: true, get: function () { return scan_behavioral_js_1.measureSelectionMatrix; } });
|
|
51
|
+
Object.defineProperty(exports, "assertNoCollision", { enumerable: true, get: function () { return scan_behavioral_js_1.assertNoCollision; } });
|
|
52
|
+
Object.defineProperty(exports, "formatSelectionReport", { enumerable: true, get: function () { return scan_behavioral_js_1.formatSelectionReport; } });
|
|
45
53
|
__exportStar(require("./adapters/claude-code/layout.js"), exports);
|
|
46
54
|
__exportStar(require("./adapters/claude-code/runtime.js"), exports);
|
|
47
55
|
__exportStar(require("./adapters/claude-code/hook-protocol.js"), exports);
|
package/dist/cli.js
CHANGED
|
@@ -676,6 +676,7 @@ function lintExitCode(report) {
|
|
|
676
676
|
report.hookScriptErrors > 0 ||
|
|
677
677
|
report.disallowedToolErrors > 0 ||
|
|
678
678
|
report.descriptionOverlapErrors > 0 ||
|
|
679
|
+
report.descriptionBudgetErrors > 0 ||
|
|
679
680
|
report.frontmatterValidErrors > 0 ||
|
|
680
681
|
report.mcpHookErrors > 0 ||
|
|
681
682
|
report.preferCompiledHookErrors > 0 ||
|
|
@@ -1027,6 +1028,10 @@ async function runLint(restArgs, flags, config) {
|
|
|
1027
1028
|
// 7k. Description-overlap — two model-invocable skills with near-identical
|
|
1028
1029
|
// descriptions collide in the selector (deterministic NCD precision proxy).
|
|
1029
1030
|
const descriptionOverlap = checkDescriptionOverlap(config, silent, adapter);
|
|
1031
|
+
// 7k². Skill-description-budget — a model-invocable skill whose description is
|
|
1032
|
+
// so long the trigger signal is buried (heuristic proxy; degrades recall +
|
|
1033
|
+
// precision). Generous 500-char budget; warn-tier, never gates.
|
|
1034
|
+
const descriptionBudget = checkDescriptionBudget(config, silent, adapter);
|
|
1030
1035
|
// 7l. Frontmatter-valid — a `---` block that isn't valid YAML (warn; js-yaml is
|
|
1031
1036
|
// stricter than some loaders, so verify before enforcing).
|
|
1032
1037
|
const frontmatterValid = checkFrontmatterValid(config, silent, adapter);
|
|
@@ -1120,6 +1125,8 @@ async function runLint(restArgs, flags, config) {
|
|
|
1120
1125
|
disallowedToolErrors: disallowedTools.errors,
|
|
1121
1126
|
descriptionOverlapIssues: descriptionOverlap.issues,
|
|
1122
1127
|
descriptionOverlapErrors: descriptionOverlap.errors,
|
|
1128
|
+
descriptionBudgetIssues: descriptionBudget.issues,
|
|
1129
|
+
descriptionBudgetErrors: descriptionBudget.errors,
|
|
1123
1130
|
frontmatterValidIssues: frontmatterValid.issues,
|
|
1124
1131
|
frontmatterValidErrors: frontmatterValid.errors,
|
|
1125
1132
|
mcpHookIssues: mcpHookTargets.issues,
|
|
@@ -2775,6 +2782,33 @@ function checkDescriptionOverlap(config, silent, adapter) {
|
|
|
2775
2782
|
}
|
|
2776
2783
|
return { issues: found.length, errors: sev === "error" ? found.length : 0 };
|
|
2777
2784
|
}
|
|
2785
|
+
/**
|
|
2786
|
+
* Apply the `skill-description-budget` rule: a model-invocable skill whose
|
|
2787
|
+
* description is so long the trigger signal is buried — the selector weighs the
|
|
2788
|
+
* opening most, so a bloated description hurts recall + precision. A
|
|
2789
|
+
* deterministic heuristic proxy (generous 500-char budget). Reuses `scanPlugin`'s
|
|
2790
|
+
* `descriptionBudgetIssues`. Warning by default; "error" gates CI.
|
|
2791
|
+
*/
|
|
2792
|
+
function checkDescriptionBudget(config, silent, adapter) {
|
|
2793
|
+
const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["skill-description-budget"]);
|
|
2794
|
+
if (!sev)
|
|
2795
|
+
return { issues: 0, errors: 0 };
|
|
2796
|
+
let found;
|
|
2797
|
+
try {
|
|
2798
|
+
found = (0, scan_js_1.scanPlugin)(process.cwd(), adapter.layout, adapter.dialect).descriptionBudgetIssues;
|
|
2799
|
+
}
|
|
2800
|
+
catch {
|
|
2801
|
+
return { issues: 0, errors: 0 };
|
|
2802
|
+
}
|
|
2803
|
+
if (found.length > 0 && !silent) {
|
|
2804
|
+
console.log("\nSkill-description-budget check:\n");
|
|
2805
|
+
for (const issue of found) {
|
|
2806
|
+
console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.message}`);
|
|
2807
|
+
ghAnnotate(sev === "error" ? "error" : "warning", issue.message);
|
|
2808
|
+
}
|
|
2809
|
+
}
|
|
2810
|
+
return { issues: found.length, errors: sev === "error" ? found.length : 0 };
|
|
2811
|
+
}
|
|
2778
2812
|
/**
|
|
2779
2813
|
* Apply the `lethal-trifecta` rule: a unit (subagent / model-invocable skill)
|
|
2780
2814
|
* whose declared tools hold all three legs (read-private + ingest-untrusted +
|
package/dist/core/rule-meta.js
CHANGED
|
@@ -202,6 +202,14 @@ exports.RULE_META = {
|
|
|
202
202
|
summary: "Two model-invocable skills aren't near-identical (wrong one fires).",
|
|
203
203
|
detector: "findDescriptionOverlaps",
|
|
204
204
|
},
|
|
205
|
+
"skill-description-budget": {
|
|
206
|
+
id: "skill-description-budget",
|
|
207
|
+
bucket: "heuristic-behavioral",
|
|
208
|
+
surface: ["skill"],
|
|
209
|
+
defaultSeverity: "warn",
|
|
210
|
+
summary: "A model-invocable skill's description isn't so long the trigger is buried.",
|
|
211
|
+
detector: "findDescriptionBudgetIssues",
|
|
212
|
+
},
|
|
205
213
|
"frontmatter-valid": {
|
|
206
214
|
id: "frontmatter-valid",
|
|
207
215
|
bucket: "heuristic-behavioral",
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Skill-description budget — a DETERMINISTIC proxy for a behavioral risk, and the
|
|
3
|
+
* deterministic sibling of {@link findDescriptionOverlaps}. A model-invocable
|
|
4
|
+
* skill is selected on its `description`, and the selector weighs the OPENING of
|
|
5
|
+
* it most; a long, buried description dilutes the trigger signal and degrades
|
|
6
|
+
* both recall ("did it fire when it should?") and precision ("did it stay quiet
|
|
7
|
+
* when it shouldn't?"). This catches a trigger-class problem with NO model.
|
|
8
|
+
*
|
|
9
|
+
* HEURISTIC-BEHAVIORAL bucket: the threshold is a PROXY (no character count
|
|
10
|
+
* PROVES a description triggers badly), so the ceiling is WARN — it never gates.
|
|
11
|
+
* Calibrated FP-safe: the default budget (500 chars) sits well above a normal
|
|
12
|
+
* one-to-three-sentence description, so only a genuinely bloated description
|
|
13
|
+
* fires. Reports the per-skill overflow, never a unilateral defect.
|
|
14
|
+
*/
|
|
15
|
+
/** A skill identified by name + its trigger-surface description. */
|
|
16
|
+
export interface BudgetedSurface {
|
|
17
|
+
readonly name: string;
|
|
18
|
+
readonly description: string;
|
|
19
|
+
}
|
|
20
|
+
export interface DescriptionBudgetIssue {
|
|
21
|
+
readonly name: string;
|
|
22
|
+
/** Length of the description in characters. */
|
|
23
|
+
readonly length: number;
|
|
24
|
+
/** The budget it exceeded. */
|
|
25
|
+
readonly budget: number;
|
|
26
|
+
readonly message: string;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* The default description-length budget, in characters. A concise what+when
|
|
30
|
+
* description is comfortably under this; only a bloated one (multiple long
|
|
31
|
+
* sentences, embedded examples, disambiguation prose) exceeds it. Generous on
|
|
32
|
+
* purpose — warn-tier, don't cry wolf. Exported so a caller / test sees it.
|
|
33
|
+
*/
|
|
34
|
+
export declare const DEFAULT_DESCRIPTION_BUDGET = 500;
|
|
35
|
+
/**
|
|
36
|
+
* Find model-invocable skills whose `description` exceeds `budget` characters.
|
|
37
|
+
* Returns one {@link DescriptionBudgetIssue} per over-budget skill, longest
|
|
38
|
+
* first. Pure; pass only the surfaces that compete for auto-selection
|
|
39
|
+
* (model-invocable, described) so a user-invoked skill isn't a false alarm.
|
|
40
|
+
*/
|
|
41
|
+
export declare function findDescriptionBudgetIssues(surfaces: readonly BudgetedSurface[], budget?: number): DescriptionBudgetIssue[];
|
|
42
|
+
//# sourceMappingURL=skill-description-budget.d.ts.map
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Skill-description budget — a DETERMINISTIC proxy for a behavioral risk, and the
|
|
4
|
+
* deterministic sibling of {@link findDescriptionOverlaps}. A model-invocable
|
|
5
|
+
* skill is selected on its `description`, and the selector weighs the OPENING of
|
|
6
|
+
* it most; a long, buried description dilutes the trigger signal and degrades
|
|
7
|
+
* both recall ("did it fire when it should?") and precision ("did it stay quiet
|
|
8
|
+
* when it shouldn't?"). This catches a trigger-class problem with NO model.
|
|
9
|
+
*
|
|
10
|
+
* HEURISTIC-BEHAVIORAL bucket: the threshold is a PROXY (no character count
|
|
11
|
+
* PROVES a description triggers badly), so the ceiling is WARN — it never gates.
|
|
12
|
+
* Calibrated FP-safe: the default budget (500 chars) sits well above a normal
|
|
13
|
+
* one-to-three-sentence description, so only a genuinely bloated description
|
|
14
|
+
* fires. Reports the per-skill overflow, never a unilateral defect.
|
|
15
|
+
*/
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.DEFAULT_DESCRIPTION_BUDGET = void 0;
|
|
18
|
+
exports.findDescriptionBudgetIssues = findDescriptionBudgetIssues;
|
|
19
|
+
/**
|
|
20
|
+
* The default description-length budget, in characters. A concise what+when
|
|
21
|
+
* description is comfortably under this; only a bloated one (multiple long
|
|
22
|
+
* sentences, embedded examples, disambiguation prose) exceeds it. Generous on
|
|
23
|
+
* purpose — warn-tier, don't cry wolf. Exported so a caller / test sees it.
|
|
24
|
+
*/
|
|
25
|
+
exports.DEFAULT_DESCRIPTION_BUDGET = 500;
|
|
26
|
+
/**
|
|
27
|
+
* Find model-invocable skills whose `description` exceeds `budget` characters.
|
|
28
|
+
* Returns one {@link DescriptionBudgetIssue} per over-budget skill, longest
|
|
29
|
+
* first. Pure; pass only the surfaces that compete for auto-selection
|
|
30
|
+
* (model-invocable, described) so a user-invoked skill isn't a false alarm.
|
|
31
|
+
*/
|
|
32
|
+
function findDescriptionBudgetIssues(surfaces, budget = exports.DEFAULT_DESCRIPTION_BUDGET) {
|
|
33
|
+
const issues = [];
|
|
34
|
+
for (const s of surfaces) {
|
|
35
|
+
const length = Array.from(s.description).length;
|
|
36
|
+
if (length <= budget)
|
|
37
|
+
continue;
|
|
38
|
+
issues.push({
|
|
39
|
+
name: s.name,
|
|
40
|
+
length,
|
|
41
|
+
budget,
|
|
42
|
+
message: `skill "${s.name}" has a ${String(length)}-char description (budget ${String(budget)}) — the selector weighs the opening most, so a long description buries the trigger signal and hurts recall + precision. Tighten it to a concise what + when.`,
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
return issues.sort((a, b) => b.length - a.length);
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=skill-description-budget.js.map
|
package/dist/core/types.d.ts
CHANGED
|
@@ -202,6 +202,15 @@ export interface RulesConfig {
|
|
|
202
202
|
* as `scan` (descriptionOverlaps).
|
|
203
203
|
*/
|
|
204
204
|
"description-overlap"?: RuleSeverity;
|
|
205
|
+
/**
|
|
206
|
+
* Flag a model-invocable skill whose `description` is so long the trigger
|
|
207
|
+
* signal is buried — the selector weighs the opening most, so a bloated
|
|
208
|
+
* description hurts recall + precision. A DETERMINISTIC heuristic proxy for a
|
|
209
|
+
* `--trigger`-class behavioral bug; calibrated FP-safe (generous default
|
|
210
|
+
* budget, 500 chars). Default "warn" — a proxy, never gates. Same detector as
|
|
211
|
+
* `scan` (descriptionBudgetIssues).
|
|
212
|
+
*/
|
|
213
|
+
"skill-description-budget"?: RuleSeverity;
|
|
205
214
|
/**
|
|
206
215
|
* Flag a skill/agent whose `---` frontmatter block EXISTS but isn't valid YAML
|
|
207
216
|
* — fields may not parse as intended. CAVEAT: a real YAML parser (js-yaml) is
|
package/dist/core/validate.js
CHANGED
|
@@ -67,6 +67,9 @@ exports.DEFAULT_RULES = {
|
|
|
67
67
|
"disallowed-tools-contract": "warn",
|
|
68
68
|
// Deterministic NCD precision proxy (near-identical skill descriptions) — warn.
|
|
69
69
|
"description-overlap": "warn",
|
|
70
|
+
// A model-invocable skill's description so long the trigger signal is buried —
|
|
71
|
+
// WARN only (heuristic proxy, generous 500-char budget); never gates.
|
|
72
|
+
"skill-description-budget": "warn",
|
|
70
73
|
// Malformed-YAML frontmatter — WARN only (js-yaml is stricter than some loaders).
|
|
71
74
|
"frontmatter-valid": "warn",
|
|
72
75
|
// A mcp_tool hook incomplete / targeting an undeclared server — on by default at warn.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/** The hidden runtime umbrella — not a human-facing verb to document. */
|
|
2
|
+
export declare const COVERAGE_EXEMPT: readonly ["hook-runtime"];
|
|
3
|
+
/**
|
|
4
|
+
* Whether `verb` appears in a COMMAND context anywhere in `content`:
|
|
5
|
+
* `vigiles <verb>` or a backtick-prefixed `` `<verb> ``. Generous on purpose
|
|
6
|
+
* (see the file header) — over-counting a verb as documented is the SAFE
|
|
7
|
+
* direction; under-counting would cry wolf.
|
|
8
|
+
*/
|
|
9
|
+
export declare function verbMentioned(verb: string, content: string): boolean;
|
|
10
|
+
/**
|
|
11
|
+
* Find public verbs not MENTIONED in any of the given doc files. Pure — the
|
|
12
|
+
* caller supplies file contents (so it runs over the repo's `docs/` in a test,
|
|
13
|
+
* or any file set). `verbs` defaults to the canonical {@link VERBS}; `exempt`
|
|
14
|
+
* drops the hidden umbrella.
|
|
15
|
+
*/
|
|
16
|
+
export declare function findUndocumentedVerbs(docs: readonly {
|
|
17
|
+
readonly path: string;
|
|
18
|
+
readonly content: string;
|
|
19
|
+
}[], verbs?: readonly string[], exempt?: readonly string[]): string[];
|
|
20
|
+
//# sourceMappingURL=doc-command-coverage.d.ts.map
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.COVERAGE_EXEMPT = void 0;
|
|
4
|
+
exports.verbMentioned = verbMentioned;
|
|
5
|
+
exports.findUndocumentedVerbs = findUndocumentedVerbs;
|
|
6
|
+
/**
|
|
7
|
+
* Doc-command coverage — the INVERSE of self-command-refs, and the deterministic
|
|
8
|
+
* FLOOR under the `document-the-why` rule. self-command-refs checks that every
|
|
9
|
+
* `vigiles <cmd>` reference in the docs resolves to a REAL command (docs → code).
|
|
10
|
+
* This checks the other direction (code → docs): every public CLI VERB must be
|
|
11
|
+
* MENTIONED somewhere under `docs/`, so a verb shipped without a doc home is a
|
|
12
|
+
* failing test, not a thing a reader discovers is missing.
|
|
13
|
+
*
|
|
14
|
+
* HIGH-PRECISION, and biased toward NOT crying wolf: the risk here is a FALSE
|
|
15
|
+
* "undocumented" alarm on a verb that IS documented, so "mentioned" is matched
|
|
16
|
+
* GENEROUSLY — a verb counts as documented if it appears in a COMMAND context:
|
|
17
|
+
* `vigiles <verb>` (covers `npx vigiles <verb>`) OR a backtick immediately
|
|
18
|
+
* followed by the verb (`` `<verb>` ``, `` `<verb> ./x` ``). A bare English word
|
|
19
|
+
* ("test", "audit", "eval", "compile" all double as prose) is NOT enough — it
|
|
20
|
+
* must sit in a command context — so the check still fires on a genuinely
|
|
21
|
+
* undocumented verb while never flagging a documented one.
|
|
22
|
+
*
|
|
23
|
+
* `hook-runtime` is excluded by default: it is the HIDDEN runtime-entrypoint
|
|
24
|
+
* umbrella (cohesive-cli-surface keeps it OUT of the human verb surface), not a
|
|
25
|
+
* verb a user is expected to read about beside `audit`/`lint`. Source of truth
|
|
26
|
+
* for the verb set: {@link VERBS}.
|
|
27
|
+
*/
|
|
28
|
+
const cli_commands_js_1 = require("./cli-commands.js");
|
|
29
|
+
/** The hidden runtime umbrella — not a human-facing verb to document. */
|
|
30
|
+
exports.COVERAGE_EXEMPT = ["hook-runtime"];
|
|
31
|
+
function escapeRegExp(s) {
|
|
32
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Whether `verb` appears in a COMMAND context anywhere in `content`:
|
|
36
|
+
* `vigiles <verb>` or a backtick-prefixed `` `<verb> ``. Generous on purpose
|
|
37
|
+
* (see the file header) — over-counting a verb as documented is the SAFE
|
|
38
|
+
* direction; under-counting would cry wolf.
|
|
39
|
+
*/
|
|
40
|
+
function verbMentioned(verb, content) {
|
|
41
|
+
const esc = escapeRegExp(verb);
|
|
42
|
+
return new RegExp(String.raw `(\bvigiles\s+|\x60)${esc}\b`).test(content);
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Find public verbs not MENTIONED in any of the given doc files. Pure — the
|
|
46
|
+
* caller supplies file contents (so it runs over the repo's `docs/` in a test,
|
|
47
|
+
* or any file set). `verbs` defaults to the canonical {@link VERBS}; `exempt`
|
|
48
|
+
* drops the hidden umbrella.
|
|
49
|
+
*/
|
|
50
|
+
function findUndocumentedVerbs(docs, verbs = cli_commands_js_1.VERBS, exempt = exports.COVERAGE_EXEMPT) {
|
|
51
|
+
const mentioned = new Set();
|
|
52
|
+
for (const { content } of docs) {
|
|
53
|
+
for (const verb of verbs) {
|
|
54
|
+
if (!mentioned.has(verb) && verbMentioned(verb, content))
|
|
55
|
+
mentioned.add(verb);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return verbs.filter((v) => !exempt.includes(v) && !mentioned.has(v));
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=doc-command-coverage.js.map
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Eval cost transparency — make what a real-model run SPENT impossible to miss.
|
|
3
|
+
* vigiles's whole affordability pitch is "runs on your Claude subscription, not a
|
|
4
|
+
* metered API," so every real-model run should say — out loud — how many tokens it
|
|
5
|
+
* spent, the API-equivalent dollar cost, and (loudly) if it was billed to a
|
|
6
|
+
* METERED API key instead of your subscription.
|
|
7
|
+
*
|
|
8
|
+
* HONEST SCOPE: we surface tokens + the API-equivalent `$` (`total_cost_usd` from
|
|
9
|
+
* the `claude` CLI) + a running session tally. We deliberately do NOT show a
|
|
10
|
+
* "% of your subscription" — Anthropic does not expose a subscription's quota or
|
|
11
|
+
* limit programmatically (and the real limits are rolling rate windows, not a
|
|
12
|
+
* dollar bucket), so any percentage would be fiction. See docs/eval-architecture.md.
|
|
13
|
+
*
|
|
14
|
+
* Pure + injectable (env + an output sink), so the whole thing is unit-tested
|
|
15
|
+
* without a model or a real key.
|
|
16
|
+
*/
|
|
17
|
+
import type { EvalUsage, ArmUsage, EvalReport } from "./eval.js";
|
|
18
|
+
/** A normalized cost/token snapshot — the common shape a report renders from. */
|
|
19
|
+
export interface CostSummary {
|
|
20
|
+
/** API-equivalent cost (`total_cost_usd`) — the number that matters. */
|
|
21
|
+
readonly costUsd: number;
|
|
22
|
+
readonly inputTokens: number;
|
|
23
|
+
readonly outputTokens: number;
|
|
24
|
+
readonly cacheCreationTokens: number;
|
|
25
|
+
readonly cacheReadTokens: number;
|
|
26
|
+
}
|
|
27
|
+
/** Total tokens across all four billing buckets. */
|
|
28
|
+
export declare function totalTokens(c: CostSummary): number;
|
|
29
|
+
/** A per-run {@link EvalUsage} → the common snapshot. */
|
|
30
|
+
export declare function costFromRun(u: EvalUsage): CostSummary;
|
|
31
|
+
/** An aggregated per-arm {@link ArmUsage} → the common snapshot. */
|
|
32
|
+
export declare function costFromArm(u: ArmUsage): CostSummary;
|
|
33
|
+
/** Sum any number of snapshots (e.g. every arm of an A/B). */
|
|
34
|
+
export declare function sumCosts(costs: readonly CostSummary[]): CostSummary;
|
|
35
|
+
/** The whole-{@link EvalReport} cost — every arm summed. */
|
|
36
|
+
export declare function costFromEvalReport(report: EvalReport): CostSummary;
|
|
37
|
+
/**
|
|
38
|
+
* How the run was billed. `metered` is true when a real Anthropic API key is in
|
|
39
|
+
* the environment — the `claude` CLI bills those PER TOKEN, whereas a
|
|
40
|
+
* subscription run (auth via `~/.claude`, no key var) costs $0 beyond the sub.
|
|
41
|
+
* The mock/deterministic tier never reaches here (it has no real cost), so a
|
|
42
|
+
* present key means a real metered run.
|
|
43
|
+
*/
|
|
44
|
+
export interface Billing {
|
|
45
|
+
readonly metered: boolean;
|
|
46
|
+
/** Which env var carried the key (for the actionable "unset X" message). */
|
|
47
|
+
readonly keyVar: string | null;
|
|
48
|
+
}
|
|
49
|
+
export declare function detectBilling(env?: NodeJS.ProcessEnv): Billing;
|
|
50
|
+
/** Add a run to the running session total and return the new total. */
|
|
51
|
+
export declare function recordSessionCost(c: CostSummary): CostSummary;
|
|
52
|
+
/** The session total so far. */
|
|
53
|
+
export declare function sessionCost(): CostSummary;
|
|
54
|
+
/** Reset the session tally (test seam). */
|
|
55
|
+
export declare function resetSessionCost(): void;
|
|
56
|
+
/**
|
|
57
|
+
* The human-readable cost block for a run. Shows tokens + API-equivalent `$`, the
|
|
58
|
+
* billed-to line (a LOUD warning + an actionable fix when metered, a green ✅ when
|
|
59
|
+
* on the subscription), and the session tally when it exceeds this run.
|
|
60
|
+
*/
|
|
61
|
+
export declare function formatCostSummary(c: CostSummary, opts: {
|
|
62
|
+
billing: Billing;
|
|
63
|
+
session?: CostSummary | null;
|
|
64
|
+
}): string;
|
|
65
|
+
/**
|
|
66
|
+
* Record `c` into the session tally and emit its cost block. The default sink is
|
|
67
|
+
* stderr (so a run's cost never pollutes `--json` stdout). Injectable env + sink
|
|
68
|
+
* keep it fully testable. Returns the emitted text (also handy for a skill to
|
|
69
|
+
* relay to the user).
|
|
70
|
+
*/
|
|
71
|
+
export declare function emitCostSummary(c: CostSummary, opts?: {
|
|
72
|
+
env?: NodeJS.ProcessEnv;
|
|
73
|
+
out?: (s: string) => void;
|
|
74
|
+
}): string;
|
|
75
|
+
//# sourceMappingURL=eval-cost.d.ts.map
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.totalTokens = totalTokens;
|
|
4
|
+
exports.costFromRun = costFromRun;
|
|
5
|
+
exports.costFromArm = costFromArm;
|
|
6
|
+
exports.sumCosts = sumCosts;
|
|
7
|
+
exports.costFromEvalReport = costFromEvalReport;
|
|
8
|
+
exports.detectBilling = detectBilling;
|
|
9
|
+
exports.recordSessionCost = recordSessionCost;
|
|
10
|
+
exports.sessionCost = sessionCost;
|
|
11
|
+
exports.resetSessionCost = resetSessionCost;
|
|
12
|
+
exports.formatCostSummary = formatCostSummary;
|
|
13
|
+
exports.emitCostSummary = emitCostSummary;
|
|
14
|
+
const ZERO = {
|
|
15
|
+
costUsd: 0,
|
|
16
|
+
inputTokens: 0,
|
|
17
|
+
outputTokens: 0,
|
|
18
|
+
cacheCreationTokens: 0,
|
|
19
|
+
cacheReadTokens: 0,
|
|
20
|
+
};
|
|
21
|
+
/** Total tokens across all four billing buckets. */
|
|
22
|
+
function totalTokens(c) {
|
|
23
|
+
return (c.inputTokens + c.outputTokens + c.cacheCreationTokens + c.cacheReadTokens);
|
|
24
|
+
}
|
|
25
|
+
/** A per-run {@link EvalUsage} → the common snapshot. */
|
|
26
|
+
function costFromRun(u) {
|
|
27
|
+
return {
|
|
28
|
+
costUsd: u.costUsd,
|
|
29
|
+
inputTokens: u.inputTokens,
|
|
30
|
+
outputTokens: u.outputTokens,
|
|
31
|
+
cacheCreationTokens: u.cacheCreationTokens,
|
|
32
|
+
cacheReadTokens: u.cacheReadTokens,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
/** An aggregated per-arm {@link ArmUsage} → the common snapshot. */
|
|
36
|
+
function costFromArm(u) {
|
|
37
|
+
return {
|
|
38
|
+
costUsd: u.totalCostUsd,
|
|
39
|
+
inputTokens: u.totalInputTokens,
|
|
40
|
+
outputTokens: u.totalOutputTokens,
|
|
41
|
+
cacheCreationTokens: u.totalCacheCreationTokens,
|
|
42
|
+
cacheReadTokens: u.totalCacheReadTokens,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/** Sum any number of snapshots (e.g. every arm of an A/B). */
|
|
46
|
+
function sumCosts(costs) {
|
|
47
|
+
return costs.reduce((a, c) => ({
|
|
48
|
+
costUsd: a.costUsd + c.costUsd,
|
|
49
|
+
inputTokens: a.inputTokens + c.inputTokens,
|
|
50
|
+
outputTokens: a.outputTokens + c.outputTokens,
|
|
51
|
+
cacheCreationTokens: a.cacheCreationTokens + c.cacheCreationTokens,
|
|
52
|
+
cacheReadTokens: a.cacheReadTokens + c.cacheReadTokens,
|
|
53
|
+
}), ZERO);
|
|
54
|
+
}
|
|
55
|
+
/** The whole-{@link EvalReport} cost — every arm summed. */
|
|
56
|
+
function costFromEvalReport(report) {
|
|
57
|
+
return sumCosts(Object.values(report.arms).map((a) => costFromArm(a.usage)));
|
|
58
|
+
}
|
|
59
|
+
const KEY_VARS = ["ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN"];
|
|
60
|
+
function detectBilling(env = process.env) {
|
|
61
|
+
for (const v of KEY_VARS) {
|
|
62
|
+
const val = env[v];
|
|
63
|
+
if (val !== undefined && val.trim() !== "")
|
|
64
|
+
return { metered: true, keyVar: v };
|
|
65
|
+
}
|
|
66
|
+
return { metered: false, keyVar: null };
|
|
67
|
+
}
|
|
68
|
+
// --- Session tally (within one process) ------------------------------------
|
|
69
|
+
let SESSION = ZERO;
|
|
70
|
+
/** Add a run to the running session total and return the new total. */
|
|
71
|
+
function recordSessionCost(c) {
|
|
72
|
+
SESSION = sumCosts([SESSION, c]);
|
|
73
|
+
return SESSION;
|
|
74
|
+
}
|
|
75
|
+
/** The session total so far. */
|
|
76
|
+
function sessionCost() {
|
|
77
|
+
return SESSION;
|
|
78
|
+
}
|
|
79
|
+
/** Reset the session tally (test seam). */
|
|
80
|
+
function resetSessionCost() {
|
|
81
|
+
SESSION = ZERO;
|
|
82
|
+
}
|
|
83
|
+
// --- Formatting ------------------------------------------------------------
|
|
84
|
+
function fmtUsd(n) {
|
|
85
|
+
return `$${n < 0.01 && n > 0 ? n.toFixed(4) : n.toFixed(2)}`;
|
|
86
|
+
}
|
|
87
|
+
function fmtInt(n) {
|
|
88
|
+
return Math.round(n).toLocaleString("en-US");
|
|
89
|
+
}
|
|
90
|
+
function fmtK(n) {
|
|
91
|
+
return n >= 1000 ? `${(n / 1000).toFixed(1)}k` : String(Math.round(n));
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* The human-readable cost block for a run. Shows tokens + API-equivalent `$`, the
|
|
95
|
+
* billed-to line (a LOUD warning + an actionable fix when metered, a green ✅ when
|
|
96
|
+
* on the subscription), and the session tally when it exceeds this run.
|
|
97
|
+
*/
|
|
98
|
+
function formatCostSummary(c, opts) {
|
|
99
|
+
const lines = [];
|
|
100
|
+
lines.push(` Spent: ${fmtInt(totalTokens(c))} tokens ` +
|
|
101
|
+
`(${fmtK(c.inputTokens)} in · ${fmtK(c.outputTokens)} out · ${fmtK(c.cacheReadTokens)} cache) ` +
|
|
102
|
+
`· ~${fmtUsd(c.costUsd)} API-equivalent`);
|
|
103
|
+
if (opts.billing.metered) {
|
|
104
|
+
const v = opts.billing.keyVar ?? "ANTHROPIC_API_KEY";
|
|
105
|
+
lines.push(` ⚠ Billed to: METERED API (${v} is set) — you paid ~${fmtUsd(c.costUsd)} this run.`, ` Run it free on your Claude subscription: unset ${v}, then \`claude login\`.`);
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
lines.push(` Billed to: your Claude subscription — $0 metered ✅`);
|
|
109
|
+
}
|
|
110
|
+
if (opts.session && opts.session.costUsd > c.costUsd) {
|
|
111
|
+
lines.push(` Session so far: ${fmtInt(totalTokens(opts.session))} tokens · ~${fmtUsd(opts.session.costUsd)} API-equivalent`);
|
|
112
|
+
}
|
|
113
|
+
return lines.join("\n");
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Record `c` into the session tally and emit its cost block. The default sink is
|
|
117
|
+
* stderr (so a run's cost never pollutes `--json` stdout). Injectable env + sink
|
|
118
|
+
* keep it fully testable. Returns the emitted text (also handy for a skill to
|
|
119
|
+
* relay to the user).
|
|
120
|
+
*/
|
|
121
|
+
function emitCostSummary(c, opts = {}) {
|
|
122
|
+
// A no-cost run (a replay, a zero-trial eval) has nothing to report.
|
|
123
|
+
if (totalTokens(c) === 0 && c.costUsd === 0)
|
|
124
|
+
return "";
|
|
125
|
+
const billing = detectBilling(opts.env ?? process.env);
|
|
126
|
+
const session = recordSessionCost(c);
|
|
127
|
+
const text = formatCostSummary(c, { billing, session });
|
|
128
|
+
(opts.out ??
|
|
129
|
+
((s) => {
|
|
130
|
+
console.error(s);
|
|
131
|
+
}))(text);
|
|
132
|
+
return text;
|
|
133
|
+
}
|
|
134
|
+
//# sourceMappingURL=eval-cost.js.map
|
package/dist/eval.d.ts
CHANGED
|
@@ -327,6 +327,8 @@ export interface CheckRate {
|
|
|
327
327
|
export interface CheckReport {
|
|
328
328
|
readonly n: number;
|
|
329
329
|
readonly perCheck: readonly CheckRate[];
|
|
330
|
+
/** Cost / latency / token totals for the run (the same source as `runEval`). */
|
|
331
|
+
readonly usage: ArmUsage;
|
|
330
332
|
}
|
|
331
333
|
/**
|
|
332
334
|
* Score a check vocabulary across trials — the scored counterpart to
|
|
@@ -714,6 +716,8 @@ export interface TriggerRateReport {
|
|
|
714
716
|
* `n` means the measurement is thin (e.g. a Codex usage limit was hit); re-run.
|
|
715
717
|
*/
|
|
716
718
|
readonly errored?: number;
|
|
719
|
+
/** Cost / tokens SPENT across all runs (relevant + irrelevant) — feeds the cost summary. */
|
|
720
|
+
readonly usage: ArmUsage;
|
|
717
721
|
}
|
|
718
722
|
/**
|
|
719
723
|
* An eval-tier transport: how to RUN a real harness turn and PARSE its output.
|
package/dist/eval.js
CHANGED
|
@@ -67,6 +67,7 @@ const node_os_1 = require("node:os");
|
|
|
67
67
|
const node_path_1 = require("node:path");
|
|
68
68
|
const plugin_loader_js_1 = require("./adapters/claude-code/plugin-loader.js");
|
|
69
69
|
const runtime_js_1 = require("./adapters/claude-code/runtime.js");
|
|
70
|
+
const eval_cost_js_1 = require("./eval-cost.js");
|
|
70
71
|
const proofs_js_1 = require("./core/proofs.js");
|
|
71
72
|
const harness_test_js_1 = require("./harness-test.js");
|
|
72
73
|
const eval_cache_js_1 = require("./eval-cache.js");
|
|
@@ -144,7 +145,11 @@ function spawnAgent(a) {
|
|
|
144
145
|
* `runEvalWith` with the real agent runner.
|
|
145
146
|
*/
|
|
146
147
|
async function runEval(spec) {
|
|
147
|
-
|
|
148
|
+
const report = await runEvalWith(spec, spawnAgent);
|
|
149
|
+
// Surface what the run spent — tokens + API-equivalent $, and a LOUD warning if
|
|
150
|
+
// it was billed to a metered API key instead of the subscription. See eval-cost.ts.
|
|
151
|
+
(0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.costFromEvalReport)(report));
|
|
152
|
+
return report;
|
|
148
153
|
}
|
|
149
154
|
/**
|
|
150
155
|
* Score a check vocabulary across trials — the scored counterpart to
|
|
@@ -201,6 +206,7 @@ async function measureWith(spec, runner) {
|
|
|
201
206
|
n: s?.n ?? 0,
|
|
202
207
|
};
|
|
203
208
|
}),
|
|
209
|
+
usage: arm?.usage ?? aggregateUsage([]),
|
|
204
210
|
};
|
|
205
211
|
}
|
|
206
212
|
finally {
|
|
@@ -211,7 +217,9 @@ async function measureWith(spec, runner) {
|
|
|
211
217
|
/* v8 ignore start -- real claude subprocess; thin wrapper over measureWith */
|
|
212
218
|
/** Score a check vocabulary across trials against the real `claude` CLI. */
|
|
213
219
|
async function measure(spec) {
|
|
214
|
-
|
|
220
|
+
const report = await measureWith(spec, spawnAgent);
|
|
221
|
+
(0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.costFromArm)(report.usage));
|
|
222
|
+
return report;
|
|
215
223
|
}
|
|
216
224
|
/** Score checks across arms (injectable runner). Reuses `runEvalWith`. */
|
|
217
225
|
async function measureArmsWith(spec, runner) {
|
|
@@ -245,6 +253,7 @@ async function measureArmsWith(spec, runner) {
|
|
|
245
253
|
n: s?.n ?? 0,
|
|
246
254
|
};
|
|
247
255
|
}),
|
|
256
|
+
usage: arm.usage,
|
|
248
257
|
};
|
|
249
258
|
}
|
|
250
259
|
return { arms };
|
|
@@ -278,7 +287,10 @@ function stubArmPluginDirs(arms) {
|
|
|
278
287
|
/* v8 ignore start -- real claude subprocess; thin wrapper over measureArmsWith */
|
|
279
288
|
/** Score checks across arms against the real `claude` CLI. */
|
|
280
289
|
async function measureArms(spec) {
|
|
281
|
-
|
|
290
|
+
const report = await measureArmsWith(spec, spawnAgent);
|
|
291
|
+
// Sum every arm's spend — an A/B run pays for both arms.
|
|
292
|
+
(0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.sumCosts)(Object.values(report.arms).map((a) => (0, eval_cost_js_1.costFromArm)(a.usage))));
|
|
293
|
+
return report;
|
|
282
294
|
}
|
|
283
295
|
/* v8 ignore stop */
|
|
284
296
|
/**
|
|
@@ -1496,13 +1508,17 @@ async function runTriggerTrial(prompt, cfg, runner) {
|
|
|
1496
1508
|
pluginDir: cfg.pluginDir,
|
|
1497
1509
|
timeoutMs: cfg.timeoutMs,
|
|
1498
1510
|
});
|
|
1511
|
+
// Usage comes from the parser (harness-neutral: Claude + Codex both fill it),
|
|
1512
|
+
// and a run costs tokens even when it errors — so accumulate it either way.
|
|
1513
|
+
const ctx = makeContext(cwd, out, cfg.parse);
|
|
1499
1514
|
// An errored/rate-limited turn is NOT a "skill didn't fire" miss — it's
|
|
1500
1515
|
// excluded from the rate, so e.g. a Codex usage limit can't read as recall 0.
|
|
1501
1516
|
if (cfg.runError?.(out))
|
|
1502
|
-
return { fired: 0, errored: true };
|
|
1517
|
+
return { fired: 0, errored: true, usage: ctx.usage };
|
|
1503
1518
|
return {
|
|
1504
|
-
fired: cfg.fired(
|
|
1519
|
+
fired: cfg.fired(ctx) ? 1 : 0,
|
|
1505
1520
|
errored: false,
|
|
1521
|
+
usage: ctx.usage,
|
|
1506
1522
|
};
|
|
1507
1523
|
}
|
|
1508
1524
|
finally {
|
|
@@ -1539,6 +1555,7 @@ async function runTriggerSet(prompts, cfg, runner) {
|
|
|
1539
1555
|
fired: firedBy.reduce((a, b) => a + b, 0),
|
|
1540
1556
|
n: trialsBy.reduce((a, b) => a + b, 0),
|
|
1541
1557
|
errored,
|
|
1558
|
+
usages: outcomes.map((o) => o.usage),
|
|
1542
1559
|
};
|
|
1543
1560
|
}
|
|
1544
1561
|
/**
|
|
@@ -1624,6 +1641,7 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
|
|
|
1624
1641
|
perPrompt: relevant.perPrompt,
|
|
1625
1642
|
competitors,
|
|
1626
1643
|
errored: positiveOrUndefined(relevant.errored),
|
|
1644
|
+
usage: aggregateUsage(relevant.usages),
|
|
1627
1645
|
};
|
|
1628
1646
|
if ((spec.irrelevantPrompts?.length ?? 0) === 0)
|
|
1629
1647
|
return base;
|
|
@@ -1635,6 +1653,8 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
|
|
|
1635
1653
|
falsePositiveRate: irrelevant.n > 0 ? irrelevant.fired / irrelevant.n : 0,
|
|
1636
1654
|
precision: fires > 0 ? relevant.fired / fires : undefined,
|
|
1637
1655
|
perIrrelevant: irrelevant.perPrompt,
|
|
1656
|
+
// Total cost across BOTH sets (the precision runs cost tokens too).
|
|
1657
|
+
usage: aggregateUsage([...relevant.usages, ...irrelevant.usages]),
|
|
1638
1658
|
};
|
|
1639
1659
|
});
|
|
1640
1660
|
}
|
|
@@ -1654,7 +1674,10 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
|
|
|
1654
1674
|
*/
|
|
1655
1675
|
async function measureTriggerRate(spec, opts = {}) {
|
|
1656
1676
|
const d = opts.evalDriver ?? exports.claudeEvalDriver;
|
|
1657
|
-
|
|
1677
|
+
const report = await measureTriggerRateWith(spec, d.runner, d.parse, d.runError, d.harness ?? "claude-code");
|
|
1678
|
+
// Surface what the run spent (tokens + API-equivalent $ + metered warning).
|
|
1679
|
+
(0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.costFromArm)(report.usage));
|
|
1680
|
+
return report;
|
|
1658
1681
|
}
|
|
1659
1682
|
/* v8 ignore stop */
|
|
1660
1683
|
/** Format a trigger-rate report: overall %, then each prompt's rate. */
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Research-index completeness — the deterministic FLOOR keeping the `research/`
|
|
3
|
+
* corpus and its index (`research/CLAUDE.md.spec.ts`) in sync. The spec's
|
|
4
|
+
* `keyFiles` map is the AGENT-FACING index of every research doc; the compiler
|
|
5
|
+
* already verifies the OTHER direction (every indexed path EXISTS, else
|
|
6
|
+
* `vigiles compile` fails), so the only open gap is a doc that was ADDED but
|
|
7
|
+
* never indexed. This check closes it: every `research/*.md` (except the
|
|
8
|
+
* human-facing `README.md`) must appear in the index, or the dogfood test fails.
|
|
9
|
+
*
|
|
10
|
+
* Pure — the caller supplies the doc filenames and the index content (the spec
|
|
11
|
+
* source, where an entry is AUTHORED), so it runs over the real `research/` dir
|
|
12
|
+
* in a test or over any file set. Bidirectional sync = compiler (index ⊆ docs)
|
|
13
|
+
* + this check (docs ⊆ index).
|
|
14
|
+
*/
|
|
15
|
+
/** Docs that are the index itself / human front-door, not indexed entries. */
|
|
16
|
+
export declare const INDEX_EXEMPT: readonly ["README.md"];
|
|
17
|
+
/**
|
|
18
|
+
* Research doc basenames (e.g. `roadmap.md`) NOT referenced anywhere in
|
|
19
|
+
* `indexContent`. A doc counts as indexed if its repo-relative path
|
|
20
|
+
* (`research/<name>.md`) appears in the index — the exact form the spec's
|
|
21
|
+
* `keyFiles` keys use. Exempt docs (the README) are never flagged.
|
|
22
|
+
*/
|
|
23
|
+
export declare function unindexedResearchDocs(docFilenames: readonly string[], indexContent: string, exempt?: readonly string[]): string[];
|
|
24
|
+
/**
|
|
25
|
+
* Index entries pointing at a `research/<name>.md` that no longer exists on
|
|
26
|
+
* disk. The compiler catches this at compile time (a missing `keyFiles` path is
|
|
27
|
+
* a compile error), so this is a belt-and-suspenders reader for a test that
|
|
28
|
+
* wants to assert it directly without invoking the compiler.
|
|
29
|
+
*/
|
|
30
|
+
export declare function deadIndexEntries(docFilenames: readonly string[], indexContent: string): string[];
|
|
31
|
+
//# sourceMappingURL=research-index.d.ts.map
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Research-index completeness — the deterministic FLOOR keeping the `research/`
|
|
4
|
+
* corpus and its index (`research/CLAUDE.md.spec.ts`) in sync. The spec's
|
|
5
|
+
* `keyFiles` map is the AGENT-FACING index of every research doc; the compiler
|
|
6
|
+
* already verifies the OTHER direction (every indexed path EXISTS, else
|
|
7
|
+
* `vigiles compile` fails), so the only open gap is a doc that was ADDED but
|
|
8
|
+
* never indexed. This check closes it: every `research/*.md` (except the
|
|
9
|
+
* human-facing `README.md`) must appear in the index, or the dogfood test fails.
|
|
10
|
+
*
|
|
11
|
+
* Pure — the caller supplies the doc filenames and the index content (the spec
|
|
12
|
+
* source, where an entry is AUTHORED), so it runs over the real `research/` dir
|
|
13
|
+
* in a test or over any file set. Bidirectional sync = compiler (index ⊆ docs)
|
|
14
|
+
* + this check (docs ⊆ index).
|
|
15
|
+
*/
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.INDEX_EXEMPT = void 0;
|
|
18
|
+
exports.unindexedResearchDocs = unindexedResearchDocs;
|
|
19
|
+
exports.deadIndexEntries = deadIndexEntries;
|
|
20
|
+
/** Docs that are the index itself / human front-door, not indexed entries. */
|
|
21
|
+
exports.INDEX_EXEMPT = ["README.md"];
|
|
22
|
+
/**
|
|
23
|
+
* Research doc basenames (e.g. `roadmap.md`) NOT referenced anywhere in
|
|
24
|
+
* `indexContent`. A doc counts as indexed if its repo-relative path
|
|
25
|
+
* (`research/<name>.md`) appears in the index — the exact form the spec's
|
|
26
|
+
* `keyFiles` keys use. Exempt docs (the README) are never flagged.
|
|
27
|
+
*/
|
|
28
|
+
function unindexedResearchDocs(docFilenames, indexContent, exempt = exports.INDEX_EXEMPT) {
|
|
29
|
+
return docFilenames.filter((name) => !exempt.includes(name) && !indexContent.includes(`research/${name}`));
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Index entries pointing at a `research/<name>.md` that no longer exists on
|
|
33
|
+
* disk. The compiler catches this at compile time (a missing `keyFiles` path is
|
|
34
|
+
* a compile error), so this is a belt-and-suspenders reader for a test that
|
|
35
|
+
* wants to assert it directly without invoking the compiler.
|
|
36
|
+
*/
|
|
37
|
+
function deadIndexEntries(docFilenames, indexContent) {
|
|
38
|
+
const present = new Set(docFilenames);
|
|
39
|
+
const refs = indexContent.matchAll(/research\/([\w.-]+\.md)/g);
|
|
40
|
+
const dead = new Set();
|
|
41
|
+
for (const m of refs) {
|
|
42
|
+
const name = m[1];
|
|
43
|
+
if (!present.has(name))
|
|
44
|
+
dead.add(name);
|
|
45
|
+
}
|
|
46
|
+
return [...dead];
|
|
47
|
+
}
|
|
48
|
+
//# sourceMappingURL=research-index.js.map
|
|
@@ -135,6 +135,48 @@ export declare function measurePluginSelectionWith(dir: string, promptSet: Trigg
|
|
|
135
135
|
export declare function measurePluginSelection(dir: string, promptSet: TriggerPromptSet, opts?: SelectionOptions): Promise<SelectionReport>;
|
|
136
136
|
/** Format the selection-collision matrix as a scan-report section. */
|
|
137
137
|
export declare function formatSelectionReport(r: SelectionReport): string;
|
|
138
|
+
/** Options for {@link measureSelectionMatrix}: {@link SelectionOptions} plus an
|
|
139
|
+
* optional explicit prompt set (auto-derived from descriptions when omitted). */
|
|
140
|
+
export interface SelectionMatrixOptions extends SelectionOptions {
|
|
141
|
+
/**
|
|
142
|
+
* Per-skill recall prompts. Omit for ZERO-SETUP — prompts are auto-derived
|
|
143
|
+
* from each skill's description (the same generator the audit trigger tier
|
|
144
|
+
* uses). Supply your own for a curated collision benchmark; only the `prompts`
|
|
145
|
+
* array per skill is read (any `irrelevant` bank is ignored here).
|
|
146
|
+
*/
|
|
147
|
+
readonly prompts?: TriggerPromptSet;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Measure a plugin's skill-SELECTION collision matrix — "when I ask for skill i's
|
|
151
|
+
* job, does ONLY skill i fire?" The first-class, assertable form of the cross-skill
|
|
152
|
+
* collision measurement (pair with {@link assertNoCollision}). The matrix diagonal
|
|
153
|
+
* is recall; off-diagonal mass is collision — skill j hijacking skill i's prompt,
|
|
154
|
+
* the failure that breaks a multi-skill plugin and that per-skill trigger-rate
|
|
155
|
+
* (each skill in ISOLATION) structurally can't see.
|
|
156
|
+
*
|
|
157
|
+
* ZERO-SETUP: with no `prompts`, they're derived from each model-invocable skill's
|
|
158
|
+
* description. Claude Code only (Codex has no skill-selection event to read); needs
|
|
159
|
+
* the `claude` CLI + model auth, else `available: false`. Thin promotion of
|
|
160
|
+
* {@link measurePluginSelection}. See research/plugin-selection-collision.md.
|
|
161
|
+
*/
|
|
162
|
+
export declare function measureSelectionMatrix(dir: string, opts?: SelectionMatrixOptions): Promise<SelectionReport>;
|
|
163
|
+
/**
|
|
164
|
+
* Injectable core of {@link measureSelectionMatrix} (for tests): auto-derive the
|
|
165
|
+
* prompts (unless supplied) and drive the matrix via a fake/real probe.
|
|
166
|
+
*/
|
|
167
|
+
export declare function measureSelectionMatrixWith(dir: string, probe: HarnessProbe, opts?: SelectionMatrixOptions): Promise<SelectionReport>;
|
|
168
|
+
/**
|
|
169
|
+
* Assert a plugin's skills don't hijack each other — the gate over a
|
|
170
|
+
* {@link SelectionReport} from {@link measureSelectionMatrix}. `maxOffDiagonal`
|
|
171
|
+
* caps EACH skill's collision rate (fraction of its own prompts on which a SIBLING
|
|
172
|
+
* fired); `maxPluginCollision` caps the plugin-wide rate. With neither set it
|
|
173
|
+
* demands ZERO collision. THROWS (never a silent green) when nothing was measured
|
|
174
|
+
* — an unavailable harness or a zero-run report is a gap, not a pass.
|
|
175
|
+
*/
|
|
176
|
+
export declare function assertNoCollision(report: SelectionReport, opts?: {
|
|
177
|
+
maxOffDiagonal?: number;
|
|
178
|
+
maxPluginCollision?: number;
|
|
179
|
+
}): void;
|
|
138
180
|
/** Does a skill description assert a hard constraint (→ an adversarial-gate candidate)? */
|
|
139
181
|
export declare function isGateDescription(description: string): boolean;
|
|
140
182
|
/** A skill considered for gate detection — name + its (model-visible) description. */
|
package/dist/scan-behavioral.js
CHANGED
|
@@ -22,6 +22,9 @@ exports.buildSelectionReport = buildSelectionReport;
|
|
|
22
22
|
exports.measurePluginSelectionWith = measurePluginSelectionWith;
|
|
23
23
|
exports.measurePluginSelection = measurePluginSelection;
|
|
24
24
|
exports.formatSelectionReport = formatSelectionReport;
|
|
25
|
+
exports.measureSelectionMatrix = measureSelectionMatrix;
|
|
26
|
+
exports.measureSelectionMatrixWith = measureSelectionMatrixWith;
|
|
27
|
+
exports.assertNoCollision = assertNoCollision;
|
|
25
28
|
exports.isGateDescription = isGateDescription;
|
|
26
29
|
exports.detectGateSkills = detectGateSkills;
|
|
27
30
|
exports.gateRubric = gateRubric;
|
|
@@ -33,6 +36,7 @@ const node_os_1 = require("node:os");
|
|
|
33
36
|
const node_path_1 = require("node:path");
|
|
34
37
|
const node_child_process_1 = require("node:child_process");
|
|
35
38
|
const scan_js_1 = require("./scan.js");
|
|
39
|
+
const audit_prompts_js_1 = require("./audit-prompts.js");
|
|
36
40
|
const judge_js_1 = require("./judge.js");
|
|
37
41
|
const eval_js_1 = require("./eval.js");
|
|
38
42
|
const harness_assert_js_1 = require("./harness-assert.js");
|
|
@@ -397,6 +401,69 @@ function formatSelectionReport(r) {
|
|
|
397
401
|
}
|
|
398
402
|
return lines.join("\n");
|
|
399
403
|
}
|
|
404
|
+
/**
|
|
405
|
+
* Measure a plugin's skill-SELECTION collision matrix — "when I ask for skill i's
|
|
406
|
+
* job, does ONLY skill i fire?" The first-class, assertable form of the cross-skill
|
|
407
|
+
* collision measurement (pair with {@link assertNoCollision}). The matrix diagonal
|
|
408
|
+
* is recall; off-diagonal mass is collision — skill j hijacking skill i's prompt,
|
|
409
|
+
* the failure that breaks a multi-skill plugin and that per-skill trigger-rate
|
|
410
|
+
* (each skill in ISOLATION) structurally can't see.
|
|
411
|
+
*
|
|
412
|
+
* ZERO-SETUP: with no `prompts`, they're derived from each model-invocable skill's
|
|
413
|
+
* description. Claude Code only (Codex has no skill-selection event to read); needs
|
|
414
|
+
* the `claude` CLI + model auth, else `available: false`. Thin promotion of
|
|
415
|
+
* {@link measurePluginSelection}. See research/plugin-selection-collision.md.
|
|
416
|
+
*/
|
|
417
|
+
async function measureSelectionMatrix(dir, opts = {}) {
|
|
418
|
+
return measurePluginSelection(dir, resolveSelectionPrompts(dir, opts), opts);
|
|
419
|
+
}
|
|
420
|
+
/**
|
|
421
|
+
* Injectable core of {@link measureSelectionMatrix} (for tests): auto-derive the
|
|
422
|
+
* prompts (unless supplied) and drive the matrix via a fake/real probe.
|
|
423
|
+
*/
|
|
424
|
+
async function measureSelectionMatrixWith(dir, probe, opts = {}) {
|
|
425
|
+
return measurePluginSelectionWith(dir, resolveSelectionPrompts(dir, opts), probe, opts);
|
|
426
|
+
}
|
|
427
|
+
/** The prompts for a selection run: explicit if given, else auto-derived from the
|
|
428
|
+
* model-invocable skills' descriptions (the zero-setup path). */
|
|
429
|
+
function resolveSelectionPrompts(dir, opts) {
|
|
430
|
+
return (opts.prompts ??
|
|
431
|
+
(0, audit_prompts_js_1.autoTriggerPrompts)((0, scan_js_1.scanPlugin)(dir)
|
|
432
|
+
.skills.filter((s) => !s.userInvoked && s.hasDescription)
|
|
433
|
+
.map((s) => ({ name: s.name, description: s.description ?? "" }))));
|
|
434
|
+
}
|
|
435
|
+
/**
|
|
436
|
+
* Assert a plugin's skills don't hijack each other — the gate over a
|
|
437
|
+
* {@link SelectionReport} from {@link measureSelectionMatrix}. `maxOffDiagonal`
|
|
438
|
+
* caps EACH skill's collision rate (fraction of its own prompts on which a SIBLING
|
|
439
|
+
* fired); `maxPluginCollision` caps the plugin-wide rate. With neither set it
|
|
440
|
+
* demands ZERO collision. THROWS (never a silent green) when nothing was measured
|
|
441
|
+
* — an unavailable harness or a zero-run report is a gap, not a pass.
|
|
442
|
+
*/
|
|
443
|
+
function assertNoCollision(report, opts = {}) {
|
|
444
|
+
if (!report.available)
|
|
445
|
+
throw new Error(`selection matrix unavailable — ${report.note ?? "n/a"}`);
|
|
446
|
+
if (report.n === 0)
|
|
447
|
+
throw new Error(`selection matrix measured nothing — ${report.note ?? "no runs"}`);
|
|
448
|
+
// Enforce the per-skill ceiling when asked, OR by default (name = NoCollision);
|
|
449
|
+
// if only the plugin-wide cap is given, don't also silently demand zero per-skill.
|
|
450
|
+
const maxOff = opts.maxOffDiagonal ??
|
|
451
|
+
(opts.maxPluginCollision === undefined ? 0 : undefined);
|
|
452
|
+
if (maxOff !== undefined) {
|
|
453
|
+
const worst = report.perSkill
|
|
454
|
+
.filter((s) => s.n > 0)
|
|
455
|
+
.reduce((w, s) => (w && w.collisionRate >= s.collisionRate ? w : s), undefined);
|
|
456
|
+
if (worst && worst.collisionRate > maxOff) {
|
|
457
|
+
const top = worst.collidesWith[0];
|
|
458
|
+
const tail = top ? ` (top collider: ${top.skill} ${pct(top.rate)})` : "";
|
|
459
|
+
throw new Error(`expected each skill's collision rate ≤ ${String(maxOff)}, but ${worst.skill} = ${worst.collisionRate.toFixed(2)}${tail}`);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
if (opts.maxPluginCollision !== undefined &&
|
|
463
|
+
report.collisionRate > opts.maxPluginCollision) {
|
|
464
|
+
throw new Error(`expected plugin collision rate ≤ ${String(opts.maxPluginCollision)}, got ${report.collisionRate.toFixed(2)}`);
|
|
465
|
+
}
|
|
466
|
+
}
|
|
400
467
|
// ─── Enforcement-gate detection (for the adversarial-gate eval) ───────────────
|
|
401
468
|
//
|
|
402
469
|
// A skill whose description states a HARD CONSTRAINT ("always write tests first",
|
package/dist/scan.d.ts
CHANGED
|
@@ -17,6 +17,7 @@ import { type ToolIssue } from "./core/tool-contract.js";
|
|
|
17
17
|
import { type HookEventIssue } from "./core/hook-events.js";
|
|
18
18
|
import { type McpIssue } from "./core/mcp-config.js";
|
|
19
19
|
import { type DescriptionOverlap } from "./core/description-overlap.js";
|
|
20
|
+
import { type DescriptionBudgetIssue } from "./core/skill-description-budget.js";
|
|
20
21
|
import { type McpToolIssue } from "./core/mcp-tool.js";
|
|
21
22
|
import { type McpContractToolError } from "./core/mcp.js";
|
|
22
23
|
import { type McpHookIssue } from "./core/mcp-hook.js";
|
|
@@ -28,8 +29,6 @@ import { type DelegationTrifectaFinding } from "./core/delegation-trifecta.js";
|
|
|
28
29
|
import { type HookBlockFinding } from "./core/hook-block-ineffective.js";
|
|
29
30
|
import { type HookMatcherFinding } from "./core/hook-matcher.js";
|
|
30
31
|
import { type PurityLevel, type EffectSurface } from "./core/effects.js";
|
|
31
|
-
/** A named writing system. The label `unexpectedScript` reports + the config's expectation parse into this. */
|
|
32
|
-
export type Script = "Latin" | "Cyrillic" | "Han" | "Japanese" | "Korean" | "Arabic" | "Hebrew" | "Greek" | "Devanagari" | "Thai";
|
|
33
32
|
export interface ScanSkill {
|
|
34
33
|
readonly name: string;
|
|
35
34
|
readonly path: string;
|
|
@@ -41,15 +40,6 @@ export interface ScanSkill {
|
|
|
41
40
|
*/
|
|
42
41
|
readonly description?: string;
|
|
43
42
|
readonly userInvoked: boolean;
|
|
44
|
-
/**
|
|
45
|
-
* The description's dominant script when it DIFFERS from the expected one
|
|
46
|
-
* (default `"Latin"`), else null. The model's skill-selection context is
|
|
47
|
-
* English-centric, so a description in another script carries a cross-language
|
|
48
|
-
* trigger risk — it may under-fire on English prompts. A RISK flag, not a
|
|
49
|
-
* defect (a language-matched audience is fine); measure the real gap with the
|
|
50
|
-
* `audit` trigger tier / `measureTriggerRate`.
|
|
51
|
-
*/
|
|
52
|
-
readonly descriptionScript: Script | null;
|
|
53
43
|
/**
|
|
54
44
|
* SKILL.md body references to a bundled file (`scripts/`/`references/`/`assets/`
|
|
55
45
|
* or a relative markdown link with an extension) that don't resolve on disk
|
|
@@ -221,6 +211,8 @@ export interface ScanReport {
|
|
|
221
211
|
readonly mcpHookIssues: readonly McpHookIssue[];
|
|
222
212
|
/** Pairs of model-invocable skills whose descriptions are near-identical (precision collision). */
|
|
223
213
|
readonly descriptionOverlaps: readonly DescriptionOverlap[];
|
|
214
|
+
/** Model-invocable skills whose description is so long the trigger signal is buried. */
|
|
215
|
+
readonly descriptionBudgetIssues: readonly DescriptionBudgetIssue[];
|
|
224
216
|
/**
|
|
225
217
|
* Lethal-trifecta findings across subagents + model-invocable skills — a unit
|
|
226
218
|
* holding all three legs (read-private + ingest-untrusted + exfiltrate). Each
|
|
@@ -299,18 +291,6 @@ export interface SurfaceClassifier {
|
|
|
299
291
|
readonly isAgent: (f: string) => boolean;
|
|
300
292
|
readonly isCommand: (f: string) => boolean;
|
|
301
293
|
}
|
|
302
|
-
/**
|
|
303
|
-
* The description's dominant alphabetic script when it DIFFERS from `expected`
|
|
304
|
-
* (default `"Latin"`) — the cross-language trigger-risk signal. The model's
|
|
305
|
-
* skill-selection context is English-centric, so a description written mostly in
|
|
306
|
-
* another script may under-fire on English prompts. `expected` is a configurable
|
|
307
|
-
* default, not a value judgement: a Russian-targeted pack sets it to `"Cyrillic"`
|
|
308
|
-
* so its Cyrillic descriptions pass and an English one is flagged instead.
|
|
309
|
-
* Returns null when the dominant script IS the expected one (or there's no
|
|
310
|
-
* alphabetic content). Shared by `scan` and the future lint rule (one detector,
|
|
311
|
-
* no drift). The ≥20% guard avoids a near-empty string tripping on one letter.
|
|
312
|
-
*/
|
|
313
|
-
export declare function unexpectedScript(text: string, expected?: Script): Script | null;
|
|
314
294
|
/**
|
|
315
295
|
* A compiled `vigiles/hook` artifact runs through the `hook-runtime run-program`
|
|
316
296
|
* runtime entrypoint; any other hook command is hand-written (a shell script or
|
package/dist/scan.js
CHANGED
|
@@ -13,7 +13,6 @@
|
|
|
13
13
|
* stack on top later; this core stays pure so it runs anywhere in CI for free.
|
|
14
14
|
*/
|
|
15
15
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
-
exports.unexpectedScript = unexpectedScript;
|
|
17
16
|
exports.isManagedHookCommand = isManagedHookCommand;
|
|
18
17
|
exports.preferCompiledHooksMessage = preferCompiledHooksMessage;
|
|
19
18
|
exports.scanPlugin = scanPlugin;
|
|
@@ -35,6 +34,7 @@ const hook_normalize_js_1 = require("./core/hook-normalize.js");
|
|
|
35
34
|
const linters_js_1 = require("./core/linters.js");
|
|
36
35
|
const frontmatter_read_js_1 = require("./core/frontmatter-read.js");
|
|
37
36
|
const description_overlap_js_1 = require("./core/description-overlap.js");
|
|
37
|
+
const skill_description_budget_js_1 = require("./core/skill-description-budget.js");
|
|
38
38
|
const mcp_tool_js_1 = require("./core/mcp-tool.js");
|
|
39
39
|
const mcp_js_1 = require("./core/mcp.js");
|
|
40
40
|
const mcp_hook_js_1 = require("./core/mcp-hook.js");
|
|
@@ -111,58 +111,6 @@ function skillName(path) {
|
|
|
111
111
|
.split("/")
|
|
112
112
|
.pop() ?? path);
|
|
113
113
|
}
|
|
114
|
-
// [Unicode \p{Script=…} property value (Node native, no dependency), our Script
|
|
115
|
-
// label]. Japanese kana fold to "Japanese". Latin is the DEFAULT expectation (the
|
|
116
|
-
// selector is English-centric), but it's just a default — a language-matched pack
|
|
117
|
-
// can declare a different expectation, and then the OTHER script is the mismatch.
|
|
118
|
-
const SCRIPTS = [
|
|
119
|
-
["Latin", "Latin"],
|
|
120
|
-
["Cyrillic", "Cyrillic"],
|
|
121
|
-
["Han", "Han"],
|
|
122
|
-
["Hiragana", "Japanese"],
|
|
123
|
-
["Katakana", "Japanese"],
|
|
124
|
-
["Hangul", "Korean"],
|
|
125
|
-
["Arabic", "Arabic"],
|
|
126
|
-
["Hebrew", "Hebrew"],
|
|
127
|
-
["Greek", "Greek"],
|
|
128
|
-
["Devanagari", "Devanagari"],
|
|
129
|
-
["Thai", "Thai"],
|
|
130
|
-
];
|
|
131
|
-
/** Letter counts per named script label (Japanese kana folded together). */
|
|
132
|
-
function scriptCounts(text) {
|
|
133
|
-
const counts = new Map();
|
|
134
|
-
for (const [script, label] of SCRIPTS) {
|
|
135
|
-
const n = (text.match(new RegExp(`\\p{Script=${script}}`, "gu")) ?? [])
|
|
136
|
-
.length;
|
|
137
|
-
if (n > 0)
|
|
138
|
-
counts.set(label, (counts.get(label) ?? 0) + n);
|
|
139
|
-
}
|
|
140
|
-
return counts;
|
|
141
|
-
}
|
|
142
|
-
/**
|
|
143
|
-
* The description's dominant alphabetic script when it DIFFERS from `expected`
|
|
144
|
-
* (default `"Latin"`) — the cross-language trigger-risk signal. The model's
|
|
145
|
-
* skill-selection context is English-centric, so a description written mostly in
|
|
146
|
-
* another script may under-fire on English prompts. `expected` is a configurable
|
|
147
|
-
* default, not a value judgement: a Russian-targeted pack sets it to `"Cyrillic"`
|
|
148
|
-
* so its Cyrillic descriptions pass and an English one is flagged instead.
|
|
149
|
-
* Returns null when the dominant script IS the expected one (or there's no
|
|
150
|
-
* alphabetic content). Shared by `scan` and the future lint rule (one detector,
|
|
151
|
-
* no drift). The ≥20% guard avoids a near-empty string tripping on one letter.
|
|
152
|
-
*/
|
|
153
|
-
function unexpectedScript(text, expected = "Latin") {
|
|
154
|
-
const counts = scriptCounts(text);
|
|
155
|
-
let total = 0;
|
|
156
|
-
let dominant = null;
|
|
157
|
-
for (const [label, count] of counts) {
|
|
158
|
-
total += count;
|
|
159
|
-
if (!dominant || count > dominant.count)
|
|
160
|
-
dominant = { label, count };
|
|
161
|
-
}
|
|
162
|
-
if (!dominant || dominant.label === expected)
|
|
163
|
-
return null;
|
|
164
|
-
return dominant.count / total >= 0.2 ? dominant.label : null;
|
|
165
|
-
}
|
|
166
114
|
/**
|
|
167
115
|
* The first prose paragraph of a SKILL.md body (after the frontmatter and any
|
|
168
116
|
* leading `#` headings) — Claude Code's FALLBACK skill description when the
|
|
@@ -236,7 +184,6 @@ function scanSkills(files, cls, ctx) {
|
|
|
236
184
|
hasDescription: Boolean(effectiveDesc && effectiveDesc.length >= 20),
|
|
237
185
|
description: effectiveDesc?.trim(),
|
|
238
186
|
userInvoked,
|
|
239
|
-
descriptionScript: effectiveDesc ? unexpectedScript(effectiveDesc) : null,
|
|
240
187
|
resourceIssues,
|
|
241
188
|
trifecta,
|
|
242
189
|
// A SKILL.md opening with `name:`/`description:` but no `---` fence loads
|
|
@@ -254,7 +201,7 @@ function scanSkills(files, cls, ctx) {
|
|
|
254
201
|
* logic as `scanSkills` (frontmatter `description` ← first body paragraph), then
|
|
255
202
|
* the NCD precision-proxy. See description-overlap.ts.
|
|
256
203
|
*/
|
|
257
|
-
function
|
|
204
|
+
function modelInvocableSkillSurfaces(files, cls) {
|
|
258
205
|
const surfaces = [];
|
|
259
206
|
for (const [path, md] of Object.entries(files)) {
|
|
260
207
|
if (!cls.isSkill(path))
|
|
@@ -267,7 +214,18 @@ function descriptionOverlapsFor(files, cls) {
|
|
|
267
214
|
continue;
|
|
268
215
|
surfaces.push({ name: fm.name ?? skillName(path), description });
|
|
269
216
|
}
|
|
270
|
-
return
|
|
217
|
+
return surfaces;
|
|
218
|
+
}
|
|
219
|
+
function descriptionOverlapsFor(files, cls) {
|
|
220
|
+
return (0, description_overlap_js_1.findDescriptionOverlaps)(modelInvocableSkillSurfaces(files, cls));
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Model-invocable skills whose description is so long the trigger signal is
|
|
224
|
+
* buried (heuristic proxy; degrades recall + precision). Same surfaces as the
|
|
225
|
+
* overlap check. See skill-description-budget.ts.
|
|
226
|
+
*/
|
|
227
|
+
function descriptionBudgetFor(files, cls) {
|
|
228
|
+
return (0, skill_description_budget_js_1.findDescriptionBudgetIssues)(modelInvocableSkillSurfaces(files, cls));
|
|
271
229
|
}
|
|
272
230
|
function scanAgents(files, dialect, declaredServers, cls) {
|
|
273
231
|
const out = [];
|
|
@@ -797,6 +755,7 @@ function scanPlugin(dir, layout, dialect = dialect_js_1.claudeCodeDialect) {
|
|
|
797
755
|
mcpIssues: (0, mcp_config_js_1.verifyMcpServers)(mcpServers),
|
|
798
756
|
mcpHookIssues: (0, mcp_hook_js_1.verifyMcpHookTargets)(loaded.settings.hooks, declaredServers, dialect),
|
|
799
757
|
descriptionOverlaps: descriptionOverlapsFor(loaded.files, cls),
|
|
758
|
+
descriptionBudgetIssues: descriptionBudgetFor(loaded.files, cls),
|
|
800
759
|
trifectaFindings,
|
|
801
760
|
skillResourceIssues: skillResourceFindings,
|
|
802
761
|
skillFenceIssues: skillFenceFindings,
|
|
@@ -923,7 +882,7 @@ function section(title, lines, count = lines.length) {
|
|
|
923
882
|
return [];
|
|
924
883
|
return [`${title} (${String(count)}):`, ...lines, ""];
|
|
925
884
|
}
|
|
926
|
-
/** One skill's report line: ✓/⚠ + name + notes (no-trigger, user-invoked
|
|
885
|
+
/** One skill's report line: ✓/⚠ + name + notes (no-trigger, user-invoked). */
|
|
927
886
|
function skillLine(s) {
|
|
928
887
|
if (!s.hasDescription) {
|
|
929
888
|
return ` ⚠ ${s.name} (no usable description — no frontmatter description and no body text — can't trigger)`;
|
|
@@ -931,11 +890,7 @@ function skillLine(s) {
|
|
|
931
890
|
const notes = [];
|
|
932
891
|
if (s.userInvoked)
|
|
933
892
|
notes.push("user-invoked");
|
|
934
|
-
|
|
935
|
-
notes.push(`description in ${s.descriptionScript} — cross-language trigger risk`);
|
|
936
|
-
}
|
|
937
|
-
const mark = s.descriptionScript ? "⚠" : "✓";
|
|
938
|
-
return ` ${mark} ${s.name}${notes.length ? ` (${notes.join("; ")})` : ""}`;
|
|
893
|
+
return ` ✓ ${s.name}${notes.length ? ` (${notes.join("; ")})` : ""}`;
|
|
939
894
|
}
|
|
940
895
|
/** One agent's report block: ✗ (broken contract) / ⚠ (inherits all) / ✓ + issues + purity. */
|
|
941
896
|
function agentLines(a) {
|
|
@@ -1002,6 +957,7 @@ function formatScanReport(r) {
|
|
|
1002
957
|
out.push(...section("MCP config", r.mcpIssues.map((i) => ` ✗ ${i.message}`)));
|
|
1003
958
|
out.push(...section("MCP hook targets", r.mcpHookIssues.map((i) => ` ✗ ${i.message}`)));
|
|
1004
959
|
out.push(...section("Description overlap (precision risk)", r.descriptionOverlaps.map((o) => ` ⚠ ${o.message}`)));
|
|
960
|
+
out.push(...section("Description budget (trigger-signal risk)", r.descriptionBudgetIssues.map((o) => ` ⚠ ${o.message}`)));
|
|
1005
961
|
out.push(...section("Lethal trifecta (prompt-injection exfil risk)", r.trifectaFindings.map((t) => ` ${t.finding.severity === "hard" ? "✗" : "⚠"} ${t.kind} ${t.name} (${t.path}): ${t.finding.message}`)));
|
|
1006
962
|
out.push(...section("Skill bundled resources", r.skillResourceIssues.map((s) => ` ✗ ${s.name}: ${s.finding.ref} (line ${String(s.finding.line)}) — bundled resource not found`)));
|
|
1007
963
|
out.push(...section("Invisible skills (missing frontmatter fence)", r.skillFenceIssues.map((s) => ` ✗ ${s.name} (${s.path}): opens with \`${s.finding.key}:\` but no \`---\` fence — loads as body, never fires`)));
|
|
@@ -1028,13 +984,6 @@ function formatScanReport(r) {
|
|
|
1028
984
|
if (warnings.length > 0) {
|
|
1029
985
|
out.push("Warnings:", ...warnings.map((w) => ` - ${w}`), "");
|
|
1030
986
|
}
|
|
1031
|
-
// Cross-language trigger risk is a RISK, not a structural defect (a
|
|
1032
|
-
// language-matched audience is fine), so it's reported separately from the
|
|
1033
|
-
// verdict — it points at the behavioral column, it doesn't fail the scan.
|
|
1034
|
-
const mismatched = r.skills.filter((s) => s.descriptionScript);
|
|
1035
|
-
if (mismatched.length > 0) {
|
|
1036
|
-
out.push(`⚠ ${String(mismatched.length)} skill(s) have descriptions in an unexpected script (cross-language trigger risk) — measure with \`vigiles measure\``, "");
|
|
1037
|
-
}
|
|
1038
987
|
// Skill-metadata is a RECOMMENDATION, not a structural defect (the skill loads
|
|
1039
988
|
// via fallbacks) — reported as a soft note, never counted in the verdict.
|
|
1040
989
|
if (r.skillMetaIssues.length > 0) {
|
package/dist/setup-plan.d.ts
CHANGED
|
@@ -95,7 +95,7 @@ export declare const WORKFLOW_RULES: readonly ["require-instructions-spec", "unt
|
|
|
95
95
|
* keep their own default severities. Named for the group taxonomy
|
|
96
96
|
* (research/install-enforcement-dx.md).
|
|
97
97
|
*/
|
|
98
|
-
export declare const NUDGE_RULES: readonly ["frontmatter-valid", "skill-frontmatter", "prefer-compiled-hooks", "unmarked-refs", "lethal-trifecta", "skill-resource-resolves", "skill-missing-fence", "plugin-dir-layout", "delegation-trifecta", "hook-block-ineffective", "hook-matcher"];
|
|
98
|
+
export declare const NUDGE_RULES: readonly ["skill-description-budget", "frontmatter-valid", "skill-frontmatter", "prefer-compiled-hooks", "unmarked-refs", "lethal-trifecta", "skill-resource-resolves", "skill-missing-fence", "plugin-dir-layout", "delegation-trifecta", "hook-block-ineffective", "hook-matcher"];
|
|
99
99
|
export declare function mergeProjectConfig(existing: Record<string, unknown>, opts: {
|
|
100
100
|
harness: string | string[];
|
|
101
101
|
strict: boolean;
|
package/dist/setup-plan.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "vigiles",
|
|
3
|
-
"version": "12.
|
|
3
|
+
"version": "12.1.0",
|
|
4
4
|
"description": "Lint & test the harness your AI agent runs on — verify the references in your CLAUDE.md / AGENTS.md and test that your hooks and skills actually work.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"claude-code",
|
|
@@ -7,7 +7,16 @@ argument-hint: <path to CLAUDE.md, defaults to CLAUDE.md>
|
|
|
7
7
|
|
|
8
8
|
Start a typed `CLAUDE.md.spec.ts` from an existing hand-written CLAUDE.md (or AGENTS.md). This is the non-destructive adoption path — you keep your existing instruction file as the starting point and get type safety going forward.
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
## Adoption rules
|
|
11
|
+
|
|
12
|
+
Adoption is the **safe, faithful on-ramp — never an upgrade in disguise.** These are non-negotiable:
|
|
13
|
+
|
|
14
|
+
- **Faithful.** Preserve every rule, command, key file, and prose section as-is. Invent nothing — the spec must compile back to ~the user's existing file.
|
|
15
|
+
- **Non-destructive.** Never edit the original `CLAUDE.md` / `AGENTS.md`. Only write the new `.spec.ts`. Never auto-`compile` over the file — switching it to spec-managed is a separate, explicit step the user runs with a diff to review.
|
|
16
|
+
- **Don't escalate enforcement.** Keep `guidance()` as `guidance()`. Upgrading to `enforce()` has a cost (config/plugins, possible false positives) and is a separate opt-in step — the `strengthen` skill. Adoption is **not** turning on strict / `workflow` gating.
|
|
17
|
+
- **Reversible.** `vigiles eject <file>` hands the file back as plain hand-owned markdown anytime — it's never a one-way door. Tell the user this.
|
|
18
|
+
- **Ask before writing.** Present the generated spec and a conversion summary first; write only on the user's yes.
|
|
19
|
+
- **A lighter touch exists.** For no spec at all, inline `<!-- vigiles:enforce ... -->` comments are verified by `vigiles lint` with the same engine.
|
|
11
20
|
|
|
12
21
|
## Instructions
|
|
13
22
|
|
|
@@ -136,6 +136,7 @@ If the vigiles plugin is installed (`/plugin marketplace add zernie/vigiles` the
|
|
|
136
136
|
|
|
137
137
|
## Important
|
|
138
138
|
|
|
139
|
+
- **Do what's asked; don't silently escalate enforcement.** Add the rule the user asked for. If making it an `enforce()` would need a linter-config edit or a plugin install (a cost), or if it could fail a clean CI, **say so and let the user choose** — don't change linter config or flip on strict / `workflow` gating on your own. A pure win (a rule that's already enabled) you can just apply. The `strengthen` skill owns `guidance()` → `enforce()` upgrades.
|
|
139
140
|
- **Never edit CLAUDE.md or AGENTS.md directly** — they have a vigiles hash comment and are build artifacts
|
|
140
141
|
- **The spec is TypeScript** — you get type checking, autocomplete, and verified references
|
|
141
142
|
- **`enforce()` rules are verified** — the compiler checks the rule exists AND is enabled in your linter config
|
|
@@ -5,6 +5,10 @@ description: Upgrade a vigiles spec's guidance() rules to enforce() — scan the
|
|
|
5
5
|
|
|
6
6
|
Scan spec files for `guidance()` rules and suggest `enforce()` replacements backed by real linter rules.
|
|
7
7
|
|
|
8
|
+
## Principle: auto the free wins, nudge for the costs
|
|
9
|
+
|
|
10
|
+
The dividing line is **cost, not strictness**. A `guidance()` → `enforce()` swap where the linter rule **already exists and is enabled** is a pure win — free, reversible, no false-positive risk — so apply it (Tier 1 below). Anything that **costs** something — editing linter config, installing a plugin, or a change that could fail a clean CI — is the user's call: **present it with the tradeoff spelled out and let them choose** (Tiers 2–4). Never silently edit config, install a dependency, or escalate the repo into strict gating. This is the `init` enforcement model (structural = on by default, workflow/strict = opt-in) applied at edit time.
|
|
11
|
+
|
|
8
12
|
## Instructions
|
|
9
13
|
|
|
10
14
|
### Step 0: Choose Mode
|
|
@@ -190,6 +190,23 @@ job that asserts the capability is present, run **`vigiles test --no-skip`** so
|
|
|
190
190
|
skipped tier fails — a green-with-skips is untested surface. Keep unit +
|
|
191
191
|
deterministic tests in CI (free); run evals locally or on a schedule with auth.
|
|
192
192
|
|
|
193
|
+
### After a real-model run: TELL THE USER WHAT IT SPENT
|
|
194
|
+
|
|
195
|
+
Whenever you run a real-model eval (`runEval` / `measureArms` / `measureTriggerRate`
|
|
196
|
+
/ `measure`), **surface the spend to the user in your reply** — don't let a paid run
|
|
197
|
+
be silent. `runEval` prints a cost block to stderr and every report carries `usage`
|
|
198
|
+
(`report.arms[*].usage`: `totalCostUsd` + token counts). Relay, in plain words:
|
|
199
|
+
|
|
200
|
+
- **tokens spent** and the **API-equivalent `$`** (`total_cost_usd` — what it _would_
|
|
201
|
+
cost at metered API rates);
|
|
202
|
+
- **how it was billed** — "on your Claude subscription (**$0 metered**)" if you're
|
|
203
|
+
logged in, or a **⚠ warning** if `ANTHROPIC_API_KEY` is set (that run was billed
|
|
204
|
+
**per token** — tell them to unset it and `claude login` to run free).
|
|
205
|
+
|
|
206
|
+
We do **not** show "% of your subscription" — Anthropic doesn't expose a plan's
|
|
207
|
+
quota, so any percentage would be invented. Tokens + API-equivalent `$` + the
|
|
208
|
+
billed-to line is the honest, complete picture. Keep the user's cost visible, always.
|
|
209
|
+
|
|
193
210
|
## Step 5 — Lock the eval so CI stays honest (you do this automatically)
|
|
194
211
|
|
|
195
212
|
Real-model evals run on the user's subscription — locally, never in CI. So **as
|