vigiles 12.0.0 → 12.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -11
- package/action.yml +73 -0
- package/dist/audit-report.d.ts +11 -0
- package/dist/audit-report.js +1 -0
- package/dist/audit-report.template.html +36 -26
- package/dist/claude-code.d.ts +2 -0
- package/dist/claude-code.js +9 -1
- package/dist/cli-commands.d.ts +1 -1
- package/dist/cli-commands.js +0 -1
- package/dist/cli.js +96 -135
- package/dist/core/rule-meta.js +8 -0
- package/dist/core/skill-description-budget.d.ts +42 -0
- package/dist/core/skill-description-budget.js +47 -0
- package/dist/core/types.d.ts +9 -0
- package/dist/core/validate.js +3 -0
- package/dist/doc-command-coverage.d.ts +20 -0
- package/dist/doc-command-coverage.js +60 -0
- package/dist/eval-cost.d.ts +75 -0
- package/dist/eval-cost.js +134 -0
- package/dist/eval.d.ts +4 -0
- package/dist/eval.js +46 -6
- package/dist/observe.d.ts +109 -0
- package/dist/observe.js +164 -0
- package/dist/research-index.d.ts +31 -0
- package/dist/research-index.js +48 -0
- package/dist/scaffold-test.js +3 -2
- package/dist/scan-behavioral.d.ts +42 -0
- package/dist/scan-behavioral.js +67 -0
- package/dist/scan.d.ts +3 -23
- package/dist/scan.js +18 -69
- package/dist/setup-plan.d.ts +1 -1
- package/dist/setup-plan.js +1 -0
- package/package.json +1 -1
- package/skills/adopt-spec/SKILL.md +10 -1
- package/skills/debug-my-harness/SKILL.md +56 -0
- package/skills/edit-spec/SKILL.md +1 -0
- package/skills/strengthen/SKILL.md +4 -0
- package/skills/test-harness/SKILL.md +17 -0
- package/dist/core/hook-spec.d.ts +0 -74
- package/dist/core/hook-spec.js +0 -130
package/dist/claude-code.d.ts
CHANGED
|
@@ -10,6 +10,8 @@ export * from "./mock-model.js";
|
|
|
10
10
|
export { claudeCodeDriver, buildClaudeArgs, parseClaudeRun, claudeAvailable, } from "./harness-test.js";
|
|
11
11
|
export * from "./adapters/claude-code/dialect.js";
|
|
12
12
|
export { agent, skill, type ClaudeCodeToolVocabulary, } from "./adapters/claude-code/typed-spec.js";
|
|
13
|
+
export { measureSelectionMatrix, assertNoCollision, formatSelectionReport, } from "./scan-behavioral.js";
|
|
14
|
+
export type { SelectionReport, SkillSelectionStat, SelectionOptions, SelectionMatrixOptions, } from "./scan-behavioral.js";
|
|
13
15
|
export * from "./adapters/claude-code/layout.js";
|
|
14
16
|
export * from "./adapters/claude-code/runtime.js";
|
|
15
17
|
export * from "./adapters/claude-code/hook-protocol.js";
|
package/dist/claude-code.js
CHANGED
|
@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.skill = exports.agent = exports.claudeAvailable = exports.parseClaudeRun = exports.buildClaudeArgs = exports.claudeCodeDriver = void 0;
|
|
17
|
+
exports.formatSelectionReport = exports.assertNoCollision = exports.measureSelectionMatrix = exports.skill = exports.agent = exports.claudeAvailable = exports.parseClaudeRun = exports.buildClaudeArgs = exports.claudeCodeDriver = void 0;
|
|
18
18
|
/**
|
|
19
19
|
* `vigiles/claude-code` — the Claude Code-specific harness pieces a *different*
|
|
20
20
|
* harness would swap out: the plugin/repo loader (reads real Claude Code plugin
|
|
@@ -42,6 +42,14 @@ __exportStar(require("./adapters/claude-code/dialect.js"), exports);
|
|
|
42
42
|
var typed_spec_js_1 = require("./adapters/claude-code/typed-spec.js");
|
|
43
43
|
Object.defineProperty(exports, "agent", { enumerable: true, get: function () { return typed_spec_js_1.agent; } });
|
|
44
44
|
Object.defineProperty(exports, "skill", { enumerable: true, get: function () { return typed_spec_js_1.skill; } });
|
|
45
|
+
// Selection-collision — a Claude-Code-ONLY behavioral measurement (Codex has no
|
|
46
|
+
// skill-selection event to read), so it lives on this surface, not the agnostic
|
|
47
|
+
// `vigiles/testing`. `measureSelectionMatrix` builds the N×N "which skill fired?"
|
|
48
|
+
// matrix (diagonal = recall, off-diagonal = collision); `assertNoCollision` gates it.
|
|
49
|
+
var scan_behavioral_js_1 = require("./scan-behavioral.js");
|
|
50
|
+
Object.defineProperty(exports, "measureSelectionMatrix", { enumerable: true, get: function () { return scan_behavioral_js_1.measureSelectionMatrix; } });
|
|
51
|
+
Object.defineProperty(exports, "assertNoCollision", { enumerable: true, get: function () { return scan_behavioral_js_1.assertNoCollision; } });
|
|
52
|
+
Object.defineProperty(exports, "formatSelectionReport", { enumerable: true, get: function () { return scan_behavioral_js_1.formatSelectionReport; } });
|
|
45
53
|
__exportStar(require("./adapters/claude-code/layout.js"), exports);
|
|
46
54
|
__exportStar(require("./adapters/claude-code/runtime.js"), exports);
|
|
47
55
|
__exportStar(require("./adapters/claude-code/hook-protocol.js"), exports);
|
package/dist/cli-commands.d.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* recognizes exactly these, so this list can't silently drift from the code.
|
|
12
12
|
*/
|
|
13
13
|
/** Human-facing verbs (printed in help; typed by a human/agent/CI). */
|
|
14
|
-
export declare const VERBS: readonly ["init", "compile", "eject", "lint", "test", "eval", "audit", "
|
|
14
|
+
export declare const VERBS: readonly ["init", "compile", "eject", "lint", "test", "eval", "audit", "generate", "hook-runtime"];
|
|
15
15
|
/** Runtime entrypoint kinds under `vigiles hook-runtime <kind>` (emitted, not typed). */
|
|
16
16
|
export declare const HOOK_RUNTIME_KINDS: readonly ["run-program", "agent", "agent-start", "agent-done", "skill", "skill-tool", "skill-start", "skill-done", "run-skill", "intercept-tool", "guard", "action", "refs", "eval-lock-nudge", "effect-enter", "effect-exit"];
|
|
17
17
|
export type Verb = (typeof VERBS)[number];
|
package/dist/cli-commands.js
CHANGED
package/dist/cli.js
CHANGED
|
@@ -22,8 +22,6 @@ const cli_flags_js_1 = require("./cli-flags.js");
|
|
|
22
22
|
const setup_plan_js_1 = require("./setup-plan.js");
|
|
23
23
|
const types_js_1 = require("./core/types.js");
|
|
24
24
|
const test_coverage_js_1 = require("./test-coverage.js");
|
|
25
|
-
const scaffold_test_js_1 = require("./scaffold-test.js");
|
|
26
|
-
const effects_js_1 = require("./core/effects.js");
|
|
27
25
|
const scan_js_1 = require("./scan.js");
|
|
28
26
|
const scan_trigger_suggest_js_1 = require("./scan-trigger-suggest.js");
|
|
29
27
|
const dialect_drift_js_1 = require("./dialect-drift.js");
|
|
@@ -52,6 +50,7 @@ const hook_install_js_1 = require("./hook-install.js");
|
|
|
52
50
|
const hook_providers_js_1 = require("./core/hook-providers.js");
|
|
53
51
|
const toml_1 = require("@iarna/toml");
|
|
54
52
|
const agent_runtime_js_1 = require("./adapters/claude-code/agent-runtime.js");
|
|
53
|
+
const observe_js_1 = require("./observe.js");
|
|
55
54
|
const effect_region_js_1 = require("./adapters/claude-code/effect-region.js");
|
|
56
55
|
const tool_intercept_js_1 = require("./tool-intercept.js");
|
|
57
56
|
const refs_js_1 = require("./core/refs.js");
|
|
@@ -676,6 +675,7 @@ function lintExitCode(report) {
|
|
|
676
675
|
report.hookScriptErrors > 0 ||
|
|
677
676
|
report.disallowedToolErrors > 0 ||
|
|
678
677
|
report.descriptionOverlapErrors > 0 ||
|
|
678
|
+
report.descriptionBudgetErrors > 0 ||
|
|
679
679
|
report.frontmatterValidErrors > 0 ||
|
|
680
680
|
report.mcpHookErrors > 0 ||
|
|
681
681
|
report.preferCompiledHookErrors > 0 ||
|
|
@@ -1027,6 +1027,10 @@ async function runLint(restArgs, flags, config) {
|
|
|
1027
1027
|
// 7k. Description-overlap — two model-invocable skills with near-identical
|
|
1028
1028
|
// descriptions collide in the selector (deterministic NCD precision proxy).
|
|
1029
1029
|
const descriptionOverlap = checkDescriptionOverlap(config, silent, adapter);
|
|
1030
|
+
// 7k². Skill-description-budget — a model-invocable skill whose description is
|
|
1031
|
+
// so long the trigger signal is buried (heuristic proxy; degrades recall +
|
|
1032
|
+
// precision). Generous 500-char budget; warn-tier, never gates.
|
|
1033
|
+
const descriptionBudget = checkDescriptionBudget(config, silent, adapter);
|
|
1030
1034
|
// 7l. Frontmatter-valid — a `---` block that isn't valid YAML (warn; js-yaml is
|
|
1031
1035
|
// stricter than some loaders, so verify before enforcing).
|
|
1032
1036
|
const frontmatterValid = checkFrontmatterValid(config, silent, adapter);
|
|
@@ -1120,6 +1124,8 @@ async function runLint(restArgs, flags, config) {
|
|
|
1120
1124
|
disallowedToolErrors: disallowedTools.errors,
|
|
1121
1125
|
descriptionOverlapIssues: descriptionOverlap.issues,
|
|
1122
1126
|
descriptionOverlapErrors: descriptionOverlap.errors,
|
|
1127
|
+
descriptionBudgetIssues: descriptionBudget.issues,
|
|
1128
|
+
descriptionBudgetErrors: descriptionBudget.errors,
|
|
1123
1129
|
frontmatterValidIssues: frontmatterValid.issues,
|
|
1124
1130
|
frontmatterValidErrors: frontmatterValid.errors,
|
|
1125
1131
|
mcpHookIssues: mcpHookTargets.issues,
|
|
@@ -2775,6 +2781,33 @@ function checkDescriptionOverlap(config, silent, adapter) {
|
|
|
2775
2781
|
}
|
|
2776
2782
|
return { issues: found.length, errors: sev === "error" ? found.length : 0 };
|
|
2777
2783
|
}
|
|
2784
|
+
/**
|
|
2785
|
+
* Apply the `skill-description-budget` rule: a model-invocable skill whose
|
|
2786
|
+
* description is so long the trigger signal is buried — the selector weighs the
|
|
2787
|
+
* opening most, so a bloated description hurts recall + precision. A
|
|
2788
|
+
* deterministic heuristic proxy (generous 500-char budget). Reuses `scanPlugin`'s
|
|
2789
|
+
* `descriptionBudgetIssues`. Warning by default; "error" gates CI.
|
|
2790
|
+
*/
|
|
2791
|
+
function checkDescriptionBudget(config, silent, adapter) {
|
|
2792
|
+
const sev = (0, types_js_1.ruleSeverity)(config?.rules?.["skill-description-budget"]);
|
|
2793
|
+
if (!sev)
|
|
2794
|
+
return { issues: 0, errors: 0 };
|
|
2795
|
+
let found;
|
|
2796
|
+
try {
|
|
2797
|
+
found = (0, scan_js_1.scanPlugin)(process.cwd(), adapter.layout, adapter.dialect).descriptionBudgetIssues;
|
|
2798
|
+
}
|
|
2799
|
+
catch {
|
|
2800
|
+
return { issues: 0, errors: 0 };
|
|
2801
|
+
}
|
|
2802
|
+
if (found.length > 0 && !silent) {
|
|
2803
|
+
console.log("\nSkill-description-budget check:\n");
|
|
2804
|
+
for (const issue of found) {
|
|
2805
|
+
console.log(` ${sev === "error" ? "✗" : "⚠"} ${issue.message}`);
|
|
2806
|
+
ghAnnotate(sev === "error" ? "error" : "warning", issue.message);
|
|
2807
|
+
}
|
|
2808
|
+
}
|
|
2809
|
+
return { issues: found.length, errors: sev === "error" ? found.length : 0 };
|
|
2810
|
+
}
|
|
2778
2811
|
/**
|
|
2779
2812
|
* Apply the `lethal-trifecta` rule: a unit (subagent / model-invocable skill)
|
|
2780
2813
|
* whose declared tools hold all three legs (read-private + ingest-untrusted +
|
|
@@ -3577,135 +3610,6 @@ function capabilitiesOfReport(report, dialect) {
|
|
|
3577
3610
|
}));
|
|
3578
3611
|
return (0, generate_harness_js_1.computeHarnessCapabilities)(agents, dialect);
|
|
3579
3612
|
}
|
|
3580
|
-
/**
|
|
3581
|
-
* The plugin's declared name for the namespaced skill id, read from the layout's
|
|
3582
|
-
* manifest (adapter-aware path, not a hardcoded `.claude-plugin/`), falling back to
|
|
3583
|
-
* the dir basename. JSON manifests only for now (a TOML/Codex manifest → basename).
|
|
3584
|
-
*/
|
|
3585
|
-
function pluginNameFor(dir, manifestPath) {
|
|
3586
|
-
try {
|
|
3587
|
-
const manifest = JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.resolve)(dir, manifestPath), "utf-8"));
|
|
3588
|
-
if (typeof manifest.name === "string" && manifest.name)
|
|
3589
|
-
return manifest.name;
|
|
3590
|
-
}
|
|
3591
|
-
catch {
|
|
3592
|
-
/* missing / non-JSON manifest → fall back */
|
|
3593
|
-
}
|
|
3594
|
-
return (0, node_path_1.basename)(dir);
|
|
3595
|
-
}
|
|
3596
|
-
/** Enrich an untested Surface with the metadata the right template needs. */
|
|
3597
|
-
/** Extract `"name": type` fields from one rendered `vigiles:ok`/`err` shape block. */
|
|
3598
|
-
function parseContractFields(block) {
|
|
3599
|
-
const fields = [];
|
|
3600
|
-
const re = /"([^"]+)"\s*:\s*(string\[\]|string|number|boolean)/g;
|
|
3601
|
-
let m;
|
|
3602
|
-
while ((m = re.exec(block)) !== null) {
|
|
3603
|
-
fields.push({ name: m[1], type: m[2] });
|
|
3604
|
-
}
|
|
3605
|
-
return fields;
|
|
3606
|
-
}
|
|
3607
|
-
/**
|
|
3608
|
-
* Parse a subagent's compiled `## Output contract` (the `vigiles:ok` / `vigiles:err`
|
|
3609
|
-
* blocks the compiler emits) back into a typed `ResultContract`, so the generator
|
|
3610
|
-
* can write an `assertAgentOk` test against the real fields. Returns null when the
|
|
3611
|
-
* agent has no result() contract.
|
|
3612
|
-
*/
|
|
3613
|
-
function parseResultContract(md) {
|
|
3614
|
-
const ok = /```vigiles:ok\n([\s\S]*?)```/.exec(md);
|
|
3615
|
-
const err = /```vigiles:err\n([\s\S]*?)```/.exec(md);
|
|
3616
|
-
if (!ok && !err)
|
|
3617
|
-
return null;
|
|
3618
|
-
const okFields = ok ? parseContractFields(ok[1]) : [];
|
|
3619
|
-
const errFields = err ? parseContractFields(err[1]) : [];
|
|
3620
|
-
if (okFields.length === 0 && errFields.length === 0)
|
|
3621
|
-
return null;
|
|
3622
|
-
return { ok: okFields, err: errFields };
|
|
3623
|
-
}
|
|
3624
|
-
function scaffoldInputFor(s, report, pluginName, dir, dialect) {
|
|
3625
|
-
const base = { kind: s.kind, name: s.name, path: s.path };
|
|
3626
|
-
switch (s.kind) {
|
|
3627
|
-
case "skill": {
|
|
3628
|
-
const sk = report.skills.find((x) => x.name === s.name);
|
|
3629
|
-
return { ...base, pluginName, userInvoked: sk?.userInvoked };
|
|
3630
|
-
}
|
|
3631
|
-
case "agent": {
|
|
3632
|
-
const ag = report.agents.find((x) => x.name === s.name);
|
|
3633
|
-
const tools = ag?.tools ?? null;
|
|
3634
|
-
const sideEffectingTools = tools
|
|
3635
|
-
? (0, effects_js_1.effectSurface)(tools, dialect).sideEffecting
|
|
3636
|
-
: undefined;
|
|
3637
|
-
let resultContract = null;
|
|
3638
|
-
try {
|
|
3639
|
-
resultContract = parseResultContract((0, node_fs_1.readFileSync)((0, node_path_1.resolve)(dir, s.path), "utf-8"));
|
|
3640
|
-
}
|
|
3641
|
-
catch {
|
|
3642
|
-
// agent .md unreadable → no contract to generate against
|
|
3643
|
-
}
|
|
3644
|
-
return { ...base, tools, sideEffectingTools, resultContract };
|
|
3645
|
-
}
|
|
3646
|
-
case "hook":
|
|
3647
|
-
return { ...base, hookCommand: `bash ${s.path}` };
|
|
3648
|
-
}
|
|
3649
|
-
}
|
|
3650
|
-
/**
|
|
3651
|
-
* `vigiles scaffold-test [dir]` — generate a runnable STARTER test for each
|
|
3652
|
-
* untested skill/agent/hook (B1, test-gen from free-form). Reuses the
|
|
3653
|
-
* untested-surface detector for the list + `scan` for the metadata, then emits the
|
|
3654
|
-
* cheapest meaningful tier per kind (hook → `runHook`, skill → `measureTriggerRate`,
|
|
3655
|
-
* subagent → `runHarnessTest`) at the surface's suggested test path. Dry-run by
|
|
3656
|
-
* default (prints the scaffolds); `--write` creates the files (never clobbering an
|
|
3657
|
-
* existing one); `--json` for the agent-consumable `{ path, content }[]`.
|
|
3658
|
-
*/
|
|
3659
|
-
function handleScaffoldTest(restArgs, args) {
|
|
3660
|
-
const dir = (0, node_path_1.resolve)(restArgs[0] ?? ".");
|
|
3661
|
-
const write = args.includes("--write");
|
|
3662
|
-
const json = args.includes("--json");
|
|
3663
|
-
const harnessFlag = harnessFlagFrom(args);
|
|
3664
|
-
const adapter = harnessFlag
|
|
3665
|
-
? (0, adapter_registry_js_1.resolveAdapter)(dir, harnessFlag)
|
|
3666
|
-
: (0, adapter_registry_js_1.detectAdapterResult)(dir).adapter;
|
|
3667
|
-
const { untested } = (0, test_coverage_js_1.findUntestedSurfaces)({
|
|
3668
|
-
basePath: dir,
|
|
3669
|
-
layout: adapter.layout,
|
|
3670
|
-
});
|
|
3671
|
-
const report = (0, scan_js_1.scanPlugin)(dir, adapter.layout, adapter.dialect);
|
|
3672
|
-
const pluginName = pluginNameFor(dir, adapter.layout.manifestPath);
|
|
3673
|
-
const scaffolds = untested.map((s) => (0, scaffold_test_js_1.scaffoldTest)(scaffoldInputFor(s, report, pluginName, dir, adapter.dialect)));
|
|
3674
|
-
if (json) {
|
|
3675
|
-
console.log(JSON.stringify(scaffolds, null, 2));
|
|
3676
|
-
return;
|
|
3677
|
-
}
|
|
3678
|
-
if (!write) {
|
|
3679
|
-
console.log((0, scaffold_test_js_1.formatScaffolds)(scaffolds));
|
|
3680
|
-
for (const s of scaffolds) {
|
|
3681
|
-
console.log(`\n# ${s.path}\n`);
|
|
3682
|
-
console.log(s.content);
|
|
3683
|
-
}
|
|
3684
|
-
if (scaffolds.length > 0) {
|
|
3685
|
-
console.log("Re-run with --write to create these files.");
|
|
3686
|
-
}
|
|
3687
|
-
return;
|
|
3688
|
-
}
|
|
3689
|
-
const written = [];
|
|
3690
|
-
const skipped = [];
|
|
3691
|
-
for (const s of scaffolds) {
|
|
3692
|
-
const target = (0, node_path_1.resolve)(dir, s.path);
|
|
3693
|
-
if ((0, node_fs_1.existsSync)(target)) {
|
|
3694
|
-
skipped.push(s.path);
|
|
3695
|
-
continue;
|
|
3696
|
-
}
|
|
3697
|
-
(0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(target), { recursive: true });
|
|
3698
|
-
(0, node_fs_1.writeFileSync)(target, s.content);
|
|
3699
|
-
written.push(s.path);
|
|
3700
|
-
}
|
|
3701
|
-
for (const p of written)
|
|
3702
|
-
console.log(`✓ wrote ${p}`);
|
|
3703
|
-
for (const p of skipped)
|
|
3704
|
-
console.log(`⊘ skipped ${p} (already exists)`);
|
|
3705
|
-
if (written.length === 0 && skipped.length === 0) {
|
|
3706
|
-
console.log("Nothing to scaffold — every surface already has a test.");
|
|
3707
|
-
}
|
|
3708
|
-
}
|
|
3709
3613
|
function printUsage(command) {
|
|
3710
3614
|
console.log("vigiles — compile typed specs to instruction files");
|
|
3711
3615
|
console.log("");
|
|
@@ -3722,7 +3626,6 @@ function printUsage(command) {
|
|
|
3722
3626
|
console.log(" vigiles eval [files...] Run *.eval.mjs real-model harness evals (--trials=N, --min=N, --no-skip)");
|
|
3723
3627
|
console.log(" --update records each named eval's result to a committed lock (run locally on your subscription)");
|
|
3724
3628
|
console.log(" --check verifies committed eval results against current inputs WITHOUT a model — the CI staleness gate");
|
|
3725
|
-
console.log(" vigiles scaffold-test [dir] Generate a starter test for each untested skill/agent/hook (--write, --json)");
|
|
3726
3629
|
console.log("");
|
|
3727
3630
|
console.log("Examples:");
|
|
3728
3631
|
console.log(" vigiles init Auto-detect project, create specs, wire CI");
|
|
@@ -3828,6 +3731,11 @@ function skillStartCommand(target) {
|
|
|
3828
3731
|
process.exit(2);
|
|
3829
3732
|
}
|
|
3830
3733
|
(0, skill_runtime_js_1.setActiveSkill)(process.cwd(), target);
|
|
3734
|
+
// Record the fire in the flight recorder: the skill NAME is the parent dir of
|
|
3735
|
+
// its SKILL.md (skills/<name>/SKILL.md), falling back to the raw target.
|
|
3736
|
+
const parts = target.replace(/\\/g, "/").split("/").filter(Boolean);
|
|
3737
|
+
const name = parts.length >= 2 ? parts[parts.length - 2] : (parts[0] ?? target);
|
|
3738
|
+
(0, observe_js_1.appendObservation)({ kind: "skill", name, fired: true });
|
|
3831
3739
|
console.log(`Active skill: ${target}`);
|
|
3832
3740
|
}
|
|
3833
3741
|
/**
|
|
@@ -3928,6 +3836,13 @@ function agentHookCommand() {
|
|
|
3928
3836
|
return;
|
|
3929
3837
|
const decision = (0, agent_runtime_js_1.evaluatePreToolUse)(cwd, tool, command);
|
|
3930
3838
|
if (!decision.allow) {
|
|
3839
|
+
(0, observe_js_1.appendObservation)({
|
|
3840
|
+
kind: "agent",
|
|
3841
|
+
name: (0, agent_runtime_js_1.readActiveAgent)(cwd) ?? "unknown",
|
|
3842
|
+
tool,
|
|
3843
|
+
allowed: false,
|
|
3844
|
+
reason: decision.message,
|
|
3845
|
+
});
|
|
3931
3846
|
console.error(decision.message);
|
|
3932
3847
|
process.exit(2);
|
|
3933
3848
|
}
|
|
@@ -4476,10 +4391,26 @@ function emitGate(decision, on, mode, file) {
|
|
|
4476
4391
|
const action = (0, hook_program_js_1.gateAction)(decision, mode);
|
|
4477
4392
|
switch (action.kind) {
|
|
4478
4393
|
case "block":
|
|
4394
|
+
(0, observe_js_1.appendObservation)({
|
|
4395
|
+
kind: "hook",
|
|
4396
|
+
event: on,
|
|
4397
|
+
decision: "deny",
|
|
4398
|
+
mode: "enforce",
|
|
4399
|
+
rule: file,
|
|
4400
|
+
reason: action.reason,
|
|
4401
|
+
});
|
|
4479
4402
|
console.error(action.reason);
|
|
4480
4403
|
process.exit(2);
|
|
4481
4404
|
return;
|
|
4482
4405
|
case "ask":
|
|
4406
|
+
(0, observe_js_1.appendObservation)({
|
|
4407
|
+
kind: "hook",
|
|
4408
|
+
event: on,
|
|
4409
|
+
decision: "ask",
|
|
4410
|
+
mode: "enforce",
|
|
4411
|
+
rule: file,
|
|
4412
|
+
reason: action.reason,
|
|
4413
|
+
});
|
|
4483
4414
|
process.stdout.write(JSON.stringify({
|
|
4484
4415
|
hookSpecificOutput: {
|
|
4485
4416
|
hookEventName: on,
|
|
@@ -4489,6 +4420,14 @@ function emitGate(decision, on, mode, file) {
|
|
|
4489
4420
|
}) + "\n");
|
|
4490
4421
|
return;
|
|
4491
4422
|
case "observe":
|
|
4423
|
+
(0, observe_js_1.appendObservation)({
|
|
4424
|
+
kind: "hook",
|
|
4425
|
+
event: on,
|
|
4426
|
+
decision: action.would,
|
|
4427
|
+
mode: "observe",
|
|
4428
|
+
rule: file,
|
|
4429
|
+
reason: action.reason,
|
|
4430
|
+
});
|
|
4492
4431
|
recordObservation(file, on, action.would, action.reason);
|
|
4493
4432
|
console.error(`⚠ [vigiles observe] ${on}: would ${action.would} — ${action.reason}`);
|
|
4494
4433
|
return; // exit 0 — observe never blocks
|
|
@@ -5078,10 +5017,14 @@ async function main() {
|
|
|
5078
5017
|
// Surfaced in the AuditReport (the report's "Create spec" command-emit
|
|
5079
5018
|
// buttons read it) and the terminal nudge below.
|
|
5080
5019
|
const adoptableSurfaces = discoverAdoptableForAudit(root, adapter.layout.instructionFile);
|
|
5020
|
+
// Read the local flight recorder ONCE — feeds both the JSON report
|
|
5021
|
+
// (structured summary, the product boundary) and the terminal render.
|
|
5022
|
+
const ledgerRecords = (0, observe_js_1.readObservations)(root);
|
|
5081
5023
|
const auditReport = (0, audit_report_js_1.buildAuditReport)(report, {
|
|
5082
5024
|
harness: adapter.name,
|
|
5083
5025
|
vigilesVersion: getVersion(),
|
|
5084
5026
|
adoptableSurfaces,
|
|
5027
|
+
observations: (0, observe_js_1.summarizeObservations)(ledgerRecords),
|
|
5085
5028
|
});
|
|
5086
5029
|
const sc = auditReport.score;
|
|
5087
5030
|
const plan = (0, optimize_js_1.optimize)(report);
|
|
@@ -5112,6 +5055,12 @@ async function main() {
|
|
|
5112
5055
|
.length);
|
|
5113
5056
|
if (fireNudge)
|
|
5114
5057
|
console.log("\n" + fireNudge);
|
|
5058
|
+
// The flight recorder: a compact summary of what the harness actually
|
|
5059
|
+
// DID in real sessions (hook/agent decisions), read off the local
|
|
5060
|
+
// agent-readable ledger. Empty (skipped) until something is recorded.
|
|
5061
|
+
const ledgerSummary = (0, observe_js_1.formatLedgerSummary)(ledgerRecords);
|
|
5062
|
+
if (ledgerSummary)
|
|
5063
|
+
console.log("\n" + ledgerSummary);
|
|
5115
5064
|
}
|
|
5116
5065
|
// ONE read-vs-run decision for the EXECUTING checks (live MCP + skill
|
|
5117
5066
|
// firing). A plain `audit` is a deterministic READ; these run only on
|
|
@@ -5145,6 +5094,21 @@ async function main() {
|
|
|
5145
5094
|
// default; `--fail-on-widen` exits non-zero (the opt-in CI gate).
|
|
5146
5095
|
const beforeReport = (0, scan_js_1.scanPlugin)((0, node_path_1.resolve)(capBase), adapter.layout, adapter.dialect);
|
|
5147
5096
|
const diff = (0, capability_diff_js_1.diffCapabilities)(capabilitiesOfReport(beforeReport, adapter.dialect), capabilitiesOfReport(report, adapter.dialect));
|
|
5097
|
+
// Feed the flight recorder: the blast-radius change (moat #2) as a record.
|
|
5098
|
+
// Write to the AUDITED root's ledger (not the caller's cwd) — the same
|
|
5099
|
+
// `root` the audit reads back via `readObservations(root)`, so a
|
|
5100
|
+
// `vigiles audit ./after --capability-diff=./before` from a parent dir
|
|
5101
|
+
// records into ./after/.vigiles/, not the parent workspace.
|
|
5102
|
+
(0, observe_js_1.appendObservation)({
|
|
5103
|
+
kind: "capability-diff",
|
|
5104
|
+
added: [
|
|
5105
|
+
...diff.addedSideEffecting,
|
|
5106
|
+
...diff.addedUnknown,
|
|
5107
|
+
...diff.addedReadOnly,
|
|
5108
|
+
],
|
|
5109
|
+
removed: [...diff.removed],
|
|
5110
|
+
widened: diff.widened,
|
|
5111
|
+
}, root);
|
|
5148
5112
|
console.log(json
|
|
5149
5113
|
? JSON.stringify({ capabilityDiff: diff }, null, 2)
|
|
5150
5114
|
: "\n" + (0, capability_diff_js_1.formatCapabilityDiff)(diff));
|
|
@@ -5243,9 +5207,6 @@ async function main() {
|
|
|
5243
5207
|
}
|
|
5244
5208
|
break;
|
|
5245
5209
|
}
|
|
5246
|
-
case "scaffold-test":
|
|
5247
|
-
handleScaffoldTest(restArgs, args);
|
|
5248
|
-
break;
|
|
5249
5210
|
// --- Plumbing ---
|
|
5250
5211
|
case "generate":
|
|
5251
5212
|
await handleGenerate(restArgs, args);
|
package/dist/core/rule-meta.js
CHANGED
|
@@ -202,6 +202,14 @@ exports.RULE_META = {
|
|
|
202
202
|
summary: "Two model-invocable skills aren't near-identical (wrong one fires).",
|
|
203
203
|
detector: "findDescriptionOverlaps",
|
|
204
204
|
},
|
|
205
|
+
"skill-description-budget": {
|
|
206
|
+
id: "skill-description-budget",
|
|
207
|
+
bucket: "heuristic-behavioral",
|
|
208
|
+
surface: ["skill"],
|
|
209
|
+
defaultSeverity: "warn",
|
|
210
|
+
summary: "A model-invocable skill's description isn't so long the trigger is buried.",
|
|
211
|
+
detector: "findDescriptionBudgetIssues",
|
|
212
|
+
},
|
|
205
213
|
"frontmatter-valid": {
|
|
206
214
|
id: "frontmatter-valid",
|
|
207
215
|
bucket: "heuristic-behavioral",
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Skill-description budget — a DETERMINISTIC proxy for a behavioral risk, and the
|
|
3
|
+
* deterministic sibling of {@link findDescriptionOverlaps}. A model-invocable
|
|
4
|
+
* skill is selected on its `description`, and the selector weighs the OPENING of
|
|
5
|
+
* it most; a long, buried description dilutes the trigger signal and degrades
|
|
6
|
+
* both recall ("did it fire when it should?") and precision ("did it stay quiet
|
|
7
|
+
* when it shouldn't?"). This catches a trigger-class problem with NO model.
|
|
8
|
+
*
|
|
9
|
+
* HEURISTIC-BEHAVIORAL bucket: the threshold is a PROXY (no character count
|
|
10
|
+
* PROVES a description triggers badly), so the ceiling is WARN — it never gates.
|
|
11
|
+
* Calibrated FP-safe: the default budget (500 chars) sits well above a normal
|
|
12
|
+
* one-to-three-sentence description, so only a genuinely bloated description
|
|
13
|
+
* fires. Reports the per-skill overflow, never a unilateral defect.
|
|
14
|
+
*/
|
|
15
|
+
/** A skill identified by name + its trigger-surface description. */
|
|
16
|
+
export interface BudgetedSurface {
|
|
17
|
+
readonly name: string;
|
|
18
|
+
readonly description: string;
|
|
19
|
+
}
|
|
20
|
+
export interface DescriptionBudgetIssue {
|
|
21
|
+
readonly name: string;
|
|
22
|
+
/** Length of the description in characters. */
|
|
23
|
+
readonly length: number;
|
|
24
|
+
/** The budget it exceeded. */
|
|
25
|
+
readonly budget: number;
|
|
26
|
+
readonly message: string;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* The default description-length budget, in characters. A concise what+when
|
|
30
|
+
* description is comfortably under this; only a bloated one (multiple long
|
|
31
|
+
* sentences, embedded examples, disambiguation prose) exceeds it. Generous on
|
|
32
|
+
* purpose — warn-tier, don't cry wolf. Exported so a caller / test sees it.
|
|
33
|
+
*/
|
|
34
|
+
export declare const DEFAULT_DESCRIPTION_BUDGET = 500;
|
|
35
|
+
/**
|
|
36
|
+
* Find model-invocable skills whose `description` exceeds `budget` characters.
|
|
37
|
+
* Returns one {@link DescriptionBudgetIssue} per over-budget skill, longest
|
|
38
|
+
* first. Pure; pass only the surfaces that compete for auto-selection
|
|
39
|
+
* (model-invocable, described) so a user-invoked skill isn't a false alarm.
|
|
40
|
+
*/
|
|
41
|
+
export declare function findDescriptionBudgetIssues(surfaces: readonly BudgetedSurface[], budget?: number): DescriptionBudgetIssue[];
|
|
42
|
+
//# sourceMappingURL=skill-description-budget.d.ts.map
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Skill-description budget — a DETERMINISTIC proxy for a behavioral risk, and the
|
|
4
|
+
* deterministic sibling of {@link findDescriptionOverlaps}. A model-invocable
|
|
5
|
+
* skill is selected on its `description`, and the selector weighs the OPENING of
|
|
6
|
+
* it most; a long, buried description dilutes the trigger signal and degrades
|
|
7
|
+
* both recall ("did it fire when it should?") and precision ("did it stay quiet
|
|
8
|
+
* when it shouldn't?"). This catches a trigger-class problem with NO model.
|
|
9
|
+
*
|
|
10
|
+
* HEURISTIC-BEHAVIORAL bucket: the threshold is a PROXY (no character count
|
|
11
|
+
* PROVES a description triggers badly), so the ceiling is WARN — it never gates.
|
|
12
|
+
* Calibrated FP-safe: the default budget (500 chars) sits well above a normal
|
|
13
|
+
* one-to-three-sentence description, so only a genuinely bloated description
|
|
14
|
+
* fires. Reports the per-skill overflow, never a unilateral defect.
|
|
15
|
+
*/
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.DEFAULT_DESCRIPTION_BUDGET = void 0;
|
|
18
|
+
exports.findDescriptionBudgetIssues = findDescriptionBudgetIssues;
|
|
19
|
+
/**
|
|
20
|
+
* The default description-length budget, in characters. A concise what+when
|
|
21
|
+
* description is comfortably under this; only a bloated one (multiple long
|
|
22
|
+
* sentences, embedded examples, disambiguation prose) exceeds it. Generous on
|
|
23
|
+
* purpose — warn-tier, don't cry wolf. Exported so a caller / test sees it.
|
|
24
|
+
*/
|
|
25
|
+
exports.DEFAULT_DESCRIPTION_BUDGET = 500;
|
|
26
|
+
/**
|
|
27
|
+
* Find model-invocable skills whose `description` exceeds `budget` characters.
|
|
28
|
+
* Returns one {@link DescriptionBudgetIssue} per over-budget skill, longest
|
|
29
|
+
* first. Pure; pass only the surfaces that compete for auto-selection
|
|
30
|
+
* (model-invocable, described) so a user-invoked skill isn't a false alarm.
|
|
31
|
+
*/
|
|
32
|
+
function findDescriptionBudgetIssues(surfaces, budget = exports.DEFAULT_DESCRIPTION_BUDGET) {
|
|
33
|
+
const issues = [];
|
|
34
|
+
for (const s of surfaces) {
|
|
35
|
+
const length = Array.from(s.description).length;
|
|
36
|
+
if (length <= budget)
|
|
37
|
+
continue;
|
|
38
|
+
issues.push({
|
|
39
|
+
name: s.name,
|
|
40
|
+
length,
|
|
41
|
+
budget,
|
|
42
|
+
message: `skill "${s.name}" has a ${String(length)}-char description (budget ${String(budget)}) — the selector weighs the opening most, so a long description buries the trigger signal and hurts recall + precision. Tighten it to a concise what + when.`,
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
return issues.sort((a, b) => b.length - a.length);
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=skill-description-budget.js.map
|
package/dist/core/types.d.ts
CHANGED
|
@@ -202,6 +202,15 @@ export interface RulesConfig {
|
|
|
202
202
|
* as `scan` (descriptionOverlaps).
|
|
203
203
|
*/
|
|
204
204
|
"description-overlap"?: RuleSeverity;
|
|
205
|
+
/**
|
|
206
|
+
* Flag a model-invocable skill whose `description` is so long the trigger
|
|
207
|
+
* signal is buried — the selector weighs the opening most, so a bloated
|
|
208
|
+
* description hurts recall + precision. A DETERMINISTIC heuristic proxy for a
|
|
209
|
+
* `--trigger`-class behavioral bug; calibrated FP-safe (generous default
|
|
210
|
+
* budget, 500 chars). Default "warn" — a proxy, never gates. Same detector as
|
|
211
|
+
* `scan` (descriptionBudgetIssues).
|
|
212
|
+
*/
|
|
213
|
+
"skill-description-budget"?: RuleSeverity;
|
|
205
214
|
/**
|
|
206
215
|
* Flag a skill/agent whose `---` frontmatter block EXISTS but isn't valid YAML
|
|
207
216
|
* — fields may not parse as intended. CAVEAT: a real YAML parser (js-yaml) is
|
package/dist/core/validate.js
CHANGED
|
@@ -67,6 +67,9 @@ exports.DEFAULT_RULES = {
|
|
|
67
67
|
"disallowed-tools-contract": "warn",
|
|
68
68
|
// Deterministic NCD precision proxy (near-identical skill descriptions) — warn.
|
|
69
69
|
"description-overlap": "warn",
|
|
70
|
+
// A model-invocable skill's description so long the trigger signal is buried —
|
|
71
|
+
// WARN only (heuristic proxy, generous 500-char budget); never gates.
|
|
72
|
+
"skill-description-budget": "warn",
|
|
70
73
|
// Malformed-YAML frontmatter — WARN only (js-yaml is stricter than some loaders).
|
|
71
74
|
"frontmatter-valid": "warn",
|
|
72
75
|
// A mcp_tool hook incomplete / targeting an undeclared server — on by default at warn.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/** The hidden runtime umbrella — not a human-facing verb to document. */
|
|
2
|
+
export declare const COVERAGE_EXEMPT: readonly ["hook-runtime"];
|
|
3
|
+
/**
|
|
4
|
+
* Whether `verb` appears in a COMMAND context anywhere in `content`:
|
|
5
|
+
* `vigiles <verb>` or a backtick-prefixed `` `<verb> ``. Generous on purpose
|
|
6
|
+
* (see the file header) — over-counting a verb as documented is the SAFE
|
|
7
|
+
* direction; under-counting would cry wolf.
|
|
8
|
+
*/
|
|
9
|
+
export declare function verbMentioned(verb: string, content: string): boolean;
|
|
10
|
+
/**
|
|
11
|
+
* Find public verbs not MENTIONED in any of the given doc files. Pure — the
|
|
12
|
+
* caller supplies file contents (so it runs over the repo's `docs/` in a test,
|
|
13
|
+
* or any file set). `verbs` defaults to the canonical {@link VERBS}; `exempt`
|
|
14
|
+
* drops the hidden umbrella.
|
|
15
|
+
*/
|
|
16
|
+
export declare function findUndocumentedVerbs(docs: readonly {
|
|
17
|
+
readonly path: string;
|
|
18
|
+
readonly content: string;
|
|
19
|
+
}[], verbs?: readonly string[], exempt?: readonly string[]): string[];
|
|
20
|
+
//# sourceMappingURL=doc-command-coverage.d.ts.map
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.COVERAGE_EXEMPT = void 0;
|
|
4
|
+
exports.verbMentioned = verbMentioned;
|
|
5
|
+
exports.findUndocumentedVerbs = findUndocumentedVerbs;
|
|
6
|
+
/**
|
|
7
|
+
* Doc-command coverage — the INVERSE of self-command-refs, and the deterministic
|
|
8
|
+
* FLOOR under the `document-the-why` rule. self-command-refs checks that every
|
|
9
|
+
* `vigiles <cmd>` reference in the docs resolves to a REAL command (docs → code).
|
|
10
|
+
* This checks the other direction (code → docs): every public CLI VERB must be
|
|
11
|
+
* MENTIONED somewhere under `docs/`, so a verb shipped without a doc home is a
|
|
12
|
+
* failing test, not a thing a reader discovers is missing.
|
|
13
|
+
*
|
|
14
|
+
* HIGH-PRECISION, and biased toward NOT crying wolf: the risk here is a FALSE
|
|
15
|
+
* "undocumented" alarm on a verb that IS documented, so "mentioned" is matched
|
|
16
|
+
* GENEROUSLY — a verb counts as documented if it appears in a COMMAND context:
|
|
17
|
+
* `vigiles <verb>` (covers `npx vigiles <verb>`) OR a backtick immediately
|
|
18
|
+
* followed by the verb (`` `<verb>` ``, `` `<verb> ./x` ``). A bare English word
|
|
19
|
+
* ("test", "audit", "eval", "compile" all double as prose) is NOT enough — it
|
|
20
|
+
* must sit in a command context — so the check still fires on a genuinely
|
|
21
|
+
* undocumented verb while never flagging a documented one.
|
|
22
|
+
*
|
|
23
|
+
* `hook-runtime` is excluded by default: it is the HIDDEN runtime-entrypoint
|
|
24
|
+
* umbrella (cohesive-cli-surface keeps it OUT of the human verb surface), not a
|
|
25
|
+
* verb a user is expected to read about beside `audit`/`lint`. Source of truth
|
|
26
|
+
* for the verb set: {@link VERBS}.
|
|
27
|
+
*/
|
|
28
|
+
const cli_commands_js_1 = require("./cli-commands.js");
|
|
29
|
+
/** The hidden runtime umbrella — not a human-facing verb to document. */
|
|
30
|
+
exports.COVERAGE_EXEMPT = ["hook-runtime"];
|
|
31
|
+
function escapeRegExp(s) {
|
|
32
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Whether `verb` appears in a COMMAND context anywhere in `content`:
|
|
36
|
+
* `vigiles <verb>` or a backtick-prefixed `` `<verb> ``. Generous on purpose
|
|
37
|
+
* (see the file header) — over-counting a verb as documented is the SAFE
|
|
38
|
+
* direction; under-counting would cry wolf.
|
|
39
|
+
*/
|
|
40
|
+
function verbMentioned(verb, content) {
|
|
41
|
+
const esc = escapeRegExp(verb);
|
|
42
|
+
return new RegExp(String.raw `(\bvigiles\s+|\x60)${esc}\b`).test(content);
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Find public verbs not MENTIONED in any of the given doc files. Pure — the
|
|
46
|
+
* caller supplies file contents (so it runs over the repo's `docs/` in a test,
|
|
47
|
+
* or any file set). `verbs` defaults to the canonical {@link VERBS}; `exempt`
|
|
48
|
+
* drops the hidden umbrella.
|
|
49
|
+
*/
|
|
50
|
+
function findUndocumentedVerbs(docs, verbs = cli_commands_js_1.VERBS, exempt = exports.COVERAGE_EXEMPT) {
|
|
51
|
+
const mentioned = new Set();
|
|
52
|
+
for (const { content } of docs) {
|
|
53
|
+
for (const verb of verbs) {
|
|
54
|
+
if (!mentioned.has(verb) && verbMentioned(verb, content))
|
|
55
|
+
mentioned.add(verb);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return verbs.filter((v) => !exempt.includes(v) && !mentioned.has(v));
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=doc-command-coverage.js.map
|