vigiles 11.0.0 → 12.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +9 -0
- package/README.md +10 -5
- package/action.yml +13 -2
- package/dist/adapter-conformance.js +6 -0
- package/dist/adapter-registry.d.ts +20 -0
- package/dist/adapter-registry.js +27 -0
- package/dist/adapters/claude-code/hook-protocol.js +4 -0
- package/dist/adapters/claude-code/runtime.js +12 -0
- package/dist/adapters/codex/eval.js +3 -0
- package/dist/adapters/codex/hook-protocol.d.ts +9 -1
- package/dist/adapters/codex/hook-protocol.js +10 -0
- package/dist/adapters/codex/runtime.js +10 -0
- package/dist/adapters/opencode/runtime.js +4 -0
- package/dist/claude-code.d.ts +2 -0
- package/dist/claude-code.js +9 -1
- package/dist/cli-commands.d.ts +1 -1
- package/dist/cli-commands.js +1 -0
- package/dist/cli.js +245 -29
- package/dist/core/hook-protocol.d.ts +15 -0
- package/dist/core/rule-meta.js +8 -0
- package/dist/core/runtime.d.ts +20 -0
- package/dist/core/skill-description-budget.d.ts +42 -0
- package/dist/core/skill-description-budget.js +47 -0
- package/dist/core/types.d.ts +21 -0
- package/dist/core/validate.js +3 -0
- package/dist/doc-command-coverage.d.ts +20 -0
- package/dist/doc-command-coverage.js +60 -0
- package/dist/eval-cache.d.ts +6 -0
- package/dist/eval-cache.js +2 -0
- package/dist/eval-cost.d.ts +75 -0
- package/dist/eval-cost.js +134 -0
- package/dist/eval-lock.d.ts +192 -0
- package/dist/eval-lock.js +286 -0
- package/dist/eval.d.ts +37 -20
- package/dist/eval.js +227 -56
- package/dist/research-index.d.ts +31 -0
- package/dist/research-index.js +48 -0
- package/dist/scan-behavioral.d.ts +42 -0
- package/dist/scan-behavioral.js +67 -0
- package/dist/scan.d.ts +3 -23
- package/dist/scan.js +18 -69
- package/dist/setup-plan.d.ts +38 -1
- package/dist/setup-plan.js +67 -4
- package/hooks/eval-lock-nudge.sh +21 -0
- package/package.json +1 -1
- package/skills/adopt-spec/SKILL.md +10 -1
- package/skills/edit-spec/SKILL.md +1 -0
- package/skills/strengthen/SKILL.md +4 -0
- package/skills/test-harness/SKILL.md +44 -0
package/dist/scan.d.ts
CHANGED
|
@@ -17,6 +17,7 @@ import { type ToolIssue } from "./core/tool-contract.js";
|
|
|
17
17
|
import { type HookEventIssue } from "./core/hook-events.js";
|
|
18
18
|
import { type McpIssue } from "./core/mcp-config.js";
|
|
19
19
|
import { type DescriptionOverlap } from "./core/description-overlap.js";
|
|
20
|
+
import { type DescriptionBudgetIssue } from "./core/skill-description-budget.js";
|
|
20
21
|
import { type McpToolIssue } from "./core/mcp-tool.js";
|
|
21
22
|
import { type McpContractToolError } from "./core/mcp.js";
|
|
22
23
|
import { type McpHookIssue } from "./core/mcp-hook.js";
|
|
@@ -28,8 +29,6 @@ import { type DelegationTrifectaFinding } from "./core/delegation-trifecta.js";
|
|
|
28
29
|
import { type HookBlockFinding } from "./core/hook-block-ineffective.js";
|
|
29
30
|
import { type HookMatcherFinding } from "./core/hook-matcher.js";
|
|
30
31
|
import { type PurityLevel, type EffectSurface } from "./core/effects.js";
|
|
31
|
-
/** A named writing system. The label `unexpectedScript` reports + the config's expectation parse into this. */
|
|
32
|
-
export type Script = "Latin" | "Cyrillic" | "Han" | "Japanese" | "Korean" | "Arabic" | "Hebrew" | "Greek" | "Devanagari" | "Thai";
|
|
33
32
|
export interface ScanSkill {
|
|
34
33
|
readonly name: string;
|
|
35
34
|
readonly path: string;
|
|
@@ -41,15 +40,6 @@ export interface ScanSkill {
|
|
|
41
40
|
*/
|
|
42
41
|
readonly description?: string;
|
|
43
42
|
readonly userInvoked: boolean;
|
|
44
|
-
/**
|
|
45
|
-
* The description's dominant script when it DIFFERS from the expected one
|
|
46
|
-
* (default `"Latin"`), else null. The model's skill-selection context is
|
|
47
|
-
* English-centric, so a description in another script carries a cross-language
|
|
48
|
-
* trigger risk — it may under-fire on English prompts. A RISK flag, not a
|
|
49
|
-
* defect (a language-matched audience is fine); measure the real gap with the
|
|
50
|
-
* `audit` trigger tier / `measureTriggerRate`.
|
|
51
|
-
*/
|
|
52
|
-
readonly descriptionScript: Script | null;
|
|
53
43
|
/**
|
|
54
44
|
* SKILL.md body references to a bundled file (`scripts/`/`references/`/`assets/`
|
|
55
45
|
* or a relative markdown link with an extension) that don't resolve on disk
|
|
@@ -221,6 +211,8 @@ export interface ScanReport {
|
|
|
221
211
|
readonly mcpHookIssues: readonly McpHookIssue[];
|
|
222
212
|
/** Pairs of model-invocable skills whose descriptions are near-identical (precision collision). */
|
|
223
213
|
readonly descriptionOverlaps: readonly DescriptionOverlap[];
|
|
214
|
+
/** Model-invocable skills whose description is so long the trigger signal is buried. */
|
|
215
|
+
readonly descriptionBudgetIssues: readonly DescriptionBudgetIssue[];
|
|
224
216
|
/**
|
|
225
217
|
* Lethal-trifecta findings across subagents + model-invocable skills — a unit
|
|
226
218
|
* holding all three legs (read-private + ingest-untrusted + exfiltrate). Each
|
|
@@ -299,18 +291,6 @@ export interface SurfaceClassifier {
|
|
|
299
291
|
readonly isAgent: (f: string) => boolean;
|
|
300
292
|
readonly isCommand: (f: string) => boolean;
|
|
301
293
|
}
|
|
302
|
-
/**
|
|
303
|
-
* The description's dominant alphabetic script when it DIFFERS from `expected`
|
|
304
|
-
* (default `"Latin"`) — the cross-language trigger-risk signal. The model's
|
|
305
|
-
* skill-selection context is English-centric, so a description written mostly in
|
|
306
|
-
* another script may under-fire on English prompts. `expected` is a configurable
|
|
307
|
-
* default, not a value judgement: a Russian-targeted pack sets it to `"Cyrillic"`
|
|
308
|
-
* so its Cyrillic descriptions pass and an English one is flagged instead.
|
|
309
|
-
* Returns null when the dominant script IS the expected one (or there's no
|
|
310
|
-
* alphabetic content). Shared by `scan` and the future lint rule (one detector,
|
|
311
|
-
* no drift). The ≥20% guard avoids a near-empty string tripping on one letter.
|
|
312
|
-
*/
|
|
313
|
-
export declare function unexpectedScript(text: string, expected?: Script): Script | null;
|
|
314
294
|
/**
|
|
315
295
|
* A compiled `vigiles/hook` artifact runs through the `hook-runtime run-program`
|
|
316
296
|
* runtime entrypoint; any other hook command is hand-written (a shell script or
|
package/dist/scan.js
CHANGED
|
@@ -13,7 +13,6 @@
|
|
|
13
13
|
* stack on top later; this core stays pure so it runs anywhere in CI for free.
|
|
14
14
|
*/
|
|
15
15
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
-
exports.unexpectedScript = unexpectedScript;
|
|
17
16
|
exports.isManagedHookCommand = isManagedHookCommand;
|
|
18
17
|
exports.preferCompiledHooksMessage = preferCompiledHooksMessage;
|
|
19
18
|
exports.scanPlugin = scanPlugin;
|
|
@@ -35,6 +34,7 @@ const hook_normalize_js_1 = require("./core/hook-normalize.js");
|
|
|
35
34
|
const linters_js_1 = require("./core/linters.js");
|
|
36
35
|
const frontmatter_read_js_1 = require("./core/frontmatter-read.js");
|
|
37
36
|
const description_overlap_js_1 = require("./core/description-overlap.js");
|
|
37
|
+
const skill_description_budget_js_1 = require("./core/skill-description-budget.js");
|
|
38
38
|
const mcp_tool_js_1 = require("./core/mcp-tool.js");
|
|
39
39
|
const mcp_js_1 = require("./core/mcp.js");
|
|
40
40
|
const mcp_hook_js_1 = require("./core/mcp-hook.js");
|
|
@@ -111,58 +111,6 @@ function skillName(path) {
|
|
|
111
111
|
.split("/")
|
|
112
112
|
.pop() ?? path);
|
|
113
113
|
}
|
|
114
|
-
// [Unicode \p{Script=…} property value (Node native, no dependency), our Script
|
|
115
|
-
// label]. Japanese kana fold to "Japanese". Latin is the DEFAULT expectation (the
|
|
116
|
-
// selector is English-centric), but it's just a default — a language-matched pack
|
|
117
|
-
// can declare a different expectation, and then the OTHER script is the mismatch.
|
|
118
|
-
const SCRIPTS = [
|
|
119
|
-
["Latin", "Latin"],
|
|
120
|
-
["Cyrillic", "Cyrillic"],
|
|
121
|
-
["Han", "Han"],
|
|
122
|
-
["Hiragana", "Japanese"],
|
|
123
|
-
["Katakana", "Japanese"],
|
|
124
|
-
["Hangul", "Korean"],
|
|
125
|
-
["Arabic", "Arabic"],
|
|
126
|
-
["Hebrew", "Hebrew"],
|
|
127
|
-
["Greek", "Greek"],
|
|
128
|
-
["Devanagari", "Devanagari"],
|
|
129
|
-
["Thai", "Thai"],
|
|
130
|
-
];
|
|
131
|
-
/** Letter counts per named script label (Japanese kana folded together). */
|
|
132
|
-
function scriptCounts(text) {
|
|
133
|
-
const counts = new Map();
|
|
134
|
-
for (const [script, label] of SCRIPTS) {
|
|
135
|
-
const n = (text.match(new RegExp(`\\p{Script=${script}}`, "gu")) ?? [])
|
|
136
|
-
.length;
|
|
137
|
-
if (n > 0)
|
|
138
|
-
counts.set(label, (counts.get(label) ?? 0) + n);
|
|
139
|
-
}
|
|
140
|
-
return counts;
|
|
141
|
-
}
|
|
142
|
-
/**
|
|
143
|
-
* The description's dominant alphabetic script when it DIFFERS from `expected`
|
|
144
|
-
* (default `"Latin"`) — the cross-language trigger-risk signal. The model's
|
|
145
|
-
* skill-selection context is English-centric, so a description written mostly in
|
|
146
|
-
* another script may under-fire on English prompts. `expected` is a configurable
|
|
147
|
-
* default, not a value judgement: a Russian-targeted pack sets it to `"Cyrillic"`
|
|
148
|
-
* so its Cyrillic descriptions pass and an English one is flagged instead.
|
|
149
|
-
* Returns null when the dominant script IS the expected one (or there's no
|
|
150
|
-
* alphabetic content). Shared by `scan` and the future lint rule (one detector,
|
|
151
|
-
* no drift). The ≥20% guard avoids a near-empty string tripping on one letter.
|
|
152
|
-
*/
|
|
153
|
-
function unexpectedScript(text, expected = "Latin") {
|
|
154
|
-
const counts = scriptCounts(text);
|
|
155
|
-
let total = 0;
|
|
156
|
-
let dominant = null;
|
|
157
|
-
for (const [label, count] of counts) {
|
|
158
|
-
total += count;
|
|
159
|
-
if (!dominant || count > dominant.count)
|
|
160
|
-
dominant = { label, count };
|
|
161
|
-
}
|
|
162
|
-
if (!dominant || dominant.label === expected)
|
|
163
|
-
return null;
|
|
164
|
-
return dominant.count / total >= 0.2 ? dominant.label : null;
|
|
165
|
-
}
|
|
166
114
|
/**
|
|
167
115
|
* The first prose paragraph of a SKILL.md body (after the frontmatter and any
|
|
168
116
|
* leading `#` headings) — Claude Code's FALLBACK skill description when the
|
|
@@ -236,7 +184,6 @@ function scanSkills(files, cls, ctx) {
|
|
|
236
184
|
hasDescription: Boolean(effectiveDesc && effectiveDesc.length >= 20),
|
|
237
185
|
description: effectiveDesc?.trim(),
|
|
238
186
|
userInvoked,
|
|
239
|
-
descriptionScript: effectiveDesc ? unexpectedScript(effectiveDesc) : null,
|
|
240
187
|
resourceIssues,
|
|
241
188
|
trifecta,
|
|
242
189
|
// A SKILL.md opening with `name:`/`description:` but no `---` fence loads
|
|
@@ -254,7 +201,7 @@ function scanSkills(files, cls, ctx) {
|
|
|
254
201
|
* logic as `scanSkills` (frontmatter `description` ← first body paragraph), then
|
|
255
202
|
* the NCD precision-proxy. See description-overlap.ts.
|
|
256
203
|
*/
|
|
257
|
-
function
|
|
204
|
+
function modelInvocableSkillSurfaces(files, cls) {
|
|
258
205
|
const surfaces = [];
|
|
259
206
|
for (const [path, md] of Object.entries(files)) {
|
|
260
207
|
if (!cls.isSkill(path))
|
|
@@ -267,7 +214,18 @@ function descriptionOverlapsFor(files, cls) {
|
|
|
267
214
|
continue;
|
|
268
215
|
surfaces.push({ name: fm.name ?? skillName(path), description });
|
|
269
216
|
}
|
|
270
|
-
return
|
|
217
|
+
return surfaces;
|
|
218
|
+
}
|
|
219
|
+
function descriptionOverlapsFor(files, cls) {
|
|
220
|
+
return (0, description_overlap_js_1.findDescriptionOverlaps)(modelInvocableSkillSurfaces(files, cls));
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Model-invocable skills whose description is so long the trigger signal is
|
|
224
|
+
* buried (heuristic proxy; degrades recall + precision). Same surfaces as the
|
|
225
|
+
* overlap check. See skill-description-budget.ts.
|
|
226
|
+
*/
|
|
227
|
+
function descriptionBudgetFor(files, cls) {
|
|
228
|
+
return (0, skill_description_budget_js_1.findDescriptionBudgetIssues)(modelInvocableSkillSurfaces(files, cls));
|
|
271
229
|
}
|
|
272
230
|
function scanAgents(files, dialect, declaredServers, cls) {
|
|
273
231
|
const out = [];
|
|
@@ -797,6 +755,7 @@ function scanPlugin(dir, layout, dialect = dialect_js_1.claudeCodeDialect) {
|
|
|
797
755
|
mcpIssues: (0, mcp_config_js_1.verifyMcpServers)(mcpServers),
|
|
798
756
|
mcpHookIssues: (0, mcp_hook_js_1.verifyMcpHookTargets)(loaded.settings.hooks, declaredServers, dialect),
|
|
799
757
|
descriptionOverlaps: descriptionOverlapsFor(loaded.files, cls),
|
|
758
|
+
descriptionBudgetIssues: descriptionBudgetFor(loaded.files, cls),
|
|
800
759
|
trifectaFindings,
|
|
801
760
|
skillResourceIssues: skillResourceFindings,
|
|
802
761
|
skillFenceIssues: skillFenceFindings,
|
|
@@ -923,7 +882,7 @@ function section(title, lines, count = lines.length) {
|
|
|
923
882
|
return [];
|
|
924
883
|
return [`${title} (${String(count)}):`, ...lines, ""];
|
|
925
884
|
}
|
|
926
|
-
/** One skill's report line: ✓/⚠ + name + notes (no-trigger, user-invoked
|
|
885
|
+
/** One skill's report line: ✓/⚠ + name + notes (no-trigger, user-invoked). */
|
|
927
886
|
function skillLine(s) {
|
|
928
887
|
if (!s.hasDescription) {
|
|
929
888
|
return ` ⚠ ${s.name} (no usable description — no frontmatter description and no body text — can't trigger)`;
|
|
@@ -931,11 +890,7 @@ function skillLine(s) {
|
|
|
931
890
|
const notes = [];
|
|
932
891
|
if (s.userInvoked)
|
|
933
892
|
notes.push("user-invoked");
|
|
934
|
-
|
|
935
|
-
notes.push(`description in ${s.descriptionScript} — cross-language trigger risk`);
|
|
936
|
-
}
|
|
937
|
-
const mark = s.descriptionScript ? "⚠" : "✓";
|
|
938
|
-
return ` ${mark} ${s.name}${notes.length ? ` (${notes.join("; ")})` : ""}`;
|
|
893
|
+
return ` ✓ ${s.name}${notes.length ? ` (${notes.join("; ")})` : ""}`;
|
|
939
894
|
}
|
|
940
895
|
/** One agent's report block: ✗ (broken contract) / ⚠ (inherits all) / ✓ + issues + purity. */
|
|
941
896
|
function agentLines(a) {
|
|
@@ -1002,6 +957,7 @@ function formatScanReport(r) {
|
|
|
1002
957
|
out.push(...section("MCP config", r.mcpIssues.map((i) => ` ✗ ${i.message}`)));
|
|
1003
958
|
out.push(...section("MCP hook targets", r.mcpHookIssues.map((i) => ` ✗ ${i.message}`)));
|
|
1004
959
|
out.push(...section("Description overlap (precision risk)", r.descriptionOverlaps.map((o) => ` ⚠ ${o.message}`)));
|
|
960
|
+
out.push(...section("Description budget (trigger-signal risk)", r.descriptionBudgetIssues.map((o) => ` ⚠ ${o.message}`)));
|
|
1005
961
|
out.push(...section("Lethal trifecta (prompt-injection exfil risk)", r.trifectaFindings.map((t) => ` ${t.finding.severity === "hard" ? "✗" : "⚠"} ${t.kind} ${t.name} (${t.path}): ${t.finding.message}`)));
|
|
1006
962
|
out.push(...section("Skill bundled resources", r.skillResourceIssues.map((s) => ` ✗ ${s.name}: ${s.finding.ref} (line ${String(s.finding.line)}) — bundled resource not found`)));
|
|
1007
963
|
out.push(...section("Invisible skills (missing frontmatter fence)", r.skillFenceIssues.map((s) => ` ✗ ${s.name} (${s.path}): opens with \`${s.finding.key}:\` but no \`---\` fence — loads as body, never fires`)));
|
|
@@ -1028,13 +984,6 @@ function formatScanReport(r) {
|
|
|
1028
984
|
if (warnings.length > 0) {
|
|
1029
985
|
out.push("Warnings:", ...warnings.map((w) => ` - ${w}`), "");
|
|
1030
986
|
}
|
|
1031
|
-
// Cross-language trigger risk is a RISK, not a structural defect (a
|
|
1032
|
-
// language-matched audience is fine), so it's reported separately from the
|
|
1033
|
-
// verdict — it points at the behavioral column, it doesn't fail the scan.
|
|
1034
|
-
const mismatched = r.skills.filter((s) => s.descriptionScript);
|
|
1035
|
-
if (mismatched.length > 0) {
|
|
1036
|
-
out.push(`⚠ ${String(mismatched.length)} skill(s) have descriptions in an unexpected script (cross-language trigger risk) — measure with \`vigiles measure\``, "");
|
|
1037
|
-
}
|
|
1038
987
|
// Skill-metadata is a RECOMMENDATION, not a structural defect (the skill loads
|
|
1039
988
|
// via fallbacks) — reported as a soft note, never counted in the verdict.
|
|
1040
989
|
if (r.skillMetaIssues.length > 0) {
|
package/dist/setup-plan.d.ts
CHANGED
|
@@ -95,7 +95,7 @@ export declare const WORKFLOW_RULES: readonly ["require-instructions-spec", "unt
|
|
|
95
95
|
* keep their own default severities. Named for the group taxonomy
|
|
96
96
|
* (research/install-enforcement-dx.md).
|
|
97
97
|
*/
|
|
98
|
-
export declare const NUDGE_RULES: readonly ["frontmatter-valid", "skill-frontmatter", "prefer-compiled-hooks", "unmarked-refs", "lethal-trifecta", "skill-resource-resolves", "skill-missing-fence", "plugin-dir-layout", "delegation-trifecta", "hook-block-ineffective", "hook-matcher"];
|
|
98
|
+
export declare const NUDGE_RULES: readonly ["skill-description-budget", "frontmatter-valid", "skill-frontmatter", "prefer-compiled-hooks", "unmarked-refs", "lethal-trifecta", "skill-resource-resolves", "skill-missing-fence", "plugin-dir-layout", "delegation-trifecta", "hook-block-ineffective", "hook-matcher"];
|
|
99
99
|
export declare function mergeProjectConfig(existing: Record<string, unknown>, opts: {
|
|
100
100
|
harness: string | string[];
|
|
101
101
|
strict: boolean;
|
|
@@ -158,6 +158,43 @@ export interface InstallPlan {
|
|
|
158
158
|
export declare function planPluginInstall(harnesses: readonly string[], opts: {
|
|
159
159
|
hasClaude: boolean;
|
|
160
160
|
}): InstallPlan[];
|
|
161
|
+
/** One vigiles-managed Codex hook: a `[[hooks.<event>]]` entry. */
|
|
162
|
+
export interface CodexPluginHook {
|
|
163
|
+
readonly event: string;
|
|
164
|
+
/** Codex matcher (anchored regex, the dialect convention). */
|
|
165
|
+
readonly matcher: string;
|
|
166
|
+
/** The shell command Codex runs (a direct `npx vigiles …`, no plugin root). */
|
|
167
|
+
readonly command: string;
|
|
168
|
+
/** A unique command substring → idempotent re-merge (replaces in place). */
|
|
169
|
+
readonly key: string;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* vigiles's proactive nudges, wired into a Codex repo's `.codex/config.toml`.
|
|
173
|
+
*
|
|
174
|
+
* Codex has no global plugin store (unlike Claude Code's marketplace), so its
|
|
175
|
+
* config is repo-committed — the idiomatic place for these. They run as DIRECT
|
|
176
|
+
* `npx vigiles hook-runtime …` commands (NOT vendored bash scripts): the runtime
|
|
177
|
+
* entrypoints read the event JSON on stdin and emit the `hookSpecificOutput.
|
|
178
|
+
* additionalContext` shape Codex honors on `PostToolUse` (confirmed against the
|
|
179
|
+
* official hooks docs + encoded in `HookProtocol.injectableEvents`). Safety: only
|
|
180
|
+
* an INTENTIONAL `exit 2` blocks an edit (the refs nudge, when `unmarked-refs` is
|
|
181
|
+
* `error`); an npx-resolution failure exits non-2, so a missing dep never blocks.
|
|
182
|
+
*
|
|
183
|
+
* Deliberately NOT here (a loud, documented deferral — no-silent-skips): the
|
|
184
|
+
* SessionStart lint summary (CC delivers it as plain stdout, whose SessionStart
|
|
185
|
+
* prepend is unconfirmed on Codex — vs the JSON `additionalContext` these use) and
|
|
186
|
+
* the compile-on-edit / pre-edit-block guards (filename-gated bash with no
|
|
187
|
+
* harness-neutral `hook-runtime` entrypoint yet). Those stay manual on Codex.
|
|
188
|
+
*/
|
|
189
|
+
export declare function codexPluginHooks(): CodexPluginHook[];
|
|
190
|
+
/**
|
|
191
|
+
* Idempotently merge {@link codexPluginHooks} into a parsed `.codex/config.toml`
|
|
192
|
+
* object. Pure — the IO (read/parse/serialize/write) stays in cli.ts's
|
|
193
|
+
* `wireCodexHooks`. Each vigiles hook is keyed by a unique command substring, so
|
|
194
|
+
* a re-run REPLACES its own entry in place and leaves the user's own Codex hooks
|
|
195
|
+
* (and every other config key) untouched. Returns a new object.
|
|
196
|
+
*/
|
|
197
|
+
export declare function applyCodexPluginHooks(existing: Record<string, unknown>): Record<string, unknown>;
|
|
161
198
|
/**
|
|
162
199
|
* Resolve the final plan: defaults, then flags, then interactive answers (each
|
|
163
200
|
* layer overrides the previous only where it has an opinion). `--target` pins a
|
package/dist/setup-plan.js
CHANGED
|
@@ -17,6 +17,8 @@ exports.mergeProjectConfig = mergeProjectConfig;
|
|
|
17
17
|
exports.shouldPrompt = shouldPrompt;
|
|
18
18
|
exports.collectSetupAnswers = collectSetupAnswers;
|
|
19
19
|
exports.planPluginInstall = planPluginInstall;
|
|
20
|
+
exports.codexPluginHooks = codexPluginHooks;
|
|
21
|
+
exports.applyCodexPluginHooks = applyCodexPluginHooks;
|
|
20
22
|
exports.resolvePlan = resolvePlan;
|
|
21
23
|
function flagValue(args, prefix) {
|
|
22
24
|
return args.find((a) => a.startsWith(prefix))?.slice(prefix.length);
|
|
@@ -119,6 +121,7 @@ exports.WORKFLOW_RULES = [
|
|
|
119
121
|
* (research/install-enforcement-dx.md).
|
|
120
122
|
*/
|
|
121
123
|
exports.NUDGE_RULES = [
|
|
124
|
+
"skill-description-budget",
|
|
122
125
|
"frontmatter-valid",
|
|
123
126
|
"skill-frontmatter",
|
|
124
127
|
"prefer-compiled-hooks",
|
|
@@ -260,8 +263,10 @@ function planPluginInstall(harnesses, opts) {
|
|
|
260
263
|
if (harness === "codex") {
|
|
261
264
|
// The cross-agent `skills` CLI with `-g -y` installs to the global store
|
|
262
265
|
// ~/.agents/skills/ (NOT the repo, and NOT ~/.codex/ — verified against
|
|
263
|
-
// the real CLI). Skills
|
|
264
|
-
// are
|
|
266
|
+
// the real CLI). Skills install globally; the proactive NUDGE hooks
|
|
267
|
+
// (eval-lock + refs) are wired into the repo's .codex/config.toml by
|
|
268
|
+
// `init` (see codexPluginHooks / wireCodexHooks) — Codex config is
|
|
269
|
+
// repo-committed, so that's the idiomatic place.
|
|
265
270
|
return {
|
|
266
271
|
harness,
|
|
267
272
|
commands: ["npx --yes skills add zernie/vigiles -a codex -g -y"],
|
|
@@ -269,9 +274,10 @@ function planPluginInstall(harnesses, opts) {
|
|
|
269
274
|
manualSteps: ["npx skills add zernie/vigiles -a codex -g -y"],
|
|
270
275
|
notes: [
|
|
271
276
|
"Codex reads AGENTS.md directly; the skills install globally to ~/.agents/skills/ (not the repo).",
|
|
272
|
-
"
|
|
277
|
+
"The eval-lock + refs NUDGE hooks are wired into .codex/config.toml (repo-committed, the Codex norm).",
|
|
278
|
+
"Still manual on Codex: the SessionStart lint summary + compile-on-edit/pre-edit guards (no harness-neutral entrypoint yet).",
|
|
273
279
|
],
|
|
274
|
-
vendors:
|
|
280
|
+
vendors: true,
|
|
275
281
|
};
|
|
276
282
|
}
|
|
277
283
|
return {
|
|
@@ -284,6 +290,63 @@ function planPluginInstall(harnesses, opts) {
|
|
|
284
290
|
};
|
|
285
291
|
});
|
|
286
292
|
}
|
|
293
|
+
/**
|
|
294
|
+
* vigiles's proactive nudges, wired into a Codex repo's `.codex/config.toml`.
|
|
295
|
+
*
|
|
296
|
+
* Codex has no global plugin store (unlike Claude Code's marketplace), so its
|
|
297
|
+
* config is repo-committed — the idiomatic place for these. They run as DIRECT
|
|
298
|
+
* `npx vigiles hook-runtime …` commands (NOT vendored bash scripts): the runtime
|
|
299
|
+
* entrypoints read the event JSON on stdin and emit the `hookSpecificOutput.
|
|
300
|
+
* additionalContext` shape Codex honors on `PostToolUse` (confirmed against the
|
|
301
|
+
* official hooks docs + encoded in `HookProtocol.injectableEvents`). Safety: only
|
|
302
|
+
* an INTENTIONAL `exit 2` blocks an edit (the refs nudge, when `unmarked-refs` is
|
|
303
|
+
* `error`); an npx-resolution failure exits non-2, so a missing dep never blocks.
|
|
304
|
+
*
|
|
305
|
+
* Deliberately NOT here (a loud, documented deferral — no-silent-skips): the
|
|
306
|
+
* SessionStart lint summary (CC delivers it as plain stdout, whose SessionStart
|
|
307
|
+
* prepend is unconfirmed on Codex — vs the JSON `additionalContext` these use) and
|
|
308
|
+
* the compile-on-edit / pre-edit-block guards (filename-gated bash with no
|
|
309
|
+
* harness-neutral `hook-runtime` entrypoint yet). Those stay manual on Codex.
|
|
310
|
+
*/
|
|
311
|
+
function codexPluginHooks() {
|
|
312
|
+
// Codex's file-edit tool is `apply_patch` (its dialect vocabulary —
|
|
313
|
+
// src/adapters/codex/dialect.ts), NOT Claude's `Edit`/`Write`. A PostToolUse
|
|
314
|
+
// matcher keyed on CC tool names would never fire on Codex, so the nudges must
|
|
315
|
+
// match the Codex tool name. (Both nudge entrypoints also self-gate on the
|
|
316
|
+
// edited file, so a non-edit event no-ops regardless.)
|
|
317
|
+
return [
|
|
318
|
+
{
|
|
319
|
+
event: "PostToolUse",
|
|
320
|
+
matcher: "^apply_patch$",
|
|
321
|
+
command: "npx --no-install vigiles hook-runtime eval-lock-nudge",
|
|
322
|
+
key: "hook-runtime eval-lock-nudge",
|
|
323
|
+
},
|
|
324
|
+
{
|
|
325
|
+
event: "PostToolUse",
|
|
326
|
+
matcher: "^apply_patch$",
|
|
327
|
+
command: "npx --no-install vigiles hook-runtime refs",
|
|
328
|
+
key: "hook-runtime refs",
|
|
329
|
+
},
|
|
330
|
+
];
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* Idempotently merge {@link codexPluginHooks} into a parsed `.codex/config.toml`
|
|
334
|
+
* object. Pure — the IO (read/parse/serialize/write) stays in cli.ts's
|
|
335
|
+
* `wireCodexHooks`. Each vigiles hook is keyed by a unique command substring, so
|
|
336
|
+
* a re-run REPLACES its own entry in place and leaves the user's own Codex hooks
|
|
337
|
+
* (and every other config key) untouched. Returns a new object.
|
|
338
|
+
*/
|
|
339
|
+
function applyCodexPluginHooks(existing) {
|
|
340
|
+
const config = existing;
|
|
341
|
+
const hooks = {
|
|
342
|
+
...(config.hooks ?? {}),
|
|
343
|
+
};
|
|
344
|
+
for (const h of codexPluginHooks()) {
|
|
345
|
+
const kept = (hooks[h.event] ?? []).filter((e) => !e.command.includes(h.key));
|
|
346
|
+
hooks[h.event] = [...kept, { matcher: h.matcher, command: h.command }];
|
|
347
|
+
}
|
|
348
|
+
return { ...existing, hooks };
|
|
349
|
+
}
|
|
287
350
|
/**
|
|
288
351
|
* Resolve the final plan: defaults, then flags, then interactive answers (each
|
|
289
352
|
* layer overrides the previous only where it has an opinion). `--target` pins a
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# PostToolUse hook — after the agent edits an eval input (a SKILL.md trigger
|
|
3
|
+
# surface or an *.eval.* script) and committed eval locks exist, inject a
|
|
4
|
+
# NON-BLOCKING reminder to re-run `vigiles eval --update`. Self-gating: silent
|
|
5
|
+
# until you've committed a lock, so it never fires in a repo that doesn't use
|
|
6
|
+
# evals. Never blocks the edit. Runs as its OWN PostToolUse entry so its stdout
|
|
7
|
+
# stays clean JSON. See docs/harness-testing.md (the eval lock).
|
|
8
|
+
|
|
9
|
+
set -uo pipefail
|
|
10
|
+
|
|
11
|
+
INPUT=$(cat)
|
|
12
|
+
|
|
13
|
+
# No npx / not a Node project → nothing to do, never disrupt the edit.
|
|
14
|
+
command -v npx >/dev/null 2>&1 || exit 0
|
|
15
|
+
[ -f package.json ] || exit 0
|
|
16
|
+
|
|
17
|
+
printf '%s' "$INPUT" | npx vigiles hook-runtime eval-lock-nudge
|
|
18
|
+
|
|
19
|
+
# Always exit 0 — this hook only nudges, it never blocks (the gate is CI's
|
|
20
|
+
# `vigiles eval --check`). Swallow any tool/setup error so it can't disrupt edits.
|
|
21
|
+
exit 0
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "vigiles",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "12.1.0",
|
|
4
4
|
"description": "Lint & test the harness your AI agent runs on — verify the references in your CLAUDE.md / AGENTS.md and test that your hooks and skills actually work.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"claude-code",
|
|
@@ -7,7 +7,16 @@ argument-hint: <path to CLAUDE.md, defaults to CLAUDE.md>
|
|
|
7
7
|
|
|
8
8
|
Start a typed `CLAUDE.md.spec.ts` from an existing hand-written CLAUDE.md (or AGENTS.md). This is the non-destructive adoption path — you keep your existing instruction file as the starting point and get type safety going forward.
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
## Adoption rules
|
|
11
|
+
|
|
12
|
+
Adoption is the **safe, faithful on-ramp — never an upgrade in disguise.** These are non-negotiable:
|
|
13
|
+
|
|
14
|
+
- **Faithful.** Preserve every rule, command, key file, and prose section as-is. Invent nothing — the spec must compile back to ~the user's existing file.
|
|
15
|
+
- **Non-destructive.** Never edit the original `CLAUDE.md` / `AGENTS.md`. Only write the new `.spec.ts`. Never auto-`compile` over the file — switching it to spec-managed is a separate, explicit step the user runs with a diff to review.
|
|
16
|
+
- **Don't escalate enforcement.** Keep `guidance()` as `guidance()`. Upgrading to `enforce()` has a cost (config/plugins, possible false positives) and is a separate opt-in step — the `strengthen` skill. Adoption is **not** turning on strict / `workflow` gating.
|
|
17
|
+
- **Reversible.** `vigiles eject <file>` hands the file back as plain hand-owned markdown anytime — it's never a one-way door. Tell the user this.
|
|
18
|
+
- **Ask before writing.** Present the generated spec and a conversion summary first; write only on the user's yes.
|
|
19
|
+
- **A lighter touch exists.** For no spec at all, inline `<!-- vigiles:enforce ... -->` comments are verified by `vigiles lint` with the same engine.
|
|
11
20
|
|
|
12
21
|
## Instructions
|
|
13
22
|
|
|
@@ -136,6 +136,7 @@ If the vigiles plugin is installed (`/plugin marketplace add zernie/vigiles` the
|
|
|
136
136
|
|
|
137
137
|
## Important
|
|
138
138
|
|
|
139
|
+
- **Do what's asked; don't silently escalate enforcement.** Add the rule the user asked for. If making it an `enforce()` would need a linter-config edit or a plugin install (a cost), or if it could fail a clean CI, **say so and let the user choose** — don't change linter config or flip on strict / `workflow` gating on your own. A pure win (a rule that's already enabled) you can just apply. The `strengthen` skill owns `guidance()` → `enforce()` upgrades.
|
|
139
140
|
- **Never edit CLAUDE.md or AGENTS.md directly** — they have a vigiles hash comment and are build artifacts
|
|
140
141
|
- **The spec is TypeScript** — you get type checking, autocomplete, and verified references
|
|
141
142
|
- **`enforce()` rules are verified** — the compiler checks the rule exists AND is enabled in your linter config
|
|
@@ -5,6 +5,10 @@ description: Upgrade a vigiles spec's guidance() rules to enforce() — scan the
|
|
|
5
5
|
|
|
6
6
|
Scan spec files for `guidance()` rules and suggest `enforce()` replacements backed by real linter rules.
|
|
7
7
|
|
|
8
|
+
## Principle: auto the free wins, nudge for the costs
|
|
9
|
+
|
|
10
|
+
The dividing line is **cost, not strictness**. A `guidance()` → `enforce()` swap where the linter rule **already exists and is enabled** is a pure win — free, reversible, no false-positive risk — so apply it (Tier 1 below). Anything that **costs** something — editing linter config, installing a plugin, or a change that could fail a clean CI — is the user's call: **present it with the tradeoff spelled out and let them choose** (Tiers 2–4). Never silently edit config, install a dependency, or escalate the repo into strict gating. This is the `init` enforcement model (structural = on by default, workflow/strict = opt-in) applied at edit time.
|
|
11
|
+
|
|
8
12
|
## Instructions
|
|
9
13
|
|
|
10
14
|
### Step 0: Choose Mode
|
|
@@ -190,6 +190,50 @@ job that asserts the capability is present, run **`vigiles test --no-skip`** so
|
|
|
190
190
|
skipped tier fails — a green-with-skips is untested surface. Keep unit +
|
|
191
191
|
deterministic tests in CI (free); run evals locally or on a schedule with auth.
|
|
192
192
|
|
|
193
|
+
### After a real-model run: TELL THE USER WHAT IT SPENT
|
|
194
|
+
|
|
195
|
+
Whenever you run a real-model eval (`runEval` / `measureArms` / `measureTriggerRate`
|
|
196
|
+
/ `measure`), **surface the spend to the user in your reply** — don't let a paid run
|
|
197
|
+
be silent. `runEval` prints a cost block to stderr and every report carries `usage`
|
|
198
|
+
(`report.arms[*].usage`: `totalCostUsd` + token counts). Relay, in plain words:
|
|
199
|
+
|
|
200
|
+
- **tokens spent** and the **API-equivalent `$`** (`total_cost_usd` — what it _would_
|
|
201
|
+
cost at metered API rates);
|
|
202
|
+
- **how it was billed** — "on your Claude subscription (**$0 metered**)" if you're
|
|
203
|
+
logged in, or a **⚠ warning** if `ANTHROPIC_API_KEY` is set (that run was billed
|
|
204
|
+
**per token** — tell them to unset it and `claude login` to run free).
|
|
205
|
+
|
|
206
|
+
We do **not** show "% of your subscription" — Anthropic doesn't expose a plan's
|
|
207
|
+
quota, so any percentage would be invented. Tokens + API-equivalent `$` + the
|
|
208
|
+
billed-to line is the honest, complete picture. Keep the user's cost visible, always.
|
|
209
|
+
|
|
210
|
+
## Step 5 — Lock the eval so CI stays honest (you do this automatically)
|
|
211
|
+
|
|
212
|
+
Real-model evals run on the user's subscription — locally, never in CI. So **as
|
|
213
|
+
part of writing an eval, you keep its result fresh for them.** Do these two things
|
|
214
|
+
without being asked:
|
|
215
|
+
|
|
216
|
+
1. **Give every eval a `name`.** That's what the lock keys on.
|
|
217
|
+
2. **After running it, commit the lock:**
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
vigiles eval --update # records the result → .vigiles/eval-locks/<name>.lock.json
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
Then commit that file. CI runs `vigiles eval --check` (no model) to verify it
|
|
224
|
+
still matches the inputs — so a later edit that forgets to re-eval fails loud
|
|
225
|
+
instead of shipping stale numbers.
|
|
226
|
+
|
|
227
|
+
**When you later change a skill's description or prompts, re-run `vigiles eval
|
|
228
|
+
--update` and commit the updated lock** — the change altered what the eval
|
|
229
|
+
measures. (vigiles also nudges you: when a lock exists, a `SKILL.md` edit triggers
|
|
230
|
+
a non-blocking reminder.)
|
|
231
|
+
|
|
232
|
+
Why it's cheap: `--check` only hashes inputs (skill text, prompts, model). A
|
|
233
|
+
**threshold** change in the test re-uses the saved numbers (no model); only an
|
|
234
|
+
**input** change needs a fresh `--update`. Full mechanics:
|
|
235
|
+
[`docs/harness-testing.md`](../../docs/harness-testing.md#keep-eval-results-fresh-in-ci-the-lock).
|
|
236
|
+
|
|
193
237
|
## When the user didn't say what to test
|
|
194
238
|
|
|
195
239
|
Don't ask them to specify — **pick something real and demonstrate.** Scan the
|