vigiles 10.0.0 → 12.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +9 -0
- package/README.md +121 -86
- package/action.yml +13 -2
- package/dist/adapter-conformance.js +6 -0
- package/dist/adapter-registry.d.ts +20 -0
- package/dist/adapter-registry.js +27 -0
- package/dist/adapters/claude-code/dialect.js +15 -0
- package/dist/adapters/claude-code/hook-protocol.js +4 -0
- package/dist/adapters/claude-code/runtime.js +12 -0
- package/dist/adapters/codex/eval.js +3 -0
- package/dist/adapters/codex/hook-protocol.d.ts +9 -1
- package/dist/adapters/codex/hook-protocol.js +10 -0
- package/dist/adapters/codex/runtime.js +10 -0
- package/dist/adapters/opencode/runtime.js +4 -0
- package/dist/audit-report.d.ts +1 -1
- package/dist/audit-report.template.html +1 -1
- package/dist/audit-score.d.ts +19 -12
- package/dist/audit-score.js +65 -11
- package/dist/cli-commands.d.ts +1 -1
- package/dist/cli-commands.js +1 -0
- package/dist/cli.js +460 -29
- package/dist/core/CLAUDE.md.spec.d.ts +3 -0
- package/dist/core/CLAUDE.md.spec.js +26 -0
- package/dist/core/delegation-trifecta.d.ts +64 -0
- package/dist/core/delegation-trifecta.js +124 -0
- package/dist/core/dialect.d.ts +18 -0
- package/dist/core/hook-block-ineffective.d.ts +62 -0
- package/dist/core/hook-block-ineffective.js +153 -0
- package/dist/core/hook-matcher.d.ts +66 -0
- package/dist/core/hook-matcher.js +182 -0
- package/dist/core/hook-normalize.d.ts +43 -0
- package/dist/core/hook-normalize.js +78 -0
- package/dist/core/hook-protocol.d.ts +15 -0
- package/dist/core/lethal-trifecta.d.ts +100 -0
- package/dist/core/lethal-trifecta.js +197 -0
- package/dist/core/plugin-dir-layout.d.ts +30 -0
- package/dist/core/plugin-dir-layout.js +73 -0
- package/dist/core/rule-meta.d.ts +82 -0
- package/dist/core/rule-meta.js +266 -0
- package/dist/core/runtime.d.ts +20 -0
- package/dist/core/skill-missing-fence.d.ts +47 -0
- package/dist/core/skill-missing-fence.js +119 -0
- package/dist/core/skill-resources.d.ts +27 -0
- package/dist/core/skill-resources.js +167 -0
- package/dist/core/types.d.ts +83 -0
- package/dist/core/validate.d.ts +1 -0
- package/dist/core/validate.js +26 -4
- package/dist/eval-cache.d.ts +6 -0
- package/dist/eval-cache.js +2 -0
- package/dist/eval-lock.d.ts +192 -0
- package/dist/eval-lock.js +286 -0
- package/dist/eval.d.ts +33 -20
- package/dist/eval.js +199 -51
- package/dist/leaderboard.d.ts +1 -0
- package/dist/leaderboard.js +42 -4
- package/dist/scan.d.ts +106 -0
- package/dist/scan.js +251 -45
- package/dist/setup-plan.d.ts +43 -3
- package/dist/setup-plan.js +78 -6
- package/hooks/eval-lock-nudge.sh +21 -0
- package/package.json +1 -1
- package/skills/test-harness/SKILL.md +27 -0
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.skillResourceIssues = skillResourceIssues;
|
|
4
|
+
/**
|
|
5
|
+
* vigiles — SKILL bundled-resource resolution (the cross-reference moat applied
|
|
6
|
+
* to a SKILL.md body).
|
|
7
|
+
*
|
|
8
|
+
* A SKILL.md body is freeform markdown that routinely points the agent at LOCAL
|
|
9
|
+
* BUNDLED files shipped beside it — `scripts/foo.sh`, `references/api.md`,
|
|
10
|
+
* `[setup](./scripts/run.py)`, an inline `run \`scripts/setup.sh\``. When a
|
|
11
|
+
* referenced file doesn't exist on disk under the skill directory, the agent
|
|
12
|
+
* reads the instruction, gets nothing, and silently continues (a documented top
|
|
13
|
+
* skill pain — one practitioner found 59 broken refs across 192 files). vigiles
|
|
14
|
+
* already verifies file/script refs inside typed specs (core/refs.ts,
|
|
15
|
+
* core/doc-refs.ts) and intra-plugin script refs in the loader
|
|
16
|
+
* (plugin-loader.ts `danglingRefs`); this extends that to the SKILL.md body.
|
|
17
|
+
*
|
|
18
|
+
* HIGH-PRECISION / FP-SAFE, by the same don't-cry-wolf discipline the rest of
|
|
19
|
+
* vigiles holds (see `danglingRefs`/`isPluginRooted`): we flag ONLY references
|
|
20
|
+
* that are UNAMBIGUOUSLY a local bundled resource — a markdown link to a
|
|
21
|
+
* relative path with a file extension, or an explicit `scripts/`/`references/`/
|
|
22
|
+
* `assets/`-prefixed path (the Agent-Skills standard bundle dirs) with an
|
|
23
|
+
* extension. Everything else is skipped: URLs, absolute paths, `${VAR}`/`$VAR`
|
|
24
|
+
* tokens, `../` escapes, bare words with no extension or known prefix. Prefer
|
|
25
|
+
* MISSING a real ref over emitting a false positive — a noisy resource check
|
|
26
|
+
* would teach users to ignore it.
|
|
27
|
+
*
|
|
28
|
+
* Pure: the only IO is an injectable `existsSync` (default node:fs), mirroring
|
|
29
|
+
* core/refs.ts and the loader so the detector is testable with a fake.
|
|
30
|
+
*/
|
|
31
|
+
const node_fs_1 = require("node:fs");
|
|
32
|
+
const node_path_1 = require("node:path");
|
|
33
|
+
// ---------------------------------------------------------------------------
|
|
34
|
+
// Shapes we match vs deliberately skip (FP-safety)
|
|
35
|
+
// ---------------------------------------------------------------------------
|
|
36
|
+
const FENCE = /^\s*(?:`{3,}|~{3,})/;
|
|
37
|
+
// The Agent-Skills standard bundle subdirectories. A path PREFIXED by one of
|
|
38
|
+
// these is unambiguously a local bundled resource, even without a `./`.
|
|
39
|
+
const BUNDLE_DIRS = ["scripts", "references", "assets"];
|
|
40
|
+
const BUNDLE_PREFIX = new RegExp(`^(?:${BUNDLE_DIRS.join("|")})/`);
|
|
41
|
+
// A markdown inline link `[text](target)` — we read its target.
|
|
42
|
+
const MD_LINK = /\[[^\]]*\]\(([^)\s]+)\)/g;
|
|
43
|
+
// An inline-code path mention: a backtick span whose whole content is a single
|
|
44
|
+
// path token. We only treat it as a ref when it is a bundle-dir-prefixed path
|
|
45
|
+
// with an extension (the high-confidence shape); a bare `scripts` or a generic
|
|
46
|
+
// `foo.ts` mention is NOT flagged.
|
|
47
|
+
const INLINE_SPAN = /`([^`\n]+)`/g;
|
|
48
|
+
// A path must carry a file extension to be a resource reference. A bare word or
|
|
49
|
+
// a directory name (`scripts/lib`) is undecidable prose — skipped.
|
|
50
|
+
const HAS_EXT = /\.[A-Za-z0-9]+$/;
|
|
51
|
+
/**
|
|
52
|
+
* A reference target is a LOCAL BUNDLED RESOURCE worth resolving iff it is a
|
|
53
|
+
* relative path with a file extension AND is not one of the skip shapes. This is
|
|
54
|
+
* the single gate; both the link path and the inline-path path run through it.
|
|
55
|
+
*/
|
|
56
|
+
function localResourceTarget(rawTarget) {
|
|
57
|
+
// Strip a markdown link title / fragment / query if present, and trim.
|
|
58
|
+
const target = rawTarget.trim();
|
|
59
|
+
if (target.length === 0)
|
|
60
|
+
return null;
|
|
61
|
+
// SKIP: URLs (http://, https://, mailto:, any scheme://) — external.
|
|
62
|
+
if (/^[a-z][a-z0-9+.-]*:\/\//i.test(target) || /^mailto:/i.test(target)) {
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
// SKIP: a pure anchor / fragment-only link (`#section`).
|
|
66
|
+
if (target.startsWith("#"))
|
|
67
|
+
return null;
|
|
68
|
+
// SKIP: absolute paths (`/etc/x`, Windows `C:\`) — not bundled-relative.
|
|
69
|
+
if (target.startsWith("/") || /^[A-Za-z]:[\\/]/.test(target))
|
|
70
|
+
return null;
|
|
71
|
+
// SKIP: variable tokens (`${CLAUDE_PLUGIN_ROOT}/x`, `$VAR/x`) — uncheckable,
|
|
72
|
+
// and almost always a plugin-root or runtime path, not a bundled file.
|
|
73
|
+
if (target.includes("$"))
|
|
74
|
+
return null;
|
|
75
|
+
// Drop a URL fragment / query suffix so `references/api.md#auth` resolves to
|
|
76
|
+
// the file. (Only after the scheme check above, so we never mangle a URL.)
|
|
77
|
+
const path = target.replace(/[?#].*$/, "");
|
|
78
|
+
if (path.length === 0)
|
|
79
|
+
return null;
|
|
80
|
+
// SKIP: a `../` escape OUT of the skill dir — undecidable / not a bundled
|
|
81
|
+
// resource (it points at a sibling skill or the repo). A leading `./` is fine.
|
|
82
|
+
const normalized = path.replace(/^\.\//, "");
|
|
83
|
+
if (normalized.startsWith("../") || normalized.includes("/../"))
|
|
84
|
+
return null;
|
|
85
|
+
// Must look like a file (have an extension), else it's a dir/prose mention.
|
|
86
|
+
if (!HAS_EXT.test(normalized))
|
|
87
|
+
return null;
|
|
88
|
+
return normalized;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Whether an inline-code path token is high-confidence enough to flag on its
|
|
92
|
+
* own (no surrounding `[..](..)` link syntax). We require a BUNDLE-DIR PREFIX
|
|
93
|
+
* (`scripts/`, `references/`, `assets/`) so a generic `` `config.json` `` or a
|
|
94
|
+
* `` `src/foo.ts` `` API mention in prose is never flagged — only the standard
|
|
95
|
+
* bundle layout, which is unambiguously a shipped resource.
|
|
96
|
+
*/
|
|
97
|
+
function isInlineBundlePath(token) {
|
|
98
|
+
const t = token.trim();
|
|
99
|
+
// A single token only — a span with spaces is a command/prose, not a path.
|
|
100
|
+
if (/\s/.test(t))
|
|
101
|
+
return false;
|
|
102
|
+
const normalized = t.replace(/^\.\//, "");
|
|
103
|
+
return BUNDLE_PREFIX.test(normalized) && HAS_EXT.test(normalized);
|
|
104
|
+
}
|
|
105
|
+
/** Collect candidate bundled-resource refs from one body line, skipping fences. */
|
|
106
|
+
function candidatesInLine(line, lineNo) {
|
|
107
|
+
const out = [];
|
|
108
|
+
// Markdown links: any relative path target that passes the local-resource gate.
|
|
109
|
+
for (const m of line.matchAll(MD_LINK)) {
|
|
110
|
+
const resolved = localResourceTarget(m[1]);
|
|
111
|
+
if (resolved !== null) {
|
|
112
|
+
out.push({ ref: m[1].trim(), resolved, kind: "link", line: lineNo });
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
// Inline-code path mentions: only the high-confidence bundle-dir-prefixed form.
|
|
116
|
+
for (const m of line.matchAll(INLINE_SPAN)) {
|
|
117
|
+
const token = m[1].trim();
|
|
118
|
+
if (!isInlineBundlePath(token))
|
|
119
|
+
continue;
|
|
120
|
+
const resolved = localResourceTarget(token);
|
|
121
|
+
if (resolved !== null) {
|
|
122
|
+
out.push({ ref: token, resolved, kind: "path", line: lineNo });
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return out;
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* The bundled-resource references in a SKILL.md body that don't resolve on disk
|
|
129
|
+
* under `skillDir`. Pure + FP-safe (see the module header). `skillDir` is the
|
|
130
|
+
* directory the SKILL.md itself lives in (resources are bundled beside it).
|
|
131
|
+
*
|
|
132
|
+
* The shared detector behind both `vigiles lint` (the `skill-resource-resolves`
|
|
133
|
+
* rule) and `vigiles audit` (the read-only report) — one detector, no drift.
|
|
134
|
+
*/
|
|
135
|
+
function skillResourceIssues(skillBody, skillDir, opts = {}) {
|
|
136
|
+
const exists = opts.existsSync ?? node_fs_1.existsSync;
|
|
137
|
+
const findings = [];
|
|
138
|
+
const seen = new Set();
|
|
139
|
+
const lines = skillBody.split("\n");
|
|
140
|
+
let inFence = false;
|
|
141
|
+
for (let i = 0; i < lines.length; i++) {
|
|
142
|
+
if (FENCE.test(lines[i])) {
|
|
143
|
+
inFence = !inFence;
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
if (inFence)
|
|
147
|
+
continue;
|
|
148
|
+
for (const c of candidatesInLine(lines[i], i + 1)) {
|
|
149
|
+
const full = (0, node_path_1.resolve)(skillDir, c.resolved);
|
|
150
|
+
if (exists(full))
|
|
151
|
+
continue;
|
|
152
|
+
// De-dupe the same missing file referenced several times in the body.
|
|
153
|
+
const key = `${c.kind}:${c.resolved}`;
|
|
154
|
+
if (seen.has(key))
|
|
155
|
+
continue;
|
|
156
|
+
seen.add(key);
|
|
157
|
+
findings.push({
|
|
158
|
+
ref: c.ref,
|
|
159
|
+
resolved: c.resolved,
|
|
160
|
+
kind: c.kind,
|
|
161
|
+
line: c.line,
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return findings;
|
|
166
|
+
}
|
|
167
|
+
//# sourceMappingURL=skill-resources.js.map
|
package/dist/core/types.d.ts
CHANGED
|
@@ -220,6 +220,77 @@ export interface RulesConfig {
|
|
|
220
220
|
* as `scan` (mcpHookIssues).
|
|
221
221
|
*/
|
|
222
222
|
"mcp-hook-target-resolves"?: RuleSeverity;
|
|
223
|
+
/**
|
|
224
|
+
* Flag a unit (subagent / model-invocable skill) whose declared tools hold all
|
|
225
|
+
* THREE legs of Simon Willison's "lethal trifecta" — read private data, ingest
|
|
226
|
+
* untrusted content, AND exfiltrate — a prompt-injection exfil path with no
|
|
227
|
+
* exploit code (Meta's Rule of Two: allow at most two). A capability SET-
|
|
228
|
+
* intersection over the declared contract, NOT a text scan; high-precision (only
|
|
229
|
+
* well-known tools map to a leg). An EXPLICIT all-three contract is a "hard"
|
|
230
|
+
* finding; an inherits-all unit (no contract → every leg) is "advisory". Default
|
|
231
|
+
* "warn" (don't-cry-wolf rollout); raise to "error" to gate CI. Same detector as
|
|
232
|
+
* `scan` (lethalTrifectaIssues). See docs/rules/lethal-trifecta.md.
|
|
233
|
+
*/
|
|
234
|
+
"lethal-trifecta"?: RuleSeverity;
|
|
235
|
+
/**
|
|
236
|
+
* Flag a SKILL.md body referencing a bundled file (`scripts/`/`references/`/
|
|
237
|
+
* `assets/`, or a relative markdown link with an extension) that doesn't exist
|
|
238
|
+
* on disk under the skill dir — the agent reads the instruction, gets nothing,
|
|
239
|
+
* and silently continues. The cross-reference moat applied to the SKILL.md body.
|
|
240
|
+
* High-precision / FP-safe (skips URLs, `$VAR` tokens, `../` escapes, extension-
|
|
241
|
+
* less mentions). Default "warn"; raise to "error" to gate CI. Same detector as
|
|
242
|
+
* `scan` (skillResourceIssues). See docs/rules/skill-resource-resolves.md.
|
|
243
|
+
*/
|
|
244
|
+
"skill-resource-resolves"?: RuleSeverity;
|
|
245
|
+
/**
|
|
246
|
+
* Flag a SKILL.md that opens with frontmatter-looking keys (`name:`,
|
|
247
|
+
* `description:`, …) but has NO opening `---` fence — the harness loads the
|
|
248
|
+
* whole file as body, so the skill has no name/description/trigger and is
|
|
249
|
+
* invisible (never fires). High-precision (a fixed key whitelist; markdown /
|
|
250
|
+
* prose lines never match). Default "warn"; raise to "error" to gate CI. Same
|
|
251
|
+
* detector as `scan` (skillFenceIssues). See docs/rules/skill-missing-fence.md.
|
|
252
|
+
*/
|
|
253
|
+
"skill-missing-fence"?: RuleSeverity;
|
|
254
|
+
/**
|
|
255
|
+
* Flag a functional surface directory (skills/agents/commands) nested INSIDE
|
|
256
|
+
* the `.claude-plugin/` manifest dir, where only `plugin.json` belongs — the
|
|
257
|
+
* harness can't see it, so the surface is invisible (the #1 plugin-author
|
|
258
|
+
* mistake). Pure filesystem check, FP-safe. Default "warn"; raise to "error"
|
|
259
|
+
* to gate CI. Same detector as `scan` (pluginLayoutIssues). See
|
|
260
|
+
* docs/rules/plugin-dir-layout.md.
|
|
261
|
+
*/
|
|
262
|
+
"plugin-dir-layout"?: RuleSeverity;
|
|
263
|
+
/**
|
|
264
|
+
* Flag a lethal trifecta that EMERGES across a delegation edge — a subagent
|
|
265
|
+
* whose effective (own ∪ delegated-to) capability holds all three legs though
|
|
266
|
+
* no single unit does (the combined blast radius). The capability-diff across
|
|
267
|
+
* the delegation tree; skips units the per-unit `lethal-trifecta` already
|
|
268
|
+
* flags (no double-report), FP-safe (explicit edges, wildcard-guarded). Default
|
|
269
|
+
* "warn"; raise to "error" to gate CI. Same detector as `scan`
|
|
270
|
+
* (delegationTrifecta). See docs/rules/delegation-trifecta.md.
|
|
271
|
+
*/
|
|
272
|
+
"delegation-trifecta"?: RuleSeverity;
|
|
273
|
+
/**
|
|
274
|
+
* Flag a hook that LOOKS like it blocks but silently doesn't — a block decision
|
|
275
|
+
* (`exit 2` / `decision` / `permissionDecision`) on an event that can't veto, or
|
|
276
|
+
* the legacy top-level `decision` field on a permission-gated event where only
|
|
277
|
+
* `hookSpecificOutput.permissionDecision` works (#19009, the #1 verified hook
|
|
278
|
+
* pain). FP-safe (conservative literal patterns; the blocking-event sets are
|
|
279
|
+
* read from the dialect, so it runs only where they're declared). Default "warn";
|
|
280
|
+
* raise to "error" to gate CI. Same detector as `scan` (hookBlockFindings). See
|
|
281
|
+
* docs/rules/hook-block-ineffective.md.
|
|
282
|
+
*/
|
|
283
|
+
"hook-block-ineffective"?: RuleSeverity;
|
|
284
|
+
/**
|
|
285
|
+
* Flag a hook `matcher` string that silently never fires — a close typo of a
|
|
286
|
+
* built-in tool (`bash`→`Bash`), or a malformed/undeclared MCP form
|
|
287
|
+
* (`mcp_memory_*` instead of `mcp__memory__.*`, or a server the plugin doesn't
|
|
288
|
+
* declare). High-precision (close-typo only; MCP gated on a declared set,
|
|
289
|
+
* built-ins allowlisted; wildcards/regex skipped). Default "warn"; raise to
|
|
290
|
+
* "error" to gate CI. Same detector as `scan` (hookMatcherFindings). See
|
|
291
|
+
* docs/rules/hook-matcher.md.
|
|
292
|
+
*/
|
|
293
|
+
"hook-matcher"?: RuleSeverity;
|
|
223
294
|
}
|
|
224
295
|
/** Extract severity from a rule value (handles both simple and tuple forms). */
|
|
225
296
|
export declare function ruleSeverity<T>(rule: RuleWithOptions<T> | undefined): RuleSeverity;
|
|
@@ -275,6 +346,18 @@ export interface VigilesConfig {
|
|
|
275
346
|
audit?: {
|
|
276
347
|
measure?: boolean;
|
|
277
348
|
};
|
|
349
|
+
/**
|
|
350
|
+
* `vigiles eval` preferences. `apiVersion` is the hand-bumped **behavior epoch**
|
|
351
|
+
* folded into the eval LOCK's input hash (`src/eval-lock.ts`): bump it when a
|
|
352
|
+
* harness-side change YOU made (a CLAUDE.md edit, a global hook) would shift
|
|
353
|
+
* eval outputs but isn't otherwise visible to the lock — so `vigiles eval
|
|
354
|
+
* --check` reports the committed eval results STALE and forces a local re-run.
|
|
355
|
+
* Default 1. Distinct from the (auto-resolved) `claude` CLI version, which is
|
|
356
|
+
* recorded as provenance but deliberately NOT hashed.
|
|
357
|
+
*/
|
|
358
|
+
eval?: {
|
|
359
|
+
apiVersion?: number;
|
|
360
|
+
};
|
|
278
361
|
}
|
|
279
362
|
/** Valid marker types for rule detection. */
|
|
280
363
|
export type MarkerType = "headings" | "checkboxes";
|
package/dist/core/validate.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { ParsedRule, ValidationError, ValidationResult, ReadResult, FileResult, ValidatePathsResult, RulesConfig, VigilesConfig, MarkerType, ParseOptions, ValidateOptions, ValidatePathsOptions, ReadOptions } from "./types.js";
|
|
2
2
|
export type { ParsedRule, ValidationError, ValidationResult, ReadResult, FileResult, ValidatePathsResult, RulesConfig, VigilesConfig, MarkerType, ParseOptions, ValidateOptions, ValidatePathsOptions, ReadOptions, };
|
|
3
|
+
export declare const DEFAULT_RULES: Required<RulesConfig>;
|
|
3
4
|
export declare function findInstructionFiles(cwd?: string, configFiles?: string[]): string[];
|
|
4
5
|
export declare function loadConfig(): VigilesConfig;
|
|
5
6
|
export declare function parseRules(content: string, { ruleMarkers }?: ParseOptions): ParsedRule[];
|
package/dist/core/validate.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_RULES = void 0;
|
|
3
4
|
exports.findInstructionFiles = findInstructionFiles;
|
|
4
5
|
exports.loadConfig = loadConfig;
|
|
5
6
|
exports.parseRules = parseRules;
|
|
@@ -28,7 +29,7 @@ const VALID_MARKERS = ["headings", "checkboxes"];
|
|
|
28
29
|
const INSTRUCTION_FILES = ["CLAUDE.md", "AGENTS.md"];
|
|
29
30
|
// The default instruction file to validate when no config names one.
|
|
30
31
|
const DEFAULT_FILES = [INSTRUCTION_FILES[0]];
|
|
31
|
-
|
|
32
|
+
exports.DEFAULT_RULES = {
|
|
32
33
|
"require-instructions-spec": "warn",
|
|
33
34
|
// Default OFF — the consistent `require-<surface>-spec` parallel. Skills are
|
|
34
35
|
// legitimately hand-written, so requiring a .spec.ts per SKILL.md is the wrong
|
|
@@ -70,10 +71,31 @@ const DEFAULT_RULES = {
|
|
|
70
71
|
"frontmatter-valid": "warn",
|
|
71
72
|
// A mcp_tool hook incomplete / targeting an undeclared server — on by default at warn.
|
|
72
73
|
"mcp-hook-target-resolves": "warn",
|
|
74
|
+
// Lethal-trifecta capability set-intersection (read-private + ingest-untrusted +
|
|
75
|
+
// exfiltrate in one unit) — WARN by default (don't-cry-wolf rollout); raise to error.
|
|
76
|
+
"lethal-trifecta": "warn",
|
|
77
|
+
// A SKILL.md body referencing a missing bundled resource — WARN by default
|
|
78
|
+
// (don't-cry-wolf rollout, FP-safe); raise to error to gate CI.
|
|
79
|
+
"skill-resource-resolves": "warn",
|
|
80
|
+
// A SKILL.md missing its opening `---` fence (invisible skill) — WARN by
|
|
81
|
+
// default (FP-safe key whitelist); raise to error to gate CI.
|
|
82
|
+
"skill-missing-fence": "warn",
|
|
83
|
+
// Functional dirs nested inside `.claude-plugin/` (invisible surfaces) — WARN
|
|
84
|
+
// by default; raise to error to gate CI.
|
|
85
|
+
"plugin-dir-layout": "warn",
|
|
86
|
+
// A lethal trifecta emerging across a delegation edge (combined blast radius) —
|
|
87
|
+
// WARN by default (don't-cry-wolf rollout); raise to error to gate CI.
|
|
88
|
+
"delegation-trifecta": "warn",
|
|
89
|
+
// A hook that looks like it blocks but silently doesn't (#19009) — WARN by
|
|
90
|
+
// default (FP-safe literal patterns); raise to error to gate CI.
|
|
91
|
+
"hook-block-ineffective": "warn",
|
|
92
|
+
// A hook matcher that never fires (tool typo / wrong MCP form) — WARN by
|
|
93
|
+
// default (high-precision); raise to error to gate CI.
|
|
94
|
+
"hook-matcher": "warn",
|
|
73
95
|
};
|
|
74
96
|
const DEFAULT_CONFIG = {
|
|
75
97
|
ruleMarkers: ["headings", "checkboxes"],
|
|
76
|
-
rules: DEFAULT_RULES,
|
|
98
|
+
rules: exports.DEFAULT_RULES,
|
|
77
99
|
files: DEFAULT_FILES,
|
|
78
100
|
};
|
|
79
101
|
// ---------------------------------------------------------------------------
|
|
@@ -99,7 +121,7 @@ function loadConfig() {
|
|
|
99
121
|
const config = {
|
|
100
122
|
...DEFAULT_CONFIG,
|
|
101
123
|
...userConfig,
|
|
102
|
-
rules: { ...DEFAULT_RULES, ...userConfig.rules },
|
|
124
|
+
rules: { ...exports.DEFAULT_RULES, ...userConfig.rules },
|
|
103
125
|
files: Array.isArray(userConfig.files) ? userConfig.files : DEFAULT_FILES,
|
|
104
126
|
};
|
|
105
127
|
if (!Array.isArray(config.ruleMarkers) ||
|
|
@@ -170,7 +192,7 @@ function parseRules(content, { ruleMarkers } = {}) {
|
|
|
170
192
|
// Core validation
|
|
171
193
|
// ---------------------------------------------------------------------------
|
|
172
194
|
function validate(content, { ruleMarkers, rules: rulesConfig, filePath, dialect } = {}) {
|
|
173
|
-
const activeRules = rulesConfig ?? DEFAULT_RULES;
|
|
195
|
+
const activeRules = rulesConfig ?? exports.DEFAULT_RULES;
|
|
174
196
|
const parsedRules = parseRules(content, { ruleMarkers });
|
|
175
197
|
const enforced = parsedRules.filter((r) => r.enforcement === "enforced").length;
|
|
176
198
|
const guidanceOnly = parsedRules.filter((r) => r.enforcement === "guidance").length;
|
package/dist/eval-cache.d.ts
CHANGED
|
@@ -43,6 +43,12 @@ export interface CacheRecord {
|
|
|
43
43
|
/** Text files present in the cwd after the run (relative path → contents). */
|
|
44
44
|
readonly files: Record<string, string>;
|
|
45
45
|
}
|
|
46
|
+
/**
|
|
47
|
+
* Canonicalize a value so the key is stable regardless of object key order —
|
|
48
|
+
* recursively sorts object keys. Arrays keep order (it's significant for tools).
|
|
49
|
+
* Exported so the eval LOCK ({@link ./eval-lock}) hashes its inputs the same way.
|
|
50
|
+
*/
|
|
51
|
+
export declare function canonical(value: unknown): unknown;
|
|
46
52
|
/**
|
|
47
53
|
* Cache record-format version, SALTED into every key (Jest `CACHE_VERSION` /
|
|
48
54
|
* webpack `cache.version` pattern). Bump when the `CacheRecord` shape — or how a
|
package/dist/eval-cache.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.CACHE_FORMAT_VERSION = void 0;
|
|
4
|
+
exports.canonical = canonical;
|
|
4
5
|
exports.cacheKey = cacheKey;
|
|
5
6
|
exports.readCache = readCache;
|
|
6
7
|
exports.writeCache = writeCache;
|
|
@@ -32,6 +33,7 @@ const SKIP_DIRS = new Set(["node_modules", ".git"]);
|
|
|
32
33
|
/**
|
|
33
34
|
* Canonicalize a value so the key is stable regardless of object key order —
|
|
34
35
|
* recursively sorts object keys. Arrays keep order (it's significant for tools).
|
|
36
|
+
* Exported so the eval LOCK ({@link ./eval-lock}) hashes its inputs the same way.
|
|
35
37
|
*/
|
|
36
38
|
function canonical(value) {
|
|
37
39
|
if (Array.isArray(value))
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import { type SHA256Hash } from "./core/hash.js";
|
|
2
|
+
/** Lock mode: never touch the lock / verify-only (CI) / record-and-write (local). */
|
|
3
|
+
export type LockMode = "off" | "check" | "update";
|
|
4
|
+
/**
|
|
5
|
+
* Per-spec lock overrides (additive on `EvalSpec`/`TriggerRateSpec`). Normally the
|
|
6
|
+
* mode comes from the CLI (`eval --check`/`--update` → `VIGILES_EVAL_LOCK`) and
|
|
7
|
+
* the dir/epoch from defaults/config; set these to drive the lock programmatically
|
|
8
|
+
* (or to point a test at a throwaway dir). Each field falls back to its env/default.
|
|
9
|
+
*/
|
|
10
|
+
export interface EvalLockOptions {
|
|
11
|
+
/** Override the lock mode (else `VIGILES_EVAL_LOCK`, else `off`). */
|
|
12
|
+
readonly mode?: LockMode;
|
|
13
|
+
/** Override the lock directory (else `<cwd>/.vigiles/eval-locks`). */
|
|
14
|
+
readonly dir?: string;
|
|
15
|
+
/** Override the behavior epoch (else `VIGILES_EVAL_API_VERSION`, else 1). */
|
|
16
|
+
readonly evalApiVersion?: number;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* On-disk lock-format version, salted into nothing (the lock is keyed by name,
|
|
20
|
+
* not by hash) but VALIDATED on read so an incompatible shape fails loud rather
|
|
21
|
+
* than deserializing into a stale structure. Bump on a breaking shape change.
|
|
22
|
+
*/
|
|
23
|
+
export declare const LOCK_VERSION = 1;
|
|
24
|
+
/** Default directory for committed eval locks (tracked, NOT gitignored). */
|
|
25
|
+
export declare const DEFAULT_LOCK_DIR = ".vigiles/eval-locks";
|
|
26
|
+
/**
|
|
27
|
+
* The model-affecting inputs hashed into a lock's `inputsHash`. Everything here
|
|
28
|
+
* is something that, if it changes, means the recorded model behavior is stale
|
|
29
|
+
* and you MUST re-drive the model (→ subscription → local). Deliberately EXCLUDED:
|
|
30
|
+
* the scoring `measure`/assertions (re-run live against the replayed report), the
|
|
31
|
+
* trial count (a sample-size knob, not a behavior input), and per-run env noise.
|
|
32
|
+
*/
|
|
33
|
+
export interface EvalLockInputs {
|
|
34
|
+
/** Model id used (folded in; a floating alias can't detect weight drift — warned). */
|
|
35
|
+
readonly model: string;
|
|
36
|
+
/**
|
|
37
|
+
* A hand-bumped behavior epoch the project owns (`.vigilesrc.json`
|
|
38
|
+
* `eval.apiVersion`), bumped when a harness-side change YOU made (a CLAUDE.md
|
|
39
|
+
* edit, a global hook) would shift eval outputs but isn't otherwise in the
|
|
40
|
+
* inputs. The escape hatch for "force a re-eval."
|
|
41
|
+
*/
|
|
42
|
+
readonly evalApiVersion: number;
|
|
43
|
+
/**
|
|
44
|
+
* The seam-specific canonical input object — the tasks/prompts/files/settings/
|
|
45
|
+
* sorted-tools/pluginDirHash/serialized-checks that steer the model. Assembled
|
|
46
|
+
* by each entry point (it knows its own shape) and hashed opaquely here.
|
|
47
|
+
*/
|
|
48
|
+
readonly inputs: unknown;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Why the harness binary version is **NOT** hashed (only recorded as provenance):
|
|
52
|
+
* `--check` runs in CI where `claude` is PINNED to a fixed version, while a dev's
|
|
53
|
+
* local `claude` is whatever they have — folding the version into the hash would
|
|
54
|
+
* false-trip `--check` on every PR where those differ. It is also the lock's
|
|
55
|
+
* honest scope: the gate verifies your committed results match your current
|
|
56
|
+
* *author-controlled inputs*, not current model/harness behavior (there is no
|
|
57
|
+
* automated live run). Harness/model drift is caught when YOU re-run `--update`
|
|
58
|
+
* locally and review the moved numbers in the git diff. Keeping the version out
|
|
59
|
+
* of the hash is what lets `--check` stay binary-free + deterministic in CI.
|
|
60
|
+
* (The eval CACHE still keys on it — that's local replay soundness, a different
|
|
61
|
+
* axis.) See research/cache-invalidation.md.
|
|
62
|
+
*/
|
|
63
|
+
/** Deterministic content hash of a lock's model-affecting inputs. */
|
|
64
|
+
export declare function evalInputsHash(input: EvalLockInputs): SHA256Hash;
|
|
65
|
+
/** A committed eval lock: the integrity stamp + the replayable recorded report. */
|
|
66
|
+
export interface EvalLock {
|
|
67
|
+
readonly version: number;
|
|
68
|
+
/** The eval's report name (human-facing; also the lock filename slug source). */
|
|
69
|
+
readonly name: string;
|
|
70
|
+
/** Hash of the model-affecting inputs ({@link evalInputsHash}). */
|
|
71
|
+
readonly inputsHash: string;
|
|
72
|
+
/** The model id the report was produced against (for the drift warning). */
|
|
73
|
+
readonly model: string;
|
|
74
|
+
/** The harness version token at record time (provenance; already in the hash). */
|
|
75
|
+
readonly harnessVersionKey: string;
|
|
76
|
+
/** The behavior epoch at record time (provenance; already in the hash). */
|
|
77
|
+
readonly evalApiVersion: number;
|
|
78
|
+
/** ISO-8601 timestamp the lock was recorded (provenance; NOT in the hash). */
|
|
79
|
+
readonly builtAt: string;
|
|
80
|
+
/**
|
|
81
|
+
* The entry point's recorded report — the model's observed behavior, REPLAYED
|
|
82
|
+
* verbatim on `--check` so the script's own assertions judge it. Stored as the
|
|
83
|
+
* exact return type of the entry point (`EvalReport` / `TriggerRateReport` /
|
|
84
|
+
* `CheckReport`) so replay is transparent to the caller.
|
|
85
|
+
*/
|
|
86
|
+
readonly report: unknown;
|
|
87
|
+
}
|
|
88
|
+
/** Filesystem-safe slug for a report name (the lock filename). */
|
|
89
|
+
export declare function lockSlug(name: string): string;
|
|
90
|
+
/** Path to a named eval's lock file under `dir`. */
|
|
91
|
+
export declare function lockPath(dir: string, name: string): string;
|
|
92
|
+
/**
|
|
93
|
+
* Read a named eval's lock. A MISS (no file) returns `null`. A CORRUPT or
|
|
94
|
+
* wrong-version file **throws** — a broken lock is a real failure the CI gate
|
|
95
|
+
* must surface, not silently treat as "no lock" (which would let a stale eval
|
|
96
|
+
* pass). The message says how to recover.
|
|
97
|
+
*/
|
|
98
|
+
export declare function readLock(dir: string, name: string): EvalLock | null;
|
|
99
|
+
/**
|
|
100
|
+
* Whether ANY lock has been committed under `dir`. The CI staleness gate
|
|
101
|
+
* (`eval --check`) uses this to stay a NO-OP until the feature is in use: a repo
|
|
102
|
+
* that has never run `eval --update` has no locks, so there is nothing to verify
|
|
103
|
+
* and CI passes green. Once the first lock is committed, every named eval is held
|
|
104
|
+
* to having a fresh one (a new unlocked eval then reads as stale). The graduated,
|
|
105
|
+
* opt-in-by-committing behavior that keeps a fresh `init` from going red.
|
|
106
|
+
*/
|
|
107
|
+
export declare function anyLocksCommitted(dir: string): boolean;
|
|
108
|
+
/**
|
|
109
|
+
* Does an edited path plausibly change an eval's INPUTS — so a committed lock may
|
|
110
|
+
* now be stale? Two surfaces feed the hash: a skill's trigger surface (`SKILL.md`)
|
|
111
|
+
* and the eval script that holds the prompts/spec (`*.eval.{mjs,cjs,js,mts,cts,ts}`).
|
|
112
|
+
* Pure (string-only) so the nudge hook stays cheap and never runs an eval script.
|
|
113
|
+
*/
|
|
114
|
+
export declare function isEvalInputFile(path: string): boolean;
|
|
115
|
+
/**
|
|
116
|
+
* The NON-BLOCKING nudge to emit after an eval-input edit when committed locks
|
|
117
|
+
* exist, or `null` for no nudge. Self-gating: it stays silent until you've opted
|
|
118
|
+
* into the lock (committed one), so it can't annoy a repo that doesn't use evals.
|
|
119
|
+
* It deliberately does NOT recompute staleness (that needs the eval script + is
|
|
120
|
+
* the job of `eval --check`) — a reminder, not a gate. The honest harness-neutral
|
|
121
|
+
* reminder; how it reaches the agent (both CC and Codex inject `additionalContext`
|
|
122
|
+
* on `PostToolUse`) is the caller's concern. See docs/harness-testing-*.md.
|
|
123
|
+
*/
|
|
124
|
+
export declare function evalLockNudge(filePath: string, lockDir: string): string | null;
|
|
125
|
+
/** Write a named eval's lock (pretty JSON for a reviewable git diff). */
|
|
126
|
+
export declare function writeLock(dir: string, lock: EvalLock): void;
|
|
127
|
+
/**
|
|
128
|
+
* Build a fresh lock envelope from a just-recorded report (the `--update` write).
|
|
129
|
+
* `builtAt` is passed in (never read from the clock here) so the module stays
|
|
130
|
+
* pure + deterministically testable; the CLI stamps the real timestamp.
|
|
131
|
+
*/
|
|
132
|
+
export declare function buildLock(args: {
|
|
133
|
+
readonly name: string;
|
|
134
|
+
readonly inputsHash: string;
|
|
135
|
+
readonly model: string;
|
|
136
|
+
readonly harnessVersionKey: string;
|
|
137
|
+
readonly evalApiVersion: number;
|
|
138
|
+
readonly builtAt: string;
|
|
139
|
+
readonly report: unknown;
|
|
140
|
+
}): EvalLock;
|
|
141
|
+
/** What the lock layer decides an entry point should do for this run. */
|
|
142
|
+
export type LockDecision =
|
|
143
|
+
/** Drive the model normally (mode `off`, or `update`, or `check` with no lock-skip). */
|
|
144
|
+
{
|
|
145
|
+
readonly kind: "run";
|
|
146
|
+
}
|
|
147
|
+
/** `check` + a matching fresh lock → return the recorded report, NO model call. */
|
|
148
|
+
| {
|
|
149
|
+
readonly kind: "replay";
|
|
150
|
+
readonly report: unknown;
|
|
151
|
+
}
|
|
152
|
+
/** `check` + a missing/stale lock → fail; the caller throws `reason`. */
|
|
153
|
+
| {
|
|
154
|
+
readonly kind: "stale";
|
|
155
|
+
readonly reason: string;
|
|
156
|
+
};
|
|
157
|
+
/**
|
|
158
|
+
* Decide what `check` mode should do given the current input hash and the
|
|
159
|
+
* committed lock. `off`/`update` always `run` (update records afterwards). `check`
|
|
160
|
+
* replays a matching lock (no model) and is `stale` on a missing lock or a hash
|
|
161
|
+
* mismatch — the deterministic CI gate.
|
|
162
|
+
*/
|
|
163
|
+
export declare function decideLock(mode: LockMode, name: string, currentHash: string, existing: EvalLock | null): LockDecision;
|
|
164
|
+
/** A single numeric leaf that moved between the prior lock and a fresh `--update`. */
|
|
165
|
+
export interface NumberDelta {
|
|
166
|
+
readonly path: string;
|
|
167
|
+
readonly before: number;
|
|
168
|
+
readonly after: number;
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Collect the numeric leaves that changed between two recorded reports — the
|
|
172
|
+
* human-facing delta printed at `--update` time (e.g. `rate: 0.900 → 0.650`).
|
|
173
|
+
* Generic over any report shape (walks numbers by dotted path), so it works for
|
|
174
|
+
* `EvalReport`, `TriggerRateReport`, and `CheckReport` without per-type code. The
|
|
175
|
+
* committed git diff is the primary review surface; this is the at-a-glance echo.
|
|
176
|
+
*/
|
|
177
|
+
export declare function diffReportNumbers(before: unknown, after: unknown): NumberDelta[];
|
|
178
|
+
/** Render the `--update` result for a human: NEW lock, or the per-number deltas. */
|
|
179
|
+
export declare function formatLockUpdate(name: string, deltas: readonly NumberDelta[], isNew: boolean): string;
|
|
180
|
+
/**
|
|
181
|
+
* Read the lock mode from the environment (`VIGILES_EVAL_LOCK`), set by the CLI's
|
|
182
|
+
* `eval --check` / `--update` flags. A run knob (like `VIGILES_TRIALS`): the CLI
|
|
183
|
+
* is the only place that should set it. Anything unrecognized → `off`.
|
|
184
|
+
*/
|
|
185
|
+
export declare function lockModeFromEnv(env?: NodeJS.ProcessEnv): LockMode;
|
|
186
|
+
/**
|
|
187
|
+
* The behavior epoch (`evalApiVersion`) for this run, read from the env the CLI
|
|
188
|
+
* populates from `.vigilesrc.json` `eval.apiVersion`. Default 1. A malformed
|
|
189
|
+
* value falls back to 1 (never throws) — the lock stays usable.
|
|
190
|
+
*/
|
|
191
|
+
export declare function evalApiVersionFromEnv(env?: NodeJS.ProcessEnv): number;
|
|
192
|
+
//# sourceMappingURL=eval-lock.d.ts.map
|