vigiles 4.1.0 → 5.0.1

package/dist/cli.js

@@ -32,6 +32,7 @@ const compose_js_1 = require("./core/compose.js");
 const compile_generator_js_1 = require("./core/compile-generator.js");
 const action_gate_js_1 = require("./action-gate.js");
 const agent_runtime_js_1 = require("./adapters/claude-code/agent-runtime.js");
+const tool_intercept_js_1 = require("./tool-intercept.js");
 const refs_js_1 = require("./core/refs.js");
 const mcp_js_1 = require("./core/mcp.js");
 const skill_runtime_js_1 = require("./adapters/claude-code/skill-runtime.js");
@@ -909,8 +910,8 @@ async function runLint(restArgs, flags, config) {
         }
     }
     // 7b. Untested-surface check — skills/agents/hooks shipping without a test or
-    // eval. Warning by default (a nudge, exit 0); set rules.untested-surface to
-    // "error" to gate CI. See src/test-coverage.ts and docs/rules/untested-surface.md.
+    // eval. Warning by default (a nudge, exit 0); set rules.untested-{skill,agent,
+    // hook} to "error" to gate CI. See src/test-coverage.ts and docs/rules/.
     const untested = checkUntestedSurfaces(config, silent);
     // 8. Validate vigiles builder calls inside markdown code blocks. Default
     // is to validate every ref; illustrative blocks opt out via
@@ -924,6 +925,16 @@ async function runLint(restArgs, flags, config) {
             console.log(`  ${line}`);
         }
     }
+    // Per-line GitHub annotations for each broken doc ref — each carries file+line,
+    // so GitHub renders it INLINE on the PR diff (not just in the summary blob).
+    // Previously this check reported to stdout only; the inline/spec checks already
+    // annotate per-line, so this closes the gap that left doc-ref findings invisible
+    // on the PR. CI-only (isGitHubActions); skipped under --json/--summary.
+    if (isGitHubActions() && !silent) {
+        for (const e of docRefReport.errors) {
+            ghAnnotate("error", `${e.kind}("${e.value}") — ${e.message}`, e.file, e.line);
+        }
+    }
     // 9. Verify code-shaped symbol references live (see src/refs.ts).
     const symbolRefErrors = verifyMarkdownSymbols(files, silent);
     // 10. Verify `vigiles:mcp server#tool` marks against live MCP servers
@@ -1877,29 +1888,47 @@ function checkIntegrityForFiles(files, severity, silent) {
     return severity === "error" ? errorCount : 0;
 }
 /**
- * Apply the `untested-surface` rule: find skills/agents/hooks with no test or
- * eval (see src/test-coverage.ts). Returns the raw untested count plus the
- * severity-gated error count — "warn" prints but never fails CI (errors=0),
- * "error" fails (exit 2), mirroring the integrity check.
+ * Apply the per-kind `untested-skill` / `untested-agent` / `untested-hook` rules:
+ * find skills/agents/hooks with no test or eval (see src/test-coverage.ts). Each
+ * kind is gated by its OWN rule severity — a kind set to `false` is not scanned;
+ * "warn" prints but never fails CI; "error" fails (exit 2). Returns the raw
+ * untested count plus the severity-gated error count.
  */
 function checkUntestedSurfaces(config, silent) {
-    const severity = (0, types_js_1.ruleSeverity)(config?.rules["untested-surface"]);
-    if (!severity)
+    const rules = config?.rules;
+    const skillSev = (0, types_js_1.ruleSeverity)(rules?.["untested-skill"]);
+    const agentSev = (0, types_js_1.ruleSeverity)(rules?.["untested-agent"]);
+    const hookSev = (0, types_js_1.ruleSeverity)(rules?.["untested-hook"]);
+    if (!skillSev && !agentSev && !hookSev)
         return { untested: 0, errors: 0 };
-    const opts = (0, types_js_1.ruleOptions)(config?.rules["untested-surface"]);
-    const report = (0, test_coverage_js_1.findUntestedSurfaces)({ basePath: process.cwd(), ...opts });
+    const sevFor = (kind) => kind === "skill" ? skillSev : kind === "agent" ? agentSev : hookSev;
+    // Test-discovery options (testGlobs/exclude) are shared; merge them from
+    // whichever of the three rules carries them.
+    const opts = {
+        ...(0, types_js_1.ruleOptions)(rules?.["untested-skill"]),
+        ...(0, types_js_1.ruleOptions)(rules?.["untested-agent"]),
+        ...(0, types_js_1.ruleOptions)(rules?.["untested-hook"]),
+    };
+    const report = (0, test_coverage_js_1.findUntestedSurfaces)({
+        basePath: process.cwd(),
+        skills: skillSev !== false,
+        agents: agentSev !== false,
+        hooks: hookSev !== false,
+        testGlobs: opts.testGlobs,
+        exclude: opts.exclude,
+    });
     if (!silent) {
         console.log("\nUntested surfaces:\n");
         for (const line of (0, test_coverage_js_1.formatUntestedReport)(report).split("\n")) {
             console.log(`  ${line}`);
         }
         for (const s of report.untested) {
-            ghAnnotate(severity === "error" ? "error" : "warning", `${s.kind} ${s.path} ships without a test or eval`, s.path);
+            ghAnnotate(sevFor(s.kind) === "error" ? "error" : "warning", `${s.kind} ${s.path} ships without a test or eval`, s.path);
         }
     }
     return {
         untested: report.untested.length,
-        errors: severity === "error" ? report.untested.length : 0,
+        errors: report.untested.filter((s) => sevFor(s.kind) === "error").length,
     };
 }
 /**
@@ -2081,6 +2110,18 @@ function handleRunScripts(kind, args, restArgs) {
     // Harness/eval scripts may be authored in JS or TS (see run-scripts.ts).
     const defaultGlob = (0, run_scripts_js_1.scriptGlob)(kind === "test" ? "harness" : "eval");
     const files = (0, run_scripts_js_1.discoverScripts)(restArgs, defaultGlob, cwd);
+    // `--min=N`: a CI gate asserts at least N scripts actually RAN — so a bad path,
+    // a renamed file, or a glob that matched nothing fails LOUD instead of passing
+    // green with zero evals executed. Default 0 (off) keeps local runs ergonomic.
+    const minFlag = args.find((a) => a.startsWith("--min="));
+    const minRequired = minFlag
+        ? Math.max(0, Number.parseInt(minFlag.split("=")[1] ?? "", 10) || 0)
+        : 0;
+    if (files.length < minRequired) {
+        console.error(`✗ vigiles ${kind}: --min=${String(minRequired)} but only ${String(files.length)} ${kind} file(s) matched — ` +
+            "evals never executed (check the paths/globs, or that the run was reached).");
+        process.exit(1);
+    }
     if (files.length === 0) {
         console.log(`No ${defaultGlob} files found.`);
         return;
@@ -2091,6 +2132,10 @@ function handleRunScripts(kind, args, restArgs) {
     if (kind === "test" && !(0, harness_test_js_1.claudeAvailable)()) {
         console.log("ℹ `claude` CLI not found — unit-tier tests run; tests that need it report SKIPPED.\n");
     }
+    // `--trials=N` (a run knob: cost/precision, doesn't change WHAT is measured) is
+    // forwarded to scripts via env. The MODEL is deliberately NOT a CLI/env knob —
+    // it's part of the measurement definition, so it belongs in the spec
+    // (`model` / `minModel`), version-controlled, not a hidden override.
     const trialsFlag = args.find((a) => a.startsWith("--trials="));
     const env = {};
     if (trialsFlag)
@@ -2117,7 +2162,7 @@ function printUsage(command) {
     console.log("  vigiles compile [files...]     Compile .spec.ts → .md");
     console.log("  vigiles lint [files...]        Verify references, find gaps in instruction files");
     console.log("  vigiles test [files...]        Run *.harness.mjs deterministic harness tests");
-    console.log("  vigiles eval [files...]        Run *.eval.mjs real-model harness evals (--trials=N)");
+    console.log("  vigiles eval [files...]        Run *.eval.mjs real-model harness evals (--trials=N, --min=N, --no-skip)");
     console.log("");
     console.log("Examples:");
     console.log("  vigiles init                 Auto-detect project, create specs, wire CI");
@@ -2231,7 +2276,7 @@ function skillStartCommand(target) {
  * PreToolUse-hook entrypoint: enforce the active subagent's allowed-tools
  * contract. Reads the tool event on stdin, parses the active agent's compiled
  * `.md` tool rail, and blocks (exit 2 + reason on stderr) any tool outside it —
- * the deterministic boundary `tools:` alone can't provide (Claude Code #54898).
+ * the deterministic boundary `tools:` alone can't provide (Claude Code #4740/#21460, SDK #172).
  */
 function agentHookCommand() {
     let raw = "";
@@ -2256,6 +2301,30 @@ function agentHookCommand() {
         process.exit(2);
     }
 }
+/**
+ * `vigiles intercept-tool-hook` — the PreToolUse interception hook for the
+ * tool-call spy. Reads the intercept list from `VIGILES_INTERCEPT_TOOLS`, decides
+ * whether the called tool should be intercepted, and if so denies the real
+ * execution (exit 2) with a block message — the call is intercepted (prevented),
+ * NOT executed. Allowing (return) lets the tool run for real. The model still
+ * emits the `tool_use`, so its arguments land in the Trace for `toolWith` /
+ * `notTool` to assert on. See src/tool-intercept.ts.
+ */
+function interceptToolHookCommand() {
+    let raw = "";
+    try {
+        raw = (0, node_fs_1.readFileSync)(0, "utf-8");
+    }
+    catch {
+        /* no stdin */
+    }
+    const intercepts = (0, tool_intercept_js_1.parseIntercepts)(process.env[tool_intercept_js_1.INTERCEPT_TOOLS_ENV] ?? "");
+    const decision = (0, tool_intercept_js_1.interceptHookDecision)(raw, intercepts);
+    if (decision.intercept) {
+        console.error(decision.denyReason);
+        process.exit(2);
+    }
+}
 /** Mark a subagent active so the PreToolUse hook enforces its tool contract. */
 function agentStartCommand(target) {
     if (!target) {
@@ -2289,6 +2358,9 @@ function handleSkillCommand(command, restArgs) {
         case "agent-hook":
             agentHookCommand();
             return true;
+        case "intercept-tool-hook":
+            interceptToolHookCommand();
+            return true;
         case "action-hook":
             actionHookCommand();
             return true;

package/dist/core/types.d.ts

@@ -69,16 +69,13 @@ export interface OrphansConfig {
      */
     exclude?: readonly string[];
 }
-/** Options for the untested-surface check. */
+/**
+ * Shared options for the per-kind untested-* rules (`untested-skill` /
+ * `untested-agent` / `untested-hook`). Which kinds are scanned is controlled by
+ * each rule's severity (set a rule to `false` to skip that kind), so only the
+ * test-discovery knobs live here.
+ */
 export interface TestCoverageConfig {
-    /** Scan skills. Default true. */
-    skills?: boolean;
-    /** Scan subagents. Default true. */
-    agents?: boolean;
-    /** Scan hook scripts referenced from plugin.json / settings.json. Default true. */
-    hooks?: boolean;
-    /** Require a test for user-invoked (disable-model-invocation) skills. Default false. */
-    includeUserInvokedSkills?: boolean;
     /** Globs of test files that count as coverage. */
     testGlobs?: readonly string[];
     /** Extra ignore globs. */
@@ -87,14 +84,22 @@ export interface TestCoverageConfig {
 export interface RulesConfig {
     /** Require .spec.ts for CLAUDE.md / AGENTS.md. Default: "warn". */
     "require-spec"?: RuleSeverity;
-    /** Require .spec.ts for SKILL.md files. Default: false. */
+    /**
+     * @deprecated Skills are legitimately hand-written; use `untested-skill`
+     * ("every skill ships with a test/eval") instead. Default: false (off). The
+     * check still runs if you set this explicitly.
+     */
     "require-skill-spec"?: RuleSeverity;
     /** Detect hand-edits to compiled markdown via SHA-256 hash. Default: "warn". */
     integrity?: RuleSeverity;
     /** Enforce minimum spec coverage thresholds. Default: false. ESLint-style: ["warn", { scripts: 50 }]. */
     coverage?: RuleWithOptions<CoverageThresholds>;
-    /** Flag skills/agents/hooks with no test or eval. Default: "warn". */
-    "untested-surface"?: RuleWithOptions<TestCoverageConfig>;
+    /** Flag a skill (SKILL.md) that ships with no test or eval. Default: "warn". */
+    "untested-skill"?: RuleWithOptions<TestCoverageConfig>;
+    /** Flag a subagent (agents/*.md) that ships with no test or eval. Default: "warn". */
+    "untested-agent"?: RuleWithOptions<TestCoverageConfig>;
+    /** Flag a hook script that ships with no test or eval. Default: "warn". */
+    "untested-hook"?: RuleWithOptions<TestCoverageConfig>;
     /**
      * Nudge (or block) when an instruction file has code-shaped references that
      * aren't expressed as vigiles marks (so the lint can't verify them), or a

package/dist/core/validate.js

@@ -32,10 +32,19 @@ const INSTRUCTION_FILES = ["CLAUDE.md", "AGENTS.md"];
 const DEFAULT_FILES = [INSTRUCTION_FILES[0]];
 const DEFAULT_RULES = {
     "require-spec": "warn",
-    "require-skill-spec": "warn",
+    // DEPRECATED — default OFF. Skills are legitimately hand-written (Level 0/1),
+    // so requiring a .spec.ts per SKILL.md was the wrong constraint and only added
+    // noise (it also nagged about vendored/fixture/bench skills). Use the
+    // `untested-*` rules instead — "every skill/agent/hook ships with a test or
+    // eval" is the coverage that matters. The implementation is kept: setting
+    // `require-skill-spec` explicitly still works for anyone who wants it.
+    "require-skill-spec": false,
     integrity: "warn",
     coverage: false,
-    "untested-surface": "warn",
+    // Per-kind surface-coverage: a skill/agent/hook must ship with a test or eval.
+    "untested-skill": "warn",
+    "untested-agent": "warn",
+    "untested-hook": "warn",
     "unmarked-refs": "warn",
 };
 const DEFAULT_CONFIG = {
@@ -175,7 +184,9 @@ function validate(content, { ruleMarkers, rules: rulesConfig, filePath, dialect
                 }
             }
         }
-        // --- require-skill-spec (SKILL.md) ---
+        // --- require-skill-spec (SKILL.md) — DEPRECATED but still honored when a
+        // user sets it explicitly, so reading the deprecated key here is intentional.
+        // eslint-disable-next-line @typescript-eslint/no-deprecated
         const skillSeverity = activeRules["require-skill-spec"];
         if (skillSeverity && isSkill && !disableComment.test(content)) {
             const specPath = filePath + ".spec.ts";

package/dist/eval-cache.d.ts

@@ -11,6 +11,29 @@ export interface CacheKeyInput {
     readonly files: Record<string, string>;
     /** The resolved `.claude/settings.json` for the arm (or undefined). */
     readonly settings: unknown;
+    /**
+     * Per-run env that affects model behaviour (e.g. `VIGILES_INTERCEPT_TOOLS`).
+     * Keyed because two intercept configs that share tool names — so produce
+     * identical merged `settings` — still differ in their `when`/`denyReason`, which
+     * lives only in the env. Omit when there's no model-affecting env.
+     */
+    readonly env?: Record<string, string>;
+    /**
+     * Content digest of a natively-installed plugin dir (`--plugin-dir`), or
+     * undefined when there is none. Folded in so editing a skill INSIDE the dir
+     * invalidates the entry — a path-only key would false-replay, since the dir's
+     * files are NOT in `files` (that holds only the materialized fixture / `plugin`
+     * arm, not a native install). See {@link hashDir}.
+     */
+    readonly pluginDirHash?: string;
+    /**
+     * The harness BINARY version (e.g. `claude --version`). The harness evolves
+     * fast — a CLI upgrade changes the system prompt + tool definitions, which steer
+     * behaviour as much as the model does — so a cached result must invalidate when
+     * the binary changes, or a replay silently serves a result from a different
+     * harness. Resolved once per run; omit when unknown (then it doesn't partition).
+     */
+    readonly harnessVersion?: string;
     /** Which trial this is — distinct trials are distinct samples, cached apart. */
     readonly trialIndex: number;
 }
@@ -20,9 +43,23 @@ export interface CacheRecord {
     /** Text files present in the cwd after the run (relative path → contents). */
     readonly files: Record<string, string>;
 }
+/**
+ * Cache record-format version, SALTED into every key (Jest `CACHE_VERSION` /
+ * webpack `cache.version` pattern). Bump when the `CacheRecord` shape — or how a
+ * record is produced in a way the key can't otherwise see — changes, so old
+ * entries become *unreachable* rather than deserializing into a stale shape (no
+ * brittle read-time version gate needed). A major bump means orphaned files on
+ * disk; reclaim them by deleting the cache dir.
+ */
+export declare const CACHE_FORMAT_VERSION = 2;
 /** Deterministic content hash of the key inputs (order-independent). */
 export declare function cacheKey(input: CacheKeyInput): SHA256Hash;
-/** Read a cached record by key, or null on miss / unreadable / malformed. */
+/**
+ * Read a cached record by key. A MISS (no file) returns `null` — normal, the run
+ * proceeds. A CORRUPT record (file present but not valid JSON) **throws** instead
+ * of silently degrading to a re-run: a broken cassette is a real failure the CI
+ * gate must surface, not mask. The message tells you how to recover.
+ */
 export declare function readCache(dir: string, key: SHA256Hash): CacheRecord | null;
 /** Write a cached record by key (creating the cache dir as needed). */
 export declare function writeCache(dir: string, key: SHA256Hash, record: CacheRecord): void;
@@ -30,4 +67,15 @@ export declare function writeCache(dir: string, key: SHA256Hash, record: CacheRe
 export declare function snapshotDir(cwd: string): Record<string, string>;
 /** Restore a snapshot into `cwd`, recreating directories as needed. */
 export declare function restoreDir(cwd: string, files: Record<string, string>): void;
+/**
+ * Content digest of a directory: a lexicographically-sorted list of
+ * `relativePath:contentHash` for every file, hashed to one value. Editing,
+ * adding, removing, or moving any file changes the digest. It hashes file
+ * CONTENT (not mtime — CI checkouts reset mtimes, the classic stale-cache
+ * anti-pattern) and includes the relative path (so a rename invalidates and two
+ * files can't swap contents undetected). A flat sorted list, NOT a Merkle tree —
+ * sufficient at plugin-dir scale; the tree's incremental-recompute payoff isn't
+ * worth the complexity here (cf. Bazel/Turborepo hash content per file).
+ */
+export declare function hashDir(dir: string): SHA256Hash;
 //# sourceMappingURL=eval-cache.d.ts.map

package/dist/eval-cache.js

@@ -1,10 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.CACHE_FORMAT_VERSION = void 0;
 exports.cacheKey = cacheKey;
 exports.readCache = readCache;
 exports.writeCache = writeCache;
 exports.snapshotDir = snapshotDir;
 exports.restoreDir = restoreDir;
+exports.hashDir = hashDir;
 /**
  * vigiles — record/replay cache for the eval tier.
  *
@@ -43,20 +45,70 @@ function canonical(value) {
     }
     return value;
 }
+/**
+ * Cache record-format version, SALTED into every key (Jest `CACHE_VERSION` /
+ * webpack `cache.version` pattern). Bump when the `CacheRecord` shape — or how a
+ * record is produced in a way the key can't otherwise see — changes, so old
+ * entries become *unreachable* rather than deserializing into a stale shape (no
+ * brittle read-time version gate needed). A major bump means orphaned files on
+ * disk; reclaim them by deleting the cache dir.
+ */
+exports.CACHE_FORMAT_VERSION = 2;
+/**
+ * Per-run env keys that are PURE NOISE for the cache key — a fresh random path
+ * every run, never model-affecting. The opt-in ephemeral run env
+ * ({@link ephemeralRunEnv} in `eval.ts`) points `HOME`/`TMPDIR` at a throwaway
+ * dir generated per trial, so folding them into the key would make every
+ * ephemeral run unique → the cache could NEVER hit. We drop them here so the
+ * exclusion holds however the env was assembled. A non-ephemeral run normally
+ * carries neither in its per-run `env` (it's an overlay over `process.env`, not
+ * a complete env), so dropping them is a no-op there.
+ */
+const CACHE_KEY_ENV_EXCLUDE = ["HOME", "TMPDIR"];
+/** Strip the per-run-noise env keys ({@link CACHE_KEY_ENV_EXCLUDE}) from a keyed
+ *  env, returning `undefined` when nothing model-affecting remains (so the key is
+ *  byte-identical to a run that had no env at all). */
+function keyedEnv(env) {
+    if (env === undefined)
+        return undefined;
+    const out = {};
+    for (const [k, v] of Object.entries(env)) {
+        if (CACHE_KEY_ENV_EXCLUDE.includes(k))
+            continue;
+        out[k] = v;
+    }
+    return Object.keys(out).length > 0 ? out : undefined;
+}
 /** Deterministic content hash of the key inputs (order-independent). */
 function cacheKey(input) {
-    return (0, hash_js_1.sha256short)(JSON.stringify(canonical(input)));
+    const normalized = {
+        ...input,
+        // The tool list is logically a SET, so ["Read","Bash"] and ["Bash","Read"]
+        // must hash the same — sort it to avoid phantom-distinct keys. (canonical()
+        // already sorts object keys; it deliberately keeps other array order.)
+        tools: [...input.tools].sort(),
+        // Drop the throwaway ephemeral HOME/TMPDIR — per-run noise, not model input.
+        env: keyedEnv(input.env),
+        cacheFormatVersion: exports.CACHE_FORMAT_VERSION,
+    };
+    return (0, hash_js_1.sha256short)(JSON.stringify(canonical(normalized)));
 }
-/** Read a cached record by key, or null on miss / unreadable / malformed. */
+/**
+ * Read a cached record by key. A MISS (no file) returns `null` — normal, the run
+ * proceeds. A CORRUPT record (file present but not valid JSON) **throws** instead
+ * of silently degrading to a re-run: a broken cassette is a real failure the CI
+ * gate must surface, not mask. The message tells you how to recover.
+ */
 function readCache(dir, key) {
     const path = (0, node_path_1.join)(dir, `${key}.json`);
     if (!(0, node_fs_1.existsSync)(path))
         return null;
+    const raw = (0, node_fs_1.readFileSync)(path, "utf-8");
     try {
-        return JSON.parse((0, node_fs_1.readFileSync)(path, "utf-8"));
+        return JSON.parse(raw);
     }
     catch {
-        return null;
+        throw new Error(`eval cache: corrupt record ${path} (invalid JSON) — delete it or clear the cache dir`);
     }
 }
 /** Write a cached record by key (creating the cache dir as needed). */
@@ -91,4 +143,32 @@ function restoreDir(cwd, files) {
         (0, node_fs_1.writeFileSync)(full, content);
     }
 }
+/**
+ * Content digest of a directory: a lexicographically-sorted list of
+ * `relativePath:contentHash` for every file, hashed to one value. Editing,
+ * adding, removing, or moving any file changes the digest. It hashes file
+ * CONTENT (not mtime — CI checkouts reset mtimes, the classic stale-cache
+ * anti-pattern) and includes the relative path (so a rename invalidates and two
+ * files can't swap contents undetected). A flat sorted list, NOT a Merkle tree —
+ * sufficient at plugin-dir scale; the tree's incremental-recompute payoff isn't
+ * worth the complexity here (cf. Bazel/Turborepo hash content per file).
+ */
+function hashDir(dir) {
+    const root = (0, node_path_1.resolve)(dir);
+    const parts = [];
+    const walk = (d) => {
+        for (const entry of (0, node_fs_1.readdirSync)(d).sort()) {
+            if (SKIP_DIRS.has(entry))
+                continue;
+            const full = (0, node_path_1.join)(d, entry);
+            const st = (0, node_fs_1.statSync)(full);
+            if (st.isDirectory())
+                walk(full);
+            else if (st.isFile())
+                parts.push(`${(0, node_path_1.relative)(root, full)}:${(0, hash_js_1.sha256short)((0, node_fs_1.readFileSync)(full))}`);
+        }
+    };
+    walk(root);
+    return (0, hash_js_1.sha256short)(parts.join("\n"));
+}
 //# sourceMappingURL=eval-cache.js.map