vigiles 11.0.0 → 12.0.0

package/dist/eval.d.ts

@@ -1,5 +1,6 @@
 import { parseToolCalls, parseHooks, parseSubagents, type ToolCall, type Trace } from "./harness-test.js";
 import { type CacheMode } from "./eval-cache.js";
+import { type EvalLockOptions } from "./eval-lock.js";
 import type { Check, CheckJSON } from "./check.js";
 import { type Comparison } from "./stats.js";
 import { type ToolIntercept } from "./tool-intercept.js";
@@ -159,6 +160,16 @@ export interface EvalSpec<M extends Metrics> {
      * to today). See {@link ToolStub} and `research/eval-coverage-and-isolation.md`.
      */
     readonly stubs?: readonly ToolStub[];
+    /**
+     * **The eval LOCK** — a committed staleness gate for CI (see `src/eval-lock.ts`).
+     * With a `name` set, `vigiles eval --update` (local, on your subscription)
+     * records the report to `.vigiles/eval-locks/<name>.lock.json`; `--check` (CI)
+     * verifies the committed result against the current inputs WITHOUT a model call,
+     * failing "stale" when they diverge. Mode normally comes from the CLI; set
+     * `lock` to drive it programmatically or point a test at a throwaway dir. The
+     * lock engages only when `name` is set (it keys the lock file).
+     */
+    readonly lock?: EvalLockOptions;
 }
 /** Per-metric summary statistics across an arm's runs. */
 export interface MetricStat {
@@ -479,16 +490,6 @@ export declare function modelTier(id: string): number | null;
  * {@link modelTier}); an unrankable model/floor is never "below" (fail-open).
  */
 export declare function belowModelFloor(model: string, floor: string): boolean;
-/**
- * Reduce a raw `--version` string to the **major.minor** cache-key token. We key
- * the cache on major.minor, NOT the patch: a patch release rarely changes agent
- * behaviour, so keying patches would churn the cache on every release for no
- * signal; a minor/major bump is where the system prompt / tool defs actually move.
- * (If a specific patch is known to matter, clear the cache or bump
- * `CACHE_FORMAT_VERSION`.) Falls back to the trimmed raw string when no semver is
- * found. Pure + tested.
- */
-export declare function harnessVersionKey(raw: string): string;
 /**
  * Build an **ephemeral run environment** for a model-driven run: a NEW env object
  * with a *fresh* `HOME` (and `TMPDIR`) pointed at the throwaway `opts.home`, only
@@ -542,15 +543,6 @@ export declare function seedEphemeralHome(throwawayHome: string, realHome: strin
 export declare function isRateLimited(out: RunOut): boolean;
 /** Map `worker` over `items` with at most `concurrency` in flight, order preserved. */
 export declare function runPool<T, R>(items: readonly T[], concurrency: number, worker: (item: T) => Promise<R>): Promise<R[]>;
-/**
- * The eval orchestration — every arm × trial via `runner`, run through the cache
- * and a rate-limit retry, with at most `concurrency` in flight and an optional
- * `maxCostUsd` budget cap; metric + usage computed per run and aggregated per
- * arm. Exported with an injectable `runner` so the loop, `measure` context,
- * caching, pooling, and aggregation are unit-testable without spawning a model
- * (pass a fake returning canned stream-json). `runEval` is this with the real
- * agent runner.
- */
 export declare function runEvalWith<M extends Metrics>(spec: EvalSpec<M>, runner: AgentRunner): Promise<EvalReport>;
 /** Format an eval report as a compact table for the console (mean ± se, pass^k). */
 export declare function formatEvalReport(report: EvalReport): string;
@@ -563,6 +555,19 @@ export declare function formatEvalReport(report: EvalReport): string;
  * (reuse the bare predicates, e.g. `(t) => skillResolved(t, "x:y")`).
  */
 export interface TriggerRateSpec {
+    /**
+     * A stable name for this trigger eval — required to engage the {@link lock}
+     * (it keys the committed `.vigiles/eval-locks/<name>.lock.json`). Two evals over
+     * the same skill with different prompts get distinct names. Optional otherwise.
+     */
+    readonly name?: string;
+    /**
+     * **The eval LOCK** — the CI staleness gate (see `src/eval-lock.ts`). With
+     * `name` set, `vigiles eval --update` records this trigger-rate report locally
+     * and `--check` verifies it against the current inputs (skill contents, prompts,
+     * model) with NO model call. Mode normally comes from the CLI flags.
+     */
+    readonly lock?: EvalLockOptions;
     /**
      * Plugin dir installed natively (`--plugin-dir`) so its skills/commands
      * activate. Provide this OR {@link skillsDir}, not both.
@@ -722,6 +727,14 @@ export interface EvalDriver {
     readonly runner: AgentRunner;
     readonly parse: ModelOutputParser;
     readonly runError?: (out: RunOut) => string | null;
+    /**
+     * The harness this driver runs (e.g. `"claude-code"`, `"codex"`). Folded into a
+     * trigger-rate eval's LOCK hash so a report recorded on one harness is marked
+     * STALE if the eval is later switched to another (a different harness can fire a
+     * skill differently). Optional for back-compat — absent defaults to
+     * `"claude-code"`, so an existing single-harness lock is unaffected.
+     */
+    readonly harness?: string;
 }
 /** The default (Claude Code) eval driver: real `claude` + stream-json parsing. */
 export declare const claudeEvalDriver: EvalDriver;
@@ -801,7 +814,7 @@ export declare function packageInstallSet(opts: {
     dir: string;
     added: number;
 };
-export declare function measureTriggerRateWith(spec: TriggerRateSpec, runner: AgentRunner, parse?: ModelOutputParser, runError?: (out: RunOut) => string | null): Promise<TriggerRateReport>;
+export declare function measureTriggerRateWith(spec: TriggerRateSpec, runner: AgentRunner, parse?: ModelOutputParser, runError?: (out: RunOut) => string | null, harness?: string): Promise<TriggerRateReport>;
 /**
  * Measure a skill/behaviour's real trigger rate across prompts × trials. Defaults
  * to the real `claude` CLI (`claudeEvalDriver`); pass `{ evalDriver }` to drive a

package/dist/eval.js

@@ -22,7 +22,6 @@ exports.aggregateUsage = aggregateUsage;
 exports.isDatedModel = isDatedModel;
 exports.modelTier = modelTier;
 exports.belowModelFloor = belowModelFloor;
-exports.harnessVersionKey = harnessVersionKey;
 exports.ephemeralRunEnv = ephemeralRunEnv;
 exports.seedEphemeralHome = seedEphemeralHome;
 exports.isRateLimited = isRateLimited;
@@ -71,6 +70,7 @@ const runtime_js_1 = require("./adapters/claude-code/runtime.js");
 const proofs_js_1 = require("./core/proofs.js");
 const harness_test_js_1 = require("./harness-test.js");
 const eval_cache_js_1 = require("./eval-cache.js");
+const eval_lock_js_1 = require("./eval-lock.js");
 const stats_js_1 = require("./stats.js");
 const tool_intercept_js_1 = require("./tool-intercept.js");
 const tool_stub_js_1 = require("./tool-stub.js");
@@ -640,32 +640,22 @@ function belowModelFloor(model, floor) {
     const f = modelTier(floor);
     return m !== null && f !== null && m < f;
 }
-/**
- * Reduce a raw `--version` string to the **major.minor** cache-key token. We key
- * the cache on major.minor, NOT the patch: a patch release rarely changes agent
- * behaviour, so keying patches would churn the cache on every release for no
- * signal; a minor/major bump is where the system prompt / tool defs actually move.
- * (If a specific patch is known to matter, clear the cache or bump
- * `CACHE_FORMAT_VERSION`.) Falls back to the trimmed raw string when no semver is
- * found. Pure + tested.
- */
-function harnessVersionKey(raw) {
-    const m = /(\d+)\.(\d+)\.\d+/.exec(raw);
-    return m ? `${m[1]}.${m[2]}` : raw.trim();
-}
 /* v8 ignore start -- spawns the real harness binary; memoized, cache-path only */
 let cachedHarnessVersion;
 /**
- * The harness binary version (`claude --version`) reduced to major.minor, for the
- * cache key — so a CLI **minor/major** upgrade (new system prompt / tool defs)
- * invalidates a stale replay, while patches don't churn it. Memoized (one spawn
- * per process), resolved only on the cache path, "unknown" if the binary isn't
- * found (then it doesn't partition the key).
+ * The harness binary version (`claude --version`), reduced to its
+ * behaviorally-significant token by the runtime port's
+ * {@link HarnessRuntime.versionKey} — so a Claude Code **minor/major** upgrade
+ * (new system prompt / tool defs) invalidates a stale replay while patches don't
+ * churn it. The reduction is per-harness (CC → `major.minor`, Codex → `""`) and
+ * lives on the adapter, not here. Memoized (one spawn per process), resolved only
+ * on the cache path, "unknown" if the binary isn't found (then it doesn't
+ * partition the key).
  */
 function harnessVersion() {
     if (cachedHarnessVersion === undefined) {
         try {
-            cachedHarnessVersion = harnessVersionKey((0, node_child_process_1.execSync)(`${runtime_js_1.claudeCodeRuntime.agentBinary} --version`, {
+            cachedHarnessVersion = runtime_js_1.claudeCodeRuntime.versionKey((0, node_child_process_1.execSync)(`${runtime_js_1.claudeCodeRuntime.agentBinary} --version`, {
                 encoding: "utf-8",
                 stdio: ["ignore", "pipe", "ignore"],
             }));
@@ -983,15 +973,141 @@ function aggregateArms(armNames, results) {
     }
     return { arms, totalCostUsd };
 }
+function resolveLock(over) {
+    return {
+        mode: over?.mode ?? (0, eval_lock_js_1.lockModeFromEnv)(),
+        dir: over?.dir ?? (0, node_path_1.resolve)(process.cwd(), eval_lock_js_1.DEFAULT_LOCK_DIR),
+        evalApiVersion: over?.evalApiVersion ?? (0, eval_lock_js_1.evalApiVersionFromEnv)(),
+    };
+}
+/** Emit a vigiles message (GitHub annotation under Actions, else stderr/stdout). */
+function emitLockMessage(msg, warn) {
+    if (process.env.GITHUB_ACTIONS)
+        console.log(`::${warn ? "warning" : "notice"}::${msg}`);
+    else if (warn)
+        console.warn(msg);
+    else
+        console.log(msg);
+}
+/**
+ * Run a named eval through the lock. `off` → just `produce()`. `check` → replay a
+ * matching committed lock (NO model call) or throw "stale". `update` → `produce()`,
+ * write the lock, print the human-facing delta. The model is driven ONLY on the
+ * run path — never on a clean `check`, which is what keeps the CI gate binary-free.
+ * A lock-on run with no `name` is a LOUD skip (the lock needs a name to key the file).
+ */
+async function withEvalLock(args, produce) {
+    const { lock } = args;
+    if (lock.mode === "off")
+        return produce();
+    if (!args.name) {
+        // An unnamed eval can't be keyed to a lock. In `check` (the CI gate) running
+        // it would call the model — violating the no-model-in-CI contract and failing
+        // for missing auth — so FAIL LOUDLY instead of silently hitting the model.
+        // `update` (local, model available) keeps producing: the eval just isn't gated.
+        if (lock.mode === "check") {
+            throw new Error(`vigiles eval --check: an unnamed eval cannot run in CI — it has no ` +
+                `committed lock to replay, and running it would call the model. Add a ` +
+                `\`name\` to the spec to gate it, or exclude it from the --check run.`);
+        }
+        emitLockMessage(`vigiles eval --${lock.mode}: skipped the lock for an unnamed eval — set ` +
+            `\`name\` on the spec to enable the staleness gate for it.`, true);
+        return produce();
+    }
+    if (!isDatedModel(args.model))
+        warnFloatingModel(args.model);
+    const inputsHash = (0, eval_lock_js_1.evalInputsHash)({
+        model: args.model,
+        evalApiVersion: lock.evalApiVersion,
+        inputs: args.inputs,
+    });
+    const existing = (0, eval_lock_js_1.readLock)(lock.dir, args.name);
+    const decision = (0, eval_lock_js_1.decideLock)(lock.mode, args.name, inputsHash, existing);
+    if (decision.kind === "stale")
+        throw new Error(decision.reason);
+    if (decision.kind === "replay")
+        return decision.report;
+    const report = await produce();
+    const builtLock = (0, eval_lock_js_1.buildLock)({
+        name: args.name,
+        inputsHash,
+        model: args.model,
+        harnessVersionKey: harnessVersion(),
+        evalApiVersion: lock.evalApiVersion,
+        builtAt: new Date().toISOString(),
+        report,
+    });
+    const deltas = existing ? (0, eval_lock_js_1.diffReportNumbers)(existing.report, report) : [];
+    (0, eval_lock_js_1.writeLock)(lock.dir, builtLock);
+    emitLockMessage((0, eval_lock_js_1.formatLockUpdate)(args.name, deltas, existing === null), false);
+    return report;
+}
+/**
+ * Strip the machine-specific plugin-root prefix from a resolved value before it
+ * enters the lock hash. `resolveHarness` expands `${PLUGIN_ROOT}` in a plugin's
+ * hook commands to the checkout's ABSOLUTE path, so a lock recorded at
+ * `/home/dev/...` would be falsely STALE when `--check` recomputes it at
+ * `/home/runner/...` in CI (or any other machine). Normalizing the prefix back to
+ * a token makes the hash location-independent. No-op when there's no plugin root.
+ */
+function stripPluginRoot(value, absRoot) {
+    if (absRoot === "")
+        return value;
+    const json = JSON.stringify(value);
+    // A hookless plugin resolves to `settings: undefined`, which `JSON.stringify`
+    // returns as `undefined` (not a string) — pass it through rather than `.split`
+    // a non-string (which would throw before the eval can run).
+    if (json === undefined)
+        return value;
+    return JSON.parse(json.split(absRoot).join("${PLUGIN_ROOT}"));
+}
 /**
- * The eval orchestration — every arm × trial via `runner`, run through the cache
- * and a rate-limit retry, with at most `concurrency` in flight and an optional
- * `maxCostUsd` budget cap; metric + usage computed per run and aggregated per
- * arm. Exported with an injectable `runner` so the loop, `measure` context,
- * caching, pooling, and aggregation are unit-testable without spawning a model
- * (pass a fake returning canned stream-json). `runEval` is this with the real
- * agent runner.
+ * Resolve each arm's model-affecting inputs into a canonical object for the lock
+ * hash — WITHOUT running the model (it reads files + hashes plugin dirs only). The
+ * trial count is excluded (a sample-size knob, not a behavior input); `measure` is
+ * excluded by design (the script re-asserts against the replayed report).
  */
+function evalArmsInputs(spec, cfg) {
+    const arms = {};
+    for (const [name, arm] of Object.entries(spec.arms)) {
+        const resolved = (0, plugin_loader_js_1.resolveHarness)({
+            plugin: arm.plugin,
+            settings: arm.settings,
+            files: { ...spec.fixture, ...arm.files },
+        });
+        // The plugin root `resolveHarness` expanded into the resolved files/settings
+        // is this checkout's absolute path — normalize it out so the hash is the same
+        // on the dev's machine and in CI (else every plugin-with-root-hooks eval is
+        // falsely stale across machines).
+        const absRoot = arm.plugin ? (0, node_path_1.resolve)(process.cwd(), arm.plugin) : "";
+        arms[name] = {
+            model: arm.model ?? cfg.model,
+            tools: [...cfg.tools].sort(),
+            files: stripPluginRoot(resolved.files, absRoot),
+            settings: stripPluginRoot(resolved.settings, absRoot),
+            pluginDirHash: arm.pluginDir ? (0, eval_cache_js_1.hashDir)(arm.pluginDir) : undefined,
+            interceptTools: arm.interceptTools
+                ? (0, tool_intercept_js_1.serializeIntercepts)(arm.interceptTools)
+                : undefined,
+        };
+    }
+    // Tool stubs (`spec.stubs`) are written onto PATH before each trial, so a
+    // change to a canned CLI output IS a model-facing input change — fold a
+    // canonical (name-sorted) view into the hash so `--check` catches it. Sorted
+    // for a stable key regardless of declaration order; an empty list is the
+    // byte-identical-to-before default. Each ToolStub is plain serializable data.
+    const stubs = [...(spec.stubs ?? [])].sort((a, b) => a.name.localeCompare(b.name));
+    // `ephemeralEnv` swaps the trial's environment (scrubbed env + throwaway HOME
+    // vs the inherited process env), which can move tool/hook/agent behavior — a
+    // model-facing input, so it belongs in the hash. Normalize to a bool so a
+    // record under one mode can't be replayed under the other with the same hash.
+    return {
+        task: spec.task,
+        arms,
+        stubs,
+        ephemeralEnv: spec.ephemeralEnv === true,
+    };
+}
 async function runEvalWith(spec, runner) {
     const trials = spec.trials ?? 5;
     const spacing = (spec.spacingSec ?? 4) * 1000;
@@ -1024,9 +1140,16 @@ async function runEvalWith(spec, runner) {
             await sleep(spacing);
         return { armName: unit.armName, skipped: false, row, usage };
     };
-    const results = await runPool(units, concurrency, worker);
-    const { arms, totalCostUsd } = aggregateArms(Object.keys(spec.arms), results);
-    return { name: spec.name ?? "eval", trials, arms, totalCostUsd, aborted };
+    const lock = resolveLock(spec.lock);
+    // Resolve the lock inputs only when the lock is active (off skips the
+    // resolveHarness/hashDir work). `check` replays the committed report below
+    // without ever entering the run pool — so no model is driven in CI.
+    const inputs = lock.mode === "off" ? undefined : evalArmsInputs(spec, cfg);
+    return withEvalLock({ name: spec.name, inputs, model: cfg.model, lock }, async () => {
+        const results = await runPool(units, concurrency, worker);
+        const { arms, totalCostUsd } = aggregateArms(Object.keys(spec.arms), results);
+        return { name: spec.name ?? "eval", trials, arms, totalCostUsd, aborted };
+    });
 }
 /** Render one metric: `name=mean±se pass^k=…` (se/pass^k shown when measured). */
 function formatMetric(name, mean, stat) {
@@ -1065,6 +1188,7 @@ function formatEvalReport(report) {
 exports.claudeEvalDriver = {
     runner: spawnAgent,
     parse: parseClaudeRun,
+    harness: "claude-code",
 };
 /**
  * Package loose `<skillsDir>/<name>/SKILL.md` skills into a throwaway plugin dir
@@ -1439,7 +1563,7 @@ function assertTriggerDiversity(spec) {
         });
     }
 }
-async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runError) {
+async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runError, harness = "claude-code") {
     // Deterministic gate FIRST — before spending a token (or packaging a skillsDir).
     assertTriggerDiversity(spec);
     // Model floor (default Sonnet): trigger-rate under-measures selection on a
@@ -1469,26 +1593,50 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
         parse,
         runError,
     };
+    const lock = resolveLock(spec.lock);
     try {
-        const relevant = await runTriggerSet(spec.prompts, cfg, runner);
-        const base = {
-            rate: relevant.n > 0 ? relevant.fired / relevant.n : 0,
-            n: relevant.n,
-            perPrompt: relevant.perPrompt,
-            competitors,
-            errored: positiveOrUndefined(relevant.errored),
-        };
-        if ((spec.irrelevantPrompts?.length ?? 0) === 0)
-            return base;
-        const irrelevant = await runTriggerSet(spec.irrelevantPrompts ?? [], cfg, runner);
-        const fires = relevant.fired + irrelevant.fired;
-        return {
-            ...base,
-            errored: positiveOrUndefined(relevant.errored + irrelevant.errored),
-            falsePositiveRate: irrelevant.n > 0 ? irrelevant.fired / irrelevant.n : 0,
-            precision: fires > 0 ? relevant.fired / fires : undefined,
-            perIrrelevant: irrelevant.perPrompt,
-        };
+        // The lock hashes the skill UNDER TEST (its dir contents) + the prompts +
+        // model + tools — the inputs that steer whether it fires. `--check` replays
+        // the committed report with no model; `--update` records it. Resolved only
+        // when the lock is active. `trials` is excluded (a sample-size knob).
+        const triggerInputs = lock.mode === "off"
+            ? undefined
+            : {
+                pluginDirHash: (0, eval_cache_js_1.hashDir)(pluginDir),
+                prompts: [...spec.prompts],
+                irrelevantPrompts: spec.irrelevantPrompts
+                    ? [...spec.irrelevantPrompts]
+                    : undefined,
+                model: cfg.model,
+                tools: [...cfg.tools].sort(),
+                fixture: spec.fixture,
+                competitors,
+                // The harness the driver runs is a model-facing input — a Claude vs
+                // Codex run can fire a skill differently — so a recorded report is
+                // STALE if the eval is switched to another harness.
+                harness,
+            };
+        return await withEvalLock({ name: spec.name, inputs: triggerInputs, model: cfg.model, lock }, async () => {
+            const relevant = await runTriggerSet(spec.prompts, cfg, runner);
+            const base = {
+                rate: relevant.n > 0 ? relevant.fired / relevant.n : 0,
+                n: relevant.n,
+                perPrompt: relevant.perPrompt,
+                competitors,
+                errored: positiveOrUndefined(relevant.errored),
+            };
+            if ((spec.irrelevantPrompts?.length ?? 0) === 0)
+                return base;
+            const irrelevant = await runTriggerSet(spec.irrelevantPrompts ?? [], cfg, runner);
+            const fires = relevant.fired + irrelevant.fired;
+            return {
+                ...base,
+                errored: positiveOrUndefined(relevant.errored + irrelevant.errored),
+                falsePositiveRate: irrelevant.n > 0 ? irrelevant.fired / irrelevant.n : 0,
+                precision: fires > 0 ? relevant.fired / fires : undefined,
+                perIrrelevant: irrelevant.perPrompt,
+            };
+        });
     }
     finally {
         // Remove the throwaway plugin dir we built from a loose `skillsDir`.
@@ -1506,7 +1654,7 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
  */
 async function measureTriggerRate(spec, opts = {}) {
     const d = opts.evalDriver ?? exports.claudeEvalDriver;
-    return measureTriggerRateWith(spec, d.runner, d.parse, d.runError);
+    return measureTriggerRateWith(spec, d.runner, d.parse, d.runError, d.harness ?? "claude-code");
 }
 /* v8 ignore stop */
 /** Format a trigger-rate report: overall %, then each prompt's rate. */

package/dist/setup-plan.d.ts

@@ -158,6 +158,43 @@ export interface InstallPlan {
 export declare function planPluginInstall(harnesses: readonly string[], opts: {
     hasClaude: boolean;
 }): InstallPlan[];
+/** One vigiles-managed Codex hook: a `[[hooks.<event>]]` entry. */
+export interface CodexPluginHook {
+    readonly event: string;
+    /** Codex matcher (anchored regex, the dialect convention). */
+    readonly matcher: string;
+    /** The shell command Codex runs (a direct `npx vigiles …`, no plugin root). */
+    readonly command: string;
+    /** A unique command substring → idempotent re-merge (replaces in place). */
+    readonly key: string;
+}
+/**
+ * vigiles's proactive nudges, wired into a Codex repo's `.codex/config.toml`.
+ *
+ * Codex has no global plugin store (unlike Claude Code's marketplace), so its
+ * config is repo-committed — the idiomatic place for these. They run as DIRECT
+ * `npx vigiles hook-runtime …` commands (NOT vendored bash scripts): the runtime
+ * entrypoints read the event JSON on stdin and emit the `hookSpecificOutput.
+ * additionalContext` shape Codex honors on `PostToolUse` (confirmed against the
+ * official hooks docs + encoded in `HookProtocol.injectableEvents`). Safety: only
+ * an INTENTIONAL `exit 2` blocks an edit (the refs nudge, when `unmarked-refs` is
+ * `error`); an npx-resolution failure exits non-2, so a missing dep never blocks.
+ *
+ * Deliberately NOT here (a loud, documented deferral — no-silent-skips): the
+ * SessionStart lint summary (CC delivers it as plain stdout, whose SessionStart
+ * prepend is unconfirmed on Codex — vs the JSON `additionalContext` these use) and
+ * the compile-on-edit / pre-edit-block guards (filename-gated bash with no
+ * harness-neutral `hook-runtime` entrypoint yet). Those stay manual on Codex.
+ */
+export declare function codexPluginHooks(): CodexPluginHook[];
+/**
+ * Idempotently merge {@link codexPluginHooks} into a parsed `.codex/config.toml`
+ * object. Pure — the IO (read/parse/serialize/write) stays in cli.ts's
+ * `wireCodexHooks`. Each vigiles hook is keyed by a unique command substring, so
+ * a re-run REPLACES its own entry in place and leaves the user's own Codex hooks
+ * (and every other config key) untouched. Returns a new object.
+ */
+export declare function applyCodexPluginHooks(existing: Record<string, unknown>): Record<string, unknown>;
 /**
  * Resolve the final plan: defaults, then flags, then interactive answers (each
  * layer overrides the previous only where it has an opinion). `--target` pins a

package/dist/setup-plan.js

@@ -17,6 +17,8 @@ exports.mergeProjectConfig = mergeProjectConfig;
 exports.shouldPrompt = shouldPrompt;
 exports.collectSetupAnswers = collectSetupAnswers;
 exports.planPluginInstall = planPluginInstall;
+exports.codexPluginHooks = codexPluginHooks;
+exports.applyCodexPluginHooks = applyCodexPluginHooks;
 exports.resolvePlan = resolvePlan;
 function flagValue(args, prefix) {
     return args.find((a) => a.startsWith(prefix))?.slice(prefix.length);
@@ -260,8 +262,10 @@ function planPluginInstall(harnesses, opts) {
         if (harness === "codex") {
             // The cross-agent `skills` CLI with `-g -y` installs to the global store
             // ~/.agents/skills/ (NOT the repo, and NOT ~/.codex/ — verified against
-            // the real CLI). Skills only; Codex hooks (.codex/config.toml [hooks])
-            // are not wired automatically.
+            // the real CLI). Skills install globally; the proactive NUDGE hooks
+            // (eval-lock + refs) are wired into the repo's .codex/config.toml by
+            // `init` (see codexPluginHooks / wireCodexHooks) — Codex config is
+            // repo-committed, so that's the idiomatic place.
             return {
                 harness,
                 commands: ["npx --yes skills add zernie/vigiles -a codex -g -y"],
@@ -269,9 +273,10 @@ function planPluginInstall(harnesses, opts) {
                 manualSteps: ["npx skills add zernie/vigiles -a codex -g -y"],
                 notes: [
                     "Codex reads AGENTS.md directly; the skills install globally to ~/.agents/skills/ (not the repo).",
-                    "Codex hooks (.codex/config.toml [hooks]) are not auto-wired yet — add them manually for compile-on-edit.",
+                    "The eval-lock + refs NUDGE hooks are wired into .codex/config.toml (repo-committed, the Codex norm).",
+                    "Still manual on Codex: the SessionStart lint summary + compile-on-edit/pre-edit guards (no harness-neutral entrypoint yet).",
                 ],
-                vendors: false,
+                vendors: true,
             };
         }
         return {
@@ -284,6 +289,63 @@ function planPluginInstall(harnesses, opts) {
         };
     });
 }
+/**
+ * vigiles's proactive nudges, wired into a Codex repo's `.codex/config.toml`.
+ *
+ * Codex has no global plugin store (unlike Claude Code's marketplace), so its
+ * config is repo-committed — the idiomatic place for these. They run as DIRECT
+ * `npx vigiles hook-runtime …` commands (NOT vendored bash scripts): the runtime
+ * entrypoints read the event JSON on stdin and emit the `hookSpecificOutput.
+ * additionalContext` shape Codex honors on `PostToolUse` (confirmed against the
+ * official hooks docs + encoded in `HookProtocol.injectableEvents`). Safety: only
+ * an INTENTIONAL `exit 2` blocks an edit (the refs nudge, when `unmarked-refs` is
+ * `error`); an npx-resolution failure exits non-2, so a missing dep never blocks.
+ *
+ * Deliberately NOT here (a loud, documented deferral — no-silent-skips): the
+ * SessionStart lint summary (CC delivers it as plain stdout, whose SessionStart
+ * prepend is unconfirmed on Codex — vs the JSON `additionalContext` these use) and
+ * the compile-on-edit / pre-edit-block guards (filename-gated bash with no
+ * harness-neutral `hook-runtime` entrypoint yet). Those stay manual on Codex.
+ */
+function codexPluginHooks() {
+    // Codex's file-edit tool is `apply_patch` (its dialect vocabulary —
+    // src/adapters/codex/dialect.ts), NOT Claude's `Edit`/`Write`. A PostToolUse
+    // matcher keyed on CC tool names would never fire on Codex, so the nudges must
+    // match the Codex tool name. (Both nudge entrypoints also self-gate on the
+    // edited file, so a non-edit event no-ops regardless.)
+    return [
+        {
+            event: "PostToolUse",
+            matcher: "^apply_patch$",
+            command: "npx --no-install vigiles hook-runtime eval-lock-nudge",
+            key: "hook-runtime eval-lock-nudge",
+        },
+        {
+            event: "PostToolUse",
+            matcher: "^apply_patch$",
+            command: "npx --no-install vigiles hook-runtime refs",
+            key: "hook-runtime refs",
+        },
+    ];
+}
+/**
+ * Idempotently merge {@link codexPluginHooks} into a parsed `.codex/config.toml`
+ * object. Pure — the IO (read/parse/serialize/write) stays in cli.ts's
+ * `wireCodexHooks`. Each vigiles hook is keyed by a unique command substring, so
+ * a re-run REPLACES its own entry in place and leaves the user's own Codex hooks
+ * (and every other config key) untouched. Returns a new object.
+ */
+function applyCodexPluginHooks(existing) {
+    const config = existing;
+    const hooks = {
+        ...(config.hooks ?? {}),
+    };
+    for (const h of codexPluginHooks()) {
+        const kept = (hooks[h.event] ?? []).filter((e) => !e.command.includes(h.key));
+        hooks[h.event] = [...kept, { matcher: h.matcher, command: h.command }];
+    }
+    return { ...existing, hooks };
+}
 /**
  * Resolve the final plan: defaults, then flags, then interactive answers (each
  * layer overrides the previous only where it has an opinion). `--target` pins a

package/hooks/eval-lock-nudge.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# PostToolUse hook — after the agent edits an eval input (a SKILL.md trigger
+# surface or an *.eval.* script) and committed eval locks exist, inject a
+# NON-BLOCKING reminder to re-run `vigiles eval --update`. Self-gating: silent
+# until you've committed a lock, so it never fires in a repo that doesn't use
+# evals. Never blocks the edit. Runs as its OWN PostToolUse entry so its stdout
+# stays clean JSON. See docs/harness-testing.md (the eval lock).
+
+set -uo pipefail
+
+INPUT=$(cat)
+
+# No npx / not a Node project → nothing to do, never disrupt the edit.
+command -v npx >/dev/null 2>&1 || exit 0
+[ -f package.json ] || exit 0
+
+printf '%s' "$INPUT" | npx vigiles hook-runtime eval-lock-nudge
+
+# Always exit 0 — this hook only nudges, it never blocks (the gate is CI's
+# `vigiles eval --check`). Swallow any tool/setup error so it can't disrupt edits.
+exit 0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vigiles",
-  "version": "11.0.0",
+  "version": "12.0.0",
   "description": "Lint & test the harness your AI agent runs on — verify the references in your CLAUDE.md / AGENTS.md and test that your hooks and skills actually work.",
   "keywords": [
     "claude-code",

package/skills/test-harness/SKILL.md

@@ -190,6 +190,33 @@ job that asserts the capability is present, run **`vigiles test --no-skip`** so
 skipped tier fails — a green-with-skips is untested surface. Keep unit +
 deterministic tests in CI (free); run evals locally or on a schedule with auth.
 
+## Step 5 — Lock the eval so CI stays honest (you do this automatically)
+
+Real-model evals run on the user's subscription — locally, never in CI. So **as
+part of writing an eval, you keep its result fresh for them.** Do these two things
+without being asked:
+
+1. **Give every eval a `name`.** That's what the lock keys on.
+2. **After running it, commit the lock:**
+
+   ```bash
+   vigiles eval --update      # records the result → .vigiles/eval-locks/<name>.lock.json
+   ```
+
+   Then commit that file. CI runs `vigiles eval --check` (no model) to verify it
+   still matches the inputs — so a later edit that forgets to re-eval fails loud
+   instead of shipping stale numbers.
+
+**When you later change a skill's description or prompts, re-run `vigiles eval
+--update` and commit the updated lock** — the change altered what the eval
+measures. (vigiles also nudges you: when a lock exists, a `SKILL.md` edit triggers
+a non-blocking reminder.)
+
+Why it's cheap: `--check` only hashes inputs (skill text, prompts, model). A
+**threshold** change in the test re-uses the saved numbers (no model); only an
+**input** change needs a fresh `--update`. Full mechanics:
+[`docs/harness-testing.md`](../../docs/harness-testing.md#keep-eval-results-fresh-in-ci-the-lock).
+
 ## When the user didn't say what to test
 
 Don't ask them to specify — **pick something real and demonstrate.** Scan the