vigiles 25.0.0 → 25.1.0

package/dist/adapters/codex/eval.d.ts

@@ -61,6 +61,24 @@ export declare function codexSkillFired(run: {
  * turn. Pure fs — unit-testable without a binary.
  */
 export declare function installCodexSkills(pluginDir: string, cwd: string): number;
+/**
+ * Refuse an `effort` this adapter cannot honour, LOUDLY.
+ *
+ * The Claude runner pins the reasoning budget through `--effort` plus the env var
+ * it sits under. Codex exposes no mapping we have measured — its config carries a
+ * `model_reasoning_effort` key, but nothing here has driven the real binary
+ * through it, so claiming support would assert something unverified.
+ *
+ * The alternative — forwarding `task`/`cwd`/`timeoutMs` and dropping `effort` on
+ * the floor, as this runner does today — is the silent CC-only path the
+ * harness-parity rule forbids: a spec would declare `effort: "low"`, the lock
+ * would RECORD `low`, and the run would happen at whatever Codex defaults to.
+ * A stale number is recoverable; a confidently mislabelled one is not.
+ *
+ * Pure so the deferral itself is tested rather than living inside the ignored
+ * subprocess region.
+ */
+export declare function refuseCodexEffort(effort: string | number | undefined): void;
 /**
  * The Codex eval-tier `AgentRunner`: install the run's skills into `.codex/skills`
  * (Codex's discovery path, vs Claude's `--plugin-dir`), then drive a real

package/dist/adapters/codex/eval.js

@@ -32,6 +32,7 @@ exports.parseCodexEvalRun = parseCodexEvalRun;
 exports.codexRunError = codexRunError;
 exports.codexSkillFired = codexSkillFired;
 exports.installCodexSkills = installCodexSkills;
+exports.refuseCodexEffort = refuseCodexEffort;
 exports.codexEvalAgentRunner = codexEvalAgentRunner;
 exports.codexEvalRunner = codexEvalRunner;
 const node_child_process_1 = require("node:child_process");
@@ -172,6 +173,31 @@ function installCodexSkills(pluginDir, cwd) {
     }
     return n;
 }
+/**
+ * Refuse an `effort` this adapter cannot honour, LOUDLY.
+ *
+ * The Claude runner pins the reasoning budget through `--effort` plus the env var
+ * it sits under. Codex exposes no mapping we have measured — its config carries a
+ * `model_reasoning_effort` key, but nothing here has driven the real binary
+ * through it, so claiming support would assert something unverified.
+ *
+ * The alternative — forwarding `task`/`cwd`/`timeoutMs` and dropping `effort` on
+ * the floor, as this runner does today — is the silent CC-only path the
+ * harness-parity rule forbids: a spec would declare `effort: "low"`, the lock
+ * would RECORD `low`, and the run would happen at whatever Codex defaults to.
+ * A stale number is recoverable; a confidently mislabelled one is not.
+ *
+ * Pure so the deferral itself is tested rather than living inside the ignored
+ * subprocess region.
+ */
+function refuseCodexEffort(effort) {
+    if (effort === undefined)
+        return;
+    throw new Error(`effort (${JSON.stringify(effort)}) is not supported on the Codex adapter: ` +
+        `vigiles has not measured a mapping for it, so honouring the spec here ` +
+        `would record an effort the run did not use. Drop \`effort\` for this ` +
+        `harness, or run the eval on Claude Code.`);
+}
 /* v8 ignore start -- real codex subprocess; validated against the binary, not the unit gate */
 /**
  * The Codex eval-tier `AgentRunner`: install the run's skills into `.codex/skills`
@@ -180,6 +206,7 @@ function installCodexSkills(pluginDir, cwd) {
  * codexEvalDriver })` dispatches through.
  */
 function codexEvalAgentRunner(args) {
+    refuseCodexEffort(args.effort);
     if (args.pluginDir)
         installCodexSkills(args.pluginDir, args.cwd);
     return Promise.resolve(codexEvalRunner({

package/dist/eval-cache.d.ts

@@ -6,6 +6,13 @@ export type CacheMode = "off" | "read" | "readwrite";
 export interface CacheKeyInput {
     readonly task: string;
     readonly model: string;
+    /**
+     * Reasoning budget (`--effort`). Keyed for the same reason `model` is: it moves
+     * the output distribution, so a replay across effort levels would serve a result
+     * the caller did not ask for. `undefined` (the harness default) drops out of the
+     * hash via JSON, so entries recorded before effort existed stay valid.
+     */
+    readonly effort?: string | number;
     readonly tools: readonly string[];
     /** The resolved fixture + arm + plugin files written before the run. */
     readonly files: Record<string, string>;

package/dist/eval-lock.d.ts

@@ -33,6 +33,18 @@ export declare const DEFAULT_LOCK_DIR = ".vigiles/eval-locks";
 export interface EvalLockInputs {
     /** Model id used (folded in; a floating alias can't detect weight drift — warned). */
     readonly model: string;
+    /**
+     * Reasoning budget (`--effort`) the run was pinned to, or undefined for the
+     * harness default. Hashed because it steers the model — the criterion this
+     * interface already states — so a committed report recorded at one effort is
+     * STALE for a run at another. `undefined` is dropped by `JSON.stringify`, so
+     * locks committed before effort existed keep their hash and still replay.
+     *
+     * Caveat kept honest: "omitted" means the harness's own default, which is
+     * per-model and can move between builds — reproducible only modulo that, the
+     * same class of provenance caveat as `harnessVersion` below.
+     */
+    readonly effort?: string | number;
     /**
      * A hand-bumped behavior epoch the project owns (`.vigilesrc.json`
      * `eval.apiVersion`), bumped when a harness-side change YOU made (a CLAUDE.md
@@ -71,6 +83,13 @@ export interface EvalLock {
     readonly inputsHash: string;
     /** The model id the report was produced against (for the drift warning). */
     readonly model: string;
+    /**
+     * The effort the report was produced at, or undefined for the harness default
+     * (provenance; already in the hash). Recorded because the complaint that
+     * motivated effort support was not only that it could not be SET — it was that
+     * nothing in the run record said which effort produced the numbers.
+     */
+    readonly effort?: string | number;
     /** The harness version token at record time (provenance; already in the hash). */
     readonly harnessVersionKey: string;
     /** The behavior epoch at record time (provenance; already in the hash). */
@@ -142,6 +161,7 @@ export declare function buildLock(args: {
     readonly name: string;
     readonly inputsHash: string;
     readonly model: string;
+    readonly effort?: string | number;
     readonly harnessVersionKey: string;
     readonly evalApiVersion: number;
     readonly builtAt: string;

package/dist/eval.d.ts

@@ -46,6 +46,13 @@ export interface EvalArm {
      * use the eval-level model. See `research/eval-architecture.md` (model strategy).
      */
     readonly model?: string;
+    /**
+     * Reasoning budget for the run (`claude --effort`, e.g. `"low"` or an integer).
+     * Lives here beside `model` because it is part of the MEASUREMENT — it moves the
+     * output distribution, not the sample size — so it is hashed into the lock and
+     * the cache, and never read from an env var. Omit for the harness default.
+     */
+    readonly effort?: string | number;
 }
 /** Per-run resource use, parsed from the terminal `result` event (0 when absent). */
 export interface EvalUsage {
@@ -95,6 +102,13 @@ export interface EvalSpec<M extends Metrics> {
     readonly trials?: number;
     /** Model alias. Default "haiku". */
     readonly model?: string;
+    /**
+     * Reasoning budget for the run (`claude --effort`, e.g. `"low"` or an integer).
+     * Lives here beside `model` because it is part of the MEASUREMENT — it moves the
+     * output distribution, not the sample size — so it is hashed into the lock and
+     * the cache, and never read from an env var. Omit for the harness default.
+     */
+    readonly effort?: string | number;
     /** Tools the agent may use. Default: Read Edit Write Bash. */
     readonly allowedTools?: readonly string[];
     /** Per-run timeout ms. Default 240000. */
@@ -229,6 +243,19 @@ export interface AgentRunArgs {
     readonly task: string;
     readonly cwd: string;
     readonly model: string;
+    /**
+     * Reasoning-budget level for the run (`claude --effort`). Part of the
+     * MEASUREMENT, not a run knob: it changes the model's output distribution, not
+     * the sample size — so it lives on the spec next to `model` (never an env),
+     * and it is hashed into both the cache key and the eval lock. Deliberately
+     * `string | number` rather than a literal union: the binary accepts an alias
+     * map, is case-insensitive, and takes an integer budget, and its own valid set
+     * MOVED between builds (2.1.42 had no `xhigh`, 2.1.257 does) — a hard-coded
+     * union would reject a valid level after any upstream addition. A wrong value
+     * is caught at RUNTIME instead, by {@link effortRejection}, which is what the
+     * binary actually tells us. Omit for the harness default.
+     */
+    readonly effort?: string | number;
     readonly tools: readonly string[];
     readonly hasSettings: boolean;
     readonly pluginDir: string | undefined;
@@ -260,10 +287,74 @@ export type AgentRunner = (args: AgentRunArgs) => Promise<RunOut>;
  * regression to an always-merge would otherwise silently defeat ephemerality and
  * leak the host environment into an untrusted, model-driven run.
  */
-export declare function resolveSpawnEnv(a: Pick<AgentRunArgs, "env" | "replaceEnv">, base?: NodeJS.ProcessEnv): NodeJS.ProcessEnv;
-/** The real `claude`-spawning runner (composition root). Exported so other
- *  real-model entries (e.g. the `audit` trigger tier) bind the same runner. */
-export declare function spawnAgent(a: AgentRunArgs): Promise<RunOut>;
+export declare function resolveSpawnEnv(a: Pick<AgentRunArgs, "env" | "replaceEnv" | "effort">, base?: NodeJS.ProcessEnv): NodeJS.ProcessEnv;
+/**
+ * The env var name the harness reads for the reasoning budget. It sits ABOVE the
+ * `--effort` flag in the CLI's own precedence chain, so passing the flag alone
+ * does NOT pin the level.
+ */
+export declare const EFFORT_ENV_VAR = "CLAUDE_CODE_EFFORT_LEVEL";
+/**
+ * Pin the effort the run actually gets, so the recorded effort is the effort
+ * that ran.
+ *
+ * WHY THIS EXISTS AND WHY IT IS NOT OPTIONAL. Effort has THREE inputs — the
+ * `--effort` flag, the `effortLevel` settings key, and `CLAUDE_CODE_EFFORT_LEVEL`
+ * — and the env var wins over the flag. `EPHEMERAL_ALLOW_PREFIXES` passes
+ * `CLAUDE_*` through by design (the CLI reads several such knobs and dropping one
+ * is the failure mode), so an ambient `CLAUDE_CODE_EFFORT_LEVEL=max` in the
+ * author's shell survives even the SCRUBBED ephemeral env. Without this pin,
+ * hashing effort into the lock would make the lock CONFIDENTLY WRONG: it would
+ * record `low` over a run that executed at `max` — the exact defect the feature
+ * exists to prevent, reintroduced by the fix for it.
+ *
+ * Both directions matter, so both are handled:
+ *  - effort DECLARED  → set the var, overriding whatever the shell had.
+ *  - effort OMITTED   → DELETE an inherited var, so "omit" means the harness
+ *                       default rather than "whatever this machine happened to
+ *                       export". An omitted effort must not be a hidden input.
+ */
+export declare function pinEffortEnv(env: NodeJS.ProcessEnv, effort: string | number | undefined): NodeJS.ProcessEnv;
+/**
+ * The harness's own rejection of an `--effort` value, or null. Pure.
+ *
+ * The CLI does NOT fail on a bad level — it prints this to stderr and silently
+ * runs at its default. That silent substitution is precisely the bug class this
+ * feature addresses (a number produced by a configuration nobody asked for), so
+ * a rejected value must never become a sample. Matched on the binary's own
+ * wording, the same shape as {@link isRateLimited}.
+ */
+export declare function effortRejection(out: RunOut): string | null;
+/**
+ * Wrap a runner so a run the harness rejected on `--effort` FAILS LOUDLY.
+ *
+ * Applied ONCE, around the real runner, rather than as a guard repeated at each
+ * of the five `runner(...)` call sites — a guard per call site is the shape that
+ * left four of five compilers unprotected in #173.
+ *
+ * It THROWS rather than counting the trial as `runError`. A `runError` trial is
+ * dropped from the denominator, which is right for a transient (a rate limit) and
+ * wrong here: an unusable effort value is deterministic and repeatable, so every
+ * trial fails it and the run would report a rate computed over ZERO samples. A
+ * configuration mistake should stop the run and name itself.
+ */
+export declare function withEffortGuard(runner: AgentRunner): AgentRunner;
+/**
+ * Build the real runner's argv. Pure and exported so the FLAGS are provable —
+ * `spawnAgentRaw` is `v8 ignore`d (it spawns a subprocess), so an argv assembled
+ * inline there could not be asserted at all. Mirrors `buildCodexArgs`.
+ */
+export declare function buildAgentArgs(a: AgentRunArgs): string[];
+/**
+ * The real `claude`-spawning runner (composition root). Exported so other
+ * real-model entries (e.g. the `audit` trigger tier) bind the same runner.
+ *
+ * The effort guard is composed in HERE, at the single definition, rather than at
+ * each of the places that bind this runner — so every consumer, including ones
+ * not yet written, is covered by construction. Guarding each call site instead is
+ * the shape that left four of five compilers unprotected in #173.
+ */
+export declare const spawnAgent: AgentRunner;
 /**
  * Run the eval: every arm × every trial against the real `claude` CLI, with the
  * metric computed per run and aggregated per arm. Requires `claude` on PATH and
@@ -320,6 +411,13 @@ export interface MeasureSpec {
     readonly trials?: number;
     /** Model alias. Default "sonnet" — measure on the model your users run. */
     readonly model?: string;
+    /**
+     * Reasoning budget for the run (`claude --effort`, e.g. `"low"` or an integer).
+     * Lives here beside `model` because it is part of the MEASUREMENT — it moves the
+     * output distribution, not the sample size — so it is hashed into the lock and
+     * the cache, and never read from an env var. Omit for the harness default.
+     */
+    readonly effort?: string | number;
     /** Tools the agent may use. */
     readonly allowedTools?: readonly string[];
     /** Per-run timeout ms. */
@@ -375,6 +473,13 @@ export interface ArmsMeasureSpec {
     readonly model?: string;
     readonly allowedTools?: readonly string[];
     readonly timeoutMs?: number;
+    /**
+     * Reasoning budget for the run (`claude --effort`, e.g. `"low"` or an integer).
+     * Lives here beside `model` because it is part of the MEASUREMENT — it moves the
+     * output distribution, not the sample size — so it is hashed into the lock and
+     * the cache, and never read from an env var. Omit for the harness default.
+     */
+    readonly effort?: string | number;
     readonly spacingSec?: number;
 }
 /** Per-arm {@link CheckReport}s — `arms[name].perCheck[i]` aligns across arms. */
@@ -471,6 +576,7 @@ export declare function runSkillSelectionTrial(args: {
     readonly runner: AgentRunner;
     readonly parse?: ModelOutputParser;
     readonly model: string;
+    readonly effort?: string | number;
     readonly tools?: readonly string[];
     readonly timeoutMs?: number;
     readonly fixture?: Record<string, string>;
@@ -658,6 +764,13 @@ export interface TriggerRateSpec {
      * 0.50 on haiku vs 0.90 on Sonnet). Override for a cheaper-but-pessimistic run.
      */
     readonly model?: string;
+    /**
+     * Reasoning budget for the run (`claude --effort`, e.g. `"low"` or an integer).
+     * Lives here beside `model` because it is part of the MEASUREMENT — it moves the
+     * output distribution, not the sample size — so it is hashed into the lock and
+     * the cache, and never read from an env var. Omit for the harness default.
+     */
+    readonly effort?: string | number;
     /**
      * Minimum model tier this eval may run on (haiku<sonnet<opus by family). The
      * run **fails** if the resolved `model` is weaker — trigger-rate under-measures

package/dist/eval.js

@@ -1,8 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.claudeEvalDriver = exports.EPHEMERAL_HOME_KEEP = void 0;
+exports.claudeEvalDriver = exports.EPHEMERAL_HOME_KEEP = exports.spawnAgent = exports.EFFORT_ENV_VAR = void 0;
 exports.resolveSpawnEnv = resolveSpawnEnv;
-exports.spawnAgent = spawnAgent;
+exports.pinEffortEnv = pinEffortEnv;
+exports.effortRejection = effortRejection;
+exports.withEffortGuard = withEffortGuard;
+exports.buildAgentArgs = buildAgentArgs;
 exports.unregisteredSkillFiles = unregisteredSkillFiles;
 exports.runEval = runEval;
 exports.measureWith = measureWith;
@@ -99,35 +102,136 @@ function writeFiles(cwd, files) {
  * leak the host environment into an untrusted, model-driven run.
  */
 function resolveSpawnEnv(a, base = process.env) {
-    return a.replaceEnv ? (a.env ?? {}) : { ...base, ...a.env };
+    const resolved = a.replaceEnv ? (a.env ?? {}) : { ...base, ...a.env };
+    return pinEffortEnv(resolved, a.effort);
 }
+/**
+ * The env var name the harness reads for the reasoning budget. It sits ABOVE the
+ * `--effort` flag in the CLI's own precedence chain, so passing the flag alone
+ * does NOT pin the level.
+ */
+exports.EFFORT_ENV_VAR = "CLAUDE_CODE_EFFORT_LEVEL";
+/**
+ * Pin the effort the run actually gets, so the recorded effort is the effort
+ * that ran.
+ *
+ * WHY THIS EXISTS AND WHY IT IS NOT OPTIONAL. Effort has THREE inputs — the
+ * `--effort` flag, the `effortLevel` settings key, and `CLAUDE_CODE_EFFORT_LEVEL`
+ * — and the env var wins over the flag. `EPHEMERAL_ALLOW_PREFIXES` passes
+ * `CLAUDE_*` through by design (the CLI reads several such knobs and dropping one
+ * is the failure mode), so an ambient `CLAUDE_CODE_EFFORT_LEVEL=max` in the
+ * author's shell survives even the SCRUBBED ephemeral env. Without this pin,
+ * hashing effort into the lock would make the lock CONFIDENTLY WRONG: it would
+ * record `low` over a run that executed at `max` — the exact defect the feature
+ * exists to prevent, reintroduced by the fix for it.
+ *
+ * Both directions matter, so both are handled:
+ *  - effort DECLARED  → set the var, overriding whatever the shell had.
+ *  - effort OMITTED   → DELETE an inherited var, so "omit" means the harness
+ *                       default rather than "whatever this machine happened to
+ *                       export". An omitted effort must not be a hidden input.
+ */
+function pinEffortEnv(env, effort) {
+    // Rebuilt WITHOUT the key rather than deleting or assigning `undefined`:
+    // omission has to be provable here, and whether a spawn drops an
+    // `undefined`-valued env entry is a Node-version detail we should not lean on.
+    const { [exports.EFFORT_ENV_VAR]: _inherited, ...rest } = env;
+    return effort === undefined
+        ? rest
+        : { ...rest, [exports.EFFORT_ENV_VAR]: String(effort) };
+}
+/**
+ * The harness's own rejection of an `--effort` value, or null. Pure.
+ *
+ * The CLI does NOT fail on a bad level — it prints this to stderr and silently
+ * runs at its default. That silent substitution is precisely the bug class this
+ * feature addresses (a number produced by a configuration nobody asked for), so
+ * a rejected value must never become a sample. Matched on the binary's own
+ * wording, the same shape as {@link isRateLimited}.
+ */
+function effortRejection(out) {
+    const text = `${out.stderr ?? ""}\n${out.stdout}`;
+    const m = /Unknown --effort value[^\n]*/.exec(text);
+    return m ? m[0].trim() : null;
+}
+/**
+ * Wrap a runner so a run the harness rejected on `--effort` FAILS LOUDLY.
+ *
+ * Applied ONCE, around the real runner, rather than as a guard repeated at each
+ * of the five `runner(...)` call sites — a guard per call site is the shape that
+ * left four of five compilers unprotected in #173.
+ *
+ * It THROWS rather than counting the trial as `runError`. A `runError` trial is
+ * dropped from the denominator, which is right for a transient (a rate limit) and
+ * wrong here: an unusable effort value is deterministic and repeatable, so every
+ * trial fails it and the run would report a rate computed over ZERO samples. A
+ * configuration mistake should stop the run and name itself.
+ */
+function withEffortGuard(runner) {
+    // 🔴 DELIBERATELY NOT `async`. An `async` wrapper turns the wrapped runner's
+    // SYNCHRONOUS throws into rejected promises, and the real runner refuses
+    // synchronously on purpose — `refuseDuringEvalLoad` / `refuseUnderForeignRunner`
+    // stop a paid eval from billing when a foreign test runner collects it. Making
+    // this `async` silently downgraded those refusals from "throws at the call" to
+    // "returns a promise that rejects", which `assert.throws` cannot see and an
+    // un-awaited caller would not notice. Caught by the full suite, not by the
+    // targeted one; pinned below by `withEffortGuard preserves a SYNCHRONOUS throw`.
+    return (a) => {
+        const pending = runner(a);
+        return pending.then((out) => {
+            const rejection = effortRejection(out);
+            if (rejection !== null) {
+                throw new Error(`the harness rejected effort ${JSON.stringify(a.effort)}: ${rejection}`);
+            }
+            return out;
+        });
+    };
+}
+/**
+ * Build the real runner's argv. Pure and exported so the FLAGS are provable —
+ * `spawnAgentRaw` is `v8 ignore`d (it spawns a subprocess), so an argv assembled
+ * inline there could not be asserted at all. Mirrors `buildCodexArgs`.
+ */
+function buildAgentArgs(a) {
+    return [
+        "-p",
+        a.task,
+        // stream-json (+ --verbose, required with -p) so the per-turn tool_use
+        // events survive into `ctx.toolCalls` — the unified Trace, same as the
+        // harness tier. The terminal `result` event still carries num_turns/output.
+        "--output-format",
+        "stream-json",
+        "--verbose",
+        "--model",
+        a.model,
+        ...(a.effort !== undefined ? ["--effort", String(a.effort)] : []),
+        "--permission-mode",
+        "acceptEdits",
+        ...(a.pluginDir !== undefined
+            ? ["--plugin-dir", (0, node_path_1.resolve)(a.pluginDir)]
+            : []),
+        ...(a.hasSettings ? ["--settings", "settings.json"] : []),
+        "--allowedTools",
+        ...a.tools,
+    ];
+}
+/**
+ * The real `claude`-spawning runner (composition root). Exported so other
+ * real-model entries (e.g. the `audit` trigger tier) bind the same runner.
+ *
+ * The effort guard is composed in HERE, at the single definition, rather than at
+ * each of the places that bind this runner — so every consumer, including ones
+ * not yet written, is covered by construction. Guarding each call site instead is
+ * the shape that left four of five compilers unprotected in #173.
+ */
+exports.spawnAgent = withEffortGuard(spawnAgentRaw);
 /* v8 ignore start -- real claude subprocess; exercised by bench/, not the unit gate */
-/** The real `claude`-spawning runner (composition root). Exported so other
- *  real-model entries (e.g. the `audit` trigger tier) bind the same runner. */
-function spawnAgent(a) {
+/** The unguarded spawn itself; wrapped by {@link spawnAgent}, never bound raw. */
+function spawnAgentRaw(a) {
     (0, eval_load_phase_js_1.refuseDuringEvalLoad)("spawning `claude`");
     (0, foreign_runner_js_1.refuseUnderForeignRunner)("spawning `claude`");
     return new Promise((resolvePromise) => {
-        const args = [
-            "-p",
-            a.task,
-            // stream-json (+ --verbose, required with -p) so the per-turn tool_use
-            // events survive into `ctx.toolCalls` — the unified Trace, same as the
-            // harness tier. The terminal `result` event still carries num_turns/output.
-            "--output-format",
-            "stream-json",
-            "--verbose",
-            "--model",
-            a.model,
-            "--permission-mode",
-            "acceptEdits",
-            ...(a.pluginDir !== undefined
-                ? ["--plugin-dir", (0, node_path_1.resolve)(a.pluginDir)]
-                : []),
-            ...(a.hasSettings ? ["--settings", "settings.json"] : []),
-            "--allowedTools",
-            ...a.tools,
-        ];
+        const args = buildAgentArgs(a);
         const child = (0, node_child_process_1.spawn)(runtime_js_1.claudeCodeRuntime.agentBinary, args, {
             cwd: a.cwd,
             // The security-critical env resolution (overlay vs. scrubbed replacement)
@@ -194,7 +298,7 @@ function warnUnregisteredSkillArms(arms) {
 }
 async function runEval(spec) {
     warnUnregisteredSkillArms(spec.arms);
-    const report = await runEvalWith(spec, spawnAgent);
+    const report = await runEvalWith(spec, exports.spawnAgent);
     // Surface what the run spent — tokens + API-equivalent $, and a LOUD warning if
     // it was billed to a metered API key instead of the subscription. See eval-cost.ts.
     (0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.costFromEvalReport)(report));
@@ -237,6 +341,7 @@ async function measureWith(spec, runner) {
             task: spec.task,
             trials: spec.trials ?? 5,
             model: spec.model ?? "sonnet",
+            effort: spec.effort,
             allowedTools: spec.allowedTools,
             timeoutMs: spec.timeoutMs,
             spacingSec: spec.spacingSec,
@@ -266,7 +371,7 @@ async function measureWith(spec, runner) {
 /* v8 ignore start -- real claude subprocess; thin wrapper over measureWith */
 /** Score a check vocabulary across trials against the real `claude` CLI. */
 async function measure(spec) {
-    const report = await measureWith(spec, spawnAgent);
+    const report = await measureWith(spec, exports.spawnAgent);
     (0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.costFromArm)(report.usage));
     return report;
 }
@@ -283,6 +388,7 @@ async function measureArmsWith(spec, runner) {
             task: spec.task,
             trials: spec.trials ?? 5,
             model: spec.model ?? "sonnet",
+            effort: spec.effort,
             allowedTools: spec.allowedTools,
             timeoutMs: spec.timeoutMs,
             spacingSec: spec.spacingSec,
@@ -337,7 +443,7 @@ function stubArmPluginDirs(arms) {
 /** Score checks across arms against the real `claude` CLI. */
 async function measureArms(spec) {
     warnUnregisteredSkillArms(spec.arms);
-    const report = await measureArmsWith(spec, spawnAgent);
+    const report = await measureArmsWith(spec, exports.spawnAgent);
     // Sum every arm's spend — an A/B run pays for both arms.
     (0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.sumCosts)(Object.values(report.arms).map((a) => (0, eval_cost_js_1.costFromArm)(a.usage))));
     return report;
@@ -552,6 +658,7 @@ async function runSkillSelectionTrial(args) {
             task: args.prompt,
             cwd,
             model: args.model,
+            effort: args.effort,
             tools: args.tools ?? ["Read", "Edit", "Write", "Bash", "Skill"],
             hasSettings: false,
             pluginDir: args.pluginDir,
@@ -637,6 +744,7 @@ async function runWithCache(runArgs, keyParts, runner, cfg) {
     const key = (0, eval_cache_js_1.cacheKey)({
         task: runArgs.task,
         model: runArgs.model,
+        effort: runArgs.effort,
         tools: runArgs.tools,
         files: keyParts.files,
         settings: keyParts.settings,
@@ -967,6 +1075,7 @@ async function executeTrial(spec, arm, trialIndex, runner, cfg) {
             cwd,
             // A model comparison is a harness A/B: an arm may override the model.
             model: arm.model ?? cfg.model,
+            effort: arm.effort ?? cfg.effort,
             tools: cfg.tools,
             hasSettings,
             pluginDir: arm.pluginDir,
@@ -1100,8 +1209,16 @@ async function withEvalLock(args, produce) {
     }
     if (!isDatedModel(args.model))
         warnFloatingModel(args.model);
+    // OVERLAP, DELIBERATE — do not delete this as dead. Effort reaches the hash
+    // twice: here (the CHOKEPOINT every seam passes through, so a future seam that
+    // forgets to fold effort into its own `inputs` is still covered) and inside
+    // each seam's `inputs` (which alone can see a PER-ARM override this line
+    // cannot). Measured 2026-09-01: removing either one alone leaves the suite
+    // green; removing BOTH fails `changing effort makes a committed lock STALE`.
+    // That is two populations covered, not one line duplicated.
     const inputsHash = (0, eval_lock_js_1.evalInputsHash)({
         model: args.model,
+        effort: args.effort,
         evalApiVersion: lock.evalApiVersion,
         inputs: args.inputs,
     });
@@ -1116,6 +1233,7 @@ async function withEvalLock(args, produce) {
         name: args.name,
         inputsHash,
         model: args.model,
+        effort: args.effort,
         harnessVersionKey: harnessVersion(),
         evalApiVersion: lock.evalApiVersion,
         builtAt: new Date().toISOString(),
@@ -1166,6 +1284,10 @@ function evalArmsInputs(spec, cfg) {
         const absRoot = arm.plugin ? (0, node_path_1.resolve)(process.cwd(), arm.plugin) : "";
         arms[name] = {
             model: arm.model ?? cfg.model,
+            // The per-ARM half of the overlap documented at `inputsHash` — the
+            // chokepoint sees only the eval-level effort, so an arm that overrides it
+            // would otherwise hash identically to its sibling.
+            effort: arm.effort ?? cfg.effort,
             tools: [...cfg.tools].sort(),
             files: stripPluginRoot(resolved.files, absRoot),
             settings: stripPluginRoot(resolved.settings, absRoot),
@@ -1203,6 +1325,7 @@ async function runEvalWith(spec, runner) {
     const backoffMs = spec.retryBackoffMs ?? 1000;
     const cfg = {
         model: spec.model ?? "haiku",
+        effort: spec.effort,
         tools: spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"],
         timeoutMs: spec.timeoutMs ?? 240000,
         cache: spec.cache ?? "off",
@@ -1232,7 +1355,7 @@ async function runEvalWith(spec, runner) {
     // resolveHarness/hashDir work). `check` replays the committed report below
     // without ever entering the run pool — so no model is driven in CI.
     const inputs = lock.mode === "off" ? undefined : evalArmsInputs(spec, cfg);
-    return withEvalLock({ name: spec.name, inputs, model: cfg.model, lock }, async () => {
+    return withEvalLock({ name: spec.name, inputs, model: cfg.model, effort: cfg.effort, lock }, async () => {
         const results = await runPool(units, concurrency, worker);
         const { arms, totalCostUsd } = aggregateArms(Object.keys(spec.arms), results);
         return { name: spec.name ?? "eval", trials, arms, totalCostUsd, aborted };
@@ -1285,7 +1408,7 @@ function formatEvalReport(report) {
  * The asymmetry reflects default-vs-injected, not a hexagonal violation.
  */
 exports.claudeEvalDriver = {
-    runner: spawnAgent,
+    runner: exports.spawnAgent,
     parse: parseClaudeRun,
     harness: "claude-code",
 };
@@ -1591,6 +1714,7 @@ async function runTriggerTrial(prompt, cfg, runner) {
             task: prompt,
             cwd,
             model: cfg.model,
+            effort: cfg.effort,
             tools: cfg.tools,
             hasSettings: false,
             pluginDir: cfg.pluginDir,
@@ -1690,6 +1814,7 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
         // Sonnet, not haiku: trigger-rate is a selection measurement and haiku
         // under-selects, producing false-negative recall (see TriggerRateSpec.model).
         model,
+        effort: spec.effort,
         tools: spec.allowedTools ?? ["Read", "Edit", "Write", "Bash", "Skill"],
         timeoutMs: spec.timeoutMs ?? 240000,
         spacing: (spec.spacingSec ?? 4) * 1000,
@@ -1715,6 +1840,7 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
                     ? [...spec.irrelevantPrompts]
                     : undefined,
                 model: cfg.model,
+                effort: cfg.effort,
                 tools: [...cfg.tools].sort(),
                 fixture: spec.fixture,
                 competitors,
@@ -1723,7 +1849,13 @@ async function measureTriggerRateWith(spec, runner, parse = parseClaudeRun, runE
                 // STALE if the eval is switched to another harness.
                 harness,
             };
-        return await withEvalLock({ name: spec.name, inputs: triggerInputs, model: cfg.model, lock }, async () => {
+        return await withEvalLock({
+            name: spec.name,
+            inputs: triggerInputs,
+            model: cfg.model,
+            effort: cfg.effort,
+            lock,
+        }, async () => {
             const relevant = await runTriggerSet(spec.prompts, cfg, runner);
             const base = {
                 rate: relevant.n > 0 ? relevant.fired / relevant.n : 0,

package/dist/scan-behavioral.d.ts

@@ -89,6 +89,16 @@ export interface SelectionOptions {
     readonly trials?: number;
     /** Selector model — defaults to Sonnet (a weaker model under-selects). */
     readonly model?: string;
+    /**
+     * Reasoning budget (`claude --effort`) for the selector, or undefined for the
+     * harness default. Present for {@link measureSelectionMatrix}, the ASSERTABLE
+     * test primitive, where the configuration a number came from has to be pinnable.
+     *
+     * Deliberately NOT exposed as an `audit` CLI flag: `audit` is a local report,
+     * not a reproducibility surface, and a flag nobody can act on is surface without
+     * a use. The audit probe therefore leaves this unset and runs at the default.
+     */
+    readonly effort?: string | number;
     /** Parallel runs across the prompts × trials grid (default 1). */
     readonly concurrency?: number;
     /** Which harness drives it (default `"claude-code"`; others report n/a). */

package/dist/scan-behavioral.js

@@ -338,6 +338,7 @@ async function measurePluginSelectionWith(dir, promptSet, probe, opts = {}) {
             parse: d.parse,
             runError: d.runError,
             model: opts.model ?? "sonnet",
+            effort: opts.effort,
         }));
         const runs = [];
         jobs.forEach((job, k) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vigiles",
-  "version": "25.0.0",
+  "version": "25.1.0",
   "description": "Audit, test and measure the harness your AI agent runs on — grade your CLAUDE.md / AGENTS.md, skills, subagents and hooks, run them against a scripted model, and measure whether they actually fire.",
   "keywords": [
     "claude-code",