vigiles 4.0.2 → 5.0.0

package/dist/eval.js

@@ -1,5 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.EPHEMERAL_HOME_KEEP = void 0;
+exports.resolveSpawnEnv = resolveSpawnEnv;
 exports.runEval = runEval;
 exports.measureWith = measureWith;
 exports.measure = measure;
@@ -13,6 +15,12 @@ exports.parseUsage = parseUsage;
 exports.aggregate = aggregate;
 exports.aggregateStats = aggregateStats;
 exports.aggregateUsage = aggregateUsage;
+exports.isDatedModel = isDatedModel;
+exports.modelTier = modelTier;
+exports.belowModelFloor = belowModelFloor;
+exports.harnessVersionKey = harnessVersionKey;
+exports.ephemeralRunEnv = ephemeralRunEnv;
+exports.seedEphemeralHome = seedEphemeralHome;
 exports.isRateLimited = isRateLimited;
 exports.runPool = runPool;
 exports.runEvalWith = runEvalWith;
@@ -23,6 +31,7 @@ exports.stubbedPluginDir = stubbedPluginDir;
 exports.promptDistance = promptDistance;
 exports.checkPromptDiversity = checkPromptDiversity;
 exports.assertPromptDiversity = assertPromptDiversity;
+exports.packageInstallSet = packageInstallSet;
 exports.measureTriggerRateWith = measureTriggerRateWith;
 exports.measureTriggerRate = measureTriggerRate;
 exports.formatTriggerRateReport = formatTriggerRateReport;
@@ -59,6 +68,8 @@ const proofs_js_1 = require("./core/proofs.js");
 const harness_test_js_1 = require("./harness-test.js");
 const eval_cache_js_1 = require("./eval-cache.js");
 const stats_js_1 = require("./stats.js");
+const tool_intercept_js_1 = require("./tool-intercept.js");
+const tool_stub_js_1 = require("./tool-stub.js");
 function writeFiles(cwd, files) {
     for (const [p, content] of Object.entries(files)) {
         const full = (0, node_path_1.resolve)(cwd, p);
@@ -66,6 +77,19 @@ function writeFiles(cwd, files) {
         (0, node_fs_1.writeFileSync)(full, content);
     }
 }
+/**
+ * Resolve the environment a trial's subprocess actually runs with — the
+ * SECURITY-CRITICAL decision behind `ephemeralEnv`. When `replaceEnv` is set, the
+ * scrubbed `env` is the COMPLETE environment, so the real `$HOME` and inherited
+ * secrets are DROPPED; otherwise `env` is an overlay on `base` (byte-identical to
+ * the pre-ephemeral behaviour). Extracted from the `v8 ignore`d `spawnAgent` so
+ * the one line that enforces the scrub is both unit- and behaviourally-tested — a
+ * regression to an always-merge would otherwise silently defeat ephemerality and
+ * leak the host environment into an untrusted, model-driven run.
+ */
+function resolveSpawnEnv(a, base = process.env) {
+    return a.replaceEnv ? (a.env ?? {}) : { ...base, ...a.env };
+}
 /* v8 ignore start -- real claude subprocess; exercised by bench/, not the unit gate */
 function spawnAgent(a) {
     return new Promise((resolvePromise) => {
@@ -91,7 +115,9 @@ function spawnAgent(a) {
         ];
         const child = (0, node_child_process_1.spawn)(runtime_js_1.claudeCodeRuntime.agentBinary, args, {
             cwd: a.cwd,
-            env: process.env,
+            // The security-critical env resolution (overlay vs. scrubbed replacement)
+            // lives in the tested `resolveSpawnEnv` seam above, not inline here.
+            env: resolveSpawnEnv(a),
             stdio: ["ignore", "pipe", "pipe"],
         });
         let stdout = "";
@@ -124,6 +150,14 @@ async function runEval(spec) {
 async function measureWith(spec, runner) {
     if (spec.stubSkillBodies && !spec.pluginDir)
         throw new Error("measure: `stubSkillBodies` requires `pluginDir`.");
+    // stubSkillBodies replaces each skill BODY with a no-op (the run stops at
+    // selection), so there is no output to grade — a `judged` check would score an
+    // empty body and mislead. The docs warn against this pairing; enforce it.
+    if (spec.stubSkillBodies &&
+        spec.checks.some((c) => c.toJSON().kind === "judged"))
+        throw new Error("measure: `stubSkillBodies` is for firing/`skill()` checks only — it stubs " +
+            "the skill body, so there's no output for a `judged` check to grade. Drop " +
+            "`stubSkillBodies`, or remove the `judged` check.");
     const stubbed = spec.stubSkillBodies
         ? stubbedPluginDir(spec.pluginDir)
         : undefined;
@@ -137,6 +171,7 @@ async function measureWith(spec, runner) {
                     settings: spec.settings,
                     plugin: spec.plugin,
                     pluginDir,
+                    interceptTools: spec.interceptTools,
                 },
             },
             task: spec.task,
@@ -273,16 +308,37 @@ function formatCheckReport(report) {
     return lines.join("\n");
 }
 /**
- * The scored gate (Phase 4): throw if any check's measured rate is below `min` —
- * the `measure` counterpart to `assertChecks` (strict). Reads the rate, not a
- * single run, so it never trips on one noisy trial.
+ * The min rate a check must clear: its per-KIND override in `per`, else the
+ * default `min`. Shared by `assertRates` (the throwing gate) and
+ * `checkReportToJUnit` (the XML report) so the two never diverge on thresholds.
+ */
+function checkRateThreshold(check, min, per) {
+    return per?.[check.kind] ?? min;
+}
+/**
+ * The scored gate (Phase 4): throw if any check's measured rate is below its
+ * threshold — the `measure` counterpart to `assertChecks` (strict). Reads the
+ * rate, not a single run, so it never trips on one noisy trial.
+ *
+ * `min` is the default threshold for every check. `per` overrides it for a check
+ * KIND (e.g. `{ min: 0.8, per: { skill: 1.0 } }` — "every check ≥ 80%, but the
+ * skill must FIRE on every trial"), so a strict firing/safety check and a
+ * lenient quality check gate in one call — the single-skill absolute oracle
+ * (`measure({ checks: [skill(), judged()] }) + assertRates`) needs exactly this.
  */
 function assertRates(report, opts) {
-    const below = report.perCheck.filter((c) => c.rate < opts.min);
+    // An empty report gates nothing — a green that tested NOTHING (the silent-pass
+    // the no-silent-skips rule forbids). Fail loudly instead of vacuously passing.
+    if (report.perCheck.length === 0) {
+        throw new Error("assertRates: the report has no checks to gate — did you call " +
+            "`measure({ checks: [...] })` with an empty list? An empty gate is a silent pass.");
+    }
+    const thresholdFor = (c) => checkRateThreshold(c.check, opts.min, opts.per);
+    const below = report.perCheck.filter((c) => c.rate < thresholdFor(c));
     if (below.length > 0) {
-        throw new Error(`${String(below.length)} check(s) below the ${(opts.min * 100).toFixed(0)}% min rate:\n` +
+        throw new Error(`${String(below.length)} check(s) below their min rate:\n` +
             below
-                .map((c) => `  ✗ ${checkLabel(c.check)}: ${(c.rate * 100).toFixed(0)}% ± ${(c.se * 100).toFixed(0)}%`)
+                .map((c) => `  ✗ ${checkLabel(c.check)}: ${(c.rate * 100).toFixed(0)}% ± ${(c.se * 100).toFixed(0)}% (min ${(thresholdFor(c) * 100).toFixed(0)}%)`)
                 .join("\n"));
     }
 }
@@ -295,18 +351,21 @@ function escapeXml(s) {
 }
 /**
  * Serialize a {@link CheckReport} to JUnit XML (Phase 4) — each check a
- * `<testcase>`, failing when its rate is below `min`. Because a check is *data*,
- * this falls out for free: CI test reporters, regression baselines, and a
- * promptfoo bridge all consume the same shape.
+ * `<testcase>`, failing when its rate is below its threshold. `min` is the
+ * default; `per` overrides it by check KIND, matching `assertRates` exactly (one
+ * shared threshold helper) so the gate and the report can never disagree about
+ * which checks failed. Because a check is *data*, this falls out for free: CI
+ * test reporters, regression baselines, and a promptfoo bridge all consume it.
  */
 function checkReportToJUnit(report, opts = {}) {
     const min = opts.min ?? 0;
-    const failures = report.perCheck.filter((c) => c.rate < min).length;
+    const thr = (c) => checkRateThreshold(c.check, min, opts.per);
+    const failures = report.perCheck.filter((c) => c.rate < thr(c)).length;
     const cases = report.perCheck
         .map((c) => {
         const name = escapeXml(checkLabel(c.check));
-        const body = c.rate < min
-            ? `\n    <failure message="rate ${(c.rate * 100).toFixed(0)}% below min ${(min * 100).toFixed(0)}% (n=${String(c.n)})"/>\n  `
+        const body = c.rate < thr(c)
+            ? `\n    <failure message="rate ${(c.rate * 100).toFixed(0)}% below min ${(thr(c) * 100).toFixed(0)}% (n=${String(c.n)})"/>\n  `
             : "";
         return `  <testcase classname="vigiles.checks" name="${name}">${body}</testcase>`;
     })
@@ -325,6 +384,8 @@ function usageFrom(result) {
         durationMs: num(result?.duration_ms),
         inputTokens: num(usage.input_tokens),
         outputTokens: num(usage.output_tokens),
+        cacheCreationTokens: num(usage.cache_creation_input_tokens),
+        cacheReadTokens: num(usage.cache_read_input_tokens),
     };
 }
 /** Parse per-run cost/latency/tokens from a stream — pure, model-free. */
@@ -424,6 +485,8 @@ function aggregateUsage(usages) {
         meanDurationMs: n > 0 ? sum((u) => u.durationMs) / n : 0,
         totalInputTokens: sum((u) => u.inputTokens),
         totalOutputTokens: sum((u) => u.outputTokens),
+        totalCacheCreationTokens: sum((u) => u.cacheCreationTokens),
+        totalCacheReadTokens: sum((u) => u.cacheReadTokens),
     };
 }
 /**
@@ -441,6 +504,13 @@ async function runWithCache(runArgs, keyParts, runner, cfg) {
         tools: runArgs.tools,
         files: keyParts.files,
         settings: keyParts.settings,
+        env: runArgs.env,
+        // A native --plugin-dir install isn't in `files`, so hash its CONTENTS into
+        // the key — otherwise editing a skill in it would false-replay.
+        pluginDirHash: runArgs.pluginDir ? (0, eval_cache_js_1.hashDir)(runArgs.pluginDir) : undefined,
+        // The harness binary evolves fast; a CLI upgrade must invalidate (stale
+        // replay otherwise serves a result from a different system prompt).
+        harnessVersion: harnessVersion(),
         trialIndex: keyParts.trialIndex,
     });
     const hit = (0, eval_cache_js_1.readCache)(cfg.cacheDir, key);
@@ -454,15 +524,313 @@ async function runWithCache(runArgs, keyParts, runner, cfg) {
     }
     return out;
 }
+/**
+ * The `vigiles intercept-tool-hook` command, as an absolute `node <cli> …`
+ * invocation — the eval runs in a throwaway cwd where `npx vigiles` wouldn't
+ * resolve, so the auto-wired PreToolUse hook must point at this CLI's own `cli.js`
+ * (resolved from `__dirname`, the same way `run-hook.ts`/`sandbox.ts` locate their
+ * entries).
+ */
+const INTERCEPT_TOOL_HOOK_CLI = [(0, node_path_1.join)(__dirname, "cli.js"), (0, node_path_1.join)(__dirname, "..", "dist", "cli.js")].find((p) => (0, node_fs_1.existsSync)(p)) ?? (0, node_path_1.join)(__dirname, "cli.js");
+const INTERCEPT_TOOL_HOOK_CMD = `"${process.execPath}" "${INTERCEPT_TOOL_HOOK_CLI}" intercept-tool-hook`;
+function isRecord(v) {
+    return v !== null && typeof v === "object";
+}
+/**
+ * A model id is "dated" (honestly pinned) when it ends in an 8-digit date stamp,
+ * e.g. `claude-haiku-4-5-20251001`. A floating alias (`haiku`, `sonnet`, or even
+ * `claude-sonnet-4-6` with no date) can change underneath you — so a cached or
+ * baselined result pinned to it can silently hide model drift. See
+ * `docs/eval-architecture.md` (honest model pinning).
+ */
+function isDatedModel(model) {
+    return /\d{8}$/.test(model);
+}
+/**
+ * Capability tier of a model by FAMILY: haiku=1 < sonnet=2 < opus=3 (version is
+ * ignored, so `claude-sonnet-4-6` and a dated Sonnet rank equal). An unrecognized
+ * family returns `null` — unrankable, so the floor never blocks a model we can't
+ * judge (fail-open on ranking). Used by the model floor; aliases and full/dated
+ * ids both work.
+ */
+function modelTier(id) {
+    const s = id.toLowerCase();
+    if (s.includes("haiku"))
+        return 1;
+    if (s.includes("sonnet"))
+        return 2;
+    if (s.includes("opus"))
+        return 3;
+    return null;
+}
+/**
+ * Is `model` a weaker tier than `floor`? Both must be rankable (see
+ * {@link modelTier}); an unrankable model/floor is never "below" (fail-open).
+ */
+function belowModelFloor(model, floor) {
+    const m = modelTier(model);
+    const f = modelTier(floor);
+    return m !== null && f !== null && m < f;
+}
+/**
+ * Reduce a raw `--version` string to the **major.minor** cache-key token. We key
+ * the cache on major.minor, NOT the patch: a patch release rarely changes agent
+ * behaviour, so keying patches would churn the cache on every release for no
+ * signal; a minor/major bump is where the system prompt / tool defs actually move.
+ * (If a specific patch is known to matter, clear the cache or bump
+ * `CACHE_FORMAT_VERSION`.) Falls back to the trimmed raw string when no semver is
+ * found. Pure + tested.
+ */
+function harnessVersionKey(raw) {
+    const m = /(\d+)\.(\d+)\.\d+/.exec(raw);
+    return m ? `${m[1]}.${m[2]}` : raw.trim();
+}
+/* v8 ignore start -- spawns the real harness binary; memoized, cache-path only */
+let cachedHarnessVersion;
+/**
+ * The harness binary version (`claude --version`) reduced to major.minor, for the
+ * cache key — so a CLI **minor/major** upgrade (new system prompt / tool defs)
+ * invalidates a stale replay, while patches don't churn it. Memoized (one spawn
+ * per process), resolved only on the cache path, "unknown" if the binary isn't
+ * found (then it doesn't partition the key).
+ */
+function harnessVersion() {
+    if (cachedHarnessVersion === undefined) {
+        try {
+            cachedHarnessVersion = harnessVersionKey((0, node_child_process_1.execSync)(`${runtime_js_1.claudeCodeRuntime.agentBinary} --version`, {
+                encoding: "utf-8",
+                stdio: ["ignore", "pipe", "ignore"],
+            }));
+        }
+        catch {
+            cachedHarnessVersion = "unknown";
+        }
+    }
+    return cachedHarnessVersion;
+}
+/* v8 ignore stop */
+/** Warn (once per run) that replaying/recording a cache on a floating alias hides drift. */
+function warnFloatingModel(model) {
+    const msg = `vigiles: eval cache is on but the model "${model}" is a floating alias — ` +
+        `a replay can serve a result computed against a since-changed model, hiding ` +
+        `drift. Pin a dated id (e.g. ...-20251001) for honest replay.`;
+    if (process.env.GITHUB_ACTIONS)
+        console.log(`::warning::${msg}`);
+    else
+        console.warn(msg);
+}
+/**
+ * Merge the tool-intercept PreToolUse hook into an arm's resolved settings
+ * (appending to any existing `PreToolUse` list). Returns the settings unchanged
+ * when there are no intercepts. The intercept list itself rides the
+ * `VIGILES_INTERCEPT_TOOLS` env, not the settings — see {@link executeTrial}.
+ */
+function withInterceptToolHook(settings, intercepts) {
+    if (intercepts.length === 0)
+        return settings;
+    const intercept = (0, tool_intercept_js_1.buildInterceptSettings)(intercepts, {
+        command: INTERCEPT_TOOL_HOOK_CMD,
+    });
+    const base = isRecord(settings) ? settings : {};
+    const baseHooks = isRecord(base.hooks) ? base.hooks : {};
+    const basePre = Array.isArray(baseHooks.PreToolUse)
+        ? baseHooks.PreToolUse
+        : [];
+    return {
+        ...base,
+        hooks: {
+            ...baseHooks,
+            PreToolUse: [...basePre, ...intercept.hooks.PreToolUse],
+        },
+    };
+}
+/**
+ * The default allowlist {@link ephemeralRunEnv} passes through from the real
+ * environment. Two groups, both load-bearing for a real-model `claude` run:
+ *
+ * - **Auth** — the harness's OWN credentials. The eval drives the real `claude`
+ *   CLI, which authenticates via the user's subscription (`~/.claude`, reached
+ *   through the fresh HOME's allowed config — see below) OR via these env vars.
+ *   We mirror the auth surface the runtime port already names
+ *   (`ANTHROPIC_API_KEY` / `ANTHROPIC_BASE_URL`, see
+ *   `adapters/claude-code/runtime.ts`) plus the OAuth/token + region variants the
+ *   CLI accepts, so a token-authed user is not broken by a too-narrow list.
+ * - **Runtime** — what any spawned process needs to *function*: `PATH` (resolve
+ *   `node` / `claude`), the locale/terminal vars (`LANG` / `LC_*` / `TERM`), and
+ *   `TMPDIR` (which we override to the fresh HOME). Mirrors what `bwrapArgs` /
+ *   `setenvArgs` in `src/sandbox.ts` set back after `--clearenv`.
+ *
+ * Notably it does NOT pass through `GIT_*`, `GH_TOKEN`, `SSH_*`, `AWS_*`, or any
+ * other non-allowlisted secret-shaped var — those are exactly what an ephemeral
+ * run must not see. `CLAUDE_*` is allowlisted by prefix because the CLI reads
+ * several `CLAUDE_*` knobs (config dir, etc.) and omitting one is the failure
+ * mode this whole guard is conservative against.
+ *
+ * Conservative by design: a too-broad allowlist is safe (it just leaks a benign
+ * var); a too-narrow one silently breaks auth — which is why the feature ships
+ * default-OFF until validated against a real run.
+ */
+const EPHEMERAL_ALLOW = [
+    // Runtime essentials (mirror sandbox.ts setenv-after-clearenv).
+    "PATH",
+    "LANG",
+    "TERM",
+    // Anthropic / Claude Code auth + endpoint (mirror runtime.ts + CLI auth vars).
+    runtime_js_1.claudeCodeRuntime.modelApiKeyEnv, // ANTHROPIC_API_KEY
+    runtime_js_1.claudeCodeRuntime.modelBaseUrlEnv, // ANTHROPIC_BASE_URL
+    "ANTHROPIC_AUTH_TOKEN",
+    "ANTHROPIC_API_URL",
+    "ANTHROPIC_MODEL",
+    "ANTHROPIC_DEFAULT_HEADERS",
+    "CLAUDE_CODE_USE_BEDROCK",
+    "CLAUDE_CODE_USE_VERTEX",
+    // Cloud-provider auth the CLI uses for Bedrock/Vertex backends (region/profile
+    // only — NOT the secret-shaped AWS_* access keys, which stay dropped).
+    "AWS_REGION",
+    "AWS_DEFAULT_REGION",
+    "AWS_PROFILE",
+    "CLOUD_ML_REGION",
+    "GOOGLE_CLOUD_PROJECT",
+    "GOOGLE_APPLICATION_CREDENTIALS",
+];
+/** Prefixes passed through wholesale — the CLI reads several `CLAUDE_*` knobs and
+ *  a `LC_*` locale family; allowlist by prefix so omitting one isn't the silent
+ *  auth/locale break this guard exists to avoid. */
+const EPHEMERAL_ALLOW_PREFIXES = ["CLAUDE_", "LC_"];
+/**
+ * Build an **ephemeral run environment** for a model-driven run: a NEW env object
+ * with a *fresh* `HOME` (and `TMPDIR`) pointed at the throwaway `opts.home`, only
+ * an allowlist of auth + runtime vars passed through from `base`, and everything
+ * else DROPPED. Pure — no fs, no spawn.
+ *
+ * The rationale is "fresh HOME + only the harness credential injected, **not** a
+ * blanket wipe": running a model-driven skill/agent is itself a side effect (the
+ * *model*, not the author, chose the actions), so it should not be able to read
+ * the real `~/.gitconfig` / `~/.ssh` / `~/.aws` or write to the real `~`. But the
+ * real `claude` CLI must still AUTHENTICATE, so the harness's own credentials
+ * ({@link EPHEMERAL_ALLOW} — `ANTHROPIC_*`, `CLAUDE_*`, locale/PATH) are
+ * re-injected; a blanket `--clearenv`-style wipe would break every eval. Because
+ * this needs no kernel features, it is the cross-platform STATE-protection floor
+ * (lands on macOS immediately), orthogonal to the Linux bubblewrap HOST
+ * confinement in `src/sandbox.ts`.
+ *
+ * @param base  the source environment to filter (usually `process.env`).
+ * @param opts.home  the throwaway dir to set as `HOME`/`TMPDIR`.
+ * @param opts.allow  extra var NAMES to pass through (e.g. the `VIGILES_*` keys
+ *   the eval already injects). Layered ON TOP of the default allowlist.
+ */
+function ephemeralRunEnv(base, opts) {
+    const out = {};
+    const allowExact = new Set([
+        ...EPHEMERAL_ALLOW,
+        ...(opts.allow ?? []),
+    ]);
+    for (const [k, v] of Object.entries(base)) {
+        if (v === undefined)
+            continue;
+        const allowed = allowExact.has(k) ||
+            EPHEMERAL_ALLOW_PREFIXES.some((p) => k.startsWith(p));
+        if (allowed)
+            out[k] = v;
+    }
+    // Fresh HOME + TMPDIR last so they always win over anything passed through.
+    out.HOME = opts.home;
+    out.TMPDIR = opts.home;
+    return out;
+}
+/**
+ * Home-relative AUTH files to carry from the real HOME into the throwaway one.
+ *
+ * A local subscription credential (OAuth token) often lives in a FILE under HOME,
+ * not an env var — so scrubbing HOME would lose it and break a local-authed run.
+ * This is the file half of the auth allowlist; {@link EPHEMERAL_ALLOW} covers the
+ * env-var / host-brokered half. Kept a named constant so it's easy to extend, and
+ * deliberately NARROW — only the explicit auth files, never `.gitconfig` / `.ssh`
+ * / `.aws`, which are exactly what an ephemeral run must not see.
+ */
+exports.EPHEMERAL_HOME_KEEP = [
+    ".claude/.credentials.json", // the Claude Code OAuth token
+];
+/**
+ * Seed the throwaway HOME with the harness's own auth FILE(s) — best-effort;
+ * covers local file-based OAuth; the env-var/host-brokered path is covered by the
+ * allowlist in {@link ephemeralRunEnv}.
+ *
+ * COPIES (never symlinks) each {@link EPHEMERAL_HOME_KEEP} path from `realHome`
+ * into `throwawayHome`, creating parent dirs as needed; a symlink would let the
+ * model-driven run write back to the real credential file, defeating ephemerality.
+ * A path that doesn't exist in the real HOME is skipped silently (that user auths
+ * via env-var / host broker instead). Pure fs — no env, no spawn.
+ */
+function seedEphemeralHome(throwawayHome, realHome, keep = exports.EPHEMERAL_HOME_KEEP) {
+    for (const rel of keep) {
+        const src = (0, node_path_1.join)(realHome, rel);
+        if (!(0, node_fs_1.existsSync)(src))
+            continue; // env-var / host-brokered auth covers this.
+        const dest = (0, node_path_1.join)(throwawayHome, rel);
+        (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(dest), { recursive: true });
+        (0, node_fs_1.cpSync)(src, dest); // copy, not symlink — keep the real credential read-only.
+    }
+}
 /** Execute one trial in a fresh sandbox; returns its metric row + usage. */
 async function executeTrial(spec, arm, trialIndex, runner, cfg) {
     const cwd = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-eval-"));
     try {
-        const { files, settings } = (0, plugin_loader_js_1.resolveHarness)({
+        const resolved = (0, plugin_loader_js_1.resolveHarness)({
             plugin: arm.plugin,
             settings: arm.settings,
             files: { ...spec.fixture, ...arm.files },
         });
+        const { files } = resolved;
+        const intercepts = arm.interceptTools ?? [];
+        const settings = withInterceptToolHook(resolved.settings, intercepts);
+        // The eval-injected overlay (VIGILES_INTERCEPT_TOOLS), if any.
+        const overlay = intercepts.length > 0
+            ? { [tool_intercept_js_1.INTERCEPT_TOOLS_ENV]: (0, tool_intercept_js_1.serializeIntercepts)(intercepts) }
+            : undefined;
+        // Opt-in (default OFF): an ephemeral run env — a throwaway HOME under the
+        // trial's own temp cwd + a scrubbed, auth-only allowlist. The eval's injected
+        // keys (e.g. VIGILES_INTERCEPT_TOOLS) are allowlisted through so interception
+        // still works. When OFF, `env`/`replaceEnv` are exactly as before.
+        // Tool stubs on PATH (rung R2): write the fake binaries into a bin dir under
+        // this trial's cwd; it is PREPENDED to whatever PATH the run uses below, so
+        // the fakes win over the real binaries. Absent → no PATH change.
+        const stubs = spec.stubs ?? [];
+        const stubDir = stubs.length > 0
+            ? (0, tool_stub_js_1.stubBinDir)(stubs, (0, node_path_1.join)(cwd, ".vigiles-stubs"))
+            : undefined;
+        const prependPath = (path) => stubDir === undefined
+            ? (path ?? "")
+            : path === undefined || path === ""
+                ? stubDir
+                : `${stubDir}${node_path_1.delimiter}${path}`;
+        const ephemeral = spec.ephemeralEnv === true;
+        let env;
+        let replaceEnv = false;
+        if (ephemeral) {
+            const home = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)(cwd, "home-"));
+            // Carry the harness's own auth FILE (local OAuth) into the fresh HOME —
+            // env-var/host-brokered auth is covered by ephemeralRunEnv's allowlist.
+            seedEphemeralHome(home, process.env.HOME ?? (0, node_os_1.homedir)());
+            env = ephemeralRunEnv(process.env, {
+                home,
+                allow: overlay ? Object.keys(overlay) : [],
+            });
+            if (overlay)
+                Object.assign(env, overlay);
+            // ephemeralRunEnv passes PATH through; prepend the stub dir over it.
+            if (stubDir !== undefined)
+                env.PATH = prependPath(env.PATH);
+            replaceEnv = true;
+        }
+        else {
+            // Legacy overlay path: `spawnAgent` spreads `{ ...process.env, ...env }`, so
+            // set PATH in the overlay to the stub dir prepended over process.env.PATH.
+            env =
+                stubDir !== undefined
+                    ? { ...overlay, PATH: prependPath(process.env.PATH) }
+                    : overlay;
+        }
         writeFiles(cwd, files);
         const hasSettings = settings !== undefined;
         if (hasSettings) {
@@ -471,11 +839,14 @@ async function executeTrial(spec, arm, trialIndex, runner, cfg) {
         const out = await runWithCache({
             task: spec.task,
             cwd,
-            model: cfg.model,
+            // A model comparison is a harness A/B: an arm may override the model.
+            model: arm.model ?? cfg.model,
             tools: cfg.tools,
             hasSettings,
             pluginDir: arm.pluginDir,
             timeoutMs: cfg.timeoutMs,
+            env,
+            replaceEnv,
         }, { files, settings, trialIndex }, runner, cfg);
         const ctx = makeContext(cwd, out);
         return { row: spec.measure(ctx), usage: ctx.usage };
@@ -566,6 +937,9 @@ async function runEvalWith(spec, runner) {
         cache: spec.cache ?? "off",
         cacheDir: spec.cacheDir ?? (0, node_path_1.resolve)(process.cwd(), ".vigiles", "eval-cache"),
     };
+    if (cfg.cache !== "off" && !isDatedModel(cfg.model)) {
+        warnFloatingModel(cfg.model);
+    }
     const retrying = (a) => runWithRetry(a, runner, retries, backoffMs);
     const units = buildUnits(spec.arms, trials);
     let spent = 0;
@@ -773,23 +1147,141 @@ function assertPromptDiversity(prompts, opts = {}) {
  * be set. `packaged` is present only when vigiles built it, so the caller knows
  * to remove it afterward.
  */
+/** The dir holding `<name>/SKILL.md` for a source that may be a plugin (`skills/`
+ *  or `.claude/skills/`) or already a loose skills dir. */
+function collectSkillsSource(dir) {
+    const abs = (0, node_path_1.resolve)(dir);
+    const pluginSkills = (0, node_path_1.join)(abs, "skills");
+    if ((0, node_fs_1.existsSync)(pluginSkills))
+        return pluginSkills;
+    const ccSkills = (0, node_path_1.join)(abs, ".claude", "skills");
+    if ((0, node_fs_1.existsSync)(ccSkills))
+        return ccSkills;
+    return abs;
+}
+/**
+ * Copy each `<name>/SKILL.md` skill from `src` into `skillsOut` (stubbing the body
+ * when asked); skip a skill whose name is already `present` so the under-test
+ * skill wins a collision. Returns how many were newly copied.
+ */
+function copySkillsInto(src, skillsOut, stub, present) {
+    const abs = (0, node_path_1.resolve)(src);
+    if (!(0, node_fs_1.existsSync)(abs))
+        throw new Error(`installSet source not found: ${src} (resolved ${abs})`);
+    let copied = 0;
+    for (const entry of (0, node_fs_1.readdirSync)(abs, { withFileTypes: true })) {
+        if (!entry.isDirectory() || present.has(entry.name))
+            continue;
+        const srcSkill = (0, node_path_1.join)(abs, entry.name, "SKILL.md");
+        if (!(0, node_fs_1.existsSync)(srcSkill))
+            continue;
+        const destDir = (0, node_path_1.join)(skillsOut, entry.name);
+        if (stub) {
+            (0, node_fs_1.mkdirSync)(destDir, { recursive: true });
+            (0, node_fs_1.writeFileSync)((0, node_path_1.join)(destDir, "SKILL.md"), stubSkillBody((0, node_fs_1.readFileSync)(srcSkill, "utf-8")));
+        }
+        else {
+            (0, node_fs_1.cpSync)((0, node_path_1.join)(abs, entry.name), destDir, { recursive: true });
+        }
+        present.add(entry.name);
+        copied++;
+    }
+    return copied;
+}
+/**
+ * Build a combined plugin: the under-test skills PLUS every `installSet` source's
+ * skills, so the skill-under-test competes for selection as in the real harness.
+ * Named after the under-test plugin so `<name>:<skill>` ids still match; the
+ * under-test skills win a name collision. Returns the dir + `added` = how many
+ * installSet skills were merged in (excludes collisions). The report's
+ * `competitors` is derived separately from the FULL pool (see `countSkills`), so
+ * sibling skills already in the under-test source count too. Caller removes the
+ * dir. Pure (filesystem only).
+ */
+function packageInstallSet(opts) {
+    const root = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-harness-"));
+    try {
+        (0, node_fs_1.mkdirSync)((0, node_path_1.join)(root, ".claude-plugin"), { recursive: true });
+        (0, node_fs_1.writeFileSync)((0, node_path_1.join)(root, ".claude-plugin", "plugin.json"), JSON.stringify({ name: opts.name, version: "0.0.0" }, null, 2));
+        const skillsOut = (0, node_path_1.join)(root, "skills");
+        (0, node_fs_1.mkdirSync)(skillsOut, { recursive: true });
+        const present = new Set();
+        const underTest = copySkillsInto(opts.underTestSrc, skillsOut, opts.stub, present);
+        if (underTest === 0)
+            throw new Error(`No <name>/SKILL.md skills under the skill-under-test source ${opts.underTestSrc}`);
+        let added = 0;
+        for (const src of opts.installSet)
+            added += copySkillsInto(collectSkillsSource(src), skillsOut, opts.stub, present);
+        return { dir: root, added };
+    }
+    catch (e) {
+        (0, node_fs_1.rmSync)(root, { recursive: true, force: true }); // don't leak the temp dir
+        throw e;
+    }
+}
+/** The under-test skills source + plugin name (the namespace `fired` matches). */
+function underTestSource(spec) {
+    if (spec.skillsDir)
+        return { src: spec.skillsDir, name: "vigiles-loose-skills" };
+    if (spec.pluginDir)
+        return {
+            src: skillsDirOf(spec.pluginDir),
+            name: pluginName(spec.pluginDir) ?? "vigiles-loose-skills",
+        };
+    throw new Error("measureTriggerRate: provide `pluginDir` or `skillsDir`.");
+}
+/** Number of `<name>/SKILL.md` skills installed in a plugin — the selection pool. */
+function countSkills(pluginDir) {
+    const dir = skillsDirOf(pluginDir);
+    if (!(0, node_fs_1.existsSync)(dir))
+        return 0;
+    let n = 0;
+    for (const e of (0, node_fs_1.readdirSync)(dir, { withFileTypes: true }))
+        if (e.isDirectory() && (0, node_fs_1.existsSync)((0, node_path_1.join)(dir, e.name, "SKILL.md")))
+            n++;
+    return n;
+}
 function resolveTriggerPluginDir(spec) {
     if (spec.pluginDir && spec.skillsDir)
         throw new Error("measureTriggerRate: set `pluginDir` OR `skillsDir`, not both.");
-    const stub = spec.stubSkillBodies ?? false;
-    if (spec.skillsDir) {
-        const packaged = packageSkillsDir(spec.skillsDir, { stub });
-        return { pluginDir: packaged, packaged };
-    }
-    if (spec.pluginDir) {
-        if (!stub)
-            return { pluginDir: spec.pluginDir };
+    const stub = spec.stubSkillBodies ?? true; // trigger = frontmatter; body never needed
+    const installSet = spec.installSet ?? [];
+    let pluginDir;
+    let packaged;
+    if (installSet.length > 0) {
+        // Whole-harness tier: merge the under-test skills with the install set so
+        // selection is competitive (the realistic, differentiated measurement).
+        const { src, name } = underTestSource(spec);
+        ({ dir: pluginDir } = packageInstallSet({
+            underTestSrc: src,
+            name,
+            installSet,
+            stub,
+        }));
+        packaged = pluginDir;
+    }
+    else if (spec.skillsDir) {
+        packaged = packageSkillsDir(spec.skillsDir, { stub });
+        pluginDir = packaged;
+    }
+    else if (spec.pluginDir) {
         // Stub a real plugin: build a minimal plugin from its skills/ with bodies
         // stripped — keep the original plugin NAME so `<name>:<skill>` still matches.
-        const packaged = stubbedPluginDir(spec.pluginDir);
-        return { pluginDir: packaged, packaged };
+        packaged = stub ? stubbedPluginDir(spec.pluginDir) : undefined;
+        pluginDir = packaged ?? spec.pluginDir;
     }
-    throw new Error("measureTriggerRate: provide `pluginDir` or `skillsDir`.");
+    else {
+        throw new Error("measureTriggerRate: provide `pluginDir` or `skillsDir`.");
+    }
+    // `competitors` is the REAL selection pressure: every OTHER skill installed in
+    // the resolved plugin (siblings already in the source + any installSet), not
+    // just the installSet delta — so a multi-skill plugin is never mislabeled
+    // "isolated". `max(0, …)` guards a 0-skill pool.
+    return {
+        pluginDir,
+        packaged,
+        competitors: Math.max(0, countSkills(pluginDir) - 1),
+    };
 }
 /** Run one prompt set × trials through `runner`, aggregating fired counts. */
 async function runTriggerSet(prompts, cfg, runner) {
@@ -851,10 +1343,23 @@ async function measureTriggerRateWith(spec, runner) {
             label: "irrelevantPrompts",
         });
     }
-    const { pluginDir, packaged } = resolveTriggerPluginDir(spec);
+    // Model floor (default Sonnet): trigger-rate under-measures selection on a
+    // weaker model, so FAIL before spending a token rather than report a
+    // false-negative recall. The floor lives in the spec (`minModel`), not an env
+    // override — model choice is part of the measurement definition. Lower it
+    // deliberately for a cheap run.
+    const model = spec.model ?? "sonnet";
+    const minModel = spec.minModel ?? "sonnet";
+    if (belowModelFloor(model, minModel))
+        throw new Error(`measureTriggerRate: model "${model}" is below the minimum "${minModel}" — ` +
+            "trigger-rate under-measures selection on a weaker model " +
+            "(raise the model, or lower `minModel` for a deliberately cheap run).");
+    const { pluginDir, packaged, competitors } = resolveTriggerPluginDir(spec);
     const cfg = {
         trials: spec.trials ?? 1,
-        model: spec.model ?? "haiku",
+        // Sonnet, not haiku: trigger-rate is a selection measurement and haiku
+        // under-selects, producing false-negative recall (see TriggerRateSpec.model).
+        model,
         tools: spec.allowedTools ?? ["Read", "Edit", "Write", "Bash", "Skill"],
         timeoutMs: spec.timeoutMs ?? 240000,
         spacing: (spec.spacingSec ?? 4) * 1000,
@@ -867,6 +1372,7 @@ async function measureTriggerRateWith(spec, runner) {
             rate: relevant.n > 0 ? relevant.fired / relevant.n : 0,
             n: relevant.n,
             perPrompt: relevant.perPrompt,
+            competitors,
         };
         if ((spec.irrelevantPrompts?.length ?? 0) === 0)
             return base;
@@ -911,6 +1417,12 @@ function formatTriggerRateReport(report) {
             lines.push(`  ${p.rate.toFixed(2)}  [irrelevant] ${p.prompt.slice(0, 48)}`);
         }
     }
+    // Honest labelling: an isolated run measures the skill alone, with no competing
+    // skills to evict or out-compete its description — so recall is an UPPER bound
+    // and false-positive a LOWER bound. Say so, or point at the whole-harness count.
+    lines.push(report.competitors > 0
+        ? `whole-harness: measured against ${String(report.competitors)} competing skill(s)`
+        : "isolated: no competing skills — recall is an upper bound, false-positive a lower bound (populate `installSet` for a release-gate measurement)");
     return lines.join("\n");
 }
 //# sourceMappingURL=eval.js.map