vigiles 36.0.0 → 38.0.0

package/dist/eval.js

@@ -16,6 +16,7 @@ exports.measureArmsWith = measureArmsWith;
 exports.measureArms = measureArms;
 exports.compareCheck = compareCheck;
 exports.formatCheckReport = formatCheckReport;
+exports.formatArmsCheckReport = formatArmsCheckReport;
 exports.assertRates = assertRates;
 exports.checkReportToJUnit = checkReportToJUnit;
 exports.parseUsage = parseUsage;
@@ -32,6 +33,8 @@ exports.ephemeralRunEnv = ephemeralRunEnv;
 exports.seedEphemeralHome = seedEphemeralHome;
 exports.isRateLimited = isRateLimited;
 exports.runPool = runPool;
+exports.unansweredInReport = unansweredInReport;
+exports.runEvalReporting = runEvalReporting;
 exports.runEvalWith = runEvalWith;
 exports.formatEvalReport = formatEvalReport;
 exports.packageSkillsDir = packageSkillsDir;
@@ -54,6 +57,7 @@ exports.formatTriggerRateReport = formatTriggerRateReport;
  * benchmark harness under `bench/` — the empirical half of testing your harness.
  *
  *   const report = await runEval({
+ *     env: { kind: "ephemeral" },
  *     fixture: { "src/billing.ts": "export function chargeCard(){}" },
  *     arms: {
  *       vanilla: {},
@@ -89,6 +93,8 @@ const check_count_js_1 = require("./check-count.js");
 const coverage_probe_js_1 = require("./coverage-probe.js");
 const tool_intercept_js_1 = require("./tool-intercept.js");
 const tool_stub_js_1 = require("./tool-stub.js");
+const stub_rules_js_1 = require("./core/stub-rules.js");
+const eval_spec_parse_js_1 = require("./core/eval-spec-parse.js");
 const tmp_root_js_1 = require("./core/tmp-root.js");
 const edit_distance_js_1 = require("./core/edit-distance.js");
 /**
@@ -108,7 +114,7 @@ function writeFiles(cwd, files) {
 }
 /**
  * Resolve the environment a trial's subprocess actually runs with — the
- * SECURITY-CRITICAL decision behind `ephemeralEnv`. When `replaceEnv` is set, the
+ * SECURITY-CRITICAL decision behind `env: { kind: "ephemeral" }`. When `replaceEnv` is set, the
  * scrubbed `env` is the COMPLETE environment, so the real `$HOME` and inherited
  * secrets are DROPPED; otherwise `env` is an overlay on `base`. In BOTH, the
  * harness's declared session identity (`runEnv.sessionIdentity`) is removed: a
@@ -345,13 +351,14 @@ function warnUnregisteredSkillArms(arms) {
     }
 }
 async function runEval(spec) {
-    warnUnregisteredSkillArms(spec.arms);
-    const report = await runEvalWith(spec, exports.spawnAgent);
-    // Surface what the run spent — tokens + API-equivalent $, and a LOUD warning if
-    // it was billed to a metered API key instead of the subscription. See eval-cost.ts.
-    (0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.costFromEvalReport)(report));
-    return report;
+    return runEvalReporting(spec, exports.spawnAgent);
 }
+/** The env `measure` / `measureArms` pass on: they have no `env` field yet. */
+const MEASURE_INHERIT_REASON = "measure and measureArms have no `env` field; they run with your HOME and environment (docs/safety.md)";
+const INHERITED_BY_MEASURE = {
+    kind: "inherit",
+    reason: MEASURE_INHERIT_REASON,
+};
 /**
  * Score a check vocabulary across trials — the scored counterpart to
  * `assertChecks` (strict). Each check yields a `rate ± se` and `pass^k` over `n`
@@ -396,6 +403,9 @@ async function measureWith(spec, runner) {
         allowedTools: spec.allowedTools,
         timeoutMs: spec.timeoutMs,
         spacingSec: spec.spacingSec,
+        // measure / measureArms have no `env` field (yet): they run with your
+        // HOME and environment, as before — said here, not defaulted silently.
+        env: INHERITED_BY_MEASURE,
         measure: (ctx) => Object.fromEntries(keyed.map(([k, c]) => [k, c.eval(ctx).pass])),
     }, runner);
     return checkReportOf(report.arms.run, keyed, installNamespace(arm));
@@ -450,6 +460,9 @@ async function measureArmsWith(spec, runner) {
         allowedTools: spec.allowedTools,
         timeoutMs: spec.timeoutMs,
         spacingSec: spec.spacingSec,
+        // measure / measureArms have no `env` field (yet): they run with your
+        // HOME and environment, as before — said here, not defaulted silently.
+        env: INHERITED_BY_MEASURE,
         measure: (ctx) => Object.fromEntries(keyed.map(([k, c]) => [k, c.eval(ctx).pass])),
     }, runner);
     const arms = {};
@@ -491,8 +504,25 @@ function checkLabel(json) {
     }
     return json.kind;
 }
-/** Format a {@link CheckReport}: one line per check with its rate ± se and pass^k. */
+/** The report line that says a run used your real HOME and environment, and why. */
+const inheritLine = (reason) => `  env: inherited your HOME and environment — ${reason}`;
+/**
+ * Format a {@link CheckReport}: one line per check with its rate ± se and
+ * pass^k, and the line saying `measure` ran with your HOME and environment
+ * (it has no `env` field; `INHERITED_BY_MEASURE`).
+ */
 function formatCheckReport(report) {
+    return [formatChecks(report), inheritLine(MEASURE_INHERIT_REASON)].join("\n");
+}
+/**
+ * Format an {@link ArmsCheckReport}: each arm's checks under `[arm: name]`, and
+ * the inherit line ONCE at the end — every arm ran the same way.
+ */
+function formatArmsCheckReport(report) {
+    const arms = Object.entries(report.arms).map(([name, arm]) => `\n[arm: ${name}]\n${formatChecks(arm)}`);
+    return [...arms, inheritLine(MEASURE_INHERIT_REASON)].join("\n");
+}
+function formatChecks(report) {
     const lines = [`measured ${String(report.n)} run(s):`];
     for (const c of report.perCheck) {
         lines.push(`  ${(c.rate * 100).toFixed(0)}% ± ${(c.se * 100).toFixed(0)}%  ${checkLabel(c.check)}` +
@@ -781,9 +811,11 @@ function aggregateUsage(usages) {
  * miss, run the agent and (in `readwrite`) record output + cwd snapshot. The
  * cache key excludes `measure`, so editing the metric still replays.
  */
-async function runWithCache(runArgs, keyParts, runner, cfg) {
-    if (cfg.cache === "off")
-        return runner(runArgs);
+async function runWithCache(runArgs, keyParts, runner, cfg, collectStubCalls) {
+    if (cfg.cache === "off") {
+        const out = await runner(runArgs);
+        return { out, stubCalls: collectStubCalls() };
+    }
     const key = (0, eval_cache_js_1.cacheKey)({
         task: runArgs.task,
         model: runArgs.model,
@@ -791,7 +823,9 @@ async function runWithCache(runArgs, keyParts, runner, cfg) {
         tools: runArgs.tools,
         files: keyParts.files,
         settings: keyParts.settings,
-        env: runArgs.env,
+        env: keyParts.env,
+        stubs: keyParts.stubs.length > 0 ? keyParts.stubs : undefined,
+        runEnv: keyParts.runEnv,
         // A native --plugin-dir install isn't in `files`, so hash its CONTENTS into
         // the key — otherwise editing a skill in it would false-replay.
         pluginDirHash: runArgs.pluginDir ? (0, eval_cache_js_1.hashDir)(runArgs.pluginDir) : undefined,
@@ -806,13 +840,20 @@ async function runWithCache(runArgs, keyParts, runner, cfg) {
     // otherwise be scored as a trial on every later run.
     if (hit && startFailure(hit.out) === null) {
         (0, eval_cache_js_1.restoreDir)(runArgs.cwd, hit.files);
-        return hit.out;
+        // The stub log lives beside the work dir, so the record carries it: a
+        // replayed trial reports the same calls, unanswered ones included.
+        return { out: hit.out, stubCalls: hit.stubCalls };
     }
     const out = await runner(runArgs);
+    const stubCalls = collectStubCalls();
     if (cfg.cache === "readwrite") {
-        (0, eval_cache_js_1.writeCache)(cfg.cacheDir, key, { out, files: (0, eval_cache_js_1.snapshotDir)(runArgs.cwd) });
+        (0, eval_cache_js_1.writeCache)(cfg.cacheDir, key, {
+            out,
+            files: (0, eval_cache_js_1.snapshotDir)(runArgs.cwd),
+            ...(stubCalls === undefined ? {} : { stubCalls }),
+        });
     }
-    return out;
+    return { out, stubCalls };
 }
 /**
  * The `vigiles hook-runtime intercept-tool` command, as an absolute `node <cli> …`
@@ -822,7 +863,15 @@ async function runWithCache(runArgs, keyParts, runner, cfg) {
  * entries).
  */
 const INTERCEPT_TOOL_HOOK_CLI = [(0, node_path_1.join)(__dirname, "cli.js"), (0, node_path_1.join)(__dirname, "..", "dist", "cli.js")].find((p) => (0, node_fs_1.existsSync)(p)) ?? (0, node_path_1.join)(__dirname, "cli.js");
-const INTERCEPT_TOOL_HOOK_CMD = `"${process.execPath}" "${INTERCEPT_TOOL_HOOK_CLI}" hook-runtime intercept-tool`;
+/** This node binary — what every command the eval writes for a child calls back into. */
+const NODE_BIN = process.execPath;
+const INTERCEPT_TOOL_HOOK_CMD = `"${NODE_BIN}" "${INTERCEPT_TOOL_HOOK_CLI}" hook-runtime intercept-tool`;
+/**
+ * How a stub shim reaches the stub runtime: the same node and `cli.js` as the
+ * intercept hook, for the same reason (`npx vigiles` does not resolve from a
+ * throwaway dir). The shim runs `cli.js hook-runtime stub`.
+ */
+const STUB_LAUNCHER = { node: NODE_BIN, cli: INTERCEPT_TOOL_HOOK_CLI };
 function isRecord(v) {
     return v !== null && typeof v === "object";
 }
@@ -988,9 +1037,91 @@ function seedEphemeralHome(throwawayHome, realHome, keep = EVAL_RUNTIME.runEnv?.
         (0, node_fs_1.copyFileSync)(src, dest);
     }
 }
+/**
+ * The parts of a {@link RunEnv} that move behaviour — the kind and the seed —
+ * for the cache key and the lock. Never the per-trial HOME path, and never an
+ * `inherit` reason (prose, not an input).
+ */
+function runEnvKeyView(env) {
+    return env.kind === "ephemeral"
+        ? { kind: env.kind, home: env.home }
+        : { kind: env.kind };
+}
+/** The run environment as a report carries it: the kind, and an inherit's reason. */
+function runEnvReportView(env) {
+    return env.kind === "ephemeral"
+        ? { kind: "ephemeral" }
+        : { kind: "inherit", reason: env.reason };
+}
+/** Write a seed's files under a throwaway HOME, parents created. Effects only. */
+function materializeHome(home, files) {
+    Object.entries(files).forEach(([path, contents]) => {
+        const dest = (0, node_path_1.join)(home, path);
+        (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(dest), { recursive: true });
+        (0, node_fs_1.writeFileSync)(dest, contents);
+    });
+}
+/**
+ * Build a trial's environment from the spec's {@link RunEnv}. `ephemeral`: a
+ * throwaway HOME (`home`, a directory BESIDE the trial cwd — never in it, or a
+ * cache snapshot of the cwd would carry the credentials and the seed), seeded
+ * (the seed first, then the harness's own auth file), and a scrubbed allowlist env — the eval's injected
+ * keys (`VIGILES_INTERCEPT_TOOLS`) allowlisted through so interception works.
+ * `inherit`: an overlay on the caller's env. Either way the stub bin dir, when
+ * there is one, goes first on PATH.
+ */
+function trialEnv(runEnv, home, overlay, stubBin) {
+    const withStubs = (path) => stubBin === undefined
+        ? (path ?? "")
+        : path === undefined || path === ""
+            ? stubBin
+            : `${stubBin}${node_path_1.delimiter}${path}`;
+    switch (runEnv.kind) {
+        case "ephemeral": {
+            // The seed was validated at the spec boundary (`parseRunEnv`): every
+            // path HOME-relative, inside HOME, and not the harness's auth file.
+            materializeHome(home, runEnv.home?.files ?? {});
+            // Carry the harness's own auth FILE (local OAuth) into the fresh HOME —
+            // env-var/host-brokered auth is covered by ephemeralRunEnv's allowlist.
+            seedEphemeralHome(home, process.env.HOME ?? (0, node_os_1.homedir)());
+            const base = {
+                ...ephemeralRunEnv(process.env, {
+                    home,
+                    allow: overlay ? Object.keys(overlay) : [],
+                }),
+                ...overlay,
+            };
+            return {
+                env: stubBin === undefined
+                    ? base
+                    : { ...base, PATH: withStubs(base.PATH) },
+                replaceEnv: true,
+                keyEnv: base,
+            };
+        }
+        case "inherit":
+            // `spawnAgent` spreads `{ ...process.env, ...env }`, so a stub PATH is
+            // set in the overlay as the stub dir prepended over process.env.PATH.
+            return {
+                env: stubBin === undefined
+                    ? overlay
+                    : { ...overlay, PATH: withStubs(process.env.PATH) },
+                replaceEnv: false,
+                keyEnv: overlay,
+            };
+    }
+}
 /** Execute one trial in a fresh sandbox; returns its metric row + usage. */
-async function executeTrial(spec, arm, trialIndex, runner, cfg) {
+async function executeTrial(spec, arm, trialIndex, retrying, cfg) {
     const cwd = (0, tmp_root_js_1.makeTmpDir)("eval");
+    // The ephemeral env's HOME: beside the work dir, so `snapshotDir(cwd)` (the
+    // cache record) never sees the credentials file or the seed, and the agent's
+    // `ls -a` of its project does not show it. An `inherit` run leaves it empty.
+    const home = (0, tmp_root_js_1.makeTmpDir)("home");
+    // Tool stubs (rung R2): a shim per stub in a temp dir BESIDE the work dir —
+    // not in it, so the agent does not find vigiles plumbing in its project.
+    const stubs = spec.stubs;
+    const stubDir = stubs.length > 0 ? (0, tool_stub_js_1.writeStubDir)(stubs, STUB_LAUNCHER) : undefined;
     try {
         const resolved = (0, plugin_loader_js_1.resolveHarness)({
             plugin: arm.plugin,
@@ -1004,55 +1135,13 @@ async function executeTrial(spec, arm, trialIndex, runner, cfg) {
         const overlay = intercepts.length > 0
             ? { [tool_intercept_js_1.INTERCEPT_TOOLS_ENV]: (0, tool_intercept_js_1.serializeIntercepts)(intercepts) }
             : undefined;
-        // Opt-in (default OFF): an ephemeral run env — a throwaway HOME under the
-        // trial's own temp cwd + a scrubbed, auth-only allowlist. The eval's injected
-        // keys (e.g. VIGILES_INTERCEPT_TOOLS) are allowlisted through so interception
-        // still works. When OFF, `env`/`replaceEnv` are exactly as before.
-        // Tool stubs on PATH (rung R2): write the fake binaries into a bin dir under
-        // this trial's cwd; it is PREPENDED to whatever PATH the run uses below, so
-        // the fakes win over the real binaries. Absent → no PATH change.
-        const stubs = spec.stubs ?? [];
-        const stubDir = stubs.length > 0
-            ? (0, tool_stub_js_1.stubBinDir)(stubs, (0, node_path_1.join)(cwd, ".vigiles-stubs"))
-            : undefined;
-        const prependPath = (path) => stubDir === undefined
-            ? (path ?? "")
-            : path === undefined || path === ""
-                ? stubDir
-                : `${stubDir}${node_path_1.delimiter}${path}`;
-        const ephemeral = spec.ephemeralEnv === true;
-        let env;
-        let replaceEnv = false;
-        if (ephemeral) {
-            const home = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)(cwd, "home-"));
-            // Carry the harness's own auth FILE (local OAuth) into the fresh HOME —
-            // env-var/host-brokered auth is covered by ephemeralRunEnv's allowlist.
-            seedEphemeralHome(home, process.env.HOME ?? (0, node_os_1.homedir)());
-            env = ephemeralRunEnv(process.env, {
-                home,
-                allow: overlay ? Object.keys(overlay) : [],
-            });
-            if (overlay)
-                Object.assign(env, overlay);
-            // ephemeralRunEnv passes PATH through; prepend the stub dir over it.
-            if (stubDir !== undefined)
-                env.PATH = prependPath(env.PATH);
-            replaceEnv = true;
-        }
-        else {
-            // Legacy overlay path: `spawnAgent` spreads `{ ...process.env, ...env }`, so
-            // set PATH in the overlay to the stub dir prepended over process.env.PATH.
-            env =
-                stubDir !== undefined
-                    ? { ...overlay, PATH: prependPath(process.env.PATH) }
-                    : overlay;
-        }
+        const run = trialEnv(spec.env, home, overlay, stubDir?.binDir);
         writeFiles(cwd, files);
         const hasSettings = settings !== undefined;
         if (hasSettings) {
             (0, node_fs_1.writeFileSync)((0, node_path_1.join)(cwd, "settings.json"), JSON.stringify(settings, null, 2).replaceAll("{cwd}", cwd));
         }
-        const out = await runWithCache({
+        const { out, stubCalls } = await runWithCache({
             task: spec.task,
             cwd,
             // A model comparison is a harness A/B: an arm may override the model.
@@ -1062,14 +1151,43 @@ async function executeTrial(spec, arm, trialIndex, runner, cfg) {
             hasSettings,
             pluginDir: arm.pluginDir,
             timeoutMs: cfg.timeoutMs,
-            env,
-            replaceEnv,
-        }, { files, settings, trialIndex }, runner, cfg);
-        const ctx = makeContext(cwd, out);
-        return { row: spec.measure(ctx), usage: ctx.usage };
+            env: run.env,
+            replaceEnv: run.replaceEnv,
+        }, {
+            files,
+            settings,
+            trialIndex,
+            env: run.keyEnv,
+            stubs: stubs.map(stub_rules_js_1.encodeStub),
+            runEnv: runEnvKeyView(spec.env),
+        }, 
+        // A rate-limited attempt is discarded and retried; its stub calls must
+        // go with it, or the retry inherits a used-up `inOrder` answer and the
+        // report counts calls from a run nobody scores.
+        retrying(stubDir === undefined
+            ? undefined
+            : () => {
+                (0, tool_stub_js_1.resetStubCalls)(stubDir);
+            }), cfg, () => (stubDir === undefined ? undefined : (0, tool_stub_js_1.readStubCalls)(stubDir)));
+        const base = makeContext(cwd, out);
+        const ctx = stubCalls === undefined ? base : { ...base, stubCalls };
+        return {
+            row: spec.measure(ctx),
+            usage: ctx.usage,
+            unanswered: [
+                ...(stubCalls ?? []),
+                // A stub that could not write its log still printed its neutral line
+                // into the agent's tool results: count those too, or a confined shell
+                // would leave the report clean.
+                ...(0, stub_rules_js_1.unloggedMisses)(ctx.toolCalls.map((c) => c.resultText), stubCalls ?? [], stubs.map((s) => s.name)),
+            ].filter((c) => (0, stub_rules_js_1.isUnanswered)(c.outcome)),
+        };
     }
     finally {
         (0, node_fs_1.rmSync)(cwd, { recursive: true, force: true });
+        (0, node_fs_1.rmSync)(home, { recursive: true, force: true });
+        if (stubDir !== undefined)
+            (0, tool_stub_js_1.removeStubDir)(stubDir);
     }
 }
 // A signal in the captured streams that the model call was rate-limited /
@@ -1095,9 +1213,14 @@ const RATE_LIMIT_RE = /rate[ -]?limit(?:ed)?\b|rate_limit_error|\b429\b|overload
 function isRateLimited(out) {
     return RATE_LIMIT_RE.test(`${out.stderr ?? ""}\n${out.stdout}`);
 }
-/** Call `runner`, retrying with exponential backoff while it looks rate-limited. */
-async function runWithRetry(runArgs, runner, retries, baseMs) {
+/**
+ * Call `runner`, retrying with exponential backoff while it looks rate-limited.
+ * `beforeAttempt` runs before every attempt, the first included: it resets the
+ * state a discarded attempt would otherwise leave to the next.
+ */
+async function runWithRetry(runArgs, runner, retries, baseMs, beforeAttempt) {
     for (let attempt = 0;; attempt++) {
+        beforeAttempt?.();
         const out = await runner(runArgs);
         if (!isRateLimited(out) || attempt >= retries)
             return out;
@@ -1155,6 +1278,7 @@ function aggregateArms(armNames, results) {
             metrics: aggregate(rows),
             stats: aggregateStats(rows),
             usage,
+            unansweredStubCalls: done.flatMap((d) => d.unanswered),
         };
     }
     return { arms, totalCostUsd };
@@ -1222,6 +1346,11 @@ async function withEvalLock(args, produce) {
     if (decision.kind === "replay")
         return decision.report;
     const report = await produce();
+    const refusal = args.refuse?.(report);
+    if (refusal !== undefined) {
+        emitLockMessage(`vigiles eval --${lock.mode}: the lock for "${args.name}" was NOT written — ${refusal}`, true);
+        return report;
+    }
     const builtLock = (0, eval_lock_js_1.buildLock)({
         name: args.name,
         inputsHash,
@@ -1290,21 +1419,22 @@ function evalArmsInputs(spec, cfg) {
                 : undefined,
         };
     }
-    // Tool stubs (`spec.stubs`) are written onto PATH before each trial, so a
-    // change to a canned CLI output IS a model-facing input change — fold a
-    // canonical (name-sorted) view into the hash so `--check` catches it. Sorted
-    // for a stable key regardless of declaration order; an empty list is the
-    // byte-identical-to-before default. Each ToolStub is plain serializable data.
-    const stubs = [...(spec.stubs ?? [])].sort((a, b) => a.name.localeCompare(b.name));
-    // `ephemeralEnv` swaps the trial's environment (scrubbed env + throwaway HOME
-    // vs the inherited process env), which can move tool/hook/agent behavior — a
-    // model-facing input, so it belongs in the hash. Normalize to a bool so a
-    // record under one mode can't be replayed under the other with the same hash.
+    // Tool stubs are a model-facing input: a changed answer changes what the agent
+    // sees. Folded in ENCODED (`encodeStub`), never as the raw rules —
+    // `JSON.stringify(/^repos/)` is `{}`, so two specs differing only in a RegExp
+    // token would hash the same and `--check` would pass a stale lock. Sorted by
+    // name for a key that ignores declaration order.
+    const stubs = [...spec.stubs]
+        .sort((a, b) => a.name.localeCompare(b.name))
+        .map(stub_rules_js_1.encodeStub);
+    // Where the trials run (ephemeral + its seed, or inherit) moves behaviour, so
+    // it is an input. The seed's CONTENTS are hashed — the author's input — never
+    // the per-trial HOME path, which differs every run.
     return {
         task: spec.task,
         arms,
         stubs,
-        ephemeralEnv: spec.ephemeralEnv === true,
+        env: runEnvKeyView(spec.env),
     };
 }
 // ---------------------------------------------------------------------------
@@ -1349,7 +1479,7 @@ const EVAL_SPEC_KEYS = {
     maxCostUsd: true,
     rateLimitRetries: true,
     retryBackoffMs: true,
-    ephemeralEnv: true,
+    env: true,
     stubs: true,
     lock: true,
 };
@@ -1443,7 +1573,45 @@ function assertKnownKeys(spec, known, at) {
         "  An unknown field is refused rather than ignored: a dropped field would " +
         "make the run measure a setup you did not ask for.");
 }
+/**
+ * The author-facing failure for a report whose stubs left a call unanswered, or
+ * undefined. One owner for the rule — `vigiles eval`'s exit 2 and the lock's
+ * refusal both read it — so a consumer whose `assert` calls no helper still
+ * fails. Pure.
+ */
+function unansweredInReport(report, stubs) {
+    const calls = Object.values(report.arms).flatMap((a) => a.unansweredStubCalls);
+    const message = (0, stub_rules_js_1.unansweredMessage)(calls, stubs);
+    return message === undefined
+        ? undefined
+        : `${message}\n  The report is not a measurement: no lock is written, and \`vigiles eval\` exits 2.`;
+}
+/**
+ * `runEval` with its author-facing output, over any runner — so the real one and
+ * a test's fake share one path. Besides the run it prints, to stderr, what the
+ * caller must not miss: a skill arm that never activates, a stub call no rule
+ * answered (the same text `vigiles eval` fails with — a script that calls
+ * `runEval` itself, as the `withServices` examples do, gets the report back and
+ * nothing else, so the library is not silent about it either), and what the
+ * run spent.
+ */
+async function runEvalReporting(spec, runner) {
+    warnUnregisteredSkillArms(spec.arms);
+    const report = await runEvalWith(spec, runner);
+    const unanswered = unansweredInReport(report, (0, eval_spec_parse_js_1.parseToolStubs)(spec.stubs, "runEval"));
+    if (unanswered !== undefined)
+        console.error(`⚠ runEval: ${unanswered}`);
+    // Surface what the run spent — tokens + API-equivalent $, and a LOUD warning if
+    // it was billed to a metered API key instead of the subscription. See eval-cost.ts.
+    (0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.costFromEvalReport)(report));
+    return report;
+}
 async function runEvalWith(input, runner) {
+    // A REMOVED field gets its rewrite, not "unknown field … did you mean".
+    if (Object.hasOwn(input, "ephemeralEnv"))
+        throw new Error("runEval: `ephemeralEnv` was replaced by `env`, which is required:\n" +
+            '    ephemeralEnv: true            → env: { kind: "ephemeral" }\n' +
+            '    ephemeralEnv: false / absent  → env: { kind: "inherit", reason: "…" }  (your real HOME; the reason is printed with the report)');
     // Refuse a stray field BEFORE spending a token: an eval file is plain JS, so a
     // typo'd or misplaced key would otherwise vanish and the run would report a
     // confident number about the wrong setup (issue #307).
@@ -1464,8 +1632,18 @@ async function runEvalWith(input, runner) {
     // `skillsDir` packaged into a throwaway plugin) is resolved HERE, once — the
     // one place that decides what `--plugin-dir` receives, so measure / measureArms
     // / runEval cannot disagree about it. The throwaways are removed afterward.
+    // `stubs` and `env` are parsed HERE, once: the old argv-blind stub shape, a
+    // stateful RegExp, a missing `env`, a seed path outside HOME — each refused
+    // with its rewrite before a token is spent. The runner trusts the result.
+    const stubs = (0, eval_spec_parse_js_1.parseToolStubs)(input.stubs, "runEval");
+    const env = (0, eval_spec_parse_js_1.parseRunEnv)(input.env, "runEval", EVAL_RUNTIME.runEnv?.keepHomeFiles ?? []);
     const { arms: resolvedArms, packaged } = resolveArmInstalls(input.arms, input.stubSkillBodies ?? false);
-    const spec = { ...input, arms: resolvedArms };
+    const spec = {
+        ...input,
+        arms: resolvedArms,
+        stubs,
+        env,
+    };
     try {
         return await runResolvedEval(spec, runner);
     }
@@ -1492,31 +1670,49 @@ async function runResolvedEval(spec, runner) {
     if (cfg.cache !== "off" && !isDatedModel(cfg.model)) {
         warnFloatingModel(cfg.model);
     }
-    const retrying = (a) => runWithRetry(a, runner, retries, backoffMs);
+    const retrying = (beforeAttempt) => (a) => runWithRetry(a, runner, retries, backoffMs, beforeAttempt);
     const units = buildUnits(spec.arms, trials);
     let spent = 0;
-    let aborted = false;
+    // Why no NEW trial starts: the budget cap, or a stub call nobody scripted —
+    // the report will be refused, so later trials would only spend. In-flight
+    // trials finish either way.
+    let stopped;
     const worker = async (unit) => {
-        if (aborted)
+        if (stopped !== undefined)
             return { armName: unit.armName, skipped: true };
-        const { row, usage } = await executeTrial(spec, unit.arm, unit.trialIndex, retrying, cfg);
+        const { row, usage, unanswered } = await executeTrial(spec, unit.arm, unit.trialIndex, retrying, cfg);
         spent += usage.costUsd;
-        if (spec.maxCostUsd !== undefined && spent >= spec.maxCostUsd) {
-            aborted = true;
-        }
+        if (spec.maxCostUsd !== undefined && spent >= spec.maxCostUsd)
+            stopped = "budget";
+        else if (unanswered.length > 0)
+            stopped = stopped ?? "unanswered";
         if (spacing > 0)
             await sleep(spacing);
-        return { armName: unit.armName, skipped: false, row, usage };
+        return { armName: unit.armName, skipped: false, row, usage, unanswered };
     };
     const lock = resolveLock(spec.lock);
     // Resolve the lock inputs only when the lock is active (off skips the
     // resolveHarness/hashDir work). `check` replays the committed report below
     // without ever entering the run pool — so no model is driven in CI.
     const inputs = lock.mode === "off" ? undefined : evalArmsInputs(spec, cfg);
-    return withEvalLock({ name: spec.name, inputs, model: cfg.model, effort: cfg.effort, lock }, async () => {
+    return withEvalLock({
+        name: spec.name,
+        inputs,
+        model: cfg.model,
+        effort: cfg.effort,
+        lock,
+        refuse: (report) => unansweredInReport(report, spec.stubs),
+    }, async () => {
         const results = await runPool(units, concurrency, worker);
         const { arms, totalCostUsd } = aggregateArms(Object.keys(spec.arms), results);
-        return { name: spec.name ?? "eval", trials, arms, totalCostUsd, aborted };
+        return {
+            name: spec.name ?? "eval",
+            trials,
+            arms,
+            totalCostUsd,
+            aborted: stopped === "budget",
+            env: runEnvReportView(spec.env),
+        };
     });
 }
 /** Render one metric: `name=mean±se pass^k=…` (se/pass^k shown when measured). */
@@ -1549,7 +1745,13 @@ function formatEvalReport(report) {
             .map(([k, v]) => formatMetric(k, v, r.stats[k]))
             .join("  ");
         lines.push(`  ${arm.padEnd(10)} ${parts}${formatUsage(r.usage)}`);
+        // One line: the calls themselves are printed once, by `runEval`
+        // (`unansweredInReport`), not again under every arm of the table.
+        if (r.unansweredStubCalls.length > 0)
+            lines.push(`  ⚠ ${arm}: ${String(r.unansweredStubCalls.length)} stub call(s) went unanswered — the report is not a measurement`);
     }
+    if (report.env?.kind === "inherit")
+        lines.push(inheritLine(report.env.reason));
     return lines.join("\n");
 }
 /**

package/dist/harness-test.d.ts

@@ -204,7 +204,7 @@ export declare function parseClaudeRun(stdout: string): ParsedRun;
  *   else from the caller: no harness auth (the scripted mock needs none, and a
  *   config-dir variable would point the run back at the real config), no
  *   identity, no secrets. The same scrubbed shape as the eval tier's
- *   `ephemeralEnv` (`scrubbedRunEnv`), without the auth.
+ *   `env: { kind: "ephemeral" }` (`scrubbedRunEnv`), without the auth.
  */
 export type HarnessRunHome = {
     readonly home: "inherit";

package/dist/services.d.ts

@@ -29,7 +29,7 @@
  *     a dev box with NO production access;
  *   - point the task at the disposable service's connection string ONLY;
  *   - keep real credentials OUT of the run (prod `DATABASE_URL`, cloud keys,
- *     `~/.ssh`) — pair it with the eval tier's `ephemeralEnv` (throwaway HOME +
+ *     `~/.ssh`) — pair it with the eval tier's `env: { kind: "ephemeral" }` (throwaway HOME +
  *     cleared env) to scrub them so the model has no real keys to misuse.
  * Treat it like running an untrusted script. A future increment adds an egress
  * wall (the skill reaches only the model + the service); until then that job is

package/dist/services.js

@@ -30,7 +30,7 @@
  *     a dev box with NO production access;
  *   - point the task at the disposable service's connection string ONLY;
  *   - keep real credentials OUT of the run (prod `DATABASE_URL`, cloud keys,
- *     `~/.ssh`) — pair it with the eval tier's `ephemeralEnv` (throwaway HOME +
+ *     `~/.ssh`) — pair it with the eval tier's `env: { kind: "ephemeral" }` (throwaway HOME +
  *     cleared env) to scrub them so the model has no real keys to misuse.
  * Treat it like running an untrusted script. A future increment adds an egress
  * wall (the skill reaches only the model + the service); until then that job is