vigiles 36.0.0 → 37.0.0

package/dist/eval.d.ts

@@ -1,3 +1,4 @@
+import { type RunEnv } from "./core/run-env.js";
 import type { HarnessLiveDriver } from "./core/live-driver.js";
 import type { AgentRunArgs, AgentRunner, EvalDriver, EvalUsage, ModelOutputParser, ParsedModelRun, RunOut } from "./core/eval-driver.js";
 export type { AgentRunArgs, AgentRunner, EvalDriver, EvalUsage, ModelOutputParser, ParsedModelRun, RunOut, } from "./core/eval-driver.js";
@@ -7,7 +8,7 @@ import { type EvalLockOptions } from "./eval-lock.js";
 import type { Check, CheckJSON } from "./check.js";
 import { type Comparison } from "./stats.js";
 import { type ToolIntercept } from "./tool-intercept.js";
-import { type ToolStub } from "./tool-stub.js";
+import { type StubCall, type ToolStub } from "./core/stub-rules.js";
 /** One arm of the comparison: fixture overrides + settings (hooks) for this arm. */
 export interface EvalArm {
     /** Files written on top of the base fixture for this arm. */
@@ -156,37 +157,41 @@ export interface EvalSpec<M extends Metrics> {
     /** Base backoff ms (doubled each retry). Default 1000. */
     readonly retryBackoffMs?: number;
     /**
-     * **Opt-in, default OFF.** Run each trial in an *ephemeral run environment* — a
-     * throwaway `$HOME` + scrubbed env, re-injecting only the harness's own auth (see
-     * `ephemeralRunEnv`). Running a model-driven skill/agent is itself a side
-     * effect (the *model*, not the author, chose the actions), so a `git push` /
-     * write to `~` should land in a disposable HOME, not the real `~/.gitconfig` /
-     * `~/.ssh` / `~/.aws`. This is the cross-platform STATE-protection floor (no
-     * kernel features), orthogonal to the bubblewrap host-confinement in
-     * `src/sandbox.ts`.
+     * **Where each trial runs — required, no default.**
      *
-     * **Ships default-OFF** because a too-narrow auth allowlist would silently break
-     * the real `claude` CLI's authentication; leaving it off keeps every existing
-     * eval (including one running right now) authenticating exactly as before. When
-     * absent / `false`, the per-trial env is the caller's (`{ ...process.env,
-     * ...arm.env }`) minus the harness's declared session identity, which no child
-     * run inherits in either mode. See `docs/safety.md` (ephemerality) and
-     * `research/cross-platform-sandboxing.md`.
-     */
-    readonly ephemeralEnv?: boolean;
-    /**
-     * **Tool stubs on PATH (rung R2).** A list of fake binaries to shadow on PATH
-     * for every trial, so a skill/hook/agent that calls a CLI tool (`gh`, `psql`,
-     * `redis-cli`, `z3`, …) and works with its RESULT can be tested against a
-     * **recorded / author-provided canned output** — no live service. vigiles writes
-     * one executable stub per {@link ToolStub} into a bin dir under the trial cwd and
-     * PREPENDS that dir to the run's PATH (both the default and the ephemeral env
-     * path), so the fake wins over the real binary.
+     * - `{ kind: "ephemeral" }` — a throwaway HOME (under the trial's temp dir) and
+     *   a scrubbed environment: the OS essentials, the harness's own auth
+     *   (`runEnv.keep`, plus its auth file copied in), the eval's own `VIGILES_*`
+     *   variables, and nothing else. `home: { kind: "files", files }` seeds that
+     *   HOME first (HOME-relative paths; the harness's auth file cannot be
+     *   overwritten). Running a model-driven skill is itself a side effect, so a
+     *   `git push` or a write to `~` lands in a disposable HOME.
+     * - `{ kind: "inherit", reason }` — your real HOME and environment, minus the
+     *   parent session's identity. `reason` is printed with the report.
      *
-     * The stubs are author/recorded fixtures, **never** model-synthesized — a
-     * synthesized tool output looks plausible but diverges from the real
-     * tool/version (false confidence). Absent → no change (the PATH is byte-identical
-     * to today). See {@link ToolStub} and `research/eval-coverage-and-isolation.md`.
+     * There is no default because both are wrong silently in one case: inheriting
+     * measures the author's machine (their config, their tokens), and an
+     * ephemeral run fails if a user's credential is not where the harness says it
+     * is — loudly, on trial 1, which is why it is the one to try first. See
+     * `docs/safety.md` (what a child run inherits).
+     */
+    readonly env: RunEnv;
+    /**
+     * **Tool stubs on PATH (rung R2)** — fake binaries that answer PER INVOCATION.
+     * Build each with {@link experimental_stub}: a list of rules over the argv
+     * tokens, first match wins. vigiles writes a shim per stub into a temp dir
+     * BESIDE the trial's working dir and puts it first on the run's PATH, so a
+     * skill/hook/agent — or a script it runs — that calls `gh`/`psql`/`git` gets
+     * the recorded answer instead of the real service.
+     *
+     * An invocation no rule answers is UNANSWERED: the model sees one neutral
+     * stderr line and a non-zero exit, no new trials start, the report lists the
+     * call (`ArmReport.unansweredStubCalls`), `vigiles eval` exits 2 and
+     * `--update` writes no lock. Every call is recorded in `RunContext.stubCalls`,
+     * so `experimental_stub.called(...)` can assert one happened.
+     *
+     * The answers are author/recorded fixtures, **never** model-synthesized — a
+     * synthesized tool output looks plausible but diverges from the real tool.
      */
     readonly stubs?: readonly ToolStub[];
     /**
@@ -236,6 +241,12 @@ export interface ArmReport {
     readonly stats: Record<string, MetricStat>;
     /** Cost / latency / token totals + means for this arm. */
     readonly usage: ArmUsage;
+    /**
+     * Stub invocations no rule answered, across this arm's trials. Non-empty means
+     * the arm measured at least one answer nobody wrote: `vigiles eval` exits 2 on
+     * it and `--update` refuses to record the lock. Empty when the eval has no stubs.
+     */
+    readonly unansweredStubCalls: readonly StubCall[];
 }
 export interface EvalReport {
     readonly name: string;
@@ -245,10 +256,21 @@ export interface EvalReport {
     readonly totalCostUsd: number;
     /** True if a `maxCostUsd` budget cap stopped the run before all trials ran. */
     readonly aborted: boolean;
+    /**
+     * The run environment the trials used, without seeded file contents — so an
+     * `inherit` run's reason travels with the report and is printed with it.
+     * Absent on a report built before `env` existed.
+     */
+    readonly env?: {
+        readonly kind: "ephemeral";
+    } | {
+        readonly kind: "inherit";
+        readonly reason: string;
+    };
 }
 /**
  * Resolve the environment a trial's subprocess actually runs with — the
- * SECURITY-CRITICAL decision behind `ephemeralEnv`. When `replaceEnv` is set, the
+ * SECURITY-CRITICAL decision behind `env: { kind: "ephemeral" }`. When `replaceEnv` is set, the
  * scrubbed `env` is the COMPLETE environment, so the real `$HOME` and inherited
  * secrets are DROPPED; otherwise `env` is an overlay on `base`. In BOTH, the
  * harness's declared session identity (`runEnv.sessionIdentity`) is removed: a
@@ -501,8 +523,17 @@ export declare function measureArms(spec: ArmsMeasureSpec): Promise<ArmsCheckRep
  * than vanilla" is a p-value, not a vibe. Reuses `welchTTest` from stats.ts.
  */
 export declare function compareCheck(report: ArmsCheckReport, baseline: string, arm: string, checkIndex: number): Comparison;
-/** Format a {@link CheckReport}: one line per check with its rate ± se and pass^k. */
+/**
+ * Format a {@link CheckReport}: one line per check with its rate ± se and
+ * pass^k, and the line saying `measure` ran with your HOME and environment
+ * (it has no `env` field; `INHERITED_BY_MEASURE`).
+ */
 export declare function formatCheckReport(report: CheckReport): string;
+/**
+ * Format an {@link ArmsCheckReport}: each arm's checks under `[arm: name]`, and
+ * the inherit line ONCE at the end — every arm ran the same way.
+ */
+export declare function formatArmsCheckReport(report: ArmsCheckReport): string;
 /**
  * The scored gate (Phase 4): throw if any check's measured rate is below its
  * threshold — the `measure` counterpart to `assertChecks` (strict). Reads the
@@ -652,6 +683,23 @@ export declare function seedEphemeralHome(throwawayHome: string, realHome: strin
 export declare function isRateLimited(out: RunOut): boolean;
 /** Map `worker` over `items` with at most `concurrency` in flight, order preserved. */
 export declare function runPool<T, R>(items: readonly T[], concurrency: number, worker: (item: T) => Promise<R>): Promise<R[]>;
+/**
+ * The author-facing failure for a report whose stubs left a call unanswered, or
+ * undefined. One owner for the rule — `vigiles eval`'s exit 2 and the lock's
+ * refusal both read it — so a consumer whose `assert` calls no helper still
+ * fails. Pure.
+ */
+export declare function unansweredInReport(report: Pick<EvalReport, "arms">, stubs: readonly ToolStub[]): string | undefined;
+/**
+ * `runEval` with its author-facing output, over any runner — so the real one and
+ * a test's fake share one path. Besides the run it prints, to stderr, what the
+ * caller must not miss: a skill arm that never activates, a stub call no rule
+ * answered (the same text `vigiles eval` fails with — a script that calls
+ * `runEval` itself, as the `withServices` examples do, gets the report back and
+ * nothing else, so the library is not silent about it either), and what the
+ * run spent.
+ */
+export declare function runEvalReporting<M extends Metrics>(spec: EvalSpec<M>, runner: AgentRunner): Promise<EvalReport>;
 export declare function runEvalWith<M extends Metrics>(input: EvalSpec<M>, runner: AgentRunner): Promise<EvalReport>;
 /** Format an eval report as a compact table for the console (mean ± se, pass^k). */
 export declare function formatEvalReport(report: EvalReport): string;