reeval 0.8.0 → 0.10.0

package/dist/{types-_LOFdPUf.d.ts → types-g17bRAzt.d.ts}

@@ -41,6 +41,10 @@ type EffortContrast = {
   agentMsP: number | null;
   /** The variant's and control's tool-call ranges don't overlap. */
   separated: boolean;
+  /** Their agent-time ranges don't overlap. */
+  timeSeparated: boolean;
+  /** Their token ranges don't overlap; null when the token test was omitted. */
+  tokensSeparated: boolean | null;
   /** On tool calls, the steadier measure; `timeVerdict` on agent time. */
   verdict: EffortVerdict;
   timeVerdict: EffortVerdict;
@@ -67,6 +71,21 @@ declare function effortVerdict(xs: number[], ys: number[]): {
   p: number | null;
   verdict: EffortVerdict;
 };
+/**
+ * The fewest attempts a side for "no overlap" to say anything: at 3 against 3 one ordering in
+ * ten separates by chance; at 1 against 1 every pair of unequal values does.
+ */
+declare const DIRECTIONAL_MIN = 3;
+/**
+ * The fewest runs per cell whose effort test can reach `EFFORT_ALPHA` (4 against 4: p = 2/70).
+ */
+declare const EFFORT_MIN_RUNS: number;
+type EffortMetric = 'tools' | 'agentS' | 'tokens';
+/**
+ * The metrics whose test is `insufficient` but whose ranges don't overlap at
+ * `DIRECTIONAL_MIN` a side or more: a direction, not an effect.
+ */
+declare function directional(c: EffortContrast): EffortMetric[];
 declare function contrastEffort(variant: EffortSample[], control: EffortSample[]): EffortContrast | null;
 /** The counted attempts of one cell of an eval, as effort samples. */
 declare const cellSamples: (e: Pick<EvalEntry, 'attempts'>, cellKey: string) => EffortSample[];
@@ -83,6 +102,7 @@ declare const VERDICT_TEXT: Record<EffortVerdict, (what: string) => string>;
  * `tools 5 vs 43 (×0.11), agent 21s vs 46s (×0.47): too few attempts to test` for a variant
  * against its control; with enough attempts, `fewer tool calls (p = 0.029)`. Tokens fold in
  * when both sides have usage samples; they are omitted (not `insufficient`) when either does not.
+ * A metric too small to test whose ranges don't overlap says so: `(×0.37, no overlap)`.
  */
 declare function effortLabel(c: EffortContrast): string;
 /** One cell's effort on its own: `tools 43 · agent 46s`, plus tokens when any attempt reported them. */
@@ -273,14 +293,18 @@ type AttemptRecord = {
     usage: TokenUsage | null;
   };
   timings: PhaseTimings;
-  /** Counts from the transcript, so a run's shape is visible without reading it. */
+  /** Counts from the transcript, so a run's shape is visible without reading it. `webSearches` includes refused searches. */
   activity: {
     tools: number;
     fileEdits: number;
-    /** Null when a turn with none couldn't report them (`unreported`), not a count of 0 that isn't one. */
+    /** Every `web-search` event, including refused ones. Null when a turn with none couldn't report them (`unreported`). */
     webSearches: number | null;
     webFetches: number | null;
     reasoningMs: number;
+    /** Transcript tool events with `stalled: true` (long shell or edit). */
+    stalledTools: number;
+    /** Longest tool `durationMs` in the transcript; null when none reported a duration. */
+    maxToolMs: number | null;
   };
   judgements: JudgementRecord[];
   /** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
@@ -291,6 +315,8 @@ type AttemptRecord = {
     durationMs: number;
     log?: FileRef;
   }>;
+  /** Every `expect` and `expect.soft` matcher call that settled, in call order; unset with none. */
+  assertions?: AssertionRecord[];
   /** Files under a dependency directory (node_modules/, .venv/, …) collapse to one entry with `count`. */
   changedFiles: Array<FileChange & {
     count?: number;
@@ -325,7 +351,7 @@ type EvalEntry = {
   outcome: Outcome;
   /** A placeholder, `evaluate.todo` or `group.todo`: it never runs, and its outcome is `skipped`. */
   todo: boolean;
-  /** pass / (pass + fail) over the gated cells' attempts; infra errors and skips excluded. */
+  /** pass@1, pass / (pass + fail), over the gated cells' attempts; infra errors and skips excluded. */
   passRate: number | null;
   cells: CellRecord[];
   attempts: AttemptRecord[];
@@ -333,11 +359,117 @@ type EvalEntry = {
    * The compare's questions, answered after its attempts by pairwise votes, one per question;
    * unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
    * may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
-   * (rates by cut, votes, ties, a failed call, a sample of reasons).
+   * (rates by cut, votes, ties, a failed call, a sample of reasons). Each answer's `input` is
+   * `<eval-slug>/questions/<n>.json`: every call its model was sent, per pair and order, secrets
+   * masked and screenshots as refs; unset where the eval's files weren't kept.
    */
   answers?: CompareAnswer[];
+  /** Set on each case of a pooled `evaluate.each`; `RunRecord.families` has the suite. */
+  family?: {
+    id: string;
+    name: string;
+    case: number;
+    cases: number;
+  };
+};
+/** A 95% interval on a pooled rate or difference. */
+type FamilyInterval = {
+  lo: number;
+  hi: number;
+  level: 0.95;
+  /**
+   * `wilson` / `newcombe-paired`: one counted attempt a case and side, so each case is one
+   * Bernoulli trial (or pair). `t-cases`: several, so a t interval over the cases' pass@1, with
+   * attempts within a case never counted as independent.
+   */
+  method: 'wilson' | 'newcombe-paired' | 't-cases';
+};
+/** One cell's pass@1 over a family's cases. */
+type FamilyRate = {
+  /** The cell's key, the same in every case. */
+  cell: string;
+  /** Cases with at least one counted attempt in this cell. */
+  cases: number;
+  /** Passes and counted attempts over those cases. */
+  pass: number;
+  counted: number;
+  /** The mean of the cases' pass@1; at one attempt a case, `pass / counted`. */
+  rate: number;
+  interval: FamilyInterval;
+};
+/**
+ * `lift` / `drop`: the interval excludes 0. `noise`: it doesn't. `insufficient`: fewer than 6
+ * discordant pairs (one attempt a case) or 6 paired cases (several), where no test can decide.
+ */
+type FamilyVerdict = 'lift' | 'drop' | 'noise' | 'insufficient';
+/** One effort metric over a family's paired cases. */
+type FamilyEffortMetric = {
+  /** Cases with a value on both sides. */
+  pairs: number;
+  /** The median over cases of the variant's per-case median, over the control's; null at 0. */
+  ratio: number | null;
+  /** Wilcoxon signed-rank over the per-case differences; null when every difference is 0. */
+  p: number | null;
+  /** `insufficient` below 6 non-zero differences, where the exact test can't reach 0.05. */
+  verdict: 'fewer' | 'more' | 'noise' | 'insufficient';
+};
+/** The variant's effort against the control's, paired by case. */
+type FamilyEffort = {
+  tools: FamilyEffortMetric;
+  agentMs: FamilyEffortMetric;
+  /** Over the cases where every counted attempt reported usage; omitted when none did. */
+  tokens?: FamilyEffortMetric;
+};
+/** A variant cell against its control over a family's cases, paired by case. */
+type FamilyLift = {
+  /** The variant cell's key. */
+  variant: string;
+  /** The control cell's key. */
+  vs: string;
+  /** Cases with counted attempts on both sides. */
+  pairs: number;
+  /** Mean pass@1 over the paired cases on each side. */
+  variantRate: number;
+  controlRate: number;
+  /** Set at one attempt a case and side: pairs where only one side passed. */
+  discordant?: {
+    variantOnly: number;
+    controlOnly: number;
+  };
+  /** Variant minus control, from -1 to 1. */
+  points: number;
+  interval: FamilyInterval;
+  /** McNemar's exact p (one attempt a case), else Wilcoxon signed-rank over the cases. */
+  p: number | null;
+  /** The interval spans 0. */
+  withinNoise: boolean;
+  verdict: FamilyVerdict;
+  /** The compare's `minLift`: on a family it gates this lower bound, not each case's. */
+  minLift?: number;
+  /** `fail` when `interval.lo` is below `minLift`. */
+  outcome: 'pass' | 'fail';
+  effort?: FamilyEffort;
 };
-/** A variant cell's pass rate against the control's, on the same harness and sandbox. */
+/** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an eval. */
+type FamilyRecord = {
+  /** `file › group › … › template`. */
+  id: string;
+  file: string;
+  group: string[];
+  /** The name template, e.g. `fixes $id`. */
+  name: string;
+  /** Cases in the list. */
+  cases: number;
+  /** Cases with an eval in this run; fewer when a filter or tag left some out. */
+  ran: number;
+  /** One per cell. */
+  rates: FamilyRate[];
+  /** One per variant cell, with `compare`. */
+  lifts: FamilyLift[];
+  /** `fail` when a lift is below its `minLift`. The cases still pass or fail on their own. */
+  outcome: 'pass' | 'fail';
+};
+/** A variant cell's pass@1 against the control's, on the same harness and sandbox. */
 type Lift = {
   /** The control cell's key. */
   vs: string;
@@ -350,7 +482,7 @@ type Lift = {
     pass: number;
     counted: number;
   };
-  /** Variant pass rate minus control pass rate, from -1 to 1. */
+  /** Variant pass@1 minus control pass@1, from -1 to 1. */
   points: number;
   /** 95% interval on the difference, by Newcombe's hybrid score method from two Wilson intervals. */
   interval: {
@@ -373,7 +505,7 @@ type FingerprintParts = {
   environment: string | null;
   image: string | null;
   agentCli: string | null;
-  /** Over the judge's model, threshold and version. */
+  /** Over the judge's model, threshold and version, and the config's `state` when it sets one. */
   judge: string;
   /** Over the attempt's context files. */
   context: string;
@@ -402,14 +534,39 @@ type CellRecord = Cell & {
   fail: number;
   infraError: number;
   skipped: number;
+  /** pass@1: pass / (pass + fail); infra errors and skips excluded. */
   passRate: number | null;
   /**
-   * By `gate`. `'all'` matches `{ passRate: 1 }` (infra only when nothing counted); `'none'`
+   * By `gate`. `'all'` (pass^k: every counted attempt passes) matches `{ passRate: 1 }`;
+   * `{ passRate }` is a pass@1 threshold. Infra only when nothing counted; `'none'`
    * reports the worst outcome. A variant below its `minLift` fails too.
    */
   outcome: Outcome;
   lift?: Lift;
+  /**
+   * Each check (assertion label) seen in any attempt of the eval, over this cell's counted
+   * attempts; on a variant cell, the control's beside it. Unset when no attempt recorded any.
+   */
+  checks?: CellCheck[];
+};
+/** How often one check passed: a counted attempt that never reached it counts it `notRun`. */
+type CheckTally = {
+  pass: number;
+  fail: number;
+  error: number;
+  notRun: number;
 };
+/**
+ * One check over a cell's counted attempts. `verdict` (variant cells): `discriminates` when it
+ * passes on every counted attempt of one side and none of the other; `both-pass` or `both-fail`
+ * when it does the same on both sides, so it can't tell them apart; `mixed` otherwise.
+ */
+type CellCheck = CheckTally & {
+  label: string;
+  control?: CheckTally;
+  verdict?: CheckVerdict;
+};
+type CheckVerdict = 'discriminates' | 'both-pass' | 'both-fail' | 'mixed';
 /** Counts attempts, except `evals`. */
 type RunSummary = {
   evals: number;
@@ -432,7 +589,7 @@ type Cost = {
   source: 'reported' | 'unreported';
 };
 type RunRecord = {
-  kind: 'es-run';
+  kind: 'reeval-run';
   schemaVersion: typeof SCHEMA_VERSION;
   /** Also the file name: `<UTC start>_<short sha>_<author or ci>`. */
   runId: string;
@@ -508,10 +665,13 @@ type RunRecord = {
   };
   summary: RunSummary;
   evals: EvalEntry[];
+  /** Each pooled `evaluate.each` that ran; unset when none did. */
+  families?: FamilyRecord[];
 };
 /** What ran and graded the agents: the machine, images, harnesses and judge. */
 type Stack = {
-  es: string;
+  /** The reeval version that wrote the run. */
+  reeval: string;
   /** `process.versions.node`: under Bun or Deno, the Node version they're compatible with. */
   node: string;
   /**
@@ -577,6 +737,8 @@ type Stack = {
     threshold: number;
     /** Changes when reeval, the AI SDK or the provider adapter (and so the judge prompt) change. */
     version: string;
+    /** What the judge sees by default, every key resolved; absent when the config and profile leave `state` at its defaults. */
+    state?: ResolvedJudgeStateOptions;
   };
   /** Union of every attempt's context files. */
   context: ContextFile[];
@@ -646,6 +808,8 @@ type HistoryEntry = {
     passRate: number | null;
     cells: HistoryCell[];
   }>;
+  /** The run's pooled families; unset when none ran. */
+  families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'>>;
 };
 //#endregion
 //#region src/types.d.ts
@@ -666,7 +830,8 @@ type FileEditEvent = {
 };
 /**
  * Normalised agent events. Each tool call is one event, merged with its result; `durationMs`
- * is null when the harness never reported the end.
+ * is wall time until the result arrived (Cursor's own `completed_at_ms` can be earlier). Null
+ * when the harness never reported the end. Long shell or edit calls set `stalled: true`.
  */
 type TranscriptEvent = EventTime & ({
   type: 'text' | 'reasoning';
@@ -674,19 +839,27 @@ type TranscriptEvent = EventTime & ({
   durationMs: number | null;
 } | (FileEditEvent & {
   durationMs: number | null;
+  /** True when a shell-like tool or edit ran at least two minutes. */
+  stalled?: boolean;
 }) | {
   type: 'web-search';
   toolCallId: string;
   query: string;
   /** How many sources the harness says it used; Cursor sends no URLs or titles. */
   references: number | null;
+  /** Whether the search succeeded. Absent means it did. */
+  ok?: boolean;
   durationMs: number | null;
+  /** True when a shell-like tool or edit ran at least two minutes. */
+  stalled?: boolean;
 } | {
   type: 'web-fetch';
   toolCallId: string;
   url: string;
   ok: boolean;
   durationMs: number | null;
+  /** True when a shell-like tool or edit ran at least two minutes. */
+  stalled?: boolean;
 } | {
   type: 'tool';
   toolCallId: string;
@@ -695,6 +868,8 @@ type TranscriptEvent = EventTime & ({
   output: unknown;
   ok: boolean;
   durationMs: number | null;
+  /** True when a shell-like tool or edit ran at least two minutes. */
+  stalled?: boolean;
 } | {
   type: 'error';
   message: string;
@@ -713,7 +888,9 @@ type ExecResult = {
 };
 /** Live tab size for `c.capture()`. Default 1280×720 when the call and cascade omit `viewport`. */
 type CaptureWindow = {
+  /** In CSS pixels. */
   width: number;
+  /** In CSS pixels. */
   height: number;
 };
 /**
@@ -775,6 +952,7 @@ type CaptureUse = {
  * pass it as the first argument to `c.capture`.
  */
 type CaptureOptions = {
+  /** The window and saved PNG size, or `'full-page'`. Default 1280×720. */
   viewport?: Viewport;
   /** How long to poll before failing. Default `'60s'`. */
   timeout?: Duration;
@@ -880,6 +1058,7 @@ type Turn = TurnData & {
   exec(command: string): Promise<ExecResult>;
   readFile(path: string): Promise<string | null>;
 };
+/** Any JSON value. */
 type JSONValue = null | boolean | number | string | JSONValue[] | {
   [key: string]: JSONValue;
 };
@@ -916,10 +1095,11 @@ type JudgeArtifactRef = {
  */
 type FakeJudgeModel = {
   kind: 'fake-judge';
+  /** The score (0–1) every call gets, or a function of the state it would see. */
   score: number | ((state: JudgeState) => number);
 };
 /**
- * `'provider/model'` (e.g. `'typesafe-ai/jev-1.13.0'`, `'google/gemini-3.8-flash'`), an AI SDK
+ * `'provider/model'` (e.g. `'typesafe-ai/jev-1.14.0'`, `'google/gemini-3.8-flash'`), an AI SDK
  * decision model object (per group, eval or call only), or a fake judge.
  */
 type JudgeModel = string | FakeJudgeModel | {
@@ -927,28 +1107,77 @@ type JudgeModel = string | FakeJudgeModel | {
 };
 /** Merged field by field down the cascade, so per-model settings can join as new fields. */
 type JudgeOptions = {
+  /** The model that grades. */
   model?: JudgeModel;
   /** Minimum score (0–1) for a `pass` verdict. */
   threshold?: number;
+  /** AI SDK provider options for the judge's calls, keyed by provider. */
   providerOptions?: Record<string, Record<string, JSONValue>>;
   /** The most judge calls to this model's provider at once, across the run. */
   maxConcurrent?: number;
   /**
-   * Which changed files the judge's diff shows: gitignore-style patterns over workspace paths,
-   * where the last matching pattern wins and a `!` pattern excludes. Default: every changed file.
+   * The judge's state: what the judge is given beside the question, not the workspace's or the
+   * attempt's state. Merged key by key down the cascade.
    */
-  include?: string[];
-  /** Workspace files the judge sees whole, as they are now, whether or not the agent changed them. */
-  files?: string[];
+  state?: JudgeStateOptions;
+};
+/**
+ * What the judge is given: each key is a key of the state in `judge-input.json`. An unset key
+ * keeps its default, which is what the judge gets with no `state` option at all.
+ */
+type JudgeStateOptions = {
+  /** The graded turn's prompt. Default true. */
+  task?: boolean;
+  /** The graded turn's final message, or `on`'s value. Default true. */
+  output?: boolean;
+  /** `status path` lines for the changed files, under `diff`'s patterns. Default true. */
+  files?: boolean;
+  /**
+   * The diff since the eval started. A list is gitignore-style patterns over workspace paths:
+   * the last matching pattern wins and `!` excludes. Default true.
+   */
+  diff?: boolean | string[];
+  /**
+   * Workspace files shown whole, as they are now, changed or not: exact paths or gitignore-style
+   * patterns (`'**'`, `'src/**'`; the last match wins, `!` excludes) over the files git sees in
+   * the workspace. A missing exact path is `null`; a pattern that matches nothing adds nothing.
+   * These files leave the diff. Default none.
+   */
+  contents?: string[];
+  /**
+   * The graded turn's transcript as text lines: `true` is messages and tool calls with tool
+   * output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output.
+   * Default false.
+   */
+  transcript?: boolean | 'tools' | 'full';
+  /**
+   * `'all'`: every turn up to the graded one under `turns`, each with its task, output and (with
+   * `transcript`) transcript. Default `'last'`.
+   */
+  turns?: 'last' | 'all';
+  /** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. Default false. */
+  checks?: boolean | string[];
+};
+/** `JudgeStateOptions` with every key set: what one judge call saw. */
+type ResolvedJudgeStateOptions = {
+  task: boolean;
+  output: boolean;
+  files: boolean;
+  diff: boolean | string[];
+  contents: string[];
+  transcript: boolean | 'tools' | 'full';
+  turns: 'last' | 'all';
+  checks: boolean | string[];
 };
 type JudgeCallOptions = JudgeOptions & {
   /** What to grade instead of the latest turn: another turn, or a value used as the output. */
   on?: Turn | JSONValue;
   /**
-   * Captures this call sends the judge as file parts. A `c.capture()` handle or its name.
-   * Call-only: the bytes are on this attempt. The provider that cannot take files fails the call.
+   * Captures this call sends the judge as file parts: `c.capture()` handles or their names, or
+   * `true` for every capture so far. Call-only: the bytes are on this attempt. Jev takes PNG
+   * captures; a provider that cannot take files fails the call.
    */
-  artifacts?: Array<Capture | string>;
+  artifacts?: true | Array<Capture | string>;
 };
 type Verdict = {
   verdict: 'pass' | 'fail';
@@ -971,12 +1200,16 @@ type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel'> & {
   score: null;
   error: string;
 };
+/**
+ * What an eval's body gets. The sandbox and the agent session start on the first call that needs
+ * them (`send`, `exec`, `readFile`, `writeFile`, `serve` or `capture`), once however many ask.
+ */
 type AttemptContext = {
-  /** Sends the next turn of this eval's agent session, starting the sandbox on first use. */
+  /** Sends the next turn of this eval's agent session. */
   send(prompt: string): Promise<Turn>;
   /**
    * Asks the judge a yes/no question about the latest turn (or `options.on`).
-   * `options.artifacts` is a list of `c.capture()` handles or names.
+   * `options.artifacts` is a list of `c.capture()` handles or names, or `true` for all of them.
    */
   judge(question: string, options?: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
   exec(command: string): Promise<ExecResult>;
@@ -995,16 +1228,16 @@ type AttemptContext = {
    * Screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`.
    * First argument is the container port or a `ServeHandle` from `c.serve`.
    * `instructions` runs a vision journey on that same tab first.
-   * Needs a sandbox, same as `c.exec`.
    */
   capture(target: number | ServeHandle, options?: CaptureCall): Promise<Capture>;
   /** Workspace-relative, or `~/` for the home directory; `null` when there's no such file. */
   readFile(path: string): Promise<string | null>;
   /**
-   * Starts the sandbox and the agent session without a turn, for checks on what the environment
-   * set up. `c.send()` starts them too; calling both starts them once.
+   * Writes a file into the sandbox (workspace-relative, or `~/`), creating its directories.
+   * Later turns' `diff` leaves it out, as it does the environment's files, but shows any change
+   * made to it afterwards.
    */
-  sandbox(): Promise<Pick<AttemptContext, 'exec' | 'readFile'>>;
+  writeFile(path: string, contents: string | Uint8Array): Promise<void>;
   /** Every turn so far, oldest first. */
   readonly turns: readonly Turn[];
   /** Aborted on timeout or Ctrl-C. */
@@ -1017,12 +1250,14 @@ type AttemptContext = {
   };
 };
 type EvalFn = (c: AttemptContext) => Promise<void>;
+/** A canned agent for tests and dry runs, made by `fakeHarness()` in `reeval/harness`. */
 type FakeHarnessDescriptor = {
   kind: 'fake';
   /** Cell id; defaults to `fake`. */
   id?: string;
   /** Files the fake agent writes into the sandbox on every turn. */
   files?: Record<string, string>;
+  /** The fake agent's final message on every turn; default `Done.`. */
   text?: string;
   /** Simulated turn length, honours the abort signal. */
   delayMs?: number;
@@ -1074,10 +1309,11 @@ type FakeHarnessDescriptor = {
    * Behave like this agent: its image, and where it reads rules and skills (`CLAUDE.md` and
    * `.claude/skills` for `claude-code`, `~/.gemini/config/skills` for `antigravity`,
    * `~/.config/opencode/skills` for `opencode`, Goose's private home for `goose`,
-   * `~/.pi/agent/skills` for `pi`, `~/.agents/skills` for `codex`). Defaults to the
-   * fake's own (`AGENTS.md`, `.agents/skills`) in Cursor's image.
+   * `~/.pi/agent/skills` for `pi`, `~/.agents/skills` for `codex`, fx's private home for
+   * `fx`, Grok's private home for `grok-build`). Defaults to the fake's own (`AGENTS.md`,
+   * `.agents/skills`) in Cursor's image.
    */
-  layout?: 'cursor' | 'claude-code' | 'antigravity' | 'opencode' | 'goose' | 'pi' | 'codex';
+  layout?: 'cursor' | 'claude-code' | 'antigravity' | 'opencode' | 'goose' | 'pi' | 'codex' | 'fx' | 'grok-build';
 };
 /**
  * One action of a scripted fake agent. `exec` is reported as a `shell` tool call whose output is
@@ -1125,16 +1361,18 @@ type CursorHarnessDescriptor = {
    */
   startTimeout?: Duration;
 };
+/** Reasoning effort the Claude Code Agent SDK `/effort` control takes. */
+type ClaudeCodeEffort = 'low' | 'medium' | 'high' | 'xhigh' | 'max';
 /** Claude Code through `@ai-sdk/harness-claude-code`, made by `claudeCode()`. */
 type ClaudeCodeHarnessDescriptor = {
   kind: 'claude-code';
   /** A model id or alias the `claude` CLI accepts, e.g. `claude-sonnet-5` or `sonnet`. */
   model: string;
   /**
-   * Reasoning effort the Agent SDK `/effort` control takes, e.g. `low`, `medium`, `high`,
+   * Reasoning effort the Agent SDK `/effort` control takes: `low`, `medium`, `high`,
    * `xhigh` (Extra in the UI) or `max`.
    */
-  effort?: string;
+  effort?: ClaudeCodeEffort;
   /** Cell id; defaults to `claude-code:<model>`, or `claude-code:<model>[effort=<value>]` when `effort` is set. */
   id?: string;
   /** The most Claude Code attempts at once, across the run, e.g. for the account's rate limits. */
@@ -1261,28 +1499,82 @@ type CodexHarnessDescriptor = {
    */
   startTimeout?: Duration;
 };
+/**
+ * fx through `@reeval/harness-fx` (`fx acp`), made by `fx()`. It runs on Vercel AI Gateway or
+ * OpenAI's API by the model id, and reports each turn's tokens.
+ */
+type FxHarnessDescriptor = {
+  kind: 'fx';
+  /**
+   * A Vercel AI Gateway model id, `<provider>/<model>`, run on fx's `gateway` provider with
+   * `AI_GATEWAY_API_KEY`, e.g. `anthropic/claude-sonnet-5`; or a bare OpenAI model id, run on
+   * OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-4.1`.
+   */
+  model: string;
+  /** Cell id; defaults to `fx:<model>`. */
+  id?: string;
+  /** The most fx attempts at once, across the run, e.g. for the key's rate limits. */
+  maxConcurrent?: number;
+  /**
+   * How long the harness's session may take to become ready (its bootstrap and first session)
+   * before the attempt is an infra error, after one retry; default 90s.
+   */
+  startTimeout?: Duration;
+};
+/**
+ * xAI's Grok Build through `@reeval/harness-grok-build` (`grok agent stdio`), made by
+ * `grokBuild()`. It runs xAI models on an xAI API key and reports each turn's tokens.
+ */
+type GrokBuildHarnessDescriptor = {
+  kind: 'grok-build';
+  /** An xAI model id Grok Build runs on the xAI API, e.g. `grok-4.6`. */
+  model: string;
+  /**
+   * Reasoning effort as Grok's `default_reasoning_effort`, e.g. `low`, `medium` or `high`, for a
+   * model that takes one.
+   */
+  effort?: string;
+  /** Cell id; defaults to `grok-build:<model>`, or `grok-build:<model>[effort=<value>]` when `effort` is set. */
+  id?: string;
+  /** The most Grok Build attempts at once, across the run, e.g. for the key's rate limits. */
+  maxConcurrent?: number;
+  /**
+   * How long the harness's session may take to become ready (its bootstrap and first session)
+   * before the attempt is an infra error, after one retry; default 90s.
+   */
+  startTimeout?: Duration;
+};
 /** A real coding agent, driven through its AI SDK harness adapter. */
-type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor;
+type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor | FxHarnessDescriptor | GrokBuildHarnessDescriptor;
+/** The agent an attempt runs: a real harness, or the fake one. */
 type HarnessDescriptor = FakeHarnessDescriptor | AgentHarnessDescriptor;
+/** A temp directory on the host instead of a container, made by `fakeSandbox()`. */
 type FakeSandboxDescriptor = {
   kind: 'fake';
+  /** Cell id; defaults to `fake`. */
   id?: string;
   /** Parent directory for the per-eval workspace (defaults to the OS temp dir). */
   dir?: string;
   /** The most attempts on this sandbox kind at once, across the run. */
   maxConcurrent?: number;
 };
+/** A local Docker container, made by `docker()`, or a Podman one, made by `podman()`. */
 type DockerSandboxDescriptor = {
   kind: 'docker';
-  /** Cell id; defaults to `docker`, or `docker:<image>` for other images, plus an install hash. */
+  /** The container CLI the sandbox runs on: `podman` for `podman()`, Docker when unset. */
+  engine?: 'docker' | 'podman';
+  /**
+   * Cell id; defaults to `docker` (`podman`), or `docker:<image>` for other images, plus an
+   * install hash.
+   */
   id?: string;
   /** An image that already has the harness baked in, used as is. */
   image?: string;
   /** A Debian or Ubuntu based toolchain image; reeval bakes Node and the harness on top, cached. */
   from?: string;
   /**
-   * An absolute path to a Dockerfile (`docker()` resolves it against the calling file), built from
-   * its directory with `REEVAL_BASE` set to reeval's sandbox image and cached by content hash.
+   * A Dockerfile, absolute or relative to the directory of the file it's written in, built from its
+   * directory with `REEVAL_BASE` set to reeval's sandbox image and cached by content hash.
    */
   dockerfile?: string;
   /** Baked onto the sandbox image in one build container: in order, no secrets, no workspace. */
@@ -1296,8 +1588,13 @@ type DockerSandboxDescriptor = {
   root?: boolean;
   /** Linux capabilities to add back, e.g. `['NET_RAW']`; every other one is dropped. */
   capabilities?: string[];
-  /** The container's memory, with no swap on top: `'4g'` (default), `'512m'`, or bytes. */
+  /** The container's memory: `'4g'` (default), `'512m'`, or bytes. */
   memory?: string | number;
+  /**
+   * Swap on top of `memory`, in the same units. Default: as much as `memory`; `0` makes `memory`
+   * the hard ceiling. A host without swap has none to give.
+   */
+  swap?: string | number;
   /** CPUs the container may use, fractions allowed. Default 2, capped at the Docker host's. */
   cpus?: number;
   /** Processes and threads at once. Default 1024. */
@@ -1335,13 +1632,17 @@ type SecretOptions = {
 /** `secret.template('Bearer ${TOKEN}')`: host variables substituted into a string, for MCP headers. */
 type SecretTemplate = {
   kind: 'secret-template';
+  /** The string, with `${NAME}` for each host variable to substitute. */
   template: string;
 };
+/** A variable's value: a literal string, or a host secret. */
 type EnvValue = string | Secret;
+/** Environment variables by name. */
 type EnvVars = Record<string, EnvValue>;
 /** A file on the host, read when an attempt starts. Made by the `file` helper. */
 type HostFile = {
   kind: 'file';
+  /** Its path on the host, absolute or relative to the directory of the file it's written in. */
   path: string;
 };
 /** Options for the `directory` helper. */
@@ -1359,62 +1660,92 @@ type DirectoryOptions = {
   /** `false` walks into `DEFAULT_DIRECTORY_IGNORES` too. `.git` is always skipped. */
   defaultIgnores?: boolean;
 };
+/** An MCP server the agent gets, made by the `mcp` helper: a command in the sandbox, or a URL. */
 type McpServer = {
   kind: 'mcp';
   transport: 'stdio';
+  /** The command that starts the server in the sandbox. */
   command: string;
   args: string[];
+  /** Variables the server process gets. */
   env: EnvVars;
 } | {
   kind: 'mcp';
+  /** Streamable HTTP, or the older SSE transport. */
   transport: 'http' | 'sse';
   url: string;
+  /** Request headers, with secrets brokered by the egress gateway. */
   headers: Record<string, string | Secret | SecretTemplate>;
 };
 /** `harness` (default): the agent's own skills, via its adapter. `workspace`: its repo skill dir. */
 type SkillPlacement = 'harness' | 'workspace';
+/** Skills to place for the agent, made by the `skill` helper. */
 type SkillSource = {
   kind: 'skill';
   placement: SkillPlacement;
 } & ({
   source: 'local';
+  /** A host directory with a `SKILL.md`, or a directory of them. */
   path: string;
+  /** The skill names to keep; default every one found. */
   only?: string[];
+  /** Paths in each skill directory to leave out, gitignore-style. */
   exclude?: string[];
 } | {
   source: 'git';
+  /** As written: a URL, `github:owner/repo`, or a GitHub `/tree/<ref>/<path>` URL. */
   repo: string;
+  /** A branch, tag or commit SHA; unset is the remote's `HEAD`. */
   ref?: string;
+  /** The directory in the repository to read skills from. */
   path?: string;
+  /** The skill names to keep; default every one found. */
   only?: string[];
+  /** Paths in each skill directory to leave out, gitignore-style. */
   exclude?: string[];
 } | {
   source: 'registry';
+  /** A skills.sh id, `owner/repo@skill`. */
   id: string;
+  /** A git ref of the skill's repository. */
   version?: string;
 } | {
   source: 'npm';
+  /** A pinned npm package spec. */
   spec: string;
+  /** The directory in the package to read skills from; default `skills`. */
   path: string;
+  /** The skill names to keep; default every one found. */
   only?: string[];
+  /** Paths in each skill directory to leave out, gitignore-style. */
   exclude?: string[];
 } | {
   source: 'inline';
+  /** The skill's name, as in `SKILL.md`'s frontmatter. */
   name: string;
+  /** The skill's description, as in `SKILL.md`'s frontmatter. */
   description: string;
+  /** The body of `SKILL.md`. */
   content: string;
+  /** Other files in the skill directory, by relative path. */
   files: Record<string, string>;
 });
 /** A `docker({ install })` command, baked into a cached image layer: no secrets, root, cwd `/`. */
 type InstallStep = string | {
+  /** The shell command. */
   run: string;
   timeout?: Duration;
 };
-/** Runs in every attempt before the agent starts, with secrets, outside the eval timeout. */
-/** A shell command, or `{ run, timeout?, cwd? }`; `cwd` is relative to the workspace or absolute. */
+/**
+ * Runs in every attempt before the agent starts, with secrets, outside the eval timeout: a shell
+ * command, or `{ run, timeout?, cwd? }`.
+ */
 type SetupStep = string | {
+  /** The shell command. */
   run: string;
+  /** Default 5m. */
   timeout?: Duration;
+  /** Relative to the workspace, or absolute; default the workspace. */
   cwd?: string;
 };
 /**
@@ -1434,6 +1765,7 @@ type GitWorkspace = {
   kind: 'git';
   /** As written: a URL, `github:owner/repo`, or a local path. */
   repo: string;
+  /** What reeval fetches: the clone URL, or an absolute path for a local repository. */
   url: string;
   /** A branch, tag or commit SHA; unset is the remote's `HEAD`. */
   ref?: string;
@@ -1459,6 +1791,7 @@ type EnvironmentSpec = {
   sandbox?: SandboxDescriptor;
   /** The code the workspace starts as, before `files`, `rules` and `setup`; a child's replaces its parent's. */
   workspace?: GitWorkspace;
+  /** Variables and secrets for the sandbox, merged by name over the parent's. */
   env?: EnvVars;
   /** Replaces the inherited policy; an `extend` child's replaces its parent's. */
   egress?: EgressPolicy;
@@ -1466,13 +1799,18 @@ type EnvironmentSpec = {
   files?: Record<string, string | HostFile>;
   /** Project rules, written where the harness's agent reads them: `AGENTS.md`, or `CLAUDE.md` for Claude Code. */
   rules?: string | HostFile;
+  /** Shell commands run in every attempt before the agent starts; a child's run after its parent's. */
   setup?: SetupStep[];
+  /** Skills placed for the agent. */
   skills?: SkillSource[];
+  /** MCP servers the agent gets, by name. */
   mcp?: Record<string, McpServer>;
 };
 /** One environment in an `extend` chain. */
 type EnvironmentLayer = {
+  /** The name passed to `defineEnvironment` or `extend`. */
   name: string;
+  /** The spec as that layer wrote it, before merging. */
   spec: EnvironmentSpec;
 };
 /**
@@ -1481,7 +1819,9 @@ type EnvironmentLayer = {
  */
 type Environment = {
   readonly kind: 'environment';
+  /** The name it was defined or extended under, which names its cells. */
   readonly name: string;
+  /** Its `extend` chain, root first. */
   readonly layers: readonly EnvironmentLayer[];
   /**
    * A child environment layered on this one; wherever it sets a key this one has, it wins.
@@ -1617,14 +1957,71 @@ type CompareOptions<E = Environment> = {
 };
 /**
  * A question for pairwise judgement: its text, or the text and the `'provider/model'` that
- * answers it. Without a model, `analysis.model` answers, else the judge's when it's a language model.
+ * answers it. Without a model, the cascaded `question.model` answers, else `analysis.model`, else the
+ * judge's when it's a language model.
  * `across` is the factor it varies when pairing; without it (and without a one-factor
- * `compare.across`) both pair kinds run.
+ * `compare.across`) both pair kinds run. `state` and `artifacts` choose what each side shows.
  */
 type CompareQuestion = string | {
   question: string;
+  /** `'provider/model'` that answers it, over the cascaded `question.model`. */
   model?: string;
+  /** The factor it varies when pairing. */
   across?: PairFactor;
+  /** What each side shows, over the cascaded `question.state`, key by key. */
+  state?: QuestionStateOptions;
+  /** Each side's own captures, by name; `true` sends all of them. Replaces `question.artifacts`. */
+  artifacts?: true | string[];
+};
+/**
+ * What each side of a compare question shows: the keys of `judge.state` but `contents` (the
+ * attempts' workspaces are gone when questions are asked). An unset key keeps its default, which
+ * is what a question shows with no `state` at all: the task and the diff.
+ */
+type QuestionStateOptions = {
+  /** The prompts the attempt was sent, once for both sides when they match. Default true. */
+  task?: boolean;
+  /** The final message: the agent's own account of its work. Default false. */
+  output?: boolean;
+  /** `status path` lines for the changed files, under `diff`'s patterns. Default false. */
+  files?: boolean;
+  /**
+   * The diff since the eval started. A list is gitignore-style patterns over workspace paths:
+   * the last matching pattern wins and `!` excludes. Default true.
+   */
+  diff?: boolean | string[];
+  /**
+   * The transcript as text lines: `true` is messages and tool calls with tool output clipped,
+   * `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each
+   * harness's tools, so a pair of two harnesses isn't blind. Default false.
+   */
+  transcript?: boolean | 'tools' | 'full';
+  /**
+   * `'all'`: every turn under `turns`, each with its task, output and (with `transcript`)
+   * transcript; `'last'`: the last turn only. Default `'all'`.
+   */
+  turns?: 'last' | 'all';
+  /** `c.exec` results; a list shows only those commands, as passed to `c.exec`. Default false. */
+  checks?: boolean | string[];
+};
+/** `QuestionStateOptions` with every key set: what each side of one question showed. */
+type ResolvedQuestionStateOptions = {
+  task: boolean;
+  output: boolean;
+  files: boolean;
+  diff: boolean | string[];
+  transcript: boolean | 'tools' | 'full';
+  turns: 'last' | 'all';
+  checks: boolean | string[];
+};
+/** Defaults for every compare question: `model` and `artifacts` replaced, `state` merged key by key. */
+type QuestionOptions = {
+  /** `'provider/model'` that answers every question below it without its own `model`. */
+  model?: string;
+  /** What each side shows. */
+  state?: QuestionStateOptions;
+  /** Each side's own captures, by name; `true` sends all of them. Default none. */
+  artifacts?: true | string[];
 };
 /** A matrix factor a compare question varies when pairing; the others are held. */
 type PairFactor = 'environment' | 'harness';
@@ -1649,6 +2046,11 @@ type PairVote = {
   swapped: boolean;
   /** The model's short reason for this pick; omitted when it gave none. Capped. */
   reason?: string;
+  /** The captures each named side showed, by name; set when the question sends `artifacts`. */
+  artifacts?: {
+    a: string[];
+    b: string[];
+  };
 };
 /** A compare question's answer: every vote, and each side's wins over decided votes per cut. */
 type CompareAnswer = {
@@ -1657,6 +2059,10 @@ type CompareAnswer = {
   model: string;
   /** The model id the provider says served the calls, when it reports one. */
   resolvedModel?: string;
+  /** What each side showed, every key resolved; unset on answers from before questions took `state`, which showed the defaults. */
+  stateOptions?: ResolvedQuestionStateOptions;
+  /** The captures it sent of each side, as asked: `true` for all, or their names. */
+  artifacts?: true | string[];
   votes: PairVote[];
   /** Wins over decided pairs, per named side, per cut. */
   rates: Array<{
@@ -1667,11 +2073,15 @@ type CompareAnswer = {
   }>;
   /** Why some or all of its votes are missing: the model call failed. Outcomes never change. */
   error?: string;
+  /** `questions/<n>.json` under the eval's directory: what the model was sent, per pair and order. */
+  input?: FileRef;
 };
+/** An option a profile's `limit` can narrow, and `pin` can protect. */
 type PinnableKey = 'harness' | 'runs' | 'compare';
 /**
  * How a cell's attempts decide its outcome: `'all'` (the default) passes only when every attempt
- * passes; `{ passRate }` passes when at least that share of its passes and fails are passes.
+ * passes (pass^k); `{ passRate }` passes when its pass@1 (passes over passes and fails) is at
+ * least that share.
  */
 type GatePolicy = 'all' | {
   passRate: number;
@@ -1682,7 +2092,7 @@ type CellGate = GatePolicy | 'none';
 type CompareSpec<E = Environment> = CompareOptions<E>;
 /** Options other than `environment` and `compare`, which are exclusive of each other. */
 type EvalOptionFields = {
-  /** The only list: each harness is its own cell. */
+  /** Which agents run the eval; each harness is its own cell. */
   harness?: HarnessOption;
   /**
    * The default machine when the resolved environment omits `sandbox`. Cascades like `harness`.
@@ -1700,11 +2110,15 @@ type EvalOptionFields = {
    * environment, then groups and the eval.
    */
   egress?: EgressPolicy;
+  /** Defaults for `c.judge()`, merged field by field down the cascade. */
   judge?: JudgeOptions;
   /** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
   capture?: CaptureOptions;
+  /** Defaults for every compare question: `model`, and `state` merged key by key, down the cascade. */
+  question?: QuestionOptions;
   /** Per attempt. */
   timeout?: Duration;
+  /** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
   tags?: string[];
   /** Keys a profile's `limit` may not narrow for this group or eval. */
   pin?: PinnableKey[];
@@ -1722,20 +2136,36 @@ type EnvironmentOrCompare<E> = {
 } | {
   /** Forbidden beside `compare.baseline`, which names the control. */
   environment?: never;
+  /** Environments to run side by side; `baseline` is the control. */
   compare: CompareSpec<E> & {
     baseline: E;
   };
 } | {
   /** The control: the compare has no `baseline`. */
   environment?: E;
+  /** Environments to run side by side against `environment`, the control. */
   compare: CompareSpec<E> & {
     baseline?: undefined;
   };
 };
 type EvalOptions = EvalOptionFields & EnvironmentOrCompare<Environment>;
 type GroupOptions = EvalOptions;
+/**
+ * `EvalOptions` in their data forms, as JSON carries them: the harness as data, the judge's model
+ * by name or a fake judge, and environments without `extend`. The same `environment` / compare
+ * object split as `EvalOptions`.
+ */
+type EvalOptionsData = Omit<EvalOptionFields, 'harness' | 'judge'> & {
+  /** Which agents run the eval; each harness is its own cell. */
+  harness?: HarnessData;
+  /** Defaults for `c.judge()`, merged field by field down the cascade. */
+  judge?: ConfigJudgeOptions;
+} & EnvironmentOrCompare<EnvironmentData>;
+/** An `Environment` as data, as JSON carries it: without `extend` / `extendEnvironment`. */
+type EnvironmentData = Omit<Environment, 'extendEnvironment' | 'extend'>;
 /** The config file is plain data sent to workers, so its judge model can't be an object. */
 type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
+  /** `'provider/model'`, or a fake judge. */
   model?: string | FakeJudgeModel;
 };
 type HistoryConfig = {
@@ -1746,22 +2176,35 @@ type HistoryConfig = {
 };
 /** Run-level caps. `attempts` is checked against the plan; `duration` while running. */
 type Budget = {
+  /** The most attempts the plan may have. */
   attempts?: number;
+  /** How long the run may go on, from the first attempt's start. */
   duration?: Duration;
 };
 /** What evals get when they don't say otherwise: under groups and evals. */
 type ProfileDefaults = {
+  /** The agents every eval runs on, unless a group or eval replaces them. */
   harness?: HarnessDescriptor | HarnessDescriptor[];
   /** The default machine when the resolved environment omits `sandbox`. */
   sandbox?: SandboxDescriptor;
+  /** What every eval runs in when it names no environment. */
   environment?: Environment;
+  /** Attempts per cell. */
   runs?: number;
+  /** How each cell's attempts decide its outcome. */
   gate?: GatePolicy;
+  /** Variables and secrets for the sandbox, merged by name over the config's. */
   env?: EnvVars;
+  /** Where the sandbox may connect; replaces the config's policy. */
   egress?: EgressPolicy;
+  /** Defaults for `c.judge()`, merged field by field over the config's. */
   judge?: ConfigJudgeOptions;
+  /** Per attempt. */
   timeout?: Duration;
+  /** Screenshot defaults for `c.capture()`, shallow-merged over the config's. */
   capture?: CaptureOptions;
+  /** Defaults for every compare question, merged over the config's. */
+  question?: QuestionOptions;
 };
 /** Over every eval, and it can only narrow; `pin` on a group or eval skips it. */
 type ProfileLimit = {
@@ -1776,10 +2219,15 @@ type ProfileLimit = {
 };
 /** How a run is sized: chosen with `--profile`, `REEVAL_PROFILE` or `defaultProfile`. */
 type Profile = {
+  /** What evals get when they don't say otherwise: under groups and evals. */
   defaults?: ProfileDefaults;
+  /** Caps over every eval, which only narrow; `pin` on a group or eval skips them. */
   limit?: ProfileLimit;
+  /** Judge settings for this profile, the same as `defaults.judge`, which wins field by field. */
   judge?: ConfigJudgeOptions;
+  /** Merged over the config's `budget`. */
   budget?: Budget;
+  /** When this profile's runs are copied into the history folder. */
   record?: RecordMode;
   /** Tag filters for this profile, like `--tag` (`'!slow'` excludes). */
   tags?: string[];
@@ -1823,12 +2271,48 @@ type FakePick = FakeVote | {
   winner: FakeVote;
   reason?: string;
 };
-/** What a compare question's model is shown: the prompt, and the two diffs in slot order. */
+/** What a compare question's model is shown: the prompt, and the two sides in slot order. */
 type BlindPair = {
   question: string;
   prompt: string;
-  a: string;
-  b: string;
+  a: BlindSide;
+  b: BlindSide;
+};
+/** A turn of one side, with the keys its question's `state` turned on. */
+type BlindTurn = {
+  task?: string;
+  output?: string;
+  transcript?: string;
+};
+/**
+ * One side of a blinded pair: only the keys its question's `state` turned on. With
+ * `turns: 'all'`, `task`, `output` and `transcript` are under `turns`; a task both sides share is
+ * the pair's prompt's, not either side's.
+ */
+type BlindSide = BlindTurn & {
+  turns?: BlindTurn[];
+  files?: string[];
+  /** The diff as the model sees it: dependency and binary noise dropped. */
+  diff?: string;
+  /** A command the other side ran and this one didn't is `{ command, notRun: true }`. */
+  checks?: Array<{
+    command: string;
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+  } | {
+    command: string;
+    notRun: true;
+  }>;
+  /** Its captures, in the order sent, labelled by position; a name it lacks is `missing`. */
+  artifacts?: Array<{
+    label: string;
+    mediaType: 'image/png';
+    bytes: number;
+  } | {
+    label: string;
+    missing: true;
+  }>;
 };
 /** `reeval analyze`: a language model (not an evaluation model) and what matters in this project. */
 type AnalysisConfig = {
@@ -1846,33 +2330,46 @@ type AnalysisConfig = {
   /** The most tokens the model may answer with (default 8000). reeval has no prices, so tokens are the cap. */
   maxOutputTokens?: number;
 };
-type EsConfig = {
+type ReevalConfig = {
+  /** Globs of eval files, relative to the config's directory; default every `.eval.ts` under `evals`. */
   include?: string[];
+  /** The agents every eval runs on, unless a group or eval replaces them; default Cursor. */
   harness?: HarnessDescriptor | HarnessDescriptor[];
   /**
    * The default machine when the resolved environment omits `sandbox`. Not an environment:
    * it does not invent skills, files or a name. Implicit `docker()` when nobody sets one.
    */
   sandbox?: SandboxDescriptor;
+  /** What every eval runs in when it names no environment, and a baseline-less compare's control. */
   environment?: Environment;
+  /** Attempts per cell; default 1. */
   runs?: number;
   /** How each cell's attempts decide its outcome; default `'all'`. */
   gate?: GatePolicy;
+  /** Variables and secrets for the sandbox, merged by name down the cascade. */
   env?: EnvVars;
+  /** Where the sandbox may connect; default `'allow-all'`. */
   egress?: EgressPolicy;
+  /** Defaults for `c.judge()`, merged field by field down the cascade. */
   judge?: ConfigJudgeOptions;
+  /** Per attempt; default `'15m'`. */
   timeout?: Duration;
   /**
    * Screenshot defaults for `c.capture()`. `REEVAL_CAPTURE_BROWSER=0` refuses the host
    * binary when `c.capture()` runs; it does not change this object.
    */
   capture?: CaptureOptions;
+  /** Defaults for every compare question: the model that answers it and what each side shows. Merged down the cascade. */
+  question?: QuestionOptions;
   /** How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it. */
   concurrency?: number;
   /** When runs are copied into the history folder: `reeval record` only, in CI, or on every run. */
   record?: RecordMode;
+  /** The history folder, and which attempts keep their artifacts there. */
   history?: HistoryConfig;
+  /** Caps on a run; a profile's `budget` is merged over it. */
   budget?: Budget;
+  /** Named presets of `defaults`, `limit`, `judge`, `budget`, `record` and `tags`. */
   profiles?: Record<string, Profile>;
   /** The profile when neither `--profile` nor `REEVAL_PROFILE` names one: `ci` when `CI` is set. */
   defaultProfile?: {
@@ -1891,6 +2388,7 @@ type EsConfig = {
     keep?: number;
     traceBufferMb?: number;
   };
+  /** `reeval analyze`, compare questions without a model, and `agent`'s notes during a run. */
   analysis?: AnalysisConfig;
 };
 /** What a `defineConfig` callback gets: the environment helpers plus the run's context. */
@@ -1921,11 +2419,23 @@ type PrepareFailureRecord = {
   log: string | null;
   setBy: string[];
 };
-/** Why an attempt failed; `prepare` when its sandbox couldn't be prepared. */
+/**
+ * The sandbox ran out of a resource during the attempt: `oom` is the kernel killing `kills`
+ * processes at its memory limit. The attempt is then an infra error, whatever its checks said.
+ */
+type ResourceFailureRecord = {
+  cause: 'oom';
+  kills: number;
+};
+/**
+ * Why an attempt failed; `prepare` when its sandbox couldn't be prepared, `resource` when it ran
+ * out of one.
+ */
 type AttemptError = {
   kind: ErrorKind;
   message: string;
   prepare?: PrepareFailureRecord;
+  resource?: ResourceFailureRecord;
 };
 /** Where an eval's time went, in ms. `agentMs` is the harness turns; `diffMs` reading changes. */
 type PhaseTimings = {
@@ -2060,6 +2570,25 @@ type StartRetry = {
   /** How long it waited before giving up on that start. */
   waitedMs: number;
 };
+/** One `expect` or `expect.soft` matcher call an attempt made. */
+type AssertionRecord = {
+  /**
+   * `expect`'s second argument, or `<matcher> <first argument>` (or the command `c.exec` ran, or
+   * the judge's question), cut at 60 characters; ` #2`, ` #3` on repeats within the attempt.
+   */
+  label: string;
+  /** The eval gave the label. */
+  labelled: boolean;
+  /** `toHaveFile`, `not.toBe`, `resolves.toBe`, `poll.toBe`. */
+  matcher: string;
+  /** `error`: the matcher couldn't answer (a subject it can't read, an event the harness doesn't report). */
+  status: 'pass' | 'fail' | 'error';
+  soft: boolean;
+  /** Why it failed or couldn't answer, cut at 1 KB. */
+  message?: string;
+  /** Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`). */
+  durationMs?: number;
+};
 /** Which environment ran as the control, so the studio can show it. */
 type EvalCompareMeta = {
   baseline: string;
@@ -2087,5 +2616,5 @@ type SandboxWarning = {
   message: string;
 };
 //#endregion
-export { HarnessDescriptor as $, TOKENS_UNREPORTED as $t, DockerSandboxDescriptor as A, TurnData as At, EvalFn as B, FileRef as Bt, CompareOptions as C, SecretTemplate as Ct, CursorHarnessDescriptor as D, TokenUsage as Dt, ConfigJudgeOptions as E, SkillSource as Et, Environment as F, WebEventType as Ft, FakeJudgeModel as G, RunRecord as Gt, ExecResult as H, HistoryEntry as Ht, EnvironmentHelpers as I, ArtifactRef as It, FakeScriptStep as J, EFFORT_ALPHA as Jt, FakeNoteInput as K, StackSummary as Kt, EnvironmentInput as L, AttemptRecord as Lt, EgressPolicy as M, Verdict as Mt, EgressRecord as N, Viewport as Nt, Diff as O, TranscriptEvent as Ot, EnvVars as P, Wait as Pt, HarnessData as Q, Spread as Qt, EnvironmentSpec as R, CellRecord as Rt, CompareCut as S, Secret as St, ConfigHelpers as T, SetupStep as Tt, FakeAnalysisModel as U, JudgementRecord as Ut, EvalOptions as V, HistoryCell as Vt, FakeHarnessDescriptor as W, Lift as Wt, GooseHarnessDescriptor as X, EffortSample as Xt, FakeVote as Y, EffortContrast as Yt, GroupOptions as Z, EffortVerdict as Zt, Cell as _, Profile as _t, BackgroundHandle as a, effortChangeLabel as an, JudgeCallOptions as at, CodexHarnessDescriptor as b, SandboxDescriptor as bt, Capture as c, effortSample as cn, JudgeOptions as ct, CaptureOptions as d, formatCount as dn, OpenCodeHarnessDescriptor as dt, VERDICT_TEXT as en, HarnessEdit as et, CaptureUse as f, formatRatio as fn, Outcome as ft, CaptureWindow as g, PinnableKey as gt, CaptureWait as h, PiHarnessDescriptor as ht, AttemptContext as i, contrastEffort as in, JudgeArtifactRef as it, Duration as j, TurnStatus as jt, DirectoryOptions as k, Turn as kt, CaptureCall as l, effortSummary as ln, JudgeState as lt, CaptureUseScript as m, tokenValues as mn, PairVote as mt, AnalysisConfig as n, cellEffort as nn, HistoryConfig as nt, BlindPair as o, effortLabel as on, JudgeFailure as ot, CaptureUseAction as p, formatSeconds as pn, PairFactor as pt, FakeSandboxDescriptor as q, CellEffort as qt, AntigravityHarnessDescriptor as r, cellSamples as rn, HostFile as rt, Budget as s, effortOf as sn, JudgeModel as st, AgentHarnessDescriptor as t, cellContrast as tn, HarnessOption as tt, CaptureInstruction as u, effortVerdict as un, McpServer as ut, CellRole as v, ProfileDefaults as vt, CompareQuestion as w, ServeHandle as wt, CompareAnswer as x, SandboxWarning as xt, ClaudeCodeHarnessDescriptor as y, ProfileLimit as yt, EsConfig as z, EvalEntry as zt };
-//# sourceMappingURL=types-_LOFdPUf.d.ts.map
+export { GrokBuildHarnessDescriptor as $, RunRecord as $t, DirectoryOptions as A, SetupStep as At, EvalFn as B, WebEventType as Bt, CompareCut as C, formatCount as Cn, ProfileLimit as Ct, ConfigJudgeOptions as D, Secret as Dt, ConfigHelpers as E, tokenValues as En, SandboxWarning as Et, EnvVars as F, TurnData as Ft, FakeHarnessDescriptor as G, FamilyLift as Gt, EvalOptionsData as H, AttemptRecord as Ht, Environment as I, TurnStatus as It, FakeSandboxDescriptor as J, FileRef as Jt, FakeJudgeModel as K, FamilyRate as Kt, EnvironmentHelpers as L, Verdict as Lt, Duration as M, TokenUsage as Mt, EgressPolicy as N, TranscriptEvent as Nt, CursorHarnessDescriptor as O, SecretTemplate as Ot, EgressRecord as P, Turn as Pt, GooseHarnessDescriptor as Q, Lift as Qt, EnvironmentInput as R, Viewport as Rt, CompareAnswer as S, effortVerdict as Sn, ProfileDefaults as St, CompareQuestion as T, formatSeconds as Tn, SandboxDescriptor as Tt, ExecResult as U, CellRecord as Ut, EvalOptions as V, ArtifactRef as Vt, FakeAnalysisModel as W, EvalEntry as Wt, FakeVote as X, HistoryEntry as Xt, FakeScriptStep as Y, HistoryCell as Yt, FxHarnessDescriptor as Z, JudgementRecord as Zt, Cell as _, effortChangeLabel as _n, PairFactor as _t, BackgroundHandle as a, EffortContrast as an, HistoryConfig as at, ClaudeCodeHarnessDescriptor as b, effortSample as bn, PinnableKey as bt, Capture as c, EffortVerdict as cn, JudgeCallOptions as ct, CaptureOptions as d, VERDICT_TEXT as dn, JudgeOptions as dt, StackSummary as en, GroupOptions as et, CaptureUse as f, cellContrast as fn, JudgeState as ft, CaptureWindow as g, directional as gn, Outcome as gt, CaptureWait as h, contrastEffort as hn, OpenCodeHarnessDescriptor as ht, AttemptContext as i, EFFORT_MIN_RUNS as in, HarnessOption as it, DockerSandboxDescriptor as j, SkillSource as jt, Diff as k, ServeHandle as kt, CaptureCall as l, Spread as ln, JudgeFailure as lt, CaptureUseScript as m, cellSamples as mn, McpServer as mt, AnalysisConfig as n, DIRECTIONAL_MIN as nn, HarnessDescriptor as nt, BlindPair as o, EffortMetric as on, HostFile as ot, CaptureUseAction as p, cellEffort as pn, JudgeStateOptions as pt, FakeNoteInput as q, FamilyRecord as qt, AntigravityHarnessDescriptor as r, EFFORT_ALPHA as rn, HarnessEdit as rt, Budget as s, EffortSample as sn, JudgeArtifactRef as st, AgentHarnessDescriptor as t, CellEffort as tn, HarnessData as tt, CaptureInstruction as u, TOKENS_UNREPORTED as un, JudgeModel as ut, CellRole as v, effortLabel as vn, PairVote as vt, CompareOptions as w, formatRatio as wn, ReevalConfig as wt, CodexHarnessDescriptor as x, effortSummary as xn, Profile as xt, ClaudeCodeEffort as y, effortOf as yn, PiHarnessDescriptor as yt, EnvironmentSpec as z, Wait as zt };
+//# sourceMappingURL=types-g17bRAzt.d.ts.map