reeval 0.9.0 → 0.10.0

package/dist/{types-D8RLfkFP.d.ts → types-g17bRAzt.d.ts}

@@ -41,6 +41,10 @@ type EffortContrast = {
   agentMsP: number | null;
   /** The variant's and control's tool-call ranges don't overlap. */
   separated: boolean;
+  /** Their agent-time ranges don't overlap. */
+  timeSeparated: boolean;
+  /** Their token ranges don't overlap; null when the token test was omitted. */
+  tokensSeparated: boolean | null;
   /** On tool calls, the steadier measure; `timeVerdict` on agent time. */
   verdict: EffortVerdict;
   timeVerdict: EffortVerdict;
@@ -67,6 +71,21 @@ declare function effortVerdict(xs: number[], ys: number[]): {
   p: number | null;
   verdict: EffortVerdict;
 };
+/**
+ * The fewest attempts a side for "no overlap" to say anything: at 3 against 3 one ordering in
+ * ten separates by chance; at 1 against 1 every pair of unequal values does.
+ */
+declare const DIRECTIONAL_MIN = 3;
+/**
+ * The fewest runs per cell whose effort test can reach `EFFORT_ALPHA` (4 against 4: p = 2/70).
+ */
+declare const EFFORT_MIN_RUNS: number;
+type EffortMetric = 'tools' | 'agentS' | 'tokens';
+/**
+ * The metrics whose test is `insufficient` but whose ranges don't overlap at
+ * `DIRECTIONAL_MIN` a side or more: a direction, not an effect.
+ */
+declare function directional(c: EffortContrast): EffortMetric[];
 declare function contrastEffort(variant: EffortSample[], control: EffortSample[]): EffortContrast | null;
 /** The counted attempts of one cell of an eval, as effort samples. */
 declare const cellSamples: (e: Pick<EvalEntry, 'attempts'>, cellKey: string) => EffortSample[];
@@ -83,6 +102,7 @@ declare const VERDICT_TEXT: Record<EffortVerdict, (what: string) => string>;
  * `tools 5 vs 43 (×0.11), agent 21s vs 46s (×0.47): too few attempts to test` for a variant
  * against its control; with enough attempts, `fewer tool calls (p = 0.029)`. Tokens fold in
  * when both sides have usage samples; they are omitted (not `insufficient`) when either does not.
+ * A metric too small to test whose ranges don't overlap says so: `(×0.37, no overlap)`.
  */
 declare function effortLabel(c: EffortContrast): string;
 /** One cell's effort on its own: `tools 43 · agent 46s`, plus tokens when any attempt reported them. */
@@ -281,6 +301,10 @@ type AttemptRecord = {
     webSearches: number | null;
     webFetches: number | null;
     reasoningMs: number;
+    /** Transcript tool events with `stalled: true` (long shell or edit). */
+    stalledTools: number;
+    /** Longest tool `durationMs` in the transcript; null when none reported a duration. */
+    maxToolMs: number | null;
   };
   judgements: JudgementRecord[];
   /** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
@@ -335,7 +359,9 @@ type EvalEntry = {
    * The compare's questions, answered after its attempts by pairwise votes, one per question;
    * unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
    * may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
-   * (rates by cut, votes, ties, a failed call, a sample of reasons).
+   * (rates by cut, votes, ties, a failed call, a sample of reasons). Each answer's `input` is
+   * `<eval-slug>/questions/<n>.json`: every call its model was sent, per pair and order, secrets
+   * masked and screenshots as refs; unset where the eval's files weren't kept.
    */
   answers?: CompareAnswer[];
   /** Set on each case of a pooled `evaluate.each`; `RunRecord.families` has the suite. */
@@ -358,7 +384,7 @@ type FamilyInterval = {
    */
   method: 'wilson' | 'newcombe-paired' | 't-cases';
 };
-/** One cell's pass rate over a family's cases. */
+/** One cell's pass@1 over a family's cases. */
 type FamilyRate = {
   /** The cell's key, the same in every case. */
   cell: string;
@@ -479,7 +505,7 @@ type FingerprintParts = {
   environment: string | null;
   image: string | null;
   agentCli: string | null;
-  /** Over the judge's model, threshold and version. */
+  /** Over the judge's model, threshold and version, and the config's `state` when it sets one. */
   judge: string;
   /** Over the attempt's context files. */
   context: string;
@@ -563,7 +589,7 @@ type Cost = {
   source: 'reported' | 'unreported';
 };
 type RunRecord = {
-  kind: 'es-run';
+  kind: 'reeval-run';
   schemaVersion: typeof SCHEMA_VERSION;
   /** Also the file name: `<UTC start>_<short sha>_<author or ci>`. */
   runId: string;
@@ -644,7 +670,8 @@ type RunRecord = {
 };
 /** What ran and graded the agents: the machine, images, harnesses and judge. */
 type Stack = {
-  es: string;
+  /** The reeval version that wrote the run. */
+  reeval: string;
   /** `process.versions.node`: under Bun or Deno, the Node version they're compatible with. */
   node: string;
   /**
@@ -710,6 +737,8 @@ type Stack = {
     threshold: number;
     /** Changes when reeval, the AI SDK or the provider adapter (and so the judge prompt) change. */
     version: string;
+    /** What the judge sees by default, every key resolved; absent when the config and profile leave `state` at its defaults. */
+    state?: ResolvedJudgeStateOptions;
   };
   /** Union of every attempt's context files. */
   context: ContextFile[];
@@ -801,7 +830,8 @@ type FileEditEvent = {
 };
 /**
  * Normalised agent events. Each tool call is one event, merged with its result; `durationMs`
- * is null when the harness never reported the end.
+ * is wall time until the result arrived (Cursor's own `completed_at_ms` can be earlier). Null
+ * when the harness never reported the end. Long shell or edit calls set `stalled: true`.
  */
 type TranscriptEvent = EventTime & ({
   type: 'text' | 'reasoning';
@@ -809,6 +839,8 @@ type TranscriptEvent = EventTime & ({
   durationMs: number | null;
 } | (FileEditEvent & {
   durationMs: number | null;
+  /** True when a shell-like tool or edit ran at least two minutes. */
+  stalled?: boolean;
 }) | {
   type: 'web-search';
   toolCallId: string;
@@ -818,12 +850,16 @@ type TranscriptEvent = EventTime & ({
   /** Whether the search succeeded. Absent means it did. */
   ok?: boolean;
   durationMs: number | null;
+  /** True when a shell-like tool or edit ran at least two minutes. */
+  stalled?: boolean;
 } | {
   type: 'web-fetch';
   toolCallId: string;
   url: string;
   ok: boolean;
   durationMs: number | null;
+  /** True when a shell-like tool or edit ran at least two minutes. */
+  stalled?: boolean;
 } | {
   type: 'tool';
   toolCallId: string;
@@ -832,6 +868,8 @@ type TranscriptEvent = EventTime & ({
   output: unknown;
   ok: boolean;
   durationMs: number | null;
+  /** True when a shell-like tool or edit ran at least two minutes. */
+  stalled?: boolean;
 } | {
   type: 'error';
   message: string;
@@ -850,7 +888,9 @@ type ExecResult = {
 };
 /** Live tab size for `c.capture()`. Default 1280×720 when the call and cascade omit `viewport`. */
 type CaptureWindow = {
+  /** In CSS pixels. */
   width: number;
+  /** In CSS pixels. */
   height: number;
 };
 /**
@@ -912,6 +952,7 @@ type CaptureUse = {
  * pass it as the first argument to `c.capture`.
  */
 type CaptureOptions = {
+  /** The window and saved PNG size, or `'full-page'`. Default 1280×720. */
   viewport?: Viewport;
   /** How long to poll before failing. Default `'60s'`. */
   timeout?: Duration;
@@ -1017,6 +1058,7 @@ type Turn = TurnData & {
   exec(command: string): Promise<ExecResult>;
   readFile(path: string): Promise<string | null>;
 };
+/** Any JSON value. */
 type JSONValue = null | boolean | number | string | JSONValue[] | {
   [key: string]: JSONValue;
 };
@@ -1053,10 +1095,11 @@ type JudgeArtifactRef = {
  */
 type FakeJudgeModel = {
   kind: 'fake-judge';
+  /** The score (0–1) every call gets, or a function of the state it would see. */
   score: number | ((state: JudgeState) => number);
 };
 /**
- * `'provider/model'` (e.g. `'typesafe-ai/jev-1.13.0'`, `'google/gemini-3.8-flash'`), an AI SDK
+ * `'provider/model'` (e.g. `'typesafe-ai/jev-1.14.0'`, `'google/gemini-3.8-flash'`), an AI SDK
  * decision model object (per group, eval or call only), or a fake judge.
  */
 type JudgeModel = string | FakeJudgeModel | {
@@ -1064,28 +1107,77 @@ type JudgeModel = string | FakeJudgeModel | {
 };
 /** Merged field by field down the cascade, so per-model settings can join as new fields. */
 type JudgeOptions = {
+  /** The model that grades. */
   model?: JudgeModel;
   /** Minimum score (0–1) for a `pass` verdict. */
   threshold?: number;
+  /** AI SDK provider options for the judge's calls, keyed by provider. */
   providerOptions?: Record<string, Record<string, JSONValue>>;
   /** The most judge calls to this model's provider at once, across the run. */
   maxConcurrent?: number;
   /**
-   * Which changed files the judge's diff shows: gitignore-style patterns over workspace paths,
-   * where the last matching pattern wins and a `!` pattern excludes. Default: every changed file.
+   * The judge's state: what the judge is given beside the question, not the workspace's or the
+   * attempt's state. Merged key by key down the cascade.
    */
-  include?: string[];
-  /** Workspace files the judge sees whole, as they are now, whether or not the agent changed them. */
-  files?: string[];
+  state?: JudgeStateOptions;
+};
+/**
+ * What the judge is given: each key is a key of the state in `judge-input.json`. An unset key
+ * keeps its default, which is what the judge gets with no `state` option at all.
+ */
+type JudgeStateOptions = {
+  /** The graded turn's prompt. Default true. */
+  task?: boolean;
+  /** The graded turn's final message, or `on`'s value. Default true. */
+  output?: boolean;
+  /** `status path` lines for the changed files, under `diff`'s patterns. Default true. */
+  files?: boolean;
+  /**
+   * The diff since the eval started. A list is gitignore-style patterns over workspace paths:
+   * the last matching pattern wins and `!` excludes. Default true.
+   */
+  diff?: boolean | string[];
+  /**
+   * Workspace files shown whole, as they are now, changed or not: exact paths or gitignore-style
+   * patterns (`'**'`, `'src/**'`; the last match wins, `!` excludes) over the files git sees in
+   * the workspace. A missing exact path is `null`; a pattern that matches nothing adds nothing.
+   * These files leave the diff. Default none.
+   */
+  contents?: string[];
+  /**
+   * The graded turn's transcript as text lines: `true` is messages and tool calls with tool
+   * output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output.
+   * Default false.
+   */
+  transcript?: boolean | 'tools' | 'full';
+  /**
+   * `'all'`: every turn up to the graded one under `turns`, each with its task, output and (with
+   * `transcript`) transcript. Default `'last'`.
+   */
+  turns?: 'last' | 'all';
+  /** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. Default false. */
+  checks?: boolean | string[];
+};
+/** `JudgeStateOptions` with every key set: what one judge call saw. */
+type ResolvedJudgeStateOptions = {
+  task: boolean;
+  output: boolean;
+  files: boolean;
+  diff: boolean | string[];
+  contents: string[];
+  transcript: boolean | 'tools' | 'full';
+  turns: 'last' | 'all';
+  checks: boolean | string[];
 };
 type JudgeCallOptions = JudgeOptions & {
   /** What to grade instead of the latest turn: another turn, or a value used as the output. */
   on?: Turn | JSONValue;
   /**
-   * Captures this call sends the judge as file parts. A `c.capture()` handle or its name.
-   * Call-only: the bytes are on this attempt. The provider that cannot take files fails the call.
+   * Captures this call sends the judge as file parts: `c.capture()` handles or their names, or
+   * `true` for every capture so far. Call-only: the bytes are on this attempt. Jev takes PNG
+   * captures; a provider that cannot take files fails the call.
    */
-  artifacts?: Array<Capture | string>;
+  artifacts?: true | Array<Capture | string>;
 };
 type Verdict = {
   verdict: 'pass' | 'fail';
@@ -1108,12 +1200,16 @@ type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel'> & {
   score: null;
   error: string;
 };
+/**
+ * What an eval's body gets. The sandbox and the agent session start on the first call that needs
+ * them (`send`, `exec`, `readFile`, `writeFile`, `serve` or `capture`), once however many ask.
+ */
 type AttemptContext = {
-  /** Sends the next turn of this eval's agent session, starting the sandbox on first use. */
+  /** Sends the next turn of this eval's agent session. */
   send(prompt: string): Promise<Turn>;
   /**
    * Asks the judge a yes/no question about the latest turn (or `options.on`).
-   * `options.artifacts` is a list of `c.capture()` handles or names.
+   * `options.artifacts` is a list of `c.capture()` handles or names, or `true` for all of them.
    */
   judge(question: string, options?: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
   exec(command: string): Promise<ExecResult>;
@@ -1132,7 +1228,6 @@ type AttemptContext = {
    * Screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`.
    * First argument is the container port or a `ServeHandle` from `c.serve`.
    * `instructions` runs a vision journey on that same tab first.
-   * Needs a sandbox, same as `c.exec`.
    */
   capture(target: number | ServeHandle, options?: CaptureCall): Promise<Capture>;
   /** Workspace-relative, or `~/` for the home directory; `null` when there's no such file. */
@@ -1140,14 +1235,9 @@ type AttemptContext = {
   /**
    * Writes a file into the sandbox (workspace-relative, or `~/`), creating its directories.
    * Later turns' `diff` leaves it out, as it does the environment's files, but shows any change
-   * made to it afterwards. Needs a sandbox, same as `c.exec`.
+   * made to it afterwards.
    */
   writeFile(path: string, contents: string | Uint8Array): Promise<void>;
-  /**
-   * Starts the sandbox and the agent session without a turn, for checks on what the environment
-   * set up. `c.send()` starts them too; calling both starts them once.
-   */
-  sandbox(): Promise<Pick<AttemptContext, 'exec' | 'readFile'>>;
   /** Every turn so far, oldest first. */
   readonly turns: readonly Turn[];
   /** Aborted on timeout or Ctrl-C. */
@@ -1160,12 +1250,14 @@ type AttemptContext = {
   };
 };
 type EvalFn = (c: AttemptContext) => Promise<void>;
+/** A canned agent for tests and dry runs, made by `fakeHarness()` in `reeval/harness`. */
 type FakeHarnessDescriptor = {
   kind: 'fake';
   /** Cell id; defaults to `fake`. */
   id?: string;
   /** Files the fake agent writes into the sandbox on every turn. */
   files?: Record<string, string>;
+  /** The fake agent's final message on every turn; default `Done.`. */
   text?: string;
   /** Simulated turn length, honours the abort signal. */
   delayMs?: number;
@@ -1217,10 +1309,11 @@ type FakeHarnessDescriptor = {
    * Behave like this agent: its image, and where it reads rules and skills (`CLAUDE.md` and
    * `.claude/skills` for `claude-code`, `~/.gemini/config/skills` for `antigravity`,
    * `~/.config/opencode/skills` for `opencode`, Goose's private home for `goose`,
-   * `~/.pi/agent/skills` for `pi`, `~/.agents/skills` for `codex`). Defaults to the
-   * fake's own (`AGENTS.md`, `.agents/skills`) in Cursor's image.
+   * `~/.pi/agent/skills` for `pi`, `~/.agents/skills` for `codex`, fx's private home for
+   * `fx`, Grok's private home for `grok-build`). Defaults to the fake's own (`AGENTS.md`,
+   * `.agents/skills`) in Cursor's image.
    */
-  layout?: 'cursor' | 'claude-code' | 'antigravity' | 'opencode' | 'goose' | 'pi' | 'codex';
+  layout?: 'cursor' | 'claude-code' | 'antigravity' | 'opencode' | 'goose' | 'pi' | 'codex' | 'fx' | 'grok-build';
 };
 /**
  * One action of a scripted fake agent. `exec` is reported as a `shell` tool call whose output is
@@ -1406,28 +1499,82 @@ type CodexHarnessDescriptor = {
    */
   startTimeout?: Duration;
 };
+/**
+ * fx through `@reeval/harness-fx` (`fx acp`), made by `fx()`. It runs on Vercel AI Gateway or
+ * OpenAI's API by the model id, and reports each turn's tokens.
+ */
+type FxHarnessDescriptor = {
+  kind: 'fx';
+  /**
+   * A Vercel AI Gateway model id, `<provider>/<model>`, run on fx's `gateway` provider with
+   * `AI_GATEWAY_API_KEY`, e.g. `anthropic/claude-sonnet-5`; or a bare OpenAI model id, run on
+   * OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-4.1`.
+   */
+  model: string;
+  /** Cell id; defaults to `fx:<model>`. */
+  id?: string;
+  /** The most fx attempts at once, across the run, e.g. for the key's rate limits. */
+  maxConcurrent?: number;
+  /**
+   * How long the harness's session may take to become ready (its bootstrap and first session)
+   * before the attempt is an infra error, after one retry; default 90s.
+   */
+  startTimeout?: Duration;
+};
+/**
+ * xAI's Grok Build through `@reeval/harness-grok-build` (`grok agent stdio`), made by
+ * `grokBuild()`. It runs xAI models on an xAI API key and reports each turn's tokens.
+ */
+type GrokBuildHarnessDescriptor = {
+  kind: 'grok-build';
+  /** An xAI model id Grok Build runs on the xAI API, e.g. `grok-4.6`. */
+  model: string;
+  /**
+   * Reasoning effort as Grok's `default_reasoning_effort`, e.g. `low`, `medium` or `high`, for a
+   * model that takes one.
+   */
+  effort?: string;
+  /** Cell id; defaults to `grok-build:<model>`, or `grok-build:<model>[effort=<value>]` when `effort` is set. */
+  id?: string;
+  /** The most Grok Build attempts at once, across the run, e.g. for the key's rate limits. */
+  maxConcurrent?: number;
+  /**
+   * How long the harness's session may take to become ready (its bootstrap and first session)
+   * before the attempt is an infra error, after one retry; default 90s.
+   */
+  startTimeout?: Duration;
+};
 /** A real coding agent, driven through its AI SDK harness adapter. */
-type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor;
+type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor | FxHarnessDescriptor | GrokBuildHarnessDescriptor;
+/** The agent an attempt runs: a real harness, or the fake one. */
 type HarnessDescriptor = FakeHarnessDescriptor | AgentHarnessDescriptor;
+/** A temp directory on the host instead of a container, made by `fakeSandbox()`. */
 type FakeSandboxDescriptor = {
   kind: 'fake';
+  /** Cell id; defaults to `fake`. */
   id?: string;
   /** Parent directory for the per-eval workspace (defaults to the OS temp dir). */
   dir?: string;
   /** The most attempts on this sandbox kind at once, across the run. */
   maxConcurrent?: number;
 };
+/** A local Docker container, made by `docker()`, or a Podman one, made by `podman()`. */
 type DockerSandboxDescriptor = {
   kind: 'docker';
-  /** Cell id; defaults to `docker`, or `docker:<image>` for other images, plus an install hash. */
+  /** The container CLI the sandbox runs on: `podman` for `podman()`, Docker when unset. */
+  engine?: 'docker' | 'podman';
+  /**
+   * Cell id; defaults to `docker` (`podman`), or `docker:<image>` for other images, plus an
+   * install hash.
+   */
   id?: string;
   /** An image that already has the harness baked in, used as is. */
   image?: string;
   /** A Debian or Ubuntu based toolchain image; reeval bakes Node and the harness on top, cached. */
   from?: string;
   /**
-   * An absolute path to a Dockerfile (`docker()` resolves it against the calling file), built from
-   * its directory with `REEVAL_BASE` set to reeval's sandbox image and cached by content hash.
+   * A Dockerfile, absolute or relative to the directory of the file it's written in, built from its
+   * directory with `REEVAL_BASE` set to reeval's sandbox image and cached by content hash.
    */
   dockerfile?: string;
   /** Baked onto the sandbox image in one build container: in order, no secrets, no workspace. */
@@ -1485,13 +1632,17 @@ type SecretOptions = {
 /** `secret.template('Bearer ${TOKEN}')`: host variables substituted into a string, for MCP headers. */
 type SecretTemplate = {
   kind: 'secret-template';
+  /** The string, with `${NAME}` for each host variable to substitute. */
   template: string;
 };
+/** A variable's value: a literal string, or a host secret. */
 type EnvValue = string | Secret;
+/** Environment variables by name. */
 type EnvVars = Record<string, EnvValue>;
 /** A file on the host, read when an attempt starts. Made by the `file` helper. */
 type HostFile = {
   kind: 'file';
+  /** Its path on the host, absolute or relative to the directory of the file it's written in. */
   path: string;
 };
 /** Options for the `directory` helper. */
@@ -1509,62 +1660,92 @@ type DirectoryOptions = {
   /** `false` walks into `DEFAULT_DIRECTORY_IGNORES` too. `.git` is always skipped. */
   defaultIgnores?: boolean;
 };
+/** An MCP server the agent gets, made by the `mcp` helper: a command in the sandbox, or a URL. */
 type McpServer = {
   kind: 'mcp';
   transport: 'stdio';
+  /** The command that starts the server in the sandbox. */
   command: string;
   args: string[];
+  /** Variables the server process gets. */
   env: EnvVars;
 } | {
   kind: 'mcp';
+  /** Streamable HTTP, or the older SSE transport. */
   transport: 'http' | 'sse';
   url: string;
+  /** Request headers, with secrets brokered by the egress gateway. */
   headers: Record<string, string | Secret | SecretTemplate>;
 };
 /** `harness` (default): the agent's own skills, via its adapter. `workspace`: its repo skill dir. */
 type SkillPlacement = 'harness' | 'workspace';
+/** Skills to place for the agent, made by the `skill` helper. */
 type SkillSource = {
   kind: 'skill';
   placement: SkillPlacement;
 } & ({
   source: 'local';
+  /** A host directory with a `SKILL.md`, or a directory of them. */
   path: string;
+  /** The skill names to keep; default every one found. */
   only?: string[];
+  /** Paths in each skill directory to leave out, gitignore-style. */
   exclude?: string[];
 } | {
   source: 'git';
+  /** As written: a URL, `github:owner/repo`, or a GitHub `/tree/<ref>/<path>` URL. */
   repo: string;
+  /** A branch, tag or commit SHA; unset is the remote's `HEAD`. */
   ref?: string;
+  /** The directory in the repository to read skills from. */
   path?: string;
+  /** The skill names to keep; default every one found. */
   only?: string[];
+  /** Paths in each skill directory to leave out, gitignore-style. */
   exclude?: string[];
 } | {
   source: 'registry';
+  /** A skills.sh id, `owner/repo@skill`. */
   id: string;
+  /** A git ref of the skill's repository. */
   version?: string;
 } | {
   source: 'npm';
+  /** A pinned npm package spec. */
   spec: string;
+  /** The directory in the package to read skills from; default `skills`. */
   path: string;
+  /** The skill names to keep; default every one found. */
   only?: string[];
+  /** Paths in each skill directory to leave out, gitignore-style. */
   exclude?: string[];
 } | {
   source: 'inline';
+  /** The skill's name, as in `SKILL.md`'s frontmatter. */
   name: string;
+  /** The skill's description, as in `SKILL.md`'s frontmatter. */
   description: string;
+  /** The body of `SKILL.md`. */
   content: string;
+  /** Other files in the skill directory, by relative path. */
   files: Record<string, string>;
 });
 /** A `docker({ install })` command, baked into a cached image layer: no secrets, root, cwd `/`. */
 type InstallStep = string | {
+  /** The shell command. */
   run: string;
   timeout?: Duration;
 };
-/** Runs in every attempt before the agent starts, with secrets, outside the eval timeout. */
-/** A shell command, or `{ run, timeout?, cwd? }`; `cwd` is relative to the workspace or absolute. */
+/**
+ * Runs in every attempt before the agent starts, with secrets, outside the eval timeout: a shell
+ * command, or `{ run, timeout?, cwd? }`.
+ */
 type SetupStep = string | {
+  /** The shell command. */
   run: string;
+  /** Default 5m. */
   timeout?: Duration;
+  /** Relative to the workspace, or absolute; default the workspace. */
   cwd?: string;
 };
 /**
@@ -1584,6 +1765,7 @@ type GitWorkspace = {
   kind: 'git';
   /** As written: a URL, `github:owner/repo`, or a local path. */
   repo: string;
+  /** What reeval fetches: the clone URL, or an absolute path for a local repository. */
   url: string;
   /** A branch, tag or commit SHA; unset is the remote's `HEAD`. */
   ref?: string;
@@ -1609,6 +1791,7 @@ type EnvironmentSpec = {
   sandbox?: SandboxDescriptor;
   /** The code the workspace starts as, before `files`, `rules` and `setup`; a child's replaces its parent's. */
   workspace?: GitWorkspace;
+  /** Variables and secrets for the sandbox, merged by name over the parent's. */
   env?: EnvVars;
   /** Replaces the inherited policy; an `extend` child's replaces its parent's. */
   egress?: EgressPolicy;
@@ -1616,13 +1799,18 @@ type EnvironmentSpec = {
   files?: Record<string, string | HostFile>;
   /** Project rules, written where the harness's agent reads them: `AGENTS.md`, or `CLAUDE.md` for Claude Code. */
   rules?: string | HostFile;
+  /** Shell commands run in every attempt before the agent starts; a child's run after its parent's. */
   setup?: SetupStep[];
+  /** Skills placed for the agent. */
   skills?: SkillSource[];
+  /** MCP servers the agent gets, by name. */
   mcp?: Record<string, McpServer>;
 };
 /** One environment in an `extend` chain. */
 type EnvironmentLayer = {
+  /** The name passed to `defineEnvironment` or `extend`. */
   name: string;
+  /** The spec as that layer wrote it, before merging. */
   spec: EnvironmentSpec;
 };
 /**
@@ -1631,7 +1819,9 @@ type EnvironmentLayer = {
  */
 type Environment = {
   readonly kind: 'environment';
+  /** The name it was defined or extended under, which names its cells. */
   readonly name: string;
+  /** Its `extend` chain, root first. */
   readonly layers: readonly EnvironmentLayer[];
   /**
    * A child environment layered on this one; wherever it sets a key this one has, it wins.
@@ -1767,14 +1957,71 @@ type CompareOptions<E = Environment> = {
 };
 /**
  * A question for pairwise judgement: its text, or the text and the `'provider/model'` that
- * answers it. Without a model, `analysis.model` answers, else the judge's when it's a language model.
+ * answers it. Without a model, the cascaded `question.model` answers, else `analysis.model`, else the
+ * judge's when it's a language model.
  * `across` is the factor it varies when pairing; without it (and without a one-factor
- * `compare.across`) both pair kinds run.
+ * `compare.across`) both pair kinds run. `state` and `artifacts` choose what each side shows.
  */
 type CompareQuestion = string | {
   question: string;
+  /** `'provider/model'` that answers it, over the cascaded `question.model`. */
   model?: string;
+  /** The factor it varies when pairing. */
   across?: PairFactor;
+  /** What each side shows, over the cascaded `question.state`, key by key. */
+  state?: QuestionStateOptions;
+  /** Each side's own captures, by name; `true` sends all of them. Replaces `question.artifacts`. */
+  artifacts?: true | string[];
+};
+/**
+ * What each side of a compare question shows: the keys of `judge.state` but `contents` (the
+ * attempts' workspaces are gone when questions are asked). An unset key keeps its default, which
+ * is what a question shows with no `state` at all: the task and the diff.
+ */
+type QuestionStateOptions = {
+  /** The prompts the attempt was sent, once for both sides when they match. Default true. */
+  task?: boolean;
+  /** The final message: the agent's own account of its work. Default false. */
+  output?: boolean;
+  /** `status path` lines for the changed files, under `diff`'s patterns. Default false. */
+  files?: boolean;
+  /**
+   * The diff since the eval started. A list is gitignore-style patterns over workspace paths:
+   * the last matching pattern wins and `!` excludes. Default true.
+   */
+  diff?: boolean | string[];
+  /**
+   * The transcript as text lines: `true` is messages and tool calls with tool output clipped,
+   * `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each
+   * harness's tools, so a pair of two harnesses isn't blind. Default false.
+   */
+  transcript?: boolean | 'tools' | 'full';
+  /**
+   * `'all'`: every turn under `turns`, each with its task, output and (with `transcript`)
+   * transcript; `'last'`: the last turn only. Default `'all'`.
+   */
+  turns?: 'last' | 'all';
+  /** `c.exec` results; a list shows only those commands, as passed to `c.exec`. Default false. */
+  checks?: boolean | string[];
+};
+/** `QuestionStateOptions` with every key set: what each side of one question showed. */
+type ResolvedQuestionStateOptions = {
+  task: boolean;
+  output: boolean;
+  files: boolean;
+  diff: boolean | string[];
+  transcript: boolean | 'tools' | 'full';
+  turns: 'last' | 'all';
+  checks: boolean | string[];
+};
+/** Defaults for every compare question: `model` and `artifacts` replaced, `state` merged key by key. */
+type QuestionOptions = {
+  /** `'provider/model'` that answers every question below it without its own `model`. */
+  model?: string;
+  /** What each side shows. */
+  state?: QuestionStateOptions;
+  /** Each side's own captures, by name; `true` sends all of them. Default none. */
+  artifacts?: true | string[];
 };
 /** A matrix factor a compare question varies when pairing; the others are held. */
 type PairFactor = 'environment' | 'harness';
@@ -1799,6 +2046,11 @@ type PairVote = {
   swapped: boolean;
   /** The model's short reason for this pick; omitted when it gave none. Capped. */
   reason?: string;
+  /** The captures each named side showed, by name; set when the question sends `artifacts`. */
+  artifacts?: {
+    a: string[];
+    b: string[];
+  };
 };
 /** A compare question's answer: every vote, and each side's wins over decided votes per cut. */
 type CompareAnswer = {
@@ -1807,6 +2059,10 @@ type CompareAnswer = {
   model: string;
   /** The model id the provider says served the calls, when it reports one. */
   resolvedModel?: string;
+  /** What each side showed, every key resolved; unset on answers from before questions took `state`, which showed the defaults. */
+  stateOptions?: ResolvedQuestionStateOptions;
+  /** The captures it sent of each side, as asked: `true` for all, or their names. */
+  artifacts?: true | string[];
   votes: PairVote[];
   /** Wins over decided pairs, per named side, per cut. */
   rates: Array<{
@@ -1817,7 +2073,10 @@ type CompareAnswer = {
   }>;
   /** Why some or all of its votes are missing: the model call failed. Outcomes never change. */
   error?: string;
+  /** `questions/<n>.json` under the eval's directory: what the model was sent, per pair and order. */
+  input?: FileRef;
 };
+/** An option a profile's `limit` can narrow, and `pin` can protect. */
 type PinnableKey = 'harness' | 'runs' | 'compare';
 /**
  * How a cell's attempts decide its outcome: `'all'` (the default) passes only when every attempt
@@ -1833,7 +2092,7 @@ type CellGate = GatePolicy | 'none';
 type CompareSpec<E = Environment> = CompareOptions<E>;
 /** Options other than `environment` and `compare`, which are exclusive of each other. */
 type EvalOptionFields = {
-  /** The only list: each harness is its own cell. */
+  /** Which agents run the eval; each harness is its own cell. */
   harness?: HarnessOption;
   /**
    * The default machine when the resolved environment omits `sandbox`. Cascades like `harness`.
@@ -1851,11 +2110,15 @@ type EvalOptionFields = {
    * environment, then groups and the eval.
    */
   egress?: EgressPolicy;
+  /** Defaults for `c.judge()`, merged field by field down the cascade. */
   judge?: JudgeOptions;
   /** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
   capture?: CaptureOptions;
+  /** Defaults for every compare question: `model`, and `state` merged key by key, down the cascade. */
+  question?: QuestionOptions;
   /** Per attempt. */
   timeout?: Duration;
+  /** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
   tags?: string[];
   /** Keys a profile's `limit` may not narrow for this group or eval. */
   pin?: PinnableKey[];
@@ -1873,20 +2136,36 @@ type EnvironmentOrCompare<E> = {
 } | {
   /** Forbidden beside `compare.baseline`, which names the control. */
   environment?: never;
+  /** Environments to run side by side; `baseline` is the control. */
   compare: CompareSpec<E> & {
     baseline: E;
   };
 } | {
   /** The control: the compare has no `baseline`. */
   environment?: E;
+  /** Environments to run side by side against `environment`, the control. */
   compare: CompareSpec<E> & {
     baseline?: undefined;
   };
 };
 type EvalOptions = EvalOptionFields & EnvironmentOrCompare<Environment>;
 type GroupOptions = EvalOptions;
+/**
+ * `EvalOptions` in their data forms, as JSON carries them: the harness as data, the judge's model
+ * by name or a fake judge, and environments without `extend`. The same `environment` / compare
+ * object split as `EvalOptions`.
+ */
+type EvalOptionsData = Omit<EvalOptionFields, 'harness' | 'judge'> & {
+  /** Which agents run the eval; each harness is its own cell. */
+  harness?: HarnessData;
+  /** Defaults for `c.judge()`, merged field by field down the cascade. */
+  judge?: ConfigJudgeOptions;
+} & EnvironmentOrCompare<EnvironmentData>;
+/** An `Environment` as data, as JSON carries it: without `extend` / `extendEnvironment`. */
+type EnvironmentData = Omit<Environment, 'extendEnvironment' | 'extend'>;
 /** The config file is plain data sent to workers, so its judge model can't be an object. */
 type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
+  /** `'provider/model'`, or a fake judge. */
   model?: string | FakeJudgeModel;
 };
 type HistoryConfig = {
@@ -1897,22 +2176,35 @@ type HistoryConfig = {
 };
 /** Run-level caps. `attempts` is checked against the plan; `duration` while running. */
 type Budget = {
+  /** The most attempts the plan may have. */
   attempts?: number;
+  /** How long the run may go on, from the first attempt's start. */
   duration?: Duration;
 };
 /** What evals get when they don't say otherwise: under groups and evals. */
 type ProfileDefaults = {
+  /** The agents every eval runs on, unless a group or eval replaces them. */
   harness?: HarnessDescriptor | HarnessDescriptor[];
   /** The default machine when the resolved environment omits `sandbox`. */
   sandbox?: SandboxDescriptor;
+  /** What every eval runs in when it names no environment. */
   environment?: Environment;
+  /** Attempts per cell. */
   runs?: number;
+  /** How each cell's attempts decide its outcome. */
   gate?: GatePolicy;
+  /** Variables and secrets for the sandbox, merged by name over the config's. */
   env?: EnvVars;
+  /** Where the sandbox may connect; replaces the config's policy. */
   egress?: EgressPolicy;
+  /** Defaults for `c.judge()`, merged field by field over the config's. */
   judge?: ConfigJudgeOptions;
+  /** Per attempt. */
   timeout?: Duration;
+  /** Screenshot defaults for `c.capture()`, shallow-merged over the config's. */
   capture?: CaptureOptions;
+  /** Defaults for every compare question, merged over the config's. */
+  question?: QuestionOptions;
 };
 /** Over every eval, and it can only narrow; `pin` on a group or eval skips it. */
 type ProfileLimit = {
@@ -1927,10 +2219,15 @@ type ProfileLimit = {
 };
 /** How a run is sized: chosen with `--profile`, `REEVAL_PROFILE` or `defaultProfile`. */
 type Profile = {
+  /** What evals get when they don't say otherwise: under groups and evals. */
   defaults?: ProfileDefaults;
+  /** Caps over every eval, which only narrow; `pin` on a group or eval skips them. */
   limit?: ProfileLimit;
+  /** Judge settings for this profile, the same as `defaults.judge`, which wins field by field. */
   judge?: ConfigJudgeOptions;
+  /** Merged over the config's `budget`. */
   budget?: Budget;
+  /** When this profile's runs are copied into the history folder. */
   record?: RecordMode;
   /** Tag filters for this profile, like `--tag` (`'!slow'` excludes). */
   tags?: string[];
@@ -1974,12 +2271,48 @@ type FakePick = FakeVote | {
   winner: FakeVote;
   reason?: string;
 };
-/** What a compare question's model is shown: the prompt, and the two diffs in slot order. */
+/** What a compare question's model is shown: the prompt, and the two sides in slot order. */
 type BlindPair = {
   question: string;
   prompt: string;
-  a: string;
-  b: string;
+  a: BlindSide;
+  b: BlindSide;
+};
+/** A turn of one side, with the keys its question's `state` turned on. */
+type BlindTurn = {
+  task?: string;
+  output?: string;
+  transcript?: string;
+};
+/**
+ * One side of a blinded pair: only the keys its question's `state` turned on. With
+ * `turns: 'all'`, `task`, `output` and `transcript` are under `turns`; a task both sides share is
+ * the pair's prompt's, not either side's.
+ */
+type BlindSide = BlindTurn & {
+  turns?: BlindTurn[];
+  files?: string[];
+  /** The diff as the model sees it: dependency and binary noise dropped. */
+  diff?: string;
+  /** A command the other side ran and this one didn't is `{ command, notRun: true }`. */
+  checks?: Array<{
+    command: string;
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+  } | {
+    command: string;
+    notRun: true;
+  }>;
+  /** Its captures, in the order sent, labelled by position; a name it lacks is `missing`. */
+  artifacts?: Array<{
+    label: string;
+    mediaType: 'image/png';
+    bytes: number;
+  } | {
+    label: string;
+    missing: true;
+  }>;
 };
 /** `reeval analyze`: a language model (not an evaluation model) and what matters in this project. */
 type AnalysisConfig = {
@@ -1997,33 +2330,46 @@ type AnalysisConfig = {
   /** The most tokens the model may answer with (default 8000). reeval has no prices, so tokens are the cap. */
   maxOutputTokens?: number;
 };
-type EsConfig = {
+type ReevalConfig = {
+  /** Globs of eval files, relative to the config's directory; default every `.eval.ts` under `evals`. */
   include?: string[];
+  /** The agents every eval runs on, unless a group or eval replaces them; default Cursor. */
   harness?: HarnessDescriptor | HarnessDescriptor[];
   /**
    * The default machine when the resolved environment omits `sandbox`. Not an environment:
    * it does not invent skills, files or a name. Implicit `docker()` when nobody sets one.
    */
   sandbox?: SandboxDescriptor;
+  /** What every eval runs in when it names no environment, and a baseline-less compare's control. */
   environment?: Environment;
+  /** Attempts per cell; default 1. */
   runs?: number;
   /** How each cell's attempts decide its outcome; default `'all'`. */
   gate?: GatePolicy;
+  /** Variables and secrets for the sandbox, merged by name down the cascade. */
   env?: EnvVars;
+  /** Where the sandbox may connect; default `'allow-all'`. */
   egress?: EgressPolicy;
+  /** Defaults for `c.judge()`, merged field by field down the cascade. */
   judge?: ConfigJudgeOptions;
+  /** Per attempt; default `'15m'`. */
   timeout?: Duration;
   /**
    * Screenshot defaults for `c.capture()`. `REEVAL_CAPTURE_BROWSER=0` refuses the host
    * binary when `c.capture()` runs; it does not change this object.
    */
   capture?: CaptureOptions;
+  /** Defaults for every compare question: the model that answers it and what each side shows. Merged down the cascade. */
+  question?: QuestionOptions;
   /** How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it. */
   concurrency?: number;
   /** When runs are copied into the history folder: `reeval record` only, in CI, or on every run. */
   record?: RecordMode;
+  /** The history folder, and which attempts keep their artifacts there. */
   history?: HistoryConfig;
+  /** Caps on a run; a profile's `budget` is merged over it. */
   budget?: Budget;
+  /** Named presets of `defaults`, `limit`, `judge`, `budget`, `record` and `tags`. */
   profiles?: Record<string, Profile>;
   /** The profile when neither `--profile` nor `REEVAL_PROFILE` names one: `ci` when `CI` is set. */
   defaultProfile?: {
@@ -2042,6 +2388,7 @@ type EsConfig = {
     keep?: number;
     traceBufferMb?: number;
   };
+  /** `reeval analyze`, compare questions without a model, and `agent`'s notes during a run. */
   analysis?: AnalysisConfig;
 };
 /** What a `defineConfig` callback gets: the environment helpers plus the run's context. */
@@ -2269,5 +2616,5 @@ type SandboxWarning = {
   message: string;
 };
 //#endregion
-export { HarnessData as $, EffortContrast as $t, DirectoryOptions as A, Turn as At, EsConfig as B, EvalEntry as Bt, CompareCut as C, Secret as Ct, ConfigJudgeOptions as D, SkillSource as Dt, ConfigHelpers as E, SetupStep as Et, EnvVars as F, Wait as Ft, FakeHarnessDescriptor as G, HistoryCell as Gt, EvalOptions as H, FamilyRate as Ht, Environment as I, WebEventType as It, FakeSandboxDescriptor as J, Lift as Jt, FakeJudgeModel as K, HistoryEntry as Kt, EnvironmentHelpers as L, ArtifactRef as Lt, Duration as M, TurnStatus as Mt, EgressPolicy as N, Verdict as Nt, CursorHarnessDescriptor as O, TokenUsage as Ot, EgressRecord as P, Viewport as Pt, GroupOptions as Q, EFFORT_ALPHA as Qt, EnvironmentInput as R, AttemptRecord as Rt, CompareAnswer as S, SandboxWarning as St, CompareQuestion as T, ServeHandle as Tt, ExecResult as U, FamilyRecord as Ut, EvalFn as V, FamilyLift as Vt, FakeAnalysisModel as W, FileRef as Wt, FakeVote as X, StackSummary as Xt, FakeScriptStep as Y, RunRecord as Yt, GooseHarnessDescriptor as Z, CellEffort as Zt, Cell as _, formatSeconds as _n, PinnableKey as _t, BackgroundHandle as a, cellContrast as an, JudgeArtifactRef as at, ClaudeCodeHarnessDescriptor as b, ProfileLimit as bt, Capture as c, contrastEffort as cn, JudgeModel as ct, CaptureOptions as d, effortOf as dn, McpServer as dt, EffortSample as en, HarnessDescriptor as et, CaptureUse as f, effortSample as fn, OpenCodeHarnessDescriptor as ft, CaptureWindow as g, formatRatio as gn, PiHarnessDescriptor as gt, CaptureWait as h, formatCount as hn, PairVote as ht, AttemptContext as i, VERDICT_TEXT as in, HostFile as it, DockerSandboxDescriptor as j, TurnData as jt, Diff as k, TranscriptEvent as kt, CaptureCall as l, effortChangeLabel as ln, JudgeOptions as lt, CaptureUseScript as m, effortVerdict as mn, PairFactor as mt, AnalysisConfig as n, Spread as nn, HarnessOption as nt, BlindPair as o, cellEffort as on, JudgeCallOptions as ot, CaptureUseAction as p, effortSummary as pn, Outcome as pt, FakeNoteInput as q, JudgementRecord as qt, AntigravityHarnessDescriptor as r, TOKENS_UNREPORTED as rn, HistoryConfig as rt, Budget as s, cellSamples as sn, JudgeFailure as st, AgentHarnessDescriptor as t, EffortVerdict as tn, HarnessEdit as tt, CaptureInstruction as u, effortLabel as un, JudgeState as ut, CellRole as v, tokenValues as vn, Profile as vt, CompareOptions as w, SecretTemplate as wt, CodexHarnessDescriptor as x, SandboxDescriptor as xt, ClaudeCodeEffort as y, ProfileDefaults as yt, EnvironmentSpec as z, CellRecord as zt };
-//# sourceMappingURL=types-D8RLfkFP.d.ts.map
+export { GrokBuildHarnessDescriptor as $, RunRecord as $t, DirectoryOptions as A, SetupStep as At, EvalFn as B, WebEventType as Bt, CompareCut as C, formatCount as Cn, ProfileLimit as Ct, ConfigJudgeOptions as D, Secret as Dt, ConfigHelpers as E, tokenValues as En, SandboxWarning as Et, EnvVars as F, TurnData as Ft, FakeHarnessDescriptor as G, FamilyLift as Gt, EvalOptionsData as H, AttemptRecord as Ht, Environment as I, TurnStatus as It, FakeSandboxDescriptor as J, FileRef as Jt, FakeJudgeModel as K, FamilyRate as Kt, EnvironmentHelpers as L, Verdict as Lt, Duration as M, TokenUsage as Mt, EgressPolicy as N, TranscriptEvent as Nt, CursorHarnessDescriptor as O, SecretTemplate as Ot, EgressRecord as P, Turn as Pt, GooseHarnessDescriptor as Q, Lift as Qt, EnvironmentInput as R, Viewport as Rt, CompareAnswer as S, effortVerdict as Sn, ProfileDefaults as St, CompareQuestion as T, formatSeconds as Tn, SandboxDescriptor as Tt, ExecResult as U, CellRecord as Ut, EvalOptions as V, ArtifactRef as Vt, FakeAnalysisModel as W, EvalEntry as Wt, FakeVote as X, HistoryEntry as Xt, FakeScriptStep as Y, HistoryCell as Yt, FxHarnessDescriptor as Z, JudgementRecord as Zt, Cell as _, effortChangeLabel as _n, PairFactor as _t, BackgroundHandle as a, EffortContrast as an, HistoryConfig as at, ClaudeCodeHarnessDescriptor as b, effortSample as bn, PinnableKey as bt, Capture as c, EffortVerdict as cn, JudgeCallOptions as ct, CaptureOptions as d, VERDICT_TEXT as dn, JudgeOptions as dt, StackSummary as en, GroupOptions as et, CaptureUse as f, cellContrast as fn, JudgeState as ft, CaptureWindow as g, directional as gn, Outcome as gt, CaptureWait as h, contrastEffort as hn, OpenCodeHarnessDescriptor as ht, AttemptContext as i, EFFORT_MIN_RUNS as in, HarnessOption as it, DockerSandboxDescriptor as j, SkillSource as jt, Diff as k, ServeHandle as kt, CaptureCall as l, Spread as ln, JudgeFailure as lt, CaptureUseScript as m, cellSamples as mn, McpServer as mt, AnalysisConfig as n, DIRECTIONAL_MIN as nn, HarnessDescriptor as nt, BlindPair as o, EffortMetric as on, HostFile as ot, CaptureUseAction as p, cellEffort as pn, JudgeStateOptions as pt, FakeNoteInput as q, FamilyRecord as qt, AntigravityHarnessDescriptor as r, EFFORT_ALPHA as rn, HarnessEdit as rt, Budget as s, EffortSample as sn, JudgeArtifactRef as st, AgentHarnessDescriptor as t, CellEffort as tn, HarnessData as tt, CaptureInstruction as u, TOKENS_UNREPORTED as un, JudgeModel as ut, CellRole as v, effortLabel as vn, PairVote as vt, CompareOptions as w, formatRatio as wn, ReevalConfig as wt, CodexHarnessDescriptor as x, effortSummary as xn, Profile as xt, ClaudeCodeEffort as y, effortOf as yn, PiHarnessDescriptor as yt, EnvironmentSpec as z, Wait as zt };
+//# sourceMappingURL=types-g17bRAzt.d.ts.map