reeval 0.7.0 → 0.9.0

package/dist/{types-BsRSknLe.d.ts → types-D8RLfkFP.d.ts}

@@ -1,10 +1,14 @@
 //#region src/view/effort.d.ts
 /** Below this two-sided p, an effort difference is an effect. */
 declare const EFFORT_ALPHA = 0.05;
+/** Some counted attempts reported no usage. Not a verdict; never printed as one. */
+declare const TOKENS_UNREPORTED = "some cells did not report usage";
 type EffortSample = {
   tools: number;
   agentMs: number;
   fileEdits: number;
+  /** Total tokens when this attempt reported usage; omitted when the harness dropped it. */
+  tokens?: number;
 };
 type Spread = {
   median: number;
@@ -17,10 +21,13 @@ type CellEffort = {
   tools: Spread;
   agentMs: Spread;
   fileEdits: number;
+  /** Over the counted attempts that reported usage; omitted when none did. */
+  tokens?: Spread;
 };
 /**
  * `fewer` / `more`: the exact rank test's p is below `EFFORT_ALPHA`. `noise`: it isn't.
- * `insufficient`: too few attempts for the test to ever reach it (2 against 2 can't).
+ * `insufficient`: too few attempts for the test to ever reach it (2 against 2 can't). Never
+ * means a harness dropped usage.
  */
 type EffortVerdict = 'fewer' | 'more' | 'noise' | 'insufficient';
 type EffortContrast = {
@@ -37,8 +44,23 @@ type EffortContrast = {
   /** On tool calls, the steadier measure; `timeVerdict` on agent time. */
   verdict: EffortVerdict;
   timeVerdict: EffortVerdict;
+  /**
+   * Token medians' ratio when both sides have at least one usage sample; null when the
+   * control's is 0 or the token test was omitted.
+   */
+  tokensRatio: number | null;
+  tokensP: number | null;
+  /**
+   * Rank-test verdict on the attempts that reported usage. Null when either side has no
+   * token sample — omitted, not `insufficient`.
+   */
+  tokensVerdict: EffortVerdict | null;
+  /** Set when some counted attempts on either side reported no usage. Not a verdict. */
+  tokensNote?: typeof TOKENS_UNREPORTED;
 };
-declare const effortSample: (a: Pick<AttemptRecord, 'activity' | 'timings'>) => EffortSample;
+declare const effortSample: (a: Pick<AttemptRecord, 'activity' | 'timings' | 'usage'>) => EffortSample;
+/** Token totals from the attempts that reported usage; empty when none did. */
+declare const tokenValues: (samples: EffortSample[]) => number[];
 declare function effortOf(samples: EffortSample[]): CellEffort | null;
 /** The verdict of an exact rank test of `xs` (variant) against `ys` (control). */
 declare function effortVerdict(xs: number[], ys: number[]): {
@@ -59,13 +81,14 @@ declare function formatSeconds(ms: number): string;
 declare const VERDICT_TEXT: Record<EffortVerdict, (what: string) => string>;
 /**
  * `tools 5 vs 43 (×0.11), agent 21s vs 46s (×0.47): too few attempts to test` for a variant
- * against its control; with enough attempts, `fewer tool calls (p = 0.029)`.
+ * against its control; with enough attempts, `fewer tool calls (p = 0.029)`. Tokens fold in
+ * when both sides have usage samples; they are omitted (not `insufficient`) when either does not.
  */
 declare function effortLabel(c: EffortContrast): string;
-/** One cell's effort on its own: `tools 43 · agent 46s`. */
-declare const effortSummary: (e: Pick<CellEffort, 'tools' | 'agentMs'>) => string;
+/** One cell's effort on its own: `tools 43 · agent 46s`, plus tokens when any attempt reported them. */
+declare const effortSummary: (e: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens'>) => string;
 /** The same cell between two runs: `tools 44 → 10 (×0.23) · agent 45s → 21s (×0.47)`. */
-declare function effortChangeLabel(base: Pick<CellEffort, 'tools' | 'agentMs'>, head: Pick<CellEffort, 'tools' | 'agentMs'>): string;
+declare function effortChangeLabel(base: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens'>, head: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens'>): string;
 //#endregion
 //#region src/history/types.d.ts
 /** Stays 1 while reeval is pre-release: the record changes in place. */
@@ -86,10 +109,19 @@ type ArtifactRef = FileRef & {
   capture?: {
     port: number;
     path: string;
+    /** The live window, never the string `'full-page'`. */
     viewport?: {
       width: number;
       height: number;
     };
+    /** Saved PNG used `captureBeyondViewport`. */
+    fullPage?: boolean;
+    /** Goal strings only; wait beats omitted. */
+    instructions?: string[];
+    use?: {
+      model: string;
+      moves: number;
+    };
   };
 };
 type GitInfo = {
@@ -149,6 +181,8 @@ type JudgementRecord = {
   key: string;
   /** Transient provider errors retried before this verdict, with the backoff before each retry. */
   retries?: JudgeRetry[];
+  /** Captures this call sent the judge as file parts, when any. */
+  artifacts?: JudgeArtifactRef[];
 };
 type AttemptFiles = {
   /** Each environment setup step's command, exit code, duration and output. */
@@ -197,6 +231,17 @@ type AttemptRecord = {
    * recorded them.
    */
   servedModels: string[] | null;
+  /**
+   * The reasoning effort the harness factory asked for (`claudeCode({ effort: 'high' })`);
+   * null when the factory left it unset. Fast is not this.
+   */
+  effort: string | null;
+  /**
+   * The reasoning effort the harness's API said it ran at, from `servedModels` /
+   * each turn's `model` (`medium` from `gpt-5.6-luna[effort=medium]`); null when those
+   * labels name none. Fast (`[fast=true]`) is not this.
+   */
+  servedEffort: string | null;
   outcome: Outcome;
   error: AttemptError | null;
   durationMs: number;
@@ -218,12 +263,21 @@ type AttemptRecord = {
   }>;
   /** Tokens across the attempt's turns; null when the harness reported none. */
   usage: TokenUsage | null;
+  /**
+   * Tokens the capture vision loop used. Infrastructure only: never folded
+   * into `usage`, effort, compare or lift.
+   */
+  captureUse?: {
+    model: string;
+    moves: number;
+    usage: TokenUsage | null;
+  };
   timings: PhaseTimings;
-  /** Counts from the transcript, so a run's shape is visible without reading it. */
+  /** Counts from the transcript, so a run's shape is visible without reading it. `webSearches` includes refused searches. */
   activity: {
     tools: number;
     fileEdits: number;
-    /** Null when a turn with none couldn't report them (`unreported`), not a count of 0 that isn't one. */
+    /** Every `web-search` event, including refused ones. Null when a turn with none couldn't report them (`unreported`). */
     webSearches: number | null;
     webFetches: number | null;
     reasoningMs: number;
@@ -237,6 +291,8 @@ type AttemptRecord = {
     durationMs: number;
     log?: FileRef;
   }>;
+  /** Every `expect` and `expect.soft` matcher call that settled, in call order; unset with none. */
+  assertions?: AssertionRecord[];
   /** Files under a dependency directory (node_modules/, .venv/, …) collapse to one entry with `count`. */
   changedFiles: Array<FileChange & {
     count?: number;
@@ -271,17 +327,123 @@ type EvalEntry = {
   outcome: Outcome;
   /** A placeholder, `evaluate.todo` or `group.todo`: it never runs, and its outcome is `skipped`. */
   todo: boolean;
-  /** pass / (pass + fail) over the gated cells' attempts; infra errors and skips excluded. */
+  /** pass@1, pass / (pass + fail), over the gated cells' attempts; infra errors and skips excluded. */
   passRate: number | null;
   cells: CellRecord[];
   attempts: AttemptRecord[];
   /**
    * The compare's questions, answered after its attempts by pairwise votes, one per question;
-   * unset without any. They annotate: `outcome` and `passRate` never depend on them.
+   * unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
+   * may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
+   * (rates by cut, votes, ties, a failed call, a sample of reasons).
    */
   answers?: CompareAnswer[];
+  /** Set on each case of a pooled `evaluate.each`; `RunRecord.families` has the suite. */
+  family?: {
+    id: string;
+    name: string;
+    case: number;
+    cases: number;
+  };
+};
+/** A 95% interval on a pooled rate or difference. */
+type FamilyInterval = {
+  lo: number;
+  hi: number;
+  level: 0.95;
+  /**
+   * `wilson` / `newcombe-paired`: one counted attempt a case and side, so each case is one
+   * Bernoulli trial (or pair). `t-cases`: several, so a t interval over the cases' pass@1, with
+   * attempts within a case never counted as independent.
+   */
+  method: 'wilson' | 'newcombe-paired' | 't-cases';
 };
-/** A variant cell's pass rate against the control's, on the same harness and sandbox. */
+/** One cell's pass rate over a family's cases. */
+type FamilyRate = {
+  /** The cell's key, the same in every case. */
+  cell: string;
+  /** Cases with at least one counted attempt in this cell. */
+  cases: number;
+  /** Passes and counted attempts over those cases. */
+  pass: number;
+  counted: number;
+  /** The mean of the cases' pass@1; at one attempt a case, `pass / counted`. */
+  rate: number;
+  interval: FamilyInterval;
+};
+/**
+ * `lift` / `drop`: the interval excludes 0. `noise`: it doesn't. `insufficient`: fewer than 6
+ * discordant pairs (one attempt a case) or 6 paired cases (several), where no test can decide.
+ */
+type FamilyVerdict = 'lift' | 'drop' | 'noise' | 'insufficient';
+/** One effort metric over a family's paired cases. */
+type FamilyEffortMetric = {
+  /** Cases with a value on both sides. */
+  pairs: number;
+  /** The median over cases of the variant's per-case median, over the control's; null at 0. */
+  ratio: number | null;
+  /** Wilcoxon signed-rank over the per-case differences; null when every difference is 0. */
+  p: number | null;
+  /** `insufficient` below 6 non-zero differences, where the exact test can't reach 0.05. */
+  verdict: 'fewer' | 'more' | 'noise' | 'insufficient';
+};
+/** The variant's effort against the control's, paired by case. */
+type FamilyEffort = {
+  tools: FamilyEffortMetric;
+  agentMs: FamilyEffortMetric;
+  /** Over the cases where every counted attempt reported usage; omitted when none did. */
+  tokens?: FamilyEffortMetric;
+};
+/** A variant cell against its control over a family's cases, paired by case. */
+type FamilyLift = {
+  /** The variant cell's key. */
+  variant: string;
+  /** The control cell's key. */
+  vs: string;
+  /** Cases with counted attempts on both sides. */
+  pairs: number;
+  /** Mean pass@1 over the paired cases on each side. */
+  variantRate: number;
+  controlRate: number;
+  /** Set at one attempt a case and side: pairs where only one side passed. */
+  discordant?: {
+    variantOnly: number;
+    controlOnly: number;
+  };
+  /** Variant minus control, from -1 to 1. */
+  points: number;
+  interval: FamilyInterval;
+  /** McNemar's exact p (one attempt a case), else Wilcoxon signed-rank over the cases. */
+  p: number | null;
+  /** The interval spans 0. */
+  withinNoise: boolean;
+  verdict: FamilyVerdict;
+  /** The compare's `minLift`: on a family it gates this lower bound, not each case's. */
+  minLift?: number;
+  /** `fail` when `interval.lo` is below `minLift`. */
+  outcome: 'pass' | 'fail';
+  effort?: FamilyEffort;
+};
+/** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an eval. */
+type FamilyRecord = {
+  /** `file › group › … › template`. */
+  id: string;
+  file: string;
+  group: string[];
+  /** The name template, e.g. `fixes $id`. */
+  name: string;
+  /** Cases in the list. */
+  cases: number;
+  /** Cases with an eval in this run; fewer when a filter or tag left some out. */
+  ran: number;
+  /** One per cell. */
+  rates: FamilyRate[];
+  /** One per variant cell, with `compare`. */
+  lifts: FamilyLift[];
+  /** `fail` when a lift is below its `minLift`. The cases still pass or fail on their own. */
+  outcome: 'pass' | 'fail';
+};
+/** A variant cell's pass@1 against the control's, on the same harness and sandbox. */
 type Lift = {
   /** The control cell's key. */
   vs: string;
@@ -294,7 +456,7 @@ type Lift = {
     pass: number;
     counted: number;
   };
-  /** Variant pass rate minus control pass rate, from -1 to 1. */
+  /** Variant pass@1 minus control pass@1, from -1 to 1. */
   points: number;
   /** 95% interval on the difference, by Newcombe's hybrid score method from two Wilson intervals. */
   interval: {
@@ -332,6 +494,11 @@ type CellRecord = Cell & {
   harnessKind: string;
   /** The configured harness model, when it has one. */
   model: string | null;
+  /**
+   * The reasoning effort the harness factory asked for; null when it left it unset.
+   * Fast is not this.
+   */
+  effort: string | null;
   /** How the attempts decide the cell's outcome; `'none'` when it doesn't fail the eval. */
   gate: CellGate;
   /** Environment keys the variant replaced on the control's (or a child on its parent's). */
@@ -341,14 +508,39 @@ type CellRecord = Cell & {
   fail: number;
   infraError: number;
   skipped: number;
+  /** pass@1: pass / (pass + fail); infra errors and skips excluded. */
   passRate: number | null;
   /**
-   * By `gate`. `'all'` matches `{ passRate: 1 }` (infra only when nothing counted); `'none'`
+   * By `gate`. `'all'` (pass^k: every counted attempt passes) matches `{ passRate: 1 }`;
+   * `{ passRate }` is a pass@1 threshold. Infra only when nothing counted; `'none'`
    * reports the worst outcome. A variant below its `minLift` fails too.
    */
   outcome: Outcome;
   lift?: Lift;
+  /**
+   * Each check (assertion label) seen in any attempt of the eval, over this cell's counted
+   * attempts; on a variant cell, the control's beside it. Unset when no attempt recorded any.
+   */
+  checks?: CellCheck[];
 };
+/** How often one check passed: a counted attempt that never reached it counts it `notRun`. */
+type CheckTally = {
+  pass: number;
+  fail: number;
+  error: number;
+  notRun: number;
+};
+/**
+ * One check over a cell's counted attempts. `verdict` (variant cells): `discriminates` when it
+ * passes on every counted attempt of one side and none of the other; `both-pass` or `both-fail`
+ * when it does the same on both sides, so it can't tell them apart; `mixed` otherwise.
+ */
+type CellCheck = CheckTally & {
+  label: string;
+  control?: CheckTally;
+  verdict?: CheckVerdict;
+};
+type CheckVerdict = 'discriminates' | 'both-pass' | 'both-fail' | 'mixed';
 /** Counts attempts, except `evals`. */
 type RunSummary = {
   evals: number;
@@ -447,6 +639,8 @@ type RunRecord = {
   };
   summary: RunSummary;
   evals: EvalEntry[];
+  /** Each pooled `evaluate.each` that ran; unset when none did. */
+  families?: FamilyRecord[];
 };
 /** What ran and graded the agents: the machine, images, harnesses and judge. */
 type Stack = {
@@ -478,21 +672,31 @@ type Stack = {
   /** Exact versions of the packages that run and grade the agent. */
   packages: Record<string, string>;
   /**
-   * Each harness: its configured `model`, and `resolvedModels`, what it reported serving across
-   * the run's turns with the variant it resolved (`composer-2.5[fast=true]` for `composer-2.5`
-   * without `params`); empty when it reports none.
+   * Each harness: its configured `model` and requested `effort`, `resolvedModels` (what it
+   * reported serving, with the variant it resolved), and `resolvedEffort` (reasoning-effort
+   * suffixes those labels named). Fast (`[fast=true]`) is not `resolvedEffort`.
    */
   harnesses: Array<{
     id: string;
     kind: string;
     /** The configured model, when the harness has one. */
     model: string | null;
+    /**
+     * The reasoning effort the harness factory asked for; null when it left it unset.
+     * Fast is not this.
+     */
+    effort: string | null;
     /**
      * The models the harness reported serving, across every turn in the run, with the variant it
      * resolved (`composer-2.5[fast=true]` for `composer-2.5` with no `params`); empty when it
      * reports none.
      */
     resolvedModels: string[];
+    /**
+     * Reasoning-effort suffixes those served models named (`medium` from
+     * `gpt-5.6-luna[effort=medium]`); empty when they name none. Fast is not this.
+     */
+    resolvedEffort: string[];
   }>;
   /**
    * The run's judge: the config's, with what the profile's `judge` and `defaults.judge` set over it.
@@ -535,7 +739,7 @@ type HistoryCell = Pick<CellRecord, 'key' | 'role' | 'variant' | 'harness' | 'mo
     hi: number;
     withinNoise: boolean;
   };
-  /** Median tool calls and agent time over the counted attempts; null when none counted. */
+  /** Median tool calls, agent time and tokens over the counted attempts; null when none counted. */
   effort: CellEffort | null;
 };
 /**
@@ -575,6 +779,8 @@ type HistoryEntry = {
     passRate: number | null;
     cells: HistoryCell[];
   }>;
+  /** The run's pooled families; unset when none ran. */
+  families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'>>;
 };
 //#endregion
 //#region src/types.d.ts
@@ -609,6 +815,8 @@ type TranscriptEvent = EventTime & ({
   query: string;
   /** How many sources the harness says it used; Cursor sends no URLs or titles. */
   references: number | null;
+  /** Whether the search succeeded. Absent means it did. */
+  ok?: boolean;
   durationMs: number | null;
 } | {
   type: 'web-fetch';
@@ -640,14 +848,67 @@ type ExecResult = {
   stdout: string;
   stderr: string;
 };
-/** CSS pixels for `c.capture()`. Default 1280×720 when the call and cascade omit it. */
-type Viewport = {
+/** Live tab size for `c.capture()`. Default 1280×720 when the call and cascade omit `viewport`. */
+type CaptureWindow = {
   width: number;
   height: number;
 };
+/**
+ * Author camera. `{ width, height }` is the live window and the saved PNG.
+ * `'full-page'` keeps the default window and saves with `captureBeyondViewport`.
+ */
+type Viewport = CaptureWindow | 'full-page';
+/**
+ * Chromium look-ready events. `'load'` is `window` load (`Page.loadEventFired`).
+ * `'idle'` is `Page.lifecycleEvent` `networkAlmostIdle`.
+ */
+type CaptureWait = 'load' | 'idle';
+/** Same shape at top-level `wait` and `{ wait }` between instructions. */
+type Wait = CaptureWait | {
+  ms: number;
+};
+type CaptureInstruction = string | {
+  wait: Wait;
+};
+type CaptureUseAction = {
+  type: 'click';
+  x: number;
+  y: number;
+} | {
+  type: 'type';
+  text: string;
+} | {
+  type: 'key';
+  key: 'Tab' | 'Enter';
+} | {
+  type: 'scroll';
+  x: number;
+  y: number;
+  dy: number;
+} | {
+  type: 'drag';
+  x: number;
+  y: number;
+  toX: number;
+  toY: number;
+} | {
+  type: 'hover';
+  x: number;
+  y: number;
+} | {
+  type: 'done';
+};
+/** One list per instruction string. Tests; never downloads a model. */
+type CaptureUseScript = CaptureUseAction[][];
+/** Resolved vision driver after cascade, call override and analysis fallback. */
+type CaptureUse = {
+  model: string;
+} | {
+  fake: CaptureUseScript;
+};
 /**
  * Camera defaults for `c.capture()`. Cascades config → profile defaults → group → eval,
- * shallow-merged. `viewport` on a level replaces the whole object. `port` is not an option:
+ * shallow-merged. `viewport` on a level replaces the whole value. `port` is not an option:
  * pass it as the first argument to `c.capture`.
  */
 type CaptureOptions = {
@@ -656,6 +917,11 @@ type CaptureOptions = {
   timeout?: Duration;
   /** `false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws. */
   browser?: boolean;
+  /**
+   * Vision model for `instructions`. Cascades like `question`. Default is the analysis
+   * model when that model accepts images; never Jev.
+   */
+  model?: string;
 };
 /** A background process started by `c.exec(..., { background: true })` or `c.serve`. */
 type BackgroundHandle = {
@@ -672,6 +938,7 @@ type Capture = {
   port: number;
   /** Path on the app, as passed in. */
   path: string;
+  /** What the author asked, including `'full-page'`. */
   viewport: Viewport;
   /** `artifacts/<name>.png` under the attempt directory. */
   file: {
@@ -683,13 +950,19 @@ type Capture = {
 /**
  * Optional camera and path for `c.capture(port | ServeHandle, options?)`.
  * `port` is the first argument, not a field here. Default `path` is `/`;
- * default `name` is derived from the path.
+ * default `name` is derived from the path. `wait` and `instructions` are call-only.
  */
 type CaptureCall = {
   name?: string;
   path?: string;
   viewport?: Viewport;
   timeout?: Duration;
+  /** Look-ready once, before the first model shot or the saved PNG. Default `'load'`. */
+  wait?: Wait;
+  /** Journey: goal strings and `{ wait }` beats. Tests may pass `fakeInstructions(...)`. */
+  instructions?: CaptureInstruction[];
+  /** Overrides cascaded `capture.model` for this call. */
+  model?: string;
 };
 type FileChange = {
   path: string;
@@ -747,19 +1020,44 @@ type Turn = TurnData & {
 type JSONValue = null | boolean | number | string | JSONValue[] | {
   [key: string]: JSONValue;
 };
+/** JSON fields the judge sees for the turn: task, output, files, diff, optional contents. */
+type JudgeJsonState = {
+  [key: string]: JSONValue;
+};
+/** A capture sent to `experimental_decide` as a file part. */
+type JudgeFilePart = {
+  type: 'file';
+  mediaType: string;
+  data: Uint8Array;
+};
+/**
+ * What `experimental_decide` (and a fake judge) receive: a JSON object when the call has no
+ * captures, or an ordered list of one JSON part plus each capture's file part.
+ */
+type JudgeState = JudgeJsonState | Array<{
+  type: 'json';
+  value: JudgeJsonState;
+} | JudgeFilePart>;
+/** A capture this judge call sent as a file part. Bytes live in `artifacts/<name>.png`. */
+type JudgeArtifactRef = {
+  name: string;
+  /** `artifacts/<name>.png` under the attempt directory. */
+  path: string;
+  sha256: string;
+  bytes: number;
+  mediaType: 'image/png';
+};
 /**
  * A canned judge for tests and dry runs; see `fakeJudge()` in `reeval/judge`. `score` as a
  * function of the state the judge would see works per group, eval or call, not in the config.
  */
 type FakeJudgeModel = {
   kind: 'fake-judge';
-  score: number | ((state: {
-    [key: string]: JSONValue;
-  }) => number);
+  score: number | ((state: JudgeState) => number);
 };
 /**
  * `'provider/model'` (e.g. `'typesafe-ai/jev-1.13.0'`, `'google/gemini-3.8-flash'`), an AI SDK
- * evaluation model object (per group, eval or call only), or a fake judge.
+ * decision model object (per group, eval or call only), or a fake judge.
  */
 type JudgeModel = string | FakeJudgeModel | {
   readonly specificationVersion: string;
@@ -783,6 +1081,11 @@ type JudgeOptions = {
 type JudgeCallOptions = JudgeOptions & {
   /** What to grade instead of the latest turn: another turn, or a value used as the output. */
   on?: Turn | JSONValue;
+  /**
+   * Captures this call sends the judge as file parts. A `c.capture()` handle or its name.
+   * Call-only: the bytes are on this attempt. The provider that cannot take files fails the call.
+   */
+  artifacts?: Array<Capture | string>;
 };
 type Verdict = {
   verdict: 'pass' | 'fail';
@@ -793,6 +1096,8 @@ type Verdict = {
   model: string;
   /** The model id the provider says served the call, when it reports one. */
   resolvedModel?: string;
+  /** Captures this call sent the judge as file parts, when any. */
+  artifacts?: JudgeArtifactRef[];
 };
 /**
  * What `c.judge()` returns when the call failed (the provider's error, after any retries): the attempt fails as a judge error once the eval's body has finished,
@@ -806,7 +1111,10 @@ type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel'> & {
 type AttemptContext = {
   /** Sends the next turn of this eval's agent session, starting the sandbox on first use. */
   send(prompt: string): Promise<Turn>;
-  /** Asks the judge a yes/no question about the latest turn (or `options.on`). */
+  /**
+   * Asks the judge a yes/no question about the latest turn (or `options.on`).
+   * `options.artifacts` is a list of `c.capture()` handles or names.
+   */
   judge(question: string, options?: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
   exec(command: string): Promise<ExecResult>;
   exec(command: string, options: {
@@ -823,11 +1131,18 @@ type AttemptContext = {
   /**
    * Screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`.
    * First argument is the container port or a `ServeHandle` from `c.serve`.
+   * `instructions` runs a vision journey on that same tab first.
    * Needs a sandbox, same as `c.exec`.
    */
   capture(target: number | ServeHandle, options?: CaptureCall): Promise<Capture>;
   /** Workspace-relative, or `~/` for the home directory; `null` when there's no such file. */
   readFile(path: string): Promise<string | null>;
+  /**
+   * Writes a file into the sandbox (workspace-relative, or `~/`), creating its directories.
+   * Later turns' `diff` leaves it out, as it does the environment's files, but shows any change
+   * made to it afterwards. Needs a sandbox, same as `c.exec`.
+   */
+  writeFile(path: string, contents: string | Uint8Array): Promise<void>;
   /**
    * Starts the sandbox and the agent session without a turn, for checks on what the environment
    * set up. `c.send()` starts them too; calling both starts them once.
@@ -867,6 +1182,11 @@ type FakeHarnessDescriptor = {
    * and variant (`composer-2.5[fast=true]`); unset, it reports none.
    */
   servedModel?: string;
+  /**
+   * Requested reasoning effort, recorded as `effort` on the cell. Use `servedModel` for what the
+   * turn reports (`fake-1[effort=medium]`). Fast is not this.
+   */
+  effort?: string;
   /** Tool calls the fake agent reports on every turn, before writing files. */
   tools?: Array<{
     tool: string;
@@ -933,7 +1253,12 @@ type CursorHarnessDescriptor = {
    * model catalog (`fast=true` for `composer-2.5`), sent explicitly; pass the ones that change cost.
    */
   params?: Record<string, string>;
-  /** Cell id; defaults to `cursor:<model>`. */
+  /**
+   * Reasoning effort, mapped onto the catalog param `Cursor.models.list()` names for this model
+   * (`reasoning_effort`, `effort`, …). Not Fast: that stays `params.fast`.
+   */
+  effort?: string;
+  /** Cell id; defaults to `cursor:<model>`, or `cursor:<model>[effort=<value>]` when `effort` is set. */
   id?: string;
   /** The most Cursor attempts at once, across the run, e.g. for the account's rate limits. */
   maxConcurrent?: number;
@@ -943,12 +1268,19 @@ type CursorHarnessDescriptor = {
    */
   startTimeout?: Duration;
 };
+/** Reasoning effort the Claude Code Agent SDK `/effort` control takes. */
+type ClaudeCodeEffort = 'low' | 'medium' | 'high' | 'xhigh' | 'max';
 /** Claude Code through `@ai-sdk/harness-claude-code`, made by `claudeCode()`. */
 type ClaudeCodeHarnessDescriptor = {
   kind: 'claude-code';
   /** A model id or alias the `claude` CLI accepts, e.g. `claude-sonnet-5` or `sonnet`. */
   model: string;
-  /** Cell id; defaults to `claude-code:<model>`. */
+  /**
+   * Reasoning effort the Agent SDK `/effort` control takes: `low`, `medium`, `high`,
+   * `xhigh` (Extra in the UI) or `max`.
+   */
+  effort?: ClaudeCodeEffort;
+  /** Cell id; defaults to `claude-code:<model>`, or `claude-code:<model>[effort=<value>]` when `effort` is set. */
   id?: string;
   /** The most Claude Code attempts at once, across the run, e.g. for the account's rate limits. */
   maxConcurrent?: number;
@@ -963,7 +1295,12 @@ type AntigravityHarnessDescriptor = {
   kind: 'antigravity';
   /** A model slug `agy models` lists for a Gemini API key, e.g. `gemini-3.8-flash-medium`. */
   model: string;
-  /** Cell id; defaults to `antigravity:<model>`. */
+  /**
+   * Reasoning effort passed as `agy --effort`, e.g. `low`, `medium`, `high` or `max`. The
+   * slug's `-low`/`-medium`/`-high` suffix is the same knob baked into `model`.
+   */
+  effort?: string;
+  /** Cell id; defaults to `antigravity:<model>`, or `antigravity:<model>[effort=<value>]` when `effort` is set. */
   id?: string;
   /** The most Antigravity attempts at once, across the run, e.g. for the key's rate limits. */
   maxConcurrent?: number;
@@ -985,7 +1322,12 @@ type OpenCodeHarnessDescriptor = {
    * `anthropic/claude-haiku-4-5`; the provider picks the key.
    */
   model: string;
-  /** Cell id; defaults to `opencode:<model>`. */
+  /**
+   * Reasoning effort for that provider: `options.reasoningEffort` (OpenAI) or `options.thinking`
+   * (Anthropic). Values are whatever that provider takes.
+   */
+  effort?: string;
+  /** Cell id; defaults to `opencode:<model>`, or `opencode:<model>[effort=<value>]` when `effort` is set. */
   id?: string;
   /** The most OpenCode attempts at once, across the run, e.g. for the key's rate limits. */
   maxConcurrent?: number;
@@ -1003,7 +1345,11 @@ type GooseHarnessDescriptor = {
   kind: 'goose';
   /** A Gemini model id, run on Goose's `google` provider, e.g. `gemini-3.8-flash`. */
   model: string;
-  /** Cell id; defaults to `goose:<model>`. */
+  /**
+   * Reasoning effort as `GOOSE_THINKING_EFFORT`, e.g. `off`, `low`, `medium`, `high` or `max`.
+   */
+  effort?: string;
+  /** Cell id; defaults to `goose:<model>`, or `goose:<model>[effort=<value>]` when `effort` is set. */
   id?: string;
   /** The most Goose attempts at once, across the run, e.g. for the key's rate limits. */
   maxConcurrent?: number;
@@ -1022,7 +1368,12 @@ type PiHarnessDescriptor = {
   kind: 'pi';
   /** A Gemini model id, run on pi's `google` provider, e.g. `gemini-3.8-flash`. */
   model: string;
-  /** Cell id; defaults to `pi:<model>`. */
+  /**
+   * Reasoning effort, passed as pi's `--thinking` (the CLI's name). Values are whatever that
+   * flag takes, e.g. `off`, `minimal`, `low`, `medium`, `high`, `xhigh` or `max`.
+   */
+  effort?: string;
+  /** Cell id; defaults to `pi:<model>`, or `pi:<model>[effort=<value>]` when `effort` is set. */
   id?: string;
   /** The most pi attempts at once, across the run, e.g. for the key's rate limits. */
   maxConcurrent?: number;
@@ -1040,7 +1391,12 @@ type CodexHarnessDescriptor = {
   kind: 'codex';
   /** An OpenAI model id Codex runs on the Responses API, e.g. `gpt-5.6-luna`. */
   model: string;
-  /** Cell id; defaults to `codex:<model>`. */
+  /**
+   * Reasoning effort as Codex's `model_reasoning_effort`, e.g. `low`, `medium`, `high`,
+   * `xhigh`, `max` or `ultra`.
+   */
+  effort?: string;
+  /** Cell id; defaults to `codex:<model>`, or `codex:<model>[effort=<value>]` when `effort` is set. */
   id?: string;
   /** The most Codex attempts at once, across the run, e.g. for the key's rate limits. */
   maxConcurrent?: number;
@@ -1085,8 +1441,13 @@ type DockerSandboxDescriptor = {
   root?: boolean;
   /** Linux capabilities to add back, e.g. `['NET_RAW']`; every other one is dropped. */
   capabilities?: string[];
-  /** The container's memory, with no swap on top: `'4g'` (default), `'512m'`, or bytes. */
+  /** The container's memory: `'4g'` (default), `'512m'`, or bytes. */
   memory?: string | number;
+  /**
+   * Swap on top of `memory`, in the same units. Default: as much as `memory`; `0` makes `memory`
+   * the hard ceiling. A host without swap has none to give.
+   */
+  swap?: string | number;
   /** CPUs the container may use, fractions allowed. Default 2, capped at the Docker host's. */
   cpus?: number;
   /** Processes and threads at once. Default 1024. */
@@ -1397,16 +1758,27 @@ type CompareOptions<E = Environment> = {
    * win rates and never change pass, fail or the exit code.
    */
   questions?: CompareQuestion[];
+  /**
+   * The factor every question varies when pairing, when a question does not name its own
+   * `across`. `'environment'` holds the harness; `'harness'` holds the environment. A list of
+   * both, or omitted, asks both pair kinds.
+   */
+  across?: PairFactor | PairFactor[];
 };
 /**
  * A question for pairwise judgement: its text, or the text and the `'provider/model'` that
  * answers it. Without a model, `analysis.model` answers, else the judge's when it's a language model.
+ * `across` is the factor it varies when pairing; without it (and without a one-factor
+ * `compare.across`) both pair kinds run.
  */
 type CompareQuestion = string | {
   question: string;
   model?: string;
+  across?: PairFactor;
 };
-/** Which pairs a vote is over: two arms on one harness, or two harnesses on one arm. */
+/** A matrix factor a compare question varies when pairing; the others are held. */
+type PairFactor = 'environment' | 'harness';
+/** Which pairs a vote is over: two environments on one harness, or two harnesses on one environment. */
 type CompareCut = 'within' | 'across';
 /** One judgement of one pair of attempts, in one order. */
 type PairVote = {
@@ -1425,6 +1797,8 @@ type PairVote = {
   winner: string | 'tie';
   /** True for the repeat that showed `b` first. */
   swapped: boolean;
+  /** The model's short reason for this pick; omitted when it gave none. Capped. */
+  reason?: string;
 };
 /** A compare question's answer: every vote, and each side's wins over decided votes per cut. */
 type CompareAnswer = {
@@ -1447,7 +1821,8 @@ type CompareAnswer = {
 type PinnableKey = 'harness' | 'runs' | 'compare';
 /**
  * How a cell's attempts decide its outcome: `'all'` (the default) passes only when every attempt
- * passes; `{ passRate }` passes when at least that share of its passes and fails are passes.
+ * passes (pass^k); `{ passRate }` passes when its pass@1 (passes over passes and fails) is at
+ * least that share.
  */
 type GatePolicy = 'all' | {
   passRate: number;
@@ -1477,7 +1852,7 @@ type EvalOptionFields = {
    */
   egress?: EgressPolicy;
   judge?: JudgeOptions;
-  /** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` replaces the whole object. */
+  /** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
   capture?: CaptureOptions;
   /** Per attempt. */
   timeout?: Duration;
@@ -1586,14 +1961,19 @@ type FakeAnalysisModel = {
   };
   /**
    * How it answers compare questions: a fixed pick, a function of the blinded pair, or
-   * `{ error }` to fail every call. Default `'tie'`.
+   * `{ error }` to fail every call. A pick may be `{ winner, reason? }`. Default `'tie'`.
    */
-  vote?: FakeVote | ((pair: BlindPair) => FakeVote) | {
+  vote?: FakePick | ((pair: BlindPair) => FakePick) | {
     error: string;
   };
 };
 /** A blinded label's pick: the attempt shown first (`A`), second (`B`), or neither. */
 type FakeVote = 'A' | 'B' | 'tie';
+/** A fake compare-question pick, optionally with a reason. */
+type FakePick = FakeVote | {
+  winner: FakeVote;
+  reason?: string;
+};
 /** What a compare question's model is shown: the prompt, and the two diffs in slot order. */
 type BlindPair = {
   question: string;
@@ -1692,11 +2072,23 @@ type PrepareFailureRecord = {
   log: string | null;
   setBy: string[];
 };
-/** Why an attempt failed; `prepare` when its sandbox couldn't be prepared. */
+/**
+ * The sandbox ran out of a resource during the attempt: `oom` is the kernel killing `kills`
+ * processes at its memory limit. The attempt is then an infra error, whatever its checks said.
+ */
+type ResourceFailureRecord = {
+  cause: 'oom';
+  kills: number;
+};
+/**
+ * Why an attempt failed; `prepare` when its sandbox couldn't be prepared, `resource` when it ran
+ * out of one.
+ */
 type AttemptError = {
   kind: ErrorKind;
   message: string;
   prepare?: PrepareFailureRecord;
+  resource?: ResourceFailureRecord;
 };
 /** Where an eval's time went, in ms. `agentMs` is the harness turns; `diffMs` reading changes. */
 type PhaseTimings = {
@@ -1831,6 +2223,25 @@ type StartRetry = {
   /** How long it waited before giving up on that start. */
   waitedMs: number;
 };
+/** One `expect` or `expect.soft` matcher call an attempt made. */
+type AssertionRecord = {
+  /**
+   * `expect`'s second argument, or `<matcher> <first argument>` (or the command `c.exec` ran, or
+   * the judge's question), cut at 60 characters; ` #2`, ` #3` on repeats within the attempt.
+   */
+  label: string;
+  /** The eval gave the label. */
+  labelled: boolean;
+  /** `toHaveFile`, `not.toBe`, `resolves.toBe`, `poll.toBe`. */
+  matcher: string;
+  /** `error`: the matcher couldn't answer (a subject it can't read, an event the harness doesn't report). */
+  status: 'pass' | 'fail' | 'error';
+  soft: boolean;
+  /** Why it failed or couldn't answer, cut at 1 KB. */
+  message?: string;
+  /** Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`). */
+  durationMs?: number;
+};
 /** Which environment ran as the control, so the studio can show it. */
 type EvalCompareMeta = {
   baseline: string;
@@ -1858,5 +2269,5 @@ type SandboxWarning = {
   message: string;
 };
 //#endregion
-export { JudgeFailure as $, effortVerdict as $t, EnvironmentHelpers as A, FileRef as At, FakeNoteInput as B, EffortSample as Bt, DirectoryOptions as C, Verdict as Ct, EgressRecord as D, AttemptRecord as Dt, EgressPolicy as E, ArtifactRef as Et, EvalOptions as F, RunRecord as Ft, GroupOptions as G, cellEffort as Gt, FakeScriptStep as H, Spread as Ht, ExecResult as I, StackSummary as It, HarnessEdit as J, effortChangeLabel as Jt, HarnessData as K, cellSamples as Kt, FakeAnalysisModel as L, CellEffort as Lt, EnvironmentSpec as M, HistoryEntry as Mt, EsConfig as N, JudgementRecord as Nt, EnvVars as O, CellRecord as Ot, EvalFn as P, Lift as Pt, JudgeCallOptions as Q, effortSummary as Qt, FakeHarnessDescriptor as R, EFFORT_ALPHA as Rt, Diff as S, TurnStatus as St, Duration as T, WebEventType as Tt, FakeVote as U, VERDICT_TEXT as Ut, FakeSandboxDescriptor as V, EffortVerdict as Vt, GooseHarnessDescriptor as W, cellContrast as Wt, HistoryConfig as X, effortOf as Xt, HarnessOption as Y, effortLabel as Yt, HostFile as Z, effortSample as Zt, CompareOptions as _, SkillSource as _t, BackgroundHandle as a, PairVote as at, ConfigJudgeOptions as b, Turn as bt, Capture as c, Profile as ct, Cell as d, SandboxDescriptor as dt, formatCount as en, JudgeModel as et, CellRole as f, SandboxWarning as ft, CompareCut as g, SetupStep as gt, CompareAnswer as h, ServeHandle as ht, AttemptContext as i, Outcome as it, EnvironmentInput as j, HistoryCell as jt, Environment as k, EvalEntry as kt, CaptureCall as l, ProfileDefaults as lt, CodexHarnessDescriptor as m, SecretTemplate as mt, AnalysisConfig as n, formatSeconds as nn, McpServer as nt, BlindPair as o, PiHarnessDescriptor as ot, ClaudeCodeHarnessDescriptor as p, Secret as pt, HarnessDescriptor as q, contrastEffort as qt, AntigravityHarnessDescriptor as r, OpenCodeHarnessDescriptor as rt, Budget as s, PinnableKey as st, AgentHarnessDescriptor as t, formatRatio as tn, JudgeOptions as tt, CaptureOptions as u, ProfileLimit as ut, CompareQuestion as v, TokenUsage as vt, DockerSandboxDescriptor as w, Viewport as wt, CursorHarnessDescriptor as x, TurnData as xt, ConfigHelpers as y, TranscriptEvent as yt, FakeJudgeModel as z, EffortContrast as zt };
-//# sourceMappingURL=types-BsRSknLe.d.ts.map
+export { HarnessData as $, EffortContrast as $t, DirectoryOptions as A, Turn as At, EsConfig as B, EvalEntry as Bt, CompareCut as C, Secret as Ct, ConfigJudgeOptions as D, SkillSource as Dt, ConfigHelpers as E, SetupStep as Et, EnvVars as F, Wait as Ft, FakeHarnessDescriptor as G, HistoryCell as Gt, EvalOptions as H, FamilyRate as Ht, Environment as I, WebEventType as It, FakeSandboxDescriptor as J, Lift as Jt, FakeJudgeModel as K, HistoryEntry as Kt, EnvironmentHelpers as L, ArtifactRef as Lt, Duration as M, TurnStatus as Mt, EgressPolicy as N, Verdict as Nt, CursorHarnessDescriptor as O, TokenUsage as Ot, EgressRecord as P, Viewport as Pt, GroupOptions as Q, EFFORT_ALPHA as Qt, EnvironmentInput as R, AttemptRecord as Rt, CompareAnswer as S, SandboxWarning as St, CompareQuestion as T, ServeHandle as Tt, ExecResult as U, FamilyRecord as Ut, EvalFn as V, FamilyLift as Vt, FakeAnalysisModel as W, FileRef as Wt, FakeVote as X, StackSummary as Xt, FakeScriptStep as Y, RunRecord as Yt, GooseHarnessDescriptor as Z, CellEffort as Zt, Cell as _, formatSeconds as _n, PinnableKey as _t, BackgroundHandle as a, cellContrast as an, JudgeArtifactRef as at, ClaudeCodeHarnessDescriptor as b, ProfileLimit as bt, Capture as c, contrastEffort as cn, JudgeModel as ct, CaptureOptions as d, effortOf as dn, McpServer as dt, EffortSample as en, HarnessDescriptor as et, CaptureUse as f, effortSample as fn, OpenCodeHarnessDescriptor as ft, CaptureWindow as g, formatRatio as gn, PiHarnessDescriptor as gt, CaptureWait as h, formatCount as hn, PairVote as ht, AttemptContext as i, VERDICT_TEXT as in, HostFile as it, DockerSandboxDescriptor as j, TurnData as jt, Diff as k, TranscriptEvent as kt, CaptureCall as l, effortChangeLabel as ln, JudgeOptions as lt, CaptureUseScript as m, effortVerdict as mn, PairFactor as mt, AnalysisConfig as n, Spread as nn, HarnessOption as nt, BlindPair as o, cellEffort as on, JudgeCallOptions as ot, CaptureUseAction as p, effortSummary as pn, Outcome as pt, FakeNoteInput as q, JudgementRecord as qt, AntigravityHarnessDescriptor as r, TOKENS_UNREPORTED as rn, HistoryConfig as rt, Budget as s, cellSamples as sn, JudgeFailure as st, AgentHarnessDescriptor as t, EffortVerdict as tn, HarnessEdit as tt, CaptureInstruction as u, effortLabel as un, JudgeState as ut, CellRole as v, tokenValues as vn, Profile as vt, CompareOptions as w, SecretTemplate as wt, CodexHarnessDescriptor as x, SandboxDescriptor as xt, ClaudeCodeEffort as y, ProfileDefaults as yt, EnvironmentSpec as z, CellRecord as zt };
+//# sourceMappingURL=types-D8RLfkFP.d.ts.map