reeval 0.6.3 → 0.8.0

package/dist/{types-CoAfC-_k.d.ts → types-_LOFdPUf.d.ts}

@@ -1,10 +1,14 @@
 //#region src/view/effort.d.ts
 /** Below this two-sided p, an effort difference is an effect. */
 declare const EFFORT_ALPHA = 0.05;
+/** Some counted attempts reported no usage. Not a verdict; never printed as one. */
+declare const TOKENS_UNREPORTED = "some cells did not report usage";
 type EffortSample = {
   tools: number;
   agentMs: number;
   fileEdits: number;
+  /** Total tokens when this attempt reported usage; omitted when the harness dropped it. */
+  tokens?: number;
 };
 type Spread = {
   median: number;
@@ -17,10 +21,13 @@ type CellEffort = {
   tools: Spread;
   agentMs: Spread;
   fileEdits: number;
+  /** Over the counted attempts that reported usage; omitted when none did. */
+  tokens?: Spread;
 };
 /**
  * `fewer` / `more`: the exact rank test's p is below `EFFORT_ALPHA`. `noise`: it isn't.
- * `insufficient`: too few attempts for the test to ever reach it (2 against 2 can't).
+ * `insufficient`: too few attempts for the test to ever reach it (2 against 2 can't). Never
+ * means a harness dropped usage.
  */
 type EffortVerdict = 'fewer' | 'more' | 'noise' | 'insufficient';
 type EffortContrast = {
@@ -37,8 +44,23 @@ type EffortContrast = {
   /** On tool calls, the steadier measure; `timeVerdict` on agent time. */
   verdict: EffortVerdict;
   timeVerdict: EffortVerdict;
+  /**
+   * Token medians' ratio when both sides have at least one usage sample; null when the
+   * control's is 0 or the token test was omitted.
+   */
+  tokensRatio: number | null;
+  tokensP: number | null;
+  /**
+   * Rank-test verdict on the attempts that reported usage. Null when either side has no
+   * token sample — omitted, not `insufficient`.
+   */
+  tokensVerdict: EffortVerdict | null;
+  /** Set when some counted attempts on either side reported no usage. Not a verdict. */
+  tokensNote?: typeof TOKENS_UNREPORTED;
 };
-declare const effortSample: (a: Pick<AttemptRecord, 'activity' | 'timings'>) => EffortSample;
+declare const effortSample: (a: Pick<AttemptRecord, 'activity' | 'timings' | 'usage'>) => EffortSample;
+/** Token totals from the attempts that reported usage; empty when none did. */
+declare const tokenValues: (samples: EffortSample[]) => number[];
 declare function effortOf(samples: EffortSample[]): CellEffort | null;
 /** The verdict of an exact rank test of `xs` (variant) against `ys` (control). */
 declare function effortVerdict(xs: number[], ys: number[]): {
@@ -59,13 +81,14 @@ declare function formatSeconds(ms: number): string;
 declare const VERDICT_TEXT: Record<EffortVerdict, (what: string) => string>;
 /**
  * `tools 5 vs 43 (×0.11), agent 21s vs 46s (×0.47): too few attempts to test` for a variant
- * against its control; with enough attempts, `fewer tool calls (p = 0.029)`.
+ * against its control; with enough attempts, `fewer tool calls (p = 0.029)`. Tokens fold in
+ * when both sides have usage samples; they are omitted (not `insufficient`) when either does not.
  */
 declare function effortLabel(c: EffortContrast): string;
-/** One cell's effort on its own: `tools 43 · agent 46s`. */
-declare const effortSummary: (e: Pick<CellEffort, 'tools' | 'agentMs'>) => string;
+/** One cell's effort on its own: `tools 43 · agent 46s`, plus tokens when any attempt reported them. */
+declare const effortSummary: (e: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens'>) => string;
 /** The same cell between two runs: `tools 44 → 10 (×0.23) · agent 45s → 21s (×0.47)`. */
-declare function effortChangeLabel(base: Pick<CellEffort, 'tools' | 'agentMs'>, head: Pick<CellEffort, 'tools' | 'agentMs'>): string;
+declare function effortChangeLabel(base: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens'>, head: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens'>): string;
 //#endregion
 //#region src/history/types.d.ts
 /** Stays 1 while reeval is pre-release: the record changes in place. */
@@ -86,10 +109,19 @@ type ArtifactRef = FileRef & {
   capture?: {
     port: number;
     path: string;
+    /** The live window, never the string `'full-page'`. */
     viewport?: {
       width: number;
       height: number;
     };
+    /** Saved PNG used `captureBeyondViewport`. */
+    fullPage?: boolean;
+    /** Goal strings only; wait beats omitted. */
+    instructions?: string[];
+    use?: {
+      model: string;
+      moves: number;
+    };
   };
 };
 type GitInfo = {
@@ -149,6 +181,8 @@ type JudgementRecord = {
   key: string;
   /** Transient provider errors retried before this verdict, with the backoff before each retry. */
   retries?: JudgeRetry[];
+  /** Captures this call sent the judge as file parts, when any. */
+  artifacts?: JudgeArtifactRef[];
 };
 type AttemptFiles = {
   /** Each environment setup step's command, exit code, duration and output. */
@@ -197,6 +231,17 @@ type AttemptRecord = {
    * recorded them.
    */
   servedModels: string[] | null;
+  /**
+   * The reasoning effort the harness factory asked for (`claudeCode({ effort: 'high' })`);
+   * null when the factory left it unset. Fast is not this.
+   */
+  effort: string | null;
+  /**
+   * The reasoning effort the harness's API said it ran at, from `servedModels` /
+   * each turn's `model` (`medium` from `gpt-5.6-luna[effort=medium]`); null when those
+   * labels name none. Fast (`[fast=true]`) is not this.
+   */
+  servedEffort: string | null;
   outcome: Outcome;
   error: AttemptError | null;
   durationMs: number;
@@ -218,6 +263,15 @@ type AttemptRecord = {
   }>;
   /** Tokens across the attempt's turns; null when the harness reported none. */
   usage: TokenUsage | null;
+  /**
+   * Tokens the capture vision loop used. Infrastructure only: never folded
+   * into `usage`, effort, compare or lift.
+   */
+  captureUse?: {
+    model: string;
+    moves: number;
+    usage: TokenUsage | null;
+  };
   timings: PhaseTimings;
   /** Counts from the transcript, so a run's shape is visible without reading it. */
   activity: {
@@ -277,7 +331,9 @@ type EvalEntry = {
   attempts: AttemptRecord[];
   /**
    * The compare's questions, answered after its attempts by pairwise votes, one per question;
-   * unset without any. They annotate: `outcome` and `passRate` never depend on them.
+   * unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
+   * may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
+   * (rates by cut, votes, ties, a failed call, a sample of reasons).
    */
   answers?: CompareAnswer[];
 };
@@ -332,6 +388,11 @@ type CellRecord = Cell & {
   harnessKind: string;
   /** The configured harness model, when it has one. */
   model: string | null;
+  /**
+   * The reasoning effort the harness factory asked for; null when it left it unset.
+   * Fast is not this.
+   */
+  effort: string | null;
   /** How the attempts decide the cell's outcome; `'none'` when it doesn't fail the eval. */
   gate: CellGate;
   /** Environment keys the variant replaced on the control's (or a child on its parent's). */
@@ -478,21 +539,31 @@ type Stack = {
   /** Exact versions of the packages that run and grade the agent. */
   packages: Record<string, string>;
   /**
-   * Each harness: its configured `model`, and `resolvedModels`, what it reported serving across
-   * the run's turns with the variant it resolved (`composer-2.5[fast=true]` for `composer-2.5`
-   * without `params`); empty when it reports none.
+   * Each harness: its configured `model` and requested `effort`, `resolvedModels` (what it
+   * reported serving, with the variant it resolved), and `resolvedEffort` (reasoning-effort
+   * suffixes those labels named). Fast (`[fast=true]`) is not `resolvedEffort`.
    */
   harnesses: Array<{
     id: string;
     kind: string;
     /** The configured model, when the harness has one. */
     model: string | null;
+    /**
+     * The reasoning effort the harness factory asked for; null when it left it unset.
+     * Fast is not this.
+     */
+    effort: string | null;
     /**
      * The models the harness reported serving, across every turn in the run, with the variant it
      * resolved (`composer-2.5[fast=true]` for `composer-2.5` with no `params`); empty when it
      * reports none.
      */
     resolvedModels: string[];
+    /**
+     * Reasoning-effort suffixes those served models named (`medium` from
+     * `gpt-5.6-luna[effort=medium]`); empty when they name none. Fast is not this.
+     */
+    resolvedEffort: string[];
   }>;
   /**
    * The run's judge: the config's, with what the profile's `judge` and `defaults.judge` set over it.
@@ -535,7 +606,7 @@ type HistoryCell = Pick<CellRecord, 'key' | 'role' | 'variant' | 'harness' | 'mo
     hi: number;
     withinNoise: boolean;
   };
-  /** Median tool calls and agent time over the counted attempts; null when none counted. */
+  /** Median tool calls, agent time and tokens over the counted attempts; null when none counted. */
   effort: CellEffort | null;
 };
 /**
@@ -640,14 +711,67 @@ type ExecResult = {
   stdout: string;
   stderr: string;
 };
-/** CSS pixels for `c.capture()`. Default 1280×720 when the call and cascade omit it. */
-type Viewport = {
+/** Live tab size for `c.capture()`. Default 1280×720 when the call and cascade omit `viewport`. */
+type CaptureWindow = {
   width: number;
   height: number;
 };
+/**
+ * Author camera. `{ width, height }` is the live window and the saved PNG.
+ * `'full-page'` keeps the default window and saves with `captureBeyondViewport`.
+ */
+type Viewport = CaptureWindow | 'full-page';
+/**
+ * Chromium look-ready events. `'load'` is `window` load (`Page.loadEventFired`).
+ * `'idle'` is `Page.lifecycleEvent` `networkAlmostIdle`.
+ */
+type CaptureWait = 'load' | 'idle';
+/** Same shape at top-level `wait` and `{ wait }` between instructions. */
+type Wait = CaptureWait | {
+  ms: number;
+};
+type CaptureInstruction = string | {
+  wait: Wait;
+};
+type CaptureUseAction = {
+  type: 'click';
+  x: number;
+  y: number;
+} | {
+  type: 'type';
+  text: string;
+} | {
+  type: 'key';
+  key: 'Tab' | 'Enter';
+} | {
+  type: 'scroll';
+  x: number;
+  y: number;
+  dy: number;
+} | {
+  type: 'drag';
+  x: number;
+  y: number;
+  toX: number;
+  toY: number;
+} | {
+  type: 'hover';
+  x: number;
+  y: number;
+} | {
+  type: 'done';
+};
+/** One list per instruction string. Tests; never downloads a model. */
+type CaptureUseScript = CaptureUseAction[][];
+/** Resolved vision driver after cascade, call override and analysis fallback. */
+type CaptureUse = {
+  model: string;
+} | {
+  fake: CaptureUseScript;
+};
 /**
  * Camera defaults for `c.capture()`. Cascades config → profile defaults → group → eval,
- * shallow-merged. `viewport` on a level replaces the whole object. `port` is not an option:
+ * shallow-merged. `viewport` on a level replaces the whole value. `port` is not an option:
  * pass it as the first argument to `c.capture`.
  */
 type CaptureOptions = {
@@ -656,6 +780,11 @@ type CaptureOptions = {
   timeout?: Duration;
   /** `false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws. */
   browser?: boolean;
+  /**
+   * Vision model for `instructions`. Cascades like `question`. Default is the analysis
+   * model when that model accepts images; never Jev.
+   */
+  model?: string;
 };
 /** A background process started by `c.exec(..., { background: true })` or `c.serve`. */
 type BackgroundHandle = {
@@ -672,6 +801,7 @@ type Capture = {
   port: number;
   /** Path on the app, as passed in. */
   path: string;
+  /** What the author asked, including `'full-page'`. */
   viewport: Viewport;
   /** `artifacts/<name>.png` under the attempt directory. */
   file: {
@@ -683,13 +813,19 @@ type Capture = {
 /**
  * Optional camera and path for `c.capture(port | ServeHandle, options?)`.
  * `port` is the first argument, not a field here. Default `path` is `/`;
- * default `name` is derived from the path.
+ * default `name` is derived from the path. `wait` and `instructions` are call-only.
  */
 type CaptureCall = {
   name?: string;
   path?: string;
   viewport?: Viewport;
   timeout?: Duration;
+  /** Look-ready once, before the first model shot or the saved PNG. Default `'load'`. */
+  wait?: Wait;
+  /** Journey: goal strings and `{ wait }` beats. Tests may pass `fakeInstructions(...)`. */
+  instructions?: CaptureInstruction[];
+  /** Overrides cascaded `capture.model` for this call. */
+  model?: string;
 };
 type FileChange = {
   path: string;
@@ -730,9 +866,10 @@ type TurnData = {
   model: string | null;
   /**
    * Event types the harness couldn't report for this turn: the agent may have searched or
-   * fetched with no event in `transcript`. `toHaveSearched()` and `toHaveFetched()` refuse such a
-   * turn. Usually `[]`; a Cursor turn whose web calls the adapter couldn't recover (its SDK drops
-   * them from the stream) is `['web-search', 'web-fetch']`.
+   * fetched with no event in `transcript`. `toHaveSearched()` and `toHaveFetched()` refuse
+   * such a turn. Usually `[]`; a Cursor turn
+   * whose web calls the adapter couldn't recover (its SDK drops them from the stream) is
+   * `['web-search', 'web-fetch']`.
    */
   unreported: WebEventType[];
   /** Workspace changes since the eval started, as of the end of this turn. */
@@ -746,19 +883,44 @@ type Turn = TurnData & {
 type JSONValue = null | boolean | number | string | JSONValue[] | {
   [key: string]: JSONValue;
 };
+/** JSON fields the judge sees for the turn: task, output, files, diff, optional contents. */
+type JudgeJsonState = {
+  [key: string]: JSONValue;
+};
+/** A capture sent to `experimental_decide` as a file part. */
+type JudgeFilePart = {
+  type: 'file';
+  mediaType: string;
+  data: Uint8Array;
+};
+/**
+ * What `experimental_decide` (and a fake judge) receive: a JSON object when the call has no
+ * captures, or an ordered list of one JSON part plus each capture's file part.
+ */
+type JudgeState = JudgeJsonState | Array<{
+  type: 'json';
+  value: JudgeJsonState;
+} | JudgeFilePart>;
+/** A capture this judge call sent as a file part. Bytes live in `artifacts/<name>.png`. */
+type JudgeArtifactRef = {
+  name: string;
+  /** `artifacts/<name>.png` under the attempt directory. */
+  path: string;
+  sha256: string;
+  bytes: number;
+  mediaType: 'image/png';
+};
 /**
  * A canned judge for tests and dry runs; see `fakeJudge()` in `reeval/judge`. `score` as a
  * function of the state the judge would see works per group, eval or call, not in the config.
  */
 type FakeJudgeModel = {
   kind: 'fake-judge';
-  score: number | ((state: {
-    [key: string]: JSONValue;
-  }) => number);
+  score: number | ((state: JudgeState) => number);
 };
 /**
  * `'provider/model'` (e.g. `'typesafe-ai/jev-1.13.0'`, `'google/gemini-3.8-flash'`), an AI SDK
- * evaluation model object (per group, eval or call only), or a fake judge.
+ * decision model object (per group, eval or call only), or a fake judge.
  */
 type JudgeModel = string | FakeJudgeModel | {
   readonly specificationVersion: string;
@@ -782,6 +944,11 @@ type JudgeOptions = {
 type JudgeCallOptions = JudgeOptions & {
   /** What to grade instead of the latest turn: another turn, or a value used as the output. */
   on?: Turn | JSONValue;
+  /**
+   * Captures this call sends the judge as file parts. A `c.capture()` handle or its name.
+   * Call-only: the bytes are on this attempt. The provider that cannot take files fails the call.
+   */
+  artifacts?: Array<Capture | string>;
 };
 type Verdict = {
   verdict: 'pass' | 'fail';
@@ -792,6 +959,8 @@ type Verdict = {
   model: string;
   /** The model id the provider says served the call, when it reports one. */
   resolvedModel?: string;
+  /** Captures this call sent the judge as file parts, when any. */
+  artifacts?: JudgeArtifactRef[];
 };
 /**
  * What `c.judge()` returns when the call failed (the provider's error, after any retries): the attempt fails as a judge error once the eval's body has finished,
@@ -805,7 +974,10 @@ type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel'> & {
 type AttemptContext = {
   /** Sends the next turn of this eval's agent session, starting the sandbox on first use. */
   send(prompt: string): Promise<Turn>;
-  /** Asks the judge a yes/no question about the latest turn (or `options.on`). */
+  /**
+   * Asks the judge a yes/no question about the latest turn (or `options.on`).
+   * `options.artifacts` is a list of `c.capture()` handles or names.
+   */
   judge(question: string, options?: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
   exec(command: string): Promise<ExecResult>;
   exec(command: string, options: {
@@ -822,6 +994,7 @@ type AttemptContext = {
   /**
    * Screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`.
    * First argument is the container port or a `ServeHandle` from `c.serve`.
+   * `instructions` runs a vision journey on that same tab first.
    * Needs a sandbox, same as `c.exec`.
    */
   capture(target: number | ServeHandle, options?: CaptureCall): Promise<Capture>;
@@ -866,6 +1039,11 @@ type FakeHarnessDescriptor = {
    * and variant (`composer-2.5[fast=true]`); unset, it reports none.
    */
   servedModel?: string;
+  /**
+   * Requested reasoning effort, recorded as `effort` on the cell. Use `servedModel` for what the
+   * turn reports (`fake-1[effort=medium]`). Fast is not this.
+   */
+  effort?: string;
   /** Tool calls the fake agent reports on every turn, before writing files. */
   tools?: Array<{
     tool: string;
@@ -932,7 +1110,12 @@ type CursorHarnessDescriptor = {
    * model catalog (`fast=true` for `composer-2.5`), sent explicitly; pass the ones that change cost.
    */
   params?: Record<string, string>;
-  /** Cell id; defaults to `cursor:<model>`. */
+  /**
+   * Reasoning effort, mapped onto the catalog param `Cursor.models.list()` names for this model
+   * (`reasoning_effort`, `effort`, …). Not Fast: that stays `params.fast`.
+   */
+  effort?: string;
+  /** Cell id; defaults to `cursor:<model>`, or `cursor:<model>[effort=<value>]` when `effort` is set. */
   id?: string;
   /** The most Cursor attempts at once, across the run, e.g. for the account's rate limits. */
   maxConcurrent?: number;
@@ -947,7 +1130,12 @@ type ClaudeCodeHarnessDescriptor = {
   kind: 'claude-code';
   /** A model id or alias the `claude` CLI accepts, e.g. `claude-sonnet-5` or `sonnet`. */
   model: string;
-  /** Cell id; defaults to `claude-code:<model>`. */
+  /**
+   * Reasoning effort the Agent SDK `/effort` control takes, e.g. `low`, `medium`, `high`,
+   * `xhigh` (Extra in the UI) or `max`.
+   */
+  effort?: string;
+  /** Cell id; defaults to `claude-code:<model>`, or `claude-code:<model>[effort=<value>]` when `effort` is set. */
   id?: string;
   /** The most Claude Code attempts at once, across the run, e.g. for the account's rate limits. */
   maxConcurrent?: number;
@@ -962,7 +1150,12 @@ type AntigravityHarnessDescriptor = {
   kind: 'antigravity';
   /** A model slug `agy models` lists for a Gemini API key, e.g. `gemini-3.8-flash-medium`. */
   model: string;
-  /** Cell id; defaults to `antigravity:<model>`. */
+  /**
+   * Reasoning effort passed as `agy --effort`, e.g. `low`, `medium`, `high` or `max`. The
+   * slug's `-low`/`-medium`/`-high` suffix is the same knob baked into `model`.
+   */
+  effort?: string;
+  /** Cell id; defaults to `antigravity:<model>`, or `antigravity:<model>[effort=<value>]` when `effort` is set. */
   id?: string;
   /** The most Antigravity attempts at once, across the run, e.g. for the key's rate limits. */
   maxConcurrent?: number;
@@ -984,7 +1177,12 @@ type OpenCodeHarnessDescriptor = {
    * `anthropic/claude-haiku-4-5`; the provider picks the key.
    */
   model: string;
-  /** Cell id; defaults to `opencode:<model>`. */
+  /**
+   * Reasoning effort for that provider: `options.reasoningEffort` (OpenAI) or `options.thinking`
+   * (Anthropic). Values are whatever that provider takes.
+   */
+  effort?: string;
+  /** Cell id; defaults to `opencode:<model>`, or `opencode:<model>[effort=<value>]` when `effort` is set. */
   id?: string;
   /** The most OpenCode attempts at once, across the run, e.g. for the key's rate limits. */
   maxConcurrent?: number;
@@ -1002,7 +1200,11 @@ type GooseHarnessDescriptor = {
   kind: 'goose';
   /** A Gemini model id, run on Goose's `google` provider, e.g. `gemini-3.8-flash`. */
   model: string;
-  /** Cell id; defaults to `goose:<model>`. */
+  /**
+   * Reasoning effort as `GOOSE_THINKING_EFFORT`, e.g. `off`, `low`, `medium`, `high` or `max`.
+   */
+  effort?: string;
+  /** Cell id; defaults to `goose:<model>`, or `goose:<model>[effort=<value>]` when `effort` is set. */
   id?: string;
   /** The most Goose attempts at once, across the run, e.g. for the key's rate limits. */
   maxConcurrent?: number;
@@ -1021,7 +1223,12 @@ type PiHarnessDescriptor = {
   kind: 'pi';
   /** A Gemini model id, run on pi's `google` provider, e.g. `gemini-3.8-flash`. */
   model: string;
-  /** Cell id; defaults to `pi:<model>`. */
+  /**
+   * Reasoning effort, passed as pi's `--thinking` (the CLI's name). Values are whatever that
+   * flag takes, e.g. `off`, `minimal`, `low`, `medium`, `high`, `xhigh` or `max`.
+   */
+  effort?: string;
+  /** Cell id; defaults to `pi:<model>`, or `pi:<model>[effort=<value>]` when `effort` is set. */
   id?: string;
   /** The most pi attempts at once, across the run, e.g. for the key's rate limits. */
   maxConcurrent?: number;
@@ -1039,7 +1246,12 @@ type CodexHarnessDescriptor = {
   kind: 'codex';
   /** An OpenAI model id Codex runs on the Responses API, e.g. `gpt-5.6-luna`. */
   model: string;
-  /** Cell id; defaults to `codex:<model>`. */
+  /**
+   * Reasoning effort as Codex's `model_reasoning_effort`, e.g. `low`, `medium`, `high`,
+   * `xhigh`, `max` or `ultra`.
+   */
+  effort?: string;
+  /** Cell id; defaults to `codex:<model>`, or `codex:<model>[effort=<value>]` when `effort` is set. */
   id?: string;
   /** The most Codex attempts at once, across the run, e.g. for the key's rate limits. */
   maxConcurrent?: number;
@@ -1396,16 +1608,27 @@ type CompareOptions<E = Environment> = {
    * win rates and never change pass, fail or the exit code.
    */
   questions?: CompareQuestion[];
+  /**
+   * The factor every question varies when pairing, when a question does not name its own
+   * `across`. `'environment'` holds the harness; `'harness'` holds the environment. A list of
+   * both, or omitted, asks both pair kinds.
+   */
+  across?: PairFactor | PairFactor[];
 };
 /**
  * A question for pairwise judgement: its text, or the text and the `'provider/model'` that
  * answers it. Without a model, `analysis.model` answers, else the judge's when it's a language model.
+ * `across` is the factor it varies when pairing; without it (and without a one-factor
+ * `compare.across`) both pair kinds run.
  */
 type CompareQuestion = string | {
   question: string;
   model?: string;
+  across?: PairFactor;
 };
-/** Which pairs a vote is over: two arms on one harness, or two harnesses on one arm. */
+/** A matrix factor a compare question varies when pairing; the others are held. */
+type PairFactor = 'environment' | 'harness';
+/** Which pairs a vote is over: two environments on one harness, or two harnesses on one environment. */
 type CompareCut = 'within' | 'across';
 /** One judgement of one pair of attempts, in one order. */
 type PairVote = {
@@ -1424,6 +1647,8 @@ type PairVote = {
   winner: string | 'tie';
   /** True for the repeat that showed `b` first. */
   swapped: boolean;
+  /** The model's short reason for this pick; omitted when it gave none. Capped. */
+  reason?: string;
 };
 /** A compare question's answer: every vote, and each side's wins over decided votes per cut. */
 type CompareAnswer = {
@@ -1476,7 +1701,7 @@ type EvalOptionFields = {
    */
   egress?: EgressPolicy;
   judge?: JudgeOptions;
-  /** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` replaces the whole object. */
+  /** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
   capture?: CaptureOptions;
   /** Per attempt. */
   timeout?: Duration;
@@ -1585,14 +1810,19 @@ type FakeAnalysisModel = {
   };
   /**
    * How it answers compare questions: a fixed pick, a function of the blinded pair, or
-   * `{ error }` to fail every call. Default `'tie'`.
+   * `{ error }` to fail every call. A pick may be `{ winner, reason? }`. Default `'tie'`.
    */
-  vote?: FakeVote | ((pair: BlindPair) => FakeVote) | {
+  vote?: FakePick | ((pair: BlindPair) => FakePick) | {
     error: string;
   };
 };
 /** A blinded label's pick: the attempt shown first (`A`), second (`B`), or neither. */
 type FakeVote = 'A' | 'B' | 'tie';
+/** A fake compare-question pick, optionally with a reason. */
+type FakePick = FakeVote | {
+  winner: FakeVote;
+  reason?: string;
+};
 /** What a compare question's model is shown: the prompt, and the two diffs in slot order. */
 type BlindPair = {
   question: string;
@@ -1857,5 +2087,5 @@ type SandboxWarning = {
   message: string;
 };
 //#endregion
-export { JudgeFailure as $, effortVerdict as $t, EnvironmentHelpers as A, FileRef as At, FakeNoteInput as B, EffortSample as Bt, DirectoryOptions as C, Verdict as Ct, EgressRecord as D, AttemptRecord as Dt, EgressPolicy as E, ArtifactRef as Et, EvalOptions as F, RunRecord as Ft, GroupOptions as G, cellEffort as Gt, FakeScriptStep as H, Spread as Ht, ExecResult as I, StackSummary as It, HarnessEdit as J, effortChangeLabel as Jt, HarnessData as K, cellSamples as Kt, FakeAnalysisModel as L, CellEffort as Lt, EnvironmentSpec as M, HistoryEntry as Mt, EsConfig as N, JudgementRecord as Nt, EnvVars as O, CellRecord as Ot, EvalFn as P, Lift as Pt, JudgeCallOptions as Q, effortSummary as Qt, FakeHarnessDescriptor as R, EFFORT_ALPHA as Rt, Diff as S, TurnStatus as St, Duration as T, WebEventType as Tt, FakeVote as U, VERDICT_TEXT as Ut, FakeSandboxDescriptor as V, EffortVerdict as Vt, GooseHarnessDescriptor as W, cellContrast as Wt, HistoryConfig as X, effortOf as Xt, HarnessOption as Y, effortLabel as Yt, HostFile as Z, effortSample as Zt, CompareOptions as _, SkillSource as _t, BackgroundHandle as a, PairVote as at, ConfigJudgeOptions as b, Turn as bt, Capture as c, Profile as ct, Cell as d, SandboxDescriptor as dt, formatCount as en, JudgeModel as et, CellRole as f, SandboxWarning as ft, CompareCut as g, SetupStep as gt, CompareAnswer as h, ServeHandle as ht, AttemptContext as i, Outcome as it, EnvironmentInput as j, HistoryCell as jt, Environment as k, EvalEntry as kt, CaptureCall as l, ProfileDefaults as lt, CodexHarnessDescriptor as m, SecretTemplate as mt, AnalysisConfig as n, formatSeconds as nn, McpServer as nt, BlindPair as o, PiHarnessDescriptor as ot, ClaudeCodeHarnessDescriptor as p, Secret as pt, HarnessDescriptor as q, contrastEffort as qt, AntigravityHarnessDescriptor as r, OpenCodeHarnessDescriptor as rt, Budget as s, PinnableKey as st, AgentHarnessDescriptor as t, formatRatio as tn, JudgeOptions as tt, CaptureOptions as u, ProfileLimit as ut, CompareQuestion as v, TokenUsage as vt, DockerSandboxDescriptor as w, Viewport as wt, CursorHarnessDescriptor as x, TurnData as xt, ConfigHelpers as y, TranscriptEvent as yt, FakeJudgeModel as z, EffortContrast as zt };
-//# sourceMappingURL=types-CoAfC-_k.d.ts.map
+export { HarnessDescriptor as $, TOKENS_UNREPORTED as $t, DockerSandboxDescriptor as A, TurnData as At, EvalFn as B, FileRef as Bt, CompareOptions as C, SecretTemplate as Ct, CursorHarnessDescriptor as D, TokenUsage as Dt, ConfigJudgeOptions as E, SkillSource as Et, Environment as F, WebEventType as Ft, FakeJudgeModel as G, RunRecord as Gt, ExecResult as H, HistoryEntry as Ht, EnvironmentHelpers as I, ArtifactRef as It, FakeScriptStep as J, EFFORT_ALPHA as Jt, FakeNoteInput as K, StackSummary as Kt, EnvironmentInput as L, AttemptRecord as Lt, EgressPolicy as M, Verdict as Mt, EgressRecord as N, Viewport as Nt, Diff as O, TranscriptEvent as Ot, EnvVars as P, Wait as Pt, HarnessData as Q, Spread as Qt, EnvironmentSpec as R, CellRecord as Rt, CompareCut as S, Secret as St, ConfigHelpers as T, SetupStep as Tt, FakeAnalysisModel as U, JudgementRecord as Ut, EvalOptions as V, HistoryCell as Vt, FakeHarnessDescriptor as W, Lift as Wt, GooseHarnessDescriptor as X, EffortSample as Xt, FakeVote as Y, EffortContrast as Yt, GroupOptions as Z, EffortVerdict as Zt, Cell as _, Profile as _t, BackgroundHandle as a, effortChangeLabel as an, JudgeCallOptions as at, CodexHarnessDescriptor as b, SandboxDescriptor as bt, Capture as c, effortSample as cn, JudgeOptions as ct, CaptureOptions as d, formatCount as dn, OpenCodeHarnessDescriptor as dt, VERDICT_TEXT as en, HarnessEdit as et, CaptureUse as f, formatRatio as fn, Outcome as ft, CaptureWindow as g, PinnableKey as gt, CaptureWait as h, PiHarnessDescriptor as ht, AttemptContext as i, contrastEffort as in, JudgeArtifactRef as it, Duration as j, TurnStatus as jt, DirectoryOptions as k, Turn as kt, CaptureCall as l, effortSummary as ln, JudgeState as lt, CaptureUseScript as m, tokenValues as mn, PairVote as mt, AnalysisConfig as n, cellEffort as nn, HistoryConfig as nt, BlindPair as o, effortLabel as on, JudgeFailure as ot, CaptureUseAction as p, formatSeconds as pn, PairFactor as pt, FakeSandboxDescriptor as q, CellEffort as qt, AntigravityHarnessDescriptor as r, cellSamples as rn, HostFile as rt, Budget as s, effortOf as sn, JudgeModel as st, AgentHarnessDescriptor as t, cellContrast as tn, HarnessOption as tt, CaptureInstruction as u, effortVerdict as un, McpServer as ut, CellRole as v, ProfileDefaults as vt, CompareQuestion as w, ServeHandle as wt, CompareAnswer as x, SandboxWarning as xt, ClaudeCodeHarnessDescriptor as y, ProfileLimit as yt, EsConfig as z, EvalEntry as zt };
+//# sourceMappingURL=types-_LOFdPUf.d.ts.map

package/dist/{usage-DsYUqIUM.js → usage-DDF_6tkV.js}

@@ -18,7 +18,9 @@ const sumUsage = (usages) => {
 };
 /** Every token the prompt side used: uncached input plus cache reads and writes. */
 const promptTokens = (u) => u.input + u.cacheRead + u.cacheWrite;
+/** Prompt side, output, and reasoning when the harness reported it. No USD. */
+const totalTokens = (u) => promptTokens(u) + u.output + (u.reasoning ?? 0);
 //#endregion
-export { promptTokens as n, sumUsage as r, addUsage as t };
+export { totalTokens as i, promptTokens as n, sumUsage as r, addUsage as t };
 
-//# sourceMappingURL=usage-DsYUqIUM.js.map
+//# sourceMappingURL=usage-DDF_6tkV.js.map

package/dist/{usage-DsYUqIUM.js.map → usage-DDF_6tkV.js.map}

@@ -1 +1 @@
-{"version":3,"file":"usage-DsYUqIUM.js","names":[],"sources":["../src/usage.ts"],"mappings":";;AAGA,SAAgB,SAAS,GAAsB,GAAyC;CACtF,IAAI,CAAC,KAAK,CAAC,GAAG,OAAO,KAAK;CAC1B,OAAO;EACL,OAAO,EAAE,QAAQ,EAAE;EACnB,QAAQ,EAAE,SAAS,EAAE;EACrB,WAAW,EAAE,YAAY,EAAE;EAC3B,YAAY,EAAE,aAAa,EAAE;EAC7B,WACE,EAAE,cAAc,QAAQ,EAAE,cAAc,OAAO,QAAQ,EAAE,aAAa,MAAM,EAAE,aAAa;CAC/F;AACF;;AAGA,MAAa,YAAY,WAAuE;CAC9F,IAAI,QAA2B;CAC/B,KAAK,MAAM,KAAK,QAAQ,QAAQ,SAAS,OAAO,KAAK,IAAI;CACzD,OAAO;AACT;;AAGA,MAAa,gBAAgB,MAAkB,EAAE,QAAQ,EAAE,YAAY,EAAE"}
+{"version":3,"file":"usage-DDF_6tkV.js","names":[],"sources":["../src/usage.ts"],"mappings":";;AAGA,SAAgB,SAAS,GAAsB,GAAyC;CACtF,IAAI,CAAC,KAAK,CAAC,GAAG,OAAO,KAAK;CAC1B,OAAO;EACL,OAAO,EAAE,QAAQ,EAAE;EACnB,QAAQ,EAAE,SAAS,EAAE;EACrB,WAAW,EAAE,YAAY,EAAE;EAC3B,YAAY,EAAE,aAAa,EAAE;EAC7B,WACE,EAAE,cAAc,QAAQ,EAAE,cAAc,OAAO,QAAQ,EAAE,aAAa,MAAM,EAAE,aAAa;CAC/F;AACF;;AAGA,MAAa,YAAY,WAAuE;CAC9F,IAAI,QAA2B;CAC/B,KAAK,MAAM,KAAK,QAAQ,QAAQ,SAAS,OAAO,KAAK,IAAI;CACzD,OAAO;AACT;;AAGA,MAAa,gBAAgB,MAAkB,EAAE,QAAQ,EAAE,YAAY,EAAE;;AAGzE,MAAa,eAAe,MAAkB,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,aAAa"}