reeval 0.0.1

package/dist/types-DLec-6Cx.d.ts

@@ -0,0 +1,1535 @@
+//#region src/view/effort.d.ts
+/** Below this two-sided p, an effort difference is an effect. */
+declare const EFFORT_ALPHA = 0.05;
+type EffortSample = {
+  tools: number;
+  agentMs: number;
+  fileEdits: number;
+};
+type Spread = {
+  median: number;
+  q1: number;
+  q3: number;
+};
+/** One cell's effort over its counted attempts (passes and fails; infra errors and skips aren't). */
+type CellEffort = {
+  n: number;
+  tools: Spread;
+  agentMs: Spread;
+  fileEdits: number;
+};
+/**
+ * `fewer` / `more`: the exact rank test's p is below `EFFORT_ALPHA`. `noise`: it isn't.
+ * `insufficient`: too few attempts for the test to ever reach it (2 against 2 can't).
+ */
+type EffortVerdict = 'fewer' | 'more' | 'noise' | 'insufficient';
+type EffortContrast = {
+  variant: CellEffort;
+  control: CellEffort;
+  /** Variant median over control median; null when the control's is 0. */
+  toolsRatio: number | null;
+  agentMsRatio: number | null;
+  agentMsDelta: number;
+  toolsP: number | null;
+  agentMsP: number | null;
+  /** The variant's and control's tool-call ranges don't overlap. */
+  separated: boolean;
+  /** On tool calls, the steadier measure; `timeVerdict` on agent time. */
+  verdict: EffortVerdict;
+  timeVerdict: EffortVerdict;
+};
+declare const effortSample: (a: Pick<AttemptRecord, 'activity' | 'timings'>) => EffortSample;
+declare function effortOf(samples: EffortSample[]): CellEffort | null;
+/** The verdict of an exact rank test of `xs` (variant) against `ys` (control). */
+declare function effortVerdict(xs: number[], ys: number[]): {
+  p: number | null;
+  verdict: EffortVerdict;
+};
+declare function contrastEffort(variant: EffortSample[], control: EffortSample[]): EffortContrast | null;
+/** The counted attempts of one cell of an eval, as effort samples. */
+declare const cellSamples: (e: Pick<EvalEntry, 'attempts'>, cellKey: string) => EffortSample[];
+declare const cellEffort: (e: Pick<EvalEntry, 'attempts'>, cellKey: string) => CellEffort | null;
+/** A variant cell's effort against its control's, in the same run; null for controls. */
+declare function cellContrast(e: Pick<EvalEntry, 'attempts' | 'cells'>, cellKey: string): EffortContrast | null;
+/** `4`, `4.5`: medians of counts are whole or halves. */
+declare const formatCount: (n: number) => string;
+declare const formatRatio: (r: number | null) => string;
+/** Seconds for agent time: `21s`, `1m 05s`. */
+declare function formatSeconds(ms: number): string;
+declare const VERDICT_TEXT: Record<EffortVerdict, (what: string) => string>;
+/**
+ * `tools 5 vs 43 (×0.11), agent 21s vs 46s (×0.47): too few attempts to test` for a variant
+ * against its control; with enough attempts, `fewer tool calls (p = 0.029)`.
+ */
+declare function effortLabel(c: EffortContrast): string;
+/** One cell's effort on its own: `tools 43 · agent 46s`. */
+declare const effortSummary: (e: Pick<CellEffort, 'tools' | 'agentMs'>) => string;
+/** The same cell between two runs: `tools 44 → 10 (×0.23) · agent 45s → 21s (×0.47)`. */
+declare function effortChangeLabel(base: Pick<CellEffort, 'tools' | 'agentMs'>, head: Pick<CellEffort, 'tools' | 'agentMs'>): string;
+//#endregion
+//#region src/history/types.d.ts
+/** Stays 1 while reeval is pre-release: the record changes in place. */
+declare const SCHEMA_VERSION = 1;
+type ArtifactLevel = 'all' | 'failures' | 'none';
+type RecordMode = 'manual' | 'ci' | 'always';
+/** `path` is relative to the run's artifact directory; `sha256` is of the bytes on disk. */
+type FileRef = {
+  path: string;
+  sha256: string;
+  bytes: number;
+};
+type GitInfo = {
+  /**
+   * The commit the run executed at: HEAD when `reeval run` started. The later commit that adds the
+   * history file is a different, descendant commit.
+   */
+  sha: string;
+  branch: string | null;
+  /** Uncommitted changes, outside `.reeval/` and the history folder, when the run started. */
+  dirty: boolean;
+  /** sha256 of the uncommitted changes (tracked and untracked, as a binary patch), when dirty. */
+  diffHash: string | null;
+  /** Those changes as `worktree.patch` in the artifact dir; apply on `sha` to reproduce. */
+  worktreePatch: FileRef | null;
+  /** Why a dirty run has no `worktree.patch`, e.g. too large or artifacts: 'none'. */
+  worktreePatchNote: string | null;
+  commitTime: string;
+  shallow: boolean;
+  /** The `origin` URL with any credentials removed. */
+  remote: string | null;
+};
+type RunnerInfo = {
+  kind: 'ci';
+  provider: string;
+  runUrl: string | null;
+  attempt: number | null;
+  /** The pull request a CI run built, when the provider says (GitHub Actions). */
+  pr: {
+    number: number | null;
+    baseRef: string | null;
+  } | null;
+} | {
+  kind: 'local';
+  hostHash: string;
+};
+/** One failed judge try that was retried, and the backoff before the next. */
+type JudgeRetry = {
+  error: string;
+  delayMs: number;
+};
+type JudgementRecord = {
+  question: string;
+  verdict: 'pass' | 'fail';
+  score: number;
+  threshold: number;
+  model: string;
+  /** The model id the provider says served the call, when it reports one. */
+  resolvedModel?: string;
+  /**
+   * `<index>:<sha256(question) prefix>`: the same question at the same place in the eval, so a
+   * verdict can be followed across runs (and an edited question starts a new series).
+   */
+  key: string;
+  /** Transient provider errors retried before this verdict, with the backoff before each retry. */
+  retries?: JudgeRetry[];
+};
+type AttemptFiles = {
+  /** Each environment setup step's command, exit code, duration and output. */
+  setupLog?: FileRef;
+  transcript?: FileRef;
+  /** Unnormalised harness stream parts, only with `REEVAL_RAW_TRANSCRIPT=1`. */
+  rawTranscript?: FileRef;
+  diff?: FileRef;
+  judgeInput?: FileRef;
+};
+type ContextFile = {
+  path: string;
+  sha256: string;
+};
+type AttemptRecord = {
+  /** `<eval key> # <cell key>#<n>`: unique in the run. */
+  key: string;
+  /** `<cell key>#<n>`: unique in its eval. */
+  id: string;
+  /** The cell's key; see `EvalEntry.cells`. */
+  cell: string;
+  /** Run number within the cell, from 1. */
+  n: number;
+  /** Start order across the run, from 1; null for attempts that never started. */
+  seq: number | null;
+  startedAt: string | null;
+  /** The image the container started from, in `stack.images`; null for the fake sandbox. */
+  imageId: string | null;
+  /** Names of the variables and secrets set in the sandbox, and the brokered ones; never values. */
+  env: {
+    vars: string[];
+    secrets: string[];
+    brokered: string[];
+  };
+  /** Skills, rules and MCP config visible to the agent in its sandbox, by content hash. */
+  context: ContextFile[];
+  /** The egress policy and, when the sandbox records it, the hosts contacted and refused; null if it never ran. */
+  egress: EgressRecord | null;
+  /**
+   * The models the harness's own API calls were served by, as the egress gateway saw them: named
+   * in the request (e.g. `gemini-3.8-flash` for the id `gemini-3.8-flash-low`, plus any the agent
+   * uses on the side) or, for Codex, in the response, with its effort
+   * (`gpt-5.4-mini-2026-03-17[effort=medium]`); null when neither names one (Cursor) or nothing
+   * recorded them.
+   */
+  servedModels: string[] | null;
+  outcome: Outcome;
+  error: AttemptError | null;
+  durationMs: number;
+  /** `text` is the agent's final message, cut at 8 KB; the whole of it is in `transcript.jsonl`. */
+  turns: Array<{
+    prompt: string;
+    status: TurnStatus;
+    startedAt: string;
+    durationMs: number;
+    text: string;
+    /** Tokens the turn used; null when the harness reports none. */
+    usage: TokenUsage | null;
+    /** The model the harness says served the turn; null when it reports none. */
+    model: string | null;
+    /** Event types the harness can't report; see `Turn.unreported`. */
+    unreported: WebEventType[];
+    /** The turn's first error, such as a provider's billing or auth error, cut at 1 KB; else null. */
+    error: string | null;
+  }>;
+  /** Tokens across the attempt's turns; null when the harness reported none. */
+  usage: TokenUsage | null;
+  timings: PhaseTimings;
+  /** Counts from the transcript, so a run's shape is visible without reading it. */
+  activity: {
+    tools: number;
+    fileEdits: number;
+    /** Null when a turn with none couldn't report them (`unreported`), not a count of 0 that isn't one. */
+    webSearches: number | null;
+    webFetches: number | null;
+    reasoningMs: number;
+  };
+  judgements: JudgementRecord[];
+  /** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
+  startRetries?: StartRetry[];
+  checks: Array<{
+    command: string;
+    exitCode: number;
+    durationMs: number;
+    log?: FileRef;
+  }>;
+  /** Files under a dependency directory (node_modules/, .venv/, …) collapse to one entry with `count`. */
+  changedFiles: Array<FileChange & {
+    count?: number;
+  }>;
+  files: AttemptFiles;
+};
+type EvalEntry = {
+  /** `file › group › … › name`, as shown. */
+  id: string;
+  /** What the eval is matched on across runs; its `id` for now. */
+  key: string;
+  /** Directory name inside the artifact directory, from `key`. */
+  slug: string;
+  file: string;
+  group: string[];
+  name: string;
+  /** Results are comparable only when fingerprints match. */
+  fingerprint: string;
+  /** The eval file and every local file it imports, directly or not, by content hash. */
+  sources: Array<{
+    path: string;
+    sha256: string;
+  }>;
+  tags: string[];
+  /**
+   * Over gated cells: `fail` if any failed, else `infra_error` if any errored, else `skipped` if
+   * all were, else `pass`. An ungated cell's failures don't fail the eval.
+   */
+  outcome: Outcome;
+  /** A placeholder, `evaluate.todo` or `group.todo`: it never runs, and its outcome is `skipped`. */
+  todo: boolean;
+  /** pass / (pass + fail) over the gated cells' attempts; infra errors and skips excluded. */
+  passRate: number | null;
+  cells: CellRecord[];
+  attempts: AttemptRecord[];
+};
+/** A variant cell's pass rate against the control's, on the same harness and sandbox. */
+type Lift = {
+  /** The control cell's key. */
+  vs: string;
+  /** Passes out of counted attempts (passes and fails) on each side. */
+  control: {
+    pass: number;
+    counted: number;
+  };
+  variant: {
+    pass: number;
+    counted: number;
+  };
+  /** Variant pass rate minus control pass rate, from -1 to 1. */
+  points: number;
+  /** 95% interval on the difference, by Newcombe's hybrid score method from two Wilson intervals. */
+  interval: {
+    lo: number;
+    hi: number;
+    level: 0.95;
+    method: 'newcombe-wilson';
+  };
+  /** The interval spans 0: the attempts can't tell the variant from the control. */
+  withinNoise: boolean;
+  /** Set when the compare has a `minLift`; the cell fails when `interval.lo` is below it. */
+  minLift?: number;
+};
+/** Each input of a cell fingerprint, hashed on its own, so a change can be named. */
+type FingerprintParts = {
+  /** Over `EvalEntry.sources`. */
+  sources: string;
+  /** Over the harness descriptor, less concurrency caps. */
+  harness: string;
+  environment: string | null;
+  image: string | null;
+  agentCli: string | null;
+  /** Over the judge's model, threshold and version. */
+  judge: string;
+  /** Over the attempt's context files. */
+  context: string;
+};
+/** One harness and environment of an eval, and how its attempts went. */
+type CellRecord = Cell & {
+  /** `sha256` of `fingerprintParts`; results are comparable only when it matches. */
+  fingerprint: string;
+  fingerprintParts: FingerprintParts;
+  /** The environment's digest: the join key into `RunRecord.environments`; null for none. */
+  environmentDigest: string | null;
+  harnessKind: string;
+  /** The configured harness model, when it has one. */
+  model: string | null;
+  /** How the attempts decide the cell's outcome; `'none'` when it doesn't fail the eval. */
+  gate: CellGate;
+  /** Environment keys the variant replaced on the control's (or a child on its parent's). */
+  overrides: string[];
+  runs: number;
+  pass: number;
+  fail: number;
+  infraError: number;
+  skipped: number;
+  passRate: number | null;
+  /** By `gate` (`'none'` reports the worst outcome); a variant below its `minLift` fails too. */
+  outcome: Outcome;
+  lift?: Lift;
+};
+/** Counts attempts, except `evals`. */
+type RunSummary = {
+  evals: number;
+  attempts: number;
+  pass: number;
+  /** Every failed attempt, including `notGated` ones. */
+  fail: number;
+  /** Failed attempts in ungated cells (controls, by default), which don't fail the run. */
+  notGated: number;
+  infraError: number;
+  skipped: number;
+};
+/** The tokens the run used, as the harnesses count them; reeval prices nothing. */
+type Cost = {
+  /** Prompt tokens, cache reads and writes included. */
+  inputTokens: number | null;
+  outputTokens: number | null;
+  /** The same totals split as each turn reports them. */
+  usage: TokenUsage | null;
+  source: 'reported' | 'unreported';
+};
+type RunRecord = {
+  kind: 'es-run';
+  schemaVersion: typeof SCHEMA_VERSION;
+  /** Also the file name: `<UTC start>_<short sha>_<author or ci>`. */
+  runId: string;
+  /** `budget` when `budget.duration` stopped new attempts from starting. */
+  status: 'complete' | 'interrupted' | 'budget';
+  startedAt: string;
+  endedAt: string;
+  durationMs: number;
+  author: string;
+  git: GitInfo | null;
+  runner: RunnerInfo;
+  dependencies: {
+    /** Lockfiles in the project root (and the repo root, if different), by content hash. */
+    lockfiles: Array<{
+      path: string;
+      sha256: string;
+    }>;
+  };
+  stack: Stack;
+  /** Each environment the run applied, fingerprinted; secret values are never included. */
+  environments: EnvironmentRecord[];
+  /** The profile the run was sized with, and why it was chosen. */
+  profile: {
+    name: string;
+    reason: string;
+    defaults: {
+      harness?: string[];
+      environment?: string;
+      sandbox?: string;
+      runs?: number;
+      timeout?: string | number;
+      env?: string[];
+    };
+    limit: {
+      runs?: number;
+      compare?: false | {
+        maxVariants: number;
+      };
+      harness?: string[];
+    };
+  } | null;
+  /**
+   * What the run planned before it started, and each level whose sandbox ran as root or with
+   * capabilities added back.
+   */
+  plan: {
+    evals: number;
+    cells: number;
+    attempts: number;
+    estimateMs: number;
+    sandboxWarnings: SandboxWarning[];
+  };
+  budget: {
+    attempts?: number;
+    duration?: string | number;
+  } | null;
+  config: {
+    sandbox: string;
+    environment: string | null;
+    runs: number;
+    judgeModel: string | null;
+    threshold: number;
+    timeoutMs: number;
+    /** Attempts that ran at once; not part of `configHash`, since it doesn't change results. */
+    concurrency: number;
+    configHash: string;
+  };
+  cost: Cost;
+  /** Which artifacts sit next to this record. Refs to files that weren't kept are removed. */
+  artifacts: ArtifactLevel;
+  redaction: {
+    version: 1;
+  };
+  summary: RunSummary;
+  evals: EvalEntry[];
+};
+/** What ran and graded the agents: the machine, images, harnesses and judge. */
+type Stack = {
+  es: string;
+  node: string;
+  os: string;
+  arch: string;
+  /** Every image an attempt started from. `id` is content-addressed; `repoDigests` only exist once pushed. */
+  images: Array<AttemptImage & {
+    repoDigests: string[];
+  }>;
+  /** Exact versions of the packages that run and grade the agent. */
+  packages: Record<string, string>;
+  /**
+   * Each harness: its configured `model`, and `resolvedModels`, what it reported serving across
+   * the run's turns with the variant it resolved (`composer-2.5[fast=true]` for `composer-2.5`
+   * without `params`); empty when it reports none.
+   */
+  harnesses: Array<{
+    id: string;
+    kind: string;
+    /** The configured model, when the harness has one. */
+    model: string | null;
+    /**
+     * The models the harness reported serving, across every turn in the run, with the variant it
+     * resolved (`composer-2.5[fast=true]` for `composer-2.5` with no `params`); empty when it
+     * reports none.
+     */
+    resolvedModels: string[];
+  }>;
+  judge: {
+    /** The config's default model; groups, evals and calls may use others (see each verdict). */
+    model: string | null;
+    /** Model ids the provider reported serving, across every judge call in the run. */
+    resolvedModels: string[];
+    threshold: number;
+    /** Changes when reeval, the AI SDK or the provider adapter (and so the judge prompt) change. */
+    version: string;
+  };
+  /** Union of every attempt's context files. */
+  context: ContextFile[];
+  /** What eval fingerprints hash; fingerprints from different scopes aren't comparable. */
+  fingerprintScope: string[];
+};
+/** The parts of a run's stack that `reeval compare` checks for drift. */
+type StackSummary = {
+  imageIds: string[];
+  agentClis: string[];
+  harnesses: string[];
+  /** `name@digest` per environment. */
+  environments: string[];
+  judgeModel: string | null;
+  /** What the judge provider reported serving; differs from `judgeModel` on silent updates. */
+  judgeServed: string[];
+  judgeVersion: string;
+  threshold: number;
+  fingerprintScope: string[];
+};
+/** A cell in a `HistoryEntry`: enough to follow cells, variants and lift across runs. */
+type HistoryCell = Pick<CellRecord, 'key' | 'role' | 'variant' | 'harness' | 'model' | 'environment' | 'environmentDigest' | 'gate' | 'fingerprint' | 'fingerprintParts' | 'runs' | 'pass' | 'fail' | 'infraError' | 'skipped' | 'passRate' | 'outcome'> & {
+  lift?: {
+    vs: string;
+    points: number;
+    lo: number;
+    hi: number;
+    withinNoise: boolean;
+  };
+  /** Median tool calls and agent time over the counted attempts; null when none counted. */
+  effort: CellEffort | null;
+};
+/**
+ * One line of the derived history listing: cached locally by the CLI, and the run header in the
+ * studio's index shards.
+ */
+type HistoryEntry = {
+  runId: string;
+  recorded: boolean;
+  status: RunRecord['status'];
+  startedAt: string;
+  durationMs: number;
+  author: string;
+  git: Pick<GitInfo, 'sha' | 'branch' | 'dirty'> | null;
+  runner: RunnerInfo['kind'];
+  /** The CI provider, its run page and pull request, for CI runs. */
+  ci: {
+    provider: string;
+    runUrl: string | null;
+    pr: {
+      number: number | null;
+      baseRef: string | null;
+    } | null;
+  } | null;
+  harnesses: string[];
+  summary: RunSummary;
+  /** The run's tokens, summed from the turns that reported them. */
+  usage: TokenUsage | null;
+  stack: StackSummary;
+  evals: Array<{
+    id: string;
+    key: string;
+    slug: string;
+    fingerprint: string;
+    sources: EvalEntry['sources'];
+    outcome: Outcome;
+    passRate: number | null;
+    cells: HistoryCell[];
+  }>;
+};
+//#endregion
+//#region src/types.d.ts
+type Duration = number | `${number}${'ms' | 's' | 'm' | 'h'}`;
+/** When an event started: wall clock, and ms since its turn started. */
+type EventTime = {
+  ts: string;
+  atMs: number;
+};
+type FileEditEvent = {
+  type: 'file-edit';
+  toolCallId: string;
+  path: string;
+  kind: 'create' | 'modify' | 'delete';
+  /** Line counts, or null when the harness didn't send the content (e.g. deletes). */
+  added: number | null;
+  removed: number | null;
+};
+/**
+ * Normalised agent events. Each tool call is one event, merged with its result; `durationMs`
+ * is null when the harness never reported the end.
+ */
+type TranscriptEvent = EventTime & ({
+  type: 'text' | 'reasoning';
+  text: string;
+  durationMs: number | null;
+} | (FileEditEvent & {
+  durationMs: number | null;
+}) | {
+  type: 'web-search';
+  toolCallId: string;
+  query: string;
+  /** How many sources the harness says it used; Cursor sends no URLs or titles. */
+  references: number | null;
+  durationMs: number | null;
+} | {
+  type: 'web-fetch';
+  toolCallId: string;
+  url: string;
+  ok: boolean;
+  durationMs: number | null;
+} | {
+  type: 'tool';
+  toolCallId: string;
+  tool: string;
+  input: unknown;
+  output: unknown;
+  ok: boolean;
+  durationMs: number | null;
+} | {
+  type: 'error';
+  message: string;
+} |
+/** A step that failed while the turn went on: a model call the harness retried. */
+{
+  type: 'retry';
+  message: string;
+});
+/** The transcript events a harness may be unable to report. */
+type WebEventType = 'web-search' | 'web-fetch';
+type ExecResult = {
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+};
+type FileChange = {
+  path: string;
+  status: 'added' | 'modified' | 'deleted';
+};
+type Diff = {
+  patch: string;
+  files: FileChange[];
+};
+type TurnStatus = 'completed' | 'aborted' | 'error';
+/**
+ * The tokens one turn used, as its harness reported them. `input` excludes cache reads and
+ * writes (so the prompt was `input + cacheRead + cacheWrite`); `reasoning` is part of `output`.
+ */
+type TokenUsage = {
+  input: number;
+  output: number;
+  cacheRead: number;
+  cacheWrite: number;
+  reasoning: number | null;
+};
+/** One agent turn. Immutable once `t.send()` resolves. */
+/** A turn as data, as a record or another language's runner carries it: `Turn` without its methods. */
+type TurnData = {
+  prompt: string;
+  /** The agent's final message for this turn. */
+  text: string;
+  status: TurnStatus;
+  startedAt: string;
+  durationMs: number;
+  transcript: TranscriptEvent[];
+  /** Tokens the turn used; null when the harness reports none. */
+  usage: TokenUsage | null;
+  /**
+   * The model the harness says served the turn, e.g. `composer-2.5[fast=false]`; null when it
+   * reports none. It can differ from the requested one (an alias, or parameters it filled in).
+   */
+  model: string | null;
+  /**
+   * Event types the harness couldn't report for this turn: the agent may have searched or
+   * fetched with no event in `transcript`. `toHaveSearched()` and `toHaveFetched()` refuse such a
+   * turn. Usually `[]`; a Cursor turn whose web calls the adapter couldn't recover (its SDK drops
+   * them from the stream) is `['web-search', 'web-fetch']`.
+   */
+  unreported: WebEventType[];
+  /** Workspace changes since the eval started, as of the end of this turn. */
+  diff: Diff;
+};
+type Turn = TurnData & {
+  /** Runs in the eval's sandbox. Same as `t.exec`. */
+  exec(command: string): Promise<ExecResult>;
+  readFile(path: string): Promise<string | null>;
+};
+type JSONValue = null | boolean | number | string | JSONValue[] | {
+  [key: string]: JSONValue;
+};
+/** A canned judge for tests and dry runs; see `fakeJudge()` in `reeval/judge`. */
+type FakeJudgeModel = {
+  kind: 'fake-judge';
+  score: number;
+};
+/**
+ * `'provider/model'` (e.g. `'typesafe-ai/jev-1.13.0'`, `'google/gemini-3.8-flash'`), an AI SDK
+ * evaluation model object (per group, eval or call only), or a fake judge.
+ */
+type JudgeModel = string | FakeJudgeModel | {
+  readonly specificationVersion: string;
+};
+/** Merged field by field down the cascade, so per-model settings can join as new fields. */
+type JudgeOptions = {
+  model?: JudgeModel;
+  /** Minimum score (0–1) for a `pass` verdict. */
+  threshold?: number;
+  providerOptions?: Record<string, Record<string, JSONValue>>;
+  /** The most judge calls to this model's provider at once, across the run. */
+  maxConcurrent?: number;
+};
+type JudgeCallOptions = JudgeOptions & {
+  /** What to grade instead of the latest turn: another turn, or a value used as the output. */
+  on?: Turn | JSONValue;
+};
+type Verdict = {
+  verdict: 'pass' | 'fail';
+  /** The judge's probability that the answer is yes, from 0 to 1. */
+  score: number;
+  threshold: number;
+  question: string;
+  model: string;
+  /** The model id the provider says served the call, when it reports one. */
+  resolvedModel?: string;
+};
+type EvalContext = {
+  /** Sends the next turn of this eval's agent session, starting the sandbox on first use. */
+  send(prompt: string): Promise<Turn>;
+  /** Asks the judge a yes/no question about the latest turn (or `options.on`). */
+  judge(question: string, options?: JudgeCallOptions): Promise<Verdict>;
+  exec(command: string): Promise<ExecResult>;
+  /** Workspace-relative, or `~/` for the home directory; `null` when there's no such file. */
+  readFile(path: string): Promise<string | null>;
+  /**
+   * Starts the sandbox and the agent session without a turn, for checks on what the environment
+   * set up. `t.send()` starts them too; calling both starts them once.
+   */
+  sandbox(): Promise<Pick<EvalContext, 'exec' | 'readFile'>>;
+  /** Every turn so far, oldest first. */
+  readonly turns: readonly Turn[];
+  /** Aborted on timeout or Ctrl-C. */
+  readonly signal: AbortSignal;
+  /** Which cell of the matrix this attempt is, and its run number (from 1). */
+  readonly attempt: {
+    id: string;
+    cell: Cell;
+    n: number;
+  };
+};
+type EvalFn = (t: EvalContext) => Promise<void>;
+type FakeHarnessDescriptor = {
+  kind: 'fake';
+  /** Cell id; defaults to `fake`. */
+  id?: string;
+  /** Files the fake agent writes into the sandbox on every turn. */
+  files?: Record<string, string>;
+  text?: string;
+  /** Simulated turn length, honours the abort signal. */
+  delayMs?: number;
+  /** Sits out `delayMs` even after an abort, like an agent that doesn't stop when it's told to. */
+  ignoreAbort?: boolean;
+  /** Throw before the first turn starts, to simulate an infra failure. */
+  failWith?: string;
+  /** The first this many session starts of an attempt never become ready, even when aborted (`true`: every one). */
+  startHangs?: number | true;
+  /** A reasoning block at the start of every turn. */
+  reasoning?: string;
+  /**
+   * The model the fake agent says served each turn, as a real harness reports its resolved model
+   * and variant (`composer-2.5[fast=true]`); unset, it reports none.
+   */
+  servedModel?: string;
+  /** Tool calls the fake agent reports on every turn, before writing files. */
+  tools?: Array<{
+    tool: string;
+    input: unknown;
+    output: unknown;
+    ok?: boolean;
+  }>;
+  /** Files and message used instead when the environment gives the agent a skill. */
+  withSkill?: {
+    files?: Record<string, string>;
+    text?: string;
+  };
+  /** Chance (0–1) that an attempt writes no files and says so, for flaky pass rates. */
+  flakeRate?: number;
+  /**
+   * Real sandbox actions, one list per `t.send` (the last list repeats), run in order before
+   * `files`. `exec` runs as the agent: with the sandbox's own variables, none of the checks'.
+   */
+  script?: FakeScriptStep[][];
+  /** The most attempts on this harness kind at once, across the run. */
+  maxConcurrent?: number;
+  /**
+   * How long the harness's session may take to become ready (its bootstrap and first session)
+   * before the attempt is an infra error, after one retry; default 90s.
+   */
+  startTimeout?: Duration;
+  /**
+   * Behave like this agent: its image, and where it reads rules and skills (`CLAUDE.md` and
+   * `.claude/skills` for `claude-code`, `~/.gemini/config/skills` for `antigravity`,
+   * `~/.config/opencode/skills` for `opencode`, Goose's private home for `goose`,
+   * `~/.pi/agent/skills` for `pi`, `~/.agents/skills` for `codex`). Defaults to the
+   * fake's own (`AGENTS.md`, `.agents/skills`) in Cursor's image.
+   */
+  layout?: 'cursor' | 'claude-code' | 'antigravity' | 'opencode' | 'goose' | 'pi' | 'codex';
+};
+/**
+ * One action of a scripted fake agent. `exec` is reported as a `shell` tool call whose output is
+ * the command's stdout, `write` as file edits, and `reply` is the turn's final message. `mcp`
+ * (`'<server>/<tool>'`) starts the environment's stdio MCP server in the sandbox, as an agent
+ * would, and calls the tool with no arguments; it's reported as `mcp__<server>__<tool>`.
+ * `error` ends the turn there with status `error` and that message, as a provider error would.
+ */
+type FakeScriptStep = {
+  exec: string;
+} | {
+  write: Record<string, string>;
+} | {
+  reply: string;
+} | {
+  mcp: string;
+} | {
+  error: string;
+};
+/**
+ * Cursor's agent through the Cursor TypeScript SDK's local runtime
+ * (`@reeval/harness-cursor-sdk`), made by `cursor()`. It reports each turn's tokens.
+ */
+type CursorHarnessDescriptor = {
+  kind: 'cursor';
+  /** A model id `Cursor.models.list()` returns for the key, e.g. `composer-2.5`. */
+  model: string;
+  /**
+   * Model parameters, e.g. `{ fast: 'false' }`. Unset ones take the default variant in Cursor's
+   * model catalog (`fast=true` for `composer-2.5`), sent explicitly; pass the ones that change cost.
+   */
+  params?: Record<string, string>;
+  /** Cell id; defaults to `cursor:<model>`. */
+  id?: string;
+  /** The most Cursor attempts at once, across the run, e.g. for the account's rate limits. */
+  maxConcurrent?: number;
+  /**
+   * How long the harness's session may take to become ready (its bootstrap and first session)
+   * before the attempt is an infra error, after one retry; default 90s.
+   */
+  startTimeout?: Duration;
+};
+/** Claude Code through `@ai-sdk/harness-claude-code`, made by `claudeCode()`. */
+type ClaudeCodeHarnessDescriptor = {
+  kind: 'claude-code';
+  /** A model id or alias the `claude` CLI accepts, e.g. `claude-sonnet-5` or `sonnet`. */
+  model: string;
+  /** Cell id; defaults to `claude-code:<model>`. */
+  id?: string;
+  /** The most Claude Code attempts at once, across the run, e.g. for the account's rate limits. */
+  maxConcurrent?: number;
+  /**
+   * How long the harness's session may take to become ready (its bootstrap and first session)
+   * before the attempt is an infra error, after one retry; default 90s.
+   */
+  startTimeout?: Duration;
+};
+/** The Antigravity CLI through `@reeval/harness-antigravity`, made by `antigravity()`. */
+type AntigravityHarnessDescriptor = {
+  kind: 'antigravity';
+  /** A model slug `agy models` lists for a Gemini API key, e.g. `gemini-3.8-flash-medium`. */
+  model: string;
+  /** Cell id; defaults to `antigravity:<model>`. */
+  id?: string;
+  /** The most Antigravity attempts at once, across the run, e.g. for the key's rate limits. */
+  maxConcurrent?: number;
+  /**
+   * How long the harness's session may take to become ready (its bootstrap and first session)
+   * before the attempt is an infra error, after one retry; default 90s.
+   */
+  startTimeout?: Duration;
+};
+/**
+ * OpenCode through `@reeval/harness-opencode` (`opencode acp`), made by `opencode()`. It runs
+ * on any provider whose key OpenCode sends in a header on known hosts, with that provider's key,
+ * and reports no usage (its ACP server counts one call).
+ */
+type OpenCodeHarnessDescriptor = {
+  kind: 'opencode';
+  /**
+   * OpenCode's model id, `<provider>/<model>`, e.g. `google/gemini-3.8-flash` or
+   * `anthropic/claude-haiku-4-5`; the provider picks the key.
+   */
+  model: string;
+  /** Cell id; defaults to `opencode:<model>`. */
+  id?: string;
+  /** The most OpenCode attempts at once, across the run, e.g. for the key's rate limits. */
+  maxConcurrent?: number;
+  /**
+   * How long the harness's session may take to become ready (its bootstrap and first session)
+   * before the attempt is an infra error, after one retry; default 90s.
+   */
+  startTimeout?: Duration;
+};
+/**
+ * Goose through `@reeval/harness-goose` (`goose acp`), made by `goose()`. It runs Google models
+ * on a Gemini API key, and reports no usage (its ACP server counts one call).
+ */
+type GooseHarnessDescriptor = {
+  kind: 'goose';
+  /** A Gemini model id, run on Goose's `google` provider, e.g. `gemini-3.8-flash`. */
+  model: string;
+  /** Cell id; defaults to `goose:<model>`. */
+  id?: string;
+  /** The most Goose attempts at once, across the run, e.g. for the key's rate limits. */
+  maxConcurrent?: number;
+  /**
+   * How long the harness's session may take to become ready (its bootstrap and first session)
+   * before the attempt is an infra error, after one retry; default 90s.
+   */
+  startTimeout?: Duration;
+};
+/**
+ * pi through `@reeval/harness-pi` (`pi --mode json` in the sandbox), made by `pi()`. It runs
+ * Google models on a Gemini API key and reports each model call's tokens. It has no MCP client.
+ */
+type PiHarnessDescriptor = {
+  kind: 'pi';
+  /** A Gemini model id, run on pi's `google` provider, e.g. `gemini-3.8-flash`. */
+  model: string;
+  /** Cell id; defaults to `pi:<model>`. */
+  id?: string;
+  /** The most pi attempts at once, across the run, e.g. for the key's rate limits. */
+  maxConcurrent?: number;
+  /**
+   * How long the harness's session may take to become ready (its bootstrap and first session)
+   * before the attempt is an infra error, after one retry; default 90s.
+   */
+  startTimeout?: Duration;
+};
+/**
+ * OpenAI's Codex CLI through `@reeval/harness-codex` (`codex exec --json`), made by `codex()`.
+ * It runs OpenAI models on an OpenAI API key and reports each turn's tokens, reasoning included.
+ */
+type CodexHarnessDescriptor = {
+  kind: 'codex';
+  /** An OpenAI model id Codex runs on the Responses API, e.g. `gpt-5.4-mini`. */
+  model: string;
+  /** Cell id; defaults to `codex:<model>`. */
+  id?: string;
+  /** The most Codex attempts at once, across the run, e.g. for the key's rate limits. */
+  maxConcurrent?: number;
+  /**
+   * How long the harness's session may take to become ready (its bootstrap and first session)
+   * before the attempt is an infra error, after one retry; default 90s.
+   */
+  startTimeout?: Duration;
+};
+/** A real coding agent, driven through its AI SDK harness adapter. */
+type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor;
+type HarnessDescriptor = FakeHarnessDescriptor | AgentHarnessDescriptor;
+type FakeSandboxDescriptor = {
+  kind: 'fake';
+  id?: string;
+  /** Parent directory for the per-eval workspace (defaults to the OS temp dir). */
+  dir?: string;
+  /** The most attempts on this sandbox kind at once, across the run. */
+  maxConcurrent?: number;
+};
+type DockerSandboxDescriptor = {
+  kind: 'docker';
+  /** Cell id; defaults to `docker`, or `docker:<image>` for other images. */
+  id?: string;
+  /** An image that already has the harness baked in, used as is. */
+  image?: string;
+  /** A Debian or Ubuntu based toolchain image; reeval bakes Node and the harness on top, cached. */
+  from?: string;
+  /**
+   * An absolute path to a Dockerfile (`docker()` resolves it against the calling file), built from
+   * its directory with `REEVAL_BASE` set to reeval's sandbox image and cached by content hash.
+   */
+  dockerfile?: string;
+  /** Baked onto the sandbox image in one build container: in order, no secrets, no workspace. */
+  install?: InstallStep[];
+  /** The most Docker sandboxes at once, across the run, whatever their image. */
+  maxConcurrent?: number;
+  /**
+   * Run the agent, setup steps and checks as root, with the capabilities `apt-get` needs, instead
+   * of the unprivileged `es` user. For evals that install system packages at run time.
+   */
+  root?: boolean;
+  /** Linux capabilities to add back, e.g. `['NET_RAW']`; every other one is dropped. */
+  capabilities?: string[];
+  /** The container's memory, with no swap on top: `'4g'` (default), `'512m'`, or bytes. */
+  memory?: string | number;
+  /** CPUs the container may use, fractions allowed. Default 2, capped at the Docker host's. */
+  cpus?: number;
+  /** Processes and threads at once. Default 1024. */
+  pids?: number;
+};
+type SandboxDescriptor = FakeSandboxDescriptor | DockerSandboxDescriptor;
+/**
+ * A host environment variable, redacted everywhere reeval writes and recorded by name only. Made by
+ * the `secret` helper of `defineEnvironment` and `defineConfig` callbacks.
+ */
+type Secret = {
+  kind: 'secret';
+  /** The host variable; defaults to the key it's set under. */
+  from?: string;
+  /** When the host variable is unset, skip it (the record says so) instead of failing. */
+  optional?: boolean;
+  /** `agent` (default): the container's environment. `checks`: only `t.exec` and `setup` steps. */
+  to?: 'agent' | 'checks';
+  /**
+   * Brokers the secret: the sandbox gets a per-attempt placeholder, and the Docker egress gateway
+   * swaps in the real value only on HTTPS requests to these hosts (`*.example.com` for subdomains).
+   */
+  hosts?: string[];
+  /** Where in a request to those hosts the gateway substitutes; defaults to `['header']`. */
+  in?: SubstituteIn[];
+};
+/** `header` values, the `query` string, or the request `body`. */
+type SubstituteIn = 'header' | 'query' | 'body';
+type SecretOptions = {
+  optional?: boolean;
+  to?: 'agent' | 'checks';
+  hosts?: readonly string[];
+  in?: readonly SubstituteIn[];
+};
+/** `secret.template('Bearer ${TOKEN}')`: host variables substituted into a string, for MCP headers. */
+type SecretTemplate = {
+  kind: 'secret-template';
+  template: string;
+};
+type EnvValue = string | Secret;
+type EnvVars = Record<string, EnvValue>;
+/** A file on the host, read when an attempt starts. Made by the `file` helper. */
+type HostFile = {
+  kind: 'file';
+  path: string;
+};
+type McpServer = {
+  kind: 'mcp';
+  transport: 'stdio';
+  command: string;
+  args: string[];
+  env: EnvVars;
+} | {
+  kind: 'mcp';
+  transport: 'http' | 'sse';
+  url: string;
+  headers: Record<string, string | Secret | SecretTemplate>;
+};
+/** `harness` (default): the agent's own skills, via its adapter. `workspace`: its repo skill dir. */
+type SkillPlacement = 'harness' | 'workspace';
+type SkillSource = {
+  kind: 'skill';
+  placement: SkillPlacement;
+} & ({
+  source: 'local';
+  path: string;
+  only?: string[];
+  exclude?: string[];
+} | {
+  source: 'git';
+  repo: string;
+  ref?: string;
+  path?: string;
+  only?: string[];
+  exclude?: string[];
+} | {
+  source: 'registry';
+  id: string;
+  version?: string;
+} | {
+  source: 'npm';
+  spec: string;
+  path: string;
+  only?: string[];
+  exclude?: string[];
+} | {
+  source: 'inline';
+  name: string;
+  description: string;
+  content: string;
+  files: Record<string, string>;
+});
+/** A `docker({ install })` command, baked into a cached image layer: no secrets, root, cwd `/`. */
+type InstallStep = string | {
+  run: string;
+  timeout?: Duration;
+};
+/** Runs in every attempt before the agent starts, with secrets, outside the eval timeout. */
+/** A shell command, or `{ run, timeout?, cwd? }`; `cwd` is relative to the workspace or absolute. */
+type SetupStep = string | {
+  run: string;
+  timeout?: Duration;
+  cwd?: string;
+};
+/**
+ * Where the sandbox may connect: anywhere (`'allow-all'`, the default), nowhere (`'deny-all'`), or
+ * only the listed hosts (`'*.example.com'` matches subdomains). Under a restricted policy the
+ * harness's own hosts, brokered secrets' `hosts` and MCP servers' hosts are allowed too; any
+ * other request gets a refusal, and the attempt goes on.
+ */
+type EgressPolicy = 'allow-all' | 'deny-all' | {
+  allow: readonly string[];
+};
+/**
+ * A git repository (or one directory of it) the workspace starts as, from the `git` helper. `url`
+ * is what reeval fetches: the clone URL, or an absolute path for a local repository.
+ */
+type GitWorkspace = {
+  kind: 'git';
+  /** As written: a URL, `github:owner/repo`, or a local path. */
+  repo: string;
+  url: string;
+  /** A branch, tag or commit SHA; unset is the remote's `HEAD`. */
+  ref?: string;
+  /** A directory in the repository whose contents become the workspace. */
+  path?: string;
+  /** Give the workspace a `.git`: one commit of the fetched tree, with no remote. */
+  init?: boolean;
+  /** The host variable holding a token for the fetch; read on the host only. */
+  token?: string;
+};
+type GitHelper = (repo: string | URL, options?: {
+  ref?: string;
+  path?: string;
+  init?: boolean;
+  token?: string;
+}) => GitWorkspace;
+/** What an eval runs in: the declarative, phased spec of one environment. */
+type EnvironmentSpec = {
+  /** The code the workspace starts as, before `files`, `rules` and `setup`; a child's replaces its parent's. */
+  workspace?: GitWorkspace;
+  env?: EnvVars;
+  /** Replaces the inherited policy; an `extend` child's replaces its parent's. */
+  egress?: EgressPolicy;
+  /** Workspace-relative, or `~/` for home. */
+  files?: Record<string, string | HostFile>;
+  /** Project rules, written where the harness's agent reads them: `AGENTS.md`, or `CLAUDE.md` for Claude Code. */
+  rules?: string | HostFile;
+  setup?: SetupStep[];
+  skills?: SkillSource[];
+  mcp?: Record<string, McpServer>;
+};
+/** One environment in an `extend` chain. */
+type EnvironmentLayer = {
+  name: string;
+  spec: EnvironmentSpec;
+};
+/**
+ * What an eval runs in: skills, MCP servers, files, rules, variables, secrets and setup. Plain
+ * data (`layers` root first), plus a non-enumerable `extend`; see `defineEnvironment`.
+ */
+type Environment = {
+  readonly kind: 'environment';
+  readonly name: string;
+  readonly layers: readonly EnvironmentLayer[];
+  /** A child environment layered on this one; wherever it sets a key this one has, it wins. */
+  extend(name: string, spec: EnvironmentInput): Environment;
+};
+/** Typed helpers for environment specs, passed to `defineEnvironment` and `extend` callbacks. */
+type EnvironmentHelpers = {
+  secret: SecretHelper;
+  skill: SkillHelper;
+  mcp: McpHelper;
+  /** A host file, relative to the file that calls it (or an absolute path or `URL`). */
+  file: (path: string | URL) => HostFile;
+  /** A git repository for `workspace`: a URL, `github:owner/repo`, or a local path. */
+  git: GitHelper;
+};
+type SecretHelper = {
+  /** The host variable of the same name as the key it's set under. */
+  (options?: SecretOptions): Secret;
+  fromEnv(host: string, options?: SecretOptions): Secret;
+  template(template: string): SecretTemplate;
+  /** The hosts well-known APIs take their keys on, for `hosts`. */
+  hosts: {
+    readonly gemini: readonly string[];
+    readonly openai: readonly string[];
+    readonly anthropic: readonly string[];
+    readonly github: readonly string[];
+  };
+};
+type SkillCommon = {
+  placement?: SkillPlacement;
+};
+type SkillDirOptions = SkillCommon & {
+  only?: string[];
+  /**
+   * Paths in each skill directory to leave out of the sandbox, gitignore-style: `*` matches within
+   * a name, a leading or inner `/` anchors to the skill root, a trailing `/` matches directories
+   * only, and anything else matches a name at any depth. Replaces `DEFAULT_SKILL_EXCLUDE`
+   * (`/evals/`, `.history/`, `*.eval.ts`); spread it to add to it. `.git` is always left out.
+   */
+  exclude?: readonly string[];
+};
+type SkillHelper = {
+  /** A directory with a `SKILL.md`, or a directory of them, relative to the calling file. */
+  local(path: string | URL, options?: SkillDirOptions): SkillSource;
+  /** `https://…`, `git@…`, `github:owner/repo` or a GitHub `/tree/<ref>/<path>` URL. */
+  git(repo: string, options?: SkillDirOptions & {
+    ref?: string;
+    path?: string;
+  }): SkillSource;
+  /** A skills.sh id, `owner/repo@skill`; `version` is a git ref. */
+  registry(id: string, options?: SkillCommon & {
+    version?: string;
+  }): SkillSource;
+  /** A pinned npm package spec; skills are read from `path` inside it (default `skills`). */
+  npm(spec: string, options?: SkillDirOptions & {
+    path?: string;
+  }): SkillSource;
+  inline(skill: {
+    name: string;
+    description: string;
+    content: string;
+    files?: Record<string, string>;
+  }, options?: SkillCommon): SkillSource;
+};
+type McpHelper = {
+  stdio(command: string, args?: string[], options?: {
+    env?: EnvVars;
+  }): McpServer;
+  http(url: string, options?: {
+    headers?: Record<string, string | Secret | SecretTemplate>;
+  }): McpServer;
+  sse(url: string, options?: {
+    headers?: Record<string, string | Secret | SecretTemplate>;
+  }): McpServer;
+};
+/** A spec, or a callback that gets the typed helpers and returns one (synchronously). */
+type EnvironmentInput = EnvironmentSpec | ((helpers: EnvironmentHelpers) => EnvironmentSpec);
+/**
+ * An edit of the inherited harness list, as data: `add` appends harnesses, `only` keeps the
+ * inherited ones with these ids, in their order.
+ */
+type HarnessEdit = {
+  add: HarnessDescriptor | HarnessDescriptor[];
+} | {
+  only: string | string[];
+};
+/** One value, a list (each is its own cell), or an edit of the inherited list, as data. */
+type HarnessData = HarnessDescriptor | HarnessDescriptor[] | HarnessEdit;
+/** `HarnessData`, or a function that edits the inherited list (TypeScript only). */
+type HarnessOption = HarnessData | ((inherited: HarnessDescriptor[]) => HarnessDescriptor | HarnessDescriptor[]);
+type CompareGate = 'variants' | 'all' | 'none';
+/** Variants run next to the inherited environment (the control), with the same prompt and checks. */
+type CompareOptions<E = Environment> = E[] | {
+  variants: E[];
+  /** Which cells decide pass/fail: `variants` (default), `all`, or `none` (report only). */
+  gate?: CompareGate;
+  /** Lift in points from -1 to 1; a variant cell below it fails. */
+  minLift?: number;
+  /** Harnesses (ids or descriptors) to compare on; the others run the control only. */
+  harness?: Array<string | HarnessDescriptor>;
+} | false;
+type PinnableKey = 'harness' | 'runs' | 'compare';
+/**
+ * How a cell's attempts decide its outcome: `'all'` (the default) passes only when every attempt
+ * passes; `{ passRate }` passes when at least that share of its passes and fails are passes.
+ */
+type GatePolicy = 'all' | {
+  passRate: number;
+};
+/** A cell's gate: its policy, or `'none'` when its failures don't fail the eval (a control's). */
+type CellGate = GatePolicy | 'none';
+type EvalOptions = {
+  /** The only list: each harness is its own cell. */
+  harness?: HarnessOption;
+  sandbox?: SandboxDescriptor;
+  /** What the eval runs in; the control of `compare`. */
+  environment?: Environment;
+  compare?: CompareOptions;
+  /** Attempts per cell. */
+  runs?: number;
+  /** How each cell's attempts decide its outcome; default `'all'`. */
+  gate?: GatePolicy;
+  /** Variables and secrets for the sandbox, merged by name down the cascade. */
+  env?: EnvVars;
+  /**
+   * Where the sandbox may connect. Replaces the inherited policy: config, profile defaults, the
+   * environment, then groups and the eval.
+   */
+  egress?: EgressPolicy;
+  judge?: JudgeOptions;
+  /** Per attempt. */
+  timeout?: Duration;
+  tags?: string[];
+  /** Keys a profile's `limit` may not narrow for this group or eval. */
+  pin?: PinnableKey[];
+};
+type GroupOptions = EvalOptions;
+/** The config file is plain data sent to workers, so its judge model can't be an object. */
+type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
+  model?: string | FakeJudgeModel;
+};
+type HistoryConfig = {
+  /** Committed history folder (default: `.history` in the directory the evals live in). */
+  dir?: string;
+  /** Artifacts recorded next to each run: every eval's, failing evals' only, or none. */
+  artifacts?: ArtifactLevel;
+};
+/** Run-level caps. `attempts` is checked against the plan; `duration` while running. */
+type Budget = {
+  attempts?: number;
+  duration?: Duration;
+};
+/** What evals get when they don't say otherwise: under groups and evals. */
+type ProfileDefaults = {
+  harness?: HarnessDescriptor | HarnessDescriptor[];
+  sandbox?: SandboxDescriptor;
+  environment?: Environment;
+  runs?: number;
+  gate?: GatePolicy;
+  env?: EnvVars;
+  egress?: EgressPolicy;
+  judge?: ConfigJudgeOptions;
+  timeout?: Duration;
+};
+/** Over every eval, and it can only narrow; `pin` on a group or eval skips it. */
+type ProfileLimit = {
+  /** The most runs per cell. */
+  runs?: number;
+  /** `false` runs the control only; `{ maxVariants }` keeps the first n variants. */
+  compare?: false | {
+    maxVariants: number;
+  };
+  /** Harness ids to allow; evals left with none are skipped. */
+  harness?: string[];
+};
+/** How a run is sized: chosen with `--profile`, `REEVAL_PROFILE` or `defaultProfile`. */
+type Profile = {
+  defaults?: ProfileDefaults;
+  limit?: ProfileLimit;
+  judge?: ConfigJudgeOptions;
+  budget?: Budget;
+  record?: RecordMode;
+  /** Tag filters for this profile, like `--tag` (`'!slow'` excludes). */
+  tags?: string[];
+};
+/** A canned analysis for tests; see `fakeAnalysis()` in `reeval/judge`. */
+type FakeAnalysisModel = {
+  kind: 'fake-analysis';
+  /** The model's structured answer, or a function of the digest it was given. */
+  output: unknown | ((digest: unknown) => unknown);
+};
+/** `reeval analyze`: a language model (not an evaluation model) and what matters in this project. */
+type AnalysisConfig = {
+  /** `'provider/model'`, e.g. `'google/gemini-3.8-flash'`; default: the judge's, if it's one. */
+  model?: string | FakeAnalysisModel;
+  /** Added to the prompt, e.g. "Fewer tool calls count as a win. The skill under test is reeval-good." */
+  instructions?: string;
+  /** The digest's cap; evidence is dropped, lowest priority first, to fit (default 20000). */
+  maxInputTokens?: number;
+  /** The most tokens the model may answer with (default 8000). reeval has no prices, so tokens are the cap. */
+  maxOutputTokens?: number;
+};
+type EsConfig = {
+  include?: string[];
+  harness?: HarnessDescriptor | HarnessDescriptor[];
+  sandbox?: SandboxDescriptor;
+  environment?: Environment;
+  runs?: number;
+  /** How each cell's attempts decide its outcome; default `'all'`. */
+  gate?: GatePolicy;
+  env?: EnvVars;
+  egress?: EgressPolicy;
+  judge?: ConfigJudgeOptions;
+  timeout?: Duration;
+  /** How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it. */
+  concurrency?: number;
+  /** When runs are copied into the history folder: `reeval record` only, in CI, or on every run. */
+  record?: RecordMode;
+  history?: HistoryConfig;
+  budget?: Budget;
+  profiles?: Record<string, Profile>;
+  /** The profile when neither `--profile` nor `REEVAL_PROFILE` names one: `ci` when `CI` is set. */
+  defaultProfile?: {
+    ci?: string;
+    local?: string;
+  };
+  /** `unbrokered: 'error'` refuses Docker secrets without `hosts` instead of warning (default `'warn'`). */
+  secrets?: {
+    unbrokered?: 'warn' | 'error';
+  };
+  /**
+   * Local debug data in `.reeval/debug`: how many runs to keep (default 20), and how many MB of raw
+   * trace each attempt buffers for trace.ndjson (default 8; the oldest entries go first).
+   */
+  debug?: {
+    keep?: number;
+    traceBufferMb?: number;
+  };
+  analysis?: AnalysisConfig;
+};
+/** What a `defineConfig` callback gets: the environment helpers plus the run's context. */
+type ConfigHelpers = EnvironmentHelpers & {
+  /** True when `CI` is set and isn't `false`. */
+  ci: boolean;
+  /** The `reeval` command that's running, e.g. `run`. */
+  command: string;
+  /** `process.env`. */
+  hostEnv: NodeJS.ProcessEnv;
+  /** The profile named by `--profile` or `REEVAL_PROFILE` (not `defaultProfile`), else undefined. */
+  profile: string | undefined;
+};
+type Outcome = 'pass' | 'fail' | 'infra_error' | 'skipped';
+type ErrorKind = 'assertion' | 'timeout' | 'aborted' | 'infra' | 'judge';
+/**
+ * The sandbox an attempt needed that couldn't be prepared before the run, on each attempt that
+ * needed it: its provider's stage (Docker's `image`, `from`, `dockerfile`, `harness` or
+ * `install`, or `workspace` for a git workspace, whose `ref` is `repo#ref:path`), what was being
+ * made, the class (`config`, `transient` after the retries, or `resource`), a named cause
+ * (`enospc`, `oom`), the build log, and the levels that set it.
+ */
+type PrepareFailureRecord = {
+  class: 'config' | 'transient' | 'resource';
+  stage: string;
+  ref: string | null;
+  cause: 'enospc' | 'oom' | null;
+  log: string | null;
+  setBy: string[];
+};
+/** Why an attempt failed; `prepare` when its sandbox couldn't be prepared. */
+type AttemptError = {
+  kind: ErrorKind;
+  message: string;
+  prepare?: PrepareFailureRecord;
+};
+/** Where an eval's time went, in ms. `agentMs` is the harness turns; `diffMs` reading changes. */
+type PhaseTimings = {
+  setupMs: number;
+  agentMs: number;
+  diffMs: number;
+  checksMs: number;
+  judgeMs: number;
+};
+type CellRole = 'control' | 'variant';
+/**
+ * One cell of an eval: a harness and an environment (`-` for none) on the eval's sandbox. With
+ * `compare`, the control runs the inherited environment and each variant runs it extended.
+ */
+type Cell = {
+  key: string;
+  harness: string;
+  environment: string;
+  sandbox: string;
+  role: CellRole;
+  /** The variant's name, for variant cells. */
+  variant?: string;
+};
+/** An environment as it was applied, without secret values. */
+type EnvironmentRecord = {
+  name: string;
+  /** The environment this one extends (for a variant cell, the control), else null. */
+  parent: string | null;
+  /** Keys a later layer replaced, e.g. `env.KEY`, `skills.genkit`, `mcp.docs`, `rules`. */
+  overrides: string[];
+  /** Covers every field but the rules file's path; secret names are included, values never are. The image is the sandbox's. */
+  digest: string;
+  setup: string[];
+  skills: Array<{
+    name: string;
+    source: string;
+    commit: string | null;
+    sha256: string;
+    placement: SkillPlacement;
+    /** Paths left out of the sandbox by `exclude` (directories end in `/`); `.git` isn't listed. */
+    excluded: string[];
+  }>;
+  mcp: string[];
+  files: Array<{
+    path: string;
+    sha256: string;
+  }>;
+  vars: string[];
+  secrets: string[];
+  /** Secrets the sandbox only had placeholders for, swapped in by the egress gateway. */
+  brokered: string[];
+  /** Optional secrets whose host variable was unset, so they weren't injected. */
+  unset: string[];
+  /** The policy the attempt ran under, from the options and the environment; in the digest unless `allow-all`. */
+  egress: EgressPolicy;
+  /**
+   * The git repository the workspace started as: the repo as written (a local one relative to the
+   * project root), the ref and path asked for (null when unset), the commit the ref resolved to
+   * and the hash of the tree that was copied in, which is in the digest.
+   */
+  workspace?: {
+    source: string;
+    ref: string | null;
+    path: string | null;
+    commit: string;
+    tree: string;
+    /** `init: true`: the workspace had a `.git` with one commit of the tree. */
+    init?: boolean;
+  };
+};
+/**
+ * An attempt's network traffic. `allowed` is every host a restricted policy let through, its own
+ * and the automatic ones; null under `allow-all`. `recorded` is false when the sandbox can't see
+ * the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between the
+ * sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the harness
+ * always tries (its CLI's telemetry, say), kept out of `blocked` so a real block stands out.
+ */
+type EgressRecord = {
+  policy: EgressPolicy;
+  allowed: string[] | null;
+  recorded: boolean;
+  contacted: Array<{
+    host: string;
+    port: number;
+    connections: number;
+    sent: number;
+    received: number;
+  }>;
+  blocked: Array<{
+    host: string;
+    port: number;
+    connections: number;
+    reason: string;
+  }>;
+  expectedBlocked: Array<{
+    host: string;
+    port: number;
+    connections: number;
+    reason: string;
+  }>;
+};
+/** The image an attempt's container started from, and how reeval made it. */
+type AttemptImage = {
+  ref: string;
+  /** Content-addressed image id (`sha256:…`). */
+  id: string;
+  agentCli: string | null;
+  /** The harness image `install` was baked onto: `ref` itself without `install`. */
+  base: string;
+  /** The `docker({ install })` commands, in order. */
+  install: string[];
+  /** `docker({ dockerfile })`: the path from the project root and the build context's hash. */
+  dockerfile: {
+    path: string;
+    sha256: string;
+  } | null;
+  /**
+   * `docker({ from })`: the base as written and the digest the image was built on (a local-only
+   * base's image id); `reeval image build` rebuilds it when the tag has moved since.
+   */
+  from?: {
+    ref: string;
+    digest: string | null;
+  };
+};
+/** A harness session start that timed out and was started again. */
+type StartRetry = {
+  /** `<harness kind>: session did not become ready within <startTimeout>`. */
+  error: string;
+  /** How long it waited before giving up on that start. */
+  waitedMs: number;
+};
+/**
+ * A sandbox with more privilege than the default `es` user: `docker({ root: true })`, or
+ * `docker({ capabilities })`. One per level that set such a sandbox, however many evals inherit it.
+ */
+type SandboxWarning = {
+  kind: 'root' | 'capabilities';
+  /** Which level set the sandbox. */
+  level: 'config' | 'profile' | 'group' | 'eval';
+  /** That level as reeval names it: `reeval.config.ts`, `profiles.pr.defaults`, `group("net")` or `evaluate("installs deps")`. */
+  where: string;
+  /** The eval file that sets it, for a group or eval; null for the config or a profile. */
+  file: string | null;
+  /** The capabilities added back, for `capabilities`. */
+  capabilities?: string[];
+  /** The line reeval prints, after `WARNING: ` (root) or `warning: ` (capabilities). */
+  message: string;
+};
+//#endregion
+export { SandboxWarning as $, FakeJudgeModel as A, contrastEffort as At, HostFile as B, EsConfig as C, EffortSample as Ct, ExecResult as D, cellContrast as Dt, EvalOptions as E, VERDICT_TEXT as Et, HarnessData as F, effortSummary as Ft, OpenCodeHarnessDescriptor as G, JudgeModel as H, HarnessDescriptor as I, effortVerdict as It, PinnableKey as J, Outcome as K, HarnessEdit as L, formatCount as Lt, FakeScriptStep as M, effortLabel as Mt, GooseHarnessDescriptor as N, effortOf as Nt, FakeAnalysisModel as O, cellEffort as Ot, GroupOptions as P, effortSample as Pt, SandboxDescriptor as Q, HarnessOption as R, formatRatio as Rt, EnvironmentSpec as S, EffortContrast as St, EvalFn as T, Spread as Tt, JudgeOptions as U, JudgeCallOptions as V, McpServer as W, ProfileDefaults as X, Profile as Y, ProfileLimit as Z, EgressRecord as _, JudgementRecord as _t, Cell as a, TranscriptEvent as at, EnvironmentHelpers as b, CellEffort as bt, CodexHarnessDescriptor as c, TurnStatus as ct, ConfigJudgeOptions as d, AttemptRecord as dt, Secret as et, CursorHarnessDescriptor as f, CellRecord as ft, EgressPolicy as g, HistoryEntry as gt, Duration as h, HistoryCell as ht, Budget as i, TokenUsage as it, FakeSandboxDescriptor as j, effortChangeLabel as jt, FakeHarnessDescriptor as k, cellSamples as kt, CompareOptions as l, Verdict as lt, DockerSandboxDescriptor as m, FileRef as mt, AnalysisConfig as n, SetupStep as nt, CellRole as o, Turn as ot, Diff as p, EvalEntry as pt, PiHarnessDescriptor as q, AntigravityHarnessDescriptor as r, SkillSource as rt, ClaudeCodeHarnessDescriptor as s, TurnData as st, AgentHarnessDescriptor as t, SecretTemplate as tt, ConfigHelpers as u, WebEventType as ut, EnvVars as v, RunRecord as vt, EvalContext as w, EffortVerdict as wt, EnvironmentInput as x, EFFORT_ALPHA as xt, Environment as y, StackSummary as yt, HistoryConfig as z, formatSeconds as zt };
+//# sourceMappingURL=types-DLec-6Cx.d.ts.map