reeval 0.11.0 → 0.12.0

package/dist/{types-BYdlwMwo.d.ts → types-GraZtfHN.d.ts}

@@ -66,7 +66,10 @@ declare const effortSample: (a: Pick<AttemptRecord, 'activity' | 'timings' | 'us
 /** Token totals from the attempts that reported usage; empty when none did. */
 declare const tokenValues: (samples: EffortSample[]) => number[];
 declare function effortOf(samples: EffortSample[]): CellEffort | null;
-/** The verdict of an exact rank test of `xs` (variant) against `ys` (control). */
+/**
+ * The verdict of an exact rank test of `xs` (variant) against `ys` (control). Its direction is
+ * the test's (which side ranks lower), not the medians', which can be equal or disagree.
+ */
 declare function effortVerdict(xs: number[], ys: number[]): {
   p: number | null;
   verdict: EffortVerdict;
@@ -268,7 +271,10 @@ type AttemptRecord = {
   outcome: Outcome;
   error: AttemptError | null;
   durationMs: number;
-  /** `text` is the agent's final message, cut at 8 KB; the whole of it is in `transcript.jsonl`. */
+  /**
+   * `text` is the agent's final message, cut at 8 KB; the whole of it is in `transcript.jsonl`.
+   * `mcpStartup` times each stdio MCP server's start in the turn (`McpStartup`).
+   */
   turns: Array<{
     prompt: string;
     status: TurnStatus;
@@ -283,6 +289,8 @@ type AttemptRecord = {
     unreported: WebEventType[];
     /** The turn's first error, such as a provider's billing or auth error, cut at 1 KB; else null. */
     error: string | null;
+    /** Each stdio MCP server start during the turn; see `McpStartup`. */
+    mcpStartup?: McpStartup[];
   }>;
   /** Tokens across the attempt's turns; null when the harness reported none. */
   usage: TokenUsage | null;
@@ -413,7 +421,10 @@ type FamilyEffortMetric = {
   ratio: number | null;
   /** Wilcoxon signed-rank over the per-case differences; null when every difference is 0. */
   p: number | null;
-  /** `insufficient` below 6 non-zero differences, where the exact test can't reach 0.05. */
+  /**
+   * `insufficient` below 6 non-zero differences, where the exact test can't reach 0.05. `fewer`
+   * and `more` are the test's direction (which side's differences rank higher), not the median's.
+   */
   verdict: 'fewer' | 'more' | 'noise' | 'insufficient';
 };
 /** The variant's effort against the control's, paired by case. */
@@ -671,6 +682,11 @@ type RunRecord = {
   evals: EvalEntry[];
   /** Each pooled `evaluate.each` that ran; unset when none did. */
   families?: FamilyRecord[];
+  /**
+   * Errors outside any attempt, redacted: an eval file that didn't load, a rejection or throw no
+   * attempt awaited, a worker that died. Any one fails the run. Unset when there were none.
+   */
+  errors?: string[];
 };
 /** What ran and graded the agents: the machine, images, harnesses and judge. */
 type Stack = {
@@ -719,7 +735,7 @@ type Stack = {
     effort: string | null;
     /**
      * The models the harness reported serving, across every turn in the run, with the variant it
-     * resolved (`composer-2.5[fast=true]` for `composer-2.5` with no `params`); empty when it
+     * resolved (`composer-2.5[fast=false]` for `composer-2.5` with no `params`); empty when it
      * reports none.
      */
     resolvedModels: string[];
@@ -774,6 +790,8 @@ type HistoryCell = Pick<CellRecord, 'key' | 'role' | 'variant' | 'harness' | 'mo
   };
   /** Median tool calls, agent time and tokens over the counted attempts; null when none counted. */
   effort: CellEffort | null;
+  /** Of `fail`, the attempts the run's interrupt cancelled (`error.kind: 'aborted'`); unset when none. */
+  cancelled?: number;
 };
 /**
  * One line of the derived history listing: cached locally by the CLI, and the run header in the
@@ -814,6 +832,592 @@ type HistoryEntry = {
   }>;
   /** The run's pooled families; unset when none ran. */
   families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'>>;
+  /** The run's errors outside any attempt; unset when there were none. */
+  errors?: string[];
+};
+//#endregion
+//#region src/view/suite.d.ts
+/**
+ * A variant against its baseline: `better` or `worse` when the stored 95% interval on the
+ * difference excludes 0, `same` when both pass at the same rate, and `unclear` otherwise,
+ * including whenever either side has fewer than `EFFECT_MIN_COUNTED` counted attempts.
+ */
+type Effect = 'better' | 'worse' | 'same' | 'unclear';
+/**
+ * What an eval's cells compare: `change` when it has variants (or several environments),
+ * `harness` when it has several harnesses, `both`, or `none`.
+ */
+type Across = 'none' | 'change' | 'harness' | 'both';
+/** The fewest counted attempts a side for an effect to be anything but `unclear`. */
+declare const EFFECT_MIN_COUNTED = 4;
+/**
+ * One finished run's facts. Every node has a `path`, a stable id to cite it by, and `paths`
+ * lists them all in the order the nodes appear.
+ *
+ * A path is `/`-separated segments, each with `%`, `/` and `+` percent-encoded:
+ *
+ * - `<eval>`: the eval's id (`evals/a.eval.ts › group › name`) or a family's
+ *   (`evals/a.eval.ts › fixes $id`);
+ * - `<eval>/<case>`: a family case, by its label (`GH-101` for `fixes GH-101`);
+ * - `<eval>[/<case>]/<harness>+<environment>+<sandbox>`: a cell;
+ * - `<cell>/checks/<label>`: a check, by its assertion label;
+ * - `<cell>/effort`: the cell's effort, only when it was tested;
+ * - `<eval>/<harness>+<environment>+<sandbox>`: on a family, a variant's pooled effect.
+ *
+ * A case segment never has a raw `+` and a cell segment always has two, so the two can't be
+ * confused. Ids, labels and cell keys are what runs are matched on, so a path names the same
+ * fact in every run that has it.
+ */
+type SuiteResult = {
+  runId: string;
+  status: RunRecord['status'];
+  /**
+   * The run's outcome, which `reeval run`'s exit code is read from: `fail` on any error outside
+   * an attempt, and `skipped` on an interrupted run.
+   */
+  outcome: Outcome;
+  /**
+   * Errors outside any attempt: an eval file that didn't load, a rejection or throw no attempt
+   * awaited, a worker that died. Any one fails the run.
+   */
+  errors: string[];
+  /**
+   * Outcomes counted per block: each family case is one, each eval with no cases is one, and a
+   * family whose pooled gate failed is one more fail. `errors` counts the run's `errors`.
+   */
+  tally: SuiteTally;
+  /** Sorted by path. */
+  files: ResultFile[];
+  /** Every node's `path`, in the order they appear. */
+  paths: string[];
+};
+type SuiteTally = {
+  errors: number;
+  pass: number;
+  fail: number;
+  infraError: number;
+  cancelled: number;
+  skipped: number;
+};
+/**
+ * An eval's or a family case's outcome: the record's, except `cancelled` where its only failed
+ * gated attempts are ones the run's interrupt cancelled, since the agent didn't fail those.
+ */
+type ResultOutcome = Outcome | 'cancelled';
+type ResultFile = {
+  path: string;
+  evals: EvalResult[];
+};
+type EvalResult = {
+  path: string;
+  /** The eval's id, or the family's. */
+  id: string;
+  group: string[];
+  /** The eval's name; on a family, its template. */
+  name: string;
+  /** A family's name template (`fixes $id`); null on a single eval. */
+  template: string | null;
+  /** Source order in the file. */
+  order: number;
+  across: Across;
+  /**
+   * A pooled `evaluate.each` with two or more cases: `cases` is set and `cells` is null. An
+   * unpooled one (`{ pool: false }`) is recorded as separate evals, each a single eval.
+   */
+  family: boolean;
+  /** A placeholder (`evaluate.todo`) that never ran. */
+  todo: boolean;
+  /** On a family: `fail` when a case failed or its pooled gate did. */
+  outcome: ResultOutcome;
+  cells: CellResult[] | null;
+  cases: CaseResult[] | null;
+  /** A family's variants against their baseline, pooled over the cases; null on a single eval. */
+  pooled: PooledArm[] | null;
+  /**
+   * Its compare questions' answers. On a family, each question asked of any case, in first-asked
+   * order, with its votes and wins summed over the cases that answered it.
+   */
+  questions: QuestionResult[];
+};
+/** A compare question: each side's wins over the pairs the model decided. Never gates. */
+type QuestionResult = {
+  question: string;
+  /** Pairs voted, each counted once per order it was asked in. */
+  votes: number;
+  rates: Array<{
+    cut: 'within' | 'across';
+    name: string;
+    wins: number;
+    n: number;
+  }>;
+  /** Why some or all of its votes are missing: the model call failed. */
+  error: string | null;
+};
+type CaseResult = {
+  path: string;
+  /** The case's own eval id. */
+  id: string;
+  /** Its name less the template's literal text: `GH-101` for `fixes GH-101`. */
+  label: string;
+  name: string;
+  /** Its place in the case list. */
+  order: number;
+  outcome: ResultOutcome;
+  cells: CellResult[];
+};
+type CellResult = {
+  path: string;
+  key: string;
+  harness: string;
+  /** `-` when no environment is set. */
+  environment: string;
+  sandbox: string;
+  /** The variant's name; null on a control. */
+  variant: string | null;
+  /** The control of an eval that has variants: the arm the variants are measured against. */
+  baseline: boolean;
+  /** On a variant, its baseline cell's key; else null. */
+  vs: string | null;
+  /** The cell's place in the eval's cells. */
+  order: number;
+  /** Its failures fail the eval (its gate isn't `'none'`). */
+  gated: boolean;
+  outcome: Outcome;
+  /** Every attempt: `passed + failed + infraError + skipped`. */
+  attempts: number;
+  passed: number;
+  failed: number;
+  /** Infrastructure errors, which are neither passes nor fails. */
+  infraError: number;
+  skipped: number;
+  fingerprint: string;
+  /** The skill sources and files the cell's environment placed, as written. */
+  context: string[];
+  /**
+   * Every check the eval's attempts recorded (on a family, any case's). A cell with no counted
+   * attempt (every one an infra error or skipped) lists each as `not run`.
+   */
+  checks: CheckResult[];
+  /** On a variant, against its baseline; null on a control. */
+  effect: Effect | null;
+  /** The stored 95% interval on the variant's difference from its baseline. Not for display. */
+  interval: Interval | null;
+  /** On a variant, the compare's `minLift`, a fraction like the interval; null without one. */
+  minLift: number | null;
+  /** Medians over the counted attempts; null below `EFFECT_MIN_COUNTED` of them. */
+  effort: EffortCounters | null;
+  /** On a variant, its tool calls, agent time or tokens differ from its baseline's by the rank test. */
+  effortDiffers: boolean;
+  /** Why attempts failed or errored where no check says: a timeout, an infra error, a throw. */
+  errors: CellError[];
+  /** Web searches and fetches per counted attempt; null where the harness can't report them. */
+  searches: Array<number | null>;
+  fetches: Array<number | null>;
+};
+/** A cell's attempts the run's interrupt cancelled: recorded as fails, but not the agent's. */
+declare const cancelledOf: (cell: CellResult) => number;
+/** A cell's counted tries: its passes and fails, less the ones the interrupt cancelled. */
+declare const countedOf: (cell: CellResult) => number;
+type CheckResult = {
+  path: string;
+  label: string;
+  /** First-seen order across the eval's attempts. */
+  order: number;
+  outcome: CheckOutcome;
+  /**
+   * On a variant, this check against the baseline's (`unclear` when the baseline never ran it);
+   * null on a control or without one.
+   */
+  effect: Effect | null;
+  /** Set when the check is a judge verdict. */
+  judge: JudgeResult | null;
+};
+/**
+ * `not run`: no counted attempt reached the check, because a hard `expect` before it failed or
+ * none counted. That is not a failure, and is never `0 of N`. `ran`: counts over the counted attempts that
+ * reached it (`errored` ones couldn't answer), with `notRun` for those that didn't, and the
+ * first failure's message.
+ */
+type CheckOutcome = {
+  status: 'not run';
+} | {
+  status: 'ran';
+  passed: number;
+  failed: number;
+  errored: number;
+  notRun: number;
+  assertion: string | null;
+};
+/**
+ * One distinct error over a cell's attempts: its kind (`turn` for a turn the harness ended with
+ * an error), its message's first line, and how often.
+ */
+type CellError = {
+  kind: ErrorKind | 'turn';
+  message: string;
+  attempts: number;
+};
+type JudgeResult = {
+  question: string;
+  threshold: number;
+  /** One per counted attempt that judged it. */
+  scores: number[];
+};
+type Interval = {
+  low: number;
+  high: number;
+};
+/** A null counter is one the harness didn't record; a recorded zero stays zero. */
+type EffortCounters = {
+  tools: number | null;
+  agentMs: number | null;
+  tokens: number | null;
+};
+type PooledArm = {
+  path: string;
+  /** The variant cell's key, the same in every case. */
+  key: string;
+  vs: string;
+  effect: Effect;
+  /** Cases with counted attempts on both sides. */
+  pairs: number;
+  interval: Interval;
+  /** The compare's `minLift`, a fraction like the interval, which gates its low end; null without one. */
+  minLift: number | null;
+  outcome: 'pass' | 'fail';
+};
+type Side = {
+  pass: number;
+  counted: number;
+};
+/**
+ * The effect of `variant` against `control`, from the interval on their difference. A side
+ * with fewer than `EFFECT_MIN_COUNTED` counted attempts is `unclear`, whatever the interval.
+ */
+declare function effectOf(variant: Side, control: Side, interval: Interval | null): Effect;
+/** The check's effect, on the same Newcombe interval the record stores for a cell's lift. */
+declare function checkEffect(variant: CheckTally, control: CheckTally): Effect;
+/** One path segment: `%`, `/` and `+` percent-encoded, so segments split back unambiguously. */
+declare const factSegment: (s: string) => string;
+declare function evalAcross(cells: readonly Pick<CellRecord, 'harness' | 'environment' | 'role'>[]): Across;
+/**
+ * An error's one line: its first line with any letter or digit in it, joined with the next one
+ * when it ends with `:`, since `Docker failed:` alone says nothing. A log line's `| ` is dropped.
+ */
+declare function messageLine(text: string): string;
+/** The facts of a run, from its record. Pure: the same record always gives the same result. */
+declare function suiteResult(record: RunRecord): SuiteResult;
+//#endregion
+//#region src/view/insights.d.ts
+/** `<history>/<run>/insights.json`, beside the record. */
+declare const INSIGHTS_FILE = "insights.json";
+/** `RunInsights.model` when no analysis model wrote the file: it holds deterministic remarks only. */
+declare const NO_MODEL = "none";
+/** Whether an analysis model wrote these insights, rather than reeval alone for its remarks. */
+declare const fromModel: (insights: RunInsights) => boolean;
+/** What one attempt tests. Only an analysis model can tell; without one every eval is `other`. */
+type Job = 'choose' | 'finish' | 'other';
+/**
+ * A run's insights, `<history>/<run>/insights.json`. Written create-only; the record is untouched.
+ * They're for the facts they were written from: a `factsHash` that isn't the run's marks them stale.
+ */
+type RunInsights = {
+  kind: 'reeval-insights';
+  schemaVersion: 1;
+  runId: string;
+  /**
+   * The analysis model, as configured (`google/gemini-3.8-flash`, or `fake`); `NO_MODEL` when
+   * reeval wrote the file without one, to hold the run's deterministic remarks only.
+   */
+  model: string;
+  /** The prompts' version: insights from another version were asked differently. */
+  promptVersion: string;
+  /** sha256 of the run's `SuiteResult` as canonical JSON: the facts the model saw. */
+  factsHash: string;
+  createdAt: string;
+  /**
+   * Over every call; `outputTokens` includes `reasoningTokens`, the model's thinking, when it
+   * reports them. `costUsd` is null when reeval doesn't know the model's price.
+   */
+  usage: {
+    inputTokens: number;
+    outputTokens: number;
+    reasoningTokens?: number;
+    costUsd: number | null;
+  };
+  /**
+   * `analysis.maxTokens`, and the evals (`SuiteResult` paths) it ran out before: they print
+   * without insights. Absent when no budget applied.
+   */
+  budget?: {
+    maxTokens: number;
+    skipped: string[];
+  };
+  /**
+   * Evals whose calls failed twice (an error, a reply cut off at the output cap, or a call past its
+   * timeout), with the last error: they print without insights. Absent when none failed.
+   */
+  failed?: Array<{
+    evalId: string;
+    error: string;
+  }>;
+  /**
+   * Set when the run was stopped (Ctrl-C, or SIGTERM) before every eval's insights had finished:
+   * the evals that hadn't, which print without insights. Absent otherwise, even when the stop came
+   * during the headline call, which leaves `headline` null.
+   */
+  stopped?: {
+    signal: 'SIGINT' | 'SIGTERM';
+    skipped: string[];
+  };
+  /** One sentence for the whole run, citing facts; null when the model's was dropped. */
+  headline: Sentence | null;
+  /** One per eval (a family is one), in the order the output prints them. */
+  evals: EvalInsight[];
+  /**
+   * Remarks about the evals' own quality, ranked: every one this run raised, including those an
+   * earlier run printed first (`firstSeenRun`), which don't print again.
+   */
+  remarks: Remark[];
+  /**
+   * What the checks dropped, and how many sentences were asked for again, so the output can say
+   * so. `drops` is each dropped sentence: its slot (`<eval> › answer`, `› row <path>`, `headline`),
+   * its text and why.
+   */
+  validation: {
+    dropped: number;
+    regenerated: number;
+    drops?: Array<{
+      slot: string;
+      text: string;
+      reason: string;
+    }>;
+  };
+  /** Set when insights failed; the facts still print. */
+  error?: string;
+};
+type EvalInsight = {
+  /** The eval's `SuiteResult` path; for a family, the family's. */
+  evalId: string;
+  intent: {
+    /** What one attempt is testing. Can't be read from the cell shape. */
+    job: Job;
+    /** What the cells compare. Deterministic, copied from the facts. */
+    across: Across;
+    /** The eval's question in plain words: "Do agents pick Genkit for a production AI feature?" */
+    question: string;
+    /** The tool a choose eval asks about, and the change a measure eval makes. */
+    subject: {
+      tool?: string;
+      change?: string;
+    };
+    /** How the eval name relates to what the eval checks. */
+    name: 'states-intent' | 'no-intent' | 'contradicts';
+    /**
+     * sha256 of the eval's id, sources, first attempt's prompts, judge questions, check labels,
+     * the model and the prompt version: the classification's cache key.
+     */
+    key: string;
+  };
+  /** The answer to `question`, at most two sentences; null when it was dropped. */
+  answer: Sentence | null;
+  /** Up to three supporting points: what moved, what broke, what the agent did. */
+  findings: Sentence[];
+  /**
+   * At most one sentence per row (a case, an arm or a harness), keyed by the row's fact path.
+   * Only rows the eval's layout opens a slot on are kept.
+   */
+  rows?: Array<Sentence & {
+    row: string;
+  }>;
+  /**
+   * Per attempt (`<cell path>#<n>`), the tools the reply recommended or used, each found
+   * verbatim in that attempt's final message. Only on `choose` evals.
+   */
+  namedTools?: Array<{
+    attempt: string;
+    tools: string[];
+  }>;
+};
+/**
+ * A plain sentence citing the facts it states: `SuiteResult` paths
+ * (`evals%2Fbilling.eval.ts › bills in pence/fake+-+fake/checks/amount`) or a transcript line
+ * (`transcript:<cell path>#<n>#L<line>`). A sentence citing nothing is dropped.
+ */
+type Sentence = {
+  text: string;
+  cites: string[];
+};
+/**
+ * Each remark kind's origin and severity: an analysis model raises the `model` kinds, through the
+ * gates (a known kind, a quote verbatim on the cited line); reeval's rules raise the others. The
+ * severity is the kind's, never the model's.
+ */
+declare const REMARK_KINDS: Record<Remark['kind'], {
+  origin: Remark['origin'];
+  severity: 'concern' | 'suggestion';
+}>;
+/** The kinds an analysis model may raise; a remark of any other kind from it is dropped. */
+declare const MODEL_REMARK_KINDS: ("check-never-fails" | "compound-judge" | "flaky-check" | "judge-leaks-answer" | "judge-too-loose" | "name-contradicts" | "name-no-intent" | "no-baseline" | "prompt-names-tool" | "slow-mcp-start" | "too-few-tries")[];
+/** A remark about the eval's quality, not about the agent. Anything that fails a gate is dropped. */
+type Remark = {
+  /**
+   * sha256(kind, evalId, the eval's sources hash, and for `slow-mcp-start` the server): one remark
+   * per kind (and server) per eval version.
+   */
+  id: string;
+  evalId: string;
+  /** Closed set; an unknown kind is dropped. There is no general or free-form kind. */
+  kind: 'name-contradicts' | 'name-no-intent' | 'judge-too-loose' | 'judge-leaks-answer' | 'prompt-names-tool' | 'compound-judge' | 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'no-baseline' | 'slow-mcp-start';
+  /** Fixed by kind (`REMARK_KINDS`), never chosen by the model. */
+  severity: 'concern' | 'suggestion';
+  /** One plain sentence: what's wrong. */
+  text: string;
+  /**
+   * Where the problem is: `source:<file>#L<n>` with `quote`, found verbatim on that line of the
+   * file at the run's source hash. Every model remark has one.
+   */
+  source?: {
+    at: string;
+    quote: string;
+  };
+  /** Facts from this run (or history facts) showing it mattered now; a model's, as it cited them. */
+  evidence: string[];
+  /**
+   * An edit at a source location (`source:<file>#L<n>`) in the eval. A model remark's `edit` is
+   * empty when the model gave none, and its `at` is the quoted line when it named no line.
+   */
+  fix: {
+    at: string;
+    edit: string;
+  };
+  origin: 'model' | 'deterministic';
+  /**
+   * The first run that printed this `id`, or this run when none has: it prints only in that run,
+   * except `slow-mcp-start`, which is about the environment, not the eval's source, and prints in
+   * every run it happens.
+   */
+  firstSeenRun: string;
+  /**
+   * Whether this run's `reeval run` output printed it: false when an earlier run printed it, or
+   * when the cap or a run without `-v` left it out, so a later run prints it. Missing: printed in
+   * `firstSeenRun` only.
+   */
+  printed?: boolean;
+  /** Set on later runs once the cited span changed: the remark was acted on. */
+  actedOn?: {
+    run: string;
+  };
+};
+//#endregion
+//#region src/insights/input.d.ts
+/** One fact the model may cite, by its path. */
+type InsightFact = {
+  path: string;
+  kind: string;
+} & Record<string, unknown>;
+type InsightAttempt = {
+  /** `<cell path>#<n>`. */
+  id: string;
+  cell: string;
+  outcome: string;
+  /** The agent's final message, clipped. */
+  lastText: string;
+  /** A failed attempt's transcript lines, `L<n> …`, each citable as `transcript:<id>#L<n>`. */
+  lines: string[];
+};
+/** Where an insight sentence can go, as the eval's layout opens them. */
+type InsightSlots = {
+  /** Row paths that take one sentence each. */
+  rows: string[];
+  findings: boolean;
+  namedTools: boolean;
+};
+/** One eval as the classifier and the insight model see it. */
+type InsightInput = {
+  call: 'classify' | 'insights';
+  eval: {
+    path: string;
+    /** The eval's name, or a family's template: the author's statement of what it asks. */
+    name: string;
+    group: string[];
+    file: string;
+    across: Across;
+    family: boolean;
+    verdict: string;
+  };
+  /** The first attempt's prompts, in order. */
+  prompts: string[];
+  judgeQuestions: string[];
+  checks: string[];
+  /** Set on the insights call: the classification. */
+  intent?: {
+    job: Job;
+    question: string;
+    subject: {
+      tool?: string;
+      change?: string;
+    };
+  };
+  /** Set on the insights call. */
+  slots?: InsightSlots;
+  facts: InsightFact[];
+  attempts: InsightAttempt[];
+  /**
+   * Set on the insights call: the eval's declaration in its file, each line `L<n> …` as the file
+   * numbers it, which a remark's `source` and `fix` cite.
+   */
+  source?: {
+    file: string;
+    lines: string[];
+  };
+  /** What was left out to fit the eval's size cap (`trim.ts`), when anything was. */
+  trimmed?: string[];
+  /** On a regeneration: why the earlier sentences were refused. */
+  retry?: string[];
+};
+/** The headline call's input: each eval's verdict and answer, and the run's tally with its total. */
+type HeadlineInput = {
+  call: 'headline';
+  /**
+   * Counts results, not evals: each case of a family counts one, as the tally line does, and
+   * `total` is their sum, so no count is the model's arithmetic; `errors` (outside any eval) aren't
+   * results and stay out of it. The eval count is `evals.length`.
+   */
+  tally: SuiteResult['tally'] & {
+    total: number;
+    unit: 'results';
+  };
+  evals: Array<{
+    path: string;
+    name: string;
+    verdict: string;
+    answer: string | null;
+  }>;
+  retry?: string[];
+};
+/** A fact as the model sees it: its short id (`F1`), which is what it cites. */
+type ModelFact = InsightFact & {
+  id: string;
+};
+/** An attempt as the model sees it: `A1`, its cell's fact id, and its lines citable as `A1#L<n>`. */
+type ModelAttempt = Omit<InsightAttempt, 'id' | 'cell'> & {
+  id: string;
+  cell: string;
+};
+/** One eval's call as the model sees it: facts, attempts and row slots by short id. */
+type ModelInsightInput = Omit<InsightInput, 'facts' | 'attempts' | 'slots'> & {
+  facts: ModelFact[];
+  attempts: ModelAttempt[];
+  slots?: Omit<InsightSlots, 'rows'> & {
+    rows: string[];
+  };
+};
+/** The headline call as the model sees it: each eval by its short id. */
+type ModelHeadlineInput = Omit<HeadlineInput, 'evals'> & {
+  evals: Array<HeadlineInput['evals'][number] & {
+    id: string;
+  }>;
 };
 //#endregion
 //#region src/types.d.ts
@@ -1346,8 +1950,8 @@ type CursorHarnessDescriptor = {
   /** A model id `Cursor.models.list()` returns for the key, e.g. `composer-2.5`. */
   model: string;
   /**
-   * Model parameters, e.g. `{ fast: 'false' }`. Unset ones take the default variant in Cursor's
-   * model catalog (`fast=true` for `composer-2.5`), sent explicitly; pass the ones that change cost.
+   * Model parameters, e.g. `{ fast: 'true' }`. Unset ones take the default variant in Cursor's
+   * model catalog, sent explicitly, except `fast`, which is `false` wherever the model takes it.
    */
   params?: Record<string, string>;
   /**
@@ -1809,6 +2413,103 @@ type EnvironmentSpec = {
   skills?: SkillSource[];
   /** MCP servers the agent gets, by name. */
   mcp?: Record<string, McpServer>;
+  /**
+   * Plugins in Claude Code's layout, made by the `plugin` helper; a child's replaces its
+   * parent's plugin of the same name.
+   */
+  plugins?: PluginSource[];
+};
+/**
+ * A part of a plugin, as Claude Code's layout bundles it: `skills/<name>/SKILL.md`,
+ * `agents/<name>.md`, `commands/<name>.md`, `hooks/hooks.json`, `.mcp.json` (or the manifest's `mcpServers`) or `rules/`.
+ */
+type PluginPart = 'skills' | 'agents' | 'commands' | 'hooks' | 'mcp' | 'rules';
+/** A plugin to place for the agent, in Claude Code's layout, made by the `plugin` helper. */
+type PluginSource = {
+  kind: 'plugin';
+  /** The parts to place; default every part the plugin has. */
+  only?: PluginPart[];
+  /** Commands that build the plugin in a Docker image layer, from its directory unless `cwd` says otherwise. */
+  build?: PluginBuild;
+} & ({
+  source: 'local';
+  /** A host directory with a `.claude-plugin/plugin.json`. */
+  path: string;
+} | {
+  source: 'git';
+  /** As written: a URL, `github:owner/repo`, or a GitHub `/tree/<ref>/<path>` URL. */
+  repo: string;
+  /** A branch, tag or commit SHA; unset is the remote's `HEAD`. */
+  ref?: string;
+  /** The plugin's directory in the repository; default its root. */
+  path?: string;
+} | {
+  source: 'npm';
+  /** A pinned npm package spec. */
+  spec: string;
+  /** The plugin's directory in the package; default its root. */
+  path?: string;
+} | {
+  source: 'marketplace';
+  /** The plugin's entry in the marketplace's `.claude-plugin/marketplace.json`. */
+  name: string;
+  /** Where the marketplace is: a git repository (at `ref`) or an npm package. */
+  marketplace: PluginMarketplace;
+});
+/** A Claude Code marketplace: the source that holds its `.claude-plugin/marketplace.json`. */
+type PluginMarketplace = {
+  source: 'git';
+  /** As written: a URL or `github:owner/repo`. */
+  repo: string;
+  /** A branch, tag or commit SHA; unset is the remote's `HEAD`. */
+  ref?: string;
+} | {
+  source: 'npm';
+  /** An npm package spec. */
+  spec: string;
+};
+/**
+ * Shell commands that build a plugin in a Docker image layer, on the cell's harness image, as root
+ * and with no secrets: one, a list, or `{ run, cwd }`. They run from the plugin's directory, or from
+ * `cwd`, relative to the source's root (`'.'` for the repository's or package's root).
+ */
+type PluginBuild = string | string[] | {
+  /** The command, or the commands in order. */
+  run: string | string[];
+  /** Where they run, relative to the source's root; default the plugin's directory. */
+  cwd?: string;
+};
+type PluginOptions = {
+  /** The parts to place, e.g. `['skills', 'mcp']`; default every part the plugin has. */
+  only?: readonly PluginPart[];
+  /**
+   * Commands that build the plugin in a Docker image layer, e.g. `'npm ci && npm run build'`, or
+   * `{ run, cwd: '.' }` to run them from the source's root. Only the plugin's directory is placed.
+   */
+  build?: string | readonly string[] | {
+    run: string | readonly string[];
+    cwd?: string;
+  };
+};
+type PluginHelper = {
+  /** A directory with a `.claude-plugin/plugin.json`, relative to the calling file. */
+  local(path: string | URL, options?: PluginOptions): PluginSource;
+  /** `https://…`, `git@…`, `github:owner/repo` or a GitHub `/tree/<ref>/<path>` URL. */
+  git(repo: string, options?: PluginOptions & {
+    ref?: string;
+    path?: string;
+  }): PluginSource;
+  /** A pinned npm package spec; the plugin is at `path` inside it (default its root). */
+  npm(spec: string, options?: PluginOptions & {
+    path?: string;
+  }): PluginSource;
+  /**
+   * `'name@source'`: the entry `name` in a Claude Code marketplace, at a git repository (a URL or
+   * `github:owner/repo`, at `ref`) or an npm package (`npm:<spec>`).
+   */
+  marketplace(entry: string, options?: PluginOptions & {
+    ref?: string;
+  }): PluginSource;
 };
 /** One environment in an `extend` chain. */
 type EnvironmentLayer = {
@@ -1842,6 +2543,7 @@ type EnvironmentHelpers = {
   secret: SecretHelper;
   skill: SkillHelper;
   mcp: McpHelper;
+  plugin: PluginHelper;
   /** A host file, relative to the file that calls it (or an absolute path or `URL`). */
   file: (path: string | URL) => HostFile;
   /**
@@ -2236,30 +2938,11 @@ type Profile = {
   /** Tag filters for this profile, like `--tag` (`'!slow'` excludes). */
   tags?: string[];
 };
-/** What a fake note is written from: the eval, the harness, and what its tools returned. */
-type FakeNoteInput = {
-  eval: string;
-  harness: string;
-  attempts: Array<{
-    id: string;
-    outcome: string;
-  }>;
-  /** Each failed attempt the fake opened beside a passing one, and the step where they diverged. */
-  compared: Array<{
-    failed: string;
-    passed: string;
-    divergedAt: number | null;
-  }>;
-};
 /** A canned analysis for tests; see `fakeAnalysis()` in `reeval/judge`. */
 type FakeAnalysisModel = {
   kind: 'fake-analysis';
   /** The model's structured answer, or a function of the digest it was given. */
   output?: unknown | ((digest: unknown) => unknown);
-  /** `analysis.agent`'s note on each eval and harness: text, a function of its tools' results, or an error to throw. */
-  note?: string | ((input: FakeNoteInput) => string) | {
-    error: string;
-  };
   /**
    * How it answers compare questions: a fixed pick, a function of the blinded pair, or
    * `{ error }` to fail every call. A pick may be `{ winner, reason? }`. Default `'tie'`.
@@ -2267,6 +2950,74 @@ type FakeAnalysisModel = {
   vote?: FakePick | ((pair: BlindPair) => FakePick) | {
     error: string;
   };
+  /**
+   * What insights get back, in `reeval run` and `reeval analyze --insights`: fixed insights, a
+   * function of each call's input (`input.call` is `classify` or `insights`, once each per eval,
+   * or `headline` once; it may return a promise), or `{ error }` to fail every call. It counts a
+   * token per 4 characters of each call's input and answer, so `analysis.maxTokens` applies.
+   */
+  insights?: FakeInsights | ((input: FakeInsightsInput) => FakeInsights | Promise<FakeInsights>) | {
+    error: string;
+  };
+};
+/** What a fake insights function is given: one eval's scoped facts, or the run's evals for the headline. */
+type FakeInsightsInput = ModelInsightInput | ModelHeadlineInput;
+/**
+ * A fake insight answer. Each call takes its part: `intent` the classification, `answer`,
+ * `findings`, `rows`, `namedTools` and `remarks` the eval's, `headline` the run's. Whatever's
+ * left out is empty, and the intent defaults to `job: 'other'`.
+ */
+type FakeInsights = {
+  intent?: {
+    job: 'choose' | 'finish' | 'other';
+    question?: string;
+    subject?: {
+      tool?: string;
+      change?: string;
+    };
+    name?: 'states-intent' | 'no-intent' | 'contradicts';
+    /** `low` falls back to `job: 'other'`, as a real classification does. */
+    confidence?: 'high' | 'medium' | 'low';
+  };
+  answer?: {
+    text: string;
+    cites: string[];
+  } | null;
+  findings?: Array<{
+    text: string;
+    cites: string[];
+  }>;
+  rows?: Array<{
+    row: string;
+    text: string;
+    cites: string[];
+  }>;
+  namedTools?: Array<{
+    attempt: string;
+    tools: string[];
+  }>;
+  /**
+   * The eval's remarks, as the model's JSON would have them: `source.at` and `fix.at` are
+   * `L<n>` lines of the eval's source, `evidence` fact ids (`F1`) or attempt lines (`A1#L3`).
+   * They go through the same gates as a model's, so one missing a part is dropped.
+   */
+  remarks?: Array<{
+    kind: string;
+    text: string;
+    source?: {
+      at: string;
+      quote: string;
+    } | null;
+    evidence?: string[];
+    fix?: {
+      at: string;
+      edit: string;
+    } | null;
+  }>;
+  headline?: {
+    text: string;
+    cites: string[];
+  } | null;
 };
 /** A blinded label's pick: the attempt shown first (`A`), second (`B`), or neither. */
 type FakeVote = 'A' | 'B' | 'tie';
@@ -2318,15 +3069,24 @@ type BlindSide = BlindTurn & {
     missing: true;
   }>;
 };
-/** `reeval analyze`: a language model (not an evaluation model) and what matters in this project. */
+/** Insights and `reeval analyze`: a language model (not an evaluation model) and what matters in this project. */
 type AnalysisConfig = {
-  /** `'provider/model'`, e.g. `'google/gemini-3.8-flash'`; default: the judge's, if it's one. */
+  /**
+   * `'provider/model'`, e.g. `'google/gemini-3.8-flash'`. Insights use it only; `reeval analyze`
+   * and compare questions fall back to the judge's when it's a language model.
+   */
   model?: string | FakeAnalysisModel;
   /**
-   * `reeval run` writes a note on each eval and harness once its attempts finish, with the analysis
-   * model (default false: one call per eval and harness, every run).
+   * `reeval run` classifies each eval and explains its result with `model` once the attempts
+   * finish (default true when `model` is set; `--no-insights` turns it off for one run).
    */
-  agent?: boolean;
+  insights?: boolean;
+  /**
+   * Tokens (in and out) that one run's insights may use, over every classification, insight and
+   * headline call (default 250000). Past it, the evals left print without insights; the run's
+   * outcome and exit code never change.
+   */
+  maxTokens?: number;
   /** Added to the prompt, e.g. "Fewer tool calls count as a win. The skill under test is reeval-good." */
   instructions?: string;
   /** The digest's cap; evidence is dropped, lowest priority first, to fit (default 20000). */
@@ -2392,7 +3152,7 @@ type ReevalConfig = {
     keep?: number;
     traceBufferMb?: number;
   };
-  /** `reeval analyze`, compare questions without a model, and `agent`'s notes during a run. */
+  /** Insights during `reeval run`, `reeval analyze`, and compare questions without a model. */
   analysis?: AnalysisConfig;
 };
 /** What a `defineConfig` callback gets: the environment helpers plus the run's context. */
@@ -2441,6 +3201,23 @@ type AttemptError = {
   prepare?: PrepareFailureRecord;
   resource?: ResourceFailureRecord;
 };
+/**
+ * One start of a stdio MCP server during a turn (`turns[].mcpStartup`, in start order), timed by
+ * reeval's MCP timer, which every stdio server runs through. An agent CLI that runs a process per
+ * turn (Antigravity, pi, Codex) starts its servers on every turn; one that keeps a session starts
+ * them once, on the first. A server that doesn't answer within 60s fails the attempt as an infra
+ * error naming it; one whose client closes it first doesn't. `mcpStartup` is absent when the
+ * environment has no stdio MCP server, or the sandbox has no `node` to run the timer.
+ */
+type McpStartup = {
+  /** The server's name in the environment's `mcp`. */
+  server: string;
+  /**
+   * Milliseconds from spawning the server to its answer to the client's `initialize`; null when it
+   * never answered (it exited first, timed out, or its client closed it first).
+   */
+  ms: number | null;
+};
 /** Where an eval's time went, in ms. `agentMs` is the harness turns; `diffMs` reading changes. */
 type PhaseTimings = {
   setupMs: number;
@@ -2466,6 +3243,38 @@ type Cell = {
   /** The variant's name, for variant cells. */
   variant?: string;
 };
+/** A plugin as an attempt placed it. */
+type PluginRecord = {
+  /** The manifest's `name`. */
+  name: string;
+  /** Where it came from, e.g. `local:plugins/review`, `github:acme/agent-plugins#v1:plugins/review` or `npm:@acme/review@1.2.0`; for a marketplace entry, the entry's own source. */
+  source: string;
+  /**
+   * The marketplace it was listed in, with its `marketplace.json` name, its source as written
+   * (`github:acme/agent-plugins#v3`, `npm:@acme/plugins@2.0.0`) and the commit a git one resolved
+   * to (null for npm); null when the plugin was named directly.
+   */
+  marketplace: {
+    name: string;
+    source: string;
+    commit: string | null;
+  } | null;
+  /** The commit a git source resolved to (the marketplace's, for an entry inside it); null for a local or npm one. */
+  commit: string | null;
+  /** The hash of the files placed. */
+  sha256: string;
+  /** The tag of the image layer its build ran in (`reeval-plugin:…`); null without a build. */
+  build: string | null;
+  /** What it placed: skill, agent, command and MCP server names, hook events, and whether it had rules. */
+  parts: {
+    skills: string[];
+    agents: string[];
+    commands: string[];
+    hooks: string[];
+    mcp: string[];
+    rules: boolean;
+  };
+};
 /** An environment as it was applied, without secret values. */
 type EnvironmentRecord = {
   name: string;
@@ -2484,8 +3293,13 @@ type EnvironmentRecord = {
     placement: SkillPlacement;
     /** Paths left out of the sandbox by `exclude` (directories end in `/`); `.git` isn't listed. */
     excluded: string[];
+    /** The plugin the skill came with; unset for a skill from `skills`. */
+    plugin?: string;
   }>;
+  /** Every MCP server, a plugin's as `<plugin>:<server>`. */
   mcp: string[];
+  /** The plugins placed, each with what it placed; in the digest. Unset when there are none. */
+  plugins?: PluginRecord[];
   files: Array<{
     path: string;
     sha256: string;
@@ -2550,7 +3364,7 @@ type AttemptImage = {
   /** Content-addressed image id (`sha256:…`). */
   id: string;
   agentCli: string | null;
-  /** The harness image `install` was baked onto: `ref` itself without `install`. */
+  /** The harness image `install` and plugin builds were baked onto: `ref` itself without either. */
   base: string;
   /** The `docker({ install })` commands, in order. */
   install: string[];
@@ -2621,5 +3435,5 @@ type SandboxWarning = {
   message: string;
 };
 //#endregion
-export { GooseHarnessDescriptor as $, JudgementRecord as $t, DirectoryOptions as A, ServeHandle as At, ErrorKind as B, Wait as Bt, CompareCut as C, effortSample as Cn, ProfileDefaults as Ct, ConfigJudgeOptions as D, formatRatio as Dn, SandboxWarning as Dt, ConfigHelpers as E, formatCount as En, SandboxDescriptor as Et, EnvVars as F, Turn as Ft, FakeAnalysisModel as G, CheckTally as Gt, EvalOptions as H, ArtifactRef as Ht, Environment as I, TurnData as It, FakeNoteInput as J, FamilyRate as Jt, FakeHarnessDescriptor as K, EvalEntry as Kt, EnvironmentHelpers as L, TurnStatus as Lt, Duration as M, SkillSource as Mt, EgressPolicy as N, TokenUsage as Nt, CursorHarnessDescriptor as O, formatSeconds as On, Secret as Ot, EgressRecord as P, TranscriptEvent as Pt, FxHarnessDescriptor as Q, HistoryEntry as Qt, EnvironmentInput as R, Verdict as Rt, CompareAnswer as S, effortOf as Sn, Profile as St, CompareQuestion as T, effortVerdict as Tn, ReevalConfig as Tt, EvalOptionsData as U, AttemptRecord as Ut, EvalFn as V, WebEventType as Vt, ExecResult as W, CellRecord as Wt, FakeScriptStep as X, FileRef as Xt, FakeSandboxDescriptor as Y, FamilyRecord as Yt, FakeVote as Z, HistoryCell as Zt, Cell as _, cellSamples as _n, Outcome as _t, BackgroundHandle as a, EFFORT_ALPHA as an, HarnessOption as at, ClaudeCodeHarnessDescriptor as b, effortChangeLabel as bn, PiHarnessDescriptor as bt, Capture as c, EffortMetric as cn, JudgeArtifactRef as ct, CaptureOptions as d, Spread as dn, JudgeModel as dt, Lift as en, GrokBuildHarnessDescriptor as et, CaptureUse as f, TOKENS_UNREPORTED as fn, JudgeOptions as ft, CaptureWindow as g, cellEffort as gn, OpenCodeHarnessDescriptor as gt, CaptureWait as h, cellContrast as hn, McpServer as ht, AttemptContext as i, DIRECTIONAL_MIN as in, HarnessEdit as it, DockerSandboxDescriptor as j, SetupStep as jt, Diff as k, tokenValues as kn, SecretTemplate as kt, CaptureCall as l, EffortSample as ln, JudgeCallOptions as lt, CaptureUseScript as m, bothUnderATenth as mn, JudgeStateOptions as mt, AnalysisConfig as n, StackSummary as nn, HarnessData as nt, BlindPair as o, EFFORT_MIN_RUNS as on, HistoryConfig as ot, CaptureUseAction as p, VERDICT_TEXT as pn, JudgeState as pt, FakeJudgeModel as q, FamilyLift as qt, AntigravityHarnessDescriptor as r, CellEffort as rn, HarnessDescriptor as rt, Budget as s, EffortContrast as sn, HostFile as st, AgentHarnessDescriptor as t, RunRecord as tn, GroupOptions as tt, CaptureInstruction as u, EffortVerdict as un, JudgeFailure as ut, CellRole as v, contrastEffort as vn, PairFactor as vt, CompareOptions as w, effortSummary as wn, ProfileLimit as wt, CodexHarnessDescriptor as x, effortLabel as xn, PinnableKey as xt, ClaudeCodeEffort as y, directional as yn, PairVote as yt, EnvironmentSpec as z, Viewport as zt };
-//# sourceMappingURL=types-BYdlwMwo.d.ts.map
+export { GooseHarnessDescriptor as $, cellEffort as $n, fromModel as $t, DirectoryOptions as A, FamilyLift as An, Secret as At, EvalFn as B, CellEffort as Bn, Verdict as Bt, CompareCut as C, factSegment as Cn, PluginSource as Ct, ConfigJudgeOptions as D, AttemptRecord as Dn, ReevalConfig as Dt, ConfigHelpers as E, ArtifactRef as En, ProfileLimit as Et, EnvVars as F, HistoryEntry as Fn, TokenUsage as Ft, FakeHarnessDescriptor as G, EffortMetric as Gn, INSIGHTS_FILE as Gt, EvalOptionsData as H, EFFORT_ALPHA as Hn, Wait as Ht, Environment as I, JudgementRecord as In, TranscriptEvent as It, FakeJudgeModel as J, Spread as Jn, NO_MODEL as Jt, FakeInsights as K, EffortSample as Kn, Job as Kt, EnvironmentHelpers as L, Lift as Ln, Turn as Lt, Duration as M, FamilyRecord as Mn, ServeHandle as Mt, EgressPolicy as N, FileRef as Nn, SetupStep as Nt, CursorHarnessDescriptor as O, CellRecord as On, SandboxDescriptor as Ot, EgressRecord as P, HistoryCell as Pn, SkillSource as Pt, FxHarnessDescriptor as Q, cellContrast as Qn, Sentence as Qt, EnvironmentInput as R, RunRecord as Rn, TurnData as Rt, CompareAnswer as S, evalAcross as Sn, PluginPart as St, CompareQuestion as T, suiteResult as Tn, ProfileDefaults as Tt, ExecResult as U, EFFORT_MIN_RUNS as Un, WebEventType as Ut, EvalOptions as V, DIRECTIONAL_MIN as Vn, Viewport as Vt, FakeAnalysisModel as W, EffortContrast as Wn, EvalInsight as Wt, FakeScriptStep as X, VERDICT_TEXT as Xn, Remark as Xt, FakeSandboxDescriptor as Y, TOKENS_UNREPORTED as Yn, REMARK_KINDS as Yt, FakeVote as Z, bothUnderATenth as Zn, RunInsights as Zt, Cell as _, SuiteTally as _n, Outcome as _t, BackgroundHandle as a, CheckResult as an, effortOf as ar, HarnessOption as at, ClaudeCodeHarnessDescriptor as b, countedOf as bn, PiHarnessDescriptor as bt, Capture as c, EffortCounters as cn, effortVerdict as cr, JudgeArtifactRef as ct, CaptureOptions as d, JudgeResult as dn, formatSeconds as dr, JudgeModel as dt, Across as en, cellSamples as er, GrokBuildHarnessDescriptor as et, CaptureUse as f, PooledArm as fn, tokenValues as fr, JudgeOptions as ft, CaptureWindow as g, SuiteResult as gn, OpenCodeHarnessDescriptor as gt, CaptureWait as h, ResultOutcome as hn, McpServer as ht, AttemptContext as i, CheckOutcome as in, effortLabel as ir, HarnessEdit as it, DockerSandboxDescriptor as j, FamilyRate as jn, SecretTemplate as jt, Diff as k, EvalEntry as kn, SandboxWarning as kt, CaptureCall as l, EvalResult as ln, formatCount as lr, JudgeCallOptions as lt, CaptureUseScript as m, ResultFile as mn, JudgeStateOptions as mt, AnalysisConfig as n, CellError as nn, directional as nr, HarnessData as nt, BlindPair as o, EFFECT_MIN_COUNTED as on, effortSample as or, HistoryConfig as ot, CaptureUseAction as p, QuestionResult as pn, JudgeState as pt, FakeInsightsInput as q, EffortVerdict as qn, MODEL_REMARK_KINDS as qt, AntigravityHarnessDescriptor as r, CellResult as rn, effortChangeLabel as rr, HarnessDescriptor as rt, Budget as s, Effect as sn, effortSummary as sr, HostFile as st, AgentHarnessDescriptor as t, CaseResult as tn, contrastEffort as tr, GroupOptions as tt, CaptureInstruction as u, Interval as un, formatRatio as ur, JudgeFailure as ut, CellRole as v, cancelledOf as vn, PairFactor as vt, CompareOptions as w, messageLine as wn, Profile as wt, CodexHarnessDescriptor as x, effectOf as xn, PinnableKey as xt, ClaudeCodeEffort as y, checkEffect as yn, PairVote as yt, EnvironmentSpec as z, StackSummary as zn, TurnStatus as zt };
+//# sourceMappingURL=types-GraZtfHN.d.ts.map