reeval 0.8.0 → 0.10.0

package/dist/{usage-DDF_6tkV.js → usage-CleNKkly.js}

@@ -20,7 +20,9 @@ const sumUsage = (usages) => {
 const promptTokens = (u) => u.input + u.cacheRead + u.cacheWrite;
 /** Prompt side, output, and reasoning when the harness reported it. No USD. */
 const totalTokens = (u) => promptTokens(u) + u.output + (u.reasoning ?? 0);
+/** A token count as people read it: `9,512`, `430k`, `1,094k`. */
+const formatTokens = (n) => n >= 1e4 ? `${Math.round(n / 1e3).toLocaleString("en-US")}k` : Math.round(n).toLocaleString("en-US");
 //#endregion
-export { totalTokens as i, promptTokens as n, sumUsage as r, addUsage as t };
+export { totalTokens as a, sumUsage as i, formatTokens as n, promptTokens as r, addUsage as t };
 
-//# sourceMappingURL=usage-DDF_6tkV.js.map
+//# sourceMappingURL=usage-CleNKkly.js.map

package/dist/{usage-DDF_6tkV.js.map → usage-CleNKkly.js.map}

@@ -1 +1 @@
-{"version":3,"file":"usage-DDF_6tkV.js","names":[],"sources":["../src/usage.ts"],"mappings":";;AAGA,SAAgB,SAAS,GAAsB,GAAyC;CACtF,IAAI,CAAC,KAAK,CAAC,GAAG,OAAO,KAAK;CAC1B,OAAO;EACL,OAAO,EAAE,QAAQ,EAAE;EACnB,QAAQ,EAAE,SAAS,EAAE;EACrB,WAAW,EAAE,YAAY,EAAE;EAC3B,YAAY,EAAE,aAAa,EAAE;EAC7B,WACE,EAAE,cAAc,QAAQ,EAAE,cAAc,OAAO,QAAQ,EAAE,aAAa,MAAM,EAAE,aAAa;CAC/F;AACF;;AAGA,MAAa,YAAY,WAAuE;CAC9F,IAAI,QAA2B;CAC/B,KAAK,MAAM,KAAK,QAAQ,QAAQ,SAAS,OAAO,KAAK,IAAI;CACzD,OAAO;AACT;;AAGA,MAAa,gBAAgB,MAAkB,EAAE,QAAQ,EAAE,YAAY,EAAE;;AAGzE,MAAa,eAAe,MAAkB,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,aAAa"}
+{"version":3,"file":"usage-CleNKkly.js","names":[],"sources":["../src/usage.ts"],"mappings":";;AAGA,SAAgB,SAAS,GAAsB,GAAyC;CACtF,IAAI,CAAC,KAAK,CAAC,GAAG,OAAO,KAAK;CAC1B,OAAO;EACL,OAAO,EAAE,QAAQ,EAAE;EACnB,QAAQ,EAAE,SAAS,EAAE;EACrB,WAAW,EAAE,YAAY,EAAE;EAC3B,YAAY,EAAE,aAAa,EAAE;EAC7B,WACE,EAAE,cAAc,QAAQ,EAAE,cAAc,OAAO,QAAQ,EAAE,aAAa,MAAM,EAAE,aAAa;CAC/F;AACF;;AAGA,MAAa,YAAY,WAAuE;CAC9F,IAAI,QAA2B;CAC/B,KAAK,MAAM,KAAK,QAAQ,QAAQ,SAAS,OAAO,KAAK,IAAI;CACzD,OAAO;AACT;;AAGA,MAAa,gBAAgB,MAAkB,EAAE,QAAQ,EAAE,YAAY,EAAE;;AAGzE,MAAa,eAAe,MAAkB,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,aAAa;;AAG3F,MAAa,gBAAgB,MAC3B,KAAK,MACD,GAAG,KAAK,MAAM,IAAI,GAAI,CAAC,CAAC,eAAe,OAAO,EAAE,KAChD,KAAK,MAAM,CAAC,CAAC,CAAC,eAAe,OAAO"}

package/dist/view/index.d.ts

@@ -1,9 +1,9 @@
-import { $t as TOKENS_UNREPORTED, Bt as FileRef, Dt as TokenUsage, Gt as RunRecord, Ht as HistoryEntry, It as ArtifactRef, Jt as EFFORT_ALPHA, Kt as StackSummary, Lt as AttemptRecord, Ot as TranscriptEvent, Qt as Spread, Rt as CellRecord, Ut as JudgementRecord, Vt as HistoryCell, Wt as Lift, Xt as EffortSample, Yt as EffortContrast, Zt as EffortVerdict, an as effortChangeLabel, cn as effortSample, dn as formatCount, en as VERDICT_TEXT, fn as formatRatio, ft as Outcome, in as contrastEffort, jt as TurnStatus, ln as effortSummary, mn as tokenValues, nn as cellEffort, on as effortLabel, pn as formatSeconds, qt as CellEffort, rn as cellSamples, sn as effortOf, tn as cellContrast, un as effortVerdict, xt as SandboxWarning, zt as EvalEntry } from "../types-_LOFdPUf.js";
+import { $t as RunRecord, Cn as formatCount, En as tokenValues, Et as SandboxWarning, Gt as FamilyLift, Ht as AttemptRecord, It as TurnStatus, Jt as FileRef, Kt as FamilyRate, Mt as TokenUsage, Nt as TranscriptEvent, Qt as Lift, Sn as effortVerdict, Tn as formatSeconds, Ut as CellRecord, Vt as ArtifactRef, Wt as EvalEntry, Xt as HistoryEntry, Yt as HistoryCell, Zt as JudgementRecord, _n as effortChangeLabel, an as EffortContrast, bn as effortSample, cn as EffortVerdict, dn as VERDICT_TEXT, en as StackSummary, fn as cellContrast, gn as directional, gt as Outcome, hn as contrastEffort, in as EFFORT_MIN_RUNS, ln as Spread, mn as cellSamples, nn as DIRECTIONAL_MIN, on as EffortMetric, pn as cellEffort, qt as FamilyRecord, rn as EFFORT_ALPHA, sn as EffortSample, tn as CellEffort, un as TOKENS_UNREPORTED, vn as effortLabel, wn as formatRatio, xn as effortSummary, yn as effortOf } from "../types-g17bRAzt.js";
 //#region src/view/analysis.d.ts
 /** Bumped when the system prompt or output schema changes: an older analysis is superseded. */
-export declare const ANALYSIS_PROMPT_VERSION = "analysis-v9";
+export declare const ANALYSIS_PROMPT_VERSION = "analysis-v10";
 /** Bumped when the digest's shape or selection rules change. */
-export declare const DIGEST_VERSION = 7;
+export declare const DIGEST_VERSION = 10;
 export type Confidence = 'high' | 'medium' | 'low';
 export type AnalysisClaim = {
   text: string;
@@ -15,7 +15,7 @@ export type AnalysisClaim = {
   /** Numbers in the text that no cited fact or evidence contains. */
   unverifiedNumbers?: string[];
 };
-export type AnomalyKind = 'es-bug' | 'eval-design' | 'judge' | 'noise' | 'other';
+export type AnomalyKind = 'reeval-bug' | 'eval-design' | 'judge' | 'noise' | 'other';
 export type AnalysisAnomaly = {
   text: string;
   kind: AnomalyKind;
@@ -23,7 +23,7 @@ export type AnalysisAnomaly = {
   /** Numbers in the text that no cited fact or evidence contains. */
   unverifiedNumbers?: string[];
 };
-export type ActionKind = 'skill' | 'docs' | 'eval' | 'judge' | 'es' | 'harness';
+export type ActionKind = 'skill' | 'docs' | 'eval' | 'judge' | 'reeval' | 'harness';
 export type Level = 'high' | 'medium' | 'low';
 export type AnalysisAction = {
   title: string;
@@ -70,7 +70,7 @@ export type AnalysisScope = {
   evals: string | null;
 };
 export type AnalysisDoc = {
-  kind: 'es-analysis';
+  kind: 'reeval-analysis';
   schemaVersion: 1;
   scope: AnalysisScope;
   /** The run the analysis belongs to: the scope's head. */
@@ -298,6 +298,29 @@ export declare function formatBytes(n: number): string;
  */
 export declare function requestLine(r: Omit<DebugRequest, 'at'>): string;
 //#endregion
+//#region src/view/family.d.ts
+/** `pass@1 2/3 · 67% (95% 21…94)`, or `pass@1 67% over 3 cases (95% 30…100)` at several runs. */
+export declare function familyRateLabel(rate: FamilyRate): string;
+/**
+ * `pass@1 88% vs 20%  lift +68 pts (95% +40…+82, paired, 18 vs 1 discordant, p <0.001)`. At
+ * one attempt a case, the counts are the discordant pairs; at several, the paired cases.
+ */
+export declare function familyLiftLabel(lift: FamilyLift): string;
+/**
+ * `tools ×0.41 per case (fewer, signed-rank p 0.002) · agent time ×0.80 per case (noise, …)`;
+ * a metric whose control median is 0 has no ratio and is left out.
+ */
+export declare function familyEffortLabel(lift: FamilyLift): string | null;
+/** `25 cases · paired` with a compare, and `3 of 25 cases` when a filter left some out. */
+export declare function familyHeadLabel(family: FamilyRecord): string;
+/**
+ * A case's short label: its name less the template's literal text before the first placeholder
+ * and after the last, so `fixes $id` labels `fixes GH-101` as `GH-101`.
+ */
+export declare function caseLabel(template: string, name: string): string;
+/** The family's cases in list order. */
+export declare const familyCases: (family: Pick<FamilyRecord, 'id'>, evals: readonly EvalEntry[]) => EvalEntry[];
+//#endregion
 //#region src/view/header.d.ts
 /** The derived header of one run: what listings, indexes and comparisons need without the record. */
 export declare function runHeader(record: RunRecord, recorded: boolean): HistoryEntry;
@@ -321,7 +344,10 @@ export declare function uniqueJudgeLabels(items: readonly {
 }[]): Map<string, string>;
 export type JudgeKeyRate = {
   key: string;
+  /** The `expect` label when the judge had one, else a short name from its question. */
   label: string;
+  /** Named by an `expect` label, so the cell's checks carry it too. */
+  labelled: boolean;
   question: string;
   pass: number;
   counted: number;
@@ -330,6 +356,11 @@ export type JudgeKeyRate = {
 };
 /** Per-key pass/counted for one cell's counted attempts, with lift on a variant when both sides qualify. */
 export declare function cellJudgeRates(e: Pick<EvalEntry, 'attempts' | 'cells'>, cellKey: string): JudgeKeyRate[];
+/**
+ * The judge rates a cell's summary line adds to its checks: a labelled judge whose label is one
+ * of the cell's checks against its control is already there.
+ */
+export declare function summaryJudgeRates(e: Pick<EvalEntry, 'attempts' | 'cells'>, cell: Pick<CellRecord, 'key' | 'checks'>): JudgeKeyRate[];
 /** `login 4/5 · checkout 0/5`, with `(+80 pts)` on a key that has lift. */
 export declare function judgeRatesLabel(rates: readonly JudgeKeyRate[]): string;
 export type CollapsedJudge = {
@@ -346,8 +377,27 @@ export declare function collapsedJudges(judgements: readonly JudgementRecord[]):
 };
 /** `fail 0.78/0.80`, or `fail 0.33/0.80 · 3 judges` when there are many. */
 export declare function collapsedJudgeLabel(judgements: readonly JudgementRecord[]): string | null;
+/**
+ * Each judgement's `expect` label, by position: the attempt's `toPass` assertions (less those on a
+ * failed judge call, which recorded no judgement) in call order, when they are exactly as many as
+ * its judgements. `check` is that assertion's label, the key of its check; a judgement no label
+ * reaches is named by its question.
+ */
+export declare function labelledJudgements(a: Pick<AttemptRecord, 'judgements' | 'assertions'>): Array<{
+  judgement: JudgementRecord;
+  label: string;
+  labelled: boolean;
+  check?: string;
+}>;
 /** Sum of counted attempts' usage; null when none of them reported any. */
 export declare function cellUsage(e: Pick<EvalEntry, 'attempts'>, cellKey: string): TokenUsage | null;
+/**
+ * Each `c.exec` check's `expect` label, by position: the attempt's `toSucceed` assertions in call
+ * order, when they are exactly as many as its checks; undefined where no label reaches.
+ */
+export declare function checkLabels(a: Pick<AttemptRecord, 'checks' | 'assertions'>): Array<string | undefined>;
+/** A check's command on one line: the first non-empty line, clipped; its log has the rest. */
+export declare function commandLine(command: string, max?: number): string;
 //#endregion
 //#region src/view/live.d.ts
 /** The progress file's name inside the run's staging directory. */
@@ -436,7 +486,7 @@ export type IndexManifest = {
 };
 export type StudioInfo = {
   mode: 'live' | 'static';
-  es: string;
+  reeval: string;
   project: string;
   generatedAt: string;
   /** `debug`: local debug data (`.reeval/debug`) can be read; never in a static build. */
@@ -463,7 +513,7 @@ export declare function formatDuration(ms: number | null | undefined): string;
 export declare const formatPercent: (rate: number | null) => string;
 /**
  * A harness of the run's stack as `cursor (model composer-2.5, served composer-2.5[fast=true])`:
- * the configured model, then the variant the harness says it resolved.
+ * the configured model and reasoning effort, then what the harness says it resolved.
  */
 export declare function harnessLabel(h: RunRecord['stack']['harnesses'][number]): string;
 /**
@@ -490,6 +540,9 @@ export declare function runOutcome(run: {
   evals: ReadonlyArray<{
     outcome: Outcome;
   }>;
+  families?: ReadonlyArray<{
+    outcome: 'pass' | 'fail';
+  }>;
 }): Outcome;
 /** Distinct judge labels that failed on a counted attempt, in first-seen order. */
 export declare function failedJudgeLabels(record: Pick<RunRecord, 'evals'>): string[];
@@ -503,6 +556,8 @@ export type RunRow = {
   passed: number;
   /** Attempts that reached a verdict or errored; skips excluded. */
   total: number;
+  /** Passes and fails, the denominator of the run's pass@1. */
+  counted: number;
   /** Stack changes in the cells it shares with earlier runs, and in the judge since the previous complete run. */
   drift: StackChange[];
   /** Evals with a cell whose fingerprint changed since the last run that had that cell. */
@@ -547,12 +602,15 @@ export type CellRow = {
   gated: boolean;
   gate: CellRecord['gate'];
   overrides: string[];
+  /** The reasoning effort the harness factory asked for; null when it left it unset. */
+  reasoningEffort: string | null;
   runs: number;
   pass: number;
   fail: number;
   infraError: number;
   skipped: number;
-  passRate: number | null;
+  /** pass@1: passes over passes and fails. */
+  passAt1: number | null;
   outcome: Outcome;
   /** A variant's lift over its control, in whole points (from the record), else null. */
   lift: number | null;
@@ -598,6 +656,8 @@ export type AnswerRow = {
   noPairs: boolean;
   /** Distinct vote reasons, in vote order, at most three. */
   reasons: string[];
+  /** `questions/<n>.json` under the run's directory: what the model was sent; null when not kept. */
+  input: string | null;
 };
 /** One row per compare question, with each cut's win rates in the record's order. */
 export declare function answerRows(e: Pick<EvalEntry, 'answers'>): AnswerRow[];
@@ -606,7 +666,8 @@ export declare const isMatrix: (e: Pick<EvalEntry, 'cells'>) => boolean;
 export type EvalHistoryPoint = {
   run: HistoryEntry;
   outcome: Outcome;
-  passRate: number | null;
+  /** pass@1 over the gated cells' attempts. */
+  passAt1: number | null;
   cells: HistoryCell[];
   /** A cell's fingerprint differs from the last run that had that cell. */
   changed: boolean;
@@ -619,9 +680,9 @@ export type CompareViewRow = {
   status: CompareStatus;
   base: Outcome | null;
   head: Outcome | null;
-  basePassRate: number | null;
-  headPassRate: number | null;
-  /** See `CompareRow`: then the pass rates are over the cells both runs have too. */
+  basePassAt1: number | null;
+  headPassAt1: number | null;
+  /** See `CompareRow`: then pass@1 is over the cells both runs have too. */
   cells?: CompareRow['cells'];
   changes?: string[];
   /** Mean judge score per run; null without judgements or without the record. */
@@ -633,8 +694,8 @@ export type CompareViewRow = {
 export declare const COMPARE_ORDER: CompareStatus[];
 /**
  * Whether some of an eval's cells aren't gated (`gate: 'none'`, e.g. controls under
- * `compare.gate: 'variants'`). Its pass rate then covers fewer attempts than its pass count, so
- * the eval line labels the rate `gated`: `2/4 · 100% gated`.
+ * `compare.gate: 'variants'`). Its pass@1 then covers fewer attempts than its pass count, so
+ * the eval line labels it `gated`: `2/4 · pass@1 100% gated`.
  */
 export declare const hasUngatedCells: (cells: readonly Pick<HistoryCell, 'gate'>[]) => boolean;
 /**
@@ -644,7 +705,12 @@ export declare const hasUngatedCells: (cells: readonly Pick<HistoryCell, 'gate'>
 export declare const isUngatedMiss: (e: {
   cells: readonly Pick<HistoryCell, 'key' | 'gate'>[];
 }, a: Pick<AttemptRecord, 'cell' | 'outcome' | 'error'>) => boolean;
-/** `reeval compare`'s classification, plus pass rates and judge-score deltas from the records. */
+/**
+ * The outcome an eval's mark shows. A pass needs a counted (passed or failed) attempt and no infra
+ * error: a gate that lets those through, such as `gate: 'none'`, still shows `!`, not `✓`.
+ */
+export declare function evalMarkOutcome(e: Pick<EvalEntry, 'outcome' | 'attempts'>): Outcome;
+/** `reeval compare`'s classification, plus pass@1 and judge-score deltas from the records. */
 export declare function compareView(base: HistoryEntry, head: HistoryEntry, records?: {
   base?: RunRecord;
   head?: RunRecord;
@@ -767,5 +833,5 @@ export declare const readableTranscriptLine: (line: TranscriptLine) => Transcrip
 /** An analysis document as every reader sees it; one from a newer reeval throws. */
 export declare const readableAnalysis: (doc: AnalysisDoc, what?: string) => AnalysisDoc;
 //#endregion
-export { type ArtifactRef, type AttemptRecord, CellEffort, type CellRecord, EFFORT_ALPHA, EffortContrast, EffortSample, EffortVerdict, type EvalEntry, type FileRef, type HistoryCell, type HistoryEntry, type JudgementRecord, type Outcome, type RunRecord, type SandboxWarning, Spread, type StackSummary, TOKENS_UNREPORTED, type TranscriptEvent, VERDICT_TEXT, cellContrast, cellEffort, cellSamples, contrastEffort, effortChangeLabel, effortLabel, effortOf, effortSample, effortSummary, effortVerdict, formatCount, formatRatio, formatSeconds, tokenValues };
+export { type ArtifactRef, type AttemptRecord, CellEffort, type CellRecord, DIRECTIONAL_MIN, EFFORT_ALPHA, EFFORT_MIN_RUNS, EffortContrast, EffortMetric, EffortSample, EffortVerdict, type EvalEntry, type FamilyLift, type FamilyRate, type FamilyRecord, type FileRef, type HistoryCell, type HistoryEntry, type JudgementRecord, type Outcome, type RunRecord, type SandboxWarning, Spread, type StackSummary, TOKENS_UNREPORTED, type TranscriptEvent, VERDICT_TEXT, cellContrast, cellEffort, cellSamples, contrastEffort, directional, effortChangeLabel, effortLabel, effortOf, effortSample, effortSummary, effortVerdict, formatCount, formatRatio, formatSeconds, tokenValues };
 //# sourceMappingURL=index.d.ts.map

package/dist/view/index.js

@@ -1,8 +1,9 @@
 import { a as parseDataPath, c as shardPath, i as manifestPath, l as studioPath, n as artifactPath, o as recordPath, r as artifactType, s as shardOf, t as RUN_ID } from "../paths-BmR-7tx3.js";
 import { a as debugApiPath, c as formatBytes, d as requestLine, f as traceApiPath, i as TRACE_FILE, l as parseDebugLog, n as DEBUG_PHASES, o as debugLogLine, r as REQUESTS_FILE, s as debugTime, t as DEBUG_LOG, u as parseNdjson } from "../debug-D24NOQH1.js";
-import { a as pinnedAnalysis, i as isSuperseded, n as DIGEST_VERSION, o as sortAnalyses, r as analysisDataPath, t as ANALYSIS_PROMPT_VERSION } from "../analysis-BTNnKh1w.js";
-import { $ as formatRatio, A as transcriptItems, B as TOKENS_UNREPORTED, C as runLead, D as shortSha, E as runtimeLabel, F as collapsedJudges, G as contrastEffort, H as cellContrast, I as judgeLabel, J as effortOf, K as effortChangeLabel, L as judgeRatesLabel, M as cellJudgeRates, N as cellUsage, O as splitEvalId, P as collapsedJudgeLabel, Q as formatCount, R as uniqueJudgeLabels, S as parseTranscript, T as runRows, U as cellEffort, V as VERDICT_TEXT, W as cellSamples, X as effortSummary, Y as effortSample, Z as effortVerdict, _ as hasUngatedCells, a as cellRows, at as cellsChanged, b as isUngatedMiss, c as evalHistory, ct as stackChanges, d as findAttempt, et as formatSeconds, f as findEval, g as harnessLabel, h as formatUsage, i as buildIndex, j as LIFT_MIN_COUNTED, k as splitPatch, l as evalTree, lt as whyChanged, m as formatPercent, n as activityLabel, o as compareView, ot as compare, p as formatDuration, q as effortLabel, r as answerRows, s as defaultAttempt, st as sourceChanges, t as COMPARE_ORDER, tt as tokenValues, u as failedJudgeLabels, v as indexFiles, w as runOutcome, x as languageLabel, y as isMatrix, z as EFFORT_ALPHA } from "../model-BndnmcLm.js";
-import { t as runHeader } from "../header-_F0EWob3.js";
-import { c as readableTranscriptLine, r as readableAnalysis, s as readableRecord, t as NewerSchemaError } from "../record-BRP9vZrj.js";
-import { a as liveApiPath, c as readableLiveProgress, i as isFinished, n as LIVE_FILE, o as liveCounts, r as LIVE_SCHEMA_VERSION, s as liveTranscriptPath, t as FINISHED } from "../live-S0qODb6o.js";
-export { ANALYSIS_PROMPT_VERSION, COMPARE_ORDER, DEBUG_LOG, DEBUG_PHASES, DIGEST_VERSION, EFFORT_ALPHA, FINISHED, LIFT_MIN_COUNTED, LIVE_FILE, LIVE_SCHEMA_VERSION, NewerSchemaError, REQUESTS_FILE, RUN_ID, TOKENS_UNREPORTED, TRACE_FILE, VERDICT_TEXT, activityLabel, analysisDataPath, answerRows, artifactPath, artifactType, buildIndex, cellContrast, cellEffort, cellJudgeRates, cellRows, cellSamples, cellUsage, cellsChanged, collapsedJudgeLabel, collapsedJudges, compare, compareView, contrastEffort, debugApiPath, debugLogLine, debugTime, defaultAttempt, effortChangeLabel, effortLabel, effortOf, effortSample, effortSummary, effortVerdict, evalHistory, evalTree, failedJudgeLabels, findAttempt, findEval, formatBytes, formatCount, formatDuration, formatPercent, formatRatio, formatSeconds, formatUsage, harnessLabel, hasUngatedCells, indexFiles, isFinished, isMatrix, isSuperseded, isUngatedMiss, judgeLabel, judgeRatesLabel, languageLabel, liveApiPath, liveCounts, liveTranscriptPath, manifestPath, parseDataPath, parseDebugLog, parseNdjson, parseTranscript, pinnedAnalysis, readableAnalysis, readableLiveProgress, readableRecord, readableTranscriptLine, recordPath, requestLine, runHeader, runLead, runOutcome, runRows, runtimeLabel, shardOf, shardPath, shortSha, sortAnalyses, sourceChanges, splitEvalId, splitPatch, stackChanges, studioPath, tokenValues, traceApiPath, transcriptItems, uniqueJudgeLabels, whyChanged };
+import { a as pinnedAnalysis, i as isSuperseded, n as DIGEST_VERSION, o as sortAnalyses, r as analysisDataPath, t as ANALYSIS_PROMPT_VERSION } from "../analysis-Bxub9LMT.js";
+import { $ as directional, A as splitPatch, B as judgeRatesLabel, C as parseTranscript, D as runtimeLabel, E as runRows, F as checkLabels, G as EFFORT_ALPHA, H as summaryJudgeRates, I as collapsedJudgeLabel, J as VERDICT_TEXT, K as EFFORT_MIN_RUNS, L as collapsedJudges, M as LIFT_MIN_COUNTED, N as cellJudgeRates, O as shortSha, P as cellUsage, Q as contrastEffort, R as commandLine, S as languageLabel, St as whyChanged, T as runOutcome, U as uniqueJudgeLabels, V as labelledJudgements, W as DIRECTIONAL_MIN, X as cellEffort, Y as cellContrast, Z as cellSamples, _ as harnessLabel, a as cellRows, at as effortVerdict, b as isMatrix, bt as sourceChanges, c as evalHistory, ct as formatSeconds, d as failedJudgeLabels, et as effortChangeLabel, f as findAttempt, g as formatUsage, h as formatPercent, i as buildIndex, it as effortSummary, j as transcriptItems, k as splitEvalId, l as evalMarkOutcome, lt as tokenValues, m as formatDuration, n as activityLabel, nt as effortOf, o as compareView, ot as formatCount, p as findEval, q as TOKENS_UNREPORTED, r as answerRows, rt as effortSample, s as defaultAttempt, st as formatRatio, t as COMPARE_ORDER, tt as effortLabel, u as evalTree, v as hasUngatedCells, vt as cellsChanged, w as runLead, x as isUngatedMiss, xt as stackChanges, y as indexFiles, yt as compare, z as judgeLabel } from "../model-oqcDplJG.js";
+import { a as familyLiftLabel, i as familyHeadLabel, n as familyCases, o as familyRateLabel, r as familyEffortLabel, t as caseLabel } from "../family-D2b7WD10.js";
+import { t as runHeader } from "../header-hK-Z_Po9.js";
+import { c as readableTranscriptLine, r as readableAnalysis, s as readableRecord, t as NewerSchemaError } from "../record-BKmMvZ6O.js";
+import { a as liveApiPath, c as readableLiveProgress, i as isFinished, n as LIVE_FILE, o as liveCounts, r as LIVE_SCHEMA_VERSION, s as liveTranscriptPath, t as FINISHED } from "../live-jlfszAbb.js";
+export { ANALYSIS_PROMPT_VERSION, COMPARE_ORDER, DEBUG_LOG, DEBUG_PHASES, DIGEST_VERSION, DIRECTIONAL_MIN, EFFORT_ALPHA, EFFORT_MIN_RUNS, FINISHED, LIFT_MIN_COUNTED, LIVE_FILE, LIVE_SCHEMA_VERSION, NewerSchemaError, REQUESTS_FILE, RUN_ID, TOKENS_UNREPORTED, TRACE_FILE, VERDICT_TEXT, activityLabel, analysisDataPath, answerRows, artifactPath, artifactType, buildIndex, caseLabel, cellContrast, cellEffort, cellJudgeRates, cellRows, cellSamples, cellUsage, cellsChanged, checkLabels, collapsedJudgeLabel, collapsedJudges, commandLine, compare, compareView, contrastEffort, debugApiPath, debugLogLine, debugTime, defaultAttempt, directional, effortChangeLabel, effortLabel, effortOf, effortSample, effortSummary, effortVerdict, evalHistory, evalMarkOutcome, evalTree, failedJudgeLabels, familyCases, familyEffortLabel, familyHeadLabel, familyLiftLabel, familyRateLabel, findAttempt, findEval, formatBytes, formatCount, formatDuration, formatPercent, formatRatio, formatSeconds, formatUsage, harnessLabel, hasUngatedCells, indexFiles, isFinished, isMatrix, isSuperseded, isUngatedMiss, judgeLabel, judgeRatesLabel, labelledJudgements, languageLabel, liveApiPath, liveCounts, liveTranscriptPath, manifestPath, parseDataPath, parseDebugLog, parseNdjson, parseTranscript, pinnedAnalysis, readableAnalysis, readableLiveProgress, readableRecord, readableTranscriptLine, recordPath, requestLine, runHeader, runLead, runOutcome, runRows, runtimeLabel, shardOf, shardPath, shortSha, sortAnalyses, sourceChanges, splitEvalId, splitPatch, stackChanges, studioPath, summaryJudgeRates, tokenValues, traceApiPath, transcriptItems, uniqueJudgeLabels, whyChanged };

package/dist/views-BW6oCDUQ.js

@@ -0,0 +1,2 @@
+import "./model-oqcDplJG.js";
+export { c as resolveRun } from "./views-BmQF4UCI.js";

package/dist/{views-DZ1YDqey.js → views-BmQF4UCI.js}

@@ -1,7 +1,7 @@
 import { r as formatDuration } from "./text-gEcKjAyp.js";
-import { K as effortChangeLabel, at as cellsChanged, ct as stackChanges } from "./model-BndnmcLm.js";
-import { o as warnings } from "./state-B72h_jEw.js";
-import { c as stateRows, l as summaryCounts, r as cellCounts, s as repeatsVerdict, t as OUTCOME_MARK } from "./summary-k3mhWWXv.js";
+import { V as labelledJudgements, et as effortChangeLabel, vt as cellsChanged, xt as stackChanges } from "./model-oqcDplJG.js";
+import { o as warnings } from "./state-C__9swoG.js";
+import { a as contrastLines, d as summaryCounts, l as repeatsVerdict, r as cellCounts, t as OUTCOME_MARK, u as stateRows } from "./summary-BLAVLySU.js";
 //#region src/history/views.ts
 var RefError = class extends Error {
 	name = "RefError";
@@ -15,20 +15,20 @@ function resolveRun(entries, ref = "latest") {
 	const back = /^latest(?:~(\d+))?$/.exec(ref);
 	if (back) {
 		const entry = entries[entries.length - 1 - Number(back[1] ?? 0)];
-		if (!entry) throw new RefError(`There are only ${entries.length} runs`);
+		if (!entry) throw new RefError(`There ${entries.length === 1 ? "is only 1 run" : `are only ${entries.length} runs`}, so ${ref} names none. reeval history lists them.`);
 		return entry;
 	}
 	const exact = entries.find((e) => e.runId === ref);
 	if (exact) return exact;
 	const byId = entries.filter((e) => e.runId.startsWith(ref));
 	if (byId.length === 1 && byId[0]) return byId[0];
-	if (byId.length > 1) throw new RefError(`"${ref}" matches ${byId.length} runs; use more of the id`);
+	if (byId.length > 1) throw new RefError(`"${ref}" matches ${byId.length} runs: use more of the id`);
 	if (/^[0-9a-f]{4,40}$/i.test(ref)) {
 		const atSha = entries.filter((e) => e.git?.sha.startsWith(ref.toLowerCase()));
 		const newest = atSha[atSha.length - 1];
 		if (newest) return newest;
 	}
-	throw new RefError(`No run matches "${ref}"`);
+	throw new RefError(`No run matches "${ref}". Pass latest, latest~N, a run id or a git SHA; reeval history lists them.`);
 }
 /**
 * The run to compare `head` against: a complete, earlier run on the first commit of `ancestry` that
@@ -205,10 +205,15 @@ function formatShow(run) {
 		const single = e.attempts.length === 1;
 		const first = e.attempts[0];
 		lines.push(`  ${OUTCOME_MARK[e.outcome]} ${e.id}${single && first ? `  ${formatDuration(first.durationMs)}` : `  ${cellCounts(e)}`}`);
+		const contrasts = contrastLines(e);
+		if (contrasts.length) {
+			lines.push("      contrasts (each pair of arms, same harness)");
+			for (const line of contrasts) lines.push(`        ${line}`);
+		}
 		for (const t of e.attempts) {
 			const indent = single ? "      " : "        ";
 			if (!single) lines.push(`      ${OUTCOME_MARK[t.outcome]} ${t.id}  ${formatDuration(t.durationMs)}`);
-			for (const j of t.judgements) lines.push(`${indent}judge ${j.verdict} ${j.score.toFixed(2)} (≥ ${j.threshold})  ${j.question}`);
+			for (const { judgement: j, label, labelled } of labelledJudgements(t)) lines.push(`${indent}judge ${j.verdict} ${j.score.toFixed(2)} (≥ ${j.threshold})  ${labelled ? `${label}: ` : ""}${j.question}`);
 			if (t.error && !repeatsVerdict(t)) lines.push(`${indent}${t.error.kind}: ${t.error.message.split("\n")[0]}`);
 		}
 	}
@@ -217,4 +222,4 @@ function formatShow(run) {
 //#endregion
 export { formatShow as a, resolveRun as c, formatNoBaseline as i, unreachable as l, formatCompare as n, history as o, formatHistory as r, pickBaseline as s, RefError as t };
 
-//# sourceMappingURL=views-DZ1YDqey.js.map
+//# sourceMappingURL=views-BmQF4UCI.js.map

package/dist/views-BmQF4UCI.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"views-BmQF4UCI.js","names":[],"sources":["../src/history/views.ts"],"mappings":";;;;;AA6BA,IAAa,WAAb,cAA8B,MAAM;CAClC,OAAgB;AAClB;;;;;AAMA,SAAgB,WAAW,SAAyB,MAAM,UAAwB;CAChF,IAAI,QAAQ,WAAW,GAAG,MAAM,IAAI,SAAS,8BAA8B;CAC3E,MAAM,OAAO,sBAAsB,KAAK,GAAG;CAC3C,IAAI,MAAM;EACR,MAAM,QAAQ,QAAQ,QAAQ,SAAS,IAAI,OAAO,KAAK,MAAM,CAAC;EAC9D,IAAI,CAAC,OACH,MAAM,IAAI,SACR,SAAS,QAAQ,WAAW,IAAI,kBAAkB,YAAY,QAAQ,OAAO,OAAO,OAAO,IAAI,wCACjG;EAEF,OAAO;CACT;CACA,MAAM,QAAQ,QAAQ,MAAM,MAAM,EAAE,UAAU,GAAG;CACjD,IAAI,OAAO,OAAO;CAClB,MAAM,OAAO,QAAQ,QAAQ,MAAM,EAAE,MAAM,WAAW,GAAG,CAAC;CAC1D,IAAI,KAAK,WAAW,KAAK,KAAK,IAAI,OAAO,KAAK;CAC9C,IAAI,KAAK,SAAS,GAChB,MAAM,IAAI,SAAS,IAAI,IAAI,YAAY,KAAK,OAAO,0BAA0B;CAC/E,IAAI,oBAAoB,KAAK,GAAG,GAAG;EACjC,MAAM,QAAQ,QAAQ,QAAQ,MAAM,EAAE,KAAK,IAAI,WAAW,IAAI,YAAY,CAAC,CAAC;EAC5E,MAAM,SAAS,MAAM,MAAM,SAAS;EACpC,IAAI,QAAQ,OAAO;CACrB;CACA,MAAM,IAAI,SACR,mBAAmB,IAAI,4EACzB;AACF;;;;;;;;AASA,SAAgB,aACd,SACA,MACA,UACqB;CACrB,MAAM,UAAU,QAAQ,QAAQ,MAAM,EAAE,QAAQ,KAAK,SAAS,EAAE,WAAW,UAAU;CACrF,IAAI,YAAY,SAAS,SAAS,GAAG;EACnC,MAAM,OAAO,IAAI,IAAI,SAAS,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;EACvD,IAAI,OAA4B;EAChC,IAAI,WAAW,OAAO;EACtB,KAAK,MAAM,KAAK,SAAS;GACvB,MAAM,IAAI,EAAE,MAAM,KAAK,IAAI,EAAE,IAAI,GAAG,IAAI,KAAA;GACxC,IAAI,MAAM,KAAA,MAAc,IAAI,YAAa,MAAM,YAAY,QAAQ,EAAE,QAAQ,KAAK,QAAS;IACzF,OAAO;IACP,WAAW;GACb;EACF;EACA,IAAI,MAAM,OAAO;CACnB;CACA,OAAO,QAAQ,QAAQ,SAAS,MAAM;AACxC;AAEA,MAAM,YAAY,MAAqB,EAAE,MAAM,EAAE,IAAI,IAAI,MAAM,GAAG,CAAC,IAAI;AAkBvE,MAAa,eAAe,OAAqB,UAC/C,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,OAAO,CAAC,MAAM,IAAI,MAAM,IAAI,GAAG;AAEpD,SAAgB,QACd,SACA,UAAqE,CAAC,GACtE;CACA,MAAM,OAAO,QAAQ,MAAM,EAAE,QAAQ,SAAS,GAAG;CACjD,MAAM,MAAgB,CAAC;CACvB,KAAK,MAAM,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,GAClC,KAAK,MAAM,KAAK,IAAI,OAAO,IAAI,CAAC,IAAI,SAAS,EAAE,EAAE,GAAG,IAAI,KAAK,EAAE,EAAE;CAEnE,MAAM,SAAS,QAAQ,QAAQ,YAAY;CAiB3C,OAAO;EAAE;EAAM,MAhBY,IACxB,QAAQ,OAAO,CAAC,UAAU,GAAG,YAAY,CAAC,CAAC,SAAS,MAAM,CAAC,CAAC,CAC5D,KAAK,OAAO;GACX,MAAM,MAAkB;IAAE;IAAI,UAAU,CAAC;IAAG,OAAO,CAAC;IAAG,MAAM;IAAG,SAAS;GAAE;GAC3E,MAAM,uBAAO,IAAI,IAAoB;GACrC,KAAK,SAAS,KAAK,MAAM;IACvB,MAAM,IAAI,IAAI,MAAM,MAAM,MAAM,EAAE,OAAO,EAAE;IAC3C,IAAI,SAAS,KAAK,GAAG,WAAW,IAAI;IACpC,IAAI,CAAC,GAAG;IACR,IAAI,aAAa,MAAM,IAAI,EAAE,KAAK,GAAG,IAAI,MAAM,KAAK,CAAC;IACrD,IAAI,EAAE,YAAY,UAAU,EAAE,YAAY,QAAQ,IAAI;IACtD,IAAI,EAAE,YAAY,QAAQ,IAAI;GAChC,CAAC;GACD,OAAO;EACT,CAEgB;EAAG,UADJ,KAAK,QAAQ,MAAM,YAAY,GAAG,QAAQ,SAAS,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,KAC5D;CAAE;AAChC;AAEA,SAAgB,cAAc,EAAE,MAAM,MAAM,YAAgD;CAC1F,IAAI,KAAK,WAAW,GAClB,OAAO;CACT,MAAM,QAAQ,KAAK;CACnB,MAAM,OAAO,KAAK,KAAK,SAAS;CAChC,MAAM,QAAQ,CACZ,GAAG,KAAK,OAAO,MAAM,KAAK,WAAW,IAAI,KAAK,IAAI,qBAAqB,MAAM,MAAM,IAAI,SAAS,KAAK,EAAE,MAAM,KAAK,MAAM,IAAI,SAAS,IAAI,EAAE,IAC3I,EACF;CACA,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,KAAK,MAAM,EAAE,SAAS,SAAS,EAAE,MAAM,MAAM,GAAG,CAAC;CAChF,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,IAAI,SACf,KAAK,GAAG,MAAM,GAAG,IAAI,MAAM,SAAS,CAAC,IAAI,MAAM,KAAK,IAAI,aAAa,KAAK,KAAK,CAAC,CAChF,KAAK,EAAE;EACV,MAAM,OAAO,IAAI,UAAU,GAAG,IAAI,KAAK,GAAG,IAAI,YAAY;EAC1D,MAAM,KAAK,KAAK,MAAM,OAAO,KAAK,EAAE,IAAI,KAAK,SAAS,CAAC,EAAE,IAAI,IAAI,IAAI;CACvE;CACA,MAAM,KAAK,IAAI,8EAA8E;CAC7F,IAAI,SAAS,SAAS,GACpB,MAAM,KACJ,IACA,GAAG,SAAS,OAAO,MAAM,SAAS,WAAW,IAAI,SAAS,SAAS,4EACnE,GAAG,SAAS,KAAK,OAAO,KAAK,IAAI,CACnC;CAEF,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,MAAM,eAA8C;CAClD,WAAW;CACX,OAAO;CACP,MAAM;CACN,OAAO;CACP,OAAO;CACP,SAAS;CACT,OAAO;AACT;;;;;AAMA,MAAM,OAAO,MACX;CACE,EAAE,SAAS,SAAS,KAAK,EAAE,QAAQ,KAAK,IAAI,MAAM;CAClD,EAAE,OAAO,MAAM,SAAS,mBAAmB,EAAE,MAAM,MAAM,KAAK,IAAI,MAAM;CACxE,EAAE,OAAO,QAAQ,SAAS,qBAAqB,EAAE,MAAM,QAAQ,KAAK,IAAI,MAAM;AAChF,CAAC,CAAC,KAAK,EAAE;AAEX,MAAM,QAAyB;CAAC;CAAa;CAAS;CAAS;CAAS;CAAS;CAAW;AAAM;AAElG,MAAM,eAAe,GAAiB,UACpC,GAAG,EAAE,MAAM,IAAI,SAAS,CAAC,IAAI,EAAE,WAAW,KAAK,mBAAmB,YAAY,GAAG,KAAK,IAAI,mCAAmC,GAAG;AAElI,SAAgB,cACd,MACA,MACA,MACA,IACA,QAAsB,MACtB;CACA,MAAM,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,GAAG,MAAM,MAAM,QAAQ,EAAE,MAAM,IAAI,MAAM,QAAQ,EAAE,MAAM,CAAC;CACzF,MAAM,QAAQ,MAAuB,IAAI,aAAa,KAAK;CAC3D,MAAM,SAAS,MAAqB,KAAK,QAAQ,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;CACvE,MAAM,QAAQ,GAAG,MAAM,WAAW,EAAE,eAAe,MAAM,OAAO,EAAE,WAAW,MAAM,OAAO,EAAE,oBAAoB,MAAM,MAAM,EAAE;CAC9H,MAAM,UAAU,aAAa,MAAM,IAAI;CACvC,MAAM,aAAa,QAAQ,KAAK,MAAM,GAAG,EAAE,MAAM,GAAG,EAAE,KAAK,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,IAAI;CACnF,MAAM,OAAO,MAAc,EAAE,QAAQ,OAAO,KAAK;CACjD,MAAM,QAAQ,OAAO,SAAS,OAAO,EAAE,WAAW,CAAC,EAAA,CAAG,KAAK,OAAO;EAAE,KAAK;EAAG,MAAM;CAAE,EAAE,CAAC;CACvF,IAAI,IACF,OAAO;EACL,eAAe,KAAK,MAAM,MAAM,KAAK;EACrC;EACA,QAAQ,YAAY,MAAM,KAAK,EAAE,UAAU,YAAY,MAAM,KAAK,EAAE,IAAI,MAAM;EAC9E;EACA,GAAI,QAAQ,SAAS,CAAC,sBAAsB,WAAW,IAAI,EAAE,IAAI,CAAC;EAClE;EACA;EACA,GAAG,OAAO,KACP,MACC,KAAK,IAAI,EAAE,EAAE,EAAE,KAAK,KAAK,EAAE,IAAI,EAAE,KAAK,KAAK,EAAE,IAAI,EAAE,KAAK,aAAa,EAAE,UAAU,IAAI,CAAC,EAAE,GAC5F;EACA;EACA,GAAI,MAAM,SACN;GACE;GACA;GACA,GAAG,MAAM,KACN,EAAE,KAAK,WACN,KAAK,IAAI,IAAI,EAAE,EAAE,KAAK,IAAI,KAAK,GAAG,EAAE,KAAK,WAAW,IAAI,EAAE,KAAK,aAAa,IAAI,EAAE,GACtF;GACA;EACF,IACA,CAAC;CACP,CAAC,CAAC,KAAK,IAAI;CAEb,OAAO;EACL,QAAQ,YAAY,MAAM,KAAK,EAAE,UAAU,YAAY,MAAM,KAAK;EAClE;EACA,GAAI,QAAQ,SAAS,CAAC,kBAAkB,YAAY,IAAI,CAAC;EACzD;EACA,GAAG,OAAO,SAAS,MAAM,CACvB,KAAK,KAAK,EAAE,IAAI,EAAE,KAAK,KAAK,EAAE,IAAI,EAAE,IAAI,aAAa,EAAE,OAAO,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,KAAK,IAAI,CAAC,KAC1F,IAAI,EAAE,WAAW,CAAC,EAAA,CAAG,KAClB,MACC,SAAS,EAAE,IAAI,IAAI,CAAC,EAAE,KAAK,QAAQ,EAAE,KAAK,OAAO,QAAQ,WAAW,CAAC,MAAM,IAAI,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK,KAAK,GAC9H,CACF,CAAC;CACH,CAAC,CAAC,KAAK,IAAI;AACb;;AAGA,SAAgB,iBAAiB,MAAoB,MAAoB,IAAqB;CAC5F,MAAM,UAAU,KAAK,QAAQ,MAAM,EAAE,SAAS,MAAM,CAAC,CAAC;CACtD,MAAM,OAAO,sCAAsC,KAAK,MAAM,0BAA0B,UAAU,SAAS,QAAQ,OAAO,YAAY,IAAI,MAAM,OAAO,GAAG;CAC1J,MAAM,QAAQ,MAAuB,IAAI,aAAa,KAAK;CAC3D,IAAI,IACF,OAAO;EACL,eAAe,KAAK;EACpB;EACA;EACA;EACA;EACA;EACA,GAAG,KAAK,KAAK,MAAM,KAAK,EAAE,GAAG,QAAQ,OAAO,KAAK,EAAE,KAAK,KAAK,EAAE,IAAI,EAAE,GAAG;EACxE;CACF,CAAC,CAAC,KAAK,IAAI;CAEb,OAAO;EAAC;EAAM;EAAI,GAAG,KAAK,KAAK,MAAM,KAAK,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI;CAAC,CAAC,CAAC,KAAK,IAAI;AACpF;AAEA,MAAM,aAAa,MACjB,IAAI,GAAG,OAAO,EAAE,MAAM,EAAE,IAAI,OAAO,EAAE,EAAE,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,cAAc,WAAW,OAAO;AAChG,MAAM,UAAU,MAAc,GAAG,KAAK,IAAI,MAAM,MAAM,KAAK,MAAM,KAAK,IAAI,CAAC,IAAI,GAAG;AAClF,MAAM,cAAc,MAAmB,GAAG,UAAU,EAAE,KAAK,IAAI,EAAE,KAAK,UAAU,EAAE,KAAK,IAAI;AAC3F,MAAM,gBAAgB,MACpB,EAAE,KAAK,UAAU,EAAE,KAAK,SAAS,kBAAkB,EAAE,KAAK,QAAQ,EAAE,KAAK,MAAM,IAAI;AAErF,SAAgB,WAAW,KAAwB;CACjD,MAAM,OAAO,UAAU,GAAG;CAC1B,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC;CACrD,MAAM,QAAQ;EACZ,OAAO,IAAI,MAAM,IAAI,cAAc,IAAI,OAAO,IAAI,IAAI,WAAW,gBAAgB,oBAAoB;EACrG,aAAa,IAAI,UAAU,KAAK,eAAe,IAAI,UAAU,EAAE,gBAAgB,IAAI;EACnF;EACA,GAAG,KAAK,KAAK,CAAC,GAAG,OAAO,KAAK,EAAE,OAAO,KAAK,EAAE,IAAI,GAAG;EACpD,GAAG,SAAS,GAAG,CAAC,CAAC,KAAK,MAAM,OAAO,GAAG;EACtC;CACF;CACA,KAAK,MAAM,KAAK,IAAI,OAAO;EACzB,MAAM,SAAS,EAAE,SAAS,WAAW;EACrC,MAAM,QAAQ,EAAE,SAAS;EACzB,MAAM,KACJ,KAAK,aAAa,EAAE,SAAS,GAAG,EAAE,KAAK,UAAU,QAAQ,KAAK,eAAe,MAAM,UAAU,MAAM,KAAK,WAAW,CAAC,KACtH;EACA,MAAM,YAAY,cAAc,CAAC;EACjC,IAAI,UAAU,QAAQ;GACpB,MAAM,KAAK,mDAAmD;GAC9D,KAAK,MAAM,QAAQ,WAAW,MAAM,KAAK,WAAW,MAAM;EAC5D;EACA,KAAK,MAAM,KAAK,EAAE,UAAU;GAC1B,MAAM,SAAS,SAAS,WAAW;GACnC,IAAI,CAAC,QACH,MAAM,KAAK,SAAS,aAAa,EAAE,SAAS,GAAG,EAAE,GAAG,IAAI,eAAe,EAAE,UAAU,GAAG;GAExF,KAAK,MAAM,EAAE,WAAW,GAAG,OAAO,cAAc,mBAAmB,CAAC,GAClE,MAAM,KACJ,GAAG,OAAO,QAAQ,EAAE,QAAQ,GAAG,EAAE,MAAM,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,KAAK,WAAW,GAAG,MAAM,MAAM,KAAK,EAAE,UAC5G;GAEF,IAAI,EAAE,SAAS,CAAC,eAAe,CAAC,GAC9B,MAAM,KAAK,GAAG,SAAS,EAAE,MAAM,KAAK,IAAI,EAAE,MAAM,QAAQ,MAAM,IAAI,CAAC,CAAC,IAAI;EAE5E;CACF;CACA,OAAO,MAAM,KAAK,IAAI;AACxB"}

package/dist/{worker-entry-DXeyWY-m.js → worker-entry-yOchGYYv.js}

@@ -1,86 +1,16 @@
-import { a as calledMcp, c as edited, d as searched, l as fetched, n as useSchedulerChannel, o as calledSkill, s as calledTool, t as acquire, u as ran } from "./client-CwzU9CiW.js";
-import { n as UsageError, t as TimeoutError } from "./errors-yDAEwmZo.js";
-import { c as relativeFile } from "./ids-Bp6ut7ZD.js";
-import { t as esJiti } from "./jiti-BcJHCtrb.js";
-import { a as useEngine, d as withAttemptExpect, l as extendExpect, o as attemptExpectError, s as createExpect, u as unawaitedError } from "./runtime-Dfb2r-cS.js";
+import { n as UsageError, t as TimeoutError } from "./errors-B6BSMs03.js";
+import { i as originalStack, t as adapterForFile } from "./adapter-DWlpOtRn.js";
+import { a as useEngine, c as createExpect, f as rawAssertions, m as withAttemptExpect, p as unawaitedError, s as attemptExpectError, u as extendExpect } from "./runtime-PX_oXiCS.js";
+import { n as useSchedulerChannel, t as acquire } from "./client-fcwgwOAD.js";
 import { t as budgetMarker } from "./budget-DBUrH4jX.js";
-import { t as codeFrame } from "./errors-C9w_ud7g.js";
-import { existsSync, readFileSync } from "node:fs";
-import { dirname } from "node:path";
+import { n as evalMatchers } from "./matchers-Ddmc2z7H.js";
+import { existsSync } from "node:fs";
+import { format } from "node:util";
 import { AsyncLocalStorage } from "node:async_hooks";
-import { format, stripVTControlCharacters } from "node:util";
-import { DecodedMap, getOriginalPosition } from "@vitest/utils/source-map";
-import { getTsconfig } from "get-tsconfig";
 import { processError } from "@vitest/utils/error";
-//#region src/matchers.ts
-/** The last `lines` lines, and at most `chars` characters of them: one line can be megabytes. */
-const tail = (s, lines = 20, chars = 2e3) => {
-	const last = s.trimEnd().split("\n").slice(-lines).join("\n");
-	return last.length <= chars ? last : `[… ${last.length - chars} earlier characters]${last.slice(-chars)}`;
-};
-const hasFiles = (value) => typeof value === "object" && value !== null && typeof value.readFile === "function";
-const matchers = {
-	async toHaveFile(actual, path, pattern) {
-		if (!hasFiles(actual)) throw new TypeError("toHaveFile() expects a turn from c.send(), or c");
-		const content = await actual.readFile(path);
-		if (content === null) return {
-			pass: false,
-			message: () => `expected file ${path} to exist`
-		};
-		if (pattern === void 0) return {
-			pass: true,
-			message: () => `expected no file at ${path}`
-		};
-		const pass = typeof pattern === "string" ? content.includes(pattern) : pattern.test(content);
-		return {
-			pass,
-			message: () => `expected ${path} ${pass ? "not " : ""}to match ${pattern}\n--- ${path} ---\n${tail(content)}`
-		};
-	},
-	async toSucceed(actual) {
-		const r = await actual;
-		if (typeof r?.exitCode !== "number") throw new TypeError("toSucceed() expects run.exec(...)");
-		return {
-			pass: r.exitCode === 0,
-			message: () => r.exitCode === 0 ? "expected command to fail" : `expected command to succeed, exit code ${r.exitCode}\n--- stderr ---\n${tail(r.stderr)}\n--- stdout ---\n${tail(r.stdout)}`
-		};
-	},
-	toHaveSearched(actual, query) {
-		return searched(actual, query);
-	},
-	toHaveFetched(actual, url) {
-		return fetched(actual, url);
-	},
-	toHaveCalledSkill(actual, name) {
-		return calledSkill(actual, name);
-	},
-	toHaveCalledMcp(actual, server, tool, input) {
-		return calledMcp(actual, server, tool, input);
-	},
-	toHaveCalledTool(actual, name, input) {
-		return calledTool(actual, name, input);
-	},
-	toHaveRun(actual, command) {
-		return ran(actual, command);
-	},
-	toHaveEdited(actual, path) {
-		return edited(actual, path);
-	},
-	toPass(actual) {
-		const failure = actual;
-		if (failure?.verdict === "error" && typeof failure.error === "string") return {
-			pass: !!this?.isNot,
-			message: () => `judge failed for "${failure.question}" (${failure.model}): ${failure.error}`
-		};
-		const v = actual;
-		if (typeof v?.score !== "number") throw new TypeError("toPass() expects a c.judge() verdict");
-		const score = v.score.toFixed(2);
-		return {
-			pass: v.verdict === "pass",
-			message: () => v.verdict === "pass" ? `expected judge to fail: score ${score} ≥ ${v.threshold} for "${v.question}"` : `judge score ${score} is below ${v.threshold} for "${v.question}" (${v.model})`
-		};
-	}
-};
+//#region src/language-runner/load.ts
+/** Loads an eval file through its language adapter while the worker is collecting. */
+const loadEvalModule = (file, root) => adapterForFile(file).load(file, root);
 //#endregion
 //#region src/engine/collector.ts
 let collecting;
@@ -181,147 +111,6 @@ function markAll(suite, mode) {
 	}
 }
 //#endregion
-//#region src/engine/loader.ts
-var TsconfigError = class extends Error {
-	name = "TsconfigError";
-};
-/** Each file jiti transformed: its source map, and how far jiti's wrapper shifts line 1. */
-const transformed = /* @__PURE__ */ new Map();
-const INLINE_MAP = /\/\/# sourceMappingURL=data:application\/json;(?:charset=utf-8;)?base64,([A-Za-z0-9+/=]+)\s*$/;
-/** Imports `file` and whatever it imports. */
-async function loadEvalFile(file, root) {
-	checkTsconfig(dirname(file), root);
-	const options = {
-		moduleCache: true,
-		fsCache: false,
-		sourceMaps: true
-	};
-	const base = await esJiti(file, options);
-	const jiti = await esJiti(file, {
-		...options,
-		transform: (opts) => {
-			const code = base.transform(opts);
-			const inline = INLINE_MAP.exec(code);
-			if (opts.filename && inline?.[1]) {
-				const map = JSON.parse(Buffer.from(inline[1], "base64").toString("utf8"));
-				const shift = `(${opts.async ? "async " : ""}function (exports, require, module, __filename, __dirname, jitiImport, jitiESMResolve) { `.length;
-				transformed.set(opts.filename, {
-					map: new DecodedMap(map, opts.filename),
-					shift
-				});
-			}
-			return { code };
-		}
-	});
-	try {
-		return await jiti.import(file);
-	} catch (error) {
-		throw syntaxError(error, root) ?? missingModule(error);
-	}
-}
-/** jiti's `Cannot find module 'x'` plus a require stack, as `… imported from <file>`. */
-function missingModule(error) {
-	const e = error;
-	if (e?.code !== "MODULE_NOT_FOUND") return error;
-	const parts = /^(Cannot find module '[^']+')\nRequire stack:\n- (.+)/.exec(e.message);
-	if (parts) e.message = `${parts[1]} imported from ${parts[2]}`;
-	return error;
-}
-/**
-* jiti reports a parse error as a plain Error, `ParseError: <reason>` then ` <file>:<line>:<col>`
-* (col 0-based): a SyntaxError again, naming the file and position, with a code frame.
-*/
-function syntaxError(error, root) {
-	const parsed = /^ParseError: (.*?)\s*\n\s*(.+):(\d+):(\d+)$/.exec(error?.message ?? "");
-	if (!parsed) return void 0;
-	const [, said, file, line, column] = parsed;
-	const reason = said.replace(/^(?:[A-Za-z]:)?[\\/]\S*?: /, "");
-	const at = {
-		line: Number(line),
-		column: Number(column) + 1
-	};
-	let frame = "";
-	try {
-		frame = stripVTControlCharacters(codeFrame(readFileSync(file, "utf8"), 0, at, 200));
-	} catch {}
-	return /* @__PURE__ */ new SyntaxError(`${reason} (${relativeFile(root, file)}:${at.line}:${at.column})${frame ? `\n${frame}` : ""}`);
-}
-/** `stack` with positions in transformed files mapped back to their sources. */
-function originalStack(stack) {
-	return stack.replace(/^(\s+at (?:.*\()?)(.+?):(\d+):(\d+)(\)?)$/gm, (whole, before, file, line, column, after) => {
-		const loaded = transformed.get(file);
-		const at0 = Number(column) - 1 - (line === "1" ? loaded?.shift ?? 0 : 0);
-		if (!loaded || at0 < 0) return whole;
-		const at = getOriginalPosition(loaded.map, {
-			line: Number(line),
-			column: at0
-		});
-		if (!at) return whole;
-		const original = at.line;
-		return `${before}${file}:${original}:${callee(file, original, at.column) + 1}${after}`;
-	});
-}
-/**
-* Where V8 would put a call's position without the transform: jiti calls an imported function as
-* `(0, _module.fn)()`, which V8 reports at the `(`, where a plain `fn()` reports at `fn`.
-*/
-function callee(file, line, column) {
-	const text = readFileSync(file, "utf8").split(/\r?\n/)[line - 1] ?? "";
-	if (text[column] !== "(") return column;
-	let start = column;
-	while (start > 0 && /[\w$]/.test(text[start - 1])) start--;
-	return start;
-}
-/** Fails when the nearest tsconfig.json isn't valid JSON with comments, or extends a missing file. */
-function checkTsconfig(dir, root) {
-	let found;
-	try {
-		found = getTsconfig(dir);
-	} catch (error) {
-		const message = error.message;
-		const missing = /^File '(.+)' not found\.$/.exec(message);
-		throw new TsconfigError(missing ? `Failed to load tsconfig '${missing[1]}': Tsconfig not found` : `Failed to load tsconfig: ${message}`);
-	}
-	if (!found) return;
-	try {
-		JSON.parse(stripJsonComments(readFileSync(found.path, "utf8")));
-	} catch (error) {
-		throw new TsconfigError(`Failed to load tsconfig '${relativeFile(root, found.path)}': ${error.message}`);
-	}
-}
-/** JSON with comments and trailing commas (tsconfig's dialect) as plain JSON. */
-function stripJsonComments(text) {
-	return scan(scan(text, (rest) => {
-		if (rest.startsWith("//")) {
-			const end = rest.indexOf("\n");
-			return [end < 0 ? rest.length : end, ""];
-		}
-		if (rest.startsWith("/*")) {
-			const end = rest.indexOf("*/", 2);
-			return [end < 0 ? rest.length : end + 2, " "];
-		}
-	}), (rest) => /^,\s*[}\]]/.test(rest) ? [1, " "] : void 0);
-}
-/** Rewrites `text` outside its strings: `at` says how much to replace here, and with what. */
-function scan(text, at) {
-	let out = "";
-	for (let i = 0; i < text.length;) {
-		if (text[i] === "\"") {
-			let end = i + 1;
-			while (end < text.length && text[end] !== "\"") end += text[end] === "\\" ? 2 : 1;
-			out += text.slice(i, end + 1);
-			i = end + 1;
-			continue;
-		}
-		const hit = at(text.slice(i));
-		if (hit) {
-			out += hit[1];
-			i += hit[0];
-		} else out += text[i++];
-	}
-	return out;
-}
-//#endregion
 //#region src/engine/run.ts
 /** An error as it crosses to the main process: its diff computed, then plain data. */
 function serializeError(error) {
@@ -413,7 +202,9 @@ async function runTest(test, { file, stagingDir, signal, reporter }, bodies) {
 		},
 		progress: (progress) => reporter.progress(index, progress),
 		debug: (line) => reporter.debug(index, line),
-		unawaited: async () => await unawaitedError(pending) ?? attemptExpectError(own, task)
+		unawaited: () => unawaitedError(pending),
+		expectError: () => attemptExpectError(own),
+		assertions: () => rawAssertions(task)
 	})));
 	bodies.add(body);
 	const release = () => lease?.release();
@@ -515,10 +306,10 @@ function startWorker(channel) {
 			getRunContext: () => run,
 			collectingFile
 		});
-		extendExpect(matchers);
+		extendExpect(evalMatchers);
 		routeConsole(reporter);
 		try {
-			const tests = await collect(file, () => loadEvalFile(file, run.config.root));
+			const tests = await collect(file, () => loadEvalModule(file, run.config.root));
 			channel.send({
 				t: "collected",
 				tests: tests.map(({ index, name, groups, mode, options }) => ({
@@ -574,4 +365,4 @@ process.on("uncaughtException", (error) => worker.unhandled(error));
 //#endregion
 export {};
 
-//# sourceMappingURL=worker-entry-DXeyWY-m.js.map
+//# sourceMappingURL=worker-entry-yOchGYYv.js.map