reeval 0.11.0 → 0.12.0

package/dist/{judge-DW_GDOmB.js → judge-Dt6KW6_O.js}

@@ -65,6 +65,8 @@ async function withRetries(call, policy, signal) {
 //#region src/judge/index.ts
 /** The judge the docs recommend, pinned so a model update never silently moves scores. */
 const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.14.0";
+/** The language model `reeval init` sets as `analysis.model` and the docs recommend for insights. */
+const RECOMMENDED_ANALYSIS_MODEL = "google/gemini-3.8-flash";
 const DEFAULT_THRESHOLD = .8;
 const GATEWAY_KEYS = ["AI_GATEWAY_API_KEY", "VERCEL_OIDC_TOKEN"];
 /** Providers reeval calls directly, through their own AI SDK package, when their key is set. */
@@ -128,7 +130,7 @@ async function runJudge(question, state, options, signal, retry = JUDGE_RETRY) {
 			const problem = checkModelString(model);
 			if (problem) throw new Error(problem);
 		}
-		const { scoreWithModel } = await import("./model-DaVCY3rb.js");
+		const { scoreWithModel } = await import("./model-C7tSwT7f.js");
 		const scored = await withRetries(() => scoreWithModel(model, question, state, options.providerOptions, signal), retry, signal);
 		({score, resolvedModel} = scored.value);
 		retries = scored.retries;
@@ -146,6 +148,6 @@ async function runJudge(question, state, options, signal, retry = JUDGE_RETRY) {
 	};
 }
 //#endregion
-export { isFakeJudge as a, judgeRoute as c, checkModelString as i, runJudge as l, DIRECT_PROVIDERS as n, judgeCredentials as o, RECOMMENDED_JUDGE_MODEL as r, judgeModelName as s, DEFAULT_THRESHOLD as t, isApiError as u };
+export { checkModelString as a, judgeModelName as c, isApiError as d, RECOMMENDED_JUDGE_MODEL as i, judgeRoute as l, DIRECT_PROVIDERS as n, isFakeJudge as o, RECOMMENDED_ANALYSIS_MODEL as r, judgeCredentials as s, DEFAULT_THRESHOLD as t, runJudge as u };
 
-//# sourceMappingURL=judge-DW_GDOmB.js.map
+//# sourceMappingURL=judge-Dt6KW6_O.js.map

package/dist/judge-Dt6KW6_O.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge-Dt6KW6_O.js","names":["sleep"],"sources":["../src/judge/retry.ts","../src/judge/index.ts"],"mappings":";;;AAOA,MAAa,cAA2B;CAAE,UAAU;CAAG,QAAQ;CAAO,OAAO;AAAO;AAEpF,MAAM,mCAAmB,IAAI,IAAI;CAAC;CAAK;CAAK;CAAK;AAAG,CAAC;AACrD,MAAM,oBACJ;;AAUF,MAAM,aAAa,UAA4B;CAC7C,MAAM,IAAI;CACV,OAAO,GAAG,SAAS,mBAAmB,EAAE,YAAY,UAAU,EAAE,SAAS,IAAI;AAC/E;;AAGA,MAAa,cAAc,UACzB,OAAQ,UAAU,KAAK,CAAC,EAAoB,eAAe;;AAG7D,SAAgB,YAAY,OAAyB;CACnD,MAAM,IAAI,UAAU,KAAK;CACzB,IAAI,CAAC,KAAK,EAAE,SAAS,gBAAgB,EAAE,SAAS,gBAAgB,OAAO;CACvE,IAAI,OAAO,EAAE,eAAe,UAC1B,OAAO,EAAE,cAAc,OAAO,iBAAiB,IAAI,EAAE,UAAU,KAAK,EAAE,gBAAgB;CAExF,OAAO,EAAE,gBAAgB,QAAQ,kBAAkB,KAAK,EAAE,WAAW,OAAO,CAAC,CAAC;AAChF;AAEA,SAAS,aAAa,OAAwB;CAC5C,MAAM,SAAU,UAAU,KAAK,CAAC,EAAoB,kBAAkB;CACtE,MAAM,UAAU,OAAO,MAAM;CAC7B,OAAO,OAAO,SAAS,OAAO,KAAK,UAAU,IAAI,UAAU,MAAO;AACpE;;AAGA,SAAgB,UAAU,QAAqB,OAAe,OAAwB;CACpF,MAAM,MAAM,KAAK,IAAI,OAAO,OAAO,OAAO,SAAS,KAAK,KAAK;CAC7D,MAAM,WAAW,MAAM,IAAI,KAAK,OAAO,KAAK,MAAM;CAClD,OAAO,KAAK,MAAM,KAAK,IAAI,OAAO,OAAO,KAAK,IAAI,UAAU,aAAa,KAAK,CAAC,CAAC,CAAC;AACnF;;AAKA,eAAsB,YACpB,MACA,QACA,QAC8C;CAC9C,MAAM,UAAwB,CAAC;CAC/B,SACE,IAAI;EACF,OAAO;GAAE,OAAO,MAAM,KAAK;GAAG;EAAQ;CACxC,SAAS,OAAO;EACd,IAAI,OAAO,WAAW,QAAQ,SAAS,KAAK,OAAO,YAAY,CAAC,YAAY,KAAK,GAAG;GAClF,MAAM,SAAU,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;GACxE,OAAO,UAAU;GACjB,MAAM;EACR;EACA,MAAM,UAAU,UAAU,QAAQ,QAAQ,QAAQ,KAAK;EACvD,MAAM,IAAI,UAAU,KAAK;EACzB,QAAQ,KAAK;GAAE,OAAO,GAAG,WAAW,OAAO,CAAC;GAAG;EAAQ,CAAC;EACxD,MAAMA,WAAM,SAAS,KAAA,GAAW,EAAE,OAAO,CAAC;CAC5C;AAEJ;;;;ACxEA,MAAa,0BAA0B;;AAEvC,MAAa,6BAA6B;AAC1C,MAAa,oBAAoB;AAEjC,MAAM,eAAe,CAAC,sBAAsB,mBAAmB;;AAG/D,MAAa,mBAAiE;CAC5E,QAAQ;EAAE,KAAK;EAAgC,KAAK;CAAiB;CACrE,eAAe;EAAE,KAAK;EAAuB,KAAK;CAAsB;AAC1E;AAMA,MAAa,eAAe,MAC1B,OAAO,MAAM,YAAY,MAAM,QAAS,EAAqB,SAAS;AAExE,SAAgB,eAAe,OAA2B;CACxD,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,IAAI,YAAY,KAAK,GAAG,OAAO;CAC/B,MAAM,IAAI;CACV,OAAO,EAAE,YAAY,EAAE,UAAU,GAAG,EAAE,SAAS,GAAG,EAAE,YAAY;AAClE;;;;;AAMA,SAAgB,iBACd,OACA,OAAO,mBACP,WAAW,CAAC,yBAAyB,UAAU,OAAO,GAClC;CACpB,IAAI,MAAM,SAAS,GAAG,GAAG,OAAO,KAAA;CAChC,OAAO,GAAG,KAAK,iCAAiC,SAAS,KAAK,MAAM,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,EAAE,SAAS,MAAM;AAC5G;AAEA,MAAM,cAAc,UAAkB,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC;;AAGvE,SAAgB,iBAAiB,OAA0C;CACzE,IAAI,YAAY,KAAK,GAAG,OAAO,CAAC;CAChC,MAAM,SAAS,iBAAiB,WAAW,KAAK;CAChD,OAAO,SAAS,CAAC,OAAO,KAAK,GAAG,YAAY,IAAI;AAClD;;;;;AAMA,SAAgB,WAAW,OAAe,MAAyB,QAAQ,KAAiB;CAC1F,MAAM,WAAW,WAAW,KAAK;CACjC,MAAM,SAAS,iBAAiB;CAChC,IAAI,UAAU,IAAI,OAAO,MAAM,OAAO;EAAE,MAAM;EAAU;EAAU,KAAK,OAAO;CAAI;CAClF,IAAI,aAAa,MAAM,MAAM,IAAI,EAAE,GAAG,OAAO,EAAE,MAAM,UAAU;CAC/D,MAAM,UAAU,SACZ,OAAO,OAAO,IAAI,WAAW,OAAO,IAAI,kIACxC;CACJ,MAAM,IAAI,MAAM,mCAAmC,MAAM,KAAK,SAAS;AACzE;;AAMA,eAAsB,SACpB,UACA,OACA,SACA,QACA,QAAqB,aACC;CACtB,MAAM,EAAE,OAAO,cAAc;CAC7B,IAAI;CACJ,IAAI;CACJ,IAAI,UAAwB,CAAC;CAC7B,IAAI,YAAY,KAAK,GACnB,QAAQ,OAAO,MAAM,UAAU,aAAa,MAAM,MAAM,KAAK,IAAI,MAAM;MAClE;EACL,IAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,UAAU,iBAAiB,KAAK;GACtC,IAAI,SAAS,MAAM,IAAI,MAAM,OAAO;EACtC;EACA,MAAM,EAAE,mBAAmB,MAAM,OAAO;EACxC,MAAM,SAAS,MAAM,kBACb,eAAe,OAAO,UAAU,OAAO,QAAQ,iBAAiB,MAAM,GAC5E,OACA,MACF;EACC,CAAC,CAAE,OAAO,iBAAkB,OAAO;EACpC,UAAU,OAAO;CACnB;CASA,OAAO;EAAE,SAAA;GAPP,SAAS,SAAS,YAAY,SAAS;GACvC;GACA;GACA;GACA,OAAO,eAAe,KAAK;GAC3B,GAAI,gBAAgB,EAAE,cAAc,IAAI,CAAC;EAE5B;EAAG;CAAQ;AAC5B"}

package/dist/judge.d.ts

@@ -1,4 +1,4 @@
-import { D as ConfigJudgeOptions, G as FakeAnalysisModel, J as FakeNoteInput, Rt as Verdict, Z as FakeVote, ct as JudgeArtifactRef, dt as JudgeModel, ft as JudgeOptions, lt as JudgeCallOptions, n as AnalysisConfig, o as BlindPair, pt as JudgeState, q as FakeJudgeModel, ut as JudgeFailure } from "./types-BYdlwMwo.js";
+import { Bt as Verdict, D as ConfigJudgeOptions, J as FakeJudgeModel, K as FakeInsights, W as FakeAnalysisModel, Z as FakeVote, ct as JudgeArtifactRef, dt as JudgeModel, ft as JudgeOptions, lt as JudgeCallOptions, n as AnalysisConfig, o as BlindPair, pt as JudgeState, q as FakeInsightsInput, ut as JudgeFailure } from "./types-GraZtfHN.js";
 //#region src/judge/index.d.ts
 /** The judge the docs recommend, pinned so a model update never silently moves scores. */
 export declare const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.14.0";
@@ -16,11 +16,12 @@ export declare const fakeJudge: ({ score }?: {
  * An analysis model for tests: `reeval analyze` gets `output` (or `output(digest)`) back as the
  * model's structured answer, and each compare question's pair gets `vote` (a fixed `'A'`, `'B'` or
  * `'tie'`, `{ winner, reason? }`, a function of the blinded pair, or `{ error }`), through the same
- * parsing and validation as a real one. With `analysis.agent`, each note calls the same tools a real
- * model
- * would, then answers with `note` (or `note(input)`), or throws `note.error`.
+ * parsing and validation as a real one. Insights (in `reeval run` and `reeval analyze --insights`)
+ * get `insights` (or `insights(input)`, called per eval for its classification and its sentences,
+ * then once for the headline), through the same validator: an uncited sentence is dropped, and one
+ * with a banned word is asked for again, then dropped.
  */
-export declare const fakeAnalysis: ({ output, note, vote }: Pick<FakeAnalysisModel, 'output' | 'note' | 'vote'>) => FakeAnalysisModel;
+export declare const fakeAnalysis: ({ output, vote, insights }: Pick<FakeAnalysisModel, 'output' | 'vote' | 'insights'>) => FakeAnalysisModel;
 //#endregion
-export type { AnalysisConfig, BlindPair, ConfigJudgeOptions, FakeAnalysisModel, FakeJudgeModel, FakeNoteInput, FakeVote, JudgeArtifactRef, JudgeCallOptions, JudgeFailure, JudgeModel, JudgeOptions, JudgeState, Verdict };
+export type { AnalysisConfig, BlindPair, ConfigJudgeOptions, FakeAnalysisModel, FakeInsights, FakeInsightsInput, FakeJudgeModel, FakeVote, JudgeArtifactRef, JudgeCallOptions, JudgeFailure, JudgeModel, JudgeOptions, JudgeState, Verdict };
 //# sourceMappingURL=judge.d.ts.map

package/dist/judge.js

@@ -1,4 +1,4 @@
-import { r as RECOMMENDED_JUDGE_MODEL, t as DEFAULT_THRESHOLD } from "./judge-DW_GDOmB.js";
+import { i as RECOMMENDED_JUDGE_MODEL, t as DEFAULT_THRESHOLD } from "./judge-Dt6KW6_O.js";
 //#region src/judge.ts
 /**
 * A judge that returns `score` without calling a model, for tests and dry runs. `score` can be a
@@ -12,15 +12,16 @@ const fakeJudge = ({ score = 1 } = {}) => ({
 * An analysis model for tests: `reeval analyze` gets `output` (or `output(digest)`) back as the
 * model's structured answer, and each compare question's pair gets `vote` (a fixed `'A'`, `'B'` or
 * `'tie'`, `{ winner, reason? }`, a function of the blinded pair, or `{ error }`), through the same
-* parsing and validation as a real one. With `analysis.agent`, each note calls the same tools a real
-* model
-* would, then answers with `note` (or `note(input)`), or throws `note.error`.
+* parsing and validation as a real one. Insights (in `reeval run` and `reeval analyze --insights`)
+* get `insights` (or `insights(input)`, called per eval for its classification and its sentences,
+* then once for the headline), through the same validator: an uncited sentence is dropped, and one
+* with a banned word is asked for again, then dropped.
 */
-const fakeAnalysis = ({ output, note, vote }) => ({
+const fakeAnalysis = ({ output, vote, insights }) => ({
 	kind: "fake-analysis",
 	...output !== void 0 ? { output } : {},
-	...note !== void 0 ? { note } : {},
-	...vote !== void 0 ? { vote } : {}
+	...vote !== void 0 ? { vote } : {},
+	...insights !== void 0 ? { insights } : {}
 });
 //#endregion
 export { DEFAULT_THRESHOLD, RECOMMENDED_JUDGE_MODEL, fakeAnalysis, fakeJudge };

package/dist/judge.js.map

@@ -1 +1 @@
-{"version":3,"file":"judge.js","names":[],"sources":["../src/judge.ts"],"mappings":";;;;;;AAQA,MAAa,aAAa,EACxB,QAAQ,MAGN,CAAC,OAAuB;CAC1B,MAAM;CACN;AACF;;;;;;;;;AAUA,MAAa,gBAAgB,EAC3B,QACA,MACA,YAC6E;CAC7E,MAAM;CACN,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;CACzC,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;CACrC,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;AACvC"}
+{"version":3,"file":"judge.js","names":[],"sources":["../src/judge.ts"],"mappings":";;;;;;AAQA,MAAa,aAAa,EACxB,QAAQ,MAGN,CAAC,OAAuB;CAC1B,MAAM;CACN;AACF;;;;;;;;;;AAWA,MAAa,gBAAgB,EAC3B,QACA,MACA,gBACiF;CACjF,MAAM;CACN,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;CACzC,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;CACrC,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;AAC/C"}

package/dist/{live-jlfszAbb.js → live-DYd2QEyR.js}

@@ -1,5 +1,5 @@
 import { n as isOneOf } from "./object-CLf2-tNq.js";
-import { n as fillDefaults } from "./record-BKmMvZ6O.js";
+import { n as fillDefaults } from "./record-kk9M1hj-.js";
 //#region src/view/live.ts
 /** The progress file's name inside the run's staging directory. */
 const LIVE_FILE = "live.json";
@@ -84,4 +84,4 @@ function readableLiveProgress(value) {
 //#endregion
 export { liveApiPath as a, readableLiveProgress as c, isFinished as i, LIVE_FILE as n, liveCounts as o, LIVE_SCHEMA_VERSION as r, liveTranscriptPath as s, FINISHED as t };
 
-//# sourceMappingURL=live-jlfszAbb.js.map
+//# sourceMappingURL=live-DYd2QEyR.js.map

package/dist/{live-jlfszAbb.js.map → live-DYd2QEyR.js.map}

@@ -1 +1 @@
-{"version":3,"file":"live-jlfszAbb.js","names":[],"sources":["../src/view/live.ts"],"mappings":";;;;AAQA,MAAa,YAAY;AACzB,MAAa,sBAAsB;AAsDnC,MAAa,cAAc;AAC3B,MAAa,sBAAsB,OAAe,eAChD,YAAY,MAAM,sBAAsB,mBAAmB,UAAU;AAEvE,MAAa,WAAkC;CAAC;CAAQ;CAAQ;CAAe;AAAS;AACxF,MAAa,cAAc,WAAuB,SAAS,SAAS,MAAM;AAE1E,SAAgB,WAAW,UAA8D;CACvF,MAAM,SAAS,GAAG,aAChB,SAAS,QAAQ,MAAM,SAAS,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;CACtD,OAAO;EACL,MAAM,MAAM,GAAG,QAAQ;EACvB,QAAQ,MAAM,MAAM;EACpB,QAAQ,MAAM,MAAM;EACpB,SAAS,MAAM,aAAa;EAC5B,SAAS,MAAM,SAAS;EACxB,SAAS,MAAM,SAAS;CAC1B;AACF;AAEA,MAAM,WAAkC;CAAC;CAAU;CAAW,GAAG;AAAQ;AACzE,MAAM,SAA+B;CAAC;CAAW;CAAc;AAAW;AAC1E,MAAM,SAAS,MACb,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC;AACzD,MAAM,OAAO,MAA4B,OAAO,MAAM;AACtD,MAAM,OAAO,MAA4B,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC;AACnF,MAAM,UAAa,GAAY,OAC7B,MAAM,QAAQ,MAAM,KAAA,KAAa,GAAG,CAAC;AAEvC,SAAS,gBAAgB,GAAgC;CACvD,IAAI,CAAC,MAAM,CAAC,GAAG,OAAO;CACtB,MAAM,EAAE,KAAK,QAAQ,MAAM,MAAM,MAAM,GAAG,QAAQ,WAAW,eAAe;CAC5E,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO;CAC3F,IAAI,CAAC,QAAQ,UAAU,MAAM,GAAG,OAAO;CACvC,IAAI,CAAC,OAAO,WAAW,GAAG,KAAK,CAAC,OAAO,YAAY,GAAG,GAAG,OAAO;CAChE,OAAO;EACL;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW,aAAa;EACxB,YAAY,cAAc;CAC5B;AACF;;;;;AAMA,SAAgB,qBAAqB,OAAqC;CACxE,IAAI,CAAC,MAAM,KAAK,KAAK,MAAM,kBAAA,GAAuC,OAAO;CACzE,aAAa,iBAAiB,KAAK;CACnC,MAAM,EAAE,OAAO,WAAW,KAAK,MAAM,aAAa,OAAO,aAAa;CACtE,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,SAAS,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,WAAW,KAAK,CAAC,IAAI,GAAG,GAAG,OAAO;CAC3F,IAAI,CAAC,QAAQ,QAAQ,KAAK,KAAK,CAAC,MAAM,QAAQ,QAAQ,GAAG,OAAO;CAChE,MAAM,OAAO,SAAS,IAAI,eAAe;CACzC,IAAI,CAAC,KAAK,OAAO,MAAM,MAAM,IAAI,GAAG,OAAO;CAC3C,OAAO;EACL,eAAA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,SAAS,KAAK;EACd,QAAQ,WAAW,IAAI;EACvB,UAAU;CACZ;AACF"}
+{"version":3,"file":"live-DYd2QEyR.js","names":[],"sources":["../src/view/live.ts"],"mappings":";;;;AAQA,MAAa,YAAY;AACzB,MAAa,sBAAsB;AAsDnC,MAAa,cAAc;AAC3B,MAAa,sBAAsB,OAAe,eAChD,YAAY,MAAM,sBAAsB,mBAAmB,UAAU;AAEvE,MAAa,WAAkC;CAAC;CAAQ;CAAQ;CAAe;AAAS;AACxF,MAAa,cAAc,WAAuB,SAAS,SAAS,MAAM;AAE1E,SAAgB,WAAW,UAA8D;CACvF,MAAM,SAAS,GAAG,aAChB,SAAS,QAAQ,MAAM,SAAS,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;CACtD,OAAO;EACL,MAAM,MAAM,GAAG,QAAQ;EACvB,QAAQ,MAAM,MAAM;EACpB,QAAQ,MAAM,MAAM;EACpB,SAAS,MAAM,aAAa;EAC5B,SAAS,MAAM,SAAS;EACxB,SAAS,MAAM,SAAS;CAC1B;AACF;AAEA,MAAM,WAAkC;CAAC;CAAU;CAAW,GAAG;AAAQ;AACzE,MAAM,SAA+B;CAAC;CAAW;CAAc;AAAW;AAC1E,MAAM,SAAS,MACb,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC;AACzD,MAAM,OAAO,MAA4B,OAAO,MAAM;AACtD,MAAM,OAAO,MAA4B,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC;AACnF,MAAM,UAAa,GAAY,OAC7B,MAAM,QAAQ,MAAM,KAAA,KAAa,GAAG,CAAC;AAEvC,SAAS,gBAAgB,GAAgC;CACvD,IAAI,CAAC,MAAM,CAAC,GAAG,OAAO;CACtB,MAAM,EAAE,KAAK,QAAQ,MAAM,MAAM,MAAM,GAAG,QAAQ,WAAW,eAAe;CAC5E,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO;CAC3F,IAAI,CAAC,QAAQ,UAAU,MAAM,GAAG,OAAO;CACvC,IAAI,CAAC,OAAO,WAAW,GAAG,KAAK,CAAC,OAAO,YAAY,GAAG,GAAG,OAAO;CAChE,OAAO;EACL;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW,aAAa;EACxB,YAAY,cAAc;CAC5B;AACF;;;;;AAMA,SAAgB,qBAAqB,OAAqC;CACxE,IAAI,CAAC,MAAM,KAAK,KAAK,MAAM,kBAAA,GAAuC,OAAO;CACzE,aAAa,iBAAiB,KAAK;CACnC,MAAM,EAAE,OAAO,WAAW,KAAK,MAAM,aAAa,OAAO,aAAa;CACtE,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,SAAS,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,WAAW,KAAK,CAAC,IAAI,GAAG,GAAG,OAAO;CAC3F,IAAI,CAAC,QAAQ,QAAQ,KAAK,KAAK,CAAC,MAAM,QAAQ,QAAQ,GAAG,OAAO;CAChE,MAAM,OAAO,SAAS,IAAI,eAAe;CACzC,IAAI,CAAC,KAAK,OAAO,MAAM,MAAM,IAAI,GAAG,OAAO;CAC3C,OAAO;EACL,eAAA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,SAAS,KAAK;EACd,QAAQ,WAAW,IAAI;EACvB,UAAU;CACZ;AACF"}

package/dist/{model-D0IcOQao.js → model-B-mel-Gv.js}

@@ -1,6 +1,6 @@
 import { n as UsageError } from "./errors-B6BSMs03.js";
-import { a as isFakeJudge, c as judgeRoute } from "./judge-DW_GDOmB.js";
-import { n as analysisModelOf } from "./config-DrVSEVN0.js";
+import { a as checkModelString, l as judgeRoute, o as isFakeJudge, s as judgeCredentials } from "./judge-Dt6KW6_O.js";
+import { n as analysisModelOf } from "./config-Dv5MG7Rp.js";
 import { z } from "zod";
 import { Output, generateText } from "ai";
 //#region src/analysis/schema.ts
@@ -100,10 +100,10 @@ function judgeAside(config, task) {
 	return ` (the judge, ${isFakeJudge(judge) ? "a fake judge" : judge}, is an evaluation model: it scores yes/no questions and can't ${task})`;
 }
 /** Which model analyses: `analysis.model`, else the judge's when it's a language model. */
-function analysisModelName(config, use = "analyze") {
+function analysisModelName(config) {
 	const model = defaultModel(config);
 	if (model !== void 0) return model;
-	throw new UsageError(`${use === "agent" ? `analysis.agent in ${config.file}` : "reeval analyze"} needs a language model. Set analysis: { model: 'google/gemini-3.8-flash' } in ${config.file}${judgeAside(config, "write an analysis")}${use === "agent" ? ", or turn agent off" : ", or run reeval analyze --context to hand the digest to your own coding agent"}.`);
+	throw new UsageError(`reeval analyze needs a language model. Set analysis: { model: 'google/gemini-3.8-flash' } in ${config.file}${judgeAside(config, "write an analysis")}, or run reeval analyze --context to hand the digest to your own coding agent.`);
 }
 /** The plan's name for the model a compare question without its own is answered by; null when none. */
 function questionModelLabel(config) {
@@ -123,9 +123,19 @@ function questionModel(config, question, where) {
 	if (typeof model === "string" && isEvaluationModel(model)) throw new UsageError(`compare.questions "${question.question}" (${where}) would be answered by analysis.model in ${config.file}, "${model}", an evaluation model: it scores yes/no questions and can't pick between two attempts. Use a language model, e.g. 'google/gemini-3.8-flash', or ${fix}.`);
 	return model;
 }
+/**
+* The variable `analysis.model` is missing when it's a provider model with no credential set: its
+* provider's own key, else the AI Gateway's. Null when one is set, or for any other model.
+*/
+function missingAnalysisKey(config, env = process.env) {
+	const model = analysisModelOf(config);
+	if (typeof model !== "string" || isEvaluationModel(model) || checkModelString(model)) return null;
+	const keys = judgeCredentials(model);
+	return keys.some((k) => env[k]) ? null : keys[0] ?? null;
+}
 /** Checks the model and its credentials before any work; throws a UsageError (exit 2). */
-function resolveAnalysisModel(config, use = "analyze") {
-	const model = analysisModelName(config, use);
+function resolveAnalysisModel(config) {
+	const model = analysisModelName(config);
 	if (isFake(model)) return {
 		name: "fake",
 		route: "fake",
@@ -139,37 +149,52 @@ function resolveAnalysisModel(config, use = "analyze") {
 		};
 	} catch (error) {
 		const message = error.message.replace("judge model", "analysis model");
-		throw new UsageError(use === "agent" ? message : `${message} Or run reeval analyze --context, which needs no key.`);
+		throw new UsageError(`${message} Or run reeval analyze --context, which needs no key.`);
 	}
 }
-/** A language model that answers `text`, through the AI SDK's own parsing, or throws `error`. */
-function cannedModel(answer) {
+/**
+* A language model that answers `text`, through the AI SDK's own parsing, or throws `error`.
+* With `countTokens`, it reports a token per 4 characters of its prompt and its answer, so a
+* token budget applies to it as it would to a real model; otherwise it reports none.
+*/
+function cannedModel(answer, countTokens = false) {
 	return {
 		specificationVersion: "v4",
 		provider: "fake",
 		modelId: "fake-analysis",
 		supportedUrls: {},
-		doGenerate: async () => {
+		doGenerate: async (options) => {
 			if ("error" in answer) throw new Error(answer.error);
+			const input = countTokens ? Math.ceil(JSON.stringify(options.prompt).length / 4) : 0;
+			let output = countTokens ? Math.ceil(answer.text.length / 4) : 0;
+			let text = answer.text;
+			const cut = countTokens && options.maxOutputTokens !== void 0 && output > options.maxOutputTokens;
+			if (cut) {
+				output = options.maxOutputTokens;
+				text = text.slice(0, output * 4);
+			}
 			return {
 				content: [{
 					type: "text",
-					text: answer.text
+					text
 				}],
-				finishReason: {
+				finishReason: cut ? {
+					unified: "length",
+					raw: "max_tokens"
+				} : {
 					unified: "stop",
 					raw: "stop"
 				},
 				usage: {
 					inputTokens: {
-						total: 0,
-						noCache: 0,
+						total: input,
+						noCache: input,
 						cacheRead: 0,
 						cacheWrite: 0
 					},
 					outputTokens: {
-						total: 0,
-						text: 0,
+						total: output,
+						text: output,
 						reasoning: 0
 					}
 				},
@@ -228,6 +253,6 @@ async function callAnalysisModel(model, digest, prompt, maxOutputTokens) {
 	};
 }
 //#endregion
-export { providerModel as a, resolveAnalysisModel as c, isFake as i, SYSTEM_PROMPT as l, callAnalysisModel as n, questionModel as o, cannedModel as r, questionModelLabel as s, analysisModelName as t };
+export { missingAnalysisKey as a, questionModelLabel as c, isFake as i, resolveAnalysisModel as l, callAnalysisModel as n, providerModel as o, cannedModel as r, questionModel as s, analysisModelName as t, SYSTEM_PROMPT as u };
 
-//# sourceMappingURL=model-D0IcOQao.js.map
+//# sourceMappingURL=model-B-mel-Gv.js.map

package/dist/model-B-mel-Gv.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-B-mel-Gv.js","names":[],"sources":["../src/analysis/schema.ts","../src/analysis/model.ts"],"mappings":";;;;;;AAGA,MAAM,aAAa,EAAE,KAAK;CAAC;CAAQ;CAAU;AAAK,CAAC;AACnD,MAAM,QAAQ,EACX,MAAM,EAAE,OAAO,CAAC,CAAC,CACjB,SAAS,uFAAuF;AAEnG,MAAM,QAAQ,EAAE,OAAO;CACrB,MAAM,EAAE,OAAO;CACf;CACA;CACA,QAAQ,EACL,MAAM,EAAE,OAAO,CAAC,CAAC,CACjB,SAAS,CAAC,CACV,SAAS,qEAAqE;AACnF,CAAC;AAED,MAAa,iBAAiB,EAAE,OAAO;CACrC,MAAM,EACH,OAAO,CAAC,CACR,SACC,2HACF;CACF,cAAc,EAAE,MAAM,KAAK;CAC3B,SAAS,EACN,MAAM,KAAK,CAAC,CACZ,SACC,uLACF;CACF,KAAK,EACF,MAAM,KAAK,CAAC,CACZ,SAAS,yEAAyE;CACrF,WAAW,EAAE,MACX,EAAE,OAAO;EACP,MAAM,EAAE,OAAO;EACf,MAAM,EAAE,KAAK;GAAC;GAAc;GAAe;GAAS;GAAS;EAAO,CAAC;EACrE;CACF,CAAC,CACH;CACA,SAAS,EACN,MACC,EAAE,OAAO;EACP,OAAO,EAAE,OAAO;EAChB,QAAQ,EAAE,OAAO,CAAC,CAAC,SAAS,mDAAmD;EAC/E,MAAM,EAAE,KAAK;GAAC;GAAS;GAAQ;GAAQ;GAAS;GAAU;EAAS,CAAC;EACpE,QAAQ,EAAE,OAAO;EACjB,gBAAgB,EAAE,OAAO,CAAC,CAAC,SAAS,+BAA+B;EACnE,QAAQ,EACL,OAAO,CAAC,CACR,SACC,sIACF;EACF,QAAQ,EAAE,KAAK;GAAC;GAAQ;GAAU;EAAK,CAAC;EACxC,QAAQ,EAAE,KAAK;GAAC;GAAO;GAAU;EAAM,CAAC;EACxC;CACF,CAAC,CACH,CAAC,CACA,IAAI,CAAC;CACR,YAAY,EAAE,OAAO;EAAE,SAAS;EAAY,KAAK,EAAE,OAAO;CAAE,CAAC;AAC/D,CAAC;AAID,MAAa,gBAAgB;;;;;;;;;;;;;;;;;;;;;AChD7B,MAAa,UAAU,MACrB,OAAO,MAAM,YAAY,MAAM,QAAS,EAAwB,SAAS;AAE3E,MAAM,qBAAqB,UAAkB,MAAM,WAAW,cAAc;;AAG5E,SAAS,aAAa,QAAgE;CACpF,MAAM,QAAQ,gBAAgB,MAAM;CACpC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,MAAM,QAAQ,OAAO,MAAM;CAC3B,OAAO,OAAO,UAAU,YAAY,CAAC,kBAAkB,KAAK,IAAI,QAAQ,KAAA;AAC1E;;AAGA,SAAS,WAAW,QAAwB,MAAsB;CAChE,MAAM,QAAQ,OAAO,MAAM;CAC3B,IAAI,OAAO,UAAU,YAAY,CAAC,YAAY,KAAK,GAAG,OAAO;CAC7D,OAAO,gBAAgB,YAAY,KAAK,IAAI,iBAAiB,MAAM,iEAAiE,KAAK;AAC3I;;AAGA,SAAgB,kBAAkB,QAAoD;CACpF,MAAM,QAAQ,aAAa,MAAM;CACjC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,MAAM,IAAI,WACR,gGAAgG,OAAO,OAAO,WAAW,QAAQ,mBAAmB,EAAE,+EACxJ;AACF;;AAGA,SAAgB,mBAAmB,QAAuC;CACxE,MAAM,QAAQ,aAAa,MAAM;CACjC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,OAAO,OAAO,KAAK,IAAI,SAAS;AAClC;;;;;AAMA,SAAgB,cACd,QACA,UACA,OAC4B;CAC5B,IAAI,SAAS,UAAU,KAAA,GAAW,OAAO,SAAS;CAClD,MAAM,QAAQ,aAAa,MAAM;CACjC,MAAM,MAAM;CACZ,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,WACR,sBAAsB,SAAS,SAAS,KAAK,MAAM,+FAA+F,OAAO,OAAO,WAAW,QAAQ,2BAA2B,EAAE,OAAO,IAAI,EAC7N;CAEF,IAAI,OAAO,UAAU,YAAY,kBAAkB,KAAK,GACtD,MAAM,IAAI,WACR,sBAAsB,SAAS,SAAS,KAAK,MAAM,2CAA2C,OAAO,KAAK,KAAK,MAAM,mJAAmJ,IAAI,EAC9Q;CAEF,OAAO;AACT;;;;;AAMA,SAAgB,mBACd,QACA,MAAyB,QAAQ,KAClB;CACf,MAAM,QAAQ,gBAAgB,MAAM;CACpC,IAAI,OAAO,UAAU,YAAY,kBAAkB,KAAK,KAAK,iBAAiB,KAAK,GAAG,OAAO;CAC7F,MAAM,OAAO,iBAAiB,KAAK;CACnC,OAAO,KAAK,MAAM,MAAM,IAAI,EAAE,IAAI,OAAQ,KAAK,MAAM;AACvD;;AAGA,SAAgB,qBAAqB,QAA+C;CAClF,MAAM,QAAQ,kBAAkB,MAAM;CACtC,IAAI,OAAO,KAAK,GAAG,OAAO;EAAE,MAAM;EAAQ,OAAO;EAAQ,MAAM;CAAM;CACrE,IAAI,kBAAkB,KAAK,GACzB,MAAM,IAAI,WACR,qBAAqB,OAAO,KAAK,OAAO,MAAM,sIAChD;CAEF,IAAI;EACF,OAAO;GAAE,MAAM;GAAO,OAAO,WAAW,KAAK,CAAC,CAAC;EAAK;CACtD,SAAS,OAAO;EACd,MAAM,UAAW,MAAgB,QAAQ,QAAQ,eAAe,gBAAgB;EAChF,MAAM,IAAI,WAAW,GAAG,QAAQ,sDAAsD;CACxF;AACF;;;;;;AAOA,SAAgB,YACd,QACA,cAAc,OACC;CACf,OAAO;EACL,sBAAsB;EACtB,UAAU;EACV,SAAS;EACT,eAAe,CAAC;EAChB,YAAY,OAAO,YAAY;GAC7B,IAAI,WAAW,QAAQ,MAAM,IAAI,MAAM,OAAO,KAAK;GACnD,MAAM,QAAQ,cAAc,KAAK,KAAK,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC,SAAS,CAAC,IAAI;GACnF,IAAI,SAAS,cAAc,KAAK,KAAK,OAAO,KAAK,SAAS,CAAC,IAAI;GAC/D,IAAI,OAAO,OAAO;GAElB,MAAM,MACJ,eAAe,QAAQ,oBAAoB,KAAA,KAAa,SAAS,QAAQ;GAC3E,IAAI,KAAK;IACP,SAAS,QAAQ;IACjB,OAAO,KAAK,MAAM,GAAG,SAAS,CAAC;GACjC;GACA,OAAO;IACL,SAAS,CAAC;KAAE,MAAM;KAAQ;IAAK,CAAC;IAChC,cAAc,MACV;KAAE,SAAS;KAAU,KAAK;IAAa,IACvC;KAAE,SAAS;KAAQ,KAAK;IAAO;IACnC,OAAO;KACL,aAAa;MAAE,OAAO;MAAO,SAAS;MAAO,WAAW;MAAG,YAAY;KAAE;KACzE,cAAc;MAAE,OAAO;MAAQ,MAAM;MAAQ,WAAW;KAAE;IAC5D;IACA,UAAU,CAAC;IACX,UAAU,EAAE,SAAS,gBAAgB;GACvC;EACF;EACA,UAAU,YAAY;GACpB,MAAM,IAAI,MAAM,yCAAyC;EAC3D;CACF;AACF;;AAGA,SAAS,UAAU,MAAyB,QAA+B;CACzE,IAAI,KAAK,WAAW,KAAA,GAClB,MAAM,IAAI,MAAM,+DAA+D;CAEjF,MAAM,SAAS,OAAO,KAAK,WAAW,aAAa,KAAK,OAAO,MAAM,IAAI,KAAK;CAC9E,OAAO,YAAY,EAAE,MAAM,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM,EAAE,CAAC;AAC3F;AAEA,eAAe,cAAc,OAA8B,QAAwC;CACjG,IAAI,MAAM,MAAM,OAAO,UAAU,MAAM,MAAM,MAAM;CACnD,OAAO,cAAc,KAAK;AAC5B;;AAGA,eAAsB,cAAc,OAAsD;CACxF,IAAI,MAAM,UAAU,WAAW,OAAO,MAAM;CAC5C,MAAM,WAAW,MAAM,KAAK,MAAM,GAAG,MAAM,KAAK,QAAQ,GAAG,CAAC;CAC5D,MAAM,KAAK,MAAM,KAAK,MAAM,SAAS,SAAS,CAAC;CAC/C,IAAI,aAAa,UAAU;EACzB,MAAM,EAAE,WAAW,MAAM,OAAO;EAChC,OAAO,OAAO,EAAE;CAClB;CACA,OAAO,MAAM;AACf;AASA,eAAsB,kBACpB,OACA,QACA,QACA,iBACsB;CACtB,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,SAAS,MAAM,aAAa;EAChC,OAAO,MAAM,cAAc,OAAO,MAAM;EACxC,QAAQ;EACR;EACA,QAAQ,OAAO,OAAO;GAAE,QAAQ;GAAgB,MAAM;EAAW,CAAC;EAClE,aAAa;EACb;EACA,YAAY;CACd,CAAC;CACD,OAAO;EACL,QAAQ,OAAO;EACf,eAAe,OAAO,UAAU,WAAW;EAC3C,OAAO;GACL,aAAa,OAAO,OAAO,eAAe;GAC1C,cAAc,OAAO,OAAO,gBAAgB;EAC9C;EACA,WAAW,KAAK,IAAI,IAAI;CAC1B;AACF"}

package/dist/model-BGKYVpoj.js

@@ -0,0 +1 @@
+export { t as analysisModelName, n as callAnalysisModel, r as cannedModel, i as isFake, a as missingAnalysisKey, o as providerModel, s as questionModel, c as questionModelLabel, l as resolveAnalysisModel } from "./model-B-mel-Gv.js";

package/dist/{model-DaVCY3rb.js → model-C7tSwT7f.js}

@@ -1,4 +1,4 @@
-import { c as judgeRoute } from "./judge-DW_GDOmB.js";
+import { l as judgeRoute } from "./judge-Dt6KW6_O.js";
 import { i as debugTrace } from "./attempt-CGPV3BZY.js";
 import { experimental_decide } from "ai";
 import { google } from "@ai-sdk/google";
@@ -41,4 +41,4 @@ async function scoreWithModel(model, question, state, providerOptions, signal) {
 //#endregion
 export { scoreWithModel };
 
-//# sourceMappingURL=model-DaVCY3rb.js.map
+//# sourceMappingURL=model-C7tSwT7f.js.map

package/dist/{model-DaVCY3rb.js.map → model-C7tSwT7f.js.map}

@@ -1 +1 @@
-{"version":3,"file":"model-DaVCY3rb.js","names":["decide"],"sources":["../src/judge/model.ts"],"mappings":";;;;;;AASA,eAAe,aACb,OACwB;CACxB,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,MAAM,QAAQ,WAAW,KAAK;CAC9B,IAAI,MAAM,SAAS,WAAW,OAAO;CACrC,MAAM,KAAK,MAAM,MAAM,MAAM,SAAS,SAAS,CAAC;CAChD,IAAI,MAAM,aAAa,UAAU,OAAO,OAAO,cAAc,EAAE;CAC/D,MAAM,EAAE,eAAe,MAAM,OAAO;CACpC,OAAO,WAAW,cAAc,EAAE;AACpC;AAEA,eAAsB,eACpB,OACA,UACA,OACA,iBACA,QAC+D;CAC/D,MAAM,SAAS,MAAMA,oBAAO;EAC1B,OAAO,MAAM,aAAa,KAAK;EAC/B;EACA,WAAW,EAAE,QAAQ;GAAE,MAAM;GAAW,cAAc;EAAS,EAAE;EACjE;EACA,aAAa;EAEb,YAAY;CACd,CAAC;CACD,MAAM,EAAE,SAAS,OAAO,UAAU,UAAU,qBAAqB;CACjE,WACE,SACA,qBACA;EAAE;EAAS;EAAO;EAAU;EAAU;CAAiB,GACvD,yBACF;CACA,OAAO;EAAE,OAAO,OAAO,QAAQ,OAAO;EAAa,eAAe,OAAO,UAAU;CAAQ;AAC7F"}
+{"version":3,"file":"model-C7tSwT7f.js","names":["decide"],"sources":["../src/judge/model.ts"],"mappings":";;;;;;AASA,eAAe,aACb,OACwB;CACxB,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,MAAM,QAAQ,WAAW,KAAK;CAC9B,IAAI,MAAM,SAAS,WAAW,OAAO;CACrC,MAAM,KAAK,MAAM,MAAM,MAAM,SAAS,SAAS,CAAC;CAChD,IAAI,MAAM,aAAa,UAAU,OAAO,OAAO,cAAc,EAAE;CAC/D,MAAM,EAAE,eAAe,MAAM,OAAO;CACpC,OAAO,WAAW,cAAc,EAAE;AACpC;AAEA,eAAsB,eACpB,OACA,UACA,OACA,iBACA,QAC+D;CAC/D,MAAM,SAAS,MAAMA,oBAAO;EAC1B,OAAO,MAAM,aAAa,KAAK;EAC/B;EACA,WAAW,EAAE,QAAQ;GAAE,MAAM;GAAW,cAAc;EAAS,EAAE;EACjE;EACA,aAAa;EAEb,YAAY;CACd,CAAC;CACD,MAAM,EAAE,SAAS,OAAO,UAAU,UAAU,qBAAqB;CACjE,WACE,SACA,qBACA;EAAE;EAAS;EAAO;EAAU;EAAU;CAAiB,GACvD,yBACF;CACA,OAAO;EAAE,OAAO,OAAO,QAAQ,OAAO;EAAa,eAAe,OAAO,UAAU;CAAQ;AAC7F"}

package/dist/{model-DaX4bNLZ.js → model-CWlAqmbU.js}

@@ -1,8 +1,8 @@
 import { c as shardPath, i as manifestPath, s as shardOf } from "./paths-BmR-7tx3.js";
-import { n as clip } from "./text-gEcKjAyp.js";
+import { n as clip } from "./text-CrVy5ioL.js";
 import { a as totalTokens, i as sumUsage, n as formatTokens, r as promptTokens } from "./usage-CleNKkly.js";
-import { c as readableTranscriptLine } from "./record-BKmMvZ6O.js";
-import { s as sampleReasons } from "./pairs-CnInRu-G.js";
+import { l as readableTranscriptLine } from "./record-kk9M1hj-.js";
+import { s as sampleReasons } from "./pairs-DP6XOJby.js";
 //#region src/view/compare.ts
 const STACK_FIELDS = [
 	{
@@ -151,6 +151,29 @@ function outcomeOf(cells) {
 	return outcomes.every((o) => o === "skipped") ? "skipped" : "pass";
 }
 /**
+* Whether a failing eval failed only on attempts the run's interrupt cancelled: every failed
+* attempt in a gated cell was cut short, so the agent didn't fail it (`reeval show`'s `cancelled`).
+*/
+const cancelledOn = (outcome, cells) => {
+	const gated = cells.filter((c) => c.gate !== "none");
+	return outcome === "fail" && gated.some((c) => c.fail > 0) && gated.every((c) => c.fail === (c.cancelled ?? 0));
+};
+function cancelledSides(outcomes, cells) {
+	const sides = ["base", "head"].filter((s) => cancelledOn(outcomes[s], cells[s] ?? []));
+	return sides.length ? { cancelled: sides } : {};
+}
+/** An eval only head has. */
+const addedRow = (h) => ({
+	id: h.id,
+	base: null,
+	head: h.outcome,
+	status: "added",
+	...cancelledSides({
+		base: null,
+		head: h.outcome
+	}, { head: h.cells })
+});
+/**
 * When two runs of an eval differ only in which cells ran (a filtered run, a harness added), its
 * outcome on the cells both have, and the cells only one has. Null when no cell is shared or a
 * shared cell changed.
@@ -160,12 +183,17 @@ function sharedScope(b, h) {
 	const shared = h.cells.filter((c) => before.has(c.key));
 	if (!shared.length || shared.some((c) => before.get(c.key)?.fingerprint !== c.fingerprint)) return null;
 	const keys = new Set(h.cells.map((c) => c.key));
+	const baseCells = shared.map((c) => before.get(c.key) ?? c);
 	return {
-		base: outcomeOf(shared.map((c) => before.get(c.key) ?? c)),
+		base: outcomeOf(baseCells),
 		head: outcomeOf(shared),
 		cells: {
 			added: h.cells.filter((c) => !before.has(c.key)).map((c) => c.key),
 			removed: b.cells.filter((c) => !keys.has(c.key)).map((c) => c.key)
+		},
+		over: {
+			base: baseCells,
+			head: shared
 		}
 	};
 }
@@ -192,27 +220,34 @@ function compare(base, head) {
 	return [.../* @__PURE__ */ new Set([...head.evals.map((e) => e.id), ...base.evals.map((e) => e.id)])].map((id) => {
 		const b = base.evals.find((e) => e.id === id);
 		const h = head.evals.find((e) => e.id === id);
-		if (!b) return {
-			id,
-			base: null,
-			head: h?.outcome ?? null,
-			status: "added"
-		};
+		if (!b) return addedRow(h);
 		if (!h) return {
 			id,
 			base: b.outcome,
 			head: null,
-			status: "removed"
+			status: "removed",
+			...cancelledSides({
+				base: b.outcome,
+				head: null
+			}, { base: b.cells })
 		};
 		const scoped = b.fingerprint === h.fingerprint ? null : sharedScope(b, h);
-		const row = scoped ? {
-			id,
-			...scoped
+		const outcomes = scoped ? {
+			base: scoped.base,
+			head: scoped.head
 		} : {
-			id,
 			base: b.outcome,
 			head: h.outcome
 		};
+		const row = {
+			id,
+			...outcomes,
+			...scoped ? { cells: scoped.cells } : {},
+			...cancelledSides(outcomes, scoped?.over ?? {
+				base: b.cells,
+				head: h.cells
+			})
+		};
 		const perCell = compareCells(b, h);
 		if (perCell.length) row.perCell = perCell;
 		if (row.base === "infra_error" || row.head === "infra_error") return {
@@ -224,6 +259,10 @@ function compare(base, head) {
 			status: "drift",
 			changes: whyChanged(b, h)
 		};
+		if (row.cancelled) return {
+			...row,
+			status: "drift"
+		};
 		if (graderChanged && row.base !== row.head) return {
 			...row,
 			status: "drift"
@@ -281,6 +320,8 @@ function liftInterval(variant, control) {
 		high: points + Math.sqrt((a.high - p1) ** 2 + (p2 - b.low) ** 2)
 	};
 }
+/** A 95% interval is within noise when it contains 0; touching 0 does not exclude it. */
+const containsZero = (low, high) => low <= 0 && high >= 0;
 /** Rounded to three places, as the record stores rates and points. */
 const round3 = (x) => Math.round(x * 1e3) / 1e3;
 /** Linear-interpolated quantile (`q` from 0 to 1) of unsorted numbers; null for none. */
@@ -304,7 +345,10 @@ const choose = (n, k) => {
 /**
 * Two-sided Mann–Whitney (Wilcoxon rank-sum) p-value for `xs` against `ys`: exact, by every split
 * of the pooled values, with mid-ranks for ties; the normal approximation with a tie correction
-* above `EXACT_LIMIT` splits. Null when either side is empty.
+* above `EXACT_LIMIT` splits. `sign` is the sign of `xs`'s rank sum minus its expectation: below 0,
+* `xs` ranks lower. `floor` is the smallest p any split of these pooled values could give, so ties
+* count: `[2, 2, 2, 3]` against `[2, 2, 3, 3]` can never go below 10/70. Null when either side is
+* empty.
 */
 function mannWhitney(xs, ys) {
 	const n1 = xs.length;
@@ -323,30 +367,57 @@ function mannWhitney(xs, ys) {
 	}
 	const n = n1 + n2;
 	const expected = n1 * (n + 1) / 2;
-	const observed = Math.abs(ranks.slice(0, n1).reduce((a, b) => a + b, 0) - expected);
+	const deviation = ranks.slice(0, n1).reduce((a, b) => a + b, 0) - expected;
+	const sign = Math.sign(deviation);
+	const observed = Math.abs(deviation);
 	if (choose(n, n1) > EXACT_LIMIT) {
 		const counts = /* @__PURE__ */ new Map();
 		for (const { v } of pooled) counts.set(v, (counts.get(v) ?? 0) + 1);
 		const ties = [...counts.values()].reduce((a, t) => a + t ** 3 - t, 0);
 		const variance = n1 * n2 / 12 * (n + 1 - ties / (n * (n - 1)));
-		if (variance <= 0) return 1;
-		const z = Math.max(0, observed - .5) / Math.sqrt(variance);
-		return Math.min(1, 2 * (1 - normalCdf(z)));
+		if (variance <= 0) return {
+			p: 1,
+			sign,
+			floor: 1
+		};
+		const pOf = (d) => Math.min(1, 2 * (1 - normalCdf(Math.max(0, d - .5) / Math.sqrt(variance))));
+		const sorted = pooled.map(({ i }) => ranks[i]);
+		const sum = (rs) => rs.reduce((a, b) => a + b, 0);
+		const widest = Math.max(Math.abs(sum(sorted.slice(0, n1)) - expected), Math.abs(sum(sorted.slice(n - n1)) - expected));
+		return {
+			p: pOf(observed),
+			sign,
+			floor: pOf(widest)
+		};
 	}
 	let extreme = 0;
+	let widest = 0;
+	let atWidest = 0;
 	let total = 0;
 	const pick = (start, left, sum) => {
 		if (left === 0) {
 			total++;
-			if (Math.abs(sum - expected) >= observed - 1e-9) extreme++;
+			const d = Math.abs(sum - expected);
+			if (d >= observed - 1e-9) extreme++;
+			if (d > widest + 1e-9) {
+				widest = d;
+				atWidest = 1;
+			} else if (d >= widest - 1e-9) atWidest++;
 			return;
 		}
 		for (let i = start; i <= n - left; i++) pick(i + 1, left - 1, sum + ranks[i]);
 	};
 	pick(0, n1, 0);
-	return extreme / total;
+	return {
+		p: extreme / total,
+		sign,
+		floor: atWidest / total
+	};
 }
-/** The smallest two-sided p an exact rank test can reach with these group sizes. */
+/**
+* The smallest two-sided p an exact rank test can reach with these group sizes and no ties: for
+* planning `runs` before there's data. On data, `mannWhitney`'s `floor` counts the ties.
+*/
 const minimumP = (n1, n2) => n1 === 0 || n2 === 0 ? 1 : Math.min(1, 2 / choose(n1 + n2, n1));
 /**
 * The difference of two paired pass rates (variant minus control) with Newcombe's score interval
@@ -382,9 +453,14 @@ function mcnemar(variantOnly, controlOnly) {
 	const n = variantOnly + controlOnly;
 	if (n === 0) return 1;
 	const low = Math.min(variantOnly, controlOnly);
-	let tail = 0;
-	for (let i = 0; i <= low; i++) tail += choose(n, i) / 2 ** n;
-	return Math.min(1, 2 * tail);
+	let logTerm = -n * Math.LN2;
+	let logTail = logTerm;
+	for (let i = 0; i < low; i++) {
+		logTerm += Math.log((n - i) / (i + 1));
+		const top = Math.max(logTail, logTerm);
+		logTail = top + Math.log1p(Math.exp(Math.min(logTail, logTerm) - top));
+	}
+	return Math.min(1, 2 * Math.exp(logTail));
 }
 /** Two-sided 95% Student t quantiles for 1 to 30 degrees of freedom. */
 const T975 = [
@@ -454,7 +530,10 @@ function meanInterval(values) {
 /**
 * Two-sided Wilcoxon signed-rank p for paired differences: zeros dropped, mid-ranks for ties,
 * exact over every sign assignment up to 20 non-zero differences, else the normal approximation.
-* Null when every difference is zero.
+* `sign` is the sign of the positive differences' rank sum minus its expectation: below 0, the
+* negative differences outrank the positive ones. Null when every difference is zero. Ties are
+* exact equality, so form each difference of two rates with one division (`(a·d − c·b) / (b·d)`),
+* never as `a/b − c/d`.
 */
 function signedRank(differences) {
 	const d = differences.filter((x) => x !== 0);
@@ -473,15 +552,22 @@ function signedRank(differences) {
 	}
 	const total = n * (n + 1) / 2;
 	const plus = d.reduce((s, v, i) => v > 0 ? s + ranks[i] : s, 0);
+	const sign = Math.sign(plus - total / 2);
 	const observed = Math.abs(plus - total / 2);
 	if (n > 20) {
 		const counts = /* @__PURE__ */ new Map();
 		for (const { a } of sorted) counts.set(a, (counts.get(a) ?? 0) + 1);
 		const ties = [...counts.values()].reduce((s, t) => s + t ** 3 - t, 0);
 		const variance = n * (n + 1) * (2 * n + 1) / 24 - ties / 48;
-		if (variance <= 0) return 1;
+		if (variance <= 0) return {
+			p: 1,
+			sign
+		};
 		const z = Math.max(0, observed - .5) / Math.sqrt(variance);
-		return Math.min(1, 2 * (1 - normalCdf(z)));
+		return {
+			p: Math.min(1, 2 * (1 - normalCdf(z))),
+			sign
+		};
 	}
 	let extreme = 0;
 	for (let mask = 0; mask < 2 ** n; mask++) {
@@ -489,7 +575,10 @@ function signedRank(differences) {
 		for (let i = 0; i < n; i++) if (mask & 1 << i) sum += ranks[i];
 		if (Math.abs(sum - total / 2) >= observed - 1e-9) extreme++;
 	}
-	return extreme / 2 ** n;
+	return {
+		p: extreme / 2 ** n,
+		sign
+	};
 }
 function normalCdf(z) {
 	const t = 1 / (1 + .3275911 * Math.abs(z) / Math.SQRT2);
@@ -527,20 +616,24 @@ function effortOf(samples) {
 		...tokens.length ? { tokens: spread(tokens) } : {}
 	};
 }
-/** The verdict of an exact rank test of `xs` (variant) against `ys` (control). */
+/**
+* The verdict of an exact rank test of `xs` (variant) against `ys` (control). Its direction is
+* the test's (which side ranks lower), not the medians', which can be equal or disagree.
+*/
 function effortVerdict(xs, ys) {
-	const p = mannWhitney(xs, ys);
-	if (minimumP(xs.length, ys.length) >= .05) return {
+	const test = mannWhitney(xs, ys);
+	const p = test?.p ?? null;
+	if (test === null || test.floor >= .05) return {
 		p,
 		verdict: "insufficient"
 	};
-	if (p === null || p >= .05) return {
+	if (test.p >= .05) return {
 		p,
 		verdict: "noise"
 	};
 	return {
 		p,
-		verdict: (median(xs) ?? 0) < (median(ys) ?? 0) ? "fewer" : "more"
+		verdict: test.sign < 0 ? "fewer" : "more"
 	};
 }
 const ratio = (a, b) => b === 0 ? null : a / b;
@@ -797,7 +890,7 @@ function liftForKey(variant, control) {
 			pass: variant.pass,
 			counted: variant.counted
 		},
-		withinNoise: low < 0 && high > 0
+		withinNoise: containsZero(low, high)
 	};
 }
 /** Per-key pass/counted for one cell's counted attempts, with lift on a variant when both sides qualify. */
@@ -1009,11 +1102,13 @@ function activityLabel(a) {
 	return `${plural(a.tools, "tool call")} · ${plural(a.fileEdits, "file edit")}`;
 }
 /**
-* A run's outcome, as `reeval run`'s exit code has it: a failed gated cell (or a budget stop) fails
-* the run, while failures in ungated cells (controls, by default) don't.
+* A run's outcome, which `reeval run`'s exit code is read from: an error outside any attempt, a
+* failed gated cell or a budget stop fails the run, while failures in ungated cells (controls, by
+* default) don't. An interrupted run is `skipped`, whatever ran.
 */
 function runOutcome(run) {
 	if (run.status === "interrupted") return "skipped";
+	if (run.errors?.length) return "fail";
 	const outcomes = [...run.evals, ...run.families ?? []].map((e) => e.outcome);
 	if (outcomes.includes("fail") || run.status === "budget") return "fail";
 	if (outcomes.includes("infra_error")) return "infra_error";
@@ -1369,6 +1464,6 @@ const findAttempt = (e, attemptId) => e.attempts.find((a) => a.id === attemptId)
 /** The attempt to open by default: the first failure, else the first attempt. */
 const defaultAttempt = (e) => e.attempts.find((a) => a.outcome === "fail" || a.outcome === "infra_error") ?? e.attempts[0];
 //#endregion
-export { contrastEffort as $, splitPatch as A, judgeRatesLabel as B, parseTranscript as C, whyChanged as Ct, runtimeLabel as D, runRows as E, checkLabels as F, EFFORT_ALPHA as G, summaryJudgeRates as H, collapsedJudgeLabel as I, VERDICT_TEXT as J, EFFORT_MIN_RUNS as K, collapsedJudges as L, LIFT_MIN_COUNTED as M, cellJudgeRates as N, shortSha as O, cellUsage as P, cellSamples as Q, commandLine as R, languageLabel as S, stackChanges as St, runOutcome as T, uniqueJudgeLabels as U, labelledJudgements as V, DIRECTIONAL_MIN as W, cellContrast as X, bothUnderATenth as Y, cellEffort as Z, harnessLabel as _, signedRank as _t, cellRows as a, effortSummary as at, isMatrix as b, compare as bt, evalHistory as c, formatRatio as ct, failedJudgeLabels as d, liftInterval as dt, directional as et, findAttempt as f, mcnemar as ft, formatUsage as g, round3 as gt, formatPercent as h, pairedLiftInterval as ht, buildIndex as i, effortSample as it, transcriptItems as j, splitEvalId as k, evalMarkOutcome as l, formatSeconds as lt, formatDuration as m, median as mt, activityLabel as n, effortLabel as nt, compareView as o, effortVerdict as ot, findEval as p, meanInterval as pt, TOKENS_UNREPORTED as q, answerRows as r, effortOf as rt, defaultAttempt as s, formatCount as st, COMPARE_ORDER as t, effortChangeLabel as tt, evalTree as u, tokenValues as ut, hasUngatedCells as v, wilson as vt, runLead as w, isUngatedMiss as x, sourceChanges as xt, indexFiles as y, cellsChanged as yt, judgeLabel as z };
+export { contrastEffort as $, splitPatch as A, judgeRatesLabel as B, parseTranscript as C, sourceChanges as Ct, runtimeLabel as D, runRows as E, checkLabels as F, EFFORT_ALPHA as G, summaryJudgeRates as H, collapsedJudgeLabel as I, VERDICT_TEXT as J, EFFORT_MIN_RUNS as K, collapsedJudges as L, LIFT_MIN_COUNTED as M, cellJudgeRates as N, shortSha as O, cellUsage as P, cellSamples as Q, commandLine as R, languageLabel as S, compare as St, runOutcome as T, whyChanged as Tt, uniqueJudgeLabels as U, labelledJudgements as V, DIRECTIONAL_MIN as W, cellContrast as X, bothUnderATenth as Y, cellEffort as Z, harnessLabel as _, round3 as _t, cellRows as a, effortSummary as at, isMatrix as b, addedRow as bt, evalHistory as c, formatRatio as ct, failedJudgeLabels as d, containsZero as dt, directional as et, findAttempt as f, liftInterval as ft, formatUsage as g, pairedLiftInterval as gt, formatPercent as h, median as ht, buildIndex as i, effortSample as it, transcriptItems as j, splitEvalId as k, evalMarkOutcome as l, formatSeconds as lt, formatDuration as m, meanInterval as mt, activityLabel as n, effortLabel as nt, compareView as o, effortVerdict as ot, findEval as p, mcnemar as pt, TOKENS_UNREPORTED as q, answerRows as r, effortOf as rt, defaultAttempt as s, formatCount as st, COMPARE_ORDER as t, effortChangeLabel as tt, evalTree as u, tokenValues as ut, hasUngatedCells as v, signedRank as vt, runLead as w, stackChanges as wt, isUngatedMiss as x, cellsChanged as xt, indexFiles as y, wilson as yt, judgeLabel as z };
 
-//# sourceMappingURL=model-DaX4bNLZ.js.map
+//# sourceMappingURL=model-CWlAqmbU.js.map