stageflow 0.44.0 → 0.46.0

package/dist/evals/manager.js

@@ -10,19 +10,14 @@ import { scoreResult } from "./scoring.js";
 import { computeStats } from "./stats.js";
 import { toGradeRecords, toResultRecord } from "./statsAdapter.js";
 import { isConversationTarget } from "./types.js";
+import { isJudged } from "./judged.js";
+import { measureCalibration } from "./calibrator.js";
+import { EvalRunError } from "./errors.js";
+export { CALIBRATION_DEFAULT_MAX_COST_USD } from "./calibrator.js";
+export { EvalRunError };
 export const EVAL_HOST_STOPPED_REASON = "Host stopped while the run was in progress";
 export const EVAL_DEFAULT_ENGINE = "promptfoo";
 export const EVAL_ORPHAN_SESSION_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
-export class EvalRunError extends Error {
-    code;
-    findings;
-    constructor(code, message, findings) {
-        super(message);
-        this.code = code;
-        this.findings = findings;
-        this.name = "EvalRunError";
-    }
-}
 const key = (caseId, repeatIndex) => `${caseId}:${repeatIndex}`;
 function isRecord(value) {
     return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -96,6 +91,7 @@ export class EvalRunManager {
     reserved = false;
     trying = false;
     tryingRunId = null;
+    calibrationCancel = null;
     constructor(deps) {
         this.deps = deps;
         this.store = deps.store;
@@ -542,6 +538,10 @@ export class EvalRunManager {
         }
     }
     async cancel(evalRunId) {
+        if (this.calibrationCancel !== null && this.tryingRunId === evalRunId) {
+            this.calibrationCancel.abort();
+            return true;
+        }
         const live = this.live;
         if (live === undefined || live.evalRunId !== evalRunId) {
             if ((await this.store.getEvalRun(evalRunId)) === null) {
@@ -676,6 +676,59 @@ export class EvalRunManager {
             this.deps.logger?.warn("eval.try_session_cleanup_failed", safeMessage(err));
         }
     }
+    logEngine(level, message) {
+        if (level === "info")
+            this.deps.logger?.info?.("eval.engine", message);
+        else
+            this.deps.logger?.[level]("eval.engine", message);
+    }
+    /**
+     * Re-judges the stored outputs of results the author labeled for one grader and compares the verdicts to the
+     * labels. No stage or agent runs: the engine's case callback replays the stored output. Nothing is added to the
+     * run list; the measurement is stored as a calibration.
+     */
+    async calibrate(input) {
+        this.claim();
+        this.reserved = false;
+        this.trying = true;
+        const evalRunId = `${EVAL_RUN_ID_PREFIX}${randomBytes(6).toString("hex")}`;
+        this.tryingRunId = evalRunId;
+        const abort = new AbortController();
+        const cancel = new AbortController();
+        this.calibrationCancel = cancel;
+        const relay = () => cancel.abort();
+        if (input.signal?.aborted)
+            relay();
+        input.signal?.addEventListener("abort", relay, { once: true });
+        try {
+            const engine = this.engines(this.engineName);
+            const engineInfo = { name: engine.name, version: await engine.version(), nodeVersion: process.versions.node };
+            const built = await buildEvalPlan({
+                projectRoot: input.projectRoot,
+                evalPath: input.evalPath,
+                overrides: {},
+                evalRunId,
+                engine: engineInfo,
+                ...(this.deps.isKnownModel ? { isKnownModel: this.deps.isKnownModel } : {}),
+                ...this.agentsInput(),
+            });
+            if (!built.ok)
+                throw new EvalRunError("EVAL_INVALID", "Eval is invalid", built.findings);
+            const base = built.plan;
+            const found = base.graders.find((g) => g.id === input.graderId);
+            if (!found || !isJudged(found)) {
+                throw new EvalRunError("EVAL_INVALID", `Grader ${input.graderId} is not a judged grader of this eval`);
+            }
+            return await measureCalibration({ store: this.store, engine, log: (level, message) => this.logEngine(level, message) }, input, base, found, abort, cancel.signal);
+        }
+        finally {
+            input.signal?.removeEventListener("abort", relay);
+            this.calibrationCancel = null;
+            this.trying = false;
+            this.tryingRunId = null;
+            await this.cleanupWorkDir(evalRunId);
+        }
+    }
     async tryOne(input) {
         this.claim();
         this.reserved = false;

package/dist/evals/plan.d.ts

@@ -1,3 +1,4 @@
+import { type LoadedEval } from "../config/loadEval.js";
 import { type AgentCatalogPort } from "../config/resolveAgentTarget.js";
 import type { ValidationFinding } from "../config/validateCatalog.js";
 import { type ConfigComponents, type ConfigSnapshot } from "./snapshot.js";
@@ -62,4 +63,5 @@ export type BuildEvalPlanResult = {
     code: "EVAL_INVALID";
     findings: ValidationFinding[];
 };
+export declare function evalDefinition(loaded: LoadedEval, projectRoot: string): Record<string, unknown>;
 export declare function buildEvalPlan(input: BuildEvalPlanInput): Promise<BuildEvalPlanResult>;

package/dist/evals/plan.js

@@ -13,6 +13,7 @@ import { EVAL_LIMITS, SIMULATOR_PROMPT_VERSION } from "./constants.js";
 import { buildComponents, buildSnapshot, computeFingerprint, } from "./snapshot.js";
 import { planInternalAgent } from "./planInternalAgent.js";
 import { isConversationTarget } from "./types.js";
+import { isJudged, isJudgedType } from "./judged.js";
 const SELECTION_KEYS = ["case_ids", "tags", "only_failed_from"];
 const RUN_SIZE_CODES = new Set(["eval.large_run", "eval.too_large_run"]);
 function posix(projectRoot, abs) {
@@ -106,10 +107,10 @@ function withOverrides(loaded, overrides) {
         ...(overrides.max_cost_usd !== undefined ? { max_cost_usd: overrides.max_cost_usd } : {}),
     };
     const graders = loaded.graders.map((grader) => {
-        if ((grader.type !== "rubric" && grader.type !== "goal_reached") || overrides.judge === undefined)
+        if (!isJudged(grader) || overrides.judge === undefined)
             return grader;
         const raw = loaded.file.graders.find((g) => g.id === grader.id);
-        if ((raw?.type === "rubric" || raw?.type === "goal_reached") && raw.judge !== undefined)
+        if (raw !== undefined && isJudgedType(raw.type) && "judge" in raw && raw.judge !== undefined)
             return grader;
         const { hash: _old, ...rest } = { ...grader, judge: overrides.judge };
         return { ...rest, hash: graderHash(rest) };
@@ -133,7 +134,7 @@ function selectCases(cases, overrides, failedIds) {
         return cases.filter((c) => failedIds.has(c.id));
     return [...cases];
 }
-function evalDefinition(loaded, projectRoot) {
+export function evalDefinition(loaded, projectRoot) {
     const { file } = loaded;
     const paths = resolveEvalPaths(file, loaded.path);
     const rel = (abs) => abs === undefined ? undefined : posix(projectRoot, abs);
@@ -508,7 +509,7 @@ async function buildAgentResult(a) {
     });
     const fingerprint = computeFingerprint(components);
     const { overrides } = a;
-    const hasRubric = loaded.graders.some((g) => g.type === "rubric" || g.type === "goal_reached");
+    const hasRubric = loaded.graders.some(isJudged);
     const plainDefinition = JSON.parse(JSON.stringify(definition));
     const snapshot = buildSnapshot({
         evalId: loaded.id,

package/dist/evals/planInternalAgent.d.ts

@@ -1,7 +1,7 @@
 import type { DatasetCase } from "../config/datasetTypes.js";
 import { type LoadedEval } from "../config/loadEval.js";
 import type { ValidationFinding } from "../config/validateCatalog.js";
-import type { BuildEvalPlanInput, BuildEvalPlanResult, EvalOverrides } from "./plan.js";
+import { type BuildEvalPlanInput, type BuildEvalPlanResult, type EvalOverrides } from "./plan.js";
 type Dataset = {
     path: string;
     contentHash: string;

package/dist/evals/planInternalAgent.js

@@ -5,6 +5,7 @@ import { EVAL_LARGE_RUN, EVAL_TOO_LARGE_RUN, evalFinding } from "../config/valid
 import { PACKAGE_VERSION } from "../package-meta.js";
 import { sha256Hex } from "./canonical.js";
 import { EVAL_LIMITS, SIMULATOR_PROMPT_VERSION } from "./constants.js";
+import { evalDefinition } from "./plan.js";
 import { buildComponents, buildSnapshot, computeFingerprint } from "./snapshot.js";
 function posix(projectRoot, abs) {
     return path.relative(projectRoot, abs).split(path.sep).join("/");
@@ -127,12 +128,7 @@ export function planInternalAgent(args) {
         evalId: loaded.id,
         evalPath: loaded.relPath,
         evalHash: loaded.fileHash,
-        evalDefinition: {
-            version: 1,
-            id: loaded.id,
-            target: { kind: "internal_agent", profile: profileId },
-            dataset: posix(projectRoot, paths.datasetPath),
-        },
+        evalDefinition: evalDefinition(loaded, projectRoot),
         overrides: snapshotOverrides,
         target: {
             profile_id: profileId,

package/dist/evals/read/api.d.ts

@@ -159,6 +159,12 @@ export declare function buildResultDetail(store: RunStore, id: string, caseId: s
         } | null;
         tags: string[];
         suggestion: import("./suggestions.js").WireSuggestion | null;
+        grader_labels: {
+            [k: string]: {
+                label: "fail" | "pass";
+                note: string | null;
+            };
+        };
     }[];
 }>;
 export declare function loadEvalListItems(input: {

package/dist/evals/read/api.js

@@ -227,11 +227,12 @@ export async function buildResultsPage(store, id, input) {
 }
 export async function buildResultDetail(store, id, caseId) {
     const row = await runByIdOr404(store, id);
-    const [cases, results, reviews, suggestions] = await Promise.all([
+    const [cases, results, reviews, suggestions, graderLabels] = await Promise.all([
         store.getEvalCases(row.evalRunId),
         store.getEvalResultsForCase(row.evalRunId, caseId),
         store.getEvalReviews(row.evalRunId),
         store.listEvalSuggestions(row.evalRunId),
+        store.listEvalGraderLabels(row.evalRunId),
     ]);
     const caseRow = cases.find((x) => x.caseId === caseId);
     if (!caseRow || results.length === 0) {
@@ -245,6 +246,9 @@ export async function buildResultDetail(store, id, caseId) {
             .map((r) => ({
             ...resultToWire(r, reviewByKey.get(resultKey(r.caseId, r.repeatIndex)), row.targetKind === "agent" ? (row.targetRef.agent_id ?? "") : undefined),
             suggestion: suggestionFor(suggestions, r.caseId, r.repeatIndex),
+            grader_labels: Object.fromEntries(graderLabels
+                .filter((l) => l.caseId === r.caseId && l.repeatIndex === r.repeatIndex)
+                .map((l) => [l.graderId, { label: l.label, note: l.note }])),
         })),
     };
 }

package/dist/evals/read/labelQueue.d.ts

@@ -0,0 +1,45 @@
+import type { RunStore } from "../../runstore/port.js";
+export type LabelCounts = {
+    pass: number;
+    fail: number;
+};
+/**
+ * Everything the console's labeling screen needs in one read: each result the judge graded for one grader, with the
+ * conversation, what the case expects, the judge's answer and the author's label so far. Results the judge errored on
+ * are left out (there is no verdict to compare with), and so are results that never ran.
+ */
+export declare function buildLabelQueue(store: RunStore, id: string, graderId: string): Promise<{
+    grader_id: string;
+    items: {
+        case: {
+            position: number;
+        };
+        result: import("./runs.js").WireResultDetail;
+        judge: {
+            passed: boolean;
+            score: number | null;
+            reason: string | null;
+        };
+        label: {
+            label: "fail" | "pass";
+            note: string | null;
+        } | null;
+    }[];
+    labeled_here: LabelCounts;
+    labeled_eval: LabelCounts;
+}>;
+/**
+ * The label counts across the eval's runs, and which results of this run the judge graded that still have no label
+ * from you. No conversations: this is for a progress line and the Cases filter.
+ */
+export declare function buildLabelSummary(store: RunStore, id: string, graderId: string): Promise<{
+    grader_id: string;
+    labeled_eval: {
+        pass: number;
+        fail: number;
+    };
+    unlabeled: {
+        case_id: string;
+        repeat_index: number;
+    }[];
+}>;

package/dist/evals/read/labelQueue.js

@@ -0,0 +1,83 @@
+import { resultKey } from "../statsTypes.js";
+import { EvalApiError, runByIdOr404 } from "./api.js";
+import { resultToWire } from "./runs.js";
+import { isJudgedType } from "../judged.js";
+/**
+ * Everything the console's labeling screen needs in one read: each result the judge graded for one grader, with the
+ * conversation, what the case expects, the judge's answer and the author's label so far. Results the judge errored on
+ * are left out (there is no verdict to compare with), and so are results that never ran.
+ */
+export async function buildLabelQueue(store, id, graderId) {
+    const row = await runByIdOr404(store, id);
+    const [cases, results, labels, evalLabels] = await Promise.all([
+        store.getEvalCases(row.evalRunId),
+        store.listEvalResults(row.evalRunId),
+        store.listEvalGraderLabels(row.evalRunId),
+        store.listEvalGraderLabelsForEval(row.projectRoot, row.evalId, graderId),
+    ]);
+    const caseById = new Map(cases.map((c) => [c.caseId, c]));
+    const labelByKey = new Map(labels.filter((l) => l.graderId === graderId).map((l) => [resultKey(l.caseId, l.repeatIndex), l]));
+    const gradeOf = (grades) => grades.find((g) => g.graderId === graderId);
+    const known = results.some((r) => gradeOf(r.grades) !== undefined);
+    if (!known) {
+        throw new EvalApiError(404, "EVAL_GRADER_NOT_FOUND", `Grader ${graderId} has no grades in run ${id}`);
+    }
+    const judged = results.find((r) => {
+        const g = gradeOf(r.grades);
+        return g !== undefined && isJudgedType(g.graderType);
+    });
+    if (!judged) {
+        throw new EvalApiError(400, "invalid_request", `Grader ${graderId} is not judged by a model, so it has nothing to label`);
+    }
+    const items = [...results]
+        .sort((a, b) => (caseById.get(a.caseId)?.position ?? 0) - (caseById.get(b.caseId)?.position ?? 0) || a.repeatIndex - b.repeatIndex)
+        .flatMap((r) => {
+        const grade = gradeOf(r.grades);
+        const caseRow = caseById.get(r.caseId);
+        if (!grade || grade.status !== "graded" || !caseRow)
+            return [];
+        const label = labelByKey.get(resultKey(r.caseId, r.repeatIndex));
+        return [
+            {
+                case: { ...caseRow.case, position: caseRow.position },
+                result: resultToWire(r, undefined),
+                judge: { passed: grade.passed === true, score: grade.score, reason: grade.reason },
+                label: label ? { label: label.label, note: label.note } : null,
+            },
+        ];
+    });
+    const count = (rows) => ({
+        pass: rows.filter((l) => l.label === "pass").length,
+        fail: rows.filter((l) => l.label === "fail").length,
+    });
+    return {
+        grader_id: graderId,
+        items,
+        labeled_here: count(labels.filter((l) => l.graderId === graderId)),
+        labeled_eval: count(evalLabels),
+    };
+}
+/**
+ * The label counts across the eval's runs, and which results of this run the judge graded that still have no label
+ * from you. No conversations: this is for a progress line and the Cases filter.
+ */
+export async function buildLabelSummary(store, id, graderId) {
+    const row = await runByIdOr404(store, id);
+    const [labels, runLabels, graded] = await Promise.all([
+        store.listEvalGraderLabelsForEval(row.projectRoot, row.evalId, graderId),
+        store.listEvalGraderLabels(row.evalRunId),
+        store.listGradedResultKeys(row.evalRunId, graderId),
+    ]);
+    const labeled = new Set(runLabels.filter((l) => l.graderId === graderId).map((l) => resultKey(l.caseId, l.repeatIndex)));
+    const unlabeled = graded
+        .filter((r) => !labeled.has(resultKey(r.caseId, r.repeatIndex)))
+        .map((r) => ({ case_id: r.caseId, repeat_index: r.repeatIndex }));
+    return {
+        grader_id: graderId,
+        labeled_eval: {
+            pass: labels.filter((l) => l.label === "pass").length,
+            fail: labels.filter((l) => l.label === "fail").length,
+        },
+        unlabeled,
+    };
+}

package/dist/evals/stats.d.ts

@@ -93,3 +93,43 @@ export declare function clusteredCi(center: number, perCase: number[], lo: numbe
 export declare function buildCaseStats(results: ResultRecord[]): CaseStat[];
 export declare function computeByGrader(graders: GraderRef[], grades: GradeRecord[], previousRun: PreviousRunLite | null | undefined): GraderStat[];
 export declare function computeStats(input: ComputeStatsInput): Stats;
+export declare const CALIBRATION_MIN_LABELS = 20;
+export declare const CALIBRATION_MINORITY_SHARE = 0.2;
+export interface CalibrationSample {
+    caseId: string;
+    repeatIndex: number;
+    label: boolean;
+    judge: boolean;
+}
+export interface CalibrationStats {
+    n: number;
+    agreement: number | null;
+    cohen_kappa: number | null;
+    tpr: number | null;
+    tnr: number | null;
+    confusion: {
+        tp: number;
+        fp: number;
+        tn: number;
+        fn: number;
+    };
+    warnings: Array<"low_n" | "imbalanced" | "no_variance">;
+}
+/** Judge pass is the positive class; the human label is the truth. */
+export declare function computeCalibration(samples: CalibrationSample[]): CalibrationStats;
+/** A strict majority of valid votes passes; a tie fails. */
+export declare function majorityPasses(passes: number, valid: number): boolean;
+export declare function meanOf(values: readonly number[]): number;
+/** Deterministic 70/30 split by a stable hash of the case id, so every repeat of a case lands on the same side. */
+export declare function calibrationSplitOf(caseId: string): "dev" | "held_out";
+export interface StabilityStats {
+    /** Judge calls per output. */
+    k: number;
+    /** Outputs with at least two valid votes. */
+    n: number;
+    /** Outputs whose valid votes were not all the same verdict. */
+    unstable: number;
+    flip_rate: number | null;
+}
+/** How often the judge changes its mind on the same output: the share of outputs with a split vote. */
+export declare function computeStability(voteSets: boolean[][], k: number): StabilityStats;

package/dist/evals/stats.js

@@ -218,3 +218,72 @@ export function computeStats(input) {
         ...(hasAgentData ? { mean_turns: mean(turnValues), mean_tool_calls: mean(toolCallValues) } : {}),
     };
 }
+export const CALIBRATION_MIN_LABELS = 20;
+export const CALIBRATION_MINORITY_SHARE = 0.2;
+/** Judge pass is the positive class; the human label is the truth. */
+export function computeCalibration(samples) {
+    let tp = 0;
+    let fp = 0;
+    let tn = 0;
+    let fn = 0;
+    for (const s of samples) {
+        if (s.judge && s.label)
+            tp++;
+        else if (s.judge && !s.label)
+            fp++;
+        else if (!s.judge && !s.label)
+            tn++;
+        else
+            fn++;
+    }
+    const n = tp + fp + tn + fn;
+    const warnings = [];
+    if (n < CALIBRATION_MIN_LABELS)
+        warnings.push("low_n");
+    const labelPass = tp + fn;
+    const labelFail = fp + tn;
+    if (n > 0 && Math.min(labelPass, labelFail) / n < CALIBRATION_MINORITY_SHARE)
+        warnings.push("imbalanced");
+    const agreement = n > 0 ? (tp + tn) / n : null;
+    const judgePass = tp + fp;
+    const judgeFail = fn + tn;
+    const expected = n > 0 ? (judgePass * labelPass + judgeFail * labelFail) / (n * n) : 0;
+    let kappa = null;
+    if (n > 0 && agreement !== null) {
+        if (expected >= 1)
+            warnings.push("no_variance");
+        else
+            kappa = (agreement - expected) / (1 - expected);
+    }
+    return {
+        n,
+        agreement,
+        cohen_kappa: kappa,
+        tpr: labelPass > 0 ? tp / labelPass : null,
+        tnr: labelFail > 0 ? tn / labelFail : null,
+        confusion: { tp, fp, tn, fn },
+        warnings,
+    };
+}
+/** A strict majority of valid votes passes; a tie fails. */
+export function majorityPasses(passes, valid) {
+    return passes * 2 > valid;
+}
+export function meanOf(values) {
+    return values.reduce((total, v) => total + v, 0) / values.length;
+}
+/** Deterministic 70/30 split by a stable hash of the case id, so every repeat of a case lands on the same side. */
+export function calibrationSplitOf(caseId) {
+    let h = 2166136261;
+    for (let i = 0; i < caseId.length; i++) {
+        h ^= caseId.charCodeAt(i);
+        h = Math.imul(h, 16777619) >>> 0;
+    }
+    return h % 10 < 7 ? "dev" : "held_out";
+}
+/** How often the judge changes its mind on the same output: the share of outputs with a split vote. */
+export function computeStability(voteSets, k) {
+    const counted = voteSets.filter((votes) => votes.length >= 2);
+    const unstable = counted.filter((votes) => votes.some((v) => v !== votes[0])).length;
+    return { k, n: counted.length, unstable, flip_rate: counted.length > 0 ? unstable / counted.length : null };
+}

package/dist/evals/storeTypes.d.ts

@@ -205,3 +205,45 @@ export type EvalGraderVersionInput = {
     graderId: string;
     graderHash: string;
 };
+export type EvalGraderLabelRow = {
+    evalRunId: string;
+    caseId: string;
+    repeatIndex: number;
+    graderId: string;
+    label: EvalReviewLabel;
+    note: string | null;
+    createdAt: string;
+    updatedAt: string;
+};
+export type EvalGraderLabelPatch = {
+    label: EvalReviewLabel | null;
+    note?: string | null;
+};
+export type EvalCalibrationSplit = "all" | "dev" | "held_out";
+export type EvalCalibrationDisagreement = {
+    /** The run the labeled result came from; absent on calibrations stored before this field existed. */
+    evalRunId?: string;
+    caseId: string;
+    repeatIndex: number;
+    label: EvalReviewLabel;
+    judge: EvalReviewLabel;
+    reason: string | null;
+};
+export type EvalCalibrationInput = {
+    projectRoot: string;
+    evalId: string;
+    graderId: string;
+    graderHash: string;
+    judge: string;
+    promptVersion: number;
+    split: EvalCalibrationSplit;
+    candidate: boolean;
+    stats: EvalJson;
+    disagreements: EvalCalibrationDisagreement[];
+    costUsd: number;
+};
+export type EvalCalibrationRow = Omit<EvalCalibrationInput, "stats"> & {
+    id: string;
+    stats: EvalJson;
+    createdAt: string;
+};

package/dist/evals/types.d.ts

@@ -57,9 +57,18 @@ export type EvalTask = {
     context?: string;
     constraints?: string;
 };
+export type JudgeSettings = {
+    /** Sampling temperature for the judge call; unset leaves the provider default. */
+    temperature?: number;
+    /** The judge writes its reason before the verdict. */
+    reasoning?: boolean;
+    /** Judge calls per grade (odd, 1 to 5); the majority verdict wins. */
+    votes?: number;
+};
 export type EvalDefaults = {
     repeats?: number;
     judge?: string;
+    judge_settings?: JudgeSettings;
     pass_threshold?: number;
     max_cost_usd?: number;
     concurrency?: number;
@@ -103,6 +112,7 @@ export type GraderConfig = (GraderCommon & {
     scale?: "pass_fail" | "1_to_5";
     pass_at?: number;
     judge?: string;
+    judge_settings?: JudgeSettings;
     examples?: RubricExample[];
 }) | (GraderCommon & {
     type: "code";
@@ -122,6 +132,7 @@ export type GraderConfig = (GraderCommon & {
 }) | (GraderCommon & {
     type: "goal_reached";
     judge?: string;
+    judge_settings?: JudgeSettings;
     guidance?: string;
 });
 export type EvalFile = {
@@ -178,6 +189,7 @@ export type ResolvedGrader = (ResolvedGraderCommon & {
     scale: "pass_fail" | "1_to_5";
     passAt: number;
     judge: string;
+    judgeSettings?: JudgeSettings;
     examples: RubricExample[];
 }) | (ResolvedGraderCommon & {
     type: "code";
@@ -197,6 +209,7 @@ export type ResolvedGrader = (ResolvedGraderCommon & {
 }) | (ResolvedGraderCommon & {
     type: "goal_reached";
     judge: string;
+    judgeSettings?: JudgeSettings;
     guidance?: string;
 });
 export type PlanCase = {

package/dist/evals/votes.d.ts

@@ -0,0 +1,10 @@
+import type { NormalizedGrade } from "./types.js";
+export declare const JUDGE_ERROR_CODE = "EVAL_JUDGE_ERROR";
+/**
+ * One grade from several votes: the majority verdict for pass/fail (a tie fails), the mean raw score for 1 to 5.
+ * Errored votes are left out; if every vote errored the grade is an error. Tokens and cost add up over all votes.
+ */
+export declare function combineVotes(grader: {
+    scale: "pass_fail" | "1_to_5";
+    passAt: number;
+}, votes: readonly NormalizedGrade[]): NormalizedGrade;

package/dist/evals/votes.js

@@ -0,0 +1,60 @@
+import { normalizeOneToFive } from "./scoring.js";
+import { majorityPasses, meanOf } from "./stats.js";
+export const JUDGE_ERROR_CODE = "EVAL_JUDGE_ERROR";
+/**
+ * One grade from several votes: the majority verdict for pass/fail (a tie fails), the mean raw score for 1 to 5.
+ * Errored votes are left out; if every vote errored the grade is an error. Tokens and cost add up over all votes.
+ */
+export function combineVotes(grader, votes) {
+    const first = votes[0];
+    if (votes.length === 1)
+        return first;
+    const valid = votes.filter((g) => g.status === "graded" && g.passed !== undefined);
+    const sum = (pick) => votes.reduce((total, g) => total + (pick(g) ?? 0), 0);
+    const totals = {
+        judgeTokensIn: sum((g) => g.judgeTokensIn),
+        judgeTokensOut: sum((g) => g.judgeTokensOut),
+        judgeCostUsd: sum((g) => g.judgeCostUsd),
+    };
+    const priceUnknown = votes.some((g) => g.details?.judgeCostUnknown);
+    const record = votes.map((g) => ({
+        status: g.status,
+        ...(g.passed !== undefined ? { passed: g.passed } : {}),
+        ...(g.rawValue !== undefined ? { raw_value: g.rawValue } : {}),
+        ...(g.reason !== undefined ? { reason: g.reason } : {}),
+    }));
+    const baseDetails = {
+        ...(first.details ?? {}),
+        ...(priceUnknown ? { judgeCostUnknown: true } : {}),
+        votes: record,
+    };
+    if (valid.length === 0) {
+        return { ...first, ...totals, status: "error", errorCode: first.errorCode ?? JUDGE_ERROR_CODE, details: baseDetails };
+    }
+    let passed;
+    let rawValue;
+    let score;
+    if (grader.scale === "1_to_5") {
+        rawValue = meanOf(valid.map((g) => g.rawValue ?? 0));
+        score = normalizeOneToFive(rawValue);
+        passed = rawValue >= grader.passAt;
+    }
+    else {
+        passed = majorityPasses(valid.filter((g) => g.passed === true).length, valid.length);
+        rawValue = passed ? 1 : 0;
+        score = rawValue;
+    }
+    const lead = valid.find((g) => g.passed === passed) ?? valid[0];
+    return {
+        graderId: first.graderId,
+        graderType: first.graderType,
+        status: "graded",
+        passed,
+        score,
+        rawValue,
+        ...(lead.reason !== undefined ? { reason: lead.reason } : {}),
+        ...(first.judgeModel !== undefined ? { judgeModel: first.judgeModel } : {}),
+        ...totals,
+        details: baseDetails,
+    };
+}