stageflow 0.44.0 → 0.46.0

package/dist/agents/evalGate.js

@@ -9,6 +9,7 @@ const ALWAYS_BLOCKED = new Set([
     "stageflow.agent_chat",
     "stageflow.fire_trigger",
     "stageflow.sort_eval_failures",
+    "stageflow.calibrate_eval_grader",
     "stageflow.cancel_eval_run",
 ]);
 const BLOCKED_UNLESS_REAL = new Set(["stageflow.probe_project_mcp"]);

package/dist/cli/evalCommand.d.ts

@@ -1,6 +1,6 @@
 import { type RunNpm } from "../evals/engine/promptfoo/install.js";
 import { type EnsureGlobalServiceResult } from "../server/ensureGlobalService.js";
-export declare const EVAL_USAGE = "Usage:\n  sf eval list [--json]\n  sf eval show <id|path> [--json]\n  sf eval validate [<id|path>] [--json] [--strict]\n  sf eval run <id|path> [--repeats <n>] [--judge <model>] [--model <id>] [--user-model <model>] [--allow-real-tools] [--max-cost <usd>] [--case <id> ...] [--tag <tag> ...] [--label <text>] [--no-wait] [--json]\n  sf eval runs [<evalId>] [--json]\n  sf eval runs show <evalRunId> [--json]\n  sf eval runs cancel <evalRunId>\n  sf eval runs delete <evalRunId>\n  sf eval sort <evalRunId> [--classifier auto|jev|judge] [--limit <n>] [--resort] [--json]\n  sf eval suggestions <evalRunId> [--status pending|accepted|dismissed] [--json]\n  sf eval compare <evalRunIdA> <evalRunIdB> [--json]\n  sf eval datasets [--json]\n  sf eval engine status [--json]\n  sf eval engine install\n\n  Agent evals: --model overrides the agent's model, --user-model the model that plays the user. An eval whose\n  conversation.tools is real also needs --allow-real-tools.\n\n  Every subcommand also accepts --project-root <id-or-path>.";
+export declare const EVAL_USAGE = "Usage:\n  sf eval list [--json]\n  sf eval show <id|path> [--json]\n  sf eval validate [<id|path>] [--json] [--strict]\n  sf eval run <id|path> [--repeats <n>] [--judge <model>] [--model <id>] [--user-model <model>] [--allow-real-tools] [--max-cost <usd>] [--case <id> ...] [--tag <tag> ...] [--label <text>] [--no-wait] [--json]\n  sf eval runs [<evalId>] [--json]\n  sf eval runs show <evalRunId> [--json]\n  sf eval runs cancel <evalRunId>\n  sf eval runs delete <evalRunId>\n  sf eval sort <evalRunId> [--classifier auto|jev|judge] [--limit <n>] [--resort] [--json]\n  sf eval calibrate <evalId> <graderId> [--split all|dev|held_out] [--max-cost <usd>] [--stability] [--json]\n  sf eval suggestions <evalRunId> [--status pending|accepted|dismissed] [--json]\n  sf eval compare <evalRunIdA> <evalRunIdB> [--json]\n  sf eval datasets [--json]\n  sf eval engine status [--json]\n  sf eval engine install\n\n  Agent evals: --model overrides the agent's model, --user-model the model that plays the user. An eval whose\n  conversation.tools is real also needs --allow-real-tools.\n\n  Every subcommand also accepts --project-root <id-or-path>.";
 export type EvalCommandIo = {
     log: (line: string) => void;
     error: (line: string) => void;

package/dist/cli/evalCommand.js

@@ -9,7 +9,7 @@ import { installPromptfoo } from "../evals/engine/promptfoo/install.js";
 import { engineStatus } from "../evals/engine/promptfoo/status.js";
 import { globalStageflowHome } from "../project/globalHome.js";
 import { ensureGlobalService, hostBaseUrl, } from "../server/ensureGlobalService.js";
-import { httpCancelEvalRun, httpCompareEvalRuns, httpDeleteEvalRun, httpEnsureProject, httpGetEval, httpGetEvalRun, httpListDatasets, httpListEvalRuns, httpListEvalSuggestions, httpSortEvalFailures, httpListEvals, httpStartEvalRun, } from "./hostClient.js";
+import { httpCancelEvalRun, httpCompareEvalRuns, httpDeleteEvalRun, httpEnsureProject, httpGetEval, httpGetEvalRun, httpListDatasets, httpListEvalRuns, httpListEvalSuggestions, httpCalibrateEvalGrader, httpSortEvalFailures, httpListEvals, httpStartEvalRun, } from "./hostClient.js";
 import { resolveCliProjectRootLocal } from "./resolveCliProjectRoot.js";
 import { exitCodeForValidation, formatValidationHuman, formatValidationJson, } from "./validateOutput.js";
 export const EVAL_USAGE = `Usage:
@@ -22,6 +22,7 @@ export const EVAL_USAGE = `Usage:
   sf eval runs cancel <evalRunId>
   sf eval runs delete <evalRunId>
   sf eval sort <evalRunId> [--classifier auto|jev|judge] [--limit <n>] [--resort] [--json]
+  sf eval calibrate <evalId> <graderId> [--split all|dev|held_out] [--max-cost <usd>] [--stability] [--json]
   sf eval suggestions <evalRunId> [--status pending|accepted|dismissed] [--json]
   sf eval compare <evalRunIdA> <evalRunIdB> [--json]
   sf eval datasets [--json]
@@ -61,6 +62,7 @@ const FLAGS = {
     runs: {},
     sort: { "--classifier": { value: true }, "--limit": { value: true }, "--resort": {} },
     suggestions: { "--status": { value: true } },
+    calibrate: { "--split": { value: true }, "--max-cost": { value: true }, "--stability": {} },
     compare: {},
     datasets: {},
     engine: {},
@@ -76,6 +78,7 @@ function parseArgs(command, rest) {
         noWait: false,
         allowRealTools: false,
         resort: false,
+        stability: false,
         cases: [],
         tags: [],
         positionals: [],
@@ -150,6 +153,15 @@ function parseArgs(command, rest) {
             case "--resort":
                 parsed.resort = true;
                 break;
+            case "--stability":
+                parsed.stability = true;
+                break;
+            case "--split":
+                if (value !== "all" && value !== "dev" && value !== "held_out") {
+                    throw new UsageError("--split must be all, dev or held_out");
+                }
+                parsed.split = value;
+                break;
             case "--status":
                 if (value !== "pending" && value !== "accepted" && value !== "dismissed") {
                     throw new UsageError("--status must be pending, accepted or dismissed");
@@ -404,7 +416,9 @@ export async function runEvalCommand(args, options = {}) {
                     ? expectArgs(2, 2, "<evalRunIdA> <evalRunIdB>")
                     : command === "sort" || command === "suggestions"
                         ? expectArgs(1, 1, "<evalRunId>")
-                        : null;
+                        : command === "calibrate"
+                            ? expectArgs(2, 2, "<evalId> <graderId>")
+                            : null;
     if (argError)
         return usageError(argError);
     let runsSub;
@@ -554,6 +568,29 @@ export async function runEvalCommand(args, options = {}) {
                 out.log(`Review and accept in the console, or POST /api/eval-runs/${positionals[0]}/failures/suggestions/accept`);
             });
         }
+        case "calibrate": {
+            const result = await httpCalibrateEvalGrader(base, positionals[0], positionals[1], {
+                project_root: wireRoot,
+                ...(parsed.split !== undefined ? { split: parsed.split } : {}),
+                ...(parsed.maxCost !== undefined ? { max_cost_usd: parsed.maxCost } : {}),
+                ...(parsed.stability ? { stability: true } : {}),
+            });
+            return show(result, (b) => {
+                const st = b.stats;
+                const num = (v) => (typeof v === "number" ? v.toFixed(2) : "n/a");
+                out.log(`${b.grader_id} (${b.judge}): ${st.n} labels, agreement ${num(st.agreement)}, kappa ${num(st.cohen_kappa)}, pass recall ${num(st.tpr)}, fail recall ${num(st.tnr)}; cost ${usd(b.cost_usd)}`);
+                const cm = st.confusion;
+                out.log(`judge pass / human pass ${cm.tp}\tjudge pass / human fail ${cm.fp}\tjudge fail / human fail ${cm.tn}\tjudge fail / human pass ${cm.fn}`);
+                if (st.stability) {
+                    out.log(`stability over ${st.stability.k} judge calls per output: ${st.stability.unstable} of ${st.stability.n} outputs split (flip rate ${num(st.stability.flip_rate)})`);
+                }
+                for (const w of st.warnings)
+                    out.log(`warning: ${w}`);
+                for (const d of b.disagreements) {
+                    out.log(`${d.case_id}#${d.repeat_index}\thuman ${d.label}\tjudge ${d.judge}\t${d.reason ?? ""}`);
+                }
+            });
+        }
         case "suggestions":
             return show(await httpListEvalSuggestions(base, positionals[0], parsed.status), (b) => {
                 for (const s of b.items)

package/dist/cli/hostClient.d.ts

@@ -119,6 +119,12 @@ export declare function httpSortEvalFailures(base: string, evalRunId: string, in
     only_unsorted?: boolean;
     limit?: number;
 }): Promise<HttpEvalResult<Record<string, any>>>;
+export declare function httpCalibrateEvalGrader(base: string, evalId: string, graderId: string, input: {
+    project_root?: string;
+    split?: "all" | "dev" | "held_out";
+    max_cost_usd?: number;
+    stability?: boolean;
+}): Promise<HttpEvalResult<Record<string, any>>>;
 export declare function httpListEvalSuggestions(base: string, evalRunId: string, status?: string): Promise<HttpEvalResult<{
     items: Array<Record<string, any>>;
     summary: Record<string, any>;

package/dist/cli/hostClient.js

@@ -342,6 +342,10 @@ export async function httpSortEvalFailures(base, evalRunId, input) {
     const { status, body } = await postJson(base, `/api/eval-runs/${enc(evalRunId)}/failures/sort`, input);
     return status === 200 ? { ok: true, body: body } : evalFailure(status, body);
 }
+export async function httpCalibrateEvalGrader(base, evalId, graderId, input) {
+    const { status, body } = await postJson(base, `/api/evals/${enc(evalId)}/graders/${enc(graderId)}/calibrate`, input);
+    return status === 200 ? { ok: true, body: body } : evalFailure(status, body);
+}
 export function httpListEvalSuggestions(base, evalRunId, status) {
     return evalGet(base, `/api/eval-runs/${enc(evalRunId)}/failures/suggestions${evalQuery({ status })}`);
 }

package/dist/config/loadEval.d.ts

@@ -1,4 +1,4 @@
-import type { EvalConversation, EvalFile, ResolvedConversation, ResolvedGrader } from "../evals/types.js";
+import type { EvalConversation, JudgeSettings, EvalFile, ResolvedConversation, ResolvedGrader } from "../evals/types.js";
 import { type LoadOutcome } from "./loadOutcome.js";
 type UnhashedGrader = ResolvedGrader extends infer G ? (G extends unknown ? Omit<G, "hash"> : never) : never;
 export type LoadedEval = {
@@ -25,6 +25,8 @@ export type EvalPaths = {
 };
 /** Paths in an eval file resolve against the eval file's directory (P6). */
 export declare function resolveEvalPaths(file: EvalFile, evalAbsPath: string): EvalPaths;
+/** Grader settings over the eval defaults, key by key; undefined when neither sets anything. */
+export declare function mergeJudgeSettings(base: JudgeSettings | undefined, over: JudgeSettings | undefined): JudgeSettings | undefined;
 export declare function graderHash(grader: UnhashedGrader): string;
 export declare function graderHashShort(grader: Pick<ResolvedGrader, "hash">): string;
 export declare function parseEvalOutcome(yamlText: string, filePath: string, projectRoot: string): LoadOutcome<LoadedEval>;

package/dist/config/loadEval.js

@@ -3,6 +3,7 @@ import path from "node:path";
 import { parse as parseYaml } from "yaml";
 import { AGENT_ID_PATTERN, isReservedAgentId } from "../agents/agentId.js";
 import { canonicalJson, hash6, sha256Hex } from "../evals/canonical.js";
+import { EVAL_LIMITS } from "../evals/constants.js";
 import { loadFailure, loadSuccess } from "./loadOutcome.js";
 import { yamlParsePosition } from "./yamlParsePosition.js";
 export const EVAL_DEFAULT_REPEATS = 3;
@@ -89,6 +90,7 @@ const TASK_KEYS = ["goal", "context", "constraints"];
 const DEFAULTS_KEYS = [
     "repeats",
     "judge",
+    "judge_settings",
     "pass_threshold",
     "max_cost_usd",
     "concurrency",
@@ -101,11 +103,11 @@ const GRADER_TYPE_KEYS = {
     schema: [],
     field: ["checks"],
     reference: ["mode", "path", "expected", "pass_at"],
-    rubric: ["rubric", "scale", "pass_at", "judge", "examples"],
+    rubric: ["rubric", "scale", "pass_at", "judge", "judge_settings", "examples"],
     code: ["code"],
     tool_calls: ["must_call", "must_not_call", "must_ask_first", "max_calls", "args"],
     conversation_check: ["max_turns", "max_agent_questions_per_turn", "final_reply"],
-    goal_reached: ["judge", "guidance"],
+    goal_reached: ["judge", "judge_settings", "guidance"],
 };
 const TOOL_ARG_KEYS = ["tool", "path", "op", "value"];
 const FINAL_REPLY_KEYS = ["op", "value"];
@@ -285,6 +287,51 @@ function parseTask(c, raw) {
     }
     return out;
 }
+const JUDGE_SETTINGS_KEYS = ["temperature", "reasoning", "votes"];
+/** A mapping of judge call settings, or undefined when absent or invalid (findings added). */
+function parseJudgeSettings(c, raw, label) {
+    if (raw === undefined)
+        return undefined;
+    if (!isPlainObject(raw)) {
+        c.add("eval.invalid_field", `${label} "judge_settings" must be a mapping`);
+        return undefined;
+    }
+    c.unknownKeys(raw, JUDGE_SETTINGS_KEYS, `${label} judge_settings`);
+    const out = {};
+    let ok = true;
+    if (raw.temperature !== undefined) {
+        if (typeof raw.temperature === "number" && raw.temperature >= 0 && raw.temperature <= 1) {
+            out.temperature = raw.temperature;
+        }
+        else {
+            c.add("eval.invalid_field", `${label} judge_settings "temperature" must be a number from 0 to 1`);
+            ok = false;
+        }
+    }
+    if (raw.reasoning !== undefined) {
+        if (typeof raw.reasoning === "boolean")
+            out.reasoning = raw.reasoning;
+        else {
+            c.add("eval.invalid_field", `${label} judge_settings "reasoning" must be true or false`);
+            ok = false;
+        }
+    }
+    if (raw.votes !== undefined) {
+        if (typeof raw.votes === "number" && Number.isInteger(raw.votes) && raw.votes >= 1 && raw.votes <= 5 && raw.votes % 2 === 1) {
+            out.votes = raw.votes;
+        }
+        else {
+            c.add("eval.invalid_field", `${label} judge_settings "votes" must be 1, 3 or 5`);
+            ok = false;
+        }
+    }
+    return ok ? out : undefined;
+}
+/** Grader settings over the eval defaults, key by key; undefined when neither sets anything. */
+export function mergeJudgeSettings(base, over) {
+    const merged = { ...(base ?? {}), ...(over ?? {}) };
+    return Object.keys(merged).length === 0 ? undefined : merged;
+}
 function parseDefaults(c, raw, kind) {
     if (raw === undefined)
         return undefined;
@@ -300,6 +347,9 @@ function parseDefaults(c, raw, kind) {
     const judge = optionalString(c, raw, "judge", "defaults", { min: 1 });
     if (judge !== undefined)
         out.judge = judge;
+    const judgeSettings = parseJudgeSettings(c, raw.judge_settings, "defaults");
+    if (judgeSettings !== undefined)
+        out.judge_settings = judgeSettings;
     const passThreshold = optionalNumber(c, raw, "pass_threshold", "defaults", "a number greater than 0 and at most 1", (n) => n > 0 && n <= 1);
     if (passThreshold !== undefined)
         out.pass_threshold = passThreshold;
@@ -483,8 +533,8 @@ function parseChecks(c, raw, where) {
     return ok ? checks : undefined;
 }
 function parseExamples(c, raw, scale, where) {
-    if (!Array.isArray(raw) || raw.length > 6) {
-        c.add("eval.grader_invalid", `${where} "examples" must be an array of at most 6 items`);
+    if (!Array.isArray(raw) || raw.length > EVAL_LIMITS.rubricExamplesMax) {
+        c.add("eval.grader_invalid", `${where} "examples" must be an array of at most ${EVAL_LIMITS.rubricExamplesMax} items`);
         return undefined;
     }
     const out = [];
@@ -610,7 +660,7 @@ function parseFinalReply(c, raw, where) {
     });
     return ok ? out : undefined;
 }
-function parseGrader(c, raw, index, defaultJudge, kind) {
+function parseGrader(c, raw, index, defaultJudge, kind, defaultJudgeSettings) {
     if (!isPlainObject(raw)) {
         c.add("eval.grader_invalid", `graders[${index}] must be a mapping`);
         return undefined;
@@ -769,6 +819,10 @@ function parseGrader(c, raw, index, defaultJudge, kind) {
                 if (examples === undefined)
                     ok = false;
             }
+            const ownSettings = parseJudgeSettings(c, raw.judge_settings, label);
+            if (raw.judge_settings !== undefined && ownSettings === undefined)
+                ok = false;
+            const judgeSettings = mergeJudgeSettings(defaultJudgeSettings, ownSettings);
             if (ok) {
                 result = {
                     config: {
@@ -778,6 +832,7 @@ function parseGrader(c, raw, index, defaultJudge, kind) {
                         ...(raw.scale !== undefined ? { scale } : {}),
                         ...(passAt !== undefined ? { pass_at: passAt } : {}),
                         ...(judge !== undefined ? { judge } : {}),
+                        ...(ownSettings !== undefined ? { judge_settings: ownSettings } : {}),
                         ...(examples !== undefined ? { examples } : {}),
                     },
                     resolved: {
@@ -787,6 +842,7 @@ function parseGrader(c, raw, index, defaultJudge, kind) {
                         scale,
                         passAt: scale === "1_to_5" ? (passAt ?? 4) : 1,
                         judge: judge ?? defaultJudge ?? "",
+                        ...(judgeSettings !== undefined ? { judgeSettings } : {}),
                         examples: examples ?? [],
                     },
                 };
@@ -882,18 +938,24 @@ function parseGrader(c, raw, index, defaultJudge, kind) {
                     ok = false;
                 }
             }
+            const goalSettings = parseJudgeSettings(c, raw.judge_settings, label);
+            if (raw.judge_settings !== undefined && goalSettings === undefined)
+                ok = false;
+            const goalMerged = mergeJudgeSettings(defaultJudgeSettings, goalSettings);
             if (ok) {
                 result = {
                     config: {
                         ...common,
                         type: "goal_reached",
                         ...(judge !== undefined ? { judge } : {}),
+                        ...(goalSettings !== undefined ? { judge_settings: goalSettings } : {}),
                         ...(guidance !== undefined ? { guidance } : {}),
                     },
                     resolved: {
                         ...resolvedCommon,
                         type: "goal_reached",
                         judge: judge ?? defaultJudge ?? "",
+                        ...(goalMerged !== undefined ? { judgeSettings: goalMerged } : {}),
                         ...(guidance !== undefined ? { guidance } : {}),
                     },
                 };
@@ -932,6 +994,7 @@ function graderHashObject(grader) {
                 scale: grader.scale,
                 ...(grader.scale === "1_to_5" ? { pass_at: grader.passAt } : {}),
                 judge: grader.judge,
+                ...(grader.judgeSettings !== undefined ? { judge_settings: grader.judgeSettings } : {}),
                 examples: grader.examples,
             };
         case "code":
@@ -953,7 +1016,12 @@ function graderHashObject(grader) {
                 final_reply: grader.finalReply,
             };
         case "goal_reached":
-            return { ...base, judge: grader.judge, guidance: grader.guidance };
+            return {
+                ...base,
+                judge: grader.judge,
+                ...(grader.judgeSettings !== undefined ? { judge_settings: grader.judgeSettings } : {}),
+                guidance: grader.guidance,
+            };
     }
 }
 export function graderHash(grader) {
@@ -1047,7 +1115,7 @@ export function parseEvalOutcome(yamlText, filePath, projectRoot) {
     else {
         const seen = new Set();
         raw.graders.forEach((rawGrader, index) => {
-            const parsed = parseGrader(c, rawGrader, index, defaults?.judge, target?.kind);
+            const parsed = parseGrader(c, rawGrader, index, defaults?.judge, target?.kind, defaults?.judge_settings);
             if (isPlainObject(rawGrader) && typeof rawGrader.id === "string") {
                 if (seen.has(rawGrader.id)) {
                     c.add("eval.grader_duplicate_id", `Duplicate grader id "${rawGrader.id}"`);

package/dist/config/validateEval.js

@@ -2,7 +2,7 @@ import { internalAgentsVisible } from "../operatorAgent/internalAgents.js";
 import { PROFILE_GRANTS } from "../toolkit/profiles.js";
 import { realpath, stat } from "node:fs/promises";
 import path from "node:path";
-import { mapJudge } from "../evals/judgeProviders.js";
+import { isFloatingModelId, mapJudge } from "../evals/judgeProviders.js";
 import { classifyTool } from "../agents/toolRisk.js";
 import { checkDatasetAgainstAgent } from "./datasetAgentCheck.js";
 import { checkDatasetAgainstStage } from "./datasetStageCheck.js";
@@ -106,6 +106,7 @@ export async function validateEval(loaded, ctx) {
     }
     const stageModel = file.defaults?.model ?? resolved?.modelResolved;
     const unknownChecked = new Set();
+    const unpinnedWarned = new Set();
     for (const grader of loaded.graders) {
         if (grader.type !== "rubric")
             continue;
@@ -122,6 +123,13 @@ export async function validateEval(loaded, ctx) {
         if (stageModel !== undefined && grader.judge === stageModel) {
             add("warning", "eval.judge_same_as_stage", `Rubric grader "${grader.id}" uses judge "${grader.judge}", the same model as the stage (self-preference bias)`);
         }
+        if (isFloatingModelId(grader.judge) && !unpinnedWarned.has(grader.judge)) {
+            unpinnedWarned.add(grader.judge);
+            add("warning", "eval.judge_unpinned", `Judge model "${grader.judge}" is a floating id; its behavior can change under stored runs. Pin a dated version`);
+        }
+        if ((grader.judgeSettings?.votes ?? 1) > 1 && grader.judgeSettings?.temperature === 0) {
+            add("warning", "eval.judge_votes_no_variation", `Grader "${grader.id}" asks for ${grader.judgeSettings?.votes} votes at temperature 0, so every vote is the same call and the cost is multiplied for nothing`);
+        }
     }
     const paths = resolveEvalPaths(file, loaded.path);
     const datasetProblem = ctx.datasetInline
@@ -267,6 +275,7 @@ async function validateAgentEval(loaded, ctx, findings) {
     }
     const agentModel = file.defaults?.model ?? resolved?.modelResolved;
     const unknownChecked = new Set();
+    const unpinnedWarned = new Set();
     for (const grader of loaded.graders) {
         if (grader.type !== "rubric" && grader.type !== "goal_reached")
             continue;
@@ -283,6 +292,13 @@ async function validateAgentEval(loaded, ctx, findings) {
         if (agentModel !== undefined && grader.judge === agentModel) {
             add("warning", "eval.judge_same_as_stage", `Grader "${grader.id}" uses judge "${grader.judge}", the same model as the agent (self-preference bias)`);
         }
+        if (isFloatingModelId(grader.judge) && !unpinnedWarned.has(grader.judge)) {
+            unpinnedWarned.add(grader.judge);
+            add("warning", "eval.judge_unpinned", `Judge model "${grader.judge}" is a floating id; its behavior can change under stored runs. Pin a dated version`);
+        }
+        if ((grader.judgeSettings?.votes ?? 1) > 1 && grader.judgeSettings?.temperature === 0) {
+            add("warning", "eval.judge_votes_no_variation", `Grader "${grader.id}" asks for ${grader.judgeSettings?.votes} votes at temperature 0, so every vote is the same call and the cost is multiplied for nothing`);
+        }
     }
     if (resolved) {
         const grantWarn = (names, where) => {

package/dist/evals/calibration.d.ts

@@ -0,0 +1,21 @@
+import type { RunStore } from "../runstore/port.js";
+import type { EvalCalibrationSplit, EvalGraderLabelRow } from "./storeTypes.js";
+import type { CaseOutput } from "./types.js";
+export type CalibrationSampleSource = {
+    caseId: string;
+    /** Position within the case's samples; the replay repeat index. */
+    repeatIndex: number;
+    label: "pass" | "fail";
+    output: CaseOutput;
+    originalRunId: string;
+    originalRepeatIndex: number;
+};
+export type CalibrationStore = Pick<RunStore, "listEvalGraderLabelsForEval" | "listEvalResults">;
+/** Rebuilds the in-memory case output from the stored result JSON. */
+export declare function replayOutput(stored: unknown): CaseOutput | null;
+/**
+ * The labeled samples for one grader across every run of the eval: the newest label per identical
+ * (case, output), grouped per case with a fresh repeat index so the engine can replay them.
+ */
+export declare function collectCalibrationSamples(store: CalibrationStore, projectRoot: string, evalId: string, graderId: string, split: EvalCalibrationSplit): Promise<CalibrationSampleSource[]>;
+export type { EvalGraderLabelRow };

package/dist/evals/calibration.js

@@ -0,0 +1,69 @@
+import { createHash } from "node:crypto";
+import { calibrationSplitOf } from "./stats.js";
+import { renderTranscript } from "./graders/transcript.js";
+function outputKey(caseId, output) {
+    return createHash("sha256")
+        .update(caseId)
+        .update("\u0000")
+        .update(JSON.stringify(output ?? null))
+        .digest("hex");
+}
+/** Rebuilds the in-memory case output from the stored result JSON. */
+export function replayOutput(stored) {
+    if (stored === null || typeof stored !== "object")
+        return null;
+    const o = stored;
+    if (o.kind === "conversation") {
+        const out = { ...o };
+        out.transcript = renderTranscript(out);
+        return out;
+    }
+    if (typeof o.status !== "string" || typeof o.summary !== "string")
+        return null;
+    return {
+        kind: "envelope",
+        status: o.status === "failure" ? "failure" : "success",
+        summary: o.summary,
+        ...(o.payload !== undefined && typeof o.payload === "object" ? { payload: o.payload } : {}),
+        artifacts: Array.isArray(o.artifacts) ? o.artifacts : [],
+    };
+}
+/**
+ * The labeled samples for one grader across every run of the eval: the newest label per identical
+ * (case, output), grouped per case with a fresh repeat index so the engine can replay them.
+ */
+export async function collectCalibrationSamples(store, projectRoot, evalId, graderId, split) {
+    const labels = await store.listEvalGraderLabelsForEval(projectRoot, evalId, graderId);
+    const resultsByRun = new Map();
+    const seen = new Set();
+    const perCase = new Map();
+    const out = [];
+    for (const label of labels) {
+        if (split !== "all" && calibrationSplitOf(label.caseId) !== split)
+            continue;
+        let results = resultsByRun.get(label.evalRunId);
+        if (!results) {
+            results = new Map((await store.listEvalResults(label.evalRunId)).map((r) => [`${r.caseId}#${r.repeatIndex}`, r]));
+            resultsByRun.set(label.evalRunId, results);
+        }
+        const result = results.get(`${label.caseId}#${label.repeatIndex}`);
+        const output = result ? replayOutput(result.output) : null;
+        if (!output)
+            continue;
+        const key = outputKey(label.caseId, result?.output);
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        const repeatIndex = perCase.get(label.caseId) ?? 0;
+        perCase.set(label.caseId, repeatIndex + 1);
+        out.push({
+            caseId: label.caseId,
+            repeatIndex,
+            label: label.label,
+            output,
+            originalRunId: label.evalRunId,
+            originalRepeatIndex: label.repeatIndex,
+        });
+    }
+    return out;
+}

package/dist/evals/calibrationSuggest.d.ts

@@ -0,0 +1,29 @@
+import type { RunStore } from "../runstore/port.js";
+import type { EvalCalibrationRow } from "./storeTypes.js";
+import type { CaseOutput, RubricExample } from "./types.js";
+export declare const EXAMPLE_SUGGESTION_LIMIT = 3;
+export type SuggestedExample = {
+    eval_run_id: string;
+    case_id: string;
+    repeat_index: number;
+    /** What the judge said, so the author can see why this example would help. */
+    judge: "pass" | "fail";
+    example: RubricExample;
+};
+export type SuggestStore = Pick<RunStore, "listEvalResults" | "listEvalGraderLabelsForEval">;
+/** The text a judge example shows for one stored output. */
+export declare function exampleOutputText(output: CaseOutput): string;
+/**
+ * Up to three few-shot examples built from where the judge disagreed with the author on the dev split (a held-out row
+ * is never offered, so the held-out check stays a fair test): one of each kind of
+ * disagreement first (judge too lenient, judge too strict), distinct outputs only. The author's label note,
+ * when there is one, becomes the example's reason. Nothing is written.
+ */
+export declare function suggestExamples(store: SuggestStore, input: {
+    projectRoot: string;
+    evalId: string;
+    graderId: string;
+    scale: "pass_fail" | "1_to_5";
+    calibration: Pick<EvalCalibrationRow, "disagreements">;
+    limit?: number;
+}): Promise<SuggestedExample[]>;

package/dist/evals/calibrationSuggest.js

@@ -0,0 +1,72 @@
+import { replayOutput } from "./calibration.js";
+import { calibrationSplitOf } from "./stats.js";
+export const EXAMPLE_SUGGESTION_LIMIT = 3;
+const EXAMPLE_OUTPUT_MAX_CHARS = 1500;
+function cut(text) {
+    return text.length <= EXAMPLE_OUTPUT_MAX_CHARS ? text : `${text.slice(0, EXAMPLE_OUTPUT_MAX_CHARS)}…`;
+}
+/** The text a judge example shows for one stored output. */
+export function exampleOutputText(output) {
+    if (output.kind === "conversation")
+        return cut(output.transcript ?? output.final_reply);
+    const payload = output.payload === undefined ? "" : `\n${JSON.stringify(output.payload)}`;
+    return cut(`${output.summary}${payload}`);
+}
+function exampleFor(scale, label, output, reason) {
+    const base = { output, ...(reason !== undefined && reason !== "" ? { reason } : {}) };
+    if (scale === "pass_fail")
+        return { verdict: label, ...base };
+    return { score: label === "pass" ? 5 : 1, ...base };
+}
+/**
+ * Up to three few-shot examples built from where the judge disagreed with the author on the dev split (a held-out row
+ * is never offered, so the held-out check stays a fair test): one of each kind of
+ * disagreement first (judge too lenient, judge too strict), distinct outputs only. The author's label note,
+ * when there is one, becomes the example's reason. Nothing is written.
+ */
+export async function suggestExamples(store, input) {
+    const limit = input.limit ?? EXAMPLE_SUGGESTION_LIMIT;
+    const labels = await store.listEvalGraderLabelsForEval(input.projectRoot, input.evalId, input.graderId);
+    const notes = new Map(labels.map((l) => [`${l.evalRunId}#${l.caseId}#${l.repeatIndex}`, l.note]));
+    const dev = input.calibration.disagreements.filter((d) => calibrationSplitOf(d.caseId) === "dev");
+    const lenient = dev.filter((d) => d.judge === "pass");
+    const strict = dev.filter((d) => d.judge === "fail");
+    const ordered = [];
+    for (let i = 0; ordered.length < lenient.length + strict.length; i++) {
+        if (lenient[i])
+            ordered.push(lenient[i]);
+        if (strict[i])
+            ordered.push(strict[i]);
+    }
+    const runs = new Map();
+    const seen = new Set();
+    const out = [];
+    for (const d of ordered) {
+        if (out.length >= limit)
+            break;
+        if (d.evalRunId === undefined)
+            continue;
+        let results = runs.get(d.evalRunId);
+        if (!results) {
+            results = await store.listEvalResults(d.evalRunId);
+            runs.set(d.evalRunId, results);
+        }
+        const result = results.find((r) => r.caseId === d.caseId && r.repeatIndex === d.repeatIndex);
+        const output = result ? replayOutput(result.output) : null;
+        if (!output)
+            continue;
+        const text = exampleOutputText(output);
+        if (seen.has(text))
+            continue;
+        seen.add(text);
+        const note = notes.get(`${d.evalRunId}#${d.caseId}#${d.repeatIndex}`) ?? undefined;
+        out.push({
+            eval_run_id: d.evalRunId,
+            case_id: d.caseId,
+            repeat_index: d.repeatIndex,
+            judge: d.judge,
+            example: exampleFor(input.scale, d.label, text, note),
+        });
+    }
+    return out;
+}

package/dist/evals/calibrator.d.ts

@@ -0,0 +1,46 @@
+import { type CalibrationStore } from "./calibration.js";
+import type { JudgedGrader } from "./judged.js";
+import { type CalibrationStats, type StabilityStats } from "./stats.js";
+import type { EvalCalibrationDisagreement, EvalCalibrationRow, EvalCalibrationSplit } from "./storeTypes.js";
+import type { EvalEngine, EvalPlan, JudgeSettings, RubricExample } from "./types.js";
+import type { RunStore } from "../runstore/port.js";
+export type CalibrateInput = {
+    projectRoot: string;
+    evalPath: string;
+    graderId: string;
+    split?: EvalCalibrationSplit;
+    /** A rubric to measure instead of the saved one; the result is stored as a candidate. */
+    candidate?: {
+        rubric?: string;
+        examples?: RubricExample[];
+        judge?: string;
+        judgeSettings?: JudgeSettings;
+    };
+    maxCostUsd?: number;
+    /** Judge each output 3 times (or the grader's own larger vote count) and report how often the verdict splits. */
+    stability?: boolean;
+    /** Cancels the check (for example when the request closes); nothing is stored when it fires. */
+    signal?: AbortSignal;
+};
+export type CalibrateResult = {
+    calibration: EvalCalibrationRow;
+    stats: CalibrationStats;
+    disagreements: EvalCalibrationDisagreement[];
+    errored: number;
+    stability?: StabilityStats;
+    costUsd: number;
+};
+export type CalibrationDeps = {
+    store: CalibrationStore & Pick<RunStore, "insertEvalCalibration">;
+    engine: EvalEngine;
+    log: (level: "info" | "warn" | "error", message: string) => void;
+};
+export declare const CALIBRATION_DEFAULT_MAX_COST_USD = 1;
+/**
+ * Re-judges the stored outputs of results the author labeled for one judged grader and compares the verdicts to the
+ * labels. No stage or agent runs: the engine's case callback replays the stored output. `base` is the eval's built
+ * plan and `found` is the saved grader; the caller owns the one-at-a-time claim and the work directory.
+ */
+export declare function measureCalibration({ store, engine, log }: CalibrationDeps, input: CalibrateInput, base: EvalPlan, found: JudgedGrader, abort: AbortController, 
+/** Fires when the caller cancels; the budget limit uses `abort` instead and still stores the partial reading. */
+cancel: AbortSignal): Promise<CalibrateResult>;