stageflow 0.42.0 → 0.46.0

package/dist/evals/calibration.d.ts

@@ -0,0 +1,21 @@
+import type { RunStore } from "../runstore/port.js";
+import type { EvalCalibrationSplit, EvalGraderLabelRow } from "./storeTypes.js";
+import type { CaseOutput } from "./types.js";
+export type CalibrationSampleSource = {
+    caseId: string;
+    /** Position within the case's samples; the replay repeat index. */
+    repeatIndex: number;
+    label: "pass" | "fail";
+    output: CaseOutput;
+    originalRunId: string;
+    originalRepeatIndex: number;
+};
+export type CalibrationStore = Pick<RunStore, "listEvalGraderLabelsForEval" | "listEvalResults">;
+/** Rebuilds the in-memory case output from the stored result JSON. */
+export declare function replayOutput(stored: unknown): CaseOutput | null;
+/**
+ * The labeled samples for one grader across every run of the eval: the newest label per identical
+ * (case, output), grouped per case with a fresh repeat index so the engine can replay them.
+ */
+export declare function collectCalibrationSamples(store: CalibrationStore, projectRoot: string, evalId: string, graderId: string, split: EvalCalibrationSplit): Promise<CalibrationSampleSource[]>;
+export type { EvalGraderLabelRow };

package/dist/evals/calibration.js

@@ -0,0 +1,69 @@
+import { createHash } from "node:crypto";
+import { calibrationSplitOf } from "./stats.js";
+import { renderTranscript } from "./graders/transcript.js";
+function outputKey(caseId, output) {
+    return createHash("sha256")
+        .update(caseId)
+        .update("\u0000")
+        .update(JSON.stringify(output ?? null))
+        .digest("hex");
+}
+/** Rebuilds the in-memory case output from the stored result JSON. */
+export function replayOutput(stored) {
+    if (stored === null || typeof stored !== "object")
+        return null;
+    const o = stored;
+    if (o.kind === "conversation") {
+        const out = { ...o };
+        out.transcript = renderTranscript(out);
+        return out;
+    }
+    if (typeof o.status !== "string" || typeof o.summary !== "string")
+        return null;
+    return {
+        kind: "envelope",
+        status: o.status === "failure" ? "failure" : "success",
+        summary: o.summary,
+        ...(o.payload !== undefined && typeof o.payload === "object" ? { payload: o.payload } : {}),
+        artifacts: Array.isArray(o.artifacts) ? o.artifacts : [],
+    };
+}
+/**
+ * The labeled samples for one grader across every run of the eval: the newest label per identical
+ * (case, output), grouped per case with a fresh repeat index so the engine can replay them.
+ */
+export async function collectCalibrationSamples(store, projectRoot, evalId, graderId, split) {
+    const labels = await store.listEvalGraderLabelsForEval(projectRoot, evalId, graderId);
+    const resultsByRun = new Map();
+    const seen = new Set();
+    const perCase = new Map();
+    const out = [];
+    for (const label of labels) {
+        if (split !== "all" && calibrationSplitOf(label.caseId) !== split)
+            continue;
+        let results = resultsByRun.get(label.evalRunId);
+        if (!results) {
+            results = new Map((await store.listEvalResults(label.evalRunId)).map((r) => [`${r.caseId}#${r.repeatIndex}`, r]));
+            resultsByRun.set(label.evalRunId, results);
+        }
+        const result = results.get(`${label.caseId}#${label.repeatIndex}`);
+        const output = result ? replayOutput(result.output) : null;
+        if (!output)
+            continue;
+        const key = outputKey(label.caseId, result?.output);
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        const repeatIndex = perCase.get(label.caseId) ?? 0;
+        perCase.set(label.caseId, repeatIndex + 1);
+        out.push({
+            caseId: label.caseId,
+            repeatIndex,
+            label: label.label,
+            output,
+            originalRunId: label.evalRunId,
+            originalRepeatIndex: label.repeatIndex,
+        });
+    }
+    return out;
+}

package/dist/evals/calibrationSuggest.d.ts

@@ -0,0 +1,29 @@
+import type { RunStore } from "../runstore/port.js";
+import type { EvalCalibrationRow } from "./storeTypes.js";
+import type { CaseOutput, RubricExample } from "./types.js";
+export declare const EXAMPLE_SUGGESTION_LIMIT = 3;
+export type SuggestedExample = {
+    eval_run_id: string;
+    case_id: string;
+    repeat_index: number;
+    /** What the judge said, so the author can see why this example would help. */
+    judge: "pass" | "fail";
+    example: RubricExample;
+};
+export type SuggestStore = Pick<RunStore, "listEvalResults" | "listEvalGraderLabelsForEval">;
+/** The text a judge example shows for one stored output. */
+export declare function exampleOutputText(output: CaseOutput): string;
+/**
+ * Up to three few-shot examples built from where the judge disagreed with the author on the dev split (a held-out row
+ * is never offered, so the held-out check stays a fair test): one of each kind of
+ * disagreement first (judge too lenient, judge too strict), distinct outputs only. The author's label note,
+ * when there is one, becomes the example's reason. Nothing is written.
+ */
+export declare function suggestExamples(store: SuggestStore, input: {
+    projectRoot: string;
+    evalId: string;
+    graderId: string;
+    scale: "pass_fail" | "1_to_5";
+    calibration: Pick<EvalCalibrationRow, "disagreements">;
+    limit?: number;
+}): Promise<SuggestedExample[]>;

package/dist/evals/calibrationSuggest.js

@@ -0,0 +1,72 @@
+import { replayOutput } from "./calibration.js";
+import { calibrationSplitOf } from "./stats.js";
+export const EXAMPLE_SUGGESTION_LIMIT = 3;
+const EXAMPLE_OUTPUT_MAX_CHARS = 1500;
+function cut(text) {
+    return text.length <= EXAMPLE_OUTPUT_MAX_CHARS ? text : `${text.slice(0, EXAMPLE_OUTPUT_MAX_CHARS)}…`;
+}
+/** The text a judge example shows for one stored output. */
+export function exampleOutputText(output) {
+    if (output.kind === "conversation")
+        return cut(output.transcript ?? output.final_reply);
+    const payload = output.payload === undefined ? "" : `\n${JSON.stringify(output.payload)}`;
+    return cut(`${output.summary}${payload}`);
+}
+function exampleFor(scale, label, output, reason) {
+    const base = { output, ...(reason !== undefined && reason !== "" ? { reason } : {}) };
+    if (scale === "pass_fail")
+        return { verdict: label, ...base };
+    return { score: label === "pass" ? 5 : 1, ...base };
+}
+/**
+ * Up to three few-shot examples built from where the judge disagreed with the author on the dev split (a held-out row
+ * is never offered, so the held-out check stays a fair test): one of each kind of
+ * disagreement first (judge too lenient, judge too strict), distinct outputs only. The author's label note,
+ * when there is one, becomes the example's reason. Nothing is written.
+ */
+export async function suggestExamples(store, input) {
+    const limit = input.limit ?? EXAMPLE_SUGGESTION_LIMIT;
+    const labels = await store.listEvalGraderLabelsForEval(input.projectRoot, input.evalId, input.graderId);
+    const notes = new Map(labels.map((l) => [`${l.evalRunId}#${l.caseId}#${l.repeatIndex}`, l.note]));
+    const dev = input.calibration.disagreements.filter((d) => calibrationSplitOf(d.caseId) === "dev");
+    const lenient = dev.filter((d) => d.judge === "pass");
+    const strict = dev.filter((d) => d.judge === "fail");
+    const ordered = [];
+    for (let i = 0; ordered.length < lenient.length + strict.length; i++) {
+        if (lenient[i])
+            ordered.push(lenient[i]);
+        if (strict[i])
+            ordered.push(strict[i]);
+    }
+    const runs = new Map();
+    const seen = new Set();
+    const out = [];
+    for (const d of ordered) {
+        if (out.length >= limit)
+            break;
+        if (d.evalRunId === undefined)
+            continue;
+        let results = runs.get(d.evalRunId);
+        if (!results) {
+            results = await store.listEvalResults(d.evalRunId);
+            runs.set(d.evalRunId, results);
+        }
+        const result = results.find((r) => r.caseId === d.caseId && r.repeatIndex === d.repeatIndex);
+        const output = result ? replayOutput(result.output) : null;
+        if (!output)
+            continue;
+        const text = exampleOutputText(output);
+        if (seen.has(text))
+            continue;
+        seen.add(text);
+        const note = notes.get(`${d.evalRunId}#${d.caseId}#${d.repeatIndex}`) ?? undefined;
+        out.push({
+            eval_run_id: d.evalRunId,
+            case_id: d.caseId,
+            repeat_index: d.repeatIndex,
+            judge: d.judge,
+            example: exampleFor(input.scale, d.label, text, note),
+        });
+    }
+    return out;
+}

package/dist/evals/calibrator.d.ts

@@ -0,0 +1,46 @@
+import { type CalibrationStore } from "./calibration.js";
+import type { JudgedGrader } from "./judged.js";
+import { type CalibrationStats, type StabilityStats } from "./stats.js";
+import type { EvalCalibrationDisagreement, EvalCalibrationRow, EvalCalibrationSplit } from "./storeTypes.js";
+import type { EvalEngine, EvalPlan, JudgeSettings, RubricExample } from "./types.js";
+import type { RunStore } from "../runstore/port.js";
+export type CalibrateInput = {
+    projectRoot: string;
+    evalPath: string;
+    graderId: string;
+    split?: EvalCalibrationSplit;
+    /** A rubric to measure instead of the saved one; the result is stored as a candidate. */
+    candidate?: {
+        rubric?: string;
+        examples?: RubricExample[];
+        judge?: string;
+        judgeSettings?: JudgeSettings;
+    };
+    maxCostUsd?: number;
+    /** Judge each output 3 times (or the grader's own larger vote count) and report how often the verdict splits. */
+    stability?: boolean;
+    /** Cancels the check (for example when the request closes); nothing is stored when it fires. */
+    signal?: AbortSignal;
+};
+export type CalibrateResult = {
+    calibration: EvalCalibrationRow;
+    stats: CalibrationStats;
+    disagreements: EvalCalibrationDisagreement[];
+    errored: number;
+    stability?: StabilityStats;
+    costUsd: number;
+};
+export type CalibrationDeps = {
+    store: CalibrationStore & Pick<RunStore, "insertEvalCalibration">;
+    engine: EvalEngine;
+    log: (level: "info" | "warn" | "error", message: string) => void;
+};
+export declare const CALIBRATION_DEFAULT_MAX_COST_USD = 1;
+/**
+ * Re-judges the stored outputs of results the author labeled for one judged grader and compares the verdicts to the
+ * labels. No stage or agent runs: the engine's case callback replays the stored output. `base` is the eval's built
+ * plan and `found` is the saved grader; the caller owns the one-at-a-time claim and the work directory.
+ */
+export declare function measureCalibration({ store, engine, log }: CalibrationDeps, input: CalibrateInput, base: EvalPlan, found: JudgedGrader, abort: AbortController, 
+/** Fires when the caller cancels; the budget limit uses `abort` instead and still stores the partial reading. */
+cancel: AbortSignal): Promise<CalibrateResult>;

package/dist/evals/calibrator.js

@@ -0,0 +1,148 @@
+import { collectCalibrationSamples } from "./calibration.js";
+import { CONVERSATION_RUBRIC_PROMPT_VERSION, RUBRIC_PROMPT_VERSION } from "./constants.js";
+import { computeCalibration, computeStability } from "./stats.js";
+import { EvalRunError } from "./errors.js";
+export const CALIBRATION_DEFAULT_MAX_COST_USD = 1;
+const key = (caseId, repeatIndex) => `${caseId}:${repeatIndex}`;
+/**
+ * Re-judges the stored outputs of results the author labeled for one judged grader and compares the verdicts to the
+ * labels. No stage or agent runs: the engine's case callback replays the stored output. `base` is the eval's built
+ * plan and `found` is the saved grader; the caller owns the one-at-a-time claim and the work directory.
+ */
+export async function measureCalibration({ store, engine, log }, input, base, found, abort, 
+/** Fires when the caller cancels; the budget limit uses `abort` instead and still stores the partial reading. */
+cancel) {
+    let grader = found;
+    if (input.candidate !== undefined) {
+        if (grader.type === "rubric") {
+            grader = {
+                ...grader,
+                ...(input.candidate.rubric !== undefined ? { rubric: input.candidate.rubric } : {}),
+                ...(input.candidate.examples !== undefined ? { examples: input.candidate.examples } : {}),
+                ...(input.candidate.judge !== undefined ? { judge: input.candidate.judge } : {}),
+                ...(input.candidate.judgeSettings !== undefined
+                    ? { judgeSettings: { ...(grader.judgeSettings ?? {}), ...input.candidate.judgeSettings } }
+                    : {}),
+            };
+        }
+        else {
+            grader = {
+                ...grader,
+                ...(input.candidate.judge !== undefined ? { judge: input.candidate.judge } : {}),
+                ...(input.candidate.judgeSettings !== undefined
+                    ? { judgeSettings: { ...(grader.judgeSettings ?? {}), ...input.candidate.judgeSettings } }
+                    : {}),
+            };
+        }
+    }
+    const stabilityVotes = input.stability === true ? Math.max(grader.judgeSettings?.votes ?? 1, 3) : undefined;
+    if (stabilityVotes !== undefined) {
+        grader = { ...grader, judgeSettings: { ...(grader.judgeSettings ?? {}), votes: stabilityVotes } };
+    }
+    const split = input.split ?? "all";
+    const samples = await collectCalibrationSamples(store, input.projectRoot, base.evalId, grader.id, split);
+    const casesById = new Map(base.cases.map((c) => [c.id, c]));
+    const usable = samples.filter((s) => casesById.has(s.caseId));
+    if (usable.length === 0) {
+        throw new EvalRunError("EVAL_CALIBRATION_NO_LABELS", `No labeled results for grader ${grader.id} can be re-judged; label results in a run first`);
+    }
+    const replay = new Map(usable.map((s) => [key(s.caseId, s.repeatIndex), s]));
+    const plan = {
+        ...base,
+        cases: [...new Set(usable.map((s) => s.caseId))].map((id, position) => ({
+            ...casesById.get(id),
+            position,
+        })),
+        graders: [{ ...grader, weight: 1, mustPass: false }],
+        repeats: Math.max(...usable.map((s) => s.repeatIndex)) + 1,
+        maxCostUsd: input.maxCostUsd ?? CALIBRATION_DEFAULT_MAX_COST_USD,
+    };
+    let spend = 0;
+    const verdicts = new Map();
+    const io = {
+        runCase: async (req) => {
+            const sample = replay.get(key(req.caseId, req.repeatIndex));
+            if (!sample) {
+                return { ok: false, code: "EVAL_ENGINE_ERROR", message: "not labeled for this grader" };
+            }
+            return { ok: true, output: sample.output, latencyMs: 0, costUsd: 0, tokensIn: 0, tokensOut: 0 };
+        },
+        onResult: (nr) => {
+            const grade = nr.grades.find((g) => g.graderId === grader.id);
+            if (grade)
+                verdicts.set(key(nr.caseId, nr.repeatIndex), grade);
+            if (grade?.judgeCostUsd !== undefined && Number.isFinite(grade.judgeCostUsd))
+                spend += grade.judgeCostUsd;
+            if (spend >= plan.maxCostUsd)
+                abort.abort();
+        },
+        reportJudgeCost: () => undefined,
+        onPhase: () => undefined,
+        log: (level, message) => {
+            if (level === "info")
+                log("info", message);
+            else
+                log(level, message);
+        },
+    };
+    const stop = () => abort.abort();
+    if (cancel.aborted)
+        stop();
+    cancel.addEventListener("abort", stop, { once: true });
+    try {
+        await engine.run(plan, io, abort.signal);
+    }
+    finally {
+        cancel.removeEventListener("abort", stop);
+    }
+    if (cancel.aborted)
+        throw new EvalRunError("EVAL_CALIBRATION_CANCELLED", "The check was cancelled");
+    const voteSets = [];
+    const measured = [];
+    const disagreements = [];
+    let errored = 0;
+    for (const s of usable) {
+        const grade = verdicts.get(key(s.caseId, s.repeatIndex));
+        if (!grade || grade.status !== "graded" || grade.passed === undefined) {
+            errored++;
+            continue;
+        }
+        const label = s.label === "pass";
+        const votes = grade.details?.votes;
+        if (Array.isArray(votes)) {
+            voteSets.push(votes.filter((v) => v.status === "graded" && typeof v.passed === "boolean").map((v) => v.passed));
+        }
+        measured.push({ caseId: s.caseId, repeatIndex: s.originalRepeatIndex, label, judge: grade.passed });
+        if (label !== grade.passed) {
+            disagreements.push({
+                evalRunId: s.originalRunId,
+                caseId: s.caseId,
+                repeatIndex: s.originalRepeatIndex,
+                label: s.label,
+                judge: grade.passed ? "pass" : "fail",
+                reason: grade.reason ?? null,
+            });
+        }
+    }
+    const stats = computeCalibration(measured);
+    const stability = stabilityVotes !== undefined ? computeStability(voteSets, stabilityVotes) : undefined;
+    const calibration = await store.insertEvalCalibration({
+        projectRoot: input.projectRoot,
+        evalId: base.evalId,
+        graderId: grader.id,
+        graderHash: found.hash,
+        judge: grader.judge,
+        promptVersion: base.target.kind === "agent" ? CONVERSATION_RUBRIC_PROMPT_VERSION : RUBRIC_PROMPT_VERSION,
+        split,
+        candidate: input.candidate !== undefined,
+        stats: {
+            ...stats,
+            errored,
+            skipped_missing_case: samples.length - usable.length,
+            ...(stability !== undefined ? { stability } : {}),
+        },
+        disagreements,
+        costUsd: spend,
+    });
+    return { calibration, stats, disagreements, errored, ...(stability !== undefined ? { stability } : {}), costUsd: spend };
+}

package/dist/evals/compare.d.ts

@@ -46,7 +46,7 @@ export interface ComponentDiff {
     a: unknown;
     b: unknown;
 }
-export type CompareWarningCode = "dataset_differs" | "graders_differ" | "engine_differs" | "few_shared_cases";
+export type CompareWarningCode = "dataset_differs" | "graders_differ" | "engine_differs" | "judge_differs" | "few_shared_cases";
 export interface CompareWarning {
     code: CompareWarningCode;
     shared_cases?: number;

package/dist/evals/compare.js

@@ -151,6 +151,9 @@ export function compareRuns(a, b) {
     if (components.find((c) => c.key === "engine")?.status === "changed") {
         warnings.push({ code: "engine_differs" });
     }
+    if (["judge", "rubric_prompt_version"].some((key) => components.find((c) => c.key === key)?.status === "changed")) {
+        warnings.push({ code: "judge_differs" });
+    }
     if (m < 5)
         warnings.push({ code: "few_shared_cases", shared_cases: m });
     const stage = !isAgent &&

package/dist/evals/constants.d.ts

@@ -16,8 +16,8 @@ export type EvalReviewLabel = (typeof EVAL_REVIEW_LABELS)[number];
 export declare const EVAL_BUDGET_WARNINGS: readonly ["latency", "cost"];
 export type EvalBudgetWarning = (typeof EVAL_BUDGET_WARNINGS)[number];
 export declare const EVAL_RESULT_ERROR_CODES: readonly ["EVAL_STAGE_FAILED", "EVAL_STAGE_NEEDS_INPUT", "EVAL_STAGE_TIMEOUT", "EVAL_ENGINE_ERROR", "EVAL_BUDGET_EXCEEDED", "EVAL_CANCELLED", "EVAL_JUDGE_ERROR", "EVAL_GRADER_CODE_ERROR", "EVAL_CASE_MISSING_EXPECTED", "EVAL_AGENT_FAILED", "EVAL_AGENT_EMPTY_REPLY", "EVAL_AGENT_TIMEOUT", "EVAL_SIMULATOR_ERROR", "EVAL_GRADER_NOT_APPLICABLE"];
-export declare const EVAL_REQUEST_ERROR_CODES: readonly ["EVAL_INVALID", "EVAL_RUN_ACTIVE", "EVAL_ENGINE_UNAVAILABLE", "EVAL_NODE_TOO_OLD", "FILE_CHANGED", "EVAL_NOT_FOUND", "DATASET_NOT_FOUND", "EVAL_RUN_NOT_FOUND", "EVAL_TAGS_ON_PASSED_RESULT", "EVAL_REAL_TOOLS_NOT_CONFIRMED", "EVAL_CLASSIFIER_UNAVAILABLE"];
-export declare const EVAL_ERROR_CODES: readonly ["EVAL_STAGE_FAILED", "EVAL_STAGE_NEEDS_INPUT", "EVAL_STAGE_TIMEOUT", "EVAL_ENGINE_ERROR", "EVAL_BUDGET_EXCEEDED", "EVAL_CANCELLED", "EVAL_JUDGE_ERROR", "EVAL_GRADER_CODE_ERROR", "EVAL_CASE_MISSING_EXPECTED", "EVAL_AGENT_FAILED", "EVAL_AGENT_EMPTY_REPLY", "EVAL_AGENT_TIMEOUT", "EVAL_SIMULATOR_ERROR", "EVAL_GRADER_NOT_APPLICABLE", "EVAL_INVALID", "EVAL_RUN_ACTIVE", "EVAL_ENGINE_UNAVAILABLE", "EVAL_NODE_TOO_OLD", "FILE_CHANGED", "EVAL_NOT_FOUND", "DATASET_NOT_FOUND", "EVAL_RUN_NOT_FOUND", "EVAL_TAGS_ON_PASSED_RESULT", "EVAL_REAL_TOOLS_NOT_CONFIRMED", "EVAL_CLASSIFIER_UNAVAILABLE"];
+export declare const EVAL_REQUEST_ERROR_CODES: readonly ["EVAL_INVALID", "EVAL_RUN_ACTIVE", "EVAL_ENGINE_UNAVAILABLE", "EVAL_NODE_TOO_OLD", "FILE_CHANGED", "EVAL_NOT_FOUND", "DATASET_NOT_FOUND", "EVAL_RUN_NOT_FOUND", "EVAL_TAGS_ON_PASSED_RESULT", "EVAL_REAL_TOOLS_NOT_CONFIRMED", "EVAL_CLASSIFIER_UNAVAILABLE", "EVAL_CALIBRATION_NO_LABELS", "EVAL_CALIBRATION_CANCELLED"];
+export declare const EVAL_ERROR_CODES: readonly ["EVAL_STAGE_FAILED", "EVAL_STAGE_NEEDS_INPUT", "EVAL_STAGE_TIMEOUT", "EVAL_ENGINE_ERROR", "EVAL_BUDGET_EXCEEDED", "EVAL_CANCELLED", "EVAL_JUDGE_ERROR", "EVAL_GRADER_CODE_ERROR", "EVAL_CASE_MISSING_EXPECTED", "EVAL_AGENT_FAILED", "EVAL_AGENT_EMPTY_REPLY", "EVAL_AGENT_TIMEOUT", "EVAL_SIMULATOR_ERROR", "EVAL_GRADER_NOT_APPLICABLE", "EVAL_INVALID", "EVAL_RUN_ACTIVE", "EVAL_ENGINE_UNAVAILABLE", "EVAL_NODE_TOO_OLD", "FILE_CHANGED", "EVAL_NOT_FOUND", "DATASET_NOT_FOUND", "EVAL_RUN_NOT_FOUND", "EVAL_TAGS_ON_PASSED_RESULT", "EVAL_REAL_TOOLS_NOT_CONFIRMED", "EVAL_CLASSIFIER_UNAVAILABLE", "EVAL_CALIBRATION_NO_LABELS", "EVAL_CALIBRATION_CANCELLED"];
 export type EvalErrorCode = (typeof EVAL_ERROR_CODES)[number];
 export declare const EVAL_LIMITS: {
     readonly maxCasesPerDataset: 5000;
@@ -35,12 +35,16 @@ export declare const EVAL_LIMITS: {
     readonly labelMaxChars: 80;
     readonly failureTypeNameMaxChars: 60;
     readonly reviewNoteMaxChars: 500;
+    /** Worked examples one rubric may hold. The console mirrors this in `tuningModel.ts`. */
+    readonly rubricExamplesMax: 6;
 };
 export declare const EVAL_RUN_ID_PREFIX = "er_";
 export declare const RUBRIC_PROMPT_VERSION = 1;
 export declare const CONFIG_SNAPSHOT_VERSION = 1;
 export declare const SIMULATOR_PROMPT_VERSION = 1;
-export declare const CONVERSATION_RUBRIC_PROMPT_VERSION = 1;
+export declare const CONVERSATION_RUBRIC_PROMPT_VERSION = 2;
+/** Room for reasoning judges, whose hidden thinking counts against the output cap. */
+export declare const JUDGE_MAX_OUTPUT_TOKENS = 8192;
 /** Bump when the failure-sorting questions, bucket criteria or judge prompt text change. */
 export declare const TRIAGE_PROMPT_VERSION = 1;
 export declare const EVAL_CONVERSATION_LIMITS: {

package/dist/evals/constants.js

@@ -55,6 +55,8 @@ export const EVAL_REQUEST_ERROR_CODES = [
     "EVAL_TAGS_ON_PASSED_RESULT",
     "EVAL_REAL_TOOLS_NOT_CONFIRMED",
     "EVAL_CLASSIFIER_UNAVAILABLE",
+    "EVAL_CALIBRATION_NO_LABELS",
+    "EVAL_CALIBRATION_CANCELLED",
 ];
 export const EVAL_ERROR_CODES = [
     ...EVAL_RESULT_ERROR_CODES,
@@ -70,12 +72,16 @@ export const EVAL_LIMITS = {
     labelMaxChars: 80,
     failureTypeNameMaxChars: 60,
     reviewNoteMaxChars: 500,
+    /** Worked examples one rubric may hold. The console mirrors this in `tuningModel.ts`. */
+    rubricExamplesMax: 6,
 };
 export const EVAL_RUN_ID_PREFIX = "er_";
 export const RUBRIC_PROMPT_VERSION = 1;
 export const CONFIG_SNAPSHOT_VERSION = 1;
 export const SIMULATOR_PROMPT_VERSION = 1;
-export const CONVERSATION_RUBRIC_PROMPT_VERSION = 1;
+export const CONVERSATION_RUBRIC_PROMPT_VERSION = 2;
+/** Room for reasoning judges, whose hidden thinking counts against the output cap. */
+export const JUDGE_MAX_OUTPUT_TOKENS = 8_192;
 /** Bump when the failure-sorting questions, bucket criteria or judge prompt text change. */
 export const TRIAGE_PROMPT_VERSION = 1;
 export const EVAL_CONVERSATION_LIMITS = {

package/dist/evals/engine/FakeEngine.d.ts

@@ -1,4 +1,4 @@
-import type { CaseOutput, EngineIo, EvalEngine, EvalPlan, PlanCase, ResolvedGrader } from "../types.js";
+import { type CaseOutput, type EngineIo, type EvalEngine, type EvalPlan, type PlanCase, type ResolvedGrader } from "../types.js";
 export type FakeJudgeReply = ({
     verdict: "pass" | "fail";
 } | {
@@ -29,5 +29,6 @@ export declare class FakeEngine implements EvalEngine {
     constructor(options?: FakeEngineOptions);
     version(): Promise<string>;
     run(plan: EvalPlan, io: EngineIo, signal: AbortSignal): Promise<void>;
+    private gradeJudged;
     private gradeRubric;
 }

package/dist/evals/engine/FakeEngine.js

@@ -1,4 +1,6 @@
 import { caseGoal, goalRubricText, gradeDeterministic, isDeterministicGrader } from "../graders/index.js";
+import { combineVotes } from "../votes.js";
+import { isConversationTarget, } from "../types.js";
 export class FakeEngineError extends Error {
     code = "EVAL_ENGINE_ERROR";
     constructor(message) {
@@ -71,7 +73,7 @@ export class FakeEngine {
                     throw new FakeEngineError(`fake engine failed after ${emitted} results`);
                 }
                 const { testCase, repeatIndex } = items[next++];
-                await phase(testCase, repeatIndex, plan.target.kind === "agent" ? "agent_running" : "stage_running");
+                await phase(testCase, repeatIndex, isConversationTarget(plan.target.kind) ? "agent_running" : "stage_running");
                 const run = await io.runCase({ evalRunId: plan.evalRunId, caseId: testCase.id, repeatIndex });
                 const grades = [];
                 if (run.ok) {
@@ -88,7 +90,7 @@ export class FakeEngine {
                     };
                     for (const grader of plan.graders) {
                         if (grader.type === "rubric") {
-                            grades.push(this.gradeRubric(grader, testCase, run.output, io));
+                            grades.push(this.gradeJudged(grader, testCase, run.output, io));
                         }
                         else if (grader.type === "goal_reached") {
                             const goal = caseGoal(testCase);
@@ -101,7 +103,7 @@ export class FakeEngine {
                                     reason: "case has no input.goal or expected.outcome",
                                 }
                                 : {
-                                    ...this.gradeRubric({
+                                    ...this.gradeJudged({
                                         ...grader,
                                         type: "rubric",
                                         rubric: goalRubricText(goal, grader.guidance),
@@ -134,6 +136,10 @@ export class FakeEngine {
         if (failure !== undefined)
             throw failure;
     }
+    gradeJudged(grader, testCase, output, io) {
+        const votes = grader.judgeSettings?.votes ?? 1;
+        return combineVotes(grader, Array.from({ length: votes }, () => this.gradeRubric(grader, testCase, output, io)));
+    }
     gradeRubric(grader, testCase, output, io) {
         const base = { graderId: grader.id, graderType: "rubric", judgeModel: grader.judge };
         const reply = this.options.judge?.(grader, testCase, output) ??

package/dist/evals/engine/promptfoo/PromptfooEngine.d.ts

@@ -1,5 +1,5 @@
 import { type ChildProcess, type SpawnOptions } from "node:child_process";
-import type { EngineIo, EvalEngine, EvalPlan } from "../../types.js";
+import { type EngineIo, type EvalEngine, type EvalPlan } from "../../types.js";
 import { type JudgeKeyResolver, type JudgePriceLoader } from "./judgeServices.js";
 import type { RunTokenRegistry } from "./runTokens.js";
 export declare const PROMPTFOO_KILL_GRACE_MS = 5000;

package/dist/evals/engine/promptfoo/PromptfooEngine.js

@@ -3,6 +3,7 @@ import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, rmS
 import path from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { globalStageflowHome } from "../../../project/globalHome.js";
+import { isConversationTarget, } from "../../types.js";
 import { EngineError } from "./errors.js";
 import { CALLBACK_URL_ENV, CONFIG_FILE, RESULTS_FILE, RESULTS_STREAM_FILE, RUN_TOKEN_ENV, generateFiles, } from "./generate.js";
 import { GUARD_FILENAME } from "./guard.js";
@@ -13,6 +14,7 @@ import { locatePromptfoo } from "./locate.js";
 import { normalizeRecord, recordIdentity, recordsFromResultsFile } from "./normalize.js";
 import { assertNode } from "./nodeVersion.js";
 import { PINNED_PROMPTFOO_VERSION } from "./pinned.js";
+import { isJudged } from "../../judged.js";
 export const PROMPTFOO_KILL_GRACE_MS = 5_000;
 export const PROMPTFOO_STREAM_POLL_MS = 100;
 export const STDERR_TAIL_BYTES = 2048;
@@ -131,7 +133,7 @@ export class PromptfooEngine {
         const judgeEnv = {};
         const judgeModels = new Set();
         for (const grader of plan.graders) {
-            if (grader.type !== "rubric" && grader.type !== "goal_reached")
+            if (!isJudged(grader))
                 continue;
             judgeModels.add(grader.judge);
             const mapping = mapJudge(grader.judge);
@@ -160,7 +162,7 @@ export class PromptfooEngine {
         const hostRuns = new Map();
         const keyOf = (caseId, repeatIndex) => `${caseId}#${repeatIndex}`;
         const runCase = async (req) => {
-            io.onPhase(req.caseId, req.repeatIndex, plan.target.kind === "agent" ? "agent_running" : "stage_running");
+            io.onPhase(req.caseId, req.repeatIndex, isConversationTarget(plan.target.kind) ? "agent_running" : "stage_running");
             const result = await io.runCase(req);
             hostRuns.set(keyOf(req.caseId, req.repeatIndex), result);
             if (result.ok)

package/dist/evals/engine/promptfoo/generate.d.ts

@@ -1,5 +1,5 @@
 import { GUARD_FILENAME } from "./guard.js";
-import type { EvalPlan } from "../../types.js";
+import { type EvalPlan } from "../../types.js";
 export declare const CALLBACK_URL_ENV = "STAGEFLOW_EVAL_CALLBACK_URL";
 export declare const RUN_TOKEN_ENV = "STAGEFLOW_EVAL_TOKEN";
 export declare const CONFIG_FILE = "promptfooconfig.json";
@@ -26,5 +26,7 @@ export declare function assertionFnName(index: number): string;
 export declare function buildProviderSource(evalRunId: string, agent?: boolean): string;
 export declare function buildHooksSource(agent?: boolean): string;
 export declare function buildAssertionsSource(plan: EvalPlan, graderModuleUrl: string): string;
+/** The metric of vote `n` (1-based) for a judged grader; vote 1 keeps the grader id. */
+export declare function voteMetric(graderId: string, vote: number): string;
 export declare function buildConfig(plan: EvalPlan, options: GenerateOptions): Record<string, unknown>;
 export declare function generateFiles(plan: EvalPlan, options: GenerateOptions): GeneratedFiles;