stageflow 0.40.0 → 0.44.0

package/dist/evals/compare.js

@@ -0,0 +1,201 @@
+import { buildCaseStats, clusteredCi, computeStats, mean, sampleSd } from "./stats.js";
+import { agentDiff } from "./agentDiff.js";
+import { stageDiff } from "./stageDiff.js";
+import { EVAL_COMPARE_MISMATCH, EXECUTED_RESULT_STATUSES, resultKey, } from "./statsTypes.js";
+export class CompareMismatchError extends Error {
+    code = EVAL_COMPARE_MISMATCH;
+    constructor(a, b, what = "different evals") {
+        super(`Cannot compare runs of ${what} (${a} vs ${b})`);
+        this.name = "CompareMismatchError";
+    }
+}
+function canonical(value) {
+    return JSON.stringify(sortKeys(value));
+}
+function sortKeys(value) {
+    if (Array.isArray(value))
+        return value.map(sortKeys);
+    if (value && typeof value === "object") {
+        const obj = value;
+        return Object.fromEntries(Object.keys(obj)
+            .sort()
+            .map((k) => [k, sortKeys(obj[k])]));
+    }
+    return value ?? null;
+}
+function delta(a, b) {
+    return {
+        a,
+        b,
+        delta: a !== null && b !== null ? b - a : null,
+        percent_change: a !== null && b !== null && a !== 0 ? ((b - a) / a) * 100 : null,
+    };
+}
+function passRateOver(grades, graderId, cases) {
+    const graded = grades.filter((g) => g.graderId === graderId && g.status === "graded" && cases.has(g.caseId));
+    if (graded.length === 0)
+        return null;
+    return graded.filter((g) => g.passed === 1).length / graded.length;
+}
+function passedRepeatsByGrader(grades, executedKeys, caseId) {
+    const out = new Map();
+    for (const g of grades) {
+        if (g.caseId !== caseId || !executedKeys.has(resultKey(g.caseId, g.repeatIndex)))
+            continue;
+        out.set(g.graderId, (out.get(g.graderId) ?? 0) + (g.passed === 1 ? 1 : 0));
+    }
+    return out;
+}
+function executedKeysOf(results) {
+    return new Set(results
+        .filter((r) => EXECUTED_RESULT_STATUSES.includes(r.status))
+        .map((r) => resultKey(r.caseId, r.repeatIndex)));
+}
+export function compareRuns(a, b) {
+    if (a.evalId !== b.evalId)
+        throw new CompareMismatchError(a.evalId, b.evalId);
+    const kindA = a.targetKind ?? "stage";
+    const kindB = b.targetKind ?? "stage";
+    if (kindA !== kindB || (kindA === "agent" && (a.agentId ?? null) !== (b.agentId ?? null))) {
+        throw new CompareMismatchError(a.evalId, b.evalId, "different targets");
+    }
+    const isAgent = kindA === "agent";
+    const statsA = computeStats(a);
+    const statsB = computeStats(b);
+    const casesA = new Map(buildCaseStats(a.results).map((c) => [c.caseId, c]));
+    const casesB = new Map(buildCaseStats(b.results).map((c) => [c.caseId, c]));
+    const sharedIds = [...casesA.keys()].filter((id) => casesB.has(id)).sort();
+    const m = sharedIds.length;
+    const shared = new Set(sharedIds);
+    const diffs = sharedIds.map((id) => (casesB.get(id)?.p ?? 0) - (casesA.get(id)?.p ?? 0));
+    const dBar = mean(diffs);
+    const ci = dBar === null ? null : clusteredCi(dBar, diffs, -1, 1);
+    const sdD = sampleSd(diffs);
+    const casesNeeded = sdD === null ? null : Math.ceil(Math.round((1.96 * sdD / 0.05) ** 2 * 1e9) / 1e9);
+    let verdict = "no_clear_difference";
+    if (m < 5 || ci === null)
+        verdict = "insufficient_data";
+    else if (ci.low > 0)
+        verdict = "improved";
+    else if (ci.high < 0)
+        verdict = "regressed";
+    const keysA = executedKeysOf(a.results);
+    const keysB = executedKeysOf(b.results);
+    const cases = sharedIds.map((id) => {
+        const ca = casesA.get(id);
+        const cb = casesB.get(id);
+        const d = cb.p - ca.p;
+        const pa = passedRepeatsByGrader(a.grades, keysA, id);
+        const pb = passedRepeatsByGrader(b.grades, keysB, id);
+        const graderIds = [...new Set([...pa.keys(), ...pb.keys()])].sort();
+        return {
+            case_id: id,
+            change: d > 0 ? "improved" : d < 0 ? "regressed" : "same",
+            p_a: ca.p,
+            p_b: cb.p,
+            d,
+            passes_a: ca.flags,
+            passes_b: cb.flags,
+            mean_score_a: ca.meanScore,
+            mean_score_b: cb.meanScore,
+            graders_changed: graderIds.filter((g) => (pa.get(g) ?? 0) !== (pb.get(g) ?? 0)),
+        };
+    });
+    const rank = { regressed: 0, improved: 1, same: 2 };
+    cases.sort((x, y) => rank[x.change] - rank[y.change] ||
+        (x.change === "improved" ? y.d - x.d : x.d - y.d) ||
+        (x.case_id < y.case_id ? -1 : x.case_id > y.case_id ? 1 : 0));
+    const hashA = new Map(a.graders.map((g) => [g.id, g.hash]));
+    const hashB = new Map(b.graders.map((g) => [g.id, g.hash]));
+    const graderIds = [...new Set([...hashA.keys(), ...hashB.keys()])];
+    const graders = graderIds.map((id) => {
+        if (!hashB.has(id)) {
+            return { grader_id: id, status: "removed", pass_rate_a: null, pass_rate_b: null, delta: null };
+        }
+        if (!hashA.has(id)) {
+            return { grader_id: id, status: "added", pass_rate_a: null, pass_rate_b: null, delta: null };
+        }
+        if (hashA.get(id) !== hashB.get(id)) {
+            return { grader_id: id, status: "changed", pass_rate_a: null, pass_rate_b: null, delta: null };
+        }
+        const ra = passRateOver(a.grades, id, shared);
+        const rb = passRateOver(b.grades, id, shared);
+        return {
+            grader_id: id,
+            status: "compared",
+            pass_rate_a: ra,
+            pass_rate_b: rb,
+            delta: ra !== null && rb !== null ? (rb - ra) * 100 : null,
+        };
+    });
+    const componentKeys = [
+        ...new Set([...Object.keys(a.configComponents), ...Object.keys(b.configComponents)]),
+    ];
+    const components = componentKeys.map((key) => ({
+        key,
+        status: canonical(a.configComponents[key]) === canonical(b.configComponents[key]) ? "same" : "changed",
+        a: a.configComponents[key] ?? null,
+        b: b.configComponents[key] ?? null,
+    }));
+    const warnings = [];
+    if (a.datasetHash !== b.datasetHash) {
+        warnings.push({
+            code: "dataset_differs",
+            shared_cases: m,
+            cases_a: casesA.size,
+            cases_b: casesB.size,
+        });
+    }
+    if (graders.some((g) => g.status !== "compared"))
+        warnings.push({ code: "graders_differ" });
+    if (components.find((c) => c.key === "engine")?.status === "changed") {
+        warnings.push({ code: "engine_differs" });
+    }
+    if (m < 5)
+        warnings.push({ code: "few_shared_cases", shared_cases: m });
+    const stage = !isAgent &&
+        a.stageVersionHash !== b.stageVersionHash && a.stageDefinition && b.stageDefinition
+        ? stageDiff(a.stageDefinition, b.stageDefinition)
+        : null;
+    return {
+        shared_cases: m,
+        cases_a: casesA.size,
+        cases_b: casesB.size,
+        mean_difference: dBar,
+        ci,
+        verdict,
+        cases,
+        graders,
+        deltas: {
+            cost_per_result: delta(statsA.cost_per_result, statsB.cost_per_result),
+            latency_p50: delta(statsA.latency_p50, statsB.latency_p50),
+            latency_p95: delta(statsA.latency_p95, statsB.latency_p95),
+            flaky_cases: delta(statsA.flaky_cases.length, statsB.flaky_cases.length),
+            mean_score: delta(statsA.mean_score, statsB.mean_score),
+            ...(isAgent
+                ? {
+                    mean_turns: delta(statsA.mean_turns ?? null, statsB.mean_turns ?? null),
+                    mean_tool_calls: delta(statsA.mean_tool_calls ?? null, statsB.mean_tool_calls ?? null),
+                }
+                : {}),
+        },
+        stage_diff: stage,
+        ...(isAgent
+            ? {
+                agent_diff: a.stageVersionHash !== b.stageVersionHash && a.agentDefinition && b.agentDefinition
+                    ? agentDiff(a.agentDefinition, b.agentDefinition, {
+                        a: a.agentRevision ?? null,
+                        b: b.agentRevision ?? null,
+                    })
+                    : null,
+            }
+            : {}),
+        components,
+        same_config: components.every((c) => c.status === "same"),
+        warnings,
+        sd_d: sdD,
+        cases_needed_5pts: casesNeeded,
+        stats_a: statsA,
+        stats_b: statsB,
+    };
+}

package/dist/evals/constants.d.ts

@@ -0,0 +1,62 @@
+/** Consecutive empty-reply or backend-failed agent cases after which a run stops early. */
+export declare const EVAL_MAX_CONSECUTIVE_PROVIDER_FAILURES = 5;
+export declare const EVAL_RUN_STATUSES: readonly ["queued", "running", "completed", "failed", "cancelled", "budget_stopped"];
+export type EvalRunStatus = (typeof EVAL_RUN_STATUSES)[number];
+export declare const EVAL_RESULT_STATUSES: readonly ["completed", "stage_failed", "needs_input", "error", "skipped_budget", "cancelled"];
+export type EvalResultStatus = (typeof EVAL_RESULT_STATUSES)[number];
+export declare const EVAL_EXECUTED_RESULT_STATUSES: readonly ["completed", "stage_failed", "needs_input", "error"];
+export declare const EVAL_GRADE_STATUSES: readonly ["graded", "error"];
+export type EvalGradeStatus = (typeof EVAL_GRADE_STATUSES)[number];
+export declare const EVAL_TARGET_KINDS: readonly ["stage", "agent"];
+export type EvalTargetKind = (typeof EVAL_TARGET_KINDS)[number];
+export declare const EVAL_STARTED_VIA: readonly ["console", "cli", "mcp"];
+export type EvalStartedVia = (typeof EVAL_STARTED_VIA)[number];
+export declare const EVAL_REVIEW_LABELS: readonly ["pass", "fail"];
+export type EvalReviewLabel = (typeof EVAL_REVIEW_LABELS)[number];
+export declare const EVAL_BUDGET_WARNINGS: readonly ["latency", "cost"];
+export type EvalBudgetWarning = (typeof EVAL_BUDGET_WARNINGS)[number];
+export declare const EVAL_RESULT_ERROR_CODES: readonly ["EVAL_STAGE_FAILED", "EVAL_STAGE_NEEDS_INPUT", "EVAL_STAGE_TIMEOUT", "EVAL_ENGINE_ERROR", "EVAL_BUDGET_EXCEEDED", "EVAL_CANCELLED", "EVAL_JUDGE_ERROR", "EVAL_GRADER_CODE_ERROR", "EVAL_CASE_MISSING_EXPECTED", "EVAL_AGENT_FAILED", "EVAL_AGENT_EMPTY_REPLY", "EVAL_AGENT_TIMEOUT", "EVAL_SIMULATOR_ERROR", "EVAL_GRADER_NOT_APPLICABLE"];
+export declare const EVAL_REQUEST_ERROR_CODES: readonly ["EVAL_INVALID", "EVAL_RUN_ACTIVE", "EVAL_ENGINE_UNAVAILABLE", "EVAL_NODE_TOO_OLD", "FILE_CHANGED", "EVAL_NOT_FOUND", "DATASET_NOT_FOUND", "EVAL_RUN_NOT_FOUND", "EVAL_TAGS_ON_PASSED_RESULT", "EVAL_REAL_TOOLS_NOT_CONFIRMED", "EVAL_CLASSIFIER_UNAVAILABLE"];
+export declare const EVAL_ERROR_CODES: readonly ["EVAL_STAGE_FAILED", "EVAL_STAGE_NEEDS_INPUT", "EVAL_STAGE_TIMEOUT", "EVAL_ENGINE_ERROR", "EVAL_BUDGET_EXCEEDED", "EVAL_CANCELLED", "EVAL_JUDGE_ERROR", "EVAL_GRADER_CODE_ERROR", "EVAL_CASE_MISSING_EXPECTED", "EVAL_AGENT_FAILED", "EVAL_AGENT_EMPTY_REPLY", "EVAL_AGENT_TIMEOUT", "EVAL_SIMULATOR_ERROR", "EVAL_GRADER_NOT_APPLICABLE", "EVAL_INVALID", "EVAL_RUN_ACTIVE", "EVAL_ENGINE_UNAVAILABLE", "EVAL_NODE_TOO_OLD", "FILE_CHANGED", "EVAL_NOT_FOUND", "DATASET_NOT_FOUND", "EVAL_RUN_NOT_FOUND", "EVAL_TAGS_ON_PASSED_RESULT", "EVAL_REAL_TOOLS_NOT_CONFIRMED", "EVAL_CLASSIFIER_UNAVAILABLE"];
+export type EvalErrorCode = (typeof EVAL_ERROR_CODES)[number];
+export declare const EVAL_LIMITS: {
+    readonly maxCasesPerDataset: 5000;
+    readonly maxCaseBytes: 1000000;
+    readonly maxUploadBytes: 50000000;
+    readonly maxResultsPerRun: 5000;
+    readonly repeats: {
+        readonly min: 1;
+        readonly max: 10;
+    };
+    readonly concurrency: {
+        readonly min: 1;
+        readonly max: 8;
+    };
+    readonly labelMaxChars: 80;
+    readonly failureTypeNameMaxChars: 60;
+    readonly reviewNoteMaxChars: 500;
+};
+export declare const EVAL_RUN_ID_PREFIX = "er_";
+export declare const RUBRIC_PROMPT_VERSION = 1;
+export declare const CONFIG_SNAPSHOT_VERSION = 1;
+export declare const SIMULATOR_PROMPT_VERSION = 1;
+export declare const CONVERSATION_RUBRIC_PROMPT_VERSION = 2;
+/** Room for reasoning judges, whose hidden thinking counts against the output cap. */
+export declare const JUDGE_MAX_OUTPUT_TOKENS = 8192;
+/** Bump when the failure-sorting questions, bucket criteria or judge prompt text change. */
+export declare const TRIAGE_PROMPT_VERSION = 1;
+export declare const EVAL_CONVERSATION_LIMITS: {
+    readonly maxMessageChars: 4000;
+    readonly maxResultSummaryChars: 1000;
+    readonly maxArgsChars: 2000;
+    readonly maxMessages: 80;
+    readonly keepFirstMessages: 4;
+    readonly maxToolCalls: 200;
+    readonly maxSubjectBytes: number;
+};
+export declare const JUDGE_TRANSCRIPT_LIMITS: {
+    readonly turns: 30;
+    readonly messageChars: 2000;
+    readonly toolArgsChars: 500;
+    readonly toolResultChars: 500;
+};

package/dist/evals/constants.js

@@ -0,0 +1,97 @@
+/** Consecutive empty-reply or backend-failed agent cases after which a run stops early. */
+export const EVAL_MAX_CONSECUTIVE_PROVIDER_FAILURES = 5;
+export const EVAL_RUN_STATUSES = [
+    "queued",
+    "running",
+    "completed",
+    "failed",
+    "cancelled",
+    "budget_stopped",
+];
+export const EVAL_RESULT_STATUSES = [
+    "completed",
+    "stage_failed",
+    "needs_input",
+    "error",
+    "skipped_budget",
+    "cancelled",
+];
+export const EVAL_EXECUTED_RESULT_STATUSES = [
+    "completed",
+    "stage_failed",
+    "needs_input",
+    "error",
+];
+export const EVAL_GRADE_STATUSES = ["graded", "error"];
+export const EVAL_TARGET_KINDS = ["stage", "agent"];
+export const EVAL_STARTED_VIA = ["console", "cli", "mcp"];
+export const EVAL_REVIEW_LABELS = ["pass", "fail"];
+export const EVAL_BUDGET_WARNINGS = ["latency", "cost"];
+export const EVAL_RESULT_ERROR_CODES = [
+    "EVAL_STAGE_FAILED",
+    "EVAL_STAGE_NEEDS_INPUT",
+    "EVAL_STAGE_TIMEOUT",
+    "EVAL_ENGINE_ERROR",
+    "EVAL_BUDGET_EXCEEDED",
+    "EVAL_CANCELLED",
+    "EVAL_JUDGE_ERROR",
+    "EVAL_GRADER_CODE_ERROR",
+    "EVAL_CASE_MISSING_EXPECTED",
+    "EVAL_AGENT_FAILED",
+    "EVAL_AGENT_EMPTY_REPLY",
+    "EVAL_AGENT_TIMEOUT",
+    "EVAL_SIMULATOR_ERROR",
+    "EVAL_GRADER_NOT_APPLICABLE",
+];
+export const EVAL_REQUEST_ERROR_CODES = [
+    "EVAL_INVALID",
+    "EVAL_RUN_ACTIVE",
+    "EVAL_ENGINE_UNAVAILABLE",
+    "EVAL_NODE_TOO_OLD",
+    "FILE_CHANGED",
+    "EVAL_NOT_FOUND",
+    "DATASET_NOT_FOUND",
+    "EVAL_RUN_NOT_FOUND",
+    "EVAL_TAGS_ON_PASSED_RESULT",
+    "EVAL_REAL_TOOLS_NOT_CONFIRMED",
+    "EVAL_CLASSIFIER_UNAVAILABLE",
+];
+export const EVAL_ERROR_CODES = [
+    ...EVAL_RESULT_ERROR_CODES,
+    ...EVAL_REQUEST_ERROR_CODES,
+];
+export const EVAL_LIMITS = {
+    maxCasesPerDataset: 5_000,
+    maxCaseBytes: 1_000_000,
+    maxUploadBytes: 50_000_000,
+    maxResultsPerRun: 5_000,
+    repeats: { min: 1, max: 10 },
+    concurrency: { min: 1, max: 8 },
+    labelMaxChars: 80,
+    failureTypeNameMaxChars: 60,
+    reviewNoteMaxChars: 500,
+};
+export const EVAL_RUN_ID_PREFIX = "er_";
+export const RUBRIC_PROMPT_VERSION = 1;
+export const CONFIG_SNAPSHOT_VERSION = 1;
+export const SIMULATOR_PROMPT_VERSION = 1;
+export const CONVERSATION_RUBRIC_PROMPT_VERSION = 2;
+/** Room for reasoning judges, whose hidden thinking counts against the output cap. */
+export const JUDGE_MAX_OUTPUT_TOKENS = 8_192;
+/** Bump when the failure-sorting questions, bucket criteria or judge prompt text change. */
+export const TRIAGE_PROMPT_VERSION = 1;
+export const EVAL_CONVERSATION_LIMITS = {
+    maxMessageChars: 4_000,
+    maxResultSummaryChars: 1_000,
+    maxArgsChars: 2_000,
+    maxMessages: 80,
+    keepFirstMessages: 4,
+    maxToolCalls: 200,
+    maxSubjectBytes: 256 * 1024,
+};
+export const JUDGE_TRANSCRIPT_LIMITS = {
+    turns: 30,
+    messageChars: 2_000,
+    toolArgsChars: 500,
+    toolResultChars: 500,
+};

package/dist/evals/engine/FakeEngine.d.ts

@@ -0,0 +1,33 @@
+import { type CaseOutput, type EngineIo, type EvalEngine, type EvalPlan, type PlanCase, type ResolvedGrader } from "../types.js";
+export type FakeJudgeReply = ({
+    verdict: "pass" | "fail";
+} | {
+    score: number;
+}) & {
+    reason?: string;
+};
+export type FakeEnginePhase = "stage_running" | "agent_running" | "grading" | "done";
+export type FakeEngineOptions = {
+    judge?: (grader: Extract<ResolvedGrader, {
+        type: "rubric";
+    }>, testCase: PlanCase, output: CaseOutput) => FakeJudgeReply;
+    judgeCostUsd?: number;
+    order?: "sequential" | "shuffled";
+    seed?: number;
+    delayMs?: Partial<Record<FakeEnginePhase, number>>;
+    failEngineAfter?: number;
+    codeTimeoutMs?: number;
+    version?: string;
+};
+export declare class FakeEngineError extends Error {
+    readonly code = "EVAL_ENGINE_ERROR";
+    constructor(message: string);
+}
+export declare class FakeEngine implements EvalEngine {
+    private readonly options;
+    readonly name = "fake";
+    constructor(options?: FakeEngineOptions);
+    version(): Promise<string>;
+    run(plan: EvalPlan, io: EngineIo, signal: AbortSignal): Promise<void>;
+    private gradeRubric;
+}

package/dist/evals/engine/FakeEngine.js

@@ -0,0 +1,177 @@
+import { caseGoal, goalRubricText, gradeDeterministic, isDeterministicGrader } from "../graders/index.js";
+import { isConversationTarget, } from "../types.js";
+export class FakeEngineError extends Error {
+    code = "EVAL_ENGINE_ERROR";
+    constructor(message) {
+        super(message);
+        this.name = "FakeEngineError";
+    }
+}
+function mulberry32(seed) {
+    let a = seed >>> 0;
+    return () => {
+        a = (a + 0x6d2b79f5) >>> 0;
+        let t = a;
+        t = Math.imul(t ^ (t >>> 15), t | 1);
+        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function toSubject(output) {
+    if (output.kind === "conversation") {
+        const { transcript: _transport, ...subject } = output;
+        return subject;
+    }
+    return {
+        status: output.status,
+        summary: output.summary,
+        ...(output.payload !== undefined ? { payload: output.payload } : {}),
+        artifacts: output.artifacts,
+    };
+}
+export class FakeEngine {
+    options;
+    name = "fake";
+    constructor(options = {}) {
+        this.options = options;
+    }
+    async version() {
+        return this.options.version ?? "fake";
+    }
+    async run(plan, io, signal) {
+        const opts = this.options;
+        const items = [];
+        for (const testCase of plan.cases) {
+            for (let repeatIndex = 0; repeatIndex < plan.repeats; repeatIndex++) {
+                items.push({ testCase, repeatIndex });
+            }
+        }
+        if (opts.order === "shuffled") {
+            const rand = mulberry32(opts.seed ?? 1);
+            for (let i = items.length - 1; i > 0; i--) {
+                const j = Math.floor(rand() * (i + 1));
+                [items[i], items[j]] = [items[j], items[i]];
+            }
+        }
+        let next = 0;
+        let emitted = 0;
+        let stop = false;
+        let failure;
+        const phase = async (testCase, repeatIndex, p) => {
+            io.onPhase(testCase.id, repeatIndex, p);
+            const delay = opts.delayMs?.[p];
+            if (delay)
+                await sleep(delay);
+        };
+        const worker = async () => {
+            while (!stop && !signal.aborted && next < items.length) {
+                if (opts.failEngineAfter !== undefined && emitted >= opts.failEngineAfter) {
+                    throw new FakeEngineError(`fake engine failed after ${emitted} results`);
+                }
+                const { testCase, repeatIndex } = items[next++];
+                await phase(testCase, repeatIndex, isConversationTarget(plan.target.kind) ? "agent_running" : "stage_running");
+                const run = await io.runCase({ evalRunId: plan.evalRunId, caseId: testCase.id, repeatIndex });
+                const grades = [];
+                if (run.ok) {
+                    io.onPhase(testCase.id, repeatIndex, "grading");
+                    if (opts.delayMs?.grading)
+                        await sleep(opts.delayMs.grading);
+                    const ctx = {
+                        evalId: plan.evalId,
+                        runId: plan.evalRunId,
+                        repeat: repeatIndex,
+                        passThreshold: plan.passThreshold,
+                        outputSchema: plan.outputSchema,
+                        ...(opts.codeTimeoutMs !== undefined ? { codeTimeoutMs: opts.codeTimeoutMs } : {}),
+                    };
+                    for (const grader of plan.graders) {
+                        if (grader.type === "rubric") {
+                            grades.push(this.gradeRubric(grader, testCase, run.output, io));
+                        }
+                        else if (grader.type === "goal_reached") {
+                            const goal = caseGoal(testCase);
+                            grades.push(goal === undefined
+                                ? {
+                                    graderId: grader.id,
+                                    graderType: "goal_reached",
+                                    status: "error",
+                                    errorCode: "EVAL_CASE_MISSING_EXPECTED",
+                                    reason: "case has no input.goal or expected.outcome",
+                                }
+                                : {
+                                    ...this.gradeRubric({
+                                        ...grader,
+                                        type: "rubric",
+                                        rubric: goalRubricText(goal, grader.guidance),
+                                        scale: "pass_fail",
+                                        passAt: 1,
+                                        examples: [],
+                                    }, testCase, run.output, io),
+                                    graderType: "goal_reached",
+                                });
+                        }
+                        else if (isDeterministicGrader(grader)) {
+                            grades.push(gradeDeterministic(grader, toSubject(run.output), testCase, ctx));
+                        }
+                    }
+                }
+                else if (run.code === "EVAL_BUDGET_EXCEEDED") {
+                    stop = true;
+                }
+                const result = { caseId: testCase.id, repeatIndex, run, grades };
+                io.onResult(result);
+                emitted++;
+                await phase(testCase, repeatIndex, "done");
+            }
+        };
+        const workers = Array.from({ length: Math.max(1, plan.concurrency) }, () => worker().catch((error) => {
+            stop = true;
+            failure ??= error;
+        }));
+        await Promise.all(workers);
+        if (failure !== undefined)
+            throw failure;
+    }
+    gradeRubric(grader, testCase, output, io) {
+        const base = { graderId: grader.id, graderType: "rubric", judgeModel: grader.judge };
+        const reply = this.options.judge?.(grader, testCase, output) ??
+            (grader.scale === "1_to_5" ? { score: 5 } : { verdict: "pass" });
+        const cost = this.options.judgeCostUsd ?? 0;
+        io.reportJudgeCost(cost);
+        const judged = { ...base, judgeCostUsd: cost };
+        const reason = reply.reason;
+        const bad = (why) => ({
+            ...judged,
+            status: "error",
+            errorCode: "EVAL_JUDGE_ERROR",
+            reason: why,
+        });
+        if (grader.scale === "1_to_5") {
+            if (!("score" in reply) || !Number.isFinite(reply.score) || reply.score < 1 || reply.score > 5) {
+                return bad("judge reply is not a score from 1 to 5");
+            }
+            return {
+                ...judged,
+                status: "graded",
+                rawValue: reply.score,
+                score: (reply.score - 1) / 4,
+                passed: reply.score >= grader.passAt,
+                ...(reason !== undefined ? { reason } : {}),
+            };
+        }
+        if (!("verdict" in reply))
+            return bad("judge reply is not a pass or fail verdict");
+        const passed = reply.verdict === "pass";
+        return {
+            ...judged,
+            status: "graded",
+            rawValue: passed ? 1 : 0,
+            score: passed ? 1 : 0,
+            passed,
+            ...(reason !== undefined ? { reason } : {}),
+        };
+    }
+}

package/dist/evals/engine/index.d.ts

@@ -0,0 +1,9 @@
+import type { EvalEngine } from "../types.js";
+export { FakeEngine, FakeEngineError } from "./FakeEngine.js";
+export type { FakeEngineOptions, FakeJudgeReply } from "./FakeEngine.js";
+export { PromptfooEngine } from "./promptfoo/PromptfooEngine.js";
+export type { PromptfooEngineOptions } from "./promptfoo/PromptfooEngine.js";
+export { RunTokenRegistry } from "./promptfoo/runTokens.js";
+export { judgeCapableModels, mapJudge } from "../judgeProviders.js";
+export declare function registerEngine(name: string, factory: (options?: unknown) => EvalEngine): void;
+export declare function getEngine(name: string, options?: unknown): EvalEngine;

package/dist/evals/engine/index.js

@@ -0,0 +1,19 @@
+import { FakeEngine } from "./FakeEngine.js";
+import { PromptfooEngine } from "./promptfoo/PromptfooEngine.js";
+export { FakeEngine, FakeEngineError } from "./FakeEngine.js";
+export { PromptfooEngine } from "./promptfoo/PromptfooEngine.js";
+export { RunTokenRegistry } from "./promptfoo/runTokens.js";
+export { judgeCapableModels, mapJudge } from "../judgeProviders.js";
+const factories = new Map([
+    ["fake", (options) => new FakeEngine(options)],
+    ["promptfoo", (options) => new PromptfooEngine(options)],
+]);
+export function registerEngine(name, factory) {
+    factories.set(name, factory);
+}
+export function getEngine(name, options) {
+    const factory = factories.get(name);
+    if (!factory)
+        throw new Error(`Unknown eval engine: ${name}`);
+    return factory(options);
+}

package/dist/evals/engine/promptfoo/PromptfooEngine.d.ts

@@ -0,0 +1,52 @@
+import { type ChildProcess, type SpawnOptions } from "node:child_process";
+import { type EngineIo, type EvalEngine, type EvalPlan } from "../../types.js";
+import { type JudgeKeyResolver, type JudgePriceLoader } from "./judgeServices.js";
+import type { RunTokenRegistry } from "./runTokens.js";
+export declare const PROMPTFOO_KILL_GRACE_MS = 5000;
+export declare const PROMPTFOO_STREAM_POLL_MS = 100;
+export declare const STDERR_TAIL_BYTES = 2048;
+export declare const KEEP_WORKDIR_ENV = "STAGEFLOW_EVALS_KEEP_WORKDIR";
+export declare const DISABLE_FLAGS: {
+    readonly PROMPTFOO_DISABLE_TELEMETRY: "1";
+    readonly PROMPTFOO_DISABLE_UPDATE: "1";
+    readonly PROMPTFOO_DISABLE_SHARING: "1";
+    readonly PROMPTFOO_DISABLE_REMOTE_GENERATION: "true";
+};
+export type SpawnFn = (command: string, args: string[], options: SpawnOptions) => ChildProcess;
+export type PromptfooEngineOptions = {
+    tokens: RunTokenRegistry;
+    callbackUrl: string | (() => string);
+    home?: string;
+    spawn?: SpawnFn;
+    resolveJudgeKey?: JudgeKeyResolver;
+    loadJudgePrices?: JudgePriceLoader;
+    graderModuleUrl?: string;
+    judgeProviderConfig?: Record<string, unknown>;
+    extraEnv?: Record<string, string>;
+    killGraceMs?: number;
+    streamPollMs?: number;
+    keepWorkDir?: boolean;
+    nodeVersion?: string;
+    binPath?: string;
+};
+export declare function quoteNodeOption(file: string): string;
+export declare function buildChildEnv(input: {
+    workDir: string;
+    guardPath: string;
+    callbackUrl: string;
+    token: string;
+    judgeEnv: Record<string, string>;
+    extraEnv?: Record<string, string>;
+    path?: string;
+}): Record<string, string>;
+export declare function buildPromptfooArgs(binPath: string, plan: Pick<EvalPlan, "repeats" | "concurrency">): string[];
+export declare function resolveGraderModuleUrl(): string;
+export declare class PromptfooEngine implements EvalEngine {
+    private readonly options;
+    readonly name = "promptfoo";
+    constructor(options: PromptfooEngineOptions);
+    private home;
+    version(): Promise<string>;
+    validatePlan(plan: EvalPlan): void;
+    run(plan: EvalPlan, io: EngineIo, signal: AbortSignal): Promise<void>;
+}