stageflow 0.42.0 → 0.46.0

package/dist/evals/plan.js

@@ -11,6 +11,9 @@ import { EVAL_LARGE_RUN, EVAL_TOO_LARGE_RUN, evalFinding, evalFindingsFromIssues
 import { canonicalJson, sha256Hex } from "./canonical.js";
 import { EVAL_LIMITS, SIMULATOR_PROMPT_VERSION } from "./constants.js";
 import { buildComponents, buildSnapshot, computeFingerprint, } from "./snapshot.js";
+import { planInternalAgent } from "./planInternalAgent.js";
+import { isConversationTarget } from "./types.js";
+import { isJudged, isJudgedType } from "./judged.js";
 const SELECTION_KEYS = ["case_ids", "tags", "only_failed_from"];
 const RUN_SIZE_CODES = new Set(["eval.large_run", "eval.too_large_run"]);
 function posix(projectRoot, abs) {
@@ -58,7 +61,7 @@ function checkConversationOverrides(relPath, file, overrides) {
     const keys = ["user_model", "tools", "approvals"].filter((key) => overrides[key] !== undefined);
     if (keys.length === 0)
         return out;
-    if (file.target.kind !== "agent") {
+    if (!isConversationTarget(file.target.kind)) {
         out.push(evalFinding(relPath, "error", "eval.conversation_not_applicable", `Override ${keys.map((k) => `"${k}"`).join(", ")} applies to agent targets only`));
         return out;
     }
@@ -104,10 +107,10 @@ function withOverrides(loaded, overrides) {
         ...(overrides.max_cost_usd !== undefined ? { max_cost_usd: overrides.max_cost_usd } : {}),
     };
     const graders = loaded.graders.map((grader) => {
-        if ((grader.type !== "rubric" && grader.type !== "goal_reached") || overrides.judge === undefined)
+        if (!isJudged(grader) || overrides.judge === undefined)
             return grader;
         const raw = loaded.file.graders.find((g) => g.id === grader.id);
-        if ((raw?.type === "rubric" || raw?.type === "goal_reached") && raw.judge !== undefined)
+        if (raw !== undefined && isJudgedType(raw.type) && "judge" in raw && raw.judge !== undefined)
             return grader;
         const { hash: _old, ...rest } = { ...grader, judge: overrides.judge };
         return { ...rest, hash: graderHash(rest) };
@@ -131,7 +134,7 @@ function selectCases(cases, overrides, failedIds) {
         return cases.filter((c) => failedIds.has(c.id));
     return [...cases];
 }
-function evalDefinition(loaded, projectRoot) {
+export function evalDefinition(loaded, projectRoot) {
     const { file } = loaded;
     const paths = resolveEvalPaths(file, loaded.path);
     const rel = (abs) => abs === undefined ? undefined : posix(projectRoot, abs);
@@ -145,12 +148,14 @@ function evalDefinition(loaded, projectRoot) {
         type: file.type ?? "capability",
         target: file.target.kind === "agent"
             ? { kind: "agent", agent: file.target.agent }
-            : {
-                kind: file.target.kind,
-                pipeline: rel(paths.pipelinePath),
-                stage: stageTarget?.stage,
-                file: rel(paths.stageFile),
-            },
+            : file.target.kind === "internal_agent"
+                ? { kind: "internal_agent", profile: file.target.profile }
+                : {
+                    kind: file.target.kind,
+                    pipeline: rel(paths.pipelinePath),
+                    stage: stageTarget?.stage,
+                    file: rel(paths.stageFile),
+                },
         dataset: rel(paths.datasetPath),
         task: file.task,
         defaults: {
@@ -160,9 +165,9 @@ function evalDefinition(loaded, projectRoot) {
             max_cost_usd: defaults.max_cost_usd ?? EVAL_DEFAULT_MAX_COST_USD,
             concurrency: defaults.concurrency ?? EVAL_DEFAULT_CONCURRENCY,
             model: defaults.model,
-            ...(file.target.kind === "agent" && defaults.user_model !== undefined ? { user_model: defaults.user_model } : {}),
+            ...(isConversationTarget(file.target.kind) && defaults.user_model !== undefined ? { user_model: defaults.user_model } : {}),
         },
-        ...(file.target.kind === "agent" ? { conversation: file.conversation } : {}),
+        ...(isConversationTarget(file.target.kind) ? { conversation: file.conversation } : {}),
         graders: file.graders.map((g) => ({ ...g, weight: g.weight ?? 1, must_pass: g.must_pass ?? false })),
         budget: file.budget,
     }));
@@ -227,6 +232,21 @@ export async function buildEvalPlan(input) {
     const findings = [...validation.filter((f) => !RUN_SIZE_CODES.has(f.code)), ...dataset.findings];
     if (findings.some((f) => f.severity === "error"))
         return fail(findings);
+    if (file.target.kind === "internal_agent") {
+        return planInternalAgent({
+            input,
+            loaded,
+            dataset: {
+                path: paths.datasetPath,
+                contentHash: dataset.contentHash,
+                format: dataset.format,
+                cases: dataset.cases,
+            },
+            findings,
+            overrides,
+            ...(failedIds ? { failedIds } : {}),
+        });
+    }
     let agentTarget;
     let stageTarget;
     if (file.target.kind === "agent") {
@@ -489,7 +509,7 @@ async function buildAgentResult(a) {
     });
     const fingerprint = computeFingerprint(components);
     const { overrides } = a;
-    const hasRubric = loaded.graders.some((g) => g.type === "rubric" || g.type === "goal_reached");
+    const hasRubric = loaded.graders.some(isJudged);
     const plainDefinition = JSON.parse(JSON.stringify(definition));
     const snapshot = buildSnapshot({
         evalId: loaded.id,

package/dist/evals/planInternalAgent.d.ts

@@ -0,0 +1,19 @@
+import type { DatasetCase } from "../config/datasetTypes.js";
+import { type LoadedEval } from "../config/loadEval.js";
+import type { ValidationFinding } from "../config/validateCatalog.js";
+import { type BuildEvalPlanInput, type BuildEvalPlanResult, type EvalOverrides } from "./plan.js";
+type Dataset = {
+    path: string;
+    contentHash: string;
+    format?: string;
+    cases: DatasetCase[];
+};
+export declare function planInternalAgent(args: {
+    input: BuildEvalPlanInput;
+    loaded: LoadedEval;
+    dataset: Dataset;
+    findings: ValidationFinding[];
+    overrides: EvalOverrides;
+    failedIds?: Set<string>;
+}): BuildEvalPlanResult;
+export {};

package/dist/evals/planInternalAgent.js

@@ -0,0 +1,201 @@
+import path from "node:path";
+import { checkDatasetAgainstAgent } from "../config/datasetAgentCheck.js";
+import { EVAL_AGENT_DEFAULT_CONCURRENCY, EVAL_AGENT_DEFAULT_MAX_COST_USD, EVAL_DEFAULT_PASS_THRESHOLD, EVAL_DEFAULT_REPEATS, resolveConversation, resolveEvalPaths, } from "../config/loadEval.js";
+import { EVAL_LARGE_RUN, EVAL_TOO_LARGE_RUN, evalFinding } from "../config/validateEval.js";
+import { PACKAGE_VERSION } from "../package-meta.js";
+import { sha256Hex } from "./canonical.js";
+import { EVAL_LIMITS, SIMULATOR_PROMPT_VERSION } from "./constants.js";
+import { evalDefinition } from "./plan.js";
+import { buildComponents, buildSnapshot, computeFingerprint } from "./snapshot.js";
+function posix(projectRoot, abs) {
+    return path.relative(projectRoot, abs).split(path.sep).join("/");
+}
+function selectCases(cases, overrides, failedIds) {
+    if (overrides.case_ids !== undefined)
+        return cases.filter((c) => overrides.case_ids.includes(c.id));
+    if (overrides.tags !== undefined) {
+        const wanted = new Set(overrides.tags);
+        return cases.filter((c) => (c.tags ?? []).some((tag) => wanted.has(tag)));
+    }
+    if (failedIds !== undefined)
+        return cases.filter((c) => failedIds.has(c.id));
+    return [...cases];
+}
+export function planInternalAgent(args) {
+    const { input, loaded, dataset, overrides, failedIds } = args;
+    const findings = [...args.findings];
+    const fail = (extra) => ({ ok: false, code: "EVAL_INVALID", findings: [...findings, ...extra] });
+    if (loaded.file.target.kind !== "internal_agent")
+        return fail([]);
+    const profileId = loaded.file.target.profile;
+    const profiles = input.internalAgents ?? [];
+    const profile = profiles.find((item) => item.id === profileId);
+    if (!profile) {
+        return fail([
+            evalFinding(loaded.relPath, "error", profiles.length === 0 ? "eval.internal_agent_unavailable" : "eval.internal_agent_unknown", profiles.length === 0
+                ? "This Host is not showing internal agents, so this eval cannot run"
+                : `This Host has no internal agent "${profileId}"`),
+        ]);
+    }
+    if (overrides.case_ids !== undefined) {
+        const known = new Set(dataset.cases.map((c) => c.id));
+        const unknown = overrides.case_ids.filter((id) => !known.has(id));
+        if (unknown.length > 0) {
+            return fail([
+                evalFinding(loaded.relPath, "error", "eval.invalid_field", `Override "case_ids" names ${unknown.length} case(s) not in the dataset; first: ${unknown[0]}`),
+            ]);
+        }
+    }
+    const conversation = resolveConversation(loaded.file.conversation);
+    if (conversation.tools === "real") {
+        return fail([
+            evalFinding(loaded.relPath, "error", "eval.internal_agent_real_tools", "An internal agent eval cannot use real tools"),
+        ]);
+    }
+    const checks = new Map(checkDatasetAgainstAgent(dataset.cases, { mode: conversation.mode }).map((r) => [r.caseId, r.valid]));
+    const validCases = dataset.cases.filter((c) => checks.get(c.id) !== false);
+    const selected = selectCases(dataset.cases, overrides, failedIds);
+    const skippedCaseIds = selected.filter((c) => checks.get(c.id) === false).map((c) => c.id);
+    const runCases = selected.filter((c) => checks.get(c.id) !== false);
+    const positionOf = new Map(dataset.cases.map((c, i) => [c.id, i]));
+    const planCases = runCases.map((c) => ({
+        id: c.id,
+        position: positionOf.get(c.id) ?? 0,
+        ...(c.title !== undefined ? { title: c.title } : {}),
+        ...(c.goal !== undefined ? { goal: c.goal } : {}),
+        input: c.input,
+        ...(c.expected !== undefined ? { expected: c.expected } : {}),
+        tags: c.tags ?? [],
+        ...(c.metadata !== undefined ? { metadata: c.metadata } : {}),
+    }));
+    const repeats = loaded.file.defaults?.repeats ?? EVAL_DEFAULT_REPEATS;
+    const results = planCases.length * repeats;
+    if (planCases.length === 0) {
+        return fail([evalFinding(loaded.relPath, "error", "eval.no_valid_cases", "No valid cases are left to run after case selection and skipping")]);
+    }
+    if (results > EVAL_LIMITS.maxResultsPerRun) {
+        return fail([
+            evalFinding(loaded.relPath, "error", "eval.too_large_run", `${planCases.length} case(s) x ${repeats} repeat(s) = ${results} results, over the ${EVAL_TOO_LARGE_RUN} limit`),
+        ]);
+    }
+    if (results > EVAL_LARGE_RUN) {
+        findings.push(evalFinding(loaded.relPath, "warning", "eval.large_run", `${planCases.length} case(s) x ${repeats} repeat(s) = ${results} results, over ${EVAL_LARGE_RUN}`));
+    }
+    const defaults = loaded.file.defaults ?? {};
+    const passThreshold = defaults.pass_threshold ?? EVAL_DEFAULT_PASS_THRESHOLD;
+    const concurrency = defaults.concurrency ?? EVAL_AGENT_DEFAULT_CONCURRENCY;
+    const maxCostUsd = defaults.max_cost_usd ?? EVAL_AGENT_DEFAULT_MAX_COST_USD;
+    const judge = defaults.judge ?? "";
+    const model = defaults.model ?? "";
+    const playbookHash = sha256Hex(profile.playbook);
+    const definitionHash = sha256Hex(JSON.stringify({ playbook: profile.playbook, tools: profile.tools }));
+    const userModel = planCases.some((c) => c.input.mode === "simulated") ? (defaults.user_model ?? null) : null;
+    const paths = resolveEvalPaths(loaded.file, loaded.path);
+    const projectRoot = input.projectRoot;
+    const components = buildComponents({
+        stageVersionHash: definitionHash,
+        model,
+        datasetHash: dataset.contentHash,
+        caseIds: runCases.length < validCases.length ? runCases.map((c) => c.id) : null,
+        graders: loaded.graders,
+        judge,
+        repeats,
+        passThreshold,
+        engine: input.engine,
+        agent: {
+            revision: 0,
+            mode: conversation.mode,
+            max_turns: conversation.maxTurns,
+            turn_timeout_ms: conversation.turnTimeoutMs,
+            tools: conversation.tools,
+            approvals: conversation.approvals,
+            read_only_tools: conversation.readOnlyTools,
+            user_model: userModel,
+            simulator_prompt_version: SIMULATOR_PROMPT_VERSION,
+        },
+    });
+    const fingerprint = computeFingerprint(components);
+    const snapshotOverrides = {
+        repeats: overrides.repeats ?? null,
+        judge: overrides.judge ?? null,
+        model: overrides.model ?? null,
+        max_cost_usd: overrides.max_cost_usd ?? null,
+        case_ids: overrides.case_ids ?? null,
+        tags: overrides.tags ?? null,
+        only_failed_from: overrides.only_failed_from ?? null,
+    };
+    const snapshot = buildSnapshot({
+        evalId: loaded.id,
+        evalPath: loaded.relPath,
+        evalHash: loaded.fileHash,
+        evalDefinition: evalDefinition(loaded, projectRoot),
+        overrides: snapshotOverrides,
+        target: {
+            profile_id: profileId,
+            title: profile.title,
+            playbook_hash: playbookHash,
+            tools: profile.tools,
+            model,
+        },
+        dataset: {
+            path: posix(projectRoot, paths.datasetPath),
+            hash: dataset.contentHash,
+            format: dataset.format ?? "jsonl",
+            case_count: dataset.cases.length,
+            cases_run: planCases.length,
+            cases_skipped: skippedCaseIds.length,
+            skipped_case_ids: skippedCaseIds,
+        },
+        graders: loaded.graders.map((g) => ({
+            id: g.id,
+            type: g.type,
+            hash: g.hash,
+            version_ordinal: input.graderOrdinals?.[g.id] ?? 1,
+            weight: g.weight,
+            must_pass: g.mustPass,
+        })),
+        settings: {
+            repeats,
+            pass_threshold: passThreshold,
+            judge,
+            judge_same_as_stage: false,
+            max_cost_usd: maxCostUsd,
+            concurrency,
+            suite_type: loaded.file.type ?? "capability",
+            conversation: {
+                mode: conversation.mode,
+                max_turns: conversation.maxTurns,
+                turn_timeout_ms: conversation.turnTimeoutMs,
+                tools: conversation.tools,
+                approvals: conversation.approvals,
+                read_only_tools: conversation.readOnlyTools,
+                real_tools_confirmed: false,
+                user_model: userModel ?? "",
+                user_model_from_judge: false,
+                simulator_prompt_version: SIMULATOR_PROMPT_VERSION,
+            },
+        },
+        engine: input.engine,
+        stageflowVersion: PACKAGE_VERSION,
+    });
+    const plan = {
+        evalRunId: input.evalRunId,
+        projectRoot,
+        evalId: loaded.id,
+        target: { kind: "internal_agent", profileId },
+        targetVersionHash: definitionHash,
+        stageModel: model,
+        task: {},
+        cases: planCases,
+        graders: loaded.graders,
+        repeats,
+        passThreshold,
+        concurrency,
+        maxCostUsd,
+        ...(model !== "" ? { modelOverride: model } : {}),
+        conversation,
+        userModel,
+        agent: { definitionHash, definition: { id: profileId, title: profile.title, tools: profile.tools } },
+    };
+    return { ok: true, plan, snapshot, fingerprint, components, skippedCaseIds, findings };
+}

package/dist/evals/read/api.d.ts

@@ -54,6 +54,12 @@ export declare function planFindings(input: {
     datasetCases?: readonly unknown[];
     isKnownModel?: IsKnownModel;
     agents?: AgentCatalogPort;
+    internalAgents?: ReadonlyArray<{
+        id: string;
+        title: string;
+        playbook: string;
+        tools: string[];
+    }>;
 }): Promise<{
     ok: boolean;
     findings: ValidationFinding[];
@@ -65,6 +71,12 @@ export declare function validateEvalText(input: {
     datasetCases?: readonly unknown[];
     isKnownModel?: IsKnownModel;
     agents?: AgentCatalogPort;
+    internalAgents?: ReadonlyArray<{
+        id: string;
+        title: string;
+        playbook: string;
+        tools: string[];
+    }>;
 }): Promise<{
     findings: ValidationFinding[];
     parsed: import("../../config/loadOutcome.js").LoadOutcome<LoadedEval>;
@@ -147,6 +159,12 @@ export declare function buildResultDetail(store: RunStore, id: string, caseId: s
         } | null;
         tags: string[];
         suggestion: import("./suggestions.js").WireSuggestion | null;
+        grader_labels: {
+            [k: string]: {
+                label: "fail" | "pass";
+                note: string | null;
+            };
+        };
     }[];
 }>;
 export declare function loadEvalListItems(input: {

package/dist/evals/read/api.js

@@ -99,6 +99,9 @@ export async function buildEvalDetail(input) {
             agent = await liveAgentOf(input.agents, projectRoot, file.target.agent);
             targetModel = file.defaults?.model ?? agent?.model ?? null;
         }
+        else if (file.target.kind === "internal_agent") {
+            targetModel = file.defaults?.model ?? null;
+        }
         else {
             const target = await resolveEvalTarget(file, entry.outcome.value.path, projectRoot);
             if (target.ok)
@@ -130,6 +133,7 @@ export async function planFindings(input) {
         engine: { name: ENGINE_NAME, version: PINNED_PROMPTFOO_VERSION, nodeVersion: process.versions.node },
         ...(input.isKnownModel ? { isKnownModel: input.isKnownModel } : {}),
         ...(input.agents ? { agents: input.agents } : {}),
+        ...(input.internalAgents ? { internalAgents: input.internalAgents } : {}),
     });
     return { ok: built.ok, findings: built.findings };
 }
@@ -143,6 +147,7 @@ export async function validateEvalText(input) {
         ...(input.datasetCases !== undefined ? { datasetCases: input.datasetCases } : {}),
         ...(input.isKnownModel ? { isKnownModel: input.isKnownModel } : {}),
         ...(input.agents ? { agents: input.agents } : {}),
+        ...(input.internalAgents ? { internalAgents: input.internalAgents } : {}),
     });
     const findings = checked.findings.length > 0 || parsed.ok
         ? checked.findings
@@ -222,11 +227,12 @@ export async function buildResultsPage(store, id, input) {
 }
 export async function buildResultDetail(store, id, caseId) {
     const row = await runByIdOr404(store, id);
-    const [cases, results, reviews, suggestions] = await Promise.all([
+    const [cases, results, reviews, suggestions, graderLabels] = await Promise.all([
         store.getEvalCases(row.evalRunId),
         store.getEvalResultsForCase(row.evalRunId, caseId),
         store.getEvalReviews(row.evalRunId),
         store.listEvalSuggestions(row.evalRunId),
+        store.listEvalGraderLabels(row.evalRunId),
     ]);
     const caseRow = cases.find((x) => x.caseId === caseId);
     if (!caseRow || results.length === 0) {
@@ -240,6 +246,9 @@ export async function buildResultDetail(store, id, caseId) {
             .map((r) => ({
             ...resultToWire(r, reviewByKey.get(resultKey(r.caseId, r.repeatIndex)), row.targetKind === "agent" ? (row.targetRef.agent_id ?? "") : undefined),
             suggestion: suggestionFor(suggestions, r.caseId, r.repeatIndex),
+            grader_labels: Object.fromEntries(graderLabels
+                .filter((l) => l.caseId === r.caseId && l.repeatIndex === r.repeatIndex)
+                .map((l) => [l.graderId, { label: l.label, note: l.note }])),
         })),
     };
 }

package/dist/evals/read/labelQueue.d.ts

@@ -0,0 +1,45 @@
+import type { RunStore } from "../../runstore/port.js";
+export type LabelCounts = {
+    pass: number;
+    fail: number;
+};
+/**
+ * Everything the console's labeling screen needs in one read: each result the judge graded for one grader, with the
+ * conversation, what the case expects, the judge's answer and the author's label so far. Results the judge errored on
+ * are left out (there is no verdict to compare with), and so are results that never ran.
+ */
+export declare function buildLabelQueue(store: RunStore, id: string, graderId: string): Promise<{
+    grader_id: string;
+    items: {
+        case: {
+            position: number;
+        };
+        result: import("./runs.js").WireResultDetail;
+        judge: {
+            passed: boolean;
+            score: number | null;
+            reason: string | null;
+        };
+        label: {
+            label: "fail" | "pass";
+            note: string | null;
+        } | null;
+    }[];
+    labeled_here: LabelCounts;
+    labeled_eval: LabelCounts;
+}>;
+/**
+ * The label counts across the eval's runs, and which results of this run the judge graded that still have no label
+ * from you. No conversations: this is for a progress line and the Cases filter.
+ */
+export declare function buildLabelSummary(store: RunStore, id: string, graderId: string): Promise<{
+    grader_id: string;
+    labeled_eval: {
+        pass: number;
+        fail: number;
+    };
+    unlabeled: {
+        case_id: string;
+        repeat_index: number;
+    }[];
+}>;

package/dist/evals/read/labelQueue.js

@@ -0,0 +1,83 @@
+import { resultKey } from "../statsTypes.js";
+import { EvalApiError, runByIdOr404 } from "./api.js";
+import { resultToWire } from "./runs.js";
+import { isJudgedType } from "../judged.js";
+/**
+ * Everything the console's labeling screen needs in one read: each result the judge graded for one grader, with the
+ * conversation, what the case expects, the judge's answer and the author's label so far. Results the judge errored on
+ * are left out (there is no verdict to compare with), and so are results that never ran.
+ */
+export async function buildLabelQueue(store, id, graderId) {
+    const row = await runByIdOr404(store, id);
+    const [cases, results, labels, evalLabels] = await Promise.all([
+        store.getEvalCases(row.evalRunId),
+        store.listEvalResults(row.evalRunId),
+        store.listEvalGraderLabels(row.evalRunId),
+        store.listEvalGraderLabelsForEval(row.projectRoot, row.evalId, graderId),
+    ]);
+    const caseById = new Map(cases.map((c) => [c.caseId, c]));
+    const labelByKey = new Map(labels.filter((l) => l.graderId === graderId).map((l) => [resultKey(l.caseId, l.repeatIndex), l]));
+    const gradeOf = (grades) => grades.find((g) => g.graderId === graderId);
+    const known = results.some((r) => gradeOf(r.grades) !== undefined);
+    if (!known) {
+        throw new EvalApiError(404, "EVAL_GRADER_NOT_FOUND", `Grader ${graderId} has no grades in run ${id}`);
+    }
+    const judged = results.find((r) => {
+        const g = gradeOf(r.grades);
+        return g !== undefined && isJudgedType(g.graderType);
+    });
+    if (!judged) {
+        throw new EvalApiError(400, "invalid_request", `Grader ${graderId} is not judged by a model, so it has nothing to label`);
+    }
+    const items = [...results]
+        .sort((a, b) => (caseById.get(a.caseId)?.position ?? 0) - (caseById.get(b.caseId)?.position ?? 0) || a.repeatIndex - b.repeatIndex)
+        .flatMap((r) => {
+        const grade = gradeOf(r.grades);
+        const caseRow = caseById.get(r.caseId);
+        if (!grade || grade.status !== "graded" || !caseRow)
+            return [];
+        const label = labelByKey.get(resultKey(r.caseId, r.repeatIndex));
+        return [
+            {
+                case: { ...caseRow.case, position: caseRow.position },
+                result: resultToWire(r, undefined),
+                judge: { passed: grade.passed === true, score: grade.score, reason: grade.reason },
+                label: label ? { label: label.label, note: label.note } : null,
+            },
+        ];
+    });
+    const count = (rows) => ({
+        pass: rows.filter((l) => l.label === "pass").length,
+        fail: rows.filter((l) => l.label === "fail").length,
+    });
+    return {
+        grader_id: graderId,
+        items,
+        labeled_here: count(labels.filter((l) => l.graderId === graderId)),
+        labeled_eval: count(evalLabels),
+    };
+}
+/**
+ * The label counts across the eval's runs, and which results of this run the judge graded that still have no label
+ * from you. No conversations: this is for a progress line and the Cases filter.
+ */
+export async function buildLabelSummary(store, id, graderId) {
+    const row = await runByIdOr404(store, id);
+    const [labels, runLabels, graded] = await Promise.all([
+        store.listEvalGraderLabelsForEval(row.projectRoot, row.evalId, graderId),
+        store.listEvalGraderLabels(row.evalRunId),
+        store.listGradedResultKeys(row.evalRunId, graderId),
+    ]);
+    const labeled = new Set(runLabels.filter((l) => l.graderId === graderId).map((l) => resultKey(l.caseId, l.repeatIndex)));
+    const unlabeled = graded
+        .filter((r) => !labeled.has(resultKey(r.caseId, r.repeatIndex)))
+        .map((r) => ({ case_id: r.caseId, repeat_index: r.repeatIndex }));
+    return {
+        grader_id: graderId,
+        labeled_eval: {
+            pass: labels.filter((l) => l.label === "pass").length,
+            fail: labels.filter((l) => l.label === "fail").length,
+        },
+        unlabeled,
+    };
+}

package/dist/evals/read/list.js

@@ -103,7 +103,16 @@ export async function buildEvalListItems(input) {
         bannerInputs.push({ evalId: loaded.id, status, last: lastSummary, previous: previousSummary });
         const agentTarget = file.target.kind === "agent"
             ? await agentListTarget(file.target.agent, last, input.agents, projectRoot)
-            : undefined;
+            : file.target.kind === "internal_agent"
+                ? {
+                    kind: "agent",
+                    agent_id: file.target.profile,
+                    agent_name: file.target.profile,
+                    model: file.defaults?.model ?? null,
+                    revision: null,
+                    label: `${file.target.profile} (internal)`,
+                }
+                : undefined;
         const conv = agentTarget ? resolveConversation(file.conversation) : undefined;
         const pipelinePath = paths.pipelinePath !== undefined ? posix(projectRoot, paths.pipelinePath) : undefined;
         const stageFile = paths.stageFile !== undefined ? posix(projectRoot, paths.stageFile) : undefined;

package/dist/evals/read/runs.d.ts

@@ -106,6 +106,11 @@ export type EvalRunDetail = {
     stats: Stats | null;
     lowest_cases_by_grader: Record<string, LowestCase[]>;
     current: RunCurrent;
+    cases: Array<{
+        id: string;
+        title: string | null;
+        position: number;
+    }>;
     live?: Array<{
         case_id: string;
         repeat_index: number;

package/dist/evals/read/runs.js

@@ -218,11 +218,16 @@ export async function currentHashes(row, agents) {
         return none;
     }
 }
+function plannedCase(row) {
+    const title = row.case.title;
+    return { id: row.caseId, title: typeof title === "string" && title !== "" ? title : null, position: row.position };
+}
 export async function buildRunDetail(input) {
     const { store, row } = input;
-    const [results, reviews] = await Promise.all([
+    const [results, reviews, storedCases] = await Promise.all([
         store.listEvalResults(row.evalRunId),
         store.getEvalReviews(row.evalRunId),
+        store.getEvalCases(row.evalRunId),
     ]);
     const graders = graderRefsOf(row);
     const grades = results.flatMap(toGradeRecords);
@@ -258,6 +263,7 @@ export async function buildRunDetail(input) {
             ? Object.fromEntries(graders.map((g) => [g.id, lowestCases(grades.filter((x) => x.graderId === g.id))]))
             : {},
         current: await currentHashes(row, input.agents),
+        cases: storedCases.map(plannedCase),
     };
     if (input.live && (row.status === "running" || row.status === "queued")) {
         detail.live = input.live.phases.map((p) => ({

package/dist/evals/runner/caseRunner.d.ts

@@ -1,7 +1,8 @@
 import type { StageCaseRunner } from "./stageCaseRunner.js";
 export type EvalCaseRunner = StageCaseRunner;
-/** Routes each case to the stage or agent runner by the plan's target kind. */
+/** Routes each case to the stage, agent, or internal-agent runner by the plan's target kind. */
 export declare function createEvalCaseRunner(deps: {
     stage: StageCaseRunner;
     agent: StageCaseRunner;
+    internalAgent?: StageCaseRunner;
 }): EvalCaseRunner;

package/dist/evals/runner/caseRunner.js

@@ -1,16 +1,27 @@
-/** Routes each case to the stage or agent runner by the plan's target kind. */
+import { isConversationTarget } from "../types.js";
+/** Routes each case to the stage, agent, or internal-agent runner by the plan's target kind. */
 export function createEvalCaseRunner(deps) {
     const kinds = new Map();
-    const runnerOf = (evalRunId) => (kinds.get(evalRunId) === "agent" ? deps.agent : deps.stage);
+    const runnerOf = (evalRunId) => {
+        const kind = kinds.get(evalRunId);
+        if (kind === "internal_agent" && deps.internalAgent)
+            return deps.internalAgent;
+        return kind === "agent" ? deps.agent : deps.stage;
+    };
     return {
         runCase(plan, req) {
-            kinds.set(req.evalRunId, plan.target.kind === "agent" ? "agent" : "stage");
+            const kind = plan.target.kind === "internal_agent" ? "internal_agent" : isConversationTarget(plan.target.kind) ? "agent" : "stage";
+            kinds.set(req.evalRunId, kind);
             return runnerOf(req.evalRunId).runCase(plan, req);
         },
         async cancelAll(evalRunId) {
-            await Promise.all([deps.stage.cancelAll(evalRunId), deps.agent.cancelAll(evalRunId)]);
+            await Promise.all([
+                deps.stage.cancelAll(evalRunId),
+                deps.agent.cancelAll(evalRunId),
+                deps.internalAgent?.cancelAll(evalRunId),
+            ]);
         },
-        spend: (evalRunId) => deps.stage.spend(evalRunId) + deps.agent.spend(evalRunId),
+        spend: (evalRunId) => deps.stage.spend(evalRunId) + deps.agent.spend(evalRunId) + (deps.internalAgent?.spend(evalRunId) ?? 0),
         addSpend: (evalRunId, usd) => runnerOf(evalRunId).addSpend(evalRunId, usd),
         stopReason: (evalRunId) => runnerOf(evalRunId).stopReason?.(evalRunId),
     };

package/dist/evals/runner/internalAgentCaseRunner.d.ts

@@ -0,0 +1,6 @@
+import type { OperatorAgentHost } from "../../operatorAgent/types.js";
+import type { StageCaseRunner } from "./stageCaseRunner.js";
+export declare function createInternalAgentCaseRunner(deps: {
+    host: () => OperatorAgentHost | undefined;
+    now?: () => number;
+}): StageCaseRunner;