stageflow 0.42.0 → 0.44.0

package/dist/evals/planInternalAgent.js

@@ -0,0 +1,205 @@
+import path from "node:path";
+import { checkDatasetAgainstAgent } from "../config/datasetAgentCheck.js";
+import { EVAL_AGENT_DEFAULT_CONCURRENCY, EVAL_AGENT_DEFAULT_MAX_COST_USD, EVAL_DEFAULT_PASS_THRESHOLD, EVAL_DEFAULT_REPEATS, resolveConversation, resolveEvalPaths, } from "../config/loadEval.js";
+import { EVAL_LARGE_RUN, EVAL_TOO_LARGE_RUN, evalFinding } from "../config/validateEval.js";
+import { PACKAGE_VERSION } from "../package-meta.js";
+import { sha256Hex } from "./canonical.js";
+import { EVAL_LIMITS, SIMULATOR_PROMPT_VERSION } from "./constants.js";
+import { buildComponents, buildSnapshot, computeFingerprint } from "./snapshot.js";
+function posix(projectRoot, abs) {
+    return path.relative(projectRoot, abs).split(path.sep).join("/");
+}
+function selectCases(cases, overrides, failedIds) {
+    if (overrides.case_ids !== undefined)
+        return cases.filter((c) => overrides.case_ids.includes(c.id));
+    if (overrides.tags !== undefined) {
+        const wanted = new Set(overrides.tags);
+        return cases.filter((c) => (c.tags ?? []).some((tag) => wanted.has(tag)));
+    }
+    if (failedIds !== undefined)
+        return cases.filter((c) => failedIds.has(c.id));
+    return [...cases];
+}
+export function planInternalAgent(args) {
+    const { input, loaded, dataset, overrides, failedIds } = args;
+    const findings = [...args.findings];
+    const fail = (extra) => ({ ok: false, code: "EVAL_INVALID", findings: [...findings, ...extra] });
+    if (loaded.file.target.kind !== "internal_agent")
+        return fail([]);
+    const profileId = loaded.file.target.profile;
+    const profiles = input.internalAgents ?? [];
+    const profile = profiles.find((item) => item.id === profileId);
+    if (!profile) {
+        return fail([
+            evalFinding(loaded.relPath, "error", profiles.length === 0 ? "eval.internal_agent_unavailable" : "eval.internal_agent_unknown", profiles.length === 0
+                ? "This Host is not showing internal agents, so this eval cannot run"
+                : `This Host has no internal agent "${profileId}"`),
+        ]);
+    }
+    if (overrides.case_ids !== undefined) {
+        const known = new Set(dataset.cases.map((c) => c.id));
+        const unknown = overrides.case_ids.filter((id) => !known.has(id));
+        if (unknown.length > 0) {
+            return fail([
+                evalFinding(loaded.relPath, "error", "eval.invalid_field", `Override "case_ids" names ${unknown.length} case(s) not in the dataset; first: ${unknown[0]}`),
+            ]);
+        }
+    }
+    const conversation = resolveConversation(loaded.file.conversation);
+    if (conversation.tools === "real") {
+        return fail([
+            evalFinding(loaded.relPath, "error", "eval.internal_agent_real_tools", "An internal agent eval cannot use real tools"),
+        ]);
+    }
+    const checks = new Map(checkDatasetAgainstAgent(dataset.cases, { mode: conversation.mode }).map((r) => [r.caseId, r.valid]));
+    const validCases = dataset.cases.filter((c) => checks.get(c.id) !== false);
+    const selected = selectCases(dataset.cases, overrides, failedIds);
+    const skippedCaseIds = selected.filter((c) => checks.get(c.id) === false).map((c) => c.id);
+    const runCases = selected.filter((c) => checks.get(c.id) !== false);
+    const positionOf = new Map(dataset.cases.map((c, i) => [c.id, i]));
+    const planCases = runCases.map((c) => ({
+        id: c.id,
+        position: positionOf.get(c.id) ?? 0,
+        ...(c.title !== undefined ? { title: c.title } : {}),
+        ...(c.goal !== undefined ? { goal: c.goal } : {}),
+        input: c.input,
+        ...(c.expected !== undefined ? { expected: c.expected } : {}),
+        tags: c.tags ?? [],
+        ...(c.metadata !== undefined ? { metadata: c.metadata } : {}),
+    }));
+    const repeats = loaded.file.defaults?.repeats ?? EVAL_DEFAULT_REPEATS;
+    const results = planCases.length * repeats;
+    if (planCases.length === 0) {
+        return fail([evalFinding(loaded.relPath, "error", "eval.no_valid_cases", "No valid cases are left to run after case selection and skipping")]);
+    }
+    if (results > EVAL_LIMITS.maxResultsPerRun) {
+        return fail([
+            evalFinding(loaded.relPath, "error", "eval.too_large_run", `${planCases.length} case(s) x ${repeats} repeat(s) = ${results} results, over the ${EVAL_TOO_LARGE_RUN} limit`),
+        ]);
+    }
+    if (results > EVAL_LARGE_RUN) {
+        findings.push(evalFinding(loaded.relPath, "warning", "eval.large_run", `${planCases.length} case(s) x ${repeats} repeat(s) = ${results} results, over ${EVAL_LARGE_RUN}`));
+    }
+    const defaults = loaded.file.defaults ?? {};
+    const passThreshold = defaults.pass_threshold ?? EVAL_DEFAULT_PASS_THRESHOLD;
+    const concurrency = defaults.concurrency ?? EVAL_AGENT_DEFAULT_CONCURRENCY;
+    const maxCostUsd = defaults.max_cost_usd ?? EVAL_AGENT_DEFAULT_MAX_COST_USD;
+    const judge = defaults.judge ?? "";
+    const model = defaults.model ?? "";
+    const playbookHash = sha256Hex(profile.playbook);
+    const definitionHash = sha256Hex(JSON.stringify({ playbook: profile.playbook, tools: profile.tools }));
+    const userModel = planCases.some((c) => c.input.mode === "simulated") ? (defaults.user_model ?? null) : null;
+    const paths = resolveEvalPaths(loaded.file, loaded.path);
+    const projectRoot = input.projectRoot;
+    const components = buildComponents({
+        stageVersionHash: definitionHash,
+        model,
+        datasetHash: dataset.contentHash,
+        caseIds: runCases.length < validCases.length ? runCases.map((c) => c.id) : null,
+        graders: loaded.graders,
+        judge,
+        repeats,
+        passThreshold,
+        engine: input.engine,
+        agent: {
+            revision: 0,
+            mode: conversation.mode,
+            max_turns: conversation.maxTurns,
+            turn_timeout_ms: conversation.turnTimeoutMs,
+            tools: conversation.tools,
+            approvals: conversation.approvals,
+            read_only_tools: conversation.readOnlyTools,
+            user_model: userModel,
+            simulator_prompt_version: SIMULATOR_PROMPT_VERSION,
+        },
+    });
+    const fingerprint = computeFingerprint(components);
+    const snapshotOverrides = {
+        repeats: overrides.repeats ?? null,
+        judge: overrides.judge ?? null,
+        model: overrides.model ?? null,
+        max_cost_usd: overrides.max_cost_usd ?? null,
+        case_ids: overrides.case_ids ?? null,
+        tags: overrides.tags ?? null,
+        only_failed_from: overrides.only_failed_from ?? null,
+    };
+    const snapshot = buildSnapshot({
+        evalId: loaded.id,
+        evalPath: loaded.relPath,
+        evalHash: loaded.fileHash,
+        evalDefinition: {
+            version: 1,
+            id: loaded.id,
+            target: { kind: "internal_agent", profile: profileId },
+            dataset: posix(projectRoot, paths.datasetPath),
+        },
+        overrides: snapshotOverrides,
+        target: {
+            profile_id: profileId,
+            title: profile.title,
+            playbook_hash: playbookHash,
+            tools: profile.tools,
+            model,
+        },
+        dataset: {
+            path: posix(projectRoot, paths.datasetPath),
+            hash: dataset.contentHash,
+            format: dataset.format ?? "jsonl",
+            case_count: dataset.cases.length,
+            cases_run: planCases.length,
+            cases_skipped: skippedCaseIds.length,
+            skipped_case_ids: skippedCaseIds,
+        },
+        graders: loaded.graders.map((g) => ({
+            id: g.id,
+            type: g.type,
+            hash: g.hash,
+            version_ordinal: input.graderOrdinals?.[g.id] ?? 1,
+            weight: g.weight,
+            must_pass: g.mustPass,
+        })),
+        settings: {
+            repeats,
+            pass_threshold: passThreshold,
+            judge,
+            judge_same_as_stage: false,
+            max_cost_usd: maxCostUsd,
+            concurrency,
+            suite_type: loaded.file.type ?? "capability",
+            conversation: {
+                mode: conversation.mode,
+                max_turns: conversation.maxTurns,
+                turn_timeout_ms: conversation.turnTimeoutMs,
+                tools: conversation.tools,
+                approvals: conversation.approvals,
+                read_only_tools: conversation.readOnlyTools,
+                real_tools_confirmed: false,
+                user_model: userModel ?? "",
+                user_model_from_judge: false,
+                simulator_prompt_version: SIMULATOR_PROMPT_VERSION,
+            },
+        },
+        engine: input.engine,
+        stageflowVersion: PACKAGE_VERSION,
+    });
+    const plan = {
+        evalRunId: input.evalRunId,
+        projectRoot,
+        evalId: loaded.id,
+        target: { kind: "internal_agent", profileId },
+        targetVersionHash: definitionHash,
+        stageModel: model,
+        task: {},
+        cases: planCases,
+        graders: loaded.graders,
+        repeats,
+        passThreshold,
+        concurrency,
+        maxCostUsd,
+        ...(model !== "" ? { modelOverride: model } : {}),
+        conversation,
+        userModel,
+        agent: { definitionHash, definition: { id: profileId, title: profile.title, tools: profile.tools } },
+    };
+    return { ok: true, plan, snapshot, fingerprint, components, skippedCaseIds, findings };
+}

package/dist/evals/read/api.d.ts

@@ -54,6 +54,12 @@ export declare function planFindings(input: {
     datasetCases?: readonly unknown[];
     isKnownModel?: IsKnownModel;
     agents?: AgentCatalogPort;
+    internalAgents?: ReadonlyArray<{
+        id: string;
+        title: string;
+        playbook: string;
+        tools: string[];
+    }>;
 }): Promise<{
     ok: boolean;
     findings: ValidationFinding[];
@@ -65,6 +71,12 @@ export declare function validateEvalText(input: {
     datasetCases?: readonly unknown[];
     isKnownModel?: IsKnownModel;
     agents?: AgentCatalogPort;
+    internalAgents?: ReadonlyArray<{
+        id: string;
+        title: string;
+        playbook: string;
+        tools: string[];
+    }>;
 }): Promise<{
     findings: ValidationFinding[];
     parsed: import("../../config/loadOutcome.js").LoadOutcome<LoadedEval>;

package/dist/evals/read/api.js

@@ -99,6 +99,9 @@ export async function buildEvalDetail(input) {
             agent = await liveAgentOf(input.agents, projectRoot, file.target.agent);
             targetModel = file.defaults?.model ?? agent?.model ?? null;
         }
+        else if (file.target.kind === "internal_agent") {
+            targetModel = file.defaults?.model ?? null;
+        }
         else {
             const target = await resolveEvalTarget(file, entry.outcome.value.path, projectRoot);
             if (target.ok)
@@ -130,6 +133,7 @@ export async function planFindings(input) {
         engine: { name: ENGINE_NAME, version: PINNED_PROMPTFOO_VERSION, nodeVersion: process.versions.node },
         ...(input.isKnownModel ? { isKnownModel: input.isKnownModel } : {}),
         ...(input.agents ? { agents: input.agents } : {}),
+        ...(input.internalAgents ? { internalAgents: input.internalAgents } : {}),
     });
     return { ok: built.ok, findings: built.findings };
 }
@@ -143,6 +147,7 @@ export async function validateEvalText(input) {
         ...(input.datasetCases !== undefined ? { datasetCases: input.datasetCases } : {}),
         ...(input.isKnownModel ? { isKnownModel: input.isKnownModel } : {}),
         ...(input.agents ? { agents: input.agents } : {}),
+        ...(input.internalAgents ? { internalAgents: input.internalAgents } : {}),
     });
     const findings = checked.findings.length > 0 || parsed.ok
         ? checked.findings

package/dist/evals/read/list.js

@@ -103,7 +103,16 @@ export async function buildEvalListItems(input) {
         bannerInputs.push({ evalId: loaded.id, status, last: lastSummary, previous: previousSummary });
         const agentTarget = file.target.kind === "agent"
             ? await agentListTarget(file.target.agent, last, input.agents, projectRoot)
-            : undefined;
+            : file.target.kind === "internal_agent"
+                ? {
+                    kind: "agent",
+                    agent_id: file.target.profile,
+                    agent_name: file.target.profile,
+                    model: file.defaults?.model ?? null,
+                    revision: null,
+                    label: `${file.target.profile} (internal)`,
+                }
+                : undefined;
         const conv = agentTarget ? resolveConversation(file.conversation) : undefined;
         const pipelinePath = paths.pipelinePath !== undefined ? posix(projectRoot, paths.pipelinePath) : undefined;
         const stageFile = paths.stageFile !== undefined ? posix(projectRoot, paths.stageFile) : undefined;

package/dist/evals/read/runs.d.ts

@@ -106,6 +106,11 @@ export type EvalRunDetail = {
     stats: Stats | null;
     lowest_cases_by_grader: Record<string, LowestCase[]>;
     current: RunCurrent;
+    cases: Array<{
+        id: string;
+        title: string | null;
+        position: number;
+    }>;
     live?: Array<{
         case_id: string;
         repeat_index: number;

package/dist/evals/read/runs.js

@@ -218,11 +218,16 @@ export async function currentHashes(row, agents) {
         return none;
     }
 }
+function plannedCase(row) {
+    const title = row.case.title;
+    return { id: row.caseId, title: typeof title === "string" && title !== "" ? title : null, position: row.position };
+}
 export async function buildRunDetail(input) {
     const { store, row } = input;
-    const [results, reviews] = await Promise.all([
+    const [results, reviews, storedCases] = await Promise.all([
         store.listEvalResults(row.evalRunId),
         store.getEvalReviews(row.evalRunId),
+        store.getEvalCases(row.evalRunId),
     ]);
     const graders = graderRefsOf(row);
     const grades = results.flatMap(toGradeRecords);
@@ -258,6 +263,7 @@ export async function buildRunDetail(input) {
             ? Object.fromEntries(graders.map((g) => [g.id, lowestCases(grades.filter((x) => x.graderId === g.id))]))
             : {},
         current: await currentHashes(row, input.agents),
+        cases: storedCases.map(plannedCase),
     };
     if (input.live && (row.status === "running" || row.status === "queued")) {
         detail.live = input.live.phases.map((p) => ({

package/dist/evals/runner/caseRunner.d.ts

@@ -1,7 +1,8 @@
 import type { StageCaseRunner } from "./stageCaseRunner.js";
 export type EvalCaseRunner = StageCaseRunner;
-/** Routes each case to the stage or agent runner by the plan's target kind. */
+/** Routes each case to the stage, agent, or internal-agent runner by the plan's target kind. */
 export declare function createEvalCaseRunner(deps: {
     stage: StageCaseRunner;
     agent: StageCaseRunner;
+    internalAgent?: StageCaseRunner;
 }): EvalCaseRunner;

package/dist/evals/runner/caseRunner.js

@@ -1,16 +1,27 @@
-/** Routes each case to the stage or agent runner by the plan's target kind. */
+import { isConversationTarget } from "../types.js";
+/** Routes each case to the stage, agent, or internal-agent runner by the plan's target kind. */
 export function createEvalCaseRunner(deps) {
     const kinds = new Map();
-    const runnerOf = (evalRunId) => (kinds.get(evalRunId) === "agent" ? deps.agent : deps.stage);
+    const runnerOf = (evalRunId) => {
+        const kind = kinds.get(evalRunId);
+        if (kind === "internal_agent" && deps.internalAgent)
+            return deps.internalAgent;
+        return kind === "agent" ? deps.agent : deps.stage;
+    };
     return {
         runCase(plan, req) {
-            kinds.set(req.evalRunId, plan.target.kind === "agent" ? "agent" : "stage");
+            const kind = plan.target.kind === "internal_agent" ? "internal_agent" : isConversationTarget(plan.target.kind) ? "agent" : "stage";
+            kinds.set(req.evalRunId, kind);
             return runnerOf(req.evalRunId).runCase(plan, req);
         },
         async cancelAll(evalRunId) {
-            await Promise.all([deps.stage.cancelAll(evalRunId), deps.agent.cancelAll(evalRunId)]);
+            await Promise.all([
+                deps.stage.cancelAll(evalRunId),
+                deps.agent.cancelAll(evalRunId),
+                deps.internalAgent?.cancelAll(evalRunId),
+            ]);
         },
-        spend: (evalRunId) => deps.stage.spend(evalRunId) + deps.agent.spend(evalRunId),
+        spend: (evalRunId) => deps.stage.spend(evalRunId) + deps.agent.spend(evalRunId) + (deps.internalAgent?.spend(evalRunId) ?? 0),
         addSpend: (evalRunId, usd) => runnerOf(evalRunId).addSpend(evalRunId, usd),
         stopReason: (evalRunId) => runnerOf(evalRunId).stopReason?.(evalRunId),
     };

package/dist/evals/runner/internalAgentCaseRunner.d.ts

@@ -0,0 +1,6 @@
+import type { OperatorAgentHost } from "../../operatorAgent/types.js";
+import type { StageCaseRunner } from "./stageCaseRunner.js";
+export declare function createInternalAgentCaseRunner(deps: {
+    host: () => OperatorAgentHost | undefined;
+    now?: () => number;
+}): StageCaseRunner;

package/dist/evals/runner/internalAgentCaseRunner.js

@@ -0,0 +1,250 @@
+import { randomUUID } from "node:crypto";
+import { createOperatorEvalGate } from "../../operatorAgent/evalGate.js";
+import { buildConversationSubject } from "./conversationSubject.js";
+import { createDriver, DriverScenarioError } from "./userDriver.js";
+const ZERO_USAGE = {
+    cost_usd: 0,
+    agent_cost_usd: 0,
+    simulator_cost_usd: 0,
+    tokens_in: 0,
+    tokens_out: 0,
+    simulator_tokens_in: 0,
+    simulator_tokens_out: 0,
+    simulator_estimated: false,
+    agent_calls: 0,
+    simulator_calls: 0,
+};
+function failure(code, message, extra = {}) {
+    return { ok: false, code, message, ...extra };
+}
+function parseMocks(value) {
+    const out = {};
+    if (typeof value !== "object" || value === null || Array.isArray(value))
+        return out;
+    for (const [tool, mock] of Object.entries(value)) {
+        if (typeof mock !== "object" || mock === null)
+            continue;
+        const { result, error } = mock;
+        if (typeof error === "string")
+            out[tool] = { error };
+        else if ("result" in mock)
+            out[tool] = { result };
+    }
+    return out;
+}
+function detailFrom(sessionId, profileId, messages) {
+    const now = new Date().toISOString();
+    return {
+        session: {
+            id: sessionId,
+            agentId: profileId,
+            revision: 0,
+            backend: "pi",
+            status: "closed",
+            source: "eval",
+            sourceRef: null,
+            threadKey: null,
+            title: null,
+            createdAt: now,
+            updatedAt: now,
+        },
+        messages,
+        turns: [],
+        spans: [],
+        approvals: [],
+        pendingApproval: null,
+        evalScores: [],
+        deliveries: [],
+    };
+}
+export function createInternalAgentCaseRunner(deps) {
+    const clock = deps.now ?? Date.now;
+    const states = new Map();
+    const stateFor = (id) => {
+        let state = states.get(id);
+        if (!state) {
+            state = { cancelled: false, abort: new AbortController() };
+            states.set(id, state);
+        }
+        return state;
+    };
+    async function runCase(plan, req) {
+        const state = stateFor(req.evalRunId);
+        if (state.cancelled)
+            return failure("EVAL_CANCELLED", "Eval run was cancelled");
+        if (plan.target.kind !== "internal_agent" || !plan.conversation) {
+            return failure("EVAL_ENGINE_ERROR", "Not an internal-agent plan");
+        }
+        if (plan.conversation.tools === "real") {
+            return failure("EVAL_ENGINE_ERROR", "Internal agents cannot run an eval with real tools");
+        }
+        const host = deps.host();
+        if (!host)
+            return failure("EVAL_ENGINE_ERROR", "Internal agents are unavailable on this host");
+        const planCase = plan.cases.find((c) => c.id === req.caseId);
+        if (!planCase)
+            return failure("EVAL_ENGINE_ERROR", `Case "${req.caseId}" is not in the plan`);
+        const scenario = planCase.input;
+        const mode = scenario.mode === "script" || scenario.mode === "simulated" ? scenario.mode : plan.conversation.mode;
+        let driver;
+        try {
+            driver = createDriver(mode, scenario, {});
+        }
+        catch (err) {
+            return failure("EVAL_ENGINE_ERROR", err instanceof DriverScenarioError ? err.message : "Scenario could not be prepared");
+        }
+        if (mode === "simulated") {
+            return failure("EVAL_ENGINE_ERROR", "A simulated internal-agent case needs a user model, which this runner does not call");
+        }
+        const conversation = plan.conversation;
+        const sessionId = `op_eval_${randomUUID()}`;
+        let turn = 0;
+        const gate = createOperatorEvalGate({
+            policy: { tools: conversation.tools, mocks: parseMocks(scenario.tool_mocks) },
+            turn: () => turn,
+        });
+        let session;
+        try {
+            session = host.openEvalSession({
+                profileId: plan.target.profileId,
+                context: { kind: "stageflow_guide" },
+                gate,
+            });
+        }
+        catch (err) {
+            return failure("EVAL_ENGINE_ERROR", err instanceof Error ? err.message : "The eval session could not be opened");
+        }
+        const startedAt = clock();
+        const messages = [];
+        const transcript = [];
+        let end = {
+            kind: "done",
+            stopReason: "max_turns",
+        };
+        const caseCtl = new AbortController();
+        let wallExpired = false;
+        let turnTimedOut = false;
+        const stopCase = () => {
+            caseCtl.abort();
+            void session.abort().catch(() => undefined);
+        };
+        const onRunAbort = () => stopCase();
+        state.abort.signal.addEventListener("abort", onRunAbort, { once: true });
+        const wallMs = conversation.maxTurns * conversation.turnTimeoutMs + 60_000;
+        const wall = setTimeout(() => {
+            wallExpired = true;
+            stopCase();
+        }, wallMs);
+        const stopped = new Promise((resolve) => {
+            if (caseCtl.signal.aborted)
+                resolve("stopped");
+            else
+                caseCtl.signal.addEventListener("abort", () => resolve("stopped"), { once: true });
+        });
+        try {
+            for (turn = 1; turn <= conversation.maxTurns; turn += 1) {
+                if (state.cancelled) {
+                    end = { kind: "cancelled" };
+                    break;
+                }
+                if (wallExpired || turnTimedOut || caseCtl.signal.aborted) {
+                    end = {
+                        kind: "failed",
+                        code: "EVAL_AGENT_TIMEOUT",
+                        message: turnTimedOut ? `Turn ${turn} exceeded ${conversation.turnTimeoutMs} ms` : "Conversation exceeded its time limit",
+                        stopReason: "timeout",
+                    };
+                    break;
+                }
+                const step = await driver.next({ turn, transcript, signal: caseCtl.signal });
+                if (step.kind === "end") {
+                    end = { kind: "done", stopReason: step.reason };
+                    break;
+                }
+                const now = new Date().toISOString();
+                messages.push({ id: randomUUID(), sessionId, seq: messages.length, turnId: String(turn), role: "user", author: null, text: step.text, createdAt: now });
+                const turnTimer = setTimeout(() => {
+                    turnTimedOut = true;
+                    stopCase();
+                }, conversation.turnTimeoutMs);
+                const sent = session.send(step.text, plan.modelOverride ? { modelId: plan.modelOverride } : {}).then((events) => ({ ok: true, events }), (err) => ({ ok: false, err }));
+                const outcome = await Promise.race([sent, stopped]);
+                clearTimeout(turnTimer);
+                if (state.cancelled) {
+                    end = { kind: "cancelled" };
+                    break;
+                }
+                if (outcome === "stopped" || wallExpired || turnTimedOut) {
+                    end = {
+                        kind: "failed",
+                        code: "EVAL_AGENT_TIMEOUT",
+                        message: turnTimedOut ? `Turn ${turn} exceeded ${conversation.turnTimeoutMs} ms` : "Conversation exceeded its time limit",
+                        stopReason: "timeout",
+                    };
+                    break;
+                }
+                if (!outcome.ok) {
+                    end = { kind: "failed", code: "EVAL_ENGINE_ERROR", message: outcome.err instanceof Error ? outcome.err.message : "Conversation failed", stopReason: "error" };
+                    break;
+                }
+                const text = outcome.events
+                    .filter((event) => event.type === "message" && event.role === "assistant")
+                    .map((event) => (event.type === "message" ? event.text : ""))
+                    .filter(Boolean)
+                    .join("\n\n");
+                if (!text && gate.records().every((call) => call.turn !== turn)) {
+                    end = { kind: "failed", code: "EVAL_AGENT_EMPTY_REPLY", message: "The agent returned an empty reply", stopReason: "error" };
+                    break;
+                }
+                messages.push({ id: randomUUID(), sessionId, seq: messages.length, turnId: String(turn), role: "assistant", author: null, text, createdAt: now });
+                transcript.push({ role: "user", text: step.text }, { role: "assistant", text });
+                if (step.endAfter !== undefined) {
+                    end = { kind: "done", stopReason: step.endAfter };
+                    break;
+                }
+                if (turn === conversation.maxTurns)
+                    end = { kind: "done", stopReason: "max_turns" };
+            }
+        }
+        catch (err) {
+            end = state.cancelled
+                ? { kind: "cancelled" }
+                : wallExpired || turnTimedOut
+                    ? { kind: "failed", code: "EVAL_AGENT_TIMEOUT", message: turnTimedOut ? `Turn ${turn} exceeded ${conversation.turnTimeoutMs} ms` : "Conversation exceeded its time limit", stopReason: "timeout" }
+                    : { kind: "failed", code: "EVAL_ENGINE_ERROR", message: err instanceof Error ? err.message : "Conversation failed", stopReason: "error" };
+        }
+        finally {
+            clearTimeout(wall);
+            state.abort.signal.removeEventListener("abort", onRunAbort);
+            session.close();
+        }
+        const usage = { ...ZERO_USAGE, agent_calls: messages.filter((m) => m.role === "assistant").length };
+        const subject = (stopReason, failed) => buildConversationSubject({
+            detail: detailFrom(sessionId, plan.target.kind === "internal_agent" ? plan.target.profileId : "", messages),
+            calls: gate.records(),
+            stopReason,
+            usage,
+            turns: messages.filter((m) => m.role === "user").length,
+            ...(failed !== undefined ? { failure: failed } : {}),
+        });
+        const latencyMs = Math.max(0, clock() - startedAt);
+        if (end.kind === "done") {
+            return { ok: true, subjectRef: sessionId, output: subject(end.stopReason), latencyMs, costUsd: 0, tokensIn: 0, tokensOut: 0 };
+        }
+        if (end.kind === "cancelled")
+            return failure("EVAL_CANCELLED", "Eval run was cancelled", { subjectRef: sessionId, latencyMs, costUsd: 0 });
+        return failure(end.code, end.message, { subjectRef: sessionId, output: subject(end.stopReason, end.message), latencyMs, costUsd: 0 });
+    }
+    return {
+        runCase,
+        async cancelAll(evalRunId) {
+            const state = states.get(evalRunId);
+            if (!state)
+                return;
+            state.cancelled = true;
+            state.abort.abort();
+        },
+        spend: () => 0,
+        addSpend: () => undefined,
+    };
+}

package/dist/evals/snapshot.d.ts

@@ -74,6 +74,14 @@ export type ConversationSettings = {
     user_model_from_judge: boolean;
     simulator_prompt_version: number;
 };
+export type InternalAgentSnapshotTarget = {
+    kind: "internal_agent";
+    profile_id: string;
+    title: string;
+    playbook_hash: string;
+    tools: string[];
+    model: string;
+};
 export type ConfigSnapshot = {
     snapshot_version: number;
     eval: {
@@ -83,7 +91,7 @@ export type ConfigSnapshot = {
         definition: Record<string, unknown>;
     };
     overrides: SnapshotOverrides;
-    target: StageSnapshotTarget | AgentSnapshotTarget;
+    target: StageSnapshotTarget | AgentSnapshotTarget | InternalAgentSnapshotTarget;
     dataset: {
         path: string;
         hash: string;
@@ -169,7 +177,7 @@ export type SnapshotInput = {
     evalHash: string;
     evalDefinition: Record<string, unknown>;
     overrides: SnapshotOverrides;
-    target: Omit<StageSnapshotTarget, "kind"> | Omit<AgentSnapshotTarget, "kind">;
+    target: Omit<StageSnapshotTarget, "kind"> | Omit<AgentSnapshotTarget, "kind"> | Omit<InternalAgentSnapshotTarget, "kind">;
     dataset: ConfigSnapshot["dataset"];
     graders: SnapshotGrader[];
     settings: ConfigSnapshot["settings"];