stageflow 0.40.0 → 0.44.0

package/dist/evals/runner/FakeAgentBackend.d.ts

@@ -0,0 +1,40 @@
+import type { CompleteInput, CompleteResult, ConversationBackend, RuntimeUsage } from "../../agents/runtime/types.js";
+export type FakeToolStep = {
+    tool: string;
+    input?: Record<string, unknown>;
+    /** What the tool does when the gate lets it run. */
+    body?: () => string | Promise<string>;
+    /** Report the tool as failed when it ran. */
+    fails?: boolean;
+};
+export type FakeTurnScript = {
+    text?: string;
+    tools?: FakeToolStep[];
+    usage?: RuntimeUsage;
+    /** Wait before replying; an abort ends the wait early. */
+    delayMs?: number;
+    /** Never reply until the turn is aborted. */
+    hang?: boolean;
+    /** Throw this from the backend. */
+    error?: Error;
+};
+export type FakeTurnReply = FakeTurnScript | ((input: CompleteInput, index: number) => FakeTurnScript);
+export declare const FAKE_USAGE: RuntimeUsage;
+/**
+ * A scriptable conversation backend for eval tests. Each model call takes the next scripted turn; tool calls go
+ * through the same gate and hooks the real runtime uses, so tool policy is exercised.
+ */
+export declare class FakeAgentBackend implements ConversationBackend {
+    private readonly script;
+    readonly id: "pi";
+    readonly inputs: CompleteInput[];
+    readonly released: string[];
+    /** Tool bodies that actually ran, by grant id. */
+    readonly executed: string[];
+    private index;
+    private callSeq;
+    constructor(script?: readonly FakeTurnReply[] | ((input: CompleteInput, index: number) => FakeTurnScript));
+    complete(input: CompleteInput): Promise<CompleteResult>;
+    release(sessionId: string): Promise<void>;
+    get calls(): number;
+}

package/dist/evals/runner/FakeAgentBackend.js

@@ -0,0 +1,82 @@
+export const FAKE_USAGE = { inputTokens: 100, outputTokens: 20, costUsd: 0.01 };
+function waitOrAbort(ms, signal) {
+    return new Promise((resolve) => {
+        if (signal.aborted)
+            return resolve();
+        const done = () => {
+            clearTimeout(timer);
+            signal.removeEventListener("abort", done);
+            resolve();
+        };
+        const timer = ms === undefined ? undefined : setTimeout(done, ms);
+        signal.addEventListener("abort", done, { once: true });
+    });
+}
+/**
+ * A scriptable conversation backend for eval tests. Each model call takes the next scripted turn; tool calls go
+ * through the same gate and hooks the real runtime uses, so tool policy is exercised.
+ */
+export class FakeAgentBackend {
+    script;
+    id = "pi";
+    inputs = [];
+    released = [];
+    /** Tool bodies that actually ran, by grant id. */
+    executed = [];
+    index = 0;
+    callSeq = 0;
+    constructor(script = []) {
+        this.script = script;
+    }
+    async complete(input) {
+        const index = this.index++;
+        this.inputs.push(input);
+        const reply = typeof this.script === "function" ? this.script : this.script[index];
+        const turn = typeof reply === "function" ? reply(input, index) : (reply ?? { text: `reply ${index + 1}` });
+        if (turn.error)
+            throw turn.error;
+        const usage = turn.usage ?? FAKE_USAGE;
+        const toolOutputs = [];
+        const startedAt = new Date();
+        if (turn.hang)
+            await waitOrAbort(undefined, input.signal);
+        else if (turn.delayMs !== undefined)
+            await waitOrAbort(turn.delayMs, input.signal);
+        if (input.signal.aborted)
+            return { text: "", usage, toolOutputs };
+        for (const step of turn.tools ?? []) {
+            const call = { id: `call_${(this.callSeq += 1)}`, tool: step.tool, input: step.input ?? {} };
+            const decision = await input.hooks.gate(call);
+            if (!decision.allow)
+                continue;
+            input.hooks.toolStarted(call);
+            let output;
+            let ok = step.fails !== true;
+            if (step.tool.startsWith("agent:")) {
+                output = await input.hooks.delegate(step.tool.slice(6), String(call.input.message ?? ""), call.id);
+            }
+            else {
+                try {
+                    this.executed.push(step.tool);
+                    output = await (step.body ? step.body() : `ran ${step.tool}`);
+                }
+                catch (error) {
+                    ok = false;
+                    output = error instanceof Error ? error.message : String(error);
+                }
+            }
+            toolOutputs.push(output);
+            input.hooks.toolFinished(call, { ok, output });
+        }
+        const text = turn.text ?? `reply ${index + 1}`;
+        input.onDelta(text);
+        input.hooks.modelFinished({ startedAt, endedAt: new Date(), ...usage });
+        return { text, usage, toolOutputs };
+    }
+    async release(sessionId) {
+        this.released.push(sessionId);
+    }
+    get calls() {
+        return this.index;
+    }
+}

package/dist/evals/runner/agentCaseRunner.d.ts

@@ -0,0 +1,24 @@
+import type { AgentHostContext } from "../../agents/hostContext.js";
+import type { AgentDefinition } from "../../config/loadAgent.js";
+import { type ModelPrice, type SimulatorModel } from "./simulatorModel.js";
+import type { StageCaseRunner } from "./stageCaseRunner.js";
+export declare const EVAL_CASE_WALL_CLOCK_GRACE_MS = 60000;
+export declare const EVAL_TURN_STOP_GRACE_MS = 10000;
+export type AgentRunnerHost = Pick<AgentHostContext, "service" | "chat" | "evalGates" | "backend">;
+export type AgentCaseRunnerDeps = {
+    /** Lazy: the manager is built before the agent host. Undefined means this Host cannot run agents. */
+    host: () => AgentRunnerHost | undefined;
+    /** Builds the simulated user's model; defaults to the agent backend's one-shot call on `plan.userModel`. */
+    simulatorModel?: (input: {
+        projectRoot: string;
+        definition: AgentDefinition;
+        modelId: string;
+        host: AgentRunnerHost;
+    }) => SimulatorModel;
+    /** USD per million tokens by model id, for pricing simulator calls the backend reports no cost for. */
+    loadPrices?: (modelIds: readonly string[], projectRoot: string) => Promise<Map<string, ModelPrice>>;
+    now?: () => number;
+    wallClockGraceMs?: number;
+    stopGraceMs?: number;
+};
+export declare function createAgentCaseRunner(deps: AgentCaseRunnerDeps): StageCaseRunner;

package/dist/evals/runner/agentCaseRunner.js

@@ -0,0 +1,388 @@
+import { isAgentError } from "../../agents/errors.js";
+import { createEvalGate } from "../../agents/evalGate.js";
+import { EVAL_MAX_CONSECUTIVE_PROVIDER_FAILURES } from "../constants.js";
+import { buildConversationSubject } from "./conversationSubject.js";
+import { createBackendSimulatorModel, SimulatorError, } from "./simulatorModel.js";
+import { createDriver, DriverScenarioError, } from "./userDriver.js";
+export const EVAL_CASE_WALL_CLOCK_GRACE_MS = 60_000;
+export const EVAL_TURN_STOP_GRACE_MS = 10_000;
+const SESSION_OVERHEAD_MINUTES = 5;
+function failure(code, message, extra = {}) {
+    return {
+        ok: false,
+        code,
+        message,
+        ...(extra.subjectRef !== undefined ? { subjectRef: extra.subjectRef } : {}),
+        ...(extra.output !== undefined ? { output: extra.output } : {}),
+        ...(extra.latencyMs !== undefined ? { latencyMs: extra.latencyMs } : {}),
+        ...(extra.costUsd !== undefined ? { costUsd: extra.costUsd } : {}),
+    };
+}
+function parseMocks(value) {
+    const out = {};
+    if (typeof value !== "object" || value === null || Array.isArray(value))
+        return out;
+    for (const [tool, mock] of Object.entries(value)) {
+        if (typeof mock !== "object" || mock === null)
+            continue;
+        const { result, error } = mock;
+        if (typeof error === "string")
+            out[tool] = { error };
+        else if ("result" in mock)
+            out[tool] = { result };
+    }
+    return out;
+}
+function sumTurns(detail) {
+    let costUsd = 0;
+    let tokensIn = 0;
+    let tokensOut = 0;
+    for (const turn of detail.turns) {
+        costUsd += turn.costUsd ?? 0;
+        tokensIn += turn.inputTokens ?? 0;
+        tokensOut += turn.outputTokens ?? 0;
+    }
+    const modelSpans = detail.spans.filter((span) => span.kind === "model").length;
+    return { costUsd, tokensIn, tokensOut, modelCalls: modelSpans > 0 ? modelSpans : detail.turns.length };
+}
+function isEmptyReply(detail, turn) {
+    if ((turn.outputTokens ?? 0) > 0)
+        return false;
+    return !detail.messages.some((m) => m.turnId === turn.id && m.role === "assistant" && m.text.trim() !== "");
+}
+function errorText(error) {
+    if (isAgentError(error))
+        return error.message === error.code ? error.code : `${error.code}: ${error.message}`;
+    return "agent.backend_failed";
+}
+export function createAgentCaseRunner(deps) {
+    const clock = deps.now ?? Date.now;
+    const wallGraceMs = deps.wallClockGraceMs ?? EVAL_CASE_WALL_CLOCK_GRACE_MS;
+    const stopGraceMs = deps.stopGraceMs ?? EVAL_TURN_STOP_GRACE_MS;
+    const states = new Map();
+    const stateFor = (evalRunId) => {
+        let state = states.get(evalRunId);
+        if (state === undefined) {
+            state = { active: new Map(), spend: 0, cancelled: false, abort: new AbortController(), providerFailures: 0 };
+            states.set(evalRunId, state);
+        }
+        return state;
+    };
+    async function simulatorFor(plan, host, definition) {
+        const modelId = plan.userModel;
+        if (deps.simulatorModel)
+            return deps.simulatorModel({ projectRoot: plan.projectRoot, definition, modelId, host });
+        const prices = (await deps.loadPrices?.([modelId], plan.projectRoot).catch(() => undefined)) ?? new Map();
+        return createBackendSimulatorModel({
+            backend: host.backend,
+            projectRoot: plan.projectRoot,
+            definition,
+            modelId,
+            priceOf: (id) => prices.get(id),
+        });
+    }
+    async function runCase(plan, req) {
+        const state = stateFor(req.evalRunId);
+        if (state.stopReason !== undefined)
+            return failure("EVAL_CANCELLED", state.stopReason);
+        const result = await runCaseInner(plan, req);
+        if (result.ok)
+            state.providerFailures = 0;
+        else if (result.code === "EVAL_AGENT_EMPTY_REPLY" || result.code === "EVAL_AGENT_FAILED") {
+            state.providerFailures += 1;
+            if (state.providerFailures >= EVAL_MAX_CONSECUTIVE_PROVIDER_FAILURES && state.stopReason === undefined) {
+                state.stopReason = `Stopped after ${EVAL_MAX_CONSECUTIVE_PROVIDER_FAILURES} consecutive provider failures; check your provider key or rate limit`;
+            }
+        }
+        return result;
+    }
+    async function runCaseInner(plan, req) {
+        const state = stateFor(req.evalRunId);
+        if (state.cancelled)
+            return failure("EVAL_CANCELLED", "Eval run was cancelled");
+        if (state.spend >= plan.maxCostUsd) {
+            return failure("EVAL_BUDGET_EXCEEDED", `Eval spend reached the $${plan.maxCostUsd} limit`);
+        }
+        const { agent, conversation, target } = plan;
+        if (target.kind !== "agent" || !agent || !conversation) {
+            return failure("EVAL_ENGINE_ERROR", "Not an agent plan");
+        }
+        const host = deps.host();
+        if (!host)
+            return failure("EVAL_ENGINE_ERROR", "Agents are unavailable on this host");
+        const planCase = plan.cases.find((c) => c.id === req.caseId);
+        if (planCase === undefined)
+            return failure("EVAL_ENGINE_ERROR", `Case "${req.caseId}" is not in the plan`);
+        const definition = agent.definition;
+        const scenario = planCase.input;
+        const mode = scenario.mode === "script" || scenario.mode === "simulated" ? scenario.mode : conversation.mode;
+        let driver;
+        try {
+            let simulatorModel;
+            if (mode === "simulated") {
+                if (!plan.userModel)
+                    return failure("EVAL_ENGINE_ERROR", "A simulated case needs a user model");
+                simulatorModel = await simulatorFor(plan, host, definition);
+            }
+            driver = createDriver(mode, scenario, simulatorModel ? { simulatorModel } : {});
+        }
+        catch (err) {
+            return failure("EVAL_ENGINE_ERROR", err instanceof DriverScenarioError ? err.message : "Scenario could not be prepared");
+        }
+        const { service, chat, evalGates } = host;
+        const startedAt = clock();
+        const sessionMinutes = Math.ceil((conversation.maxTurns * conversation.turnTimeoutMs) / 60_000) + SESSION_OVERHEAD_MINUTES;
+        let sessionId;
+        try {
+            const session = await service.openSession(plan.projectRoot, target.agentId, {
+                source: "eval",
+                sourceRef: plan.evalRunId,
+                title: `eval ${plan.evalId} ${plan.evalRunId} ${req.caseId}.${req.repeatIndex}`,
+                eval: {
+                    ...(target.revision !== undefined ? { revision: target.revision } : {}),
+                    ...(plan.modelOverride !== undefined ? { definition: { model: { id: plan.modelOverride } } } : {}),
+                    limits: { max_turns: conversation.maxTurns + 1, session_minutes: sessionMinutes, cost_usd: null },
+                    capture: "full",
+                },
+            });
+            sessionId = session.id;
+        }
+        catch (err) {
+            const code = isAgentError(err) && (err.code === "agent.archived" || err.code === "agent.revision_not_found") ? "EVAL_AGENT_FAILED" : "EVAL_ENGINE_ERROR";
+            return failure(code, isAgentError(err) ? errorText(err) : "The eval session could not be opened");
+        }
+        let resolveDone;
+        state.active.set(sessionId, { done: new Promise((resolve) => (resolveDone = resolve)) });
+        let currentTurn = 0;
+        const gate = createEvalGate({
+            policy: {
+                tools: conversation.tools,
+                approvals: conversation.approvals,
+                readOnlyTools: conversation.readOnlyTools,
+                mocks: parseMocks(scenario.tool_mocks),
+            },
+            definition,
+            projectRoot: plan.projectRoot,
+            turn: () => currentTurn,
+        });
+        evalGates.set(sessionId, gate);
+        const usage = { simCost: 0, simIn: 0, simOut: 0, simCalls: 0, simEstimated: false };
+        const addSimulator = (u) => {
+            if (u === undefined)
+                return;
+            usage.simCost += u.costUsd;
+            usage.simIn += u.tokensIn;
+            usage.simOut += u.tokensOut;
+            usage.simCalls += 1;
+            usage.simEstimated ||= u.estimated === true;
+            state.spend += u.costUsd;
+        };
+        let countedRootCost = 0;
+        const syncSpend = (detail) => {
+            const cost = sumTurns(detail).costUsd;
+            state.spend += cost - countedRootCost;
+            countedRootCost = cost;
+        };
+        const caseCtl = new AbortController();
+        let wallExpired = false;
+        let stuckTimer;
+        let releaseStuck = () => undefined;
+        const stuck = new Promise((resolve) => (releaseStuck = () => resolve("stuck")));
+        const stopTurn = () => {
+            chat.stop(sessionId);
+            stuckTimer ??= setTimeout(releaseStuck, stopGraceMs);
+        };
+        const onCancel = () => {
+            caseCtl.abort();
+            stopTurn();
+        };
+        state.abort.signal.addEventListener("abort", onCancel, { once: true });
+        const wallMs = conversation.maxTurns * conversation.turnTimeoutMs + wallGraceMs;
+        const wallTimer = setTimeout(() => {
+            wallExpired = true;
+            caseCtl.abort();
+            stopTurn();
+        }, wallMs);
+        let turnsDone = 0;
+        const converse = async () => {
+            const transcript = [];
+            for (let turn = 1; turn <= conversation.maxTurns; turn++) {
+                if (state.cancelled)
+                    return { kind: "cancelled" };
+                if (wallExpired)
+                    return { kind: "failed", code: "EVAL_AGENT_TIMEOUT", message: "Conversation exceeded its time limit", stopReason: "timeout" };
+                let step;
+                try {
+                    step = await driver.next({ turn, transcript, signal: caseCtl.signal });
+                }
+                catch (err) {
+                    if (state.cancelled)
+                        return { kind: "cancelled" };
+                    if (wallExpired)
+                        return { kind: "failed", code: "EVAL_AGENT_TIMEOUT", message: "Conversation exceeded its time limit", stopReason: "timeout" };
+                    const message = err instanceof SimulatorError ? err.message : "The simulated user failed";
+                    return { kind: "failed", code: "EVAL_SIMULATOR_ERROR", message, stopReason: "error" };
+                }
+                addSimulator(step.usage);
+                if (step.kind === "end")
+                    return { kind: "done", stopReason: step.reason };
+                currentTurn = turn;
+                let turnTimedOut = false;
+                const turnTimer = setTimeout(() => {
+                    turnTimedOut = true;
+                    stopTurn();
+                }, conversation.turnTimeoutMs);
+                const sent = chat
+                    .send({ projectRoot: plan.projectRoot, agentId: target.agentId, sessionId, text: step.text, source: "eval" })
+                    .then((detail) => ({ ok: true, detail }), (err) => ({ ok: false, err }));
+                const outcome = await Promise.race([sent, stuck]);
+                clearTimeout(turnTimer);
+                turnsDone = turn;
+                if (state.cancelled)
+                    return { kind: "cancelled" };
+                if (outcome === "stuck" || turnTimedOut || wallExpired) {
+                    const message = turnTimedOut ? `Turn ${turn} exceeded ${conversation.turnTimeoutMs} ms` : "Conversation exceeded its time limit";
+                    return { kind: "failed", code: "EVAL_AGENT_TIMEOUT", message, stopReason: "timeout" };
+                }
+                if (!outcome.ok) {
+                    return { kind: "failed", code: "EVAL_AGENT_FAILED", message: errorText(outcome.err), stopReason: "error" };
+                }
+                const { detail } = outcome;
+                syncSpend(detail);
+                const latest = detail.turns[detail.turns.length - 1];
+                if (latest?.status === "waiting") {
+                    return { kind: "failed", code: "EVAL_ENGINE_ERROR", message: "A turn is waiting for approval, which an eval cannot give", stopReason: "error" };
+                }
+                if (latest !== undefined && latest.status !== "succeeded") {
+                    return { kind: "failed", code: "EVAL_AGENT_FAILED", message: latest.errorCode ?? "agent.backend_failed", stopReason: "error" };
+                }
+                if (latest !== undefined && isEmptyReply(detail, latest)) {
+                    return {
+                        kind: "failed",
+                        code: "EVAL_AGENT_EMPTY_REPLY",
+                        message: "The agent returned an empty reply with no output tokens. The model provider may be rate limiting or failing; check your provider key and limits",
+                        stopReason: "error",
+                    };
+                }
+                transcript.splice(0, transcript.length, ...detail.messages.map((m) => ({ role: m.role, text: m.text })));
+                if (step.endAfter !== undefined)
+                    return { kind: "done", stopReason: step.endAfter };
+            }
+            return { kind: "done", stopReason: "max_turns" };
+        };
+        let end;
+        try {
+            end = await converse();
+        }
+        catch (err) {
+            end = { kind: "failed", code: "EVAL_ENGINE_ERROR", message: err instanceof Error ? err.message : "Conversation failed", stopReason: "error" };
+        }
+        clearTimeout(wallTimer);
+        if (stuckTimer)
+            clearTimeout(stuckTimer);
+        state.abort.signal.removeEventListener("abort", onCancel);
+        let result;
+        try {
+            const detail = await service.getSession(plan.projectRoot, sessionId);
+            syncSpend(detail);
+            const root = sumTurns(detail);
+            let childCost = 0;
+            let childIn = 0;
+            let childOut = 0;
+            let childCalls = 0;
+            for (const child of service.listChildSessions(sessionId)) {
+                const childDetail = await service.getSession(plan.projectRoot, child.id);
+                const sums = sumTurns(childDetail);
+                childCost += sums.costUsd;
+                childIn += sums.tokensIn;
+                childOut += sums.tokensOut;
+                childCalls += sums.modelCalls;
+            }
+            state.spend += childCost;
+            const agentCost = root.costUsd + childCost;
+            const conversationUsage = {
+                cost_usd: agentCost + usage.simCost,
+                agent_cost_usd: agentCost,
+                simulator_cost_usd: usage.simCost,
+                tokens_in: root.tokensIn + childIn,
+                tokens_out: root.tokensOut + childOut,
+                simulator_tokens_in: usage.simIn,
+                simulator_tokens_out: usage.simOut,
+                simulator_estimated: usage.simEstimated,
+                agent_calls: root.modelCalls + childCalls,
+                simulator_calls: usage.simCalls,
+            };
+            const latencyMs = Math.max(0, clock() - startedAt);
+            const costUsd = conversationUsage.cost_usd;
+            const subject = (stopReason, failed) => buildConversationSubject({
+                detail,
+                calls: gate.records(),
+                stopReason,
+                usage: conversationUsage,
+                turns: turnsDone,
+                ...(failed !== undefined ? { failure: failed } : {}),
+            });
+            if (end.kind === "done") {
+                result = {
+                    ok: true,
+                    subjectRef: sessionId,
+                    output: subject(end.stopReason),
+                    latencyMs,
+                    costUsd,
+                    tokensIn: conversationUsage.tokens_in,
+                    tokensOut: conversationUsage.tokens_out,
+                };
+            }
+            else if (end.kind === "cancelled") {
+                result = failure("EVAL_CANCELLED", "Eval run was cancelled", { subjectRef: sessionId, latencyMs, costUsd });
+            }
+            else {
+                result = failure(end.code, end.message, {
+                    subjectRef: sessionId,
+                    output: subject(end.stopReason, end.message),
+                    latencyMs,
+                    costUsd,
+                });
+            }
+        }
+        catch (err) {
+            result = failure("EVAL_ENGINE_ERROR", err instanceof Error ? err.message : "Conversation could not be read", { subjectRef: sessionId });
+        }
+        finally {
+            try {
+                const children = service.listChildSessions(sessionId);
+                await chat.closeEvalSession(plan.projectRoot, sessionId);
+                for (const child of children) {
+                    await chat.release(child.id);
+                    service.forgetEvalSession(child.id);
+                    evalGates.delete(child.id);
+                }
+            }
+            catch {
+                /* the runtime session is dropped with the Host at the latest */
+            }
+            evalGates.delete(sessionId);
+            state.active.delete(sessionId);
+            resolveDone();
+        }
+        return result;
+    }
+    return {
+        runCase,
+        async cancelAll(evalRunId) {
+            const state = stateFor(evalRunId);
+            state.cancelled = true;
+            state.abort.abort();
+            const host = deps.host();
+            await Promise.all([...state.active.entries()].map(async ([sessionId, entry]) => {
+                host?.chat.stop(sessionId);
+                await entry.done;
+            }));
+        },
+        spend: (evalRunId) => states.get(evalRunId)?.spend ?? 0,
+        stopReason: (evalRunId) => states.get(evalRunId)?.stopReason,
+        addSpend(evalRunId, usd) {
+            stateFor(evalRunId).spend += usd;
+        },
+    };
+}

package/dist/evals/runner/caseRunner.d.ts

@@ -0,0 +1,8 @@
+import type { StageCaseRunner } from "./stageCaseRunner.js";
+export type EvalCaseRunner = StageCaseRunner;
+/** Routes each case to the stage, agent, or internal-agent runner by the plan's target kind. */
+export declare function createEvalCaseRunner(deps: {
+    stage: StageCaseRunner;
+    agent: StageCaseRunner;
+    internalAgent?: StageCaseRunner;
+}): EvalCaseRunner;

package/dist/evals/runner/caseRunner.js

@@ -0,0 +1,28 @@
+import { isConversationTarget } from "../types.js";
+/** Routes each case to the stage, agent, or internal-agent runner by the plan's target kind. */
+export function createEvalCaseRunner(deps) {
+    const kinds = new Map();
+    const runnerOf = (evalRunId) => {
+        const kind = kinds.get(evalRunId);
+        if (kind === "internal_agent" && deps.internalAgent)
+            return deps.internalAgent;
+        return kind === "agent" ? deps.agent : deps.stage;
+    };
+    return {
+        runCase(plan, req) {
+            const kind = plan.target.kind === "internal_agent" ? "internal_agent" : isConversationTarget(plan.target.kind) ? "agent" : "stage";
+            kinds.set(req.evalRunId, kind);
+            return runnerOf(req.evalRunId).runCase(plan, req);
+        },
+        async cancelAll(evalRunId) {
+            await Promise.all([
+                deps.stage.cancelAll(evalRunId),
+                deps.agent.cancelAll(evalRunId),
+                deps.internalAgent?.cancelAll(evalRunId),
+            ]);
+        },
+        spend: (evalRunId) => deps.stage.spend(evalRunId) + deps.agent.spend(evalRunId) + (deps.internalAgent?.spend(evalRunId) ?? 0),
+        addSpend: (evalRunId, usd) => runnerOf(evalRunId).addSpend(evalRunId, usd),
+        stopReason: (evalRunId) => runnerOf(evalRunId).stopReason?.(evalRunId),
+    };
+}

package/dist/evals/runner/conversationSubject.d.ts

@@ -0,0 +1,18 @@
+import type { EvalToolCallRecord } from "../../agents/evalGate.js";
+import type { AgentSessionDetail } from "../../agents/service.js";
+import type { ConversationOutput, ConversationStopReason, ConversationUsage } from "../types.js";
+export type ConversationSubjectInput = {
+    detail: AgentSessionDetail;
+    calls: readonly EvalToolCallRecord[];
+    stopReason: ConversationStopReason;
+    usage: ConversationUsage;
+    /** User turns completed (a turn counts once the agent replied or failed). */
+    turns: number;
+    /** Set for the partial subject of a failed case. */
+    failure?: string;
+};
+/**
+ * The normalized subject of an agent result. Every text is redacted before it is capped, because stored agent
+ * messages are not redacted.
+ */
+export declare function buildConversationSubject(input: ConversationSubjectInput): ConversationOutput;