stageflow 0.40.0 → 0.42.0

package/dist/evals/runner/agentCaseRunner.js

@@ -0,0 +1,388 @@
+import { isAgentError } from "../../agents/errors.js";
+import { createEvalGate } from "../../agents/evalGate.js";
+import { EVAL_MAX_CONSECUTIVE_PROVIDER_FAILURES } from "../constants.js";
+import { buildConversationSubject } from "./conversationSubject.js";
+import { createBackendSimulatorModel, SimulatorError, } from "./simulatorModel.js";
+import { createDriver, DriverScenarioError, } from "./userDriver.js";
+export const EVAL_CASE_WALL_CLOCK_GRACE_MS = 60_000;
+export const EVAL_TURN_STOP_GRACE_MS = 10_000;
+const SESSION_OVERHEAD_MINUTES = 5;
+function failure(code, message, extra = {}) {
+    return {
+        ok: false,
+        code,
+        message,
+        ...(extra.subjectRef !== undefined ? { subjectRef: extra.subjectRef } : {}),
+        ...(extra.output !== undefined ? { output: extra.output } : {}),
+        ...(extra.latencyMs !== undefined ? { latencyMs: extra.latencyMs } : {}),
+        ...(extra.costUsd !== undefined ? { costUsd: extra.costUsd } : {}),
+    };
+}
+function parseMocks(value) {
+    const out = {};
+    if (typeof value !== "object" || value === null || Array.isArray(value))
+        return out;
+    for (const [tool, mock] of Object.entries(value)) {
+        if (typeof mock !== "object" || mock === null)
+            continue;
+        const { result, error } = mock;
+        if (typeof error === "string")
+            out[tool] = { error };
+        else if ("result" in mock)
+            out[tool] = { result };
+    }
+    return out;
+}
+function sumTurns(detail) {
+    let costUsd = 0;
+    let tokensIn = 0;
+    let tokensOut = 0;
+    for (const turn of detail.turns) {
+        costUsd += turn.costUsd ?? 0;
+        tokensIn += turn.inputTokens ?? 0;
+        tokensOut += turn.outputTokens ?? 0;
+    }
+    const modelSpans = detail.spans.filter((span) => span.kind === "model").length;
+    return { costUsd, tokensIn, tokensOut, modelCalls: modelSpans > 0 ? modelSpans : detail.turns.length };
+}
+function isEmptyReply(detail, turn) {
+    if ((turn.outputTokens ?? 0) > 0)
+        return false;
+    return !detail.messages.some((m) => m.turnId === turn.id && m.role === "assistant" && m.text.trim() !== "");
+}
+function errorText(error) {
+    if (isAgentError(error))
+        return error.message === error.code ? error.code : `${error.code}: ${error.message}`;
+    return "agent.backend_failed";
+}
+export function createAgentCaseRunner(deps) {
+    const clock = deps.now ?? Date.now;
+    const wallGraceMs = deps.wallClockGraceMs ?? EVAL_CASE_WALL_CLOCK_GRACE_MS;
+    const stopGraceMs = deps.stopGraceMs ?? EVAL_TURN_STOP_GRACE_MS;
+    const states = new Map();
+    const stateFor = (evalRunId) => {
+        let state = states.get(evalRunId);
+        if (state === undefined) {
+            state = { active: new Map(), spend: 0, cancelled: false, abort: new AbortController(), providerFailures: 0 };
+            states.set(evalRunId, state);
+        }
+        return state;
+    };
+    async function simulatorFor(plan, host, definition) {
+        const modelId = plan.userModel;
+        if (deps.simulatorModel)
+            return deps.simulatorModel({ projectRoot: plan.projectRoot, definition, modelId, host });
+        const prices = (await deps.loadPrices?.([modelId], plan.projectRoot).catch(() => undefined)) ?? new Map();
+        return createBackendSimulatorModel({
+            backend: host.backend,
+            projectRoot: plan.projectRoot,
+            definition,
+            modelId,
+            priceOf: (id) => prices.get(id),
+        });
+    }
+    async function runCase(plan, req) {
+        const state = stateFor(req.evalRunId);
+        if (state.stopReason !== undefined)
+            return failure("EVAL_CANCELLED", state.stopReason);
+        const result = await runCaseInner(plan, req);
+        if (result.ok)
+            state.providerFailures = 0;
+        else if (result.code === "EVAL_AGENT_EMPTY_REPLY" || result.code === "EVAL_AGENT_FAILED") {
+            state.providerFailures += 1;
+            if (state.providerFailures >= EVAL_MAX_CONSECUTIVE_PROVIDER_FAILURES && state.stopReason === undefined) {
+                state.stopReason = `Stopped after ${EVAL_MAX_CONSECUTIVE_PROVIDER_FAILURES} consecutive provider failures; check your provider key or rate limit`;
+            }
+        }
+        return result;
+    }
+    async function runCaseInner(plan, req) {
+        const state = stateFor(req.evalRunId);
+        if (state.cancelled)
+            return failure("EVAL_CANCELLED", "Eval run was cancelled");
+        if (state.spend >= plan.maxCostUsd) {
+            return failure("EVAL_BUDGET_EXCEEDED", `Eval spend reached the $${plan.maxCostUsd} limit`);
+        }
+        const { agent, conversation, target } = plan;
+        if (target.kind !== "agent" || !agent || !conversation) {
+            return failure("EVAL_ENGINE_ERROR", "Not an agent plan");
+        }
+        const host = deps.host();
+        if (!host)
+            return failure("EVAL_ENGINE_ERROR", "Agents are unavailable on this host");
+        const planCase = plan.cases.find((c) => c.id === req.caseId);
+        if (planCase === undefined)
+            return failure("EVAL_ENGINE_ERROR", `Case "${req.caseId}" is not in the plan`);
+        const definition = agent.definition;
+        const scenario = planCase.input;
+        const mode = scenario.mode === "script" || scenario.mode === "simulated" ? scenario.mode : conversation.mode;
+        let driver;
+        try {
+            let simulatorModel;
+            if (mode === "simulated") {
+                if (!plan.userModel)
+                    return failure("EVAL_ENGINE_ERROR", "A simulated case needs a user model");
+                simulatorModel = await simulatorFor(plan, host, definition);
+            }
+            driver = createDriver(mode, scenario, simulatorModel ? { simulatorModel } : {});
+        }
+        catch (err) {
+            return failure("EVAL_ENGINE_ERROR", err instanceof DriverScenarioError ? err.message : "Scenario could not be prepared");
+        }
+        const { service, chat, evalGates } = host;
+        const startedAt = clock();
+        const sessionMinutes = Math.ceil((conversation.maxTurns * conversation.turnTimeoutMs) / 60_000) + SESSION_OVERHEAD_MINUTES;
+        let sessionId;
+        try {
+            const session = await service.openSession(plan.projectRoot, target.agentId, {
+                source: "eval",
+                sourceRef: plan.evalRunId,
+                title: `eval ${plan.evalId} ${plan.evalRunId} ${req.caseId}.${req.repeatIndex}`,
+                eval: {
+                    ...(target.revision !== undefined ? { revision: target.revision } : {}),
+                    ...(plan.modelOverride !== undefined ? { definition: { model: { id: plan.modelOverride } } } : {}),
+                    limits: { max_turns: conversation.maxTurns + 1, session_minutes: sessionMinutes, cost_usd: null },
+                    capture: "full",
+                },
+            });
+            sessionId = session.id;
+        }
+        catch (err) {
+            const code = isAgentError(err) && (err.code === "agent.archived" || err.code === "agent.revision_not_found") ? "EVAL_AGENT_FAILED" : "EVAL_ENGINE_ERROR";
+            return failure(code, isAgentError(err) ? errorText(err) : "The eval session could not be opened");
+        }
+        let resolveDone;
+        state.active.set(sessionId, { done: new Promise((resolve) => (resolveDone = resolve)) });
+        let currentTurn = 0;
+        const gate = createEvalGate({
+            policy: {
+                tools: conversation.tools,
+                approvals: conversation.approvals,
+                readOnlyTools: conversation.readOnlyTools,
+                mocks: parseMocks(scenario.tool_mocks),
+            },
+            definition,
+            projectRoot: plan.projectRoot,
+            turn: () => currentTurn,
+        });
+        evalGates.set(sessionId, gate);
+        const usage = { simCost: 0, simIn: 0, simOut: 0, simCalls: 0, simEstimated: false };
+        const addSimulator = (u) => {
+            if (u === undefined)
+                return;
+            usage.simCost += u.costUsd;
+            usage.simIn += u.tokensIn;
+            usage.simOut += u.tokensOut;
+            usage.simCalls += 1;
+            usage.simEstimated ||= u.estimated === true;
+            state.spend += u.costUsd;
+        };
+        let countedRootCost = 0;
+        const syncSpend = (detail) => {
+            const cost = sumTurns(detail).costUsd;
+            state.spend += cost - countedRootCost;
+            countedRootCost = cost;
+        };
+        const caseCtl = new AbortController();
+        let wallExpired = false;
+        let stuckTimer;
+        let releaseStuck = () => undefined;
+        const stuck = new Promise((resolve) => (releaseStuck = () => resolve("stuck")));
+        const stopTurn = () => {
+            chat.stop(sessionId);
+            stuckTimer ??= setTimeout(releaseStuck, stopGraceMs);
+        };
+        const onCancel = () => {
+            caseCtl.abort();
+            stopTurn();
+        };
+        state.abort.signal.addEventListener("abort", onCancel, { once: true });
+        const wallMs = conversation.maxTurns * conversation.turnTimeoutMs + wallGraceMs;
+        const wallTimer = setTimeout(() => {
+            wallExpired = true;
+            caseCtl.abort();
+            stopTurn();
+        }, wallMs);
+        let turnsDone = 0;
+        const converse = async () => {
+            const transcript = [];
+            for (let turn = 1; turn <= conversation.maxTurns; turn++) {
+                if (state.cancelled)
+                    return { kind: "cancelled" };
+                if (wallExpired)
+                    return { kind: "failed", code: "EVAL_AGENT_TIMEOUT", message: "Conversation exceeded its time limit", stopReason: "timeout" };
+                let step;
+                try {
+                    step = await driver.next({ turn, transcript, signal: caseCtl.signal });
+                }
+                catch (err) {
+                    if (state.cancelled)
+                        return { kind: "cancelled" };
+                    if (wallExpired)
+                        return { kind: "failed", code: "EVAL_AGENT_TIMEOUT", message: "Conversation exceeded its time limit", stopReason: "timeout" };
+                    const message = err instanceof SimulatorError ? err.message : "The simulated user failed";
+                    return { kind: "failed", code: "EVAL_SIMULATOR_ERROR", message, stopReason: "error" };
+                }
+                addSimulator(step.usage);
+                if (step.kind === "end")
+                    return { kind: "done", stopReason: step.reason };
+                currentTurn = turn;
+                let turnTimedOut = false;
+                const turnTimer = setTimeout(() => {
+                    turnTimedOut = true;
+                    stopTurn();
+                }, conversation.turnTimeoutMs);
+                const sent = chat
+                    .send({ projectRoot: plan.projectRoot, agentId: target.agentId, sessionId, text: step.text, source: "eval" })
+                    .then((detail) => ({ ok: true, detail }), (err) => ({ ok: false, err }));
+                const outcome = await Promise.race([sent, stuck]);
+                clearTimeout(turnTimer);
+                turnsDone = turn;
+                if (state.cancelled)
+                    return { kind: "cancelled" };
+                if (outcome === "stuck" || turnTimedOut || wallExpired) {
+                    const message = turnTimedOut ? `Turn ${turn} exceeded ${conversation.turnTimeoutMs} ms` : "Conversation exceeded its time limit";
+                    return { kind: "failed", code: "EVAL_AGENT_TIMEOUT", message, stopReason: "timeout" };
+                }
+                if (!outcome.ok) {
+                    return { kind: "failed", code: "EVAL_AGENT_FAILED", message: errorText(outcome.err), stopReason: "error" };
+                }
+                const { detail } = outcome;
+                syncSpend(detail);
+                const latest = detail.turns[detail.turns.length - 1];
+                if (latest?.status === "waiting") {
+                    return { kind: "failed", code: "EVAL_ENGINE_ERROR", message: "A turn is waiting for approval, which an eval cannot give", stopReason: "error" };
+                }
+                if (latest !== undefined && latest.status !== "succeeded") {
+                    return { kind: "failed", code: "EVAL_AGENT_FAILED", message: latest.errorCode ?? "agent.backend_failed", stopReason: "error" };
+                }
+                if (latest !== undefined && isEmptyReply(detail, latest)) {
+                    return {
+                        kind: "failed",
+                        code: "EVAL_AGENT_EMPTY_REPLY",
+                        message: "The agent returned an empty reply with no output tokens. The model provider may be rate limiting or failing; check your provider key and limits",
+                        stopReason: "error",
+                    };
+                }
+                transcript.splice(0, transcript.length, ...detail.messages.map((m) => ({ role: m.role, text: m.text })));
+                if (step.endAfter !== undefined)
+                    return { kind: "done", stopReason: step.endAfter };
+            }
+            return { kind: "done", stopReason: "max_turns" };
+        };
+        let end;
+        try {
+            end = await converse();
+        }
+        catch (err) {
+            end = { kind: "failed", code: "EVAL_ENGINE_ERROR", message: err instanceof Error ? err.message : "Conversation failed", stopReason: "error" };
+        }
+        clearTimeout(wallTimer);
+        if (stuckTimer)
+            clearTimeout(stuckTimer);
+        state.abort.signal.removeEventListener("abort", onCancel);
+        let result;
+        try {
+            const detail = await service.getSession(plan.projectRoot, sessionId);
+            syncSpend(detail);
+            const root = sumTurns(detail);
+            let childCost = 0;
+            let childIn = 0;
+            let childOut = 0;
+            let childCalls = 0;
+            for (const child of service.listChildSessions(sessionId)) {
+                const childDetail = await service.getSession(plan.projectRoot, child.id);
+                const sums = sumTurns(childDetail);
+                childCost += sums.costUsd;
+                childIn += sums.tokensIn;
+                childOut += sums.tokensOut;
+                childCalls += sums.modelCalls;
+            }
+            state.spend += childCost;
+            const agentCost = root.costUsd + childCost;
+            const conversationUsage = {
+                cost_usd: agentCost + usage.simCost,
+                agent_cost_usd: agentCost,
+                simulator_cost_usd: usage.simCost,
+                tokens_in: root.tokensIn + childIn,
+                tokens_out: root.tokensOut + childOut,
+                simulator_tokens_in: usage.simIn,
+                simulator_tokens_out: usage.simOut,
+                simulator_estimated: usage.simEstimated,
+                agent_calls: root.modelCalls + childCalls,
+                simulator_calls: usage.simCalls,
+            };
+            const latencyMs = Math.max(0, clock() - startedAt);
+            const costUsd = conversationUsage.cost_usd;
+            const subject = (stopReason, failed) => buildConversationSubject({
+                detail,
+                calls: gate.records(),
+                stopReason,
+                usage: conversationUsage,
+                turns: turnsDone,
+                ...(failed !== undefined ? { failure: failed } : {}),
+            });
+            if (end.kind === "done") {
+                result = {
+                    ok: true,
+                    subjectRef: sessionId,
+                    output: subject(end.stopReason),
+                    latencyMs,
+                    costUsd,
+                    tokensIn: conversationUsage.tokens_in,
+                    tokensOut: conversationUsage.tokens_out,
+                };
+            }
+            else if (end.kind === "cancelled") {
+                result = failure("EVAL_CANCELLED", "Eval run was cancelled", { subjectRef: sessionId, latencyMs, costUsd });
+            }
+            else {
+                result = failure(end.code, end.message, {
+                    subjectRef: sessionId,
+                    output: subject(end.stopReason, end.message),
+                    latencyMs,
+                    costUsd,
+                });
+            }
+        }
+        catch (err) {
+            result = failure("EVAL_ENGINE_ERROR", err instanceof Error ? err.message : "Conversation could not be read", { subjectRef: sessionId });
+        }
+        finally {
+            try {
+                const children = service.listChildSessions(sessionId);
+                await chat.closeEvalSession(plan.projectRoot, sessionId);
+                for (const child of children) {
+                    await chat.release(child.id);
+                    service.forgetEvalSession(child.id);
+                    evalGates.delete(child.id);
+                }
+            }
+            catch {
+                /* the runtime session is dropped with the Host at the latest */
+            }
+            evalGates.delete(sessionId);
+            state.active.delete(sessionId);
+            resolveDone();
+        }
+        return result;
+    }
+    return {
+        runCase,
+        async cancelAll(evalRunId) {
+            const state = stateFor(evalRunId);
+            state.cancelled = true;
+            state.abort.abort();
+            const host = deps.host();
+            await Promise.all([...state.active.entries()].map(async ([sessionId, entry]) => {
+                host?.chat.stop(sessionId);
+                await entry.done;
+            }));
+        },
+        spend: (evalRunId) => states.get(evalRunId)?.spend ?? 0,
+        stopReason: (evalRunId) => states.get(evalRunId)?.stopReason,
+        addSpend(evalRunId, usd) {
+            stateFor(evalRunId).spend += usd;
+        },
+    };
+}

package/dist/evals/runner/caseRunner.d.ts

@@ -0,0 +1,7 @@
+import type { StageCaseRunner } from "./stageCaseRunner.js";
+export type EvalCaseRunner = StageCaseRunner;
+/** Routes each case to the stage or agent runner by the plan's target kind. */
+export declare function createEvalCaseRunner(deps: {
+    stage: StageCaseRunner;
+    agent: StageCaseRunner;
+}): EvalCaseRunner;

package/dist/evals/runner/caseRunner.js

@@ -0,0 +1,17 @@
+/** Routes each case to the stage or agent runner by the plan's target kind. */
+export function createEvalCaseRunner(deps) {
+    const kinds = new Map();
+    const runnerOf = (evalRunId) => (kinds.get(evalRunId) === "agent" ? deps.agent : deps.stage);
+    return {
+        runCase(plan, req) {
+            kinds.set(req.evalRunId, plan.target.kind === "agent" ? "agent" : "stage");
+            return runnerOf(req.evalRunId).runCase(plan, req);
+        },
+        async cancelAll(evalRunId) {
+            await Promise.all([deps.stage.cancelAll(evalRunId), deps.agent.cancelAll(evalRunId)]);
+        },
+        spend: (evalRunId) => deps.stage.spend(evalRunId) + deps.agent.spend(evalRunId),
+        addSpend: (evalRunId, usd) => runnerOf(evalRunId).addSpend(evalRunId, usd),
+        stopReason: (evalRunId) => runnerOf(evalRunId).stopReason?.(evalRunId),
+    };
+}

package/dist/evals/runner/conversationSubject.d.ts

@@ -0,0 +1,18 @@
+import type { EvalToolCallRecord } from "../../agents/evalGate.js";
+import type { AgentSessionDetail } from "../../agents/service.js";
+import type { ConversationOutput, ConversationStopReason, ConversationUsage } from "../types.js";
+export type ConversationSubjectInput = {
+    detail: AgentSessionDetail;
+    calls: readonly EvalToolCallRecord[];
+    stopReason: ConversationStopReason;
+    usage: ConversationUsage;
+    /** User turns completed (a turn counts once the agent replied or failed). */
+    turns: number;
+    /** Set for the partial subject of a failed case. */
+    failure?: string;
+};
+/**
+ * The normalized subject of an agent result. Every text is redacted before it is capped, because stored agent
+ * messages are not redacted.
+ */
+export declare function buildConversationSubject(input: ConversationSubjectInput): ConversationOutput;

package/dist/evals/runner/conversationSubject.js

@@ -0,0 +1,123 @@
+import { redactSecrets } from "../../agent/streamLogRedact.js";
+import { EVAL_CONVERSATION_LIMITS } from "../constants.js";
+import { renderTranscript } from "../graders/transcript.js";
+const L = EVAL_CONVERSATION_LIMITS;
+function cut(text, max, count) {
+    if (text.length <= max)
+        return text;
+    count();
+    return `${text.slice(0, max)}…[truncated ${text.length - max} chars]`;
+}
+function redactedJson(value) {
+    try {
+        return redactSecrets(JSON.stringify(value ?? {}));
+    }
+    catch {
+        return "{}";
+    }
+}
+function subAgentSessions(detail) {
+    const out = new Map();
+    for (const span of detail.spans) {
+        const callId = span.attributes["gen_ai.tool.call.id"];
+        const child = span.attributes["stageflow.sub_agent.session_id"];
+        if (typeof callId === "string" && typeof child === "string")
+            out.set(callId, child);
+    }
+    return out;
+}
+function messagesOf(detail) {
+    let turn = 0;
+    const out = [];
+    for (const message of detail.messages) {
+        if (message.role === "user")
+            turn += 1;
+        out.push({
+            role: message.role,
+            text: cut(redactSecrets(message.text), L.maxMessageChars, () => undefined),
+            turn: Math.max(turn, 1),
+        });
+    }
+    return out;
+}
+function toolCallsOf(detail, calls, truncation) {
+    const children = subAgentSessions(detail);
+    const kept = calls.slice(0, L.maxToolCalls);
+    truncation.tool_calls_dropped = calls.length - kept.length;
+    return kept.map((call) => {
+        let args;
+        const json = redactedJson(call.args);
+        if (json.length > L.maxArgsChars) {
+            truncation.tool_args += 1;
+            args = { _truncated: json.slice(0, L.maxArgsChars) };
+        }
+        else {
+            args = JSON.parse(json);
+        }
+        const result = cut(redactSecrets(call.result ?? ""), L.maxResultSummaryChars, () => (truncation.tool_results += 1));
+        const sessionId = children.get(call.id);
+        return {
+            id: call.id,
+            turn: call.turn,
+            name: call.name,
+            args,
+            result_summary: result,
+            status: call.status,
+            policy_outcome: call.policyOutcome,
+            ...(sessionId !== undefined ? { session_id: sessionId } : {}),
+        };
+    });
+}
+function sizeOf(output) {
+    return Buffer.byteLength(JSON.stringify(output));
+}
+/**
+ * The normalized subject of an agent result. Every text is redacted before it is capped, because stored agent
+ * messages are not redacted.
+ */
+export function buildConversationSubject(input) {
+    const truncation = { messages: 0, tool_results: 0, tool_args: 0, tool_calls_dropped: 0 };
+    let messages = messagesOf(input.detail);
+    const finalReply = [...messages].reverse().find((m) => m.role === "assistant")?.text ?? "";
+    const tool_calls = toolCallsOf(input.detail, input.calls, truncation);
+    let droppedMessages = 0;
+    if (messages.length > L.maxMessages) {
+        const tail = L.maxMessages - L.keepFirstMessages;
+        droppedMessages = messages.length - L.maxMessages;
+        messages = [...messages.slice(0, L.keepFirstMessages), ...messages.slice(messages.length - tail)];
+    }
+    const summary = input.failure !== undefined
+        ? `${input.turns} turns, ${tool_calls.length} tool calls, ended ${input.stopReason}: ${input.failure}`
+        : `${input.turns} turns, ${tool_calls.length} tool calls, ended ${input.stopReason}`;
+    const output = {
+        kind: "conversation",
+        status: input.failure !== undefined ? "failure" : "success",
+        summary,
+        messages,
+        tool_calls,
+        final_reply: finalReply,
+        turns: input.turns,
+        stop_reason: input.stopReason,
+        usage: input.usage,
+    };
+    while (sizeOf(output) > L.maxSubjectBytes && output.messages.length > 2) {
+        const middle = Math.max(1, Math.floor(output.messages.length / 2));
+        output.messages.splice(middle, 1);
+        droppedMessages += 1;
+    }
+    truncation.messages = droppedMessages + countTruncatedText(output.messages);
+    const counts = Object.entries(truncation).filter(([, value]) => value > 0);
+    if (counts.length > 0) {
+        output.truncated = {
+            messages: truncation.messages,
+            tool_results: truncation.tool_results,
+            tool_args: truncation.tool_args,
+            tool_calls_dropped: truncation.tool_calls_dropped,
+        };
+    }
+    output.transcript = renderTranscript(output);
+    return output;
+}
+function countTruncatedText(messages) {
+    return messages.filter((m) => /…\[truncated \d+ chars\]$/.test(m.text)).length;
+}

package/dist/evals/runner/simulatorModel.d.ts

@@ -0,0 +1,51 @@
+import type { AgentDefinition } from "../../config/loadAgent.js";
+import type { ConversationBackend } from "../../agents/runtime/types.js";
+export type DriverUsage = {
+    costUsd: number;
+    tokensIn: number;
+    tokensOut: number;
+    model: string;
+    estimated?: boolean;
+};
+export type SimulatorModelInput = {
+    system: string;
+    user: string;
+    signal: AbortSignal;
+};
+export type SimulatorModelReply = {
+    text: string;
+    usage?: DriverUsage;
+};
+export interface SimulatorModel {
+    complete(input: SimulatorModelInput): Promise<SimulatorModelReply>;
+}
+export declare class SimulatorError extends Error {
+    readonly code = "EVAL_SIMULATOR_ERROR";
+    constructor(message?: string);
+}
+export declare const CHARS_PER_TOKEN_ESTIMATE = 4;
+/** Price in USD per one million tokens. */
+export type ModelPrice = {
+    inputPerMillion: number;
+    outputPerMillion: number;
+};
+export declare function estimateTokens(text: string): number;
+export declare function createBackendSimulatorModel(options: {
+    backend: ConversationBackend;
+    projectRoot: string;
+    definition: AgentDefinition;
+    modelId: string;
+    priceOf?: (modelId: string) => ModelPrice | undefined;
+}): SimulatorModel;
+export type FakeSimulatorReply = string | {
+    text: string;
+    usage?: DriverUsage;
+} | Error;
+export declare class FakeSimulatorModel implements SimulatorModel {
+    private readonly replies;
+    private readonly defaultUsage;
+    readonly calls: SimulatorModelInput[];
+    private index;
+    constructor(replies: readonly FakeSimulatorReply[] | ((call: SimulatorModelInput, index: number) => FakeSimulatorReply), defaultUsage?: DriverUsage);
+    complete(input: SimulatorModelInput): Promise<SimulatorModelReply>;
+}

package/dist/evals/runner/simulatorModel.js

@@ -0,0 +1,58 @@
+export class SimulatorError extends Error {
+    code = "EVAL_SIMULATOR_ERROR";
+    constructor(message = "The simulated user model failed to reply.") {
+        super(message);
+        this.name = "SimulatorError";
+    }
+}
+export const CHARS_PER_TOKEN_ESTIMATE = 4;
+export function estimateTokens(text) {
+    return Math.ceil(text.length / CHARS_PER_TOKEN_ESTIMATE);
+}
+export function createBackendSimulatorModel(options) {
+    const { backend, projectRoot, modelId } = options;
+    const definition = { ...options.definition, model: { ...options.definition.model, id: modelId } };
+    return {
+        async complete(input) {
+            if (backend.judge === undefined)
+                throw new SimulatorError("The agent backend cannot run a one-shot model call.");
+            const prompt = `${input.system}\n\n${input.user}`;
+            let text;
+            try {
+                text = await backend.judge({ projectRoot, definition, prompt: input.user, systemPrompt: input.system, signal: input.signal });
+            }
+            catch (error) {
+                if (input.signal.aborted)
+                    throw error;
+                throw new SimulatorError();
+            }
+            const tokensIn = estimateTokens(prompt);
+            const tokensOut = estimateTokens(text);
+            const price = options.priceOf?.(modelId);
+            const costUsd = price === undefined ? 0 : (tokensIn * price.inputPerMillion + tokensOut * price.outputPerMillion) / 1_000_000;
+            return { text, usage: { costUsd, tokensIn, tokensOut, model: modelId, estimated: true } };
+        },
+    };
+}
+export class FakeSimulatorModel {
+    replies;
+    defaultUsage;
+    calls = [];
+    index = 0;
+    constructor(replies, defaultUsage = { costUsd: 0.001, tokensIn: 100, tokensOut: 10, model: "fake/simulator" }) {
+        this.replies = replies;
+        this.defaultUsage = defaultUsage;
+    }
+    async complete(input) {
+        const index = this.index++;
+        this.calls.push(input);
+        const reply = typeof this.replies === "function" ? this.replies(input, index) : this.replies[index];
+        if (reply === undefined)
+            throw new Error("FakeSimulatorModel has no scripted reply left");
+        if (reply instanceof Error)
+            throw reply;
+        if (typeof reply === "string")
+            return { text: reply, usage: this.defaultUsage };
+        return { text: reply.text, usage: reply.usage ?? this.defaultUsage };
+    }
+}