stageflow 0.40.0 → 0.42.0

package/README.md

@@ -35,6 +35,7 @@ The same pipeline runs three ways without rewriting anything:
 - **Verified stage execution** — after-phase `verify` checks with durable evidence, plus `on_verify_fail` repair or manual operator recovery ([guide](docs/verified-stage-execution.md))
 - **HITL gates** — operator questions in the console; CI exits `2` when a run is waiting
 - **Agents** — project-owned, revisioned agents (`agents/*.agent.yaml`) with granted tools, ask-first approvals, traces, channels, schedules and a public web chat page ([guide](docs/agents.md))
+- **Evals** — check a stage or an agent against a dataset: graders, repeats, honest statistics, compare two runs, and sort failures into what to fix ([guide](docs/evals.md))
 - **Operator console** — an Inbox for gates and agent approvals, pipeline runs, agents and their sessions, connections, MCP servers and settings at `http://127.0.0.1:3847` ([guide](docs/operator-console.md))
 - **MCP endpoint** — Streamable HTTP at `/mcp` when `sf ui` or `sf mcp` is running
 - **MCP servers for agents and stages** — add servers, secrets and OAuth sign-in from Settings → Project MCP, or in the project `.mcp.json` ([guide](docs/mcp-servers.md))
@@ -288,7 +289,7 @@ Full docs: **[tejasghutukade.github.io/stageflow](https://tejasghutukade.github.
 | [docs/hosted.md](docs/hosted.md) | Embed Stageflow from another application and receive ordered run events |
 | [docs/quickstart.md](docs/quickstart.md) | Expanded quick start |
 | [docs/yaml-catalog.md](docs/yaml-catalog.md) | Pipelines, stages, tasks schema |
-| [docs/cli-reference.md](docs/cli-reference.md) | `sf init`, `sf run`, `sf run-stage`, `sf graph`, `sf validate`, `sf ui`, `sf mcp`, `sf envelope`, `sf export-run`, `sf artifact`, `sf skills`, `sf providers` |
+| [docs/cli-reference.md](docs/cli-reference.md) | `sf init`, `sf run`, `sf run-stage`, `sf graph`, `sf validate`, `sf ui`, `sf mcp`, `sf envelope`, `sf export-run`, `sf artifact`, `sf skills`, `sf providers`, `sf eval` |
 | [docs/envelopes.md](docs/envelopes.md) | Handoff envelope contract |
 | [docs/verified-stage-execution.md](docs/verified-stage-execution.md) | After-phase `verify`, evidence, and `on_verify_fail` repair / manual recovery |
 | [docs/hitl.md](docs/hitl.md) | Gate kinds, `--skip-gates`, exit code `2` |
@@ -298,6 +299,7 @@ Full docs: **[tejasghutukade.github.io/stageflow](https://tejasghutukade.github.
 | [docs/providers.md](docs/providers.md) | Pi providers, `sf providers` |
 | [docs/docker.md](docs/docker.md) | Docker Compose local try, image pull/run, backup, self-hosting |
 | [docs/operator-console.md](docs/operator-console.md) | Console navigation, command palette, shortcuts and routes; one page per area ([Inbox](docs/console-inbox.md), [Runs](docs/console-runs.md), [Agents](docs/console-agents.md), [Connections](docs/console-connections.md), [Settings](docs/console-settings.md)) |
+| [docs/evals.md](docs/evals.md) | Evals for stages and agents: datasets, graders, runs, compare, failure sorting, the engine |
 | [docs/agents.md](docs/agents.md) | Agents: definition, runtime, approvals, channels, schedules, web chat |
 | [docs/mcp-servers.md](docs/mcp-servers.md) | Add MCP servers, secrets and OAuth; use them from agents and stages |
 | [docs/channels.md](docs/channels.md) | Connections and communication channels |

package/dist/agents/authorChat.d.ts

@@ -1,3 +1,4 @@
+import { type AuthorSessionRecord } from "./authorSessionStore.js";
 import { type SummaryLine } from "./definitionSummary.js";
 import { type AgentAuthorDraft, type OperatorAgentHost } from "../operatorAgent/index.js";
 export type AgentAuthorTurnResult = {
@@ -14,6 +15,9 @@ export declare function runAgentAuthorTurn(input: {
     message: string;
     sessionId?: string;
     draft?: AgentAuthorDraft;
+    /** Composer override. Blank uses the same default as Workshop chat. */
+    model?: string | null;
 }): Promise<AgentAuthorTurnResult>;
 export declare function undoAgentAuthorTurn(sessionId: string): AgentAuthorDraft;
+export declare function authorSessionForApi(sessionId: string): AuthorSessionRecord;
 export declare function parseAgentAuthorDraft(body: unknown): AgentAuthorDraft;

package/dist/agents/authorChat.js

@@ -1,7 +1,53 @@
 import { randomUUID } from "node:crypto";
+import { resolveWorkshopModel } from "../workshop/modelSettings.js";
+import { createAuthorSession, readAuthorSession, saveAuthorSession, } from "./authorSessionStore.js";
 import { describeChanges } from "./definitionSummary.js";
 import { AGENT_AUTHOR_PROFILE_ID, createAgentAuthorContext, isAgentAuthorContext, readAgentAuthorDraft, } from "../operatorAgent/index.js";
 const sessions = new Map();
+function isPlainObject(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function sameDraftValue(a, b) {
+    return JSON.stringify(a) === JSON.stringify(b);
+}
+/** Objects walk key by key; arrays and scalars are wholes. Turn changes win, including removals. */
+function mergeTurnDraft(start, end, disk) {
+    const keys = new Set([...Object.keys(start), ...Object.keys(end), ...Object.keys(disk)]);
+    const next = {};
+    for (const key of keys) {
+        const startHas = Object.prototype.hasOwnProperty.call(start, key);
+        const endHas = Object.prototype.hasOwnProperty.call(end, key);
+        const diskHas = Object.prototype.hasOwnProperty.call(disk, key);
+        if (startHas && !endHas)
+            continue;
+        if (!endHas) {
+            if (diskHas)
+                next[key] = structuredClone(disk[key]);
+            continue;
+        }
+        if (!startHas) {
+            next[key] = structuredClone(end[key]);
+            continue;
+        }
+        const startVal = start[key];
+        const endVal = end[key];
+        const diskVal = disk[key];
+        if (isPlainObject(startVal) && isPlainObject(endVal) && (!diskHas || isPlainObject(diskVal))) {
+            if (!diskHas && sameDraftValue(startVal, endVal))
+                continue;
+            next[key] = mergeTurnDraft(startVal, endVal, diskHas && isPlainObject(diskVal) ? diskVal : {});
+            continue;
+        }
+        if (sameDraftValue(startVal, endVal)) {
+            if (diskHas)
+                next[key] = structuredClone(diskVal);
+        }
+        else {
+            next[key] = structuredClone(endVal);
+        }
+    }
+    return next;
+}
 export function resetAgentAuthorSessionsForTests() {
     for (const session of sessions.values())
         session.close();
@@ -12,26 +58,49 @@ export async function runAgentAuthorTurn(input) {
     if (!message)
         throw new Error("agent.invalid");
     let sessionId = input.sessionId?.trim();
+    let record = sessionId ? readAuthorSession(sessionId) : null;
     let session = sessionId ? sessions.get(sessionId) : undefined;
-    if (sessionId && !session)
+    if (sessionId && !session && !record)
         throw new Error("agent.session_not_found");
     const posted = createAgentAuthorContext({
         projectRoot: input.projectRoot,
-        draft: input.draft,
+        draft: input.draft ?? record?.draft,
     });
+    if (!record) {
+        const draft = readAgentAuthorDraft(posted);
+        if (!sessionId) {
+            record = createAuthorSession({ draft, model: input.model ?? undefined });
+            sessionId = record.id;
+        }
+        else {
+            const now = new Date().toISOString();
+            record = saveAuthorSession({
+                version: 1,
+                id: sessionId,
+                createdAt: now,
+                updatedAt: now,
+                draft,
+                transcript: [],
+                ...(input.model?.trim() ? { model: input.model.trim() } : {}),
+            });
+        }
+    }
+    if (!sessionId || !record)
+        throw new Error("agent.invalid");
     if (!session) {
-        sessionId = randomUUID();
         session = input.host.openSession({
             profileId: AGENT_AUTHOR_PROFILE_ID,
             context: posted,
         });
         sessions.set(sessionId, session);
+        if (record.transcript.length > 0) {
+            await session.prepareRestart?.(record.transcript.map((entry) => ({ role: entry.role, text: entry.text })));
+        }
     }
-    else {
-        session.setContext(posted);
-    }
+    session.setContext(posted);
     const before = readAgentAuthorDraft(session.getContext());
-    const events = await session.send(message);
+    const modelId = resolveWorkshopModel({ sessionOverride: input.model ?? record.model });
+    const events = await session.send(message, { modelId });
     const text = events
         .filter((event) => event.type === "message" && event.role === "assistant")
         .map((event) => (event.type === "message" ? event.text : ""))
@@ -39,11 +108,33 @@ export async function runAgentAuthorTurn(input) {
         .join("\n\n");
     const error = events.find((event) => event.type === "error");
     const draft = readAgentAuthorDraft(session.getContext());
+    const summaryLines = describeChanges(before, draft);
+    const latest = readAuthorSession(sessionId);
+    if (!latest)
+        throw new Error("agent.session_not_found");
+    const merged = mergeTurnDraft(before, draft, latest.draft);
+    const now = new Date().toISOString();
+    const saved = saveAuthorSession({
+        ...latest,
+        draft: merged,
+        ...(input.model?.trim() ? { model: input.model.trim() } : {}),
+        transcript: [
+            ...latest.transcript,
+            { id: randomUUID(), role: "user", text: message, createdAt: now },
+            {
+                id: randomUUID(),
+                role: "assistant",
+                text: text || (error && error.type === "error" ? error.message : ""),
+                createdAt: now,
+                ...(summaryLines.length > 0 ? { summaryLines } : {}),
+            },
+        ],
+    });
     return {
-        sessionId: sessionId,
-        text: text || (error && error.type === "error" ? error.message : ""),
-        draft,
-        summaryLines: describeChanges(before, draft),
+        sessionId: saved.id,
+        text: saved.transcript.at(-1)?.text ?? "",
+        draft: merged,
+        summaryLines,
     };
 }
 export function undoAgentAuthorTurn(sessionId) {
@@ -56,7 +147,17 @@ export function undoAgentAuthorTurn(sessionId) {
     const context = session.getContext();
     if (!isAgentAuthorContext(context))
         throw new Error("agent.invalid");
-    return readAgentAuthorDraft(context);
+    const draft = readAgentAuthorDraft(context);
+    const record = readAuthorSession(sessionId);
+    if (record)
+        saveAuthorSession({ ...record, draft });
+    return draft;
+}
+export function authorSessionForApi(sessionId) {
+    const record = readAuthorSession(sessionId);
+    if (!record)
+        throw new Error("agent.session_not_found");
+    return record;
 }
 export function parseAgentAuthorDraft(body) {
     return body !== null && typeof body === "object" && !Array.isArray(body) ? body : {};

package/dist/agents/authorSessionStore.d.ts

@@ -0,0 +1,35 @@
+import type { SummaryLine } from "./definitionSummary.js";
+import type { AgentAuthorDraft } from "../operatorAgent/index.js";
+export type AuthorSessionMessage = {
+    id: string;
+    role: "user" | "assistant";
+    text: string;
+    createdAt: string;
+    summaryLines?: SummaryLine[];
+};
+export type AuthorSessionRecord = {
+    version: 1;
+    id: string;
+    createdAt: string;
+    updatedAt: string;
+    draft: AgentAuthorDraft;
+    transcript: AuthorSessionMessage[];
+    model?: string;
+};
+export declare function authorSessionsDir(storeRoot?: string): string;
+export declare function authorSessionFilePath(sessionId: string, storeRoot?: string): string;
+export declare function parseAuthorSession(value: unknown): AuthorSessionRecord | null;
+export declare function createAuthorSession(input?: {
+    draft?: AgentAuthorDraft;
+    model?: string;
+}, storeRoot?: string): AuthorSessionRecord;
+export declare function readAuthorSession(sessionId: string, storeRoot?: string): AuthorSessionRecord | null;
+export declare function saveAuthorSession(record: AuthorSessionRecord, storeRoot?: string): AuthorSessionRecord;
+export type AuthorDraftSummary = {
+    id: string;
+    name: string;
+    updatedAt: string;
+    messages: number;
+};
+/** Unsaved author sessions, newest first. Empty shells with no name and no chat are left out. */
+export declare function listAuthorDrafts(storeRoot?: string): AuthorDraftSummary[];

package/dist/agents/authorSessionStore.js

@@ -0,0 +1,152 @@
+import { randomUUID } from "node:crypto";
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { globalStageflowHome } from "../project/globalHome.js";
+function guardSessionId(sessionId) {
+    const trimmed = sessionId.trim();
+    if (!trimmed || trimmed.includes("/") || trimmed.includes("\\") || trimmed.includes("..") || trimmed === ".") {
+        throw new Error("agent.invalid");
+    }
+    return trimmed;
+}
+export function authorSessionsDir(storeRoot = globalStageflowHome()) {
+    return path.join(storeRoot, "agents", "author-sessions");
+}
+export function authorSessionFilePath(sessionId, storeRoot = globalStageflowHome()) {
+    return path.join(authorSessionsDir(storeRoot), guardSessionId(sessionId), "session.json");
+}
+function isPlainObject(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function parseMessage(value) {
+    if (!isPlainObject(value))
+        return null;
+    if (typeof value.id !== "string" || !value.id)
+        return null;
+    if (value.role !== "user" && value.role !== "assistant")
+        return null;
+    if (typeof value.text !== "string")
+        return null;
+    if (typeof value.createdAt !== "string" || !value.createdAt)
+        return null;
+    const summaryLines = Array.isArray(value.summaryLines)
+        ? value.summaryLines.flatMap((line) => {
+            if (!isPlainObject(line) || typeof line.label !== "string" || typeof line.value !== "string")
+                return [];
+            return [{ label: line.label, value: line.value }];
+        })
+        : [];
+    return {
+        id: value.id,
+        role: value.role,
+        text: value.text,
+        createdAt: value.createdAt,
+        ...(summaryLines.length > 0 ? { summaryLines } : {}),
+    };
+}
+export function parseAuthorSession(value) {
+    if (!isPlainObject(value) || value.version !== 1)
+        return null;
+    if (typeof value.id !== "string" || !value.id)
+        return null;
+    if (typeof value.createdAt !== "string" || typeof value.updatedAt !== "string")
+        return null;
+    if (!isPlainObject(value.draft))
+        return null;
+    if (!Array.isArray(value.transcript))
+        return null;
+    const transcript = [];
+    for (const entry of value.transcript) {
+        const message = parseMessage(entry);
+        if (!message)
+            return null;
+        transcript.push(message);
+    }
+    const model = typeof value.model === "string" && value.model.trim() ? value.model.trim() : undefined;
+    return {
+        version: 1,
+        id: value.id,
+        createdAt: value.createdAt,
+        updatedAt: value.updatedAt,
+        draft: value.draft,
+        transcript,
+        ...(model ? { model } : {}),
+    };
+}
+function writeRecord(record, storeRoot) {
+    const file = authorSessionFilePath(record.id, storeRoot);
+    mkdirSync(path.dirname(file), { recursive: true });
+    writeFileSync(file, JSON.stringify(record));
+}
+export function createAuthorSession(input = {}, storeRoot) {
+    const now = new Date().toISOString();
+    const record = {
+        version: 1,
+        id: randomUUID(),
+        createdAt: now,
+        updatedAt: now,
+        draft: { ...(input.draft ?? {}) },
+        transcript: [],
+        ...(input.model?.trim() ? { model: input.model.trim() } : {}),
+    };
+    writeRecord(record, storeRoot);
+    return record;
+}
+export function readAuthorSession(sessionId, storeRoot) {
+    let file;
+    try {
+        file = authorSessionFilePath(sessionId, storeRoot);
+    }
+    catch {
+        return null;
+    }
+    if (!existsSync(file))
+        return null;
+    try {
+        return parseAuthorSession(JSON.parse(readFileSync(file, "utf8")));
+    }
+    catch {
+        return null;
+    }
+}
+export function saveAuthorSession(record, storeRoot) {
+    const next = { ...record, updatedAt: new Date().toISOString() };
+    writeRecord(next, storeRoot);
+    return next;
+}
+function draftTitle(draft) {
+    const name = typeof draft.name === "string" ? draft.name.trim() : "";
+    return name || "Untitled draft";
+}
+function draftHasWork(record) {
+    if (record.transcript.length > 0)
+        return true;
+    for (const key of ["name", "instructions", "description"]) {
+        const value = record.draft[key];
+        if (typeof value === "string" && value.trim())
+            return true;
+    }
+    return false;
+}
+/** Unsaved author sessions, newest first. Empty shells with no name and no chat are left out. */
+export function listAuthorDrafts(storeRoot = globalStageflowHome()) {
+    const dir = authorSessionsDir(storeRoot);
+    if (!existsSync(dir))
+        return [];
+    const drafts = [];
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+        if (!entry.isDirectory())
+            continue;
+        const record = readAuthorSession(entry.name, storeRoot);
+        if (!record || !draftHasWork(record))
+            continue;
+        drafts.push({
+            id: record.id,
+            name: draftTitle(record.draft),
+            updatedAt: record.updatedAt,
+            messages: record.transcript.length,
+        });
+    }
+    drafts.sort((a, b) => (a.updatedAt < b.updatedAt ? 1 : a.updatedAt > b.updatedAt ? -1 : 0));
+    return drafts;
+}

package/dist/agents/chatTurn.d.ts

@@ -18,6 +18,8 @@ export declare class AgentChatController {
     stop(sessionId: string): boolean;
     /** Let the runtime drop what it holds for a session (model session, isolated MCP processes). Never throws. */
     release(sessionId: string): Promise<void>;
+    /** An eval conversation is over: stop any running turn, release the runtime session, drop its in-memory overrides. */
+    closeEvalSession(projectRoot: string, sessionId: string): Promise<void>;
     /** Stop a turn that is still opening its session (client left before the session id arrived). */
     stopOpening(agentId: string): boolean;
 }

package/dist/agents/chatTurn.js

@@ -1,3 +1,5 @@
+import { AgentError } from "./errors.js";
+import { EVAL_SESSION_SOURCE, isEvalSession } from "./evalSession.js";
 import { AgentConversation } from "./sessions.js";
 /**
  * Serializes turns per session (or per agent while a session is being
@@ -19,7 +21,12 @@ export class AgentChatController {
         this.turnChains.set(key, run.then(() => undefined, () => undefined));
         return run;
     }
-    send(input) {
+    async send(input) {
+        if (input.sessionId && input.source !== EVAL_SESSION_SOURCE) {
+            const { session } = await this.options.service.getSession(input.projectRoot, input.sessionId);
+            if (isEvalSession(session))
+                throw new AgentError("agent.session_eval_readonly", "Eval sessions are read only");
+        }
         return this.runExclusive(input.sessionId ?? `agent:${input.agentId}`, async () => {
             const chat = new AgentConversation(this.options);
             if (input.sessionId)
@@ -66,6 +73,15 @@ export class AgentChatController {
             this.options.logError?.(`releasing agent session ${sessionId} failed: ${error instanceof Error ? error.message : String(error)}`);
         }
     }
+    /** An eval conversation is over: stop any running turn, release the runtime session, drop its in-memory overrides. */
+    async closeEvalSession(projectRoot, sessionId) {
+        const { session } = await this.options.service.getSession(projectRoot, sessionId);
+        if (!isEvalSession(session))
+            throw new AgentError("agent.invalid", `Session ${sessionId} is not an eval session`);
+        this.stop(sessionId);
+        await this.release(sessionId);
+        this.options.service.forgetEvalSession(sessionId);
+    }
     /** Stop a turn that is still opening its session (client left before the session id arrived). */
     stopOpening(agentId) {
         const chat = this.activeByAgent.get(agentId);

package/dist/agents/errors.d.ts

@@ -1,5 +1,5 @@
 import type { AgentIssue } from "../config/loadAgent.js";
-export type AgentErrorCode = "agent.invalid" | "agent.not_found" | "agent.exists" | "agent.archived" | "agent.disabled" | "agent.conflict" | "agent.revision_not_found" | "agent.session_not_found" | "agent.session_busy" | "agent.tool_not_granted" | "agent.approval_not_required" | "agent.approval_not_pending" | "agent.approval_expired" | "agent.approval_forbidden" | "agent.unbound" | "agent.sender_not_allowed" | "agent.outbound_unknown" | "agent.trace_metadata_body" | "agent.guardrail_blocked" | "agent.delegate_not_allowed" | "agent.delegate_depth_exceeded" | "agent.sandbox_unavailable" | "agent.limit_reached" | "agent.model_timeout" | "agent.backend_unavailable" | "agent.approval_denied";
+export type AgentErrorCode = "agent.invalid" | "agent.not_found" | "agent.exists" | "agent.archived" | "agent.disabled" | "agent.conflict" | "agent.revision_not_found" | "agent.session_not_found" | "agent.session_busy" | "agent.session_eval_readonly" | "agent.tool_not_granted" | "agent.approval_not_required" | "agent.approval_not_pending" | "agent.approval_expired" | "agent.approval_forbidden" | "agent.unbound" | "agent.sender_not_allowed" | "agent.outbound_unknown" | "agent.trace_metadata_body" | "agent.guardrail_blocked" | "agent.delegate_not_allowed" | "agent.delegate_depth_exceeded" | "agent.sandbox_unavailable" | "agent.limit_reached" | "agent.model_timeout" | "agent.backend_unavailable" | "agent.approval_denied";
 /** Domain failure with a stable code; the HTTP, CLI and MCP layers map it, they don't parse messages. */
 export declare class AgentError extends Error {
     readonly code: AgentErrorCode;

package/dist/agents/errors.js

@@ -8,6 +8,7 @@ const STATUS = {
     "agent.disabled": 409,
     "agent.conflict": 409,
     "agent.session_busy": 409,
+    "agent.session_eval_readonly": 403,
     "agent.limit_reached": 409,
     "agent.model_timeout": 504,
     "agent.approval_forbidden": 403,

package/dist/agents/evalGate.d.ts

@@ -0,0 +1,61 @@
+import type { AgentDefinition } from "../config/loadAgent.js";
+import type { GateDecision, ToolCall } from "./runtime/types.js";
+export type EvalToolPolicy = "real" | "read_only" | "mocked";
+export type EvalApprovalPolicy = "deny" | "auto_approve_read" | "record_only";
+export type EvalPolicyOutcome = "ran" | "stubbed" | "mocked" | "would_ask_first" | "denied";
+export type EvalGatePolicy = {
+    tools: EvalToolPolicy;
+    approvals: EvalApprovalPolicy;
+    /** Grant ids the eval author declares read-only (`<server>.<tool>`, `<server>.*`, or a builtin/Stageflow grant id). */
+    readOnlyTools: string[];
+    /** Canned results by tool grant id: `{ result }` or `{ error }`. */
+    mocks: Record<string, {
+        result?: unknown;
+        error?: string;
+    }>;
+};
+export type EvalToolCallRecord = {
+    id: string;
+    turn: number;
+    name: string;
+    args: Record<string, unknown>;
+    policyOutcome: EvalPolicyOutcome;
+    status: "ok" | "error" | "not_run";
+    result?: string;
+};
+export type EvalGate = {
+    readonly policy: EvalGatePolicy;
+    /** Set when tools run for real; the console shows it as a warning. */
+    readonly warning: string | undefined;
+    decide(call: ToolCall, definition?: AgentDefinition): GateDecision;
+    observeFinished(call: ToolCall, result: {
+        ok: boolean;
+        output: string;
+    }): void;
+    recordOf(callId: string): EvalToolCallRecord | undefined;
+    records(): EvalToolCallRecord[];
+};
+export declare const APPROVAL_NOT_GRANTED: (tool: string) => string;
+export declare const STUB_RESULT: (tool: string) => string;
+/**
+ * The eval-only tool gate. Order: denied (not granted, forbidden path, operator-only, always-blocked), then the approval
+ * policy for ask-first tools, then the tool policy. A canned result can reach the model only as the reason of a blocked
+ * call, which the runtime turns into an error-flagged tool result.
+ */
+export declare function createEvalGate(options: {
+    policy: EvalGatePolicy;
+    definition: AgentDefinition;
+    /** Called when a call is decided, and again when a call that ran finishes (same id; the later record wins). */
+    onCall?: (record: EvalToolCallRecord) => void;
+    /** The conversation turn a call belongs to; defaults to 0. */
+    turn?: () => number;
+    projectRoot?: string;
+    stageflowHome?: string;
+}): EvalGate;
+/** Which eval gate governs which eval session. An eval session with no entry is denied every tool. */
+export declare class EvalGateRegistry {
+    private readonly gates;
+    set(sessionId: string, gate: EvalGate): void;
+    get(sessionId: string): EvalGate | undefined;
+    delete(sessionId: string): void;
+}

package/dist/agents/evalGate.js

@@ -0,0 +1,103 @@
+import { redactSecrets } from "../agent/streamLogRedact.js";
+import { globalStageflowHome } from "../project/globalHome.js";
+import { forbiddenWriteTarget, isToolGranted, requiresAskFirst } from "./policy.js";
+import { classifyTool } from "./toolRisk.js";
+export const APPROVAL_NOT_GRANTED = (tool) => `Approval for ${tool} was not granted in this evaluation. Continue without it.`;
+export const STUB_RESULT = (tool) => `${tool} was not executed in this evaluation.`;
+const ALWAYS_BLOCKED = new Set([
+    "stageflow.run_eval",
+    "stageflow.agent_chat",
+    "stageflow.fire_trigger",
+    "stageflow.sort_eval_failures",
+    "stageflow.cancel_eval_run",
+]);
+const BLOCKED_UNLESS_REAL = new Set(["stageflow.probe_project_mcp"]);
+function redactedArgs(input) {
+    try {
+        return JSON.parse(redactSecrets(JSON.stringify(input)));
+    }
+    catch {
+        return {};
+    }
+}
+function mockText(mock) {
+    if (mock.error !== undefined)
+        return mock.error;
+    return typeof mock.result === "string" ? mock.result : JSON.stringify(mock.result ?? null);
+}
+/**
+ * The eval-only tool gate. Order: denied (not granted, forbidden path, operator-only, always-blocked), then the approval
+ * policy for ask-first tools, then the tool policy. A canned result can reach the model only as the reason of a blocked
+ * call, which the runtime turns into an error-flagged tool result.
+ */
+export function createEvalGate(options) {
+    const { policy } = options;
+    const byId = new Map();
+    const emit = (record) => {
+        byId.set(record.id, record);
+        options.onCall?.(record);
+    };
+    const decide = (call, definition = options.definition) => {
+        const base = { id: call.id, turn: options.turn?.() ?? 0, name: call.tool, args: redactedArgs(call.input) };
+        const refuse = (policyOutcome, reason) => {
+            emit({ ...base, policyOutcome, status: "not_run", result: reason });
+            return { allow: false, reason };
+        };
+        if (!isToolGranted(definition, call.tool))
+            return refuse("denied", `Tool ${call.tool} is not granted to this agent.`);
+        if (options.projectRoot && forbiddenWriteTarget(options.projectRoot, options.stageflowHome ?? globalStageflowHome(), call.tool, call.input) !== undefined) {
+            return refuse("denied", "Agent definitions and Stageflow state cannot be written by an agent.");
+        }
+        const kind = classifyTool(definition, call.tool, policy.readOnlyTools);
+        const blocked = kind.class === "operator_only" ||
+            ALWAYS_BLOCKED.has(call.tool) ||
+            (policy.tools !== "real" && (BLOCKED_UNLESS_REAL.has(call.tool) || kind.class === "sub_agent"));
+        if (blocked)
+            return refuse("denied", `${call.tool} cannot run in an evaluation.`);
+        if (requiresAskFirst(definition, call.tool)) {
+            if (policy.approvals === "deny")
+                return refuse("denied", APPROVAL_NOT_GRANTED(call.tool));
+            if (policy.approvals === "record_only" || !kind.readOnly)
+                return refuse("would_ask_first", APPROVAL_NOT_GRANTED(call.tool));
+        }
+        const mock = policy.mocks[call.tool];
+        if (policy.tools === "real" || (kind.readOnly && !(policy.tools === "mocked" && mock))) {
+            emit({ ...base, policyOutcome: "ran", status: "ok" });
+            return { allow: true };
+        }
+        if (mock) {
+            const text = mockText(mock);
+            emit({ ...base, policyOutcome: "mocked", status: mock.error !== undefined ? "error" : "ok", result: redactSecrets(text) });
+            return { allow: false, reason: text };
+        }
+        const text = STUB_RESULT(call.tool);
+        emit({ ...base, policyOutcome: "stubbed", status: "not_run", result: text });
+        return { allow: false, reason: text };
+    };
+    return {
+        policy,
+        warning: policy.tools === "real" ? "Tools run for real in this evaluation: they can change files, call services and spend money." : undefined,
+        decide,
+        observeFinished(call, result) {
+            const record = byId.get(call.id);
+            if (!record || record.policyOutcome !== "ran")
+                return;
+            emit({ ...record, status: result.ok ? "ok" : "error", result: redactSecrets(result.output) });
+        },
+        recordOf: (callId) => byId.get(callId),
+        records: () => [...byId.values()],
+    };
+}
+/** Which eval gate governs which eval session. An eval session with no entry is denied every tool. */
+export class EvalGateRegistry {
+    gates = new Map();
+    set(sessionId, gate) {
+        this.gates.set(sessionId, gate);
+    }
+    get(sessionId) {
+        return this.gates.get(sessionId);
+    }
+    delete(sessionId) {
+        this.gates.delete(sessionId);
+    }
+}

package/dist/agents/evalSession.d.ts

@@ -0,0 +1,26 @@
+import type { AgentDefinition } from "../config/loadAgent.js";
+export declare const EVAL_SESSION_SOURCE = "eval";
+export type EvalSessionOverrides = {
+    /** Open this revision instead of the current one. Pinned for the life of the session. */
+    revision?: number;
+    definition?: {
+        model?: Partial<AgentDefinition["model"]>;
+        limits?: Partial<AgentDefinition["limits"]>;
+        observability?: Partial<AgentDefinition["observability"]>;
+        evals?: Partial<AgentDefinition["evals"]>;
+    };
+    limits?: {
+        max_turns?: number;
+        session_minutes?: number;
+        cost_usd?: number | null;
+    };
+    capture?: AgentDefinition["observability"]["capture"];
+};
+export declare function isEvalSession(session: {
+    source: string;
+}): boolean;
+/**
+ * The definition an eval session runs on: the pinned revision with the eval's model, limits and capture, and with
+ * every side effect off (always-on, OTLP push, groundedness judge). Capture defaults to full so tool calls are recorded.
+ */
+export declare function applyEvalOverrides(base: AgentDefinition, overrides: EvalSessionOverrides): AgentDefinition;