stageflow 0.39.0 → 0.42.0

package/dist/agents/hostContext.js

@@ -12,6 +12,7 @@ import { buildToolkit } from "../toolkit/index.js";
 import { createPiAgentRuntime } from "./runtime/piRuntime.js";
 import { AgentService } from "./service.js";
 import { getModelCatalog } from "../models/catalog.js";
+import { EvalGateRegistry } from "./evalGate.js";
 import { defaultAgentProjectRoot } from "./projectRoot.js";
 import { PublicWebChat } from "./webChat/publicChat.js";
 import { AgentStore } from "./store.js";
@@ -82,9 +83,11 @@ export function createAgentHostContext(options) {
                 }
                 : {}),
         });
+    const evalGates = new EvalGateRegistry();
     const chat = new AgentChatController({
         service,
         backend,
+        evalGates,
         otlpSink: "otlpSink" in options ? options.otlpSink : otlpSinkFromEnv(),
         logError: (message) => console.error(message),
     });
@@ -147,7 +150,7 @@ export function createAgentHostContext(options) {
         delivery.start();
     if (options.listenOnChannels)
         void inbound.resumeInterrupted().catch((error) => console.error(`agent resume failed: ${String(error)}`));
-    return { service, chat, backend, inbound, publicChat, channelContact, ...(channelListeners ? { channelListeners } : {}), delivery, stop: async () => {
+    return { service, chat, evalGates, backend, inbound, publicChat, channelContact, ...(channelListeners ? { channelListeners } : {}), delivery, stop: async () => {
             delivery.stop();
             await channelListeners?.stop();
             // The Host owns every runtime session's lifetime: nothing outlives it.

package/dist/agents/runtime/piRuntime.d.ts

@@ -34,7 +34,7 @@ export type OpenSessionInput = {
         reason: string;
         terminate?: boolean;
     } | undefined>;
-    delegateTool?: (agentId: string, message: string) => Promise<string>;
+    delegateTool?: (agentId: string, message: string, callId: string) => Promise<string>;
     /** Stageflow tools this agent is granted. */
     stageflowTools?: ToolkitTool[];
     memory?: StageMemory;

package/dist/agents/runtime/piRuntime.js

@@ -149,8 +149,8 @@ async function openPiSession(options, input) {
                     label: `Ask ${agentId}`,
                     description: `Hand a task to the "${agentId}" agent and get its reply.`,
                     parameters: Type.Object({ message: Type.String({ description: "The task or question for the agent." }) }),
-                    execute: async (_id, args) => {
-                        const reply = await input.delegateTool(agentId, args.message);
+                    execute: async (callId, args) => {
+                        const reply = await input.delegateTool(agentId, args.message, callId);
                         return { content: [{ type: "text", text: reply }], details: {} };
                     },
                 }));
@@ -296,7 +296,7 @@ export function createPiAgentRuntime(options) {
             hooks: () => state.hooks,
             ...(memory ? { memory } : {}),
             ...(stageflowTools.length > 0 ? { stageflowTools } : {}),
-            ...(subAgents.length > 0 ? { delegateTool: (agentId, message) => state.hooks.delegate(agentId, message) } : {}),
+            ...(subAgents.length > 0 ? { delegateTool: (agentId, message, callId) => state.hooks.delegate(agentId, message, callId) } : {}),
             gate: async (call) => {
                 const hooks = state.hooks;
                 if (!hooks)
@@ -433,7 +433,7 @@ export function createPiAgentRuntime(options) {
             const opened = await open({
                 projectRoot: input.projectRoot,
                 definition: { ...input.definition, tools: { ...input.definition.tools, builtin: [], builtin_tools: [], mcp: {}, skills: [] }, sub_agents: [] },
-                systemPrompt: "You are a strict evaluator. Answer only with the requested number.",
+                systemPrompt: input.systemPrompt ?? "You are a strict evaluator. Answer only with the requested number.",
                 toolNames: [],
                 gate: async () => ({ block: true, reason: "No tools" }),
                 hooks: () => undefined,

package/dist/agents/runtime/types.d.ts

@@ -39,7 +39,7 @@ export type TurnHooks = {
         model?: string;
     }): void;
     /** Run another agent for a `agent:<id>` call and return its reply. */
-    delegate(agentId: string, message: string): Promise<string>;
+    delegate(agentId: string, message: string, callId?: string): Promise<string>;
 };
 export type CompleteInput = {
     sessionId: string;
@@ -64,6 +64,8 @@ export type ConversationBackend = {
         projectRoot: string;
         definition: AgentDefinition;
         prompt: string;
+        /** Replaces the default "strict evaluator" system prompt. */
+        systemPrompt?: string;
         signal: AbortSignal;
     }): Promise<string>;
     /** Drop any cached model session for a conversation session. */

package/dist/agents/service.d.ts

@@ -3,6 +3,7 @@ import { type AgentInboundMessage } from "./communications/port.js";
 import { ApprovalBroker } from "./approvalBroker.js";
 import { type DeliveryLedger, type ScheduleLedger } from "./ledgers.js";
 import { type FinishInput } from "./turnLifecycle.js";
+import { type EvalSessionOverrides } from "./evalSession.js";
 import { type DiffLine } from "./lineDiff.js";
 import { AgentStore, type AgentApproval, type AgentEvalScore, type AgentMessage, type AgentRevision, type AgentRevisionMeta, type AgentSession, type AgentSessionListItem, type AgentDelivery, type AgentSpan, type AgentTurn, type ApprovalActor, type RevisionSource, type SessionQuery, type SessionStats, type SessionStatus } from "./store.js";
 /** Lookups outside the agents directory, used only to produce non-blocking warnings. */
@@ -88,6 +89,8 @@ export declare class AgentService {
     private readonly environment;
     private readonly now;
     private readonly tails;
+    /** Overrides of open eval sessions. Memory only: eval sessions do not survive a Host restart. */
+    private readonly evalOverrides;
     /** Wakes a turn paused at an ask-first tool when its approval is decided. */
     readonly approvalBroker: ApprovalBroker;
     /** Owns every legal change of a turn, its session and its approval. */
@@ -140,6 +143,25 @@ export declare class AgentService {
     get(projectRoot: string, agentId: string): Promise<AgentDetail>;
     /** The definition a session is pinned to. */
     definitionAt(projectRoot: string, agentId: string, revision: number): Promise<AgentDefinition>;
+    /** The pinned definition, with an eval session's overrides applied. Real sessions get it unchanged. */
+    private effectiveDefinition;
+    /** The overrides an eval session was opened with (sub-agent sessions inherit them), or undefined for a real session. */
+    evalOverridesOf(sessionId: string): EvalSessionOverrides | undefined;
+    /** Every eval session below this one (sub-agents and theirs). */
+    listChildSessions(sessionId: string): AgentSession[];
+    /** Delete eval sessions and everything below them. Refuses a session that is not an eval session. */
+    deleteEvalSessions(projectRoot: string, sessionIds: string[]): Promise<string[]>;
+    /**
+     * Boot cleanup of eval sessions: those of the given (recovered) eval runs, and older root sessions no eval result
+     * references. Returns how many sessions were deleted, descendants included.
+     */
+    sweepEvalSessions(input: {
+        runIds: string[];
+        olderThanMs: number;
+        isReferenced: (sessionId: string) => Promise<boolean> | boolean;
+    }): Promise<number>;
+    /** Forget a finished eval session's in-memory overrides. The rows stay readable. */
+    forgetEvalSession(sessionId: string): void;
     private warningsFor;
     /**
      * Web chat defaults and links: a new agent starts with web chat on, an enabled block without a link
@@ -182,10 +204,11 @@ export declare class AgentService {
         sourceRef?: string | null;
         threadKey?: string | null;
         title?: string | null;
+        eval?: EvalSessionOverrides;
     }): Promise<AgentSession>;
     /** Id of the session's pending approval, if a turn is waiting on a person. */
     pendingApprovalFor(_projectRoot: string, sessionId: string): string | null;
-    listSessionsForRef(projectRoot: string, agentId: string, source: string, sourceRef: string, limit: number): AgentSession[];
+    listSessionsForRef(projectRoot: string, agentId: string, source: string, sourceRef: string, limit: number, includeEval?: boolean): AgentSession[];
     private sessionInProject;
     getSession(projectRoot: string, sessionId: string): Promise<AgentSessionDetail>;
     listSessions(projectRoot: string, agentId: string, query: SessionQuery): Promise<{
@@ -274,7 +297,9 @@ export declare class AgentService {
         turnId?: string | null;
     }): Promise<AgentApproval>;
     decideApproval(projectRoot: string, sessionId: string, decision: "approve" | "deny", actor: ApprovalActor): Promise<AgentApproval>;
-    pendingApprovals(projectRoot: string): Promise<Array<AgentApproval & {
+    pendingApprovals(projectRoot: string, options?: {
+        includeEval?: boolean;
+    }): Promise<Array<AgentApproval & {
         agentId: string;
     }>>;
     receive(projectRoot: string, message: AgentInboundMessage): Promise<{

package/dist/agents/service.js

@@ -11,6 +11,7 @@ import { ApprovalBroker } from "./approvalBroker.js";
 import { deliveryLedger, scheduleLedger } from "./ledgers.js";
 import { TurnLifecycle } from "./turnLifecycle.js";
 import { AgentError } from "./errors.js";
+import { applyEvalOverrides, EVAL_SESSION_SOURCE, isEvalSession } from "./evalSession.js";
 import { agentMemoryIssues } from "../memory/agentIssues.js";
 import { diffLines } from "./lineDiff.js";
 import { exportTurnsToOtlp } from "./otlpExport.js";
@@ -38,6 +39,8 @@ export class AgentService {
     environment;
     now;
     tails = new Map();
+    /** Overrides of open eval sessions. Memory only: eval sessions do not survive a Host restart. */
+    evalOverrides = new Map();
     /** Wakes a turn paused at an ask-first tool when its approval is decided. */
     approvalBroker = new ApprovalBroker();
     /** Owns every legal change of a turn, its session and its approval. */
@@ -311,6 +314,64 @@ export class AgentService {
             throw new AgentError("agent.revision_not_found");
         return found.definition;
     }
+    /** The pinned definition, with an eval session's overrides applied. Real sessions get it unchanged. */
+    async effectiveDefinition(projectRoot, session) {
+        const base = await this.definitionAt(projectRoot, session.agentId, session.revision);
+        return isEvalSession(session) ? applyEvalOverrides(base, this.evalOverrides.get(session.id) ?? {}) : base;
+    }
+    /** The overrides an eval session was opened with (sub-agent sessions inherit them), or undefined for a real session. */
+    evalOverridesOf(sessionId) {
+        return this.evalOverrides.get(sessionId);
+    }
+    /** Every eval session below this one (sub-agents and theirs). */
+    listChildSessions(sessionId) {
+        return this.store.listChildSessions(sessionId);
+    }
+    /** Delete eval sessions and everything below them. Refuses a session that is not an eval session. */
+    async deleteEvalSessions(projectRoot, sessionIds) {
+        const roots = sessionIds.map((sessionId) => this.sessionInProject(projectRoot, sessionId));
+        const refused = roots.find((session) => !isEvalSession(session));
+        if (refused)
+            throw new AgentError("agent.invalid", `Session ${refused.id} is not an eval session`);
+        const all = new Set(roots.map((session) => session.id));
+        for (const root of roots)
+            for (const child of this.store.listChildSessions(root.id))
+                all.add(child.id);
+        const ids = [...all];
+        this.store.deleteSessions(ids);
+        for (const sessionId of ids)
+            this.evalOverrides.delete(sessionId);
+        return ids;
+    }
+    /**
+     * Boot cleanup of eval sessions: those of the given (recovered) eval runs, and older root sessions no eval result
+     * references. Returns how many sessions were deleted, descendants included.
+     */
+    async sweepEvalSessions(input) {
+        const roots = new Map();
+        for (const session of this.store.listEvalRootSessions({ sourceRefs: input.runIds }))
+            roots.set(session.id, session);
+        const cutoff = new Date(this.clockMs() - input.olderThanMs).toISOString();
+        for (const session of this.store.listEvalRootSessions({ createdBefore: cutoff })) {
+            if (!roots.has(session.id) && !(await input.isReferenced(session.id)))
+                roots.set(session.id, session);
+        }
+        const all = new Set();
+        for (const root of roots.values()) {
+            all.add(root.id);
+            for (const child of this.store.listChildSessions(root.id))
+                all.add(child.id);
+        }
+        const ids = [...all];
+        this.store.deleteSessions(ids);
+        for (const sessionId of ids)
+            this.evalOverrides.delete(sessionId);
+        return ids.length;
+    }
+    /** Forget a finished eval session's in-memory overrides. The rows stay readable. */
+    forgetEvalSession(sessionId) {
+        this.evalOverrides.delete(sessionId);
+    }
     /* ---------- validation ---------- */
     async warningsFor(projectRoot, definition) {
         const warnings = [];
@@ -636,10 +697,20 @@ export class AgentService {
     }
     /* ---------- sessions ---------- */
     async openSession(projectRoot, agentId, input) {
+        if (input.eval && input.source !== EVAL_SESSION_SOURCE)
+            throw new AgentError("agent.invalid", "Eval overrides need an eval session");
         return this.exclusive(projectRoot, async () => {
             await this.sync(projectRoot);
-            const { revision } = this.currentRevision(projectRoot, agentId);
-            if (!revision.definition.enabled)
+            const isEval = input.source === EVAL_SESSION_SOURCE;
+            const { revision: current } = this.currentRevision(projectRoot, agentId);
+            let revision = current;
+            if (input.eval?.revision !== undefined && input.eval.revision !== current.revision) {
+                const pinned = this.store.getRevision(agentProjectKey(projectRoot), agentId, input.eval.revision);
+                if (!pinned)
+                    throw new AgentError("agent.revision_not_found");
+                revision = pinned;
+            }
+            if (!isEval && !revision.definition.enabled)
                 throw new AgentError("agent.disabled");
             const now = this.iso();
             const session = {
@@ -656,6 +727,8 @@ export class AgentService {
                 updatedAt: now,
             };
             this.store.insertSession(agentProjectKey(projectRoot), session);
+            if (isEval)
+                this.evalOverrides.set(session.id, input.eval ?? {});
             return session;
         });
     }
@@ -663,8 +736,8 @@ export class AgentService {
     pendingApprovalFor(_projectRoot, sessionId) {
         return this.store.listApprovals(sessionId).find((approval) => approval.status === "pending")?.id ?? null;
     }
-    listSessionsForRef(projectRoot, agentId, source, sourceRef, limit) {
-        return this.store.listSessionsForRef(agentProjectKey(projectRoot), agentId, source, sourceRef, limit);
+    listSessionsForRef(projectRoot, agentId, source, sourceRef, limit, includeEval) {
+        return this.store.listSessionsForRef(agentProjectKey(projectRoot), agentId, source, sourceRef, limit, includeEval);
     }
     sessionInProject(projectRoot, sessionId) {
         const session = this.store.getSession(sessionId);
@@ -698,7 +771,7 @@ export class AgentService {
             const since = new Date(this.now().getTime() - RECENT_WINDOW_MS).toISOString();
             return {
                 ...this.store.listSessions(projectKey, agentId, query),
-                stats: this.store.sessionStats(projectKey, agentId, since),
+                stats: this.store.sessionStats(projectKey, agentId, since, query.includeEval),
             };
         });
     }
@@ -711,7 +784,7 @@ export class AgentService {
             const since = new Date(this.now().getTime() - RECENT_WINDOW_MS).toISOString();
             return {
                 ...this.store.listSessions(projectKey, agentId, query),
-                stats: this.store.sessionStats(projectKey, agentId, since),
+                stats: this.store.sessionStats(projectKey, agentId, since, query.includeEval),
             };
         });
     }
@@ -745,11 +818,11 @@ export class AgentService {
             await this.sync(projectRoot);
             const session = this.sessionInProject(projectRoot, sessionId);
             const { revision: live } = this.currentRevision(projectRoot, session.agentId);
-            if (!live.definition.enabled)
+            if (!live.definition.enabled && !isEvalSession(session))
                 throw new AgentError("agent.disabled");
             if (session.status === "running" || session.status === "waiting")
                 throw new AgentError("agent.session_busy");
-            const definition = await this.definitionAt(projectRoot, session.agentId, session.revision);
+            const definition = await this.effectiveDefinition(projectRoot, session);
             this.assertWithinLimits(definition, session);
             const now = this.iso();
             return this.store.transaction(() => {
@@ -930,9 +1003,9 @@ export class AgentService {
             this.expireApprovals();
             const session = this.sessionInProject(projectRoot, sessionId);
             const { revision: live } = this.currentRevision(projectRoot, session.agentId);
-            if (!live.definition.enabled)
+            if (!live.definition.enabled && !isEvalSession(session))
                 throw new AgentError("agent.disabled");
-            const definition = await this.definitionAt(projectRoot, session.agentId, session.revision);
+            const definition = await this.effectiveDefinition(projectRoot, session);
             if (!isToolGranted(definition, input.tool))
                 throw new AgentError("agent.tool_not_granted");
             if (!requiresAskFirst(definition, input.tool))
@@ -975,7 +1048,9 @@ export class AgentService {
         return this.exclusive(projectRoot, async () => {
             await this.sync(projectRoot);
             const session = this.sessionInProject(projectRoot, sessionId);
-            const definition = await this.definitionAt(projectRoot, session.agentId, session.revision);
+            if (isEvalSession(session))
+                throw new AgentError("agent.session_eval_readonly", "Eval sessions are read only");
+            const definition = await this.effectiveDefinition(projectRoot, session);
             if (!canApprove(definition, actor))
                 throw new AgentError("agent.approval_forbidden");
             const pending = this.store.getPendingApproval(sessionId);
@@ -997,9 +1072,9 @@ export class AgentService {
             return decided.approval;
         });
     }
-    async pendingApprovals(projectRoot) {
+    async pendingApprovals(projectRoot, options = {}) {
         this.expireApprovals();
-        return this.store.listPendingApprovals(agentProjectKey(projectRoot));
+        return this.store.listPendingApprovals(agentProjectKey(projectRoot), options.includeEval);
     }
     /* ---------- communications ---------- */
     async receive(projectRoot, message) {

package/dist/agents/sessions.d.ts

@@ -1,3 +1,5 @@
+import type { EvalGateRegistry } from "./evalGate.js";
+import { type EvalSessionOverrides } from "./evalSession.js";
 import type { OtlpSink } from "./otlpSink.js";
 import type { ConversationBackend } from "./runtime/types.js";
 import type { AgentService, AgentSessionDetail } from "./service.js";
@@ -11,6 +13,10 @@ export type SendTurnInput = {
     author?: string | null;
     /** Where a new session came from: console, cli, mcp, schedule, or a provider id. */
     source?: string;
+    /** Where a new session points to; defaults to the author. Eval sessions point at their eval run or parent session. */
+    sourceRef?: string | null;
+    /** Overrides for a new eval session (needs `source: "eval"`). */
+    eval?: EvalSessionOverrides;
     onDelta?: (text: string) => void;
     onSessionOpened?: (sessionId: string) => void;
     /** Retry this earlier, interrupted turn: its user message is reused, not asked again. */
@@ -20,6 +26,8 @@ export type ConversationOptions = {
     service: AgentService;
     backend: ConversationBackend;
     otlpSink?: OtlpSink | undefined;
+    /** Eval tool gates by session id. An eval session without one is denied every tool. */
+    evalGates?: EvalGateRegistry;
     logError?: (message: string) => void;
 };
 /** One turn: open (if needed) → record user message → model + tools → guardrails → record reply. */

package/dist/agents/sessions.js

@@ -1,6 +1,7 @@
 import { randomBytes } from "node:crypto";
 import { assertDelegateAllowed } from "./delegate.js";
 import { AgentError, isAgentError } from "./errors.js";
+import { EVAL_SESSION_SOURCE, isEvalSession } from "./evalSession.js";
 import { guardedStream } from "./guardrails.js";
 import { fitHistory } from "./historyFit.js";
 import { isToolGranted, requiresAskFirst } from "./policy.js";
@@ -51,7 +52,8 @@ export class AgentConversation {
             if (!sessionId) {
                 const opened = await service.openSession(input.projectRoot, input.agentId, {
                     source: input.source ?? "console",
-                    sourceRef: input.author ?? null,
+                    sourceRef: input.sourceRef !== undefined ? input.sourceRef : (input.author ?? null),
+                    ...(input.eval ? { eval: input.eval } : {}),
                 });
                 sessionId = opened.id;
                 input.onSessionOpened?.(sessionId);
@@ -98,9 +100,31 @@ export class AgentConversation {
                 service.recordSpan(input.projectRoot, span);
         };
         const toolStarts = new Map();
+        const childSessions = new Map();
+        const childSessionOf = (call) => (isEvalSession(active.session) ? (childSessions.get(call.id) ?? childSessions.get(call.tool)) : undefined);
         let stopReason;
         let gateChain = Promise.resolve();
         const gateOne = async (call) => {
+            if (active.session.source === EVAL_SESSION_SOURCE) {
+                const gate = this.options.evalGates?.get(sessionId);
+                if (!gate)
+                    return { allow: false, reason: "No eval policy is registered for this session." };
+                const decision = gate.decide(call, definition);
+                if (gate.recordOf(call.id)?.policyOutcome === "would_ask_first") {
+                    const at = new Date();
+                    record({
+                        turnId: turn.id,
+                        parentSpanId: rootId,
+                        name: `approval ${call.tool}`,
+                        kind: "gate",
+                        status: "error",
+                        startedAt: at,
+                        endedAt: at,
+                        attributes: { "stageflow.approval.outcome": "would_ask_first", "gen_ai.tool.name": call.tool },
+                    });
+                }
+                return decision;
+            }
             if (!isToolGranted(definition, call.tool)) {
                 return { allow: false, reason: `Tool ${call.tool} is not granted to this agent.` };
             }
@@ -162,6 +186,8 @@ export class AgentConversation {
                 toolStarts.set(call.id, new Date());
             },
             toolFinished: (call, result) => {
+                if (active.session.source === EVAL_SESSION_SOURCE)
+                    this.options.evalGates?.get(sessionId)?.observeFinished(call, result);
                 const isAgent = call.tool.startsWith("agent:");
                 record({
                     turnId: turn.id,
@@ -171,7 +197,11 @@ export class AgentConversation {
                     status: result.ok ? "ok" : "error",
                     startedAt: toolStarts.get(call.id) ?? new Date(),
                     endedAt: new Date(),
-                    attributes: { "gen_ai.tool.name": call.tool },
+                    attributes: {
+                        "gen_ai.tool.name": call.tool,
+                        ...(isEvalSession(active.session) ? { "gen_ai.tool.call.id": call.id } : {}),
+                        ...(childSessionOf(call) ? { "stageflow.sub_agent.session_id": childSessionOf(call) } : {}),
+                    },
                     input: toJson(call.input),
                     output: result.output,
                 });
@@ -193,16 +223,26 @@ export class AgentConversation {
                     },
                 });
             },
-            delegate: async (agentId, message) => {
+            delegate: async (agentId, message, callId) => {
                 assertDelegateAllowed(definition, agentId, this.depth);
                 const child = new AgentConversation(this.options, this.depth + 1, controller.signal);
+                const inEval = isEvalSession(active.session);
+                const inherited = { ...service.evalOverridesOf(active.session.id) };
+                delete inherited.revision;
                 try {
                     const detail = await child.send({
                         projectRoot: input.projectRoot,
                         agentId,
                         text: message,
-                        source: "agent",
+                        source: inEval ? EVAL_SESSION_SOURCE : "agent",
                         author: active.session.agentId,
+                        ...(inEval ? { sourceRef: active.session.id, eval: inherited } : {}),
+                        onSessionOpened: (childId) => {
+                            childSessions.set(callId ?? `agent:${agentId}`, childId);
+                            const parentGate = inEval ? this.options.evalGates?.get(sessionId) : undefined;
+                            if (parentGate)
+                                this.options.evalGates.set(childId, parentGate);
+                        },
                     });
                     return [...detail.messages].reverse().find((m) => m.role === "assistant")?.text ?? "(no reply)";
                 }
@@ -230,7 +270,7 @@ export class AgentConversation {
             const code = isAgentError(error) ? error.code : "agent.backend_failed";
             record({ turnId: turn.id, name: `invoke_agent ${definition.id}`, kind: "agent", status: "error", startedAt, endedAt: new Date(), attributes: { "error.type": code } });
             await service.completeTurn(input.projectRoot, turn.id, { status: "failed", errorCode: code });
-            this.exportLive(input.projectRoot, definition, turn.id);
+            this.exportLive(input.projectRoot, active.session, definition, turn.id);
             throw error;
         }
         const cancelled = controller.signal.aborted;
@@ -264,10 +304,10 @@ export class AgentConversation {
                 errorCode: "agent.guardrail_blocked",
                 ...(usage ? { usage, costUsd: usage.costUsd } : {}),
             });
-            this.exportLive(input.projectRoot, definition, turn.id);
+            this.exportLive(input.projectRoot, active.session, definition, turn.id);
             throw new AgentError("agent.guardrail_blocked", `Reply blocked by guardrail ${guard.pattern}`);
         }
-        const evalScores = await this.score(input.projectRoot, definition, turn.id, rootId, result, controller.signal, cancelled || stopReason !== undefined);
+        const evalScores = await this.score(input.projectRoot, definition, turn.id, rootId, result, controller.signal, cancelled || stopReason !== undefined || isEvalSession(active.session));
         const failed = stopReason !== undefined;
         endRoot(cancelled ? "cancelled" : failed ? "error" : "ok");
         const detail = await service.completeTurn(input.projectRoot, turn.id, {
@@ -277,7 +317,7 @@ export class AgentConversation {
             ...(usage ? { usage, costUsd: usage.costUsd } : {}),
             ...(evalScores.length > 0 ? { evalScores } : {}),
         });
-        this.exportLive(input.projectRoot, definition, turn.id);
+        this.exportLive(input.projectRoot, active.session, definition, turn.id);
         return detail;
     }
     async score(projectRoot, definition, turnId, rootId, result, signal, skip) {
@@ -305,9 +345,9 @@ export class AgentConversation {
         }
     }
     /** Best-effort push to the configured OTLP collector; a collector outage never fails a turn. */
-    exportLive(projectRoot, definition, turnId) {
+    exportLive(projectRoot, session, definition, turnId) {
         const sink = this.options.otlpSink;
-        if (!sink || !definition.observability.otlp || definition.observability.capture === "off")
+        if (!sink || isEvalSession(session) || !definition.observability.otlp || definition.observability.capture === "off")
             return;
         void (async () => {
             try {

package/dist/agents/store.d.ts

@@ -149,6 +149,8 @@ export type SessionQuery = {
     limit: number;
     /** Opaque cursor from a previous page. */
     cursor?: string;
+    /** Eval sessions are hidden unless an internal reader (eval read models, tests) asks for them. */
+    includeEval?: boolean;
 };
 export type SessionStats = {
     total: number;
@@ -204,9 +206,18 @@ export declare class AgentStore {
         nextCursor: string | null;
     };
     /** One visitor's sessions on one agent, newest first (public web chat). */
-    listSessionsForRef(projectKey: string, agentId: string, source: string, sourceRef: string, limit: number): AgentSession[];
+    listSessionsForRef(projectKey: string, agentId: string, source: string, sourceRef: string, limit: number, includeEval?: boolean): AgentSession[];
     /** Stats for one agent, or for every agent when `agentId` is omitted. */
-    sessionStats(projectKey: string, agentId: string | undefined, since: string): SessionStats;
+    sessionStats(projectKey: string, agentId: string | undefined, since: string, includeEval?: boolean): SessionStats;
+    /** Eval sessions below a session or an eval run: those whose `source_ref` is it, and theirs, recursively. */
+    listChildSessions(parentId: string): AgentSession[];
+    /** Root eval sessions (those pointing at an eval run, not at a parent session), optionally of given runs or older than a time. */
+    listEvalRootSessions(filter?: {
+        sourceRefs?: string[];
+        createdBefore?: string;
+    }): AgentSession[];
+    /** Delete eval sessions; messages, turns, spans, scores and approvals go with them. Never touches other sessions. */
+    deleteSessions(sessionIds: string[]): void;
     /** Ids of every session an agent has had, for releasing what a runtime holds for them. */
     sessionIdsOf(projectKey: string, agentId: string): string[];
     appendMessage(message: Omit<AgentMessage, "seq">): AgentMessage;
@@ -250,7 +261,7 @@ export declare class AgentStore {
     getPendingApproval(sessionId: string): AgentApproval | undefined;
     findApprovalByPayload(sessionId: string, payloadHash: string): AgentApproval | undefined;
     listApprovals(sessionId: string): AgentApproval[];
-    listPendingApprovals(projectKey: string): Array<AgentApproval & {
+    listPendingApprovals(projectKey: string, includeEval?: boolean): Array<AgentApproval & {
         agentId: string;
     }>;
     decideApproval(approvalId: string, patch: {

package/dist/agents/store.js

@@ -9,6 +9,7 @@ export const SESSION_STATUSES = ["open", "running", "waiting", "held", "closed"]
 export function isSessionStatus(value) {
     return SESSION_STATUSES.includes(value);
 }
+const HIDE_EVAL = "source <> 'eval'";
 export function agentProjectKey(projectRoot) {
     return createHash("sha256").update(path.resolve(projectRoot)).digest("hex").slice(0, 16);
 }
@@ -274,6 +275,8 @@ export class AgentStore {
     /** One agent's sessions, or every agent's when `agentId` is omitted. */
     listSessions(projectKey, agentId, query) {
         const where = ["project_key = @projectKey"];
+        if (!query.includeEval)
+            where.push(HIDE_EVAL);
         const params = { projectKey, limit: query.limit + 1 };
         if (agentId !== undefined) {
             where.push("agent_id = @agentId");
@@ -327,17 +330,17 @@ export class AgentStore {
         };
     }
     /** One visitor's sessions on one agent, newest first (public web chat). */
-    listSessionsForRef(projectKey, agentId, source, sourceRef, limit) {
+    listSessionsForRef(projectKey, agentId, source, sourceRef, limit, includeEval = false) {
         return this.db
             .prepare(`SELECT * FROM agent_sessions
-            WHERE project_key = ? AND agent_id = ? AND source = ? AND source_ref = ?
+            WHERE project_key = ? AND agent_id = ? AND source = ? AND source_ref = ?${includeEval ? "" : ` AND ${HIDE_EVAL}`}
             ORDER BY updated_at DESC, session_id DESC LIMIT ?`)
             .all(projectKey, agentId, source, sourceRef, limit).map(toSession);
     }
     /** Stats for one agent, or for every agent when `agentId` is omitted. */
-    sessionStats(projectKey, agentId, since) {
-        const agentFilter = agentId === undefined ? "" : " AND agent_id = @agentId";
-        const sessionAgentFilter = agentId === undefined ? "" : " AND s.agent_id = @agentId";
+    sessionStats(projectKey, agentId, since, includeEval = false) {
+        const agentFilter = (agentId === undefined ? "" : " AND agent_id = @agentId") + (includeEval ? "" : ` AND ${HIDE_EVAL}`);
+        const sessionAgentFilter = (agentId === undefined ? "" : " AND s.agent_id = @agentId") + (includeEval ? "" : ` AND s.${HIDE_EVAL}`);
         const bind = (extra) => (agentId === undefined ? extra : { ...extra, agentId });
         const row = this.db
             .prepare(`SELECT COUNT(*) AS total,
@@ -367,6 +370,41 @@ export class AgentStore {
             costTotal: numOrNull(turns.cost_all),
         };
     }
+    /** Eval sessions below a session or an eval run: those whose `source_ref` is it, and theirs, recursively. */
+    listChildSessions(parentId) {
+        return this.db
+            .prepare(`WITH RECURSIVE below(session_id) AS (
+             SELECT session_id FROM agent_sessions WHERE source = 'eval' AND source_ref = ?
+             UNION
+             SELECT s.session_id FROM agent_sessions s JOIN below b ON s.source_ref = b.session_id WHERE s.source = 'eval'
+           )
+           SELECT s.* FROM agent_sessions s JOIN below b ON b.session_id = s.session_id ORDER BY s.created_at, s.session_id`)
+            .all(parentId).map(toSession);
+    }
+    /** Root eval sessions (those pointing at an eval run, not at a parent session), optionally of given runs or older than a time. */
+    listEvalRootSessions(filter = {}) {
+        const clauses = [`source = 'eval'`, `(source_ref IS NULL OR source_ref NOT IN (SELECT session_id FROM agent_sessions))`];
+        const params = [];
+        if (filter.sourceRefs) {
+            if (filter.sourceRefs.length === 0)
+                return [];
+            clauses.push(`source_ref IN (${filter.sourceRefs.map(() => "?").join(", ")})`);
+            params.push(...filter.sourceRefs);
+        }
+        if (filter.createdBefore) {
+            clauses.push(`created_at < ?`);
+            params.push(filter.createdBefore);
+        }
+        return this.db.prepare(`SELECT * FROM agent_sessions WHERE ${clauses.join(" AND ")} ORDER BY created_at, session_id`).all(...params).map(toSession);
+    }
+    /** Delete eval sessions; messages, turns, spans, scores and approvals go with them. Never touches other sessions. */
+    deleteSessions(sessionIds) {
+        this.transaction(() => {
+            const remove = this.db.prepare(`DELETE FROM agent_sessions WHERE session_id = ? AND source = 'eval'`);
+            for (const sessionId of sessionIds)
+                remove.run(sessionId);
+        });
+    }
     /** Ids of every session an agent has had, for releasing what a runtime holds for them. */
     sessionIdsOf(projectKey, agentId) {
         return this.db.prepare(`SELECT session_id FROM agent_sessions WHERE project_key = ? AND agent_id = ?`).all(projectKey, agentId).map((row) => str(row.session_id));
@@ -562,10 +600,10 @@ export class AgentStore {
             .prepare(`SELECT * FROM agent_approvals WHERE session_id = ? ORDER BY created_at`)
             .all(sessionId).map(toApproval);
     }
-    listPendingApprovals(projectKey) {
+    listPendingApprovals(projectKey, includeEval = false) {
         return this.db
             .prepare(`SELECT a.*, s.agent_id FROM agent_approvals a JOIN agent_sessions s ON s.session_id = a.session_id
-            WHERE s.project_key = ? AND a.status = 'pending' ORDER BY a.created_at`)
+            WHERE s.project_key = ? AND a.status = 'pending'${includeEval ? "" : ` AND s.${HIDE_EVAL}`} ORDER BY a.created_at`)
             .all(projectKey).map((row) => ({ ...toApproval(row), agentId: str(row.agent_id) }));
     }
     decideApproval(approvalId, patch) {