stageflow 0.40.0 → 0.44.0

package/dist/agents/sessions.js

@@ -1,9 +1,12 @@
 import { randomBytes } from "node:crypto";
 import { assertDelegateAllowed } from "./delegate.js";
 import { AgentError, isAgentError } from "./errors.js";
+import { EVAL_SESSION_SOURCE, isEvalSession } from "./evalSession.js";
 import { guardedStream } from "./guardrails.js";
 import { fitHistory } from "./historyFit.js";
 import { isToolGranted, requiresAskFirst } from "./policy.js";
+import { scrubSecrets } from "../compaction/scrub.js";
+import { WORKING_STATE_TOOL_NAME } from "../compaction/types.js";
 import { prepareSpan } from "./trace.js";
 const GROUNDEDNESS_PROMPT = (reply, evidence) => [
     "Rate how well the ANSWER is supported by the TOOL OUTPUTS, from 0 (unsupported or contradicted) to 1 (fully supported).",
@@ -51,7 +54,8 @@ export class AgentConversation {
             if (!sessionId) {
                 const opened = await service.openSession(input.projectRoot, input.agentId, {
                     source: input.source ?? "console",
-                    sourceRef: input.author ?? null,
+                    sourceRef: input.sourceRef !== undefined ? input.sourceRef : (input.author ?? null),
+                    ...(input.eval ? { eval: input.eval } : {}),
                 });
                 sessionId = opened.id;
                 input.onSessionOpened?.(sessionId);
@@ -98,9 +102,31 @@ export class AgentConversation {
                 service.recordSpan(input.projectRoot, span);
         };
         const toolStarts = new Map();
+        const childSessions = new Map();
+        const childSessionOf = (call) => (isEvalSession(active.session) ? (childSessions.get(call.id) ?? childSessions.get(call.tool)) : undefined);
         let stopReason;
         let gateChain = Promise.resolve();
         const gateOne = async (call) => {
+            if (active.session.source === EVAL_SESSION_SOURCE) {
+                const gate = this.options.evalGates?.get(sessionId);
+                if (!gate)
+                    return { allow: false, reason: "No eval policy is registered for this session." };
+                const decision = gate.decide(call, definition);
+                if (gate.recordOf(call.id)?.policyOutcome === "would_ask_first") {
+                    const at = new Date();
+                    record({
+                        turnId: turn.id,
+                        parentSpanId: rootId,
+                        name: `approval ${call.tool}`,
+                        kind: "gate",
+                        status: "error",
+                        startedAt: at,
+                        endedAt: at,
+                        attributes: { "stageflow.approval.outcome": "would_ask_first", "gen_ai.tool.name": call.tool },
+                    });
+                }
+                return decision;
+            }
             if (!isToolGranted(definition, call.tool)) {
                 return { allow: false, reason: `Tool ${call.tool} is not granted to this agent.` };
             }
@@ -162,6 +188,8 @@ export class AgentConversation {
                 toolStarts.set(call.id, new Date());
             },
             toolFinished: (call, result) => {
+                if (active.session.source === EVAL_SESSION_SOURCE)
+                    this.options.evalGates?.get(sessionId)?.observeFinished(call, result);
                 const isAgent = call.tool.startsWith("agent:");
                 record({
                     turnId: turn.id,
@@ -171,8 +199,12 @@ export class AgentConversation {
                     status: result.ok ? "ok" : "error",
                     startedAt: toolStarts.get(call.id) ?? new Date(),
                     endedAt: new Date(),
-                    attributes: { "gen_ai.tool.name": call.tool },
-                    input: toJson(call.input),
+                    attributes: {
+                        "gen_ai.tool.name": call.tool,
+                        ...(isEvalSession(active.session) ? { "gen_ai.tool.call.id": call.id } : {}),
+                        ...(childSessionOf(call) ? { "stageflow.sub_agent.session_id": childSessionOf(call) } : {}),
+                    },
+                    input: call.tool === WORKING_STATE_TOOL_NAME ? scrubSecrets(toJson(call.input)) : toJson(call.input),
                     output: result.output,
                 });
             },
@@ -193,16 +225,26 @@ export class AgentConversation {
                     },
                 });
             },
-            delegate: async (agentId, message) => {
+            delegate: async (agentId, message, callId) => {
                 assertDelegateAllowed(definition, agentId, this.depth);
                 const child = new AgentConversation(this.options, this.depth + 1, controller.signal);
+                const inEval = isEvalSession(active.session);
+                const inherited = { ...service.evalOverridesOf(active.session.id) };
+                delete inherited.revision;
                 try {
                     const detail = await child.send({
                         projectRoot: input.projectRoot,
                         agentId,
                         text: message,
-                        source: "agent",
+                        source: inEval ? EVAL_SESSION_SOURCE : "agent",
                         author: active.session.agentId,
+                        ...(inEval ? { sourceRef: active.session.id, eval: inherited } : {}),
+                        onSessionOpened: (childId) => {
+                            childSessions.set(callId ?? `agent:${agentId}`, childId);
+                            const parentGate = inEval ? this.options.evalGates?.get(sessionId) : undefined;
+                            if (parentGate)
+                                this.options.evalGates.set(childId, parentGate);
+                        },
                     });
                     return [...detail.messages].reverse().find((m) => m.role === "assistant")?.text ?? "(no reply)";
                 }
@@ -230,7 +272,7 @@ export class AgentConversation {
             const code = isAgentError(error) ? error.code : "agent.backend_failed";
             record({ turnId: turn.id, name: `invoke_agent ${definition.id}`, kind: "agent", status: "error", startedAt, endedAt: new Date(), attributes: { "error.type": code } });
             await service.completeTurn(input.projectRoot, turn.id, { status: "failed", errorCode: code });
-            this.exportLive(input.projectRoot, definition, turn.id);
+            this.exportLive(input.projectRoot, active.session, definition, turn.id);
             throw error;
         }
         const cancelled = controller.signal.aborted;
@@ -264,10 +306,10 @@ export class AgentConversation {
                 errorCode: "agent.guardrail_blocked",
                 ...(usage ? { usage, costUsd: usage.costUsd } : {}),
             });
-            this.exportLive(input.projectRoot, definition, turn.id);
+            this.exportLive(input.projectRoot, active.session, definition, turn.id);
             throw new AgentError("agent.guardrail_blocked", `Reply blocked by guardrail ${guard.pattern}`);
         }
-        const evalScores = await this.score(input.projectRoot, definition, turn.id, rootId, result, controller.signal, cancelled || stopReason !== undefined);
+        const evalScores = await this.score(input.projectRoot, definition, turn.id, rootId, result, controller.signal, cancelled || stopReason !== undefined || isEvalSession(active.session));
         const failed = stopReason !== undefined;
         endRoot(cancelled ? "cancelled" : failed ? "error" : "ok");
         const detail = await service.completeTurn(input.projectRoot, turn.id, {
@@ -277,7 +319,7 @@ export class AgentConversation {
             ...(usage ? { usage, costUsd: usage.costUsd } : {}),
             ...(evalScores.length > 0 ? { evalScores } : {}),
         });
-        this.exportLive(input.projectRoot, definition, turn.id);
+        this.exportLive(input.projectRoot, active.session, definition, turn.id);
         return detail;
     }
     async score(projectRoot, definition, turnId, rootId, result, signal, skip) {
@@ -305,9 +347,9 @@ export class AgentConversation {
         }
     }
     /** Best-effort push to the configured OTLP collector; a collector outage never fails a turn. */
-    exportLive(projectRoot, definition, turnId) {
+    exportLive(projectRoot, session, definition, turnId) {
         const sink = this.options.otlpSink;
-        if (!sink || !definition.observability.otlp || definition.observability.capture === "off")
+        if (!sink || isEvalSession(session) || !definition.observability.otlp || definition.observability.capture === "off")
             return;
         void (async () => {
             try {

package/dist/agents/store.d.ts

@@ -149,6 +149,8 @@ export type SessionQuery = {
     limit: number;
     /** Opaque cursor from a previous page. */
     cursor?: string;
+    /** Eval sessions are hidden unless an internal reader (eval read models, tests) asks for them. */
+    includeEval?: boolean;
 };
 export type SessionStats = {
     total: number;
@@ -204,9 +206,18 @@ export declare class AgentStore {
         nextCursor: string | null;
     };
     /** One visitor's sessions on one agent, newest first (public web chat). */
-    listSessionsForRef(projectKey: string, agentId: string, source: string, sourceRef: string, limit: number): AgentSession[];
+    listSessionsForRef(projectKey: string, agentId: string, source: string, sourceRef: string, limit: number, includeEval?: boolean): AgentSession[];
     /** Stats for one agent, or for every agent when `agentId` is omitted. */
-    sessionStats(projectKey: string, agentId: string | undefined, since: string): SessionStats;
+    sessionStats(projectKey: string, agentId: string | undefined, since: string, includeEval?: boolean): SessionStats;
+    /** Eval sessions below a session or an eval run: those whose `source_ref` is it, and theirs, recursively. */
+    listChildSessions(parentId: string): AgentSession[];
+    /** Root eval sessions (those pointing at an eval run, not at a parent session), optionally of given runs or older than a time. */
+    listEvalRootSessions(filter?: {
+        sourceRefs?: string[];
+        createdBefore?: string;
+    }): AgentSession[];
+    /** Delete eval sessions; messages, turns, spans, scores and approvals go with them. Never touches other sessions. */
+    deleteSessions(sessionIds: string[]): void;
     /** Ids of every session an agent has had, for releasing what a runtime holds for them. */
     sessionIdsOf(projectKey: string, agentId: string): string[];
     appendMessage(message: Omit<AgentMessage, "seq">): AgentMessage;
@@ -250,7 +261,7 @@ export declare class AgentStore {
     getPendingApproval(sessionId: string): AgentApproval | undefined;
     findApprovalByPayload(sessionId: string, payloadHash: string): AgentApproval | undefined;
     listApprovals(sessionId: string): AgentApproval[];
-    listPendingApprovals(projectKey: string): Array<AgentApproval & {
+    listPendingApprovals(projectKey: string, includeEval?: boolean): Array<AgentApproval & {
         agentId: string;
     }>;
     decideApproval(approvalId: string, patch: {

package/dist/agents/store.js

@@ -9,6 +9,7 @@ export const SESSION_STATUSES = ["open", "running", "waiting", "held", "closed"]
 export function isSessionStatus(value) {
     return SESSION_STATUSES.includes(value);
 }
+const HIDE_EVAL = "source <> 'eval'";
 export function agentProjectKey(projectRoot) {
     return createHash("sha256").update(path.resolve(projectRoot)).digest("hex").slice(0, 16);
 }
@@ -274,6 +275,8 @@ export class AgentStore {
     /** One agent's sessions, or every agent's when `agentId` is omitted. */
     listSessions(projectKey, agentId, query) {
         const where = ["project_key = @projectKey"];
+        if (!query.includeEval)
+            where.push(HIDE_EVAL);
         const params = { projectKey, limit: query.limit + 1 };
         if (agentId !== undefined) {
             where.push("agent_id = @agentId");
@@ -327,17 +330,17 @@ export class AgentStore {
         };
     }
     /** One visitor's sessions on one agent, newest first (public web chat). */
-    listSessionsForRef(projectKey, agentId, source, sourceRef, limit) {
+    listSessionsForRef(projectKey, agentId, source, sourceRef, limit, includeEval = false) {
         return this.db
             .prepare(`SELECT * FROM agent_sessions
-            WHERE project_key = ? AND agent_id = ? AND source = ? AND source_ref = ?
+            WHERE project_key = ? AND agent_id = ? AND source = ? AND source_ref = ?${includeEval ? "" : ` AND ${HIDE_EVAL}`}
             ORDER BY updated_at DESC, session_id DESC LIMIT ?`)
             .all(projectKey, agentId, source, sourceRef, limit).map(toSession);
     }
     /** Stats for one agent, or for every agent when `agentId` is omitted. */
-    sessionStats(projectKey, agentId, since) {
-        const agentFilter = agentId === undefined ? "" : " AND agent_id = @agentId";
-        const sessionAgentFilter = agentId === undefined ? "" : " AND s.agent_id = @agentId";
+    sessionStats(projectKey, agentId, since, includeEval = false) {
+        const agentFilter = (agentId === undefined ? "" : " AND agent_id = @agentId") + (includeEval ? "" : ` AND ${HIDE_EVAL}`);
+        const sessionAgentFilter = (agentId === undefined ? "" : " AND s.agent_id = @agentId") + (includeEval ? "" : ` AND s.${HIDE_EVAL}`);
         const bind = (extra) => (agentId === undefined ? extra : { ...extra, agentId });
         const row = this.db
             .prepare(`SELECT COUNT(*) AS total,
@@ -367,6 +370,41 @@ export class AgentStore {
             costTotal: numOrNull(turns.cost_all),
         };
     }
+    /** Eval sessions below a session or an eval run: those whose `source_ref` is it, and theirs, recursively. */
+    listChildSessions(parentId) {
+        return this.db
+            .prepare(`WITH RECURSIVE below(session_id) AS (
+             SELECT session_id FROM agent_sessions WHERE source = 'eval' AND source_ref = ?
+             UNION
+             SELECT s.session_id FROM agent_sessions s JOIN below b ON s.source_ref = b.session_id WHERE s.source = 'eval'
+           )
+           SELECT s.* FROM agent_sessions s JOIN below b ON b.session_id = s.session_id ORDER BY s.created_at, s.session_id`)
+            .all(parentId).map(toSession);
+    }
+    /** Root eval sessions (those pointing at an eval run, not at a parent session), optionally of given runs or older than a time. */
+    listEvalRootSessions(filter = {}) {
+        const clauses = [`source = 'eval'`, `(source_ref IS NULL OR source_ref NOT IN (SELECT session_id FROM agent_sessions))`];
+        const params = [];
+        if (filter.sourceRefs) {
+            if (filter.sourceRefs.length === 0)
+                return [];
+            clauses.push(`source_ref IN (${filter.sourceRefs.map(() => "?").join(", ")})`);
+            params.push(...filter.sourceRefs);
+        }
+        if (filter.createdBefore) {
+            clauses.push(`created_at < ?`);
+            params.push(filter.createdBefore);
+        }
+        return this.db.prepare(`SELECT * FROM agent_sessions WHERE ${clauses.join(" AND ")} ORDER BY created_at, session_id`).all(...params).map(toSession);
+    }
+    /** Delete eval sessions; messages, turns, spans, scores and approvals go with them. Never touches other sessions. */
+    deleteSessions(sessionIds) {
+        this.transaction(() => {
+            const remove = this.db.prepare(`DELETE FROM agent_sessions WHERE session_id = ? AND source = 'eval'`);
+            for (const sessionId of sessionIds)
+                remove.run(sessionId);
+        });
+    }
     /** Ids of every session an agent has had, for releasing what a runtime holds for them. */
     sessionIdsOf(projectKey, agentId) {
         return this.db.prepare(`SELECT session_id FROM agent_sessions WHERE project_key = ? AND agent_id = ?`).all(projectKey, agentId).map((row) => str(row.session_id));
@@ -562,10 +600,10 @@ export class AgentStore {
             .prepare(`SELECT * FROM agent_approvals WHERE session_id = ? ORDER BY created_at`)
             .all(sessionId).map(toApproval);
     }
-    listPendingApprovals(projectKey) {
+    listPendingApprovals(projectKey, includeEval = false) {
         return this.db
             .prepare(`SELECT a.*, s.agent_id FROM agent_approvals a JOIN agent_sessions s ON s.session_id = a.session_id
-            WHERE s.project_key = ? AND a.status = 'pending' ORDER BY a.created_at`)
+            WHERE s.project_key = ? AND a.status = 'pending'${includeEval ? "" : ` AND s.${HIDE_EVAL}`} ORDER BY a.created_at`)
             .all(projectKey).map((row) => ({ ...toApproval(row), agentId: str(row.agent_id) }));
     }
     decideApproval(approvalId, patch) {

package/dist/agents/toolRisk.d.ts

@@ -0,0 +1,21 @@
+import { type AgentDefinition } from "../config/loadAgent.js";
+export type ToolClass = "read" | "write" | "shell" | "destructive" | "external" | "mcp_unknown" | "sub_agent" | "browser" | "operator_only";
+export type ToolClassification = {
+    class: ToolClass;
+    readOnly: boolean;
+};
+type Grants = Pick<AgentDefinition, "tools" | "sub_agents">;
+/**
+ * What an eval may assume about a tool. `readOnly` is true only for what is provably read-only (builtin read tools,
+ * Stageflow tools with risk `read`) or an MCP tool the eval author declares in `read_only_tools` (a declaration never changes a builtin or Stageflow tool). Sub-agents and
+ * operator-only tools are never read-only.
+ */
+export declare function classifyTool(_definition: Grants | undefined, toolId: string, readOnlyExtra?: readonly string[]): ToolClassification;
+export type ToolRiskEntry = {
+    tool: string;
+    class: ToolClass;
+    readOnly: boolean;
+};
+/** Every tool the agent is granted with how the `read_only` policy would treat it, for the eval picker. MCP wildcards stay as `<server>.*`. */
+export declare function toolRiskSummary(definition: Grants, readOnlyExtra?: readonly string[]): ToolRiskEntry[];
+export {};

package/dist/agents/toolRisk.js

@@ -0,0 +1,50 @@
+import { grantedToolIds } from "../config/loadAgent.js";
+import { isOperatorOnlyTool, STAGEFLOW_GRANT_PREFIX, TOOL_META } from "../toolkit/meta.js";
+import { BUILTIN_TOOL_GROUP_OF } from "./runtime/types.js";
+function listed(toolId, readOnlyExtra) {
+    if (readOnlyExtra.includes(toolId))
+        return true;
+    const dot = toolId.indexOf(".");
+    return dot > 0 && readOnlyExtra.includes(`${toolId.slice(0, dot)}.*`);
+}
+function baseClass(toolId) {
+    if (toolId === "browser")
+        return "browser";
+    if (toolId.startsWith("agent:"))
+        return "sub_agent";
+    if (toolId.startsWith(STAGEFLOW_GRANT_PREFIX)) {
+        const name = toolId.slice(STAGEFLOW_GRANT_PREFIX.length);
+        const meta = TOOL_META[name];
+        if (!meta)
+            return "mcp_unknown";
+        if (isOperatorOnlyTool(name))
+            return "operator_only";
+        return meta.risk;
+    }
+    const builtin = BUILTIN_TOOL_GROUP_OF[toolId];
+    if (builtin)
+        return builtin;
+    return "mcp_unknown";
+}
+/**
+ * What an eval may assume about a tool. `readOnly` is true only for what is provably read-only (builtin read tools,
+ * Stageflow tools with risk `read`) or an MCP tool the eval author declares in `read_only_tools` (a declaration never changes a builtin or Stageflow tool). Sub-agents and
+ * operator-only tools are never read-only.
+ */
+export function classifyTool(_definition, toolId, readOnlyExtra = []) {
+    let cls = baseClass(toolId);
+    if (cls === "operator_only" || cls === "sub_agent")
+        return { class: cls, readOnly: false };
+    const declared = listed(toolId, readOnlyExtra);
+    if (declared && cls === "mcp_unknown")
+        cls = "read";
+    return { class: cls, readOnly: cls === "read" };
+}
+/** Every tool the agent is granted with how the `read_only` policy would treat it, for the eval picker. MCP wildcards stay as `<server>.*`. */
+export function toolRiskSummary(definition, readOnlyExtra = []) {
+    const ids = new Set(grantedToolIds(definition));
+    for (const [server, tools] of Object.entries(definition.tools.mcp))
+        if (tools.includes("*"))
+            ids.add(`${server}.*`);
+    return [...ids].sort().map((tool) => ({ tool, ...classifyTool(definition, tool, readOnlyExtra) }));
+}

package/dist/cli/doctorCommand.js

@@ -9,6 +9,8 @@ import { warnMissingCaPaths } from "../preflight/tls.js";
 import { runPipelinePreflight, } from "../preflight/pipelinePreflight.js";
 import { loadPipelineOutcome } from "../config/loadPipeline.js";
 import { MCP_CATALOG_FILENAME } from "../config/resolveStageMcpServers.js";
+import { MIN_NODE } from "../evals/engine/promptfoo/pinned.js";
+import { engineStatus } from "../evals/engine/promptfoo/status.js";
 import { globalStageflowHome } from "../project/globalHome.js";
 import { createRunStore } from "../runstore/createStore.js";
 import { readFilesystemSize, resolveMinFreeDiskFloor, } from "../runstore/diskUsage.js";
@@ -142,6 +144,29 @@ function nodeVersionCheck() {
         message: `Node ${process.version}; engines floor may differ from Pi/better-sqlite3 (not bumping engines.node)`,
     };
 }
+function evalEngineCheck(home) {
+    const status = engineStatus({ home });
+    if (!status.installed) {
+        return {
+            id: "eval_engine",
+            status: "skipped",
+            message: `Eval engine not installed (optional; promptfoo ${status.pinned}). Run \`${status.installCommand}\``,
+        };
+    }
+    if (!status.nodeOk) {
+        return {
+            id: "eval_engine",
+            status: "warn",
+            code: "EVAL_NODE_TOO_OLD",
+            message: `Eval engine promptfoo ${status.version} installed, but Node ${status.node} is below the ${MIN_NODE} evals need`,
+        };
+    }
+    return {
+        id: "eval_engine",
+        status: "pass",
+        message: `Eval engine promptfoo ${status.version}`,
+    };
+}
 function discoverProviderEnvIds(env) {
     const byLower = new Set();
     for (const key of Object.keys(env)) {
@@ -355,6 +380,7 @@ export async function runDoctorChecks(options) {
     checks.push(tlsCheck(env));
     checks.push(await freeDiskCheck(home, env));
     checks.push(...(await mcpCommandsCheck(cwd)));
+    checks.push(evalEngineCheck(home));
     let preflight;
     if (options?.pipeline !== undefined) {
         const outcome = await loadPipelineOutcome(options.pipeline, { cwd });

package/dist/cli/evalCommand.d.ts

@@ -0,0 +1,20 @@
+import { type RunNpm } from "../evals/engine/promptfoo/install.js";
+import { type EnsureGlobalServiceResult } from "../server/ensureGlobalService.js";
+export declare const EVAL_USAGE = "Usage:\n  sf eval list [--json]\n  sf eval show <id|path> [--json]\n  sf eval validate [<id|path>] [--json] [--strict]\n  sf eval run <id|path> [--repeats <n>] [--judge <model>] [--model <id>] [--user-model <model>] [--allow-real-tools] [--max-cost <usd>] [--case <id> ...] [--tag <tag> ...] [--label <text>] [--no-wait] [--json]\n  sf eval runs [<evalId>] [--json]\n  sf eval runs show <evalRunId> [--json]\n  sf eval runs cancel <evalRunId>\n  sf eval runs delete <evalRunId>\n  sf eval sort <evalRunId> [--classifier auto|jev|judge] [--limit <n>] [--resort] [--json]\n  sf eval suggestions <evalRunId> [--status pending|accepted|dismissed] [--json]\n  sf eval compare <evalRunIdA> <evalRunIdB> [--json]\n  sf eval datasets [--json]\n  sf eval engine status [--json]\n  sf eval engine install\n\n  Agent evals: --model overrides the agent's model, --user-model the model that plays the user. An eval whose\n  conversation.tools is real also needs --allow-real-tools.\n\n  Every subcommand also accepts --project-root <id-or-path>.";
+export type EvalCommandIo = {
+    log: (line: string) => void;
+    error: (line: string) => void;
+};
+export type EvalCommandOptions = {
+    cwd?: string;
+    projectRoot?: string;
+    io?: Partial<EvalCommandIo>;
+    hostBaseUrl?: string;
+    ensureService?: () => Promise<EnsureGlobalServiceResult>;
+    home?: string;
+    runNpm?: RunNpm;
+    pollIntervalMs?: number;
+    stderrIsTTY?: boolean;
+    onInterrupt?: (handler: () => void) => () => void;
+};
+export declare function runEvalCommand(args: string[], options?: EvalCommandOptions): Promise<number>;