stageflow 0.39.0 → 0.42.0

package/dist/agents/toolRisk.d.ts

@@ -0,0 +1,21 @@
+import { type AgentDefinition } from "../config/loadAgent.js";
+export type ToolClass = "read" | "write" | "shell" | "destructive" | "external" | "mcp_unknown" | "sub_agent" | "browser" | "operator_only";
+export type ToolClassification = {
+    class: ToolClass;
+    readOnly: boolean;
+};
+type Grants = Pick<AgentDefinition, "tools" | "sub_agents">;
+/**
+ * What an eval may assume about a tool. `readOnly` is true only for what is provably read-only (builtin read tools,
+ * Stageflow tools with risk `read`) or an MCP tool the eval author declares in `read_only_tools` (a declaration never changes a builtin or Stageflow tool). Sub-agents and
+ * operator-only tools are never read-only.
+ */
+export declare function classifyTool(_definition: Grants | undefined, toolId: string, readOnlyExtra?: readonly string[]): ToolClassification;
+export type ToolRiskEntry = {
+    tool: string;
+    class: ToolClass;
+    readOnly: boolean;
+};
+/** Every tool the agent is granted with how the `read_only` policy would treat it, for the eval picker. MCP wildcards stay as `<server>.*`. */
+export declare function toolRiskSummary(definition: Grants, readOnlyExtra?: readonly string[]): ToolRiskEntry[];
+export {};

package/dist/agents/toolRisk.js

@@ -0,0 +1,50 @@
+import { grantedToolIds } from "../config/loadAgent.js";
+import { isOperatorOnlyTool, STAGEFLOW_GRANT_PREFIX, TOOL_META } from "../toolkit/meta.js";
+import { BUILTIN_TOOL_GROUP_OF } from "./runtime/types.js";
+function listed(toolId, readOnlyExtra) {
+    if (readOnlyExtra.includes(toolId))
+        return true;
+    const dot = toolId.indexOf(".");
+    return dot > 0 && readOnlyExtra.includes(`${toolId.slice(0, dot)}.*`);
+}
+function baseClass(toolId) {
+    if (toolId === "browser")
+        return "browser";
+    if (toolId.startsWith("agent:"))
+        return "sub_agent";
+    if (toolId.startsWith(STAGEFLOW_GRANT_PREFIX)) {
+        const name = toolId.slice(STAGEFLOW_GRANT_PREFIX.length);
+        const meta = TOOL_META[name];
+        if (!meta)
+            return "mcp_unknown";
+        if (isOperatorOnlyTool(name))
+            return "operator_only";
+        return meta.risk;
+    }
+    const builtin = BUILTIN_TOOL_GROUP_OF[toolId];
+    if (builtin)
+        return builtin;
+    return "mcp_unknown";
+}
+/**
+ * What an eval may assume about a tool. `readOnly` is true only for what is provably read-only (builtin read tools,
+ * Stageflow tools with risk `read`) or an MCP tool the eval author declares in `read_only_tools` (a declaration never changes a builtin or Stageflow tool). Sub-agents and
+ * operator-only tools are never read-only.
+ */
+export function classifyTool(_definition, toolId, readOnlyExtra = []) {
+    let cls = baseClass(toolId);
+    if (cls === "operator_only" || cls === "sub_agent")
+        return { class: cls, readOnly: false };
+    const declared = listed(toolId, readOnlyExtra);
+    if (declared && cls === "mcp_unknown")
+        cls = "read";
+    return { class: cls, readOnly: cls === "read" };
+}
+/** Every tool the agent is granted with how the `read_only` policy would treat it, for the eval picker. MCP wildcards stay as `<server>.*`. */
+export function toolRiskSummary(definition, readOnlyExtra = []) {
+    const ids = new Set(grantedToolIds(definition));
+    for (const [server, tools] of Object.entries(definition.tools.mcp))
+        if (tools.includes("*"))
+            ids.add(`${server}.*`);
+    return [...ids].sort().map((tool) => ({ tool, ...classifyTool(definition, tool, readOnlyExtra) }));
+}

package/dist/cli/doctorCommand.js

@@ -9,6 +9,8 @@ import { warnMissingCaPaths } from "../preflight/tls.js";
 import { runPipelinePreflight, } from "../preflight/pipelinePreflight.js";
 import { loadPipelineOutcome } from "../config/loadPipeline.js";
 import { MCP_CATALOG_FILENAME } from "../config/resolveStageMcpServers.js";
+import { MIN_NODE } from "../evals/engine/promptfoo/pinned.js";
+import { engineStatus } from "../evals/engine/promptfoo/status.js";
 import { globalStageflowHome } from "../project/globalHome.js";
 import { createRunStore } from "../runstore/createStore.js";
 import { readFilesystemSize, resolveMinFreeDiskFloor, } from "../runstore/diskUsage.js";
@@ -142,6 +144,29 @@ function nodeVersionCheck() {
         message: `Node ${process.version}; engines floor may differ from Pi/better-sqlite3 (not bumping engines.node)`,
     };
 }
+function evalEngineCheck(home) {
+    const status = engineStatus({ home });
+    if (!status.installed) {
+        return {
+            id: "eval_engine",
+            status: "skipped",
+            message: `Eval engine not installed (optional; promptfoo ${status.pinned}). Run \`${status.installCommand}\``,
+        };
+    }
+    if (!status.nodeOk) {
+        return {
+            id: "eval_engine",
+            status: "warn",
+            code: "EVAL_NODE_TOO_OLD",
+            message: `Eval engine promptfoo ${status.version} installed, but Node ${status.node} is below the ${MIN_NODE} evals need`,
+        };
+    }
+    return {
+        id: "eval_engine",
+        status: "pass",
+        message: `Eval engine promptfoo ${status.version}`,
+    };
+}
 function discoverProviderEnvIds(env) {
     const byLower = new Set();
     for (const key of Object.keys(env)) {
@@ -355,6 +380,7 @@ export async function runDoctorChecks(options) {
     checks.push(tlsCheck(env));
     checks.push(await freeDiskCheck(home, env));
     checks.push(...(await mcpCommandsCheck(cwd)));
+    checks.push(evalEngineCheck(home));
     let preflight;
     if (options?.pipeline !== undefined) {
         const outcome = await loadPipelineOutcome(options.pipeline, { cwd });

package/dist/cli/evalCommand.d.ts

@@ -0,0 +1,20 @@
+import { type RunNpm } from "../evals/engine/promptfoo/install.js";
+import { type EnsureGlobalServiceResult } from "../server/ensureGlobalService.js";
+export declare const EVAL_USAGE = "Usage:\n  sf eval list [--json]\n  sf eval show <id|path> [--json]\n  sf eval validate [<id|path>] [--json] [--strict]\n  sf eval run <id|path> [--repeats <n>] [--judge <model>] [--model <id>] [--user-model <model>] [--allow-real-tools] [--max-cost <usd>] [--case <id> ...] [--tag <tag> ...] [--label <text>] [--no-wait] [--json]\n  sf eval runs [<evalId>] [--json]\n  sf eval runs show <evalRunId> [--json]\n  sf eval runs cancel <evalRunId>\n  sf eval runs delete <evalRunId>\n  sf eval sort <evalRunId> [--classifier auto|jev|judge] [--limit <n>] [--resort] [--json]\n  sf eval suggestions <evalRunId> [--status pending|accepted|dismissed] [--json]\n  sf eval compare <evalRunIdA> <evalRunIdB> [--json]\n  sf eval datasets [--json]\n  sf eval engine status [--json]\n  sf eval engine install\n\n  Agent evals: --model overrides the agent's model, --user-model the model that plays the user. An eval whose\n  conversation.tools is real also needs --allow-real-tools.\n\n  Every subcommand also accepts --project-root <id-or-path>.";
+export type EvalCommandIo = {
+    log: (line: string) => void;
+    error: (line: string) => void;
+};
+export type EvalCommandOptions = {
+    cwd?: string;
+    projectRoot?: string;
+    io?: Partial<EvalCommandIo>;
+    hostBaseUrl?: string;
+    ensureService?: () => Promise<EnsureGlobalServiceResult>;
+    home?: string;
+    runNpm?: RunNpm;
+    pollIntervalMs?: number;
+    stderrIsTTY?: boolean;
+    onInterrupt?: (handler: () => void) => () => void;
+};
+export declare function runEvalCommand(args: string[], options?: EvalCommandOptions): Promise<number>;