stageflow 0.40.0 → 0.42.0

package/dist/server/evalTriageRoutes.d.ts

@@ -0,0 +1,2 @@
+import { type Ctx } from "./evalRouteContext.js";
+export declare function matchEvalTriageRoute(c: Ctx, seg: (value: string | undefined) => string): (() => Promise<void>) | undefined;

package/dist/server/evalTriageRoutes.js

@@ -0,0 +1,95 @@
+import { EvalApiError } from "../evals/read/api.js";
+import { SUGGESTION_STATUSES, TRIAGE_BUCKETS, TRIAGE_MAX_LIMIT } from "../evals/triage/constants.js";
+import { isTriageBucket, TriageService } from "../evals/triage/service.js";
+import { audit, bad, optStringArray, readOptionalBody, runByIdOr404 } from "./evalRouteContext.js";
+function triageOf(c) {
+    return new TriageService({ store: c.store, ...(c.triage ?? {}), loadPrices: c.triage?.loadPrices ?? c.loadPrices });
+}
+async function handleSort(c, id) {
+    const body = await readOptionalBody(c);
+    const row = await runByIdOr404(c, id);
+    const classifier = body.classifier ?? "auto";
+    if (classifier !== "auto" && classifier !== "jev" && classifier !== "judge") {
+        throw bad(`"classifier" must be auto, jev or judge`);
+    }
+    if (body.only_unsorted !== undefined && typeof body.only_unsorted !== "boolean")
+        throw bad(`"only_unsorted" must be a boolean`);
+    if (body.limit !== undefined &&
+        (typeof body.limit !== "number" || !Number.isInteger(body.limit) || body.limit < 1 || body.limit > TRIAGE_MAX_LIMIT)) {
+        throw bad(`"limit" must be an integer from 1 to ${TRIAGE_MAX_LIMIT}`);
+    }
+    try {
+        const result = await triageOf(c).sort(row, {
+            classifier: classifier,
+            ...(body.only_unsorted !== undefined ? { onlyUnsorted: body.only_unsorted } : {}),
+            ...(body.limit !== undefined ? { limit: body.limit } : {}),
+        });
+        audit(c, "eval_failures_sort", "ok", id);
+        c.json(c.res, 200, result);
+    }
+    catch (err) {
+        audit(c, "eval_failures_sort", "error", id, err instanceof EvalApiError ? err.code : "internal_error");
+        throw err;
+    }
+}
+async function handleSuggestionsList(c, id) {
+    const row = await runByIdOr404(c, id);
+    const status = c.url.searchParams.get("status") ?? undefined;
+    if (status !== undefined && !SUGGESTION_STATUSES.includes(status)) {
+        throw bad(`"status" must be one of ${SUGGESTION_STATUSES.join(", ")}`);
+    }
+    c.json(c.res, 200, await triageOf(c).list(row, status));
+}
+function idsOf(body, required) {
+    const ids = optStringArray(body, "ids");
+    if (ids === undefined && required)
+        throw bad(`"ids" is required`);
+    return ids;
+}
+async function handleSuggestionsAccept(c, id) {
+    const body = await readOptionalBody(c);
+    const row = await runByIdOr404(c, id);
+    const ids = idsOf(body, false);
+    if (body.min_confidence !== undefined && (typeof body.min_confidence !== "number" || body.min_confidence < 0 || body.min_confidence > 1)) {
+        throw bad(`"min_confidence" must be a number from 0 to 1`);
+    }
+    if (body.bucket !== undefined && !isTriageBucket(body.bucket))
+        throw bad(`"bucket" must be one of ${TRIAGE_BUCKETS.join(", ")}`);
+    const result = await triageOf(c).accept(row, {
+        ...(ids !== undefined ? { ids } : {}),
+        ...(typeof body.min_confidence === "number" ? { minConfidence: body.min_confidence } : {}),
+        ...(isTriageBucket(body.bucket) ? { bucket: body.bucket } : {}),
+    });
+    audit(c, "eval_suggestions_accept", "ok", id);
+    c.json(c.res, 200, result);
+}
+async function handleSuggestionsDismiss(c, id) {
+    const body = await readOptionalBody(c);
+    const row = await runByIdOr404(c, id);
+    const result = await triageOf(c).dismiss(row, idsOf(body, true));
+    audit(c, "eval_suggestions_dismiss", "ok", id);
+    c.json(c.res, 200, result);
+}
+async function handleSortStatus(c, id) {
+    c.json(c.res, 200, await triageOf(c).status(await runByIdOr404(c, id)));
+}
+export function matchEvalTriageRoute(c, seg) {
+    const { method, pathname } = c;
+    const m = (re) => pathname.match(re);
+    let match = m(/^\/api\/eval-runs\/([^/]+)\/failures\/sort$/);
+    if (method === "POST" && match)
+        return () => handleSort(c, seg(match[1]));
+    match = m(/^\/api\/eval-runs\/([^/]+)\/failures\/suggestions$/);
+    if (method === "GET" && match)
+        return () => handleSuggestionsList(c, seg(match[1]));
+    match = m(/^\/api\/eval-runs\/([^/]+)\/failures\/suggestions\/accept$/);
+    if (method === "POST" && match)
+        return () => handleSuggestionsAccept(c, seg(match[1]));
+    match = m(/^\/api\/eval-runs\/([^/]+)\/failures\/suggestions\/dismiss$/);
+    if (method === "POST" && match)
+        return () => handleSuggestionsDismiss(c, seg(match[1]));
+    match = m(/^\/api\/eval-runs\/([^/]+)\/failures\/status$/);
+    if (method === "GET" && match)
+        return () => handleSortStatus(c, seg(match[1]));
+    return undefined;
+}

package/dist/server/http.d.ts

@@ -1,5 +1,6 @@
 import type { StagePort } from "../agent/port.js";
 import { type ProviderAuthContext } from "../agent/providerAuth.js";
+import type { EvalRunManager } from "../evals/manager.js";
 import { type LiveViewRoutesOptions } from "./liveViewRoutes.js";
 import { WorkshopChatSessionRegistry } from "../workshop/chatTurn.js";
 import { type AgentPort } from "../operatorAgent/index.js";
@@ -10,6 +11,7 @@ import type { RunChangeBus } from "../runtime/runChangeBus.js";
 import type { StageBrowserSupport } from "../browser/browserHost.js";
 import { type AgentHostContext } from "../agents/hostContext.js";
 import type { ConversationBackend } from "../agents/sessions.js";
+import { type EvalServices } from "./evalInternalRoutes.js";
 import { EmailTriggers } from "../email/triggers.js";
 import { type AllowedHosts } from "./allowedHosts.js";
 import { type ControlTokens } from "./controlToken.js";
@@ -68,6 +70,9 @@ export type OperatorRouteDeps = {
     agentConversationBackend?: ConversationBackend;
     /** Shared with MCP when set by bootstrap. */
     agentHost?: AgentHostContext;
+    evalRunManager?: EvalRunManager;
+    /** Wired by the eval manager; until then the internal eval callback answers 404. */
+    evalServices?: EvalServices;
 };
 /**
  * The full operator-console route surface: REST API (runs, stages,

package/dist/server/http.js

@@ -8,6 +8,9 @@ import { mapProviderAuthError } from "../agent/providerInspect.js";
 import { handleProviderRoutes } from "./providerRoutes.js";
 import { handleProjectMcpRoutes } from "./projectMcpRoutes.js";
 import { handleTriggerRoutes } from "./triggerRoutes.js";
+import { handleEvalRoutes } from "./evalRoutes.js";
+import { defaultLoadJudgePrices } from "../evals/engine/promptfoo/judgeServices.js";
+import { engineStatus as promptfooEngineStatus } from "../evals/engine/promptfoo/status.js";
 import { readCatalogFileForHttp, writeCatalogFileForHttp, } from "./readCatalogFile.js";
 import { applyLiveViewHeaders, createLiveViewRoutes, isLiveViewDialogPath, isLiveViewInputPath, isLiveViewPath, isLiveViewReopenTabPath, } from "./liveViewRoutes.js";
 import { createPipeline, parseCreatePipelineBody } from "../config/createPipeline.js";
@@ -61,6 +64,7 @@ import { AgentScheduleSource, agentScheduleTickIntervalMsFromEnv, } from "../age
 import { handleAgentAuthorRoutes } from "./agentAuthorRoutes.js";
 import { handleAgentRoutes } from "./agentRoutes.js";
 import { handleChannelRoutes } from "./channelRoutes.js";
+import { handleEvalInternalRoutes, isEvalInternalPath } from "./evalInternalRoutes.js";
 import { handleMemoryRoutes } from "./memoryRoutes.js";
 import { bootstrapStageflowHost, } from "./bootstrap.js";
 import { assertAllowedHttpAccess, isTrustedLocalHttpRequest, resolveAllowedHosts, } from "./allowedHosts.js";
@@ -183,7 +187,7 @@ export function defaultUiDistDir() {
     return path.resolve(here, "../ui");
 }
 export function createOperatorRoutes(deps) {
-    const { manager, store, cwd, agentDir, rootDir, providerAuthContext, uiDistDir, workshopOperatorHost, agentConversationBackend, agentHost: injectedHost, } = deps;
+    const { manager, store, cwd, agentDir, rootDir, providerAuthContext, uiDistDir, workshopOperatorHost, agentConversationBackend, agentHost: injectedHost, evalServices, } = deps;
     const agentConnection = injectedHost ? undefined : sqliteConnectionFromStore(store);
     const agentHost = injectedHost ??
         (agentConnection
@@ -250,7 +254,12 @@ export function createOperatorRoutes(deps) {
             try {
                 if (await liveView.handle(req, res))
                     return true;
-                if (pathname === "/api/agent-author/chat" || pathname === "/api/agent-author/undo") {
+                if (isEvalInternalPath(pathname) && (await handleEvalInternalRoutes(req, method, pathname, evalServices, res, json)))
+                    return true;
+                if (pathname === "/api/agent-author/chat" ||
+                    pathname === "/api/agent-author/undo" ||
+                    pathname === "/api/agent-author/sessions" ||
+                    pathname.startsWith("/api/agent-author/sessions/")) {
                     if (await handleAgentAuthorRoutes(req, method, pathname, cwd, operatorHost, res, json))
                         return true;
                 }
@@ -377,6 +386,9 @@ export function createOperatorRoutes(deps) {
                         filter.pipeline = pipeline;
                     if (callerId !== null)
                         filter.caller_id = callerId;
+                    const origin = url.searchParams.get("origin");
+                    if (origin === "eval" || origin === "any")
+                        filter.origin = origin;
                     json(res, 200, {
                         runs: await store.listRuns(Object.keys(filter).length > 0 ? filter : undefined),
                     });
@@ -2894,6 +2906,21 @@ export function createOperatorRoutes(deps) {
                 })) {
                     return true;
                 }
+                if (await handleEvalRoutes(req, res, {
+                    cwd,
+                    store,
+                    manager,
+                    evalRunManager: deps.evalRunManager,
+                    json,
+                    readJsonBody,
+                    auditLog,
+                    engineStatus: () => promptfooEngineStatus({ home: globalStageflowHome() }),
+                    agents: () => agentHost?.service,
+                    loadPrices: defaultLoadJudgePrices,
+                    triage: { judgeBackend: () => agentHost?.backend },
+                })) {
+                    return true;
+                }
                 if (await handleTriggerRoutes(req, res, {
                     cwd,
                     manager,
@@ -3151,6 +3178,8 @@ export async function startUiServer(options) {
                 ? { workshopOperatorHost: options.workshopOperatorHost }
                 : {}),
             ...(boot.agentHost ? { agentHost: boot.agentHost } : {}),
+            ...(boot.evalRunManager ? { evalRunManager: boot.evalRunManager } : {}),
+            ...(boot.evalServices ? { evalServices: boot.evalServices } : {}),
             ...(options.agentConversationBackend
                 ? { agentConversationBackend: options.agentConversationBackend }
                 : {}),
@@ -3163,6 +3192,7 @@ export async function startUiServer(options) {
         controlTokens,
         routes,
     });
+    boot.setEvalCallbackBase?.(envelope.url);
     const closeServer = envelope.server.close.bind(envelope.server);
     envelope.server.close = ((callback) => closeServer(error => {
         void (boot.email?.stop() ?? Promise.resolve()).then(() => callback?.(error), () => callback?.(error ?? new Error("Email host shutdown failed")));

package/dist/server/mcpHost.js

@@ -28,6 +28,8 @@ export async function startMcpServer(options) {
             controlTokens,
             getShutdown: () => shutdown,
             ...(boot.agentHost ? { agentHost: boot.agentHost } : {}),
+            ...(boot.evalRunManager ? { evalRunManager: boot.evalRunManager } : {}),
+            ...(boot.evalServices ? { evalServices: boot.evalServices } : {}),
         });
     const envelope = await createHttpHost({
         boot,
@@ -39,6 +41,7 @@ export async function startMcpServer(options) {
         maxConnections: options.maxConnections,
         routes,
     });
+    boot.setEvalCallbackBase?.(envelope.url);
     shutdown = installShutdownController({
         server: envelope.server,
         host: makeDrainableHostFromOptional(envelope.manager, envelope.store),

package/dist/toolkit/meta.d.ts

@@ -4,7 +4,7 @@
  * can import it without pulling in the runtime.
  */
 export type ToolRisk = "read" | "write" | "destructive" | "external";
-export type ToolGroupId = "discover" | "runs.read" | "runs.control" | "runs.cleanup" | "gates.read" | "gates.decide" | "triggers" | "agents.read" | "agents.manage" | "agents.cleanup" | "agents.converse" | "pipelines.author";
+export type ToolGroupId = "discover" | "runs.read" | "runs.control" | "runs.cleanup" | "gates.read" | "gates.decide" | "triggers" | "agents.read" | "agents.manage" | "agents.cleanup" | "agents.converse" | "evals.read" | "evals.run" | "pipelines.author";
 export type ToolGroup = {
     id: ToolGroupId;
     title: string;

package/dist/toolkit/meta.js

@@ -16,6 +16,8 @@ export const TOOL_GROUPS = [
     { id: "agents.cleanup", title: "Archive agents", summary: "Archive an agent. Destructive." },
     { id: "agents.converse", title: "Talk to agents", summary: "Chat with an agent and stop its running turn." },
     { id: "pipelines.author", title: "Author pipelines and tasks", summary: "Read, create and edit pipelines, stages and tasks directly on disk." },
+    { id: "evals.read", title: "Read evals", summary: "List evals, runs and results; validate and compare. Free." },
+    { id: "evals.run", title: "Run evals", summary: "Start and cancel eval runs. Calls models and costs money." },
 ];
 const t = (group, risk, operatorOnly) => ({ group, risk, ...(operatorOnly ? { operatorOnly } : {}) });
 export const TOOL_META = {
@@ -87,6 +89,19 @@ export const TOOL_META = {
     edit_stage: t("pipelines.author", "write"),
     create_task: t("pipelines.author", "write"),
     edit_task: t("pipelines.author", "write"),
+    list_evals: t("evals.read", "read"),
+    get_eval: t("evals.read", "read"),
+    validate_eval: t("evals.read", "read"),
+    get_eval_run: t("evals.read", "read"),
+    list_eval_runs: t("evals.read", "read"),
+    get_eval_results: t("evals.read", "read"),
+    get_eval_result: t("evals.read", "read"),
+    compare_eval_runs: t("evals.read", "read"),
+    get_eval_failure_suggestions: t("evals.read", "read"),
+    get_eval_failure_sort_status: t("evals.read", "read"),
+    run_eval: t("evals.run", "external"),
+    sort_eval_failures: t("evals.run", "external"),
+    cancel_eval_run: t("evals.run", "write"),
 };
 /** The grant id the policy gate and `approvals.ask_first` use for a Stageflow tool. */
 export const STAGEFLOW_GRANT_PREFIX = "stageflow.";

package/dist/toolkit/profiles.js

@@ -11,7 +11,7 @@ export const PROFILE_GRANTS = {
     },
     "agent-author": {
         prefix: "agent_author.",
-        tools: ["read_project_file", "list_project_files", "list_mcp_servers", "update_agent_draft"],
+        tools: ["read_agent_draft", "read_project_file", "list_project_files", "list_mcp_servers", "list_author_catalog", "update_agent_draft"],
     },
 };
 export function profileGrantIds(profileId) {

package/dist/types/stageflowManifest.d.ts

@@ -3,10 +3,14 @@ export type StageflowManifestCatalog = {
     pipelines: string[];
     tasks: string[];
     triggers?: string[];
+    evals?: string[];
+    datasets?: string[];
     patterns?: {
         pipeline?: string;
         task?: string;
         trigger?: string;
+        eval?: string;
+        dataset?: string | string[];
     };
     exclude?: string[];
 };
@@ -26,5 +30,7 @@ export type LoadedManifest = {
         pipeline: string;
         task: string;
         trigger: string;
+        eval: string;
+        dataset: string[];
     };
 };