stageflow 0.40.0 → 0.44.0

package/dist/server/agentAuthorRoutes.js

@@ -1,4 +1,6 @@
-import { parseAgentAuthorDraft, runAgentAuthorTurn, undoAgentAuthorTurn } from "../agents/authorChat.js";
+import { authorSessionForApi, forgetAgentAuthorSession, parseAgentAuthorDraft, runAgentAuthorTurn, undoAgentAuthorTurn, } from "../agents/authorChat.js";
+import { createAuthorSession, deleteAuthorSession, listAuthorDrafts, readAuthorSession, saveAuthorSession, } from "../agents/authorSessionStore.js";
+import { parseWorkshopModel } from "../workshop/modelSettings.js";
 async function readBody(req) {
     const chunks = [];
     let size = 0;
@@ -24,6 +26,60 @@ async function readBody(req) {
 export async function handleAgentAuthorRoutes(req, method, pathname, projectRoot, host, res, json) {
     const chat = pathname === "/api/agent-author/chat";
     const undo = pathname === "/api/agent-author/undo";
+    const sessions = pathname === "/api/agent-author/sessions";
+    const sessionMatch = /^\/api\/agent-author\/sessions\/([^/]+)$/.exec(pathname);
+    if (!chat && !undo && !sessions && !sessionMatch)
+        return false;
+    if (sessions && method === "POST") {
+        const body = await readBody(req);
+        if (body.__tooLarge || body.__invalid) {
+            json(res, 400, { code: "agent.invalid" });
+            return true;
+        }
+        json(res, 200, createAuthorSession({
+            draft: parseAgentAuthorDraft(body.draft),
+            ...(parseWorkshopModel(body.model) ? { model: parseWorkshopModel(body.model) } : {}),
+        }));
+        return true;
+    }
+    if (sessions && method === "GET") {
+        json(res, 200, { drafts: listAuthorDrafts() });
+        return true;
+    }
+    if (sessionMatch && (method === "GET" || method === "PUT" || method === "DELETE")) {
+        const sessionId = decodeURIComponent(sessionMatch[1] ?? "");
+        try {
+            if (method === "GET") {
+                json(res, 200, authorSessionForApi(sessionId));
+                return true;
+            }
+            if (method === "DELETE") {
+                forgetAgentAuthorSession(sessionId);
+                if (!deleteAuthorSession(sessionId)) {
+                    json(res, 404, { code: "agent.session_not_found" });
+                    return true;
+                }
+                json(res, 200, { removed: true });
+                return true;
+            }
+            const body = await readBody(req);
+            if (body.__tooLarge || body.__invalid) {
+                json(res, 400, { code: "agent.invalid" });
+                return true;
+            }
+            const current = readAuthorSession(sessionId);
+            if (!current) {
+                json(res, 404, { code: "agent.session_not_found" });
+                return true;
+            }
+            json(res, 200, saveAuthorSession({ ...current, draft: parseAgentAuthorDraft(body.draft) }));
+        }
+        catch (err) {
+            const code = err instanceof Error ? err.message : "agent.internal";
+            json(res, code === "agent.session_not_found" ? 404 : 400, { code: code.startsWith("agent.") ? code : "agent.internal" });
+        }
+        return true;
+    }
     if (!chat && !undo)
         return false;
     if (method !== "POST") {
@@ -46,12 +102,14 @@ export async function handleAgentAuthorRoutes(req, method, pathname, projectRoot
             return true;
         }
         const message = typeof body.message === "string" ? body.message : "";
+        const model = parseWorkshopModel(body.model);
         const result = await runAgentAuthorTurn({
             host,
             projectRoot,
             message,
             ...(typeof body.sessionId === "string" ? { sessionId: body.sessionId } : {}),
             draft: parseAgentAuthorDraft(body.draft),
+            ...(model ? { model } : {}),
         });
         json(res, 200, result);
     }

package/dist/server/bootstrap.d.ts

@@ -7,6 +7,8 @@ import type { RunStore } from "../runstore/port.js";
 import type { StoreFilesystemClassification } from "../runstore/storeFilesystem.js";
 import type { StageBrowserSupport } from "../browser/browserHost.js";
 import { RunManager } from "../runtime/runManager.js";
+import { EvalRunManager } from "../evals/manager.js";
+import type { EvalInternalServices } from "./evalInternalRoutes.js";
 import { emailHostFor } from "../email/host.js";
 import { EmailTriggers } from "../email/triggers.js";
 import { type ChannelService } from "../channels/service.js";
@@ -15,6 +17,7 @@ import { type RunChangeBus } from "../runtime/runChangeBus.js";
 import { type McpHttpHandler } from "../mcp/server.js";
 import { type HostConfig } from "../config/hostConfig.js";
 import { bootProviderConfig } from "../agent/bootProviderConfig.js";
+import type { OperatorAgentHost } from "../operatorAgent/types.js";
 import { type AgentHostContext } from "../agents/hostContext.js";
 import type { ConversationBackend } from "../agents/sessions.js";
 export declare const DEFAULT_GC_INTERVAL_MS: number;
@@ -48,6 +51,12 @@ export type StageflowHostBootstrap = {
     store?: RunStore;
     /** Absent when `serveBlocked` — no RunManager lifecycle/resume. */
     manager?: RunManager;
+    /** Eval runs: shared with HTTP routes and MCP. Absent when `manager` is. */
+    evalRunManager?: EvalRunManager;
+    /** Services for the internal eval callback route; absent when `evalRunManager` is. */
+    evalServices?: EvalInternalServices;
+    /** Set once the HTTP host is listening so the promptfoo provider knows where to call back. */
+    setEvalCallbackBase?: (baseUrl: string) => void;
     runChangeBus: RunChangeBus;
     mcpStateless: boolean;
     providerAuthContext: ProviderAuthContext | undefined;
@@ -70,6 +79,10 @@ export type StageflowHostBootstrap = {
     memories?: MemoryHost;
     /** Shared agent store, chat controller, and conversation backend for HTTP/MCP. */
     agentHost?: AgentHostContext;
+    /** Filled when the operator console host is created, so internal-agent evals can open a gated session. */
+    operatorHostSlot?: {
+        current?: OperatorAgentHost;
+    };
     email?: ReturnType<typeof emailHostFor> & {
         triggers: EmailTriggers;
         stop: () => Promise<void>;

package/dist/server/bootstrap.js

@@ -14,6 +14,12 @@ import { assertStoreFilesystemSupported } from "../runstore/storeFilesystem.js";
 import { assertStoreQuickCheck, isSqliteCorruptError, maybeCheckpointResidualWal, rethrowAsStoreIntegrityFailed, } from "../runstore/sqlite/applyStorePragmas.js";
 import { consoleStageBrowserSupport } from "../browser/stageBrowserEnv.js";
 import { RunManager } from "../runtime/runManager.js";
+import { EvalRunManager } from "../evals/manager.js";
+import { createStageCaseRunner } from "../evals/runner/stageCaseRunner.js";
+import { getEngine } from "../evals/engine/index.js";
+import { RunTokenRegistry } from "../evals/engine/promptfoo/runTokens.js";
+import { EVAL_DEFAULT_ENGINE } from "../evals/manager.js";
+import { PACKAGE_VERSION } from "../package-meta.js";
 import { PI_CODING_AGENT_DIR_ENV } from "../runtime/stageRoots.js";
 import { ScheduleSource, triggerTickIntervalMsFromEnv, } from "../runtime/scheduleSource.js";
 import { GithubPollSource, githubPollIntervalMsFromEnv, } from "../runtime/githubPollSource.js";
@@ -32,6 +38,11 @@ import { applyPendingRestoreAtBoot, } from "../runstore/restore.js";
 import { assertTmpdirUsable } from "../runstore/assertTmpdir.js";
 import { logger as rootLogger } from "../logging/logger.js";
 import { getModelCatalog } from "../models/catalog.js";
+import { createAgentCaseRunner } from "../evals/runner/agentCaseRunner.js";
+import { createEvalCaseRunner } from "../evals/runner/caseRunner.js";
+import { createInternalAgentCaseRunner } from "../evals/runner/internalAgentCaseRunner.js";
+import { describeInternalAgents, internalAgentsVisible } from "../operatorAgent/internalAgents.js";
+import { defaultLoadJudgePrices } from "../evals/engine/promptfoo/judgeServices.js";
 import { createAgentHostContext } from "../agents/hostContext.js";
 export const DEFAULT_GC_INTERVAL_MS = 60 * 60 * 1000;
 /** Parse `STAGEFLOW_GC_INTERVAL_MS`; default 1h; `0` disables. */
@@ -257,6 +268,52 @@ export async function bootstrapStageflowHost(options) {
     await manager.autoResumeInterruptedStages();
     await manager.resumeStalledSchedules();
     await manager.reenqueuePersistedQueuedRuns();
+    const evalLog = rootLogger.child({ component: "evals" });
+    const evalStageRunner = createStageCaseRunner({ manager, store });
+    const operatorHostSlot = {};
+    const evalAgentRunner = createAgentCaseRunner({
+        host: () => agentHost,
+        loadPrices: defaultLoadJudgePrices,
+    });
+    const evalInternalRunner = createInternalAgentCaseRunner({ host: () => operatorHostSlot.current });
+    const evalRunner = createEvalCaseRunner({ stage: evalStageRunner, agent: evalAgentRunner, internalAgent: evalInternalRunner });
+    const evalTokens = new RunTokenRegistry();
+    let evalCallbackBase = "";
+    const evalRunManager = new EvalRunManager({
+        store,
+        stageRunner: evalRunner,
+        agents: () => agentHost?.service,
+        internalAgents: () => operatorHostSlot.current && internalAgentsVisible()
+            ? describeInternalAgents(operatorHostSlot.current).map(({ id, title, playbook, tools }) => ({ id, title, playbook, tools }))
+            : [],
+        engines: (name, engineOptions) => getEngine(name, name === EVAL_DEFAULT_ENGINE
+            ? {
+                tokens: evalTokens,
+                callbackUrl: () => `${evalCallbackBase}/api/evals/internal/run-case`,
+                home: ctx.globalHome,
+                ...engineOptions,
+            }
+            : engineOptions),
+        stageflowVersion: PACKAGE_VERSION,
+        workDirRoot: path.join(ctx.globalHome, "eval-runs"),
+        logger: {
+            info: (event, msg) => evalLog.info(event, msg),
+            warn: (event, msg) => evalLog.warn(event, msg),
+            error: (event, msg) => evalLog.error(event, msg),
+        },
+    });
+    const recoveredEvalRuns = await evalRunManager.recoverOnBoot().catch((err) => {
+        evalLog.warn("eval.recover_failed", err instanceof Error ? err.message : String(err));
+        return [];
+    });
+    const evalServices = {
+        tokens: evalTokens,
+        runner: evalRunner,
+        isRunning: (evalRunId) => evalRunManager.isExecuting(evalRunId),
+    };
+    const setEvalCallbackBase = (baseUrl) => {
+        evalCallbackBase = baseUrl.replace(/\/+$/, "");
+    };
     await warnDurableRootDiskIfNeeded(ctx.globalHome, { env });
     const gcInterval = startPeriodicRunGc(manager, gcIntervalMsFromEnv(env));
     const stopGcInterval = () => {
@@ -350,11 +407,16 @@ export async function bootstrapStageflowHost(options) {
                 store,
                 agentDir,
                 projectRoot: rootDir,
+                channels,
+                memories: memoryHostFor(),
+                evalRunManager,
                 ...(options.providerAuthContext ? { providerAuthContext: options.providerAuthContext } : {}),
                 ...(providerBoot !== undefined ? { providerBoot } : {}),
             },
         })
         : undefined;
+    if (agentHost)
+        void evalRunManager.sweepAgentSessions(recoveredEvalRuns);
     const mcpStateless = resolveMcpStateless({
         mcpStateless: options.mcpStateless,
     });
@@ -366,6 +428,9 @@ export async function bootstrapStageflowHost(options) {
         runChangeBus,
         providerAuthContext: options.providerAuthContext,
         projectRoot: rootDir,
+        evalRunManager,
+        channels,
+        memories: memoryHostFor(),
         ...(agentHost
             ? {
                 agentService: agentHost.service,
@@ -389,6 +454,9 @@ export async function bootstrapStageflowHost(options) {
         isGitProject,
         store,
         manager,
+        evalRunManager,
+        evalServices,
+        setEvalCallbackBase,
         runChangeBus,
         mcpStateless,
         providerAuthContext: options.providerAuthContext,
@@ -406,5 +474,6 @@ export async function bootstrapStageflowHost(options) {
         storeFilesystem,
         workspaceCatalog,
         ...(agentHost ? { agentHost } : {}),
+        operatorHostSlot,
     };
 }

package/dist/server/controlToken.js

@@ -1,6 +1,7 @@
 import { createHash, timingSafeEqual } from "node:crypto";
 import { readSecretFromEnvOrFile } from "../config/secretFromEnvOrFile.js";
 import { isLoopbackHostname } from "./allowedHosts.js";
+import { isEvalInternalPath } from "./evalInternalRoutes.js";
 import { matchLiveViewRoute } from "./liveViewPath.js";
 const CONTROL_TOKEN_PREFIX = "STAGEFLOW_CONTROL_TOKEN_";
 const DEFAULT_CALLER_ID = "default";
@@ -200,6 +201,8 @@ export function requiredScopeFor(method, pathname) {
         return null;
     if (isLiveViewRoute(pathname))
         return null;
+    if (isEvalInternalPath(pathname))
+        return null;
     if (pathname === "/api/backup" || pathname.startsWith("/api/backup/")) {
         return "drive";
     }

package/dist/server/evalInternalRoutes.d.ts

@@ -0,0 +1,14 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { RunTokenRegistry } from "../evals/engine/promptfoo/runTokens.js";
+import type { StageCaseRunner } from "../evals/runner/stageCaseRunner.js";
+export declare const EVAL_INTERNAL_RUN_CASE_PATH = "/api/evals/internal/run-case";
+export type EvalInternalServices = {
+    tokens: Pick<RunTokenRegistry, "verify">;
+    runner: Pick<StageCaseRunner, "runCase">;
+    isRunning(evalRunId: string): boolean;
+};
+export type EvalServices = EvalInternalServices | (() => EvalInternalServices | undefined);
+type Json = (res: ServerResponse, status: number, body: unknown) => void;
+export declare function isEvalInternalPath(pathname: string): boolean;
+export declare function handleEvalInternalRoutes(req: IncomingMessage, method: string, pathname: string, services: EvalServices | undefined, res: ServerResponse, json: Json): Promise<boolean>;
+export {};

package/dist/server/evalInternalRoutes.js

@@ -0,0 +1,87 @@
+export const EVAL_INTERNAL_RUN_CASE_PATH = "/api/evals/internal/run-case";
+const MAX_BODY_BYTES = 8192;
+export function isEvalInternalPath(pathname) {
+    return pathname === EVAL_INTERNAL_RUN_CASE_PATH;
+}
+function resolveServices(services) {
+    return typeof services === "function" ? services() : services;
+}
+async function readBody(req) {
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of req) {
+        const buffer = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
+        size += buffer.length;
+        if (size > MAX_BODY_BYTES)
+            throw new Error("body too large");
+        chunks.push(buffer);
+    }
+    return JSON.parse(Buffer.concat(chunks).toString("utf8"));
+}
+function parseRequest(body) {
+    if (typeof body !== "object" || body === null)
+        return undefined;
+    const { eval_run_id: evalRunId, case_id: caseId, repeat_index: repeatIndex } = body;
+    if (typeof evalRunId !== "string" || evalRunId.length === 0)
+        return undefined;
+    if (typeof caseId !== "string" || caseId.length === 0)
+        return undefined;
+    if (typeof repeatIndex !== "number" || !Number.isInteger(repeatIndex) || repeatIndex < 0)
+        return undefined;
+    return { evalRunId, caseId, repeatIndex };
+}
+function bearerToken(req) {
+    const header = req.headers.authorization;
+    const match = typeof header === "string" ? /^Bearer ([^\s]+)$/i.exec(header) : null;
+    return match?.[1];
+}
+export async function handleEvalInternalRoutes(req, method, pathname, services, res, json) {
+    if (!isEvalInternalPath(pathname))
+        return false;
+    const deps = resolveServices(services);
+    if (deps === undefined || method !== "POST") {
+        json(res, 404, { error: "Not found" });
+        return true;
+    }
+    const entry = deps.tokens.verify(bearerToken(req));
+    if (entry === undefined) {
+        json(res, 401, { code: "unauthorized" });
+        return true;
+    }
+    let request;
+    try {
+        request = parseRequest(await readBody(req));
+    }
+    catch {
+        request = undefined;
+    }
+    if (request === undefined) {
+        json(res, 400, { code: "invalid_request" });
+        return true;
+    }
+    if (request.evalRunId !== entry.evalRunId) {
+        json(res, 401, { code: "unauthorized" });
+        return true;
+    }
+    if (!deps.isRunning(request.evalRunId)) {
+        json(res, 409, { code: "eval_run_not_running" });
+        return true;
+    }
+    try {
+        const { binding } = entry;
+        const result = binding.runCase
+            ? await binding.runCase(request)
+            : binding.plan
+                ? await deps.runner.runCase(binding.plan, request)
+                : undefined;
+        if (result === undefined) {
+            json(res, 500, { code: "EVAL_ENGINE_ERROR" });
+            return true;
+        }
+        json(res, 200, result);
+    }
+    catch {
+        json(res, 500, { code: "EVAL_ENGINE_ERROR" });
+    }
+    return true;
+}

package/dist/server/evalRouteContext.d.ts

@@ -0,0 +1,46 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { SessionsSource } from "../config/datasetFromSessions.js";
+import type { EvalSampleAgents } from "../config/installEvalSample.js";
+import type { EngineStatus } from "../evals/engine/promptfoo/status.js";
+import type { TokenPrice } from "../evals/estimate.js";
+import type { EvalAgentsPort, EvalRunManager } from "../evals/manager.js";
+import { EvalApiError } from "../evals/read/api.js";
+import type { EvalRunRow } from "../evals/storeTypes.js";
+import type { TriageDeps } from "../evals/triage/service.js";
+import type { Logger } from "../logging/logger.js";
+import type { RunStore } from "../runstore/port.js";
+import type { RunManager } from "../runtime/runManager.js";
+export type EvalRoutesDeps = {
+    cwd: string;
+    store: RunStore;
+    manager: RunManager;
+    evalRunManager: EvalRunManager | undefined;
+    json: (res: ServerResponse, status: number, body: unknown) => void;
+    readJsonBody: (req: IncomingMessage) => Promise<unknown>;
+    auditLog: Logger;
+    engineStatus: () => EngineStatus;
+    isKnownModel?: (id: string) => Promise<boolean>;
+    sampleSourceDir?: string;
+    now?: () => Date;
+    /** Lazy view of the agent service; absent or undefined means agents are unavailable on this host. */
+    agents?: () => RouteAgents | undefined;
+    /** Model prices (USD per million tokens) for the agent cost estimate; unknown models are simply absent. */
+    loadPrices?: (modelIds: string[], projectRoot: string) => Promise<Map<string, TokenPrice>>;
+    /** Failure sorting: the Host's agent backend (judge classifier) and test seams (fetch, key lookup, base URL). */
+    triage?: Omit<TriageDeps, "store">;
+};
+export type Ctx = EvalRoutesDeps & {
+    req: IncomingMessage;
+    res: ServerResponse;
+    url: URL;
+    method: string;
+    pathname: string;
+    evalRunManager: EvalRunManager;
+};
+export type RouteAgents = EvalAgentsPort & SessionsSource & Partial<Pick<EvalSampleAgents, "create" | "setEnabled">>;
+export declare function isRecord(value: unknown): value is Record<string, unknown>;
+export declare function bad(message: string, code?: string): EvalApiError;
+export declare function audit(c: Ctx, action: string, outcome: "ok" | "error", target?: string, code?: string): void;
+export declare function readOptionalBody(c: Ctx): Promise<Record<string, unknown>>;
+export declare function optStringArray(body: Record<string, unknown>, key: string): string[] | undefined;
+export declare function runByIdOr404(c: Ctx, id: string): Promise<EvalRunRow>;

package/dist/server/evalRouteContext.js

@@ -0,0 +1,43 @@
+import { badRequest, runByIdOr404 as runByIdOr404Shared } from "../evals/read/api.js";
+import { writeAudit } from "../logging/audit.js";
+import { callerIdFromRequestAuth, getRequestAuth } from "./requestAuthContext.js";
+export function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+export function bad(message, code = "invalid_request") {
+    return badRequest(message, code);
+}
+function surface() {
+    return getRequestAuth()?.surface ?? "rest";
+}
+export function audit(c, action, outcome, target, code) {
+    writeAudit(c.auditLog, {
+        caller_id: callerIdFromRequestAuth(),
+        surface: surface(),
+        action,
+        outcome,
+        ...(target !== undefined ? { target_run_id: target } : {}),
+        ...(code !== undefined ? { error_code: code } : {}),
+    });
+}
+export async function readOptionalBody(c) {
+    try {
+        const body = await c.readJsonBody(c.req);
+        return isRecord(body) ? body : {};
+    }
+    catch {
+        return {};
+    }
+}
+export function optStringArray(body, key) {
+    const value = body[key];
+    if (value === undefined)
+        return undefined;
+    if (!Array.isArray(value) || value.some((v) => typeof v !== "string" || v === "")) {
+        throw bad(`"${key}" must be an array of non-empty strings`);
+    }
+    return value;
+}
+export async function runByIdOr404(c, id) {
+    return runByIdOr404Shared(c.store, id);
+}

package/dist/server/evalRoutes.d.ts

@@ -0,0 +1,5 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import { type EvalRoutesDeps } from "./evalRouteContext.js";
+export type { EvalRoutesDeps };
+export declare function isEvalRoutePath(pathname: string): boolean;
+export declare function handleEvalRoutes(req: IncomingMessage, res: ServerResponse, deps: EvalRoutesDeps): Promise<boolean>;