stageflow 0.40.0 → 0.42.0

package/dist/server/bootstrap.js

@@ -14,6 +14,12 @@ import { assertStoreFilesystemSupported } from "../runstore/storeFilesystem.js";
 import { assertStoreQuickCheck, isSqliteCorruptError, maybeCheckpointResidualWal, rethrowAsStoreIntegrityFailed, } from "../runstore/sqlite/applyStorePragmas.js";
 import { consoleStageBrowserSupport } from "../browser/stageBrowserEnv.js";
 import { RunManager } from "../runtime/runManager.js";
+import { EvalRunManager } from "../evals/manager.js";
+import { createStageCaseRunner } from "../evals/runner/stageCaseRunner.js";
+import { getEngine } from "../evals/engine/index.js";
+import { RunTokenRegistry } from "../evals/engine/promptfoo/runTokens.js";
+import { EVAL_DEFAULT_ENGINE } from "../evals/manager.js";
+import { PACKAGE_VERSION } from "../package-meta.js";
 import { PI_CODING_AGENT_DIR_ENV } from "../runtime/stageRoots.js";
 import { ScheduleSource, triggerTickIntervalMsFromEnv, } from "../runtime/scheduleSource.js";
 import { GithubPollSource, githubPollIntervalMsFromEnv, } from "../runtime/githubPollSource.js";
@@ -32,6 +38,9 @@ import { applyPendingRestoreAtBoot, } from "../runstore/restore.js";
 import { assertTmpdirUsable } from "../runstore/assertTmpdir.js";
 import { logger as rootLogger } from "../logging/logger.js";
 import { getModelCatalog } from "../models/catalog.js";
+import { createAgentCaseRunner } from "../evals/runner/agentCaseRunner.js";
+import { createEvalCaseRunner } from "../evals/runner/caseRunner.js";
+import { defaultLoadJudgePrices } from "../evals/engine/promptfoo/judgeServices.js";
 import { createAgentHostContext } from "../agents/hostContext.js";
 export const DEFAULT_GC_INTERVAL_MS = 60 * 60 * 1000;
 /** Parse `STAGEFLOW_GC_INTERVAL_MS`; default 1h; `0` disables. */
@@ -257,6 +266,47 @@ export async function bootstrapStageflowHost(options) {
     await manager.autoResumeInterruptedStages();
     await manager.resumeStalledSchedules();
     await manager.reenqueuePersistedQueuedRuns();
+    const evalLog = rootLogger.child({ component: "evals" });
+    const evalStageRunner = createStageCaseRunner({ manager, store });
+    const evalAgentRunner = createAgentCaseRunner({
+        host: () => agentHost,
+        loadPrices: defaultLoadJudgePrices,
+    });
+    const evalRunner = createEvalCaseRunner({ stage: evalStageRunner, agent: evalAgentRunner });
+    const evalTokens = new RunTokenRegistry();
+    let evalCallbackBase = "";
+    const evalRunManager = new EvalRunManager({
+        store,
+        stageRunner: evalRunner,
+        agents: () => agentHost?.service,
+        engines: (name, engineOptions) => getEngine(name, name === EVAL_DEFAULT_ENGINE
+            ? {
+                tokens: evalTokens,
+                callbackUrl: () => `${evalCallbackBase}/api/evals/internal/run-case`,
+                home: ctx.globalHome,
+                ...engineOptions,
+            }
+            : engineOptions),
+        stageflowVersion: PACKAGE_VERSION,
+        workDirRoot: path.join(ctx.globalHome, "eval-runs"),
+        logger: {
+            info: (event, msg) => evalLog.info(event, msg),
+            warn: (event, msg) => evalLog.warn(event, msg),
+            error: (event, msg) => evalLog.error(event, msg),
+        },
+    });
+    const recoveredEvalRuns = await evalRunManager.recoverOnBoot().catch((err) => {
+        evalLog.warn("eval.recover_failed", err instanceof Error ? err.message : String(err));
+        return [];
+    });
+    const evalServices = {
+        tokens: evalTokens,
+        runner: evalRunner,
+        isRunning: (evalRunId) => evalRunManager.isExecuting(evalRunId),
+    };
+    const setEvalCallbackBase = (baseUrl) => {
+        evalCallbackBase = baseUrl.replace(/\/+$/, "");
+    };
     await warnDurableRootDiskIfNeeded(ctx.globalHome, { env });
     const gcInterval = startPeriodicRunGc(manager, gcIntervalMsFromEnv(env));
     const stopGcInterval = () => {
@@ -355,6 +405,8 @@ export async function bootstrapStageflowHost(options) {
             },
         })
         : undefined;
+    if (agentHost)
+        void evalRunManager.sweepAgentSessions(recoveredEvalRuns);
     const mcpStateless = resolveMcpStateless({
         mcpStateless: options.mcpStateless,
     });
@@ -366,6 +418,7 @@ export async function bootstrapStageflowHost(options) {
         runChangeBus,
         providerAuthContext: options.providerAuthContext,
         projectRoot: rootDir,
+        evalRunManager,
         ...(agentHost
             ? {
                 agentService: agentHost.service,
@@ -389,6 +442,9 @@ export async function bootstrapStageflowHost(options) {
         isGitProject,
         store,
         manager,
+        evalRunManager,
+        evalServices,
+        setEvalCallbackBase,
         runChangeBus,
         mcpStateless,
         providerAuthContext: options.providerAuthContext,

package/dist/server/controlToken.js

@@ -1,6 +1,7 @@
 import { createHash, timingSafeEqual } from "node:crypto";
 import { readSecretFromEnvOrFile } from "../config/secretFromEnvOrFile.js";
 import { isLoopbackHostname } from "./allowedHosts.js";
+import { isEvalInternalPath } from "./evalInternalRoutes.js";
 import { matchLiveViewRoute } from "./liveViewPath.js";
 const CONTROL_TOKEN_PREFIX = "STAGEFLOW_CONTROL_TOKEN_";
 const DEFAULT_CALLER_ID = "default";
@@ -200,6 +201,8 @@ export function requiredScopeFor(method, pathname) {
         return null;
     if (isLiveViewRoute(pathname))
         return null;
+    if (isEvalInternalPath(pathname))
+        return null;
     if (pathname === "/api/backup" || pathname.startsWith("/api/backup/")) {
         return "drive";
     }

package/dist/server/evalInternalRoutes.d.ts

@@ -0,0 +1,14 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { RunTokenRegistry } from "../evals/engine/promptfoo/runTokens.js";
+import type { StageCaseRunner } from "../evals/runner/stageCaseRunner.js";
+export declare const EVAL_INTERNAL_RUN_CASE_PATH = "/api/evals/internal/run-case";
+export type EvalInternalServices = {
+    tokens: Pick<RunTokenRegistry, "verify">;
+    runner: Pick<StageCaseRunner, "runCase">;
+    isRunning(evalRunId: string): boolean;
+};
+export type EvalServices = EvalInternalServices | (() => EvalInternalServices | undefined);
+type Json = (res: ServerResponse, status: number, body: unknown) => void;
+export declare function isEvalInternalPath(pathname: string): boolean;
+export declare function handleEvalInternalRoutes(req: IncomingMessage, method: string, pathname: string, services: EvalServices | undefined, res: ServerResponse, json: Json): Promise<boolean>;
+export {};

package/dist/server/evalInternalRoutes.js

@@ -0,0 +1,87 @@
+export const EVAL_INTERNAL_RUN_CASE_PATH = "/api/evals/internal/run-case";
+const MAX_BODY_BYTES = 8192;
+export function isEvalInternalPath(pathname) {
+    return pathname === EVAL_INTERNAL_RUN_CASE_PATH;
+}
+function resolveServices(services) {
+    return typeof services === "function" ? services() : services;
+}
+async function readBody(req) {
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of req) {
+        const buffer = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
+        size += buffer.length;
+        if (size > MAX_BODY_BYTES)
+            throw new Error("body too large");
+        chunks.push(buffer);
+    }
+    return JSON.parse(Buffer.concat(chunks).toString("utf8"));
+}
+function parseRequest(body) {
+    if (typeof body !== "object" || body === null)
+        return undefined;
+    const { eval_run_id: evalRunId, case_id: caseId, repeat_index: repeatIndex } = body;
+    if (typeof evalRunId !== "string" || evalRunId.length === 0)
+        return undefined;
+    if (typeof caseId !== "string" || caseId.length === 0)
+        return undefined;
+    if (typeof repeatIndex !== "number" || !Number.isInteger(repeatIndex) || repeatIndex < 0)
+        return undefined;
+    return { evalRunId, caseId, repeatIndex };
+}
+function bearerToken(req) {
+    const header = req.headers.authorization;
+    const match = typeof header === "string" ? /^Bearer ([^\s]+)$/i.exec(header) : null;
+    return match?.[1];
+}
+export async function handleEvalInternalRoutes(req, method, pathname, services, res, json) {
+    if (!isEvalInternalPath(pathname))
+        return false;
+    const deps = resolveServices(services);
+    if (deps === undefined || method !== "POST") {
+        json(res, 404, { error: "Not found" });
+        return true;
+    }
+    const entry = deps.tokens.verify(bearerToken(req));
+    if (entry === undefined) {
+        json(res, 401, { code: "unauthorized" });
+        return true;
+    }
+    let request;
+    try {
+        request = parseRequest(await readBody(req));
+    }
+    catch {
+        request = undefined;
+    }
+    if (request === undefined) {
+        json(res, 400, { code: "invalid_request" });
+        return true;
+    }
+    if (request.evalRunId !== entry.evalRunId) {
+        json(res, 401, { code: "unauthorized" });
+        return true;
+    }
+    if (!deps.isRunning(request.evalRunId)) {
+        json(res, 409, { code: "eval_run_not_running" });
+        return true;
+    }
+    try {
+        const { binding } = entry;
+        const result = binding.runCase
+            ? await binding.runCase(request)
+            : binding.plan
+                ? await deps.runner.runCase(binding.plan, request)
+                : undefined;
+        if (result === undefined) {
+            json(res, 500, { code: "EVAL_ENGINE_ERROR" });
+            return true;
+        }
+        json(res, 200, result);
+    }
+    catch {
+        json(res, 500, { code: "EVAL_ENGINE_ERROR" });
+    }
+    return true;
+}

package/dist/server/evalRouteContext.d.ts

@@ -0,0 +1,46 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { SessionsSource } from "../config/datasetFromSessions.js";
+import type { EvalSampleAgents } from "../config/installEvalSample.js";
+import type { EngineStatus } from "../evals/engine/promptfoo/status.js";
+import type { TokenPrice } from "../evals/estimate.js";
+import type { EvalAgentsPort, EvalRunManager } from "../evals/manager.js";
+import { EvalApiError } from "../evals/read/api.js";
+import type { EvalRunRow } from "../evals/storeTypes.js";
+import type { TriageDeps } from "../evals/triage/service.js";
+import type { Logger } from "../logging/logger.js";
+import type { RunStore } from "../runstore/port.js";
+import type { RunManager } from "../runtime/runManager.js";
+export type EvalRoutesDeps = {
+    cwd: string;
+    store: RunStore;
+    manager: RunManager;
+    evalRunManager: EvalRunManager | undefined;
+    json: (res: ServerResponse, status: number, body: unknown) => void;
+    readJsonBody: (req: IncomingMessage) => Promise<unknown>;
+    auditLog: Logger;
+    engineStatus: () => EngineStatus;
+    isKnownModel?: (id: string) => Promise<boolean>;
+    sampleSourceDir?: string;
+    now?: () => Date;
+    /** Lazy view of the agent service; absent or undefined means agents are unavailable on this host. */
+    agents?: () => RouteAgents | undefined;
+    /** Model prices (USD per million tokens) for the agent cost estimate; unknown models are simply absent. */
+    loadPrices?: (modelIds: string[], projectRoot: string) => Promise<Map<string, TokenPrice>>;
+    /** Failure sorting: the Host's agent backend (judge classifier) and test seams (fetch, key lookup, base URL). */
+    triage?: Omit<TriageDeps, "store">;
+};
+export type Ctx = EvalRoutesDeps & {
+    req: IncomingMessage;
+    res: ServerResponse;
+    url: URL;
+    method: string;
+    pathname: string;
+    evalRunManager: EvalRunManager;
+};
+export type RouteAgents = EvalAgentsPort & SessionsSource & Partial<Pick<EvalSampleAgents, "create" | "setEnabled">>;
+export declare function isRecord(value: unknown): value is Record<string, unknown>;
+export declare function bad(message: string, code?: string): EvalApiError;
+export declare function audit(c: Ctx, action: string, outcome: "ok" | "error", target?: string, code?: string): void;
+export declare function readOptionalBody(c: Ctx): Promise<Record<string, unknown>>;
+export declare function optStringArray(body: Record<string, unknown>, key: string): string[] | undefined;
+export declare function runByIdOr404(c: Ctx, id: string): Promise<EvalRunRow>;

package/dist/server/evalRouteContext.js

@@ -0,0 +1,43 @@
+import { badRequest, runByIdOr404 as runByIdOr404Shared } from "../evals/read/api.js";
+import { writeAudit } from "../logging/audit.js";
+import { callerIdFromRequestAuth, getRequestAuth } from "./requestAuthContext.js";
+export function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+export function bad(message, code = "invalid_request") {
+    return badRequest(message, code);
+}
+function surface() {
+    return getRequestAuth()?.surface ?? "rest";
+}
+export function audit(c, action, outcome, target, code) {
+    writeAudit(c.auditLog, {
+        caller_id: callerIdFromRequestAuth(),
+        surface: surface(),
+        action,
+        outcome,
+        ...(target !== undefined ? { target_run_id: target } : {}),
+        ...(code !== undefined ? { error_code: code } : {}),
+    });
+}
+export async function readOptionalBody(c) {
+    try {
+        const body = await c.readJsonBody(c.req);
+        return isRecord(body) ? body : {};
+    }
+    catch {
+        return {};
+    }
+}
+export function optStringArray(body, key) {
+    const value = body[key];
+    if (value === undefined)
+        return undefined;
+    if (!Array.isArray(value) || value.some((v) => typeof v !== "string" || v === "")) {
+        throw bad(`"${key}" must be an array of non-empty strings`);
+    }
+    return value;
+}
+export async function runByIdOr404(c, id) {
+    return runByIdOr404Shared(c.store, id);
+}

package/dist/server/evalRoutes.d.ts

@@ -0,0 +1,5 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import { type EvalRoutesDeps } from "./evalRouteContext.js";
+export type { EvalRoutesDeps };
+export declare function isEvalRoutePath(pathname: string): boolean;
+export declare function handleEvalRoutes(req: IncomingMessage, res: ServerResponse, deps: EvalRoutesDeps): Promise<boolean>;