stageflow 0.40.0 → 0.44.0

package/dist/evals/manager.d.ts

@@ -0,0 +1,187 @@
+import type { AgentCatalogPort } from "../config/resolveAgentTarget.js";
+import type { ValidationFinding } from "../config/validateCatalog.js";
+import type { RunStore } from "../runstore/port.js";
+import { type EvalBudgetWarning, type EvalErrorCode, type EvalResultStatus, type EvalStartedVia } from "./constants.js";
+import { type EvalOverrides } from "./plan.js";
+import type { StageCaseRunner } from "./runner/stageCaseRunner.js";
+import type { EvalJson, EvalRunRow } from "./storeTypes.js";
+import type { EvalEngine, EvalFile, EvalPlan, RunCaseRequest, RunCaseResult } from "./types.js";
+export declare const EVAL_HOST_STOPPED_REASON = "Host stopped while the run was in progress";
+export declare const EVAL_DEFAULT_ENGINE = "promptfoo";
+export type EvalManagerStore = Pick<RunStore, "createEvalRun" | "insertEvalCases" | "upsertEvalResult" | "updateEvalRunProgress" | "finishEvalRun" | "getEvalRun" | "listEvalRuns" | "listEvalResults" | "recordGraderVersions">;
+export type EvalLogger = {
+    info?(code: string, message: string): void;
+    warn(code: string, message: string): void;
+    error(code: string, message: string): void;
+};
+export type EvalRunContext = {
+    plan: EvalPlan;
+    engineName: string;
+    runCase: (req: RunCaseRequest) => Promise<RunCaseResult>;
+    signal: AbortSignal;
+};
+/** What the manager needs from the agent service: plan resolution and cleanup of eval sessions. */
+export type EvalAgentsPort = AgentCatalogPort & {
+    deleteEvalSessions(projectRoot: string, sessionIds: string[]): Promise<string[]>;
+    sweepEvalSessions(input: {
+        runIds: string[];
+        olderThanMs: number;
+        isReferenced: (sessionId: string) => Promise<boolean> | boolean;
+    }): Promise<number>;
+};
+export declare const EVAL_ORPHAN_SESSION_MAX_AGE_MS: number;
+export type EvalRunManagerDeps = {
+    store: EvalManagerStore;
+    stageRunner: StageCaseRunner;
+    /** Lazy: the manager is built before the agent host. Absent or undefined means agents are unavailable. */
+    agents?: () => EvalAgentsPort | undefined;
+    /** Built-in operator agents currently visible. Empty when the Host hides them. */
+    internalAgents?: () => Array<{
+        id: string;
+        title: string;
+        playbook: string;
+        tools: string[];
+    }>;
+    engines?: (name: string, options?: unknown) => EvalEngine;
+    engineName?: string;
+    stageflowVersion: string;
+    now?: () => Date;
+    logger?: EvalLogger;
+    workDirRoot: string;
+    isKnownModel?: (id: string) => Promise<boolean>;
+    onRunStarted?(evalRunId: string, ctx: EvalRunContext): void | Promise<void>;
+    onRunFinished?(evalRunId: string): void | Promise<void>;
+};
+export declare class EvalRunError extends Error {
+    readonly code: EvalErrorCode;
+    readonly findings?: ValidationFinding[] | undefined;
+    constructor(code: EvalErrorCode, message: string, findings?: ValidationFinding[] | undefined);
+}
+export type StartEvalRunInput = {
+    projectRoot: string;
+    evalPath: string;
+    overrides?: EvalOverrides;
+    startedVia: EvalStartedVia;
+    callerId?: string;
+    label?: string;
+};
+export type TryCaseInput = {
+    id?: string;
+    input: Record<string, unknown>;
+    expected?: unknown;
+    tags?: string[];
+    title?: string;
+    goal?: string;
+    metadata?: unknown;
+};
+export type TryOneInput = {
+    projectRoot: string;
+    evalPath: string;
+    definition?: EvalFile | string;
+    datasetCases?: readonly unknown[];
+    caseId?: string;
+    case?: TryCaseInput;
+    /** The per-run consent to `conversation.tools: real`. */
+    allowRealTools?: boolean;
+};
+export type TryGrade = {
+    graderId: string;
+    graderType: string;
+    status: "graded" | "error";
+    passed: boolean | null;
+    score: number | null;
+    rawValue: number | null;
+    reason: string | null;
+    errorCode: string | null;
+    details: unknown;
+    judgeCostUsd: number | null;
+};
+export type TryResult = {
+    result: {
+        caseId: string;
+        status: EvalResultStatus;
+        errorCode: string | null;
+        errorMessage: string | null;
+        stageRunId: string | null;
+        subjectRef: string | null;
+        output: EvalJson | null;
+        latencyMs: number | null;
+        costUsd: number | null;
+        tokensIn: number | null;
+        tokensOut: number | null;
+        score: number | null;
+        passed: boolean;
+        hasGraderErrors: boolean;
+        budgetWarnings: EvalBudgetWarning[];
+        grades: TryGrade[];
+    };
+    grades: TryGrade[];
+    stageRunId: string | null;
+    costUsd: number;
+};
+export type LivePhase = {
+    caseId: string;
+    repeatIndex: number;
+    phase: "stage_running" | "agent_running" | "grading";
+};
+export type EvalLiveState = {
+    evalRunId: string;
+    phases: LivePhase[];
+};
+export type AwaitFinishedResult = {
+    finished: boolean;
+    run: EvalRunRow;
+};
+export declare class EvalRunManager {
+    private readonly deps;
+    private readonly store;
+    private readonly runner;
+    private readonly engines;
+    private readonly engineName;
+    private readonly now;
+    private live;
+    private reserved;
+    private trying;
+    private tryingRunId;
+    constructor(deps: EvalRunManagerDeps);
+    activeRunId(): string | null;
+    isExecuting(evalRunId: string): boolean;
+    getLive(evalRunId: string): EvalLiveState | null;
+    private claim;
+    start(input: StartEvalRunInput): Promise<{
+        evalRunId: string;
+        runNumber: number;
+    }>;
+    private failedCaseIds;
+    private makeIo;
+    private runHook;
+    private execute;
+    private enqueuePersist;
+    private resultInput;
+    private statsFor;
+    private summaryOf;
+    private fillRemainder;
+    private finish;
+    private finishFailed;
+    private cleanupWorkDir;
+    cancel(evalRunId: string): Promise<boolean>;
+    awaitFinished(evalRunId: string, timeoutMs: number): Promise<AwaitFinishedResult>;
+    private requireRealToolsConsent;
+    /**
+     * Validation and the real-tools consent for an agent eval, without claiming the run slot, so a transport can answer
+     * them before it checks the engine. Does nothing for stage evals and unreadable files.
+     */
+    preflightAgentStart(input: StartEvalRunInput): Promise<void>;
+    visibleInternalAgents(): Array<{
+        id: string;
+        title: string;
+        playbook: string;
+        tools: string[];
+    }>;
+    private agentsInput;
+    /** Delete the agent sessions of recovered runs and eval sessions older than 7 days that no result references. */
+    sweepAgentSessions(recoveredRunIds: string[]): Promise<number>;
+    recoverOnBoot(): Promise<string[]>;
+    private dropTrySession;
+    tryOne(input: TryOneInput): Promise<TryResult>;
+}