stageflow 0.40.0 → 0.44.0

package/dist/evals/runner/stageCaseRunner.js

@@ -0,0 +1,279 @@
+import path from "node:path";
+import { isStageTimeoutReason } from "../../agent/stageTimeout.js";
+import { loadPipelineOutcome } from "../../config/loadPipeline.js";
+import { readYamlObject } from "../../config/readYamlObject.js";
+import { projectRunForMcp } from "../../mcp/projectRun.js";
+import { isWaitingProjection } from "../../mcp/waitRun.js";
+import { PipelineValidationError } from "../../runtime/pipelineValidationError.js";
+import { caseToTask } from "../caseTask.js";
+export const EVAL_STAGE_TIMEOUT_GRACE_MS = 60_000;
+export const EVAL_STAGE_DEFAULT_TIMEOUT_MS = 30 * 60_000;
+export const EVAL_RUNNER_POLL_INTERVAL_MS = 250;
+export const EVAL_RUNNER_BUSY_BACKOFF_MS = { initial: 1_000, max: 5_000 };
+class EngineError extends Error {
+}
+function sleep(ms, signal) {
+    return new Promise((resolve) => {
+        if (signal.aborted) {
+            resolve();
+            return;
+        }
+        const finish = () => {
+            clearTimeout(timer);
+            signal.removeEventListener("abort", finish);
+            resolve();
+        };
+        const timer = setTimeout(finish, ms);
+        signal.addEventListener("abort", finish, { once: true });
+    });
+}
+async function readStageBody(plan) {
+    const { target, projectRoot } = plan;
+    if (target.kind !== "stage")
+        throw new EngineError("Not a stage target");
+    const abs = (rel) => path.resolve(projectRoot, rel);
+    let body;
+    let stageId;
+    if (target.file !== undefined) {
+        body = await readYamlObject(abs(target.file));
+        stageId = typeof body.id === "string" ? body.id : undefined;
+    }
+    else if (target.pipelinePath !== undefined && target.stageId !== undefined) {
+        stageId = target.stageId;
+        const pipelineAbs = abs(target.pipelinePath);
+        const loaded = await loadPipelineOutcome(pipelineAbs, { cwd: projectRoot, projectRoot, requireIo: false });
+        if (!loaded.ok)
+            throw new EngineError(`Target pipeline ${target.pipelinePath} does not load`);
+        const source = loaded.value.stageSources?.[stageId];
+        if (source?.kind === "file") {
+            body = await readYamlObject(source.path);
+        }
+        else {
+            const raw = await readYamlObject(pipelineAbs);
+            const entries = Array.isArray(raw.stages) ? raw.stages : [];
+            const entry = entries.find((e) => typeof e === "object" && e !== null && e.id === stageId);
+            if (entry !== undefined) {
+                const { needs: _needs, route: _route, route_if: _routeIf, entry: _entry, ...rest } = entry;
+                body = rest;
+            }
+        }
+    }
+    if (body === undefined || stageId === undefined) {
+        throw new EngineError("Target stage could not be read");
+    }
+    return { body: { ...body, id: stageId }, stageId };
+}
+function sumTokens(usage) {
+    let tokensIn = 0;
+    let tokensOut = 0;
+    for (const model of Object.values(usage?.models ?? {})) {
+        tokensIn += model.inputTokens;
+        tokensOut += model.outputTokens;
+    }
+    return { tokensIn, tokensOut };
+}
+function failure(code, message, extra = {}) {
+    return {
+        ok: false,
+        code,
+        message,
+        ...(extra.stageRunId !== undefined ? { stageRunId: extra.stageRunId } : {}),
+        ...(extra.latencyMs !== undefined ? { latencyMs: extra.latencyMs } : {}),
+        ...(extra.costUsd !== undefined ? { costUsd: extra.costUsd } : {}),
+    };
+}
+export function createStageCaseRunner(deps) {
+    const { manager, store } = deps;
+    const pollMs = deps.pollIntervalMs ?? EVAL_RUNNER_POLL_INTERVAL_MS;
+    const backoff = deps.busyBackoffMs ?? EVAL_RUNNER_BUSY_BACKOFF_MS;
+    const graceMs = deps.timeoutGraceMs ?? EVAL_STAGE_TIMEOUT_GRACE_MS;
+    const defaultTimeoutMs = deps.defaultTimeoutMs ?? EVAL_STAGE_DEFAULT_TIMEOUT_MS;
+    const states = new Map();
+    const stateFor = (evalRunId) => {
+        let state = states.get(evalRunId);
+        if (state === undefined) {
+            state = { active: new Map(), stops: new Map(), spend: 0, cancelled: false, abort: new AbortController() };
+            states.set(evalRunId, state);
+        }
+        return state;
+    };
+    async function cancelStageRun(state, runId, reason) {
+        if (!state.stops.has(runId))
+            state.stops.set(runId, reason);
+        await manager.cancelRun(runId, `eval_${reason}`).catch(() => undefined);
+    }
+    async function startWithRetry(state, input) {
+        let delay = backoff.initial;
+        for (;;) {
+            const started = await manager.startRun(input);
+            if (started.ok || started.code !== "busy_capacity" || state.cancelled)
+                return started;
+            await sleep(delay, state.abort.signal);
+            if (state.cancelled)
+                return started;
+            delay = Math.min(delay * 2, backoff.max);
+        }
+    }
+    async function awaitRun(state, runId, done, timeoutMs) {
+        let stop = false;
+        const wakeUp = new AbortController();
+        let timer;
+        const settled = done.then(() => "done", () => "done");
+        const timedOut = new Promise((resolve) => {
+            timer = setTimeout(() => resolve("timeout"), timeoutMs);
+        });
+        const waiting = (async () => {
+            while (!stop) {
+                await sleep(pollMs, wakeUp.signal);
+                if (stop)
+                    break;
+                try {
+                    const detail = await store.readRun(runId);
+                    if (detail.status === "running" && isWaitingProjection(projectRunForMcp(detail)))
+                        return "waiting";
+                }
+                catch {
+                    /* run row not readable yet; retry on next tick */
+                }
+            }
+            return "stopped";
+        })();
+        try {
+            const wake = await Promise.race([settled, timedOut, waiting]);
+            if (wake === "waiting")
+                await cancelStageRun(state, runId, "needs_input");
+            else if (wake === "timeout")
+                await cancelStageRun(state, runId, "timeout");
+            await settled;
+        }
+        finally {
+            stop = true;
+            wakeUp.abort();
+            if (timer !== undefined)
+                clearTimeout(timer);
+        }
+    }
+    async function collect(state, runId, stageId, fallbackLatencyMs) {
+        const detail = await store.readRun(runId);
+        const stage = detail.stages.find((s) => s.stage_id === stageId) ?? detail.stages[0];
+        const costUsd = stage?.cost_usd ?? stage?.usage?.costUsd ?? detail.total_cost_usd ?? 0;
+        let latencyMs = fallbackLatencyMs;
+        try {
+            const executions = await store.listStageExecutions(runId, stage?.stage_id ?? stageId);
+            const latest = executions[executions.length - 1];
+            if (latest?.started_at !== undefined && latest.finished_at !== undefined) {
+                latencyMs = Math.max(0, Date.parse(latest.finished_at) - Date.parse(latest.started_at));
+            }
+        }
+        catch {
+            /* keep wall-clock fallback */
+        }
+        const extra = { stageRunId: runId, latencyMs, costUsd };
+        state.spend += costUsd;
+        if (detail.status === "cancelled") {
+            const reason = state.stops.get(runId) ?? "cancelled";
+            if (reason === "needs_input") {
+                return failure("EVAL_STAGE_NEEDS_INPUT", "Stage stopped to ask the operator for input", extra);
+            }
+            if (reason === "timeout") {
+                return failure("EVAL_STAGE_TIMEOUT", "Stage exceeded the eval safety timeout", extra);
+            }
+            return failure("EVAL_CANCELLED", "Eval run was cancelled", extra);
+        }
+        if (detail.status === "failed") {
+            const reason = detail.failed_reason ?? "Stage failed";
+            return failure(isStageTimeoutReason(reason) ? "EVAL_STAGE_TIMEOUT" : "EVAL_STAGE_FAILED", reason, extra);
+        }
+        if (detail.status !== "succeeded") {
+            return failure("EVAL_ENGINE_ERROR", `Stage run ended in unexpected state "${detail.status}"`, extra);
+        }
+        const envelope = stage?.envelope;
+        if (!envelope)
+            return failure("EVAL_ENGINE_ERROR", "Stage run finished without an envelope", extra);
+        const output = {
+            kind: "envelope",
+            status: envelope.status === "success" ? "success" : "failure",
+            summary: envelope.summary,
+            ...(envelope.payload !== undefined ? { payload: envelope.payload } : {}),
+            artifacts: envelope.artifacts ?? [],
+        };
+        const { tokensIn, tokensOut } = sumTokens(stage?.usage);
+        return { ok: true, stageRunId: runId, output, latencyMs, costUsd, tokensIn, tokensOut };
+    }
+    async function runCase(plan, req) {
+        const state = stateFor(req.evalRunId);
+        if (state.cancelled)
+            return failure("EVAL_CANCELLED", "Eval run was cancelled");
+        if (state.spend >= plan.maxCostUsd) {
+            return failure("EVAL_BUDGET_EXCEEDED", `Eval spend reached the $${plan.maxCostUsd} limit`);
+        }
+        const planCase = plan.cases.find((c) => c.id === req.caseId);
+        if (planCase === undefined)
+            return failure("EVAL_ENGINE_ERROR", `Case "${req.caseId}" is not in the plan`);
+        let stageBody;
+        try {
+            stageBody = await readStageBody(plan);
+        }
+        catch (err) {
+            return failure("EVAL_ENGINE_ERROR", err instanceof Error ? err.message : "Target stage could not be read");
+        }
+        const { body, stageId } = stageBody;
+        const pipeline = {
+            id: `standalone-${stageId}`,
+            stages: [{ ...body, model: plan.modelOverride ?? plan.stageModel }],
+        };
+        const task = caseToTask(plan, planCase, req.evalRunId, req.repeatIndex);
+        const startedAt = Date.now();
+        let started;
+        try {
+            started = await startWithRetry(state, { pipeline, task, projectRoot: plan.projectRoot, origin: "eval" });
+        }
+        catch (err) {
+            if (err instanceof PipelineValidationError) {
+                const first = err.result.findings.find((f) => f.severity === "error") ?? err.result.findings[0];
+                return failure("EVAL_ENGINE_ERROR", first?.message ?? "Stage validation failed");
+            }
+            return failure("EVAL_ENGINE_ERROR", err instanceof Error ? err.message : "Stage run could not start");
+        }
+        if (!started.ok) {
+            if (state.cancelled)
+                return failure("EVAL_CANCELLED", "Eval run was cancelled");
+            return failure("EVAL_ENGINE_ERROR", started.reason);
+        }
+        const { runId, done } = started;
+        state.active.set(runId, { done });
+        try {
+            if (state.cancelled)
+                await cancelStageRun(state, runId, "cancelled");
+            const declared = typeof body.timeout_ms === "number" ? body.timeout_ms : undefined;
+            const timeoutMs = declared !== undefined ? declared + graceMs : defaultTimeoutMs;
+            await awaitRun(state, runId, done, timeoutMs);
+            return await collect(state, runId, stageId, Date.now() - startedAt);
+        }
+        catch (err) {
+            return failure("EVAL_ENGINE_ERROR", err instanceof Error ? err.message : "Stage run failed", {
+                stageRunId: runId,
+            });
+        }
+        finally {
+            state.active.delete(runId);
+            state.stops.delete(runId);
+        }
+    }
+    return {
+        runCase,
+        async cancelAll(evalRunId) {
+            const state = stateFor(evalRunId);
+            state.cancelled = true;
+            state.abort.abort();
+            await Promise.all([...state.active.entries()].map(async ([runId, entry]) => {
+                await cancelStageRun(state, runId, "cancelled");
+                await Promise.resolve(entry.done).catch(() => undefined);
+            }));
+        },
+        spend: (evalRunId) => states.get(evalRunId)?.spend ?? 0,
+        addSpend(evalRunId, usd) {
+            stateFor(evalRunId).spend += usd;
+        },
+    };
+}

package/dist/evals/runner/userDriver.d.ts

@@ -0,0 +1,71 @@
+import { type DriverUsage, type SimulatorModel } from "./simulatorModel.js";
+export type { DriverUsage };
+export type DriverMode = "script" | "simulated";
+export type DriverScenario = {
+    mode?: DriverMode;
+    script?: readonly string[];
+    opening?: string;
+    persona?: string;
+    goal?: string;
+    facts?: readonly string[];
+};
+export type DriverContext = {
+    turn: number;
+    transcript: Array<{
+        role: "user" | "assistant";
+        text: string;
+    }>;
+    signal: AbortSignal;
+};
+export type DriverEndReason = "script_end" | "resolved" | "abandoned" | "no_progress" | "empty_message";
+export type DriverStep = {
+    kind: "message";
+    text: string;
+    endAfter?: "resolved" | "abandoned";
+    usage?: DriverUsage;
+} | {
+    kind: "end";
+    reason: DriverEndReason;
+    usage?: DriverUsage;
+};
+export interface UserDriver {
+    next(ctx: DriverContext): Promise<DriverStep>;
+}
+export declare class DriverScenarioError extends Error {
+    readonly code = "EVAL_INVALID";
+    constructor(message: string);
+}
+export declare const SCRIPT_LIMITS: {
+    readonly minMessages: 1;
+    readonly maxMessages: 40;
+};
+export declare class ScriptDriver implements UserDriver {
+    private readonly script;
+    constructor(script: readonly string[]);
+    next(ctx: DriverContext): Promise<DriverStep>;
+}
+export declare function parseSimulatorReply(raw: string): {
+    text: string;
+    end?: "resolved" | "abandoned";
+};
+export declare class SimulatedUser implements UserDriver {
+    private readonly scenario;
+    private readonly model;
+    private readonly maxHistoryTurns;
+    private readonly maxMessageChars;
+    constructor(options: {
+        scenario: DriverScenario;
+        model: SimulatorModel;
+        maxHistoryTurns?: number;
+        maxMessageChars?: number;
+    });
+    next(ctx: DriverContext): Promise<DriverStep>;
+}
+export type DriverDeps = {
+    simulatorModel?: SimulatorModel;
+    limits?: {
+        maxHistoryTurns?: number;
+        maxMessageChars?: number;
+    };
+};
+export declare function createDriver(mode: DriverMode, scenario: DriverScenario, deps?: DriverDeps): UserDriver;

package/dist/evals/runner/userDriver.js

@@ -0,0 +1,104 @@
+import { SimulatorError } from "./simulatorModel.js";
+import { renderSimulatorPrompt, SIMULATOR_LIMITS } from "./simulatorPrompt.js";
+export class DriverScenarioError extends Error {
+    code = "EVAL_INVALID";
+    constructor(message) {
+        super(message);
+        this.name = "DriverScenarioError";
+    }
+}
+export const SCRIPT_LIMITS = { minMessages: 1, maxMessages: 40 };
+export class ScriptDriver {
+    script;
+    constructor(script) {
+        this.script = script;
+    }
+    async next(ctx) {
+        const text = this.script[ctx.turn - 1];
+        return text === undefined ? { kind: "end", reason: "script_end" } : { kind: "message", text };
+    }
+}
+const END_MARKER = /\[\[END:(resolved|abandoned)\]\]/g;
+const LAST_LINE_MARKER = /^\[\[END:(resolved|abandoned)\]\]$/;
+export function parseSimulatorReply(raw) {
+    const lines = raw.trim().split(/\r?\n/);
+    let end;
+    const last = lines[lines.length - 1]?.trim() ?? "";
+    const match = LAST_LINE_MARKER.exec(last);
+    if (match) {
+        end = match[1];
+        lines.pop();
+    }
+    const text = lines.join("\n").replace(END_MARKER, "").trim();
+    return end === undefined ? { text } : { text, end };
+}
+export class SimulatedUser {
+    scenario;
+    model;
+    maxHistoryTurns;
+    maxMessageChars;
+    constructor(options) {
+        const { opening, persona, goal } = options.scenario;
+        if (!opening || !persona || !goal) {
+            throw new DriverScenarioError("A simulated scenario needs opening, persona and goal.");
+        }
+        this.scenario = { opening, persona, goal, facts: options.scenario.facts ?? [] };
+        this.model = options.model;
+        this.maxHistoryTurns = options.maxHistoryTurns ?? SIMULATOR_LIMITS.maxHistoryTurns;
+        this.maxMessageChars = options.maxMessageChars ?? SIMULATOR_LIMITS.maxMessageChars;
+    }
+    async next(ctx) {
+        if (ctx.turn <= 1)
+            return { kind: "message", text: this.scenario.opening };
+        const { system, user } = renderSimulatorPrompt({
+            persona: this.scenario.persona,
+            goal: this.scenario.goal,
+            facts: this.scenario.facts,
+            transcript: ctx.transcript,
+            limits: { maxHistoryTurns: this.maxHistoryTurns, maxMessageChars: this.maxMessageChars },
+        });
+        let reply;
+        try {
+            reply = await this.model.complete({ system, user, signal: ctx.signal });
+        }
+        catch (error) {
+            if (ctx.signal.aborted || error instanceof SimulatorError)
+                throw error;
+            throw new SimulatorError();
+        }
+        const usage = reply.usage;
+        const parsed = parseSimulatorReply(reply.text);
+        const text = parsed.text.length > SIMULATOR_LIMITS.maxReplyChars ? parsed.text.slice(0, SIMULATOR_LIMITS.maxReplyChars) : parsed.text;
+        const withUsage = usage === undefined ? {} : { usage };
+        if (text.length === 0) {
+            return parsed.end === undefined
+                ? { kind: "end", reason: "empty_message", ...withUsage }
+                : { kind: "end", reason: parsed.end, ...withUsage };
+        }
+        const previous = [...ctx.transcript].reverse().find((entry) => entry.role === "user")?.text;
+        if (previous !== undefined && previous.trim().toLowerCase() === text.trim().toLowerCase()) {
+            return { kind: "end", reason: "no_progress", ...withUsage };
+        }
+        return parsed.end === undefined
+            ? { kind: "message", text, ...withUsage }
+            : { kind: "message", text, endAfter: parsed.end, ...withUsage };
+    }
+}
+export function createDriver(mode, scenario, deps = {}) {
+    if (mode === "script") {
+        const script = scenario.script;
+        if (!Array.isArray(script) || script.length < SCRIPT_LIMITS.minMessages || script.length > SCRIPT_LIMITS.maxMessages) {
+            throw new DriverScenarioError(`A scripted scenario needs ${SCRIPT_LIMITS.minMessages}-${SCRIPT_LIMITS.maxMessages} messages.`);
+        }
+        if (script.some((message) => typeof message !== "string" || message.trim().length === 0)) {
+            throw new DriverScenarioError("A scripted scenario cannot contain an empty message.");
+        }
+        return new ScriptDriver(script);
+    }
+    if (mode === "simulated") {
+        if (deps.simulatorModel === undefined)
+            throw new DriverScenarioError("A simulated scenario needs a simulator model.");
+        return new SimulatedUser({ scenario, model: deps.simulatorModel, ...deps.limits });
+    }
+    throw new DriverScenarioError(`Unknown conversation mode "${String(mode)}".`);
+}

package/dist/evals/scoring.d.ts

@@ -0,0 +1,34 @@
+import type { BudgetWarning, GradeStatus, ResultStatus, RunBudget } from "./statsTypes.js";
+export interface ScoringGrader {
+    id: string;
+    type?: string;
+    weight: number;
+    mustPass: boolean;
+    scale?: "pass_fail" | "1_to_5";
+    passAt?: number;
+}
+export interface ScoringGrade {
+    graderId: string;
+    status: GradeStatus;
+    passed: 0 | 1 | boolean | null;
+    score: number | null;
+    rawValue?: number | null;
+}
+export interface ScoreResultInput {
+    graders: ScoringGrader[];
+    grades: ScoringGrade[];
+    passThreshold: number;
+    status: ResultStatus;
+    latencyMs: number | null;
+    costUsd: number | null;
+    budget?: RunBudget | null;
+}
+export interface ScoreResultOutput {
+    score: number | null;
+    passed: boolean;
+    hasGraderErrors: boolean;
+    budgetWarnings: BudgetWarning[];
+}
+export declare function normalizeOneToFive(raw: number): number;
+export declare function graderPassed(grader: ScoringGrader, grade: Pick<ScoringGrade, "passed" | "score" | "rawValue">, passThreshold: number): boolean;
+export declare function scoreResult(input: ScoreResultInput): ScoreResultOutput;

package/dist/evals/scoring.js

@@ -0,0 +1,67 @@
+const EPS = 1e-9;
+export function normalizeOneToFive(raw) {
+    return (raw - 1) / 4;
+}
+export function graderPassed(grader, grade, passThreshold) {
+    if (grade.passed !== null && grade.passed !== undefined)
+        return Boolean(grade.passed);
+    const score = grade.score ?? 0;
+    if (grader.type === "reference")
+        return score + EPS >= (grader.passAt ?? 1);
+    if (grader.type === "rubric" && grader.scale === "1_to_5") {
+        return (grade.rawValue ?? 0) + EPS >= (grader.passAt ?? 4);
+    }
+    if (grader.type === "code")
+        return score + EPS >= passThreshold;
+    return score + EPS >= 1;
+}
+function budgetWarnings(latencyMs, costUsd, budget) {
+    const warnings = [];
+    if (budget?.latencyMs != null && latencyMs != null && latencyMs > budget.latencyMs) {
+        warnings.push("latency");
+    }
+    if (budget?.costUsd != null && costUsd != null && costUsd > budget.costUsd) {
+        warnings.push("cost");
+    }
+    return warnings;
+}
+export function scoreResult(input) {
+    const warnings = budgetWarnings(input.latencyMs, input.costUsd, input.budget);
+    if (input.status !== "completed") {
+        return { score: null, passed: false, hasGraderErrors: false, budgetWarnings: warnings };
+    }
+    const gradeById = new Map();
+    for (const grade of input.grades)
+        gradeById.set(grade.graderId, grade);
+    let mustPassFailed = false;
+    let hasGraderErrors = false;
+    let weightedSum = 0;
+    let weightTotal = 0;
+    for (const grader of input.graders) {
+        const grade = gradeById.get(grader.id);
+        if (grader.mustPass) {
+            if (!grade) {
+                mustPassFailed = true;
+            }
+            else if (grade.status === "error") {
+                mustPassFailed = true;
+                hasGraderErrors = true;
+            }
+            else if (!graderPassed(grader, grade, input.passThreshold)) {
+                mustPassFailed = true;
+            }
+            continue;
+        }
+        if (!grade)
+            continue;
+        if (grade.status === "error") {
+            hasGraderErrors = true;
+            continue;
+        }
+        weightedSum += grader.weight * (grade.score ?? 0);
+        weightTotal += grader.weight;
+    }
+    const score = weightTotal > 0 ? weightedSum / weightTotal : mustPassFailed ? 0 : 1;
+    const passed = !mustPassFailed && score + EPS >= input.passThreshold;
+    return { score, passed, hasGraderErrors, budgetWarnings: warnings };
+}