stageflow 0.40.0 → 0.42.0

package/dist/evals/engine/promptfoo/PromptfooEngine.js

@@ -0,0 +1,311 @@
+import { spawn as nodeSpawn } from "node:child_process";
+import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, rmSync, statSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { globalStageflowHome } from "../../../project/globalHome.js";
+import { EngineError } from "./errors.js";
+import { CALLBACK_URL_ENV, CONFIG_FILE, RESULTS_FILE, RESULTS_STREAM_FILE, RUN_TOKEN_ENV, generateFiles, } from "./generate.js";
+import { GUARD_FILENAME } from "./guard.js";
+import { mapJudge } from "../../judgeProviders.js";
+import { assertJudgesSupported } from "./judgeProviders.js";
+import { defaultLoadJudgePrices, defaultResolveJudgeKey, } from "./judgeServices.js";
+import { locatePromptfoo } from "./locate.js";
+import { normalizeRecord, recordIdentity, recordsFromResultsFile } from "./normalize.js";
+import { assertNode } from "./nodeVersion.js";
+import { PINNED_PROMPTFOO_VERSION } from "./pinned.js";
+export const PROMPTFOO_KILL_GRACE_MS = 5_000;
+export const PROMPTFOO_STREAM_POLL_MS = 100;
+export const STDERR_TAIL_BYTES = 2048;
+export const KEEP_WORKDIR_ENV = "STAGEFLOW_EVALS_KEEP_WORKDIR";
+export const DISABLE_FLAGS = {
+    PROMPTFOO_DISABLE_TELEMETRY: "1",
+    PROMPTFOO_DISABLE_UPDATE: "1",
+    PROMPTFOO_DISABLE_SHARING: "1",
+    PROMPTFOO_DISABLE_REMOTE_GENERATION: "true",
+};
+export function quoteNodeOption(file) {
+    return `"${file.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
+}
+export function buildChildEnv(input) {
+    return {
+        PATH: input.path ?? process.env.PATH ?? "",
+        HOME: input.workDir,
+        PROMPTFOO_CONFIG_DIR: path.join(input.workDir, "pf-home"),
+        NODE_OPTIONS: `--require ${quoteNodeOption(input.guardPath)}`,
+        ...DISABLE_FLAGS,
+        [CALLBACK_URL_ENV]: input.callbackUrl,
+        [RUN_TOKEN_ENV]: input.token,
+        ...input.judgeEnv,
+        ...input.extraEnv,
+    };
+}
+export function buildPromptfooArgs(binPath, plan) {
+    return [
+        binPath,
+        "eval",
+        "-c",
+        CONFIG_FILE,
+        "--output",
+        RESULTS_FILE,
+        "--no-cache",
+        "--no-progress-bar",
+        "--no-write",
+        "--repeat",
+        String(plan.repeats),
+        "-j",
+        String(Math.max(1, plan.concurrency)),
+    ];
+}
+export function resolveGraderModuleUrl() {
+    const here = path.dirname(fileURLToPath(import.meta.url));
+    const candidates = [
+        path.resolve(here, "../../graders/index.js"),
+        path.resolve(here, "../../../../dist/evals/graders/index.js"),
+    ];
+    const found = candidates.find((candidate) => existsSync(candidate));
+    if (found === undefined) {
+        throw new EngineError("EVAL_ENGINE_UNAVAILABLE", "Compiled graders were not found. Run `npm run build` before running evals from source.");
+    }
+    return pathToFileURL(found).href;
+}
+class StreamTail {
+    file;
+    offset = 0;
+    constructor(file) {
+        this.file = file;
+    }
+    lines() {
+        let size;
+        try {
+            size = statSync(this.file).size;
+        }
+        catch {
+            return [];
+        }
+        if (size <= this.offset)
+            return [];
+        const fd = openSync(this.file, "r");
+        let chunk;
+        try {
+            chunk = Buffer.alloc(size - this.offset);
+            readSync(fd, chunk, 0, chunk.length, this.offset);
+        }
+        finally {
+            closeSync(fd);
+        }
+        const lastNewline = chunk.lastIndexOf(0x0a);
+        if (lastNewline < 0)
+            return [];
+        this.offset += lastNewline + 1;
+        return chunk
+            .subarray(0, lastNewline)
+            .toString("utf8")
+            .split("\n")
+            .filter((line) => line.length > 0);
+    }
+}
+export class PromptfooEngine {
+    options;
+    name = "promptfoo";
+    constructor(options) {
+        this.options = options;
+    }
+    home() {
+        return this.options.home ?? globalStageflowHome();
+    }
+    async version() {
+        return PINNED_PROMPTFOO_VERSION;
+    }
+    validatePlan(plan) {
+        assertJudgesSupported(plan.graders);
+    }
+    async run(plan, io, signal) {
+        const opts = this.options;
+        assertNode(opts.nodeVersion);
+        assertJudgesSupported(plan.graders);
+        const located = locatePromptfoo({ home: this.home() });
+        if (!located.installed && opts.binPath === undefined) {
+            throw new EngineError("EVAL_ENGINE_UNAVAILABLE", located.reason);
+        }
+        const binPath = opts.binPath ?? (located.installed ? located.binPath : "");
+        const judgeEnv = {};
+        const judgeModels = new Set();
+        for (const grader of plan.graders) {
+            if (grader.type !== "rubric" && grader.type !== "goal_reached")
+                continue;
+            judgeModels.add(grader.judge);
+            const mapping = mapJudge(grader.judge);
+            if (judgeEnv[mapping.envVar] !== undefined)
+                continue;
+            const key = await (opts.resolveJudgeKey ?? defaultResolveJudgeKey)(mapping, plan.projectRoot);
+            if (key === undefined) {
+                throw new EngineError("EVAL_INVALID", `No API key is configured for judge provider "${mapping.stageflowProvider}" (model "${grader.judge}").`);
+            }
+            judgeEnv[mapping.envVar] = key;
+        }
+        const prices = await (opts.loadJudgePrices ?? defaultLoadJudgePrices)([...judgeModels], plan.projectRoot);
+        const judgePrice = (model) => prices.get(model);
+        const graderModuleUrl = opts.graderModuleUrl ?? resolveGraderModuleUrl();
+        const files = generateFiles(plan, {
+            graderModuleUrl,
+            ...(opts.judgeProviderConfig !== undefined ? { judgeProviderConfig: opts.judgeProviderConfig } : {}),
+        });
+        if (signal.aborted)
+            return;
+        const workDir = path.join(this.home(), "eval-runs", plan.evalRunId);
+        rmSync(workDir, { recursive: true, force: true });
+        mkdirSync(path.join(workDir, "pf-home"), { recursive: true, mode: 0o700 });
+        for (const [name, content] of Object.entries(files))
+            writeFileSync(path.join(workDir, name), content);
+        const hostRuns = new Map();
+        const keyOf = (caseId, repeatIndex) => `${caseId}#${repeatIndex}`;
+        const runCase = async (req) => {
+            io.onPhase(req.caseId, req.repeatIndex, plan.target.kind === "agent" ? "agent_running" : "stage_running");
+            const result = await io.runCase(req);
+            hostRuns.set(keyOf(req.caseId, req.repeatIndex), result);
+            if (result.ok)
+                io.onPhase(req.caseId, req.repeatIndex, "grading");
+            return result;
+        };
+        const token = opts.tokens.issue(plan.evalRunId, { plan, runCase });
+        const callbackUrl = typeof opts.callbackUrl === "function" ? opts.callbackUrl() : opts.callbackUrl;
+        const env = buildChildEnv({
+            workDir,
+            guardPath: path.join(workDir, GUARD_FILENAME),
+            callbackUrl,
+            token,
+            judgeEnv,
+            ...(opts.extraEnv !== undefined ? { extraEnv: opts.extraEnv } : {}),
+        });
+        const emitted = new Set();
+        const warnedPrice = new Set();
+        let stopping = false;
+        let killTimer;
+        let child;
+        let exited = false;
+        const terminate = () => {
+            if (child === undefined || exited || stopping)
+                return;
+            stopping = true;
+            child.kill("SIGTERM");
+            killTimer = setTimeout(() => {
+                if (!exited)
+                    child?.kill("SIGKILL");
+            }, opts.killGraceMs ?? PROMPTFOO_KILL_GRACE_MS);
+        };
+        const handleRecord = (record) => {
+            const identity = recordIdentity(record, plan);
+            if (identity === undefined)
+                return;
+            const key = keyOf(identity.caseId, identity.repeatIndex);
+            if (emitted.has(key))
+                return;
+            emitted.add(key);
+            const outcome = normalizeRecord(record, identity, plan, {
+                hostRun: (caseId, repeatIndex) => hostRuns.get(keyOf(caseId, repeatIndex)),
+                judgePrice,
+            });
+            for (const model of outcome.priceUnknownModels) {
+                if (warnedPrice.has(model))
+                    continue;
+                warnedPrice.add(model);
+                io.log("warn", `No catalog price for judge model ${model}; its cost is counted as $0.`);
+            }
+            io.onResult(outcome.result);
+            if (outcome.judgeCostUsd > 0)
+                io.reportJudgeCost(outcome.judgeCostUsd);
+            io.onPhase(identity.caseId, identity.repeatIndex, "done");
+            const run = outcome.result.run;
+            if (!run.ok && run.code === "EVAL_BUDGET_EXCEEDED")
+                terminate();
+        };
+        const tail = new StreamTail(path.join(workDir, RESULTS_STREAM_FILE));
+        const drain = () => {
+            for (const line of tail.lines()) {
+                try {
+                    handleRecord(JSON.parse(line));
+                }
+                catch {
+                    io.log("warn", "Skipped an unreadable progress line from the eval engine.");
+                }
+            }
+        };
+        const onAbort = () => terminate();
+        let poll;
+        let stderrTail = "";
+        try {
+            const exit = await new Promise((resolve) => {
+                try {
+                    child = (opts.spawn ?? nodeSpawn)(process.execPath, buildPromptfooArgs(binPath, plan), {
+                        cwd: workDir,
+                        env,
+                        stdio: ["ignore", "pipe", "pipe"],
+                    });
+                }
+                catch (error) {
+                    resolve({ code: null, signal: null, spawnError: error instanceof Error ? error : new Error(String(error)) });
+                    return;
+                }
+                child.stdout?.on("data", () => undefined);
+                child.stderr?.on("data", (data) => {
+                    stderrTail = (stderrTail + data.toString()).slice(-STDERR_TAIL_BYTES);
+                });
+                child.on("error", (error) => {
+                    exited = true;
+                    resolve({ code: null, signal: null, spawnError: error });
+                });
+                child.on("close", (code, sig) => {
+                    exited = true;
+                    resolve({ code, signal: sig });
+                });
+                poll = setInterval(drain, opts.streamPollMs ?? PROMPTFOO_STREAM_POLL_MS);
+                if (signal.aborted)
+                    terminate();
+                else
+                    signal.addEventListener("abort", onAbort, { once: true });
+            });
+            if (poll !== undefined)
+                clearInterval(poll);
+            if (killTimer !== undefined)
+                clearTimeout(killTimer);
+            signal.removeEventListener("abort", onAbort);
+            drain();
+            if (stopping || signal.aborted)
+                return;
+            const fail = (message) => {
+                if (stderrTail.length > 0)
+                    io.log("error", `promptfoo stderr (last ${STDERR_TAIL_BYTES} bytes): ${stderrTail}`);
+                throw new EngineError("EVAL_ENGINE_ERROR", message);
+            };
+            if (exit.spawnError !== undefined)
+                fail("The eval engine could not be started");
+            if (exit.code !== 0 && exit.code !== 100)
+                fail(`The eval engine exited with code ${exit.code ?? exit.signal ?? "unknown"}`);
+            const resultsFile = path.join(workDir, RESULTS_FILE);
+            if (!existsSync(resultsFile))
+                fail("The eval engine wrote no results");
+            let records;
+            try {
+                records = recordsFromResultsFile(JSON.parse(readFileSync(resultsFile, "utf8")));
+            }
+            catch {
+                records = undefined;
+            }
+            if (records === undefined)
+                fail("The eval engine results could not be read");
+            for (const record of records ?? [])
+                handleRecord(record);
+        }
+        finally {
+            if (poll !== undefined)
+                clearInterval(poll);
+            if (killTimer !== undefined)
+                clearTimeout(killTimer);
+            signal.removeEventListener("abort", onAbort);
+            opts.tokens.revoke(plan.evalRunId);
+            if (!(opts.keepWorkDir ?? process.env[KEEP_WORKDIR_ENV] === "1")) {
+                rmSync(workDir, { recursive: true, force: true });
+            }
+        }
+    }
+}

package/dist/evals/engine/promptfoo/errors.d.ts

@@ -0,0 +1,5 @@
+export type EngineErrorCode = "EVAL_NODE_TOO_OLD" | "EVAL_ENGINE_UNAVAILABLE" | "EVAL_ENGINE_ERROR" | "EVAL_INVALID";
+export declare class EngineError extends Error {
+    readonly code: EngineErrorCode;
+    constructor(code: EngineErrorCode, message: string);
+}

package/dist/evals/engine/promptfoo/errors.js

@@ -0,0 +1,8 @@
+export class EngineError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+        this.name = "EngineError";
+    }
+}

package/dist/evals/engine/promptfoo/generate.d.ts

@@ -0,0 +1,30 @@
+import { GUARD_FILENAME } from "./guard.js";
+import type { EvalPlan } from "../../types.js";
+export declare const CALLBACK_URL_ENV = "STAGEFLOW_EVAL_CALLBACK_URL";
+export declare const RUN_TOKEN_ENV = "STAGEFLOW_EVAL_TOKEN";
+export declare const CONFIG_FILE = "promptfooconfig.json";
+export declare const PROVIDER_FILE = "provider.mjs";
+export declare const ASSERTIONS_FILE = "assertions.mjs";
+export declare const HOOKS_FILE = "hooks.mjs";
+export declare const DATASET_FILE = "dataset.json";
+export declare const RESULTS_STREAM_FILE = "results.ndjson";
+export declare const RESULTS_FILE = "results.json";
+export declare const EMPTY_EXPECTED = "(none)";
+export type GenerateOptions = {
+    graderModuleUrl: string;
+    judgeProviderConfig?: Record<string, unknown>;
+};
+export type GeneratedFiles = {
+    [CONFIG_FILE]: string;
+    [PROVIDER_FILE]: string;
+    [ASSERTIONS_FILE]: string;
+    [HOOKS_FILE]: string;
+    [DATASET_FILE]: string;
+    [GUARD_FILENAME]: string;
+};
+export declare function assertionFnName(index: number): string;
+export declare function buildProviderSource(evalRunId: string, agent?: boolean): string;
+export declare function buildHooksSource(agent?: boolean): string;
+export declare function buildAssertionsSource(plan: EvalPlan, graderModuleUrl: string): string;
+export declare function buildConfig(plan: EvalPlan, options: GenerateOptions): Record<string, unknown>;
+export declare function generateFiles(plan: EvalPlan, options: GenerateOptions): GeneratedFiles;