stageflow 0.40.0 → 0.42.0

package/dist/evals/engine/promptfoo/generate.js

@@ -0,0 +1,342 @@
+import { GUARD_FILENAME, GUARD_SOURCE } from "./guard.js";
+import { EngineError } from "./errors.js";
+import { mapJudge, unsupportedJudgeMessage } from "../../judgeProviders.js";
+import { caseGoal, goalRubricText } from "../../graders/goal.js";
+import { escapeTemplate, renderConversationRubricPrompt, renderRubricPrompt } from "./rubricPrompt.js";
+export const CALLBACK_URL_ENV = "STAGEFLOW_EVAL_CALLBACK_URL";
+export const RUN_TOKEN_ENV = "STAGEFLOW_EVAL_TOKEN";
+export const CONFIG_FILE = "promptfooconfig.json";
+export const PROVIDER_FILE = "provider.mjs";
+export const ASSERTIONS_FILE = "assertions.mjs";
+export const HOOKS_FILE = "hooks.mjs";
+export const DATASET_FILE = "dataset.json";
+export const RESULTS_STREAM_FILE = "results.ndjson";
+export const RESULTS_FILE = "results.json";
+export const EMPTY_EXPECTED = "(none)";
+export function assertionFnName(index) {
+    return `grader${index}`;
+}
+export function buildProviderSource(evalRunId, agent = false) {
+    const okMeta = agent
+        ? `
+            subjectRef: reply.subjectRef,
+            turns: reply.output && reply.output.turns,
+            toolCallCount: reply.output && Array.isArray(reply.output.tool_calls) ? reply.output.tool_calls.length : undefined,`
+        : "";
+    const failKeys = agent ? '["stageRunId", "subjectRef", "latencyMs", "costUsd"]' : '["stageRunId", "latencyMs", "costUsd"]';
+    return `import http from "node:http";
+
+const EVAL_RUN_ID = ${JSON.stringify(evalRunId)};
+
+function post(url, token, payload) {
+  return new Promise((resolve, reject) => {
+    const body = JSON.stringify(payload);
+    const req = http.request(
+      url,
+      {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          "content-length": Buffer.byteLength(body),
+          authorization: "Bearer " + token,
+        },
+      },
+      (res) => {
+        const chunks = [];
+        res.on("data", (chunk) => chunks.push(chunk));
+        res.on("end", () => resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString("utf8") }));
+        res.on("error", reject);
+      },
+    );
+    req.on("error", reject);
+    req.end(body);
+  });
+}
+
+export default class StageflowHostProvider {
+  id() {
+    return "stageflow-host";
+  }
+
+  async callApi(_prompt, context) {
+    const caseId = String(context?.vars?.caseId ?? "");
+    const index = context?.repeatIndex ?? context?.vars?.__repeatIndex;
+    const repeatIndex = Number.isInteger(index) ? index : 0;
+    const base = { caseId, repeatIndex };
+    const failure = (code, message, extra = {}) => ({
+      error: code + ": " + message,
+      metadata: { ...base, errorCode: code, ...extra },
+    });
+    try {
+      const url = process.env.${CALLBACK_URL_ENV};
+      const token = process.env.${RUN_TOKEN_ENV};
+      if (!url || !token) return failure("EVAL_ENGINE_ERROR", "host callback is not configured");
+      const res = await post(url, token, { eval_run_id: EVAL_RUN_ID, case_id: caseId, repeat_index: repeatIndex });
+      if (res.status !== 200) return failure("EVAL_ENGINE_ERROR", "host callback answered " + res.status);
+      let reply;
+      try {
+        reply = JSON.parse(res.body);
+      } catch {
+        return failure("EVAL_ENGINE_ERROR", "host callback reply is not JSON");
+      }
+      if (reply && reply.ok === true) {
+        return {
+          output: JSON.stringify(reply.output),
+          metadata: {
+            ...base,
+            stageRunId: reply.stageRunId,${okMeta}
+            latencyMs: reply.latencyMs,
+            costUsd: reply.costUsd,
+            tokensIn: reply.tokensIn,
+            tokensOut: reply.tokensOut,
+          },
+        };
+      }
+      const extra = {};
+      for (const key of ${failKeys}) {
+        if (reply && reply[key] !== undefined) extra[key] = reply[key];
+      }
+      return failure(
+        typeof reply?.code === "string" ? reply.code : "EVAL_ENGINE_ERROR",
+        typeof reply?.message === "string" ? reply.message : "stage case failed",
+        extra,
+      );
+    } catch {
+      return failure("EVAL_ENGINE_ERROR", "host callback failed");
+    }
+  }
+}
+`;
+}
+export function buildHooksSource(agent = false) {
+    const metaKeys = agent
+        ? '["caseId", "repeatIndex", "stageRunId", "subjectRef", "turns", "toolCallCount", "latencyMs", "costUsd", "tokensIn", "tokensOut", "errorCode"]'
+        : '["caseId", "repeatIndex", "stageRunId", "latencyMs", "costUsd", "tokensIn", "tokensOut", "errorCode"]';
+    return `import { appendFileSync } from "node:fs";
+
+const STREAM = new URL(${JSON.stringify(RESULTS_STREAM_FILE)}, import.meta.url);
+const META_KEYS = ${metaKeys};
+
+function pickMeta(meta) {
+  const out = {};
+  if (meta && typeof meta === "object") {
+    for (const key of META_KEYS) if (meta[key] !== undefined) out[key] = meta[key];
+  }
+  return out;
+}
+
+function pickComponent(c) {
+  const graderError = c?.metadata?.graderError === true;
+  return {
+    metric: c?.assertion?.metric,
+    type: c?.assertion?.type,
+    pass: c?.pass === true,
+    score: typeof c?.score === "number" ? c.score : undefined,
+    reason: graderError || typeof c?.reason !== "string" ? undefined : c.reason,
+    tokensIn: typeof c?.tokensUsed?.prompt === "number" ? c.tokensUsed.prompt : undefined,
+    tokensOut: typeof c?.tokensUsed?.completion === "number" ? c.tokensUsed.completion : undefined,
+    graderError,
+    errorCode: typeof c?.metadata?.errorCode === "string" ? c.metadata.errorCode : undefined,
+    stageflow: c?.metadata?.stageflow,
+  };
+}
+
+export async function afterEach(context) {
+  try {
+    const r = context?.result ?? {};
+    const record = {
+      testIdx: r.testIdx,
+      caseId: typeof r.vars?.caseId === "string" ? r.vars.caseId : undefined,
+      failureReason: r.failureReason,
+      metadata: pickMeta(r.response?.metadata ?? r.metadata),
+      output: typeof r.response?.output === "string" ? r.response.output : undefined,
+      providerError: typeof r.response?.error === "string" ? r.response.error : undefined,
+      components: (r.gradingResult?.componentResults ?? []).map(pickComponent),
+    };
+    appendFileSync(STREAM, JSON.stringify(record) + "\\n");
+  } catch {
+    return;
+  }
+}
+`;
+}
+const STAGE_TO_SUBJECT = `function toSubject(output) {
+  const value = typeof output === "string" ? JSON.parse(output) : output;
+  return {
+    status: value.status,
+    summary: value.summary,
+    ...(value.payload !== undefined ? { payload: value.payload } : {}),
+    artifacts: Array.isArray(value.artifacts) ? value.artifacts : [],
+  };
+}`;
+const AGENT_TO_SUBJECT = `function toSubject(output) {
+  const value = typeof output === "string" ? JSON.parse(output) : output;
+  if (value.kind === "conversation") {
+    const { transcript: _transport, ...subject } = value;
+    return subject;
+  }
+  return {
+    status: value.status,
+    summary: value.summary,
+    ...(value.payload !== undefined ? { payload: value.payload } : {}),
+    artifacts: Array.isArray(value.artifacts) ? value.artifacts : [],
+  };
+}`;
+const MISSING_GOAL_GRADE = `function missingGoal(index) {
+  return {
+    pass: false,
+    score: 0,
+    reason: "EVAL_CASE_MISSING_EXPECTED",
+    metadata: {
+      graderError: true,
+      errorCode: "EVAL_CASE_MISSING_EXPECTED",
+      stageflow: {
+        graderId: GRADERS[index].id,
+        graderType: GRADERS[index].type,
+        status: "error",
+        errorCode: "EVAL_CASE_MISSING_EXPECTED",
+        reason: "case has no input.goal or expected.outcome",
+      },
+    },
+  };
+}
+
+`;
+function isJudged(grader) {
+    return grader.type === "rubric" || grader.type === "goal_reached";
+}
+export function buildAssertionsSource(plan, graderModuleUrl) {
+    const agent = plan.target.kind === "agent";
+    const graders = plan.graders.map((g, index) => ({ index, grader: g }));
+    const exports = graders
+        .filter(({ grader }) => grader.type !== "rubric" && (agent || grader.type !== "goal_reached"))
+        .map(({ index }) => `export const ${assertionFnName(index)} = (output, context) => run(${index}, output, context);`);
+    return `import { readFileSync } from "node:fs";
+import { gradeDeterministic } from ${JSON.stringify(graderModuleUrl)};
+
+const GRADERS = ${JSON.stringify(plan.graders)};
+const BASE_CONTEXT = ${JSON.stringify({
+        evalId: plan.evalId,
+        runId: plan.evalRunId,
+        passThreshold: plan.passThreshold,
+        ...(plan.outputSchema !== undefined ? { outputSchema: plan.outputSchema } : {}),
+    })};
+const CASES = new Map(
+  JSON.parse(readFileSync(new URL(${JSON.stringify(DATASET_FILE)}, import.meta.url), "utf8")).map((c) => [c.id, c]),
+);
+
+${agent ? AGENT_TO_SUBJECT : STAGE_TO_SUBJECT}
+
+function engineError(index) {
+  return {
+    pass: false,
+    score: 0,
+    reason: "grader failed to run",
+    metadata: {
+      graderError: true,
+      errorCode: "EVAL_ENGINE_ERROR",
+      stageflow: { graderId: GRADERS[index].id, graderType: GRADERS[index].type, status: "error", errorCode: "EVAL_ENGINE_ERROR" },
+    },
+  };
+}
+
+${agent ? MISSING_GOAL_GRADE : ""}function run(index, output, context) {
+  try {
+    const grader = GRADERS[index];
+${agent ? '    if (grader.type === "goal_reached") return missingGoal(index);\n' : ""}    const testCase = CASES.get(context?.vars?.caseId);
+    if (!testCase) return engineError(index);
+    const repeat = context?.providerResponse?.metadata?.repeatIndex;
+    const grade = gradeDeterministic(grader, toSubject(output), testCase, {
+      ...BASE_CONTEXT,
+      repeat: Number.isInteger(repeat) ? repeat : 0,
+    });
+    if (grade.status === "error") {
+      return {
+        pass: false,
+        score: 0,
+        reason: grade.errorCode ?? "EVAL_ENGINE_ERROR",
+        metadata: { graderError: true, errorCode: grade.errorCode ?? "EVAL_ENGINE_ERROR", stageflow: grade },
+      };
+    }
+    return {
+      pass: grade.passed === true,
+      score: typeof grade.score === "number" ? grade.score : 0,
+      reason: grade.reason ?? (grade.passed === true ? "passed" : "failed"),
+      metadata: { stageflow: grade },
+    };
+  } catch {
+    return engineError(index);
+  }
+}
+
+${exports.join("\n")}
+`;
+}
+function judgeAssertion(grader, options, agent, testCase) {
+    const mapping = mapJudge(grader.judge);
+    if (mapping === null)
+        throw new EngineError("EVAL_INVALID", unsupportedJudgeMessage(grader.judge));
+    let rubric;
+    let prompt;
+    if (grader.type === "goal_reached") {
+        rubric = escapeTemplate(goalRubricText(caseGoal(testCase) ?? "", grader.guidance));
+        prompt = renderConversationRubricPrompt({ scale: "pass_fail", examples: [] });
+    }
+    else {
+        rubric = grader.rubric;
+        prompt = agent ? renderConversationRubricPrompt(grader) : renderRubricPrompt(grader);
+    }
+    return {
+        type: "llm-rubric",
+        value: rubric,
+        metric: grader.id,
+        weight: 1,
+        rubricPrompt: prompt,
+        provider: options.judgeProviderConfig === undefined
+            ? mapping.providerSpec
+            : { id: mapping.providerSpec, config: options.judgeProviderConfig },
+    };
+}
+function assertionsFor(plan, options, testCase) {
+    const agent = plan.target.kind === "agent";
+    return plan.graders.map((grader, index) => {
+        if (isJudged(grader) && !(grader.type === "goal_reached" && caseGoal(testCase) === undefined)) {
+            return judgeAssertion(grader, options, agent, testCase);
+        }
+        return {
+            type: "javascript",
+            value: `file://${ASSERTIONS_FILE}:${assertionFnName(index)}`,
+            metric: grader.id,
+            weight: 1,
+        };
+    });
+}
+export function buildConfig(plan, options) {
+    const agent = plan.target.kind === "agent";
+    const shared = agent ? undefined : assertionsFor(plan, options, plan.cases[0] ?? { input: {} });
+    return {
+        description: `Stageflow eval ${plan.evalId}`,
+        prompts: ["{{caseId}}"],
+        providers: [{ id: `file://${PROVIDER_FILE}` }],
+        extensions: [`file://${HOOKS_FILE}:afterEach`],
+        tests: plan.cases.map((c) => ({
+            description: c.id,
+            vars: {
+                caseId: c.id,
+                input: JSON.stringify(c.input),
+                expected: c.expected === undefined ? EMPTY_EXPECTED : JSON.stringify(c.expected),
+            },
+            assert: shared ?? assertionsFor(plan, options, c),
+        })),
+    };
+}
+export function generateFiles(plan, options) {
+    return {
+        [CONFIG_FILE]: `${JSON.stringify(buildConfig(plan, options), null, 2)}\n`,
+        [PROVIDER_FILE]: buildProviderSource(plan.evalRunId, plan.target.kind === "agent"),
+        [ASSERTIONS_FILE]: buildAssertionsSource(plan, options.graderModuleUrl),
+        [HOOKS_FILE]: buildHooksSource(plan.target.kind === "agent"),
+        [DATASET_FILE]: `${JSON.stringify(plan.cases, null, 2)}\n`,
+        [GUARD_FILENAME]: GUARD_SOURCE,
+    };
+}

package/dist/evals/engine/promptfoo/guard.d.ts

@@ -0,0 +1,10 @@
+export declare const GUARD_FILENAME = "guard.cjs";
+export declare const GUARD_SOURCE = "\"use strict\";\nconst http = require(\"node:http\");\nconst https = require(\"node:https\");\nconst { Writable } = require(\"node:stream\");\n\nconst BLOCKED = [\"promptfoo.app\", \"promptfoo.dev\"];\n\nfunction isBlocked(host) {\n  if (typeof host !== \"string\") return false;\n  const h = host.toLowerCase().replace(/\\.$/, \"\").replace(/^\\[|\\]$/g, \"\");\n  return BLOCKED.some((d) => h === d || h.endsWith(\".\" + d));\n}\n\nfunction hostOf(input) {\n  if (typeof input === \"string\") {\n    try { return new URL(input).hostname; } catch { return undefined; }\n  }\n  if (input instanceof URL) return input.hostname;\n  if (input && typeof input === \"object\") {\n    if (typeof input.hostname === \"string\") return input.hostname;\n    if (typeof input.host === \"string\") return input.host.replace(/:\\d+$/, \"\");\n    if (typeof input.url === \"string\") return hostOf(input.url);\n  }\n  return undefined;\n}\n\nfunction blockedError(host) {\n  const err = new Error(\"Stageflow blocked network access to \" + host);\n  err.code = \"ECONNREFUSED\";\n  return err;\n}\n\nfunction failedRequest(host) {\n  const req = new Writable({ write(_c, _e, cb) { cb(); } });\n  for (const name of [\"setHeader\", \"setTimeout\", \"setNoDelay\", \"setSocketKeepAlive\", \"flushHeaders\", \"abort\"]) {\n    req[name] = () => req;\n  }\n  process.nextTick(() => req.emit(\"error\", blockedError(host)));\n  return req;\n}\n\nfunction wrap(mod, name) {\n  const original = mod[name];\n  mod[name] = function (...args) {\n    const host = hostOf(args[0]);\n    if (isBlocked(host)) return failedRequest(host);\n    return original.apply(this, args);\n  };\n}\n\nfor (const mod of [http, https]) {\n  wrap(mod, \"request\");\n  wrap(mod, \"get\");\n}\n\nif (typeof globalThis.fetch === \"function\") {\n  const originalFetch = globalThis.fetch;\n  globalThis.fetch = function (input, init) {\n    const host = hostOf(input);\n    if (isBlocked(host)) return Promise.reject(blockedError(host));\n    return originalFetch.call(this, input, init);\n  };\n}\n\ntry { require(\"node:module\").syncBuiltinESMExports(); } catch {}\n";
+export declare function writeGuardFile(dir: string): string;
+export type GuardVia = "fetch" | "http" | "https";
+export type GuardProbeResult = {
+    blocked: boolean;
+    status?: number;
+    code?: string;
+};
+export declare function probeGuard(via: GuardVia, url: string): Promise<GuardProbeResult>;

package/dist/evals/engine/promptfoo/guard.js

@@ -0,0 +1,116 @@
+import { spawn } from "node:child_process";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+export const GUARD_FILENAME = "guard.cjs";
+export const GUARD_SOURCE = `"use strict";
+const http = require("node:http");
+const https = require("node:https");
+const { Writable } = require("node:stream");
+
+const BLOCKED = ["promptfoo.app", "promptfoo.dev"];
+
+function isBlocked(host) {
+  if (typeof host !== "string") return false;
+  const h = host.toLowerCase().replace(/\\.$/, "").replace(/^\\[|\\]$/g, "");
+  return BLOCKED.some((d) => h === d || h.endsWith("." + d));
+}
+
+function hostOf(input) {
+  if (typeof input === "string") {
+    try { return new URL(input).hostname; } catch { return undefined; }
+  }
+  if (input instanceof URL) return input.hostname;
+  if (input && typeof input === "object") {
+    if (typeof input.hostname === "string") return input.hostname;
+    if (typeof input.host === "string") return input.host.replace(/:\\d+$/, "");
+    if (typeof input.url === "string") return hostOf(input.url);
+  }
+  return undefined;
+}
+
+function blockedError(host) {
+  const err = new Error("Stageflow blocked network access to " + host);
+  err.code = "ECONNREFUSED";
+  return err;
+}
+
+function failedRequest(host) {
+  const req = new Writable({ write(_c, _e, cb) { cb(); } });
+  for (const name of ["setHeader", "setTimeout", "setNoDelay", "setSocketKeepAlive", "flushHeaders", "abort"]) {
+    req[name] = () => req;
+  }
+  process.nextTick(() => req.emit("error", blockedError(host)));
+  return req;
+}
+
+function wrap(mod, name) {
+  const original = mod[name];
+  mod[name] = function (...args) {
+    const host = hostOf(args[0]);
+    if (isBlocked(host)) return failedRequest(host);
+    return original.apply(this, args);
+  };
+}
+
+for (const mod of [http, https]) {
+  wrap(mod, "request");
+  wrap(mod, "get");
+}
+
+if (typeof globalThis.fetch === "function") {
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = function (input, init) {
+    const host = hostOf(input);
+    if (isBlocked(host)) return Promise.reject(blockedError(host));
+    return originalFetch.call(this, input, init);
+  };
+}
+
+try { require("node:module").syncBuiltinESMExports(); } catch {}
+`;
+export function writeGuardFile(dir) {
+    const file = path.join(dir, GUARD_FILENAME);
+    writeFileSync(file, GUARD_SOURCE);
+    return file;
+}
+const PROBE_SCRIPT = `
+const [via, url] = process.argv.slice(1);
+const out = (o) => { process.stdout.write(JSON.stringify(o)); process.exit(0); };
+const fail = (e) => out({ blocked: true, code: e && e.code });
+if (via === "fetch") {
+  fetch(url).then((r) => out({ blocked: false, status: r.status }), fail);
+} else {
+  const mod = require(via);
+  try {
+    const req = mod.request(url, (res) => { res.resume(); out({ blocked: false, status: res.statusCode }); });
+    req.on("error", fail);
+    req.end();
+  } catch (e) { fail(e); }
+}
+`;
+export async function probeGuard(via, url) {
+    const dir = mkdtempSync(path.join(tmpdir(), "sf-guard-"));
+    try {
+        const guard = writeGuardFile(dir);
+        return await new Promise((resolve, reject) => {
+            const child = spawn(process.execPath, ["--require", guard, "-e", PROBE_SCRIPT, via, url], { stdio: ["ignore", "pipe", "pipe"] });
+            let stdout = "";
+            let stderr = "";
+            child.stdout.on("data", (d) => (stdout += d));
+            child.stderr.on("data", (d) => (stderr += d));
+            child.on("error", reject);
+            child.on("close", () => {
+                try {
+                    resolve(JSON.parse(stdout));
+                }
+                catch {
+                    reject(new Error(`guard probe failed: ${stderr || stdout}`));
+                }
+            });
+        });
+    }
+    finally {
+        rmSync(dir, { recursive: true, force: true });
+    }
+}

package/dist/evals/engine/promptfoo/install.d.ts

@@ -0,0 +1,17 @@
+export type RunNpm = (args: string[], opts: {
+    cwd: string;
+    log: (line: string) => void;
+}) => Promise<{
+    code: number;
+}>;
+export type InstallResult = {
+    status: "installed" | "already_installed";
+    version: string;
+    dir: string;
+};
+export declare const realRunNpm: RunNpm;
+export declare function installPromptfoo(opts: {
+    home: string;
+    runNpm?: RunNpm;
+    log?: (line: string) => void;
+}): Promise<InstallResult>;

package/dist/evals/engine/promptfoo/install.js

@@ -0,0 +1,73 @@
+import { spawn } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { createInterface } from "node:readline";
+import { EngineError } from "./errors.js";
+import { engineDir, engineRoot, locatePromptfoo } from "./locate.js";
+import { assertNode } from "./nodeVersion.js";
+import { ENGINE_INSTALL_COMMAND, ENGINE_MARKER_FILE, PINNED_PROMPTFOO_VERSION, } from "./pinned.js";
+export const realRunNpm = (args, { cwd, log }) => new Promise((resolve, reject) => {
+    const nodeDir = path.dirname(process.execPath);
+    const npm = path.join(nodeDir, process.platform === "win32" ? "npm.cmd" : "npm");
+    const child = spawn(npm, args, {
+        cwd,
+        env: { ...process.env, PATH: `${nodeDir}${path.delimiter}${process.env.PATH ?? ""}` },
+        stdio: ["ignore", "pipe", "pipe"],
+        shell: process.platform === "win32",
+    });
+    for (const stream of [child.stdout, child.stderr]) {
+        createInterface({ input: stream }).on("line", log);
+    }
+    child.on("error", reject);
+    child.on("close", (code) => resolve({ code: code ?? 1 }));
+});
+function readIntegrity(dir) {
+    try {
+        const lock = JSON.parse(readFileSync(path.join(dir, "package-lock.json"), "utf8"));
+        return lock.packages?.["node_modules/promptfoo"]?.integrity ?? null;
+    }
+    catch {
+        return null;
+    }
+}
+export async function installPromptfoo(opts) {
+    const log = opts.log ?? (() => { });
+    const runNpm = opts.runNpm ?? realRunNpm;
+    assertNode();
+    const dir = engineDir(opts.home);
+    const existing = locatePromptfoo({ home: opts.home });
+    if (existing.installed) {
+        log(`promptfoo ${existing.version} is already installed`);
+        return { status: "already_installed", version: existing.version, dir };
+    }
+    const root = engineRoot(opts.home);
+    const rootExisted = existsSync(root);
+    rmSync(dir, { recursive: true, force: true });
+    mkdirSync(dir, { recursive: true });
+    try {
+        writeFileSync(path.join(dir, "package.json"), `${JSON.stringify({ name: "stageflow-eval-engine", private: true }, null, 2)}\n`);
+        const { code } = await runNpm(["install", "--no-audit", "--no-fund", `promptfoo@${PINNED_PROMPTFOO_VERSION}`], { cwd: dir, log });
+        if (code !== 0) {
+            throw new EngineError("EVAL_ENGINE_UNAVAILABLE", `npm install promptfoo@${PINNED_PROMPTFOO_VERSION} failed (exit ${code}). Retry with \`${ENGINE_INSTALL_COMMAND}\`.`);
+        }
+        const marker = {
+            version: PINNED_PROMPTFOO_VERSION,
+            node: process.versions.node,
+            installedAt: new Date().toISOString(),
+            integrity: readIntegrity(dir),
+        };
+        writeFileSync(path.join(dir, ENGINE_MARKER_FILE), `${JSON.stringify(marker, null, 2)}\n`);
+        const verified = locatePromptfoo({ home: opts.home });
+        if (!verified.installed) {
+            throw new EngineError("EVAL_ENGINE_UNAVAILABLE", verified.reason);
+        }
+    }
+    catch (err) {
+        rmSync(dir, { recursive: true, force: true });
+        if (!rootExisted)
+            rmSync(root, { recursive: true, force: true });
+        throw err;
+    }
+    log(`installed promptfoo ${PINNED_PROMPTFOO_VERSION}`);
+    return { status: "installed", version: PINNED_PROMPTFOO_VERSION, dir };
+}

package/dist/evals/engine/promptfoo/judgeProviders.d.ts

@@ -0,0 +1,2 @@
+import type { ResolvedGrader } from "../../types.js";
+export declare function assertJudgesSupported(graders: readonly ResolvedGrader[]): void;

package/dist/evals/engine/promptfoo/judgeProviders.js

@@ -0,0 +1,9 @@
+import { mapJudge, unsupportedJudgeMessage } from "../../judgeProviders.js";
+import { EngineError } from "./errors.js";
+export function assertJudgesSupported(graders) {
+    for (const grader of graders) {
+        if ((grader.type === "rubric" || grader.type === "goal_reached") && mapJudge(grader.judge) === null) {
+            throw new EngineError("EVAL_INVALID", unsupportedJudgeMessage(grader.judge));
+        }
+    }
+}

package/dist/evals/engine/promptfoo/judgeServices.d.ts

@@ -0,0 +1,6 @@
+import type { JudgeMapping } from "../../judgeProviders.js";
+import type { JudgePrice } from "./normalize.js";
+export type JudgeKeyResolver = (mapping: JudgeMapping, projectRoot: string) => Promise<string | undefined> | string | undefined;
+export type JudgePriceLoader = (modelIds: readonly string[], projectRoot: string) => Promise<Map<string, JudgePrice>>;
+export declare const defaultResolveJudgeKey: JudgeKeyResolver;
+export declare const defaultLoadJudgePrices: JudgePriceLoader;

package/dist/evals/engine/promptfoo/judgeServices.js

@@ -0,0 +1,41 @@
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { ModelRuntime } from "@earendil-works/pi-coding-agent";
+import { resolveCredentialBinding } from "../../../runtime/credentialBinding.js";
+import { readAuthKey } from "../../providerKeys.js";
+export const defaultResolveJudgeKey = (mapping, projectRoot) => {
+    const fromStore = readAuthKey(resolveCredentialBinding(projectRoot).authPath, mapping.stageflowProvider);
+    if (fromStore !== undefined)
+        return fromStore;
+    const fromEnv = process.env[mapping.envVar];
+    return fromEnv !== undefined && fromEnv.length > 0 ? fromEnv : undefined;
+};
+export const defaultLoadJudgePrices = async (modelIds, projectRoot) => {
+    const prices = new Map();
+    if (modelIds.length === 0)
+        return prices;
+    try {
+        const authPath = resolveCredentialBinding(projectRoot).authPath;
+        if (!existsSync(authPath))
+            return prices;
+        const runtime = await ModelRuntime.create({
+            authPath,
+            modelsPath: path.join(path.dirname(authPath), "models.json"),
+            refreshOnCreate: false,
+        });
+        for (const id of modelIds) {
+            const slash = id.indexOf("/");
+            const model = runtime
+                .getModels(id.slice(0, slash))
+                .find((candidate) => candidate.id === id.slice(slash + 1));
+            const cost = model?.cost;
+            if (cost !== undefined && typeof cost.input === "number" && typeof cost.output === "number") {
+                prices.set(id, { inputPerMillion: cost.input, outputPerMillion: cost.output });
+            }
+        }
+    }
+    catch {
+        return prices;
+    }
+    return prices;
+};

package/dist/evals/engine/promptfoo/locate.d.ts

@@ -0,0 +1,21 @@
+export type EngineMarker = {
+    version: string;
+    node: string;
+    installedAt: string;
+    integrity: string | null;
+};
+export type LocateResult = {
+    installed: true;
+    version: string;
+    binPath: string;
+    dir: string;
+} | {
+    installed: false;
+    reason: string;
+};
+export declare function engineRoot(home: string): string;
+export declare function engineDir(home: string, version?: string): string;
+export declare function engineBinPath(dir: string): string;
+export declare function locatePromptfoo(opts: {
+    home: string;
+}): LocateResult;