stageflow 0.40.0 → 0.44.0

package/dist/evals/read/api.js

@@ -0,0 +1,363 @@
+import { readFile, stat } from "node:fs/promises";
+import path from "node:path";
+import { contentHash } from "../../config/atomicWrite.js";
+import { resolveDefaultWriteRoot, resolveCatalogStartInput, } from "../../config/catalogRelativePath.js";
+import { listDatasetFiles, loadEvalCatalog } from "../../config/evalCatalog.js";
+import { resolveCatalogRoots, } from "../../config/resolveCatalogRoots.js";
+import { loadDatasetFile } from "../../config/loadDataset.js";
+import { evalStem, findDatasetFile, findEvalFile, manifestForWrite } from "../../config/evalWriteRoots.js";
+import { resolveEvalTarget } from "../../config/resolveEvalTarget.js";
+import { parseEvalOutcome, resolveEvalPaths } from "../../config/loadEval.js";
+import { evalFindingsFromIssues } from "../../config/validateEval.js";
+import { EVAL_RUN_STATUSES } from "../constants.js";
+import { PINNED_PROMPTFOO_VERSION } from "../engine/promptfoo/pinned.js";
+import { buildEvalPlan } from "../plan.js";
+import { liveAgentOf } from "./agentTargets.js";
+import { buildEvalListItems, costLastSevenDays } from "./list.js";
+import { resultKey } from "../statsTypes.js";
+import { regressionBanner } from "../listStatus.js";
+import { buildResultRows, RESULT_ROW_STATUSES } from "./results.js";
+import { activeRunOf, completedRunsNewestFirst, resultToWire, runHeadline } from "./runs.js";
+import { suggestionFor } from "./suggestions.js";
+export const DEFAULT_LIMIT = 50;
+export const MAX_LIMIT = 200;
+export const ENGINE_NAME = "promptfoo";
+export class EvalApiError extends Error {
+    status;
+    code;
+    extra;
+    constructor(status, code, message, extra = {}) {
+        super(message);
+        this.status = status;
+        this.code = code;
+        this.extra = extra;
+        this.name = "EvalApiError";
+    }
+}
+export function rel(projectRoot, abs) {
+    return path.relative(projectRoot, abs).split(path.sep).join("/");
+}
+export function badRequest(message, code = "invalid_request") {
+    return new EvalApiError(400, code, message);
+}
+export function engineCheck(status) {
+    if (!status.nodeOk) {
+        throw new EvalApiError(503, "EVAL_NODE_TOO_OLD", `Evals need a newer Node.js; found ${status.node}`);
+    }
+    if (!status.installed) {
+        throw new EvalApiError(503, "EVAL_ENGINE_UNAVAILABLE", `The eval engine is not installed. Run \`${status.installCommand}\`.`);
+    }
+}
+export async function findEvalOr404(projectRoot, id) {
+    const ref = await findEvalFile(projectRoot, id);
+    if (ref === undefined)
+        throw new EvalApiError(404, "EVAL_NOT_FOUND", `Eval not found: ${id}`);
+    return ref;
+}
+export async function runByIdOr404(store, id) {
+    const row = await store.getEvalRun(id);
+    if (row === null)
+        throw new EvalApiError(404, "EVAL_RUN_NOT_FOUND", `Eval run not found: ${id}`);
+    return row;
+}
+export function encodeOffset(offset) {
+    return Buffer.from(JSON.stringify({ offset }), "utf8").toString("base64url");
+}
+export function decodeOffset(cursor) {
+    if (cursor === null || cursor === undefined || cursor === "")
+        return 0;
+    try {
+        const parsed = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
+        if (typeof parsed.offset === "number" && Number.isInteger(parsed.offset) && parsed.offset >= 0)
+            return parsed.offset;
+    }
+    catch {
+        // fall through to the error below
+    }
+    throw badRequest("Invalid cursor", "invalid_cursor");
+}
+export async function buildEvalDetail(input) {
+    const { store, projectRoot, absPath } = input;
+    const manifest = await manifestForWrite(projectRoot);
+    const entries = await loadEvalCatalog(manifest, input.isKnownModel ? { isKnownModel: input.isKnownModel } : {});
+    const entry = entries.find((e) => path.resolve(e.absPath) === path.resolve(absPath));
+    const bytes = await readFile(absPath);
+    const id = entry?.outcome.ok ? entry.outcome.value.id : evalStem(path.basename(absPath));
+    let runsCount = 0;
+    let cursor;
+    do {
+        const page = await store.listEvalRuns({ projectRoot, evalId: id, limit: MAX_LIMIT, ...(cursor ? { cursor } : {}) });
+        runsCount += page.items.length;
+        cursor = page.nextCursor ?? undefined;
+    } while (cursor !== undefined);
+    const last = (await completedRunsNewestFirst(store, projectRoot, id, 1))[0];
+    let targetModel = null;
+    let agent;
+    if (entry?.outcome.ok) {
+        const file = entry.outcome.value.file;
+        if (file.target.kind === "agent") {
+            agent = await liveAgentOf(input.agents, projectRoot, file.target.agent);
+            targetModel = file.defaults?.model ?? agent?.model ?? null;
+        }
+        else if (file.target.kind === "internal_agent") {
+            targetModel = file.defaults?.model ?? null;
+        }
+        else {
+            const target = await resolveEvalTarget(file, entry.outcome.value.path, projectRoot);
+            if (target.ok)
+                targetModel = target.value.modelResolved;
+        }
+    }
+    return {
+        id,
+        path: rel(projectRoot, absPath),
+        file_hash: contentHash(bytes),
+        definition: entry?.outcome.ok ? entry.outcome.value.file : null,
+        yaml: bytes.toString("utf8"),
+        findings: entry?.findings ?? [],
+        runs_count: runsCount,
+        active_run: await activeRunOf(store, projectRoot, id),
+        last_run: last ? runHeadline(last) : null,
+        target_model: targetModel,
+        judge_default: entry?.outcome.ok ? (entry.outcome.value.file.defaults?.judge ?? null) : null,
+        ...(agent !== undefined ? { agent } : {}),
+    };
+}
+export async function planFindings(input) {
+    const built = await buildEvalPlan({
+        projectRoot: input.projectRoot,
+        evalPath: input.evalPath,
+        evalText: input.evalText,
+        ...(input.datasetCases !== undefined ? { datasetCases: input.datasetCases } : {}),
+        evalRunId: "er_validate",
+        engine: { name: ENGINE_NAME, version: PINNED_PROMPTFOO_VERSION, nodeVersion: process.versions.node },
+        ...(input.isKnownModel ? { isKnownModel: input.isKnownModel } : {}),
+        ...(input.agents ? { agents: input.agents } : {}),
+        ...(input.internalAgents ? { internalAgents: input.internalAgents } : {}),
+    });
+    return { ok: built.ok, findings: built.findings };
+}
+export async function validateEvalText(input) {
+    const { projectRoot, evalAbs, derived } = input;
+    const parsed = parseEvalOutcome(derived, evalAbs, projectRoot);
+    const checked = await planFindings({
+        projectRoot,
+        evalPath: evalAbs,
+        evalText: derived,
+        ...(input.datasetCases !== undefined ? { datasetCases: input.datasetCases } : {}),
+        ...(input.isKnownModel ? { isKnownModel: input.isKnownModel } : {}),
+        ...(input.agents ? { agents: input.agents } : {}),
+        ...(input.internalAgents ? { internalAgents: input.internalAgents } : {}),
+    });
+    const findings = checked.findings.length > 0 || parsed.ok
+        ? checked.findings
+        : evalFindingsFromIssues(rel(projectRoot, evalAbs), parsed.issues);
+    return { findings, parsed };
+}
+export async function listEvalRunsPage(store, input) {
+    const { projectRoot, evalId, status, stageRunId, subjectRef, limit, cursor } = input;
+    if (status !== undefined && !EVAL_RUN_STATUSES.includes(status)) {
+        throw badRequest(`"status" must be one of ${EVAL_RUN_STATUSES.join(", ")}`);
+    }
+    let page;
+    try {
+        page = await store.listEvalRuns({
+            ...(projectRoot !== undefined ? { projectRoot } : {}),
+            ...(evalId !== undefined ? { evalId } : {}),
+            ...(status !== undefined ? { status: status } : {}),
+            ...(stageRunId !== undefined ? { stageRunId } : {}),
+            ...(subjectRef !== undefined ? { subjectRef } : {}),
+            limit,
+            ...(cursor !== undefined ? { cursor } : {}),
+        });
+    }
+    catch (err) {
+        if (cursor !== undefined && err instanceof SyntaxError)
+            throw badRequest("Invalid cursor", "invalid_cursor");
+        throw err;
+    }
+    const items = await Promise.all(page.items.map(async (row) => {
+        const headline = {
+            ...runHeadline(row),
+            created_at: row.createdAt,
+            started_at: row.startedAt,
+            results_total: row.resultsTotal,
+            project_root: input.wireRootOf?.(row.projectRoot) ?? row.projectRoot,
+        };
+        if (stageRunId === undefined && subjectRef === undefined)
+            return headline;
+        const match = (await store.listEvalResults(row.evalRunId)).find((r) => stageRunId !== undefined ? r.stageRunId === stageRunId : r.subjectRef === subjectRef);
+        return {
+            ...headline,
+            ...(match ? { result: { case_id: match.caseId, repeat_index: match.repeatIndex } } : {}),
+        };
+    }));
+    return { items, next_cursor: page.nextCursor };
+}
+export async function buildResultsPage(store, id, input) {
+    const row = await runByIdOr404(store, id);
+    const { status, grader, q, tag, limit } = input;
+    if (status !== undefined && !RESULT_ROW_STATUSES.includes(status)) {
+        throw badRequest(`"status" must be one of ${RESULT_ROW_STATUSES.join(", ")}`);
+    }
+    const offset = decodeOffset(input.cursor);
+    const [cases, results, reviews] = await Promise.all([
+        store.getEvalCases(row.evalRunId),
+        store.listEvalResults(row.evalRunId),
+        store.getEvalReviews(row.evalRunId),
+    ]);
+    const { rows, counts } = buildResultRows({
+        targetKind: row.targetKind,
+        cases,
+        results,
+        reviews,
+        filter: {
+            ...(status !== undefined ? { status: status } : {}),
+            ...(grader !== undefined ? { grader } : {}),
+            ...(q !== undefined ? { q } : {}),
+            ...(tag !== undefined ? { tag } : {}),
+        },
+    });
+    const page = rows.slice(offset, offset + limit);
+    return {
+        items: page,
+        next_cursor: offset + limit < rows.length ? encodeOffset(offset + limit) : null,
+        counts,
+    };
+}
+export async function buildResultDetail(store, id, caseId) {
+    const row = await runByIdOr404(store, id);
+    const [cases, results, reviews, suggestions] = await Promise.all([
+        store.getEvalCases(row.evalRunId),
+        store.getEvalResultsForCase(row.evalRunId, caseId),
+        store.getEvalReviews(row.evalRunId),
+        store.listEvalSuggestions(row.evalRunId),
+    ]);
+    const caseRow = cases.find((x) => x.caseId === caseId);
+    if (!caseRow || results.length === 0) {
+        throw new EvalApiError(404, "EVAL_RUN_NOT_FOUND", `Eval result not found: ${id}/${caseId}`);
+    }
+    const reviewByKey = new Map(reviews.map((r) => [resultKey(r.caseId, r.repeatIndex), r]));
+    return {
+        case: { ...caseRow.case, position: caseRow.position },
+        repeats: [...results]
+            .sort((a, b) => a.repeatIndex - b.repeatIndex)
+            .map((r) => ({
+            ...resultToWire(r, reviewByKey.get(resultKey(r.caseId, r.repeatIndex)), row.targetKind === "agent" ? (row.targetRef.agent_id ?? "") : undefined),
+            suggestion: suggestionFor(suggestions, r.caseId, r.repeatIndex),
+        })),
+    };
+}
+async function exists(abs) {
+    try {
+        await stat(abs);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export async function loadEvalListItems(input) {
+    const { store, projectRoot } = input;
+    const wireRoot = input.wireRoot ?? projectRoot;
+    const manifest = await manifestForWrite(projectRoot);
+    const counts = new Map();
+    const entries = await loadEvalCatalog(manifest, {
+        ...(input.isKnownModel ? { isKnownModel: input.isKnownModel } : {}),
+        datasetReportFor: async (loaded) => {
+            const { datasetPath } = resolveEvalPaths(loaded.file, loaded.path);
+            if (!(await exists(datasetPath)))
+                return undefined;
+            const report = await loadDatasetFile(datasetPath, { root: projectRoot });
+            counts.set(datasetPath, report.cases.length);
+            return report;
+        },
+    });
+    const built = await buildEvalListItems({
+        store,
+        projectRoot,
+        entries,
+        datasetCases: (abs) => counts.get(abs) ?? 0,
+        ...(input.agents ? { agents: input.agents } : {}),
+    });
+    const items = built.items.map((item) => ({ ...item, project_root: wireRoot }));
+    return { items, banner: built.banner, bannerInputs: built.bannerInputs, manifest };
+}
+export async function evalScopeRoots(options, wire) {
+    if (wire !== undefined)
+        return [(await resolveCatalogStartInput(options, wire)).wireRoot];
+    const roots = await resolveCatalogRoots(options);
+    return roots.filter((r) => !r.read_only && r.available !== false);
+}
+export async function wireRootOf(options) {
+    const roots = await resolveCatalogRoots(options);
+    return (projectRoot) => roots.find((r) => r.path === projectRoot)?.project_root ?? projectRoot;
+}
+export async function loadAggregatedEvalList(input) {
+    const roots = await evalScopeRoots(input.options, input.wire);
+    const groups = [];
+    const bannerInputs = [];
+    let datasets = 0;
+    let cost = 0;
+    for (const root of roots) {
+        const loaded = await loadEvalListItems({
+            store: input.options.store,
+            projectRoot: root.path,
+            wireRoot: root.project_root,
+            ...(input.isKnownModel ? { isKnownModel: input.isKnownModel } : {}),
+            ...(input.agents ? { agents: input.agents } : {}),
+        });
+        groups.push({ root, items: loaded.items });
+        bannerInputs.push(...loaded.bannerInputs);
+        datasets += (await listDatasetFiles(loaded.manifest)).length;
+        cost += await costLastSevenDays(input.options.store, root.path, input.now);
+    }
+    return {
+        groups,
+        items: groups.flatMap((g) => g.items),
+        banner: regressionBanner(bannerInputs),
+        datasets,
+        cost,
+    };
+}
+function ambiguous(code, noun, id, roots) {
+    return new EvalApiError(409, code, `${noun} "${id}" exists in more than one project; pass project_root`, { project_roots: roots });
+}
+async function locate(options, wire, write, find, fail, missing) {
+    const notFound = () => new EvalApiError(404, fail.notFoundCode, `${fail.noun} not found: ${fail.id}`);
+    if (wire !== undefined) {
+        const root = write
+            ? (await resolveDefaultWriteRoot(options, wire)).wireRoot
+            : (await resolveCatalogStartInput(options, wire)).wireRoot;
+        const ref = await find(root.path);
+        if (ref === undefined && missing === "throw")
+            throw notFound();
+        return { root, ref };
+    }
+    const matches = [];
+    for (const root of await evalScopeRoots(options, undefined)) {
+        const ref = await find(root.path);
+        if (ref !== undefined)
+            matches.push({ root, ref });
+    }
+    if (matches.length > 1) {
+        throw ambiguous(fail.ambiguousCode, fail.noun, fail.id, matches.map((m) => m.root.project_root));
+    }
+    if (matches[0] !== undefined)
+        return matches[0];
+    if (missing === "throw")
+        throw notFound();
+    return { root: (await resolveDefaultWriteRoot(options)).wireRoot, ref: undefined };
+}
+export async function locateEval(options, wire, id, write = false) {
+    const { root, ref } = await locate(options, wire, write, (projectRoot) => findEvalFile(projectRoot, id), { ambiguousCode: "EVAL_AMBIGUOUS", notFoundCode: "EVAL_NOT_FOUND", noun: "Eval", id }, "throw");
+    return { root, projectRoot: root.path, ref: ref };
+}
+export async function locateEvalRoot(options, wire, id) {
+    const { root } = await locate(options, wire, false, (projectRoot) => findEvalFile(projectRoot, id), { ambiguousCode: "EVAL_AMBIGUOUS", notFoundCode: "EVAL_NOT_FOUND", noun: "Eval", id }, "default");
+    return root;
+}
+export async function locateDataset(options, wire, id, opts) {
+    const { root, ref } = await locate(options, wire, opts.write, (projectRoot) => findDatasetFile(projectRoot, id), { ambiguousCode: "DATASET_AMBIGUOUS", notFoundCode: "DATASET_NOT_FOUND", noun: "Dataset", id }, opts.missing);
+    return { root, projectRoot: root.path, ref };
+}

package/dist/evals/read/compare.d.ts

@@ -0,0 +1,4 @@
+import { type CompareResult } from "../compare.js";
+import type { EvalRunRow } from "../storeTypes.js";
+import { type ReadStore } from "./runs.js";
+export declare function buildCompare(store: ReadStore, a: EvalRunRow, b: EvalRunRow): Promise<CompareResult>;

package/dist/evals/read/compare.js

@@ -0,0 +1,39 @@
+import { compareRuns } from "../compare.js";
+import { toGradeRecords, toResultRecord } from "../statsAdapter.js";
+import { graderRefsOf } from "./runs.js";
+async function compareInput(store, row) {
+    const results = await store.listEvalResults(row.evalRunId);
+    const target = row.config
+        .target;
+    const isAgent = row.targetKind === "agent";
+    const definition = isAgent ? undefined : target?.stage_definition;
+    const agentDefinition = isAgent ? target?.definition : undefined;
+    const plain = (v) => (typeof v === "object" && v !== null && !Array.isArray(v) ? v : null);
+    return {
+        targetKind: row.targetKind === "agent" ? "agent" : "stage",
+        ...(isAgent
+            ? {
+                agentId: row.targetRef.agent_id ?? null,
+                agentRevision: typeof target?.revision === "number" ? target.revision : null,
+                agentDefinition: plain(agentDefinition),
+            }
+            : {}),
+        evalId: row.evalId,
+        runNumber: row.runNumber,
+        datasetHash: row.datasetHash,
+        stageVersionHash: row.targetVersionHash,
+        stageDefinition: typeof definition === "object" && definition !== null && !Array.isArray(definition)
+            ? definition
+            : null,
+        configComponents: row.configComponents,
+        results: results.map(toResultRecord),
+        grades: results.flatMap(toGradeRecords),
+        graders: graderRefsOf(row),
+        repeats: row.repeats,
+        passThreshold: row.passThreshold,
+    };
+}
+export async function buildCompare(store, a, b) {
+    const [left, right] = await Promise.all([compareInput(store, a), compareInput(store, b)]);
+    return compareRuns(left, right);
+}

package/dist/evals/read/datasets.d.ts

@@ -0,0 +1,105 @@
+import { type UploadMapping, type UploadPreviewRow } from "../../config/datasetImport.js";
+import type { DatasetCase, DatasetFormat, DatasetLoadResult } from "../../config/datasetTypes.js";
+import type { ValidationFinding } from "../../config/validateCatalog.js";
+import type { AgentCatalogPort } from "../../config/resolveAgentTarget.js";
+import type { ConversationMode } from "../types.js";
+export type InputField = {
+    name: string;
+    required: boolean;
+    type?: string;
+};
+export type DatasetTarget = {
+    pipeline_path?: string;
+    stage_id?: string;
+    file?: string;
+    agent_id?: string;
+};
+/** What an agent eval reads from a dataset: the effective conversation mode decides which scenario keys are required. */
+export type AgentDatasetUse = {
+    mode: ConversationMode;
+};
+export declare function agentInputFields(mode: ConversationMode): InputField[];
+/** The upload mapping targets an agent dataset cannot take from a column: scripts and tool mocks are JSON or JSONL only. */
+export declare const AGENT_UNSUPPORTED_MAPPING_TARGETS: readonly ["input.script", "input.tool_mocks"];
+export type DatasetListItem = {
+    project_root: string;
+    id: string;
+    path: string;
+    format: DatasetFormat | null;
+    cases: number;
+    hash: string;
+    editable: boolean;
+    used_by: string[];
+    findings_count: number;
+};
+export type DatasetQuery = {
+    offset: number;
+    limit: number;
+    tag?: string;
+    q?: string;
+    filter: "all" | "invalid" | "tagged";
+};
+export type DatasetDetail = {
+    id: string;
+    path: string;
+    format: DatasetFormat | null;
+    hash: string;
+    editable: boolean;
+    cases_total: number;
+    invalid_total: number;
+    tagged_total: number;
+    matched_total: number;
+    tags: Record<string, number>;
+    input_fields: InputField[];
+    cases: Array<DatasetCase & {
+        input_check: {
+            valid: boolean;
+            message?: string;
+        } | null;
+    }>;
+    findings: ValidationFinding[];
+    used_by: string[];
+};
+export declare function inputFieldsOf(schema: unknown): InputField[];
+export type ResolvedDatasetTarget = {
+    inputSchema: unknown;
+    stageId: string | undefined;
+    pipelineAbsPath: string | undefined;
+    agentId?: string;
+};
+export declare function resolveDatasetTarget(projectRoot: string, target: DatasetTarget, agents?: AgentCatalogPort): Promise<{
+    ok: true;
+    value: ResolvedDatasetTarget;
+} | {
+    ok: false;
+    message: string;
+}>;
+export declare function buildDatasetDetail(input: {
+    id: string;
+    loaded: DatasetLoadResult;
+    inputSchema: unknown;
+    hasEval: boolean;
+    usedBy: string[];
+    query: DatasetQuery;
+    agent?: AgentDatasetUse;
+}): DatasetDetail;
+export type PreviewInput = {
+    format: DatasetFormat;
+    content: string;
+    delimiter?: string;
+    mapping?: UploadMapping;
+    inputSchema?: unknown;
+    existingIds?: readonly string[];
+    /** Agent dataset: rows are checked as scenarios, in this mode unless the row maps `input.mode`. */
+    agent?: AgentDatasetUse;
+};
+export type PreviewResult = {
+    columns?: string[];
+    rows: UploadPreviewRow[];
+    findings: ValidationFinding[];
+    counts: {
+        ready: number;
+        skipped: number;
+    };
+};
+export declare function previewUpload(input: PreviewInput): PreviewResult;