stageflow 0.40.0 → 0.42.0

package/dist/evals/read/agentTargets.d.ts

@@ -0,0 +1,37 @@
+import { type ToolClass } from "../../agents/toolRisk.js";
+import { type AgentCatalogPort, type AgentToolRisk } from "../../config/resolveAgentTarget.js";
+export type AgentToolWire = {
+    id: string;
+    risk: AgentToolRisk;
+    class: ToolClass;
+};
+export type AgentWire = {
+    id: string;
+    name: string;
+    model: string | null;
+    revision: number;
+    enabled: boolean;
+    archived: boolean;
+    tools: AgentToolWire[];
+    ask_first: string[];
+    issues: string[];
+};
+export type AgentTargetItem = {
+    id: string;
+    name: string;
+    model: string | null;
+    revision: number;
+    enabled: boolean;
+    tools: AgentToolWire[];
+    ask_first: string[];
+    evals_count: number;
+    issues: string[];
+    unavailable_reason: "archived" | "invalid" | null;
+};
+/** The live agent behind an eval's target, for the editor and warnings; null when it cannot be read. */
+export declare function liveAgentOf(agents: AgentCatalogPort | undefined, projectRoot: string, agentId: string): Promise<AgentWire | null>;
+/**
+ * Agents an eval can target. `AgentService.list` hides archived agents, so an archived agent shows up only while an
+ * eval still names it (greyed in the picker). Never reads eval sessions.
+ */
+export declare function listAgentTargets(agents: AgentCatalogPort, projectRoot: string): Promise<AgentTargetItem[]>;

package/dist/evals/read/agentTargets.js

@@ -0,0 +1,101 @@
+import { classifyTool } from "../../agents/toolRisk.js";
+import { isAgentError } from "../../agents/errors.js";
+import { listEvalFiles } from "../../config/evalCatalog.js";
+import { manifestForWrite } from "../../config/evalWriteRoots.js";
+import { loadEvalOutcome } from "../../config/loadEval.js";
+import { agentToolGrants } from "../../config/resolveAgentTarget.js";
+function toolsOf(detail) {
+    const { definition } = detail;
+    if (definition === null)
+        return [];
+    return agentToolGrants(definition).map((tool) => ({
+        id: tool.id,
+        risk: tool.risk,
+        class: classifyTool(definition, tool.id).class,
+    }));
+}
+function issueTexts(detail) {
+    return detail.agent.issues.map((issue) => (issue.path ? `${issue.path}: ${issue.message}` : issue.message));
+}
+function toWire(detail, archived) {
+    const { agent, definition } = detail;
+    return {
+        id: agent.id,
+        name: agent.name,
+        model: definition?.model.id ?? agent.model?.id ?? null,
+        revision: agent.revision,
+        enabled: agent.enabled,
+        archived,
+        tools: toolsOf(detail),
+        ask_first: definition?.approvals.ask_first ?? agent.askFirst,
+        issues: issueTexts(detail),
+    };
+}
+/** The live agent behind an eval's target, for the editor and warnings; null when it cannot be read. */
+export async function liveAgentOf(agents, projectRoot, agentId) {
+    if (!agents)
+        return null;
+    try {
+        const detail = await agents.get(projectRoot, agentId);
+        const listed = await agents.list(projectRoot);
+        return toWire(detail, !listed.some((agent) => agent.id === agentId));
+    }
+    catch (err) {
+        if (isAgentError(err))
+            return null;
+        throw err;
+    }
+}
+async function agentEvalCounts(projectRoot) {
+    const manifest = await manifestForWrite(projectRoot);
+    const counts = new Map();
+    for (const ref of await listEvalFiles(manifest)) {
+        const outcome = await loadEvalOutcome(ref.absPath, projectRoot);
+        if (!outcome.ok || outcome.value.file.target.kind !== "agent")
+            continue;
+        const id = outcome.value.file.target.agent;
+        counts.set(id, (counts.get(id) ?? 0) + 1);
+    }
+    return { counts, referenced: [...counts.keys()] };
+}
+/**
+ * Agents an eval can target. `AgentService.list` hides archived agents, so an archived agent shows up only while an
+ * eval still names it (greyed in the picker). Never reads eval sessions.
+ */
+export async function listAgentTargets(agents, projectRoot) {
+    const [listed, { counts, referenced }] = await Promise.all([agents.list(projectRoot), agentEvalCounts(projectRoot)]);
+    const items = [];
+    const seen = new Set();
+    const add = async (id, archived) => {
+        let detail;
+        try {
+            detail = await agents.get(projectRoot, id);
+        }
+        catch (err) {
+            if (isAgentError(err))
+                return;
+            throw err;
+        }
+        seen.add(id);
+        const wire = toWire(detail, archived);
+        items.push({
+            id: wire.id,
+            name: wire.name,
+            model: wire.model,
+            revision: wire.revision,
+            enabled: wire.enabled,
+            tools: wire.tools,
+            ask_first: wire.ask_first,
+            evals_count: counts.get(id) ?? 0,
+            issues: wire.issues,
+            unavailable_reason: archived ? "archived" : detail.definition === null || wire.issues.length > 0 ? "invalid" : null,
+        });
+    };
+    for (const agent of listed)
+        await add(agent.id, false);
+    for (const id of referenced) {
+        if (!seen.has(id))
+            await add(id, true);
+    }
+    return items.sort((a, b) => a.id.localeCompare(b.id));
+}

package/dist/evals/read/api.d.ts

@@ -0,0 +1,196 @@
+import { type CatalogRoot, type ResolveCatalogRootsOptions } from "../../config/resolveCatalogRoots.js";
+import type { AgentCatalogPort } from "../../config/resolveAgentTarget.js";
+import { type LoadedEval } from "../../config/loadEval.js";
+import type { ValidationFinding } from "../../config/validateCatalog.js";
+import type { RunStore } from "../../runstore/port.js";
+import type { EngineStatus } from "../engine/promptfoo/status.js";
+import { type AgentWire } from "./agentTargets.js";
+import { type EvalListItem } from "./list.js";
+import type { EvalRunRow } from "../storeTypes.js";
+import { type BannerEvalInput } from "../listStatus.js";
+import { runHeadline, type ActiveRun } from "./runs.js";
+export declare const DEFAULT_LIMIT = 50;
+export declare const MAX_LIMIT = 200;
+export declare const ENGINE_NAME = "promptfoo";
+export declare class EvalApiError extends Error {
+    readonly status: number;
+    readonly code: string;
+    readonly extra: Record<string, unknown>;
+    constructor(status: number, code: string, message: string, extra?: Record<string, unknown>);
+}
+export type IsKnownModel = (id: string) => Promise<boolean>;
+export declare function rel(projectRoot: string, abs: string): string;
+export declare function badRequest(message: string, code?: string): EvalApiError;
+export declare function engineCheck(status: EngineStatus): void;
+export declare function findEvalOr404(projectRoot: string, id: string): Promise<import("../../config/evalCatalog.js").CatalogFileRef>;
+export declare function runByIdOr404(store: RunStore, id: string): Promise<EvalRunRow>;
+export declare function encodeOffset(offset: number): string;
+export declare function decodeOffset(cursor: string | null | undefined): number;
+export type EvalDetailWire = {
+    id: string;
+    path: string;
+    file_hash: string;
+    definition: unknown;
+    yaml: string;
+    findings: ValidationFinding[];
+    runs_count: number;
+    active_run: ActiveRun | null;
+    last_run: ReturnType<typeof runHeadline> | null;
+    target_model: string | null;
+    judge_default: string | null;
+    agent?: AgentWire | null;
+};
+export declare function buildEvalDetail(input: {
+    store: RunStore;
+    projectRoot: string;
+    absPath: string;
+    isKnownModel?: IsKnownModel;
+    agents?: AgentCatalogPort;
+}): Promise<EvalDetailWire>;
+export declare function planFindings(input: {
+    projectRoot: string;
+    evalPath: string;
+    evalText: string;
+    datasetCases?: readonly unknown[];
+    isKnownModel?: IsKnownModel;
+    agents?: AgentCatalogPort;
+}): Promise<{
+    ok: boolean;
+    findings: ValidationFinding[];
+}>;
+export declare function validateEvalText(input: {
+    projectRoot: string;
+    evalAbs: string;
+    derived: string;
+    datasetCases?: readonly unknown[];
+    isKnownModel?: IsKnownModel;
+    agents?: AgentCatalogPort;
+}): Promise<{
+    findings: ValidationFinding[];
+    parsed: import("../../config/loadOutcome.js").LoadOutcome<LoadedEval>;
+}>;
+export declare function listEvalRunsPage(store: RunStore, input: {
+    projectRoot?: string;
+    wireRootOf?: (projectRoot: string) => string;
+    evalId?: string;
+    status?: string;
+    stageRunId?: string;
+    subjectRef?: string;
+    limit: number;
+    cursor?: string;
+}): Promise<{
+    items: {
+        eval_run_id: string;
+        eval_id: string;
+        run_number: number;
+        status: string;
+        passed: boolean | null;
+        pass_rate: number | null;
+        pass_rate_ci: {
+            low: number;
+            high: number;
+        } | null;
+        results_executed: number;
+        cost_total: number;
+        finished_at: string | null;
+        fingerprint: string;
+        label: string | null;
+        created_at: string;
+        started_at: string | null;
+        results_total: number;
+        project_root: string;
+    }[];
+    next_cursor: string | null;
+}>;
+export declare function buildResultsPage(store: RunStore, id: string, input: {
+    status?: string;
+    grader?: string;
+    q?: string;
+    tag?: string;
+    limit: number;
+    cursor?: string | null;
+}): Promise<{
+    items: import("./results.js").ResultRow[];
+    next_cursor: string | null;
+    counts: import("./results.js").ResultRowCounts;
+}>;
+export declare function buildResultDetail(store: RunStore, id: string, caseId: string): Promise<{
+    case: {
+        position: number;
+    };
+    repeats: {
+        case_id: string;
+        repeat_index: number;
+        status: string;
+        error_code: string | null;
+        error_message: string | null;
+        stage_run_id: string | null;
+        subject_ref: string | null;
+        turns: number | null;
+        tool_call_count: number | null;
+        agent?: {
+            id: string;
+            session_id: string | null;
+        } | null;
+        output: unknown;
+        latency_ms: number | null;
+        cost_usd: number | null;
+        tokens_in: number | null;
+        tokens_out: number | null;
+        score: number | null;
+        passed: boolean;
+        grades: import("./runs.js").WireGrade[];
+        budget_warnings: string[];
+        review: {
+            label: string | null;
+            note: string | null;
+        } | null;
+        tags: string[];
+        suggestion: import("./suggestions.js").WireSuggestion | null;
+    }[];
+}>;
+export declare function loadEvalListItems(input: {
+    store: RunStore;
+    projectRoot: string;
+    wireRoot?: string;
+    isKnownModel?: IsKnownModel;
+    agents?: AgentCatalogPort;
+}): Promise<{
+    items: EvalListItem[];
+    banner: import("../listStatus.js").RegressionBanner | null;
+    bannerInputs: BannerEvalInput[];
+    manifest: import("../../types/stageflowManifest.js").LoadedManifest;
+}>;
+export type RootsOptions = ResolveCatalogRootsOptions;
+export declare function evalScopeRoots(options: RootsOptions, wire: string | undefined): Promise<CatalogRoot[]>;
+export declare function wireRootOf(options: RootsOptions): Promise<(projectRoot: string) => string>;
+export declare function loadAggregatedEvalList(input: {
+    options: RootsOptions;
+    wire: string | undefined;
+    now: Date;
+    isKnownModel?: IsKnownModel;
+    agents?: AgentCatalogPort;
+}): Promise<{
+    groups: {
+        root: CatalogRoot;
+        items: EvalListItem[];
+    }[];
+    items: EvalListItem[];
+    banner: import("../listStatus.js").RegressionBanner | null;
+    datasets: number;
+    cost: number;
+}>;
+export declare function locateEval(options: RootsOptions, wire: string | undefined, id: string, write?: boolean): Promise<{
+    root: CatalogRoot;
+    projectRoot: string;
+    ref: NonNullable<import("../../config/evalCatalog.js").CatalogFileRef | undefined>;
+}>;
+export declare function locateEvalRoot(options: RootsOptions, wire: string | undefined, id: string): Promise<CatalogRoot>;
+export declare function locateDataset(options: RootsOptions, wire: string | undefined, id: string, opts: {
+    write: boolean;
+    missing: "throw" | "default";
+}): Promise<{
+    root: CatalogRoot;
+    projectRoot: string;
+    ref: import("../../config/evalCatalog.js").CatalogFileRef | undefined;
+}>;

package/dist/evals/read/api.js

@@ -0,0 +1,358 @@
+import { readFile, stat } from "node:fs/promises";
+import path from "node:path";
+import { contentHash } from "../../config/atomicWrite.js";
+import { resolveDefaultWriteRoot, resolveCatalogStartInput, } from "../../config/catalogRelativePath.js";
+import { listDatasetFiles, loadEvalCatalog } from "../../config/evalCatalog.js";
+import { resolveCatalogRoots, } from "../../config/resolveCatalogRoots.js";
+import { loadDatasetFile } from "../../config/loadDataset.js";
+import { evalStem, findDatasetFile, findEvalFile, manifestForWrite } from "../../config/evalWriteRoots.js";
+import { resolveEvalTarget } from "../../config/resolveEvalTarget.js";
+import { parseEvalOutcome, resolveEvalPaths } from "../../config/loadEval.js";
+import { evalFindingsFromIssues } from "../../config/validateEval.js";
+import { EVAL_RUN_STATUSES } from "../constants.js";
+import { PINNED_PROMPTFOO_VERSION } from "../engine/promptfoo/pinned.js";
+import { buildEvalPlan } from "../plan.js";
+import { liveAgentOf } from "./agentTargets.js";
+import { buildEvalListItems, costLastSevenDays } from "./list.js";
+import { resultKey } from "../statsTypes.js";
+import { regressionBanner } from "../listStatus.js";
+import { buildResultRows, RESULT_ROW_STATUSES } from "./results.js";
+import { activeRunOf, completedRunsNewestFirst, resultToWire, runHeadline } from "./runs.js";
+import { suggestionFor } from "./suggestions.js";
+export const DEFAULT_LIMIT = 50;
+export const MAX_LIMIT = 200;
+export const ENGINE_NAME = "promptfoo";
+export class EvalApiError extends Error {
+    status;
+    code;
+    extra;
+    constructor(status, code, message, extra = {}) {
+        super(message);
+        this.status = status;
+        this.code = code;
+        this.extra = extra;
+        this.name = "EvalApiError";
+    }
+}
+export function rel(projectRoot, abs) {
+    return path.relative(projectRoot, abs).split(path.sep).join("/");
+}
+export function badRequest(message, code = "invalid_request") {
+    return new EvalApiError(400, code, message);
+}
+export function engineCheck(status) {
+    if (!status.nodeOk) {
+        throw new EvalApiError(503, "EVAL_NODE_TOO_OLD", `Evals need a newer Node.js; found ${status.node}`);
+    }
+    if (!status.installed) {
+        throw new EvalApiError(503, "EVAL_ENGINE_UNAVAILABLE", `The eval engine is not installed. Run \`${status.installCommand}\`.`);
+    }
+}
+export async function findEvalOr404(projectRoot, id) {
+    const ref = await findEvalFile(projectRoot, id);
+    if (ref === undefined)
+        throw new EvalApiError(404, "EVAL_NOT_FOUND", `Eval not found: ${id}`);
+    return ref;
+}
+export async function runByIdOr404(store, id) {
+    const row = await store.getEvalRun(id);
+    if (row === null)
+        throw new EvalApiError(404, "EVAL_RUN_NOT_FOUND", `Eval run not found: ${id}`);
+    return row;
+}
+export function encodeOffset(offset) {
+    return Buffer.from(JSON.stringify({ offset }), "utf8").toString("base64url");
+}
+export function decodeOffset(cursor) {
+    if (cursor === null || cursor === undefined || cursor === "")
+        return 0;
+    try {
+        const parsed = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
+        if (typeof parsed.offset === "number" && Number.isInteger(parsed.offset) && parsed.offset >= 0)
+            return parsed.offset;
+    }
+    catch {
+        // fall through to the error below
+    }
+    throw badRequest("Invalid cursor", "invalid_cursor");
+}
+export async function buildEvalDetail(input) {
+    const { store, projectRoot, absPath } = input;
+    const manifest = await manifestForWrite(projectRoot);
+    const entries = await loadEvalCatalog(manifest, input.isKnownModel ? { isKnownModel: input.isKnownModel } : {});
+    const entry = entries.find((e) => path.resolve(e.absPath) === path.resolve(absPath));
+    const bytes = await readFile(absPath);
+    const id = entry?.outcome.ok ? entry.outcome.value.id : evalStem(path.basename(absPath));
+    let runsCount = 0;
+    let cursor;
+    do {
+        const page = await store.listEvalRuns({ projectRoot, evalId: id, limit: MAX_LIMIT, ...(cursor ? { cursor } : {}) });
+        runsCount += page.items.length;
+        cursor = page.nextCursor ?? undefined;
+    } while (cursor !== undefined);
+    const last = (await completedRunsNewestFirst(store, projectRoot, id, 1))[0];
+    let targetModel = null;
+    let agent;
+    if (entry?.outcome.ok) {
+        const file = entry.outcome.value.file;
+        if (file.target.kind === "agent") {
+            agent = await liveAgentOf(input.agents, projectRoot, file.target.agent);
+            targetModel = file.defaults?.model ?? agent?.model ?? null;
+        }
+        else {
+            const target = await resolveEvalTarget(file, entry.outcome.value.path, projectRoot);
+            if (target.ok)
+                targetModel = target.value.modelResolved;
+        }
+    }
+    return {
+        id,
+        path: rel(projectRoot, absPath),
+        file_hash: contentHash(bytes),
+        definition: entry?.outcome.ok ? entry.outcome.value.file : null,
+        yaml: bytes.toString("utf8"),
+        findings: entry?.findings ?? [],
+        runs_count: runsCount,
+        active_run: await activeRunOf(store, projectRoot, id),
+        last_run: last ? runHeadline(last) : null,
+        target_model: targetModel,
+        judge_default: entry?.outcome.ok ? (entry.outcome.value.file.defaults?.judge ?? null) : null,
+        ...(agent !== undefined ? { agent } : {}),
+    };
+}
+export async function planFindings(input) {
+    const built = await buildEvalPlan({
+        projectRoot: input.projectRoot,
+        evalPath: input.evalPath,
+        evalText: input.evalText,
+        ...(input.datasetCases !== undefined ? { datasetCases: input.datasetCases } : {}),
+        evalRunId: "er_validate",
+        engine: { name: ENGINE_NAME, version: PINNED_PROMPTFOO_VERSION, nodeVersion: process.versions.node },
+        ...(input.isKnownModel ? { isKnownModel: input.isKnownModel } : {}),
+        ...(input.agents ? { agents: input.agents } : {}),
+    });
+    return { ok: built.ok, findings: built.findings };
+}
+export async function validateEvalText(input) {
+    const { projectRoot, evalAbs, derived } = input;
+    const parsed = parseEvalOutcome(derived, evalAbs, projectRoot);
+    const checked = await planFindings({
+        projectRoot,
+        evalPath: evalAbs,
+        evalText: derived,
+        ...(input.datasetCases !== undefined ? { datasetCases: input.datasetCases } : {}),
+        ...(input.isKnownModel ? { isKnownModel: input.isKnownModel } : {}),
+        ...(input.agents ? { agents: input.agents } : {}),
+    });
+    const findings = checked.findings.length > 0 || parsed.ok
+        ? checked.findings
+        : evalFindingsFromIssues(rel(projectRoot, evalAbs), parsed.issues);
+    return { findings, parsed };
+}
+export async function listEvalRunsPage(store, input) {
+    const { projectRoot, evalId, status, stageRunId, subjectRef, limit, cursor } = input;
+    if (status !== undefined && !EVAL_RUN_STATUSES.includes(status)) {
+        throw badRequest(`"status" must be one of ${EVAL_RUN_STATUSES.join(", ")}`);
+    }
+    let page;
+    try {
+        page = await store.listEvalRuns({
+            ...(projectRoot !== undefined ? { projectRoot } : {}),
+            ...(evalId !== undefined ? { evalId } : {}),
+            ...(status !== undefined ? { status: status } : {}),
+            ...(stageRunId !== undefined ? { stageRunId } : {}),
+            ...(subjectRef !== undefined ? { subjectRef } : {}),
+            limit,
+            ...(cursor !== undefined ? { cursor } : {}),
+        });
+    }
+    catch (err) {
+        if (cursor !== undefined && err instanceof SyntaxError)
+            throw badRequest("Invalid cursor", "invalid_cursor");
+        throw err;
+    }
+    const items = await Promise.all(page.items.map(async (row) => {
+        const headline = {
+            ...runHeadline(row),
+            created_at: row.createdAt,
+            started_at: row.startedAt,
+            results_total: row.resultsTotal,
+            project_root: input.wireRootOf?.(row.projectRoot) ?? row.projectRoot,
+        };
+        if (stageRunId === undefined && subjectRef === undefined)
+            return headline;
+        const match = (await store.listEvalResults(row.evalRunId)).find((r) => stageRunId !== undefined ? r.stageRunId === stageRunId : r.subjectRef === subjectRef);
+        return {
+            ...headline,
+            ...(match ? { result: { case_id: match.caseId, repeat_index: match.repeatIndex } } : {}),
+        };
+    }));
+    return { items, next_cursor: page.nextCursor };
+}
+export async function buildResultsPage(store, id, input) {
+    const row = await runByIdOr404(store, id);
+    const { status, grader, q, tag, limit } = input;
+    if (status !== undefined && !RESULT_ROW_STATUSES.includes(status)) {
+        throw badRequest(`"status" must be one of ${RESULT_ROW_STATUSES.join(", ")}`);
+    }
+    const offset = decodeOffset(input.cursor);
+    const [cases, results, reviews] = await Promise.all([
+        store.getEvalCases(row.evalRunId),
+        store.listEvalResults(row.evalRunId),
+        store.getEvalReviews(row.evalRunId),
+    ]);
+    const { rows, counts } = buildResultRows({
+        targetKind: row.targetKind,
+        cases,
+        results,
+        reviews,
+        filter: {
+            ...(status !== undefined ? { status: status } : {}),
+            ...(grader !== undefined ? { grader } : {}),
+            ...(q !== undefined ? { q } : {}),
+            ...(tag !== undefined ? { tag } : {}),
+        },
+    });
+    const page = rows.slice(offset, offset + limit);
+    return {
+        items: page,
+        next_cursor: offset + limit < rows.length ? encodeOffset(offset + limit) : null,
+        counts,
+    };
+}
+export async function buildResultDetail(store, id, caseId) {
+    const row = await runByIdOr404(store, id);
+    const [cases, results, reviews, suggestions] = await Promise.all([
+        store.getEvalCases(row.evalRunId),
+        store.getEvalResultsForCase(row.evalRunId, caseId),
+        store.getEvalReviews(row.evalRunId),
+        store.listEvalSuggestions(row.evalRunId),
+    ]);
+    const caseRow = cases.find((x) => x.caseId === caseId);
+    if (!caseRow || results.length === 0) {
+        throw new EvalApiError(404, "EVAL_RUN_NOT_FOUND", `Eval result not found: ${id}/${caseId}`);
+    }
+    const reviewByKey = new Map(reviews.map((r) => [resultKey(r.caseId, r.repeatIndex), r]));
+    return {
+        case: { ...caseRow.case, position: caseRow.position },
+        repeats: [...results]
+            .sort((a, b) => a.repeatIndex - b.repeatIndex)
+            .map((r) => ({
+            ...resultToWire(r, reviewByKey.get(resultKey(r.caseId, r.repeatIndex)), row.targetKind === "agent" ? (row.targetRef.agent_id ?? "") : undefined),
+            suggestion: suggestionFor(suggestions, r.caseId, r.repeatIndex),
+        })),
+    };
+}
+async function exists(abs) {
+    try {
+        await stat(abs);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export async function loadEvalListItems(input) {
+    const { store, projectRoot } = input;
+    const wireRoot = input.wireRoot ?? projectRoot;
+    const manifest = await manifestForWrite(projectRoot);
+    const counts = new Map();
+    const entries = await loadEvalCatalog(manifest, {
+        ...(input.isKnownModel ? { isKnownModel: input.isKnownModel } : {}),
+        datasetReportFor: async (loaded) => {
+            const { datasetPath } = resolveEvalPaths(loaded.file, loaded.path);
+            if (!(await exists(datasetPath)))
+                return undefined;
+            const report = await loadDatasetFile(datasetPath, { root: projectRoot });
+            counts.set(datasetPath, report.cases.length);
+            return report;
+        },
+    });
+    const built = await buildEvalListItems({
+        store,
+        projectRoot,
+        entries,
+        datasetCases: (abs) => counts.get(abs) ?? 0,
+        ...(input.agents ? { agents: input.agents } : {}),
+    });
+    const items = built.items.map((item) => ({ ...item, project_root: wireRoot }));
+    return { items, banner: built.banner, bannerInputs: built.bannerInputs, manifest };
+}
+export async function evalScopeRoots(options, wire) {
+    if (wire !== undefined)
+        return [(await resolveCatalogStartInput(options, wire)).wireRoot];
+    const roots = await resolveCatalogRoots(options);
+    return roots.filter((r) => !r.read_only && r.available !== false);
+}
+export async function wireRootOf(options) {
+    const roots = await resolveCatalogRoots(options);
+    return (projectRoot) => roots.find((r) => r.path === projectRoot)?.project_root ?? projectRoot;
+}
+export async function loadAggregatedEvalList(input) {
+    const roots = await evalScopeRoots(input.options, input.wire);
+    const groups = [];
+    const bannerInputs = [];
+    let datasets = 0;
+    let cost = 0;
+    for (const root of roots) {
+        const loaded = await loadEvalListItems({
+            store: input.options.store,
+            projectRoot: root.path,
+            wireRoot: root.project_root,
+            ...(input.isKnownModel ? { isKnownModel: input.isKnownModel } : {}),
+            ...(input.agents ? { agents: input.agents } : {}),
+        });
+        groups.push({ root, items: loaded.items });
+        bannerInputs.push(...loaded.bannerInputs);
+        datasets += (await listDatasetFiles(loaded.manifest)).length;
+        cost += await costLastSevenDays(input.options.store, root.path, input.now);
+    }
+    return {
+        groups,
+        items: groups.flatMap((g) => g.items),
+        banner: regressionBanner(bannerInputs),
+        datasets,
+        cost,
+    };
+}
+function ambiguous(code, noun, id, roots) {
+    return new EvalApiError(409, code, `${noun} "${id}" exists in more than one project; pass project_root`, { project_roots: roots });
+}
+async function locate(options, wire, write, find, fail, missing) {
+    const notFound = () => new EvalApiError(404, fail.notFoundCode, `${fail.noun} not found: ${fail.id}`);
+    if (wire !== undefined) {
+        const root = write
+            ? (await resolveDefaultWriteRoot(options, wire)).wireRoot
+            : (await resolveCatalogStartInput(options, wire)).wireRoot;
+        const ref = await find(root.path);
+        if (ref === undefined && missing === "throw")
+            throw notFound();
+        return { root, ref };
+    }
+    const matches = [];
+    for (const root of await evalScopeRoots(options, undefined)) {
+        const ref = await find(root.path);
+        if (ref !== undefined)
+            matches.push({ root, ref });
+    }
+    if (matches.length > 1) {
+        throw ambiguous(fail.ambiguousCode, fail.noun, fail.id, matches.map((m) => m.root.project_root));
+    }
+    if (matches[0] !== undefined)
+        return matches[0];
+    if (missing === "throw")
+        throw notFound();
+    return { root: (await resolveDefaultWriteRoot(options)).wireRoot, ref: undefined };
+}
+export async function locateEval(options, wire, id, write = false) {
+    const { root, ref } = await locate(options, wire, write, (projectRoot) => findEvalFile(projectRoot, id), { ambiguousCode: "EVAL_AMBIGUOUS", notFoundCode: "EVAL_NOT_FOUND", noun: "Eval", id }, "throw");
+    return { root, projectRoot: root.path, ref: ref };
+}
+export async function locateEvalRoot(options, wire, id) {
+    const { root } = await locate(options, wire, false, (projectRoot) => findEvalFile(projectRoot, id), { ambiguousCode: "EVAL_AMBIGUOUS", notFoundCode: "EVAL_NOT_FOUND", noun: "Eval", id }, "default");
+    return root;
+}
+export async function locateDataset(options, wire, id, opts) {
+    const { root, ref } = await locate(options, wire, opts.write, (projectRoot) => findDatasetFile(projectRoot, id), { ambiguousCode: "DATASET_AMBIGUOUS", notFoundCode: "DATASET_NOT_FOUND", noun: "Dataset", id }, opts.missing);
+    return { root, projectRoot: root.path, ref };
+}