stageflow 0.40.0 → 0.44.0

package/dist/evals/planInternalAgent.d.ts

@@ -0,0 +1,19 @@
+import type { DatasetCase } from "../config/datasetTypes.js";
+import { type LoadedEval } from "../config/loadEval.js";
+import type { ValidationFinding } from "../config/validateCatalog.js";
+import type { BuildEvalPlanInput, BuildEvalPlanResult, EvalOverrides } from "./plan.js";
+type Dataset = {
+    path: string;
+    contentHash: string;
+    format?: string;
+    cases: DatasetCase[];
+};
+export declare function planInternalAgent(args: {
+    input: BuildEvalPlanInput;
+    loaded: LoadedEval;
+    dataset: Dataset;
+    findings: ValidationFinding[];
+    overrides: EvalOverrides;
+    failedIds?: Set<string>;
+}): BuildEvalPlanResult;
+export {};

package/dist/evals/planInternalAgent.js

@@ -0,0 +1,205 @@
+import path from "node:path";
+import { checkDatasetAgainstAgent } from "../config/datasetAgentCheck.js";
+import { EVAL_AGENT_DEFAULT_CONCURRENCY, EVAL_AGENT_DEFAULT_MAX_COST_USD, EVAL_DEFAULT_PASS_THRESHOLD, EVAL_DEFAULT_REPEATS, resolveConversation, resolveEvalPaths, } from "../config/loadEval.js";
+import { EVAL_LARGE_RUN, EVAL_TOO_LARGE_RUN, evalFinding } from "../config/validateEval.js";
+import { PACKAGE_VERSION } from "../package-meta.js";
+import { sha256Hex } from "./canonical.js";
+import { EVAL_LIMITS, SIMULATOR_PROMPT_VERSION } from "./constants.js";
+import { buildComponents, buildSnapshot, computeFingerprint } from "./snapshot.js";
+function posix(projectRoot, abs) {
+    return path.relative(projectRoot, abs).split(path.sep).join("/");
+}
+function selectCases(cases, overrides, failedIds) {
+    if (overrides.case_ids !== undefined)
+        return cases.filter((c) => overrides.case_ids.includes(c.id));
+    if (overrides.tags !== undefined) {
+        const wanted = new Set(overrides.tags);
+        return cases.filter((c) => (c.tags ?? []).some((tag) => wanted.has(tag)));
+    }
+    if (failedIds !== undefined)
+        return cases.filter((c) => failedIds.has(c.id));
+    return [...cases];
+}
+export function planInternalAgent(args) {
+    const { input, loaded, dataset, overrides, failedIds } = args;
+    const findings = [...args.findings];
+    const fail = (extra) => ({ ok: false, code: "EVAL_INVALID", findings: [...findings, ...extra] });
+    if (loaded.file.target.kind !== "internal_agent")
+        return fail([]);
+    const profileId = loaded.file.target.profile;
+    const profiles = input.internalAgents ?? [];
+    const profile = profiles.find((item) => item.id === profileId);
+    if (!profile) {
+        return fail([
+            evalFinding(loaded.relPath, "error", profiles.length === 0 ? "eval.internal_agent_unavailable" : "eval.internal_agent_unknown", profiles.length === 0
+                ? "This Host is not showing internal agents, so this eval cannot run"
+                : `This Host has no internal agent "${profileId}"`),
+        ]);
+    }
+    if (overrides.case_ids !== undefined) {
+        const known = new Set(dataset.cases.map((c) => c.id));
+        const unknown = overrides.case_ids.filter((id) => !known.has(id));
+        if (unknown.length > 0) {
+            return fail([
+                evalFinding(loaded.relPath, "error", "eval.invalid_field", `Override "case_ids" names ${unknown.length} case(s) not in the dataset; first: ${unknown[0]}`),
+            ]);
+        }
+    }
+    const conversation = resolveConversation(loaded.file.conversation);
+    if (conversation.tools === "real") {
+        return fail([
+            evalFinding(loaded.relPath, "error", "eval.internal_agent_real_tools", "An internal agent eval cannot use real tools"),
+        ]);
+    }
+    const checks = new Map(checkDatasetAgainstAgent(dataset.cases, { mode: conversation.mode }).map((r) => [r.caseId, r.valid]));
+    const validCases = dataset.cases.filter((c) => checks.get(c.id) !== false);
+    const selected = selectCases(dataset.cases, overrides, failedIds);
+    const skippedCaseIds = selected.filter((c) => checks.get(c.id) === false).map((c) => c.id);
+    const runCases = selected.filter((c) => checks.get(c.id) !== false);
+    const positionOf = new Map(dataset.cases.map((c, i) => [c.id, i]));
+    const planCases = runCases.map((c) => ({
+        id: c.id,
+        position: positionOf.get(c.id) ?? 0,
+        ...(c.title !== undefined ? { title: c.title } : {}),
+        ...(c.goal !== undefined ? { goal: c.goal } : {}),
+        input: c.input,
+        ...(c.expected !== undefined ? { expected: c.expected } : {}),
+        tags: c.tags ?? [],
+        ...(c.metadata !== undefined ? { metadata: c.metadata } : {}),
+    }));
+    const repeats = loaded.file.defaults?.repeats ?? EVAL_DEFAULT_REPEATS;
+    const results = planCases.length * repeats;
+    if (planCases.length === 0) {
+        return fail([evalFinding(loaded.relPath, "error", "eval.no_valid_cases", "No valid cases are left to run after case selection and skipping")]);
+    }
+    if (results > EVAL_LIMITS.maxResultsPerRun) {
+        return fail([
+            evalFinding(loaded.relPath, "error", "eval.too_large_run", `${planCases.length} case(s) x ${repeats} repeat(s) = ${results} results, over the ${EVAL_TOO_LARGE_RUN} limit`),
+        ]);
+    }
+    if (results > EVAL_LARGE_RUN) {
+        findings.push(evalFinding(loaded.relPath, "warning", "eval.large_run", `${planCases.length} case(s) x ${repeats} repeat(s) = ${results} results, over ${EVAL_LARGE_RUN}`));
+    }
+    const defaults = loaded.file.defaults ?? {};
+    const passThreshold = defaults.pass_threshold ?? EVAL_DEFAULT_PASS_THRESHOLD;
+    const concurrency = defaults.concurrency ?? EVAL_AGENT_DEFAULT_CONCURRENCY;
+    const maxCostUsd = defaults.max_cost_usd ?? EVAL_AGENT_DEFAULT_MAX_COST_USD;
+    const judge = defaults.judge ?? "";
+    const model = defaults.model ?? "";
+    const playbookHash = sha256Hex(profile.playbook);
+    const definitionHash = sha256Hex(JSON.stringify({ playbook: profile.playbook, tools: profile.tools }));
+    const userModel = planCases.some((c) => c.input.mode === "simulated") ? (defaults.user_model ?? null) : null;
+    const paths = resolveEvalPaths(loaded.file, loaded.path);
+    const projectRoot = input.projectRoot;
+    const components = buildComponents({
+        stageVersionHash: definitionHash,
+        model,
+        datasetHash: dataset.contentHash,
+        caseIds: runCases.length < validCases.length ? runCases.map((c) => c.id) : null,
+        graders: loaded.graders,
+        judge,
+        repeats,
+        passThreshold,
+        engine: input.engine,
+        agent: {
+            revision: 0,
+            mode: conversation.mode,
+            max_turns: conversation.maxTurns,
+            turn_timeout_ms: conversation.turnTimeoutMs,
+            tools: conversation.tools,
+            approvals: conversation.approvals,
+            read_only_tools: conversation.readOnlyTools,
+            user_model: userModel,
+            simulator_prompt_version: SIMULATOR_PROMPT_VERSION,
+        },
+    });
+    const fingerprint = computeFingerprint(components);
+    const snapshotOverrides = {
+        repeats: overrides.repeats ?? null,
+        judge: overrides.judge ?? null,
+        model: overrides.model ?? null,
+        max_cost_usd: overrides.max_cost_usd ?? null,
+        case_ids: overrides.case_ids ?? null,
+        tags: overrides.tags ?? null,
+        only_failed_from: overrides.only_failed_from ?? null,
+    };
+    const snapshot = buildSnapshot({
+        evalId: loaded.id,
+        evalPath: loaded.relPath,
+        evalHash: loaded.fileHash,
+        evalDefinition: {
+            version: 1,
+            id: loaded.id,
+            target: { kind: "internal_agent", profile: profileId },
+            dataset: posix(projectRoot, paths.datasetPath),
+        },
+        overrides: snapshotOverrides,
+        target: {
+            profile_id: profileId,
+            title: profile.title,
+            playbook_hash: playbookHash,
+            tools: profile.tools,
+            model,
+        },
+        dataset: {
+            path: posix(projectRoot, paths.datasetPath),
+            hash: dataset.contentHash,
+            format: dataset.format ?? "jsonl",
+            case_count: dataset.cases.length,
+            cases_run: planCases.length,
+            cases_skipped: skippedCaseIds.length,
+            skipped_case_ids: skippedCaseIds,
+        },
+        graders: loaded.graders.map((g) => ({
+            id: g.id,
+            type: g.type,
+            hash: g.hash,
+            version_ordinal: input.graderOrdinals?.[g.id] ?? 1,
+            weight: g.weight,
+            must_pass: g.mustPass,
+        })),
+        settings: {
+            repeats,
+            pass_threshold: passThreshold,
+            judge,
+            judge_same_as_stage: false,
+            max_cost_usd: maxCostUsd,
+            concurrency,
+            suite_type: loaded.file.type ?? "capability",
+            conversation: {
+                mode: conversation.mode,
+                max_turns: conversation.maxTurns,
+                turn_timeout_ms: conversation.turnTimeoutMs,
+                tools: conversation.tools,
+                approvals: conversation.approvals,
+                read_only_tools: conversation.readOnlyTools,
+                real_tools_confirmed: false,
+                user_model: userModel ?? "",
+                user_model_from_judge: false,
+                simulator_prompt_version: SIMULATOR_PROMPT_VERSION,
+            },
+        },
+        engine: input.engine,
+        stageflowVersion: PACKAGE_VERSION,
+    });
+    const plan = {
+        evalRunId: input.evalRunId,
+        projectRoot,
+        evalId: loaded.id,
+        target: { kind: "internal_agent", profileId },
+        targetVersionHash: definitionHash,
+        stageModel: model,
+        task: {},
+        cases: planCases,
+        graders: loaded.graders,
+        repeats,
+        passThreshold,
+        concurrency,
+        maxCostUsd,
+        ...(model !== "" ? { modelOverride: model } : {}),
+        conversation,
+        userModel,
+        agent: { definitionHash, definition: { id: profileId, title: profile.title, tools: profile.tools } },
+    };
+    return { ok: true, plan, snapshot, fingerprint, components, skippedCaseIds, findings };
+}

package/dist/evals/providerKeys.d.ts

@@ -0,0 +1 @@
+export declare function readAuthKey(authPath: string, provider: string): string | undefined;

package/dist/evals/providerKeys.js

@@ -0,0 +1,21 @@
+import { existsSync, readFileSync } from "node:fs";
+export function readAuthKey(authPath, provider) {
+    try {
+        if (!existsSync(authPath))
+            return undefined;
+        const parsed = JSON.parse(readFileSync(authPath, "utf8"));
+        if (typeof parsed !== "object" || parsed === null)
+            return undefined;
+        const entry = parsed[provider];
+        if (typeof entry !== "object" || entry === null)
+            return undefined;
+        const { type, key } = entry;
+        if (type !== "api_key" || typeof key !== "string" || key.length === 0 || key.startsWith("!")) {
+            return undefined;
+        }
+        return key;
+    }
+    catch {
+        return undefined;
+    }
+}

package/dist/evals/read/agentTargets.d.ts

@@ -0,0 +1,37 @@
+import { type ToolClass } from "../../agents/toolRisk.js";
+import { type AgentCatalogPort, type AgentToolRisk } from "../../config/resolveAgentTarget.js";
+export type AgentToolWire = {
+    id: string;
+    risk: AgentToolRisk;
+    class: ToolClass;
+};
+export type AgentWire = {
+    id: string;
+    name: string;
+    model: string | null;
+    revision: number;
+    enabled: boolean;
+    archived: boolean;
+    tools: AgentToolWire[];
+    ask_first: string[];
+    issues: string[];
+};
+export type AgentTargetItem = {
+    id: string;
+    name: string;
+    model: string | null;
+    revision: number;
+    enabled: boolean;
+    tools: AgentToolWire[];
+    ask_first: string[];
+    evals_count: number;
+    issues: string[];
+    unavailable_reason: "archived" | "invalid" | null;
+};
+/** The live agent behind an eval's target, for the editor and warnings; null when it cannot be read. */
+export declare function liveAgentOf(agents: AgentCatalogPort | undefined, projectRoot: string, agentId: string): Promise<AgentWire | null>;
+/**
+ * Agents an eval can target. `AgentService.list` hides archived agents, so an archived agent shows up only while an
+ * eval still names it (greyed in the picker). Never reads eval sessions.
+ */
+export declare function listAgentTargets(agents: AgentCatalogPort, projectRoot: string): Promise<AgentTargetItem[]>;

package/dist/evals/read/agentTargets.js

@@ -0,0 +1,101 @@
+import { classifyTool } from "../../agents/toolRisk.js";
+import { isAgentError } from "../../agents/errors.js";
+import { listEvalFiles } from "../../config/evalCatalog.js";
+import { manifestForWrite } from "../../config/evalWriteRoots.js";
+import { loadEvalOutcome } from "../../config/loadEval.js";
+import { agentToolGrants } from "../../config/resolveAgentTarget.js";
+function toolsOf(detail) {
+    const { definition } = detail;
+    if (definition === null)
+        return [];
+    return agentToolGrants(definition).map((tool) => ({
+        id: tool.id,
+        risk: tool.risk,
+        class: classifyTool(definition, tool.id).class,
+    }));
+}
+function issueTexts(detail) {
+    return detail.agent.issues.map((issue) => (issue.path ? `${issue.path}: ${issue.message}` : issue.message));
+}
+function toWire(detail, archived) {
+    const { agent, definition } = detail;
+    return {
+        id: agent.id,
+        name: agent.name,
+        model: definition?.model.id ?? agent.model?.id ?? null,
+        revision: agent.revision,
+        enabled: agent.enabled,
+        archived,
+        tools: toolsOf(detail),
+        ask_first: definition?.approvals.ask_first ?? agent.askFirst,
+        issues: issueTexts(detail),
+    };
+}
+/** The live agent behind an eval's target, for the editor and warnings; null when it cannot be read. */
+export async function liveAgentOf(agents, projectRoot, agentId) {
+    if (!agents)
+        return null;
+    try {
+        const detail = await agents.get(projectRoot, agentId);
+        const listed = await agents.list(projectRoot);
+        return toWire(detail, !listed.some((agent) => agent.id === agentId));
+    }
+    catch (err) {
+        if (isAgentError(err))
+            return null;
+        throw err;
+    }
+}
+async function agentEvalCounts(projectRoot) {
+    const manifest = await manifestForWrite(projectRoot);
+    const counts = new Map();
+    for (const ref of await listEvalFiles(manifest)) {
+        const outcome = await loadEvalOutcome(ref.absPath, projectRoot);
+        if (!outcome.ok || outcome.value.file.target.kind !== "agent")
+            continue;
+        const id = outcome.value.file.target.agent;
+        counts.set(id, (counts.get(id) ?? 0) + 1);
+    }
+    return { counts, referenced: [...counts.keys()] };
+}
+/**
+ * Agents an eval can target. `AgentService.list` hides archived agents, so an archived agent shows up only while an
+ * eval still names it (greyed in the picker). Never reads eval sessions.
+ */
+export async function listAgentTargets(agents, projectRoot) {
+    const [listed, { counts, referenced }] = await Promise.all([agents.list(projectRoot), agentEvalCounts(projectRoot)]);
+    const items = [];
+    const seen = new Set();
+    const add = async (id, archived) => {
+        let detail;
+        try {
+            detail = await agents.get(projectRoot, id);
+        }
+        catch (err) {
+            if (isAgentError(err))
+                return;
+            throw err;
+        }
+        seen.add(id);
+        const wire = toWire(detail, archived);
+        items.push({
+            id: wire.id,
+            name: wire.name,
+            model: wire.model,
+            revision: wire.revision,
+            enabled: wire.enabled,
+            tools: wire.tools,
+            ask_first: wire.ask_first,
+            evals_count: counts.get(id) ?? 0,
+            issues: wire.issues,
+            unavailable_reason: archived ? "archived" : detail.definition === null || wire.issues.length > 0 ? "invalid" : null,
+        });
+    };
+    for (const agent of listed)
+        await add(agent.id, false);
+    for (const id of referenced) {
+        if (!seen.has(id))
+            await add(id, true);
+    }
+    return items.sort((a, b) => a.id.localeCompare(b.id));
+}

package/dist/evals/read/api.d.ts

@@ -0,0 +1,208 @@
+import { type CatalogRoot, type ResolveCatalogRootsOptions } from "../../config/resolveCatalogRoots.js";
+import type { AgentCatalogPort } from "../../config/resolveAgentTarget.js";
+import { type LoadedEval } from "../../config/loadEval.js";
+import type { ValidationFinding } from "../../config/validateCatalog.js";
+import type { RunStore } from "../../runstore/port.js";
+import type { EngineStatus } from "../engine/promptfoo/status.js";
+import { type AgentWire } from "./agentTargets.js";
+import { type EvalListItem } from "./list.js";
+import type { EvalRunRow } from "../storeTypes.js";
+import { type BannerEvalInput } from "../listStatus.js";
+import { runHeadline, type ActiveRun } from "./runs.js";
+export declare const DEFAULT_LIMIT = 50;
+export declare const MAX_LIMIT = 200;
+export declare const ENGINE_NAME = "promptfoo";
+export declare class EvalApiError extends Error {
+    readonly status: number;
+    readonly code: string;
+    readonly extra: Record<string, unknown>;
+    constructor(status: number, code: string, message: string, extra?: Record<string, unknown>);
+}
+export type IsKnownModel = (id: string) => Promise<boolean>;
+export declare function rel(projectRoot: string, abs: string): string;
+export declare function badRequest(message: string, code?: string): EvalApiError;
+export declare function engineCheck(status: EngineStatus): void;
+export declare function findEvalOr404(projectRoot: string, id: string): Promise<import("../../config/evalCatalog.js").CatalogFileRef>;
+export declare function runByIdOr404(store: RunStore, id: string): Promise<EvalRunRow>;
+export declare function encodeOffset(offset: number): string;
+export declare function decodeOffset(cursor: string | null | undefined): number;
+export type EvalDetailWire = {
+    id: string;
+    path: string;
+    file_hash: string;
+    definition: unknown;
+    yaml: string;
+    findings: ValidationFinding[];
+    runs_count: number;
+    active_run: ActiveRun | null;
+    last_run: ReturnType<typeof runHeadline> | null;
+    target_model: string | null;
+    judge_default: string | null;
+    agent?: AgentWire | null;
+};
+export declare function buildEvalDetail(input: {
+    store: RunStore;
+    projectRoot: string;
+    absPath: string;
+    isKnownModel?: IsKnownModel;
+    agents?: AgentCatalogPort;
+}): Promise<EvalDetailWire>;
+export declare function planFindings(input: {
+    projectRoot: string;
+    evalPath: string;
+    evalText: string;
+    datasetCases?: readonly unknown[];
+    isKnownModel?: IsKnownModel;
+    agents?: AgentCatalogPort;
+    internalAgents?: ReadonlyArray<{
+        id: string;
+        title: string;
+        playbook: string;
+        tools: string[];
+    }>;
+}): Promise<{
+    ok: boolean;
+    findings: ValidationFinding[];
+}>;
+export declare function validateEvalText(input: {
+    projectRoot: string;
+    evalAbs: string;
+    derived: string;
+    datasetCases?: readonly unknown[];
+    isKnownModel?: IsKnownModel;
+    agents?: AgentCatalogPort;
+    internalAgents?: ReadonlyArray<{
+        id: string;
+        title: string;
+        playbook: string;
+        tools: string[];
+    }>;
+}): Promise<{
+    findings: ValidationFinding[];
+    parsed: import("../../config/loadOutcome.js").LoadOutcome<LoadedEval>;
+}>;
+export declare function listEvalRunsPage(store: RunStore, input: {
+    projectRoot?: string;
+    wireRootOf?: (projectRoot: string) => string;
+    evalId?: string;
+    status?: string;
+    stageRunId?: string;
+    subjectRef?: string;
+    limit: number;
+    cursor?: string;
+}): Promise<{
+    items: {
+        eval_run_id: string;
+        eval_id: string;
+        run_number: number;
+        status: string;
+        passed: boolean | null;
+        pass_rate: number | null;
+        pass_rate_ci: {
+            low: number;
+            high: number;
+        } | null;
+        results_executed: number;
+        cost_total: number;
+        finished_at: string | null;
+        fingerprint: string;
+        label: string | null;
+        created_at: string;
+        started_at: string | null;
+        results_total: number;
+        project_root: string;
+    }[];
+    next_cursor: string | null;
+}>;
+export declare function buildResultsPage(store: RunStore, id: string, input: {
+    status?: string;
+    grader?: string;
+    q?: string;
+    tag?: string;
+    limit: number;
+    cursor?: string | null;
+}): Promise<{
+    items: import("./results.js").ResultRow[];
+    next_cursor: string | null;
+    counts: import("./results.js").ResultRowCounts;
+}>;
+export declare function buildResultDetail(store: RunStore, id: string, caseId: string): Promise<{
+    case: {
+        position: number;
+    };
+    repeats: {
+        case_id: string;
+        repeat_index: number;
+        status: string;
+        error_code: string | null;
+        error_message: string | null;
+        stage_run_id: string | null;
+        subject_ref: string | null;
+        turns: number | null;
+        tool_call_count: number | null;
+        agent?: {
+            id: string;
+            session_id: string | null;
+        } | null;
+        output: unknown;
+        latency_ms: number | null;
+        cost_usd: number | null;
+        tokens_in: number | null;
+        tokens_out: number | null;
+        score: number | null;
+        passed: boolean;
+        grades: import("./runs.js").WireGrade[];
+        budget_warnings: string[];
+        review: {
+            label: string | null;
+            note: string | null;
+        } | null;
+        tags: string[];
+        suggestion: import("./suggestions.js").WireSuggestion | null;
+    }[];
+}>;
+export declare function loadEvalListItems(input: {
+    store: RunStore;
+    projectRoot: string;
+    wireRoot?: string;
+    isKnownModel?: IsKnownModel;
+    agents?: AgentCatalogPort;
+}): Promise<{
+    items: EvalListItem[];
+    banner: import("../listStatus.js").RegressionBanner | null;
+    bannerInputs: BannerEvalInput[];
+    manifest: import("../../types/stageflowManifest.js").LoadedManifest;
+}>;
+export type RootsOptions = ResolveCatalogRootsOptions;
+export declare function evalScopeRoots(options: RootsOptions, wire: string | undefined): Promise<CatalogRoot[]>;
+export declare function wireRootOf(options: RootsOptions): Promise<(projectRoot: string) => string>;
+export declare function loadAggregatedEvalList(input: {
+    options: RootsOptions;
+    wire: string | undefined;
+    now: Date;
+    isKnownModel?: IsKnownModel;
+    agents?: AgentCatalogPort;
+}): Promise<{
+    groups: {
+        root: CatalogRoot;
+        items: EvalListItem[];
+    }[];
+    items: EvalListItem[];
+    banner: import("../listStatus.js").RegressionBanner | null;
+    datasets: number;
+    cost: number;
+}>;
+export declare function locateEval(options: RootsOptions, wire: string | undefined, id: string, write?: boolean): Promise<{
+    root: CatalogRoot;
+    projectRoot: string;
+    ref: NonNullable<import("../../config/evalCatalog.js").CatalogFileRef | undefined>;
+}>;
+export declare function locateEvalRoot(options: RootsOptions, wire: string | undefined, id: string): Promise<CatalogRoot>;
+export declare function locateDataset(options: RootsOptions, wire: string | undefined, id: string, opts: {
+    write: boolean;
+    missing: "throw" | "default";
+}): Promise<{
+    root: CatalogRoot;
+    projectRoot: string;
+    ref: import("../../config/evalCatalog.js").CatalogFileRef | undefined;
+}>;