stageflow 0.40.0 → 0.42.0

package/dist/evals/read/list.js

@@ -0,0 +1,173 @@
+import path from "node:path";
+import { EVAL_DEFAULT_REPEATS, resolveConversation, resolveEvalPaths } from "../../config/loadEval.js";
+import { evalStem } from "../../config/evalWriteRoots.js";
+import { listStatus, regressionBanner, } from "../listStatus.js";
+import { activeRunOf, completedRunsNewestFirst, runHeadline, statsOfRun, } from "./runs.js";
+const TREND_RUNS = 7;
+function posix(projectRoot, abs) {
+    return path.relative(projectRoot, abs).split(path.sep).join("/");
+}
+function pipelineStem(pipelinePath) {
+    return path.basename(pipelinePath).replace(/\.pipeline\.ya?ml$/, "");
+}
+function str(value) {
+    return typeof value === "string" && value !== "" ? value : null;
+}
+async function agentListTarget(agentId, last, agents, projectRoot) {
+    const snap = last && last.targetKind === "agent" ? last.config.target : undefined;
+    let name = null;
+    let model = null;
+    let revision = null;
+    if (snap) {
+        const def = (snap.definition ?? {});
+        name = str(def.name);
+        model = str(snap.model);
+        revision = typeof snap.revision === "number" ? snap.revision : null;
+    }
+    else if (agents) {
+        try {
+            const detail = await agents.get(projectRoot, agentId);
+            name = str(detail.agent.name);
+            model = str(detail.agent.model?.id);
+            revision = detail.agent.revision;
+        }
+        catch {
+            // the agent may be deleted or unreadable: fall back to the id
+        }
+    }
+    const agentName = name ?? agentId;
+    return { kind: "agent", agent_id: agentId, agent_name: agentName, model, revision, label: agentName };
+}
+export function filterEvalListItems(items, filter) {
+    return items.filter((item) => {
+        if (filter.target_kind && item.target.kind !== filter.target_kind)
+            return false;
+        if (filter.agent && !(item.target.kind === "agent" && item.target.agent_id === filter.agent))
+            return false;
+        return true;
+    });
+}
+async function listRunSummary(store, row) {
+    const stats = statsOfRun(row, await store.listEvalResults(row.evalRunId));
+    return {
+        runNumber: row.runNumber,
+        passed: row.passed === true,
+        passRate: stats.pass_rate,
+        finishedAt: row.finishedAt,
+        graders: stats.by_grader.map((g) => ({ graderId: g.grader_id, hash: g.hash, passRate: g.pass_rate })),
+    };
+}
+export async function buildEvalListItems(input) {
+    const { store, projectRoot } = input;
+    const items = [];
+    const bannerInputs = [];
+    for (const entry of input.entries) {
+        const findingCount = entry.findings.length;
+        const valid = !entry.findings.some((f) => f.severity === "error");
+        if (!entry.outcome.ok) {
+            const id = evalStem(path.basename(entry.absPath));
+            items.push({
+                id,
+                name: id,
+                description: null,
+                type: "capability",
+                path: entry.relPath,
+                target: { kind: "stage", label: "Unknown target" },
+                dataset_path: "",
+                dataset_cases: 0,
+                repeats: EVAL_DEFAULT_REPEATS,
+                grader_count: 0,
+                rubric_count: 0,
+                valid: false,
+                finding_count: findingCount,
+                status: "never_run",
+                active_run: null,
+                last_run: null,
+                last_run_delta_pts: null,
+                trend: [],
+            });
+            continue;
+        }
+        const loaded = entry.outcome.value;
+        const { file } = loaded;
+        const paths = resolveEvalPaths(file, loaded.path);
+        const type = file.type ?? "capability";
+        const stageId = file.target.kind === "stage" ? file.target.stage : undefined;
+        const completed = await completedRunsNewestFirst(store, projectRoot, loaded.id);
+        const last = completed[0];
+        const previous = completed[1];
+        const lastSummary = last ? await listRunSummary(store, last) : null;
+        const previousSummary = previous ? await listRunSummary(store, previous) : null;
+        const status = listStatus([lastSummary, previousSummary].filter((s) => s !== null), type);
+        const activeRun = await activeRunOf(store, projectRoot, loaded.id);
+        bannerInputs.push({ evalId: loaded.id, status, last: lastSummary, previous: previousSummary });
+        const agentTarget = file.target.kind === "agent"
+            ? await agentListTarget(file.target.agent, last, input.agents, projectRoot)
+            : undefined;
+        const conv = agentTarget ? resolveConversation(file.conversation) : undefined;
+        const pipelinePath = paths.pipelinePath !== undefined ? posix(projectRoot, paths.pipelinePath) : undefined;
+        const stageFile = paths.stageFile !== undefined ? posix(projectRoot, paths.stageFile) : undefined;
+        const label = pipelinePath !== undefined
+            ? `${pipelineStem(pipelinePath)} · ${stageId ?? ""}`
+            : path.basename(stageFile ?? "").replace(/\.(stage\.)?ya?ml$/, "");
+        const delta = lastSummary?.passRate != null && previousSummary?.passRate != null
+            ? (lastSummary.passRate - previousSummary.passRate) * 100
+            : null;
+        items.push({
+            id: loaded.id,
+            name: file.name ?? loaded.id,
+            description: file.description ?? null,
+            type,
+            path: entry.relPath,
+            target: agentTarget ?? {
+                kind: "stage",
+                ...(pipelinePath !== undefined ? { pipeline_path: pipelinePath } : {}),
+                ...(stageId !== undefined ? { stage_id: stageId } : {}),
+                ...(stageFile !== undefined ? { file: stageFile } : {}),
+                label,
+            },
+            ...(conv
+                ? { conversation: { mode: conv.mode, max_turns: conv.maxTurns, tools: conv.tools, approvals: conv.approvals } }
+                : {}),
+            dataset_path: posix(projectRoot, paths.datasetPath),
+            dataset_cases: input.datasetCases(paths.datasetPath),
+            repeats: file.defaults?.repeats ?? EVAL_DEFAULT_REPEATS,
+            grader_count: loaded.graders.length,
+            rubric_count: loaded.graders.filter((g) => g.type === "rubric").length,
+            valid,
+            finding_count: findingCount,
+            status: activeRun ? "running" : status,
+            active_run: activeRun,
+            last_run: last ? runHeadline(last) : null,
+            last_run_delta_pts: delta,
+            trend: completed
+                .slice(0, TREND_RUNS)
+                .reverse()
+                .map((r) => ({
+                run_number: r.runNumber,
+                passed: r.passed === true,
+                pass_rate: (r.summary ?? {}).pass_rate ?? null,
+                pass_threshold: r.passThreshold,
+            })),
+        });
+    }
+    return { items, banner: regressionBanner(bannerInputs), bannerInputs };
+}
+export async function costLastSevenDays(store, projectRoot, now) {
+    const cutoff = now.getTime() - 7 * 24 * 3600 * 1000;
+    const createdFloor = cutoff - 24 * 3600 * 1000;
+    let total = 0;
+    let cursor;
+    for (let pages = 0; pages < 50; pages += 1) {
+        const page = await store.listEvalRuns({ projectRoot, limit: 200, ...(cursor ? { cursor } : {}) });
+        for (const run of page.items) {
+            if (run.finishedAt !== null && Date.parse(run.finishedAt) >= cutoff)
+                total += run.costUsd;
+        }
+        const oldest = page.items[page.items.length - 1];
+        if (page.nextCursor === null || !oldest || Date.parse(oldest.createdAt) < createdFloor)
+            break;
+        cursor = page.nextCursor;
+    }
+    return total;
+}

package/dist/evals/read/results.d.ts

@@ -0,0 +1,46 @@
+import type { EvalCaseRow, EvalResultWithGrades, EvalReviewRow } from "../storeTypes.js";
+export declare const RESULT_ROW_STATUSES: readonly ["passed", "failed", "errored", "flaky"];
+export type ResultRowStatus = (typeof RESULT_ROW_STATUSES)[number];
+export type ResultRow = {
+    case_id: string;
+    position: number;
+    title: string | null;
+    tags: string[];
+    repeats: Array<{
+        repeat_index: number;
+        status: string;
+        passed: boolean;
+        score: number | null;
+    }>;
+    pass_fraction: number | null;
+    mean_score: number | null;
+    failed_graders: string[];
+    latency_ms_p50: number | null;
+    cost_usd: number;
+    failure_tags: string[];
+    turns?: number | null;
+    tool_calls?: number | null;
+};
+export type ResultRowFilter = {
+    status?: ResultRowStatus;
+    grader?: string;
+    q?: string;
+    tag?: string;
+};
+export type ResultRowCounts = {
+    all: number;
+    passed: number;
+    failed: number;
+    errored: number;
+    flaky: number;
+};
+export declare function buildResultRows(input: {
+    cases: readonly EvalCaseRow[];
+    results: readonly EvalResultWithGrades[];
+    reviews: readonly EvalReviewRow[];
+    filter?: ResultRowFilter;
+    targetKind?: string;
+}): {
+    rows: ResultRow[];
+    counts: ResultRowCounts;
+};

package/dist/evals/read/results.js

@@ -0,0 +1,107 @@
+import { mean, passFraction, percentile } from "../stats.js";
+import { EXECUTED_RESULT_STATUSES, resultKey } from "../statsTypes.js";
+export const RESULT_ROW_STATUSES = ["passed", "failed", "errored", "flaky"];
+function caseField(c, key) {
+    return c ? c.case[key] : undefined;
+}
+function buildFacts(caseRow, caseId, results, reviews, targetKind) {
+    const sorted = [...results].sort((a, b) => a.repeatIndex - b.repeatIndex);
+    const executedResults = sorted.filter((r) => EXECUTED_RESULT_STATUSES.includes(r.status));
+    const completed = executedResults.filter((r) => r.status === "completed");
+    const failedGraders = new Set();
+    const failureTags = new Set();
+    let cost = 0;
+    for (const r of sorted) {
+        cost += r.costUsd ?? 0;
+        for (const g of r.grades)
+            cost += g.judgeCostUsd ?? 0;
+        for (const tag of reviews.get(resultKey(caseId, r.repeatIndex))?.tags ?? [])
+            failureTags.add(tag);
+    }
+    for (const r of executedResults) {
+        for (const g of r.grades) {
+            if (g.status === "error" || g.passed === false)
+                failedGraders.add(g.graderId);
+        }
+    }
+    const passed = executedResults.filter((r) => r.passed).length;
+    const latencies = completed
+        .map((r) => r.latencyMs)
+        .filter((v) => v !== null)
+        .sort((a, b) => a - b);
+    const turns = executedResults.flatMap((r) => (r.turns !== null ? [r.turns] : []));
+    const toolCalls = executedResults.flatMap((r) => (r.toolCallCount !== null ? [r.toolCallCount] : []));
+    const agentData = targetKind === "agent" || turns.length > 0 || toolCalls.length > 0;
+    const title = caseField(caseRow, "title");
+    const tags = caseField(caseRow, "tags") ?? [];
+    return {
+        executed: executedResults.length,
+        passed,
+        errored: executedResults.some((r) => r.status !== "completed"),
+        haystack: [caseId, title ?? "", JSON.stringify(caseField(caseRow, "input") ?? {})].join("\n").toLowerCase(),
+        row: {
+            case_id: caseId,
+            position: caseRow?.position ?? 0,
+            title: title ?? null,
+            tags,
+            repeats: sorted.map((r) => ({
+                repeat_index: r.repeatIndex,
+                status: r.status,
+                passed: r.passed,
+                score: r.score,
+            })),
+            pass_fraction: passFraction(passed, executedResults.length),
+            mean_score: mean(completed.filter((r) => r.score !== null).map((r) => r.score)),
+            failed_graders: [...failedGraders].sort(),
+            latency_ms_p50: percentile(latencies, 0.5),
+            cost_usd: cost,
+            failure_tags: [...failureTags].sort(),
+            ...(agentData ? { turns: mean(turns), tool_calls: mean(toolCalls) } : {}),
+        },
+    };
+}
+function matchesStatus(f, status) {
+    if (status === "errored")
+        return f.errored;
+    if (f.executed === 0)
+        return false;
+    if (status === "passed")
+        return f.passed === f.executed;
+    if (status === "failed")
+        return f.passed < f.executed;
+    return f.passed > 0 && f.passed < f.executed && f.executed >= 2;
+}
+export function buildResultRows(input) {
+    const { filter = {} } = input;
+    const reviews = new Map(input.reviews.map((r) => [resultKey(r.caseId, r.repeatIndex), r]));
+    const byCase = new Map();
+    for (const r of input.results) {
+        const list = byCase.get(r.caseId);
+        if (list)
+            list.push(r);
+        else
+            byCase.set(r.caseId, [r]);
+    }
+    const caseById = new Map(input.cases.map((c) => [c.caseId, c]));
+    const ordered = [...byCase.keys()].sort((a, b) => (caseById.get(a)?.position ?? 0) - (caseById.get(b)?.position ?? 0) || (a < b ? -1 : a > b ? 1 : 0));
+    const q = filter.q?.trim().toLowerCase();
+    const facts = ordered
+        .map((id) => buildFacts(caseById.get(id), id, byCase.get(id) ?? [], reviews, input.targetKind))
+        .filter((f) => {
+        if (filter.grader !== undefined && !f.row.failed_graders.includes(filter.grader))
+            return false;
+        if (filter.tag !== undefined && !f.row.failure_tags.includes(filter.tag) && !f.row.tags.includes(filter.tag)) {
+            return false;
+        }
+        return q === undefined || q === "" || f.haystack.includes(q);
+    });
+    const counts = {
+        all: facts.length,
+        passed: facts.filter((f) => matchesStatus(f, "passed")).length,
+        failed: facts.filter((f) => matchesStatus(f, "failed")).length,
+        errored: facts.filter((f) => matchesStatus(f, "errored")).length,
+        flaky: facts.filter((f) => matchesStatus(f, "flaky")).length,
+    };
+    const selected = filter.status === undefined ? facts : facts.filter((f) => matchesStatus(f, filter.status));
+    return { rows: selected.map((f) => f.row), counts };
+}

package/dist/evals/read/runs.d.ts

@@ -0,0 +1,149 @@
+import { type AgentCatalogPort } from "../../config/resolveAgentTarget.js";
+import type { RunStore } from "../../runstore/port.js";
+import type { EvalLiveState } from "../manager.js";
+import { type LowestCase, type Stats } from "../stats.js";
+import { type GraderRef, type PreviousRunLite } from "../statsTypes.js";
+import type { EvalGradeRow, EvalResultWithGrades, EvalReviewRow, EvalRunRow } from "../storeTypes.js";
+export type ReadStore = Pick<RunStore, "listEvalRuns" | "getEvalRun" | "getEvalRunByNumber" | "listEvalResults" | "getEvalResultsForCase" | "getEvalCases" | "getEvalReviews">;
+export type RunHeadline = {
+    eval_run_id: string;
+    eval_id: string;
+    run_number: number;
+    status: string;
+    passed: boolean | null;
+    pass_rate: number | null;
+    pass_rate_ci: {
+        low: number;
+        high: number;
+    } | null;
+    results_executed: number;
+    cost_total: number;
+    finished_at: string | null;
+    fingerprint: string;
+    label: string | null;
+};
+export type WireGrade = {
+    grader_id: string;
+    grader_type: string;
+    status: string;
+    passed: boolean | null;
+    score: number | null;
+    raw_value: number | null;
+    reason: string | null;
+    details: unknown;
+    error_code: string | null;
+    judge_model: string | null;
+    judge_cost_usd: number | null;
+    judge_tokens_in: number | null;
+    judge_tokens_out: number | null;
+};
+export type WireResultDetail = {
+    case_id: string;
+    repeat_index: number;
+    status: string;
+    error_code: string | null;
+    error_message: string | null;
+    stage_run_id: string | null;
+    subject_ref: string | null;
+    turns: number | null;
+    tool_call_count: number | null;
+    agent?: {
+        id: string;
+        session_id: string | null;
+    } | null;
+    output: unknown;
+    latency_ms: number | null;
+    cost_usd: number | null;
+    tokens_in: number | null;
+    tokens_out: number | null;
+    score: number | null;
+    passed: boolean;
+    grades: WireGrade[];
+    budget_warnings: string[];
+    review: {
+        label: string | null;
+        note: string | null;
+    } | null;
+    tags: string[];
+};
+export type RunCurrent = {
+    stage_version_hash: string | null;
+    dataset_hash: string | null;
+    grader_hashes: Record<string, string> | null;
+    agent_revision?: number | null;
+    agent_definition_hash?: string | null;
+};
+export type ConversationView = {
+    messages: unknown[];
+    tool_calls: unknown[];
+    final_reply: string;
+    stop_reason: string;
+};
+export declare function conversationView(output: unknown): ConversationView | null;
+export type EvalRunDetail = {
+    eval_run_id: string;
+    run_number: number;
+    eval_id: string;
+    eval_path: string;
+    status: string;
+    status_reason: string | null;
+    passed: boolean | null;
+    label: string | null;
+    started_via: string;
+    created_at: string;
+    started_at: string | null;
+    finished_at: string | null;
+    progress: {
+        results_done: number;
+        results_total: number;
+        cost_usd: number;
+        max_cost_usd: number;
+        cases_total: number;
+    };
+    config: Record<string, unknown>;
+    fingerprint: string;
+    components: Record<string, unknown>;
+    stats: Stats | null;
+    lowest_cases_by_grader: Record<string, LowestCase[]>;
+    current: RunCurrent;
+    live?: Array<{
+        case_id: string;
+        repeat_index: number;
+        phase: string;
+    }>;
+};
+export declare function runHeadline(row: EvalRunRow): RunHeadline;
+export type ActiveRun = {
+    run_number: number;
+    eval_run_id: string;
+    status: "queued" | "running";
+    results_done: number;
+    results_total: number;
+};
+/** The newest queued or running run of an eval, or null. */
+export declare function activeRunOf(store: Pick<ReadStore, "listEvalRuns">, projectRoot: string, evalId: string): Promise<ActiveRun | null>;
+export declare function graderRefsOf(row: Pick<EvalRunRow, "config">): GraderRef[];
+export declare function reviewTagMap(reviews: readonly EvalReviewRow[]): Map<string, string[]>;
+export declare function statsOfRun(row: EvalRunRow, results: readonly EvalResultWithGrades[], extra?: {
+    reviews?: readonly EvalReviewRow[];
+    previousRun?: PreviousRunLite | null;
+}): Stats;
+export declare function completedRunsNewestFirst(store: Pick<ReadStore, "listEvalRuns">, projectRoot: string, evalId: string, limit?: number): Promise<EvalRunRow[]>;
+export declare function previousRunLite(store: ReadStore, row: EvalRunRow, graders: readonly GraderRef[]): Promise<PreviousRunLite | null>;
+export declare function gradeToWire(g: EvalGradeRow): WireGrade;
+export declare function resultToWire(r: EvalResultWithGrades, review: EvalReviewRow | undefined, agentId?: string): WireResultDetail;
+export declare function currentHashes(row: EvalRunRow, agents?: AgentCatalogPort): Promise<RunCurrent>;
+export declare function buildRunDetail(input: {
+    store: ReadStore;
+    row: EvalRunRow;
+    live?: EvalLiveState | null;
+    agents?: AgentCatalogPort;
+}): Promise<EvalRunDetail>;
+export declare function buildExport(input: {
+    store: ReadStore;
+    row: EvalRunRow;
+    what: "results" | "config";
+}): Promise<{
+    filename: string;
+    body: unknown;
+}>;