stageflow 0.40.0 → 0.42.0

package/dist/evals/read/runs.js

@@ -0,0 +1,289 @@
+import path from "node:path";
+import { loadDatasetFile } from "../../config/loadDataset.js";
+import { loadEvalOutcome, resolveEvalPaths } from "../../config/loadEval.js";
+import { resolveAgentTarget } from "../../config/resolveAgentTarget.js";
+import { resolveEvalTarget } from "../../config/resolveEvalTarget.js";
+import { computeStats, lowestCases } from "../stats.js";
+import { toGradeRecords, toResultRecord } from "../statsAdapter.js";
+import { resultKey } from "../statsTypes.js";
+export function conversationView(output) {
+    const rec = asRecord(output);
+    if (rec.kind !== "conversation")
+        return null;
+    if (!Array.isArray(rec.messages) || !Array.isArray(rec.tool_calls))
+        return null;
+    return {
+        messages: rec.messages,
+        tool_calls: rec.tool_calls,
+        final_reply: typeof rec.final_reply === "string" ? rec.final_reply : "",
+        stop_reason: typeof rec.stop_reason === "string" ? rec.stop_reason : "",
+    };
+}
+function asRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value)
+        ? value
+        : {};
+}
+export function runHeadline(row) {
+    const summary = (row.summary ?? {});
+    return {
+        eval_run_id: row.evalRunId,
+        eval_id: row.evalId,
+        run_number: row.runNumber,
+        status: row.status,
+        passed: row.passed,
+        pass_rate: summary.pass_rate ?? null,
+        pass_rate_ci: summary.pass_rate_ci ?? null,
+        results_executed: summary.results_executed ?? row.resultsDone,
+        cost_total: summary.cost_total ?? row.costUsd,
+        finished_at: row.finishedAt,
+        fingerprint: row.configFingerprint,
+        label: row.label,
+    };
+}
+/** The newest queued or running run of an eval, or null. */
+export async function activeRunOf(store, projectRoot, evalId) {
+    const found = [];
+    for (const status of ["running", "queued"]) {
+        const page = await store.listEvalRuns({ projectRoot, evalId, status, limit: 50 });
+        found.push(...page.items);
+    }
+    const row = found.sort((a, b) => b.runNumber - a.runNumber)[0];
+    if (!row || (row.status !== "queued" && row.status !== "running"))
+        return null;
+    return {
+        run_number: row.runNumber,
+        eval_run_id: row.evalRunId,
+        status: row.status,
+        results_done: row.resultsDone,
+        results_total: row.resultsTotal,
+    };
+}
+export function graderRefsOf(row) {
+    const graders = asRecord(row.config).graders;
+    if (!Array.isArray(graders))
+        return [];
+    return graders.flatMap((g) => {
+        const rec = asRecord(g);
+        return typeof rec.id === "string" && typeof rec.hash === "string"
+            ? [{ id: rec.id, type: typeof rec.type === "string" ? rec.type : "", hash: rec.hash }]
+            : [];
+    });
+}
+export function reviewTagMap(reviews) {
+    const map = new Map();
+    for (const review of reviews) {
+        if (review.tags.length > 0)
+            map.set(resultKey(review.caseId, review.repeatIndex), review.tags);
+    }
+    return map;
+}
+export function statsOfRun(row, results, extra = {}) {
+    return computeStats({
+        ...(row.targetKind === "agent" ? { targetKind: "agent" } : {}),
+        results: results.map(toResultRecord),
+        grades: results.flatMap(toGradeRecords),
+        graders: graderRefsOf(row),
+        repeats: row.repeats,
+        passThreshold: row.passThreshold,
+        ...(extra.reviews ? { reviewsTags: reviewTagMap(extra.reviews) } : {}),
+        ...(extra.previousRun !== undefined ? { previousRun: extra.previousRun } : {}),
+    });
+}
+export async function completedRunsNewestFirst(store, projectRoot, evalId, limit = 50) {
+    const page = await store.listEvalRuns({ projectRoot, evalId, status: "completed", limit });
+    return [...page.items].sort((a, b) => b.runNumber - a.runNumber);
+}
+export async function previousRunLite(store, row, graders) {
+    const earlier = (await completedRunsNewestFirst(store, row.projectRoot, row.evalId)).filter((r) => r.runNumber < row.runNumber);
+    if (earlier.length === 0)
+        return null;
+    const rates = new Map();
+    const rateOf = async (run, graderId) => {
+        let byGrader = rates.get(run.evalRunId);
+        if (!byGrader) {
+            const stats = statsOfRun(run, await store.listEvalResults(run.evalRunId));
+            byGrader = new Map(stats.by_grader.map((g) => [g.grader_id, g.pass_rate]));
+            rates.set(run.evalRunId, byGrader);
+        }
+        return byGrader.get(graderId) ?? null;
+    };
+    const lite = [];
+    for (const g of graders) {
+        let changedHash;
+        let match;
+        for (const run of earlier) {
+            const found = graderRefsOf(run).find((x) => x.id === g.id);
+            if (!found)
+                continue;
+            if (found.hash === g.hash) {
+                match = run;
+                break;
+            }
+            changedHash ??= found.hash;
+        }
+        if (match)
+            lite.push({ graderId: g.id, hash: g.hash, passRate: await rateOf(match, g.id) });
+        else if (changedHash !== undefined)
+            lite.push({ graderId: g.id, hash: changedHash, passRate: null });
+    }
+    return { graders: lite };
+}
+export function gradeToWire(g) {
+    return {
+        grader_id: g.graderId,
+        grader_type: g.graderType,
+        status: g.status,
+        passed: g.passed,
+        score: g.score,
+        raw_value: g.rawValue,
+        reason: g.reason,
+        details: g.details,
+        error_code: g.errorCode,
+        judge_model: g.judgeModel,
+        judge_cost_usd: g.judgeCostUsd,
+        judge_tokens_in: g.judgeTokensIn,
+        judge_tokens_out: g.judgeTokensOut,
+    };
+}
+export function resultToWire(r, review, agentId) {
+    const hasReview = review !== undefined && (review.label !== null || review.note !== null);
+    return {
+        case_id: r.caseId,
+        repeat_index: r.repeatIndex,
+        status: r.status,
+        error_code: r.errorCode,
+        error_message: r.errorMessage,
+        stage_run_id: r.stageRunId,
+        subject_ref: r.subjectRef,
+        turns: r.turns,
+        tool_call_count: r.toolCallCount,
+        ...(agentId !== undefined
+            ? { agent: r.subjectRef !== null ? { id: agentId, session_id: r.subjectRef } : null }
+            : {}),
+        output: r.output,
+        latency_ms: r.latencyMs,
+        cost_usd: r.costUsd,
+        tokens_in: r.tokensIn,
+        tokens_out: r.tokensOut,
+        score: r.score,
+        passed: r.passed,
+        grades: r.grades.map(gradeToWire),
+        budget_warnings: r.budgetWarnings ?? [],
+        review: hasReview ? { label: review.label, note: review.note } : null,
+        tags: review?.tags ?? [],
+    };
+}
+export async function currentHashes(row, agents) {
+    const isAgent = row.targetKind === "agent";
+    const none = {
+        stage_version_hash: null,
+        dataset_hash: null,
+        grader_hashes: null,
+        ...(isAgent ? { agent_revision: null, agent_definition_hash: null } : {}),
+    };
+    const evalAbs = path.resolve(row.projectRoot, row.evalPath);
+    try {
+        const outcome = await loadEvalOutcome(evalAbs, row.projectRoot);
+        if (!outcome.ok)
+            return none;
+        const loaded = outcome.value;
+        const paths = resolveEvalPaths(loaded.file, loaded.path);
+        const datasetHashOf = async () => {
+            const dataset = await loadDatasetFile(paths.datasetPath, { root: row.projectRoot });
+            return dataset.findings.some((f) => f.severity === "error") ? null : dataset.contentHash;
+        };
+        const graderHashes = Object.fromEntries(loaded.graders.map((g) => [g.id, g.hash]));
+        if (isAgent) {
+            const resolved = agents ? await resolveAgentTarget(loaded.file, row.projectRoot, agents) : null;
+            return {
+                stage_version_hash: null,
+                dataset_hash: await datasetHashOf(),
+                grader_hashes: graderHashes,
+                agent_revision: resolved?.ok ? resolved.value.revision : null,
+                agent_definition_hash: resolved?.ok ? resolved.value.definitionHash : null,
+            };
+        }
+        const [target, datasetHash] = await Promise.all([
+            resolveEvalTarget(loaded.file, loaded.path, row.projectRoot),
+            datasetHashOf(),
+        ]);
+        return {
+            stage_version_hash: target.ok ? target.value.stageVersionHash : null,
+            dataset_hash: datasetHash,
+            grader_hashes: graderHashes,
+        };
+    }
+    catch {
+        return none;
+    }
+}
+export async function buildRunDetail(input) {
+    const { store, row } = input;
+    const [results, reviews] = await Promise.all([
+        store.listEvalResults(row.evalRunId),
+        store.getEvalReviews(row.evalRunId),
+    ]);
+    const graders = graderRefsOf(row);
+    const grades = results.flatMap(toGradeRecords);
+    const executed = results.some((r) => ["completed", "stage_failed", "needs_input", "error"].includes(r.status));
+    const stats = executed
+        ? statsOfRun(row, results, { reviews, previousRun: await previousRunLite(store, row, graders) })
+        : null;
+    const detail = {
+        eval_run_id: row.evalRunId,
+        run_number: row.runNumber,
+        eval_id: row.evalId,
+        eval_path: row.evalPath,
+        status: row.status,
+        status_reason: row.statusReason,
+        passed: row.passed,
+        label: row.label,
+        started_via: row.startedVia,
+        created_at: row.createdAt,
+        started_at: row.startedAt,
+        finished_at: row.finishedAt,
+        progress: {
+            results_done: row.resultsDone,
+            results_total: row.resultsTotal,
+            cost_usd: row.costUsd,
+            max_cost_usd: row.maxCostUsd,
+            cases_total: row.casesTotal,
+        },
+        config: row.config,
+        fingerprint: row.configFingerprint,
+        components: row.configComponents,
+        stats,
+        lowest_cases_by_grader: executed
+            ? Object.fromEntries(graders.map((g) => [g.id, lowestCases(grades.filter((x) => x.graderId === g.id))]))
+            : {},
+        current: await currentHashes(row, input.agents),
+    };
+    if (input.live && (row.status === "running" || row.status === "queued")) {
+        detail.live = input.live.phases.map((p) => ({
+            case_id: p.caseId,
+            repeat_index: p.repeatIndex,
+            phase: p.phase,
+        }));
+    }
+    return detail;
+}
+export async function buildExport(input) {
+    const { store, row, what } = input;
+    const filename = `${row.evalId}-run-${row.runNumber}-${what}.json`;
+    if (what === "config")
+        return { filename, body: row.config };
+    const [detail, results, reviews] = await Promise.all([
+        buildRunDetail({ store, row }),
+        store.listEvalResults(row.evalRunId),
+        store.getEvalReviews(row.evalRunId),
+    ]);
+    const reviewByKey = new Map(reviews.map((r) => [resultKey(r.caseId, r.repeatIndex), r]));
+    return {
+        filename,
+        body: {
+            ...detail,
+            results: results.map((r) => resultToWire(r, reviewByKey.get(resultKey(r.caseId, r.repeatIndex)))),
+        },
+    };
+}

package/dist/evals/read/suggestions.d.ts

@@ -0,0 +1,43 @@
+import { type TriageBucket } from "../triage/constants.js";
+import type { SuggestionRow } from "../triage/types.js";
+export type WireSuggestion = {
+    id: string;
+    case_id: string;
+    repeat_index: number;
+    source: "jev" | "judge";
+    bucket: TriageBucket;
+    failure_type: string | null;
+    confidence: number;
+    bucket_probabilities: Record<TriageBucket, number>;
+    grader_false_alarms: Array<{
+        grader_id: string;
+        probability: number;
+    }>;
+    model: string;
+    prompt_version: number;
+    status: "pending" | "accepted" | "dismissed";
+    created_at: string;
+};
+export type SuggestionSummary = {
+    by_bucket: Record<TriageBucket, number>;
+    by_type: Record<string, number>;
+    pending: number;
+    accepted: number;
+    dismissed: number;
+    low_confidence: number;
+};
+export declare function suggestionToWire(row: SuggestionRow): WireSuggestion;
+export declare function isLowConfidence(row: Pick<SuggestionRow, "bucket" | "confidence">): boolean;
+/**
+ * One suggestion stands for a result: the newest that is not dismissed, else the newest dismissed. A result sorted by
+ * both backends therefore counts once. Ties on time keep the later source name so the choice is stable.
+ */
+export declare function representatives(rows: readonly SuggestionRow[]): SuggestionRow[];
+/**
+ * Counts per result (see `representatives`). `by_bucket`, `by_type` and `low_confidence` count suggestions that are not
+ * dismissed; `pending`, `accepted` and `dismissed` count every representative by status. A null failure type is not
+ * counted in `by_type`. Low confidence means the bucket is `unclear` or its confidence is below 0.7.
+ */
+export declare function buildSuggestionSummary(rows: readonly SuggestionRow[]): SuggestionSummary;
+/** The suggestion shown for one result in the case drawer: its representative, or null. */
+export declare function suggestionFor(rows: readonly SuggestionRow[], caseId: string, repeatIndex: number): WireSuggestion | null;

package/dist/evals/read/suggestions.js

@@ -0,0 +1,73 @@
+import { LOW_CONFIDENCE_THRESHOLD, TRIAGE_BUCKETS } from "../triage/constants.js";
+export function suggestionToWire(row) {
+    return {
+        id: row.id,
+        case_id: row.caseId,
+        repeat_index: row.repeatIndex,
+        source: row.source,
+        bucket: row.bucket,
+        failure_type: row.failureType,
+        confidence: row.confidence,
+        bucket_probabilities: row.bucketProbabilities,
+        grader_false_alarms: row.graderFalseAlarms.map((a) => ({ grader_id: a.graderId, probability: a.probability })),
+        model: row.model,
+        prompt_version: row.promptVersion,
+        status: row.status,
+        created_at: row.createdAt,
+    };
+}
+export function isLowConfidence(row) {
+    return row.bucket === "unclear" || row.confidence < LOW_CONFIDENCE_THRESHOLD;
+}
+/**
+ * One suggestion stands for a result: the newest that is not dismissed, else the newest dismissed. A result sorted by
+ * both backends therefore counts once. Ties on time keep the later source name so the choice is stable.
+ */
+export function representatives(rows) {
+    const best = new Map();
+    const rank = (row) => [row.status === "dismissed" ? 0 : 1, row.createdAt, row.source];
+    for (const row of rows) {
+        const key = `${row.caseId}\u0000${row.repeatIndex}`;
+        const current = best.get(key);
+        if (current === undefined) {
+            best.set(key, row);
+            continue;
+        }
+        const [a, b] = [rank(row), rank(current)];
+        if (a[0] > b[0] || (a[0] === b[0] && (a[1] > b[1] || (a[1] === b[1] && a[2] > b[2]))))
+            best.set(key, row);
+    }
+    return [...best.values()];
+}
+/**
+ * Counts per result (see `representatives`). `by_bucket`, `by_type` and `low_confidence` count suggestions that are not
+ * dismissed; `pending`, `accepted` and `dismissed` count every representative by status. A null failure type is not
+ * counted in `by_type`. Low confidence means the bucket is `unclear` or its confidence is below 0.7.
+ */
+export function buildSuggestionSummary(rows) {
+    const summary = {
+        by_bucket: Object.fromEntries(TRIAGE_BUCKETS.map((b) => [b, 0])),
+        by_type: {},
+        pending: 0,
+        accepted: 0,
+        dismissed: 0,
+        low_confidence: 0,
+    };
+    for (const row of representatives(rows)) {
+        summary[row.status] += 1;
+        if (row.status === "dismissed")
+            continue;
+        summary.by_bucket[row.bucket] += 1;
+        if (row.failureType !== null)
+            summary.by_type[row.failureType] = (summary.by_type[row.failureType] ?? 0) + 1;
+        if (isLowConfidence(row))
+            summary.low_confidence += 1;
+    }
+    return summary;
+}
+/** The suggestion shown for one result in the case drawer: its representative, or null. */
+export function suggestionFor(rows, caseId, repeatIndex) {
+    const own = rows.filter((r) => r.caseId === caseId && r.repeatIndex === repeatIndex);
+    const [rep] = representatives(own);
+    return rep === undefined ? null : suggestionToWire(rep);
+}

package/dist/evals/runner/FakeAgentBackend.d.ts

@@ -0,0 +1,40 @@
+import type { CompleteInput, CompleteResult, ConversationBackend, RuntimeUsage } from "../../agents/runtime/types.js";
+export type FakeToolStep = {
+    tool: string;
+    input?: Record<string, unknown>;
+    /** What the tool does when the gate lets it run. */
+    body?: () => string | Promise<string>;
+    /** Report the tool as failed when it ran. */
+    fails?: boolean;
+};
+export type FakeTurnScript = {
+    text?: string;
+    tools?: FakeToolStep[];
+    usage?: RuntimeUsage;
+    /** Wait before replying; an abort ends the wait early. */
+    delayMs?: number;
+    /** Never reply until the turn is aborted. */
+    hang?: boolean;
+    /** Throw this from the backend. */
+    error?: Error;
+};
+export type FakeTurnReply = FakeTurnScript | ((input: CompleteInput, index: number) => FakeTurnScript);
+export declare const FAKE_USAGE: RuntimeUsage;
+/**
+ * A scriptable conversation backend for eval tests. Each model call takes the next scripted turn; tool calls go
+ * through the same gate and hooks the real runtime uses, so tool policy is exercised.
+ */
+export declare class FakeAgentBackend implements ConversationBackend {
+    private readonly script;
+    readonly id: "pi";
+    readonly inputs: CompleteInput[];
+    readonly released: string[];
+    /** Tool bodies that actually ran, by grant id. */
+    readonly executed: string[];
+    private index;
+    private callSeq;
+    constructor(script?: readonly FakeTurnReply[] | ((input: CompleteInput, index: number) => FakeTurnScript));
+    complete(input: CompleteInput): Promise<CompleteResult>;
+    release(sessionId: string): Promise<void>;
+    get calls(): number;
+}

package/dist/evals/runner/FakeAgentBackend.js

@@ -0,0 +1,82 @@
+export const FAKE_USAGE = { inputTokens: 100, outputTokens: 20, costUsd: 0.01 };
+function waitOrAbort(ms, signal) {
+    return new Promise((resolve) => {
+        if (signal.aborted)
+            return resolve();
+        const done = () => {
+            clearTimeout(timer);
+            signal.removeEventListener("abort", done);
+            resolve();
+        };
+        const timer = ms === undefined ? undefined : setTimeout(done, ms);
+        signal.addEventListener("abort", done, { once: true });
+    });
+}
+/**
+ * A scriptable conversation backend for eval tests. Each model call takes the next scripted turn; tool calls go
+ * through the same gate and hooks the real runtime uses, so tool policy is exercised.
+ */
+export class FakeAgentBackend {
+    script;
+    id = "pi";
+    inputs = [];
+    released = [];
+    /** Tool bodies that actually ran, by grant id. */
+    executed = [];
+    index = 0;
+    callSeq = 0;
+    constructor(script = []) {
+        this.script = script;
+    }
+    async complete(input) {
+        const index = this.index++;
+        this.inputs.push(input);
+        const reply = typeof this.script === "function" ? this.script : this.script[index];
+        const turn = typeof reply === "function" ? reply(input, index) : (reply ?? { text: `reply ${index + 1}` });
+        if (turn.error)
+            throw turn.error;
+        const usage = turn.usage ?? FAKE_USAGE;
+        const toolOutputs = [];
+        const startedAt = new Date();
+        if (turn.hang)
+            await waitOrAbort(undefined, input.signal);
+        else if (turn.delayMs !== undefined)
+            await waitOrAbort(turn.delayMs, input.signal);
+        if (input.signal.aborted)
+            return { text: "", usage, toolOutputs };
+        for (const step of turn.tools ?? []) {
+            const call = { id: `call_${(this.callSeq += 1)}`, tool: step.tool, input: step.input ?? {} };
+            const decision = await input.hooks.gate(call);
+            if (!decision.allow)
+                continue;
+            input.hooks.toolStarted(call);
+            let output;
+            let ok = step.fails !== true;
+            if (step.tool.startsWith("agent:")) {
+                output = await input.hooks.delegate(step.tool.slice(6), String(call.input.message ?? ""), call.id);
+            }
+            else {
+                try {
+                    this.executed.push(step.tool);
+                    output = await (step.body ? step.body() : `ran ${step.tool}`);
+                }
+                catch (error) {
+                    ok = false;
+                    output = error instanceof Error ? error.message : String(error);
+                }
+            }
+            toolOutputs.push(output);
+            input.hooks.toolFinished(call, { ok, output });
+        }
+        const text = turn.text ?? `reply ${index + 1}`;
+        input.onDelta(text);
+        input.hooks.modelFinished({ startedAt, endedAt: new Date(), ...usage });
+        return { text, usage, toolOutputs };
+    }
+    async release(sessionId) {
+        this.released.push(sessionId);
+    }
+    get calls() {
+        return this.index;
+    }
+}

package/dist/evals/runner/agentCaseRunner.d.ts

@@ -0,0 +1,24 @@
+import type { AgentHostContext } from "../../agents/hostContext.js";
+import type { AgentDefinition } from "../../config/loadAgent.js";
+import { type ModelPrice, type SimulatorModel } from "./simulatorModel.js";
+import type { StageCaseRunner } from "./stageCaseRunner.js";
+export declare const EVAL_CASE_WALL_CLOCK_GRACE_MS = 60000;
+export declare const EVAL_TURN_STOP_GRACE_MS = 10000;
+export type AgentRunnerHost = Pick<AgentHostContext, "service" | "chat" | "evalGates" | "backend">;
+export type AgentCaseRunnerDeps = {
+    /** Lazy: the manager is built before the agent host. Undefined means this Host cannot run agents. */
+    host: () => AgentRunnerHost | undefined;
+    /** Builds the simulated user's model; defaults to the agent backend's one-shot call on `plan.userModel`. */
+    simulatorModel?: (input: {
+        projectRoot: string;
+        definition: AgentDefinition;
+        modelId: string;
+        host: AgentRunnerHost;
+    }) => SimulatorModel;
+    /** USD per million tokens by model id, for pricing simulator calls the backend reports no cost for. */
+    loadPrices?: (modelIds: readonly string[], projectRoot: string) => Promise<Map<string, ModelPrice>>;
+    now?: () => number;
+    wallClockGraceMs?: number;
+    stopGraceMs?: number;
+};
+export declare function createAgentCaseRunner(deps: AgentCaseRunnerDeps): StageCaseRunner;