stageflow 0.40.0 → 0.44.0

package/dist/evals/graders/referenceGrader.js

@@ -0,0 +1,59 @@
+import { isDeepStrictEqual } from "node:util";
+import { resolveValue } from "./path.js";
+import { show, subjectRoot, truncate } from "./subject.js";
+const norm = (s) => s.replace(/\s+/g, " ").trim().toLowerCase();
+function toText(value) {
+    if (typeof value === "string")
+        return value;
+    return JSON.stringify(value) ?? "";
+}
+export function gradeReference(grader, subject, testCase, _context) {
+    const base = { graderId: grader.id, graderType: "reference" };
+    const missing = (reason) => ({
+        ...base,
+        status: "error",
+        errorCode: "EVAL_CASE_MISSING_EXPECTED",
+        reason,
+    });
+    const expectedRef = resolveValue(testCase, grader.expected);
+    if (!expectedRef.found || expectedRef.value === undefined || expectedRef.value === null) {
+        return missing(`case has no value at ${grader.expected}`);
+    }
+    const actual = resolveValue(subjectRoot(subject), grader.path);
+    if (grader.mode === "equals") {
+        const passed = actual.found && isDeepStrictEqual(actual.value, expectedRef.value);
+        return {
+            ...base,
+            status: "graded",
+            passed,
+            score: passed ? 1 : 0,
+            rawValue: passed ? 1 : 0,
+            ...(passed
+                ? {}
+                : {
+                    reason: truncate(`${grader.path} is ${actual.found ? show(actual.value) : "missing"}, expected ${show(expectedRef.value)}`),
+                }),
+        };
+    }
+    const expected = expectedRef.value;
+    if (!Array.isArray(expected) || !expected.every((e) => typeof e === "string")) {
+        return missing(`${grader.expected} must be an array of strings`);
+    }
+    const text = actual.found ? norm(toText(actual.value)) : "";
+    const items = expected.map((t) => ({ text: t, found: text.includes(norm(t)) }));
+    const foundCount = items.filter((i) => i.found).length;
+    const score = items.length === 0 ? 1 : foundCount / items.length;
+    const passed = score + 1e-9 >= grader.passAt;
+    const firstMissing = items.find((i) => !i.found);
+    return {
+        ...base,
+        status: "graded",
+        passed,
+        score,
+        rawValue: score,
+        ...(firstMissing
+            ? { reason: truncate(`found ${foundCount} of ${items.length}; missing ${show(firstMissing.text)}`) }
+            : {}),
+        details: { items },
+    };
+}

package/dist/evals/graders/schemaGrader.d.ts

@@ -0,0 +1,5 @@
+import type { ResolvedGrader } from "../types.js";
+import type { GradeData, GraderContext, Subject } from "./types.js";
+export declare function gradeSchema(grader: Extract<ResolvedGrader, {
+    type: "schema";
+}>, subject: Subject, _testCase: unknown, context: GraderContext): GradeData;

package/dist/evals/graders/schemaGrader.js

@@ -0,0 +1,29 @@
+import { payloadInstanceMismatch } from "../../envelope/payloadSchema.js";
+import { notApplicable } from "./notApplicable.js";
+import { isConversation, truncate } from "./subject.js";
+export function gradeSchema(grader, subject, _testCase, context) {
+    if (isConversation(subject))
+        return notApplicable(grader, "conversation");
+    const base = { graderId: grader.id, graderType: "schema" };
+    const fail = (reason) => ({
+        ...base,
+        status: "graded",
+        passed: false,
+        score: 0,
+        rawValue: 0,
+        reason: truncate(reason, 500),
+    });
+    if (subject.status !== "success") {
+        return fail(`envelope status is ${subject.status}, expected success`);
+    }
+    let mismatch;
+    try {
+        mismatch = payloadInstanceMismatch(subject.payload ?? {}, context.outputSchema);
+    }
+    catch (error) {
+        return fail(error instanceof Error ? error.message : String(error));
+    }
+    if (mismatch !== undefined)
+        return fail(mismatch);
+    return { ...base, status: "graded", passed: true, score: 1, rawValue: 1 };
+}

package/dist/evals/graders/subject.d.ts

@@ -0,0 +1,6 @@
+import type { ConversationSubject, EnvelopeSubject, Subject } from "./types.js";
+export declare function isConversation(subject: Subject): subject is ConversationSubject;
+export declare function subjectEnvelope(subject: EnvelopeSubject): Record<string, unknown>;
+export declare function subjectRoot(subject: Subject): Record<string, unknown>;
+export declare function truncate(text: string, max?: number): string;
+export declare function show(value: unknown): string;

package/dist/evals/graders/subject.js

@@ -0,0 +1,33 @@
+export function isConversation(subject) {
+    return subject.kind === "conversation";
+}
+export function subjectEnvelope(subject) {
+    const env = {
+        status: subject.status,
+        summary: subject.summary,
+        artifacts: subject.artifacts,
+    };
+    if (subject.payload !== undefined)
+        env.payload = subject.payload;
+    return env;
+}
+export function subjectRoot(subject) {
+    if (isConversation(subject)) {
+        const { transcript: _transport, ...root } = subject;
+        return root;
+    }
+    return subjectEnvelope(subject);
+}
+export function truncate(text, max = 200) {
+    return text.length > max ? `${text.slice(0, max - 1)}…` : text;
+}
+export function show(value) {
+    if (value === undefined)
+        return "undefined";
+    try {
+        return JSON.stringify(value) ?? String(value);
+    }
+    catch {
+        return String(value);
+    }
+}

package/dist/evals/graders/toolCallsGrader.d.ts

@@ -0,0 +1,6 @@
+import type { ResolvedGrader } from "../types.js";
+import type { GradeCase, GradeData, GraderContext, Subject } from "./types.js";
+export declare function toolNameMatches(pattern: string, name: string): boolean;
+export declare function gradeToolCalls(grader: Extract<ResolvedGrader, {
+    type: "tool_calls";
+}>, subject: Subject, testCase: GradeCase, _context?: GraderContext): GradeData;

package/dist/evals/graders/toolCallsGrader.js

@@ -0,0 +1,100 @@
+import { describeCheck, runCheck } from "./fieldGrader.js";
+import { notApplicable } from "./notApplicable.js";
+import { isConversation, truncate } from "./subject.js";
+function isRecord(v) {
+    return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+function expectedList(testCase, key) {
+    const expected = testCase.expected;
+    if (!isRecord(expected))
+        return undefined;
+    const list = expected[key];
+    return Array.isArray(list) && list.every((x) => typeof x === "string") ? list : undefined;
+}
+export function toolNameMatches(pattern, name) {
+    if (pattern.endsWith(".*")) {
+        const prefix = pattern.slice(0, -2);
+        return name === prefix || name.startsWith(`${prefix}.`);
+    }
+    return name === pattern;
+}
+function plural(n, word) {
+    return `${n} ${word}${n === 1 ? "" : "s"}`;
+}
+export function gradeToolCalls(grader, subject, testCase, _context) {
+    if (!isConversation(subject))
+        return notApplicable(grader, "envelope");
+    const mustCall = grader.mustCall ?? expectedList(testCase, "tools");
+    const mustNotCall = grader.mustNotCall ?? expectedList(testCase, "forbidden_tools");
+    const { mustAskFirst, maxCalls } = grader;
+    if (mustCall === undefined &&
+        mustNotCall === undefined &&
+        mustAskFirst === undefined &&
+        maxCalls === undefined &&
+        grader.args.length === 0) {
+        return {
+            graderId: grader.id,
+            graderType: "tool_calls",
+            status: "error",
+            errorCode: "EVAL_CASE_MISSING_EXPECTED",
+            reason: "no tool rule: set a rule on the grader or expected.tools / expected.forbidden_tools on the case",
+        };
+    }
+    const calls = subject.tool_calls;
+    const named = (pattern) => calls.filter((c) => toolNameMatches(pattern, c.name));
+    const checks = [];
+    for (const name of mustCall ?? []) {
+        const n = named(name).length;
+        checks.push({ rule: `must_call ${name}`, passed: n > 0, found: n > 0 ? plural(n, "call") : "not called" });
+    }
+    for (const name of mustNotCall ?? []) {
+        const n = named(name).length;
+        checks.push({ rule: `must_not_call ${name}`, passed: n === 0, found: n === 0 ? "not called" : plural(n, "call") });
+    }
+    for (const name of mustAskFirst ?? []) {
+        const matching = named(name);
+        const asked = matching.filter((c) => c.policy_outcome === "would_ask_first").length;
+        checks.push({
+            rule: `must_ask_first ${name}`,
+            passed: asked > 0,
+            found: asked > 0 ? plural(asked, "approval request") : matching.length > 0 ? "called without asking" : "not called",
+        });
+    }
+    if (maxCalls !== undefined) {
+        checks.push({
+            rule: `max_calls ${maxCalls}`,
+            passed: calls.length <= maxCalls,
+            found: plural(calls.length, "call"),
+        });
+    }
+    for (const arg of grader.args) {
+        const check = { path: arg.path, op: arg.op, ...(arg.value !== undefined ? { value: arg.value } : {}) };
+        const matching = named(arg.tool);
+        const rule = `args ${arg.tool} ${describeCheck(check)}`;
+        let failure;
+        for (const call of matching) {
+            const outcome = runCheck(check, call.args);
+            if (!outcome.passed) {
+                failure = `call ${call.id}: ${outcome.found}`;
+                break;
+            }
+        }
+        checks.push({
+            rule,
+            passed: failure === undefined,
+            found: failure ?? (matching.length === 0 ? "no call to check" : plural(matching.length, "call")),
+        });
+    }
+    const first = checks.find((c) => !c.passed);
+    const passed = first === undefined;
+    return {
+        graderId: grader.id,
+        graderType: "tool_calls",
+        status: "graded",
+        passed,
+        score: passed ? 1 : 0,
+        rawValue: passed ? 1 : 0,
+        ...(first ? { reason: truncate(`${first.rule}: ${first.found}`) } : {}),
+        details: { checks: checks.map((c) => ({ ...c, found: truncate(c.found) })) },
+    };
+}

package/dist/evals/graders/transcript.d.ts

@@ -0,0 +1,12 @@
+import type { ConversationOutput } from "../types.js";
+export type TranscriptLimits = {
+    turns: number;
+    messageChars: number;
+    toolArgsChars: number;
+    toolResultChars: number;
+};
+/**
+ * The bounded transcript the judge reads: one line block per message or tool call, tool arguments and results
+ * capped, and only the first and last turns kept past the turn limit.
+ */
+export declare function renderTranscript(subject: Pick<ConversationOutput, "messages" | "tool_calls">, limits?: TranscriptLimits): string;

package/dist/evals/graders/transcript.js

@@ -0,0 +1,57 @@
+import { JUDGE_TRANSCRIPT_LIMITS } from "../constants.js";
+const KEEP_FIRST_TURNS = 3;
+function cut(text, max) {
+    if (text.length <= max)
+        return text;
+    return `${text.slice(0, max)}…[truncated ${text.length - max} chars]`;
+}
+function neutralize(text) {
+    return text.replace(/<(\/?)(transcript|rubric|examples|case_input|case_expected)\b/gi, "<\\$1$2");
+}
+function json(value) {
+    try {
+        return JSON.stringify(value ?? {}) ?? "{}";
+    }
+    catch {
+        return "{}";
+    }
+}
+/**
+ * The bounded transcript the judge reads: one line block per message or tool call, tool arguments and results
+ * capped, and only the first and last turns kept past the turn limit.
+ */
+export function renderTranscript(subject, limits = JUDGE_TRANSCRIPT_LIMITS) {
+    const turns = [
+        ...new Set([...subject.messages.map((m) => m.turn), ...subject.tool_calls.map((c) => c.turn)]),
+    ].sort((a, b) => a - b);
+    let kept = turns;
+    let omitted = 0;
+    if (turns.length > limits.turns) {
+        const tail = Math.max(0, limits.turns - KEEP_FIRST_TURNS);
+        omitted = turns.length - limits.turns;
+        kept = [...turns.slice(0, KEEP_FIRST_TURNS), ...turns.slice(turns.length - tail)];
+    }
+    const lines = [];
+    kept.forEach((turn, index) => {
+        if (omitted > 0 && index === KEEP_FIRST_TURNS)
+            lines.push(`[${omitted} turns omitted]`);
+        const say = (role) => {
+            for (const m of subject.messages) {
+                if (m.turn === turn && m.role === role) {
+                    lines.push(`[turn ${turn}] ${role === "user" ? "USER" : "AGENT"}: ${neutralize(cut(m.text, limits.messageChars))}`);
+                }
+            }
+        };
+        say("user");
+        for (const call of subject.tool_calls) {
+            if (call.turn !== turn)
+                continue;
+            const outcome = call.status === "error" ? `${call.policy_outcome}, error` : call.policy_outcome;
+            const args = neutralize(cut(json(call.args), limits.toolArgsChars));
+            const result = neutralize(cut(call.result_summary, limits.toolResultChars));
+            lines.push(`[turn ${turn}] TOOL ${call.name} (${outcome}) args=${args} -> ${result}`);
+        }
+        say("assistant");
+    });
+    return lines.join("\n");
+}

package/dist/evals/graders/types.d.ts

@@ -0,0 +1,20 @@
+import type { ConversationOutput, NormalizedGrade, PlanCase } from "../types.js";
+export type EnvelopeSubject = {
+    kind?: "envelope";
+    status: "success" | "failure";
+    summary: string;
+    payload?: Record<string, unknown>;
+    artifacts: string[];
+};
+export type ConversationSubject = ConversationOutput;
+export type Subject = EnvelopeSubject | ConversationSubject;
+export type GraderContext = {
+    evalId: string;
+    runId: string;
+    repeat: number;
+    passThreshold: number;
+    outputSchema?: unknown;
+    codeTimeoutMs?: number;
+};
+export type GradeData = NormalizedGrade;
+export type GradeCase = PlanCase;

package/dist/evals/graders/types.js

@@ -0,0 +1 @@
+export {};

package/dist/evals/judgeProviders.d.ts

@@ -0,0 +1,16 @@
+export type JudgeMapping = {
+    providerSpec: string;
+    envVar: string;
+    stageflowProvider: string;
+};
+type JudgeProviderRow = {
+    stageflowProvider: string;
+    envVar: string;
+    spec: (model: string) => string;
+};
+export declare const JUDGE_PROVIDERS: readonly JudgeProviderRow[];
+export declare function supportedJudgeProviders(): string[];
+export declare function mapJudge(modelId: string): JudgeMapping | null;
+export declare function judgeCapableModels(modelIds: readonly string[]): string[];
+export declare function unsupportedJudgeMessage(modelId: string): string;
+export {};

package/dist/evals/judgeProviders.js

@@ -0,0 +1,27 @@
+export const JUDGE_PROVIDERS = [
+    { stageflowProvider: "anthropic", envVar: "ANTHROPIC_API_KEY", spec: (m) => `anthropic:messages:${m}` },
+    { stageflowProvider: "openai", envVar: "OPENAI_API_KEY", spec: (m) => `openai:chat:${m}` },
+    { stageflowProvider: "openrouter", envVar: "OPENROUTER_API_KEY", spec: (m) => `openrouter:${m}` },
+];
+export function supportedJudgeProviders() {
+    return JUDGE_PROVIDERS.map((row) => row.stageflowProvider);
+}
+export function mapJudge(modelId) {
+    const slash = modelId.indexOf("/");
+    if (slash <= 0)
+        return null;
+    const provider = modelId.slice(0, slash);
+    const model = modelId.slice(slash + 1);
+    if (model.length === 0)
+        return null;
+    const row = JUDGE_PROVIDERS.find((entry) => entry.stageflowProvider === provider);
+    if (row === undefined)
+        return null;
+    return { providerSpec: row.spec(model), envVar: row.envVar, stageflowProvider: provider };
+}
+export function judgeCapableModels(modelIds) {
+    return modelIds.filter((id) => mapJudge(id) !== null);
+}
+export function unsupportedJudgeMessage(modelId) {
+    return `Judge model "${modelId}" cannot be used: supported judge providers are ${supportedJudgeProviders().join(", ")}.`;
+}

package/dist/evals/lineDiff.d.ts

@@ -0,0 +1,5 @@
+export interface DiffLine {
+    op: "same" | "add" | "remove";
+    text: string;
+}
+export declare function lineDiff(a: string, b: string): DiffLine[];

package/dist/evals/lineDiff.js

@@ -0,0 +1,34 @@
+function toLines(text) {
+    return text === "" ? [] : text.split("\n");
+}
+export function lineDiff(a, b) {
+    const x = toLines(a);
+    const y = toLines(b);
+    const lcs = Array.from({ length: x.length + 1 }, () => new Array(y.length + 1).fill(0));
+    for (let i = x.length - 1; i >= 0; i--) {
+        for (let j = y.length - 1; j >= 0; j--) {
+            lcs[i][j] = x[i] === y[j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
+        }
+    }
+    const out = [];
+    let i = 0;
+    let j = 0;
+    while (i < x.length && j < y.length) {
+        if (x[i] === y[j]) {
+            out.push({ op: "same", text: x[i] });
+            i++;
+            j++;
+        }
+        else if (lcs[i + 1][j] >= lcs[i][j + 1]) {
+            out.push({ op: "remove", text: x[i++] });
+        }
+        else {
+            out.push({ op: "add", text: y[j++] });
+        }
+    }
+    while (i < x.length)
+        out.push({ op: "remove", text: x[i++] });
+    while (j < y.length)
+        out.push({ op: "add", text: y[j++] });
+    return out;
+}

package/dist/evals/listStatus.d.ts

@@ -0,0 +1,31 @@
+import type { EvalType } from "./statsTypes.js";
+export type ListStatus = "never_run" | "saturated" | "passing" | "regressed" | "failing";
+export interface ListRunGrader {
+    graderId: string;
+    hash: string;
+    passRate: number | null;
+}
+export interface ListRunSummary {
+    runNumber: number;
+    passed: boolean;
+    passRate: number | null;
+    finishedAt: string | null;
+    graders: ListRunGrader[];
+}
+export declare function listStatus(completedRunsNewestFirst: ListRunSummary[], type: EvalType): ListStatus;
+export interface BannerEvalInput {
+    evalId: string;
+    status: ListStatus;
+    last: ListRunSummary | null;
+    previous: ListRunSummary | null;
+}
+export interface RegressionBanner {
+    eval_id: string;
+    previous_pass_rate: number | null;
+    last_pass_rate: number | null;
+    criteria_dropped: number;
+    last_run_number: number;
+    previous_run_number: number;
+}
+export declare function criteriaDropped(last: ListRunSummary, previous: ListRunSummary): number;
+export declare function regressionBanner(evals: BannerEvalInput[]): RegressionBanner | null;

package/dist/evals/listStatus.js

@@ -0,0 +1,48 @@
+const EPS = 1e-9;
+const SATURATED_PASS_RATE = 0.98;
+const CRITERIA_DROP_POINTS = -5;
+export function listStatus(completedRunsNewestFirst, type) {
+    const last = completedRunsNewestFirst[0];
+    if (!last)
+        return "never_run";
+    const previous = completedRunsNewestFirst[1];
+    if (last.passed) {
+        if (type === "capability" && (last.passRate ?? 0) + EPS >= SATURATED_PASS_RATE) {
+            return "saturated";
+        }
+        return "passing";
+    }
+    return previous?.passed ? "regressed" : "failing";
+}
+export function criteriaDropped(last, previous) {
+    let dropped = 0;
+    for (const g of last.graders) {
+        const prev = previous.graders.find((p) => p.graderId === g.graderId);
+        if (!prev || prev.hash !== g.hash)
+            continue;
+        if (g.passRate === null || prev.passRate === null)
+            continue;
+        if ((g.passRate - prev.passRate) * 100 <= CRITERIA_DROP_POINTS + EPS)
+            dropped += 1;
+    }
+    return dropped;
+}
+export function regressionBanner(evals) {
+    let best = null;
+    for (const e of evals) {
+        if (e.status !== "regressed" || !e.last || !e.previous)
+            continue;
+        if (!best || (e.last.finishedAt ?? "") > (best.last?.finishedAt ?? ""))
+            best = e;
+    }
+    if (!best?.last || !best.previous)
+        return null;
+    return {
+        eval_id: best.evalId,
+        previous_pass_rate: best.previous.passRate,
+        last_pass_rate: best.last.passRate,
+        criteria_dropped: criteriaDropped(best.last, best.previous),
+        last_run_number: best.last.runNumber,
+        previous_run_number: best.previous.runNumber,
+    };
+}