stageflow 0.40.0 → 0.42.0

package/dist/config/writeEval.js

@@ -0,0 +1,398 @@
+import { access, readFile } from "node:fs/promises";
+import { isDeepStrictEqual } from "node:util";
+import path from "node:path";
+import { Document, isMap, isNode, isScalar, isSeq, parseDocument, Scalar, } from "yaml";
+import { assertBaseHash, contentHash, EvalsWriteError, FileChangedError, writeFileAtomic, } from "./atomicWrite.js";
+import { ensureCatalogScanRoot } from "./ensureCatalogScanRoot.js";
+import { assertInsideProject, explicitRoots, findEvalFile, manifestForWrite, resolveWriteDir, } from "./evalWriteRoots.js";
+import { EVAL_AGENT_DEFAULT_CONCURRENCY, EVAL_AGENT_DEFAULT_MAX_COST_USD, EVAL_CONVERSATION_DEFAULTS, EVAL_DEFAULT_CONCURRENCY, EVAL_DEFAULT_MAX_COST_USD, EVAL_DEFAULT_PASS_THRESHOLD, EVAL_DEFAULT_REPEATS, parseEvalOutcome, } from "./loadEval.js";
+import { evalFindingsFromIssues } from "./validateEval.js";
+const TOP_ORDER = [
+    "version",
+    "id",
+    "name",
+    "description",
+    "type",
+    "target",
+    "dataset",
+    "task",
+    "defaults",
+    "conversation",
+    "graders",
+    "budget",
+];
+const TARGET_ORDER = ["kind", "pipeline", "stage", "file", "agent"];
+const CONVERSATION_ORDER = [
+    "mode",
+    "max_turns",
+    "turn_timeout_ms",
+    "tools",
+    "approvals",
+    "read_only_tools",
+    "allow_real_tools",
+];
+const TASK_ORDER = ["goal", "context", "constraints"];
+const DEFAULTS_ORDER = ["repeats", "judge", "user_model", "pass_threshold", "max_cost_usd", "concurrency", "model"];
+const BUDGET_ORDER = ["latency_ms", "cost_usd"];
+const GRADER_COMMON_ORDER = ["id", "type", "description", "weight", "must_pass"];
+const GRADER_TYPE_ORDER = {
+    schema: [],
+    field: ["checks"],
+    reference: ["mode", "path", "expected", "pass_at"],
+    rubric: ["rubric", "scale", "pass_at", "judge", "examples"],
+    code: ["code"],
+    tool_calls: ["must_call", "must_not_call", "must_ask_first", "max_calls", "args"],
+    conversation_check: ["max_turns", "max_agent_questions_per_turn", "final_reply"],
+    goal_reached: ["judge", "guidance"],
+};
+const CHECK_ORDER = ["path", "op", "value"];
+const TOOL_ARG_ORDER = ["tool", "path", "op", "value"];
+const FINAL_REPLY_ORDER = ["op", "value"];
+const FLOW_LISTS = new Set(["checks", "args", "final_reply"]);
+function isPlain(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function ordered(value, order) {
+    const out = {};
+    for (const key of order) {
+        if (value[key] !== undefined)
+            out[key] = value[key];
+    }
+    for (const key of Object.keys(value)) {
+        if (!(key in out) && value[key] !== undefined)
+            out[key] = value[key];
+    }
+    return out;
+}
+function graderOrder(grader) {
+    const typeKeys = typeof grader.type === "string" ? (GRADER_TYPE_ORDER[grader.type] ?? []) : [];
+    return [...GRADER_COMMON_ORDER, ...typeKeys];
+}
+function orderedGrader(grader) {
+    const out = ordered(grader, graderOrder(grader));
+    if (Array.isArray(out.checks)) {
+        out.checks = out.checks.map((c) => (isPlain(c) ? ordered(c, CHECK_ORDER) : c));
+    }
+    if (Array.isArray(out.args)) {
+        out.args = out.args.map((c) => (isPlain(c) ? ordered(c, TOOL_ARG_ORDER) : c));
+    }
+    if (Array.isArray(out.final_reply)) {
+        out.final_reply = out.final_reply.map((c) => (isPlain(c) ? ordered(c, FINAL_REPLY_ORDER) : c));
+    }
+    if (Array.isArray(out.examples)) {
+        out.examples = out.examples.map((e) => isPlain(e) ? ordered(e, ["verdict", "score", "output", "reason"]) : e);
+    }
+    return out;
+}
+function orderedDefinition(definition) {
+    if (!isPlain(definition))
+        return definition;
+    const out = ordered(definition, TOP_ORDER);
+    const subs = [
+        ["target", TARGET_ORDER],
+        ["task", TASK_ORDER],
+        ["defaults", DEFAULTS_ORDER],
+        ["conversation", CONVERSATION_ORDER],
+        ["budget", BUDGET_ORDER],
+    ];
+    for (const [key, order] of subs) {
+        if (isPlain(out[key]))
+            out[key] = ordered(out[key], order);
+    }
+    if (Array.isArray(out.graders)) {
+        out.graders = out.graders.map((g) => (isPlain(g) ? orderedGrader(g) : g));
+    }
+    return out;
+}
+function styleNode(node, keys) {
+    if (isMap(node)) {
+        for (const pair of node.items) {
+            const key = isScalar(pair.key) ? String(pair.key.value) : String(pair.key);
+            styleNode(pair.value, [...keys, key]);
+        }
+    }
+    else if (isSeq(node)) {
+        for (const item of node.items) {
+            if (FLOW_LISTS.has(keys[keys.length - 1] ?? "") && isMap(item))
+                item.flow = true;
+            styleNode(item, [...keys, "[]"]);
+        }
+    }
+    else if (isScalar(node) && typeof node.value === "string") {
+        if (keys[keys.length - 1] === "output" && keys[keys.length - 3] === "examples") {
+            node.type = Scalar.QUOTE_DOUBLE;
+        }
+        else if (node.value.includes("\n")) {
+            node.type = Scalar.BLOCK_LITERAL;
+        }
+    }
+}
+const STRINGIFY = { indent: 2, lineWidth: 0 };
+function toText(doc) {
+    const text = doc.toString(STRINGIFY);
+    return text.endsWith("\n") ? text : `${text}\n`;
+}
+export function renderEvalYaml(definition) {
+    const doc = new Document(orderedDefinition(definition));
+    styleNode(doc.contents, []);
+    return toText(doc);
+}
+function fillCreateDefaults(definition) {
+    if (definition.target?.kind === "agent") {
+        const conversation = { ...definition.conversation };
+        const d = EVAL_CONVERSATION_DEFAULTS;
+        const explicit = {
+            mode: d.mode,
+            max_turns: d.maxTurns,
+            turn_timeout_ms: d.turnTimeoutMs,
+            tools: d.tools,
+            approvals: d.approvals,
+        };
+        for (const [key, value] of Object.entries(explicit))
+            conversation[key] ??= value;
+        if (Array.isArray(conversation.read_only_tools) && conversation.read_only_tools.length === 0) {
+            delete conversation.read_only_tools;
+        }
+        if (conversation.allow_real_tools === false)
+            delete conversation.allow_real_tools;
+        return {
+            ...definition,
+            defaults: {
+                repeats: EVAL_DEFAULT_REPEATS,
+                pass_threshold: EVAL_DEFAULT_PASS_THRESHOLD,
+                max_cost_usd: EVAL_AGENT_DEFAULT_MAX_COST_USD,
+                concurrency: EVAL_AGENT_DEFAULT_CONCURRENCY,
+                ...definition.defaults,
+            },
+            conversation: conversation,
+        };
+    }
+    return {
+        ...definition,
+        defaults: {
+            repeats: EVAL_DEFAULT_REPEATS,
+            pass_threshold: EVAL_DEFAULT_PASS_THRESHOLD,
+            max_cost_usd: EVAL_DEFAULT_MAX_COST_USD,
+            ...definition.defaults,
+        },
+    };
+}
+export function renderEvalCreateYaml(definition) {
+    return renderEvalYaml(fillCreateDefaults(definition));
+}
+function isOmittableDefault(keys, value, parent, agent) {
+    const key = keys[keys.length - 1];
+    if (keys.length === 1)
+        return key === "type" && value === "capability";
+    if (keys.length === 2 && keys[0] === "defaults") {
+        return ((key === "repeats" && value === EVAL_DEFAULT_REPEATS) ||
+            (key === "pass_threshold" && value === EVAL_DEFAULT_PASS_THRESHOLD) ||
+            (key === "max_cost_usd" && value === (agent ? EVAL_AGENT_DEFAULT_MAX_COST_USD : EVAL_DEFAULT_MAX_COST_USD)) ||
+            (key === "concurrency" && value === (agent ? EVAL_AGENT_DEFAULT_CONCURRENCY : EVAL_DEFAULT_CONCURRENCY)));
+    }
+    if (keys.length === 3 && keys[0] === "graders") {
+        if (key === "weight")
+            return value === 1;
+        if (key === "must_pass")
+            return value === false;
+        if (key === "scale")
+            return value === "pass_fail";
+        if (key === "path")
+            return parent.type === "reference" && value === (agent ? "final_reply" : "payload");
+        if (key === "pass_at") {
+            if (parent.type === "reference")
+                return value === 1;
+            if (parent.type === "rubric")
+                return parent.scale === "1_to_5" && value === 4;
+        }
+    }
+    return false;
+}
+function create(doc, value, keys) {
+    const node = doc.createNode(value);
+    styleNode(node, keys);
+    return node;
+}
+function carryComments(from, to) {
+    if (!isNode(from) || !isNode(to))
+        return;
+    if (from.comment !== undefined)
+        to.comment = from.comment;
+    if (from.commentBefore !== undefined)
+        to.commentBefore = from.commentBefore;
+    if (from.spaceBefore)
+        to.spaceBefore = true;
+}
+function replace(doc, old, value, keys) {
+    const node = create(doc, value, keys);
+    carryComments(old, node);
+    return node;
+}
+function pairKey(pair) {
+    return isScalar(pair.key) ? String(pair.key.value) : String(pair.key);
+}
+function syncValue(doc, node, value, keys, agent) {
+    if (isPlain(value)) {
+        if (!isMap(node))
+            return replace(doc, node, value, keys);
+        syncMap(doc, node, value, keys, agent);
+        return node;
+    }
+    if (Array.isArray(value)) {
+        if (!isSeq(node))
+            return replace(doc, node, value, keys);
+        syncSeq(doc, node, value, keys, agent);
+        return node;
+    }
+    if (isScalar(node) && (value === null || typeof value !== "object")) {
+        if (node.value !== value) {
+            node.value = value;
+            if (typeof value === "string" && value.includes("\n")) {
+                if (node.type !== Scalar.BLOCK_FOLDED)
+                    node.type = Scalar.BLOCK_LITERAL;
+            }
+            else if (node.type === Scalar.BLOCK_LITERAL || node.type === Scalar.BLOCK_FOLDED) {
+                node.type = undefined;
+            }
+        }
+        return node;
+    }
+    return replace(doc, node, value, keys);
+}
+function syncMap(doc, map, value, keys, agent) {
+    const desired = Object.keys(value).filter((k) => value[k] !== undefined);
+    map.items = map.items.filter((pair) => desired.includes(pairKey(pair)));
+    let lastIndex = -1;
+    for (const key of desired) {
+        const childKeys = [...keys, key];
+        const index = map.items.findIndex((pair) => pairKey(pair) === key);
+        if (index >= 0) {
+            const pair = map.items[index];
+            pair.value = syncValue(doc, pair.value, value[key], childKeys, agent);
+            lastIndex = index;
+            continue;
+        }
+        if (isOmittableDefault(childKeys, value[key], value, agent))
+            continue;
+        const pair = doc.createPair(key, create(doc, value[key], childKeys));
+        map.items.splice(lastIndex + 1, 0, pair);
+        lastIndex += 1;
+    }
+}
+function itemId(node) {
+    if (!isMap(node))
+        return undefined;
+    const id = node.get("id");
+    return typeof id === "string" ? id : undefined;
+}
+function syncSeq(doc, seq, value, keys, agent) {
+    const byId = keys[keys.length - 1] === "graders";
+    const childKeys = [...keys, "[]"];
+    const used = new Set();
+    const next = value.map((item, index) => {
+        let existing;
+        if (byId) {
+            const id = isPlain(item) && typeof item.id === "string" ? item.id : undefined;
+            existing = seq.items.find((n) => !used.has(n) && id !== undefined && itemId(n) === id);
+        }
+        else {
+            existing = seq.items[index];
+        }
+        if (existing === undefined)
+            return create(doc, item, childKeys);
+        used.add(existing);
+        return syncValue(doc, existing, item, childKeys, agent);
+    });
+    seq.items = next;
+}
+export function previewEvalUpdate(currentText, definition) {
+    const doc = parseDocument(currentText);
+    if (doc.errors.length > 0 || !isMap(doc.contents))
+        return renderEvalYaml(definition);
+    const target = orderedDefinition(definition);
+    if (!isPlain(target))
+        return renderEvalYaml(definition);
+    if (isDeepStrictEqual(doc.toJS(), JSON.parse(JSON.stringify(target))))
+        return currentText;
+    const targetKind = isPlain(target.target) ? target.target.kind : undefined;
+    syncMap(doc, doc.contents, target, [], targetKind === "agent");
+    return toText(doc);
+}
+function relOf(projectRoot, abs) {
+    return path.relative(projectRoot, abs).split(path.sep).join("/");
+}
+function validateEvalText(text, absPath, projectRoot) {
+    const outcome = parseEvalOutcome(text, absPath, projectRoot);
+    const relPath = relOf(projectRoot, absPath);
+    if (!outcome.ok) {
+        const findings = evalFindingsFromIssues(relPath, outcome.issues);
+        throw new EvalsWriteError("EVAL_INVALID", findings.find((f) => f.severity === "error")?.message ?? "Eval definition is invalid", findings);
+    }
+    return evalFindingsFromIssues(relPath, outcome.issues ?? []);
+}
+async function exists(abs) {
+    try {
+        await access(abs);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export async function createEvalFile(input) {
+    const { projectRoot, definition } = input;
+    const manifest = await manifestForWrite(projectRoot);
+    const dir = await resolveWriteDir(projectRoot, manifest, "evals", input.directory);
+    const rawId = definition.id;
+    const fileName = `${typeof rawId === "string" ? rawId : "new"}.eval.yaml`;
+    const absPath = path.join(dir.absDir, fileName);
+    assertInsideProject(projectRoot, absPath, "eval path");
+    if (typeof rawId === "string") {
+        if ((await findEvalFile(projectRoot, rawId, manifest)) !== undefined || (await exists(absPath))) {
+            throw new EvalsWriteError("EVAL_EXISTS", `Eval "${rawId}" already exists`);
+        }
+    }
+    const text = renderEvalYaml(fillCreateDefaults(definition));
+    const findings = validateEvalText(text, absPath, projectRoot);
+    await writeFileAtomic(absPath, text);
+    const named = explicitRoots(manifest, "evals");
+    if (named !== undefined)
+        await ensureCatalogScanRoot(projectRoot, dir.relDir, ["evals"]);
+    return {
+        id: rawId,
+        path: absPath,
+        relPath: relOf(projectRoot, absPath),
+        text,
+        fileHash: contentHash(text),
+        findings,
+    };
+}
+export async function updateEvalFile(input) {
+    const { projectRoot, id, baseHash } = input;
+    if ((input.definition === undefined) === (input.yaml === undefined)) {
+        throw new EvalsWriteError("EVAL_INVALID", "Provide exactly one of definition or yaml");
+    }
+    const ref = await findEvalFile(projectRoot, id);
+    if (ref === undefined)
+        throw new EvalsWriteError("EVAL_NOT_FOUND", `Eval "${id}" not found`);
+    const current = await readFile(ref.absPath);
+    const currentHash = contentHash(current);
+    if (currentHash !== baseHash)
+        throw new FileChangedError(currentHash);
+    const text = input.yaml !== undefined
+        ? input.yaml.endsWith("\n")
+            ? input.yaml
+            : `${input.yaml}\n`
+        : previewEvalUpdate(current.toString("utf8"), input.definition);
+    const findings = validateEvalText(text, ref.absPath, projectRoot);
+    await assertBaseHash(ref.absPath, baseHash);
+    await writeFileAtomic(ref.absPath, text);
+    return {
+        id,
+        path: ref.absPath,
+        relPath: ref.relPath,
+        text,
+        fileHash: contentHash(text),
+        findings,
+    };
+}

package/dist/diagnostics/checks.js

@@ -49,7 +49,7 @@ function checkHomeWritable(home) {
 }
 async function checkStoreOpenable(store) {
     try {
-        await store.listRuns();
+        await store.listRuns({ origin: "any" });
         return true;
     }
     catch {

package/dist/evals/agentDiff.d.ts

@@ -0,0 +1,13 @@
+import { type DiffLine } from "./lineDiff.js";
+export interface AgentDiff {
+    changed_keys: string[];
+    instructions_diff: DiffLine[] | null;
+    revision: {
+        a: number | null;
+        b: number | null;
+    };
+}
+export declare function agentDiff(defA: Record<string, unknown>, defB: Record<string, unknown>, revision: {
+    a: number | null;
+    b: number | null;
+}): AgentDiff;

package/dist/evals/agentDiff.js

@@ -0,0 +1,11 @@
+import { lineDiff } from "./lineDiff.js";
+import { stageDiff } from "./stageDiff.js";
+export function agentDiff(defA, defB, revision) {
+    const changed = stageDiff(defA, defB).changed_keys;
+    const text = (v) => (typeof v === "string" ? v : v === undefined || v === null ? "" : JSON.stringify(v));
+    return {
+        changed_keys: changed,
+        instructions_diff: changed.includes("instructions") ? lineDiff(text(defA.instructions), text(defB.instructions)) : null,
+        revision,
+    };
+}

package/dist/evals/canonical.d.ts

@@ -0,0 +1,3 @@
+export declare function canonicalJson(value: unknown): string;
+export declare function sha256Hex(input: string | Uint8Array): string;
+export declare function hash6(fullHash: string): string;

package/dist/evals/canonical.js

@@ -0,0 +1,24 @@
+import { createHash } from "node:crypto";
+function sortValue(value) {
+    if (Array.isArray(value))
+        return value.map(sortValue);
+    if (value !== null && typeof value === "object") {
+        const out = {};
+        for (const key of Object.keys(value).sort()) {
+            const v = value[key];
+            if (v !== undefined)
+                out[key] = sortValue(v);
+        }
+        return out;
+    }
+    return value;
+}
+export function canonicalJson(value) {
+    return JSON.stringify(sortValue(value));
+}
+export function sha256Hex(input) {
+    return createHash("sha256").update(input).digest("hex");
+}
+export function hash6(fullHash) {
+    return fullHash.slice(0, 6);
+}

package/dist/evals/caseTask.d.ts

@@ -0,0 +1,3 @@
+import type { TaskFile } from "../types/task.js";
+import type { EvalPlan, PlanCase } from "./types.js";
+export declare function caseToTask(plan: Pick<EvalPlan, "task">, testCase: Pick<PlanCase, "id" | "goal" | "input">, evalRunId: string, repeatIndex: number): TaskFile;

package/dist/evals/caseTask.js

@@ -0,0 +1,9 @@
+export function caseToTask(plan, testCase, evalRunId, repeatIndex) {
+    return {
+        id: `eval-${evalRunId}-${testCase.id}-${repeatIndex}`,
+        goal: testCase.goal ?? plan.task.goal ?? `Evaluate case ${testCase.id}`,
+        ...(plan.task.context !== undefined ? { context: plan.task.context } : {}),
+        ...(plan.task.constraints !== undefined ? { constraints: plan.task.constraints } : {}),
+        input: testCase.input,
+    };
+}

package/dist/evals/compare.d.ts

@@ -0,0 +1,94 @@
+import { type Interval, type Stats } from "./stats.js";
+import { type AgentDiff } from "./agentDiff.js";
+import { type StageDiff } from "./stageDiff.js";
+import { type GradeRecord, type GraderRef, type ResultRecord } from "./statsTypes.js";
+export interface CompareRunInput {
+    evalId: string;
+    runNumber: number;
+    datasetHash: string;
+    stageVersionHash: string;
+    stageDefinition: Record<string, unknown> | null;
+    targetKind?: "stage" | "agent";
+    agentId?: string | null;
+    agentRevision?: number | null;
+    agentDefinition?: Record<string, unknown> | null;
+    configComponents: Record<string, unknown>;
+    results: ResultRecord[];
+    grades: GradeRecord[];
+    graders: GraderRef[];
+    repeats: number;
+    passThreshold: number;
+}
+export type Verdict = "improved" | "regressed" | "no_clear_difference" | "insufficient_data";
+export type CaseChange = "improved" | "regressed" | "same";
+export interface CaseDetail {
+    case_id: string;
+    change: CaseChange;
+    p_a: number;
+    p_b: number;
+    d: number;
+    passes_a: boolean[];
+    passes_b: boolean[];
+    mean_score_a: number | null;
+    mean_score_b: number | null;
+    graders_changed: string[];
+}
+export interface GraderComparison {
+    grader_id: string;
+    status: "compared" | "changed" | "added" | "removed";
+    pass_rate_a: number | null;
+    pass_rate_b: number | null;
+    delta: number | null;
+}
+export interface ComponentDiff {
+    key: string;
+    status: "same" | "changed";
+    a: unknown;
+    b: unknown;
+}
+export type CompareWarningCode = "dataset_differs" | "graders_differ" | "engine_differs" | "few_shared_cases";
+export interface CompareWarning {
+    code: CompareWarningCode;
+    shared_cases?: number;
+    cases_a?: number;
+    cases_b?: number;
+}
+export interface NumberDelta {
+    a: number | null;
+    b: number | null;
+    delta: number | null;
+    percent_change: number | null;
+}
+export interface CompareResult {
+    shared_cases: number;
+    cases_a: number;
+    cases_b: number;
+    mean_difference: number | null;
+    ci: Interval | null;
+    verdict: Verdict;
+    cases: CaseDetail[];
+    graders: GraderComparison[];
+    deltas: {
+        cost_per_result: NumberDelta;
+        latency_p50: NumberDelta;
+        latency_p95: NumberDelta;
+        flaky_cases: NumberDelta;
+        mean_score: NumberDelta;
+        mean_turns?: NumberDelta;
+        mean_tool_calls?: NumberDelta;
+    };
+    stage_diff: StageDiff | null;
+    agent_diff?: AgentDiff | null;
+    components: ComponentDiff[];
+    same_config: boolean;
+    warnings: CompareWarning[];
+    sd_d: number | null;
+    cases_needed_5pts: number | null;
+    stats_a: Stats;
+    stats_b: Stats;
+}
+export declare class CompareMismatchError extends Error {
+    readonly code = "EVAL_COMPARE_MISMATCH";
+    constructor(a: string, b: string, what?: string);
+}
+export declare function compareRuns(a: CompareRunInput, b: CompareRunInput): CompareResult;