stageflow 0.40.0 → 0.42.0

package/dist/evals/read/compare.d.ts

@@ -0,0 +1,4 @@
+import { type CompareResult } from "../compare.js";
+import type { EvalRunRow } from "../storeTypes.js";
+import { type ReadStore } from "./runs.js";
+export declare function buildCompare(store: ReadStore, a: EvalRunRow, b: EvalRunRow): Promise<CompareResult>;

package/dist/evals/read/compare.js

@@ -0,0 +1,39 @@
+import { compareRuns } from "../compare.js";
+import { toGradeRecords, toResultRecord } from "../statsAdapter.js";
+import { graderRefsOf } from "./runs.js";
+async function compareInput(store, row) {
+    const results = await store.listEvalResults(row.evalRunId);
+    const target = row.config
+        .target;
+    const isAgent = row.targetKind === "agent";
+    const definition = isAgent ? undefined : target?.stage_definition;
+    const agentDefinition = isAgent ? target?.definition : undefined;
+    const plain = (v) => (typeof v === "object" && v !== null && !Array.isArray(v) ? v : null);
+    return {
+        targetKind: row.targetKind === "agent" ? "agent" : "stage",
+        ...(isAgent
+            ? {
+                agentId: row.targetRef.agent_id ?? null,
+                agentRevision: typeof target?.revision === "number" ? target.revision : null,
+                agentDefinition: plain(agentDefinition),
+            }
+            : {}),
+        evalId: row.evalId,
+        runNumber: row.runNumber,
+        datasetHash: row.datasetHash,
+        stageVersionHash: row.targetVersionHash,
+        stageDefinition: typeof definition === "object" && definition !== null && !Array.isArray(definition)
+            ? definition
+            : null,
+        configComponents: row.configComponents,
+        results: results.map(toResultRecord),
+        grades: results.flatMap(toGradeRecords),
+        graders: graderRefsOf(row),
+        repeats: row.repeats,
+        passThreshold: row.passThreshold,
+    };
+}
+export async function buildCompare(store, a, b) {
+    const [left, right] = await Promise.all([compareInput(store, a), compareInput(store, b)]);
+    return compareRuns(left, right);
+}

package/dist/evals/read/datasets.d.ts

@@ -0,0 +1,105 @@
+import { type UploadMapping, type UploadPreviewRow } from "../../config/datasetImport.js";
+import type { DatasetCase, DatasetFormat, DatasetLoadResult } from "../../config/datasetTypes.js";
+import type { ValidationFinding } from "../../config/validateCatalog.js";
+import type { AgentCatalogPort } from "../../config/resolveAgentTarget.js";
+import type { ConversationMode } from "../types.js";
+export type InputField = {
+    name: string;
+    required: boolean;
+    type?: string;
+};
+export type DatasetTarget = {
+    pipeline_path?: string;
+    stage_id?: string;
+    file?: string;
+    agent_id?: string;
+};
+/** What an agent eval reads from a dataset: the effective conversation mode decides which scenario keys are required. */
+export type AgentDatasetUse = {
+    mode: ConversationMode;
+};
+export declare function agentInputFields(mode: ConversationMode): InputField[];
+/** The upload mapping targets an agent dataset cannot take from a column: scripts and tool mocks are JSON or JSONL only. */
+export declare const AGENT_UNSUPPORTED_MAPPING_TARGETS: readonly ["input.script", "input.tool_mocks"];
+export type DatasetListItem = {
+    project_root: string;
+    id: string;
+    path: string;
+    format: DatasetFormat | null;
+    cases: number;
+    hash: string;
+    editable: boolean;
+    used_by: string[];
+    findings_count: number;
+};
+export type DatasetQuery = {
+    offset: number;
+    limit: number;
+    tag?: string;
+    q?: string;
+    filter: "all" | "invalid" | "tagged";
+};
+export type DatasetDetail = {
+    id: string;
+    path: string;
+    format: DatasetFormat | null;
+    hash: string;
+    editable: boolean;
+    cases_total: number;
+    invalid_total: number;
+    tagged_total: number;
+    matched_total: number;
+    tags: Record<string, number>;
+    input_fields: InputField[];
+    cases: Array<DatasetCase & {
+        input_check: {
+            valid: boolean;
+            message?: string;
+        } | null;
+    }>;
+    findings: ValidationFinding[];
+    used_by: string[];
+};
+export declare function inputFieldsOf(schema: unknown): InputField[];
+export type ResolvedDatasetTarget = {
+    inputSchema: unknown;
+    stageId: string | undefined;
+    pipelineAbsPath: string | undefined;
+    agentId?: string;
+};
+export declare function resolveDatasetTarget(projectRoot: string, target: DatasetTarget, agents?: AgentCatalogPort): Promise<{
+    ok: true;
+    value: ResolvedDatasetTarget;
+} | {
+    ok: false;
+    message: string;
+}>;
+export declare function buildDatasetDetail(input: {
+    id: string;
+    loaded: DatasetLoadResult;
+    inputSchema: unknown;
+    hasEval: boolean;
+    usedBy: string[];
+    query: DatasetQuery;
+    agent?: AgentDatasetUse;
+}): DatasetDetail;
+export type PreviewInput = {
+    format: DatasetFormat;
+    content: string;
+    delimiter?: string;
+    mapping?: UploadMapping;
+    inputSchema?: unknown;
+    existingIds?: readonly string[];
+    /** Agent dataset: rows are checked as scenarios, in this mode unless the row maps `input.mode`. */
+    agent?: AgentDatasetUse;
+};
+export type PreviewResult = {
+    columns?: string[];
+    rows: UploadPreviewRow[];
+    findings: ValidationFinding[];
+    counts: {
+        ready: number;
+        skipped: number;
+    };
+};
+export declare function previewUpload(input: PreviewInput): PreviewResult;

package/dist/evals/read/datasets.js

@@ -0,0 +1,318 @@
+import path from "node:path";
+import { isInsideProjectRoot } from "../../config/catalogRelativePath.js";
+import { parseCsv, CsvParseError } from "../../config/csv.js";
+import { checkDatasetAgainstAgent } from "../../config/datasetAgentCheck.js";
+import { checkDatasetAgainstStage } from "../../config/datasetStageCheck.js";
+import { applyUploadMapping, formatId, nextIdNumber, } from "../../config/datasetImport.js";
+import { DATASET_LIMITS } from "../../config/loadDataset.js";
+import { resolveEvalTarget } from "../../config/resolveEvalTarget.js";
+import { AGENT_ID_PATTERN, isReservedAgentId } from "../../agents/agentId.js";
+import { isAgentError } from "../../agents/errors.js";
+export function agentInputFields(mode) {
+    const simulated = mode === "simulated";
+    return [
+        { name: "mode", required: false },
+        { name: "opening", required: simulated },
+        { name: "persona", required: simulated },
+        { name: "goal", required: simulated },
+        { name: "facts", required: false },
+        { name: "script", required: !simulated },
+        { name: "tool_mocks", required: false },
+    ];
+}
+/** The upload mapping targets an agent dataset cannot take from a column: scripts and tool mocks are JSON or JSONL only. */
+export const AGENT_UNSUPPORTED_MAPPING_TARGETS = ["input.script", "input.tool_mocks"];
+export function inputFieldsOf(schema) {
+    if (typeof schema !== "object" || schema === null)
+        return [];
+    const { properties, required } = schema;
+    if (typeof properties !== "object" || properties === null)
+        return [];
+    const requiredSet = new Set(Array.isArray(required) ? required.filter((r) => typeof r === "string") : []);
+    return Object.entries(properties).map(([name, prop]) => {
+        const type = typeof prop === "object" && prop !== null ? prop.type : undefined;
+        return { name, required: requiredSet.has(name), ...(typeof type === "string" ? { type } : {}) };
+    });
+}
+export async function resolveDatasetTarget(projectRoot, target, agents) {
+    if (target.agent_id !== undefined) {
+        if (target.file !== undefined || target.pipeline_path !== undefined || target.stage_id !== undefined) {
+            return { ok: false, message: "target needs either agent_id, file, or pipeline_path with stage_id" };
+        }
+        const agentId = target.agent_id;
+        if (!AGENT_ID_PATTERN.test(agentId) || isReservedAgentId(agentId)) {
+            return { ok: false, message: `"${agentId}" is not a valid agent id` };
+        }
+        if (agents) {
+            try {
+                await agents.get(projectRoot, agentId);
+            }
+            catch (err) {
+                if (isAgentError(err))
+                    return { ok: false, message: `Agent "${agentId}" was not found` };
+                throw err;
+            }
+        }
+        return { ok: true, value: { inputSchema: undefined, stageId: undefined, pipelineAbsPath: undefined, agentId } };
+    }
+    const kindFile = target.file !== undefined;
+    const kindPipeline = target.pipeline_path !== undefined && target.stage_id !== undefined;
+    if (kindFile === kindPipeline) {
+        return { ok: false, message: "target needs either file or pipeline_path with stage_id" };
+    }
+    const given = (kindFile ? target.file : target.pipeline_path);
+    if (path.isAbsolute(given) || !isInsideProjectRoot(projectRoot, path.resolve(projectRoot, given))) {
+        return { ok: false, message: "target path must be relative and inside the project" };
+    }
+    const evalAbs = path.join(projectRoot, "target-lookup.eval.yaml");
+    const file = {
+        version: 1,
+        id: "target-lookup",
+        target: kindFile
+            ? { kind: "stage", file: target.file }
+            : { kind: "stage", pipeline: target.pipeline_path, stage: target.stage_id },
+        dataset: "target-lookup.dataset.jsonl",
+        graders: [],
+    };
+    const outcome = await resolveEvalTarget(file, evalAbs, projectRoot);
+    if (!outcome.ok)
+        return { ok: false, message: outcome.issues[0]?.message ?? "target does not resolve" };
+    const { inputSchema, stageId, pipelinePath } = outcome.value;
+    return { ok: true, value: { inputSchema, stageId, pipelineAbsPath: pipelinePath } };
+}
+export function buildDatasetDetail(input) {
+    const { loaded, query } = input;
+    const checks = input.hasEval
+        ? new Map((input.agent
+            ? checkDatasetAgainstAgent(loaded.cases, { mode: input.agent.mode })
+            : checkDatasetAgainstStage(loaded.cases, input.inputSchema)).map((c) => [c.caseId, c]))
+        : undefined;
+    const tags = {};
+    let tagged = 0;
+    let invalid = 0;
+    for (const c of loaded.cases) {
+        if ((c.tags ?? []).length > 0)
+            tagged += 1;
+        for (const tag of c.tags ?? [])
+            tags[tag] = (tags[tag] ?? 0) + 1;
+        if (checks?.get(c.id)?.valid === false)
+            invalid += 1;
+    }
+    const q = query.q?.trim().toLowerCase();
+    const matched = loaded.cases.filter((c) => {
+        const check = checks?.get(c.id);
+        if (query.filter === "invalid" && check?.valid !== false)
+            return false;
+        if (query.filter === "tagged" && (c.tags ?? []).length === 0)
+            return false;
+        if (query.tag !== undefined && !(c.tags ?? []).includes(query.tag))
+            return false;
+        if (q === undefined || q === "")
+            return true;
+        return `${c.id}\n${c.title ?? ""}\n${JSON.stringify(c.input)}`.toLowerCase().includes(q);
+    });
+    const page = matched.slice(query.offset, query.offset + query.limit);
+    return {
+        id: input.id,
+        path: loaded.relPath,
+        format: loaded.format ?? null,
+        hash: loaded.contentHash,
+        editable: loaded.format === "jsonl",
+        cases_total: loaded.cases.length,
+        invalid_total: invalid,
+        tagged_total: tagged,
+        matched_total: matched.length,
+        tags,
+        input_fields: input.agent ? agentInputFields(input.agent.mode) : inputFieldsOf(input.inputSchema),
+        cases: page.map((c) => {
+            const check = checks?.get(c.id);
+            return {
+                ...c,
+                input_check: check === undefined ? null : { valid: check.valid, ...(check.message !== undefined ? { message: check.message } : {}) },
+            };
+        }),
+        findings: input.agent && input.hasEval ? [...loaded.findings, ...scenarioNotes(loaded.relPath, checks)] : loaded.findings,
+        used_by: input.usedBy,
+    };
+}
+function scenarioNotes(relPath, checks) {
+    const out = [];
+    for (const check of checks?.values() ?? []) {
+        for (const note of check.notes ?? []) {
+            out.push({ severity: "warning", code: note.code, path: relPath, message: note.message, category: "dataset" });
+        }
+    }
+    return out;
+}
+const CASE_KEYS = new Set(["id", "input", "title", "expected", "tags", "goal", "metadata"]);
+const ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
+function uploadFinding(code, message, line) {
+    return {
+        severity: "error",
+        code,
+        path: "",
+        message,
+        category: "dataset",
+        ...(line !== undefined ? { line } : {}),
+    };
+}
+function isPlain(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function parseObjectRows(format, content, findings) {
+    if (format === "jsonl") {
+        const rows = [];
+        content.split(/\r?\n/).forEach((line, idx) => {
+            if (line.trim() === "")
+                return;
+            try {
+                const value = JSON.parse(line);
+                if (isPlain(value))
+                    rows.push(value);
+                else
+                    findings.push(uploadFinding("dataset.not_object", `line ${idx + 1}: not a JSON object`, idx + 1));
+            }
+            catch {
+                findings.push(uploadFinding("dataset.load_error", `line ${idx + 1}: invalid JSON`, idx + 1));
+            }
+        });
+        return rows;
+    }
+    try {
+        const parsed = JSON.parse(content);
+        const list = Array.isArray(parsed) ? parsed : isPlain(parsed) && Array.isArray(parsed.cases) ? parsed.cases : undefined;
+        if (list === undefined) {
+            findings.push(uploadFinding("dataset.invalid_shape", "JSON must be an array of cases or { cases: [...] }"));
+            return [];
+        }
+        return list.filter(isPlain);
+    }
+    catch {
+        findings.push(uploadFinding("dataset.load_error", "content is not valid JSON"));
+        return [];
+    }
+}
+function stringCell(value) {
+    return typeof value === "string" ? value : JSON.stringify(value);
+}
+function defaultMapping(columns, inputSchema, agent) {
+    const mapping = {};
+    if (agent) {
+        for (const field of agentInputFields(agent.mode)) {
+            if (field.name === "script" || field.name === "tool_mocks")
+                continue;
+            const column = columns.find((c) => c === field.name || c === `input.${field.name}`);
+            mapping[`input.${field.name}`] =
+                column === undefined ? null : field.name === "facts" ? { column, split: "|" } : { column };
+        }
+        return mapping;
+    }
+    const fields = inputFieldsOf(inputSchema);
+    if (fields.length === 0) {
+        for (const column of columns)
+            mapping[`input.${column}`] = { column };
+        return mapping;
+    }
+    for (const field of fields) {
+        const column = columns.find((c) => c === field.name || c === `input.${field.name}`);
+        mapping[`input.${field.name}`] = column === undefined ? null : { column };
+    }
+    return mapping;
+}
+function scenarioReason(draft, mode) {
+    const [result] = checkDatasetAgainstAgent([draft], { mode });
+    return result !== undefined && !result.valid ? result.message : undefined;
+}
+function directRows(objects, inputSchema, existingIds, agent) {
+    const shaped = objects.map((raw) => (isPlain(raw.input) ? raw : { input: raw }));
+    let next = nextIdNumber([...existingIds, ...shaped.flatMap((o) => (typeof o.id === "string" ? [o.id] : []))]);
+    const cases = shaped.map((o, i) => ({ id: String(i), input: o.input }));
+    const checks = agent
+        ? checkDatasetAgainstAgent(cases, { mode: agent.mode })
+        : checkDatasetAgainstStage(cases, inputSchema);
+    return shaped.map((row, index) => {
+        const draft = { ...row, id: typeof row.id === "string" ? row.id : "" };
+        const unknownKey = Object.keys(row).find((k) => !CASE_KEYS.has(k));
+        let reason;
+        if (unknownKey !== undefined)
+            reason = `unknown key "${unknownKey}"`;
+        else if (draft.id !== "" && !ID_RE.test(draft.id))
+            reason = `id "${draft.id}" is not valid`;
+        else if (!checks[index]?.valid)
+            reason = checks[index]?.message?.replace(/\s+/g, " ").slice(0, 160);
+        if (reason !== undefined)
+            return { index, case: draft, status: "skipped", reason };
+        if (draft.id === "") {
+            draft.id = formatId(next);
+            next += 1;
+        }
+        return { index, case: draft, status: "ready" };
+    });
+}
+export function previewUpload(input) {
+    const findings = [];
+    const existingIds = input.existingIds ?? [];
+    const { agent } = input;
+    const mapOptions = agent
+        ? { caseMeta: true, check: (draft) => scenarioReason(draft, agent.mode) }
+        : {};
+    if (agent && input.mapping !== undefined) {
+        for (const target of AGENT_UNSUPPORTED_MAPPING_TARGETS) {
+            if (input.mapping[target] != null) {
+                findings.push(uploadFinding("dataset.mapping_unsupported_field", `${target} cannot be mapped from a file column; scripts and tool mocks need a JSON or JSONL file`));
+            }
+        }
+        if (findings.length > 0)
+            return { rows: [], findings, counts: { ready: 0, skipped: 0 } };
+    }
+    if (Buffer.byteLength(input.content, "utf8") > DATASET_LIMITS.maxFileBytes) {
+        findings.push(uploadFinding("dataset.file_too_large", `file is larger than ${DATASET_LIMITS.maxFileBytes} bytes`));
+        return { rows: [], findings, counts: { ready: 0, skipped: 0 } };
+    }
+    let columns;
+    let rows = [];
+    if (input.format === "csv") {
+        let records = [];
+        try {
+            records = parseCsv(input.content, input.delimiter !== undefined ? { delimiter: input.delimiter } : {});
+        }
+        catch (err) {
+            findings.push(uploadFinding("dataset.load_error", err instanceof CsvParseError ? err.message : "invalid CSV", err instanceof CsvParseError ? err.record : undefined));
+        }
+        const headerIdx = records.findIndex((r) => !(r.length === 1 && r[0] === ""));
+        if (headerIdx !== -1) {
+            const headers = records[headerIdx].map((h) => h.trim());
+            columns = headers;
+            const body = records
+                .slice(headerIdx + 1)
+                .filter((r) => !(r.length === 1 && r[0] === ""))
+                .map((cells) => Object.fromEntries(headers.map((h, i) => [h, cells[i] ?? ""])));
+            rows = applyUploadMapping(body, input.mapping ?? defaultMapping(headers, input.inputSchema, agent), input.inputSchema, existingIds, mapOptions);
+        }
+    }
+    else {
+        const objects = parseObjectRows(input.format, input.content, findings);
+        if (input.mapping !== undefined) {
+            const keys = [...new Set(objects.flatMap((o) => Object.keys(o)))];
+            columns = keys;
+            const flat = objects.map((o) => Object.fromEntries(keys.filter((k) => k in o).map((k) => [k, stringCell(o[k])])));
+            rows = applyUploadMapping(flat, input.mapping, input.inputSchema, existingIds, mapOptions);
+        }
+        else {
+            rows = directRows(objects, input.inputSchema, existingIds, agent);
+        }
+    }
+    if (rows.length === 0 && !findings.some((f) => f.severity === "error")) {
+        findings.push(uploadFinding("dataset.empty", "no rows found"));
+    }
+    if (rows.length > DATASET_LIMITS.maxCases) {
+        findings.push(uploadFinding("dataset.too_many_cases", `more than ${DATASET_LIMITS.maxCases} rows`));
+    }
+    const ready = rows.filter((r) => r.status === "ready").length;
+    return {
+        ...(columns !== undefined ? { columns } : {}),
+        rows,
+        findings,
+        counts: { ready, skipped: rows.length - ready },
+    };
+}

package/dist/evals/read/list.d.ts

@@ -0,0 +1,66 @@
+import type { EvalCatalogEntry } from "../../config/evalCatalog.js";
+import type { AgentCatalogPort } from "../../config/resolveAgentTarget.js";
+import { type BannerEvalInput, type ListStatus, type RegressionBanner } from "../listStatus.js";
+import { type ActiveRun, type ReadStore, type RunHeadline } from "./runs.js";
+export type EvalListTarget = {
+    kind: "stage";
+    pipeline_path?: string;
+    stage_id?: string;
+    file?: string;
+    label: string;
+} | {
+    kind: "agent";
+    agent_id: string;
+    agent_name: string;
+    model: string | null;
+    revision: number | null;
+    label: string;
+};
+export type EvalListItem = {
+    project_root?: string;
+    id: string;
+    name: string;
+    description: string | null;
+    type: string;
+    path: string;
+    target: EvalListTarget;
+    conversation?: {
+        mode: string;
+        max_turns: number;
+        tools: string;
+        approvals: string;
+    };
+    dataset_path: string;
+    dataset_cases: number;
+    repeats: number;
+    grader_count: number;
+    rubric_count: number;
+    valid: boolean;
+    finding_count: number;
+    status: ListStatus | "running";
+    active_run: ActiveRun | null;
+    last_run: RunHeadline | null;
+    last_run_delta_pts: number | null;
+    trend: Array<{
+        run_number: number;
+        passed: boolean;
+        pass_rate: number | null;
+        pass_threshold: number;
+    }>;
+};
+export declare function filterEvalListItems(items: readonly EvalListItem[], filter: {
+    target_kind?: string | null;
+    agent?: string | null;
+}): EvalListItem[];
+export declare function buildEvalListItems(input: {
+    store: ReadStore;
+    projectRoot: string;
+    entries: readonly EvalCatalogEntry[];
+    datasetCases: (absPath: string) => number;
+    agents?: AgentCatalogPort;
+}): Promise<{
+    items: EvalListItem[];
+    banner: RegressionBanner | null;
+    bannerInputs: BannerEvalInput[];
+}>;
+export declare function costLastSevenDays(store: Pick<ReadStore, "listEvalRuns">, projectRoot: string, now: Date): Promise<number>;