stageflow 0.40.0 → 0.44.0

package/dist/config/loadDataset.js

@@ -0,0 +1,318 @@
+import { readFile, stat } from "node:fs/promises";
+import path from "node:path";
+import { CsvParseError, parseCsv } from "./csv.js";
+import { datasetContentHash } from "./datasetHash.js";
+export const DATASET_LIMITS = {
+    maxCases: 5000,
+    maxCaseBytes: 1024 * 1024,
+    maxFileBytes: 50 * 1024 * 1024,
+};
+export const DATASET_MIN_CASES = 5;
+const ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
+const TAG_RE = /^[a-z0-9][a-z0-9-]{0,31}$/;
+const MAX_TAGS = 10;
+const MAX_TITLE = 120;
+const CASE_KEYS = new Set(["id", "input", "title", "expected", "tags", "goal", "metadata"]);
+function isPlainObject(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+export function datasetFormatFromPath(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    if (ext === ".jsonl")
+        return "jsonl";
+    if (ext === ".json")
+        return "json";
+    if (ext === ".csv")
+        return "csv";
+    return undefined;
+}
+function finding(severity, code, filePath, message, pos) {
+    const out = {
+        severity,
+        code,
+        path: filePath,
+        message: pos ? `${pos.label}: ${message}` : message,
+        category: "dataset",
+    };
+    if (pos && pos.line > 0)
+        out.line = pos.line;
+    return out;
+}
+function parseJsonl(text, filePath, findings) {
+    const entries = [];
+    const lines = text.charCodeAt(0) === 0xfeff ? text.slice(1).split(/\r?\n/) : text.split(/\r?\n/);
+    lines.forEach((raw, idx) => {
+        if (raw.trim() === "")
+            return;
+        const pos = { line: idx + 1, label: `line ${idx + 1}` };
+        try {
+            entries.push({ value: JSON.parse(raw), pos });
+        }
+        catch (err) {
+            const detail = err instanceof Error ? err.message : String(err);
+            findings.push(finding("error", "dataset.load_error", filePath, `invalid JSON (${detail})`, pos));
+        }
+    });
+    return entries;
+}
+function parseJson(text, filePath, findings) {
+    let doc;
+    try {
+        doc = JSON.parse(text.charCodeAt(0) === 0xfeff ? text.slice(1) : text);
+    }
+    catch (err) {
+        const detail = err instanceof Error ? err.message : String(err);
+        findings.push(finding("error", "dataset.load_error", filePath, `invalid JSON (${detail})`));
+        return [];
+    }
+    let list;
+    if (Array.isArray(doc))
+        list = doc;
+    else if (isPlainObject(doc) && Array.isArray(doc.cases))
+        list = doc.cases;
+    if (!Array.isArray(list)) {
+        findings.push(finding("error", "dataset.invalid_shape", filePath, 'JSON dataset must be an array of cases or an object { "cases": [...] }'));
+        return [];
+    }
+    return list.map((value, idx) => ({
+        value,
+        pos: { line: 0, label: `case ${idx + 1}` },
+    }));
+}
+const STRING_COLUMNS = new Set(["id", "title", "goal"]);
+const JSON_BASES = new Set(["id", "title", "goal", "tags", "input", "expected", "metadata"]);
+function classifyHeader(header) {
+    const json = header.endsWith(":json");
+    const name = json ? header.slice(0, -":json".length) : header;
+    if (json && JSON_BASES.has(name))
+        return { base: name, json };
+    if (!json && (STRING_COLUMNS.has(name) || name === "tags" || name === "expected")) {
+        return { base: name, json };
+    }
+    for (const prefix of ["input.", "expected."]) {
+        if (name.startsWith(prefix) && name.length > prefix.length) {
+            return { base: prefix.slice(0, -1), key: name.slice(prefix.length), json };
+        }
+    }
+    return undefined;
+}
+function parseCsvEntries(text, filePath, findings, delimiter) {
+    let records;
+    try {
+        records = parseCsv(text, { delimiter });
+    }
+    catch (err) {
+        if (err instanceof CsvParseError) {
+            findings.push(finding("error", "dataset.load_error", filePath, err.message, {
+                line: err.record,
+                label: `row ${err.record}`,
+            }));
+            return [];
+        }
+        throw err;
+    }
+    const headerIdx = records.findIndex((r) => !(r.length === 1 && r[0] === ""));
+    if (headerIdx === -1)
+        return [];
+    const headers = records[headerIdx].map((h) => h.trim());
+    const targets = headers.map((h, col) => {
+        const target = classifyHeader(h);
+        if (!target) {
+            findings.push(finding("error", "dataset.csv_unknown_column", filePath, `unknown CSV column "${h}" (column ${col + 1}); use id, title, goal, tags, input.<name>, expected.<name> or a :json suffix`, { line: headerIdx + 1, label: `row ${headerIdx + 1}` }));
+        }
+        return target;
+    });
+    const entries = [];
+    for (let r = headerIdx + 1; r < records.length; r += 1) {
+        const cells = records[r];
+        if (cells.length === 1 && cells[0] === "")
+            continue;
+        const pos = { line: r + 1, label: `row ${r + 1}` };
+        const obj = {};
+        let broken = false;
+        targets.forEach((target, col) => {
+            const cell = cells[col];
+            if (!target || cell === undefined || cell === "")
+                return;
+            let value = cell;
+            if (target.json) {
+                try {
+                    value = JSON.parse(cell);
+                }
+                catch {
+                    findings.push(finding("error", "dataset.load_error", filePath, `column "${headers[col]}" holds invalid JSON`, pos));
+                    broken = true;
+                    return;
+                }
+            }
+            else if (target.base === "tags") {
+                value = cell
+                    .split(";")
+                    .map((t) => t.trim())
+                    .filter((t) => t !== "");
+            }
+            if (target.key !== undefined) {
+                const parent = (obj[target.base] ??= {});
+                parent[target.key] = value;
+            }
+            else {
+                obj[target.base] = value;
+            }
+        });
+        if (!broken)
+            entries.push({ value: obj, pos });
+    }
+    return entries;
+}
+function validateEntries(entries, filePath, limits, findings) {
+    const cases = [];
+    const seen = new Set();
+    let list = entries;
+    if (entries.length > limits.maxCases) {
+        findings.push(finding("error", "dataset.too_many_cases", filePath, `dataset has ${entries.length} cases; the limit is ${limits.maxCases}`));
+        list = entries.slice(0, limits.maxCases);
+    }
+    for (const { value, pos } of list) {
+        const report = (code, message) => {
+            findings.push(finding("error", code, filePath, message, pos));
+        };
+        if (!isPlainObject(value)) {
+            report("dataset.not_object", "case must be a JSON object");
+            continue;
+        }
+        let ok = true;
+        const fail = (code, message) => {
+            report(code, message);
+            ok = false;
+        };
+        for (const key of Object.keys(value)) {
+            if (!CASE_KEYS.has(key))
+                fail("dataset.unknown_key", `unknown case key "${key}"`);
+        }
+        const id = value.id;
+        if (id === undefined) {
+            fail("dataset.case_missing_id", "case has no id (hand-authored datasets must carry ids)");
+        }
+        else if (typeof id !== "string" || !ID_RE.test(id)) {
+            fail("dataset.case_invalid_id", `id ${JSON.stringify(id)} must match ${ID_RE.source}`);
+        }
+        else if (seen.has(id)) {
+            fail("dataset.duplicate_id", `duplicate id "${id}"`);
+        }
+        else {
+            seen.add(id);
+        }
+        if (value.input === undefined) {
+            fail("dataset.case_missing_input", "case has no input");
+        }
+        else if (!isPlainObject(value.input)) {
+            fail("dataset.invalid_field", "input must be an object");
+        }
+        if (value.title !== undefined && (typeof value.title !== "string" || value.title.length > MAX_TITLE)) {
+            fail("dataset.invalid_field", `title must be a string of at most ${MAX_TITLE} characters`);
+        }
+        if (value.goal !== undefined && typeof value.goal !== "string") {
+            fail("dataset.invalid_field", "goal must be a string");
+        }
+        if (value.metadata !== undefined && !isPlainObject(value.metadata)) {
+            fail("dataset.invalid_field", "metadata must be an object");
+        }
+        if (value.tags !== undefined) {
+            const tags = value.tags;
+            if (!Array.isArray(tags) || tags.length > MAX_TAGS) {
+                fail("dataset.invalid_field", `tags must be an array of at most ${MAX_TAGS} strings`);
+            }
+            else {
+                for (const tag of tags) {
+                    if (typeof tag !== "string" || !TAG_RE.test(tag)) {
+                        fail("dataset.invalid_field", `tag ${JSON.stringify(tag)} must match ${TAG_RE.source}`);
+                    }
+                }
+            }
+        }
+        if (ok && Buffer.byteLength(JSON.stringify(value), "utf8") > limits.maxCaseBytes) {
+            fail("dataset.case_too_large", `case "${String(id)}" is larger than ${limits.maxCaseBytes} bytes`);
+        }
+        if (ok)
+            cases.push(value);
+    }
+    return cases;
+}
+export function parseDatasetText(text, format, options = {}) {
+    const limits = { ...DATASET_LIMITS, ...options.limits };
+    const filePath = options.path ?? "";
+    const findings = [];
+    if (Buffer.byteLength(text, "utf8") > limits.maxFileBytes) {
+        findings.push(finding("error", "dataset.file_too_large", filePath, `dataset file is larger than ${limits.maxFileBytes} bytes`));
+        return { cases: [], findings, contentHash: datasetContentHash([]) };
+    }
+    let entries;
+    if (format === "jsonl")
+        entries = parseJsonl(text, filePath, findings);
+    else if (format === "json")
+        entries = parseJson(text, filePath, findings);
+    else
+        entries = parseCsvEntries(text, filePath, findings, options.csvDelimiter);
+    return finishParse(entries, filePath, limits, findings);
+}
+function finishParse(entries, filePath, limits, findings) {
+    const cases = validateEntries(entries, filePath, limits, findings);
+    if (entries.length === 0 && !findings.some((f) => f.severity === "error")) {
+        findings.push(finding("error", "dataset.empty", filePath, "dataset has no cases"));
+    }
+    else if (entries.length > 0 &&
+        cases.length < DATASET_MIN_CASES &&
+        !findings.some((f) => f.severity === "error")) {
+        findings.push(finding("warning", "dataset.few_cases", filePath, `dataset has ${cases.length} cases; fewer than ${DATASET_MIN_CASES} gives wide error bars`));
+    }
+    return { cases, findings, contentHash: datasetContentHash(cases) };
+}
+export function parseDatasetCases(rawCases, options = {}) {
+    const limits = { ...DATASET_LIMITS, ...options.limits };
+    const entries = rawCases.map((value, idx) => ({
+        value,
+        pos: { line: 0, label: `case ${idx + 1}` },
+    }));
+    return finishParse(entries, options.path ?? "", limits, []);
+}
+export async function loadDatasetFile(absPath, options = {}) {
+    const limits = { ...DATASET_LIMITS, ...options.limits };
+    const relPath = (options.root ? path.relative(options.root, absPath) : path.basename(absPath))
+        .split(path.sep)
+        .join("/");
+    const base = {
+        path: absPath,
+        relPath,
+        format: datasetFormatFromPath(absPath),
+        cases: [],
+        contentHash: datasetContentHash([]),
+        fileBytes: 0,
+    };
+    const fail = (code, message, fileBytes = 0) => ({
+        ...base,
+        fileBytes,
+        findings: [finding("error", code, relPath, message)],
+    });
+    if (base.format === undefined) {
+        return fail("dataset.load_error", `unknown dataset format "${path.extname(absPath)}"; use .jsonl, .json or .csv`);
+    }
+    let size;
+    try {
+        size = (await stat(absPath)).size;
+    }
+    catch (err) {
+        return fail("dataset.load_error", `cannot read dataset: ${err.message}`);
+    }
+    if (size > limits.maxFileBytes) {
+        return fail("dataset.file_too_large", `dataset file is larger than ${limits.maxFileBytes} bytes`, size);
+    }
+    let text;
+    try {
+        text = await readFile(absPath, "utf8");
+    }
+    catch (err) {
+        return fail("dataset.load_error", `cannot read dataset: ${err.message}`, size);
+    }
+    const parsed = parseDatasetText(text, base.format, { ...options, path: relPath });
+    return { ...base, ...parsed, fileBytes: size };
+}

package/dist/config/loadEval.d.ts

@@ -0,0 +1,32 @@
+import type { EvalConversation, EvalFile, ResolvedConversation, ResolvedGrader } from "../evals/types.js";
+import { type LoadOutcome } from "./loadOutcome.js";
+type UnhashedGrader = ResolvedGrader extends infer G ? (G extends unknown ? Omit<G, "hash"> : never) : never;
+export type LoadedEval = {
+    path: string;
+    relPath: string;
+    id: string;
+    fileHash: string;
+    file: EvalFile;
+    graders: ResolvedGrader[];
+};
+export declare const EVAL_DEFAULT_REPEATS = 3;
+export declare const EVAL_DEFAULT_PASS_THRESHOLD = 0.8;
+export declare const EVAL_DEFAULT_MAX_COST_USD = 2;
+export declare const EVAL_DEFAULT_CONCURRENCY = 4;
+export declare const EVAL_AGENT_DEFAULT_CONCURRENCY = 2;
+export declare const EVAL_AGENT_DEFAULT_MAX_COST_USD = 1;
+export declare const EVAL_CONVERSATION_DEFAULTS: ResolvedConversation;
+export declare function resolveConversation(conversation: EvalConversation | undefined): ResolvedConversation;
+export declare function evalFileStem(filePath: string): string;
+export type EvalPaths = {
+    pipelinePath?: string;
+    stageFile?: string;
+    datasetPath: string;
+};
+/** Paths in an eval file resolve against the eval file's directory (P6). */
+export declare function resolveEvalPaths(file: EvalFile, evalAbsPath: string): EvalPaths;
+export declare function graderHash(grader: UnhashedGrader): string;
+export declare function graderHashShort(grader: Pick<ResolvedGrader, "hash">): string;
+export declare function parseEvalOutcome(yamlText: string, filePath: string, projectRoot: string): LoadOutcome<LoadedEval>;
+export declare function loadEvalOutcome(absPath: string, projectRoot: string): Promise<LoadOutcome<LoadedEval>>;
+export {};