stageflow 0.40.0 → 0.44.0

package/dist/config/validateEval.js

@@ -0,0 +1,446 @@
+import { internalAgentsVisible } from "../operatorAgent/internalAgents.js";
+import { PROFILE_GRANTS } from "../toolkit/profiles.js";
+import { realpath, stat } from "node:fs/promises";
+import path from "node:path";
+import { mapJudge } from "../evals/judgeProviders.js";
+import { classifyTool } from "../agents/toolRisk.js";
+import { checkDatasetAgainstAgent } from "./datasetAgentCheck.js";
+import { checkDatasetAgainstStage } from "./datasetStageCheck.js";
+import { EVAL_DEFAULT_REPEATS, resolveConversation, resolveEvalPaths, } from "./loadEval.js";
+import { resolveAgentTarget, resolveAgentTargetFromFile, } from "./resolveAgentTarget.js";
+import { resolveEvalTarget } from "./resolveEvalTarget.js";
+export const EVAL_LARGE_RUN = 500;
+export const EVAL_TOO_LARGE_RUN = 5000;
+const CONVERSATION_GRADER_TYPES = ["tool_calls", "conversation_check", "goal_reached"];
+export function evalFinding(relPath, severity, code, message, location = {}) {
+    return {
+        severity,
+        code,
+        path: relPath,
+        message,
+        category: "eval",
+        ...(location.line !== undefined ? { line: location.line } : {}),
+        ...(location.column !== undefined ? { column: location.column } : {}),
+    };
+}
+export function evalFindingsFromIssues(relPath, issues) {
+    return issues.flatMap((issue) => issue.category === "eval"
+        ? [evalFinding(relPath, issue.severity ?? "error", issue.code, issue.message, issue)]
+        : []);
+}
+function valueAtPath(root, dotted) {
+    let current = root;
+    for (const segment of dotted.split(".")) {
+        if (typeof current !== "object" || current === null)
+            return undefined;
+        current = current[segment];
+    }
+    return current;
+}
+async function datasetFileFinding(loaded, datasetPath, projectRoot) {
+    const missing = (reason) => evalFinding(loaded.relPath, "error", "eval.dataset_unresolved", `Dataset "${loaded.file.dataset}" ${reason}`);
+    try {
+        const info = await stat(datasetPath);
+        if (!info.isFile())
+            return missing("is not a file");
+        const [realFile, realRoot] = await Promise.all([realpath(datasetPath), realpath(projectRoot)]);
+        const rel = path.relative(realRoot, realFile);
+        if (rel.startsWith("..") || path.isAbsolute(rel))
+            return missing("resolves outside the project root");
+    }
+    catch {
+        return missing("was not found");
+    }
+    return undefined;
+}
+function stageNeedsCheckout(target) {
+    return target.completion?.checks.some((check) => check.type === "checkout_changes") === true;
+}
+export async function validateEval(loaded, ctx) {
+    const findings = [];
+    const add = (severity, code, message) => {
+        findings.push(evalFinding(loaded.relPath, severity, code, message));
+    };
+    const { file } = loaded;
+    const sameId = (ctx.evals ?? []).filter((other) => other.id === loaded.id && path.resolve(other.absPath) !== path.resolve(loaded.path));
+    if (sameId.length > 0) {
+        const others = sameId
+            .map((other) => path.relative(ctx.projectRoot, other.absPath).split(path.sep).join("/"))
+            .join(", ");
+        add("error", "eval.duplicate_id", `Duplicate eval id "${loaded.id}" also declared in: ${others}`);
+    }
+    const targetKind = file.target.kind;
+    for (const grader of loaded.graders) {
+        const conversationTarget = targetKind === "agent" || targetKind === "internal_agent";
+        const wrongKind = conversationTarget
+            ? grader.type === "schema"
+            : CONVERSATION_GRADER_TYPES.includes(grader.type);
+        if (wrongKind) {
+            add("error", "eval.grader_not_applicable", `Grader "${grader.id}" (${grader.type}) cannot grade ${conversationTarget ? "an agent" : "a stage"} target`);
+        }
+    }
+    if (file.target.kind === "agent") {
+        return validateAgentEval(loaded, ctx, findings);
+    }
+    if (file.target.kind === "internal_agent") {
+        return validateInternalAgentEval(loaded, findings);
+    }
+    const target = await resolveEvalTarget(file, loaded.path, ctx.projectRoot);
+    let resolved;
+    if (!target.ok) {
+        findings.push(...evalFindingsFromIssues(loaded.relPath, target.issues));
+    }
+    else {
+        resolved = target.value;
+    }
+    if (resolved) {
+        if (loaded.graders.some((g) => g.type === "schema") && resolved.outputSchema === undefined) {
+            add("error", "eval.schema_grader_without_output_schema", `A schema grader needs the target stage to declare io.output.schema`);
+        }
+        if ((resolved.stageConfig.gate_kinds?.length ?? 0) > 0) {
+            add("warning", "eval.stage_has_gates", `Target stage declares gate_kinds (${resolved.stageConfig.gate_kinds?.join(", ")}); a case that stops to ask the operator is recorded as an error`);
+        }
+        if (stageNeedsCheckout(resolved)) {
+            add("warning", "eval.stage_needs_checkout", "Target stage needs a repository checkout, which eval runs do not provide in release 1");
+        }
+    }
+    const stageModel = file.defaults?.model ?? resolved?.modelResolved;
+    const unknownChecked = new Set();
+    for (const grader of loaded.graders) {
+        if (grader.type !== "rubric")
+            continue;
+        if (grader.judge === "") {
+            add("error", "eval.judge_missing", `Rubric grader "${grader.id}" has no judge: set defaults.judge or the grader's judge`);
+            continue;
+        }
+        if (ctx.isKnownModel && !unknownChecked.has(grader.judge)) {
+            unknownChecked.add(grader.judge);
+            if (!(await ctx.isKnownModel(grader.judge))) {
+                add("warning", "eval.judge_unknown_model", `Judge model "${grader.judge}" is not in the model catalog`);
+            }
+        }
+        if (stageModel !== undefined && grader.judge === stageModel) {
+            add("warning", "eval.judge_same_as_stage", `Rubric grader "${grader.id}" uses judge "${grader.judge}", the same model as the stage (self-preference bias)`);
+        }
+    }
+    const paths = resolveEvalPaths(file, loaded.path);
+    const datasetProblem = ctx.datasetInline
+        ? undefined
+        : await datasetFileFinding(loaded, paths.datasetPath, ctx.projectRoot);
+    if (datasetProblem) {
+        findings.push(datasetProblem);
+        return findings;
+    }
+    const report = ctx.datasetReport;
+    if (!report)
+        return findings;
+    const datasetErrors = report.findings.filter((f) => f.severity === "error");
+    if (datasetErrors.length > 0) {
+        add("error", "eval.dataset_invalid", `Dataset ${report.relPath} has ${datasetErrors.length} error(s); first: ${datasetErrors[0]?.message}`);
+    }
+    const mismatches = resolved
+        ? checkDatasetAgainstStage(report.cases, resolved.inputSchema).filter((r) => !r.valid)
+        : [];
+    const skipped = new Set(mismatches.map((m) => m.caseId));
+    for (const mismatch of mismatches) {
+        findings.push({
+            severity: "warning",
+            code: "dataset.input_mismatch",
+            path: report.relPath,
+            message: `case ${mismatch.caseId}: input does not match the stage's io.input.schema: ${mismatch.message}`,
+            category: "dataset",
+        });
+    }
+    if (mismatches.length > 0) {
+        add("warning", "eval.cases_skipped", `${mismatches.length} case(s) fail the stage's io.input.schema and will be skipped when a run starts; first: ${mismatches[0]?.caseId}`);
+    }
+    const runnable = report.cases.filter((c) => !skipped.has(c.id));
+    if (datasetErrors.length === 0 && runnable.length === 0) {
+        add("error", "eval.no_valid_cases", `Dataset ${report.relPath} has no valid cases to run`);
+    }
+    for (const grader of loaded.graders) {
+        if (grader.type !== "reference")
+            continue;
+        const missingIds = runnable
+            .filter((c) => {
+            const value = valueAtPath(c, grader.expected);
+            return value === undefined || value === null;
+        })
+            .map((c) => c.id);
+        if (missingIds.length > 0) {
+            add("warning", "eval.cases_missing_expected", `Reference grader "${grader.id}": ${missingIds.length} of ${runnable.length} case(s) have no value at "${grader.expected}"; first: ${missingIds[0]}`);
+        }
+    }
+    const repeats = file.defaults?.repeats ?? EVAL_DEFAULT_REPEATS;
+    const results = runnable.length * repeats;
+    if (results > EVAL_TOO_LARGE_RUN) {
+        add("error", "eval.too_large_run", `${runnable.length} case(s) x ${repeats} repeat(s) = ${results} results, over the ${EVAL_TOO_LARGE_RUN} limit`);
+    }
+    else if (results > EVAL_LARGE_RUN) {
+        add("warning", "eval.large_run", `${runnable.length} case(s) x ${repeats} repeat(s) = ${results} results, over ${EVAL_LARGE_RUN}`);
+    }
+    return findings;
+}
+function toolGranted(target, name) {
+    const ids = target.tools.map((tool) => tool.id);
+    if (ids.includes(name))
+        return true;
+    if (name.endsWith(".*")) {
+        const prefix = name.slice(0, -1);
+        return ids.some((id) => id.startsWith(prefix));
+    }
+    const dot = name.indexOf(".");
+    return dot > 0 && ids.includes(`${name.slice(0, dot)}.*`);
+}
+function isReadClass(tool, listed) {
+    return tool.risk === "read" || classifyTool(undefined, tool.id, listed).readOnly;
+}
+function nameList(names) {
+    return names.slice(0, 5).join(", ") + (names.length > 5 ? ", ..." : "");
+}
+function hasExpectedTools(c) {
+    const expected = c.expected;
+    if (typeof expected !== "object" || expected === null)
+        return false;
+    const record = expected;
+    return record.tools !== undefined || record.forbidden_tools !== undefined;
+}
+function validateInternalAgentEval(loaded, findings) {
+    const add = (severity, code, message) => {
+        findings.push(evalFinding(loaded.relPath, severity, code, message));
+    };
+    const profile = loaded.file.target.kind === "internal_agent" ? loaded.file.target.profile : "";
+    if (!internalAgentsVisible()) {
+        add("error", "eval.internal_agent_hidden", "Internal agents are hidden. Set STAGEFLOW_SHOW_INTERNAL_AGENTS=1 on the Host to eval one.");
+    }
+    const grant = PROFILE_GRANTS[profile];
+    if (!grant) {
+        add("error", "eval.internal_agent_unknown", `No internal agent has id "${profile}"`);
+    }
+    const conversation = resolveConversation(loaded.file.conversation);
+    if (conversation.tools === "real") {
+        add("error", "eval.internal_agent_real_tools", "An internal agent eval cannot use real tools. They write the operator's project and do not pass the project-agent tool gate. Use read_only or mocked.");
+    }
+    if (grant) {
+        const names = new Set(grant.tools);
+        for (const grader of loaded.graders) {
+            if (grader.type !== "tool_calls")
+                continue;
+            const mentioned = [
+                ...(grader.mustCall ?? []),
+                ...(grader.mustNotCall ?? []),
+                ...(grader.mustAskFirst ?? []),
+                ...grader.args.map((arg) => arg.tool),
+            ];
+            const missing = [...new Set(mentioned)].filter((name) => !names.has(name));
+            if (missing.length > 0) {
+                add("warning", "eval.tool_not_granted", `Grader "${grader.id}" names tool(s) ${profile} does not have: ${missing.join(", ")}`);
+            }
+        }
+    }
+    return findings;
+}
+async function validateAgentEval(loaded, ctx, findings) {
+    const add = (severity, code, message) => {
+        findings.push(evalFinding(loaded.relPath, severity, code, message));
+    };
+    const { file } = loaded;
+    const conversation = resolveConversation(file.conversation);
+    const outcome = ctx.agents
+        ? await resolveAgentTarget(file, ctx.projectRoot, ctx.agents)
+        : await resolveAgentTargetFromFile(file, ctx.projectRoot);
+    let resolved;
+    if (outcome.ok) {
+        resolved = outcome.value;
+        findings.push(...evalFindingsFromIssues(loaded.relPath, outcome.issues ?? []));
+    }
+    else {
+        findings.push(...evalFindingsFromIssues(loaded.relPath, outcome.issues));
+    }
+    if (conversation.tools === "real") {
+        if (!conversation.allowRealTools) {
+            add("error", "eval.real_tools_not_allowed", "conversation.tools is real: set allow_real_tools: true to consent to real tools");
+        }
+        else {
+            add("warning", "eval.tools_real", "Real tools can change files and call services from this machine.");
+        }
+    }
+    const agentModel = file.defaults?.model ?? resolved?.modelResolved;
+    const unknownChecked = new Set();
+    for (const grader of loaded.graders) {
+        if (grader.type !== "rubric" && grader.type !== "goal_reached")
+            continue;
+        if (grader.judge === "") {
+            add("error", "eval.judge_missing", `${grader.type === "rubric" ? "Rubric" : "Goal"} grader "${grader.id}" has no judge: set defaults.judge or the grader's judge`);
+            continue;
+        }
+        if (ctx.isKnownModel && !unknownChecked.has(grader.judge)) {
+            unknownChecked.add(grader.judge);
+            if (!(await ctx.isKnownModel(grader.judge))) {
+                add("warning", "eval.judge_unknown_model", `Judge model "${grader.judge}" is not in the model catalog`);
+            }
+        }
+        if (agentModel !== undefined && grader.judge === agentModel) {
+            add("warning", "eval.judge_same_as_stage", `Grader "${grader.id}" uses judge "${grader.judge}", the same model as the agent (self-preference bias)`);
+        }
+    }
+    if (resolved) {
+        const grantWarn = (names, where) => {
+            const missing = [...new Set(names)].filter((name) => !toolGranted(resolved, name));
+            if (missing.length > 0) {
+                add("warning", "eval.tool_not_granted", `${where} names tool(s) the agent is not granted: ${nameList(missing)}`);
+            }
+        };
+        for (const grader of loaded.graders) {
+            if (grader.type !== "tool_calls")
+                continue;
+            grantWarn([
+                ...(grader.mustCall ?? []),
+                ...(grader.mustNotCall ?? []),
+                ...(grader.mustAskFirst ?? []),
+                ...grader.args.map((arg) => arg.tool),
+            ], `Grader "${grader.id}"`);
+        }
+        grantWarn(conversation.readOnlyTools, "conversation.read_only_tools");
+        const ineffective = [...new Set(conversation.readOnlyTools)].filter((name) => {
+            const cls = classifyTool(undefined, name).class;
+            return cls !== "mcp_unknown" && cls !== "read";
+        });
+        if (ineffective.length > 0) {
+            add("error", "eval.read_only_tools_not_mcp", `conversation.read_only_tools can only declare MCP tools read-only; these built-in or Stageflow tools stay stubbed: ${nameList(ineffective)}`);
+        }
+        if (conversation.tools === "read_only") {
+            const stubbed = resolved.tools
+                .filter((tool) => !isReadClass(tool, conversation.readOnlyTools))
+                .map((tool) => tool.id);
+            if (stubbed.length > 0) {
+                add("warning", "eval.tools_unstubbed_list", `Tools that are not read-only: ${nameList(stubbed)}. These are stubbed and recorded, not executed`);
+            }
+        }
+    }
+    const paths = resolveEvalPaths(file, loaded.path);
+    const datasetProblem = ctx.datasetInline
+        ? undefined
+        : await datasetFileFinding(loaded, paths.datasetPath, ctx.projectRoot);
+    const simulatedAnywhere = (inputs) => inputs === undefined
+        ? conversation.mode === "simulated"
+        : inputs.some((input) => (input.mode ?? conversation.mode) === "simulated");
+    const checkSimulator = async (usesSimulator) => {
+        if (!usesSimulator)
+            return;
+        const userModel = file.defaults?.user_model ?? file.defaults?.judge;
+        if (userModel === undefined) {
+            add("error", "eval.user_model_missing", "Simulated conversations need a user model: set defaults.user_model or defaults.judge");
+            return;
+        }
+        if (mapJudge(userModel) === null) {
+            add("error", "eval.simulator_unsupported", `User model "${userModel}" cannot be used: supported providers are anthropic, openai, openrouter`);
+            return;
+        }
+        if (ctx.isKnownModel && !(await ctx.isKnownModel(userModel))) {
+            add("warning", "eval.simulator_unknown_model", `User model "${userModel}" is not in the model catalog`);
+        }
+        if (agentModel !== undefined && userModel === agentModel) {
+            add("warning", "eval.simulator_same_as_agent", `User model "${userModel}" is the same model as the agent`);
+        }
+    };
+    if (datasetProblem) {
+        findings.push(datasetProblem);
+        await checkSimulator(simulatedAnywhere(undefined));
+        return findings;
+    }
+    const report = ctx.datasetReport;
+    if (!report) {
+        await checkSimulator(simulatedAnywhere(undefined));
+        return findings;
+    }
+    const datasetErrors = report.findings.filter((f) => f.severity === "error");
+    if (datasetErrors.length > 0) {
+        add("error", "eval.dataset_invalid", `Dataset ${report.relPath} has ${datasetErrors.length} error(s); first: ${datasetErrors[0]?.message}`);
+    }
+    const checks = checkDatasetAgainstAgent(report.cases, { mode: conversation.mode });
+    const mismatches = checks.filter((r) => !r.valid);
+    const skipped = new Set(mismatches.map((m) => m.caseId));
+    for (const mismatch of mismatches) {
+        findings.push({
+            severity: "warning",
+            code: "dataset.input_mismatch",
+            path: report.relPath,
+            message: `case ${mismatch.caseId}: ${mismatch.message}`,
+            category: "dataset",
+        });
+    }
+    for (const note of checks.flatMap((r) => r.notes)) {
+        findings.push({
+            severity: "warning",
+            code: note.code,
+            path: report.relPath,
+            message: note.message,
+            category: "dataset",
+        });
+    }
+    if (mismatches.length > 0) {
+        add("warning", "eval.cases_skipped", `${mismatches.length} case(s) have an invalid scenario and will be skipped when a run starts; first: ${mismatches[0]?.caseId}`);
+    }
+    const runnable = report.cases.filter((c) => !skipped.has(c.id));
+    if (datasetErrors.length === 0 && runnable.length === 0) {
+        add("error", "eval.no_valid_cases", `Dataset ${report.relPath} has no valid cases to run`);
+    }
+    await checkSimulator(simulatedAnywhere(runnable.map((c) => c.input)));
+    if (conversation.tools === "mocked" && !runnable.some((c) => c.input.tool_mocks !== undefined)) {
+        add("warning", "eval.tools_mocked_no_mocks", "conversation.tools is mocked but no case has tool_mocks");
+    }
+    if (resolved) {
+        const mockNames = runnable.flatMap((c) => typeof c.input.tool_mocks === "object" && c.input.tool_mocks !== null
+            ? Object.keys(c.input.tool_mocks)
+            : []);
+        const missing = [...new Set(mockNames)].filter((name) => !toolGranted(resolved, name));
+        if (missing.length > 0) {
+            add("warning", "eval.tool_not_granted", `Case tool_mocks name tool(s) the agent is not granted: ${nameList(missing)}`);
+        }
+    }
+    for (const grader of loaded.graders) {
+        if (grader.type === "goal_reached") {
+            const missingGoal = runnable.filter((c) => {
+                const expected = c.expected;
+                const hasGoal = typeof c.input.goal === "string" && c.input.goal !== "";
+                const hasOutcome = typeof expected?.outcome === "string" && expected.outcome !== "";
+                return !hasGoal && !hasOutcome;
+            });
+            if (missingGoal.length > 0) {
+                add("warning", "eval.cases_missing_goal", `Grader "${grader.id}": ${missingGoal.length} of ${runnable.length} case(s) have no input.goal or expected.outcome; first: ${missingGoal[0]?.id}`);
+            }
+        }
+        if (grader.type === "tool_calls" &&
+            grader.mustCall === undefined &&
+            grader.mustNotCall === undefined &&
+            grader.mustAskFirst === undefined &&
+            grader.maxCalls === undefined &&
+            grader.args.length === 0) {
+            const missingIds = runnable.filter((c) => !hasExpectedTools(c)).map((c) => c.id);
+            if (missingIds.length > 0) {
+                add("warning", "eval.cases_missing_expected", `Grader "${grader.id}": ${missingIds.length} of ${runnable.length} case(s) have no expected.tools or expected.forbidden_tools; first: ${missingIds[0]}`);
+            }
+        }
+        if (grader.type === "reference") {
+            const missingIds = runnable
+                .filter((c) => {
+                const value = valueAtPath(c, grader.expected);
+                return value === undefined || value === null;
+            })
+                .map((c) => c.id);
+            if (missingIds.length > 0) {
+                add("warning", "eval.cases_missing_expected", `Reference grader "${grader.id}": ${missingIds.length} of ${runnable.length} case(s) have no value at "${grader.expected}"; first: ${missingIds[0]}`);
+            }
+        }
+    }
+    const repeats = file.defaults?.repeats ?? EVAL_DEFAULT_REPEATS;
+    const results = runnable.length * repeats;
+    if (results > EVAL_TOO_LARGE_RUN) {
+        add("error", "eval.too_large_run", `${runnable.length} case(s) x ${repeats} repeat(s) = ${results} results, over the ${EVAL_TOO_LARGE_RUN} limit`);
+    }
+    else if (results > EVAL_LARGE_RUN) {
+        add("warning", "eval.large_run", `${runnable.length} case(s) x ${repeats} repeat(s) = ${results} results, over ${EVAL_LARGE_RUN}`);
+    }
+    return findings;
+}

package/dist/config/writeDataset.d.ts

@@ -0,0 +1,54 @@
+import type { DatasetCase } from "./datasetTypes.js";
+import type { ValidationFinding } from "./validateCatalog.js";
+export type NewDatasetCase = Omit<DatasetCase, "id"> & {
+    id?: string;
+};
+export type CaseOp = {
+    op: "add";
+    case: NewDatasetCase;
+} | {
+    op: "update";
+    id: string;
+    case: NewDatasetCase;
+} | {
+    op: "delete";
+    id: string;
+} | {
+    op: "set_tags";
+    id: string;
+    tags: string[];
+};
+export type DatasetWriteResult = {
+    id: string;
+    path: string;
+    relPath: string;
+    text: string;
+    hash: string;
+    cases: DatasetCase[];
+    findings: ValidationFinding[];
+};
+export declare function renderDatasetJsonl(cases: readonly NewDatasetCase[]): string;
+export declare function createDatasetFile(input: {
+    projectRoot: string;
+    name: string;
+    cases: readonly NewDatasetCase[];
+    directory?: string;
+}): Promise<DatasetWriteResult>;
+export declare function applyCaseOps(input: {
+    projectRoot: string;
+    id: string;
+    baseHash: string;
+    ops: readonly CaseOp[];
+}): Promise<DatasetWriteResult>;
+export declare function appendOrReplaceCases(input: {
+    projectRoot: string;
+    id: string;
+    baseHash: string;
+    mode: "append" | "replace";
+    cases: readonly NewDatasetCase[];
+}): Promise<DatasetWriteResult>;
+export declare function convertDatasetToJsonl(input: {
+    projectRoot: string;
+    id: string;
+    baseHash: string;
+}): Promise<DatasetWriteResult>;

package/dist/config/writeDataset.js

@@ -0,0 +1,186 @@
+import { access } from "node:fs/promises";
+import path from "node:path";
+import { EvalsWriteError, FileChangedError, writeFileAtomic } from "./atomicWrite.js";
+import { ensureCatalogScanRoot } from "./ensureCatalogScanRoot.js";
+import { formatId, nextIdNumber } from "./datasetImport.js";
+import { assertInsideProject, datasetStem, explicitRoots, findDatasetFile, manifestForWrite, resolveWriteDir, } from "./evalWriteRoots.js";
+import { datasetFormatFromPath, loadDatasetFile, parseDatasetText } from "./loadDataset.js";
+const NAME_PATTERN = /^[a-z0-9][a-z0-9-]{0,62}$/;
+const CASE_KEY_ORDER = ["id", "title", "input", "expected", "tags", "goal", "metadata"];
+function orderedCase(c) {
+    const source = c;
+    const out = {};
+    for (const key of CASE_KEY_ORDER) {
+        if (source[key] !== undefined)
+            out[key] = source[key];
+    }
+    for (const key of Object.keys(source)) {
+        if (!(key in out) && source[key] !== undefined)
+            out[key] = source[key];
+    }
+    return out;
+}
+export function renderDatasetJsonl(cases) {
+    return cases.map((c) => `${JSON.stringify(orderedCase(c))}\n`).join("");
+}
+function assignIds(cases, existingIds) {
+    const taken = [...existingIds, ...cases.flatMap((c) => (c.id !== undefined ? [c.id] : []))];
+    let next = nextIdNumber(taken);
+    return cases.map((c) => {
+        if (c.id !== undefined)
+            return c;
+        const id = formatId(next);
+        next += 1;
+        return { ...c, id };
+    });
+}
+function assertUniqueIds(cases) {
+    const seen = new Set();
+    for (const c of cases) {
+        if (seen.has(c.id)) {
+            throw new EvalsWriteError("DATASET_CASE_OP_INVALID", `duplicate case id "${c.id}"`);
+        }
+        seen.add(c.id);
+    }
+}
+function checkedCases(text, relPath) {
+    const parsed = parseDatasetText(text, "jsonl", { path: relPath });
+    const errors = parsed.findings.filter((f) => f.severity === "error");
+    if (errors.length > 0) {
+        throw new EvalsWriteError("DATASET_INVALID", errors[0].message, parsed.findings);
+    }
+    return { cases: parsed.cases, hash: parsed.contentHash, findings: parsed.findings };
+}
+function relOf(projectRoot, abs) {
+    return path.relative(projectRoot, abs).split(path.sep).join("/");
+}
+async function exists(abs) {
+    try {
+        await access(abs);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export async function createDatasetFile(input) {
+    const { projectRoot, name } = input;
+    const manifest = await manifestForWrite(projectRoot);
+    const dir = await resolveWriteDir(projectRoot, manifest, "datasets", input.directory);
+    const absPath = path.join(dir.absDir, `${name}.dataset.jsonl`);
+    assertInsideProject(projectRoot, absPath, "dataset path");
+    if (!NAME_PATTERN.test(name)) {
+        throw new EvalsWriteError("DATASET_NAME_INVALID", `Dataset name must match ${NAME_PATTERN.source}`);
+    }
+    if ((await findDatasetFile(projectRoot, name, manifest)) !== undefined || (await exists(absPath))) {
+        throw new EvalsWriteError("DATASET_EXISTS", `Dataset "${name}" already exists`);
+    }
+    const cases = assignIds(input.cases, []);
+    assertUniqueIds(cases);
+    const text = renderDatasetJsonl(cases);
+    const relPath = relOf(projectRoot, absPath);
+    const checked = checkedCases(text, relPath);
+    await writeFileAtomic(absPath, text);
+    if (explicitRoots(manifest, "datasets") !== undefined) {
+        await ensureCatalogScanRoot(projectRoot, dir.relDir, ["datasets"]);
+    }
+    return { id: name, path: absPath, relPath, text, ...checked };
+}
+async function loadEditable(projectRoot, id, baseHash, requireJsonl) {
+    const ref = await findDatasetFile(projectRoot, id);
+    if (ref === undefined)
+        throw new EvalsWriteError("DATASET_NOT_FOUND", `Dataset "${id}" not found`);
+    const format = datasetFormatFromPath(ref.absPath);
+    if (format === undefined)
+        throw new EvalsWriteError("DATASET_INVALID", "Unknown dataset format");
+    if (requireJsonl && format !== "jsonl") {
+        throw new EvalsWriteError("DATASET_READ_ONLY", `Dataset "${id}" is .${format}; only .dataset.jsonl is editable. Convert it to JSONL first`);
+    }
+    const loaded = await loadDatasetFile(ref.absPath, { root: projectRoot });
+    if (loaded.contentHash !== baseHash)
+        throw new FileChangedError(loaded.contentHash);
+    const errors = loaded.findings.filter((f) => f.severity === "error");
+    if (errors.length > 0) {
+        throw new EvalsWriteError("DATASET_INVALID", errors[0].message, loaded.findings);
+    }
+    return { absPath: ref.absPath, relPath: ref.relPath, cases: loaded.cases, format };
+}
+async function commit(projectRoot, target, cases, baseHash) {
+    assertUniqueIds(cases);
+    const text = renderDatasetJsonl(cases);
+    const checked = checkedCases(text, target.relPath);
+    const recheck = await loadDatasetFile(target.absPath, { root: projectRoot });
+    if (recheck.contentHash !== baseHash)
+        throw new FileChangedError(recheck.contentHash);
+    await writeFileAtomic(target.absPath, text);
+    return { id: target.id, path: target.absPath, relPath: target.relPath, text, ...checked };
+}
+function opError(index, op, message) {
+    return new EvalsWriteError("DATASET_CASE_OP_INVALID", `op ${index} (${op.op}): ${message}`);
+}
+function applyOps(cases, ops) {
+    let working = [...cases];
+    ops.forEach((op, index) => {
+        if (op.op === "add") {
+            if (op.case.id !== undefined && working.some((c) => c.id === op.case.id)) {
+                throw opError(index, op, `case "${op.case.id}" already exists`);
+            }
+            const [added] = assignIds([op.case], working.map((c) => c.id));
+            working = [...working, added];
+            return;
+        }
+        const at = working.findIndex((c) => c.id === op.id);
+        if (at === -1)
+            throw opError(index, op, `case "${op.id}" not found`);
+        if (op.op === "delete") {
+            working = working.filter((_, i) => i !== at);
+        }
+        else if (op.op === "update") {
+            if (op.case.id !== undefined && op.case.id !== op.id) {
+                throw opError(index, op, `case id cannot change from "${op.id}" to "${op.case.id}"`);
+            }
+            working = working.map((c, i) => (i === at ? { ...op.case, id: op.id } : c));
+        }
+        else if (op.op === "set_tags") {
+            working = working.map((c, i) => {
+                if (i !== at)
+                    return c;
+                const { tags: _drop, ...rest } = c;
+                return op.tags.length > 0 ? { ...rest, tags: op.tags } : rest;
+            });
+        }
+        else {
+            throw opError(index, op, "unknown op");
+        }
+    });
+    return working;
+}
+export async function applyCaseOps(input) {
+    const { projectRoot, id, baseHash } = input;
+    const current = await loadEditable(projectRoot, id, baseHash, true);
+    const next = applyOps(current.cases, input.ops);
+    return commit(projectRoot, { ...current, id }, next, baseHash);
+}
+export async function appendOrReplaceCases(input) {
+    const { projectRoot, id, baseHash, mode } = input;
+    const current = await loadEditable(projectRoot, id, baseHash, true);
+    const kept = mode === "append" ? current.cases : [];
+    const added = assignIds(input.cases, kept.map((c) => c.id));
+    return commit(projectRoot, { ...current, id }, [...kept, ...added], baseHash);
+}
+export async function convertDatasetToJsonl(input) {
+    const { projectRoot, id, baseHash } = input;
+    const current = await loadEditable(projectRoot, id, baseHash, false);
+    if (current.format === "jsonl") {
+        throw new EvalsWriteError("DATASET_INVALID", `Dataset "${id}" is already .dataset.jsonl`);
+    }
+    const absPath = path.join(path.dirname(current.absPath), `${datasetStem(path.basename(current.absPath))}.dataset.jsonl`);
+    if (await exists(absPath)) {
+        throw new EvalsWriteError("DATASET_EXISTS", `${path.basename(absPath)} already exists`);
+    }
+    const relPath = relOf(projectRoot, absPath);
+    const text = renderDatasetJsonl(current.cases);
+    const checked = checkedCases(text, relPath);
+    await writeFileAtomic(absPath, text);
+    return { id: datasetStem(path.basename(absPath)), path: absPath, relPath, text, ...checked };
+}