stageflow 0.40.0 → 0.44.0

package/dist/compaction/types.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Shared contracts for Stageflow's built-in compaction (docs/plans/compaction-strategy.md).
+ * Pure data types only: the layers (mask, pin, state, summary, guards) import from here.
+ */
+export type PinnedFactKind = "id" | "number" | "timestamp" | "url" | "path" | "error" | "other";
+/** A value copied verbatim out of a message so a summary cannot lose or reword it. */
+export type PinnedFact = {
+    kind: PinnedFactKind;
+    /** The exact text, e.g. "INC-4471", "48211", "2026-03-14T05:52:03Z". */
+    value: string;
+    /** A short verbatim snippet around the value (single line, capped), for orientation. */
+    context: string;
+    /** Position in the message list the facts were extracted from, and the tool when from a tool result. */
+    source: {
+        messageIndex: number;
+        toolName?: string;
+    };
+};
+/** A tool result replaced by a placeholder in the model's context. */
+export type ClearedResult = {
+    toolName: string;
+    toolCallId: string;
+    /** Where the full output can be read again (a file path or a tool + args hint). */
+    recoverHint: string;
+    bytes: number;
+    /** One line describing what the result was, for the recovery index. */
+    descriptor: string;
+};
+export type WorkingStateKind = "fact" | "decision" | "progress" | "todo" | "blocker";
+/** One keyed entry in the model's working state (the `working_state` tool). */
+export type WorkingStateEntry = {
+    key: string;
+    kind: WorkingStateKind;
+    value: string;
+    updatedAt: string;
+};
+export declare const WORKING_STATE_TOOL_NAME = "working_state";
+/** Everything the compaction summary is built from, besides the model-written part. */
+export type CompactionSections = {
+    pinned: readonly PinnedFact[];
+    cleared: readonly ClearedResult[];
+    state: readonly WorkingStateEntry[];
+};
+/** Bumped whenever the focus prompt or section layout changes (like RUBRIC_PROMPT_VERSION). */
+export declare const COMPACTION_PROMPT_VERSION = 1;

package/dist/compaction/types.js

@@ -0,0 +1,7 @@
+/**
+ * Shared contracts for Stageflow's built-in compaction (docs/plans/compaction-strategy.md).
+ * Pure data types only: the layers (mask, pin, state, summary, guards) import from here.
+ */
+export const WORKING_STATE_TOOL_NAME = "working_state";
+/** Bumped whenever the focus prompt or section layout changes (like RUBRIC_PROMPT_VERSION). */
+export const COMPACTION_PROMPT_VERSION = 1;

package/dist/compaction/workingState.d.ts

@@ -0,0 +1,56 @@
+import type { WorkingStateEntry, WorkingStateKind } from "./types.js";
+export declare const WORKING_STATE_CUSTOM_TYPE = "stageflow.working_state";
+export declare const WORKING_STATE_KINDS: readonly WorkingStateKind[];
+export declare const WORKING_STATE_MAX_KEY = 80;
+export declare const WORKING_STATE_MAX_VALUE = 600;
+export declare const WORKING_STATE_MAX_ENTRIES = 60;
+export declare const WORKING_STATE_DEFAULT_RENDER_CHARS = 6000;
+export declare const WORKING_STATE_GUIDANCE = "Older messages and tool results are trimmed automatically as the conversation grows, so keep your own working state with the working_state tool. As soon as you learn or decide something you will need later, record it: exact values and identifiers (fact), choices and their reasons (decision), what is done (progress), what remains (todo) and what is stuck (blocker). Keep each entry to one or two short lines, update an existing key instead of adding a duplicate, and delete entries that no longer matter. Never store secrets or credentials in it.";
+export type WorkingStateEvent = {
+    op: "set";
+    key: string;
+    kind: WorkingStateKind;
+    value: string;
+    at: string;
+} | {
+    op: "delete";
+    key: string;
+    at: string;
+};
+export type WorkingStateSetResult = {
+    ok: true;
+    dropped: string[];
+    replaced: boolean;
+    redacted: boolean;
+    entry: WorkingStateEntry;
+} | {
+    ok: false;
+    error: string;
+};
+export type WorkingStateInput = {
+    key: string;
+    kind: WorkingStateKind;
+    value: string;
+    updatedAt?: string;
+};
+export declare class WorkingStateStore {
+    private readonly entries;
+    get size(): number;
+    set(input: WorkingStateInput): WorkingStateSetResult;
+    delete(key: string): boolean;
+    list(): WorkingStateEntry[];
+    apply(event: WorkingStateEvent): void;
+    load(events: readonly WorkingStateEvent[]): void;
+    render(maxChars?: number): string;
+    private oldestOfKind;
+}
+export declare function reduceWorkingState(events: readonly WorkingStateEvent[]): WorkingStateStore;
+type SessionEntryLike = {
+    type?: unknown;
+    customType?: unknown;
+    data?: unknown;
+};
+export declare function workingStateEventsFromEntries(entries: readonly SessionEntryLike[]): WorkingStateEvent[];
+export declare function restoreWorkingState(entries: readonly SessionEntryLike[]): WorkingStateStore;
+export declare function getWorkingStateSections(entries: readonly SessionEntryLike[]): WorkingStateEntry[];
+export {};

package/dist/compaction/workingState.js

@@ -0,0 +1,198 @@
+import { scrubSecrets } from "./scrub.js";
+export const WORKING_STATE_CUSTOM_TYPE = "stageflow.working_state";
+export const WORKING_STATE_KINDS = [
+    "fact",
+    "decision",
+    "progress",
+    "todo",
+    "blocker",
+];
+export const WORKING_STATE_MAX_KEY = 80;
+export const WORKING_STATE_MAX_VALUE = 600;
+export const WORKING_STATE_MAX_ENTRIES = 60;
+export const WORKING_STATE_DEFAULT_RENDER_CHARS = 6000;
+export const WORKING_STATE_GUIDANCE = "Older messages and tool results are trimmed automatically as the conversation grows, so keep your own working state with the working_state tool. As soon as you learn or decide something you will need later, record it: exact values and identifiers (fact), choices and their reasons (decision), what is done (progress), what remains (todo) and what is stuck (blocker). Keep each entry to one or two short lines, update an existing key instead of adding a duplicate, and delete entries that no longer matter. Never store secrets or credentials in it.";
+const RENDER_ORDER = [
+    "blocker",
+    "todo",
+    "decision",
+    "fact",
+    "progress",
+];
+const EVICTION_ORDER = [
+    "progress",
+    "fact",
+    "decision",
+    "todo",
+    "blocker",
+];
+const HEADINGS = {
+    blocker: "Blockers",
+    todo: "Todo",
+    decision: "Decisions",
+    fact: "Facts",
+    progress: "Progress",
+};
+function isKind(value) {
+    return (typeof value === "string" &&
+        WORKING_STATE_KINDS.includes(value));
+}
+function oneLine(text) {
+    return text.replace(/\s+/g, " ").trim();
+}
+export class WorkingStateStore {
+    entries = new Map();
+    get size() {
+        return this.entries.size;
+    }
+    set(input) {
+        if (!isKind(input.kind)) {
+            return {
+                ok: false,
+                error: `kind must be one of ${WORKING_STATE_KINDS.join(", ")}`,
+            };
+        }
+        const rawKey = oneLine(String(input.key ?? ""));
+        const rawValue = oneLine(String(input.value ?? ""));
+        const key = oneLine(scrubSecrets(rawKey));
+        const value = oneLine(scrubSecrets(rawValue));
+        const redacted = key !== rawKey || value !== rawValue;
+        if (key.length === 0)
+            return { ok: false, error: "key is required" };
+        if (key.length > WORKING_STATE_MAX_KEY) {
+            return {
+                ok: false,
+                error: `key is ${key.length} chars; max ${WORKING_STATE_MAX_KEY}`,
+            };
+        }
+        if (value.length === 0)
+            return { ok: false, error: "value is required" };
+        if (value.length > WORKING_STATE_MAX_VALUE) {
+            return {
+                ok: false,
+                error: `value is ${value.length} chars; max ${WORKING_STATE_MAX_VALUE}. Shorten it.`,
+            };
+        }
+        const replaced = this.entries.delete(key);
+        const dropped = [];
+        while (this.entries.size >= WORKING_STATE_MAX_ENTRIES) {
+            const victim = this.oldestOfKind("progress");
+            if (victim === undefined) {
+                return {
+                    ok: false,
+                    error: `working state is full (${WORKING_STATE_MAX_ENTRIES} entries, none are progress). Delete entries first.`,
+                };
+            }
+            this.entries.delete(victim);
+            dropped.push(victim);
+        }
+        const entry = {
+            key,
+            kind: input.kind,
+            value,
+            updatedAt: input.updatedAt ?? new Date().toISOString(),
+        };
+        this.entries.set(key, entry);
+        return { ok: true, dropped, replaced, redacted, entry };
+    }
+    delete(key) {
+        const raw = oneLine(String(key ?? ""));
+        return this.entries.delete(raw) || this.entries.delete(oneLine(scrubSecrets(raw)));
+    }
+    list() {
+        return [...this.entries.values()];
+    }
+    apply(event) {
+        if (event.op === "set") {
+            this.set({
+                key: event.key,
+                kind: event.kind,
+                value: event.value,
+                updatedAt: event.at,
+            });
+        }
+        else if (event.op === "delete") {
+            this.delete(event.key);
+        }
+    }
+    load(events) {
+        this.entries.clear();
+        for (const event of events)
+            this.apply(event);
+    }
+    render(maxChars = WORKING_STATE_DEFAULT_RENDER_CHARS) {
+        const all = this.list();
+        if (all.length === 0)
+            return "";
+        const keep = new Set(all);
+        let omitted = 0;
+        const build = () => {
+            const lines = ["## Working state"];
+            for (const kind of RENDER_ORDER) {
+                const group = all.filter((e) => keep.has(e) && e.kind === kind);
+                if (group.length === 0)
+                    continue;
+                lines.push("", `### ${HEADINGS[kind]}`);
+                for (const e of group)
+                    lines.push(`- ${e.key}: ${e.value}`);
+            }
+            if (omitted > 0) {
+                lines.push("", `(${omitted} older entries omitted for space)`);
+            }
+            return lines.join("\n");
+        };
+        let text = build();
+        for (const kind of EVICTION_ORDER) {
+            for (const e of all) {
+                if (text.length <= maxChars)
+                    return text;
+                if (e.kind !== kind)
+                    continue;
+                keep.delete(e);
+                omitted += 1;
+                text = build();
+            }
+        }
+        return text.length <= maxChars ? text : text.slice(0, Math.max(0, maxChars));
+    }
+    oldestOfKind(kind) {
+        for (const e of this.entries.values()) {
+            if (e.kind === kind)
+                return e.key;
+        }
+        return undefined;
+    }
+}
+export function reduceWorkingState(events) {
+    const store = new WorkingStateStore();
+    for (const event of events)
+        store.apply(event);
+    return store;
+}
+function isEvent(data) {
+    if (data === null || typeof data !== "object")
+        return false;
+    const d = data;
+    if (typeof d.key !== "string")
+        return false;
+    if (d.op === "delete")
+        return true;
+    return d.op === "set" && isKind(d.kind) && typeof d.value === "string";
+}
+export function workingStateEventsFromEntries(entries) {
+    const events = [];
+    for (const entry of entries) {
+        if (entry.type === "custom" &&
+            entry.customType === WORKING_STATE_CUSTOM_TYPE &&
+            isEvent(entry.data)) {
+            events.push(entry.data);
+        }
+    }
+    return events;
+}
+export function restoreWorkingState(entries) {
+    return reduceWorkingState(workingStateEventsFromEntries(entries));
+}
+export function getWorkingStateSections(entries) {
+    return restoreWorkingState(entries).list();
+}

package/dist/config/atomicWrite.d.ts

@@ -0,0 +1,15 @@
+import type { ValidationFinding } from "./validateCatalog.js";
+export type EvalsWriteErrorCode = "EVAL_EXISTS" | "EVAL_NOT_FOUND" | "EVAL_INVALID" | "EVAL_DEFINITION_ID_MISMATCH" | "DATASET_EXISTS" | "DATASET_NOT_FOUND" | "DATASET_INVALID" | "DATASET_READ_ONLY" | "DATASET_CASE_OP_INVALID" | "DATASET_NAME_INVALID" | "SAMPLE_EXISTS" | "SAMPLE_SOURCE_MISSING" | "SAMPLE_NO_DEFAULT_MODEL" | "SAMPLE_AGENTS_UNAVAILABLE";
+export declare class EvalsWriteError extends Error {
+    readonly code: EvalsWriteErrorCode;
+    readonly findings: ValidationFinding[];
+    constructor(code: EvalsWriteErrorCode, message: string, findings?: ValidationFinding[]);
+}
+export declare class FileChangedError extends Error {
+    readonly code: "FILE_CHANGED";
+    readonly currentHash: string;
+    constructor(currentHash: string);
+}
+export declare function contentHash(bytes: string | Uint8Array): string;
+export declare function assertBaseHash(filePath: string, baseHash: string): Promise<string>;
+export declare function writeFileAtomic(filePath: string, text: string): Promise<void>;

package/dist/config/atomicWrite.js

@@ -0,0 +1,43 @@
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { sha256Hex } from "../evals/canonical.js";
+export class EvalsWriteError extends Error {
+    code;
+    findings;
+    constructor(code, message, findings = []) {
+        super(message);
+        this.name = "EvalsWriteError";
+        this.code = code;
+        this.findings = findings;
+    }
+}
+export class FileChangedError extends Error {
+    code = "FILE_CHANGED";
+    currentHash;
+    constructor(currentHash) {
+        super("The file changed on disk since it was loaded");
+        this.name = "FileChangedError";
+        this.currentHash = currentHash;
+    }
+}
+export function contentHash(bytes) {
+    return sha256Hex(bytes);
+}
+export async function assertBaseHash(filePath, baseHash) {
+    const current = contentHash(await readFile(filePath));
+    if (current !== baseHash)
+        throw new FileChangedError(current);
+    return current;
+}
+export async function writeFileAtomic(filePath, text) {
+    await mkdir(path.dirname(filePath), { recursive: true });
+    const tmp = `${filePath}.tmp-${process.pid}`;
+    try {
+        await writeFile(tmp, text, "utf8");
+        await rename(tmp, filePath);
+    }
+    catch (err) {
+        await rm(tmp, { force: true });
+        throw err;
+    }
+}

package/dist/config/browseCatalog.d.ts

@@ -40,6 +40,8 @@ export type CatalogScanPaths = {
     pipelinePaths: string[];
     taskPaths: string[];
     triggerPaths: string[];
+    evalPaths: string[];
+    datasetPaths: string[];
 };
 export declare function getCatalogScanPaths(ctx: CatalogContext): Promise<CatalogScanPaths | null>;
 export declare function browseCatalog(invocationCwd: string | StageflowContext): Promise<CatalogBrowseResult>;

package/dist/config/browseCatalog.js

@@ -35,12 +35,14 @@ export async function getCatalogScanPaths(ctx) {
     if (!catalogReady(ctx)) {
         return null;
     }
-    const [pipelinePaths, taskPaths, triggerPaths] = await Promise.all([
+    const [pipelinePaths, taskPaths, triggerPaths, evalPaths, datasetPaths] = await Promise.all([
         scanCatalogPaths(ctx.manifest, "pipeline"),
         scanCatalogPaths(ctx.manifest, "task"),
         scanCatalogPaths(ctx.manifest, "trigger"),
+        scanCatalogPaths(ctx.manifest, "eval"),
+        scanCatalogPaths(ctx.manifest, "dataset"),
     ]);
-    return { pipelinePaths, taskPaths, triggerPaths };
+    return { pipelinePaths, taskPaths, triggerPaths, evalPaths, datasetPaths };
 }
 async function listTasksFromManifest(projectRoot, manifest, warnings) {
     const filePaths = await scanCatalogPaths(manifest, "task");

package/dist/config/createPipeline.js

@@ -1,5 +1,6 @@
 import { access, mkdir, readdir, unlink, writeFile } from "node:fs/promises";
 import path from "node:path";
+import { ensureCatalogScanRoot } from "./ensureCatalogScanRoot.js";
 import { invertPredecessorEdgesToRoute, invertRouteToPredecessorEdges, parsePipelineNeeds, toPipelineNeeds, } from "./pipelineNeeds.js";
 import { STAGE_ID_PATTERN } from "./createStage.js";
 import { inferIdFromUsesPath, normalizePipelineStageEntries, toWiringRefs, } from "./normalizePipelineStageEntry.js";
@@ -548,6 +549,7 @@ export async function createPipeline(projectRoot, input) {
             error: errorFinding?.message ?? "Pipeline validation failed",
         };
     }
+    await ensureCatalogScanRoot(projectRoot, path.relative(projectRoot, pipelineDirectory));
     const loadOutcome = await loadPipelineOutcome(relPath, { cwd: projectRoot });
     if (!loadOutcome.ok) {
         await unlink(filePath).catch(() => { });

package/dist/config/createTrigger.js

@@ -6,6 +6,7 @@ import { STAGE_ID_PATTERN } from "./createStage.js";
 import { getCatalogScanPaths } from "./browseCatalog.js";
 import { catalogContextFromStageflow } from "./resolveCatalogContext.js";
 import { resolveStageflowContext } from "../project/resolveStageflowContext.js";
+import { githubMatchActionError } from "./githubTriggerAction.js";
 import { loadTriggerOutcome } from "./loadTrigger.js";
 import { collectPipelineIdsFromPaths, collectTaskIdsFromPaths } from "./validateCatalog.js";
 import { findCatalogPathById } from "../runtime/triggerCatalog.js";
@@ -56,11 +57,16 @@ export function parseEventField(raw) {
     if (raw.config !== undefined && !isPlainObject(raw.config)) {
         return { ok: false, message: "event.config must be an object" };
     }
+    const match = isPlainObject(raw.match) ? raw.match : undefined;
+    const actionError = githubMatchActionError(raw.source, match);
+    if (actionError !== undefined) {
+        return { ok: false, message: actionError };
+    }
     return {
         ok: true,
         value: {
             source: raw.source,
-            ...(isPlainObject(raw.match) ? { match: raw.match } : {}),
+            ...(match !== undefined ? { match } : {}),
             ...(isPlainObject(raw.config) ? { config: raw.config } : {}),
         },
     };

package/dist/config/csv.d.ts

@@ -0,0 +1,9 @@
+export declare class CsvParseError extends Error {
+    readonly record: number;
+    constructor(message: string, record: number);
+}
+export type ParseCsvOptions = {
+    delimiter?: string;
+};
+/** RFC 4180 parser. Returns raw records; blank lines are kept as `[""]`. */
+export declare function parseCsv(text: string, options?: ParseCsvOptions): string[][];

package/dist/config/csv.js

@@ -0,0 +1,81 @@
+export class CsvParseError extends Error {
+    record;
+    constructor(message, record) {
+        super(message);
+        this.name = "CsvParseError";
+        this.record = record;
+    }
+}
+/** RFC 4180 parser. Returns raw records; blank lines are kept as `[""]`. */
+export function parseCsv(text, options = {}) {
+    const delimiter = options.delimiter ?? ",";
+    if (delimiter.length !== 1 || delimiter === '"' || delimiter === "\n" || delimiter === "\r") {
+        throw new Error(`Invalid CSV delimiter: ${JSON.stringify(delimiter)}`);
+    }
+    const src = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+    const records = [];
+    let record = [];
+    let field = "";
+    let inQuotes = false;
+    let fieldStarted = false;
+    let i = 0;
+    const endField = () => {
+        record.push(field);
+        field = "";
+        fieldStarted = false;
+    };
+    const endRecord = () => {
+        endField();
+        records.push(record);
+        record = [];
+    };
+    while (i < src.length) {
+        const ch = src[i];
+        if (inQuotes) {
+            if (ch === '"') {
+                if (src[i + 1] === '"') {
+                    field += '"';
+                    i += 2;
+                    continue;
+                }
+                inQuotes = false;
+                i += 1;
+                const next = src[i];
+                if (next !== undefined && next !== delimiter && next !== "\n" && next !== "\r") {
+                    throw new CsvParseError(`unexpected character after closing quote in row ${records.length + 1}`, records.length + 1);
+                }
+                continue;
+            }
+            field += ch;
+            i += 1;
+            continue;
+        }
+        if (ch === '"') {
+            if (fieldStarted) {
+                throw new CsvParseError(`unexpected quote inside unquoted field in row ${records.length + 1}`, records.length + 1);
+            }
+            inQuotes = true;
+            fieldStarted = true;
+            i += 1;
+        }
+        else if (ch === delimiter) {
+            endField();
+            i += 1;
+        }
+        else if (ch === "\n" || ch === "\r") {
+            endRecord();
+            i += ch === "\r" && src[i + 1] === "\n" ? 2 : 1;
+        }
+        else {
+            field += ch;
+            fieldStarted = true;
+            i += 1;
+        }
+    }
+    if (inQuotes) {
+        throw new CsvParseError(`unterminated quoted field in row ${records.length + 1}`, records.length + 1);
+    }
+    if (fieldStarted || field !== "" || record.length > 0)
+        endRecord();
+    return records;
+}

package/dist/config/datasetAgentCheck.d.ts

@@ -0,0 +1,18 @@
+import type { ConversationMode } from "../evals/types.js";
+export type DatasetAgentNote = {
+    code: "dataset.scenario_unknown_field" | "dataset.expected_invalid";
+    message: string;
+};
+export type DatasetAgentCheckResult = {
+    caseId: string;
+    valid: boolean;
+    message?: string;
+    notes: DatasetAgentNote[];
+};
+export declare function checkDatasetAgainstAgent(cases: ReadonlyArray<{
+    id: string;
+    input: Record<string, unknown>;
+    expected?: unknown;
+}>, conversation?: {
+    mode?: ConversationMode;
+}): DatasetAgentCheckResult[];

package/dist/config/datasetAgentCheck.js

@@ -0,0 +1,109 @@
+const INPUT_KEYS = ["mode", "script", "opening", "persona", "goal", "facts", "tool_mocks"];
+const MOCK_MAX_BYTES = 20 * 1024;
+function isPlainObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function boundedString(value, min, max) {
+    return typeof value === "string" && value.length >= min && value.length <= max;
+}
+function stringList(value, maxItems, maxLength) {
+    return (Array.isArray(value) &&
+        value.length <= maxItems &&
+        value.every((item) => typeof item === "string" && (maxLength === undefined || item.length <= maxLength)));
+}
+function toolMocksProblem(mocks) {
+    if (!isPlainObject(mocks))
+        return "scenario tool_mocks must be an object";
+    const entries = Object.entries(mocks);
+    if (entries.length > 50)
+        return "scenario tool_mocks has more than 50 entries";
+    for (const [tool, mock] of entries) {
+        if (tool.length < 1 || tool.length > 128 || /\s/.test(tool)) {
+            return `scenario tool_mocks key ${JSON.stringify(tool)} must be 1-128 characters with no whitespace`;
+        }
+        if (!isPlainObject(mock))
+            return `scenario tool_mocks "${tool}" must be an object`;
+        const hasResult = "result" in mock;
+        const hasError = "error" in mock;
+        if (hasResult === hasError || Object.keys(mock).length !== 1) {
+            return `scenario tool_mocks "${tool}" must have exactly one of result or error`;
+        }
+        if (hasError && !boundedString(mock.error, 0, 1000)) {
+            return `scenario tool_mocks "${tool}" error must be text of at most 1000 characters`;
+        }
+        if (Buffer.byteLength(JSON.stringify(mock) ?? "", "utf8") > MOCK_MAX_BYTES) {
+            return `scenario tool_mocks "${tool}" is larger than 20 KB`;
+        }
+    }
+    return undefined;
+}
+function scenarioProblem(input, defaultMode) {
+    let mode = defaultMode;
+    if (input.mode !== undefined) {
+        if (input.mode !== "script" && input.mode !== "simulated") {
+            return `scenario mode must be "script" or "simulated"`;
+        }
+        mode = input.mode;
+    }
+    if (mode === "script") {
+        const { script } = input;
+        if (!Array.isArray(script) ||
+            script.length < 1 ||
+            script.length > 40 ||
+            !script.every((item) => boundedString(item, 1, 4000))) {
+            return "scenario script must be 1-40 messages of 1-4000 characters (script mode)";
+        }
+    }
+    else {
+        if (!boundedString(input.opening, 1, 2000)) {
+            return "scenario opening must be a message of 1-2000 characters (simulated mode)";
+        }
+        if (!boundedString(input.persona, 1, 1000)) {
+            return "scenario persona must be 1-1000 characters (simulated mode)";
+        }
+        if (!boundedString(input.goal, 1, 1000)) {
+            return "scenario goal must be 1-1000 characters (simulated mode)";
+        }
+        if (input.facts !== undefined && !stringList(input.facts, 30, 300)) {
+            return "scenario facts must be at most 30 strings of at most 300 characters";
+        }
+    }
+    if (input.tool_mocks !== undefined)
+        return toolMocksProblem(input.tool_mocks);
+    return undefined;
+}
+function expectedNotes(caseId, expected) {
+    if (!isPlainObject(expected))
+        return [];
+    const notes = [];
+    const note = (message) => {
+        notes.push({ code: "dataset.expected_invalid", message: `case ${caseId}: ${message}` });
+    };
+    if (expected.outcome !== undefined && !boundedString(expected.outcome, 0, 500)) {
+        note("expected.outcome must be a string of at most 500 characters");
+    }
+    for (const key of ["tools", "forbidden_tools"]) {
+        if (expected[key] !== undefined && !stringList(expected[key], 50)) {
+            note(`expected.${key} must be an array of at most 50 strings`);
+        }
+    }
+    return notes;
+}
+export function checkDatasetAgainstAgent(cases, conversation = {}) {
+    const defaultMode = conversation.mode ?? "script";
+    return cases.map((c) => {
+        const notes = [];
+        const unknown = Object.keys(c.input).filter((key) => !INPUT_KEYS.includes(key));
+        if (unknown.length > 0) {
+            notes.push({
+                code: "dataset.scenario_unknown_field",
+                message: `case ${c.id}: scenario has unknown field(s): ${unknown.join(", ")}`,
+            });
+        }
+        notes.push(...expectedNotes(c.id, c.expected));
+        const problem = scenarioProblem(c.input, defaultMode);
+        return problem === undefined
+            ? { caseId: c.id, valid: true, notes }
+            : { caseId: c.id, valid: false, message: problem, notes };
+    });
+}