stageflow 0.40.0 → 0.42.0

package/dist/cli/exportAllCommand.js

@@ -135,7 +135,7 @@ function parseExportArgs(args) {
 }
 export async function* iterateExportNdjson(options) {
     yield `${JSON.stringify(buildExportHeader(options.filter, options.now ?? new Date()))}\n`;
-    const runs = await options.store.listRuns(options.filter);
+    const runs = await options.store.listRuns({ ...options.filter, origin: "any" });
     for (const summary of runs) {
         const detail = await options.store.readRun(summary.run_id);
         const pipelineBody = await options.store.readPipelineBody(summary.run_id);

package/dist/cli/hostClient.d.ts

@@ -77,5 +77,65 @@ export type HttpListTriggersResult = {
 export declare function httpFireTrigger(base: string, triggerId: string, task?: Record<string, unknown>): Promise<StartRunResult>;
 export declare function httpListTriggers(base: string): Promise<HttpListTriggersResult>;
 export declare function httpGetTrigger(base: string, triggerId: string): Promise<HttpTriggerLookupResult>;
+export type HttpEvalResult<T> = {
+    ok: true;
+    body: T;
+} | {
+    ok: false;
+    reason: string;
+    status: number;
+    code?: string;
+    findings?: unknown[];
+};
+export declare function httpListEvals(base: string, projectRoot?: string): Promise<HttpEvalResult<Record<string, unknown>>>;
+export declare function httpGetEval(base: string, evalId: string, projectRoot?: string): Promise<HttpEvalResult<Record<string, unknown>>>;
+export declare function httpValidateEval(base: string, input: {
+    project_root?: string;
+    yaml?: string;
+    path?: string;
+}): Promise<HttpEvalResult<Record<string, unknown>>>;
+export type HttpStartEvalRunInput = {
+    project_root?: string;
+    repeats?: number;
+    judge?: string;
+    model?: string;
+    user_model?: string;
+    allow_real_tools?: boolean;
+    max_cost_usd?: number;
+    case_ids?: string[];
+    tags?: string[];
+    label?: string;
+    started_via?: "console" | "cli" | "mcp";
+};
+export declare function httpStartEvalRun(base: string, evalId: string, input: HttpStartEvalRunInput): Promise<HttpEvalResult<{
+    eval_run_id: string;
+    status: string;
+    run_number?: number;
+}>>;
+export declare function httpGetEvalRun(base: string, evalRunId: string): Promise<HttpEvalResult<Record<string, any>>>;
+export declare function httpCancelEvalRun(base: string, evalRunId: string): Promise<HttpEvalResult<Record<string, any>>>;
+export declare function httpSortEvalFailures(base: string, evalRunId: string, input: {
+    classifier?: "auto" | "jev" | "judge";
+    only_unsorted?: boolean;
+    limit?: number;
+}): Promise<HttpEvalResult<Record<string, any>>>;
+export declare function httpListEvalSuggestions(base: string, evalRunId: string, status?: string): Promise<HttpEvalResult<{
+    items: Array<Record<string, any>>;
+    summary: Record<string, any>;
+}>>;
+export declare function httpListEvalRuns(base: string, query?: {
+    eval_id?: string;
+    project_root?: string;
+    limit?: number;
+    cursor?: string;
+}): Promise<HttpEvalResult<{
+    items: Array<Record<string, unknown>>;
+    next_cursor: string | null;
+}>>;
+export declare function httpDeleteEvalRun(base: string, evalRunId: string): Promise<HttpEvalResult<null>>;
+export declare function httpCompareEvalRuns(base: string, a: string, b: string): Promise<HttpEvalResult<Record<string, unknown>>>;
+export declare function httpListDatasets(base: string, projectRoot?: string): Promise<HttpEvalResult<{
+    items: Array<Record<string, unknown>>;
+}>>;
 export declare function resolveAbsolute(cwd: string, maybeRelative: string): string;
 export {};

package/dist/cli/hostClient.js

@@ -292,6 +292,77 @@ export async function httpGetTrigger(base, triggerId) {
     }
     return { ok: true, trigger: body };
 }
+function evalFailure(status, body) {
+    const rec = body && typeof body === "object" ? body : {};
+    return {
+        ok: false,
+        reason: extractError(body, status),
+        status,
+        ...(typeof rec.code === "string" ? { code: rec.code } : {}),
+        ...(Array.isArray(rec.findings) ? { findings: rec.findings } : {}),
+    };
+}
+function evalQuery(params) {
+    const parts = [];
+    for (const [key, value] of Object.entries(params)) {
+        if (value === undefined || value === "")
+            continue;
+        parts.push(`${key}=${enc(String(value))}`);
+    }
+    return parts.length > 0 ? `?${parts.join("&")}` : "";
+}
+async function evalGet(base, urlPath, expected = 200) {
+    const { status, body } = await getJson(base, urlPath);
+    return status === expected ? { ok: true, body: body } : evalFailure(status, body);
+}
+export function httpListEvals(base, projectRoot) {
+    return evalGet(base, `/api/evals${evalQuery({ project_root: projectRoot })}`);
+}
+export function httpGetEval(base, evalId, projectRoot) {
+    return evalGet(base, `/api/evals/${enc(evalId)}${evalQuery({ project_root: projectRoot })}`);
+}
+export async function httpValidateEval(base, input) {
+    const { status, body } = await postJson(base, "/api/evals/validate", input);
+    return status === 200 ? { ok: true, body: body } : evalFailure(status, body);
+}
+export async function httpStartEvalRun(base, evalId, input) {
+    const { status, body } = await postJson(base, `/api/evals/${enc(evalId)}/runs`, input);
+    return status === 202
+        ? { ok: true, body: body }
+        : evalFailure(status, body);
+}
+export function httpGetEvalRun(base, evalRunId) {
+    return evalGet(base, `/api/eval-runs/${enc(evalRunId)}`);
+}
+export async function httpCancelEvalRun(base, evalRunId) {
+    const { status, body } = await postJson(base, `/api/eval-runs/${enc(evalRunId)}/cancel`, {});
+    return status === 200 ? { ok: true, body: body } : evalFailure(status, body);
+}
+export async function httpSortEvalFailures(base, evalRunId, input) {
+    const { status, body } = await postJson(base, `/api/eval-runs/${enc(evalRunId)}/failures/sort`, input);
+    return status === 200 ? { ok: true, body: body } : evalFailure(status, body);
+}
+export function httpListEvalSuggestions(base, evalRunId, status) {
+    return evalGet(base, `/api/eval-runs/${enc(evalRunId)}/failures/suggestions${evalQuery({ status })}`);
+}
+export function httpListEvalRuns(base, query = {}) {
+    return evalGet(base, `/api/eval-runs${evalQuery(query)}`);
+}
+export async function httpDeleteEvalRun(base, evalRunId) {
+    const res = await fetch(`${base}/api/eval-runs/${enc(evalRunId)}`, {
+        method: "DELETE",
+        headers: { ...clientAuthorizationHeaders("DELETE") },
+    });
+    if (res.status === 204)
+        return { ok: true, body: null };
+    return evalFailure(res.status, await parseJsonBody(res));
+}
+export function httpCompareEvalRuns(base, a, b) {
+    return evalGet(base, `/api/eval-compare${evalQuery({ a, b })}`);
+}
+export function httpListDatasets(base, projectRoot) {
+    return evalGet(base, `/api/datasets${evalQuery({ project_root: projectRoot })}`);
+}
 export function resolveAbsolute(cwd, maybeRelative) {
     return path.resolve(cwd, maybeRelative);
 }

package/dist/cli/validateOutput.d.ts

@@ -1,6 +1,6 @@
 import type { ValidationResult } from "../config/validateCatalog.js";
-export declare const VALIDATION_SCOPE_LINE = "Scope: catalog YAML (pipelines, stages, and tasks as selected by flags). Provider auth and checkout paths are not checked.";
-export declare const VALIDATION_CHECKS = "catalog YAML (pipelines, stages, tasks as selected by flags)";
+export declare const VALIDATION_SCOPE_LINE = "Scope: catalog YAML (pipelines, stages, tasks, and evals as selected by flags). Provider auth and checkout paths are not checked.";
+export declare const VALIDATION_CHECKS = "catalog YAML (pipelines, stages, tasks, evals as selected by flags)";
 export declare function formatValidationHuman(result: ValidationResult, options?: {
     strict?: boolean;
 }): string;

package/dist/cli/validateOutput.js

@@ -1,6 +1,6 @@
 import { effectiveSeverity } from "../config/validateCatalog.js";
-export const VALIDATION_SCOPE_LINE = "Scope: catalog YAML (pipelines, stages, and tasks as selected by flags). Provider auth and checkout paths are not checked.";
-export const VALIDATION_CHECKS = "catalog YAML (pipelines, stages, tasks as selected by flags)";
+export const VALIDATION_SCOPE_LINE = "Scope: catalog YAML (pipelines, stages, tasks, and evals as selected by flags). Provider auth and checkout paths are not checked.";
+export const VALIDATION_CHECKS = "catalog YAML (pipelines, stages, tasks, evals as selected by flags)";
 function effectiveSeverityRank(finding, strict) {
     return effectiveSeverity(finding, strict) === "error" ? 0 : 1;
 }

package/dist/cli.js

@@ -20,6 +20,7 @@ import { RUN_STAGE_USAGE, runRunStageCommand } from "./cli/runStageCommand.js";
 import { RUNS_USAGE, runRunsCommand } from "./cli/runsCommand.js";
 import { AGENTS_USAGE, runAgentsCommand } from "./cli/agentsCommand.js";
 import { TRIGGER_USAGE, runTriggerCommand } from "./cli/triggerCommand.js";
+import { EVAL_USAGE, runEvalCommand } from "./cli/evalCommand.js";
 import { resolveOperatorCatalog } from "./cli/operatorCatalog.js";
 import { SKILLS_USAGE, runSkillsCommand } from "./cli/skillsCommand.js";
 import { VALIDATE_USAGE, runValidateCommand } from "./cli/validateCommand.js";
@@ -125,6 +126,8 @@ ${RUNS_USAGE}
 
 ${TRIGGER_USAGE}
 
+${EVAL_USAGE}
+
 ${AGENTS_USAGE}
 
 ${PROVIDERS_USAGE}
@@ -168,6 +171,7 @@ function parseArgs(argv) {
         command === "export" ||
         command === "runs" ||
         command === "trigger" ||
+        command === "eval" ||
         command === "agents" ||
         command === "skills" ||
         command === "browser") {
@@ -495,6 +499,12 @@ async function main(argv) {
                 cwd: ctx.invocationCwd,
             });
         }
+        if (parsed.command === "eval") {
+            return runEvalCommand(argv.slice(3), {
+                cwd: ctx.invocationCwd,
+                projectRoot: ctx.projectRoot,
+            });
+        }
         if (parsed.command === "agents") {
             return runAgentsCommand(argv.slice(3), {
                 cwd: ctx.invocationCwd,

package/dist/config/atomicWrite.d.ts

@@ -0,0 +1,15 @@
+import type { ValidationFinding } from "./validateCatalog.js";
+export type EvalsWriteErrorCode = "EVAL_EXISTS" | "EVAL_NOT_FOUND" | "EVAL_INVALID" | "EVAL_DEFINITION_ID_MISMATCH" | "DATASET_EXISTS" | "DATASET_NOT_FOUND" | "DATASET_INVALID" | "DATASET_READ_ONLY" | "DATASET_CASE_OP_INVALID" | "DATASET_NAME_INVALID" | "SAMPLE_EXISTS" | "SAMPLE_SOURCE_MISSING" | "SAMPLE_NO_DEFAULT_MODEL" | "SAMPLE_AGENTS_UNAVAILABLE";
+export declare class EvalsWriteError extends Error {
+    readonly code: EvalsWriteErrorCode;
+    readonly findings: ValidationFinding[];
+    constructor(code: EvalsWriteErrorCode, message: string, findings?: ValidationFinding[]);
+}
+export declare class FileChangedError extends Error {
+    readonly code: "FILE_CHANGED";
+    readonly currentHash: string;
+    constructor(currentHash: string);
+}
+export declare function contentHash(bytes: string | Uint8Array): string;
+export declare function assertBaseHash(filePath: string, baseHash: string): Promise<string>;
+export declare function writeFileAtomic(filePath: string, text: string): Promise<void>;

package/dist/config/atomicWrite.js

@@ -0,0 +1,43 @@
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { sha256Hex } from "../evals/canonical.js";
+export class EvalsWriteError extends Error {
+    code;
+    findings;
+    constructor(code, message, findings = []) {
+        super(message);
+        this.name = "EvalsWriteError";
+        this.code = code;
+        this.findings = findings;
+    }
+}
+export class FileChangedError extends Error {
+    code = "FILE_CHANGED";
+    currentHash;
+    constructor(currentHash) {
+        super("The file changed on disk since it was loaded");
+        this.name = "FileChangedError";
+        this.currentHash = currentHash;
+    }
+}
+export function contentHash(bytes) {
+    return sha256Hex(bytes);
+}
+export async function assertBaseHash(filePath, baseHash) {
+    const current = contentHash(await readFile(filePath));
+    if (current !== baseHash)
+        throw new FileChangedError(current);
+    return current;
+}
+export async function writeFileAtomic(filePath, text) {
+    await mkdir(path.dirname(filePath), { recursive: true });
+    const tmp = `${filePath}.tmp-${process.pid}`;
+    try {
+        await writeFile(tmp, text, "utf8");
+        await rename(tmp, filePath);
+    }
+    catch (err) {
+        await rm(tmp, { force: true });
+        throw err;
+    }
+}

package/dist/config/browseCatalog.d.ts

@@ -40,6 +40,8 @@ export type CatalogScanPaths = {
     pipelinePaths: string[];
     taskPaths: string[];
     triggerPaths: string[];
+    evalPaths: string[];
+    datasetPaths: string[];
 };
 export declare function getCatalogScanPaths(ctx: CatalogContext): Promise<CatalogScanPaths | null>;
 export declare function browseCatalog(invocationCwd: string | StageflowContext): Promise<CatalogBrowseResult>;

package/dist/config/browseCatalog.js

@@ -35,12 +35,14 @@ export async function getCatalogScanPaths(ctx) {
     if (!catalogReady(ctx)) {
         return null;
     }
-    const [pipelinePaths, taskPaths, triggerPaths] = await Promise.all([
+    const [pipelinePaths, taskPaths, triggerPaths, evalPaths, datasetPaths] = await Promise.all([
         scanCatalogPaths(ctx.manifest, "pipeline"),
         scanCatalogPaths(ctx.manifest, "task"),
         scanCatalogPaths(ctx.manifest, "trigger"),
+        scanCatalogPaths(ctx.manifest, "eval"),
+        scanCatalogPaths(ctx.manifest, "dataset"),
     ]);
-    return { pipelinePaths, taskPaths, triggerPaths };
+    return { pipelinePaths, taskPaths, triggerPaths, evalPaths, datasetPaths };
 }
 async function listTasksFromManifest(projectRoot, manifest, warnings) {
     const filePaths = await scanCatalogPaths(manifest, "task");

package/dist/config/csv.d.ts

@@ -0,0 +1,9 @@
+export declare class CsvParseError extends Error {
+    readonly record: number;
+    constructor(message: string, record: number);
+}
+export type ParseCsvOptions = {
+    delimiter?: string;
+};
+/** RFC 4180 parser. Returns raw records; blank lines are kept as `[""]`. */
+export declare function parseCsv(text: string, options?: ParseCsvOptions): string[][];

package/dist/config/csv.js

@@ -0,0 +1,81 @@
+export class CsvParseError extends Error {
+    record;
+    constructor(message, record) {
+        super(message);
+        this.name = "CsvParseError";
+        this.record = record;
+    }
+}
+/** RFC 4180 parser. Returns raw records; blank lines are kept as `[""]`. */
+export function parseCsv(text, options = {}) {
+    const delimiter = options.delimiter ?? ",";
+    if (delimiter.length !== 1 || delimiter === '"' || delimiter === "\n" || delimiter === "\r") {
+        throw new Error(`Invalid CSV delimiter: ${JSON.stringify(delimiter)}`);
+    }
+    const src = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+    const records = [];
+    let record = [];
+    let field = "";
+    let inQuotes = false;
+    let fieldStarted = false;
+    let i = 0;
+    const endField = () => {
+        record.push(field);
+        field = "";
+        fieldStarted = false;
+    };
+    const endRecord = () => {
+        endField();
+        records.push(record);
+        record = [];
+    };
+    while (i < src.length) {
+        const ch = src[i];
+        if (inQuotes) {
+            if (ch === '"') {
+                if (src[i + 1] === '"') {
+                    field += '"';
+                    i += 2;
+                    continue;
+                }
+                inQuotes = false;
+                i += 1;
+                const next = src[i];
+                if (next !== undefined && next !== delimiter && next !== "\n" && next !== "\r") {
+                    throw new CsvParseError(`unexpected character after closing quote in row ${records.length + 1}`, records.length + 1);
+                }
+                continue;
+            }
+            field += ch;
+            i += 1;
+            continue;
+        }
+        if (ch === '"') {
+            if (fieldStarted) {
+                throw new CsvParseError(`unexpected quote inside unquoted field in row ${records.length + 1}`, records.length + 1);
+            }
+            inQuotes = true;
+            fieldStarted = true;
+            i += 1;
+        }
+        else if (ch === delimiter) {
+            endField();
+            i += 1;
+        }
+        else if (ch === "\n" || ch === "\r") {
+            endRecord();
+            i += ch === "\r" && src[i + 1] === "\n" ? 2 : 1;
+        }
+        else {
+            field += ch;
+            fieldStarted = true;
+            i += 1;
+        }
+    }
+    if (inQuotes) {
+        throw new CsvParseError(`unterminated quoted field in row ${records.length + 1}`, records.length + 1);
+    }
+    if (fieldStarted || field !== "" || record.length > 0)
+        endRecord();
+    return records;
+}

package/dist/config/datasetAgentCheck.d.ts

@@ -0,0 +1,18 @@
+import type { ConversationMode } from "../evals/types.js";
+export type DatasetAgentNote = {
+    code: "dataset.scenario_unknown_field" | "dataset.expected_invalid";
+    message: string;
+};
+export type DatasetAgentCheckResult = {
+    caseId: string;
+    valid: boolean;
+    message?: string;
+    notes: DatasetAgentNote[];
+};
+export declare function checkDatasetAgainstAgent(cases: ReadonlyArray<{
+    id: string;
+    input: Record<string, unknown>;
+    expected?: unknown;
+}>, conversation?: {
+    mode?: ConversationMode;
+}): DatasetAgentCheckResult[];

package/dist/config/datasetAgentCheck.js

@@ -0,0 +1,109 @@
+const INPUT_KEYS = ["mode", "script", "opening", "persona", "goal", "facts", "tool_mocks"];
+const MOCK_MAX_BYTES = 20 * 1024;
+function isPlainObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function boundedString(value, min, max) {
+    return typeof value === "string" && value.length >= min && value.length <= max;
+}
+function stringList(value, maxItems, maxLength) {
+    return (Array.isArray(value) &&
+        value.length <= maxItems &&
+        value.every((item) => typeof item === "string" && (maxLength === undefined || item.length <= maxLength)));
+}
+function toolMocksProblem(mocks) {
+    if (!isPlainObject(mocks))
+        return "scenario tool_mocks must be an object";
+    const entries = Object.entries(mocks);
+    if (entries.length > 50)
+        return "scenario tool_mocks has more than 50 entries";
+    for (const [tool, mock] of entries) {
+        if (tool.length < 1 || tool.length > 128 || /\s/.test(tool)) {
+            return `scenario tool_mocks key ${JSON.stringify(tool)} must be 1-128 characters with no whitespace`;
+        }
+        if (!isPlainObject(mock))
+            return `scenario tool_mocks "${tool}" must be an object`;
+        const hasResult = "result" in mock;
+        const hasError = "error" in mock;
+        if (hasResult === hasError || Object.keys(mock).length !== 1) {
+            return `scenario tool_mocks "${tool}" must have exactly one of result or error`;
+        }
+        if (hasError && !boundedString(mock.error, 0, 1000)) {
+            return `scenario tool_mocks "${tool}" error must be text of at most 1000 characters`;
+        }
+        if (Buffer.byteLength(JSON.stringify(mock) ?? "", "utf8") > MOCK_MAX_BYTES) {
+            return `scenario tool_mocks "${tool}" is larger than 20 KB`;
+        }
+    }
+    return undefined;
+}
+function scenarioProblem(input, defaultMode) {
+    let mode = defaultMode;
+    if (input.mode !== undefined) {
+        if (input.mode !== "script" && input.mode !== "simulated") {
+            return `scenario mode must be "script" or "simulated"`;
+        }
+        mode = input.mode;
+    }
+    if (mode === "script") {
+        const { script } = input;
+        if (!Array.isArray(script) ||
+            script.length < 1 ||
+            script.length > 40 ||
+            !script.every((item) => boundedString(item, 1, 4000))) {
+            return "scenario script must be 1-40 messages of 1-4000 characters (script mode)";
+        }
+    }
+    else {
+        if (!boundedString(input.opening, 1, 2000)) {
+            return "scenario opening must be a message of 1-2000 characters (simulated mode)";
+        }
+        if (!boundedString(input.persona, 1, 1000)) {
+            return "scenario persona must be 1-1000 characters (simulated mode)";
+        }
+        if (!boundedString(input.goal, 1, 1000)) {
+            return "scenario goal must be 1-1000 characters (simulated mode)";
+        }
+        if (input.facts !== undefined && !stringList(input.facts, 30, 300)) {
+            return "scenario facts must be at most 30 strings of at most 300 characters";
+        }
+    }
+    if (input.tool_mocks !== undefined)
+        return toolMocksProblem(input.tool_mocks);
+    return undefined;
+}
+function expectedNotes(caseId, expected) {
+    if (!isPlainObject(expected))
+        return [];
+    const notes = [];
+    const note = (message) => {
+        notes.push({ code: "dataset.expected_invalid", message: `case ${caseId}: ${message}` });
+    };
+    if (expected.outcome !== undefined && !boundedString(expected.outcome, 0, 500)) {
+        note("expected.outcome must be a string of at most 500 characters");
+    }
+    for (const key of ["tools", "forbidden_tools"]) {
+        if (expected[key] !== undefined && !stringList(expected[key], 50)) {
+            note(`expected.${key} must be an array of at most 50 strings`);
+        }
+    }
+    return notes;
+}
+export function checkDatasetAgainstAgent(cases, conversation = {}) {
+    const defaultMode = conversation.mode ?? "script";
+    return cases.map((c) => {
+        const notes = [];
+        const unknown = Object.keys(c.input).filter((key) => !INPUT_KEYS.includes(key));
+        if (unknown.length > 0) {
+            notes.push({
+                code: "dataset.scenario_unknown_field",
+                message: `case ${c.id}: scenario has unknown field(s): ${unknown.join(", ")}`,
+            });
+        }
+        notes.push(...expectedNotes(c.id, c.expected));
+        const problem = scenarioProblem(c.input, defaultMode);
+        return problem === undefined
+            ? { caseId: c.id, valid: true, notes }
+            : { caseId: c.id, valid: false, message: problem, notes };
+    });
+}

package/dist/config/datasetCandidates.d.ts

@@ -0,0 +1,22 @@
+import type { RunStore } from "../runstore/port.js";
+import type { NewDatasetCase } from "./writeDataset.js";
+export type CandidateTarget = {
+    stageId: string;
+    pipelinePath?: string;
+};
+export type DatasetCandidate = {
+    runId: string;
+    finishedAt: string;
+    preview: string;
+};
+export declare function standalonePipelineId(stageId: string): string;
+export declare function listDatasetCandidates(input: {
+    store: RunStore;
+    target: CandidateTarget;
+    limit: number;
+}): Promise<DatasetCandidate[]>;
+export declare function casesFromRuns(input: {
+    store: RunStore;
+    runIds: readonly string[];
+    target: CandidateTarget;
+}): Promise<NewDatasetCase[]>;

package/dist/config/datasetCandidates.js

@@ -0,0 +1,86 @@
+import path from "node:path";
+import { parse as parseYaml } from "yaml";
+import { EvalsWriteError } from "./atomicWrite.js";
+const PREVIEW_CHARS = 160;
+const SHORT_ID_CHARS = 8;
+export function standalonePipelineId(stageId) {
+    return `standalone-${stageId}`;
+}
+function isPlain(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isCandidateRun(meta, target) {
+    if (meta.origin === "eval" || meta.status !== "succeeded")
+        return false;
+    if (meta.pipeline_id === standalonePipelineId(target.stageId))
+        return true;
+    if (target.pipelinePath === undefined || meta.pipeline_path === undefined)
+        return false;
+    if (path.resolve(meta.pipeline_path) !== path.resolve(target.pipelinePath))
+        return false;
+    return meta.pipeline_dag?.roots.includes(target.stageId) ?? false;
+}
+async function recordedInput(store, runId) {
+    const detail = await store.readRun(runId);
+    let task;
+    try {
+        task = parseYaml(detail.task_yaml);
+    }
+    catch {
+        return undefined;
+    }
+    if (!isPlain(task) || !isPlain(task.input))
+        return undefined;
+    return task.input;
+}
+export async function listDatasetCandidates(input) {
+    const { store, target, limit } = input;
+    const pipelines = [standalonePipelineId(target.stageId)];
+    if (target.pipelinePath !== undefined)
+        pipelines.push(target.pipelinePath);
+    const summaries = (await Promise.all(pipelines.map((pipeline) => store.listRuns({ pipeline, status: "succeeded" })))).flat();
+    const seen = new Set();
+    const ordered = summaries
+        .filter((s) => (seen.has(s.run_id) ? false : (seen.add(s.run_id), true)))
+        .sort((a, b) => (b.finished_at ?? b.created_at).localeCompare(a.finished_at ?? a.created_at));
+    const items = [];
+    for (const summary of ordered) {
+        if (items.length >= limit)
+            break;
+        const meta = await store.readRunMeta(summary.run_id);
+        if (!isCandidateRun(meta, target))
+            continue;
+        const recorded = await recordedInput(store, summary.run_id);
+        if (recorded === undefined)
+            continue;
+        items.push({
+            runId: summary.run_id,
+            finishedAt: summary.finished_at ?? summary.created_at,
+            preview: JSON.stringify(recorded).slice(0, PREVIEW_CHARS),
+        });
+    }
+    return items;
+}
+export async function casesFromRuns(input) {
+    const { store, target } = input;
+    const cases = [];
+    for (const runId of input.runIds) {
+        let meta;
+        try {
+            meta = await store.readRunMeta(runId);
+        }
+        catch {
+            throw new EvalsWriteError("DATASET_CASE_OP_INVALID", `run ${runId} not found`);
+        }
+        const recorded = isCandidateRun(meta, target) ? await recordedInput(store, runId) : undefined;
+        if (recorded === undefined) {
+            throw new EvalsWriteError("DATASET_CASE_OP_INVALID", `run ${runId} is not a completed run of stage "${target.stageId}" with a recorded input`);
+        }
+        cases.push({
+            title: `Run ${runId.slice(0, SHORT_ID_CHARS)}`,
+            input: recorded,
+            metadata: { source_run: runId },
+        });
+    }
+    return cases;
+}

package/dist/config/datasetFromSessions.d.ts

@@ -0,0 +1,34 @@
+import type { AgentSessionDetail } from "../agents/service.js";
+import type { AgentSessionListItem, SessionQuery } from "../agents/store.js";
+import type { NewDatasetCase } from "./writeDataset.js";
+/** The slice of `AgentService` that turns past sessions into scenarios. */
+export type SessionsSource = {
+    listSessions(projectRoot: string, agentId: string, query: SessionQuery): Promise<{
+        sessions: AgentSessionListItem[];
+        nextCursor: string | null;
+    }>;
+    getSession(projectRoot: string, sessionId: string): Promise<AgentSessionDetail>;
+};
+export type SessionCandidate = {
+    session_id: string;
+    updated_at: string;
+    preview: string;
+    turns: number;
+    tools: string[];
+};
+export declare class SessionCaseError extends Error {
+}
+/** Finished (idle or closed), non-eval sessions of the agent that have at least one user message, newest first. */
+export declare function listSessionCandidates(input: {
+    sessions: SessionsSource;
+    projectRoot: string;
+    agentId: string;
+    limit: number;
+}): Promise<SessionCandidate[]>;
+/** One `script` scenario per session: its user messages, plus the tools it called as the expected tools. */
+export declare function casesFromSessions(input: {
+    sessions: SessionsSource;
+    projectRoot: string;
+    agentId: string;
+    sessionIds: readonly string[];
+}): Promise<NewDatasetCase[]>;