stageflow 0.40.0 → 0.44.0

package/dist/config/datasetCandidates.d.ts

@@ -0,0 +1,22 @@
+import type { RunStore } from "../runstore/port.js";
+import type { NewDatasetCase } from "./writeDataset.js";
+export type CandidateTarget = {
+    stageId: string;
+    pipelinePath?: string;
+};
+export type DatasetCandidate = {
+    runId: string;
+    finishedAt: string;
+    preview: string;
+};
+export declare function standalonePipelineId(stageId: string): string;
+export declare function listDatasetCandidates(input: {
+    store: RunStore;
+    target: CandidateTarget;
+    limit: number;
+}): Promise<DatasetCandidate[]>;
+export declare function casesFromRuns(input: {
+    store: RunStore;
+    runIds: readonly string[];
+    target: CandidateTarget;
+}): Promise<NewDatasetCase[]>;

package/dist/config/datasetCandidates.js

@@ -0,0 +1,86 @@
+import path from "node:path";
+import { parse as parseYaml } from "yaml";
+import { EvalsWriteError } from "./atomicWrite.js";
+const PREVIEW_CHARS = 160;
+const SHORT_ID_CHARS = 8;
+export function standalonePipelineId(stageId) {
+    return `standalone-${stageId}`;
+}
+function isPlain(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isCandidateRun(meta, target) {
+    if (meta.origin === "eval" || meta.status !== "succeeded")
+        return false;
+    if (meta.pipeline_id === standalonePipelineId(target.stageId))
+        return true;
+    if (target.pipelinePath === undefined || meta.pipeline_path === undefined)
+        return false;
+    if (path.resolve(meta.pipeline_path) !== path.resolve(target.pipelinePath))
+        return false;
+    return meta.pipeline_dag?.roots.includes(target.stageId) ?? false;
+}
+async function recordedInput(store, runId) {
+    const detail = await store.readRun(runId);
+    let task;
+    try {
+        task = parseYaml(detail.task_yaml);
+    }
+    catch {
+        return undefined;
+    }
+    if (!isPlain(task) || !isPlain(task.input))
+        return undefined;
+    return task.input;
+}
+export async function listDatasetCandidates(input) {
+    const { store, target, limit } = input;
+    const pipelines = [standalonePipelineId(target.stageId)];
+    if (target.pipelinePath !== undefined)
+        pipelines.push(target.pipelinePath);
+    const summaries = (await Promise.all(pipelines.map((pipeline) => store.listRuns({ pipeline, status: "succeeded" })))).flat();
+    const seen = new Set();
+    const ordered = summaries
+        .filter((s) => (seen.has(s.run_id) ? false : (seen.add(s.run_id), true)))
+        .sort((a, b) => (b.finished_at ?? b.created_at).localeCompare(a.finished_at ?? a.created_at));
+    const items = [];
+    for (const summary of ordered) {
+        if (items.length >= limit)
+            break;
+        const meta = await store.readRunMeta(summary.run_id);
+        if (!isCandidateRun(meta, target))
+            continue;
+        const recorded = await recordedInput(store, summary.run_id);
+        if (recorded === undefined)
+            continue;
+        items.push({
+            runId: summary.run_id,
+            finishedAt: summary.finished_at ?? summary.created_at,
+            preview: JSON.stringify(recorded).slice(0, PREVIEW_CHARS),
+        });
+    }
+    return items;
+}
+export async function casesFromRuns(input) {
+    const { store, target } = input;
+    const cases = [];
+    for (const runId of input.runIds) {
+        let meta;
+        try {
+            meta = await store.readRunMeta(runId);
+        }
+        catch {
+            throw new EvalsWriteError("DATASET_CASE_OP_INVALID", `run ${runId} not found`);
+        }
+        const recorded = isCandidateRun(meta, target) ? await recordedInput(store, runId) : undefined;
+        if (recorded === undefined) {
+            throw new EvalsWriteError("DATASET_CASE_OP_INVALID", `run ${runId} is not a completed run of stage "${target.stageId}" with a recorded input`);
+        }
+        cases.push({
+            title: `Run ${runId.slice(0, SHORT_ID_CHARS)}`,
+            input: recorded,
+            metadata: { source_run: runId },
+        });
+    }
+    return cases;
+}

package/dist/config/datasetFromSessions.d.ts

@@ -0,0 +1,34 @@
+import type { AgentSessionDetail } from "../agents/service.js";
+import type { AgentSessionListItem, SessionQuery } from "../agents/store.js";
+import type { NewDatasetCase } from "./writeDataset.js";
+/** The slice of `AgentService` that turns past sessions into scenarios. */
+export type SessionsSource = {
+    listSessions(projectRoot: string, agentId: string, query: SessionQuery): Promise<{
+        sessions: AgentSessionListItem[];
+        nextCursor: string | null;
+    }>;
+    getSession(projectRoot: string, sessionId: string): Promise<AgentSessionDetail>;
+};
+export type SessionCandidate = {
+    session_id: string;
+    updated_at: string;
+    preview: string;
+    turns: number;
+    tools: string[];
+};
+export declare class SessionCaseError extends Error {
+}
+/** Finished (idle or closed), non-eval sessions of the agent that have at least one user message, newest first. */
+export declare function listSessionCandidates(input: {
+    sessions: SessionsSource;
+    projectRoot: string;
+    agentId: string;
+    limit: number;
+}): Promise<SessionCandidate[]>;
+/** One `script` scenario per session: its user messages, plus the tools it called as the expected tools. */
+export declare function casesFromSessions(input: {
+    sessions: SessionsSource;
+    projectRoot: string;
+    agentId: string;
+    sessionIds: readonly string[];
+}): Promise<NewDatasetCase[]>;

package/dist/config/datasetFromSessions.js

@@ -0,0 +1,105 @@
+import { isAgentError } from "../agents/errors.js";
+export class SessionCaseError extends Error {
+}
+const PREVIEW_CHARS = 160;
+const SCRIPT_MAX_MESSAGES = 40;
+const SCRIPT_MAX_CHARS = 4000;
+const SESSION_TAG = "from-session";
+function userMessages(detail) {
+    return detail.messages
+        .filter((m) => m.role === "user" && m.text.trim() !== "")
+        .map((m) => m.text.slice(0, SCRIPT_MAX_CHARS))
+        .slice(0, SCRIPT_MAX_MESSAGES);
+}
+function toolNames(detail) {
+    const names = new Set();
+    for (const span of detail.spans) {
+        if (span.kind !== "tool")
+            continue;
+        const name = span.attributes["gen_ai.tool.name"];
+        if (typeof name === "string" && name !== "")
+            names.add(name);
+    }
+    return [...names];
+}
+function shortId(sessionId) {
+    return sessionId.replace(/^ses_/, "").slice(0, 8);
+}
+function isFinishedStatus(status) {
+    return status === "open" || status === "closed";
+}
+function isFinishedUserSession(detail, agentId) {
+    const { session } = detail;
+    return session.agentId === agentId && session.source !== "eval" && isFinishedStatus(session.status);
+}
+/** Finished (idle or closed), non-eval sessions of the agent that have at least one user message, newest first. */
+export async function listSessionCandidates(input) {
+    const { sessions, projectRoot, agentId, limit } = input;
+    const items = [];
+    let cursor;
+    do {
+        let page;
+        try {
+            page = await sessions.listSessions(projectRoot, agentId, {
+                limit: Math.max(limit, 25),
+                ...(cursor !== undefined ? { cursor } : {}),
+            });
+        }
+        catch (err) {
+            if (isAgentError(err))
+                throw new SessionCaseError(`Agent "${agentId}" was not found`);
+            throw err;
+        }
+        for (const row of page.sessions) {
+            if (row.source === "eval" || !isFinishedStatus(row.status))
+                continue;
+            if (row.lastTurnStatus === "running" || row.lastTurnStatus === "waiting")
+                continue;
+            const detail = await sessions.getSession(projectRoot, row.id);
+            const messages = userMessages(detail);
+            if (messages.length === 0)
+                continue;
+            items.push({
+                session_id: row.id,
+                updated_at: row.updatedAt,
+                preview: messages[0].replace(/\s+/g, " ").slice(0, PREVIEW_CHARS),
+                turns: messages.length,
+                tools: toolNames(detail),
+            });
+            if (items.length >= limit)
+                return items;
+        }
+        cursor = page.nextCursor ?? undefined;
+    } while (cursor !== undefined);
+    return items;
+}
+/** One `script` scenario per session: its user messages, plus the tools it called as the expected tools. */
+export async function casesFromSessions(input) {
+    const { sessions, projectRoot, agentId } = input;
+    const cases = [];
+    for (const sessionId of input.sessionIds) {
+        let detail;
+        try {
+            detail = await sessions.getSession(projectRoot, sessionId);
+        }
+        catch (err) {
+            if (isAgentError(err))
+                throw new SessionCaseError(`Session "${sessionId}" was not found`);
+            throw err;
+        }
+        if (!isFinishedUserSession(detail, agentId)) {
+            throw new SessionCaseError(`Session "${sessionId}" is not a finished session of agent "${agentId}"`);
+        }
+        const script = userMessages(detail);
+        if (script.length === 0)
+            throw new SessionCaseError(`Session "${sessionId}" has no user messages`);
+        cases.push({
+            title: detail.session.title ?? `Session ${shortId(sessionId)}`,
+            input: { mode: "script", script, opening: script[0] },
+            expected: { tools: toolNames(detail) },
+            tags: [SESSION_TAG],
+            metadata: { source_session: sessionId },
+        });
+    }
+    return cases;
+}

package/dist/config/datasetHash.d.ts

@@ -0,0 +1 @@
+export declare function datasetContentHash(cases: readonly unknown[]): string;

package/dist/config/datasetHash.js

@@ -0,0 +1,4 @@
+import { canonicalJson, sha256Hex } from "../evals/canonical.js";
+export function datasetContentHash(cases) {
+    return sha256Hex(cases.map((c) => canonicalJson(c)).join("\n"));
+}

package/dist/config/datasetImport.d.ts

@@ -0,0 +1,21 @@
+import type { DatasetCase } from "./datasetTypes.js";
+export type UploadMappingEntry = {
+    column: string;
+    split?: string;
+} | null;
+export type UploadMapping = Record<string, UploadMappingEntry>;
+export type UploadPreviewRow = {
+    index: number;
+    case: DatasetCase;
+    status: "ready" | "skipped";
+    reason?: string;
+};
+export declare function nextIdNumber(existingIds: Iterable<string>): number;
+export declare function formatId(n: number): string;
+export type UploadMappingOptions = {
+    /** Also map the top-level `title` and `tags` targets (agent datasets). */
+    caseMeta?: boolean;
+    /** Extra check on the built case; a returned text skips the row with that reason. */
+    check?: (draft: DatasetCase) => string | undefined;
+};
+export declare function applyUploadMapping(rows: ReadonlyArray<Record<string, string>>, mapping: UploadMapping, inputSchema: unknown, existingIds: Iterable<string>, options?: UploadMappingOptions): UploadPreviewRow[];

package/dist/config/datasetImport.js

@@ -0,0 +1,120 @@
+import { payloadInstanceMismatch } from "../envelope/payloadSchema.js";
+const REASON_MAX = 160;
+const ID_NUMBER_RE = /^c(\d+)$/;
+function trimReason(message) {
+    const flat = message.replace(/\s+/g, " ").trim();
+    return flat.length > REASON_MAX ? flat.slice(0, REASON_MAX) : flat;
+}
+export function nextIdNumber(existingIds) {
+    let max = 0;
+    for (const id of existingIds) {
+        const m = ID_NUMBER_RE.exec(id);
+        if (m)
+            max = Math.max(max, Number(m[1]));
+    }
+    return max + 1;
+}
+export function formatId(n) {
+    return `c${String(n).padStart(3, "0")}`;
+}
+function schemaPropertyType(inputSchema, name) {
+    if (inputSchema === null || typeof inputSchema !== "object")
+        return undefined;
+    const props = inputSchema.properties;
+    if (props === null || typeof props !== "object")
+        return undefined;
+    const prop = props[name];
+    if (prop === null || typeof prop !== "object")
+        return undefined;
+    const type = prop.type;
+    return typeof type === "string" ? type : undefined;
+}
+function parseCell(cell, schemaType, split, name) {
+    if (split !== undefined) {
+        return {
+            value: cell
+                .split(split)
+                .map((part) => part.trim())
+                .filter((part) => part !== ""),
+        };
+    }
+    if (schemaType === "number" || schemaType === "integer") {
+        const trimmed = cell.trim();
+        const n = trimmed === "" ? Number.NaN : Number(trimmed);
+        if (!Number.isFinite(n) || (schemaType === "integer" && !Number.isInteger(n))) {
+            return { error: `${name} is not ${schemaType === "integer" ? "an integer" : "a number"}` };
+        }
+        return { value: n };
+    }
+    if (schemaType === "boolean") {
+        const lowered = cell.trim().toLowerCase();
+        if (lowered === "true")
+            return { value: true };
+        if (lowered === "false")
+            return { value: false };
+        return { error: `${name} is not a boolean` };
+    }
+    return { value: cell };
+}
+export function applyUploadMapping(rows, mapping, inputSchema, existingIds, options = {}) {
+    let nextId = nextIdNumber(existingIds);
+    const entries = Object.entries(mapping).filter((e) => e[1] !== null);
+    return rows.map((row, index) => {
+        const input = {};
+        const expected = {};
+        let parseError;
+        let title;
+        let tags;
+        for (const [target, entry] of entries) {
+            if (options.caseMeta === true && (target === "title" || target === "tags")) {
+                const raw = row[entry.column];
+                if (raw === undefined || raw.trim() === "")
+                    continue;
+                if (target === "title")
+                    title = raw.trim();
+                else
+                    tags = (entry.split !== undefined ? raw.split(entry.split) : raw.split(",")).map((t) => t.trim()).filter((t) => t !== "");
+                continue;
+            }
+            const dot = target.indexOf(".");
+            const base = dot === -1 ? target : target.slice(0, dot);
+            const name = dot === -1 ? "" : target.slice(dot + 1);
+            if ((base !== "input" && base !== "expected") || name === "")
+                continue;
+            const cell = row[entry.column];
+            if (cell === undefined || cell === "")
+                continue;
+            const schemaType = base === "input" ? schemaPropertyType(inputSchema, name) : undefined;
+            const parsed = parseCell(cell, schemaType, entry.split, name);
+            if ("error" in parsed) {
+                parseError ??= parsed.error;
+                continue;
+            }
+            (base === "input" ? input : expected)[name] = parsed.value;
+        }
+        const draft = { id: "", input };
+        if (Object.keys(expected).length > 0)
+            draft.expected = expected;
+        if (title !== undefined)
+            draft.title = title;
+        if (tags !== undefined && tags.length > 0)
+            draft.tags = tags;
+        let reason = parseError;
+        if (reason === undefined && inputSchema !== undefined) {
+            try {
+                reason = payloadInstanceMismatch(input, inputSchema);
+            }
+            catch (err) {
+                reason = err instanceof Error ? err.message : String(err);
+            }
+        }
+        if (reason === undefined && options.check)
+            reason = options.check(draft);
+        if (reason !== undefined) {
+            return { index, case: draft, status: "skipped", reason: trimReason(reason) };
+        }
+        draft.id = formatId(nextId);
+        nextId += 1;
+        return { index, case: draft, status: "ready" };
+    });
+}

package/dist/config/datasetStageCheck.d.ts

@@ -0,0 +1,9 @@
+export type DatasetStageCheckResult = {
+    caseId: string;
+    valid: boolean;
+    message?: string;
+};
+export declare function checkDatasetAgainstStage(cases: ReadonlyArray<{
+    id: string;
+    input: Record<string, unknown>;
+}>, inputSchema: unknown): DatasetStageCheckResult[];

package/dist/config/datasetStageCheck.js

@@ -0,0 +1,17 @@
+import { payloadInstanceMismatch } from "../envelope/payloadSchema.js";
+export function checkDatasetAgainstStage(cases, inputSchema) {
+    return cases.map((c) => {
+        if (inputSchema === undefined)
+            return { caseId: c.id, valid: true };
+        let message;
+        try {
+            message = payloadInstanceMismatch(c.input, inputSchema);
+        }
+        catch (err) {
+            message = err instanceof Error ? err.message : String(err);
+        }
+        return message === undefined
+            ? { caseId: c.id, valid: true }
+            : { caseId: c.id, valid: false, message };
+    });
+}

package/dist/config/datasetTypes.d.ts

@@ -0,0 +1,27 @@
+import type { ValidationFinding } from "./validateCatalog.js";
+export type DatasetFormat = "jsonl" | "json" | "csv";
+export type DatasetCase = {
+    id: string;
+    input: Record<string, unknown>;
+    title?: string;
+    expected?: unknown;
+    tags?: string[];
+    goal?: string;
+    metadata?: Record<string, unknown>;
+};
+export type DatasetLimits = {
+    maxCases: number;
+    maxCaseBytes: number;
+    maxFileBytes: number;
+};
+export type ParseDatasetResult = {
+    cases: DatasetCase[];
+    findings: ValidationFinding[];
+    contentHash: string;
+};
+export type DatasetLoadResult = ParseDatasetResult & {
+    path: string;
+    relPath: string;
+    format: DatasetFormat | undefined;
+    fileBytes: number;
+};

package/dist/config/datasetTypes.js

@@ -0,0 +1 @@
+export {};

package/dist/config/ensureCatalogScanRoot.d.ts

@@ -1,6 +1,8 @@
 import type { CatalogRootKind } from "./resolveCatalogRoots.js";
 /** Directory scanned for workshop-authored pipelines and tasks. */
 export declare const WORKSHOP_PACKAGE_ROOT = "workshop";
+/** Directory the Guide always uses for pipeline packages and their stage files. */
+export declare const GUIDE_PIPELINES_ROOT = "pipelines";
 export declare function normalizeCatalogDir(directory: string): string;
 /** Untitled workshop saves land here so Run can see them without a new catalog edit. */
 export declare function defaultWorkshopPackageDirectory(pipelineId: string | undefined, catalogRootKind?: CatalogRootKind): string;
@@ -14,4 +16,5 @@ export declare function catalogScanRootForDirectory(directory: string): string;
  * Make sure `stageflow.yaml` scans `directory` for both pipelines and tasks.
  * Returns true when the manifest changed. Missing or unreadable manifests are left alone.
  */
-export declare function ensureCatalogScanRoot(projectRoot: string, directory: string): Promise<boolean>;
+export type CatalogScanKey = "pipelines" | "tasks" | "evals" | "datasets";
+export declare function ensureCatalogScanRoot(projectRoot: string, directory: string, keys?: readonly CatalogScanKey[]): Promise<boolean>;

package/dist/config/ensureCatalogScanRoot.js

@@ -3,6 +3,8 @@ import { isSeq, parseDocument } from "yaml";
 import { manifestPathForProject } from "./loadStageflowManifest.js";
 /** Directory scanned for workshop-authored pipelines and tasks. */
 export const WORKSHOP_PACKAGE_ROOT = "workshop";
+/** Directory the Guide always uses for pipeline packages and their stage files. */
+export const GUIDE_PIPELINES_ROOT = "pipelines";
 export function normalizeCatalogDir(directory) {
     return directory.replace(/\\/g, "/").replace(/^\.\/+/, "").replace(/\/+$/, "");
 }
@@ -30,6 +32,10 @@ export function catalogScanRootForDirectory(directory) {
         normalized.startsWith(`${WORKSHOP_PACKAGE_ROOT}/`)) {
         return WORKSHOP_PACKAGE_ROOT;
     }
+    if (normalized === GUIDE_PIPELINES_ROOT ||
+        normalized.startsWith(`${GUIDE_PIPELINES_ROOT}/`)) {
+        return GUIDE_PIPELINES_ROOT;
+    }
     return normalized;
 }
 function isCovered(entries, root) {
@@ -38,11 +44,7 @@ function isCovered(entries, root) {
         return root === normalized || root.startsWith(`${normalized}/`);
     });
 }
-/**
- * Make sure `stageflow.yaml` scans `directory` for both pipelines and tasks.
- * Returns true when the manifest changed. Missing or unreadable manifests are left alone.
- */
-export async function ensureCatalogScanRoot(projectRoot, directory) {
+export async function ensureCatalogScanRoot(projectRoot, directory, keys = ["pipelines", "tasks"]) {
     const root = catalogScanRootForDirectory(directory);
     if (!root || root.split("/").includes(".."))
         return false;
@@ -56,7 +58,7 @@ export async function ensureCatalogScanRoot(projectRoot, directory) {
     }
     const doc = parseDocument(text);
     let changed = false;
-    for (const key of ["pipelines", "tasks"]) {
+    for (const key of keys) {
         const seq = doc.getIn(["catalog", key]);
         if (!isSeq(seq))
             continue;

package/dist/config/evalCatalog.d.ts

@@ -0,0 +1,22 @@
+import type { LoadedManifest } from "../types/stageflowManifest.js";
+import { type LoadedEval } from "./loadEval.js";
+import type { LoadOutcome } from "./loadOutcome.js";
+import type { ValidationFinding } from "./validateCatalog.js";
+import { type DatasetReport } from "./validateEval.js";
+export type CatalogFileRef = {
+    absPath: string;
+    relPath: string;
+};
+export declare function listEvalFiles(manifest: LoadedManifest): Promise<CatalogFileRef[]>;
+export declare function listDatasetFiles(manifest: LoadedManifest): Promise<CatalogFileRef[]>;
+export type EvalCatalogEntry = {
+    absPath: string;
+    relPath: string;
+    outcome: LoadOutcome<LoadedEval>;
+    findings: ValidationFinding[];
+};
+export type LoadEvalCatalogOptions = {
+    isKnownModel?: (id: string) => Promise<boolean>;
+    datasetReportFor?: (loaded: LoadedEval) => Promise<DatasetReport | undefined>;
+};
+export declare function loadEvalCatalog(manifest: LoadedManifest, options?: LoadEvalCatalogOptions): Promise<EvalCatalogEntry[]>;

package/dist/config/evalCatalog.js

@@ -0,0 +1,95 @@
+import { stat } from "node:fs/promises";
+import path from "node:path";
+import { isInsideProjectRoot } from "./catalogRelativePath.js";
+import { getModelCatalog } from "../models/catalog.js";
+import { loadDatasetFile } from "./loadDataset.js";
+import { loadEvalOutcome, resolveEvalPaths } from "./loadEval.js";
+import { scanCatalogPaths } from "./scanCatalogPaths.js";
+import { evalFindingsFromIssues, validateEval, } from "./validateEval.js";
+async function listFiles(manifest, kind) {
+    const absPaths = await scanCatalogPaths(manifest, kind);
+    return absPaths.map((absPath) => ({
+        absPath,
+        relPath: path.relative(manifest.projectRoot, absPath).split(path.sep).join("/"),
+    }));
+}
+export function listEvalFiles(manifest) {
+    return listFiles(manifest, "eval");
+}
+export async function listDatasetFiles(manifest) {
+    const scanned = await listFiles(manifest, "dataset");
+    const referenced = await datasetsReferencedByEvals(manifest, scanned);
+    return [...scanned, ...referenced].sort((a, b) => (a.relPath < b.relPath ? -1 : a.relPath > b.relPath ? 1 : 0));
+}
+async function datasetsReferencedByEvals(manifest, scanned) {
+    const seen = new Set(scanned.map((ref) => path.resolve(ref.absPath)));
+    const extra = [];
+    for (const ref of await listFiles(manifest, "eval")) {
+        const outcome = await loadEvalOutcome(ref.absPath, manifest.projectRoot);
+        if (!outcome.ok)
+            continue;
+        const abs = path.resolve(resolveEvalPaths(outcome.value.file, outcome.value.path).datasetPath);
+        if (seen.has(abs) || !isInsideProjectRoot(manifest.projectRoot, abs))
+            continue;
+        try {
+            if (!(await stat(abs)).isFile())
+                continue;
+        }
+        catch {
+            continue;
+        }
+        seen.add(abs);
+        extra.push({
+            absPath: abs,
+            relPath: path.relative(manifest.projectRoot, abs).split(path.sep).join("/"),
+        });
+    }
+    return extra;
+}
+export async function loadEvalCatalog(manifest, options = {}) {
+    const refs = await listEvalFiles(manifest);
+    const { projectRoot } = manifest;
+    const isKnownModel = options.isKnownModel ??
+        (async (id) => {
+            try {
+                const { models } = await getModelCatalog(projectRoot).snapshot();
+                return models.length === 0 || models.includes(id);
+            }
+            catch {
+                return true;
+            }
+        });
+    const datasetReportFor = options.datasetReportFor ??
+        (async (evalFile) => {
+            const { datasetPath } = resolveEvalPaths(evalFile.file, evalFile.path);
+            try {
+                await stat(datasetPath);
+            }
+            catch {
+                return undefined;
+            }
+            return loadDatasetFile(datasetPath, { root: projectRoot });
+        });
+    const loaded = await Promise.all(refs.map(async (ref) => ({ ref, outcome: await loadEvalOutcome(ref.absPath, projectRoot) })));
+    const evals = loaded.flatMap(({ ref, outcome }) => outcome.ok ? [{ id: outcome.value.id, absPath: ref.absPath }] : []);
+    const entries = [];
+    for (const { ref, outcome } of loaded) {
+        let findings;
+        if (outcome.ok) {
+            const datasetReport = await datasetReportFor(outcome.value);
+            findings = await validateEval(outcome.value, {
+                projectRoot,
+                evals,
+                isKnownModel,
+                ...(datasetReport !== undefined ? { datasetReport } : {}),
+            });
+            if (datasetReport !== undefined)
+                findings.push(...datasetReport.findings);
+        }
+        else {
+            findings = evalFindingsFromIssues(ref.relPath, outcome.issues);
+        }
+        entries.push({ absPath: ref.absPath, relPath: ref.relPath, outcome, findings });
+    }
+    return entries;
+}

package/dist/config/evalWriteRoots.d.ts

@@ -0,0 +1,17 @@
+import type { LoadedManifest } from "../types/stageflowManifest.js";
+import { type CatalogFileRef } from "./evalCatalog.js";
+export type WriteRootKind = "evals" | "datasets" | "pipelines";
+export declare function manifestForWrite(projectRoot: string): Promise<LoadedManifest>;
+export declare function explicitRoots(manifest: LoadedManifest, kind: WriteRootKind): string[] | undefined;
+export type WriteDir = {
+    absDir: string;
+    relDir: string;
+    explicitDirectory: boolean;
+};
+export declare function resolveWriteDir(projectRoot: string, manifest: LoadedManifest, kind: WriteRootKind, directory?: string): Promise<WriteDir>;
+export declare function assertInsideProject(projectRoot: string, absPath: string, what: string): void;
+export declare function evalStem(fileName: string): string;
+export declare function datasetStem(fileName: string): string;
+export declare function findEvalFile(projectRoot: string, id: string, manifest?: LoadedManifest): Promise<CatalogFileRef | undefined>;
+export declare function datasetIdsFor(refs: readonly CatalogFileRef[]): Map<string, string>;
+export declare function findDatasetFile(projectRoot: string, id: string, manifest?: LoadedManifest): Promise<CatalogFileRef | undefined>;