stageflow 0.40.0 → 0.42.0

package/dist/config/loadOutcome.d.ts

@@ -26,6 +26,11 @@ export type LoadIssue = ({
     message: string;
     category: "trigger";
     triggerId?: string;
+} & LoadIssueLocation) | ({
+    code: string;
+    message: string;
+    category: "eval";
+    severity?: "error" | "warning";
 } & LoadIssueLocation) | ({
     code: ValidationFindingCode | string;
     message: string;

package/dist/config/loadStageflowManifest.js

@@ -8,6 +8,8 @@ import { parseModelField } from "./modelField.js";
 const DEFAULT_PIPELINE_PATTERN = "*.pipeline.yaml";
 const DEFAULT_TASK_PATTERN = "*.task.yaml";
 const DEFAULT_TRIGGER_PATTERN = "*.trigger.yaml";
+const DEFAULT_EVAL_PATTERN = "*.eval.yaml";
+const DEFAULT_DATASET_PATTERNS = ["*.dataset.jsonl", "*.dataset.json", "*.dataset.csv"];
 export function manifestPathForProject(projectRoot) {
     return path.join(projectRoot, "stageflow.yaml");
 }
@@ -37,11 +39,27 @@ function parseStringArray(raw, field, issues) {
     }
     return entries;
 }
+function parseDatasetPattern(raw) {
+    const entries = typeof raw === "string" ? [raw] : Array.isArray(raw) ? raw : null;
+    if (entries === null || entries.length === 0) {
+        return null;
+    }
+    const parsed = [];
+    for (const entry of entries) {
+        if (typeof entry !== "string" || entry.trim().length === 0) {
+            return null;
+        }
+        parsed.push(entry.trim());
+    }
+    return parsed;
+}
 function parsePatterns(raw, issues) {
     const defaults = {
         pipeline: DEFAULT_PIPELINE_PATTERN,
         task: DEFAULT_TASK_PATTERN,
         trigger: DEFAULT_TRIGGER_PATTERN,
+        eval: DEFAULT_EVAL_PATTERN,
+        dataset: DEFAULT_DATASET_PATTERNS,
     };
     if (raw === undefined || raw === null) {
         return defaults;
@@ -66,6 +84,12 @@ function parsePatterns(raw, issues) {
         : typeof record.trigger === "string" && record.trigger.trim().length > 0
             ? record.trigger.trim()
             : null;
+    const evalPattern = record.eval === undefined
+        ? defaults.eval
+        : typeof record.eval === "string" && record.eval.trim().length > 0
+            ? record.eval.trim()
+            : null;
+    const dataset = record.dataset === undefined ? defaults.dataset : parseDatasetPattern(record.dataset);
     if (pipeline === null) {
         issues.push(catalogIssue("catalog.manifest_invalid", "catalog.patterns.pipeline must be a string"));
     }
@@ -75,10 +99,18 @@ function parsePatterns(raw, issues) {
     if (trigger === null) {
         issues.push(catalogIssue("catalog.manifest_invalid", "catalog.patterns.trigger must be a string"));
     }
+    if (evalPattern === null) {
+        issues.push(catalogIssue("catalog.manifest_invalid", "catalog.patterns.eval must be a string"));
+    }
+    if (dataset === null) {
+        issues.push(catalogIssue("catalog.manifest_invalid", "catalog.patterns.dataset must be a non-empty string or an array of non-empty strings"));
+    }
     return {
         pipeline: pipeline ?? defaults.pipeline,
         task: task ?? defaults.task,
         trigger: trigger ?? defaults.trigger,
+        eval: evalPattern ?? defaults.eval,
+        dataset: dataset ?? defaults.dataset,
     };
 }
 export function parseStageflowManifestOutcome(yamlText, manifestPath, projectRoot) {
@@ -131,6 +163,22 @@ export function parseStageflowManifestOutcome(yamlText, manifestPath, projectRoo
             triggers = parsed;
         }
     }
+    const evalsRaw = catalogRecord.evals;
+    let evals;
+    if (evalsRaw !== undefined) {
+        const parsed = parseStringArray(evalsRaw, "evals", issues);
+        if (parsed !== null) {
+            evals = parsed;
+        }
+    }
+    const datasetsRaw = catalogRecord.datasets;
+    let datasets;
+    if (datasetsRaw !== undefined) {
+        const parsed = parseStringArray(datasetsRaw, "datasets", issues);
+        if (parsed !== null) {
+            datasets = parsed;
+        }
+    }
     const patterns = parsePatterns(catalogRecord.patterns, issues);
     let agent;
     const agentField = parseAgentField(record.agent);
@@ -161,6 +209,12 @@ export function parseStageflowManifestOutcome(yamlText, manifestPath, projectRoo
     if (triggers !== undefined) {
         catalog.triggers = triggers;
     }
+    if (evals !== undefined) {
+        catalog.evals = evals;
+    }
+    if (datasets !== undefined) {
+        catalog.datasets = datasets;
+    }
     if (catalogRecord.patterns !== undefined) {
         catalog.patterns = {};
         if (typeof catalogRecord.patterns.pipeline === "string") {
@@ -172,6 +226,16 @@ export function parseStageflowManifestOutcome(yamlText, manifestPath, projectRoo
         if (typeof catalogRecord.patterns.trigger === "string") {
             catalog.patterns.trigger = patterns.trigger;
         }
+        if (typeof catalogRecord.patterns.eval === "string") {
+            catalog.patterns.eval = patterns.eval;
+        }
+        const datasetRaw = catalogRecord.patterns.dataset;
+        if (typeof datasetRaw === "string") {
+            catalog.patterns.dataset = patterns.dataset[0];
+        }
+        else if (Array.isArray(datasetRaw)) {
+            catalog.patterns.dataset = patterns.dataset;
+        }
     }
     const manifest = {
         version: 1,

package/dist/config/resolveAgentTarget.d.ts

@@ -0,0 +1,32 @@
+import type { AgentDetail, AgentSummary } from "../agents/service.js";
+import type { EvalFile } from "../evals/types.js";
+import { type AgentDefinition } from "./loadAgent.js";
+import { type LoadIssue, type LoadOutcome } from "./loadOutcome.js";
+export type AgentToolRisk = "read" | "write" | "shell" | "destructive" | "external" | "mcp_unknown" | "sub_agent" | "browser";
+export type AgentToolGrant = {
+    id: string;
+    risk: AgentToolRisk;
+};
+export type ResolvedAgentTarget = {
+    agentId: string;
+    /** The revision read now; null when resolved offline from the file alone. */
+    revision: number | null;
+    definitionHash: string;
+    /** The definition after the eval's `defaults.model` override. */
+    definition: AgentDefinition;
+    modelResolved: string | undefined;
+    tools: AgentToolGrant[];
+    askFirst: string[];
+    issues: LoadIssue[];
+};
+/** The slice of `AgentService` that eval resolution reads. */
+export type AgentCatalogPort = {
+    get(projectRoot: string, agentId: string): Promise<AgentDetail>;
+    list(projectRoot: string): Promise<AgentSummary[]>;
+};
+export declare function agentToolGrants(definition: AgentDefinition): AgentToolGrant[];
+/** sha256 of the canonical JSON of the definition after the eval's `defaults.model` override. */
+export declare function agentDefinitionHash(definition: AgentDefinition): string;
+export declare function resolveAgentTarget(file: EvalFile, projectRoot: string, agents: AgentCatalogPort): Promise<LoadOutcome<ResolvedAgentTarget>>;
+/** Offline resolution (`sf eval validate` without an agent service): parses `agents/<id>.agent.yaml` only. */
+export declare function resolveAgentTargetFromFile(file: EvalFile, projectRoot: string): Promise<LoadOutcome<ResolvedAgentTarget>>;

package/dist/config/resolveAgentTarget.js

@@ -0,0 +1,118 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { isAgentError } from "../agents/errors.js";
+import { effectiveAskFirst } from "../agents/policy.js";
+import { canonicalJson, sha256Hex } from "../evals/canonical.js";
+import { grantedToolIds, parseAgentYaml } from "./loadAgent.js";
+import { loadFailure, loadSuccess } from "./loadOutcome.js";
+import { TOOL_META, STAGEFLOW_GRANT_PREFIX } from "../toolkit/meta.js";
+const BUILTIN_RISK = {
+    read: "read",
+    grep: "read",
+    find: "read",
+    ls: "read",
+    write: "write",
+    edit: "write",
+    bash: "shell",
+};
+function agentIssue(code, message, severity = "error") {
+    return { code, message, category: "eval", severity };
+}
+function grantRisk(id) {
+    if (id in BUILTIN_RISK)
+        return BUILTIN_RISK[id];
+    if (id.startsWith("agent:"))
+        return "sub_agent";
+    if (id.startsWith(STAGEFLOW_GRANT_PREFIX)) {
+        return TOOL_META[id.slice(STAGEFLOW_GRANT_PREFIX.length)]?.risk ?? "mcp_unknown";
+    }
+    return "mcp_unknown";
+}
+export function agentToolGrants(definition) {
+    const grants = [...grantedToolIds(definition)].sort().map((id) => ({ id, risk: grantRisk(id) }));
+    if (definition.tools.browser)
+        grants.push({ id: "browser", risk: "browser" });
+    return grants;
+}
+/** sha256 of the canonical JSON of the definition after the eval's `defaults.model` override. */
+export function agentDefinitionHash(definition) {
+    return sha256Hex(canonicalJson(definition));
+}
+function withModelOverride(definition, model) {
+    return model === undefined ? definition : { ...definition, model: { ...definition.model, id: model } };
+}
+function firstIssue(issues) {
+    const issue = issues[0];
+    if (!issue)
+        return "the definition does not load";
+    return issue.path ? `${issue.path}: ${issue.message}` : issue.message;
+}
+function resolved(agentId, revision, base, file, warnings) {
+    const definition = withModelOverride(base, file.defaults?.model);
+    const issues = [...warnings];
+    if (!definition.enabled) {
+        issues.push(agentIssue("eval.agent_disabled", `Agent "${agentId}" is disabled; eval sessions run anyway`, "warning"));
+    }
+    return loadSuccess({
+        agentId,
+        revision,
+        definitionHash: agentDefinitionHash(definition),
+        definition,
+        modelResolved: definition.model.id,
+        tools: agentToolGrants(definition),
+        askFirst: effectiveAskFirst(definition),
+        issues,
+    }, issues);
+}
+export async function resolveAgentTarget(file, projectRoot, agents) {
+    if (file.target.kind !== "agent") {
+        return loadFailure([agentIssue("eval.target_unresolved", "Not an agent target")]);
+    }
+    const agentId = file.target.agent;
+    let detail;
+    try {
+        detail = await agents.get(projectRoot, agentId);
+    }
+    catch (err) {
+        if (isAgentError(err) && err.code === "agent.not_found") {
+            return loadFailure([agentIssue("eval.agent_not_found", `Agent "${agentId}" was not found`)]);
+        }
+        throw err;
+    }
+    const listed = await agents.list(projectRoot);
+    if (!listed.some((agent) => agent.id === agentId)) {
+        return loadFailure([agentIssue("eval.agent_archived", `Agent "${agentId}" is archived`)]);
+    }
+    if (detail.agent.issues.length > 0 || detail.definition === null) {
+        return loadFailure([
+            agentIssue("eval.agent_invalid", `Agent "${agentId}" has problems in its definition: ${firstIssue(detail.agent.issues)}`),
+        ]);
+    }
+    return resolved(agentId, detail.agent.revision, detail.definition, file, []);
+}
+/** Offline resolution (`sf eval validate` without an agent service): parses `agents/<id>.agent.yaml` only. */
+export async function resolveAgentTargetFromFile(file, projectRoot) {
+    if (file.target.kind !== "agent") {
+        return loadFailure([agentIssue("eval.target_unresolved", "Not an agent target")]);
+    }
+    const agentId = file.target.agent;
+    let text;
+    try {
+        text = await readFile(path.join(projectRoot, "agents", `${agentId}.agent.yaml`), "utf8");
+    }
+    catch {
+        return loadFailure([agentIssue("eval.agent_not_found", `Agent "${agentId}" was not found`)]);
+    }
+    const parsed = parseAgentYaml(text);
+    if (!parsed.ok) {
+        return loadFailure([
+            agentIssue("eval.agent_invalid", `Agent "${agentId}" has problems in its definition: ${firstIssue(parsed.issues)}`),
+        ]);
+    }
+    if (parsed.value.id !== agentId) {
+        return loadFailure([
+            agentIssue("eval.agent_invalid", `Agent "${agentId}" has problems in its definition: id must match the file name`),
+        ]);
+    }
+    return resolved(agentId, null, parsed.value, file, []);
+}

package/dist/config/resolveEvalTarget.d.ts

@@ -0,0 +1,21 @@
+import type { EvalFile } from "../evals/types.js";
+import type { CompletionContract } from "../types/completion.js";
+import type { LoadedStageConfig } from "../types/stage.js";
+import { type LoadOutcome } from "./loadOutcome.js";
+export type ResolvedEvalTarget = {
+    stageConfig: LoadedStageConfig;
+    pipelinePath?: string;
+    stageId?: string;
+    file?: string;
+    stageVersionHash: string;
+    completion?: CompletionContract;
+    inputSchema?: unknown;
+    outputSchema?: unknown;
+    modelResolved: string;
+    modelTier: "stage" | "pipeline" | "global";
+};
+/** The exact object the stage version hash is computed from. */
+export declare function stageDefinition(stage: LoadedStageConfig, completion: CompletionContract | undefined): Record<string, unknown>;
+/** sha256 of the canonical JSON of the materialized stage config plus its after-phase verify contract. */
+export declare function stageVersionHash(stage: LoadedStageConfig, completion: CompletionContract | undefined): string;
+export declare function resolveEvalTarget(file: EvalFile, evalAbsPath: string, projectRoot: string): Promise<LoadOutcome<ResolvedEvalTarget>>;

package/dist/config/resolveEvalTarget.js

@@ -0,0 +1,89 @@
+import path from "node:path";
+import { canonicalJson, sha256Hex } from "../evals/canonical.js";
+import { resolveEvalPaths } from "./loadEval.js";
+import { loadFailure, loadSuccess } from "./loadOutcome.js";
+import { loadPipelineOutcome } from "./loadPipeline.js";
+import { afterCompletionForStage, loadStageOutcome } from "./loadStage.js";
+import { materializeStageModels } from "./materializeStageModels.js";
+function unresolved(message) {
+    return { code: "eval.target_unresolved", message, category: "eval" };
+}
+function describeIssues(issues) {
+    return issues.map((issue) => issue.message).join("; ");
+}
+/** The exact object the stage version hash is computed from. */
+export function stageDefinition(stage, completion) {
+    const { model_tier: _tier, ...config } = stage;
+    return { ...config, ...(completion !== undefined ? { completion } : {}) };
+}
+/** sha256 of the canonical JSON of the materialized stage config plus its after-phase verify contract. */
+export function stageVersionHash(stage, completion) {
+    return sha256Hex(canonicalJson(stageDefinition(stage, completion)));
+}
+export async function resolveEvalTarget(file, evalAbsPath, projectRoot) {
+    if (file.target.kind !== "stage") {
+        return loadFailure([unresolved("Agent targets are resolved by resolveAgentTarget")]);
+    }
+    const paths = resolveEvalPaths(file, evalAbsPath);
+    const rel = (abs) => path.relative(projectRoot, abs).split(path.sep).join("/");
+    if (paths.pipelinePath !== undefined) {
+        const pipelineRel = rel(paths.pipelinePath);
+        const outcome = await loadPipelineOutcome(paths.pipelinePath, { cwd: projectRoot, projectRoot, requireIo: false });
+        if (!outcome.ok) {
+            return loadFailure([
+                unresolved(`Target pipeline ${pipelineRel} does not load: ${describeIssues(outcome.issues)}`),
+            ]);
+        }
+        const { value } = outcome;
+        const stageId = file.target.stage;
+        const stage = value.stages.find((s) => s.id === stageId);
+        if (!stage) {
+            const known = value.stages.map((s) => s.id).join(", ");
+            return loadFailure([
+                unresolved(`Target stage "${stageId}" not found in pipeline ${pipelineRel} (stages: ${known})`),
+            ]);
+        }
+        const completion = value.dag.nodes.find((node) => node.id === stageId)?.completion;
+        return loadSuccess({
+            stageConfig: stage,
+            pipelinePath: paths.pipelinePath,
+            stageId,
+            stageVersionHash: stageVersionHash(stage, completion),
+            ...(completion !== undefined ? { completion } : {}),
+            inputSchema: stage.clone_input_schema,
+            outputSchema: stage.payload_schema,
+            modelResolved: stage.model,
+            modelTier: stage.model_tier ?? "global",
+        });
+    }
+    const stageFile = paths.stageFile;
+    const fileRel = rel(stageFile);
+    const loaded = await loadStageOutcome(stageFile, { requireIo: false });
+    if (!loaded.ok) {
+        return loadFailure([
+            unresolved(`Target stage file ${fileRel} does not load: ${describeIssues(loaded.issues)}`),
+        ]);
+    }
+    const materialized = await materializeStageModels([loaded.value], {
+        pipelineId: `standalone-${loaded.value.id}`,
+        projectRoot,
+    });
+    if (!materialized.ok) {
+        return loadFailure([
+            unresolved(`Target stage file ${fileRel} has no resolvable model: ${describeIssues(materialized.issues)}`),
+        ]);
+    }
+    const stage = materialized.value[0];
+    const fileCompletion = afterCompletionForStage(loaded.value);
+    return loadSuccess({
+        stageConfig: stage,
+        file: stageFile,
+        stageId: stage.id,
+        stageVersionHash: stageVersionHash(stage, fileCompletion),
+        ...(fileCompletion !== undefined ? { completion: fileCompletion } : {}),
+        inputSchema: stage.clone_input_schema,
+        outputSchema: stage.payload_schema,
+        modelResolved: stage.model,
+        modelTier: stage.model_tier ?? "global",
+    });
+}

package/dist/config/scanCatalogPaths.d.ts

@@ -1,4 +1,4 @@
 import type { LoadedManifest } from "../types/stageflowManifest.js";
-export type CatalogPathKind = "pipeline" | "task" | "trigger";
+export type CatalogPathKind = "pipeline" | "task" | "trigger" | "eval" | "dataset";
 export declare function isExcluded(repoRelPath: string, exclude: string[] | undefined): boolean;
 export declare function scanCatalogPaths(manifest: LoadedManifest, kind: CatalogPathKind): Promise<string[]>;

package/dist/config/scanCatalogPaths.js

@@ -40,7 +40,7 @@ async function walkDirectory(absDir) {
     }
     return files;
 }
-async function expandCatalogEntry(projectRoot, entry, kind, pattern) {
+async function expandCatalogEntry(projectRoot, entry, patterns) {
     const absPath = path.join(projectRoot, entry);
     let entryStat;
     try {
@@ -56,7 +56,7 @@ async function expandCatalogEntry(projectRoot, entry, kind, pattern) {
         return [];
     }
     const files = await walkDirectory(absPath);
-    return files.filter((filePath) => matchesPattern(path.basename(filePath), pattern));
+    return files.filter((filePath) => patterns.some((pattern) => matchesPattern(path.basename(filePath), pattern)));
 }
 export async function scanCatalogPaths(manifest, kind) {
     const { projectRoot, manifest: doc, patterns } = manifest;
@@ -64,13 +64,25 @@ export async function scanCatalogPaths(manifest, kind) {
         ? doc.catalog.pipelines
         : kind === "task"
             ? doc.catalog.tasks
-            : (doc.catalog.triggers ?? []);
-    const pattern = kind === "pipeline" ? patterns.pipeline : kind === "task" ? patterns.task : patterns.trigger;
+            : kind === "trigger"
+                ? (doc.catalog.triggers ?? [])
+                : kind === "eval"
+                    ? (doc.catalog.evals ?? ["evals"])
+                    : (doc.catalog.datasets ?? ["datasets"]);
+    const matchPatterns = kind === "pipeline"
+        ? [patterns.pipeline]
+        : kind === "task"
+            ? [patterns.task]
+            : kind === "trigger"
+                ? [patterns.trigger]
+                : kind === "eval"
+                    ? [patterns.eval]
+                    : patterns.dataset;
     const exclude = doc.catalog.exclude;
     const seen = new Set();
     const resolved = [];
     for (const entry of entries) {
-        const absPaths = await expandCatalogEntry(projectRoot, entry, kind, pattern);
+        const absPaths = await expandCatalogEntry(projectRoot, entry, matchPatterns);
         for (const absPath of absPaths) {
             const normalized = path.resolve(absPath);
             if (seen.has(normalized))

package/dist/config/validateCatalog.d.ts

@@ -3,7 +3,7 @@ import type { TriggerFile } from "../types/trigger.js";
 import type { LoadIssue } from "./loadOutcome.js";
 import type { AnyChannelProvider } from "../channels/types.js";
 export type ValidationSeverity = "error" | "warning";
-export type ValidationCategory = "pipeline" | "stage" | "catalog" | "task" | "trigger";
+export type ValidationCategory = "pipeline" | "stage" | "catalog" | "task" | "trigger" | "eval" | "dataset";
 export type ValidationFindingCode = "pipeline.invalid_shape" | "pipeline.dag_error" | "pipeline.missing_stage" | "pipeline.load_error" | "pipeline.string_stage_ref" | "pipeline.stage_uses_inline_conflict" | "pipeline.stage_missing_body" | "pipeline.stage_id_mismatch" | "pipeline.stage_wrapper_skill_rejected" | "pipeline.invalid_completion" | "catalog.path_case_mismatch" | "pipeline.invalid_recovery" | "pipeline.include_cycle" | "pipeline.include_invalid" | "pipeline.include_duplicate_stage" | "pipeline.invalid_agent" | "pipeline.invalid_model" | "pipeline.invalid_verify" | "pipeline.io_incompatible" | "pipeline.model_applies" | "pipeline.route_if_invalid" | "pipeline.route_all_gated" | "pipeline.invalid_requires" | "pipeline.requires_conflict" | "pipeline.invalid_channels" | "pipeline.invalid_memories" | "pipeline.memory_unavailable" | "stage.invalid_shape" | "stage.invalid_model" | "stage.missing_model" | "stage.invalid_payload_schema" | "stage.invalid_clone_input_schema" | "stage.unresolved_schema_ref" | "stage.invalid_clone_actions" | "stage.invalid_gate_kinds" | "stage.invalid_pre_emit_checks" | "stage.invalid_timeout_ms" | "stage.invalid_skill" | "stage.invalid_mcp" | "stage.invalid_channels" | "stage.channel_unavailable" | "stage.invalid_browser" | "stage.invalid_secrets" | "stage.unknown_secret" | "stage.denied_secret" | "stage.invalid_requires" | "stage.invalid_agent" | "stage.invalid_io" | "stage.load_error" | "stage.id_filename_mismatch" | "task.invalid_shape" | "task.load_error" | "task.entry_input_unmet" | "task.binding_conflict" | "task.repository_ref_required" | "task.ref_without_repository" | "task.repository_invalid" | "trigger.invalid_shape" | "trigger.load_error" | "trigger.unknown_pipeline" | "trigger.unknown_task" | "catalog.duplicate_pipeline_id" | "catalog.manifest_missing" | "catalog.manifest_invalid" | "catalog.empty_catalog" | "catalog.manifest_load_error" | "catalog.invalid_mcp" | "catalog.mcp_command_missing" | "catalog.mixed_yaml_dialect" | "catalog.legacy_yaml" | "catalog.stageflow_home_absolute_path";
 export type ValidationFinding = {
     severity: ValidationSeverity;

package/dist/config/validateCatalog.js

@@ -6,6 +6,8 @@ import { INLINE_PIPELINE_PATH, loadPipelineFromObjectOutcome, loadPipelineOutcom
 import { loadStageOutcome } from "./loadStage.js";
 import { loadTaskOutcome } from "./loadTask.js";
 import { loadTriggerOutcome } from "./loadTrigger.js";
+import { loadEvalCatalog } from "./evalCatalog.js";
+import { evalFindingsFromIssues } from "./validateEval.js";
 import { readYamlObject } from "./readYamlObject.js";
 import { resolveCatalogContext } from "./resolveCatalogContext.js";
 import { getCatalogScanPaths } from "./browseCatalog.js";
@@ -46,7 +48,7 @@ function sortFindings(findings) {
 export function effectiveSeverity(finding, strict) {
     if (strict &&
         finding.severity === "warning" &&
-        STRICT_CATALOG_WARNINGS.has(finding.code)) {
+        (STRICT_CATALOG_WARNINGS.has(finding.code) || finding.category === "eval")) {
         return "error";
     }
     return finding.severity;
@@ -172,6 +174,9 @@ export function findingsFromLoadIssues(cwd, absPath, issues) {
                 withIssueLocation(findingTriggerError(cwd, absPath, issue.message, issue.code), issue),
             ];
         }
+        if (issue.category === "eval") {
+            return evalFindingsFromIssues(relPath(cwd, absPath), [issue]);
+        }
         if (issue.category === "catalog") {
             const code = issue.code;
             const severity = code === "catalog.manifest_missing" ||
@@ -647,6 +652,9 @@ async function validateManifestAll(invocationCwd) {
             }));
         }
     }
+    for (const entry of await loadEvalCatalog(manifest)) {
+        findings.push(...entry.findings);
+    }
     return findings;
 }
 /** Confirms a trigger's `pipeline`/`task` refs resolve against known catalog ids. */

package/dist/config/validateEval.d.ts

@@ -0,0 +1,36 @@
+import { type LoadedEval } from "./loadEval.js";
+import type { LoadIssue } from "./loadOutcome.js";
+import { type AgentCatalogPort } from "./resolveAgentTarget.js";
+import type { ValidationFinding, ValidationSeverity } from "./validateCatalog.js";
+export declare const EVAL_LARGE_RUN = 500;
+export declare const EVAL_TOO_LARGE_RUN = 5000;
+/** The slice of the dataset loader's report that eval validation reads. */
+export type DatasetReport = {
+    path: string;
+    relPath: string;
+    cases: Array<{
+        id: string;
+        input: Record<string, unknown>;
+        expected?: unknown;
+        tags?: string[];
+    }>;
+    findings: ValidationFinding[];
+    contentHash: string;
+};
+export type ValidateEvalContext = {
+    projectRoot: string;
+    evals?: ReadonlyArray<{
+        id: string;
+        absPath: string;
+    }>;
+    datasetReport?: DatasetReport;
+    datasetInline?: boolean;
+    isKnownModel?: (id: string) => Promise<boolean>;
+    agents?: AgentCatalogPort;
+};
+export declare function evalFinding(relPath: string, severity: ValidationSeverity, code: string, message: string, location?: {
+    line?: number;
+    column?: number;
+}): ValidationFinding;
+export declare function evalFindingsFromIssues(relPath: string, issues: LoadIssue[]): ValidationFinding[];
+export declare function validateEval(loaded: LoadedEval, ctx: ValidateEvalContext): Promise<ValidationFinding[]>;