veryfront 0.1.1128 → 0.1.1133

package/esm/cli/commands/eval/command.js

@@ -2,16 +2,17 @@
  * Eval command - Discover and run eval definitions from the evals/ directory.
  */
 import * as dntShim from "../../../_dnt.shims.js";
-import { dirname, isAbsolute, join, relative, resolve } from "../../../deps/jsr.io/@std/path/1.1.4/mod.js";
+import { dirname, isAbsolute, relative, resolve } from "../../../deps/jsr.io/@std/path/1.1.4/mod.js";
 import { isErroredToolExecutionResult, } from "../../../src/tool/index.js";
 import { orchestrateExtensions } from "../../../src/extensions/index.js";
 import { createEvalReportExporterRegistry, EvalReportExporterRegistryName, } from "../../../src/extensions/eval/index.js";
 import { createLLMProviderRegistry, LLMProviderRegistryName } from "../../../src/extensions/llm/index.js";
-import { compareEvalModelReports, compareEvalReports, createEvalModelComparisonMarkdown, createEvalRunId, EVAL_REPORT_SCHEMA_VERSION, exportEvalReport, resolveEvalRunProvenance, runEval, } from "../../../src/eval/index.js";
+import { exportEvalReport, resolveEvalRunProvenance, runEval } from "../../../src/eval/index.js";
 import { getCurrentVeryfrontCloudContext, getVeryfrontCloudBootstrap, runWithVeryfrontCloudContextAsync, } from "../../../src/provider/index.js";
 import { applyRuntimeAuthContext } from "../../shared/runtime-auth.js";
 import { cliLogger, exitProcess, VERSION } from "../../utils/index.js";
 import { discoverProjectAgentRuntime, runWithProjectAgentRuntime, } from "../../../src/agent/project/agent-runtime.js";
+import { runEvalReport } from "../../../src/eval/run-report.js";
 import { createErrorEnvelope, createSuccessEnvelope, isJsonMode, outputJson, } from "../../shared/json-output.js";
 import { withProjectSourceContext } from "../../shared/project-source-context.js";
 import { createEvalCliBuiltinExtensions } from "../../../src/extensions/builtin-extensions.js";
@@ -47,74 +48,14 @@ const MODEL_COMPARISON_METRICS = [
 ];
 const MODEL_COMPARISON_METRIC_SET = new Set(MODEL_COMPARISON_METRICS);
 const MODEL_COMPARISON_OBJECTIVE_DIRECTIONS = new Set(["minimize", "maximize"]);
-function xmlEscape(value) {
-    return value
-        .replaceAll("&", "&")
-        .replaceAll("<", "&lt;")
-        .replaceAll(">", "&gt;")
-        .replaceAll('"', "&quot;")
-        .replaceAll("'", "&apos;");
-}
 function stripFileProtocol(path) {
     if (!path.startsWith("file://"))
         return path;
     return decodeURIComponent(new URL(path).pathname);
 }
-function createEvalReportDirTimestamp(runId) {
-    return runId.startsWith("evalrun_") ? runId.slice("evalrun_".length) : runId;
-}
-function sanitizeEvalReportDirLabel(label) {
-    const normalized = label.startsWith("eval:") ? label.slice("eval:".length) : label;
-    return normalized.trim().replace(/[^A-Za-z0-9._-]+/g, "-").replace(/-+/g, "-").replace(/^[-._]+|[-._]+$/g, "");
-}
-export function createDefaultEvalReportDir(runId, label) {
-    const timestamp = createEvalReportDirTimestamp(runId);
-    const suffix = label ? sanitizeEvalReportDirLabel(label) : "";
-    return join(".veryfront", "evals", suffix ? `${timestamp}-${suffix}` : timestamp);
-}
-export function createEvalArtifactPaths(reportDir) {
-    return {
-        directory: reportDir,
-        summary: join(reportDir, "summary.json"),
-        results: join(reportDir, "results.jsonl"),
-        reportMarkdown: join(reportDir, "report.md"),
-    };
-}
-function createEvalSuiteArtifactPaths(reportDir) {
-    return {
-        directory: reportDir,
-        summary: join(reportDir, "summary.json"),
-        results: join(reportDir, "results.jsonl"),
-        reportMarkdown: join(reportDir, "report.md"),
-    };
-}
 function sortEvals(evals) {
     return [...evals].sort((left, right) => left.id.localeCompare(right.id) || left.filePath.localeCompare(right.filePath));
 }
-function createEvalSuiteChildDirectory(suiteDirectory, index, evalId) {
-    return join(suiteDirectory, `${String(index + 1).padStart(3, "0")}-${sanitizeEvalReportDirLabel(evalId)}`);
-}
-function sanitizeModelIdForPath(model) {
-    return model.trim().replace(/[^A-Za-z0-9._-]+/g, "__").replace(/^_+|_+$/g, "") || "model";
-}
-export function createEvalModelArtifactPaths(reportDir, model) {
-    const directory = join(reportDir, "models", sanitizeModelIdForPath(model));
-    return {
-        directory,
-        summary: join(directory, "summary.json"),
-        results: join(directory, "results.jsonl"),
-        reportMarkdown: join(directory, "report.md"),
-        junit: join(directory, "junit.xml"),
-    };
-}
-function createEvalModelComparisonArtifactPaths(reportDir, models) {
-    return {
-        directory: reportDir,
-        comparisonJson: join(reportDir, "comparison.json"),
-        comparisonMarkdown: join(reportDir, "comparison.md"),
-        models: Object.fromEntries(models.map((model) => [model, createEvalModelArtifactPaths(reportDir, model)])),
-    };
-}
 function displaySourcePath(filePath, projectDir) {
     const normalized = stripFileProtocol(filePath);
     if (normalized.startsWith(projectDir)) {
@@ -122,26 +63,6 @@ function displaySourcePath(filePath, projectDir) {
     }
     return normalized;
 }
-function blockingResults(record) {
-    return [...(record.metrics ?? []), ...(record.checks ?? [])].filter((result) => !result.skipped && result.pass === false &&
-        (result.severity === "gate" || result.severity === "budget"));
-}
-function skippedResults(record) {
-    return [...(record.metrics ?? []), ...(record.checks ?? [])].filter((result) => result.skipped);
-}
-function testcaseName(record) {
-    return `${record.exampleId}#${record.repetition}`;
-}
-function failureMessage(result) {
-    return `${result.name} failed`;
-}
-function failureBody(result) {
-    if (result.explanation)
-        return result.explanation;
-    if (result.evidence)
-        return JSON.stringify(result.evidence);
-    return failureMessage(result);
-}
 export function normalizeEvalCliId(id) {
     return id.startsWith("eval:") ? id : `eval:${id}`;
 }
@@ -168,35 +89,6 @@ export function normalizeEvalInputForAgent(input) {
     }
     return JSON.stringify(input);
 }
-export function summarizeReportForCli(report) {
-    return {
-        runId: report.runId,
-        evalId: report.definitionId,
-        target: report.target,
-        records: report.summary.records,
-        passed: report.summary.passed,
-        failed: report.summary.failed,
-        passRate: report.summary.passRate,
-        metrics: report.summary.metrics,
-    };
-}
-export function createSummaryArtifact(report, baseline) {
-    return {
-        kind: "eval-summary",
-        schemaVersion: report.schemaVersion ?? EVAL_REPORT_SCHEMA_VERSION,
-        runId: report.runId,
-        definitionId: report.definitionId,
-        targetKind: report.targetKind,
-        target: report.target,
-        ...(report.dataset ? { dataset: report.dataset } : {}),
-        startedAt: report.startedAt,
-        endedAt: report.endedAt,
-        summary: report.summary,
-        ...(report.metadata ? { metadata: report.metadata } : {}),
-        ...(report.exports ? { exports: report.exports } : {}),
-        ...(baseline ? { baseline } : {}),
-    };
-}
 function createEvalBaselineComparisonPolicy(options) {
     return {
         ...(options.baselinePassRateDropThreshold !== undefined
@@ -490,199 +382,12 @@ export async function createResolvedEvalModelComparisonConfig(projectDir, option
     const policy = await loadEvalModelComparisonPolicy(projectDir, config.comparisonPolicy);
     return { config, policy };
 }
-export function createEvalModelComparisonArtifact(reports, baselineModel, policy = {}) {
-    return compareEvalModelReports(reports, { ...policy, baselineModel });
-}
-export function createEvalModelComparisonExitCode(reports, exportRequired = false) {
-    return reports.some((report) => createEvalExitCode(report, undefined, exportRequired) !== 0)
-        ? 1
-        : 0;
-}
-export function createResultsJsonl(report) {
-    if (report.records.length === 0)
-        return "";
-    return `${report.records.map((record) => JSON.stringify(record)).join("\n")}\n`;
-}
-function markdownCell(value) {
-    return value.replaceAll("|", "\\|").replaceAll("\n", " ");
-}
-function numberCell(value) {
-    return value === undefined ? "-" : String(Math.round(value));
-}
-function decimalCell(value) {
-    if (value === undefined)
-        return "-";
-    if (Number.isInteger(value))
-        return String(value);
-    return value.toFixed(4).replace(/0+$/, "").replace(/\.$/, "");
-}
-function percentCell(value) {
-    return `${Math.round(value * 100)}%`;
-}
-function durationCell(valueMs) {
-    return valueMs === undefined ? "-" : `${(valueMs / 1000).toFixed(3)}s`;
-}
-function usdCell(value) {
-    if (value === undefined)
-        return "-";
-    const absolute = Math.abs(value);
-    if (absolute >= 0.01)
-        return `$${value.toFixed(2)}`;
-    return `$${value.toFixed(6).replace(/0+$/, "").replace(/\.$/, "")}`;
-}
-function isBlockingEvalResultFailure(result) {
-    return !result.skipped && result.pass === false &&
-        (result.severity === "gate" || result.severity === "budget");
-}
-function examplePassed(record) {
-    if (!record.completed || record.error)
-        return false;
-    return [...(record.metrics ?? []), ...(record.checks ?? [])].every((result) => !isBlockingEvalResultFailure(result));
-}
-function usageRows(usage) {
-    if (!usage)
-        return [];
-    const rows = [
-        ["Input tokens", numberCell(usage.inputTokens)],
-        ["Output tokens", numberCell(usage.outputTokens)],
-        ["Total tokens", numberCell(usage.totalTokens)],
-        ["Billable input tokens", numberCell(usage.billableInputTokens)],
-        ["Billable output tokens", numberCell(usage.billableOutputTokens)],
-        ["Provider input cost USD", usdCell(usage.providerInputCostUsd)],
-        ["Provider output cost USD", usdCell(usage.providerOutputCostUsd)],
-        ["Provider cost USD", usdCell(usage.providerCostUsd ?? usage.costUsd)],
-        ["Veryfront input charge USD", usdCell(usage.veryfrontInputChargeUsd)],
-        ["Veryfront output charge USD", usdCell(usage.veryfrontOutputChargeUsd)],
-        ["Veryfront charge USD", usdCell(usage.veryfrontChargeUsd)],
-        ["Veryfront billed USD", usdCell(usage.veryfrontBilledUsd)],
-        ["Cost credits", decimalCell(usage.costCredits)],
-        ["Cost source", usage.costSource ?? "-"],
-        ["Billing mode", usage.billingMode ?? "-"],
-        ["Usage capture status", usage.usageCaptureStatus ?? "-"],
-    ];
-    return rows.filter(([, value]) => value !== "-");
-}
-/** Render a human-reviewable markdown report for a single eval run. */
-export function createEvalMarkdownReport(report, baseline) {
-    const lines = [
-        `# Eval report: ${markdownCell(report.definitionId)}`,
-        "",
-        `Run: \`${markdownCell(report.runId)}\``,
-        `Target: \`${markdownCell(report.target)}\``,
-        ...(report.metadata?.model ? [`Model: \`${markdownCell(report.metadata.model)}\``] : []),
-        `Result: \`${report.summary.passed}/${report.summary.records} passed (${percentCell(report.summary.passRate)})\``,
-        "",
-        "## Metrics",
-        "",
-        "| Metric | Severity | Passed | Failed | Pass rate |",
-        "| --- | --- | ---: | ---: | ---: |",
-    ];
-    for (const metric of report.summary.metrics) {
-        lines.push(`| \`${markdownCell(metric.name)}\` | ${metric.severity} | ${metric.passed} | ${metric.failed} | ${percentCell(metric.passRate)} |`);
-    }
-    const rows = usageRows(report.summary.usage);
-    if (rows.length > 0) {
-        lines.push("", "## Usage", "", "| Usage | Value |", "| --- | ---: |");
-        for (const [label, value] of rows) {
-            lines.push(`| ${label} | ${value.startsWith("$") ? `\`${value}\`` : value} |`);
-        }
-    }
-    lines.push("", "## Examples", "", "| Example | Result | Duration | Tokens | Billed USD | Credits |", "| --- | ---: | ---: | ---: | ---: | ---: |");
-    for (const record of report.records) {
-        lines.push(`| \`${markdownCell(record.id)}\` | ${examplePassed(record) ? "PASS" : "FAIL"} | ${durationCell(record.durationMs)} | ${numberCell(record.usage.totalTokens)} | ${record.usage.veryfrontBilledUsd === undefined
-            ? "-"
-            : `\`${usdCell(record.usage.veryfrontBilledUsd)}\``} | ${decimalCell(record.usage.costCredits)} |`);
-    }
-    if (baseline) {
-        const direction = baseline.passRateDelta >= 0 ? "+" : "";
-        lines.push("", "## Baseline", "", `Status: \`${baseline.regressed ? "regressed" : "ok"}\``, `Pass rate delta: \`${direction}${Math.round(baseline.passRateDelta * 100)} pp\``);
-        if (baseline.newFailedExamples.length > 0) {
-            lines.push(`New failed examples: ${baseline.newFailedExamples.map(markdownCell).join(", ")}`);
-        }
-    }
-    if (report.exports?.length) {
-        lines.push("", "## Exports", "");
-        for (const result of report.exports) {
-            lines.push(`- \`${markdownCell(result.exporterId)}\`: ${result.ok ? "ok" : `failed, ${markdownCell(result.error)}`}`);
-        }
-    }
-    lines.push("");
-    return lines.join("\n");
-}
-export function createJunitXml(report) {
-    const skipped = report.records.reduce((count, record) => count + skippedResults(record).length, 0);
-    const lines = [
-        '<?xml version="1.0" encoding="UTF-8"?>',
-        `<testsuite name="${xmlEscape(report.definitionId)}" tests="${report.summary.records}" failures="${report.summary.failed}" skipped="${skipped}">`,
-    ];
-    for (const record of report.records) {
-        const failures = blockingResults(record);
-        const skips = skippedResults(record);
-        const attrs = `classname="${xmlEscape(report.definitionId)}" name="${xmlEscape(testcaseName(record))}" time="${(record.durationMs / 1000).toFixed(3)}"`;
-        if (failures.length === 0 && skips.length === 0) {
-            lines.push(`  <testcase ${attrs} />`);
-            continue;
-        }
-        lines.push(`  <testcase ${attrs}>`);
-        for (const failure of failures) {
-            lines.push(`    <failure message="${xmlEscape(failureMessage(failure))}">${xmlEscape(failureBody(failure))}</failure>`);
-        }
-        for (const skip of skips) {
-            lines.push(`    <skipped message="${xmlEscape(skip.explanation ?? `${skip.name} skipped`)}" />`);
-        }
-        lines.push("  </testcase>");
-    }
-    lines.push("</testsuite>");
-    return `${lines.join("\n")}\n`;
-}
-function createEvalSuiteResultsJsonl(summary) {
-    return summary.results.map((result) => JSON.stringify(result)).join("\n") +
-        (summary.results.length > 0 ? "\n" : "");
-}
-export function createEvalSuiteJunitXml(summary) {
-    const lines = [
-        '<?xml version="1.0" encoding="UTF-8"?>',
-        `<testsuites tests="${summary.total}" failures="${summary.failed}" skipped="0">`,
-        `  <testsuite name="veryfront eval suite" tests="${summary.total}" failures="${summary.failed}" skipped="0">`,
-    ];
-    for (const result of summary.results) {
-        const attrs = `classname="eval" name="${xmlEscape(result.id)}"`;
-        if (result.status === "passed") {
-            lines.push(`    <testcase ${attrs} />`);
-            continue;
-        }
-        const message = result.error ??
-            (result.summary?.failed
-                ? `${result.summary.failed} record(s) failed`
-                : "A required eval export failed.");
-        lines.push(`    <testcase ${attrs}>`);
-        lines.push(`      <failure message="${xmlEscape(message)}">${xmlEscape(message)}</failure>`);
-        lines.push("    </testcase>");
-    }
-    lines.push("  </testsuite>");
-    lines.push("</testsuites>");
-    return `${lines.join("\n")}\n`;
-}
 async function writeTextFileEnsuringDir(path, content) {
     const dir = dirname(path);
     if (dir && dir !== ".")
         await dntShim.Deno.mkdir(dir, { recursive: true });
     await dntShim.Deno.writeTextFile(path, content);
 }
-export async function writeEvalArtifacts(report, paths, baseline) {
-    await dntShim.Deno.mkdir(paths.directory, { recursive: true });
-    await dntShim.Deno.writeTextFile(paths.summary, JSON.stringify(createSummaryArtifact(report, baseline), null, 2));
-    await dntShim.Deno.writeTextFile(paths.results, createResultsJsonl(report));
-    await dntShim.Deno.writeTextFile(paths.reportMarkdown, createEvalMarkdownReport(report, baseline));
-}
-async function readEvalReport(path) {
-    return JSON.parse(await dntShim.Deno.readTextFile(path));
-}
-export function createEvalExitCode(report, baseline, exportRequired = false) {
-    const exportFailed = exportRequired &&
-        (!(report.exports?.length) || report.exports.some((result) => !result.ok));
-    return report.summary.failed === 0 && baseline?.regressed !== true && !exportFailed ? 0 : 1;
-}
 function resolveAgentTargetId(target) {
     return target.startsWith("agent:") ? target.slice("agent:".length) : target;
 }
@@ -894,38 +599,6 @@ function unsupportedEvalSuiteOption(options) {
         return undefined;
     return `${flags.join(", ")} require a named eval. Run \`veryfront eval <eval-id>\`.`;
 }
-function createEvalSuiteSummary(runId, startedAt, results) {
-    const passed = results.filter((result) => result.status === "passed").length;
-    return {
-        kind: "eval-suite-summary",
-        runId,
-        startedAt: startedAt.toISOString(),
-        endedAt: new Date().toISOString(),
-        total: results.length,
-        passed,
-        failed: results.length - passed,
-        results,
-    };
-}
-function createEvalSuiteMarkdown(summary) {
-    const rows = summary.results.map((result) => `| ${markdownCell(result.id)} | ${result.status} | ${result.summary ? `${result.summary.passed}/${result.summary.records}` : "n/a"} | ${result.error ? markdownCell(result.error) : ""} |`);
-    return [
-        "# Eval suite report",
-        "",
-        `Run: \`${markdownCell(summary.runId)}\``,
-        `Result: \`${summary.passed}/${summary.total} passed\``,
-        "",
-        "| Eval | Status | Records | Error |",
-        "| --- | --- | --- | --- |",
-        ...rows,
-        "",
-    ].join("\n");
-}
-async function writeEvalSuiteArtifacts(summary, artifacts) {
-    await writeTextFileEnsuringDir(artifacts.summary, JSON.stringify(summary, null, 2));
-    await writeTextFileEnsuringDir(artifacts.results, createEvalSuiteResultsJsonl(summary));
-    await writeTextFileEnsuringDir(artifacts.reportMarkdown, createEvalSuiteMarkdown(summary));
-}
 function getDiscoveredEvals(runtime) {
     return [...runtime.evals.entries()].map(([id, definition]) => {
         if (!definition.source) {
@@ -963,7 +636,7 @@ function printReport(report, baseline) {
         }
     }
 }
-export function createEvalCliExportConfig(evalItem, options, projectDir, artifactPaths, registry, config) {
+function createEvalCliBaseExportConfig(options, registry, config) {
     const exporterIds = resolveEvalExporterIds(options);
     if (exporterIds.length === 0)
         return undefined;
@@ -973,12 +646,7 @@ export function createEvalCliExportConfig(evalItem, options, projectDir, artifac
         exporterIds,
         required: resolveEvalExportRequired(options),
         context: {
-            evalId: evalItem.definition.id,
             ...(projectReference ? { projectReference } : {}),
-            sourcePath: displaySourcePath(evalItem.filePath, projectDir),
-            reportPath: options.report ?? artifactPaths.summary,
-            tags: evalItem.definition.tags,
-            metadata: evalItem.definition.metadata,
             redaction: resolveEvalExportRedactionFromEnv(),
         },
     };
@@ -1095,75 +763,6 @@ function resolveEvalModelComparisonConfig(options) {
         models: uniqueValues([baselineModel, ...candidateModels]),
     };
 }
-function createAgentAdapterForModel(agent, options, model) {
-    return createAgentAdapter(agent, { ...options, model });
-}
-async function writeEvalModelComparisonArtifacts(comparison, paths) {
-    await dntShim.Deno.mkdir(paths.directory, { recursive: true });
-    await dntShim.Deno.writeTextFile(paths.comparisonJson, JSON.stringify(comparison, null, 2));
-    await dntShim.Deno.writeTextFile(paths.comparisonMarkdown, createEvalModelComparisonMarkdown(comparison));
-}
-async function runEvalModelComparison(input) {
-    const runId = createEvalRunId();
-    const reportDir = input.options.reportDir ??
-        createDefaultEvalReportDir(runId, input.evalItem.id);
-    const paths = createEvalModelComparisonArtifactPaths(reportDir, input.config.models);
-    const provenance = await resolveEvalRunProvenance({
-        projectDir: input.projectDir,
-        frameworkVersion: VERSION,
-    });
-    const reports = [];
-    for (const model of input.config.models) {
-        const modelPaths = paths.models[model];
-        const modelOptions = { ...input.options, model, report: undefined };
-        const modelRunId = `${runId}_${sanitizeModelIdForPath(model)}`;
-        const exportConfig = createEvalCliExportConfig(input.evalItem, modelOptions, input.projectDir, modelPaths, input.exporterRegistry, input.projectReference ? { projectSlug: input.projectReference } : undefined);
-        const finalizedReport = await runEvalWithGatewayBillingGroup(modelRunId, () => runEval(input.evalItem.definition, {
-            baseDir: input.options.datasetBase ?? input.projectDir,
-            runId: modelRunId,
-            adapters: {
-                agent: createAgentAdapterForModel(input.agent, input.options, model),
-            },
-            metadata: {
-                model,
-                provenance,
-            },
-        }));
-        const report = await exportEvalReportForCli(finalizedReport, exportConfig);
-        reports.push(report);
-        await writeEvalArtifacts(report, modelPaths);
-        await writeTextFileEnsuringDir(modelPaths.junit, createJunitXml(report));
-    }
-    const comparison = createEvalModelComparisonArtifact(reports, input.config.baselineModel, input.policy);
-    await writeEvalModelComparisonArtifacts(comparison, paths);
-    if (input.options.report) {
-        await writeTextFileEnsuringDir(input.options.report, JSON.stringify(comparison, null, 2));
-    }
-    if (isJsonMode()) {
-        await outputJson(createSuccessEnvelope("eval", {
-            reports,
-            comparison,
-            artifacts: paths,
-        }));
-    }
-    else {
-        for (const report of reports) {
-            const model = report.metadata?.model ?? report.runId;
-            cliLogger.info(`Model: ${model}`);
-            printReport(report);
-        }
-        cliLogger.info(`Recommendation: ${comparison.recommendation.decision}${comparison.recommendation.model ? ` (${comparison.recommendation.model})` : ""}`);
-        for (const reason of comparison.recommendation.reasons) {
-            cliLogger.info(`  - ${reason}`);
-        }
-        cliLogger.info(`Report directory: ${paths.directory}`);
-        cliLogger.info(`Comparison: ${paths.comparisonJson}`);
-        cliLogger.info(`Comparison markdown: ${paths.comparisonMarkdown}`);
-        if (input.options.report)
-            cliLogger.info(`Report: ${input.options.report}`);
-    }
-    return createEvalModelComparisonExitCode(reports, resolveEvalExportRequired(input.options));
-}
 async function outputEvalNotFound(id, evals) {
     if (isJsonMode()) {
         await outputJson(createErrorEnvelope("eval", {
@@ -1225,88 +824,57 @@ async function outputEvalUsageError(message) {
     }
     return 2;
 }
-async function runEvalSuite(input) {
-    const runId = createEvalRunId();
-    const startedAt = new Date();
-    const artifacts = createEvalSuiteArtifactPaths(input.options.reportDir ?? createDefaultEvalReportDir(runId));
-    const provenance = await resolveEvalRunProvenance({
-        projectDir: input.projectDir,
-        frameworkVersion: VERSION,
-    });
-    const results = [];
-    for (const [index, evalItem] of sortEvals(input.evals).entries()) {
-        const evalArtifacts = createEvalArtifactPaths(createEvalSuiteChildDirectory(artifacts.directory, index, evalItem.id));
-        try {
-            const agentId = evalItem.definition.targetKind === "agent"
-                ? resolveAgentTargetId(evalItem.definition.target)
-                : undefined;
-            const toolId = evalItem.definition.targetKind === "tool"
-                ? resolveToolTargetId(evalItem.definition.target)
-                : undefined;
-            const agent = agentId ? input.projectRuntime.agents.get(agentId) : undefined;
-            const tool = toolId ? input.projectRuntime.tools.get(toolId) : undefined;
-            if (agentId && !agent)
-                throw new Error(`Agent "${agentId}" not found for eval target.`);
-            if (toolId && !tool)
-                throw new Error(`Tool "${toolId}" not found for eval target.`);
-            const evalRunId = `${runId}_${String(index + 1).padStart(3, "0")}`;
-            const exportConfig = createEvalCliExportConfig(evalItem, input.options, input.projectDir, evalArtifacts, input.exporterRegistry, input.config);
-            const finalizedReport = await runEvalWithGatewayBillingGroup(evalRunId, () => runEval(evalItem.definition, {
-                baseDir: input.options.datasetBase ?? input.projectDir,
-                runId: evalRunId,
-                adapters: evalItem.definition.targetKind === "tool"
-                    ? { tool: createToolAdapter(tool, createEvalToolExecutionContext(input.config)) }
-                    : { agent: createAgentAdapter(agent, input.options) },
-                metadata: { provenance },
-            }));
-            const report = await exportEvalReportForCli(finalizedReport, exportConfig);
-            await writeEvalArtifacts(report, evalArtifacts);
-            const status = createEvalExitCode(report, undefined, exportConfig?.required) === 0
-                ? "passed"
-                : "failed";
-            results.push({
-                id: evalItem.id,
-                name: evalItem.name,
-                target: evalItem.definition.target,
-                status,
-                artifacts: evalArtifacts,
-                summary: summarizeReportForCli(report),
-            });
-            if (!isJsonMode()) {
-                printReport(report);
-                cliLogger.info(`Report directory: ${evalArtifacts.directory}`);
-            }
-        }
-        catch (error) {
-            const message = error instanceof Error ? error.message : String(error);
-            results.push({
-                id: evalItem.id,
-                name: evalItem.name,
-                target: evalItem.definition.target,
-                status: "error",
-                artifacts: evalArtifacts,
-                error: message,
-            });
-            if (!isJsonMode())
-                cliLogger.error(`Eval ${evalItem.id}: ${message}`);
-        }
-    }
-    const summary = createEvalSuiteSummary(runId, startedAt, results);
-    await writeEvalSuiteArtifacts(summary, artifacts);
-    if (input.options.junit) {
-        await writeTextFileEnsuringDir(input.options.junit, createEvalSuiteJunitXml(summary));
-    }
-    if (isJsonMode()) {
-        await outputJson(createSuccessEnvelope("eval", { suite: summary, artifacts }));
-    }
-    else {
-        cliLogger.info(`Eval suite: ${summary.passed}/${summary.total} passed`);
-        cliLogger.info(`Report directory: ${artifacts.directory}`);
-        cliLogger.info(`Suite report: ${artifacts.reportMarkdown}`);
-        if (input.options.junit)
-            cliLogger.info(`JUnit: ${input.options.junit}`);
-    }
-    return summary.failed === 0 ? 0 : 1;
+function createEvalReportCommandAdapters(input) {
+    return {
+        targets: {
+            runEval: (evalItem, options) => runEval(evalItem.definition, {
+                baseDir: options.baseDir,
+                runId: options.runId,
+                adapters: options.targetKind === "tool"
+                    ? { tool: options.targetAdapter }
+                    : { agent: options.targetAdapter },
+                metadata: options.metadata,
+            }),
+            resolveTarget: (evalItem) => {
+                const agentId = evalItem.definition.targetKind === "agent"
+                    ? resolveAgentTargetId(evalItem.definition.target)
+                    : undefined;
+                const toolId = evalItem.definition.targetKind === "tool"
+                    ? resolveToolTargetId(evalItem.definition.target)
+                    : undefined;
+                const agent = agentId ? input.projectRuntime.agents.get(agentId) : undefined;
+                const tool = toolId ? input.projectRuntime.tools.get(toolId) : undefined;
+                if (agentId && !agent)
+                    throw new Error(`Agent "${agentId}" not found for eval target.`);
+                if (toolId && !tool)
+                    throw new Error(`Tool "${toolId}" not found for eval target.`);
+                return {
+                    targetKind: evalItem.definition.targetKind,
+                    target: evalItem.definition.target,
+                    targetAdapter: evalItem.definition.targetKind === "tool"
+                        ? createToolAdapter(tool, createEvalToolExecutionContext(input.config))
+                        : createAgentAdapter(agent, input.options),
+                };
+            },
+            createModelTargetAdapter: (model) => {
+                const options = { ...input.options, model };
+                if (!input.modelComparisonAgent) {
+                    throw new Error("Model comparison agent is not configured.");
+                }
+                return createAgentAdapter(input.modelComparisonAgent, options);
+            },
+        },
+        artifacts: {
+            readTextFile: (path) => dntShim.Deno.readTextFile(path),
+            writeTextFileEnsuringDir,
+        },
+        billing: {
+            runWithGatewayBillingGroup: runEvalWithGatewayBillingGroup,
+        },
+        exporters: {
+            exportReport: (report, config) => exportEvalReportForCli(report, config),
+        },
+    };
 }
 export async function runEvalCommand(options, dependencies = {}) {
     const projectDir = options.projectDir ?? dntShim.Deno.cwd();
@@ -1375,14 +943,52 @@ export async function runEvalCommand(options, dependencies = {}) {
             const selectedExporterIds = resolveEvalExporterIds(options);
             const extensionSetup = await setupEvalCliExtensions(projectDir, config, selectedExporterIds);
             try {
-                return await runWithProjectAgentRuntime(projectRuntime, () => runEvalSuite({
-                    evals,
-                    options,
-                    projectDir,
-                    projectRuntime,
-                    config,
-                    exporterRegistry: extensionSetup.exporterRegistry,
-                }));
+                return await runWithProjectAgentRuntime(projectRuntime, async () => {
+                    const outcome = await runEvalReport({
+                        kind: "suite",
+                        projectDir,
+                        frameworkVersion: VERSION,
+                        ...(options.datasetBase ? { datasetBase: options.datasetBase } : {}),
+                        ...(options.reportDir ? { reportDir: options.reportDir } : {}),
+                        ...(options.junit ? { junit: options.junit } : {}),
+                        exportConfig: createEvalCliBaseExportConfig(options, extensionSetup.exporterRegistry, config),
+                        provenance: await resolveEvalRunProvenance({
+                            projectDir,
+                            frameworkVersion: VERSION,
+                        }),
+                        evalItems: evals,
+                    }, createEvalReportCommandAdapters({
+                        options,
+                        config,
+                        projectRuntime,
+                    }));
+                    if (outcome.kind !== "suite") {
+                        throw new Error(`Unexpected eval report outcome: ${outcome.kind}`);
+                    }
+                    if (isJsonMode()) {
+                        await outputJson(createSuccessEnvelope("eval", {
+                            suite: outcome.suite,
+                            artifacts: outcome.artifacts,
+                        }));
+                    }
+                    else {
+                        for (const child of outcome.outputHints.children ?? []) {
+                            if (child.kind === "report") {
+                                printReport(child.report);
+                                cliLogger.info(`Report directory: ${child.reportDirectory}`);
+                            }
+                            else {
+                                cliLogger.error(`Eval ${child.evalId}: ${child.error}`);
+                            }
+                        }
+                        cliLogger.info(`Eval suite: ${outcome.suite.passed}/${outcome.suite.total} passed`);
+                        cliLogger.info(`Report directory: ${outcome.outputHints.reportDirectory}`);
+                        cliLogger.info(`Suite report: ${outcome.outputHints.reportMarkdown}`);
+                        if (outcome.outputHints.junit)
+                            cliLogger.info(`JUnit: ${outcome.outputHints.junit}`);
+                    }
+                    return outcome.exitCode;
+                });
             }
             finally {
                 await extensionSetup.loader.teardownAll();
@@ -1426,72 +1032,123 @@ export async function runEvalCommand(options, dependencies = {}) {
                 return await outputToolNotFound(toolId);
             }
             if (modelComparisonConfig) {
-                return await runWithProjectAgentRuntime(projectRuntime, () => runEvalModelComparison({
-                    evalItem,
-                    agent: agent,
-                    options,
-                    projectDir,
-                    config: modelComparisonConfig.config,
-                    policy: modelComparisonConfig.policy,
-                    exporterRegistry: extensionSetup.exporterRegistry,
-                    projectReference: resolveEvalRuntimeProjectSlug(config),
-                }));
+                return await runWithProjectAgentRuntime(projectRuntime, async () => {
+                    const outcome = await runEvalReport({
+                        kind: "model-comparison",
+                        projectDir,
+                        frameworkVersion: VERSION,
+                        ...(options.datasetBase ? { datasetBase: options.datasetBase } : {}),
+                        ...(options.reportDir ? { reportDir: options.reportDir } : {}),
+                        ...(options.report ? { report: options.report } : {}),
+                        exportConfig: createEvalCliBaseExportConfig(options, extensionSetup.exporterRegistry, config),
+                        provenance: await resolveEvalRunProvenance({
+                            projectDir,
+                            frameworkVersion: VERSION,
+                        }),
+                        evalItem,
+                        target: evalItem.definition.target,
+                        baselineModel: modelComparisonConfig.config.baselineModel,
+                        candidateModels: modelComparisonConfig.config.candidateModels,
+                        comparisonPolicy: modelComparisonConfig.policy,
+                        ...(options.maxOutputTokens !== undefined
+                            ? { maxOutputTokens: options.maxOutputTokens }
+                            : {}),
+                    }, createEvalReportCommandAdapters({
+                        options,
+                        config,
+                        projectRuntime,
+                        modelComparisonAgent: agent,
+                    }));
+                    if (outcome.kind !== "model-comparison") {
+                        throw new Error(`Unexpected eval report outcome: ${outcome.kind}`);
+                    }
+                    if (isJsonMode()) {
+                        await outputJson(createSuccessEnvelope("eval", {
+                            reports: outcome.reports,
+                            comparison: outcome.comparison,
+                            artifacts: outcome.artifacts,
+                        }));
+                    }
+                    else {
+                        for (const model of outcome.outputHints.models ?? []) {
+                            cliLogger.info(`Model: ${model.model}`);
+                            printReport(model.report);
+                        }
+                        const recommendation = outcome.comparison.recommendation;
+                        cliLogger.info(`Recommendation: ${recommendation.decision}${recommendation.model ? ` (${recommendation.model})` : ""}`);
+                        for (const reason of recommendation.reasons) {
+                            cliLogger.info(`  - ${reason}`);
+                        }
+                        cliLogger.info(`Report directory: ${outcome.outputHints.reportDirectory}`);
+                        if (outcome.outputHints.comparisonJson) {
+                            cliLogger.info(`Comparison: ${outcome.outputHints.comparisonJson}`);
+                        }
+                        if (outcome.outputHints.comparisonMarkdown) {
+                            cliLogger.info(`Comparison markdown: ${outcome.outputHints.comparisonMarkdown}`);
+                        }
+                        if (outcome.outputHints.report) {
+                            cliLogger.info(`Report: ${outcome.outputHints.report}`);
+                        }
+                    }
+                    return outcome.exitCode;
+                });
             }
-            const runId = createEvalRunId();
-            const artifactPaths = createEvalArtifactPaths(options.reportDir ?? createDefaultEvalReportDir(runId, evalItem.id));
-            const provenance = await resolveEvalRunProvenance({
+            const targetAdapter = evalItem.definition.targetKind === "tool"
+                ? createToolAdapter(tool, createEvalToolExecutionContext(config))
+                : createAgentAdapter(agent, options);
+            const outcome = await runWithProjectAgentRuntime(projectRuntime, async () => await runEvalReport({
+                kind: "single",
                 projectDir,
                 frameworkVersion: VERSION,
-            });
-            const billingGroupId = options.model
-                ? `${runId}_${sanitizeModelIdForPath(options.model)}`
-                : runId;
-            const exportConfig = createEvalCliExportConfig(evalItem, options, projectDir, artifactPaths, extensionSetup.exporterRegistry, config);
-            const finalizedReport = await runWithProjectAgentRuntime(projectRuntime, () => runEvalWithGatewayBillingGroup(billingGroupId, () => runEval(evalItem.definition, {
-                baseDir: options.datasetBase ?? projectDir,
-                runId,
-                adapters: evalItem.definition.targetKind === "tool"
-                    ? { tool: createToolAdapter(tool, createEvalToolExecutionContext(config)) }
-                    : { agent: createAgentAdapter(agent, options) },
-                metadata: {
-                    provenance,
-                    ...(options.model ? { model: options.model } : {}),
-                },
+                ...(options.datasetBase ? { datasetBase: options.datasetBase } : {}),
+                ...(options.reportDir ? { reportDir: options.reportDir } : {}),
+                ...(options.report ? { report: options.report } : {}),
+                ...(options.junit ? { junit: options.junit } : {}),
+                ...(options.baseline ? { baseline: options.baseline } : {}),
+                ...(options.writeBaseline ? { writeBaseline: options.writeBaseline } : {}),
+                baselinePolicy: createEvalBaselineComparisonPolicy(options),
+                exportConfig: createEvalCliBaseExportConfig(options, extensionSetup.exporterRegistry, config),
+                provenance: await resolveEvalRunProvenance({
+                    projectDir,
+                    frameworkVersion: VERSION,
+                }),
+                evalItem,
+                targetKind: evalItem.definition.targetKind,
+                target: evalItem.definition.target,
+                targetAdapter,
+                ...(options.model ? { selectedModel: options.model } : {}),
+                ...(options.maxOutputTokens !== undefined
+                    ? { maxOutputTokens: options.maxOutputTokens }
+                    : {}),
+            }, createEvalReportCommandAdapters({
+                options,
+                config,
+                projectRuntime,
             })));
-            const report = await exportEvalReportForCli(finalizedReport, exportConfig);
-            const baseline = options.baseline
-                ? compareEvalReports(report, await readEvalReport(options.baseline), createEvalBaselineComparisonPolicy(options))
-                : undefined;
-            await writeEvalArtifacts(report, artifactPaths, baseline);
-            if (options.report) {
-                await writeTextFileEnsuringDir(options.report, JSON.stringify(report, null, 2));
-            }
-            if (options.junit) {
-                await writeTextFileEnsuringDir(options.junit, createJunitXml(report));
-            }
-            if (options.writeBaseline) {
-                await writeTextFileEnsuringDir(options.writeBaseline, JSON.stringify(report, null, 2));
+            if (outcome.kind !== "single") {
+                throw new Error(`Unexpected eval report outcome: ${outcome.kind}`);
             }
             if (isJsonMode()) {
                 await outputJson(createSuccessEnvelope("eval", {
-                    report,
-                    summary: summarizeReportForCli(report),
-                    baseline,
-                    artifacts: artifactPaths,
+                    report: outcome.report,
+                    summary: outcome.summary,
+                    baseline: outcome.baseline,
+                    artifacts: outcome.artifacts,
                 }));
             }
             else {
-                printReport(report, baseline);
-                cliLogger.info(`Report directory: ${artifactPaths.directory}`);
-                cliLogger.info(`Report markdown: ${artifactPaths.reportMarkdown}`);
-                if (options.report)
-                    cliLogger.info(`Report: ${options.report}`);
-                if (options.junit)
-                    cliLogger.info(`JUnit: ${options.junit}`);
-                if (options.writeBaseline)
-                    cliLogger.info(`Baseline written: ${options.writeBaseline}`);
+                printReport(outcome.report, outcome.baseline);
+                cliLogger.info(`Report directory: ${outcome.outputHints.reportDirectory}`);
+                cliLogger.info(`Report markdown: ${outcome.outputHints.reportMarkdown}`);
+                if (outcome.outputHints.report)
+                    cliLogger.info(`Report: ${outcome.outputHints.report}`);
+                if (outcome.outputHints.junit)
+                    cliLogger.info(`JUnit: ${outcome.outputHints.junit}`);
+                if (outcome.outputHints.baselineWritten) {
+                    cliLogger.info(`Baseline written: ${outcome.outputHints.baselineWritten}`);
+                }
             }
-            return createEvalExitCode(report, baseline, exportConfig?.required);
+            return outcome.exitCode;
         }
         finally {
             await extensionSetup.loader.teardownAll();