veryfront 0.1.1127 → 0.1.1132

package/esm/src/eval/run-report.js

@@ -0,0 +1,603 @@
+import { join, relative } from "../../deps/jsr.io/@std/path/1.1.4/mod.js";
+import { compareEvalReports } from "./baseline.js";
+import { compareEvalModelReports, createEvalModelComparisonMarkdown } from "./model-comparison.js";
+import { EVAL_REPORT_SCHEMA_VERSION } from "./report.js";
+import { createEvalRunId } from "./run-id.js";
+function createEvalReportDirTimestamp(runId) {
+    return runId.startsWith("evalrun_") ? runId.slice("evalrun_".length) : runId;
+}
+function sanitizeEvalReportDirLabel(label) {
+    const normalized = label.startsWith("eval:") ? label.slice("eval:".length) : label;
+    return normalized.trim().replace(/[^A-Za-z0-9._-]+/g, "-").replace(/-+/g, "-").replace(/^[-._]+|[-._]+$/g, "");
+}
+function createDefaultEvalReportDir(runId, label) {
+    const timestamp = createEvalReportDirTimestamp(runId);
+    const suffix = label ? sanitizeEvalReportDirLabel(label) : "";
+    return join(".veryfront", "evals", suffix ? `${timestamp}-${suffix}` : timestamp);
+}
+function createEvalArtifactPaths(reportDir) {
+    return {
+        directory: reportDir,
+        summary: join(reportDir, "summary.json"),
+        results: join(reportDir, "results.jsonl"),
+        reportMarkdown: join(reportDir, "report.md"),
+    };
+}
+function createEvalSuiteArtifactPaths(reportDir) {
+    return {
+        directory: reportDir,
+        summary: join(reportDir, "summary.json"),
+        results: join(reportDir, "results.jsonl"),
+        reportMarkdown: join(reportDir, "report.md"),
+    };
+}
+function createEvalModelArtifactPaths(reportDir, model) {
+    const directory = join(reportDir, "models", sanitizeModelIdForPath(model));
+    return {
+        directory,
+        summary: join(directory, "summary.json"),
+        results: join(directory, "results.jsonl"),
+        reportMarkdown: join(directory, "report.md"),
+        junit: join(directory, "junit.xml"),
+    };
+}
+function createEvalModelComparisonArtifactPaths(reportDir, models) {
+    return {
+        directory: reportDir,
+        comparisonJson: join(reportDir, "comparison.json"),
+        comparisonMarkdown: join(reportDir, "comparison.md"),
+        models: Object.fromEntries(models.map((model) => [model, createEvalModelArtifactPaths(reportDir, model)])),
+    };
+}
+function createEvalSuiteChildDirectory(suiteDirectory, index, evalId) {
+    return join(suiteDirectory, `${String(index + 1).padStart(3, "0")}-${sanitizeEvalReportDirLabel(evalId)}`);
+}
+function sanitizeModelIdForPath(model) {
+    return model.trim().replace(/[^A-Za-z0-9._-]+/g, "__").replace(/^_+|_+$/g, "") || "model";
+}
+function uniqueTrimmedValues(values) {
+    return Array.from(new Set(values.map((value) => value.trim()).filter(Boolean)));
+}
+function summarizeReportForCli(report) {
+    return {
+        runId: report.runId,
+        evalId: report.definitionId,
+        target: report.target,
+        records: report.summary.records,
+        passed: report.summary.passed,
+        failed: report.summary.failed,
+        passRate: report.summary.passRate,
+        metrics: report.summary.metrics,
+    };
+}
+function sortEvals(evals) {
+    return [...evals].sort((left, right) => left.id.localeCompare(right.id) || left.filePath.localeCompare(right.filePath));
+}
+function createSummaryArtifact(report, baseline) {
+    return {
+        kind: "eval-summary",
+        schemaVersion: report.schemaVersion ?? EVAL_REPORT_SCHEMA_VERSION,
+        runId: report.runId,
+        definitionId: report.definitionId,
+        targetKind: report.targetKind,
+        target: report.target,
+        ...(report.dataset ? { dataset: report.dataset } : {}),
+        startedAt: report.startedAt,
+        endedAt: report.endedAt,
+        summary: report.summary,
+        ...(report.metadata ? { metadata: report.metadata } : {}),
+        ...(report.exports ? { exports: report.exports } : {}),
+        ...(baseline ? { baseline } : {}),
+    };
+}
+function createResultsJsonl(report) {
+    if (report.records.length === 0)
+        return "";
+    return `${report.records.map((record) => JSON.stringify(record)).join("\n")}\n`;
+}
+function markdownCell(value) {
+    return value.replaceAll("|", "\\|").replaceAll("\n", " ");
+}
+function numberCell(value) {
+    return value === undefined ? "-" : String(Math.round(value));
+}
+function decimalCell(value) {
+    if (value === undefined)
+        return "-";
+    if (Number.isInteger(value))
+        return String(value);
+    return value.toFixed(4).replace(/0+$/, "").replace(/\.$/, "");
+}
+function percentCell(value) {
+    return `${Math.round(value * 100)}%`;
+}
+function durationCell(valueMs) {
+    return valueMs === undefined ? "-" : `${(valueMs / 1000).toFixed(3)}s`;
+}
+function usdCell(value) {
+    if (value === undefined)
+        return "-";
+    const absolute = Math.abs(value);
+    if (absolute >= 0.01)
+        return `$${value.toFixed(2)}`;
+    return `$${value.toFixed(6).replace(/0+$/, "").replace(/\.$/, "")}`;
+}
+function usageRows(usage) {
+    if (!usage)
+        return [];
+    const rows = [
+        ["Input tokens", numberCell(usage.inputTokens)],
+        ["Output tokens", numberCell(usage.outputTokens)],
+        ["Total tokens", numberCell(usage.totalTokens)],
+        ["Billable input tokens", numberCell(usage.billableInputTokens)],
+        ["Billable output tokens", numberCell(usage.billableOutputTokens)],
+        ["Provider input cost USD", usdCell(usage.providerInputCostUsd)],
+        ["Provider output cost USD", usdCell(usage.providerOutputCostUsd)],
+        ["Provider cost USD", usdCell(usage.providerCostUsd ?? usage.costUsd)],
+        ["Veryfront input charge USD", usdCell(usage.veryfrontInputChargeUsd)],
+        ["Veryfront output charge USD", usdCell(usage.veryfrontOutputChargeUsd)],
+        ["Veryfront charge USD", usdCell(usage.veryfrontChargeUsd)],
+        ["Veryfront billed USD", usdCell(usage.veryfrontBilledUsd)],
+        ["Cost credits", decimalCell(usage.costCredits)],
+        ["Cost source", usage.costSource ?? "-"],
+        ["Billing mode", usage.billingMode ?? "-"],
+        ["Usage capture status", usage.usageCaptureStatus ?? "-"],
+    ];
+    return rows.filter(([, value]) => value !== "-");
+}
+function isBlockingEvalResultFailure(result) {
+    return !result.skipped && result.pass === false &&
+        (result.severity === "gate" || result.severity === "budget");
+}
+function examplePassed(record) {
+    if (!record.completed || record.error)
+        return false;
+    return [...(record.metrics ?? []), ...(record.checks ?? [])].every((result) => !isBlockingEvalResultFailure(result));
+}
+function createEvalMarkdownReport(report, baseline) {
+    const lines = [
+        `# Eval report: ${markdownCell(report.definitionId)}`,
+        "",
+        `Run: \`${markdownCell(report.runId)}\``,
+        `Target: \`${markdownCell(report.target)}\``,
+        ...(report.metadata?.model ? [`Model: \`${markdownCell(report.metadata.model)}\``] : []),
+        `Result: \`${report.summary.passed}/${report.summary.records} passed (${percentCell(report.summary.passRate)})\``,
+        "",
+        "## Metrics",
+        "",
+        "| Metric | Severity | Passed | Failed | Pass rate |",
+        "| --- | --- | ---: | ---: | ---: |",
+    ];
+    for (const metric of report.summary.metrics) {
+        lines.push(`| \`${markdownCell(metric.name)}\` | ${metric.severity} | ${metric.passed} | ${metric.failed} | ${percentCell(metric.passRate)} |`);
+    }
+    const rows = usageRows(report.summary.usage);
+    if (rows.length > 0) {
+        lines.push("", "## Usage", "", "| Usage | Value |", "| --- | ---: |");
+        for (const [label, value] of rows) {
+            lines.push(`| ${label} | ${value.startsWith("$") ? `\`${value}\`` : value} |`);
+        }
+    }
+    lines.push("", "## Examples", "", "| Example | Result | Duration | Tokens | Billed USD | Credits |", "| --- | ---: | ---: | ---: | ---: | ---: |");
+    for (const record of report.records) {
+        lines.push(`| \`${markdownCell(record.id)}\` | ${examplePassed(record) ? "PASS" : "FAIL"} | ${durationCell(record.durationMs)} | ${numberCell(record.usage.totalTokens)} | ${record.usage.veryfrontBilledUsd === undefined
+            ? "-"
+            : `\`${usdCell(record.usage.veryfrontBilledUsd)}\``} | ${decimalCell(record.usage.costCredits)} |`);
+    }
+    if (baseline) {
+        const direction = baseline.passRateDelta >= 0 ? "+" : "";
+        lines.push("", "## Baseline", "", `Status: \`${baseline.regressed ? "regressed" : "ok"}\``, `Pass rate delta: \`${direction}${Math.round(baseline.passRateDelta * 100)} pp\``);
+        if (baseline.newFailedExamples.length > 0) {
+            lines.push(`New failed examples: ${baseline.newFailedExamples.map(markdownCell).join(", ")}`);
+        }
+    }
+    if (report.exports?.length) {
+        lines.push("", "## Exports", "");
+        for (const result of report.exports) {
+            lines.push(`- \`${markdownCell(result.exporterId)}\`: ${result.ok ? "ok" : `failed, ${markdownCell(result.error)}`}`);
+        }
+    }
+    lines.push("");
+    return lines.join("\n");
+}
+function xmlEscape(value) {
+    return value
+        .replaceAll("&", "&")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&apos;");
+}
+function blockingResults(record) {
+    return [...(record.metrics ?? []), ...(record.checks ?? [])].filter((result) => !result.skipped && result.pass === false &&
+        (result.severity === "gate" || result.severity === "budget"));
+}
+function skippedResults(record) {
+    return [...(record.metrics ?? []), ...(record.checks ?? [])].filter((result) => result.skipped);
+}
+function testcaseName(record) {
+    return `${record.exampleId}#${record.repetition}`;
+}
+function failureMessage(result) {
+    return `${result.name} failed`;
+}
+function failureBody(result) {
+    if (result.explanation)
+        return result.explanation;
+    if (result.evidence)
+        return JSON.stringify(result.evidence);
+    return failureMessage(result);
+}
+function createJunitXml(report) {
+    const skipped = report.records.reduce((count, record) => count + skippedResults(record).length, 0);
+    const lines = [
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        `<testsuite name="${xmlEscape(report.definitionId)}" tests="${report.summary.records}" failures="${report.summary.failed}" skipped="${skipped}">`,
+    ];
+    for (const record of report.records) {
+        const failures = blockingResults(record);
+        const skips = skippedResults(record);
+        const attrs = `classname="${xmlEscape(report.definitionId)}" name="${xmlEscape(testcaseName(record))}" time="${(record.durationMs / 1000).toFixed(3)}"`;
+        if (failures.length === 0 && skips.length === 0) {
+            lines.push(`  <testcase ${attrs} />`);
+            continue;
+        }
+        lines.push(`  <testcase ${attrs}>`);
+        for (const failure of failures) {
+            lines.push(`    <failure message="${xmlEscape(failureMessage(failure))}">${xmlEscape(failureBody(failure))}</failure>`);
+        }
+        for (const skip of skips) {
+            lines.push(`    <skipped message="${xmlEscape(skip.explanation ?? `${skip.name} skipped`)}" />`);
+        }
+        lines.push("  </testcase>");
+    }
+    lines.push("</testsuite>");
+    return `${lines.join("\n")}\n`;
+}
+async function writeEvalArtifacts(report, paths, artifacts, baseline) {
+    await artifacts.writeTextFileEnsuringDir(paths.summary, JSON.stringify(createSummaryArtifact(report, baseline), null, 2));
+    await artifacts.writeTextFileEnsuringDir(paths.results, createResultsJsonl(report));
+    await artifacts.writeTextFileEnsuringDir(paths.reportMarkdown, createEvalMarkdownReport(report, baseline));
+}
+function createEvalSuiteResultsJsonl(summary) {
+    if (summary.results.length === 0)
+        return "";
+    return `${summary.results.map((result) => JSON.stringify(result)).join("\n")}\n`;
+}
+function createEvalSuiteMarkdown(summary) {
+    const rows = summary.results.map((result) => `| ${markdownCell(result.id)} | ${result.status} | ${result.summary ? `${result.summary.passed}/${result.summary.records}` : "n/a"} | ${result.error ? markdownCell(result.error) : ""} |`);
+    return [
+        "# Eval suite report",
+        "",
+        `Run: \`${markdownCell(summary.runId)}\``,
+        `Result: \`${summary.passed}/${summary.total} passed\``,
+        "",
+        "| Eval | Status | Records | Error |",
+        "| --- | --- | --- | --- |",
+        ...rows,
+        "",
+    ].join("\n");
+}
+function createEvalSuiteJunitXml(summary) {
+    const lines = [
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        `<testsuites tests="${summary.total}" failures="${summary.failed}" skipped="0">`,
+        `  <testsuite name="veryfront eval suite" tests="${summary.total}" failures="${summary.failed}" skipped="0">`,
+    ];
+    for (const result of summary.results) {
+        const attrs = `classname="eval" name="${xmlEscape(result.id)}"`;
+        if (result.status === "passed") {
+            lines.push(`    <testcase ${attrs} />`);
+            continue;
+        }
+        const message = result.error ??
+            (result.summary?.failed
+                ? `${result.summary.failed} record(s) failed`
+                : "A required eval export failed.");
+        lines.push(`    <testcase ${attrs}>`);
+        lines.push(`      <failure message="${xmlEscape(message)}">${xmlEscape(message)}</failure>`);
+        lines.push("    </testcase>");
+    }
+    lines.push("  </testsuite>");
+    lines.push("</testsuites>");
+    return `${lines.join("\n")}\n`;
+}
+async function writeEvalSuiteArtifacts(summary, paths, artifacts) {
+    await artifacts.writeTextFileEnsuringDir(paths.summary, JSON.stringify(summary, null, 2));
+    await artifacts.writeTextFileEnsuringDir(paths.results, createEvalSuiteResultsJsonl(summary));
+    await artifacts.writeTextFileEnsuringDir(paths.reportMarkdown, createEvalSuiteMarkdown(summary));
+}
+function createEvalExitCode(report, baseline, exportIsRequired = false) {
+    const exportFailed = exportIsRequired &&
+        (!(report.exports?.length) || report.exports.some((result) => !result.ok));
+    return report.summary.failed === 0 && baseline?.regressed !== true && !exportFailed ? 0 : 1;
+}
+function createOutputHints(paths, input) {
+    return {
+        reportDirectory: paths.directory,
+        reportMarkdown: paths.reportMarkdown,
+        ...(input.report ? { report: input.report } : {}),
+        ...(input.junit ? { junit: input.junit } : {}),
+        ...(input.writeBaseline ? { baselineWritten: input.writeBaseline } : {}),
+    };
+}
+function createSuiteOutputHints(paths, input, children) {
+    return {
+        reportDirectory: paths.directory,
+        reportMarkdown: paths.reportMarkdown,
+        ...(input.junit ? { junit: input.junit } : {}),
+        children,
+    };
+}
+function createRunId(clock) {
+    return createEvalRunId(clock?.now?.() ?? new Date(), clock?.createSuffix);
+}
+function stripFileProtocol(path) {
+    if (!path.startsWith("file://"))
+        return path;
+    return decodeURIComponent(new URL(path).pathname);
+}
+function displaySourcePath(filePath, projectDir) {
+    const normalized = stripFileProtocol(filePath);
+    if (normalized.startsWith(projectDir)) {
+        return relative(projectDir, normalized);
+    }
+    return normalized;
+}
+function createMetadata(input, model) {
+    return {
+        ...(input.provenance ? { provenance: input.provenance } : {}),
+        ...(input.kind === "single" && input.selectedModel ? { model: input.selectedModel } : {}),
+        ...(model ? { model } : {}),
+    };
+}
+async function readBaselineComparison(report, input, artifacts) {
+    if (!input.baseline)
+        return undefined;
+    const baseline = JSON.parse(await artifacts.readTextFile(input.baseline));
+    return compareEvalReports(report, baseline, input.baselinePolicy);
+}
+function createEvalReportExportConfig(input, artifactPaths) {
+    if (!input.exportConfig)
+        return undefined;
+    const tags = input.evalItem.definition.tags ?? [];
+    const metadata = input.evalItem.definition.metadata ?? {};
+    const reportPath = input.report ?? artifactPaths.summary;
+    return {
+        ...input.exportConfig,
+        context: {
+            evalId: input.evalItem.id,
+            sourcePath: displaySourcePath(input.evalItem.filePath, input.projectDir),
+            reportPath,
+            ...(tags.length > 0 ? { tags } : {}),
+            ...(Object.keys(metadata).length > 0 ? { metadata } : {}),
+            ...input.exportConfig.context,
+        },
+    };
+}
+function createModelComparisonOutputHints(paths, input, reports, comparison) {
+    return {
+        reportDirectory: paths.directory,
+        reportMarkdown: paths.comparisonMarkdown,
+        ...(input.report ? { report: input.report } : {}),
+        models: reports.map((report) => ({
+            model: report.metadata?.model ?? report.runId,
+            report,
+        })),
+        comparisonJson: paths.comparisonJson,
+        comparisonMarkdown: paths.comparisonMarkdown,
+        recommendation: comparison.recommendation,
+    };
+}
+function createEvalModelComparisonExitCode(reports, exportIsRequired = false) {
+    return reports.some((report) => createEvalExitCode(report, undefined, exportIsRequired) !== 0)
+        ? 1
+        : 0;
+}
+async function writeEvalModelComparisonArtifacts(comparison, paths, artifacts) {
+    await artifacts.writeTextFileEnsuringDir(paths.comparisonJson, JSON.stringify(comparison, null, 2));
+    await artifacts.writeTextFileEnsuringDir(paths.comparisonMarkdown, createEvalModelComparisonMarkdown(comparison));
+}
+function createEvalSuiteSummary(runId, startedAt, endedAt, results) {
+    const passed = results.filter((result) => result.status === "passed").length;
+    return {
+        kind: "eval-suite-summary",
+        runId,
+        startedAt: startedAt.toISOString(),
+        endedAt: endedAt.toISOString(),
+        total: results.length,
+        passed,
+        failed: results.length - passed,
+        results,
+    };
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+async function runEvalReportSuite(input, adapters) {
+    const startedAt = adapters.clock?.now?.() ?? new Date();
+    const runId = createEvalRunId(startedAt, adapters.clock?.createSuffix);
+    const artifacts = createEvalSuiteArtifactPaths(input.reportDir ?? createDefaultEvalReportDir(runId));
+    const metadata = createMetadata(input);
+    const results = [];
+    const children = [];
+    for (const [index, evalItem] of sortEvals(input.evalItems).entries()) {
+        const childRunId = `${runId}_${String(index + 1).padStart(3, "0")}`;
+        const childArtifacts = createEvalArtifactPaths(createEvalSuiteChildDirectory(artifacts.directory, index, evalItem.id));
+        try {
+            if (!adapters.targets.resolveTarget) {
+                throw new Error("Suite eval target resolution is not configured.");
+            }
+            const target = await adapters.targets.resolveTarget(evalItem);
+            const finalizedReport = await adapters.billing.runWithGatewayBillingGroup(childRunId, () => adapters.targets.runEval(evalItem, {
+                baseDir: input.datasetBase ?? input.projectDir,
+                runId: childRunId,
+                frameworkVersion: input.frameworkVersion,
+                targetKind: target.targetKind,
+                target: target.target,
+                targetAdapter: target.targetAdapter,
+                metadata,
+            }));
+            const exportConfig = createEvalReportExportConfig({
+                projectDir: input.projectDir,
+                evalItem,
+                exportConfig: input.exportConfig,
+            }, childArtifacts);
+            const report = await adapters.exporters.exportReport(finalizedReport, exportConfig);
+            await writeEvalArtifacts(report, childArtifacts, adapters.artifacts);
+            const summary = summarizeReportForCli(report);
+            const status = createEvalExitCode(report, undefined, exportConfig?.required) === 0
+                ? "passed"
+                : "failed";
+            results.push({
+                id: evalItem.id,
+                name: evalItem.name,
+                target: evalItem.definition.target,
+                status,
+                artifacts: childArtifacts,
+                summary,
+            });
+            children.push({
+                kind: "report",
+                evalId: evalItem.id,
+                reportDirectory: childArtifacts.directory,
+                report,
+            });
+        }
+        catch (error) {
+            const message = errorMessage(error);
+            results.push({
+                id: evalItem.id,
+                name: evalItem.name,
+                target: evalItem.definition.target,
+                status: "error",
+                artifacts: childArtifacts,
+                error: message,
+            });
+            children.push({
+                kind: "error",
+                evalId: evalItem.id,
+                error: message,
+            });
+        }
+    }
+    const endedAt = adapters.clock?.now?.() ?? new Date();
+    const suite = createEvalSuiteSummary(runId, startedAt, endedAt, results);
+    await writeEvalSuiteArtifacts(suite, artifacts, adapters.artifacts);
+    if (input.junit) {
+        await adapters.artifacts.writeTextFileEnsuringDir(input.junit, createEvalSuiteJunitXml(suite));
+    }
+    return {
+        kind: "suite",
+        suite,
+        artifacts,
+        exitCode: suite.failed === 0 ? 0 : 1,
+        outputHints: createSuiteOutputHints(artifacts, input, children),
+    };
+}
+async function runEvalReportModelComparison(input, adapters) {
+    const runId = createRunId(adapters.clock);
+    const reportDir = input.reportDir ?? createDefaultEvalReportDir(runId, input.evalItem.id);
+    const baselineModel = input.baselineModel.trim();
+    const models = uniqueTrimmedValues([baselineModel, ...input.candidateModels]);
+    const paths = createEvalModelComparisonArtifactPaths(reportDir, models);
+    const reports = [];
+    for (const model of models) {
+        if (!adapters.targets.createModelTargetAdapter) {
+            throw new Error("Model comparison target adapter creation is not configured.");
+        }
+        const modelPaths = paths.models[model];
+        const modelRunId = `${runId}_${sanitizeModelIdForPath(model)}`;
+        const targetAdapter = await adapters.targets.createModelTargetAdapter(model);
+        const metadata = createMetadata(input, model);
+        const finalizedReport = await adapters.billing.runWithGatewayBillingGroup(modelRunId, () => adapters.targets.runEval(input.evalItem, {
+            baseDir: input.datasetBase ?? input.projectDir,
+            runId: modelRunId,
+            frameworkVersion: input.frameworkVersion,
+            targetKind: "agent",
+            target: input.target,
+            targetAdapter,
+            selectedModel: model,
+            ...(input.maxOutputTokens !== undefined
+                ? { maxOutputTokens: input.maxOutputTokens }
+                : {}),
+            metadata,
+        }));
+        const exportConfig = createEvalReportExportConfig({
+            projectDir: input.projectDir,
+            evalItem: input.evalItem,
+            exportConfig: input.exportConfig,
+        }, modelPaths);
+        const report = await adapters.exporters.exportReport(finalizedReport, exportConfig);
+        reports.push(report);
+        await writeEvalArtifacts(report, modelPaths, adapters.artifacts);
+        await adapters.artifacts.writeTextFileEnsuringDir(modelPaths.junit, createJunitXml(report));
+    }
+    const comparison = compareEvalModelReports(reports, {
+        ...input.comparisonPolicy,
+        baselineModel,
+    });
+    await writeEvalModelComparisonArtifacts(comparison, paths, adapters.artifacts);
+    if (input.report) {
+        await adapters.artifacts.writeTextFileEnsuringDir(input.report, JSON.stringify(comparison, null, 2));
+    }
+    return {
+        kind: "model-comparison",
+        reports,
+        comparison,
+        artifacts: paths,
+        exitCode: createEvalModelComparisonExitCode(reports, input.exportConfig?.required),
+        outputHints: createModelComparisonOutputHints(paths, input, reports, comparison),
+    };
+}
+export async function runEvalReport(input, adapters) {
+    switch (input.kind) {
+        case "suite":
+            return await runEvalReportSuite(input, adapters);
+        case "model-comparison":
+            return await runEvalReportModelComparison(input, adapters);
+        case "single":
+            break;
+        default:
+            throw new Error(`Unsupported eval report kind: ${input.kind}`);
+    }
+    const runId = createRunId(adapters.clock);
+    const artifactPaths = createEvalArtifactPaths(input.reportDir ?? createDefaultEvalReportDir(runId, input.evalItem.id));
+    const billingGroupId = input.selectedModel
+        ? `${runId}_${sanitizeModelIdForPath(input.selectedModel)}`
+        : runId;
+    const metadata = createMetadata(input);
+    const finalizedReport = await adapters.billing.runWithGatewayBillingGroup(billingGroupId, () => adapters.targets.runEval(input.evalItem, {
+        baseDir: input.datasetBase ?? input.projectDir,
+        runId,
+        frameworkVersion: input.frameworkVersion,
+        targetKind: input.targetKind,
+        target: input.target,
+        targetAdapter: input.targetAdapter,
+        ...(input.selectedModel ? { selectedModel: input.selectedModel } : {}),
+        ...(input.maxOutputTokens !== undefined ? { maxOutputTokens: input.maxOutputTokens } : {}),
+        metadata,
+    }));
+    const exportConfig = createEvalReportExportConfig(input, artifactPaths);
+    const report = await adapters.exporters.exportReport(finalizedReport, exportConfig);
+    const baseline = await readBaselineComparison(report, input, adapters.artifacts);
+    await writeEvalArtifacts(report, artifactPaths, adapters.artifacts, baseline);
+    if (input.report) {
+        await adapters.artifacts.writeTextFileEnsuringDir(input.report, JSON.stringify(report, null, 2));
+    }
+    if (input.junit) {
+        await adapters.artifacts.writeTextFileEnsuringDir(input.junit, createJunitXml(report));
+    }
+    if (input.writeBaseline) {
+        await adapters.artifacts.writeTextFileEnsuringDir(input.writeBaseline, JSON.stringify(report, null, 2));
+    }
+    return {
+        kind: "single",
+        report,
+        summary: summarizeReportForCli(report),
+        ...(baseline ? { baseline } : {}),
+        artifacts: artifactPaths,
+        exitCode: createEvalExitCode(report, baseline, exportConfig?.required),
+        outputHints: createOutputHints(artifactPaths, input),
+    };
+}

package/esm/src/html/hydration-script-builder/dev-client-renderer.js

@@ -6,7 +6,7 @@ export function generateDevClientRendererScript(nonce) {
   <script type="module"${nonceAttr}>
     import * as React from 'react';
     import { createRoot } from 'react-dom/client';
-    import { RouterProvider, useRouter as useRouterFromModule } from 'veryfront/router';
+    import { RouterProvider, useRouter as useRouterFromModule, getNavigationStore } from 'veryfront/router';
     import { PageContextProvider } from 'veryfront/context';
 
     ${getRouterScript()}

package/esm/src/html/hydration-script-builder/prod-scripts.js

@@ -8,7 +8,7 @@ export function generateProdHydrationModule() {
     return [
         `import * as React from 'react';`,
         `import { createRoot } from 'react-dom/client';`,
-        `import { RouterProvider, useRouter as useRouterFromModule } from 'veryfront/router';`,
+        `import { RouterProvider, useRouter as useRouterFromModule, getNavigationStore } from 'veryfront/router';`,
         `import { PageContextProvider } from 'veryfront/context';`,
         getRouterScript().trim(),
         getLoaderScript().trim(),

package/esm/src/html/hydration-script-builder/templates/router.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../../../../../src/src/html/hydration-script-builder/templates/router.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,eAAe,cAstC3B,CAAC"}
+{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../../../../../src/src/html/hydration-script-builder/templates/router.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,eAAe,cA8uC3B,CAAC"}