veryfront 0.1.963 → 0.1.964

package/esm/cli/commands/eval/command-help.js

@@ -18,7 +18,7 @@ export const evalHelp = {
         },
         {
             flag: "--report-dir <path>",
-            description: "Write summary.json and results.jsonl artifacts to a directory",
+            description: "Write summary.json, results.jsonl, and report.md artifacts to a directory",
         },
         {
             flag: "--junit <path>",

package/esm/cli/commands/eval/command.d.ts

@@ -18,6 +18,7 @@ type EvalArtifactPaths = {
     directory: string;
     summary: string;
     results: string;
+    reportMarkdown: string;
 };
 type EvalModelArtifactPaths = EvalArtifactPaths & {
     junit: string;
@@ -49,6 +50,8 @@ export declare function createResolvedEvalModelComparisonConfig(projectDir: stri
 export declare function createEvalModelComparisonArtifact(reports: EvalReport[], baselineModel: string, policy?: EvalModelComparisonPolicy): EvalModelComparison;
 export declare function createEvalModelComparisonExitCode(reports: EvalReport[]): 0 | 1;
 export declare function createResultsJsonl(report: EvalReport): string;
+/** Render a human-reviewable markdown report for a single eval run. */
+export declare function createEvalMarkdownReport(report: EvalReport, baseline?: EvalReportComparison): string;
 export declare function createJunitXml(report: EvalReport): string;
 export declare function writeEvalArtifacts(report: EvalReport, paths: EvalArtifactPaths, baseline?: EvalReportComparison): Promise<void>;
 export declare function createEvalExitCode(report: EvalReport, baseline?: EvalReportComparison): 0 | 1;

package/esm/cli/commands/eval/command.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EAEpB,YAAY,EACb,MAAM,4BAA4B,CAAC;AAmBpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AAiCnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEhE;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAM5E;AAMD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CAQxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAcrB;AAkGD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAOf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,MA2CrD;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AA+FD,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AAkLF,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CA6IrE"}
+{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EAEpB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AAmBpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AAiCnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEhE;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAO5E;AAMD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CASxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAcrB;AAkGD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAuDD,uEAAuE;AACvE,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,MAAM,CAkFR;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,MA2CrD;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AA+FD,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AAkLF,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CA8IrE"}

package/esm/cli/commands/eval/command.js

@@ -44,6 +44,7 @@ export function createEvalArtifactPaths(reportDir) {
         directory: reportDir,
         summary: join(reportDir, "summary.json"),
         results: join(reportDir, "results.jsonl"),
+        reportMarkdown: join(reportDir, "report.md"),
     };
 }
 function sanitizeModelIdForPath(model) {
@@ -55,6 +56,7 @@ export function createEvalModelArtifactPaths(reportDir, model) {
         directory,
         summary: join(directory, "summary.json"),
         results: join(directory, "results.jsonl"),
+        reportMarkdown: join(directory, "report.md"),
         junit: join(directory, "junit.xml"),
     };
 }
@@ -284,6 +286,101 @@ export function createResultsJsonl(report) {
         return "";
     return `${report.records.map((record) => JSON.stringify(record)).join("\n")}\n`;
 }
+function markdownCell(value) {
+    return value.replaceAll("|", "\\|").replaceAll("\n", " ");
+}
+function numberCell(value) {
+    return value === undefined ? "-" : String(Math.round(value));
+}
+function decimalCell(value) {
+    if (value === undefined)
+        return "-";
+    if (Number.isInteger(value))
+        return String(value);
+    return value.toFixed(4).replace(/0+$/, "").replace(/\.$/, "");
+}
+function percentCell(value) {
+    return `${Math.round(value * 100)}%`;
+}
+function durationCell(valueMs) {
+    return valueMs === undefined ? "-" : `${(valueMs / 1000).toFixed(3)}s`;
+}
+function usdCell(value) {
+    if (value === undefined)
+        return "-";
+    const absolute = Math.abs(value);
+    if (absolute >= 0.01)
+        return `$${value.toFixed(2)}`;
+    return `$${value.toFixed(6).replace(/0+$/, "").replace(/\.$/, "")}`;
+}
+function examplePassed(record) {
+    return [...(record.metrics ?? []), ...(record.checks ?? [])].every((result) => result.skipped || result.pass !== false);
+}
+function usageRows(usage) {
+    if (!usage)
+        return [];
+    const rows = [
+        ["Input tokens", numberCell(usage.inputTokens)],
+        ["Output tokens", numberCell(usage.outputTokens)],
+        ["Total tokens", numberCell(usage.totalTokens)],
+        ["Billable input tokens", numberCell(usage.billableInputTokens)],
+        ["Billable output tokens", numberCell(usage.billableOutputTokens)],
+        ["Provider cost USD", usdCell(usage.providerCostUsd ?? usage.costUsd)],
+        ["Veryfront charge USD", usdCell(usage.veryfrontChargeUsd)],
+        ["Veryfront billed USD", usdCell(usage.veryfrontBilledUsd)],
+        ["Cost credits", decimalCell(usage.costCredits)],
+        ["Cost source", usage.costSource ?? "-"],
+        ["Usage capture status", usage.usageCaptureStatus ?? "-"],
+    ];
+    return rows.filter(([, value]) => value !== "-");
+}
+/** Render a human-reviewable markdown report for a single eval run. */
+export function createEvalMarkdownReport(report, baseline) {
+    const lines = [
+        `# Eval report: ${markdownCell(report.definitionId)}`,
+        "",
+        `Run: \`${markdownCell(report.runId)}\``,
+        `Target: \`${markdownCell(report.target)}\``,
+        ...(report.metadata?.model ? [`Model: \`${markdownCell(report.metadata.model)}\``] : []),
+        `Result: \`${report.summary.passed}/${report.summary.records} passed (${percentCell(report.summary.passRate)})\``,
+        "",
+        "## Metrics",
+        "",
+        "| Metric | Severity | Passed | Failed | Pass rate |",
+        "| --- | --- | ---: | ---: | ---: |",
+    ];
+    for (const metric of report.summary.metrics) {
+        lines.push(`| \`${markdownCell(metric.name)}\` | ${metric.severity} | ${metric.passed} | ${metric.failed} | ${percentCell(metric.passRate)} |`);
+    }
+    const rows = usageRows(report.summary.usage);
+    if (rows.length > 0) {
+        lines.push("", "## Usage", "", "| Usage | Value |", "| --- | ---: |");
+        for (const [label, value] of rows) {
+            lines.push(`| ${label} | ${value.startsWith("$") ? `\`${value}\`` : value} |`);
+        }
+    }
+    lines.push("", "## Examples", "", "| Example | Result | Duration | Tokens | Billed USD | Credits |", "| --- | ---: | ---: | ---: | ---: | ---: |");
+    for (const record of report.records) {
+        lines.push(`| \`${markdownCell(record.id)}\` | ${examplePassed(record) ? "PASS" : "FAIL"} | ${durationCell(record.durationMs)} | ${numberCell(record.usage.totalTokens)} | ${record.usage.veryfrontBilledUsd === undefined
+            ? "-"
+            : `\`${usdCell(record.usage.veryfrontBilledUsd)}\``} | ${decimalCell(record.usage.costCredits)} |`);
+    }
+    if (baseline) {
+        const direction = baseline.passRateDelta >= 0 ? "+" : "";
+        lines.push("", "## Baseline", "", `Status: \`${baseline.regressed ? "regressed" : "ok"}\``, `Pass rate delta: \`${direction}${Math.round(baseline.passRateDelta * 100)} pp\``);
+        if (baseline.newFailedExamples.length > 0) {
+            lines.push(`New failed examples: ${baseline.newFailedExamples.map(markdownCell).join(", ")}`);
+        }
+    }
+    if (report.exports?.length) {
+        lines.push("", "## Exports", "");
+        for (const result of report.exports) {
+            lines.push(`- \`${markdownCell(result.exporterId)}\`: ${result.ok ? "ok" : `failed, ${markdownCell(result.error)}`}`);
+        }
+    }
+    lines.push("");
+    return lines.join("\n");
+}
 export function createJunitXml(report) {
     const skipped = report.records.reduce((count, record) => count + skippedResults(record).length, 0);
     const lines = [
@@ -320,6 +417,7 @@ export async function writeEvalArtifacts(report, paths, baseline) {
     await dntShim.Deno.mkdir(paths.directory, { recursive: true });
     await dntShim.Deno.writeTextFile(paths.summary, JSON.stringify(createSummaryArtifact(report, baseline), null, 2));
     await dntShim.Deno.writeTextFile(paths.results, createResultsJsonl(report));
+    await dntShim.Deno.writeTextFile(paths.reportMarkdown, createEvalMarkdownReport(report, baseline));
 }
 async function readEvalReport(path) {
     return JSON.parse(await dntShim.Deno.readTextFile(path));
@@ -731,6 +829,7 @@ export async function evalCommand(options) {
         else {
             printReport(report, baseline);
             cliLogger.info(`Report directory: ${artifactPaths.directory}`);
+            cliLogger.info(`Report markdown: ${artifactPaths.reportMarkdown}`);
             if (options.report)
                 cliLogger.info(`Report: ${options.report}`);
             if (options.junit)

package/esm/deno.js

@@ -1,6 +1,6 @@
 export default {
     "name": "veryfront",
-    "version": "0.1.963",
+    "version": "0.1.964",
     "license": "Apache-2.0",
     "nodeModulesDir": "auto",
     "minimumDependencyAge": {

package/esm/src/utils/version-constant.d.ts

@@ -1,3 +1,3 @@
 /** Shared version value. */
-export declare const VERSION = "0.1.963";
+export declare const VERSION = "0.1.964";
 //# sourceMappingURL=version-constant.d.ts.map

package/esm/src/utils/version-constant.js

@@ -1,4 +1,4 @@
 // Keep in sync with deno.json version.
 // scripts/release.ts updates this constant during releases.
 /** Shared version value. */
-export const VERSION = "0.1.963";
+export const VERSION = "0.1.964";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "veryfront",
-  "version": "0.1.963",
+  "version": "0.1.964",
   "description": "The simplest way to build AI-powered apps",
   "keywords": [
     "react",