veryfront 0.1.955 → 0.1.957

package/esm/cli/commands/eval/command-help.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"command-help.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command-help.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,eAAO,MAAM,QAAQ,EAAE,WAsEtB,CAAC"}
+{"version":3,"file":"command-help.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command-help.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,eAAO,MAAM,QAAQ,EAAE,WA2EtB,CAAC"}

package/esm/cli/commands/eval/command-help.js

@@ -48,6 +48,10 @@ export const evalHelp = {
             flag: "--candidate-model <provider/model>",
             description: "Candidate model to compare against --baseline-model; repeat for multiple candidates",
         },
+        {
+            flag: "--comparison-policy <path>",
+            description: "Read model comparison constraints and weighted objectives from JSON",
+        },
         {
             flag: "--max-output-tokens <count>",
             description: "Limit target agent output tokens",
@@ -64,6 +68,7 @@ export const evalHelp = {
         "veryfront eval eval:deep-research --report .veryfront/evals/deep-research/report.json --junit .veryfront/evals/deep-research/junit.xml",
         "veryfront eval deep-research --baseline .veryfront/evals/baseline.json --json",
         "veryfront eval deep-research --baseline-model anthropic/claude-sonnet-4-6 --candidate-model moonshotai/kimi-k2.6",
+        "veryfront eval deep-research --baseline-model anthropic/claude-sonnet-4-6 --candidate-model moonshotai/kimi-k2.6 --comparison-policy evals/model-comparison.policy.json",
         "veryfront eval deep-research --export braintrust,langfuse --json",
         "veryfront eval deep-research --json",
     ],

package/esm/cli/commands/eval/command.d.ts

@@ -1,6 +1,6 @@
 import type { AgentResponse } from "../../../src/agent/index.js";
 import type { VeryfrontConfig } from "../../../src/config/index.js";
-import type { DiscoveredEval, EvalModelComparison, EvalReport, EvalReportComparison, EvalToolCall } from "../../../src/eval/index.js";
+import type { DiscoveredEval, EvalModelComparison, EvalModelComparisonOptions, EvalReport, EvalReportComparison, EvalToolCall } from "../../../src/eval/index.js";
 import type { EvalArgs } from "./handler.js";
 export interface EvalOptions extends EvalArgs {
 }
@@ -35,6 +35,7 @@ type EvalSummaryArtifact = {
     exports?: EvalReport["exports"];
     baseline?: EvalReportComparison;
 };
+type EvalModelComparisonPolicy = Omit<EvalModelComparisonOptions, "baselineModel">;
 export declare function createDefaultEvalReportDir(runId: string): string;
 export declare function createEvalArtifactPaths(reportDir: string): EvalArtifactPaths;
 export declare function createEvalModelArtifactPaths(reportDir: string, model: string): EvalModelArtifactPaths;
@@ -43,7 +44,9 @@ export declare function findEvalForCliId(evals: DiscoveredEval[], id: string): D
 export declare function normalizeEvalInputForAgent(input: unknown): string;
 export declare function summarizeReportForCli(report: EvalReport): CliEvalSummary;
 export declare function createSummaryArtifact(report: EvalReport, baseline?: EvalReportComparison): EvalSummaryArtifact;
-export declare function createEvalModelComparisonArtifact(reports: EvalReport[], baselineModel: string): EvalModelComparison;
+export declare function loadEvalModelComparisonPolicy(projectDir: string, policyPath?: string): Promise<EvalModelComparisonPolicy>;
+export declare function createResolvedEvalModelComparisonConfig(projectDir: string, options: EvalOptions): Promise<ResolvedEvalModelComparisonConfig | undefined>;
+export declare function createEvalModelComparisonArtifact(reports: EvalReport[], baselineModel: string, policy?: EvalModelComparisonPolicy): EvalModelComparison;
 export declare function createEvalModelComparisonExitCode(reports: EvalReport[]): 0 | 1;
 export declare function createResultsJsonl(report: EvalReport): string;
 export declare function createJunitXml(report: EvalReport): string;
@@ -54,6 +57,16 @@ type EvalRuntimeAuthConfig = Pick<VeryfrontConfig, "projectSlug" | "fs"> & {
 };
 export declare function hydrateEvalRuntimeAuth(projectDir: string, config: EvalRuntimeAuthConfig | null | undefined): Promise<import("../../shared/runtime-auth.js").RuntimeAuthContext>;
 export declare function normalizeToolCalls(response: AgentResponse): EvalToolCall[];
+type EvalModelComparisonConfig = {
+    baselineModel: string;
+    candidateModels: string[];
+    comparisonPolicy?: string;
+    models: string[];
+};
+type ResolvedEvalModelComparisonConfig = {
+    config: EvalModelComparisonConfig;
+    policy: EvalModelComparisonPolicy;
+};
 export declare function evalCommand(options: EvalOptions): Promise<void>;
 export {};
 //# sourceMappingURL=command.d.ts.map

package/esm/cli/commands/eval/command.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,UAAU,EACV,oBAAoB,EAEpB,YAAY,EACb,MAAM,4BAA4B,CAAC;AAmBpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAoBF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEhE;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAM5E;AAMD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CAQxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAcrB;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,GACpB,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAOf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAYD,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AA0QD,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CA4IrE"}
+{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EAEpB,YAAY,EACb,MAAM,4BAA4B,CAAC;AAmBpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AAiCnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEhE;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAM5E;AAMD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CAQxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAcrB;AAkGD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAOf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAYD,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AA+FD,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AAkLF,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CA6IrE"}

package/esm/cli/commands/eval/command.js

@@ -2,13 +2,24 @@
  * Eval command - Discover and run eval definitions from the evals/ directory.
  */
 import * as dntShim from "../../../_dnt.shims.js";
-import { dirname, join, relative } from "../../../deps/jsr.io/@std/path/1.1.4/mod.js";
+import { dirname, isAbsolute, join, relative, resolve } from "../../../deps/jsr.io/@std/path/1.1.4/mod.js";
 import { createProjectDiscoveryConfig, discoverAll } from "../../../src/discovery/index.js";
 import { compareEvalModelReports, compareEvalReports, createEvalModelComparisonMarkdown, discoverEvals, resolveEvalRunProvenance, runEval, } from "../../../src/eval/index.js";
 import { applyRuntimeAuthContext } from "../../shared/runtime-auth.js";
 import { cliLogger, exitProcess, VERSION } from "../../utils/index.js";
 import { createErrorEnvelope, createSuccessEnvelope, isJsonMode, outputJson, } from "../../shared/json-output.js";
 import { withProjectSourceContext } from "../../shared/project-source-context.js";
+const MODEL_COMPARISON_METRICS = [
+    "passRate",
+    "failed",
+    "gateFailures",
+    "groundednessScore",
+    "totalTokens",
+    "costUsd",
+    "p95Ms",
+];
+const MODEL_COMPARISON_METRIC_SET = new Set(MODEL_COMPARISON_METRICS);
+const MODEL_COMPARISON_OBJECTIVE_DIRECTIONS = new Set(["minimize", "maximize"]);
 function xmlEscape(value) {
     return value
         .replaceAll("&", "&")
@@ -135,8 +146,135 @@ export function createSummaryArtifact(report, baseline) {
         ...(baseline ? { baseline } : {}),
     };
 }
-export function createEvalModelComparisonArtifact(reports, baselineModel) {
-    return compareEvalModelReports(reports, { baselineModel });
+function isRecord(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function readNumberField(record, field, path) {
+    const value = record[field];
+    if (value === undefined)
+        return undefined;
+    if (typeof value !== "number" || !Number.isFinite(value)) {
+        throw new Error(`Invalid --comparison-policy: ${path}.${field} must be a finite number.`);
+    }
+    return value;
+}
+function assertKnownComparisonMetric(metric, path) {
+    if (MODEL_COMPARISON_METRIC_SET.has(metric))
+        return metric;
+    throw new Error(`Invalid --comparison-policy: ${path} uses unknown metric "${metric}".`);
+}
+function parseComparisonConstraints(value) {
+    if (value === undefined)
+        return undefined;
+    if (!isRecord(value)) {
+        throw new Error("Invalid --comparison-policy: constraints must be an object.");
+    }
+    const constraints = {};
+    for (const [metricName, rawConstraint] of Object.entries(value)) {
+        const metric = assertKnownComparisonMetric(metricName, `constraints.${metricName}`);
+        if (!isRecord(rawConstraint)) {
+            throw new Error(`Invalid --comparison-policy: constraints.${metricName} must be an object.`);
+        }
+        const maxRegressionPct = readNumberField(rawConstraint, "maxRegressionPct", `constraints.${metricName}`);
+        if (maxRegressionPct !== undefined && maxRegressionPct < 0) {
+            throw new Error(`Invalid --comparison-policy: constraints.${metricName}.maxRegressionPct must be at least 0.`);
+        }
+        constraints[metric] = {
+            ...(rawConstraint.min !== undefined
+                ? { min: readNumberField(rawConstraint, "min", `constraints.${metricName}`) }
+                : {}),
+            ...(rawConstraint.max !== undefined
+                ? { max: readNumberField(rawConstraint, "max", `constraints.${metricName}`) }
+                : {}),
+            ...(maxRegressionPct !== undefined ? { maxRegressionPct } : {}),
+        };
+    }
+    return constraints;
+}
+function parseComparisonObjectives(value) {
+    if (value === undefined)
+        return undefined;
+    if (!isRecord(value)) {
+        throw new Error("Invalid --comparison-policy: objectives must be an object.");
+    }
+    const objectives = {};
+    for (const [metricName, rawObjective] of Object.entries(value)) {
+        const metric = assertKnownComparisonMetric(metricName, `objectives.${metricName}`);
+        if (!isRecord(rawObjective)) {
+            throw new Error(`Invalid --comparison-policy: objectives.${metricName} must be an object.`);
+        }
+        const weight = readNumberField(rawObjective, "weight", `objectives.${metricName}`);
+        if (weight === undefined) {
+            throw new Error(`Invalid --comparison-policy: objectives.${metricName}.weight is required.`);
+        }
+        if (weight <= 0) {
+            throw new Error(`Invalid --comparison-policy: objectives.${metricName}.weight must be greater than 0.`);
+        }
+        const direction = rawObjective.direction;
+        if (typeof direction !== "string" || !MODEL_COMPARISON_OBJECTIVE_DIRECTIONS.has(direction)) {
+            throw new Error(`Invalid --comparison-policy: objectives.${metricName}.direction must be "minimize" or "maximize".`);
+        }
+        objectives[metric] = {
+            weight,
+            direction: direction,
+        };
+    }
+    return objectives;
+}
+export async function loadEvalModelComparisonPolicy(projectDir, policyPath) {
+    if (!policyPath)
+        return {};
+    const resolvedPath = isAbsolute(policyPath) ? policyPath : resolve(projectDir, policyPath);
+    let rawPolicy;
+    try {
+        rawPolicy = await dntShim.Deno.readTextFile(resolvedPath);
+    }
+    catch (error) {
+        if (error instanceof dntShim.Deno.errors.NotFound) {
+            throw new Error("Invalid --comparison-policy: file not found.");
+        }
+        throw error;
+    }
+    let policy;
+    try {
+        policy = JSON.parse(rawPolicy);
+    }
+    catch {
+        throw new Error("Invalid --comparison-policy: file must contain valid JSON.");
+    }
+    if (!isRecord(policy)) {
+        throw new Error("Invalid --comparison-policy: root value must be an object.");
+    }
+    return {
+        ...("minGroundedness" in policy
+            ? { minGroundedness: readNumberField(policy, "minGroundedness", "root") }
+            : {}),
+        ...("minCostImprovementPct" in policy
+            ? { minCostImprovementPct: readNumberField(policy, "minCostImprovementPct", "root") }
+            : {}),
+        ...("minTokenImprovementPct" in policy
+            ? { minTokenImprovementPct: readNumberField(policy, "minTokenImprovementPct", "root") }
+            : {}),
+        ...("minLatencyImprovementPct" in policy
+            ? { minLatencyImprovementPct: readNumberField(policy, "minLatencyImprovementPct", "root") }
+            : {}),
+        ...(policy.constraints !== undefined
+            ? { constraints: parseComparisonConstraints(policy.constraints) }
+            : {}),
+        ...(policy.objectives !== undefined
+            ? { objectives: parseComparisonObjectives(policy.objectives) }
+            : {}),
+    };
+}
+export async function createResolvedEvalModelComparisonConfig(projectDir, options) {
+    const config = resolveEvalModelComparisonConfig(options);
+    if (!config)
+        return undefined;
+    const policy = await loadEvalModelComparisonPolicy(projectDir, config.comparisonPolicy);
+    return { config, policy };
+}
+export function createEvalModelComparisonArtifact(reports, baselineModel, policy = {}) {
+    return compareEvalModelReports(reports, { ...policy, baselineModel });
 }
 export function createEvalModelComparisonExitCode(reports) {
     return reports.some((report) => report.summary.failed > 0) ? 1 : 0;
@@ -299,8 +437,12 @@ function uniqueValues(values) {
 function resolveEvalModelComparisonConfig(options) {
     const candidateModels = uniqueValues(options.candidateModels);
     const baselineModel = options.baselineModel?.trim();
-    if (!baselineModel && candidateModels.length === 0)
+    if (!baselineModel && candidateModels.length === 0) {
+        if (options.comparisonPolicy) {
+            throw new Error("--comparison-policy requires --baseline-model and --candidate-model.");
+        }
         return undefined;
+    }
     if (!baselineModel) {
         throw new Error("--baseline-model is required when using --candidate-model.");
     }
@@ -316,6 +458,7 @@ function resolveEvalModelComparisonConfig(options) {
     return {
         baselineModel,
         candidateModels: candidateModels.filter((model) => model !== baselineModel),
+        ...(options.comparisonPolicy ? { comparisonPolicy: options.comparisonPolicy } : {}),
         models: uniqueValues([baselineModel, ...candidateModels]),
     };
 }
@@ -355,7 +498,7 @@ async function runEvalModelComparison(input) {
         await writeEvalArtifacts(report, modelPaths);
         await writeTextFileEnsuringDir(modelPaths.junit, createJunitXml(report));
     }
-    const comparison = createEvalModelComparisonArtifact(reports, input.config.baselineModel);
+    const comparison = createEvalModelComparisonArtifact(reports, input.config.baselineModel, input.policy);
     await writeEvalModelComparisonArtifacts(comparison, paths);
     if (input.options.report) {
         await writeTextFileEnsuringDir(input.options.report, JSON.stringify(comparison, null, 2));
@@ -479,6 +622,14 @@ export async function evalCommand(options) {
             await outputEvalNotFound(evalId, evalDiscovery.evals);
             return;
         }
+        let modelComparisonConfig;
+        try {
+            modelComparisonConfig = await createResolvedEvalModelComparisonConfig(projectDir, options);
+        }
+        catch (error) {
+            await outputEvalUsageError(error instanceof Error ? error.message : String(error));
+            return;
+        }
         const discoveryConfig = createProjectDiscoveryConfig({
             projectDir,
             config,
@@ -492,21 +643,14 @@ export async function evalCommand(options) {
             await outputAgentNotFound(agentId);
             return;
         }
-        let modelComparisonConfig;
-        try {
-            modelComparisonConfig = resolveEvalModelComparisonConfig(options);
-        }
-        catch (error) {
-            await outputEvalUsageError(error instanceof Error ? error.message : String(error));
-            return;
-        }
         if (modelComparisonConfig) {
             await runEvalModelComparison({
                 evalItem,
                 agent,
                 options,
                 projectDir,
-                config: modelComparisonConfig,
+                config: modelComparisonConfig.config,
+                policy: modelComparisonConfig.policy,
             });
             return;
         }

package/esm/cli/commands/eval/handler.d.ts

@@ -14,6 +14,7 @@ declare const getEvalArgsSchema: () => import("../../../src/internal-agents/sche
     model: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
     baselineModel: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
     candidateModels: import("../../../src/internal-agents/schema.js").Schema<string[]>;
+    comparisonPolicy: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
     maxOutputTokens: import("../../../src/internal-agents/schema.js").Schema<number | undefined>;
 }>>;
 export type EvalArgs = InferSchema<ReturnType<typeof getEvalArgsSchema>>;
@@ -31,6 +32,7 @@ export declare const parseEvalArgs: (args: ParsedArgs) => import("../../shared/a
     model: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
     baselineModel: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
     candidateModels: import("../../../src/internal-agents/schema.js").Schema<string[]>;
+    comparisonPolicy: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
     maxOutputTokens: import("../../../src/internal-agents/schema.js").Schema<number | undefined>;
 }>>;
 export declare function handleEvalCommand(args: ParsedArgs): Promise<void>;

package/esm/cli/commands/eval/handler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/handler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yCAAyC,CAAC;AAE3E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAExD,QAAA,MAAM,iBAAiB;;;;;;;;;;;;;;;GAiBtB,CAAC;AAIF,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,iBAAiB,CAAC,CAAC,CAAC;AAEzE,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;GAexB,CAAC;AAEH,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAIvE"}
+{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/handler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yCAAyC,CAAC;AAE3E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAExD,QAAA,MAAM,iBAAiB;;;;;;;;;;;;;;;;GAkBtB,CAAC;AAIF,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,iBAAiB,CAAC,CAAC,CAAC;AAEzE,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;GAgBxB,CAAC;AAEH,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAIvE"}

package/esm/cli/commands/eval/handler.js

@@ -14,6 +14,7 @@ const getEvalArgsSchema = defineSchema((v) => v.object({
     model: v.string().optional(),
     baselineModel: v.string().optional(),
     candidateModels: v.array(v.string()).default([]),
+    comparisonPolicy: v.string().optional(),
     maxOutputTokens: v.number().int().positive().optional(),
 }));
 const EvalArgsSchema = lazySchema(getEvalArgsSchema);
@@ -31,6 +32,7 @@ export const parseEvalArgs = createArgParser(EvalArgsSchema, {
     model: { keys: ["model"], type: "string" },
     baselineModel: { keys: ["baseline-model"], type: "string" },
     candidateModels: { keys: ["candidate-model", "candidate-models"], type: "array" },
+    comparisonPolicy: { keys: ["comparison-policy"], type: "string" },
     maxOutputTokens: { keys: ["max-output-tokens"], type: "number" },
 });
 export async function handleEvalCommand(args) {

package/esm/deno.js

@@ -1,6 +1,6 @@
 export default {
     "name": "veryfront",
-    "version": "0.1.955",
+    "version": "0.1.957",
     "license": "Apache-2.0",
     "nodeModulesDir": "auto",
     "minimumDependencyAge": {

package/esm/extensions/ext-llm-openai/src/openai-chat-stream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"openai-chat-stream.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-llm-openai/src/openai-chat-stream.ts"],"names":[],"mappings":"AAsGA,wBAAuB,2BAA2B,CAChD,MAAM,EAAE,cAAc,CAAC,UAAU,CAAC,GACjC,aAAa,CAAC,OAAO,CAAC,CAsJxB"}
+{"version":3,"file":"openai-chat-stream.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-llm-openai/src/openai-chat-stream.ts"],"names":[],"mappings":"AAuIA,wBAAuB,2BAA2B,CAChD,MAAM,EAAE,cAAc,CAAC,UAAU,CAAC,GACjC,aAAa,CAAC,OAAO,CAAC,CAsJxB"}

package/esm/extensions/ext-llm-openai/src/openai-chat-stream.js

@@ -24,12 +24,37 @@ function extractOpenAIUsage(payload) {
     const cachedTokens = promptTokensDetails?.cached_tokens;
     const completionTokensDetails = readRecord(usage.completion_tokens_details);
     const reasoningTokens = completionTokensDetails?.reasoning_tokens;
+    const veryfront = readRecord(usage.veryfront);
+    const costSource = veryfront?.cost_source;
+    const usageCaptureStatus = veryfront?.usage_capture_status;
     return {
         inputTokens: typeof inputTokens === "number" ? inputTokens : undefined,
         outputTokens: typeof outputTokens === "number" ? outputTokens : undefined,
         totalTokens: typeof totalTokens === "number" ? totalTokens : undefined,
         ...(typeof cachedTokens === "number" ? { cacheReadInputTokens: cachedTokens } : {}),
         ...(typeof reasoningTokens === "number" ? { reasoningTokens } : {}),
+        ...(typeof veryfront?.billable_input_tokens === "number"
+            ? { billableInputTokens: veryfront.billable_input_tokens }
+            : {}),
+        ...(typeof veryfront?.billable_output_tokens === "number"
+            ? { billableOutputTokens: veryfront.billable_output_tokens }
+            : {}),
+        ...(typeof veryfront?.cost_usd === "number" ? { costUsd: veryfront.cost_usd } : {}),
+        ...(typeof veryfront?.provider_cost_usd === "number"
+            ? { providerCostUsd: veryfront.provider_cost_usd }
+            : {}),
+        ...(typeof veryfront?.veryfront_charge_usd === "number"
+            ? { veryfrontChargeUsd: veryfront.veryfront_charge_usd }
+            : {}),
+        ...(typeof veryfront?.cost_credits === "number" ? { costCredits: veryfront.cost_credits } : {}),
+        ...(costSource === "gateway" || costSource === "missing" || costSource === "partial"
+            ? { costSource }
+            : {}),
+        ...(usageCaptureStatus === "complete" ||
+            usageCaptureStatus === "missing" ||
+            usageCaptureStatus === "partial"
+            ? { usageCaptureStatus }
+            : {}),
     };
 }
 function extractOpenAIContentText(content) {

package/esm/extensions/ext-llm-openai/src/openai-provider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"openai-provider.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-llm-openai/src/openai-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AAC3F,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACrF,OAAO,EACL,kBAAkB,EAMlB,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EAKpB,+BAA+B,EAC/B,8BAA8B,EAC/B,MAAM,uCAAuC,CAAC;AAc/C,OAAO,EACL,kBAAkB,EAClB,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EACpB,+BAA+B,EAC/B,8BAA8B,GAC/B,CAAC;AAEF,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACjC;AA0RD,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,MAAM,GACd,YAAY,CAsEd;AAED,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,MAAM,GACd,YAAY,CAsEd;AAED,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,MAAM,GACd,gBAAgB,CAuClB;AAED,qBAAa,cAAe,YAAW,WAAW;IAChD,QAAQ,CAAC,EAAE,YAAY;IAEvB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,GAAG,YAAY;IAYrE,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,GAAG,gBAAgB;IAY7E,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,GAAG,YAAY;CAW1E"}
+{"version":3,"file":"openai-provider.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-llm-openai/src/openai-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AAC3F,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACrF,OAAO,EACL,kBAAkB,EAMlB,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EAKpB,+BAA+B,EAC/B,8BAA8B,EAC/B,MAAM,uCAAuC,CAAC;AAc/C,OAAO,EACL,kBAAkB,EAClB,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EACpB,+BAA+B,EAC/B,8BAA8B,GAC/B,CAAC;AAEF,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACjC;AA2TD,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,MAAM,GACd,YAAY,CAsEd;AAED,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,MAAM,GACd,YAAY,CAsEd;AAED,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,MAAM,GACd,gBAAgB,CAuClB;AAED,qBAAa,cAAe,YAAW,WAAW;IAChD,QAAQ,CAAC,EAAE,YAAY;IAEvB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,GAAG,YAAY;IAYrE,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,GAAG,gBAAgB;IAY7E,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,GAAG,YAAY;CAW1E"}

package/esm/extensions/ext-llm-openai/src/openai-provider.js

@@ -68,12 +68,37 @@ function extractOpenAIUsage(payload) {
     const cachedTokens = promptTokensDetails?.cached_tokens;
     const completionTokensDetails = readRecord(usage.completion_tokens_details);
     const reasoningTokens = completionTokensDetails?.reasoning_tokens;
+    const veryfront = readRecord(usage.veryfront);
+    const costSource = veryfront?.cost_source;
+    const usageCaptureStatus = veryfront?.usage_capture_status;
     return {
         inputTokens: typeof inputTokens === "number" ? inputTokens : undefined,
         outputTokens: typeof outputTokens === "number" ? outputTokens : undefined,
         totalTokens: typeof totalTokens === "number" ? totalTokens : undefined,
         ...(typeof cachedTokens === "number" ? { cacheReadInputTokens: cachedTokens } : {}),
         ...(typeof reasoningTokens === "number" ? { reasoningTokens } : {}),
+        ...(typeof veryfront?.billable_input_tokens === "number"
+            ? { billableInputTokens: veryfront.billable_input_tokens }
+            : {}),
+        ...(typeof veryfront?.billable_output_tokens === "number"
+            ? { billableOutputTokens: veryfront.billable_output_tokens }
+            : {}),
+        ...(typeof veryfront?.cost_usd === "number" ? { costUsd: veryfront.cost_usd } : {}),
+        ...(typeof veryfront?.provider_cost_usd === "number"
+            ? { providerCostUsd: veryfront.provider_cost_usd }
+            : {}),
+        ...(typeof veryfront?.veryfront_charge_usd === "number"
+            ? { veryfrontChargeUsd: veryfront.veryfront_charge_usd }
+            : {}),
+        ...(typeof veryfront?.cost_credits === "number" ? { costCredits: veryfront.cost_credits } : {}),
+        ...(costSource === "gateway" || costSource === "missing" || costSource === "partial"
+            ? { costSource }
+            : {}),
+        ...(usageCaptureStatus === "complete" ||
+            usageCaptureStatus === "missing" ||
+            usageCaptureStatus === "partial"
+            ? { usageCaptureStatus }
+            : {}),
     };
 }
 function extractOpenAIContentText(content) {

package/esm/src/agent/ag-ui/browser-chunk-encoder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"browser-chunk-encoder.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/ag-ui/browser-chunk-encoder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,uBAAuB,EACvB,uBAAuB,EACvB,8BAA8B,EAC9B,sBAAsB,EACvB,MAAM,sBAAsB,CAAC;AAK9B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,2DAA2D;AAC3D,MAAM,WAAW,uBAAuB,CAAC,MAAM;IAC7C,KAAK,EAAE,uBAAuB,CAAC;IAC/B,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,uBAAuB,EAAE,CAAC;IACrD,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,KAAK,uBAAuB,EAAE,CAAC;CACzE;AAED,8DAA8D;AAC9D,MAAM,WAAW,oCAAoC,CAAC,MAAM;IAC1D,gBAAgB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,SAAS,sBAAsB,EAAE,CAAC;IACvE,oBAAoB,CAAC,EAAE,CACrB,KAAK,EAAE,MAAM,KACV,OAAO,CAAC,8BAA8B,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAChE,eAAe,CAAC,EAAE,OAAO,CAAC,8BAA8B,CAAC,CAAC;CAC3D;AAkBD,0CAA0C;AAC1C,wBAAgB,6BAA6B,CAAC,MAAM,EAClD,OAAO,EAAE,oCAAoC,CAAC,MAAM,CAAC,GACpD,uBAAuB,CAAC,MAAM,CAAC,CAajC"}
+{"version":3,"file":"browser-chunk-encoder.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/ag-ui/browser-chunk-encoder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,uBAAuB,EACvB,uBAAuB,EACvB,8BAA8B,EAC9B,sBAAsB,EACvB,MAAM,sBAAsB,CAAC;AAK9B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,2DAA2D;AAC3D,MAAM,WAAW,uBAAuB,CAAC,MAAM;IAC7C,KAAK,EAAE,uBAAuB,CAAC;IAC/B,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,uBAAuB,EAAE,CAAC;IACrD,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,KAAK,uBAAuB,EAAE,CAAC;CACzE;AAED,8DAA8D;AAC9D,MAAM,WAAW,oCAAoC,CAAC,MAAM;IAC1D,gBAAgB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,SAAS,sBAAsB,EAAE,CAAC;IACvE,oBAAoB,CAAC,EAAE,CACrB,KAAK,EAAE,MAAM,KACV,OAAO,CAAC,8BAA8B,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAChE,eAAe,CAAC,EAAE,OAAO,CAAC,8BAA8B,CAAC,CAAC;CAC3D;AA8CD,0CAA0C;AAC1C,wBAAgB,6BAA6B,CAAC,MAAM,EAClD,OAAO,EAAE,oCAAoC,CAAC,MAAM,CAAC,GACpD,uBAAuB,CAAC,MAAM,CAAC,CAajC"}

package/esm/src/agent/ag-ui/browser-chunk-encoder.js

@@ -13,6 +13,38 @@ function mergeMetadata(target, metadata) {
         target.outputTokens = metadata.outputTokens;
     if (typeof metadata.totalTokens === "number")
         target.totalTokens = metadata.totalTokens;
+    if (typeof metadata.cachedInputTokens === "number") {
+        target.cachedInputTokens = metadata.cachedInputTokens;
+    }
+    if (typeof metadata.cacheCreationInputTokens === "number") {
+        target.cacheCreationInputTokens = metadata.cacheCreationInputTokens;
+    }
+    if (typeof metadata.cacheReadInputTokens === "number") {
+        target.cacheReadInputTokens = metadata.cacheReadInputTokens;
+    }
+    if (typeof metadata.reasoningTokens === "number") {
+        target.reasoningTokens = metadata.reasoningTokens;
+    }
+    if (typeof metadata.billableInputTokens === "number") {
+        target.billableInputTokens = metadata.billableInputTokens;
+    }
+    if (typeof metadata.billableOutputTokens === "number") {
+        target.billableOutputTokens = metadata.billableOutputTokens;
+    }
+    if (typeof metadata.costUsd === "number")
+        target.costUsd = metadata.costUsd;
+    if (typeof metadata.providerCostUsd === "number") {
+        target.providerCostUsd = metadata.providerCostUsd;
+    }
+    if (typeof metadata.veryfrontChargeUsd === "number") {
+        target.veryfrontChargeUsd = metadata.veryfrontChargeUsd;
+    }
+    if (typeof metadata.costCredits === "number")
+        target.costCredits = metadata.costCredits;
+    if (metadata.costSource)
+        target.costSource = metadata.costSource;
+    if (metadata.usageCaptureStatus)
+        target.usageCaptureStatus = metadata.usageCaptureStatus;
     if (typeof metadata.finishReason === "string")
         target.finishReason = metadata.finishReason;
 }

package/esm/src/agent/ag-ui/browser-encoder.d.ts

@@ -14,7 +14,13 @@ export interface AgUiBrowserRunFinishedMetadata {
     cacheCreationInputTokens?: number;
     cacheReadInputTokens?: number;
     reasoningTokens?: number;
+    billableInputTokens?: number;
+    billableOutputTokens?: number;
     costUsd?: number;
+    providerCostUsd?: number;
+    veryfrontChargeUsd?: number;
+    costCredits?: number;
+    costSource?: "gateway" | "missing" | "partial";
     finishReason?: string;
     usageCaptureStatus?: "complete" | "partial" | "missing";
 }

package/esm/src/agent/ag-ui/browser-encoder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"browser-encoder.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/ag-ui/browser-encoder.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,8CAA8C;AAC9C,MAAM,MAAM,sBAAsB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAEhF,mEAAmE;AACnE,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,kBAAkB,CAAC,EAAE,UAAU,GAAG,SAAS,GAAG,SAAS,CAAC;CACzD;AAED,uCAAuC;AACvC,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,OAAO,CAAC;IAClB,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,gBAAgB,EAAE,OAAO,CAAC;IAC1B,QAAQ,EAAE,8BAA8B,CAAC;CAC1C;AAED,+CAA+C;AAC/C,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED,8CAA8C;AAC9C,wBAAgB,6BAA6B,IAAI,uBAAuB,CAcvE;AAgLD,yDAAyD;AACzD,wBAAgB,gCAAgC,CAC9C,QAAQ,EAAE,8BAA8B,GACvC,aAAa,GAAG,IAAI,CA2DtB;AAoID,wDAAwD;AACxD,wBAAgB,wCAAwC,CACtD,KAAK,EAAE,uBAAuB,EAC9B,KAAK,EAAE,sBAAsB,GAC5B,uBAAuB,EAAE,CA+N3B;AAED,4CAA4C;AAC5C,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,uBAAuB,EAC9B,QAAQ,EAAE,aAAa,GAAG,IAAI,GAC7B,uBAAuB,EAAE,CA8B3B"}
+{"version":3,"file":"browser-encoder.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/ag-ui/browser-encoder.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,8CAA8C;AAC9C,MAAM,MAAM,sBAAsB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAEhF,mEAAmE;AACnE,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,SAAS,GAAG,SAAS,GAAG,SAAS,CAAC;IAC/C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,kBAAkB,CAAC,EAAE,UAAU,GAAG,SAAS,GAAG,SAAS,CAAC;CACzD;AAED,uCAAuC;AACvC,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,OAAO,CAAC;IAClB,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,gBAAgB,EAAE,OAAO,CAAC;IAC1B,QAAQ,EAAE,8BAA8B,CAAC;CAC1C;AAED,+CAA+C;AAC/C,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED,8CAA8C;AAC9C,wBAAgB,6BAA6B,IAAI,uBAAuB,CAcvE;AA4OD,yDAAyD;AACzD,wBAAgB,gCAAgC,CAC9C,QAAQ,EAAE,8BAA8B,GACvC,aAAa,GAAG,IAAI,CA6EtB;AAoID,wDAAwD;AACxD,wBAAgB,wCAAwC,CACtD,KAAK,EAAE,uBAAuB,EAC9B,KAAK,EAAE,sBAAsB,GAC5B,uBAAuB,EAAE,CA+N3B;AAED,4CAA4C;AAC5C,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,uBAAuB,EAC9B,QAAQ,EAAE,aAAa,GAAG,IAAI,GAC7B,uBAAuB,EAAE,CA8B3B"}

package/esm/src/agent/ag-ui/browser-encoder.js

@@ -121,6 +121,7 @@ function applyResponseMetadata(state, response) {
         state.metadata.inputTokens = response.usage.promptTokens;
         state.metadata.outputTokens = response.usage.completionTokens;
         state.metadata.totalTokens = response.usage.totalTokens;
+        const usage = response.usage;
         if (typeof response.usage.cachedInputTokens === "number") {
             state.metadata.cachedInputTokens = response.usage.cachedInputTokens;
         }
@@ -136,6 +137,30 @@ function applyResponseMetadata(state, response) {
         if (typeof response.usage.reasoningTokens === "number") {
             state.metadata.reasoningTokens = response.usage.reasoningTokens;
         }
+        if (typeof usage.billableInputTokens === "number") {
+            state.metadata.billableInputTokens = usage.billableInputTokens;
+        }
+        if (typeof usage.billableOutputTokens === "number") {
+            state.metadata.billableOutputTokens = usage.billableOutputTokens;
+        }
+        if (typeof usage.costUsd === "number") {
+            state.metadata.costUsd = usage.costUsd;
+        }
+        if (typeof usage.providerCostUsd === "number") {
+            state.metadata.providerCostUsd = usage.providerCostUsd;
+        }
+        if (typeof usage.veryfrontChargeUsd === "number") {
+            state.metadata.veryfrontChargeUsd = usage.veryfrontChargeUsd;
+        }
+        if (typeof usage.costCredits === "number") {
+            state.metadata.costCredits = usage.costCredits;
+        }
+        if (usage.costSource) {
+            state.metadata.costSource = usage.costSource;
+        }
+        if (usage.usageCaptureStatus) {
+            state.metadata.usageCaptureStatus = usage.usageCaptureStatus;
+        }
     }
     const metadata = response.metadata && typeof response.metadata === "object"
         ? response.metadata
@@ -148,6 +173,33 @@ function applyResponseMetadata(state, response) {
     if (typeof costUsd === "number" && Number.isFinite(costUsd) && costUsd >= 0) {
         state.metadata.costUsd = costUsd;
     }
+    const providerCostUsd = metadata?.providerCostUsd;
+    if (typeof providerCostUsd === "number" && Number.isFinite(providerCostUsd) && providerCostUsd >= 0) {
+        state.metadata.providerCostUsd = providerCostUsd;
+    }
+    const veryfrontChargeUsd = metadata?.veryfrontChargeUsd;
+    if (typeof veryfrontChargeUsd === "number" && Number.isFinite(veryfrontChargeUsd) &&
+        veryfrontChargeUsd >= 0) {
+        state.metadata.veryfrontChargeUsd = veryfrontChargeUsd;
+    }
+    const costCredits = metadata?.costCredits;
+    if (typeof costCredits === "number" && Number.isFinite(costCredits) && costCredits >= 0) {
+        state.metadata.costCredits = costCredits;
+    }
+    const billableInputTokens = metadata?.billableInputTokens;
+    if (typeof billableInputTokens === "number" && Number.isFinite(billableInputTokens) &&
+        billableInputTokens >= 0) {
+        state.metadata.billableInputTokens = billableInputTokens;
+    }
+    const billableOutputTokens = metadata?.billableOutputTokens;
+    if (typeof billableOutputTokens === "number" && Number.isFinite(billableOutputTokens) &&
+        billableOutputTokens >= 0) {
+        state.metadata.billableOutputTokens = billableOutputTokens;
+    }
+    const costSource = metadata?.costSource;
+    if (costSource === "gateway" || costSource === "missing" || costSource === "partial") {
+        state.metadata.costSource = costSource;
+    }
     const usageCaptureStatus = metadata?.usageCaptureStatus;
     if (usageCaptureStatus === "complete" ||
         usageCaptureStatus === "partial" ||
@@ -173,9 +225,27 @@ export function buildAgUiBrowserFinalizeResponse(metadata) {
     if (typeof metadata.reasoningTokens === "number") {
         responseMetadata.reasoningTokens = metadata.reasoningTokens;
     }
+    if (typeof metadata.billableInputTokens === "number") {
+        responseMetadata.billableInputTokens = metadata.billableInputTokens;
+    }
+    if (typeof metadata.billableOutputTokens === "number") {
+        responseMetadata.billableOutputTokens = metadata.billableOutputTokens;
+    }
     if (typeof metadata.costUsd === "number") {
         responseMetadata.costUsd = metadata.costUsd;
     }
+    if (typeof metadata.providerCostUsd === "number") {
+        responseMetadata.providerCostUsd = metadata.providerCostUsd;
+    }
+    if (typeof metadata.veryfrontChargeUsd === "number") {
+        responseMetadata.veryfrontChargeUsd = metadata.veryfrontChargeUsd;
+    }
+    if (typeof metadata.costCredits === "number") {
+        responseMetadata.costCredits = metadata.costCredits;
+    }
+    if (metadata.costSource) {
+        responseMetadata.costSource = metadata.costSource;
+    }
     if (metadata.usageCaptureStatus) {
         responseMetadata.usageCaptureStatus = metadata.usageCaptureStatus;
     }

package/esm/src/agent/ag-ui/browser-finalize-tracker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"browser-finalize-tracker.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/ag-ui/browser-finalize-tracker.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,uBAAuB,EAC5B,KAAK,8BAA8B,EAEpC,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,8DAA8D;AAC9D,MAAM,WAAW,0BAA0B,CAAC,MAAM;IAChD,YAAY,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACtC,oBAAoB,EAAE,CAAC,MAAM,EAAE,SAAS,uBAAuB,EAAE,KAAK,IAAI,CAAC;IAC3E,gBAAgB,EAAE,MAAM,aAAa,GAAG,IAAI,CAAC;CAC9C;AAED,iEAAiE;AACjE,MAAM,WAAW,uCAAuC,CAAC,MAAM;IAC7D,oBAAoB,EAAE,CACpB,KAAK,EAAE,MAAM,KACV,OAAO,CAAC,8BAA8B,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;CACjE;AAED,6CAA6C;AAC7C,wBAAgB,gCAAgC,CAAC,MAAM,EACrD,OAAO,EAAE,uCAAuC,CAAC,MAAM,CAAC,GACvD,0BAA0B,CAAC,MAAM,CAAC,CA6DpC"}
+{"version":3,"file":"browser-finalize-tracker.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/ag-ui/browser-finalize-tracker.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,uBAAuB,EAC5B,KAAK,8BAA8B,EAEpC,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,8DAA8D;AAC9D,MAAM,WAAW,0BAA0B,CAAC,MAAM;IAChD,YAAY,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACtC,oBAAoB,EAAE,CAAC,MAAM,EAAE,SAAS,uBAAuB,EAAE,KAAK,IAAI,CAAC;IAC3E,gBAAgB,EAAE,MAAM,aAAa,GAAG,IAAI,CAAC;CAC9C;AAED,iEAAiE;AACjE,MAAM,WAAW,uCAAuC,CAAC,MAAM;IAC7D,oBAAoB,EAAE,CACpB,KAAK,EAAE,MAAM,KACV,OAAO,CAAC,8BAA8B,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;CACjE;AAED,6CAA6C;AAC7C,wBAAgB,gCAAgC,CAAC,MAAM,EACrD,OAAO,EAAE,uCAAuC,CAAC,MAAM,CAAC,GACvD,0BAA0B,CAAC,MAAM,CAAC,CA+EpC"}