veryfront 0.1.979 → 0.1.981
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/cli/commands/eval/command.d.ts +5 -0
- package/esm/cli/commands/eval/command.d.ts.map +1 -1
- package/esm/cli/commands/eval/command.js +72 -26
- package/esm/deno.js +1 -1
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +1 -1
|
@@ -45,6 +45,10 @@ type GatewayBillingGroupFinalization = {
|
|
|
45
45
|
veryfront_charge_usd: number;
|
|
46
46
|
veryfront_billed_usd: number;
|
|
47
47
|
};
|
|
48
|
+
type GatewayBillingFinalizeOptions = {
|
|
49
|
+
retryDelaysMs?: readonly number[];
|
|
50
|
+
sleep?: (ms: number) => Promise<void>;
|
|
51
|
+
};
|
|
48
52
|
type EvalModelComparisonPolicy = Omit<EvalModelComparisonOptions, "baselineModel">;
|
|
49
53
|
export declare function createDefaultEvalReportDir(runId: string, label?: string): string;
|
|
50
54
|
export declare function createEvalArtifactPaths(reportDir: string): EvalArtifactPaths;
|
|
@@ -55,6 +59,7 @@ export declare function normalizeEvalInputForAgent(input: unknown): string;
|
|
|
55
59
|
export declare function summarizeReportForCli(report: EvalReport): CliEvalSummary;
|
|
56
60
|
export declare function createSummaryArtifact(report: EvalReport, baseline?: EvalReportComparison): EvalSummaryArtifact;
|
|
57
61
|
export declare function applyGatewayBillingGroupFinalization(report: EvalReport, finalization: GatewayBillingGroupFinalization): EvalReport;
|
|
62
|
+
export declare function finalizeGatewayBillingGroup(billingGroupId: string, options?: GatewayBillingFinalizeOptions): Promise<GatewayBillingGroupFinalization | undefined>;
|
|
58
63
|
export declare function runEvalWithGatewayBillingGroup(billingGroupId: string, operation: () => Promise<EvalReport>): Promise<EvalReport>;
|
|
59
64
|
export declare function exportEvalReportForCli(report: EvalReport, config?: EvalReportExportConfig): Promise<EvalReport>;
|
|
60
65
|
export declare function loadEvalModelComparisonPolicy(projectDir: string, policyPath?: string): Promise<EvalModelComparisonPolicy>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EACpB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AA0BpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,+BAA+B,GAAG;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EACpB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AA0BpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,+BAA+B,GAAG;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAOF,KAAK,6BAA6B,GAAG;IACnC,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AAoDnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAO5E;AAMD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CASxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAcrB;AA0FD,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,+BAA+B,GAC5C,UAAU,CAiBZ;AAWD,wBAAsB,2BAA2B,CAC/C,cAAc,EAAE,MAAM,EACtB,OAAO,GAAE,6BAAkC,GAC1C,OAAO,CAAC,+BAA+B,GAAG,SAAS,CAAC,CAgDtD;AAED,wBAAsB,8BAA8B,CAClD,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,GACnC,OAAO,CAAC,UAAU,CAAC,CAerB;AAED,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,UAAU,CAAC,CAGrB;AA8FD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAkED,uEAAuE;AACvE,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,MAAM,CAkFR;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,MA0DrD;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AA+FD,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AA8LF,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAsJrE"}
|
|
@@ -10,6 +10,16 @@ import { applyRuntimeAuthContext } from "../../shared/runtime-auth.js";
|
|
|
10
10
|
import { cliLogger, exitProcess, VERSION } from "../../utils/index.js";
|
|
11
11
|
import { createErrorEnvelope, createSuccessEnvelope, isJsonMode, outputJson, } from "../../shared/json-output.js";
|
|
12
12
|
import { withProjectSourceContext } from "../../shared/project-source-context.js";
|
|
13
|
+
const GATEWAY_BILLING_GROUP_USAGE_NOT_READY_CODE = "gateway_billing_group_usage_not_ready";
|
|
14
|
+
// Gateway usage capture is eventually consistent after model streams close.
|
|
15
|
+
const DEFAULT_GATEWAY_BILLING_FINALIZE_RETRY_DELAYS_MS = [
|
|
16
|
+
500,
|
|
17
|
+
1_000,
|
|
18
|
+
2_000,
|
|
19
|
+
4_000,
|
|
20
|
+
8_000,
|
|
21
|
+
15_000,
|
|
22
|
+
];
|
|
13
23
|
const MODEL_COMPARISON_METRICS = [
|
|
14
24
|
"passRate",
|
|
15
25
|
"failed",
|
|
@@ -192,6 +202,33 @@ function parseGatewayBillingGroupFinalization(payload) {
|
|
|
192
202
|
veryfront_billed_usd: veryfrontBilledUsd,
|
|
193
203
|
};
|
|
194
204
|
}
|
|
205
|
+
function sleep(ms) {
|
|
206
|
+
return new Promise((resolve) => dntShim.setTimeout(resolve, ms));
|
|
207
|
+
}
|
|
208
|
+
async function readGatewayBillingFinalizeError(response) {
|
|
209
|
+
const bodyText = await response.text().catch(() => "");
|
|
210
|
+
if (!bodyText)
|
|
211
|
+
return { bodyText };
|
|
212
|
+
try {
|
|
213
|
+
const payload = JSON.parse(bodyText);
|
|
214
|
+
if (isRecord(payload) && typeof payload.code === "string") {
|
|
215
|
+
return { bodyText, code: payload.code };
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
catch {
|
|
219
|
+
// Keep the raw body text for the warning below.
|
|
220
|
+
}
|
|
221
|
+
return { bodyText };
|
|
222
|
+
}
|
|
223
|
+
function isGatewayBillingUsageNotReady(response, error) {
|
|
224
|
+
if (response.status !== 409)
|
|
225
|
+
return false;
|
|
226
|
+
return error.code === GATEWAY_BILLING_GROUP_USAGE_NOT_READY_CODE;
|
|
227
|
+
}
|
|
228
|
+
function formatGatewayBillingFinalizeWarning(billingGroupId, response, error) {
|
|
229
|
+
const body = error.bodyText ? ` ${error.bodyText}` : "";
|
|
230
|
+
return `Gateway billing finalization skipped for ${billingGroupId}: ${response.status}${body}`;
|
|
231
|
+
}
|
|
195
232
|
export function applyGatewayBillingGroupFinalization(report, finalization) {
|
|
196
233
|
return {
|
|
197
234
|
...report,
|
|
@@ -216,36 +253,45 @@ function hasGatewayUsage(report) {
|
|
|
216
253
|
usage?.veryfrontChargeUsd !== undefined ||
|
|
217
254
|
usage?.veryfrontBilledUsd !== undefined);
|
|
218
255
|
}
|
|
219
|
-
async function finalizeGatewayBillingGroup(billingGroupId) {
|
|
256
|
+
export async function finalizeGatewayBillingGroup(billingGroupId, options = {}) {
|
|
220
257
|
const bootstrap = getVeryfrontCloudBootstrap();
|
|
221
258
|
if (!bootstrap.apiToken)
|
|
222
259
|
return undefined;
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
260
|
+
const retryDelaysMs = options.retryDelaysMs ?? DEFAULT_GATEWAY_BILLING_FINALIZE_RETRY_DELAYS_MS;
|
|
261
|
+
const sleepFn = options.sleep ?? sleep;
|
|
262
|
+
for (let attempt = 0;; attempt += 1) {
|
|
263
|
+
let response;
|
|
264
|
+
try {
|
|
265
|
+
response = await fetch(joinApiUrl(bootstrap.apiBaseUrl, "ai/gateway/billing/finalize"), {
|
|
266
|
+
method: "POST",
|
|
267
|
+
headers: {
|
|
268
|
+
Authorization: `Bearer ${bootstrap.apiToken}`,
|
|
269
|
+
"Content-Type": "application/json",
|
|
270
|
+
...(bootstrap.projectSlug ? { "x-veryfront-project-slug": bootstrap.projectSlug } : {}),
|
|
271
|
+
},
|
|
272
|
+
body: JSON.stringify({ billing_group_id: billingGroupId }),
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
catch (error) {
|
|
276
|
+
cliLogger.warn(`Gateway billing finalization skipped for ${billingGroupId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
277
|
+
return undefined;
|
|
278
|
+
}
|
|
279
|
+
if (!response.ok) {
|
|
280
|
+
const error = await readGatewayBillingFinalizeError(response);
|
|
281
|
+
const retryDelayMs = retryDelaysMs[attempt];
|
|
282
|
+
if (isGatewayBillingUsageNotReady(response, error) && retryDelayMs !== undefined) {
|
|
283
|
+
await sleepFn(retryDelayMs);
|
|
284
|
+
continue;
|
|
285
|
+
}
|
|
286
|
+
cliLogger.warn(formatGatewayBillingFinalizeWarning(billingGroupId, response, error));
|
|
287
|
+
return undefined;
|
|
288
|
+
}
|
|
289
|
+
const finalization = parseGatewayBillingGroupFinalization(await response.json());
|
|
290
|
+
if (!finalization) {
|
|
291
|
+
cliLogger.warn(`Gateway billing finalization skipped for ${billingGroupId}: invalid response`);
|
|
292
|
+
}
|
|
293
|
+
return finalization;
|
|
247
294
|
}
|
|
248
|
-
return finalization;
|
|
249
295
|
}
|
|
250
296
|
export async function runEvalWithGatewayBillingGroup(billingGroupId, operation) {
|
|
251
297
|
const currentContext = getCurrentVeryfrontCloudContext();
|
package/esm/deno.js
CHANGED