veryfront 0.1.979 → 0.1.980
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/cli/commands/eval/command.d.ts +5 -0
- package/esm/cli/commands/eval/command.d.ts.map +1 -1
- package/esm/cli/commands/eval/command.js +64 -26
- package/esm/deno.js +1 -1
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +1 -1
|
@@ -45,6 +45,10 @@ type GatewayBillingGroupFinalization = {
|
|
|
45
45
|
veryfront_charge_usd: number;
|
|
46
46
|
veryfront_billed_usd: number;
|
|
47
47
|
};
|
|
48
|
+
type GatewayBillingFinalizeOptions = {
|
|
49
|
+
retryDelaysMs?: readonly number[];
|
|
50
|
+
sleep?: (ms: number) => Promise<void>;
|
|
51
|
+
};
|
|
48
52
|
type EvalModelComparisonPolicy = Omit<EvalModelComparisonOptions, "baselineModel">;
|
|
49
53
|
export declare function createDefaultEvalReportDir(runId: string, label?: string): string;
|
|
50
54
|
export declare function createEvalArtifactPaths(reportDir: string): EvalArtifactPaths;
|
|
@@ -55,6 +59,7 @@ export declare function normalizeEvalInputForAgent(input: unknown): string;
|
|
|
55
59
|
export declare function summarizeReportForCli(report: EvalReport): CliEvalSummary;
|
|
56
60
|
export declare function createSummaryArtifact(report: EvalReport, baseline?: EvalReportComparison): EvalSummaryArtifact;
|
|
57
61
|
export declare function applyGatewayBillingGroupFinalization(report: EvalReport, finalization: GatewayBillingGroupFinalization): EvalReport;
|
|
62
|
+
export declare function finalizeGatewayBillingGroup(billingGroupId: string, options?: GatewayBillingFinalizeOptions): Promise<GatewayBillingGroupFinalization | undefined>;
|
|
58
63
|
export declare function runEvalWithGatewayBillingGroup(billingGroupId: string, operation: () => Promise<EvalReport>): Promise<EvalReport>;
|
|
59
64
|
export declare function exportEvalReportForCli(report: EvalReport, config?: EvalReportExportConfig): Promise<EvalReport>;
|
|
60
65
|
export declare function loadEvalModelComparisonPolicy(projectDir: string, policyPath?: string): Promise<EvalModelComparisonPolicy>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EACpB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AA0BpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,+BAA+B,GAAG;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EACpB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AA0BpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,+BAA+B,GAAG;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAOF,KAAK,6BAA6B,GAAG;IACnC,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AA4CnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAO5E;AAMD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CASxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAcrB;AA0FD,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,+BAA+B,GAC5C,UAAU,CAiBZ;AAWD,wBAAsB,2BAA2B,CAC/C,cAAc,EAAE,MAAM,EACtB,OAAO,GAAE,6BAAkC,GAC1C,OAAO,CAAC,+BAA+B,GAAG,SAAS,CAAC,CAgDtD;AAED,wBAAsB,8BAA8B,CAClD,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,GACnC,OAAO,CAAC,UAAU,CAAC,CAerB;AAED,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,UAAU,CAAC,CAGrB;AA8FD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAkED,uEAAuE;AACvE,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,MAAM,CAkFR;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,MA0DrD;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AA+FD,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AA8LF,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAsJrE"}
|
|
@@ -10,6 +10,8 @@ import { applyRuntimeAuthContext } from "../../shared/runtime-auth.js";
|
|
|
10
10
|
import { cliLogger, exitProcess, VERSION } from "../../utils/index.js";
|
|
11
11
|
import { createErrorEnvelope, createSuccessEnvelope, isJsonMode, outputJson, } from "../../shared/json-output.js";
|
|
12
12
|
import { withProjectSourceContext } from "../../shared/project-source-context.js";
|
|
13
|
+
const GATEWAY_BILLING_GROUP_USAGE_NOT_READY_CODE = "gateway_billing_group_usage_not_ready";
|
|
14
|
+
const DEFAULT_GATEWAY_BILLING_FINALIZE_RETRY_DELAYS_MS = [250, 500, 1_000, 2_000, 4_000];
|
|
13
15
|
const MODEL_COMPARISON_METRICS = [
|
|
14
16
|
"passRate",
|
|
15
17
|
"failed",
|
|
@@ -192,6 +194,33 @@ function parseGatewayBillingGroupFinalization(payload) {
|
|
|
192
194
|
veryfront_billed_usd: veryfrontBilledUsd,
|
|
193
195
|
};
|
|
194
196
|
}
|
|
197
|
+
function sleep(ms) {
|
|
198
|
+
return new Promise((resolve) => dntShim.setTimeout(resolve, ms));
|
|
199
|
+
}
|
|
200
|
+
async function readGatewayBillingFinalizeError(response) {
|
|
201
|
+
const bodyText = await response.text().catch(() => "");
|
|
202
|
+
if (!bodyText)
|
|
203
|
+
return { bodyText };
|
|
204
|
+
try {
|
|
205
|
+
const payload = JSON.parse(bodyText);
|
|
206
|
+
if (isRecord(payload) && typeof payload.code === "string") {
|
|
207
|
+
return { bodyText, code: payload.code };
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
211
|
+
// Keep the raw body text for the warning below.
|
|
212
|
+
}
|
|
213
|
+
return { bodyText };
|
|
214
|
+
}
|
|
215
|
+
function isGatewayBillingUsageNotReady(response, error) {
|
|
216
|
+
if (response.status !== 409)
|
|
217
|
+
return false;
|
|
218
|
+
return error.code === GATEWAY_BILLING_GROUP_USAGE_NOT_READY_CODE;
|
|
219
|
+
}
|
|
220
|
+
function formatGatewayBillingFinalizeWarning(billingGroupId, response, error) {
|
|
221
|
+
const body = error.bodyText ? ` ${error.bodyText}` : "";
|
|
222
|
+
return `Gateway billing finalization skipped for ${billingGroupId}: ${response.status}${body}`;
|
|
223
|
+
}
|
|
195
224
|
export function applyGatewayBillingGroupFinalization(report, finalization) {
|
|
196
225
|
return {
|
|
197
226
|
...report,
|
|
@@ -216,36 +245,45 @@ function hasGatewayUsage(report) {
|
|
|
216
245
|
usage?.veryfrontChargeUsd !== undefined ||
|
|
217
246
|
usage?.veryfrontBilledUsd !== undefined);
|
|
218
247
|
}
|
|
219
|
-
async function finalizeGatewayBillingGroup(billingGroupId) {
|
|
248
|
+
export async function finalizeGatewayBillingGroup(billingGroupId, options = {}) {
|
|
220
249
|
const bootstrap = getVeryfrontCloudBootstrap();
|
|
221
250
|
if (!bootstrap.apiToken)
|
|
222
251
|
return undefined;
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
252
|
+
const retryDelaysMs = options.retryDelaysMs ?? DEFAULT_GATEWAY_BILLING_FINALIZE_RETRY_DELAYS_MS;
|
|
253
|
+
const sleepFn = options.sleep ?? sleep;
|
|
254
|
+
for (let attempt = 0;; attempt += 1) {
|
|
255
|
+
let response;
|
|
256
|
+
try {
|
|
257
|
+
response = await fetch(joinApiUrl(bootstrap.apiBaseUrl, "ai/gateway/billing/finalize"), {
|
|
258
|
+
method: "POST",
|
|
259
|
+
headers: {
|
|
260
|
+
Authorization: `Bearer ${bootstrap.apiToken}`,
|
|
261
|
+
"Content-Type": "application/json",
|
|
262
|
+
...(bootstrap.projectSlug ? { "x-veryfront-project-slug": bootstrap.projectSlug } : {}),
|
|
263
|
+
},
|
|
264
|
+
body: JSON.stringify({ billing_group_id: billingGroupId }),
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
catch (error) {
|
|
268
|
+
cliLogger.warn(`Gateway billing finalization skipped for ${billingGroupId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
269
|
+
return undefined;
|
|
270
|
+
}
|
|
271
|
+
if (!response.ok) {
|
|
272
|
+
const error = await readGatewayBillingFinalizeError(response);
|
|
273
|
+
const retryDelayMs = retryDelaysMs[attempt];
|
|
274
|
+
if (isGatewayBillingUsageNotReady(response, error) && retryDelayMs !== undefined) {
|
|
275
|
+
await sleepFn(retryDelayMs);
|
|
276
|
+
continue;
|
|
277
|
+
}
|
|
278
|
+
cliLogger.warn(formatGatewayBillingFinalizeWarning(billingGroupId, response, error));
|
|
279
|
+
return undefined;
|
|
280
|
+
}
|
|
281
|
+
const finalization = parseGatewayBillingGroupFinalization(await response.json());
|
|
282
|
+
if (!finalization) {
|
|
283
|
+
cliLogger.warn(`Gateway billing finalization skipped for ${billingGroupId}: invalid response`);
|
|
284
|
+
}
|
|
285
|
+
return finalization;
|
|
247
286
|
}
|
|
248
|
-
return finalization;
|
|
249
287
|
}
|
|
250
288
|
export async function runEvalWithGatewayBillingGroup(billingGroupId, operation) {
|
|
251
289
|
const currentContext = getCurrentVeryfrontCloudContext();
|
package/esm/deno.js
CHANGED