veryfront 0.1.971 → 0.1.972
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/cli/commands/eval/command.d.ts +13 -1
- package/esm/cli/commands/eval/command.d.ts.map +1 -1
- package/esm/cli/commands/eval/command.js +126 -8
- package/esm/deno.js +1 -1
- package/esm/src/eval/index.d.ts +1 -1
- package/esm/src/eval/index.d.ts.map +1 -1
- package/esm/src/eval/index.js +1 -1
- package/esm/src/eval/runner.d.ts +4 -1
- package/esm/src/eval/runner.d.ts.map +1 -1
- package/esm/src/eval/runner.js +1 -1
- package/esm/src/platform/cloud/resolver.d.ts.map +1 -1
- package/esm/src/platform/cloud/resolver.js +5 -2
- package/esm/src/provider/index.d.ts +3 -1
- package/esm/src/provider/index.d.ts.map +1 -1
- package/esm/src/provider/index.js +2 -1
- package/esm/src/provider/veryfront-cloud/context.d.ts +3 -0
- package/esm/src/provider/veryfront-cloud/context.d.ts.map +1 -1
- package/esm/src/provider/veryfront-cloud/context.js +6 -0
- package/esm/src/provider/veryfront-cloud/shared.d.ts.map +1 -1
- package/esm/src/provider/veryfront-cloud/shared.js +6 -0
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { AgentResponse } from "../../../src/agent/index.js";
|
|
2
2
|
import type { VeryfrontConfig } from "../../../src/config/index.js";
|
|
3
|
-
import type { DiscoveredEval, EvalModelComparison, EvalModelComparisonOptions, EvalReport, EvalReportComparison, EvalToolCall } from "../../../src/eval/index.js";
|
|
3
|
+
import type { DiscoveredEval, EvalModelComparison, EvalModelComparisonOptions, EvalReport, EvalReportComparison, EvalReportExportConfig, EvalToolCall } from "../../../src/eval/index.js";
|
|
4
4
|
import type { EvalArgs } from "./handler.js";
|
|
5
5
|
export interface EvalOptions extends EvalArgs {
|
|
6
6
|
}
|
|
@@ -36,6 +36,15 @@ type EvalSummaryArtifact = {
|
|
|
36
36
|
exports?: EvalReport["exports"];
|
|
37
37
|
baseline?: EvalReportComparison;
|
|
38
38
|
};
|
|
39
|
+
type GatewayBillingGroupFinalization = {
|
|
40
|
+
billing_group_id: string;
|
|
41
|
+
charged_credits: number;
|
|
42
|
+
target_credits: number;
|
|
43
|
+
adjustment_credits: number;
|
|
44
|
+
provider_cost_usd: number;
|
|
45
|
+
veryfront_charge_usd: number;
|
|
46
|
+
veryfront_billed_usd: number;
|
|
47
|
+
};
|
|
39
48
|
type EvalModelComparisonPolicy = Omit<EvalModelComparisonOptions, "baselineModel">;
|
|
40
49
|
export declare function createDefaultEvalReportDir(runId: string, label?: string): string;
|
|
41
50
|
export declare function createEvalArtifactPaths(reportDir: string): EvalArtifactPaths;
|
|
@@ -45,6 +54,9 @@ export declare function findEvalForCliId(evals: DiscoveredEval[], id: string): D
|
|
|
45
54
|
export declare function normalizeEvalInputForAgent(input: unknown): string;
|
|
46
55
|
export declare function summarizeReportForCli(report: EvalReport): CliEvalSummary;
|
|
47
56
|
export declare function createSummaryArtifact(report: EvalReport, baseline?: EvalReportComparison): EvalSummaryArtifact;
|
|
57
|
+
export declare function applyGatewayBillingGroupFinalization(report: EvalReport, finalization: GatewayBillingGroupFinalization): EvalReport;
|
|
58
|
+
export declare function runEvalWithGatewayBillingGroup(billingGroupId: string, operation: () => Promise<EvalReport>): Promise<EvalReport>;
|
|
59
|
+
export declare function exportEvalReportForCli(report: EvalReport, config?: EvalReportExportConfig): Promise<EvalReport>;
|
|
48
60
|
export declare function loadEvalModelComparisonPolicy(projectDir: string, policyPath?: string): Promise<EvalModelComparisonPolicy>;
|
|
49
61
|
export declare function createResolvedEvalModelComparisonConfig(projectDir: string, options: EvalOptions): Promise<ResolvedEvalModelComparisonConfig | undefined>;
|
|
50
62
|
export declare function createEvalModelComparisonArtifact(reports: EvalReport[], baselineModel: string, policy?: EvalModelComparisonPolicy): EvalModelComparison;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,
|
|
1
|
+
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EACpB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AAyBpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,+BAA+B,GAAG;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AA6CnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAO5E;AAMD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CASxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAcrB;AAmDD,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,+BAA+B,GAC5C,UAAU,CAiBZ;AAsDD,wBAAsB,8BAA8B,CAClD,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,GACnC,OAAO,CAAC,UAAU,CAAC,CAerB;AAED,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,UAAU,CAAC,CAGrB;AA8FD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAkED,uEAAuE;AACvE,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,MAAM,CAkFR;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,MA0DrD;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AA+FD,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AA8LF,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAsJrE"}
|
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
import * as dntShim from "../../../_dnt.shims.js";
|
|
5
5
|
import { dirname, isAbsolute, join, relative, resolve } from "../../../deps/jsr.io/@std/path/1.1.4/mod.js";
|
|
6
6
|
import { createProjectDiscoveryConfig, discoverAll } from "../../../src/discovery/index.js";
|
|
7
|
-
import { compareEvalModelReports, compareEvalReports, createEvalModelComparisonMarkdown, discoverEvals, resolveEvalRunProvenance, runEval, } from "../../../src/eval/index.js";
|
|
7
|
+
import { compareEvalModelReports, compareEvalReports, createEvalModelComparisonMarkdown, discoverEvals, exportEvalReport, resolveEvalRunProvenance, runEval, } from "../../../src/eval/index.js";
|
|
8
|
+
import { getCurrentVeryfrontCloudContext, getVeryfrontCloudBootstrap, runWithVeryfrontCloudContextAsync, } from "../../../src/provider/index.js";
|
|
8
9
|
import { applyRuntimeAuthContext } from "../../shared/runtime-auth.js";
|
|
9
10
|
import { cliLogger, exitProcess, VERSION } from "../../utils/index.js";
|
|
10
11
|
import { createErrorEnvelope, createSuccessEnvelope, isJsonMode, outputJson, } from "../../shared/json-output.js";
|
|
@@ -160,6 +161,117 @@ export function createSummaryArtifact(report, baseline) {
|
|
|
160
161
|
function isRecord(value) {
|
|
161
162
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
162
163
|
}
|
|
164
|
+
function joinApiUrl(baseUrl, path) {
|
|
165
|
+
return `${baseUrl.replace(/\/+$/, "")}/${path.replace(/^\/+/, "")}`;
|
|
166
|
+
}
|
|
167
|
+
function readFiniteNumber(record, field) {
|
|
168
|
+
const value = record[field];
|
|
169
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
170
|
+
}
|
|
171
|
+
function parseGatewayBillingGroupFinalization(payload) {
|
|
172
|
+
if (!isRecord(payload) || typeof payload.billing_group_id !== "string")
|
|
173
|
+
return undefined;
|
|
174
|
+
const chargedCredits = readFiniteNumber(payload, "charged_credits");
|
|
175
|
+
const targetCredits = readFiniteNumber(payload, "target_credits");
|
|
176
|
+
const adjustmentCredits = readFiniteNumber(payload, "adjustment_credits");
|
|
177
|
+
const providerCostUsd = readFiniteNumber(payload, "provider_cost_usd");
|
|
178
|
+
const veryfrontChargeUsd = readFiniteNumber(payload, "veryfront_charge_usd");
|
|
179
|
+
const veryfrontBilledUsd = readFiniteNumber(payload, "veryfront_billed_usd");
|
|
180
|
+
if (chargedCredits === undefined ||
|
|
181
|
+
targetCredits === undefined ||
|
|
182
|
+
adjustmentCredits === undefined ||
|
|
183
|
+
providerCostUsd === undefined ||
|
|
184
|
+
veryfrontChargeUsd === undefined ||
|
|
185
|
+
veryfrontBilledUsd === undefined) {
|
|
186
|
+
return undefined;
|
|
187
|
+
}
|
|
188
|
+
return {
|
|
189
|
+
billing_group_id: payload.billing_group_id,
|
|
190
|
+
charged_credits: chargedCredits,
|
|
191
|
+
target_credits: targetCredits,
|
|
192
|
+
adjustment_credits: adjustmentCredits,
|
|
193
|
+
provider_cost_usd: providerCostUsd,
|
|
194
|
+
veryfront_charge_usd: veryfrontChargeUsd,
|
|
195
|
+
veryfront_billed_usd: veryfrontBilledUsd,
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
export function applyGatewayBillingGroupFinalization(report, finalization) {
|
|
199
|
+
return {
|
|
200
|
+
...report,
|
|
201
|
+
summary: {
|
|
202
|
+
...report.summary,
|
|
203
|
+
usage: {
|
|
204
|
+
...(report.summary.usage ?? {}),
|
|
205
|
+
providerCostUsd: finalization.provider_cost_usd,
|
|
206
|
+
veryfrontChargeUsd: finalization.veryfront_charge_usd,
|
|
207
|
+
veryfrontBilledUsd: finalization.veryfront_billed_usd,
|
|
208
|
+
costCredits: finalization.target_credits,
|
|
209
|
+
costSource: "gateway",
|
|
210
|
+
billingMode: "direct",
|
|
211
|
+
usageCaptureStatus: "complete",
|
|
212
|
+
},
|
|
213
|
+
},
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
function hasGatewayUsage(report) {
|
|
217
|
+
const usage = report.summary.usage;
|
|
218
|
+
return Boolean(usage?.costSource === "gateway" ||
|
|
219
|
+
usage?.veryfrontChargeUsd !== undefined ||
|
|
220
|
+
usage?.veryfrontBilledUsd !== undefined);
|
|
221
|
+
}
|
|
222
|
+
async function finalizeGatewayBillingGroup(billingGroupId) {
|
|
223
|
+
const bootstrap = getVeryfrontCloudBootstrap();
|
|
224
|
+
if (!bootstrap.apiToken)
|
|
225
|
+
return undefined;
|
|
226
|
+
let response;
|
|
227
|
+
try {
|
|
228
|
+
response = await fetch(joinApiUrl(bootstrap.apiBaseUrl, "ai/gateway/billing/finalize"), {
|
|
229
|
+
method: "POST",
|
|
230
|
+
headers: {
|
|
231
|
+
Authorization: `Bearer ${bootstrap.apiToken}`,
|
|
232
|
+
"Content-Type": "application/json",
|
|
233
|
+
...(bootstrap.projectSlug ? { "x-veryfront-project-slug": bootstrap.projectSlug } : {}),
|
|
234
|
+
},
|
|
235
|
+
body: JSON.stringify({ billing_group_id: billingGroupId }),
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
catch (error) {
|
|
239
|
+
cliLogger.warn(`Gateway billing finalization skipped for ${billingGroupId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
240
|
+
return undefined;
|
|
241
|
+
}
|
|
242
|
+
if (!response.ok) {
|
|
243
|
+
const message = await response.text().catch(() => "");
|
|
244
|
+
cliLogger.warn(`Gateway billing finalization skipped for ${billingGroupId}: ${response.status}${message ? ` ${message}` : ""}`);
|
|
245
|
+
return undefined;
|
|
246
|
+
}
|
|
247
|
+
const finalization = parseGatewayBillingGroupFinalization(await response.json());
|
|
248
|
+
if (!finalization) {
|
|
249
|
+
cliLogger.warn(`Gateway billing finalization skipped for ${billingGroupId}: invalid response`);
|
|
250
|
+
}
|
|
251
|
+
return finalization;
|
|
252
|
+
}
|
|
253
|
+
export async function runEvalWithGatewayBillingGroup(billingGroupId, operation) {
|
|
254
|
+
const currentContext = getCurrentVeryfrontCloudContext();
|
|
255
|
+
const billingContext = { ...(currentContext ?? {}), billingGroupId };
|
|
256
|
+
let report;
|
|
257
|
+
try {
|
|
258
|
+
report = await runWithVeryfrontCloudContextAsync(billingContext, operation);
|
|
259
|
+
}
|
|
260
|
+
catch (error) {
|
|
261
|
+
if (billingContext.billingGroupUsed) {
|
|
262
|
+
await finalizeGatewayBillingGroup(billingGroupId);
|
|
263
|
+
}
|
|
264
|
+
throw error;
|
|
265
|
+
}
|
|
266
|
+
if (!billingContext.billingGroupUsed && !hasGatewayUsage(report))
|
|
267
|
+
return report;
|
|
268
|
+
const finalization = await finalizeGatewayBillingGroup(billingGroupId);
|
|
269
|
+
return finalization ? applyGatewayBillingGroupFinalization(report, finalization) : report;
|
|
270
|
+
}
|
|
271
|
+
export async function exportEvalReportForCli(report, config) {
|
|
272
|
+
const exports = await exportEvalReport(report, config);
|
|
273
|
+
return exports === undefined ? report : { ...report, exports };
|
|
274
|
+
}
|
|
163
275
|
function readNumberField(record, field, path) {
|
|
164
276
|
const value = record[field];
|
|
165
277
|
if (value === undefined)
|
|
@@ -651,9 +763,11 @@ async function runEvalModelComparison(input) {
|
|
|
651
763
|
for (const model of input.config.models) {
|
|
652
764
|
const modelPaths = paths.models[model];
|
|
653
765
|
const modelOptions = { ...input.options, model, report: undefined };
|
|
654
|
-
const
|
|
766
|
+
const modelRunId = `${runId}_${sanitizeModelIdForPath(model)}`;
|
|
767
|
+
const exportConfig = createEvalCliExportConfig(input.evalItem, modelOptions, input.projectDir, modelPaths);
|
|
768
|
+
const finalizedReport = await runEvalWithGatewayBillingGroup(modelRunId, () => runEval(input.evalItem.definition, {
|
|
655
769
|
baseDir: input.options.datasetBase ?? input.projectDir,
|
|
656
|
-
runId:
|
|
770
|
+
runId: modelRunId,
|
|
657
771
|
adapters: {
|
|
658
772
|
agent: createAgentAdapterForModel(input.agent, input.options, model),
|
|
659
773
|
},
|
|
@@ -661,8 +775,8 @@ async function runEvalModelComparison(input) {
|
|
|
661
775
|
model,
|
|
662
776
|
provenance,
|
|
663
777
|
},
|
|
664
|
-
|
|
665
|
-
|
|
778
|
+
}));
|
|
779
|
+
const report = await exportEvalReportForCli(finalizedReport, exportConfig);
|
|
666
780
|
reports.push(report);
|
|
667
781
|
await writeEvalArtifacts(report, modelPaths);
|
|
668
782
|
await writeTextFileEnsuringDir(modelPaths.junit, createJunitXml(report));
|
|
@@ -829,7 +943,11 @@ export async function evalCommand(options) {
|
|
|
829
943
|
projectDir,
|
|
830
944
|
frameworkVersion: VERSION,
|
|
831
945
|
});
|
|
832
|
-
const
|
|
946
|
+
const billingGroupId = options.model
|
|
947
|
+
? `${runId}_${sanitizeModelIdForPath(options.model)}`
|
|
948
|
+
: runId;
|
|
949
|
+
const exportConfig = createEvalCliExportConfig(evalItem, options, projectDir, artifactPaths);
|
|
950
|
+
const finalizedReport = await runEvalWithGatewayBillingGroup(billingGroupId, () => runEval(evalItem.definition, {
|
|
833
951
|
baseDir: options.datasetBase ?? projectDir,
|
|
834
952
|
runId,
|
|
835
953
|
adapters: {
|
|
@@ -839,8 +957,8 @@ export async function evalCommand(options) {
|
|
|
839
957
|
provenance,
|
|
840
958
|
...(options.model ? { model: options.model } : {}),
|
|
841
959
|
},
|
|
842
|
-
|
|
843
|
-
|
|
960
|
+
}));
|
|
961
|
+
const report = await exportEvalReportForCli(finalizedReport, exportConfig);
|
|
844
962
|
const baseline = options.baseline
|
|
845
963
|
? compareEvalReports(report, await readEvalReport(options.baseline))
|
|
846
964
|
: undefined;
|
package/esm/deno.js
CHANGED
package/esm/src/eval/index.d.ts
CHANGED
|
@@ -51,7 +51,7 @@ export { createEvalReport, summarizeEvalRecords } from "./report.js";
|
|
|
51
51
|
export { compareEvalReports } from "./baseline.js";
|
|
52
52
|
export { compareEvalModelReports, createEvalModelComparisonMarkdown } from "./model-comparison.js";
|
|
53
53
|
export { createEvalRunProvenance, resolveEvalRunProvenance } from "./provenance.js";
|
|
54
|
-
export { runEval } from "./runner.js";
|
|
54
|
+
export { exportEvalReport, runEval } from "./runner.js";
|
|
55
55
|
export { deriveEvalId, discoverEvals, findEvalById } from "./discovery.js";
|
|
56
56
|
export { createEvalSourceDocument, getEvalEditableFieldSchema, getEvalRunSchema, getEvalSourceDocumentSchema, getEvalSourcePatchSchema, getEvalSourceReferenceSchema, getEvalStudioCapabilitySchema, } from "./studio.js";
|
|
57
57
|
export type { DiscoveredEval, EvalDiscoveryOptions, EvalDiscoveryResult } from "./discovery.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,iCAAiC,EAAE,MAAM,uBAAuB,CAAC;AACnG,OAAO,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AACpF,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,iCAAiC,EAAE,MAAM,uBAAuB,CAAC;AACnG,OAAO,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AACpF,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EACL,wBAAwB,EACxB,0BAA0B,EAC1B,gBAAgB,EAChB,2BAA2B,EAC3B,wBAAwB,EACxB,4BAA4B,EAC5B,6BAA6B,GAC9B,MAAM,aAAa,CAAC;AAErB,YAAY,EAAE,cAAc,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAChG,YAAY,EAAE,+BAA+B,EAAE,MAAM,aAAa,CAAC;AAEnE,YAAY,EACV,gBAAgB,EAChB,uBAAuB,EACvB,sBAAsB,EACtB,cAAc,EACd,mCAAmC,EACnC,gBAAgB,EAChB,WAAW,EACX,sBAAsB,EACtB,cAAc,EACd,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,eAAe,EACf,wBAAwB,EACxB,gBAAgB,EAChB,sBAAsB,EACtB,2BAA2B,EAC3B,6BAA6B,EAC7B,mCAAmC,EACnC,UAAU,EACV,iBAAiB,EACjB,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,4BAA4B,EAC5B,mBAAmB,EACnB,6BAA6B,EAC7B,2BAA2B,EAC3B,6BAA6B,EAC7B,4BAA4B,EAC5B,0BAA0B,EAC1B,sBAAsB,EACtB,UAAU,EACV,UAAU,EACV,oBAAoB,EACpB,sBAAsB,EACtB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,EACZ,UAAU,EACV,cAAc,EACd,YAAY,EACZ,wBAAwB,EACxB,wBAAwB,EACxB,kBAAkB,EAClB,sBAAsB,EACtB,SAAS,EACT,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,+BAA+B,EAC/B,iBAAiB,EACjB,OAAO,EACP,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,aAAa,CAAC"}
|
package/esm/src/eval/index.js
CHANGED
|
@@ -51,6 +51,6 @@ export { createEvalReport, summarizeEvalRecords } from "./report.js";
|
|
|
51
51
|
export { compareEvalReports } from "./baseline.js";
|
|
52
52
|
export { compareEvalModelReports, createEvalModelComparisonMarkdown } from "./model-comparison.js";
|
|
53
53
|
export { createEvalRunProvenance, resolveEvalRunProvenance } from "./provenance.js";
|
|
54
|
-
export { runEval } from "./runner.js";
|
|
54
|
+
export { exportEvalReport, runEval } from "./runner.js";
|
|
55
55
|
export { deriveEvalId, discoverEvals, findEvalById } from "./discovery.js";
|
|
56
56
|
export { createEvalSourceDocument, getEvalEditableFieldSchema, getEvalRunSchema, getEvalSourceDocumentSchema, getEvalSourcePatchSchema, getEvalSourceReferenceSchema, getEvalStudioCapabilitySchema, } from "./studio.js";
|
package/esm/src/eval/runner.d.ts
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { createEvalReport } from "./report.js";
|
|
2
|
+
import { type EvalReportExportResult } from "../extensions/eval/index.js";
|
|
3
|
+
import type { EvalDefinition, EvalReportExportConfig, RunEvalOptions } from "./types.js";
|
|
4
|
+
export declare function exportEvalReport(report: ReturnType<typeof createEvalReport>, config?: EvalReportExportConfig): Promise<EvalReportExportResult[] | undefined>;
|
|
2
5
|
/** Execute an eval locally with injected target adapters. */
|
|
3
6
|
export declare function runEval(definition: EvalDefinition, options: RunEvalOptions): Promise<import("./types.js").EvalReport>;
|
|
4
7
|
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/src/eval/runner.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/src/eval/runner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/C,OAAO,EAKL,KAAK,sBAAsB,EAE5B,MAAM,6BAA6B,CAAC;AAGrC,OAAO,KAAK,EAEV,cAAc,EAGd,sBAAsB,EAGtB,cAAc,EACf,MAAM,YAAY,CAAC;AA+KpB,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,EAC3C,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,sBAAsB,EAAE,GAAG,SAAS,CAAC,CAgB/C;AAiED,6DAA6D;AAC7D,wBAAsB,OAAO,CAC3B,UAAU,EAAE,cAAc,EAC1B,OAAO,EAAE,cAAc,4CAyBxB"}
|
package/esm/src/eval/runner.js
CHANGED
|
@@ -154,7 +154,7 @@ async function exportWithSelectedExporter(registry, report, config, exporterId)
|
|
|
154
154
|
error: `EvalReportExporter "${exporterId}" did not return an export result.`,
|
|
155
155
|
};
|
|
156
156
|
}
|
|
157
|
-
async function exportEvalReport(report, config) {
|
|
157
|
+
export async function exportEvalReport(report, config) {
|
|
158
158
|
if (!config)
|
|
159
159
|
return undefined;
|
|
160
160
|
const exporterIds = config.exporterIds?.filter((id) => id.length > 0) ?? [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../../../src/src/platform/cloud/resolver.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../../../src/src/platform/cloud/resolver.ts"],"names":[],"mappings":"AAuCA,eAAO,MAAM,6BAA6B,gDAAgD,CAAC;AAC3F,eAAO,MAAM,uCAAuC,kDACH,CAAC;AAElD,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,eAAe,EAAE,OAAO,CAAC;CAC1B;AAyDD,wBAAgB,0BAA0B,IAAI,MAAM,GAAG,SAAS,CAE/D;AAED,wBAAgB,4BAA4B,IAAI,MAAM,GAAG,SAAS,CAEjE;AAED,wBAAgB,0BAA0B,IAAI,uBAAuB,CAOpE;AAED,wBAAgB,uBAAuB,IAAI,OAAO,CAgBjD;AAED,wBAAgB,6BAA6B,IAAI,MAAM,CAKtD;AAED,wBAAgB,sCAAsC,IAAI,MAAM,CAK/D"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as dntShim from "../../../_dnt.shims.js";
|
|
2
2
|
import { getCurrentRequestContext } from "../adapters/fs/veryfront/request-context.js";
|
|
3
3
|
import { getHostEnv } from "../compat/process.js";
|
|
4
|
-
import { getCurrentVeryfrontCloudContext } from "../../provider/veryfront-cloud/context.js";
|
|
4
|
+
import { getCurrentVeryfrontCloudContext, } from "../../provider/veryfront-cloud/context.js";
|
|
5
5
|
function getRuntimeConfig() {
|
|
6
6
|
const getter = dntShim.dntGlobalThis.__vfGetRuntimeConfig;
|
|
7
7
|
return getter?.() ?? {};
|
|
@@ -36,6 +36,9 @@ function normalizeServiceLayer(value) {
|
|
|
36
36
|
const normalized = value?.trim().toLowerCase();
|
|
37
37
|
return normalized?.length ? normalized : undefined;
|
|
38
38
|
}
|
|
39
|
+
function hasScopedRuntimeContext(context) {
|
|
40
|
+
return Boolean(context?.apiBaseUrl || context?.apiToken || context?.projectSlug || context?.serviceLayer);
|
|
41
|
+
}
|
|
39
42
|
function getResolvedVeryfrontCloudContext() {
|
|
40
43
|
const requestContext = getCurrentRequestContext();
|
|
41
44
|
const scopedContext = getCurrentVeryfrontCloudContext();
|
|
@@ -51,7 +54,7 @@ function getResolvedVeryfrontCloudContext() {
|
|
|
51
54
|
runtimeBootstrap.projectSlug,
|
|
52
55
|
serviceLayer: normalizeServiceLayer(scopedContext?.serviceLayer) ??
|
|
53
56
|
normalizeServiceLayer(getHostEnv("VERYFRONT_SERVICE_LAYER")),
|
|
54
|
-
hasRequestContext: requestContext !== null || scopedContext
|
|
57
|
+
hasRequestContext: requestContext !== null || hasScopedRuntimeContext(scopedContext),
|
|
55
58
|
usesVeryfrontFs: runtimeBootstrap.usesVeryfrontFs,
|
|
56
59
|
};
|
|
57
60
|
}
|
|
@@ -14,8 +14,10 @@ import "../../_dnt.polyfills.js";
|
|
|
14
14
|
export { clearModelProviders, ensureModelReady, getRegisteredModelProviders, hasModelProvider, registerModelProvider, resolveModel, } from "./model-registry.js";
|
|
15
15
|
export type { ModelProviderFactory } from "./model-registry.js";
|
|
16
16
|
export type { ModelRuntime } from "./types.js";
|
|
17
|
-
export { runWithVeryfrontCloudContext, runWithVeryfrontCloudContextAsync, } from "./veryfront-cloud/context.js";
|
|
17
|
+
export { getCurrentVeryfrontCloudContext, markCurrentVeryfrontCloudBillingGroupUsed, runWithVeryfrontCloudContext, runWithVeryfrontCloudContextAsync, } from "./veryfront-cloud/context.js";
|
|
18
18
|
export type { VeryfrontCloudContext } from "./veryfront-cloud/context.js";
|
|
19
|
+
export { getVeryfrontCloudBootstrap } from "../platform/cloud/resolver.js";
|
|
20
|
+
export type { VeryfrontCloudBootstrap } from "../platform/cloud/resolver.js";
|
|
19
21
|
export type { VeryfrontCloudProviderId } from "./veryfront-cloud/shared.js";
|
|
20
22
|
export { DEFAULT_VERYFRONT_CLOUD_MODEL_ID, findVeryfrontCloudModel, findVeryfrontCloudModelByModelId, getVeryfrontCloudProviderFromModelId, groupVeryfrontCloudModelsByProvider, normalizeVeryfrontCloudModelId, resolveHostedVeryfrontCloudModelId, resolveVeryfrontCloudGatewayModelId, resolveVeryfrontCloudModelId, resolveVeryfrontCloudModelThinking, resolveVeryfrontCloudThinkingProviderOptions, tryGetVeryfrontCloudProviderFromModelId, VERYFRONT_CLOUD_CHAT_MODELS, VERYFRONT_CLOUD_MODEL_PREFIX, } from "./veryfront-cloud/model-catalog.js";
|
|
21
23
|
export type { VeryfrontCloudChatModel, VeryfrontCloudModelThinkingConfig, } from "./veryfront-cloud/model-catalog.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/provider/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EACL,mBAAmB,EACnB,gBAAgB,EAChB,2BAA2B,EAC3B,gBAAgB,EAChB,qBAAqB,EACrB,YAAY,GACb,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAChE,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EACL,4BAA4B,EAC5B,iCAAiC,GAClC,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAC1E,YAAY,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AAC5E,OAAO,EACL,gCAAgC,EAChC,uBAAuB,EACvB,gCAAgC,EAChC,oCAAoC,EACpC,mCAAmC,EACnC,8BAA8B,EAC9B,kCAAkC,EAClC,mCAAmC,EACnC,4BAA4B,EAC5B,kCAAkC,EAClC,4CAA4C,EAC5C,uCAAuC,EACvC,2BAA2B,EAC3B,4BAA4B,GAC7B,MAAM,oCAAoC,CAAC;AAC5C,YAAY,EACV,uBAAuB,EACvB,iCAAiC,GAClC,MAAM,oCAAoC,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/provider/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EACL,mBAAmB,EACnB,gBAAgB,EAChB,2BAA2B,EAC3B,gBAAgB,EAChB,qBAAqB,EACrB,YAAY,GACb,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAChE,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EACL,+BAA+B,EAC/B,yCAAyC,EACzC,4BAA4B,EAC5B,iCAAiC,GAClC,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAC1E,OAAO,EAAE,0BAA0B,EAAE,MAAM,+BAA+B,CAAC;AAC3E,YAAY,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAC7E,YAAY,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AAC5E,OAAO,EACL,gCAAgC,EAChC,uBAAuB,EACvB,gCAAgC,EAChC,oCAAoC,EACpC,mCAAmC,EACnC,8BAA8B,EAC9B,kCAAkC,EAClC,mCAAmC,EACnC,4BAA4B,EAC5B,kCAAkC,EAClC,4CAA4C,EAC5C,uCAAuC,EACvC,2BAA2B,EAC3B,4BAA4B,GAC7B,MAAM,oCAAoC,CAAC;AAC5C,YAAY,EACV,uBAAuB,EACvB,iCAAiC,GAClC,MAAM,oCAAoC,CAAC"}
|
|
@@ -12,5 +12,6 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import "../../_dnt.polyfills.js";
|
|
14
14
|
export { clearModelProviders, ensureModelReady, getRegisteredModelProviders, hasModelProvider, registerModelProvider, resolveModel, } from "./model-registry.js";
|
|
15
|
-
export { runWithVeryfrontCloudContext, runWithVeryfrontCloudContextAsync, } from "./veryfront-cloud/context.js";
|
|
15
|
+
export { getCurrentVeryfrontCloudContext, markCurrentVeryfrontCloudBillingGroupUsed, runWithVeryfrontCloudContext, runWithVeryfrontCloudContextAsync, } from "./veryfront-cloud/context.js";
|
|
16
|
+
export { getVeryfrontCloudBootstrap } from "../platform/cloud/resolver.js";
|
|
16
17
|
export { DEFAULT_VERYFRONT_CLOUD_MODEL_ID, findVeryfrontCloudModel, findVeryfrontCloudModelByModelId, getVeryfrontCloudProviderFromModelId, groupVeryfrontCloudModelsByProvider, normalizeVeryfrontCloudModelId, resolveHostedVeryfrontCloudModelId, resolveVeryfrontCloudGatewayModelId, resolveVeryfrontCloudModelId, resolveVeryfrontCloudModelThinking, resolveVeryfrontCloudThinkingProviderOptions, tryGetVeryfrontCloudProviderFromModelId, VERYFRONT_CLOUD_CHAT_MODELS, VERYFRONT_CLOUD_MODEL_PREFIX, } from "./veryfront-cloud/model-catalog.js";
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
export interface VeryfrontCloudContext {
|
|
3
3
|
apiBaseUrl?: string;
|
|
4
4
|
apiToken?: string;
|
|
5
|
+
billingGroupId?: string;
|
|
6
|
+
billingGroupUsed?: boolean;
|
|
5
7
|
projectSlug?: string;
|
|
6
8
|
serviceLayer?: string;
|
|
7
9
|
}
|
|
@@ -10,4 +12,5 @@ export declare function runWithVeryfrontCloudContext<T>(context: VeryfrontCloudC
|
|
|
10
12
|
/** Run with Veryfront Cloud context async. */
|
|
11
13
|
export declare function runWithVeryfrontCloudContextAsync<T>(context: VeryfrontCloudContext, fn: () => Promise<T>): Promise<T>;
|
|
12
14
|
export declare function getCurrentVeryfrontCloudContext(): VeryfrontCloudContext | undefined;
|
|
15
|
+
export declare function markCurrentVeryfrontCloudBillingGroupUsed(): void;
|
|
13
16
|
//# sourceMappingURL=context.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../../../src/src/provider/veryfront-cloud/context.ts"],"names":[],"mappings":"AAEA,mCAAmC;AACnC,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAID,4CAA4C;AAC5C,wBAAgB,4BAA4B,CAAC,CAAC,EAC5C,OAAO,EAAE,qBAAqB,EAC9B,EAAE,EAAE,MAAM,CAAC,GACV,CAAC,CAEH;AAED,8CAA8C;AAC9C,wBAAgB,iCAAiC,CAAC,CAAC,EACjD,OAAO,EAAE,qBAAqB,EAC9B,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACnB,OAAO,CAAC,CAAC,CAAC,CAEZ;AAED,wBAAgB,+BAA+B,IAAI,qBAAqB,GAAG,SAAS,CAEnF"}
|
|
1
|
+
{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../../../src/src/provider/veryfront-cloud/context.ts"],"names":[],"mappings":"AAEA,mCAAmC;AACnC,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAID,4CAA4C;AAC5C,wBAAgB,4BAA4B,CAAC,CAAC,EAC5C,OAAO,EAAE,qBAAqB,EAC9B,EAAE,EAAE,MAAM,CAAC,GACV,CAAC,CAEH;AAED,8CAA8C;AAC9C,wBAAgB,iCAAiC,CAAC,CAAC,EACjD,OAAO,EAAE,qBAAqB,EAC9B,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACnB,OAAO,CAAC,CAAC,CAAC,CAEZ;AAED,wBAAgB,+BAA+B,IAAI,qBAAqB,GAAG,SAAS,CAEnF;AAED,wBAAgB,yCAAyC,IAAI,IAAI,CAKhE"}
|
|
@@ -11,3 +11,9 @@ export function runWithVeryfrontCloudContextAsync(context, fn) {
|
|
|
11
11
|
export function getCurrentVeryfrontCloudContext() {
|
|
12
12
|
return veryfrontCloudContextStorage.getStore();
|
|
13
13
|
}
|
|
14
|
+
export function markCurrentVeryfrontCloudBillingGroupUsed() {
|
|
15
|
+
const context = veryfrontCloudContextStorage.getStore();
|
|
16
|
+
if (context?.billingGroupId) {
|
|
17
|
+
context.billingGroupUsed = true;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../../../src/src/provider/veryfront-cloud/shared.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../../../src/src/provider/veryfront-cloud/shared.ts"],"names":[],"mappings":"AAQA,2DAA2D;AAC3D,MAAM,MAAM,wBAAwB,GAChC,WAAW,GACX,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,YAAY,CAAC;AAEjB,UAAU,2BAA2B;IACnC,QAAQ,EAAE,wBAAwB,CAAC;IACnC,OAAO,EAAE,MAAM,CAAC;CACjB;AAiCD,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,UAAU,GAAG,WAAW,GAC7B,2BAA2B,CA2C7B;AAED,wBAAgB,8BAA8B,IAAI;IAChD,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAmBA;AAED,wBAAgB,+BAA+B,CAC7C,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,wBAAwB,GACjC,MAAM,CAER;AAED;;;;;;;;GAQG;AACH,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,MAAM,EAChB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,KAAK,CAqBd"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createError, toError } from "../../errors/veryfront-error.js";
|
|
2
2
|
import { getVeryfrontCloudBootstrap } from "../../platform/cloud/resolver.js";
|
|
3
|
+
import { getCurrentVeryfrontCloudContext, markCurrentVeryfrontCloudBillingGroupUsed, } from "./context.js";
|
|
3
4
|
import { isSupportedMistralModelId } from "./model-catalog.js";
|
|
4
5
|
const PROVIDER_ALIASES = {
|
|
5
6
|
anthropic: "anthropic",
|
|
@@ -94,6 +95,11 @@ export function createVeryfrontCloudFetch(apiToken, projectSlug) {
|
|
|
94
95
|
if (projectSlug) {
|
|
95
96
|
headers.set("x-veryfront-project-slug", projectSlug);
|
|
96
97
|
}
|
|
98
|
+
const billingGroupId = getCurrentVeryfrontCloudContext()?.billingGroupId?.trim();
|
|
99
|
+
if (billingGroupId) {
|
|
100
|
+
headers.set("x-veryfront-billing-group-id", billingGroupId);
|
|
101
|
+
markCurrentVeryfrontCloudBillingGroupUsed();
|
|
102
|
+
}
|
|
97
103
|
return fetch(new Request(request, { headers }));
|
|
98
104
|
};
|
|
99
105
|
}
|