veryfront 0.1.971 → 0.1.973
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/cli/commands/eval/command.d.ts +13 -1
- package/esm/cli/commands/eval/command.d.ts.map +1 -1
- package/esm/cli/commands/eval/command.js +128 -13
- package/esm/deno.js +1 -1
- package/esm/src/eval/index.d.ts +2 -1
- package/esm/src/eval/index.d.ts.map +1 -1
- package/esm/src/eval/index.js +2 -1
- package/esm/src/eval/run-id.d.ts +3 -0
- package/esm/src/eval/run-id.d.ts.map +1 -0
- package/esm/src/eval/run-id.js +11 -0
- package/esm/src/eval/runner.d.ts +4 -1
- package/esm/src/eval/runner.d.ts.map +1 -1
- package/esm/src/eval/runner.js +3 -5
- package/esm/src/platform/cloud/resolver.d.ts.map +1 -1
- package/esm/src/platform/cloud/resolver.js +5 -2
- package/esm/src/provider/index.d.ts +3 -1
- package/esm/src/provider/index.d.ts.map +1 -1
- package/esm/src/provider/index.js +2 -1
- package/esm/src/provider/veryfront-cloud/context.d.ts +3 -0
- package/esm/src/provider/veryfront-cloud/context.d.ts.map +1 -1
- package/esm/src/provider/veryfront-cloud/context.js +6 -0
- package/esm/src/provider/veryfront-cloud/shared.d.ts.map +1 -1
- package/esm/src/provider/veryfront-cloud/shared.js +6 -0
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { AgentResponse } from "../../../src/agent/index.js";
|
|
2
2
|
import type { VeryfrontConfig } from "../../../src/config/index.js";
|
|
3
|
-
import type { DiscoveredEval, EvalModelComparison, EvalModelComparisonOptions, EvalReport, EvalReportComparison, EvalToolCall } from "../../../src/eval/index.js";
|
|
3
|
+
import type { DiscoveredEval, EvalModelComparison, EvalModelComparisonOptions, EvalReport, EvalReportComparison, EvalReportExportConfig, EvalToolCall } from "../../../src/eval/index.js";
|
|
4
4
|
import type { EvalArgs } from "./handler.js";
|
|
5
5
|
export interface EvalOptions extends EvalArgs {
|
|
6
6
|
}
|
|
@@ -36,6 +36,15 @@ type EvalSummaryArtifact = {
|
|
|
36
36
|
exports?: EvalReport["exports"];
|
|
37
37
|
baseline?: EvalReportComparison;
|
|
38
38
|
};
|
|
39
|
+
type GatewayBillingGroupFinalization = {
|
|
40
|
+
billing_group_id: string;
|
|
41
|
+
charged_credits: number;
|
|
42
|
+
target_credits: number;
|
|
43
|
+
adjustment_credits: number;
|
|
44
|
+
provider_cost_usd: number;
|
|
45
|
+
veryfront_charge_usd: number;
|
|
46
|
+
veryfront_billed_usd: number;
|
|
47
|
+
};
|
|
39
48
|
type EvalModelComparisonPolicy = Omit<EvalModelComparisonOptions, "baselineModel">;
|
|
40
49
|
export declare function createDefaultEvalReportDir(runId: string, label?: string): string;
|
|
41
50
|
export declare function createEvalArtifactPaths(reportDir: string): EvalArtifactPaths;
|
|
@@ -45,6 +54,9 @@ export declare function findEvalForCliId(evals: DiscoveredEval[], id: string): D
|
|
|
45
54
|
export declare function normalizeEvalInputForAgent(input: unknown): string;
|
|
46
55
|
export declare function summarizeReportForCli(report: EvalReport): CliEvalSummary;
|
|
47
56
|
export declare function createSummaryArtifact(report: EvalReport, baseline?: EvalReportComparison): EvalSummaryArtifact;
|
|
57
|
+
export declare function applyGatewayBillingGroupFinalization(report: EvalReport, finalization: GatewayBillingGroupFinalization): EvalReport;
|
|
58
|
+
export declare function runEvalWithGatewayBillingGroup(billingGroupId: string, operation: () => Promise<EvalReport>): Promise<EvalReport>;
|
|
59
|
+
export declare function exportEvalReportForCli(report: EvalReport, config?: EvalReportExportConfig): Promise<EvalReport>;
|
|
48
60
|
export declare function loadEvalModelComparisonPolicy(projectDir: string, policyPath?: string): Promise<EvalModelComparisonPolicy>;
|
|
49
61
|
export declare function createResolvedEvalModelComparisonConfig(projectDir: string, options: EvalOptions): Promise<ResolvedEvalModelComparisonConfig | undefined>;
|
|
50
62
|
export declare function createEvalModelComparisonArtifact(reports: EvalReport[], baselineModel: string, policy?: EvalModelComparisonPolicy): EvalModelComparison;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,
|
|
1
|
+
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EACpB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AA0BpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,+BAA+B,GAAG;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AAyCnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAO5E;AAMD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CASxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAcrB;AAmDD,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,+BAA+B,GAC5C,UAAU,CAiBZ;AAsDD,wBAAsB,8BAA8B,CAClD,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,GACnC,OAAO,CAAC,UAAU,CAAC,CAerB;AAED,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,UAAU,CAAC,CAGrB;AA8FD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAkED,uEAAuE;AACvE,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,MAAM,CAkFR;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,MA0DrD;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AA+FD,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AA8LF,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAsJrE"}
|
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
import * as dntShim from "../../../_dnt.shims.js";
|
|
5
5
|
import { dirname, isAbsolute, join, relative, resolve } from "../../../deps/jsr.io/@std/path/1.1.4/mod.js";
|
|
6
6
|
import { createProjectDiscoveryConfig, discoverAll } from "../../../src/discovery/index.js";
|
|
7
|
-
import { compareEvalModelReports, compareEvalReports, createEvalModelComparisonMarkdown, discoverEvals, resolveEvalRunProvenance, runEval, } from "../../../src/eval/index.js";
|
|
7
|
+
import { compareEvalModelReports, compareEvalReports, createEvalModelComparisonMarkdown, createEvalRunId, discoverEvals, exportEvalReport, resolveEvalRunProvenance, runEval, } from "../../../src/eval/index.js";
|
|
8
|
+
import { getCurrentVeryfrontCloudContext, getVeryfrontCloudBootstrap, runWithVeryfrontCloudContextAsync, } from "../../../src/provider/index.js";
|
|
8
9
|
import { applyRuntimeAuthContext } from "../../shared/runtime-auth.js";
|
|
9
10
|
import { cliLogger, exitProcess, VERSION } from "../../utils/index.js";
|
|
10
11
|
import { createErrorEnvelope, createSuccessEnvelope, isJsonMode, outputJson, } from "../../shared/json-output.js";
|
|
@@ -33,9 +34,6 @@ function stripFileProtocol(path) {
|
|
|
33
34
|
return path;
|
|
34
35
|
return decodeURIComponent(new URL(path).pathname);
|
|
35
36
|
}
|
|
36
|
-
function createCliRunId(now = new Date()) {
|
|
37
|
-
return `evalrun_${now.toISOString().replace(/[-:.]/g, "").replace("T", "_").replace("Z", "")}`;
|
|
38
|
-
}
|
|
39
37
|
function createEvalReportDirTimestamp(runId) {
|
|
40
38
|
return runId.startsWith("evalrun_") ? runId.slice("evalrun_".length) : runId;
|
|
41
39
|
}
|
|
@@ -160,6 +158,117 @@ export function createSummaryArtifact(report, baseline) {
|
|
|
160
158
|
function isRecord(value) {
|
|
161
159
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
162
160
|
}
|
|
161
|
+
function joinApiUrl(baseUrl, path) {
|
|
162
|
+
return `${baseUrl.replace(/\/+$/, "")}/${path.replace(/^\/+/, "")}`;
|
|
163
|
+
}
|
|
164
|
+
function readFiniteNumber(record, field) {
|
|
165
|
+
const value = record[field];
|
|
166
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
167
|
+
}
|
|
168
|
+
function parseGatewayBillingGroupFinalization(payload) {
|
|
169
|
+
if (!isRecord(payload) || typeof payload.billing_group_id !== "string")
|
|
170
|
+
return undefined;
|
|
171
|
+
const chargedCredits = readFiniteNumber(payload, "charged_credits");
|
|
172
|
+
const targetCredits = readFiniteNumber(payload, "target_credits");
|
|
173
|
+
const adjustmentCredits = readFiniteNumber(payload, "adjustment_credits");
|
|
174
|
+
const providerCostUsd = readFiniteNumber(payload, "provider_cost_usd");
|
|
175
|
+
const veryfrontChargeUsd = readFiniteNumber(payload, "veryfront_charge_usd");
|
|
176
|
+
const veryfrontBilledUsd = readFiniteNumber(payload, "veryfront_billed_usd");
|
|
177
|
+
if (chargedCredits === undefined ||
|
|
178
|
+
targetCredits === undefined ||
|
|
179
|
+
adjustmentCredits === undefined ||
|
|
180
|
+
providerCostUsd === undefined ||
|
|
181
|
+
veryfrontChargeUsd === undefined ||
|
|
182
|
+
veryfrontBilledUsd === undefined) {
|
|
183
|
+
return undefined;
|
|
184
|
+
}
|
|
185
|
+
return {
|
|
186
|
+
billing_group_id: payload.billing_group_id,
|
|
187
|
+
charged_credits: chargedCredits,
|
|
188
|
+
target_credits: targetCredits,
|
|
189
|
+
adjustment_credits: adjustmentCredits,
|
|
190
|
+
provider_cost_usd: providerCostUsd,
|
|
191
|
+
veryfront_charge_usd: veryfrontChargeUsd,
|
|
192
|
+
veryfront_billed_usd: veryfrontBilledUsd,
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
export function applyGatewayBillingGroupFinalization(report, finalization) {
|
|
196
|
+
return {
|
|
197
|
+
...report,
|
|
198
|
+
summary: {
|
|
199
|
+
...report.summary,
|
|
200
|
+
usage: {
|
|
201
|
+
...(report.summary.usage ?? {}),
|
|
202
|
+
providerCostUsd: finalization.provider_cost_usd,
|
|
203
|
+
veryfrontChargeUsd: finalization.veryfront_charge_usd,
|
|
204
|
+
veryfrontBilledUsd: finalization.veryfront_billed_usd,
|
|
205
|
+
costCredits: finalization.target_credits,
|
|
206
|
+
costSource: "gateway",
|
|
207
|
+
billingMode: "direct",
|
|
208
|
+
usageCaptureStatus: "complete",
|
|
209
|
+
},
|
|
210
|
+
},
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
function hasGatewayUsage(report) {
|
|
214
|
+
const usage = report.summary.usage;
|
|
215
|
+
return Boolean(usage?.costSource === "gateway" ||
|
|
216
|
+
usage?.veryfrontChargeUsd !== undefined ||
|
|
217
|
+
usage?.veryfrontBilledUsd !== undefined);
|
|
218
|
+
}
|
|
219
|
+
async function finalizeGatewayBillingGroup(billingGroupId) {
|
|
220
|
+
const bootstrap = getVeryfrontCloudBootstrap();
|
|
221
|
+
if (!bootstrap.apiToken)
|
|
222
|
+
return undefined;
|
|
223
|
+
let response;
|
|
224
|
+
try {
|
|
225
|
+
response = await fetch(joinApiUrl(bootstrap.apiBaseUrl, "ai/gateway/billing/finalize"), {
|
|
226
|
+
method: "POST",
|
|
227
|
+
headers: {
|
|
228
|
+
Authorization: `Bearer ${bootstrap.apiToken}`,
|
|
229
|
+
"Content-Type": "application/json",
|
|
230
|
+
...(bootstrap.projectSlug ? { "x-veryfront-project-slug": bootstrap.projectSlug } : {}),
|
|
231
|
+
},
|
|
232
|
+
body: JSON.stringify({ billing_group_id: billingGroupId }),
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
catch (error) {
|
|
236
|
+
cliLogger.warn(`Gateway billing finalization skipped for ${billingGroupId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
237
|
+
return undefined;
|
|
238
|
+
}
|
|
239
|
+
if (!response.ok) {
|
|
240
|
+
const message = await response.text().catch(() => "");
|
|
241
|
+
cliLogger.warn(`Gateway billing finalization skipped for ${billingGroupId}: ${response.status}${message ? ` ${message}` : ""}`);
|
|
242
|
+
return undefined;
|
|
243
|
+
}
|
|
244
|
+
const finalization = parseGatewayBillingGroupFinalization(await response.json());
|
|
245
|
+
if (!finalization) {
|
|
246
|
+
cliLogger.warn(`Gateway billing finalization skipped for ${billingGroupId}: invalid response`);
|
|
247
|
+
}
|
|
248
|
+
return finalization;
|
|
249
|
+
}
|
|
250
|
+
export async function runEvalWithGatewayBillingGroup(billingGroupId, operation) {
|
|
251
|
+
const currentContext = getCurrentVeryfrontCloudContext();
|
|
252
|
+
const billingContext = { ...(currentContext ?? {}), billingGroupId };
|
|
253
|
+
let report;
|
|
254
|
+
try {
|
|
255
|
+
report = await runWithVeryfrontCloudContextAsync(billingContext, operation);
|
|
256
|
+
}
|
|
257
|
+
catch (error) {
|
|
258
|
+
if (billingContext.billingGroupUsed) {
|
|
259
|
+
await finalizeGatewayBillingGroup(billingGroupId);
|
|
260
|
+
}
|
|
261
|
+
throw error;
|
|
262
|
+
}
|
|
263
|
+
if (!billingContext.billingGroupUsed && !hasGatewayUsage(report))
|
|
264
|
+
return report;
|
|
265
|
+
const finalization = await finalizeGatewayBillingGroup(billingGroupId);
|
|
266
|
+
return finalization ? applyGatewayBillingGroupFinalization(report, finalization) : report;
|
|
267
|
+
}
|
|
268
|
+
export async function exportEvalReportForCli(report, config) {
|
|
269
|
+
const exports = await exportEvalReport(report, config);
|
|
270
|
+
return exports === undefined ? report : { ...report, exports };
|
|
271
|
+
}
|
|
163
272
|
function readNumberField(record, field, path) {
|
|
164
273
|
const value = record[field];
|
|
165
274
|
if (value === undefined)
|
|
@@ -639,7 +748,7 @@ async function writeEvalModelComparisonArtifacts(comparison, paths) {
|
|
|
639
748
|
await dntShim.Deno.writeTextFile(paths.comparisonMarkdown, createEvalModelComparisonMarkdown(comparison));
|
|
640
749
|
}
|
|
641
750
|
async function runEvalModelComparison(input) {
|
|
642
|
-
const runId =
|
|
751
|
+
const runId = createEvalRunId();
|
|
643
752
|
const reportDir = input.options.reportDir ??
|
|
644
753
|
createDefaultEvalReportDir(runId, input.evalItem.id);
|
|
645
754
|
const paths = createEvalModelComparisonArtifactPaths(reportDir, input.config.models);
|
|
@@ -651,9 +760,11 @@ async function runEvalModelComparison(input) {
|
|
|
651
760
|
for (const model of input.config.models) {
|
|
652
761
|
const modelPaths = paths.models[model];
|
|
653
762
|
const modelOptions = { ...input.options, model, report: undefined };
|
|
654
|
-
const
|
|
763
|
+
const modelRunId = `${runId}_${sanitizeModelIdForPath(model)}`;
|
|
764
|
+
const exportConfig = createEvalCliExportConfig(input.evalItem, modelOptions, input.projectDir, modelPaths);
|
|
765
|
+
const finalizedReport = await runEvalWithGatewayBillingGroup(modelRunId, () => runEval(input.evalItem.definition, {
|
|
655
766
|
baseDir: input.options.datasetBase ?? input.projectDir,
|
|
656
|
-
runId:
|
|
767
|
+
runId: modelRunId,
|
|
657
768
|
adapters: {
|
|
658
769
|
agent: createAgentAdapterForModel(input.agent, input.options, model),
|
|
659
770
|
},
|
|
@@ -661,8 +772,8 @@ async function runEvalModelComparison(input) {
|
|
|
661
772
|
model,
|
|
662
773
|
provenance,
|
|
663
774
|
},
|
|
664
|
-
|
|
665
|
-
|
|
775
|
+
}));
|
|
776
|
+
const report = await exportEvalReportForCli(finalizedReport, exportConfig);
|
|
666
777
|
reports.push(report);
|
|
667
778
|
await writeEvalArtifacts(report, modelPaths);
|
|
668
779
|
await writeTextFileEnsuringDir(modelPaths.junit, createJunitXml(report));
|
|
@@ -823,13 +934,17 @@ export async function evalCommand(options) {
|
|
|
823
934
|
});
|
|
824
935
|
return;
|
|
825
936
|
}
|
|
826
|
-
const runId =
|
|
937
|
+
const runId = createEvalRunId();
|
|
827
938
|
const artifactPaths = createEvalArtifactPaths(options.reportDir ?? createDefaultEvalReportDir(runId, evalItem.id));
|
|
828
939
|
const provenance = await resolveEvalRunProvenance({
|
|
829
940
|
projectDir,
|
|
830
941
|
frameworkVersion: VERSION,
|
|
831
942
|
});
|
|
832
|
-
const
|
|
943
|
+
const billingGroupId = options.model
|
|
944
|
+
? `${runId}_${sanitizeModelIdForPath(options.model)}`
|
|
945
|
+
: runId;
|
|
946
|
+
const exportConfig = createEvalCliExportConfig(evalItem, options, projectDir, artifactPaths);
|
|
947
|
+
const finalizedReport = await runEvalWithGatewayBillingGroup(billingGroupId, () => runEval(evalItem.definition, {
|
|
833
948
|
baseDir: options.datasetBase ?? projectDir,
|
|
834
949
|
runId,
|
|
835
950
|
adapters: {
|
|
@@ -839,8 +954,8 @@ export async function evalCommand(options) {
|
|
|
839
954
|
provenance,
|
|
840
955
|
...(options.model ? { model: options.model } : {}),
|
|
841
956
|
},
|
|
842
|
-
|
|
843
|
-
|
|
957
|
+
}));
|
|
958
|
+
const report = await exportEvalReportForCli(finalizedReport, exportConfig);
|
|
844
959
|
const baseline = options.baseline
|
|
845
960
|
? compareEvalReports(report, await readEvalReport(options.baseline))
|
|
846
961
|
: undefined;
|
package/esm/deno.js
CHANGED
package/esm/src/eval/index.d.ts
CHANGED
|
@@ -51,7 +51,8 @@ export { createEvalReport, summarizeEvalRecords } from "./report.js";
|
|
|
51
51
|
export { compareEvalReports } from "./baseline.js";
|
|
52
52
|
export { compareEvalModelReports, createEvalModelComparisonMarkdown } from "./model-comparison.js";
|
|
53
53
|
export { createEvalRunProvenance, resolveEvalRunProvenance } from "./provenance.js";
|
|
54
|
-
export {
|
|
54
|
+
export { createEvalRunId } from "./run-id.js";
|
|
55
|
+
export { exportEvalReport, runEval } from "./runner.js";
|
|
55
56
|
export { deriveEvalId, discoverEvals, findEvalById } from "./discovery.js";
|
|
56
57
|
export { createEvalSourceDocument, getEvalEditableFieldSchema, getEvalRunSchema, getEvalSourceDocumentSchema, getEvalSourcePatchSchema, getEvalSourceReferenceSchema, getEvalStudioCapabilitySchema, } from "./studio.js";
|
|
57
58
|
export type { DiscoveredEval, EvalDiscoveryOptions, EvalDiscoveryResult } from "./discovery.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,iCAAiC,EAAE,MAAM,uBAAuB,CAAC;AACnG,OAAO,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AACpF,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,iCAAiC,EAAE,MAAM,uBAAuB,CAAC;AACnG,OAAO,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AACpF,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EACL,wBAAwB,EACxB,0BAA0B,EAC1B,gBAAgB,EAChB,2BAA2B,EAC3B,wBAAwB,EACxB,4BAA4B,EAC5B,6BAA6B,GAC9B,MAAM,aAAa,CAAC;AAErB,YAAY,EAAE,cAAc,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAChG,YAAY,EAAE,+BAA+B,EAAE,MAAM,aAAa,CAAC;AAEnE,YAAY,EACV,gBAAgB,EAChB,uBAAuB,EACvB,sBAAsB,EACtB,cAAc,EACd,mCAAmC,EACnC,gBAAgB,EAChB,WAAW,EACX,sBAAsB,EACtB,cAAc,EACd,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,eAAe,EACf,wBAAwB,EACxB,gBAAgB,EAChB,sBAAsB,EACtB,2BAA2B,EAC3B,6BAA6B,EAC7B,mCAAmC,EACnC,UAAU,EACV,iBAAiB,EACjB,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,4BAA4B,EAC5B,mBAAmB,EACnB,6BAA6B,EAC7B,2BAA2B,EAC3B,6BAA6B,EAC7B,4BAA4B,EAC5B,0BAA0B,EAC1B,sBAAsB,EACtB,UAAU,EACV,UAAU,EACV,oBAAoB,EACpB,sBAAsB,EACtB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,EACZ,UAAU,EACV,cAAc,EACd,YAAY,EACZ,wBAAwB,EACxB,wBAAwB,EACxB,kBAAkB,EAClB,sBAAsB,EACtB,SAAS,EACT,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,+BAA+B,EAC/B,iBAAiB,EACjB,OAAO,EACP,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,aAAa,CAAC"}
|
package/esm/src/eval/index.js
CHANGED
|
@@ -51,6 +51,7 @@ export { createEvalReport, summarizeEvalRecords } from "./report.js";
|
|
|
51
51
|
export { compareEvalReports } from "./baseline.js";
|
|
52
52
|
export { compareEvalModelReports, createEvalModelComparisonMarkdown } from "./model-comparison.js";
|
|
53
53
|
export { createEvalRunProvenance, resolveEvalRunProvenance } from "./provenance.js";
|
|
54
|
-
export {
|
|
54
|
+
export { createEvalRunId } from "./run-id.js";
|
|
55
|
+
export { exportEvalReport, runEval } from "./runner.js";
|
|
55
56
|
export { deriveEvalId, discoverEvals, findEvalById } from "./discovery.js";
|
|
56
57
|
export { createEvalSourceDocument, getEvalEditableFieldSchema, getEvalRunSchema, getEvalSourceDocumentSchema, getEvalSourcePatchSchema, getEvalSourceReferenceSchema, getEvalStudioCapabilitySchema, } from "./studio.js";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-id.d.ts","sourceRoot":"","sources":["../../../src/src/eval/run-id.ts"],"names":[],"mappings":"AAOA,iFAAiF;AACjF,wBAAgB,eAAe,CAC7B,GAAG,OAAa,EAChB,YAAY,GAAE,MAAM,MAA8B,GACjD,MAAM,CAGR"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import * as dntShim from "../../_dnt.shims.js";
|
|
2
|
+
function createEvalRunIdSuffix() {
|
|
3
|
+
const bytes = new Uint8Array(8);
|
|
4
|
+
dntShim.crypto.getRandomValues(bytes);
|
|
5
|
+
return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
6
|
+
}
|
|
7
|
+
/** Create a timestamp-sortable eval run id with a collision-resistant suffix. */
|
|
8
|
+
export function createEvalRunId(now = new Date(), createSuffix = createEvalRunIdSuffix) {
|
|
9
|
+
const timestamp = now.toISOString().replace(/[-:.]/g, "").replace("T", "_").replace("Z", "");
|
|
10
|
+
return `evalrun_${timestamp}_${createSuffix()}`;
|
|
11
|
+
}
|
package/esm/src/eval/runner.d.ts
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { createEvalReport } from "./report.js";
|
|
2
|
+
import { type EvalReportExportResult } from "../extensions/eval/index.js";
|
|
3
|
+
import type { EvalDefinition, EvalReportExportConfig, RunEvalOptions } from "./types.js";
|
|
4
|
+
export declare function exportEvalReport(report: ReturnType<typeof createEvalReport>, config?: EvalReportExportConfig): Promise<EvalReportExportResult[] | undefined>;
|
|
2
5
|
/** Execute an eval locally with injected target adapters. */
|
|
3
6
|
export declare function runEval(definition: EvalDefinition, options: RunEvalOptions): Promise<import("./types.js").EvalReport>;
|
|
4
7
|
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/src/eval/runner.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/src/eval/runner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAG/C,OAAO,EAKL,KAAK,sBAAsB,EAE5B,MAAM,6BAA6B,CAAC;AAGrC,OAAO,KAAK,EAEV,cAAc,EAGd,sBAAsB,EAGtB,cAAc,EACf,MAAM,YAAY,CAAC;AA2KpB,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,EAC3C,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,sBAAsB,EAAE,GAAG,SAAS,CAAC,CAgB/C;AAiED,6DAA6D;AAC7D,wBAAsB,OAAO,CAC3B,UAAU,EAAE,cAAc,EAC1B,OAAO,EAAE,cAAc,4CAyBxB"}
|
package/esm/src/eval/runner.js
CHANGED
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
import * as dntShim from "../../_dnt.shims.js";
|
|
2
2
|
import { createEvalCheckContext } from "./expect.js";
|
|
3
3
|
import { createEvalReport } from "./report.js";
|
|
4
|
+
import { createEvalRunId } from "./run-id.js";
|
|
4
5
|
import { metrics as runtimeMetrics } from "../metrics/index.js";
|
|
5
6
|
import { createEvalReportExporterRegistry, EvalReportExporterRegistryName, } from "../extensions/eval/index.js";
|
|
6
7
|
import { trace } from "../observability/tracing/api-shim.js";
|
|
7
8
|
import { tryResolve } from "../extensions/contracts.js";
|
|
8
|
-
function createRunId(now) {
|
|
9
|
-
return `evalrun_${now.toISOString().replace(/[-:.]/g, "").replace("T", "_").replace("Z", "")}`;
|
|
10
|
-
}
|
|
11
9
|
function normalizeTrace(trace) {
|
|
12
10
|
return {
|
|
13
11
|
events: trace?.events ?? [],
|
|
@@ -154,7 +152,7 @@ async function exportWithSelectedExporter(registry, report, config, exporterId)
|
|
|
154
152
|
error: `EvalReportExporter "${exporterId}" did not return an export result.`,
|
|
155
153
|
};
|
|
156
154
|
}
|
|
157
|
-
async function exportEvalReport(report, config) {
|
|
155
|
+
export async function exportEvalReport(report, config) {
|
|
158
156
|
if (!config)
|
|
159
157
|
return undefined;
|
|
160
158
|
const exporterIds = config.exporterIds?.filter((id) => id.length > 0) ?? [];
|
|
@@ -235,7 +233,7 @@ export async function runEval(definition, options) {
|
|
|
235
233
|
const report = createEvalReport({
|
|
236
234
|
definition,
|
|
237
235
|
records,
|
|
238
|
-
runId: options.runId ??
|
|
236
|
+
runId: options.runId ?? createEvalRunId(startedAt),
|
|
239
237
|
startedAt,
|
|
240
238
|
endedAt,
|
|
241
239
|
metadata: options.metadata,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../../../src/src/platform/cloud/resolver.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../../../src/src/platform/cloud/resolver.ts"],"names":[],"mappings":"AAuCA,eAAO,MAAM,6BAA6B,gDAAgD,CAAC;AAC3F,eAAO,MAAM,uCAAuC,kDACH,CAAC;AAElD,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,eAAe,EAAE,OAAO,CAAC;CAC1B;AAyDD,wBAAgB,0BAA0B,IAAI,MAAM,GAAG,SAAS,CAE/D;AAED,wBAAgB,4BAA4B,IAAI,MAAM,GAAG,SAAS,CAEjE;AAED,wBAAgB,0BAA0B,IAAI,uBAAuB,CAOpE;AAED,wBAAgB,uBAAuB,IAAI,OAAO,CAgBjD;AAED,wBAAgB,6BAA6B,IAAI,MAAM,CAKtD;AAED,wBAAgB,sCAAsC,IAAI,MAAM,CAK/D"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as dntShim from "../../../_dnt.shims.js";
|
|
2
2
|
import { getCurrentRequestContext } from "../adapters/fs/veryfront/request-context.js";
|
|
3
3
|
import { getHostEnv } from "../compat/process.js";
|
|
4
|
-
import { getCurrentVeryfrontCloudContext } from "../../provider/veryfront-cloud/context.js";
|
|
4
|
+
import { getCurrentVeryfrontCloudContext, } from "../../provider/veryfront-cloud/context.js";
|
|
5
5
|
function getRuntimeConfig() {
|
|
6
6
|
const getter = dntShim.dntGlobalThis.__vfGetRuntimeConfig;
|
|
7
7
|
return getter?.() ?? {};
|
|
@@ -36,6 +36,9 @@ function normalizeServiceLayer(value) {
|
|
|
36
36
|
const normalized = value?.trim().toLowerCase();
|
|
37
37
|
return normalized?.length ? normalized : undefined;
|
|
38
38
|
}
|
|
39
|
+
function hasScopedRuntimeContext(context) {
|
|
40
|
+
return Boolean(context?.apiBaseUrl || context?.apiToken || context?.projectSlug || context?.serviceLayer);
|
|
41
|
+
}
|
|
39
42
|
function getResolvedVeryfrontCloudContext() {
|
|
40
43
|
const requestContext = getCurrentRequestContext();
|
|
41
44
|
const scopedContext = getCurrentVeryfrontCloudContext();
|
|
@@ -51,7 +54,7 @@ function getResolvedVeryfrontCloudContext() {
|
|
|
51
54
|
runtimeBootstrap.projectSlug,
|
|
52
55
|
serviceLayer: normalizeServiceLayer(scopedContext?.serviceLayer) ??
|
|
53
56
|
normalizeServiceLayer(getHostEnv("VERYFRONT_SERVICE_LAYER")),
|
|
54
|
-
hasRequestContext: requestContext !== null || scopedContext
|
|
57
|
+
hasRequestContext: requestContext !== null || hasScopedRuntimeContext(scopedContext),
|
|
55
58
|
usesVeryfrontFs: runtimeBootstrap.usesVeryfrontFs,
|
|
56
59
|
};
|
|
57
60
|
}
|
|
@@ -14,8 +14,10 @@ import "../../_dnt.polyfills.js";
|
|
|
14
14
|
export { clearModelProviders, ensureModelReady, getRegisteredModelProviders, hasModelProvider, registerModelProvider, resolveModel, } from "./model-registry.js";
|
|
15
15
|
export type { ModelProviderFactory } from "./model-registry.js";
|
|
16
16
|
export type { ModelRuntime } from "./types.js";
|
|
17
|
-
export { runWithVeryfrontCloudContext, runWithVeryfrontCloudContextAsync, } from "./veryfront-cloud/context.js";
|
|
17
|
+
export { getCurrentVeryfrontCloudContext, markCurrentVeryfrontCloudBillingGroupUsed, runWithVeryfrontCloudContext, runWithVeryfrontCloudContextAsync, } from "./veryfront-cloud/context.js";
|
|
18
18
|
export type { VeryfrontCloudContext } from "./veryfront-cloud/context.js";
|
|
19
|
+
export { getVeryfrontCloudBootstrap } from "../platform/cloud/resolver.js";
|
|
20
|
+
export type { VeryfrontCloudBootstrap } from "../platform/cloud/resolver.js";
|
|
19
21
|
export type { VeryfrontCloudProviderId } from "./veryfront-cloud/shared.js";
|
|
20
22
|
export { DEFAULT_VERYFRONT_CLOUD_MODEL_ID, findVeryfrontCloudModel, findVeryfrontCloudModelByModelId, getVeryfrontCloudProviderFromModelId, groupVeryfrontCloudModelsByProvider, normalizeVeryfrontCloudModelId, resolveHostedVeryfrontCloudModelId, resolveVeryfrontCloudGatewayModelId, resolveVeryfrontCloudModelId, resolveVeryfrontCloudModelThinking, resolveVeryfrontCloudThinkingProviderOptions, tryGetVeryfrontCloudProviderFromModelId, VERYFRONT_CLOUD_CHAT_MODELS, VERYFRONT_CLOUD_MODEL_PREFIX, } from "./veryfront-cloud/model-catalog.js";
|
|
21
23
|
export type { VeryfrontCloudChatModel, VeryfrontCloudModelThinkingConfig, } from "./veryfront-cloud/model-catalog.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/provider/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EACL,mBAAmB,EACnB,gBAAgB,EAChB,2BAA2B,EAC3B,gBAAgB,EAChB,qBAAqB,EACrB,YAAY,GACb,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAChE,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EACL,4BAA4B,EAC5B,iCAAiC,GAClC,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAC1E,YAAY,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AAC5E,OAAO,EACL,gCAAgC,EAChC,uBAAuB,EACvB,gCAAgC,EAChC,oCAAoC,EACpC,mCAAmC,EACnC,8BAA8B,EAC9B,kCAAkC,EAClC,mCAAmC,EACnC,4BAA4B,EAC5B,kCAAkC,EAClC,4CAA4C,EAC5C,uCAAuC,EACvC,2BAA2B,EAC3B,4BAA4B,GAC7B,MAAM,oCAAoC,CAAC;AAC5C,YAAY,EACV,uBAAuB,EACvB,iCAAiC,GAClC,MAAM,oCAAoC,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/provider/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EACL,mBAAmB,EACnB,gBAAgB,EAChB,2BAA2B,EAC3B,gBAAgB,EAChB,qBAAqB,EACrB,YAAY,GACb,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAChE,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EACL,+BAA+B,EAC/B,yCAAyC,EACzC,4BAA4B,EAC5B,iCAAiC,GAClC,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAC1E,OAAO,EAAE,0BAA0B,EAAE,MAAM,+BAA+B,CAAC;AAC3E,YAAY,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAC7E,YAAY,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AAC5E,OAAO,EACL,gCAAgC,EAChC,uBAAuB,EACvB,gCAAgC,EAChC,oCAAoC,EACpC,mCAAmC,EACnC,8BAA8B,EAC9B,kCAAkC,EAClC,mCAAmC,EACnC,4BAA4B,EAC5B,kCAAkC,EAClC,4CAA4C,EAC5C,uCAAuC,EACvC,2BAA2B,EAC3B,4BAA4B,GAC7B,MAAM,oCAAoC,CAAC;AAC5C,YAAY,EACV,uBAAuB,EACvB,iCAAiC,GAClC,MAAM,oCAAoC,CAAC"}
|
|
@@ -12,5 +12,6 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import "../../_dnt.polyfills.js";
|
|
14
14
|
export { clearModelProviders, ensureModelReady, getRegisteredModelProviders, hasModelProvider, registerModelProvider, resolveModel, } from "./model-registry.js";
|
|
15
|
-
export { runWithVeryfrontCloudContext, runWithVeryfrontCloudContextAsync, } from "./veryfront-cloud/context.js";
|
|
15
|
+
export { getCurrentVeryfrontCloudContext, markCurrentVeryfrontCloudBillingGroupUsed, runWithVeryfrontCloudContext, runWithVeryfrontCloudContextAsync, } from "./veryfront-cloud/context.js";
|
|
16
|
+
export { getVeryfrontCloudBootstrap } from "../platform/cloud/resolver.js";
|
|
16
17
|
export { DEFAULT_VERYFRONT_CLOUD_MODEL_ID, findVeryfrontCloudModel, findVeryfrontCloudModelByModelId, getVeryfrontCloudProviderFromModelId, groupVeryfrontCloudModelsByProvider, normalizeVeryfrontCloudModelId, resolveHostedVeryfrontCloudModelId, resolveVeryfrontCloudGatewayModelId, resolveVeryfrontCloudModelId, resolveVeryfrontCloudModelThinking, resolveVeryfrontCloudThinkingProviderOptions, tryGetVeryfrontCloudProviderFromModelId, VERYFRONT_CLOUD_CHAT_MODELS, VERYFRONT_CLOUD_MODEL_PREFIX, } from "./veryfront-cloud/model-catalog.js";
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
export interface VeryfrontCloudContext {
|
|
3
3
|
apiBaseUrl?: string;
|
|
4
4
|
apiToken?: string;
|
|
5
|
+
billingGroupId?: string;
|
|
6
|
+
billingGroupUsed?: boolean;
|
|
5
7
|
projectSlug?: string;
|
|
6
8
|
serviceLayer?: string;
|
|
7
9
|
}
|
|
@@ -10,4 +12,5 @@ export declare function runWithVeryfrontCloudContext<T>(context: VeryfrontCloudC
|
|
|
10
12
|
/** Run with Veryfront Cloud context async. */
|
|
11
13
|
export declare function runWithVeryfrontCloudContextAsync<T>(context: VeryfrontCloudContext, fn: () => Promise<T>): Promise<T>;
|
|
12
14
|
export declare function getCurrentVeryfrontCloudContext(): VeryfrontCloudContext | undefined;
|
|
15
|
+
export declare function markCurrentVeryfrontCloudBillingGroupUsed(): void;
|
|
13
16
|
//# sourceMappingURL=context.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../../../src/src/provider/veryfront-cloud/context.ts"],"names":[],"mappings":"AAEA,mCAAmC;AACnC,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAID,4CAA4C;AAC5C,wBAAgB,4BAA4B,CAAC,CAAC,EAC5C,OAAO,EAAE,qBAAqB,EAC9B,EAAE,EAAE,MAAM,CAAC,GACV,CAAC,CAEH;AAED,8CAA8C;AAC9C,wBAAgB,iCAAiC,CAAC,CAAC,EACjD,OAAO,EAAE,qBAAqB,EAC9B,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACnB,OAAO,CAAC,CAAC,CAAC,CAEZ;AAED,wBAAgB,+BAA+B,IAAI,qBAAqB,GAAG,SAAS,CAEnF"}
|
|
1
|
+
{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../../../src/src/provider/veryfront-cloud/context.ts"],"names":[],"mappings":"AAEA,mCAAmC;AACnC,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAID,4CAA4C;AAC5C,wBAAgB,4BAA4B,CAAC,CAAC,EAC5C,OAAO,EAAE,qBAAqB,EAC9B,EAAE,EAAE,MAAM,CAAC,GACV,CAAC,CAEH;AAED,8CAA8C;AAC9C,wBAAgB,iCAAiC,CAAC,CAAC,EACjD,OAAO,EAAE,qBAAqB,EAC9B,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACnB,OAAO,CAAC,CAAC,CAAC,CAEZ;AAED,wBAAgB,+BAA+B,IAAI,qBAAqB,GAAG,SAAS,CAEnF;AAED,wBAAgB,yCAAyC,IAAI,IAAI,CAKhE"}
|
|
@@ -11,3 +11,9 @@ export function runWithVeryfrontCloudContextAsync(context, fn) {
|
|
|
11
11
|
export function getCurrentVeryfrontCloudContext() {
|
|
12
12
|
return veryfrontCloudContextStorage.getStore();
|
|
13
13
|
}
|
|
14
|
+
export function markCurrentVeryfrontCloudBillingGroupUsed() {
|
|
15
|
+
const context = veryfrontCloudContextStorage.getStore();
|
|
16
|
+
if (context?.billingGroupId) {
|
|
17
|
+
context.billingGroupUsed = true;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../../../src/src/provider/veryfront-cloud/shared.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../../../src/src/provider/veryfront-cloud/shared.ts"],"names":[],"mappings":"AAQA,2DAA2D;AAC3D,MAAM,MAAM,wBAAwB,GAChC,WAAW,GACX,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,YAAY,CAAC;AAEjB,UAAU,2BAA2B;IACnC,QAAQ,EAAE,wBAAwB,CAAC;IACnC,OAAO,EAAE,MAAM,CAAC;CACjB;AAiCD,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,UAAU,GAAG,WAAW,GAC7B,2BAA2B,CA2C7B;AAED,wBAAgB,8BAA8B,IAAI;IAChD,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAmBA;AAED,wBAAgB,+BAA+B,CAC7C,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,wBAAwB,GACjC,MAAM,CAER;AAED;;;;;;;;GAQG;AACH,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,MAAM,EAChB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,KAAK,CAqBd"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createError, toError } from "../../errors/veryfront-error.js";
|
|
2
2
|
import { getVeryfrontCloudBootstrap } from "../../platform/cloud/resolver.js";
|
|
3
|
+
import { getCurrentVeryfrontCloudContext, markCurrentVeryfrontCloudBillingGroupUsed, } from "./context.js";
|
|
3
4
|
import { isSupportedMistralModelId } from "./model-catalog.js";
|
|
4
5
|
const PROVIDER_ALIASES = {
|
|
5
6
|
anthropic: "anthropic",
|
|
@@ -94,6 +95,11 @@ export function createVeryfrontCloudFetch(apiToken, projectSlug) {
|
|
|
94
95
|
if (projectSlug) {
|
|
95
96
|
headers.set("x-veryfront-project-slug", projectSlug);
|
|
96
97
|
}
|
|
98
|
+
const billingGroupId = getCurrentVeryfrontCloudContext()?.billingGroupId?.trim();
|
|
99
|
+
if (billingGroupId) {
|
|
100
|
+
headers.set("x-veryfront-billing-group-id", billingGroupId);
|
|
101
|
+
markCurrentVeryfrontCloudBillingGroupUsed();
|
|
102
|
+
}
|
|
97
103
|
return fetch(new Request(request, { headers }));
|
|
98
104
|
};
|
|
99
105
|
}
|