veryfront 0.1.972 → 0.1.974
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/cli/commands/eval/command.d.ts.map +1 -1
- package/esm/cli/commands/eval/command.js +3 -6
- package/esm/deno.js +1 -1
- package/esm/src/agent/runtime/provider-tool-compat.d.ts.map +1 -1
- package/esm/src/agent/runtime/provider-tool-compat.js +33 -2
- package/esm/src/eval/index.d.ts +1 -0
- package/esm/src/eval/index.d.ts.map +1 -1
- package/esm/src/eval/index.js +1 -0
- package/esm/src/eval/run-id.d.ts +3 -0
- package/esm/src/eval/run-id.d.ts.map +1 -0
- package/esm/src/eval/run-id.js +11 -0
- package/esm/src/eval/runner.d.ts.map +1 -1
- package/esm/src/eval/runner.js +2 -4
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EACpB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EACpB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AA0BpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,+BAA+B,GAAG;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AAyCnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAO5E;AAMD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CASxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAcrB;AAmDD,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,+BAA+B,GAC5C,UAAU,CAiBZ;AAsDD,wBAAsB,8BAA8B,CAClD,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,GACnC,OAAO,CAAC,UAAU,CAAC,CAerB;AAED,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,UAAU,CAAC,CAGrB;AA8FD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAkED,uEAAuE;AACvE,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,MAAM,CAkFR;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,MA0DrD;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AA+FD,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AA8LF,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAsJrE"}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import * as dntShim from "../../../_dnt.shims.js";
|
|
5
5
|
import { dirname, isAbsolute, join, relative, resolve } from "../../../deps/jsr.io/@std/path/1.1.4/mod.js";
|
|
6
6
|
import { createProjectDiscoveryConfig, discoverAll } from "../../../src/discovery/index.js";
|
|
7
|
-
import { compareEvalModelReports, compareEvalReports, createEvalModelComparisonMarkdown, discoverEvals, exportEvalReport, resolveEvalRunProvenance, runEval, } from "../../../src/eval/index.js";
|
|
7
|
+
import { compareEvalModelReports, compareEvalReports, createEvalModelComparisonMarkdown, createEvalRunId, discoverEvals, exportEvalReport, resolveEvalRunProvenance, runEval, } from "../../../src/eval/index.js";
|
|
8
8
|
import { getCurrentVeryfrontCloudContext, getVeryfrontCloudBootstrap, runWithVeryfrontCloudContextAsync, } from "../../../src/provider/index.js";
|
|
9
9
|
import { applyRuntimeAuthContext } from "../../shared/runtime-auth.js";
|
|
10
10
|
import { cliLogger, exitProcess, VERSION } from "../../utils/index.js";
|
|
@@ -34,9 +34,6 @@ function stripFileProtocol(path) {
|
|
|
34
34
|
return path;
|
|
35
35
|
return decodeURIComponent(new URL(path).pathname);
|
|
36
36
|
}
|
|
37
|
-
function createCliRunId(now = new Date()) {
|
|
38
|
-
return `evalrun_${now.toISOString().replace(/[-:.]/g, "").replace("T", "_").replace("Z", "")}`;
|
|
39
|
-
}
|
|
40
37
|
function createEvalReportDirTimestamp(runId) {
|
|
41
38
|
return runId.startsWith("evalrun_") ? runId.slice("evalrun_".length) : runId;
|
|
42
39
|
}
|
|
@@ -751,7 +748,7 @@ async function writeEvalModelComparisonArtifacts(comparison, paths) {
|
|
|
751
748
|
await dntShim.Deno.writeTextFile(paths.comparisonMarkdown, createEvalModelComparisonMarkdown(comparison));
|
|
752
749
|
}
|
|
753
750
|
async function runEvalModelComparison(input) {
|
|
754
|
-
const runId =
|
|
751
|
+
const runId = createEvalRunId();
|
|
755
752
|
const reportDir = input.options.reportDir ??
|
|
756
753
|
createDefaultEvalReportDir(runId, input.evalItem.id);
|
|
757
754
|
const paths = createEvalModelComparisonArtifactPaths(reportDir, input.config.models);
|
|
@@ -937,7 +934,7 @@ export async function evalCommand(options) {
|
|
|
937
934
|
});
|
|
938
935
|
return;
|
|
939
936
|
}
|
|
940
|
-
const runId =
|
|
937
|
+
const runId = createEvalRunId();
|
|
941
938
|
const artifactPaths = createEvalArtifactPaths(options.reportDir ?? createDefaultEvalReportDir(runId, evalItem.id));
|
|
942
939
|
const provenance = await resolveEvalRunProvenance({
|
|
943
940
|
projectDir,
|
package/esm/deno.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"provider-tool-compat.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/runtime/provider-tool-compat.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAE7D,6DAA6D;AAC7D,MAAM,MAAM,0BAA0B,GAClC,WAAW,GACX,QAAQ,GACR,UAAU,GACV,QAAQ,GACR,SAAS,CAAC;AAEd,qDAAqD;AACrD,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,0BAA0B,CAAC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,OAAO,CAAC;CACzB;AAED,gDAAgD;AAChD,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iBAAiB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACvC;AA2BD,oCAAoC;AACpC,wBAAgB,sBAAsB,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,mBAAmB,CAsB1E;AAeD,oDAAoD;AACpD,wBAAgB,iCAAiC,CAC/C,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,OAAO,GAAE,yBAA8B,GACtC,MAAM,EAAE,CAsBV;AAED,+CAA+C;AAC/C,wBAAgB,6BAA6B,CAC3C,KAAK,EAAE,SAAS,cAAc,EAAE,EAChC,OAAO,GAAE,yBAA8B,GACtC,cAAc,EAAE,CAclB;
|
|
1
|
+
{"version":3,"file":"provider-tool-compat.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/runtime/provider-tool-compat.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAE7D,6DAA6D;AAC7D,MAAM,MAAM,0BAA0B,GAClC,WAAW,GACX,QAAQ,GACR,UAAU,GACV,QAAQ,GACR,SAAS,CAAC;AAEd,qDAAqD;AACrD,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,0BAA0B,CAAC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,OAAO,CAAC;CACzB;AAED,gDAAgD;AAChD,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iBAAiB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACvC;AA2BD,oCAAoC;AACpC,wBAAgB,sBAAsB,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,mBAAmB,CAsB1E;AAeD,oDAAoD;AACpD,wBAAgB,iCAAiC,CAC/C,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,OAAO,GAAE,yBAA8B,GACtC,MAAM,EAAE,CAsBV;AAED,+CAA+C;AAC/C,wBAAgB,6BAA6B,CAC3C,KAAK,EAAE,SAAS,cAAc,EAAE,EAChC,OAAO,GAAE,yBAA8B,GACtC,cAAc,EAAE,CAclB;AA6MD;;;;GAIG;AACH,wBAAgB,gCAAgC,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,CAa/E;AAED,6CAA6C;AAC7C,wBAAgB,0BAA0B,CACxC,MAAM,EAAE,UAAU,EAClB,OAAO,GAAE,IAAI,CAAC,yBAAyB,EAAE,OAAO,CAAM,GACrD,UAAU,CAeZ"}
|
|
@@ -34,8 +34,8 @@ export function getProviderToolProfile(model) {
|
|
|
34
34
|
if (provider === "anthropic") {
|
|
35
35
|
return { provider: "anthropic", sanitizeSchema: true };
|
|
36
36
|
}
|
|
37
|
-
if (provider === "moonshot") {
|
|
38
|
-
return { provider: "moonshot", sanitizeSchema:
|
|
37
|
+
if (provider === "moonshot" || provider === "moonshotai") {
|
|
38
|
+
return { provider: "moonshot", sanitizeSchema: true };
|
|
39
39
|
}
|
|
40
40
|
return { provider: "unknown", sanitizeSchema: false };
|
|
41
41
|
}
|
|
@@ -227,6 +227,34 @@ function sanitizeGoogleSchemaValue(value) {
|
|
|
227
227
|
}
|
|
228
228
|
return sanitized;
|
|
229
229
|
}
|
|
230
|
+
function sanitizeMoonshotSchemaValue(value) {
|
|
231
|
+
if (Array.isArray(value)) {
|
|
232
|
+
return value.map((item) => sanitizeMoonshotSchemaValue(item));
|
|
233
|
+
}
|
|
234
|
+
if (!isPlainRecord(value)) {
|
|
235
|
+
return value;
|
|
236
|
+
}
|
|
237
|
+
const sanitized = {};
|
|
238
|
+
for (const [key, child] of Object.entries(value)) {
|
|
239
|
+
if (key === "$ref" && typeof child === "string") {
|
|
240
|
+
sanitized.$ref = child.replace(/^#\/definitions\//, "#/$defs/");
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
243
|
+
if (key === "definitions") {
|
|
244
|
+
sanitized.$defs = sanitizeMoonshotSchemaValue(child);
|
|
245
|
+
continue;
|
|
246
|
+
}
|
|
247
|
+
if (key === "properties" && isPlainRecord(child)) {
|
|
248
|
+
sanitized.properties = Object.fromEntries(Object.entries(child).map(([propertyName, propertySchema]) => [
|
|
249
|
+
propertyName,
|
|
250
|
+
sanitizeMoonshotSchemaValue(propertySchema),
|
|
251
|
+
]));
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
sanitized[key] = sanitizeMoonshotSchemaValue(child);
|
|
255
|
+
}
|
|
256
|
+
return sanitized;
|
|
257
|
+
}
|
|
230
258
|
/**
|
|
231
259
|
* Normalize a provider tool input schema so every function tool has a
|
|
232
260
|
* provider-safe JSON Schema object at the root. Remote/MCP tools can omit the
|
|
@@ -253,5 +281,8 @@ export function sanitizeProviderToolSchema(schema, options = {}) {
|
|
|
253
281
|
if (profile.provider === "google") {
|
|
254
282
|
return sanitizeGoogleSchemaValue(propertyKeySafeSchema);
|
|
255
283
|
}
|
|
284
|
+
if (profile.provider === "moonshot") {
|
|
285
|
+
return sanitizeMoonshotSchemaValue(propertyKeySafeSchema);
|
|
286
|
+
}
|
|
256
287
|
return propertyKeySafeSchema;
|
|
257
288
|
}
|
package/esm/src/eval/index.d.ts
CHANGED
|
@@ -51,6 +51,7 @@ export { createEvalReport, summarizeEvalRecords } from "./report.js";
|
|
|
51
51
|
export { compareEvalReports } from "./baseline.js";
|
|
52
52
|
export { compareEvalModelReports, createEvalModelComparisonMarkdown } from "./model-comparison.js";
|
|
53
53
|
export { createEvalRunProvenance, resolveEvalRunProvenance } from "./provenance.js";
|
|
54
|
+
export { createEvalRunId } from "./run-id.js";
|
|
54
55
|
export { exportEvalReport, runEval } from "./runner.js";
|
|
55
56
|
export { deriveEvalId, discoverEvals, findEvalById } from "./discovery.js";
|
|
56
57
|
export { createEvalSourceDocument, getEvalEditableFieldSchema, getEvalRunSchema, getEvalSourceDocumentSchema, getEvalSourcePatchSchema, getEvalSourceReferenceSchema, getEvalStudioCapabilitySchema, } from "./studio.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,iCAAiC,EAAE,MAAM,uBAAuB,CAAC;AACnG,OAAO,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AACpF,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EACL,wBAAwB,EACxB,0BAA0B,EAC1B,gBAAgB,EAChB,2BAA2B,EAC3B,wBAAwB,EACxB,4BAA4B,EAC5B,6BAA6B,GAC9B,MAAM,aAAa,CAAC;AAErB,YAAY,EAAE,cAAc,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAChG,YAAY,EAAE,+BAA+B,EAAE,MAAM,aAAa,CAAC;AAEnE,YAAY,EACV,gBAAgB,EAChB,uBAAuB,EACvB,sBAAsB,EACtB,cAAc,EACd,mCAAmC,EACnC,gBAAgB,EAChB,WAAW,EACX,sBAAsB,EACtB,cAAc,EACd,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,eAAe,EACf,wBAAwB,EACxB,gBAAgB,EAChB,sBAAsB,EACtB,2BAA2B,EAC3B,6BAA6B,EAC7B,mCAAmC,EACnC,UAAU,EACV,iBAAiB,EACjB,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,4BAA4B,EAC5B,mBAAmB,EACnB,6BAA6B,EAC7B,2BAA2B,EAC3B,6BAA6B,EAC7B,4BAA4B,EAC5B,0BAA0B,EAC1B,sBAAsB,EACtB,UAAU,EACV,UAAU,EACV,oBAAoB,EACpB,sBAAsB,EACtB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,EACZ,UAAU,EACV,cAAc,EACd,YAAY,EACZ,wBAAwB,EACxB,wBAAwB,EACxB,kBAAkB,EAClB,sBAAsB,EACtB,SAAS,EACT,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,+BAA+B,EAC/B,iBAAiB,EACjB,OAAO,EACP,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,aAAa,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,iCAAiC,EAAE,MAAM,uBAAuB,CAAC;AACnG,OAAO,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AACpF,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EACL,wBAAwB,EACxB,0BAA0B,EAC1B,gBAAgB,EAChB,2BAA2B,EAC3B,wBAAwB,EACxB,4BAA4B,EAC5B,6BAA6B,GAC9B,MAAM,aAAa,CAAC;AAErB,YAAY,EAAE,cAAc,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAChG,YAAY,EAAE,+BAA+B,EAAE,MAAM,aAAa,CAAC;AAEnE,YAAY,EACV,gBAAgB,EAChB,uBAAuB,EACvB,sBAAsB,EACtB,cAAc,EACd,mCAAmC,EACnC,gBAAgB,EAChB,WAAW,EACX,sBAAsB,EACtB,cAAc,EACd,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,eAAe,EACf,wBAAwB,EACxB,gBAAgB,EAChB,sBAAsB,EACtB,2BAA2B,EAC3B,6BAA6B,EAC7B,mCAAmC,EACnC,UAAU,EACV,iBAAiB,EACjB,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,4BAA4B,EAC5B,mBAAmB,EACnB,6BAA6B,EAC7B,2BAA2B,EAC3B,6BAA6B,EAC7B,4BAA4B,EAC5B,0BAA0B,EAC1B,sBAAsB,EACtB,UAAU,EACV,UAAU,EACV,oBAAoB,EACpB,sBAAsB,EACtB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,EACZ,UAAU,EACV,cAAc,EACd,YAAY,EACZ,wBAAwB,EACxB,wBAAwB,EACxB,kBAAkB,EAClB,sBAAsB,EACtB,SAAS,EACT,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,+BAA+B,EAC/B,iBAAiB,EACjB,OAAO,EACP,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,aAAa,CAAC"}
|
package/esm/src/eval/index.js
CHANGED
|
@@ -51,6 +51,7 @@ export { createEvalReport, summarizeEvalRecords } from "./report.js";
|
|
|
51
51
|
export { compareEvalReports } from "./baseline.js";
|
|
52
52
|
export { compareEvalModelReports, createEvalModelComparisonMarkdown } from "./model-comparison.js";
|
|
53
53
|
export { createEvalRunProvenance, resolveEvalRunProvenance } from "./provenance.js";
|
|
54
|
+
export { createEvalRunId } from "./run-id.js";
|
|
54
55
|
export { exportEvalReport, runEval } from "./runner.js";
|
|
55
56
|
export { deriveEvalId, discoverEvals, findEvalById } from "./discovery.js";
|
|
56
57
|
export { createEvalSourceDocument, getEvalEditableFieldSchema, getEvalRunSchema, getEvalSourceDocumentSchema, getEvalSourcePatchSchema, getEvalSourceReferenceSchema, getEvalStudioCapabilitySchema, } from "./studio.js";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-id.d.ts","sourceRoot":"","sources":["../../../src/src/eval/run-id.ts"],"names":[],"mappings":"AAOA,iFAAiF;AACjF,wBAAgB,eAAe,CAC7B,GAAG,OAAa,EAChB,YAAY,GAAE,MAAM,MAA8B,GACjD,MAAM,CAGR"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import * as dntShim from "../../_dnt.shims.js";
|
|
2
|
+
function createEvalRunIdSuffix() {
|
|
3
|
+
const bytes = new Uint8Array(8);
|
|
4
|
+
dntShim.crypto.getRandomValues(bytes);
|
|
5
|
+
return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
6
|
+
}
|
|
7
|
+
/** Create a timestamp-sortable eval run id with a collision-resistant suffix. */
|
|
8
|
+
export function createEvalRunId(now = new Date(), createSuffix = createEvalRunIdSuffix) {
|
|
9
|
+
const timestamp = now.toISOString().replace(/[-:.]/g, "").replace("T", "_").replace("Z", "");
|
|
10
|
+
return `evalrun_${timestamp}_${createSuffix()}`;
|
|
11
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/src/eval/runner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/src/eval/runner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAG/C,OAAO,EAKL,KAAK,sBAAsB,EAE5B,MAAM,6BAA6B,CAAC;AAGrC,OAAO,KAAK,EAEV,cAAc,EAGd,sBAAsB,EAGtB,cAAc,EACf,MAAM,YAAY,CAAC;AA2KpB,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,EAC3C,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,sBAAsB,EAAE,GAAG,SAAS,CAAC,CAgB/C;AAiED,6DAA6D;AAC7D,wBAAsB,OAAO,CAC3B,UAAU,EAAE,cAAc,EAC1B,OAAO,EAAE,cAAc,4CAyBxB"}
|
package/esm/src/eval/runner.js
CHANGED
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
import * as dntShim from "../../_dnt.shims.js";
|
|
2
2
|
import { createEvalCheckContext } from "./expect.js";
|
|
3
3
|
import { createEvalReport } from "./report.js";
|
|
4
|
+
import { createEvalRunId } from "./run-id.js";
|
|
4
5
|
import { metrics as runtimeMetrics } from "../metrics/index.js";
|
|
5
6
|
import { createEvalReportExporterRegistry, EvalReportExporterRegistryName, } from "../extensions/eval/index.js";
|
|
6
7
|
import { trace } from "../observability/tracing/api-shim.js";
|
|
7
8
|
import { tryResolve } from "../extensions/contracts.js";
|
|
8
|
-
function createRunId(now) {
|
|
9
|
-
return `evalrun_${now.toISOString().replace(/[-:.]/g, "").replace("T", "_").replace("Z", "")}`;
|
|
10
|
-
}
|
|
11
9
|
function normalizeTrace(trace) {
|
|
12
10
|
return {
|
|
13
11
|
events: trace?.events ?? [],
|
|
@@ -235,7 +233,7 @@ export async function runEval(definition, options) {
|
|
|
235
233
|
const report = createEvalReport({
|
|
236
234
|
definition,
|
|
237
235
|
records,
|
|
238
|
-
runId: options.runId ??
|
|
236
|
+
runId: options.runId ?? createEvalRunId(startedAt),
|
|
239
237
|
startedAt,
|
|
240
238
|
endedAt,
|
|
241
239
|
metadata: options.metadata,
|