veryfront 0.1.908 → 0.1.910
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/deno.js +1 -1
- package/esm/src/eval/runner.d.ts.map +1 -1
- package/esm/src/eval/runner.js +66 -0
- package/esm/src/extensions/builtin-extensions.d.ts.map +1 -1
- package/esm/src/extensions/builtin-extensions.js +6 -0
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +1 -1
package/esm/deno.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/src/eval/runner.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/src/eval/runner.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAEV,cAAc,EAMd,cAAc,EACf,MAAM,YAAY,CAAC;AAmQpB,6DAA6D;AAC7D,wBAAsB,OAAO,CAC3B,UAAU,EAAE,cAAc,EAC1B,OAAO,EAAE,cAAc,4CAwBxB"}
|
package/esm/src/eval/runner.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as dntShim from "../../_dnt.shims.js";
|
|
2
2
|
import { createEvalCheckContext } from "./expect.js";
|
|
3
3
|
import { createEvalReport } from "./report.js";
|
|
4
|
+
import { metrics as runtimeMetrics } from "../metrics/index.js";
|
|
4
5
|
import { createEvalReportExporterRegistry, EvalReportExporterRegistryName, } from "../extensions/eval/index.js";
|
|
5
6
|
import { trace } from "../observability/tracing/api-shim.js";
|
|
6
7
|
import { tryResolve } from "../extensions/contracts.js";
|
|
@@ -32,6 +33,70 @@ function isBlockingFailure(record) {
|
|
|
32
33
|
return [...(record.metrics ?? []), ...(record.checks ?? [])].some((result) => !result.skipped && result.pass === false &&
|
|
33
34
|
(result.severity === "gate" || result.severity === "budget"));
|
|
34
35
|
}
|
|
36
|
+
function recordPassed(record) {
|
|
37
|
+
if (!record.completed || record.error)
|
|
38
|
+
return false;
|
|
39
|
+
return !isBlockingFailure(record);
|
|
40
|
+
}
|
|
41
|
+
function emitEvalRuntimeMetrics(report) {
|
|
42
|
+
const baseAttributes = {
|
|
43
|
+
eval_id: report.definitionId,
|
|
44
|
+
target_kind: report.targetKind,
|
|
45
|
+
};
|
|
46
|
+
for (const metric of report.summary.metrics) {
|
|
47
|
+
const common = {
|
|
48
|
+
...baseAttributes,
|
|
49
|
+
metric: metric.name,
|
|
50
|
+
family: metric.family,
|
|
51
|
+
severity: metric.severity,
|
|
52
|
+
};
|
|
53
|
+
if (metric.passed > 0) {
|
|
54
|
+
runtimeMetrics.counter("vf_eval_result_total", metric.passed, {
|
|
55
|
+
...common,
|
|
56
|
+
outcome: "pass",
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
if (metric.failed > 0) {
|
|
60
|
+
runtimeMetrics.counter("vf_eval_result_total", metric.failed, {
|
|
61
|
+
...common,
|
|
62
|
+
outcome: "fail",
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
if (metric.skipped > 0) {
|
|
66
|
+
runtimeMetrics.counter("vf_eval_result_total", metric.skipped, {
|
|
67
|
+
...common,
|
|
68
|
+
outcome: "skipped",
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (report.summary.metrics.length === 0) {
|
|
73
|
+
if (report.summary.passed > 0) {
|
|
74
|
+
runtimeMetrics.counter("vf_eval_result_total", report.summary.passed, {
|
|
75
|
+
...baseAttributes,
|
|
76
|
+
metric: "record",
|
|
77
|
+
family: "record",
|
|
78
|
+
severity: "gate",
|
|
79
|
+
outcome: "pass",
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
if (report.summary.failed > 0) {
|
|
83
|
+
runtimeMetrics.counter("vf_eval_result_total", report.summary.failed, {
|
|
84
|
+
...baseAttributes,
|
|
85
|
+
metric: "record",
|
|
86
|
+
family: "record",
|
|
87
|
+
severity: "gate",
|
|
88
|
+
outcome: "fail",
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
for (const record of report.records) {
|
|
93
|
+
runtimeMetrics.histogram("vf_eval_duration_ms", record.durationMs, {
|
|
94
|
+
...baseAttributes,
|
|
95
|
+
metric: "duration",
|
|
96
|
+
outcome: recordPassed(record) ? "pass" : "fail",
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
}
|
|
35
100
|
function createMissingRegistryResults(exporterIds) {
|
|
36
101
|
return exporterIds.map((exporterId) => ({
|
|
37
102
|
exporterId,
|
|
@@ -174,6 +239,7 @@ export async function runEval(definition, options) {
|
|
|
174
239
|
startedAt,
|
|
175
240
|
endedAt,
|
|
176
241
|
});
|
|
242
|
+
emitEvalRuntimeMetrics(report);
|
|
177
243
|
const exports = await exportEvalReport(report, options.export);
|
|
178
244
|
return exports === undefined ? report : { ...report, exports };
|
|
179
245
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"builtin-extensions.d.ts","sourceRoot":"","sources":["../../../src/src/extensions/builtin-extensions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEpD,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAElE,OAAO,KAAK,EAAe,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"builtin-extensions.d.ts","sourceRoot":"","sources":["../../../src/src/extensions/builtin-extensions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEpD,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAElE,OAAO,KAAK,EAAe,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAkDvE,wBAAgB,uCAAuC,IAAI,0BAA0B,CAOpF;AAWD,wBAAgB,yBAAyB,IAAI,mBAAmB,CAM/D;AAED,wBAAgB,4BAA4B,IAAI,IAAI,CAInD;AAoCD,wBAAgB,uBAAuB,IAAI,iBAAiB,EAAE,CAyD7D"}
|
|
@@ -11,6 +11,7 @@ import extMdx from "../../extensions/ext-content-mdx/src/index.js";
|
|
|
11
11
|
import extTailwind from "../../extensions/ext-css-tailwind/src/index.js";
|
|
12
12
|
import extDocumentKreuzberg from "../../extensions/ext-document-kreuzberg/src/index.js";
|
|
13
13
|
import extDbSqlite from "../../extensions/ext-db-sqlite/src/index.js";
|
|
14
|
+
import extOpenTelemetry from "../../extensions/ext-observability-opentelemetry/src/index.js";
|
|
14
15
|
import extSandboxShellTools from "../../extensions/ext-sandbox-shell-tools/src/index.js";
|
|
15
16
|
import extZod from "../../extensions/ext-schema-zod/src/index.js";
|
|
16
17
|
import { createZodAdapter } from "../../extensions/ext-schema-zod/src/adapter.js";
|
|
@@ -110,6 +111,11 @@ export function createBuiltinExtensions() {
|
|
|
110
111
|
origin: "veryfront/ext-auth-jwt",
|
|
111
112
|
extension: extJwt(),
|
|
112
113
|
},
|
|
114
|
+
{
|
|
115
|
+
source: "builtin",
|
|
116
|
+
origin: "veryfront/ext-observability-opentelemetry",
|
|
117
|
+
extension: extOpenTelemetry(),
|
|
118
|
+
},
|
|
113
119
|
{
|
|
114
120
|
source: "builtin",
|
|
115
121
|
origin: "veryfront/ext-bundler-esbuild",
|