veryfront 0.1.521 → 0.1.523
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/cli/templates/manifest.d.ts +405 -405
- package/esm/cli/templates/manifest.js +454 -454
- package/esm/deno.d.ts +2 -7
- package/esm/deno.js +4 -16
- package/esm/extensions/{ext-tracing-opentelemetry → ext-observability-opentelemetry}/src/index.d.ts +3 -3
- package/esm/extensions/ext-observability-opentelemetry/src/index.d.ts.map +1 -0
- package/esm/extensions/{ext-tracing-opentelemetry → ext-observability-opentelemetry}/src/index.js +10 -10
- package/esm/src/agent/service/config.d.ts.map +1 -1
- package/esm/src/agent/service/config.js +2 -0
- package/esm/src/agent/service/node-telemetry.d.ts +1 -1
- package/esm/src/agent/service/node-telemetry.d.ts.map +1 -1
- package/esm/src/agent/service/node-telemetry.js +1 -1
- package/esm/src/agent/testing/index.d.ts +1 -1
- package/esm/src/agent/testing/index.d.ts.map +1 -1
- package/esm/src/agent/testing/index.js +1 -1
- package/esm/src/agent/testing/live-evals/index.d.ts +2 -1
- package/esm/src/agent/testing/live-evals/index.d.ts.map +1 -1
- package/esm/src/agent/testing/live-evals/index.js +2 -1
- package/esm/src/agent/testing/live-evals/request.d.ts +16 -17
- package/esm/src/agent/testing/live-evals/request.d.ts.map +1 -1
- package/esm/src/agent/testing/live-evals/runner.d.ts +124 -0
- package/esm/src/agent/testing/live-evals/runner.d.ts.map +1 -0
- package/esm/src/agent/testing/live-evals/runner.js +391 -0
- package/esm/src/agent/veryfront-cloud-agent-service.js +2 -2
- package/esm/src/extensions/{tracing → observability}/index.d.ts +2 -2
- package/esm/src/extensions/observability/index.d.ts.map +1 -0
- package/esm/src/extensions/{tracing → observability}/index.js +2 -2
- package/esm/src/extensions/{tracing → observability}/node-telemetry-provider.d.ts +2 -2
- package/esm/src/extensions/observability/node-telemetry-provider.d.ts.map +1 -0
- package/esm/src/extensions/{tracing → observability}/node-telemetry-provider.js +2 -2
- package/esm/src/extensions/{tracing → observability}/tracing-exporter.d.ts +2 -2
- package/esm/src/extensions/observability/tracing-exporter.d.ts.map +1 -0
- package/esm/src/extensions/observability/tracing-exporter.js +8 -0
- package/esm/src/extensions/recommendations.js +2 -2
- package/esm/src/observability/metrics/manager.js +1 -1
- package/esm/src/observability/simple-metrics/otel-instruments.js +1 -1
- package/esm/src/observability/tracing/api-shim.d.ts +4 -4
- package/esm/src/observability/tracing/api-shim.js +7 -7
- package/esm/src/observability/tracing/manager.js +2 -2
- package/esm/src/observability/tracing/otlp-setup.d.ts +1 -1
- package/esm/src/observability/tracing/otlp-setup.js +4 -4
- package/esm/src/proxy/tracing.d.ts +1 -1
- package/esm/src/proxy/tracing.js +2 -2
- package/esm/src/react/components/chat/theme.d.ts.map +1 -1
- package/esm/src/react/components/chat/theme.js +4 -2
- package/esm/src/server/dev-ui/manifest.d.ts +17 -17
- package/esm/src/server/dev-ui/manifest.js +17 -17
- package/esm/src/server/handlers/dev/framework-candidates.generated.d.ts.map +1 -1
- package/esm/src/server/handlers/dev/framework-candidates.generated.js +4 -1
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +4 -4
- package/src/cli/templates/manifest.js +454 -454
- package/src/deno.js +4 -16
- package/src/extensions/{ext-tracing-opentelemetry → ext-observability-opentelemetry}/src/index.ts +12 -12
- package/src/src/agent/service/config.ts +2 -0
- package/src/src/agent/service/node-telemetry.ts +1 -1
- package/src/src/agent/testing/index.ts +12 -0
- package/src/src/agent/testing/live-evals/index.ts +18 -1
- package/src/src/agent/testing/live-evals/request.ts +19 -1
- package/src/src/agent/testing/live-evals/runner.ts +629 -0
- package/src/src/agent/veryfront-cloud-agent-service.ts +2 -2
- package/src/src/extensions/{tracing → observability}/index.ts +2 -2
- package/src/src/extensions/{tracing → observability}/node-telemetry-provider.ts +2 -2
- package/src/src/extensions/{tracing → observability}/tracing-exporter.ts +2 -2
- package/src/src/extensions/recommendations.ts +2 -2
- package/src/src/observability/metrics/manager.ts +1 -1
- package/src/src/observability/simple-metrics/otel-instruments.ts +1 -1
- package/src/src/observability/tracing/api-shim.ts +7 -7
- package/src/src/observability/tracing/manager.ts +2 -2
- package/src/src/observability/tracing/otlp-setup.ts +4 -4
- package/src/src/proxy/tracing.ts +2 -2
- package/src/src/react/components/chat/theme.ts +4 -2
- package/src/src/server/bootstrap.ts +1 -1
- package/src/src/server/dev-ui/manifest.js +17 -17
- package/src/src/server/handlers/dev/framework-candidates.generated.ts +4 -1
- package/src/src/utils/version-constant.ts +1 -1
- package/esm/extensions/ext-tracing-opentelemetry/src/index.d.ts.map +0 -1
- package/esm/src/extensions/tracing/index.d.ts.map +0 -1
- package/esm/src/extensions/tracing/node-telemetry-provider.d.ts.map +0 -1
- package/esm/src/extensions/tracing/tracing-exporter.d.ts.map +0 -1
- package/esm/src/extensions/tracing/tracing-exporter.js +0 -8
package/esm/deno.d.ts
CHANGED
|
@@ -58,7 +58,7 @@ declare namespace _default {
|
|
|
58
58
|
"./extensions/parser": string;
|
|
59
59
|
"./extensions/sandbox": string;
|
|
60
60
|
"./extensions/schema": string;
|
|
61
|
-
"./extensions/
|
|
61
|
+
"./extensions/observability": string;
|
|
62
62
|
"./testing": string;
|
|
63
63
|
"./testing/assert": string;
|
|
64
64
|
"./testing/bdd": string;
|
|
@@ -153,7 +153,7 @@ declare namespace _default {
|
|
|
153
153
|
"veryfront/extensions/parser": string;
|
|
154
154
|
"veryfront/extensions/sandbox": string;
|
|
155
155
|
"veryfront/extensions/schema": string;
|
|
156
|
-
"veryfront/extensions/
|
|
156
|
+
"veryfront/extensions/observability": string;
|
|
157
157
|
"#veryfront": string;
|
|
158
158
|
"#veryfront/agent": string;
|
|
159
159
|
"#veryfront/agent/react": string;
|
|
@@ -269,7 +269,6 @@ declare namespace _default {
|
|
|
269
269
|
let strict: boolean;
|
|
270
270
|
let noImplicitAny: boolean;
|
|
271
271
|
let noUncheckedIndexedAccess: boolean;
|
|
272
|
-
let types: string[];
|
|
273
272
|
let lib: string[];
|
|
274
273
|
}
|
|
275
274
|
let tasks: {
|
|
@@ -376,10 +375,6 @@ declare namespace _default {
|
|
|
376
375
|
export let singleQuote: boolean;
|
|
377
376
|
export let proseWrap: string;
|
|
378
377
|
}
|
|
379
|
-
namespace allowScripts {
|
|
380
|
-
let allow: string[];
|
|
381
|
-
let deny: string[];
|
|
382
|
-
}
|
|
383
378
|
}
|
|
384
379
|
export default _default;
|
|
385
380
|
//# sourceMappingURL=deno.d.ts.map
|
package/esm/deno.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export default {
|
|
2
2
|
"name": "veryfront",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.523",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"nodeModulesDir": "auto",
|
|
6
6
|
"workspace": [
|
|
@@ -16,7 +16,7 @@ export default {
|
|
|
16
16
|
"./extensions/ext-db-sqlite",
|
|
17
17
|
"./extensions/ext-parser-babel",
|
|
18
18
|
"./extensions/ext-sandbox-shell-tools",
|
|
19
|
-
"./extensions/ext-
|
|
19
|
+
"./extensions/ext-observability-opentelemetry",
|
|
20
20
|
"./extensions/ext-content-mdx",
|
|
21
21
|
"./extensions/ext-schema-zod"
|
|
22
22
|
],
|
|
@@ -85,7 +85,7 @@ export default {
|
|
|
85
85
|
"./extensions/parser": "./src/extensions/parser/index.ts",
|
|
86
86
|
"./extensions/sandbox": "./src/extensions/sandbox/index.ts",
|
|
87
87
|
"./extensions/schema": "./src/extensions/schema/index.ts",
|
|
88
|
-
"./extensions/
|
|
88
|
+
"./extensions/observability": "./src/extensions/observability/index.ts",
|
|
89
89
|
"./testing": "./src/testing/index.ts",
|
|
90
90
|
"./testing/assert": "./src/testing/assert.ts",
|
|
91
91
|
"./testing/bdd": "./src/testing/bdd.ts",
|
|
@@ -180,7 +180,7 @@ export default {
|
|
|
180
180
|
"veryfront/extensions/parser": "./src/extensions/parser/index.ts",
|
|
181
181
|
"veryfront/extensions/sandbox": "./src/extensions/sandbox/index.ts",
|
|
182
182
|
"veryfront/extensions/schema": "./src/extensions/schema/index.ts",
|
|
183
|
-
"veryfront/extensions/
|
|
183
|
+
"veryfront/extensions/observability": "./src/extensions/observability/index.ts",
|
|
184
184
|
"#veryfront": "./src/index.ts",
|
|
185
185
|
"#veryfront/agent": "./src/agent/index.ts",
|
|
186
186
|
"#veryfront/agent/react": "./src/agent/react/index.ts",
|
|
@@ -296,9 +296,6 @@ export default {
|
|
|
296
296
|
"strict": true,
|
|
297
297
|
"noImplicitAny": true,
|
|
298
298
|
"noUncheckedIndexedAccess": true,
|
|
299
|
-
"types": [
|
|
300
|
-
"npm:@types/react@19.2.14"
|
|
301
|
-
],
|
|
302
299
|
"lib": [
|
|
303
300
|
"deno.window",
|
|
304
301
|
"dom",
|
|
@@ -431,14 +428,5 @@ export default {
|
|
|
431
428
|
"semiColons": true,
|
|
432
429
|
"singleQuote": false,
|
|
433
430
|
"proseWrap": "preserve"
|
|
434
|
-
},
|
|
435
|
-
"allowScripts": {
|
|
436
|
-
"allow": [
|
|
437
|
-
"npm:sharp@0.33.5",
|
|
438
|
-
"npm:onnxruntime-node@1.21.0"
|
|
439
|
-
],
|
|
440
|
-
"deny": [
|
|
441
|
-
"npm:protobufjs@7.5.4"
|
|
442
|
-
]
|
|
443
431
|
}
|
|
444
432
|
};
|
package/esm/extensions/{ext-tracing-opentelemetry → ext-observability-opentelemetry}/src/index.d.ts
RENAMED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { ExtensionFactory } from "../../../src/extensions/index.js";
|
|
2
|
-
import type { NodeTelemetryInitializeOptions, NodeTelemetryProvider, SpanData, TracingExporter } from "../../../src/extensions/
|
|
2
|
+
import type { NodeTelemetryInitializeOptions, NodeTelemetryProvider, SpanData, TracingExporter } from "../../../src/extensions/observability/index.js";
|
|
3
3
|
/**
|
|
4
4
|
* The TracerProvider interface as expected by the core shim.
|
|
5
|
-
* Using structural typing
|
|
5
|
+
* Using structural typing because the real SDK provider satisfies this shape.
|
|
6
6
|
*/
|
|
7
7
|
interface ShimTracerProvider {
|
|
8
8
|
getTracer(name: string, version?: string): unknown;
|
|
@@ -38,7 +38,7 @@ declare class OpenTelemetryNodeTelemetryProvider implements NodeTelemetryProvide
|
|
|
38
38
|
shutdown(): Promise<void>;
|
|
39
39
|
}
|
|
40
40
|
/**
|
|
41
|
-
* Default export
|
|
41
|
+
* Default export for the ext-observability-opentelemetry extension factory.
|
|
42
42
|
*
|
|
43
43
|
* Produces an extension that registers a `TracingExporter` contract
|
|
44
44
|
* implementation backed by the OpenTelemetry JS SDK.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-observability-opentelemetry/src/index.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,KAAK,EACV,8BAA8B,EAC9B,qBAAqB,EACrB,QAAQ,EACR,eAAe,EAChB,MAAM,gDAAgD,CAAC;AAiBxD;;;GAGG;AACH,UAAU,kBAAkB;IAC1B,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CACpD;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE;QACL,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC3C,CAAC;CACH;AAqDD,cAAM,mBAAoB,YAAW,eAAe;IAClD,OAAO,CAAC,WAAW,CAAoC;IAEjD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAyCxD,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAKzC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B,WAAW,IAAI,kBAAkB;IAOjC,aAAa,IAAI;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,GAAG,IAAI;IAKzF,WAAW,IAAI;QAAE,aAAa,IAAI,OAAO,CAAC;QAAC,OAAO,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAAA;KAAE,GAAG,IAAI;CAOnF;AAED,cAAM,kCAAmC,YAAW,qBAAqB;IACvE,OAAO,CAAC,GAAG,CAAwB;IAE7B,UAAU,CAAC,OAAO,EAAE,8BAA8B,GAAG,OAAO,CAAC,OAAO,CAAC;IA0CrE,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAQhC;AAED;;;;;GAKG;AACH,QAAA,MAAM,gBAAgB,EAAE,gBAgCvB,CAAC;AAEF,eAAe,gBAAgB,CAAC;AAChC,OAAO,EAAE,kCAAkC,EAAE,mBAAmB,EAAE,CAAC"}
|
package/esm/extensions/{ext-tracing-opentelemetry → ext-observability-opentelemetry}/src/index.js
RENAMED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* ext-
|
|
2
|
+
* ext-observability-opentelemetry: OpenTelemetry observability extension backed by the
|
|
3
3
|
* official OpenTelemetry JS SDK.
|
|
4
4
|
*
|
|
5
5
|
* Provides the `TracingExporter` and `NodeTelemetryProvider` contracts:
|
|
6
|
-
* - `start(config)
|
|
7
|
-
* - `export(spans)
|
|
8
|
-
* - `shutdown()
|
|
9
|
-
* - `getProvider()
|
|
10
|
-
* - `initialize(options)
|
|
6
|
+
* - `start(config)`: builds the SDK provider and OTLP HTTP exporter
|
|
7
|
+
* - `export(spans)`: no-op, the SDK handles export via BatchSpanProcessor
|
|
8
|
+
* - `shutdown()`: flushes and shuts down the provider
|
|
9
|
+
* - `getProvider()`: returns the SDK TracerProvider for shim wiring
|
|
10
|
+
* - `initialize(options)`: starts NodeSDK auto-instrumentation
|
|
11
11
|
*
|
|
12
12
|
* Configuration is read from `ctx.config` (see `OtlpExtConfig`) and falls
|
|
13
13
|
* back to standard OTEL environment variables.
|
|
14
14
|
*
|
|
15
|
-
* @module extensions/ext-
|
|
15
|
+
* @module extensions/ext-observability-opentelemetry
|
|
16
16
|
*/
|
|
17
17
|
import * as dntShim from "../../../_dnt.shims.js";
|
|
18
18
|
import { metrics, trace } from "@opentelemetry/api";
|
|
@@ -182,7 +182,7 @@ class OpenTelemetryNodeTelemetryProvider {
|
|
|
182
182
|
}
|
|
183
183
|
}
|
|
184
184
|
/**
|
|
185
|
-
* Default export
|
|
185
|
+
* Default export for the ext-observability-opentelemetry extension factory.
|
|
186
186
|
*
|
|
187
187
|
* Produces an extension that registers a `TracingExporter` contract
|
|
188
188
|
* implementation backed by the OpenTelemetry JS SDK.
|
|
@@ -191,7 +191,7 @@ const extOpenTelemetry = () => {
|
|
|
191
191
|
const exporterImpl = new OtlpTracingExporter();
|
|
192
192
|
const nodeTelemetryProvider = new OpenTelemetryNodeTelemetryProvider();
|
|
193
193
|
return {
|
|
194
|
-
name: "ext-
|
|
194
|
+
name: "ext-observability-opentelemetry",
|
|
195
195
|
version: "0.1.0",
|
|
196
196
|
capabilities: [
|
|
197
197
|
{ type: "contract", name: "TracingExporter" },
|
|
@@ -211,7 +211,7 @@ const extOpenTelemetry = () => {
|
|
|
211
211
|
await exporterImpl.start(ctx.config);
|
|
212
212
|
ctx.provide("TracingExporter", exporterImpl);
|
|
213
213
|
ctx.provide("NodeTelemetryProvider", nodeTelemetryProvider);
|
|
214
|
-
ctx.logger.info("[ext-
|
|
214
|
+
ctx.logger.info("[ext-observability-opentelemetry] TracingExporter registered");
|
|
215
215
|
},
|
|
216
216
|
async teardown() {
|
|
217
217
|
await nodeTelemetryProvider.shutdown();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/service/config.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/service/config.ts"],"names":[],"mappings":"AAYA,MAAM,MAAM,4BAA4B,GAAG,MAAM,GAAG,SAAS,GAAG,UAAU,CAAC;AAE3E,MAAM,MAAM,kBAAkB,GAAG;IAC/B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,oCAAoC,EAAE,4BAA4B,CAAC;IACnE,6CAA6C,EAAE,MAAM,CAAC;IACtD,8BAA8B,CAAC,EAAE,MAAM,CAAC;IACxC,wBAAwB,EAAE,MAAM,CAAC;IACjC,qCAAqC,EAAE,OAAO,CAAC;IAC/C,6BAA6B,EAAE,OAAO,CAAC;IACvC,QAAQ,EAAE,aAAa,GAAG,MAAM,GAAG,YAAY,CAAC;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,YAAY,EAAE,OAAO,CAAC;IACtB,2BAA2B,CAAC,EAAE,MAAM,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC,CAAC;AAsDlF,eAAO,MAAM,wBAAwB,sEAA0C,CAAC;AAEhF,eAAO,MAAM,8BAA8B,sEAA2B,CAAC;AACvE,MAAM,MAAM,wBAAwB,GAAG,kBAAkB,CAAC;AAC1D,MAAM,MAAM,6BAA6B,GAAG,uBAAuB,CAAC;AAEpE,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,uBAAuB,GAC7B,kBAAkB,CAGpB;AAED,wBAAgB,6BAA6B,CAC3C,KAAK,EAAE,6BAA6B,GACnC,wBAAwB,CAE1B"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { ensureBuiltinSchemaValidator } from "../../extensions/builtin-extensions.js";
|
|
1
2
|
import { defineSchema } from "../../schemas/define.js";
|
|
2
3
|
import { lazySchema } from "../../schemas/lazy.js";
|
|
3
4
|
function parseBooleanFlag(value) {
|
|
@@ -60,6 +61,7 @@ const getAgentServiceConfigSchema = defineSchema((v) => {
|
|
|
60
61
|
export const agentServiceConfigSchema = lazySchema(getAgentServiceConfigSchema);
|
|
61
62
|
export const hostedAgentServiceConfigSchema = agentServiceConfigSchema;
|
|
62
63
|
export function parseAgentServiceConfig(input) {
|
|
64
|
+
ensureBuiltinSchemaValidator();
|
|
63
65
|
return agentServiceConfigSchema.parse(input);
|
|
64
66
|
}
|
|
65
67
|
export function parseHostedAgentServiceConfig(input) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type NodeTelemetryInstrumentationConfig, type NodeTelemetryLogger, type NodeTelemetryProcessTarget, type NodeTelemetryProvider } from "../../extensions/
|
|
1
|
+
import { type NodeTelemetryInstrumentationConfig, type NodeTelemetryLogger, type NodeTelemetryProcessTarget, type NodeTelemetryProvider } from "../../extensions/observability/index.js";
|
|
2
2
|
export type NodeHostedAgentServiceTelemetryEnv = Record<string, string | undefined>;
|
|
3
3
|
export type NodeAgentServiceTelemetryEnv = NodeHostedAgentServiceTelemetryEnv;
|
|
4
4
|
export type NodeHostedAgentServiceInstrumentationConfig = NodeTelemetryInstrumentationConfig;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"node-telemetry.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/service/node-telemetry.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,kCAAkC,EACvC,KAAK,mBAAmB,EACxB,KAAK,0BAA0B,EAC/B,KAAK,qBAAqB,EAE3B,MAAM,
|
|
1
|
+
{"version":3,"file":"node-telemetry.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/service/node-telemetry.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,kCAAkC,EACvC,KAAK,mBAAmB,EACxB,KAAK,0BAA0B,EAC/B,KAAK,qBAAqB,EAE3B,MAAM,yCAAyC,CAAC;AAEjD,MAAM,MAAM,kCAAkC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;AAEpF,MAAM,MAAM,4BAA4B,GAAG,kCAAkC,CAAC;AAE9E,MAAM,MAAM,2CAA2C,GAAG,kCAAkC,CAAC;AAE7F,MAAM,MAAM,qCAAqC,GAAG,2CAA2C,CAAC;AAEhG,MAAM,MAAM,qCAAqC,GAAG;IAClD,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,eAAe,EAAE,2CAA2C,CAAC;CAC9D,CAAC;AAEF,MAAM,MAAM,+BAA+B,GAAG,qCAAqC,CAAC;AAEpF,MAAM,MAAM,mDAAmD,GAAG;IAChE,GAAG,EAAE,kCAAkC,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,6CAA6C,GACvD,mDAAmD,CAAC;AAEtD,MAAM,MAAM,qCAAqC,GAAG,mBAAmB,CAAC;AAExE,MAAM,MAAM,+BAA+B,GAAG,qCAAqC,CAAC;AAEpF,MAAM,MAAM,4CAA4C,GAAG,0BAA0B,CAAC;AAEtF,MAAM,MAAM,sCAAsC,GAAG,4CAA4C,CAAC;AAElG,MAAM,MAAM,gDAAgD,GACxD,qCAAqC,GACrC;IACA,MAAM,CAAC,EAAE,qCAAqC,CAAC;IAC/C,aAAa,CAAC,EAAE,4CAA4C,CAAC;IAC7D,iBAAiB,CAAC,EAAE,qBAAqB,CAAC;CAC3C,CAAC;AAEJ,MAAM,MAAM,0CAA0C,GACpD,gDAAgD,CAAC;AA6CnD,wBAAgB,4CAA4C,CAC1D,OAAO,EAAE,mDAAmD,GAC3D,qCAAqC,CAYvC;AAED,wBAAgB,sCAAsC,CACpD,OAAO,EAAE,6CAA6C,GACrD,+BAA+B,CAEjC;AAgCD,wBAAsB,6CAA6C,CACjE,OAAO,EAAE,gDAAgD,GACxD,OAAO,CAAC,OAAO,CAAC,CAkClB;AAED,wBAAsB,uCAAuC,CAC3D,OAAO,EAAE,0CAA0C,GAClD,OAAO,CAAC,OAAO,CAAC,CAElB"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { tryResolve } from "../../extensions/contracts.js";
|
|
2
|
-
import { NodeTelemetryProviderName, } from "../../extensions/
|
|
2
|
+
import { NodeTelemetryProviderName, } from "../../extensions/observability/index.js";
|
|
3
3
|
function resolveEnabled(env, defaultEnabled) {
|
|
4
4
|
const envValue = env.OTEL_ENABLED;
|
|
5
5
|
if (envValue !== undefined) {
|
|
@@ -5,5 +5,5 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import "../../../_dnt.polyfills.js";
|
|
7
7
|
export { assertCompleted, assertContains, assertToolCalled, printTestResults, testAgent, type TestCase, type TestResult, type TestSuite, } from "./agent-tester.js";
|
|
8
|
-
export { buildFailureSuffix, buildLiveEvalCaseTagSummary, buildLiveEvalRequestBody, type BuildLiveEvalRequestBodyInput, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, buildProgressLine, buildRuntimePerformanceSummary, containsOrderedSubsequence, createFailedEvalResult, createPassedEvalResult, createPlainTextPdf, createSkippedEvalResult, hasEveryLiveEvalTag, type LiveEvalCaseMetadata, type LiveEvalCaseSelectionInput, type LiveEvalResultForPerformance, type LiveEvalResultForReport, type LiveEvalResultRecord, type LiveEvalRuntime, resolveLiveEvalRequestedCaseIds, type RuntimePerformanceSummary, selectLiveEvalCases, } from "./live-evals/index.js";
|
|
8
|
+
export { buildFailureSuffix, buildLiveEvalCaseTagSummary, buildLiveEvalRequestBody, type BuildLiveEvalRequestBodyInput, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, buildProgressLine, buildRuntimePerformanceSummary, containsOrderedSubsequence, containsSkillLoad, countStepStartedEvents, createFailedEvalResult, createLiveEvalCaseSupport, createPassedEvalResult, createPlainTextPdf, createSkippedEvalResult, hasEveryLiveEvalTag, hasFinished, type LiveEvalCase, type LiveEvalCaseMetadata, type LiveEvalCaseSelectionInput, type LiveEvalContext, type LiveEvalProjectFile, type LiveEvalProjectFileReaderInput, type LiveEvalRequestBody, type LiveEvalResultForPerformance, type LiveEvalResultForReport, type LiveEvalResultRecord, type LiveEvalRunnerConfig, liveEvalRunnerInternals, type LiveEvalRuntime, type PreparedLiveEvalInput, resolveLiveEvalRequestedCaseIds, type RuntimePerformanceSummary, selectLiveEvalCases, } from "./live-evals/index.js";
|
|
9
9
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/testing/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,4BAA4B,CAAC;AAGpC,OAAO,EACL,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,SAAS,EACT,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,SAAS,GACf,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,kBAAkB,EAClB,2BAA2B,EAC3B,wBAAwB,EACxB,KAAK,6BAA6B,EAClC,2BAA2B,EAC3B,0BAA0B,EAC1B,iBAAiB,EACjB,8BAA8B,EAC9B,0BAA0B,EAC1B,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,uBAAuB,EACvB,mBAAmB,EACnB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,EAC/B,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAC5B,KAAK,oBAAoB,EACzB,KAAK,eAAe,EACpB,+BAA+B,EAC/B,KAAK,yBAAyB,EAC9B,mBAAmB,GACpB,MAAM,uBAAuB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/testing/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,4BAA4B,CAAC;AAGpC,OAAO,EACL,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,SAAS,EACT,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,SAAS,GACf,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,kBAAkB,EAClB,2BAA2B,EAC3B,wBAAwB,EACxB,KAAK,6BAA6B,EAClC,2BAA2B,EAC3B,0BAA0B,EAC1B,iBAAiB,EACjB,8BAA8B,EAC9B,0BAA0B,EAC1B,iBAAiB,EACjB,sBAAsB,EACtB,sBAAsB,EACtB,yBAAyB,EACzB,sBAAsB,EACtB,kBAAkB,EAClB,uBAAuB,EACvB,mBAAmB,EACnB,WAAW,EACX,KAAK,YAAY,EACjB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,EAC/B,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,8BAA8B,EACnC,KAAK,mBAAmB,EACxB,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAC5B,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,uBAAuB,EACvB,KAAK,eAAe,EACpB,KAAK,qBAAqB,EAC1B,+BAA+B,EAC/B,KAAK,yBAAyB,EAC9B,mBAAmB,GACpB,MAAM,uBAAuB,CAAC"}
|
|
@@ -5,4 +5,4 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import "../../../_dnt.polyfills.js";
|
|
7
7
|
export { assertCompleted, assertContains, assertToolCalled, printTestResults, testAgent, } from "./agent-tester.js";
|
|
8
|
-
export { buildFailureSuffix, buildLiveEvalCaseTagSummary, buildLiveEvalRequestBody, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, buildProgressLine, buildRuntimePerformanceSummary, containsOrderedSubsequence, createFailedEvalResult, createPassedEvalResult, createPlainTextPdf, createSkippedEvalResult, hasEveryLiveEvalTag, resolveLiveEvalRequestedCaseIds, selectLiveEvalCases, } from "./live-evals/index.js";
|
|
8
|
+
export { buildFailureSuffix, buildLiveEvalCaseTagSummary, buildLiveEvalRequestBody, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, buildProgressLine, buildRuntimePerformanceSummary, containsOrderedSubsequence, containsSkillLoad, countStepStartedEvents, createFailedEvalResult, createLiveEvalCaseSupport, createPassedEvalResult, createPlainTextPdf, createSkippedEvalResult, hasEveryLiveEvalTag, hasFinished, liveEvalRunnerInternals, resolveLiveEvalRequestedCaseIds, selectLiveEvalCases, } from "./live-evals/index.js";
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export { buildFailureSuffix, buildProgressLine, containsOrderedSubsequence, createPlainTextPdf, } from "./formatting.js";
|
|
2
2
|
export { buildRuntimePerformanceSummary, type LiveEvalResultForPerformance, type LiveEvalRuntime, type RuntimePerformanceSummary, } from "./performance.js";
|
|
3
|
-
export { buildLiveEvalRequestBody, type BuildLiveEvalRequestBodyInput } from "./request.js";
|
|
3
|
+
export { buildLiveEvalRequestBody, type BuildLiveEvalRequestBodyInput, type LiveEvalRequestBody, } from "./request.js";
|
|
4
4
|
export { buildLiveEvalCaseTagSummary, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, hasEveryLiveEvalTag, type LiveEvalCaseMetadata, type LiveEvalCaseSelectionInput, type LiveEvalResultForReport, resolveLiveEvalRequestedCaseIds, selectLiveEvalCases, } from "./report.js";
|
|
5
5
|
export { createFailedEvalResult, createPassedEvalResult, createSkippedEvalResult, type LiveEvalResultRecord, } from "./result.js";
|
|
6
|
+
export { containsSkillLoad, countStepStartedEvents, createLiveEvalCaseSupport, hasFinished, type LiveEvalCase, type LiveEvalContext, type LiveEvalProjectFile, type LiveEvalProjectFileReaderInput, type LiveEvalRunnerConfig, liveEvalRunnerInternals, type PreparedLiveEvalInput, } from "./runner.js";
|
|
6
7
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,0BAA0B,EAC1B,kBAAkB,GACnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,8BAA8B,EAC9B,KAAK,4BAA4B,EACjC,KAAK,eAAe,EACpB,KAAK,yBAAyB,GAC/B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,0BAA0B,EAC1B,kBAAkB,GACnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,8BAA8B,EAC9B,KAAK,4BAA4B,EACjC,KAAK,eAAe,EACpB,KAAK,yBAAyB,GAC/B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,wBAAwB,EACxB,KAAK,6BAA6B,EAClC,KAAK,mBAAmB,GACzB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,2BAA2B,EAC3B,2BAA2B,EAC3B,0BAA0B,EAC1B,mBAAmB,EACnB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,EAC/B,KAAK,uBAAuB,EAC5B,+BAA+B,EAC/B,mBAAmB,GACpB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,KAAK,oBAAoB,GAC1B,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,iBAAiB,EACjB,sBAAsB,EACtB,yBAAyB,EACzB,WAAW,EACX,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,8BAA8B,EACnC,KAAK,oBAAoB,EACzB,uBAAuB,EACvB,KAAK,qBAAqB,GAC3B,MAAM,aAAa,CAAC"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export { buildFailureSuffix, buildProgressLine, containsOrderedSubsequence, createPlainTextPdf, } from "./formatting.js";
|
|
2
2
|
export { buildRuntimePerformanceSummary, } from "./performance.js";
|
|
3
|
-
export { buildLiveEvalRequestBody } from "./request.js";
|
|
3
|
+
export { buildLiveEvalRequestBody, } from "./request.js";
|
|
4
4
|
export { buildLiveEvalCaseTagSummary, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, hasEveryLiveEvalTag, resolveLiveEvalRequestedCaseIds, selectLiveEvalCases, } from "./report.js";
|
|
5
5
|
export { createFailedEvalResult, createPassedEvalResult, createSkippedEvalResult, } from "./result.js";
|
|
6
|
+
export { containsSkillLoad, countStepStartedEvents, createLiveEvalCaseSupport, hasFinished, liveEvalRunnerInternals, } from "./runner.js";
|
|
@@ -1,3 +1,18 @@
|
|
|
1
|
+
export interface LiveEvalRequestBody {
|
|
2
|
+
threadId: string;
|
|
3
|
+
runId: string;
|
|
4
|
+
state: Record<string, string>;
|
|
5
|
+
tools: unknown[];
|
|
6
|
+
context: unknown[];
|
|
7
|
+
forwardedProps?: {
|
|
8
|
+
veryfront: Record<string, unknown>;
|
|
9
|
+
};
|
|
10
|
+
messages: Array<{
|
|
11
|
+
id: string;
|
|
12
|
+
role: "user";
|
|
13
|
+
content: string;
|
|
14
|
+
}>;
|
|
15
|
+
}
|
|
1
16
|
export interface BuildLiveEvalRequestBodyInput {
|
|
2
17
|
testCaseId: string;
|
|
3
18
|
prompt: string;
|
|
@@ -10,21 +25,5 @@ export interface BuildLiveEvalRequestBodyInput {
|
|
|
10
25
|
forceRuntimeOverrides?: boolean;
|
|
11
26
|
maxSteps?: number;
|
|
12
27
|
}
|
|
13
|
-
export declare function buildLiveEvalRequestBody(input: BuildLiveEvalRequestBodyInput):
|
|
14
|
-
messages: {
|
|
15
|
-
id: any;
|
|
16
|
-
role: "user";
|
|
17
|
-
content: string;
|
|
18
|
-
}[];
|
|
19
|
-
forwardedProps?: {
|
|
20
|
-
veryfront: Record<string, unknown>;
|
|
21
|
-
} | undefined;
|
|
22
|
-
threadId: any;
|
|
23
|
-
runId: string;
|
|
24
|
-
state: {
|
|
25
|
-
evalCase: string;
|
|
26
|
-
};
|
|
27
|
-
tools: never[];
|
|
28
|
-
context: never[];
|
|
29
|
-
};
|
|
28
|
+
export declare function buildLiveEvalRequestBody(input: BuildLiveEvalRequestBodyInput): LiveEvalRequestBody;
|
|
30
29
|
//# sourceMappingURL=request.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/request.ts"],"names":[],"mappings":"AACA,MAAM,WAAW,6BAA6B;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,wBAAwB,
|
|
1
|
+
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/request.ts"],"names":[],"mappings":"AACA,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,KAAK,EAAE,OAAO,EAAE,CAAC;IACjB,OAAO,EAAE,OAAO,EAAE,CAAC;IACnB,cAAc,CAAC,EAAE;QACf,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,CAAC;IACF,QAAQ,EAAE,KAAK,CAAC;QACd,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,6BAA6B;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,6BAA6B,GACnC,mBAAmB,CA6CrB"}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { type AgUiSseProgressSnapshot as EvalProgressSnapshot, type ParsedAgUiSseRun as ParsedRun } from "../../index.js";
|
|
2
|
+
import { type LiveEvalRuntime } from "./performance.js";
|
|
3
|
+
import { type LiveEvalCaseMetadata } from "./report.js";
|
|
4
|
+
import { type LiveEvalResultRecord } from "./result.js";
|
|
5
|
+
export interface PreparedLiveEvalInput {
|
|
6
|
+
prompt?: string;
|
|
7
|
+
metadata?: Record<string, string>;
|
|
8
|
+
verificationContext?: LiveEvalContext;
|
|
9
|
+
cleanup?: () => Promise<void>;
|
|
10
|
+
startSidecar?: () => Promise<(() => Promise<void>) | void>;
|
|
11
|
+
}
|
|
12
|
+
export interface LiveEvalContext {
|
|
13
|
+
apiUrl: string;
|
|
14
|
+
authToken: string;
|
|
15
|
+
projectId: string | null;
|
|
16
|
+
}
|
|
17
|
+
export interface LiveEvalCase {
|
|
18
|
+
readonly id: string;
|
|
19
|
+
readonly label: string;
|
|
20
|
+
readonly prompt?: string;
|
|
21
|
+
allowedTools?: string[];
|
|
22
|
+
forceRuntimeOverrides?: boolean;
|
|
23
|
+
requireProject?: boolean;
|
|
24
|
+
maxSteps?: number;
|
|
25
|
+
expectedEventSubsequence?: string[];
|
|
26
|
+
metadata?: LiveEvalCaseMetadata;
|
|
27
|
+
prepare?: (context: LiveEvalContext) => Promise<PreparedLiveEvalInput>;
|
|
28
|
+
verify: (run: ParsedRun, prepared: PreparedLiveEvalInput | null) => string | null | Promise<string | null>;
|
|
29
|
+
}
|
|
30
|
+
interface FileCheckInput {
|
|
31
|
+
filePath: string;
|
|
32
|
+
requiredContent?: string[];
|
|
33
|
+
description?: string;
|
|
34
|
+
}
|
|
35
|
+
export interface LiveEvalProjectFile {
|
|
36
|
+
path: string;
|
|
37
|
+
content: string;
|
|
38
|
+
}
|
|
39
|
+
export interface LiveEvalProjectFileReaderInput {
|
|
40
|
+
filePath: string;
|
|
41
|
+
requestTimeoutMs: number;
|
|
42
|
+
}
|
|
43
|
+
export interface LiveEvalRunnerConfig {
|
|
44
|
+
endpoint: string;
|
|
45
|
+
authToken: string;
|
|
46
|
+
apiUrl: string;
|
|
47
|
+
projectId: string | null;
|
|
48
|
+
branchId: string | null;
|
|
49
|
+
model: string | null;
|
|
50
|
+
requestTimeoutMs: number;
|
|
51
|
+
progressLogIntervalMs: number;
|
|
52
|
+
enableLlmJudge: boolean;
|
|
53
|
+
fetch?: (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
|
|
54
|
+
log?: (message: string) => void;
|
|
55
|
+
readProjectFile?: (input: LiveEvalProjectFileReaderInput) => Promise<LiveEvalProjectFile | null>;
|
|
56
|
+
}
|
|
57
|
+
interface LiveEvalJudgeInput {
|
|
58
|
+
question: string;
|
|
59
|
+
criteria: string;
|
|
60
|
+
}
|
|
61
|
+
interface LiveEvalJudgeRequest extends LiveEvalJudgeInput {
|
|
62
|
+
answer: string;
|
|
63
|
+
}
|
|
64
|
+
interface LiveEvalJudgeResult {
|
|
65
|
+
pass: boolean;
|
|
66
|
+
reason: string;
|
|
67
|
+
}
|
|
68
|
+
declare function collectPreparedArtifactPaths(prepared: PreparedLiveEvalInput | null): string[];
|
|
69
|
+
interface LiveEvalResultContext {
|
|
70
|
+
id: string;
|
|
71
|
+
label: string;
|
|
72
|
+
runtime: LiveEvalRuntime;
|
|
73
|
+
startedAt: number;
|
|
74
|
+
conversationId?: string | null;
|
|
75
|
+
artifactPaths?: string[];
|
|
76
|
+
}
|
|
77
|
+
interface LiveEvalRunArtifactsInput {
|
|
78
|
+
run: ParsedRun;
|
|
79
|
+
runId?: string;
|
|
80
|
+
traceSignature: string;
|
|
81
|
+
}
|
|
82
|
+
interface LiveEvalRunArtifacts {
|
|
83
|
+
runId?: string;
|
|
84
|
+
traceSignature: string;
|
|
85
|
+
toolStarts: string[];
|
|
86
|
+
toolArgsPreview: string;
|
|
87
|
+
textPreview: string;
|
|
88
|
+
}
|
|
89
|
+
declare function createLiveEvalRunArtifacts(input: LiveEvalRunArtifactsInput): LiveEvalRunArtifacts;
|
|
90
|
+
declare function createFailedRunEvalResult(input: {
|
|
91
|
+
details: string;
|
|
92
|
+
context: LiveEvalResultContext;
|
|
93
|
+
runArtifacts: LiveEvalRunArtifacts;
|
|
94
|
+
}): LiveEvalResultRecord;
|
|
95
|
+
declare function createPassedRunEvalResult(input: {
|
|
96
|
+
details: string;
|
|
97
|
+
context: LiveEvalResultContext;
|
|
98
|
+
runArtifacts: LiveEvalRunArtifacts;
|
|
99
|
+
}): LiveEvalResultRecord;
|
|
100
|
+
declare function createStreamingFailureEvalResult(input: {
|
|
101
|
+
details: string;
|
|
102
|
+
context: LiveEvalResultContext;
|
|
103
|
+
progress: EvalProgressSnapshot;
|
|
104
|
+
}): LiveEvalResultRecord;
|
|
105
|
+
declare function extractRunId(run: ParsedRun): string | null;
|
|
106
|
+
export declare function hasFinished(run: ParsedRun): boolean;
|
|
107
|
+
export declare function containsSkillLoad(run: ParsedRun, skillId: string): boolean;
|
|
108
|
+
export declare function countStepStartedEvents(run: ParsedRun): number;
|
|
109
|
+
export declare function createLiveEvalCaseSupport(config: LiveEvalRunnerConfig): {
|
|
110
|
+
runEval: (testCase: LiveEvalCase, runtime: LiveEvalRuntime) => Promise<LiveEvalResultRecord>;
|
|
111
|
+
verifyFileExists: (input: FileCheckInput) => Promise<string | null>;
|
|
112
|
+
withJudge: (structuralVerify: (run: ParsedRun) => string | null, judgeInput: LiveEvalJudgeInput) => (run: ParsedRun) => Promise<string | null>;
|
|
113
|
+
judgeLlm: (input: LiveEvalJudgeRequest) => Promise<LiveEvalJudgeResult>;
|
|
114
|
+
};
|
|
115
|
+
export declare const liveEvalRunnerInternals: {
|
|
116
|
+
collectPreparedArtifactPaths: typeof collectPreparedArtifactPaths;
|
|
117
|
+
createFailedRunEvalResult: typeof createFailedRunEvalResult;
|
|
118
|
+
createLiveEvalRunArtifacts: typeof createLiveEvalRunArtifacts;
|
|
119
|
+
createPassedRunEvalResult: typeof createPassedRunEvalResult;
|
|
120
|
+
createStreamingFailureEvalResult: typeof createStreamingFailureEvalResult;
|
|
121
|
+
extractRunId: typeof extractRunId;
|
|
122
|
+
};
|
|
123
|
+
export {};
|
|
124
|
+
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/runner.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,uBAAuB,IAAI,oBAAoB,EAIpD,KAAK,gBAAgB,IAAI,SAAS,EACnC,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAExD,OAAO,EAAE,KAAK,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAIL,KAAK,oBAAoB,EAC1B,MAAM,aAAa,CAAC;AAErB,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,mBAAmB,CAAC,EAAE,eAAe,CAAC;IACtC,OAAO,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;CAC5D;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wBAAwB,CAAC,EAAE,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IAChC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACvE,MAAM,EAAE,CACN,GAAG,EAAE,SAAS,EACd,QAAQ,EAAE,qBAAqB,GAAG,IAAI,KACnC,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAC7C;AAED,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,cAAc,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjF,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,8BAA8B,KAAK,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;CAClG;AAED,UAAU,kBAAkB;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,oBAAqB,SAAQ,kBAAkB;IACvD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,mBAAmB;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAyJD,iBAAS,4BAA4B,CAAC,QAAQ,EAAE,qBAAqB,GAAG,IAAI,GAAG,MAAM,EAAE,CAYtF;AASD,UAAU,qBAAqB;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,eAAe,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,UAAU,yBAAyB;IACjC,GAAG,EAAE,SAAS,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,UAAU,oBAAoB;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,iBAAS,0BAA0B,CAAC,KAAK,EAAE,yBAAyB,GAAG,oBAAoB,CAQ1F;AAED,iBAAS,yBAAyB,CAAC,KAAK,EAAE;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,YAAY,EAAE,oBAAoB,CAAC;CACpC,GAAG,oBAAoB,CAevB;AAED,iBAAS,yBAAyB,CAAC,KAAK,EAAE;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,YAAY,EAAE,oBAAoB,CAAC;CACpC,GAAG,oBAAoB,CAevB;AAED,iBAAS,gCAAgC,CAAC,KAAK,EAAE;IAC/C,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,QAAQ,EAAE,oBAAoB,CAAC;CAChC,GAAG,oBAAoB,CAcvB;AA+FD,iBAAS,YAAY,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,GAAG,IAAI,CASnD;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAEnD;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAE1E;AAED,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,CAE7D;AAED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,oBAAoB,GAAG;IACvE,OAAO,EAAE,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC7F,gBAAgB,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACpE,SAAS,EAAE,CACT,gBAAgB,EAAE,CAAC,GAAG,EAAE,SAAS,KAAK,MAAM,GAAG,IAAI,EACnD,UAAU,EAAE,kBAAkB,KAC3B,CAAC,GAAG,EAAE,SAAS,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAChD,QAAQ,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,OAAO,CAAC,mBAAmB,CAAC,CAAC;CACzE,CAoIA;AAED,eAAO,MAAM,uBAAuB;;;;;;;CAOnC,CAAC"}
|