veryfront 0.1.521 → 0.1.523

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/esm/cli/templates/manifest.d.ts +405 -405
  2. package/esm/cli/templates/manifest.js +454 -454
  3. package/esm/deno.d.ts +2 -7
  4. package/esm/deno.js +4 -16
  5. package/esm/extensions/{ext-tracing-opentelemetry → ext-observability-opentelemetry}/src/index.d.ts +3 -3
  6. package/esm/extensions/ext-observability-opentelemetry/src/index.d.ts.map +1 -0
  7. package/esm/extensions/{ext-tracing-opentelemetry → ext-observability-opentelemetry}/src/index.js +10 -10
  8. package/esm/src/agent/service/config.d.ts.map +1 -1
  9. package/esm/src/agent/service/config.js +2 -0
  10. package/esm/src/agent/service/node-telemetry.d.ts +1 -1
  11. package/esm/src/agent/service/node-telemetry.d.ts.map +1 -1
  12. package/esm/src/agent/service/node-telemetry.js +1 -1
  13. package/esm/src/agent/testing/index.d.ts +1 -1
  14. package/esm/src/agent/testing/index.d.ts.map +1 -1
  15. package/esm/src/agent/testing/index.js +1 -1
  16. package/esm/src/agent/testing/live-evals/index.d.ts +2 -1
  17. package/esm/src/agent/testing/live-evals/index.d.ts.map +1 -1
  18. package/esm/src/agent/testing/live-evals/index.js +2 -1
  19. package/esm/src/agent/testing/live-evals/request.d.ts +16 -17
  20. package/esm/src/agent/testing/live-evals/request.d.ts.map +1 -1
  21. package/esm/src/agent/testing/live-evals/runner.d.ts +124 -0
  22. package/esm/src/agent/testing/live-evals/runner.d.ts.map +1 -0
  23. package/esm/src/agent/testing/live-evals/runner.js +391 -0
  24. package/esm/src/agent/veryfront-cloud-agent-service.js +2 -2
  25. package/esm/src/extensions/{tracing → observability}/index.d.ts +2 -2
  26. package/esm/src/extensions/observability/index.d.ts.map +1 -0
  27. package/esm/src/extensions/{tracing → observability}/index.js +2 -2
  28. package/esm/src/extensions/{tracing → observability}/node-telemetry-provider.d.ts +2 -2
  29. package/esm/src/extensions/observability/node-telemetry-provider.d.ts.map +1 -0
  30. package/esm/src/extensions/{tracing → observability}/node-telemetry-provider.js +2 -2
  31. package/esm/src/extensions/{tracing → observability}/tracing-exporter.d.ts +2 -2
  32. package/esm/src/extensions/observability/tracing-exporter.d.ts.map +1 -0
  33. package/esm/src/extensions/observability/tracing-exporter.js +8 -0
  34. package/esm/src/extensions/recommendations.js +2 -2
  35. package/esm/src/observability/metrics/manager.js +1 -1
  36. package/esm/src/observability/simple-metrics/otel-instruments.js +1 -1
  37. package/esm/src/observability/tracing/api-shim.d.ts +4 -4
  38. package/esm/src/observability/tracing/api-shim.js +7 -7
  39. package/esm/src/observability/tracing/manager.js +2 -2
  40. package/esm/src/observability/tracing/otlp-setup.d.ts +1 -1
  41. package/esm/src/observability/tracing/otlp-setup.js +4 -4
  42. package/esm/src/proxy/tracing.d.ts +1 -1
  43. package/esm/src/proxy/tracing.js +2 -2
  44. package/esm/src/react/components/chat/theme.d.ts.map +1 -1
  45. package/esm/src/react/components/chat/theme.js +4 -2
  46. package/esm/src/server/dev-ui/manifest.d.ts +17 -17
  47. package/esm/src/server/dev-ui/manifest.js +17 -17
  48. package/esm/src/server/handlers/dev/framework-candidates.generated.d.ts.map +1 -1
  49. package/esm/src/server/handlers/dev/framework-candidates.generated.js +4 -1
  50. package/esm/src/utils/version-constant.d.ts +1 -1
  51. package/esm/src/utils/version-constant.js +1 -1
  52. package/package.json +4 -4
  53. package/src/cli/templates/manifest.js +454 -454
  54. package/src/deno.js +4 -16
  55. package/src/extensions/{ext-tracing-opentelemetry → ext-observability-opentelemetry}/src/index.ts +12 -12
  56. package/src/src/agent/service/config.ts +2 -0
  57. package/src/src/agent/service/node-telemetry.ts +1 -1
  58. package/src/src/agent/testing/index.ts +12 -0
  59. package/src/src/agent/testing/live-evals/index.ts +18 -1
  60. package/src/src/agent/testing/live-evals/request.ts +19 -1
  61. package/src/src/agent/testing/live-evals/runner.ts +629 -0
  62. package/src/src/agent/veryfront-cloud-agent-service.ts +2 -2
  63. package/src/src/extensions/{tracing → observability}/index.ts +2 -2
  64. package/src/src/extensions/{tracing → observability}/node-telemetry-provider.ts +2 -2
  65. package/src/src/extensions/{tracing → observability}/tracing-exporter.ts +2 -2
  66. package/src/src/extensions/recommendations.ts +2 -2
  67. package/src/src/observability/metrics/manager.ts +1 -1
  68. package/src/src/observability/simple-metrics/otel-instruments.ts +1 -1
  69. package/src/src/observability/tracing/api-shim.ts +7 -7
  70. package/src/src/observability/tracing/manager.ts +2 -2
  71. package/src/src/observability/tracing/otlp-setup.ts +4 -4
  72. package/src/src/proxy/tracing.ts +2 -2
  73. package/src/src/react/components/chat/theme.ts +4 -2
  74. package/src/src/server/bootstrap.ts +1 -1
  75. package/src/src/server/dev-ui/manifest.js +17 -17
  76. package/src/src/server/handlers/dev/framework-candidates.generated.ts +4 -1
  77. package/src/src/utils/version-constant.ts +1 -1
  78. package/esm/extensions/ext-tracing-opentelemetry/src/index.d.ts.map +0 -1
  79. package/esm/src/extensions/tracing/index.d.ts.map +0 -1
  80. package/esm/src/extensions/tracing/node-telemetry-provider.d.ts.map +0 -1
  81. package/esm/src/extensions/tracing/tracing-exporter.d.ts.map +0 -1
  82. package/esm/src/extensions/tracing/tracing-exporter.js +0 -8
package/esm/deno.d.ts CHANGED
@@ -58,7 +58,7 @@ declare namespace _default {
58
58
  "./extensions/parser": string;
59
59
  "./extensions/sandbox": string;
60
60
  "./extensions/schema": string;
61
- "./extensions/tracing": string;
61
+ "./extensions/observability": string;
62
62
  "./testing": string;
63
63
  "./testing/assert": string;
64
64
  "./testing/bdd": string;
@@ -153,7 +153,7 @@ declare namespace _default {
153
153
  "veryfront/extensions/parser": string;
154
154
  "veryfront/extensions/sandbox": string;
155
155
  "veryfront/extensions/schema": string;
156
- "veryfront/extensions/tracing": string;
156
+ "veryfront/extensions/observability": string;
157
157
  "#veryfront": string;
158
158
  "#veryfront/agent": string;
159
159
  "#veryfront/agent/react": string;
@@ -269,7 +269,6 @@ declare namespace _default {
269
269
  let strict: boolean;
270
270
  let noImplicitAny: boolean;
271
271
  let noUncheckedIndexedAccess: boolean;
272
- let types: string[];
273
272
  let lib: string[];
274
273
  }
275
274
  let tasks: {
@@ -376,10 +375,6 @@ declare namespace _default {
376
375
  export let singleQuote: boolean;
377
376
  export let proseWrap: string;
378
377
  }
379
- namespace allowScripts {
380
- let allow: string[];
381
- let deny: string[];
382
- }
383
378
  }
384
379
  export default _default;
385
380
  //# sourceMappingURL=deno.d.ts.map
package/esm/deno.js CHANGED
@@ -1,6 +1,6 @@
1
1
  export default {
2
2
  "name": "veryfront",
3
- "version": "0.1.521",
3
+ "version": "0.1.523",
4
4
  "license": "Apache-2.0",
5
5
  "nodeModulesDir": "auto",
6
6
  "workspace": [
@@ -16,7 +16,7 @@ export default {
16
16
  "./extensions/ext-db-sqlite",
17
17
  "./extensions/ext-parser-babel",
18
18
  "./extensions/ext-sandbox-shell-tools",
19
- "./extensions/ext-tracing-opentelemetry",
19
+ "./extensions/ext-observability-opentelemetry",
20
20
  "./extensions/ext-content-mdx",
21
21
  "./extensions/ext-schema-zod"
22
22
  ],
@@ -85,7 +85,7 @@ export default {
85
85
  "./extensions/parser": "./src/extensions/parser/index.ts",
86
86
  "./extensions/sandbox": "./src/extensions/sandbox/index.ts",
87
87
  "./extensions/schema": "./src/extensions/schema/index.ts",
88
- "./extensions/tracing": "./src/extensions/tracing/index.ts",
88
+ "./extensions/observability": "./src/extensions/observability/index.ts",
89
89
  "./testing": "./src/testing/index.ts",
90
90
  "./testing/assert": "./src/testing/assert.ts",
91
91
  "./testing/bdd": "./src/testing/bdd.ts",
@@ -180,7 +180,7 @@ export default {
180
180
  "veryfront/extensions/parser": "./src/extensions/parser/index.ts",
181
181
  "veryfront/extensions/sandbox": "./src/extensions/sandbox/index.ts",
182
182
  "veryfront/extensions/schema": "./src/extensions/schema/index.ts",
183
- "veryfront/extensions/tracing": "./src/extensions/tracing/index.ts",
183
+ "veryfront/extensions/observability": "./src/extensions/observability/index.ts",
184
184
  "#veryfront": "./src/index.ts",
185
185
  "#veryfront/agent": "./src/agent/index.ts",
186
186
  "#veryfront/agent/react": "./src/agent/react/index.ts",
@@ -296,9 +296,6 @@ export default {
296
296
  "strict": true,
297
297
  "noImplicitAny": true,
298
298
  "noUncheckedIndexedAccess": true,
299
- "types": [
300
- "npm:@types/react@19.2.14"
301
- ],
302
299
  "lib": [
303
300
  "deno.window",
304
301
  "dom",
@@ -431,14 +428,5 @@ export default {
431
428
  "semiColons": true,
432
429
  "singleQuote": false,
433
430
  "proseWrap": "preserve"
434
- },
435
- "allowScripts": {
436
- "allow": [
437
- "npm:sharp@0.33.5",
438
- "npm:onnxruntime-node@1.21.0"
439
- ],
440
- "deny": [
441
- "npm:protobufjs@7.5.4"
442
- ]
443
431
  }
444
432
  };
@@ -1,8 +1,8 @@
1
1
  import type { ExtensionFactory } from "../../../src/extensions/index.js";
2
- import type { NodeTelemetryInitializeOptions, NodeTelemetryProvider, SpanData, TracingExporter } from "../../../src/extensions/tracing/index.js";
2
+ import type { NodeTelemetryInitializeOptions, NodeTelemetryProvider, SpanData, TracingExporter } from "../../../src/extensions/observability/index.js";
3
3
  /**
4
4
  * The TracerProvider interface as expected by the core shim.
5
- * Using structural typing the real SDK provider satisfies this shape.
5
+ * Using structural typing because the real SDK provider satisfies this shape.
6
6
  */
7
7
  interface ShimTracerProvider {
8
8
  getTracer(name: string, version?: string): unknown;
@@ -38,7 +38,7 @@ declare class OpenTelemetryNodeTelemetryProvider implements NodeTelemetryProvide
38
38
  shutdown(): Promise<void>;
39
39
  }
40
40
  /**
41
- * Default export the ext-tracing-opentelemetry extension factory.
41
+ * Default export for the ext-observability-opentelemetry extension factory.
42
42
  *
43
43
  * Produces an extension that registers a `TracingExporter` contract
44
44
  * implementation backed by the OpenTelemetry JS SDK.
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-observability-opentelemetry/src/index.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,KAAK,EACV,8BAA8B,EAC9B,qBAAqB,EACrB,QAAQ,EACR,eAAe,EAChB,MAAM,gDAAgD,CAAC;AAiBxD;;;GAGG;AACH,UAAU,kBAAkB;IAC1B,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CACpD;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE;QACL,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC3C,CAAC;CACH;AAqDD,cAAM,mBAAoB,YAAW,eAAe;IAClD,OAAO,CAAC,WAAW,CAAoC;IAEjD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAyCxD,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAKzC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B,WAAW,IAAI,kBAAkB;IAOjC,aAAa,IAAI;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,GAAG,IAAI;IAKzF,WAAW,IAAI;QAAE,aAAa,IAAI,OAAO,CAAC;QAAC,OAAO,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAAA;KAAE,GAAG,IAAI;CAOnF;AAED,cAAM,kCAAmC,YAAW,qBAAqB;IACvE,OAAO,CAAC,GAAG,CAAwB;IAE7B,UAAU,CAAC,OAAO,EAAE,8BAA8B,GAAG,OAAO,CAAC,OAAO,CAAC;IA0CrE,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAQhC;AAED;;;;;GAKG;AACH,QAAA,MAAM,gBAAgB,EAAE,gBAgCvB,CAAC;AAEF,eAAe,gBAAgB,CAAC;AAChC,OAAO,EAAE,kCAAkC,EAAE,mBAAmB,EAAE,CAAC"}
@@ -1,18 +1,18 @@
1
1
  /**
2
- * ext-tracing-opentelemetry TracingExporter implementation backed by the
2
+ * ext-observability-opentelemetry: OpenTelemetry observability extension backed by the
3
3
  * official OpenTelemetry JS SDK.
4
4
  *
5
5
  * Provides the `TracingExporter` and `NodeTelemetryProvider` contracts:
6
- * - `start(config)` builds the SDK provider + OTLP HTTP exporter
7
- * - `export(spans)` no-op (SDK handles export via BatchSpanProcessor)
8
- * - `shutdown()` flushes + shuts down the provider
9
- * - `getProvider()` returns the SDK TracerProvider for shim wiring
10
- * - `initialize(options)` starts NodeSDK auto-instrumentation
6
+ * - `start(config)`: builds the SDK provider and OTLP HTTP exporter
7
+ * - `export(spans)`: no-op, the SDK handles export via BatchSpanProcessor
8
+ * - `shutdown()`: flushes and shuts down the provider
9
+ * - `getProvider()`: returns the SDK TracerProvider for shim wiring
10
+ * - `initialize(options)`: starts NodeSDK auto-instrumentation
11
11
  *
12
12
  * Configuration is read from `ctx.config` (see `OtlpExtConfig`) and falls
13
13
  * back to standard OTEL environment variables.
14
14
  *
15
- * @module extensions/ext-tracing-opentelemetry
15
+ * @module extensions/ext-observability-opentelemetry
16
16
  */
17
17
  import * as dntShim from "../../../_dnt.shims.js";
18
18
  import { metrics, trace } from "@opentelemetry/api";
@@ -182,7 +182,7 @@ class OpenTelemetryNodeTelemetryProvider {
182
182
  }
183
183
  }
184
184
  /**
185
- * Default export the ext-tracing-opentelemetry extension factory.
185
+ * Default export for the ext-observability-opentelemetry extension factory.
186
186
  *
187
187
  * Produces an extension that registers a `TracingExporter` contract
188
188
  * implementation backed by the OpenTelemetry JS SDK.
@@ -191,7 +191,7 @@ const extOpenTelemetry = () => {
191
191
  const exporterImpl = new OtlpTracingExporter();
192
192
  const nodeTelemetryProvider = new OpenTelemetryNodeTelemetryProvider();
193
193
  return {
194
- name: "ext-tracing-opentelemetry",
194
+ name: "ext-observability-opentelemetry",
195
195
  version: "0.1.0",
196
196
  capabilities: [
197
197
  { type: "contract", name: "TracingExporter" },
@@ -211,7 +211,7 @@ const extOpenTelemetry = () => {
211
211
  await exporterImpl.start(ctx.config);
212
212
  ctx.provide("TracingExporter", exporterImpl);
213
213
  ctx.provide("NodeTelemetryProvider", nodeTelemetryProvider);
214
- ctx.logger.info("[ext-tracing-opentelemetry] TracingExporter registered");
214
+ ctx.logger.info("[ext-observability-opentelemetry] TracingExporter registered");
215
215
  },
216
216
  async teardown() {
217
217
  await nodeTelemetryProvider.shutdown();
@@ -1 +1 @@
1
- {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/service/config.ts"],"names":[],"mappings":"AAWA,MAAM,MAAM,4BAA4B,GAAG,MAAM,GAAG,SAAS,GAAG,UAAU,CAAC;AAE3E,MAAM,MAAM,kBAAkB,GAAG;IAC/B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,oCAAoC,EAAE,4BAA4B,CAAC;IACnE,6CAA6C,EAAE,MAAM,CAAC;IACtD,8BAA8B,CAAC,EAAE,MAAM,CAAC;IACxC,wBAAwB,EAAE,MAAM,CAAC;IACjC,qCAAqC,EAAE,OAAO,CAAC;IAC/C,6BAA6B,EAAE,OAAO,CAAC;IACvC,QAAQ,EAAE,aAAa,GAAG,MAAM,GAAG,YAAY,CAAC;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,YAAY,EAAE,OAAO,CAAC;IACtB,2BAA2B,CAAC,EAAE,MAAM,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC,CAAC;AAsDlF,eAAO,MAAM,wBAAwB,sEAA0C,CAAC;AAEhF,eAAO,MAAM,8BAA8B,sEAA2B,CAAC;AACvE,MAAM,MAAM,wBAAwB,GAAG,kBAAkB,CAAC;AAC1D,MAAM,MAAM,6BAA6B,GAAG,uBAAuB,CAAC;AAEpE,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,uBAAuB,GAC7B,kBAAkB,CAEpB;AAED,wBAAgB,6BAA6B,CAC3C,KAAK,EAAE,6BAA6B,GACnC,wBAAwB,CAE1B"}
1
+ {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/service/config.ts"],"names":[],"mappings":"AAYA,MAAM,MAAM,4BAA4B,GAAG,MAAM,GAAG,SAAS,GAAG,UAAU,CAAC;AAE3E,MAAM,MAAM,kBAAkB,GAAG;IAC/B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,oCAAoC,EAAE,4BAA4B,CAAC;IACnE,6CAA6C,EAAE,MAAM,CAAC;IACtD,8BAA8B,CAAC,EAAE,MAAM,CAAC;IACxC,wBAAwB,EAAE,MAAM,CAAC;IACjC,qCAAqC,EAAE,OAAO,CAAC;IAC/C,6BAA6B,EAAE,OAAO,CAAC;IACvC,QAAQ,EAAE,aAAa,GAAG,MAAM,GAAG,YAAY,CAAC;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,YAAY,EAAE,OAAO,CAAC;IACtB,2BAA2B,CAAC,EAAE,MAAM,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC,CAAC;AAsDlF,eAAO,MAAM,wBAAwB,sEAA0C,CAAC;AAEhF,eAAO,MAAM,8BAA8B,sEAA2B,CAAC;AACvE,MAAM,MAAM,wBAAwB,GAAG,kBAAkB,CAAC;AAC1D,MAAM,MAAM,6BAA6B,GAAG,uBAAuB,CAAC;AAEpE,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,uBAAuB,GAC7B,kBAAkB,CAGpB;AAED,wBAAgB,6BAA6B,CAC3C,KAAK,EAAE,6BAA6B,GACnC,wBAAwB,CAE1B"}
@@ -1,3 +1,4 @@
1
+ import { ensureBuiltinSchemaValidator } from "../../extensions/builtin-extensions.js";
1
2
  import { defineSchema } from "../../schemas/define.js";
2
3
  import { lazySchema } from "../../schemas/lazy.js";
3
4
  function parseBooleanFlag(value) {
@@ -60,6 +61,7 @@ const getAgentServiceConfigSchema = defineSchema((v) => {
60
61
  export const agentServiceConfigSchema = lazySchema(getAgentServiceConfigSchema);
61
62
  export const hostedAgentServiceConfigSchema = agentServiceConfigSchema;
62
63
  export function parseAgentServiceConfig(input) {
64
+ ensureBuiltinSchemaValidator();
63
65
  return agentServiceConfigSchema.parse(input);
64
66
  }
65
67
  export function parseHostedAgentServiceConfig(input) {
@@ -1,4 +1,4 @@
1
- import { type NodeTelemetryInstrumentationConfig, type NodeTelemetryLogger, type NodeTelemetryProcessTarget, type NodeTelemetryProvider } from "../../extensions/tracing/index.js";
1
+ import { type NodeTelemetryInstrumentationConfig, type NodeTelemetryLogger, type NodeTelemetryProcessTarget, type NodeTelemetryProvider } from "../../extensions/observability/index.js";
2
2
  export type NodeHostedAgentServiceTelemetryEnv = Record<string, string | undefined>;
3
3
  export type NodeAgentServiceTelemetryEnv = NodeHostedAgentServiceTelemetryEnv;
4
4
  export type NodeHostedAgentServiceInstrumentationConfig = NodeTelemetryInstrumentationConfig;
@@ -1 +1 @@
1
- {"version":3,"file":"node-telemetry.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/service/node-telemetry.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,kCAAkC,EACvC,KAAK,mBAAmB,EACxB,KAAK,0BAA0B,EAC/B,KAAK,qBAAqB,EAE3B,MAAM,mCAAmC,CAAC;AAE3C,MAAM,MAAM,kCAAkC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;AAEpF,MAAM,MAAM,4BAA4B,GAAG,kCAAkC,CAAC;AAE9E,MAAM,MAAM,2CAA2C,GAAG,kCAAkC,CAAC;AAE7F,MAAM,MAAM,qCAAqC,GAAG,2CAA2C,CAAC;AAEhG,MAAM,MAAM,qCAAqC,GAAG;IAClD,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,eAAe,EAAE,2CAA2C,CAAC;CAC9D,CAAC;AAEF,MAAM,MAAM,+BAA+B,GAAG,qCAAqC,CAAC;AAEpF,MAAM,MAAM,mDAAmD,GAAG;IAChE,GAAG,EAAE,kCAAkC,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,6CAA6C,GACvD,mDAAmD,CAAC;AAEtD,MAAM,MAAM,qCAAqC,GAAG,mBAAmB,CAAC;AAExE,MAAM,MAAM,+BAA+B,GAAG,qCAAqC,CAAC;AAEpF,MAAM,MAAM,4CAA4C,GAAG,0BAA0B,CAAC;AAEtF,MAAM,MAAM,sCAAsC,GAAG,4CAA4C,CAAC;AAElG,MAAM,MAAM,gDAAgD,GACxD,qCAAqC,GACrC;IACA,MAAM,CAAC,EAAE,qCAAqC,CAAC;IAC/C,aAAa,CAAC,EAAE,4CAA4C,CAAC;IAC7D,iBAAiB,CAAC,EAAE,qBAAqB,CAAC;CAC3C,CAAC;AAEJ,MAAM,MAAM,0CAA0C,GACpD,gDAAgD,CAAC;AA6CnD,wBAAgB,4CAA4C,CAC1D,OAAO,EAAE,mDAAmD,GAC3D,qCAAqC,CAYvC;AAED,wBAAgB,sCAAsC,CACpD,OAAO,EAAE,6CAA6C,GACrD,+BAA+B,CAEjC;AAgCD,wBAAsB,6CAA6C,CACjE,OAAO,EAAE,gDAAgD,GACxD,OAAO,CAAC,OAAO,CAAC,CAkClB;AAED,wBAAsB,uCAAuC,CAC3D,OAAO,EAAE,0CAA0C,GAClD,OAAO,CAAC,OAAO,CAAC,CAElB"}
1
+ {"version":3,"file":"node-telemetry.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/service/node-telemetry.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,kCAAkC,EACvC,KAAK,mBAAmB,EACxB,KAAK,0BAA0B,EAC/B,KAAK,qBAAqB,EAE3B,MAAM,yCAAyC,CAAC;AAEjD,MAAM,MAAM,kCAAkC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;AAEpF,MAAM,MAAM,4BAA4B,GAAG,kCAAkC,CAAC;AAE9E,MAAM,MAAM,2CAA2C,GAAG,kCAAkC,CAAC;AAE7F,MAAM,MAAM,qCAAqC,GAAG,2CAA2C,CAAC;AAEhG,MAAM,MAAM,qCAAqC,GAAG;IAClD,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,eAAe,EAAE,2CAA2C,CAAC;CAC9D,CAAC;AAEF,MAAM,MAAM,+BAA+B,GAAG,qCAAqC,CAAC;AAEpF,MAAM,MAAM,mDAAmD,GAAG;IAChE,GAAG,EAAE,kCAAkC,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,6CAA6C,GACvD,mDAAmD,CAAC;AAEtD,MAAM,MAAM,qCAAqC,GAAG,mBAAmB,CAAC;AAExE,MAAM,MAAM,+BAA+B,GAAG,qCAAqC,CAAC;AAEpF,MAAM,MAAM,4CAA4C,GAAG,0BAA0B,CAAC;AAEtF,MAAM,MAAM,sCAAsC,GAAG,4CAA4C,CAAC;AAElG,MAAM,MAAM,gDAAgD,GACxD,qCAAqC,GACrC;IACA,MAAM,CAAC,EAAE,qCAAqC,CAAC;IAC/C,aAAa,CAAC,EAAE,4CAA4C,CAAC;IAC7D,iBAAiB,CAAC,EAAE,qBAAqB,CAAC;CAC3C,CAAC;AAEJ,MAAM,MAAM,0CAA0C,GACpD,gDAAgD,CAAC;AA6CnD,wBAAgB,4CAA4C,CAC1D,OAAO,EAAE,mDAAmD,GAC3D,qCAAqC,CAYvC;AAED,wBAAgB,sCAAsC,CACpD,OAAO,EAAE,6CAA6C,GACrD,+BAA+B,CAEjC;AAgCD,wBAAsB,6CAA6C,CACjE,OAAO,EAAE,gDAAgD,GACxD,OAAO,CAAC,OAAO,CAAC,CAkClB;AAED,wBAAsB,uCAAuC,CAC3D,OAAO,EAAE,0CAA0C,GAClD,OAAO,CAAC,OAAO,CAAC,CAElB"}
@@ -1,5 +1,5 @@
1
1
  import { tryResolve } from "../../extensions/contracts.js";
2
- import { NodeTelemetryProviderName, } from "../../extensions/tracing/index.js";
2
+ import { NodeTelemetryProviderName, } from "../../extensions/observability/index.js";
3
3
  function resolveEnabled(env, defaultEnabled) {
4
4
  const envValue = env.OTEL_ENABLED;
5
5
  if (envValue !== undefined) {
@@ -5,5 +5,5 @@
5
5
  */
6
6
  import "../../../_dnt.polyfills.js";
7
7
  export { assertCompleted, assertContains, assertToolCalled, printTestResults, testAgent, type TestCase, type TestResult, type TestSuite, } from "./agent-tester.js";
8
- export { buildFailureSuffix, buildLiveEvalCaseTagSummary, buildLiveEvalRequestBody, type BuildLiveEvalRequestBodyInput, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, buildProgressLine, buildRuntimePerformanceSummary, containsOrderedSubsequence, createFailedEvalResult, createPassedEvalResult, createPlainTextPdf, createSkippedEvalResult, hasEveryLiveEvalTag, type LiveEvalCaseMetadata, type LiveEvalCaseSelectionInput, type LiveEvalResultForPerformance, type LiveEvalResultForReport, type LiveEvalResultRecord, type LiveEvalRuntime, resolveLiveEvalRequestedCaseIds, type RuntimePerformanceSummary, selectLiveEvalCases, } from "./live-evals/index.js";
8
+ export { buildFailureSuffix, buildLiveEvalCaseTagSummary, buildLiveEvalRequestBody, type BuildLiveEvalRequestBodyInput, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, buildProgressLine, buildRuntimePerformanceSummary, containsOrderedSubsequence, containsSkillLoad, countStepStartedEvents, createFailedEvalResult, createLiveEvalCaseSupport, createPassedEvalResult, createPlainTextPdf, createSkippedEvalResult, hasEveryLiveEvalTag, hasFinished, type LiveEvalCase, type LiveEvalCaseMetadata, type LiveEvalCaseSelectionInput, type LiveEvalContext, type LiveEvalProjectFile, type LiveEvalProjectFileReaderInput, type LiveEvalRequestBody, type LiveEvalResultForPerformance, type LiveEvalResultForReport, type LiveEvalResultRecord, type LiveEvalRunnerConfig, liveEvalRunnerInternals, type LiveEvalRuntime, type PreparedLiveEvalInput, resolveLiveEvalRequestedCaseIds, type RuntimePerformanceSummary, selectLiveEvalCases, } from "./live-evals/index.js";
9
9
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/testing/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,4BAA4B,CAAC;AAGpC,OAAO,EACL,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,SAAS,EACT,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,SAAS,GACf,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,kBAAkB,EAClB,2BAA2B,EAC3B,wBAAwB,EACxB,KAAK,6BAA6B,EAClC,2BAA2B,EAC3B,0BAA0B,EAC1B,iBAAiB,EACjB,8BAA8B,EAC9B,0BAA0B,EAC1B,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,uBAAuB,EACvB,mBAAmB,EACnB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,EAC/B,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAC5B,KAAK,oBAAoB,EACzB,KAAK,eAAe,EACpB,+BAA+B,EAC/B,KAAK,yBAAyB,EAC9B,mBAAmB,GACpB,MAAM,uBAAuB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/testing/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,4BAA4B,CAAC;AAGpC,OAAO,EACL,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,SAAS,EACT,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,SAAS,GACf,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,kBAAkB,EAClB,2BAA2B,EAC3B,wBAAwB,EACxB,KAAK,6BAA6B,EAClC,2BAA2B,EAC3B,0BAA0B,EAC1B,iBAAiB,EACjB,8BAA8B,EAC9B,0BAA0B,EAC1B,iBAAiB,EACjB,sBAAsB,EACtB,sBAAsB,EACtB,yBAAyB,EACzB,sBAAsB,EACtB,kBAAkB,EAClB,uBAAuB,EACvB,mBAAmB,EACnB,WAAW,EACX,KAAK,YAAY,EACjB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,EAC/B,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,8BAA8B,EACnC,KAAK,mBAAmB,EACxB,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAC5B,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,uBAAuB,EACvB,KAAK,eAAe,EACpB,KAAK,qBAAqB,EAC1B,+BAA+B,EAC/B,KAAK,yBAAyB,EAC9B,mBAAmB,GACpB,MAAM,uBAAuB,CAAC"}
@@ -5,4 +5,4 @@
5
5
  */
6
6
  import "../../../_dnt.polyfills.js";
7
7
  export { assertCompleted, assertContains, assertToolCalled, printTestResults, testAgent, } from "./agent-tester.js";
8
- export { buildFailureSuffix, buildLiveEvalCaseTagSummary, buildLiveEvalRequestBody, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, buildProgressLine, buildRuntimePerformanceSummary, containsOrderedSubsequence, createFailedEvalResult, createPassedEvalResult, createPlainTextPdf, createSkippedEvalResult, hasEveryLiveEvalTag, resolveLiveEvalRequestedCaseIds, selectLiveEvalCases, } from "./live-evals/index.js";
8
+ export { buildFailureSuffix, buildLiveEvalCaseTagSummary, buildLiveEvalRequestBody, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, buildProgressLine, buildRuntimePerformanceSummary, containsOrderedSubsequence, containsSkillLoad, countStepStartedEvents, createFailedEvalResult, createLiveEvalCaseSupport, createPassedEvalResult, createPlainTextPdf, createSkippedEvalResult, hasEveryLiveEvalTag, hasFinished, liveEvalRunnerInternals, resolveLiveEvalRequestedCaseIds, selectLiveEvalCases, } from "./live-evals/index.js";
@@ -1,6 +1,7 @@
1
1
  export { buildFailureSuffix, buildProgressLine, containsOrderedSubsequence, createPlainTextPdf, } from "./formatting.js";
2
2
  export { buildRuntimePerformanceSummary, type LiveEvalResultForPerformance, type LiveEvalRuntime, type RuntimePerformanceSummary, } from "./performance.js";
3
- export { buildLiveEvalRequestBody, type BuildLiveEvalRequestBodyInput } from "./request.js";
3
+ export { buildLiveEvalRequestBody, type BuildLiveEvalRequestBodyInput, type LiveEvalRequestBody, } from "./request.js";
4
4
  export { buildLiveEvalCaseTagSummary, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, hasEveryLiveEvalTag, type LiveEvalCaseMetadata, type LiveEvalCaseSelectionInput, type LiveEvalResultForReport, resolveLiveEvalRequestedCaseIds, selectLiveEvalCases, } from "./report.js";
5
5
  export { createFailedEvalResult, createPassedEvalResult, createSkippedEvalResult, type LiveEvalResultRecord, } from "./result.js";
6
+ export { containsSkillLoad, countStepStartedEvents, createLiveEvalCaseSupport, hasFinished, type LiveEvalCase, type LiveEvalContext, type LiveEvalProjectFile, type LiveEvalProjectFileReaderInput, type LiveEvalRunnerConfig, liveEvalRunnerInternals, type PreparedLiveEvalInput, } from "./runner.js";
6
7
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,0BAA0B,EAC1B,kBAAkB,GACnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,8BAA8B,EAC9B,KAAK,4BAA4B,EACjC,KAAK,eAAe,EACpB,KAAK,yBAAyB,GAC/B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,wBAAwB,EAAE,KAAK,6BAA6B,EAAE,MAAM,cAAc,CAAC;AAC5F,OAAO,EACL,2BAA2B,EAC3B,2BAA2B,EAC3B,0BAA0B,EAC1B,mBAAmB,EACnB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,EAC/B,KAAK,uBAAuB,EAC5B,+BAA+B,EAC/B,mBAAmB,GACpB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,KAAK,oBAAoB,GAC1B,MAAM,aAAa,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,0BAA0B,EAC1B,kBAAkB,GACnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,8BAA8B,EAC9B,KAAK,4BAA4B,EACjC,KAAK,eAAe,EACpB,KAAK,yBAAyB,GAC/B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,wBAAwB,EACxB,KAAK,6BAA6B,EAClC,KAAK,mBAAmB,GACzB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,2BAA2B,EAC3B,2BAA2B,EAC3B,0BAA0B,EAC1B,mBAAmB,EACnB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,EAC/B,KAAK,uBAAuB,EAC5B,+BAA+B,EAC/B,mBAAmB,GACpB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,KAAK,oBAAoB,GAC1B,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,iBAAiB,EACjB,sBAAsB,EACtB,yBAAyB,EACzB,WAAW,EACX,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,8BAA8B,EACnC,KAAK,oBAAoB,EACzB,uBAAuB,EACvB,KAAK,qBAAqB,GAC3B,MAAM,aAAa,CAAC"}
@@ -1,5 +1,6 @@
1
1
  export { buildFailureSuffix, buildProgressLine, containsOrderedSubsequence, createPlainTextPdf, } from "./formatting.js";
2
2
  export { buildRuntimePerformanceSummary, } from "./performance.js";
3
- export { buildLiveEvalRequestBody } from "./request.js";
3
+ export { buildLiveEvalRequestBody, } from "./request.js";
4
4
  export { buildLiveEvalCaseTagSummary, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, hasEveryLiveEvalTag, resolveLiveEvalRequestedCaseIds, selectLiveEvalCases, } from "./report.js";
5
5
  export { createFailedEvalResult, createPassedEvalResult, createSkippedEvalResult, } from "./result.js";
6
+ export { containsSkillLoad, countStepStartedEvents, createLiveEvalCaseSupport, hasFinished, liveEvalRunnerInternals, } from "./runner.js";
@@ -1,3 +1,18 @@
1
+ export interface LiveEvalRequestBody {
2
+ threadId: string;
3
+ runId: string;
4
+ state: Record<string, string>;
5
+ tools: unknown[];
6
+ context: unknown[];
7
+ forwardedProps?: {
8
+ veryfront: Record<string, unknown>;
9
+ };
10
+ messages: Array<{
11
+ id: string;
12
+ role: "user";
13
+ content: string;
14
+ }>;
15
+ }
1
16
  export interface BuildLiveEvalRequestBodyInput {
2
17
  testCaseId: string;
3
18
  prompt: string;
@@ -10,21 +25,5 @@ export interface BuildLiveEvalRequestBodyInput {
10
25
  forceRuntimeOverrides?: boolean;
11
26
  maxSteps?: number;
12
27
  }
13
- export declare function buildLiveEvalRequestBody(input: BuildLiveEvalRequestBodyInput): {
14
- messages: {
15
- id: any;
16
- role: "user";
17
- content: string;
18
- }[];
19
- forwardedProps?: {
20
- veryfront: Record<string, unknown>;
21
- } | undefined;
22
- threadId: any;
23
- runId: string;
24
- state: {
25
- evalCase: string;
26
- };
27
- tools: never[];
28
- context: never[];
29
- };
28
+ export declare function buildLiveEvalRequestBody(input: BuildLiveEvalRequestBodyInput): LiveEvalRequestBody;
30
29
  //# sourceMappingURL=request.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/request.ts"],"names":[],"mappings":"AACA,MAAM,WAAW,6BAA6B;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,6BAA6B;;;;;;;;;;;;;;;;EA6C5E"}
1
+ {"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/request.ts"],"names":[],"mappings":"AACA,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,KAAK,EAAE,OAAO,EAAE,CAAC;IACjB,OAAO,EAAE,OAAO,EAAE,CAAC;IACnB,cAAc,CAAC,EAAE;QACf,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,CAAC;IACF,QAAQ,EAAE,KAAK,CAAC;QACd,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,6BAA6B;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,6BAA6B,GACnC,mBAAmB,CA6CrB"}
@@ -0,0 +1,124 @@
1
+ import { type AgUiSseProgressSnapshot as EvalProgressSnapshot, type ParsedAgUiSseRun as ParsedRun } from "../../index.js";
2
+ import { type LiveEvalRuntime } from "./performance.js";
3
+ import { type LiveEvalCaseMetadata } from "./report.js";
4
+ import { type LiveEvalResultRecord } from "./result.js";
5
+ export interface PreparedLiveEvalInput {
6
+ prompt?: string;
7
+ metadata?: Record<string, string>;
8
+ verificationContext?: LiveEvalContext;
9
+ cleanup?: () => Promise<void>;
10
+ startSidecar?: () => Promise<(() => Promise<void>) | void>;
11
+ }
12
+ export interface LiveEvalContext {
13
+ apiUrl: string;
14
+ authToken: string;
15
+ projectId: string | null;
16
+ }
17
+ export interface LiveEvalCase {
18
+ readonly id: string;
19
+ readonly label: string;
20
+ readonly prompt?: string;
21
+ allowedTools?: string[];
22
+ forceRuntimeOverrides?: boolean;
23
+ requireProject?: boolean;
24
+ maxSteps?: number;
25
+ expectedEventSubsequence?: string[];
26
+ metadata?: LiveEvalCaseMetadata;
27
+ prepare?: (context: LiveEvalContext) => Promise<PreparedLiveEvalInput>;
28
+ verify: (run: ParsedRun, prepared: PreparedLiveEvalInput | null) => string | null | Promise<string | null>;
29
+ }
30
+ interface FileCheckInput {
31
+ filePath: string;
32
+ requiredContent?: string[];
33
+ description?: string;
34
+ }
35
+ export interface LiveEvalProjectFile {
36
+ path: string;
37
+ content: string;
38
+ }
39
+ export interface LiveEvalProjectFileReaderInput {
40
+ filePath: string;
41
+ requestTimeoutMs: number;
42
+ }
43
+ export interface LiveEvalRunnerConfig {
44
+ endpoint: string;
45
+ authToken: string;
46
+ apiUrl: string;
47
+ projectId: string | null;
48
+ branchId: string | null;
49
+ model: string | null;
50
+ requestTimeoutMs: number;
51
+ progressLogIntervalMs: number;
52
+ enableLlmJudge: boolean;
53
+ fetch?: (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
54
+ log?: (message: string) => void;
55
+ readProjectFile?: (input: LiveEvalProjectFileReaderInput) => Promise<LiveEvalProjectFile | null>;
56
+ }
57
+ interface LiveEvalJudgeInput {
58
+ question: string;
59
+ criteria: string;
60
+ }
61
+ interface LiveEvalJudgeRequest extends LiveEvalJudgeInput {
62
+ answer: string;
63
+ }
64
+ interface LiveEvalJudgeResult {
65
+ pass: boolean;
66
+ reason: string;
67
+ }
68
+ declare function collectPreparedArtifactPaths(prepared: PreparedLiveEvalInput | null): string[];
69
+ interface LiveEvalResultContext {
70
+ id: string;
71
+ label: string;
72
+ runtime: LiveEvalRuntime;
73
+ startedAt: number;
74
+ conversationId?: string | null;
75
+ artifactPaths?: string[];
76
+ }
77
+ interface LiveEvalRunArtifactsInput {
78
+ run: ParsedRun;
79
+ runId?: string;
80
+ traceSignature: string;
81
+ }
82
+ interface LiveEvalRunArtifacts {
83
+ runId?: string;
84
+ traceSignature: string;
85
+ toolStarts: string[];
86
+ toolArgsPreview: string;
87
+ textPreview: string;
88
+ }
89
+ declare function createLiveEvalRunArtifacts(input: LiveEvalRunArtifactsInput): LiveEvalRunArtifacts;
90
+ declare function createFailedRunEvalResult(input: {
91
+ details: string;
92
+ context: LiveEvalResultContext;
93
+ runArtifacts: LiveEvalRunArtifacts;
94
+ }): LiveEvalResultRecord;
95
+ declare function createPassedRunEvalResult(input: {
96
+ details: string;
97
+ context: LiveEvalResultContext;
98
+ runArtifacts: LiveEvalRunArtifacts;
99
+ }): LiveEvalResultRecord;
100
+ declare function createStreamingFailureEvalResult(input: {
101
+ details: string;
102
+ context: LiveEvalResultContext;
103
+ progress: EvalProgressSnapshot;
104
+ }): LiveEvalResultRecord;
105
+ declare function extractRunId(run: ParsedRun): string | null;
106
+ export declare function hasFinished(run: ParsedRun): boolean;
107
+ export declare function containsSkillLoad(run: ParsedRun, skillId: string): boolean;
108
+ export declare function countStepStartedEvents(run: ParsedRun): number;
109
+ export declare function createLiveEvalCaseSupport(config: LiveEvalRunnerConfig): {
110
+ runEval: (testCase: LiveEvalCase, runtime: LiveEvalRuntime) => Promise<LiveEvalResultRecord>;
111
+ verifyFileExists: (input: FileCheckInput) => Promise<string | null>;
112
+ withJudge: (structuralVerify: (run: ParsedRun) => string | null, judgeInput: LiveEvalJudgeInput) => (run: ParsedRun) => Promise<string | null>;
113
+ judgeLlm: (input: LiveEvalJudgeRequest) => Promise<LiveEvalJudgeResult>;
114
+ };
115
+ export declare const liveEvalRunnerInternals: {
116
+ collectPreparedArtifactPaths: typeof collectPreparedArtifactPaths;
117
+ createFailedRunEvalResult: typeof createFailedRunEvalResult;
118
+ createLiveEvalRunArtifacts: typeof createLiveEvalRunArtifacts;
119
+ createPassedRunEvalResult: typeof createPassedRunEvalResult;
120
+ createStreamingFailureEvalResult: typeof createStreamingFailureEvalResult;
121
+ extractRunId: typeof extractRunId;
122
+ };
123
+ export {};
124
+ //# sourceMappingURL=runner.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/runner.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,uBAAuB,IAAI,oBAAoB,EAIpD,KAAK,gBAAgB,IAAI,SAAS,EACnC,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAExD,OAAO,EAAE,KAAK,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAIL,KAAK,oBAAoB,EAC1B,MAAM,aAAa,CAAC;AAErB,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,mBAAmB,CAAC,EAAE,eAAe,CAAC;IACtC,OAAO,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;CAC5D;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wBAAwB,CAAC,EAAE,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IAChC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACvE,MAAM,EAAE,CACN,GAAG,EAAE,SAAS,EACd,QAAQ,EAAE,qBAAqB,GAAG,IAAI,KACnC,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAC7C;AAED,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,cAAc,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjF,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,8BAA8B,KAAK,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;CAClG;AAED,UAAU,kBAAkB;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,oBAAqB,SAAQ,kBAAkB;IACvD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,mBAAmB;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAyJD,iBAAS,4BAA4B,CAAC,QAAQ,EAAE,qBAAqB,GAAG,IAAI,GAAG,MAAM,EAAE,CAYtF;AASD,UAAU,qBAAqB;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,eAAe,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,UAAU,yBAAyB;IACjC,GAAG,EAAE,SAAS,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,UAAU,oBAAoB;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,iBAAS,0BAA0B,CAAC,KAAK,EAAE,yBAAyB,GAAG,oBAAoB,CAQ1F;AAED,iBAAS,yBAAyB,CAAC,KAAK,EAAE;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,YAAY,EAAE,oBAAoB,CAAC;CACpC,GAAG,oBAAoB,CAevB;AAED,iBAAS,yBAAyB,CAAC,KAAK,EAAE;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,YAAY,EAAE,oBAAoB,CAAC;CACpC,GAAG,oBAAoB,CAevB;AAED,iBAAS,gCAAgC,CAAC,KAAK,EAAE;IAC/C,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,QAAQ,EAAE,oBAAoB,CAAC;CAChC,GAAG,oBAAoB,CAcvB;AA+FD,iBAAS,YAAY,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,GAAG,IAAI,CASnD;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAEnD;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAE1E;AAED,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,CAE7D;AAED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,oBAAoB,GAAG;IACvE,OAAO,EAAE,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC7F,gBAAgB,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACpE,SAAS,EAAE,CACT,gBAAgB,EAAE,CAAC,GAAG,EAAE,SAAS,KAAK,MAAM,GAAG,IAAI,EACnD,UAAU,EAAE,kBAAkB,KAC3B,CAAC,GAAG,EAAE,SAAS,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAChD,QAAQ,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,OAAO,CAAC,mBAAmB,CAAC,CAAC;CACzE,CAoIA;AAED,eAAO,MAAM,uBAAuB;;;;;;;CAOnC,CAAC"}