veryfront 0.1.871 → 0.1.873
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/cli/commands/eval/command-help.d.ts +3 -0
- package/esm/cli/commands/eval/command-help.d.ts.map +1 -0
- package/esm/cli/commands/eval/command-help.js +42 -0
- package/esm/cli/commands/eval/command.d.ts +21 -0
- package/esm/cli/commands/eval/command.d.ts.map +1 -0
- package/esm/cli/commands/eval/command.js +289 -0
- package/esm/cli/commands/eval/handler.d.ts +26 -0
- package/esm/cli/commands/eval/handler.d.ts.map +1 -0
- package/esm/cli/commands/eval/handler.js +28 -0
- package/esm/cli/help/command-definitions.d.ts.map +1 -1
- package/esm/cli/help/command-definitions.js +2 -0
- package/esm/cli/router.d.ts.map +1 -1
- package/esm/cli/router.js +2 -0
- package/esm/deno.d.ts +5 -2
- package/esm/deno.js +7 -4
- package/esm/src/agent/runtime/client-profile.d.ts.map +1 -1
- package/esm/src/agent/runtime/client-profile.js +10 -1
- package/esm/src/chat/ag-ui.d.ts.map +1 -1
- package/esm/src/chat/protocol.d.ts.map +1 -1
- package/esm/src/chat/protocol.js +0 -1
- package/esm/src/config/schemas/config.schema.d.ts +12 -0
- package/esm/src/config/schemas/config.schema.d.ts.map +1 -1
- package/esm/src/config/schemas/config.schema.js +12 -0
- package/esm/src/discovery/discovery-engine.d.ts.map +1 -1
- package/esm/src/discovery/discovery-engine.js +6 -1
- package/esm/src/discovery/handlers/eval-handler.d.ts +7 -0
- package/esm/src/discovery/handlers/eval-handler.d.ts.map +1 -0
- package/esm/src/discovery/handlers/eval-handler.js +16 -0
- package/esm/src/discovery/handlers/index.d.ts +1 -0
- package/esm/src/discovery/handlers/index.d.ts.map +1 -1
- package/esm/src/discovery/handlers/index.js +1 -0
- package/esm/src/discovery/import-rewriter.d.ts +1 -1
- package/esm/src/discovery/import-rewriter.d.ts.map +1 -1
- package/esm/src/discovery/import-rewriter.js +1 -0
- package/esm/src/discovery/project-discovery-config.d.ts +2 -0
- package/esm/src/discovery/project-discovery-config.d.ts.map +1 -1
- package/esm/src/discovery/project-discovery-config.js +2 -0
- package/esm/src/discovery/transpiler.d.ts.map +1 -1
- package/esm/src/discovery/transpiler.js +2 -0
- package/esm/src/discovery/types.d.ts +3 -0
- package/esm/src/discovery/types.d.ts.map +1 -1
- package/esm/src/eval/datasets.d.ts +8 -0
- package/esm/src/eval/datasets.d.ts.map +1 -0
- package/esm/src/eval/datasets.js +66 -0
- package/esm/src/eval/discovery.d.ts +43 -0
- package/esm/src/eval/discovery.d.ts.map +1 -0
- package/esm/src/eval/discovery.js +123 -0
- package/esm/src/eval/expect.d.ts +16 -0
- package/esm/src/eval/expect.d.ts.map +1 -0
- package/esm/src/eval/expect.js +62 -0
- package/esm/src/eval/factory.d.ts +6 -0
- package/esm/src/eval/factory.d.ts.map +1 -0
- package/esm/src/eval/factory.js +50 -0
- package/esm/src/eval/index.d.ts +33 -0
- package/esm/src/eval/index.d.ts.map +1 -0
- package/esm/src/eval/index.js +29 -0
- package/esm/src/eval/metrics.d.ts +55 -0
- package/esm/src/eval/metrics.d.ts.map +1 -0
- package/esm/src/eval/metrics.js +223 -0
- package/esm/src/eval/report.d.ts +12 -0
- package/esm/src/eval/report.d.ts.map +1 -0
- package/esm/src/eval/report.js +64 -0
- package/esm/src/eval/runner.d.ts +4 -0
- package/esm/src/eval/runner.d.ts.map +1 -0
- package/esm/src/eval/runner.js +101 -0
- package/esm/src/eval/studio.d.ts +177 -0
- package/esm/src/eval/studio.d.ts.map +1 -0
- package/esm/src/eval/studio.js +176 -0
- package/esm/src/eval/types.d.ts +221 -0
- package/esm/src/eval/types.d.ts.map +1 -0
- package/esm/src/eval/types.js +6 -0
- package/esm/src/eval/validation.d.ts +4 -0
- package/esm/src/eval/validation.d.ts.map +1 -0
- package/esm/src/eval/validation.js +41 -0
- package/esm/src/runs/index.d.ts +1 -1
- package/esm/src/runs/index.js +1 -1
- package/esm/src/runs/schemas.d.ts +9 -9
- package/esm/src/runs/schemas.d.ts.map +1 -1
- package/esm/src/runs/schemas.js +1 -1
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +12 -4
- package/esm/src/agent/testing/agent-tester.d.ts +0 -53
- package/esm/src/agent/testing/agent-tester.d.ts.map +0 -1
- package/esm/src/agent/testing/agent-tester.js +0 -132
- package/esm/src/agent/testing/durable-run-canaries/cli-runner.d.ts +0 -21
- package/esm/src/agent/testing/durable-run-canaries/cli-runner.d.ts.map +0 -1
- package/esm/src/agent/testing/durable-run-canaries/cli-runner.js +0 -66
- package/esm/src/agent/testing/durable-run-canaries/environment.d.ts +0 -14
- package/esm/src/agent/testing/durable-run-canaries/environment.d.ts.map +0 -1
- package/esm/src/agent/testing/durable-run-canaries/environment.js +0 -15
- package/esm/src/agent/testing/durable-run-canaries/index.d.ts +0 -5
- package/esm/src/agent/testing/durable-run-canaries/index.d.ts.map +0 -1
- package/esm/src/agent/testing/durable-run-canaries/index.js +0 -4
- package/esm/src/agent/testing/durable-run-canaries/runner.d.ts +0 -118
- package/esm/src/agent/testing/durable-run-canaries/runner.d.ts.map +0 -1
- package/esm/src/agent/testing/durable-run-canaries/runner.js +0 -391
- package/esm/src/agent/testing/durable-run-canaries/validation.d.ts +0 -12
- package/esm/src/agent/testing/durable-run-canaries/validation.d.ts.map +0 -1
- package/esm/src/agent/testing/durable-run-canaries/validation.js +0 -71
- package/esm/src/agent/testing/index.d.ts +0 -10
- package/esm/src/agent/testing/index.d.ts.map +0 -1
- package/esm/src/agent/testing/index.js +0 -9
- package/esm/src/agent/testing/live-evals/api-client.d.ts +0 -102
- package/esm/src/agent/testing/live-evals/api-client.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/api-client.js +0 -313
- package/esm/src/agent/testing/live-evals/cli-runner.d.ts +0 -40
- package/esm/src/agent/testing/live-evals/cli-runner.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/cli-runner.js +0 -144
- package/esm/src/agent/testing/live-evals/environment.d.ts +0 -15
- package/esm/src/agent/testing/live-evals/environment.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/environment.js +0 -20
- package/esm/src/agent/testing/live-evals/formatting.d.ts +0 -15
- package/esm/src/agent/testing/live-evals/formatting.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/formatting.js +0 -75
- package/esm/src/agent/testing/live-evals/index.d.ts +0 -12
- package/esm/src/agent/testing/live-evals/index.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/index.js +0 -11
- package/esm/src/agent/testing/live-evals/metadata.d.ts +0 -34
- package/esm/src/agent/testing/live-evals/metadata.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/metadata.js +0 -116
- package/esm/src/agent/testing/live-evals/performance.d.ts +0 -19
- package/esm/src/agent/testing/live-evals/performance.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/performance.js +0 -22
- package/esm/src/agent/testing/live-evals/preflight.d.ts +0 -12
- package/esm/src/agent/testing/live-evals/preflight.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/preflight.js +0 -27
- package/esm/src/agent/testing/live-evals/report.d.ts +0 -55
- package/esm/src/agent/testing/live-evals/report.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/report.js +0 -79
- package/esm/src/agent/testing/live-evals/request.d.ts +0 -31
- package/esm/src/agent/testing/live-evals/request.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/request.js +0 -48
- package/esm/src/agent/testing/live-evals/result.d.ts +0 -56
- package/esm/src/agent/testing/live-evals/result.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/result.js +0 -36
- package/esm/src/agent/testing/live-evals/runner.d.ts +0 -135
- package/esm/src/agent/testing/live-evals/runner.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/runner.js +0 -396
package/esm/src/runs/index.d.ts
CHANGED
package/esm/src/runs/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { InferSchema } from "../extensions/schema/index.js";
|
|
2
|
-
export declare const getRunKindSchema: () => import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
|
|
2
|
+
export declare const getRunKindSchema: () => import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
|
|
3
3
|
export declare const getRunStatusSchema: () => import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
|
|
4
4
|
export declare const getRunOwnerSchema: () => import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
5
5
|
kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
|
|
@@ -14,7 +14,7 @@ export declare const getRunExecutionErrorSchema: () => import("../internal-agent
|
|
|
14
14
|
}>>;
|
|
15
15
|
export declare const getRunSchema: () => import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
16
16
|
run_id: import("../internal-agents/schema.js").Schema<string>;
|
|
17
|
-
kind: import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
|
|
17
|
+
kind: import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
|
|
18
18
|
status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
|
|
19
19
|
owner: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
20
20
|
kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
|
|
@@ -59,7 +59,7 @@ export declare const getCreateRunResponseSchema: () => import("../internal-agent
|
|
|
59
59
|
duplicate: import("../internal-agents/schema.js").Schema<boolean | undefined>;
|
|
60
60
|
run: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
61
61
|
run_id: import("../internal-agents/schema.js").Schema<string>;
|
|
62
|
-
kind: import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
|
|
62
|
+
kind: import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
|
|
63
63
|
status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
|
|
64
64
|
owner: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
65
65
|
kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
|
|
@@ -104,7 +104,7 @@ export declare const getCancelRunResponseSchema: () => import("../internal-agent
|
|
|
104
104
|
cancelled: import("../internal-agents/schema.js").Schema<boolean>;
|
|
105
105
|
run: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
106
106
|
run_id: import("../internal-agents/schema.js").Schema<string>;
|
|
107
|
-
kind: import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
|
|
107
|
+
kind: import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
|
|
108
108
|
status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
|
|
109
109
|
owner: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
110
110
|
kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
|
|
@@ -174,7 +174,7 @@ export declare const getRunEventListSchema: () => import("../internal-agents/sch
|
|
|
174
174
|
export declare const getRunListSchema: () => import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
175
175
|
data: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
176
176
|
run_id: import("../internal-agents/schema.js").Schema<string>;
|
|
177
|
-
kind: import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
|
|
177
|
+
kind: import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
|
|
178
178
|
status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
|
|
179
179
|
owner: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
180
180
|
kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
|
|
@@ -224,7 +224,7 @@ export declare const getRunListSchema: () => import("../internal-agents/schema.j
|
|
|
224
224
|
/** Zod schema for a canonical durable run. */
|
|
225
225
|
export declare const RunSchema: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
226
226
|
run_id: import("../internal-agents/schema.js").Schema<string>;
|
|
227
|
-
kind: import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
|
|
227
|
+
kind: import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
|
|
228
228
|
status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
|
|
229
229
|
owner: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
230
230
|
kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
|
|
@@ -270,7 +270,7 @@ export declare const CreateRunResponseSchema: import("../internal-agents/schema.
|
|
|
270
270
|
duplicate: import("../internal-agents/schema.js").Schema<boolean | undefined>;
|
|
271
271
|
run: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
272
272
|
run_id: import("../internal-agents/schema.js").Schema<string>;
|
|
273
|
-
kind: import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
|
|
273
|
+
kind: import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
|
|
274
274
|
status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
|
|
275
275
|
owner: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
276
276
|
kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
|
|
@@ -316,7 +316,7 @@ export declare const CancelRunResponseSchema: import("../internal-agents/schema.
|
|
|
316
316
|
cancelled: import("../internal-agents/schema.js").Schema<boolean>;
|
|
317
317
|
run: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
318
318
|
run_id: import("../internal-agents/schema.js").Schema<string>;
|
|
319
|
-
kind: import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
|
|
319
|
+
kind: import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
|
|
320
320
|
status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
|
|
321
321
|
owner: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
322
322
|
kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
|
|
@@ -383,7 +383,7 @@ export declare const RunEventListSchema: import("../internal-agents/schema.js").
|
|
|
383
383
|
export declare const RunListSchema: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
384
384
|
data: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
385
385
|
run_id: import("../internal-agents/schema.js").Schema<string>;
|
|
386
|
-
kind: import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
|
|
386
|
+
kind: import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
|
|
387
387
|
status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
|
|
388
388
|
owner: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
|
|
389
389
|
kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schemas.d.ts","sourceRoot":"","sources":["../../../src/src/runs/schemas.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAEjE,eAAO,MAAM,gBAAgB,
|
|
1
|
+
{"version":3,"file":"schemas.d.ts","sourceRoot":"","sources":["../../../src/src/runs/schemas.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAEjE,eAAO,MAAM,gBAAgB,6FAE5B,CAAC;AAEF,eAAO,MAAM,kBAAkB,+HAE9B,CAAC;AAEF,eAAO,MAAM,iBAAiB;;;GAK7B,CAAC;AAEF,eAAO,MAAM,6BAA6B,uGAEzC,CAAC;AAEF,eAAO,MAAM,uBAAuB,gGAEnC,CAAC;AAEF,eAAO,MAAM,0BAA0B;;;;GAMtC,CAAC;AAEF,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCxB,CAAC;AAEF,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAMtC,CAAC;AAEF,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAKtC,CAAC;AAEF,eAAO,MAAM,iBAAiB;;;;;GAO7B,CAAC;AAEF,eAAO,MAAM,iBAAiB;;;;;GAO7B,CAAC;AAEF,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;GAKjC,CAAC;AAEF,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAK5B,CAAC;AAEF,8CAA8C;AAC9C,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAA2B,CAAC;AAClD,4CAA4C;AAC5C,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAAyC,CAAC;AAC9E,4CAA4C;AAC5C,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAAyC,CAAC;AAC9E,kCAAkC;AAClC,eAAO,MAAM,cAAc;;;;;GAAgC,CAAC;AAC5D,qDAAqD;AACrD,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;GAAoC,CAAC;AACpE,uDAAuD;AACvD,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAA+B,CAAC;AAE1D,kCAAkC;AAClC,MAAM,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC,CAAC;AACvE,oCAAoC;AACpC,MAAM,MAAM,SAAS,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,kBAAkB,CAAC,CAAC,CAAC;AAC3E,mCAAmC;AACnC,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,iBAAiB,CAAC,CAAC,CAAC;AACzE,8DAA8D;AAC9D,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,6BAA6B,CAAC,CAAC,CAAC;AACjG,qEAAqE;AACrE,MAAM,MAAM,cAAc,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,uBAAuB,CAAC,CAAC,CAAC;AACrF,gEAAgE;AAChE,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC;AAC3F,6BAA6B;AAC7B,MAAM,MAAM,GAAG,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC;AAC/D,gDAAgD;AAChD,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC;AAC3F,iDAAiD;AACjD,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC;AAC3F,8BAA8B;AAC9B,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,iBAAiB,CAAC,CAAC,CAAC;AACzE,oCAAoC;AACpC,MAAM,MAAM,YAAY,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,qBAAqB,CAAC,CAAC,CAAC;AACjF,sCAAsC;AACtC,MAAM,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC,CAAC"}
|
package/esm/src/runs/schemas.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { defineSchema, lazySchema } from "../schemas/index.js";
|
|
2
|
-
export const getRunKindSchema = defineSchema((v) => v.enum(["agent", "workflow", "task"]));
|
|
2
|
+
export const getRunKindSchema = defineSchema((v) => v.enum(["agent", "workflow", "task", "eval"]));
|
|
3
3
|
export const getRunStatusSchema = defineSchema((v) => v.enum(["pending", "running", "waiting", "completed", "failed", "cancelled"]));
|
|
4
4
|
export const getRunOwnerSchema = defineSchema((v) => v.object({
|
|
5
5
|
kind: v.enum(["conversation", "project"]),
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "veryfront",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.873",
|
|
4
4
|
"description": "The simplest way to build AI-powered apps",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"react",
|
|
@@ -49,6 +49,14 @@
|
|
|
49
49
|
"import": "./esm/src/chat/index.js",
|
|
50
50
|
"types": "./esm/src/chat/index.d.ts"
|
|
51
51
|
},
|
|
52
|
+
"./chat/ag-ui": {
|
|
53
|
+
"import": "./esm/src/chat/ag-ui.js",
|
|
54
|
+
"types": "./esm/src/chat/ag-ui.d.ts"
|
|
55
|
+
},
|
|
56
|
+
"./chat/protocol": {
|
|
57
|
+
"import": "./esm/src/chat/protocol.js",
|
|
58
|
+
"types": "./esm/src/chat/protocol.d.ts"
|
|
59
|
+
},
|
|
52
60
|
"./chat/types": {
|
|
53
61
|
"import": "./esm/src/chat/types.js",
|
|
54
62
|
"types": "./esm/src/chat/types.d.ts"
|
|
@@ -69,9 +77,9 @@
|
|
|
69
77
|
"import": "./esm/src/agent/identity-contracts.js",
|
|
70
78
|
"types": "./esm/src/agent/identity-contracts.d.ts"
|
|
71
79
|
},
|
|
72
|
-
"./
|
|
73
|
-
"import": "./esm/src/
|
|
74
|
-
"types": "./esm/src/
|
|
80
|
+
"./eval": {
|
|
81
|
+
"import": "./esm/src/eval/index.js",
|
|
82
|
+
"types": "./esm/src/eval/index.d.ts"
|
|
75
83
|
},
|
|
76
84
|
"./channels/control-plane": {
|
|
77
85
|
"import": "./esm/src/channels/control-plane.js",
|
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
import type { Agent, AgentResponse, Message } from "../types.js";
|
|
2
|
-
/** Public API contract for test case. */
|
|
3
|
-
export interface TestCase {
|
|
4
|
-
/** Test name */
|
|
5
|
-
name: string;
|
|
6
|
-
/** Input to agent */
|
|
7
|
-
input: string | Message[];
|
|
8
|
-
/** Expected output pattern (regex or string) */
|
|
9
|
-
expected?: RegExp | string;
|
|
10
|
-
/** Expected tool calls */
|
|
11
|
-
expectToolCalls?: string[];
|
|
12
|
-
/** Maximum execution time (ms) */
|
|
13
|
-
timeout?: number;
|
|
14
|
-
/** Custom validator */
|
|
15
|
-
validate?: (response: AgentResponse) => boolean | Promise<boolean>;
|
|
16
|
-
}
|
|
17
|
-
/** Result returned from test. */
|
|
18
|
-
export interface TestResult {
|
|
19
|
-
/** Test case name */
|
|
20
|
-
name: string;
|
|
21
|
-
/** Pass/fail */
|
|
22
|
-
passed: boolean;
|
|
23
|
-
/** Response from agent */
|
|
24
|
-
response?: AgentResponse;
|
|
25
|
-
/** Error if test failed */
|
|
26
|
-
error?: string;
|
|
27
|
-
/** Execution time */
|
|
28
|
-
executionTime: number;
|
|
29
|
-
/** Tool calls made */
|
|
30
|
-
toolCalls: string[];
|
|
31
|
-
}
|
|
32
|
-
/** Public API contract for test suite. */
|
|
33
|
-
export interface TestSuite {
|
|
34
|
-
/** Suite name */
|
|
35
|
-
name: string;
|
|
36
|
-
/** Test results */
|
|
37
|
-
results: TestResult[];
|
|
38
|
-
/** Overall pass/fail */
|
|
39
|
-
passed: boolean;
|
|
40
|
-
/** Total execution time */
|
|
41
|
-
totalTime: number;
|
|
42
|
-
}
|
|
43
|
-
/** Test agent helper. */
|
|
44
|
-
export declare function testAgent(agent: Agent, testCases: TestCase[]): Promise<TestSuite>;
|
|
45
|
-
/** Print test results helper. */
|
|
46
|
-
export declare function printTestResults(suite: TestSuite): void;
|
|
47
|
-
/** Assert contains helper. */
|
|
48
|
-
export declare function assertContains(response: AgentResponse, text: string): boolean;
|
|
49
|
-
/** Assert tool called helper. */
|
|
50
|
-
export declare function assertToolCalled(response: AgentResponse, toolName: string): boolean;
|
|
51
|
-
/** Assert completed helper. */
|
|
52
|
-
export declare function assertCompleted(response: AgentResponse): boolean;
|
|
53
|
-
//# sourceMappingURL=agent-tester.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"agent-tester.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/testing/agent-tester.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEjE,yCAAyC;AACzC,MAAM,WAAW,QAAQ;IACvB,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IAEb,qBAAqB;IACrB,KAAK,EAAE,MAAM,GAAG,OAAO,EAAE,CAAC;IAE1B,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAE3B,0BAA0B;IAC1B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAE3B,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uBAAuB;IACvB,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACpE;AAED,iCAAiC;AACjC,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC;IAEb,gBAAgB;IAChB,MAAM,EAAE,OAAO,CAAC;IAEhB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,aAAa,CAAC;IAEzB,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,qBAAqB;IACrB,aAAa,EAAE,MAAM,CAAC;IAEtB,sBAAsB;IACtB,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,0CAA0C;AAC1C,MAAM,WAAW,SAAS;IACxB,iBAAiB;IACjB,IAAI,EAAE,MAAM,CAAC;IAEb,mBAAmB;IACnB,OAAO,EAAE,UAAU,EAAE,CAAC;IAEtB,wBAAwB;IACxB,MAAM,EAAE,OAAO,CAAC;IAEhB,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,yBAAyB;AACzB,wBAAsB,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAWvF;AA6FD,iCAAiC;AACjC,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,SAAS,GAAG,IAAI,CAmBvD;AAED,8BAA8B;AAC9B,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAE7E;AAED,iCAAiC;AACjC,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAEnF;AAED,+BAA+B;AAC/B,wBAAgB,eAAe,CAAC,QAAQ,EAAE,aAAa,GAAG,OAAO,CAEhE"}
|
|
@@ -1,132 +0,0 @@
|
|
|
1
|
-
/**************************
|
|
2
|
-
* Agent Testing Utilities
|
|
3
|
-
*
|
|
4
|
-
* Utilities for testing agents in development and CI/CD.
|
|
5
|
-
*
|
|
6
|
-
* @module veryfront/agent/testing
|
|
7
|
-
**************************/
|
|
8
|
-
import * as dntShim from "../../../_dnt.shims.js";
|
|
9
|
-
/** Test agent helper. */
|
|
10
|
-
export async function testAgent(agent, testCases) {
|
|
11
|
-
const suiteStartTime = Date.now();
|
|
12
|
-
const results = await Promise.all(testCases.map((testCase) => runTestCase(agent, testCase)));
|
|
13
|
-
const passed = results.every((r) => r.passed);
|
|
14
|
-
return {
|
|
15
|
-
name: agent.id,
|
|
16
|
-
results,
|
|
17
|
-
passed,
|
|
18
|
-
totalTime: Date.now() - suiteStartTime,
|
|
19
|
-
};
|
|
20
|
-
}
|
|
21
|
-
async function runTestCase(agent, testCase) {
|
|
22
|
-
const startTime = Date.now();
|
|
23
|
-
try {
|
|
24
|
-
const timeoutMs = testCase.timeout ?? 30000;
|
|
25
|
-
let timeoutId;
|
|
26
|
-
let response;
|
|
27
|
-
try {
|
|
28
|
-
response = await Promise.race([
|
|
29
|
-
agent.generate({ input: testCase.input }),
|
|
30
|
-
new Promise((_, reject) => {
|
|
31
|
-
timeoutId = dntShim.setTimeout(() => reject(new Error("Test timeout")), timeoutMs);
|
|
32
|
-
}),
|
|
33
|
-
]);
|
|
34
|
-
}
|
|
35
|
-
finally {
|
|
36
|
-
clearTimeout(timeoutId);
|
|
37
|
-
}
|
|
38
|
-
const executionTime = Date.now() - startTime;
|
|
39
|
-
const toolCalls = response.toolCalls.map((tc) => tc.name);
|
|
40
|
-
const { passed, error } = await validateTestCase(testCase, response, toolCalls);
|
|
41
|
-
return {
|
|
42
|
-
name: testCase.name,
|
|
43
|
-
passed,
|
|
44
|
-
response,
|
|
45
|
-
error,
|
|
46
|
-
executionTime,
|
|
47
|
-
toolCalls,
|
|
48
|
-
};
|
|
49
|
-
}
|
|
50
|
-
catch (error) {
|
|
51
|
-
return {
|
|
52
|
-
name: testCase.name,
|
|
53
|
-
passed: false,
|
|
54
|
-
error: error instanceof Error ? error.message : String(error),
|
|
55
|
-
executionTime: Date.now() - startTime,
|
|
56
|
-
toolCalls: [],
|
|
57
|
-
};
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
async function validateTestCase(testCase, response, toolCalls) {
|
|
61
|
-
const expected = testCase.expected;
|
|
62
|
-
if (expected instanceof RegExp) {
|
|
63
|
-
if (!expected.test(response.text)) {
|
|
64
|
-
return {
|
|
65
|
-
passed: false,
|
|
66
|
-
error: `Output "${response.text}" does not match pattern ${expected}`,
|
|
67
|
-
};
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
else if (typeof expected === "string") {
|
|
71
|
-
if (!response.text.includes(expected)) {
|
|
72
|
-
return {
|
|
73
|
-
passed: false,
|
|
74
|
-
error: `Output does not contain expected text: "${expected}"`,
|
|
75
|
-
};
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
const expectedTools = testCase.expectToolCalls;
|
|
79
|
-
if (expectedTools?.length) {
|
|
80
|
-
const missingTools = expectedTools.filter((t) => !toolCalls.includes(t));
|
|
81
|
-
if (missingTools.length) {
|
|
82
|
-
return {
|
|
83
|
-
passed: false,
|
|
84
|
-
error: `Expected tool calls not found: ${missingTools.join(", ")}`,
|
|
85
|
-
};
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
if (testCase.validate) {
|
|
89
|
-
try {
|
|
90
|
-
const passed = await testCase.validate(response);
|
|
91
|
-
if (!passed)
|
|
92
|
-
return { passed: false, error: "Custom validation failed" };
|
|
93
|
-
}
|
|
94
|
-
catch (error) {
|
|
95
|
-
return {
|
|
96
|
-
passed: false,
|
|
97
|
-
error: `Custom validation error: ${error instanceof Error ? error.message : String(error)}`,
|
|
98
|
-
};
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
return { passed: true };
|
|
102
|
-
}
|
|
103
|
-
/** Print test results helper. */
|
|
104
|
-
export function printTestResults(suite) {
|
|
105
|
-
console.log(`\n=== Test Suite: ${suite.name} ===\n`);
|
|
106
|
-
const passed = suite.results.filter((r) => r.passed).length;
|
|
107
|
-
const total = suite.results.length;
|
|
108
|
-
for (const [index, result] of suite.results.entries()) {
|
|
109
|
-
const icon = result.passed ? "✅" : "❌";
|
|
110
|
-
console.log(`${icon} ${index + 1}. ${result.name}`);
|
|
111
|
-
if (!result.passed && result.error)
|
|
112
|
-
console.log(` Error: ${result.error}`);
|
|
113
|
-
if (result.toolCalls.length)
|
|
114
|
-
console.log(` Tools used: ${result.toolCalls.join(", ")}`);
|
|
115
|
-
console.log(` Time: ${result.executionTime}ms\n`);
|
|
116
|
-
}
|
|
117
|
-
console.log(`Results: ${passed}/${total} passed`);
|
|
118
|
-
console.log(`Total time: ${suite.totalTime}ms`);
|
|
119
|
-
console.log(`Status: ${suite.passed ? "✅ PASSED" : "❌ FAILED"}\n`);
|
|
120
|
-
}
|
|
121
|
-
/** Assert contains helper. */
|
|
122
|
-
export function assertContains(response, text) {
|
|
123
|
-
return response.text.toLowerCase().includes(text.toLowerCase());
|
|
124
|
-
}
|
|
125
|
-
/** Assert tool called helper. */
|
|
126
|
-
export function assertToolCalled(response, toolName) {
|
|
127
|
-
return response.toolCalls.some((tc) => tc.name === toolName);
|
|
128
|
-
}
|
|
129
|
-
/** Assert completed helper. */
|
|
130
|
-
export function assertCompleted(response) {
|
|
131
|
-
return response.status === "completed";
|
|
132
|
-
}
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import { type LiveEvalApiContext } from "../live-evals/api-client.js";
|
|
2
|
-
import { createDurableRunCanaryRunner, type DurableRunCanaryCase, type DurableRunCanaryRunnerConfig } from "./runner.js";
|
|
3
|
-
type EnvRecord = Record<string, string | undefined>;
|
|
4
|
-
/** Input payload for durable run canary cli case factory. */
|
|
5
|
-
export interface DurableRunCanaryCliCaseFactoryInput {
|
|
6
|
-
context: LiveEvalApiContext;
|
|
7
|
-
requestTimeoutMs: number;
|
|
8
|
-
}
|
|
9
|
-
/** Input payload for run durable run canary cli. */
|
|
10
|
-
export interface RunDurableRunCanaryCliInput {
|
|
11
|
-
env: EnvRecord;
|
|
12
|
-
agentId: string;
|
|
13
|
-
createCases: (input: DurableRunCanaryCliCaseFactoryInput) => DurableRunCanaryCase[];
|
|
14
|
-
cwd?: string;
|
|
15
|
-
log?: (message: string) => void;
|
|
16
|
-
createRunner?: (config: DurableRunCanaryRunnerConfig) => ReturnType<typeof createDurableRunCanaryRunner>;
|
|
17
|
-
}
|
|
18
|
-
/** Run durable run canary cli. */
|
|
19
|
-
export declare function runDurableRunCanaryCli(input: RunDurableRunCanaryCliInput): Promise<number>;
|
|
20
|
-
export {};
|
|
21
|
-
//# sourceMappingURL=cli-runner.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cli-runner.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/durable-run-canaries/cli-runner.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,kBAAkB,EAAE,MAAM,6BAA6B,CAAC;AAEtE,OAAO,EACL,4BAA4B,EAC5B,KAAK,oBAAoB,EAEzB,KAAK,4BAA4B,EAClC,MAAM,aAAa,CAAC;AAErB,KAAK,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;AAEpD,6DAA6D;AAC7D,MAAM,WAAW,mCAAmC;IAClD,OAAO,EAAE,kBAAkB,CAAC;IAC5B,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,oDAAoD;AACpD,MAAM,WAAW,2BAA2B;IAC1C,GAAG,EAAE,SAAS,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,CAAC,KAAK,EAAE,mCAAmC,KAAK,oBAAoB,EAAE,CAAC;IACpF,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,YAAY,CAAC,EAAE,CACb,MAAM,EAAE,4BAA4B,KACjC,UAAU,CAAC,OAAO,4BAA4B,CAAC,CAAC;CACtD;AAcD,kCAAkC;AAClC,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,2BAA2B,GACjC,OAAO,CAAC,MAAM,CAAC,CAwEjB"}
|
|
@@ -1,66 +0,0 @@
|
|
|
1
|
-
import { mkdir, writeTextFile } from "../../../platform/compat/fs.js";
|
|
2
|
-
import { dirname, resolve } from "../../../platform/compat/path/index.js";
|
|
3
|
-
import { cwd as getProcessCwd } from "../../../platform/compat/process.js";
|
|
4
|
-
import { resolveDurableRunCanaryEnvironment } from "./environment.js";
|
|
5
|
-
import { createDurableRunCanaryRunner, } from "./runner.js";
|
|
6
|
-
function createTimestampedReportPath(input) {
|
|
7
|
-
return resolve(input.cwd, ".omx/logs", input.directory, `${new Date().toISOString().replaceAll(":", "-").replaceAll(".", "-")}.json`);
|
|
8
|
-
}
|
|
9
|
-
/** Run durable run canary cli. */
|
|
10
|
-
export async function runDurableRunCanaryCli(input) {
|
|
11
|
-
const log = input.log ?? console.log;
|
|
12
|
-
const cwd = input.cwd ?? getProcessCwd();
|
|
13
|
-
const { apiUrl, authToken, projectId, requestTimeoutMs, keepSuccessfulEvidence } = resolveDurableRunCanaryEnvironment(input.env);
|
|
14
|
-
const reportPath = input.env.DURABLE_CANARY_REPORT_PATH ??
|
|
15
|
-
createTimestampedReportPath({ cwd, directory: "durable-run-staging-canaries" });
|
|
16
|
-
if (!authToken) {
|
|
17
|
-
throw new Error("Missing VERYFRONT_TOKEN");
|
|
18
|
-
}
|
|
19
|
-
if (!projectId) {
|
|
20
|
-
throw new Error("Missing AG_UI_EVAL_PROJECT_ID");
|
|
21
|
-
}
|
|
22
|
-
const context = {
|
|
23
|
-
apiUrl,
|
|
24
|
-
authToken,
|
|
25
|
-
projectId,
|
|
26
|
-
};
|
|
27
|
-
const createRunner = input.createRunner ?? createDurableRunCanaryRunner;
|
|
28
|
-
const { runCase } = createRunner({
|
|
29
|
-
apiUrl,
|
|
30
|
-
authToken,
|
|
31
|
-
agentId: input.agentId,
|
|
32
|
-
projectId,
|
|
33
|
-
requestTimeoutMs,
|
|
34
|
-
keepSuccessfulEvidence,
|
|
35
|
-
});
|
|
36
|
-
const testCases = input.createCases({
|
|
37
|
-
context,
|
|
38
|
-
requestTimeoutMs,
|
|
39
|
-
});
|
|
40
|
-
log(`Durable run canaries -> ${apiUrl}`);
|
|
41
|
-
log(`Project scope -> ${projectId}`);
|
|
42
|
-
const results = [];
|
|
43
|
-
for (const testCase of testCases) {
|
|
44
|
-
log(`\n[run] ${testCase.label}`);
|
|
45
|
-
const result = await runCase(testCase);
|
|
46
|
-
results.push(result);
|
|
47
|
-
log(`[${result.status}] ${result.id}: ${result.details}`);
|
|
48
|
-
}
|
|
49
|
-
const summary = {
|
|
50
|
-
passed: results.filter((result) => result.status === "pass").length,
|
|
51
|
-
failed: results.filter((result) => result.status === "fail").length,
|
|
52
|
-
};
|
|
53
|
-
await mkdir(dirname(reportPath), { recursive: true });
|
|
54
|
-
await writeTextFile(reportPath, JSON.stringify({
|
|
55
|
-
generatedAt: new Date().toISOString(),
|
|
56
|
-
apiUrl,
|
|
57
|
-
projectId,
|
|
58
|
-
results,
|
|
59
|
-
summary,
|
|
60
|
-
}, null, 2));
|
|
61
|
-
log("\nSummary");
|
|
62
|
-
log(`passed: ${summary.passed}`);
|
|
63
|
-
log(`failed: ${summary.failed}`);
|
|
64
|
-
log(`report: ${reportPath}`);
|
|
65
|
-
return summary.failed > 0 ? 1 : 0;
|
|
66
|
-
}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import { type AgentServiceConfigInput } from "../../service/config.js";
|
|
2
|
-
/** Public API contract for durable run canary environment. */
|
|
3
|
-
export interface DurableRunCanaryEnvironment {
|
|
4
|
-
apiUrl: string;
|
|
5
|
-
authToken: string;
|
|
6
|
-
projectId: string;
|
|
7
|
-
requestTimeoutMs: number;
|
|
8
|
-
keepSuccessfulEvidence: boolean;
|
|
9
|
-
}
|
|
10
|
-
/** Default value for durable run canary timeout ms. */
|
|
11
|
-
export declare const DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS = 240000;
|
|
12
|
-
/** Resolves durable run canary environment. */
|
|
13
|
-
export declare function resolveDurableRunCanaryEnvironment(env?: AgentServiceConfigInput): DurableRunCanaryEnvironment;
|
|
14
|
-
//# sourceMappingURL=environment.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"environment.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/durable-run-canaries/environment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,uBAAuB,EAA2B,MAAM,yBAAyB,CAAC;AAEhG,8DAA8D;AAC9D,MAAM,WAAW,2BAA2B;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,EAAE,OAAO,CAAC;CACjC;AAED,uDAAuD;AACvD,eAAO,MAAM,qCAAqC,SAAU,CAAC;AAE7D,+CAA+C;AAC/C,wBAAgB,kCAAkC,CAChD,GAAG,GAAE,uBAA4B,GAChC,2BAA2B,CAY7B"}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import { parseAgentServiceConfig } from "../../service/config.js";
|
|
2
|
-
/** Default value for durable run canary timeout ms. */
|
|
3
|
-
export const DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS = 240_000;
|
|
4
|
-
/** Resolves durable run canary environment. */
|
|
5
|
-
export function resolveDurableRunCanaryEnvironment(env = {}) {
|
|
6
|
-
return {
|
|
7
|
-
apiUrl: typeof env.VERYFRONT_API_URL === "string"
|
|
8
|
-
? env.VERYFRONT_API_URL
|
|
9
|
-
: parseAgentServiceConfig(env).VERYFRONT_API_URL,
|
|
10
|
-
authToken: typeof env.VERYFRONT_TOKEN === "string" ? env.VERYFRONT_TOKEN : "",
|
|
11
|
-
projectId: typeof env.AG_UI_EVAL_PROJECT_ID === "string" ? env.AG_UI_EVAL_PROJECT_ID : "",
|
|
12
|
-
requestTimeoutMs: Number(env.DURABLE_CANARY_TIMEOUT_MS ?? DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS),
|
|
13
|
-
keepSuccessfulEvidence: env.DURABLE_CANARY_KEEP_SUCCESS === "1",
|
|
14
|
-
};
|
|
15
|
-
}
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
export { type DurableRunCanaryCliCaseFactoryInput, runDurableRunCanaryCli, type RunDurableRunCanaryCliInput, } from "./cli-runner.js";
|
|
2
|
-
export { DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS, type DurableRunCanaryEnvironment, resolveDurableRunCanaryEnvironment, } from "./environment.js";
|
|
3
|
-
export { createDurableRunCanaryApiClient, createDurableRunCanaryRunner, type DurableRunCanaryApiClient, type DurableRunCanaryApiConfig, type DurableRunCanaryCase, type DurableRunCanaryCreateRootRunInput, type DurableRunCanaryMessage, type DurableRunCanaryPreparedCase, type DurableRunCanaryResult, type DurableRunCanaryRunnerConfig, durableRunCanaryRunnerInternals, type DurableRunCanaryRunSummary, type DurableRunCanarySendUserMessageInput, type DurableRunCanaryStartRunInput, getDurableRunCanaryMessageSchema, parseDurableRunCanaryRunSummary, } from "./runner.js";
|
|
4
|
-
export { assertCompleted, assertNoMalformedCreateFileToolCalls, collectAssistantText, findAssistantMessage, stringifyUnknown, } from "./validation.js";
|
|
5
|
-
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/durable-run-canaries/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,mCAAmC,EACxC,sBAAsB,EACtB,KAAK,2BAA2B,GACjC,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,qCAAqC,EACrC,KAAK,2BAA2B,EAChC,kCAAkC,GACnC,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,+BAA+B,EAC/B,4BAA4B,EAC5B,KAAK,yBAAyB,EAC9B,KAAK,yBAAyB,EAC9B,KAAK,oBAAoB,EACzB,KAAK,kCAAkC,EACvC,KAAK,uBAAuB,EAC5B,KAAK,4BAA4B,EACjC,KAAK,sBAAsB,EAC3B,KAAK,4BAA4B,EACjC,+BAA+B,EAC/B,KAAK,0BAA0B,EAC/B,KAAK,oCAAoC,EACzC,KAAK,6BAA6B,EAClC,gCAAgC,EAChC,+BAA+B,GAChC,MAAM,aAAa,CAAC;AAErB,OAAO,EACL,eAAe,EACf,oCAAoC,EACpC,oBAAoB,EACpB,oBAAoB,EACpB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC"}
|
|
@@ -1,4 +0,0 @@
|
|
|
1
|
-
export { runDurableRunCanaryCli, } from "./cli-runner.js";
|
|
2
|
-
export { DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS, resolveDurableRunCanaryEnvironment, } from "./environment.js";
|
|
3
|
-
export { createDurableRunCanaryApiClient, createDurableRunCanaryRunner, durableRunCanaryRunnerInternals, getDurableRunCanaryMessageSchema, parseDurableRunCanaryRunSummary, } from "./runner.js";
|
|
4
|
-
export { assertCompleted, assertNoMalformedCreateFileToolCalls, collectAssistantText, findAssistantMessage, stringifyUnknown, } from "./validation.js";
|
|
@@ -1,118 +0,0 @@
|
|
|
1
|
-
import type { InferSchema } from "../../../extensions/schema/index.js";
|
|
2
|
-
/** Configuration used by durable run canary API. */
|
|
3
|
-
export interface DurableRunCanaryApiConfig {
|
|
4
|
-
apiUrl: string;
|
|
5
|
-
authToken: string;
|
|
6
|
-
agentId: string;
|
|
7
|
-
projectId: string | null;
|
|
8
|
-
branchId?: string | null;
|
|
9
|
-
requestTimeoutMs: number;
|
|
10
|
-
fetch?: (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
|
|
11
|
-
}
|
|
12
|
-
/** Input payload for durable run canary create root run. */
|
|
13
|
-
export interface DurableRunCanaryCreateRootRunInput {
|
|
14
|
-
conversationId: string;
|
|
15
|
-
runId: string;
|
|
16
|
-
}
|
|
17
|
-
/** Input payload for durable run canary send user message. */
|
|
18
|
-
export interface DurableRunCanarySendUserMessageInput {
|
|
19
|
-
conversationId: string;
|
|
20
|
-
prompt: string;
|
|
21
|
-
}
|
|
22
|
-
/** Input payload for durable run canary start run. */
|
|
23
|
-
export interface DurableRunCanaryStartRunInput extends DurableRunCanaryCreateRootRunInput {
|
|
24
|
-
messageId: string;
|
|
25
|
-
prompt: string;
|
|
26
|
-
userMessageId: string;
|
|
27
|
-
}
|
|
28
|
-
/** Zod schema for get durable run canary message. */
|
|
29
|
-
export declare const getDurableRunCanaryMessageSchema: () => import("../../../internal-agents/schema.js").Schema<{
|
|
30
|
-
id: string;
|
|
31
|
-
role: "tool" | "user" | "assistant" | "system";
|
|
32
|
-
parts: ({
|
|
33
|
-
type: string;
|
|
34
|
-
} & {} & Record<string, unknown>)[];
|
|
35
|
-
} & {
|
|
36
|
-
status?: string | undefined;
|
|
37
|
-
} & Record<string, unknown>>;
|
|
38
|
-
/** Message shape for durable run canary. */
|
|
39
|
-
export type DurableRunCanaryMessage = InferSchema<ReturnType<typeof getDurableRunCanaryMessageSchema>>;
|
|
40
|
-
/** Public API contract for durable run canary run summary. */
|
|
41
|
-
export interface DurableRunCanaryRunSummary {
|
|
42
|
-
runId: string;
|
|
43
|
-
conversationId: string;
|
|
44
|
-
messageId: string;
|
|
45
|
-
agentId: string;
|
|
46
|
-
status: string;
|
|
47
|
-
latestEventId: number;
|
|
48
|
-
latestExternalEventSequence: number | null;
|
|
49
|
-
waitingToolCallId: string | null;
|
|
50
|
-
waitingToolName: string | null;
|
|
51
|
-
terminalErrorCode: string | null;
|
|
52
|
-
terminalErrorMessage: string | null;
|
|
53
|
-
startedAt: string | null;
|
|
54
|
-
finishedAt: string | null;
|
|
55
|
-
}
|
|
56
|
-
/** Parses durable run canary run summary. */
|
|
57
|
-
export declare function parseDurableRunCanaryRunSummary(value: unknown): DurableRunCanaryRunSummary;
|
|
58
|
-
/** Public API contract for durable run canary API client. */
|
|
59
|
-
export interface DurableRunCanaryApiClient {
|
|
60
|
-
createDurableRootRun: (input: DurableRunCanaryCreateRootRunInput) => Promise<void>;
|
|
61
|
-
getRunSummary: (input: DurableRunCanaryCreateRootRunInput) => Promise<DurableRunCanaryRunSummary>;
|
|
62
|
-
listMessagesForCanary: (input: {
|
|
63
|
-
conversationId: string;
|
|
64
|
-
}) => Promise<DurableRunCanaryMessage[]>;
|
|
65
|
-
sendUserMessageForCanary: (input: DurableRunCanarySendUserMessageInput) => Promise<DurableRunCanaryMessage>;
|
|
66
|
-
startDurableRun: (input: DurableRunCanaryStartRunInput) => Promise<void>;
|
|
67
|
-
}
|
|
68
|
-
/** Create durable run canary API client. */
|
|
69
|
-
export declare function createDurableRunCanaryApiClient(config: DurableRunCanaryApiConfig): DurableRunCanaryApiClient;
|
|
70
|
-
/** Result returned from durable run canary. */
|
|
71
|
-
export interface DurableRunCanaryResult {
|
|
72
|
-
id: string;
|
|
73
|
-
label: string;
|
|
74
|
-
status: "pass" | "fail";
|
|
75
|
-
details: string;
|
|
76
|
-
durationMs: number;
|
|
77
|
-
conversationId: string;
|
|
78
|
-
runId: string;
|
|
79
|
-
artifactPaths?: string[];
|
|
80
|
-
}
|
|
81
|
-
/** Public API contract for durable run canary prepared case. */
|
|
82
|
-
export interface DurableRunCanaryPreparedCase {
|
|
83
|
-
artifactPaths?: string[] | ((runId: string) => string[]);
|
|
84
|
-
cleanup: (input?: {
|
|
85
|
-
runId: string;
|
|
86
|
-
}) => Promise<void>;
|
|
87
|
-
conversationId: string;
|
|
88
|
-
prompt: string;
|
|
89
|
-
startSidecar?: () => Promise<(() => Promise<void>) | void>;
|
|
90
|
-
title: string;
|
|
91
|
-
validate: (input: {
|
|
92
|
-
messages: DurableRunCanaryMessage[];
|
|
93
|
-
run: DurableRunCanaryRunSummary;
|
|
94
|
-
}) => Promise<void> | void;
|
|
95
|
-
}
|
|
96
|
-
/** Public API contract for durable run canary case. */
|
|
97
|
-
export interface DurableRunCanaryCase {
|
|
98
|
-
id: string;
|
|
99
|
-
label: string;
|
|
100
|
-
prepare: () => Promise<DurableRunCanaryPreparedCase>;
|
|
101
|
-
}
|
|
102
|
-
/** Configuration used by durable run canary runner. */
|
|
103
|
-
export interface DurableRunCanaryRunnerConfig extends DurableRunCanaryApiConfig {
|
|
104
|
-
keepSuccessfulEvidence: boolean;
|
|
105
|
-
}
|
|
106
|
-
declare function collectReferencedChildConversationIds(messages: DurableRunCanaryMessage[]): string[];
|
|
107
|
-
declare function isTerminalRunStatus(status: string): boolean;
|
|
108
|
-
/** Create durable run canary runner. */
|
|
109
|
-
export declare function createDurableRunCanaryRunner(config: DurableRunCanaryRunnerConfig, apiClient?: DurableRunCanaryApiClient): {
|
|
110
|
-
runCase: (testCase: DurableRunCanaryCase) => Promise<DurableRunCanaryResult>;
|
|
111
|
-
};
|
|
112
|
-
/** White-box helpers used by durable run canary tests. */
|
|
113
|
-
export declare const durableRunCanaryRunnerInternals: {
|
|
114
|
-
collectReferencedChildConversationIds: typeof collectReferencedChildConversationIds;
|
|
115
|
-
isTerminalRunStatus: typeof isTerminalRunStatus;
|
|
116
|
-
};
|
|
117
|
-
export {};
|
|
118
|
-
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/durable-run-canaries/runner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qCAAqC,CAAC;AAIvE,oDAAoD;AACpD,MAAM,WAAW,yBAAyB;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;CAClF;AAED,4DAA4D;AAC5D,MAAM,WAAW,kCAAkC;IACjD,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,8DAA8D;AAC9D,MAAM,WAAW,oCAAoC;IACnD,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,sDAAsD;AACtD,MAAM,WAAW,6BAA8B,SAAQ,kCAAkC;IACvF,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,qDAAqD;AACrD,eAAO,MAAM,gCAAgC;;;;;;;;4BAO5C,CAAC;AAEF,4CAA4C;AAC5C,MAAM,MAAM,uBAAuB,GAAG,WAAW,CAC/C,UAAU,CAAC,OAAO,gCAAgC,CAAC,CACpD,CAAC;AAEF,8DAA8D;AAC9D,MAAM,WAAW,0BAA0B;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AA4CD,6CAA6C;AAC7C,wBAAgB,+BAA+B,CAAC,KAAK,EAAE,OAAO,GAAG,0BAA0B,CAoC1F;AAkGD,6DAA6D;AAC7D,MAAM,WAAW,yBAAyB;IACxC,oBAAoB,EAAE,CAAC,KAAK,EAAE,kCAAkC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACnF,aAAa,EAAE,CAAC,KAAK,EAAE,kCAAkC,KAAK,OAAO,CAAC,0BAA0B,CAAC,CAAC;IAClG,qBAAqB,EAAE,CAAC,KAAK,EAAE;QAAE,cAAc,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,uBAAuB,EAAE,CAAC,CAAC;IACjG,wBAAwB,EAAE,CACxB,KAAK,EAAE,oCAAoC,KACxC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACtC,eAAe,EAAE,CAAC,KAAK,EAAE,6BAA6B,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1E;AAED,4CAA4C;AAC5C,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,yBAAyB,GAChC,yBAAyB,CAgF3B;AAED,+CAA+C;AAC/C,MAAM,WAAW,sBAAsB;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,gEAAgE;AAChE,MAAM,WAAW,4BAA4B;IAC3C,aAAa,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC,CAAC;IACzD,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAC3D,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,CAAC,KAAK,EAAE;QAChB,QAAQ,EAAE,uBAAuB,EAAE,CAAC;QACpC,GAAG,EAAE,0BAA0B,CAAC;KACjC,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CAC5B;AAED,uDAAuD;AACvD,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,OAAO,CAAC,4BAA4B,CAAC,CAAC;CACtD;AAED,uDAAuD;AACvD,MAAM,WAAW,4BAA6B,SAAQ,yBAAyB;IAC7E,sBAAsB,EAAE,OAAO,CAAC;CACjC;AA0DD,iBAAS,qCAAqC,CAAC,QAAQ,EAAE,uBAAuB,EAAE,GAAG,MAAM,EAAE,CAc5F;AAQD,iBAAS,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEpD;AA2CD,wCAAwC;AACxC,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,4BAA4B,EACpC,SAAS,GAAE,yBAAmE;wBAkB7C,oBAAoB,KAAG,OAAO,CAAC,sBAAsB,CAAC;EAqFxF;AAED,0DAA0D;AAC1D,eAAO,MAAM,+BAA+B;;;CAG3C,CAAC"}
|