veryfront 0.1.871 → 0.1.873

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/esm/cli/commands/eval/command-help.d.ts +3 -0
  2. package/esm/cli/commands/eval/command-help.d.ts.map +1 -0
  3. package/esm/cli/commands/eval/command-help.js +42 -0
  4. package/esm/cli/commands/eval/command.d.ts +21 -0
  5. package/esm/cli/commands/eval/command.d.ts.map +1 -0
  6. package/esm/cli/commands/eval/command.js +289 -0
  7. package/esm/cli/commands/eval/handler.d.ts +26 -0
  8. package/esm/cli/commands/eval/handler.d.ts.map +1 -0
  9. package/esm/cli/commands/eval/handler.js +28 -0
  10. package/esm/cli/help/command-definitions.d.ts.map +1 -1
  11. package/esm/cli/help/command-definitions.js +2 -0
  12. package/esm/cli/router.d.ts.map +1 -1
  13. package/esm/cli/router.js +2 -0
  14. package/esm/deno.d.ts +5 -2
  15. package/esm/deno.js +7 -4
  16. package/esm/src/agent/runtime/client-profile.d.ts.map +1 -1
  17. package/esm/src/agent/runtime/client-profile.js +10 -1
  18. package/esm/src/chat/ag-ui.d.ts.map +1 -1
  19. package/esm/src/chat/protocol.d.ts.map +1 -1
  20. package/esm/src/chat/protocol.js +0 -1
  21. package/esm/src/config/schemas/config.schema.d.ts +12 -0
  22. package/esm/src/config/schemas/config.schema.d.ts.map +1 -1
  23. package/esm/src/config/schemas/config.schema.js +12 -0
  24. package/esm/src/discovery/discovery-engine.d.ts.map +1 -1
  25. package/esm/src/discovery/discovery-engine.js +6 -1
  26. package/esm/src/discovery/handlers/eval-handler.d.ts +7 -0
  27. package/esm/src/discovery/handlers/eval-handler.d.ts.map +1 -0
  28. package/esm/src/discovery/handlers/eval-handler.js +16 -0
  29. package/esm/src/discovery/handlers/index.d.ts +1 -0
  30. package/esm/src/discovery/handlers/index.d.ts.map +1 -1
  31. package/esm/src/discovery/handlers/index.js +1 -0
  32. package/esm/src/discovery/import-rewriter.d.ts +1 -1
  33. package/esm/src/discovery/import-rewriter.d.ts.map +1 -1
  34. package/esm/src/discovery/import-rewriter.js +1 -0
  35. package/esm/src/discovery/project-discovery-config.d.ts +2 -0
  36. package/esm/src/discovery/project-discovery-config.d.ts.map +1 -1
  37. package/esm/src/discovery/project-discovery-config.js +2 -0
  38. package/esm/src/discovery/transpiler.d.ts.map +1 -1
  39. package/esm/src/discovery/transpiler.js +2 -0
  40. package/esm/src/discovery/types.d.ts +3 -0
  41. package/esm/src/discovery/types.d.ts.map +1 -1
  42. package/esm/src/eval/datasets.d.ts +8 -0
  43. package/esm/src/eval/datasets.d.ts.map +1 -0
  44. package/esm/src/eval/datasets.js +66 -0
  45. package/esm/src/eval/discovery.d.ts +43 -0
  46. package/esm/src/eval/discovery.d.ts.map +1 -0
  47. package/esm/src/eval/discovery.js +123 -0
  48. package/esm/src/eval/expect.d.ts +16 -0
  49. package/esm/src/eval/expect.d.ts.map +1 -0
  50. package/esm/src/eval/expect.js +62 -0
  51. package/esm/src/eval/factory.d.ts +6 -0
  52. package/esm/src/eval/factory.d.ts.map +1 -0
  53. package/esm/src/eval/factory.js +50 -0
  54. package/esm/src/eval/index.d.ts +33 -0
  55. package/esm/src/eval/index.d.ts.map +1 -0
  56. package/esm/src/eval/index.js +29 -0
  57. package/esm/src/eval/metrics.d.ts +55 -0
  58. package/esm/src/eval/metrics.d.ts.map +1 -0
  59. package/esm/src/eval/metrics.js +223 -0
  60. package/esm/src/eval/report.d.ts +12 -0
  61. package/esm/src/eval/report.d.ts.map +1 -0
  62. package/esm/src/eval/report.js +64 -0
  63. package/esm/src/eval/runner.d.ts +4 -0
  64. package/esm/src/eval/runner.d.ts.map +1 -0
  65. package/esm/src/eval/runner.js +101 -0
  66. package/esm/src/eval/studio.d.ts +177 -0
  67. package/esm/src/eval/studio.d.ts.map +1 -0
  68. package/esm/src/eval/studio.js +176 -0
  69. package/esm/src/eval/types.d.ts +221 -0
  70. package/esm/src/eval/types.d.ts.map +1 -0
  71. package/esm/src/eval/types.js +6 -0
  72. package/esm/src/eval/validation.d.ts +4 -0
  73. package/esm/src/eval/validation.d.ts.map +1 -0
  74. package/esm/src/eval/validation.js +41 -0
  75. package/esm/src/runs/index.d.ts +1 -1
  76. package/esm/src/runs/index.js +1 -1
  77. package/esm/src/runs/schemas.d.ts +9 -9
  78. package/esm/src/runs/schemas.d.ts.map +1 -1
  79. package/esm/src/runs/schemas.js +1 -1
  80. package/esm/src/utils/version-constant.d.ts +1 -1
  81. package/esm/src/utils/version-constant.js +1 -1
  82. package/package.json +12 -4
  83. package/esm/src/agent/testing/agent-tester.d.ts +0 -53
  84. package/esm/src/agent/testing/agent-tester.d.ts.map +0 -1
  85. package/esm/src/agent/testing/agent-tester.js +0 -132
  86. package/esm/src/agent/testing/durable-run-canaries/cli-runner.d.ts +0 -21
  87. package/esm/src/agent/testing/durable-run-canaries/cli-runner.d.ts.map +0 -1
  88. package/esm/src/agent/testing/durable-run-canaries/cli-runner.js +0 -66
  89. package/esm/src/agent/testing/durable-run-canaries/environment.d.ts +0 -14
  90. package/esm/src/agent/testing/durable-run-canaries/environment.d.ts.map +0 -1
  91. package/esm/src/agent/testing/durable-run-canaries/environment.js +0 -15
  92. package/esm/src/agent/testing/durable-run-canaries/index.d.ts +0 -5
  93. package/esm/src/agent/testing/durable-run-canaries/index.d.ts.map +0 -1
  94. package/esm/src/agent/testing/durable-run-canaries/index.js +0 -4
  95. package/esm/src/agent/testing/durable-run-canaries/runner.d.ts +0 -118
  96. package/esm/src/agent/testing/durable-run-canaries/runner.d.ts.map +0 -1
  97. package/esm/src/agent/testing/durable-run-canaries/runner.js +0 -391
  98. package/esm/src/agent/testing/durable-run-canaries/validation.d.ts +0 -12
  99. package/esm/src/agent/testing/durable-run-canaries/validation.d.ts.map +0 -1
  100. package/esm/src/agent/testing/durable-run-canaries/validation.js +0 -71
  101. package/esm/src/agent/testing/index.d.ts +0 -10
  102. package/esm/src/agent/testing/index.d.ts.map +0 -1
  103. package/esm/src/agent/testing/index.js +0 -9
  104. package/esm/src/agent/testing/live-evals/api-client.d.ts +0 -102
  105. package/esm/src/agent/testing/live-evals/api-client.d.ts.map +0 -1
  106. package/esm/src/agent/testing/live-evals/api-client.js +0 -313
  107. package/esm/src/agent/testing/live-evals/cli-runner.d.ts +0 -40
  108. package/esm/src/agent/testing/live-evals/cli-runner.d.ts.map +0 -1
  109. package/esm/src/agent/testing/live-evals/cli-runner.js +0 -144
  110. package/esm/src/agent/testing/live-evals/environment.d.ts +0 -15
  111. package/esm/src/agent/testing/live-evals/environment.d.ts.map +0 -1
  112. package/esm/src/agent/testing/live-evals/environment.js +0 -20
  113. package/esm/src/agent/testing/live-evals/formatting.d.ts +0 -15
  114. package/esm/src/agent/testing/live-evals/formatting.d.ts.map +0 -1
  115. package/esm/src/agent/testing/live-evals/formatting.js +0 -75
  116. package/esm/src/agent/testing/live-evals/index.d.ts +0 -12
  117. package/esm/src/agent/testing/live-evals/index.d.ts.map +0 -1
  118. package/esm/src/agent/testing/live-evals/index.js +0 -11
  119. package/esm/src/agent/testing/live-evals/metadata.d.ts +0 -34
  120. package/esm/src/agent/testing/live-evals/metadata.d.ts.map +0 -1
  121. package/esm/src/agent/testing/live-evals/metadata.js +0 -116
  122. package/esm/src/agent/testing/live-evals/performance.d.ts +0 -19
  123. package/esm/src/agent/testing/live-evals/performance.d.ts.map +0 -1
  124. package/esm/src/agent/testing/live-evals/performance.js +0 -22
  125. package/esm/src/agent/testing/live-evals/preflight.d.ts +0 -12
  126. package/esm/src/agent/testing/live-evals/preflight.d.ts.map +0 -1
  127. package/esm/src/agent/testing/live-evals/preflight.js +0 -27
  128. package/esm/src/agent/testing/live-evals/report.d.ts +0 -55
  129. package/esm/src/agent/testing/live-evals/report.d.ts.map +0 -1
  130. package/esm/src/agent/testing/live-evals/report.js +0 -79
  131. package/esm/src/agent/testing/live-evals/request.d.ts +0 -31
  132. package/esm/src/agent/testing/live-evals/request.d.ts.map +0 -1
  133. package/esm/src/agent/testing/live-evals/request.js +0 -48
  134. package/esm/src/agent/testing/live-evals/result.d.ts +0 -56
  135. package/esm/src/agent/testing/live-evals/result.d.ts.map +0 -1
  136. package/esm/src/agent/testing/live-evals/result.js +0 -36
  137. package/esm/src/agent/testing/live-evals/runner.d.ts +0 -135
  138. package/esm/src/agent/testing/live-evals/runner.d.ts.map +0 -1
  139. package/esm/src/agent/testing/live-evals/runner.js +0 -396
@@ -1,5 +1,5 @@
1
1
  /**
2
- * Canonical durable runs for task and workflow execution.
2
+ * Canonical durable runs for task, workflow, and eval execution.
3
3
  *
4
4
  * @module
5
5
  *
@@ -1,5 +1,5 @@
1
1
  /**
2
- * Canonical durable runs for task and workflow execution.
2
+ * Canonical durable runs for task, workflow, and eval execution.
3
3
  *
4
4
  * @module
5
5
  *
@@ -1,5 +1,5 @@
1
1
  import type { InferSchema } from "../extensions/schema/index.js";
2
- export declare const getRunKindSchema: () => import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
2
+ export declare const getRunKindSchema: () => import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
3
3
  export declare const getRunStatusSchema: () => import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
4
4
  export declare const getRunOwnerSchema: () => import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
5
5
  kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
@@ -14,7 +14,7 @@ export declare const getRunExecutionErrorSchema: () => import("../internal-agent
14
14
  }>>;
15
15
  export declare const getRunSchema: () => import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
16
16
  run_id: import("../internal-agents/schema.js").Schema<string>;
17
- kind: import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
17
+ kind: import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
18
18
  status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
19
19
  owner: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
20
20
  kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
@@ -59,7 +59,7 @@ export declare const getCreateRunResponseSchema: () => import("../internal-agent
59
59
  duplicate: import("../internal-agents/schema.js").Schema<boolean | undefined>;
60
60
  run: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
61
61
  run_id: import("../internal-agents/schema.js").Schema<string>;
62
- kind: import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
62
+ kind: import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
63
63
  status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
64
64
  owner: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
65
65
  kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
@@ -104,7 +104,7 @@ export declare const getCancelRunResponseSchema: () => import("../internal-agent
104
104
  cancelled: import("../internal-agents/schema.js").Schema<boolean>;
105
105
  run: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
106
106
  run_id: import("../internal-agents/schema.js").Schema<string>;
107
- kind: import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
107
+ kind: import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
108
108
  status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
109
109
  owner: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
110
110
  kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
@@ -174,7 +174,7 @@ export declare const getRunEventListSchema: () => import("../internal-agents/sch
174
174
  export declare const getRunListSchema: () => import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
175
175
  data: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
176
176
  run_id: import("../internal-agents/schema.js").Schema<string>;
177
- kind: import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
177
+ kind: import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
178
178
  status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
179
179
  owner: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
180
180
  kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
@@ -224,7 +224,7 @@ export declare const getRunListSchema: () => import("../internal-agents/schema.j
224
224
  /** Zod schema for a canonical durable run. */
225
225
  export declare const RunSchema: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
226
226
  run_id: import("../internal-agents/schema.js").Schema<string>;
227
- kind: import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
227
+ kind: import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
228
228
  status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
229
229
  owner: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
230
230
  kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
@@ -270,7 +270,7 @@ export declare const CreateRunResponseSchema: import("../internal-agents/schema.
270
270
  duplicate: import("../internal-agents/schema.js").Schema<boolean | undefined>;
271
271
  run: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
272
272
  run_id: import("../internal-agents/schema.js").Schema<string>;
273
- kind: import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
273
+ kind: import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
274
274
  status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
275
275
  owner: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
276
276
  kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
@@ -316,7 +316,7 @@ export declare const CancelRunResponseSchema: import("../internal-agents/schema.
316
316
  cancelled: import("../internal-agents/schema.js").Schema<boolean>;
317
317
  run: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
318
318
  run_id: import("../internal-agents/schema.js").Schema<string>;
319
- kind: import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
319
+ kind: import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
320
320
  status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
321
321
  owner: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
322
322
  kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
@@ -383,7 +383,7 @@ export declare const RunEventListSchema: import("../internal-agents/schema.js").
383
383
  export declare const RunListSchema: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
384
384
  data: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
385
385
  run_id: import("../internal-agents/schema.js").Schema<string>;
386
- kind: import("../internal-agents/schema.js").Schema<"agent" | "task" | "workflow">;
386
+ kind: import("../internal-agents/schema.js").Schema<"eval" | "agent" | "task" | "workflow">;
387
387
  status: import("../internal-agents/schema.js").Schema<"waiting" | "failed" | "completed" | "pending" | "cancelled" | "running">;
388
388
  owner: import("../internal-agents/schema.js").Schema<import("../extensions/schema/schema-validator.js").InferShape<{
389
389
  kind: import("../internal-agents/schema.js").Schema<"project" | "conversation">;
@@ -1 +1 @@
1
- {"version":3,"file":"schemas.d.ts","sourceRoot":"","sources":["../../../src/src/runs/schemas.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAEjE,eAAO,MAAM,gBAAgB,oFAAsE,CAAC;AAEpG,eAAO,MAAM,kBAAkB,+HAE9B,CAAC;AAEF,eAAO,MAAM,iBAAiB;;;GAK7B,CAAC;AAEF,eAAO,MAAM,6BAA6B,uGAEzC,CAAC;AAEF,eAAO,MAAM,uBAAuB,gGAEnC,CAAC;AAEF,eAAO,MAAM,0BAA0B;;;;GAMtC,CAAC;AAEF,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCxB,CAAC;AAEF,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAMtC,CAAC;AAEF,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAKtC,CAAC;AAEF,eAAO,MAAM,iBAAiB;;;;;GAO7B,CAAC;AAEF,eAAO,MAAM,iBAAiB;;;;;GAO7B,CAAC;AAEF,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;GAKjC,CAAC;AAEF,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAK5B,CAAC;AAEF,8CAA8C;AAC9C,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAA2B,CAAC;AAClD,4CAA4C;AAC5C,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAAyC,CAAC;AAC9E,4CAA4C;AAC5C,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAAyC,CAAC;AAC9E,kCAAkC;AAClC,eAAO,MAAM,cAAc;;;;;GAAgC,CAAC;AAC5D,qDAAqD;AACrD,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;GAAoC,CAAC;AACpE,uDAAuD;AACvD,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAA+B,CAAC;AAE1D,kCAAkC;AAClC,MAAM,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC,CAAC;AACvE,oCAAoC;AACpC,MAAM,MAAM,SAAS,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,kBAAkB,CAAC,CAAC,CAAC;AAC3E,mCAAmC;AACnC,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,iBAAiB,CAAC,CAAC,CAAC;AACzE,8DAA8D;AAC9D,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,6BAA6B,CAAC,CAAC,CAAC;AACjG,qEAAqE;AACrE,MAAM,MAAM,cAAc,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,uBAAuB,CAAC,CAAC,CAAC;AACrF,gEAAgE;AAChE,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC;AAC3F,6BAA6B;AAC7B,MAAM,MAAM,GAAG,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC;AAC/D,gDAAgD;AAChD,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC;AAC3F,iDAAiD;AACjD,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC;AAC3F,8BAA8B;AAC9B,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,iBAAiB,CAAC,CAAC,CAAC;AACzE,oCAAoC;AACpC,MAAM,MAAM,YAAY,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,qBAAqB,CAAC,CAAC,CAAC;AACjF,sCAAsC;AACtC,MAAM,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC,CAAC"}
1
+ {"version":3,"file":"schemas.d.ts","sourceRoot":"","sources":["../../../src/src/runs/schemas.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAEjE,eAAO,MAAM,gBAAgB,6FAE5B,CAAC;AAEF,eAAO,MAAM,kBAAkB,+HAE9B,CAAC;AAEF,eAAO,MAAM,iBAAiB;;;GAK7B,CAAC;AAEF,eAAO,MAAM,6BAA6B,uGAEzC,CAAC;AAEF,eAAO,MAAM,uBAAuB,gGAEnC,CAAC;AAEF,eAAO,MAAM,0BAA0B;;;;GAMtC,CAAC;AAEF,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCxB,CAAC;AAEF,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAMtC,CAAC;AAEF,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAKtC,CAAC;AAEF,eAAO,MAAM,iBAAiB;;;;;GAO7B,CAAC;AAEF,eAAO,MAAM,iBAAiB;;;;;GAO7B,CAAC;AAEF,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;GAKjC,CAAC;AAEF,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAK5B,CAAC;AAEF,8CAA8C;AAC9C,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAA2B,CAAC;AAClD,4CAA4C;AAC5C,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAAyC,CAAC;AAC9E,4CAA4C;AAC5C,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAAyC,CAAC;AAC9E,kCAAkC;AAClC,eAAO,MAAM,cAAc;;;;;GAAgC,CAAC;AAC5D,qDAAqD;AACrD,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;GAAoC,CAAC;AACpE,uDAAuD;AACvD,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAA+B,CAAC;AAE1D,kCAAkC;AAClC,MAAM,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC,CAAC;AACvE,oCAAoC;AACpC,MAAM,MAAM,SAAS,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,kBAAkB,CAAC,CAAC,CAAC;AAC3E,mCAAmC;AACnC,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,iBAAiB,CAAC,CAAC,CAAC;AACzE,8DAA8D;AAC9D,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,6BAA6B,CAAC,CAAC,CAAC;AACjG,qEAAqE;AACrE,MAAM,MAAM,cAAc,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,uBAAuB,CAAC,CAAC,CAAC;AACrF,gEAAgE;AAChE,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC;AAC3F,6BAA6B;AAC7B,MAAM,MAAM,GAAG,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC;AAC/D,gDAAgD;AAChD,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC;AAC3F,iDAAiD;AACjD,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC;AAC3F,8BAA8B;AAC9B,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,iBAAiB,CAAC,CAAC,CAAC;AACzE,oCAAoC;AACpC,MAAM,MAAM,YAAY,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,qBAAqB,CAAC,CAAC,CAAC;AACjF,sCAAsC;AACtC,MAAM,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC,CAAC"}
@@ -1,5 +1,5 @@
1
1
  import { defineSchema, lazySchema } from "../schemas/index.js";
2
- export const getRunKindSchema = defineSchema((v) => v.enum(["agent", "workflow", "task"]));
2
+ export const getRunKindSchema = defineSchema((v) => v.enum(["agent", "workflow", "task", "eval"]));
3
3
  export const getRunStatusSchema = defineSchema((v) => v.enum(["pending", "running", "waiting", "completed", "failed", "cancelled"]));
4
4
  export const getRunOwnerSchema = defineSchema((v) => v.object({
5
5
  kind: v.enum(["conversation", "project"]),
@@ -1,3 +1,3 @@
1
1
  /** Shared version value. */
2
- export declare const VERSION = "0.1.871";
2
+ export declare const VERSION = "0.1.873";
3
3
  //# sourceMappingURL=version-constant.d.ts.map
@@ -1,4 +1,4 @@
1
1
  // Keep in sync with deno.json version.
2
2
  // scripts/release.ts updates this constant during releases.
3
3
  /** Shared version value. */
4
- export const VERSION = "0.1.871";
4
+ export const VERSION = "0.1.873";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "veryfront",
3
- "version": "0.1.871",
3
+ "version": "0.1.873",
4
4
  "description": "The simplest way to build AI-powered apps",
5
5
  "keywords": [
6
6
  "react",
@@ -49,6 +49,14 @@
49
49
  "import": "./esm/src/chat/index.js",
50
50
  "types": "./esm/src/chat/index.d.ts"
51
51
  },
52
+ "./chat/ag-ui": {
53
+ "import": "./esm/src/chat/ag-ui.js",
54
+ "types": "./esm/src/chat/ag-ui.d.ts"
55
+ },
56
+ "./chat/protocol": {
57
+ "import": "./esm/src/chat/protocol.js",
58
+ "types": "./esm/src/chat/protocol.d.ts"
59
+ },
52
60
  "./chat/types": {
53
61
  "import": "./esm/src/chat/types.js",
54
62
  "types": "./esm/src/chat/types.d.ts"
@@ -69,9 +77,9 @@
69
77
  "import": "./esm/src/agent/identity-contracts.js",
70
78
  "types": "./esm/src/agent/identity-contracts.d.ts"
71
79
  },
72
- "./agent/testing": {
73
- "import": "./esm/src/agent/testing/index.js",
74
- "types": "./esm/src/agent/testing/index.d.ts"
80
+ "./eval": {
81
+ "import": "./esm/src/eval/index.js",
82
+ "types": "./esm/src/eval/index.d.ts"
75
83
  },
76
84
  "./channels/control-plane": {
77
85
  "import": "./esm/src/channels/control-plane.js",
@@ -1,53 +0,0 @@
1
- import type { Agent, AgentResponse, Message } from "../types.js";
2
- /** Public API contract for test case. */
3
- export interface TestCase {
4
- /** Test name */
5
- name: string;
6
- /** Input to agent */
7
- input: string | Message[];
8
- /** Expected output pattern (regex or string) */
9
- expected?: RegExp | string;
10
- /** Expected tool calls */
11
- expectToolCalls?: string[];
12
- /** Maximum execution time (ms) */
13
- timeout?: number;
14
- /** Custom validator */
15
- validate?: (response: AgentResponse) => boolean | Promise<boolean>;
16
- }
17
- /** Result returned from test. */
18
- export interface TestResult {
19
- /** Test case name */
20
- name: string;
21
- /** Pass/fail */
22
- passed: boolean;
23
- /** Response from agent */
24
- response?: AgentResponse;
25
- /** Error if test failed */
26
- error?: string;
27
- /** Execution time */
28
- executionTime: number;
29
- /** Tool calls made */
30
- toolCalls: string[];
31
- }
32
- /** Public API contract for test suite. */
33
- export interface TestSuite {
34
- /** Suite name */
35
- name: string;
36
- /** Test results */
37
- results: TestResult[];
38
- /** Overall pass/fail */
39
- passed: boolean;
40
- /** Total execution time */
41
- totalTime: number;
42
- }
43
- /** Test agent helper. */
44
- export declare function testAgent(agent: Agent, testCases: TestCase[]): Promise<TestSuite>;
45
- /** Print test results helper. */
46
- export declare function printTestResults(suite: TestSuite): void;
47
- /** Assert contains helper. */
48
- export declare function assertContains(response: AgentResponse, text: string): boolean;
49
- /** Assert tool called helper. */
50
- export declare function assertToolCalled(response: AgentResponse, toolName: string): boolean;
51
- /** Assert completed helper. */
52
- export declare function assertCompleted(response: AgentResponse): boolean;
53
- //# sourceMappingURL=agent-tester.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"agent-tester.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/testing/agent-tester.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEjE,yCAAyC;AACzC,MAAM,WAAW,QAAQ;IACvB,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IAEb,qBAAqB;IACrB,KAAK,EAAE,MAAM,GAAG,OAAO,EAAE,CAAC;IAE1B,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAE3B,0BAA0B;IAC1B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAE3B,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uBAAuB;IACvB,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACpE;AAED,iCAAiC;AACjC,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC;IAEb,gBAAgB;IAChB,MAAM,EAAE,OAAO,CAAC;IAEhB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,aAAa,CAAC;IAEzB,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,qBAAqB;IACrB,aAAa,EAAE,MAAM,CAAC;IAEtB,sBAAsB;IACtB,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,0CAA0C;AAC1C,MAAM,WAAW,SAAS;IACxB,iBAAiB;IACjB,IAAI,EAAE,MAAM,CAAC;IAEb,mBAAmB;IACnB,OAAO,EAAE,UAAU,EAAE,CAAC;IAEtB,wBAAwB;IACxB,MAAM,EAAE,OAAO,CAAC;IAEhB,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,yBAAyB;AACzB,wBAAsB,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAWvF;AA6FD,iCAAiC;AACjC,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,SAAS,GAAG,IAAI,CAmBvD;AAED,8BAA8B;AAC9B,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAE7E;AAED,iCAAiC;AACjC,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAEnF;AAED,+BAA+B;AAC/B,wBAAgB,eAAe,CAAC,QAAQ,EAAE,aAAa,GAAG,OAAO,CAEhE"}
@@ -1,132 +0,0 @@
1
- /**************************
2
- * Agent Testing Utilities
3
- *
4
- * Utilities for testing agents in development and CI/CD.
5
- *
6
- * @module veryfront/agent/testing
7
- **************************/
8
- import * as dntShim from "../../../_dnt.shims.js";
9
- /** Test agent helper. */
10
- export async function testAgent(agent, testCases) {
11
- const suiteStartTime = Date.now();
12
- const results = await Promise.all(testCases.map((testCase) => runTestCase(agent, testCase)));
13
- const passed = results.every((r) => r.passed);
14
- return {
15
- name: agent.id,
16
- results,
17
- passed,
18
- totalTime: Date.now() - suiteStartTime,
19
- };
20
- }
21
- async function runTestCase(agent, testCase) {
22
- const startTime = Date.now();
23
- try {
24
- const timeoutMs = testCase.timeout ?? 30000;
25
- let timeoutId;
26
- let response;
27
- try {
28
- response = await Promise.race([
29
- agent.generate({ input: testCase.input }),
30
- new Promise((_, reject) => {
31
- timeoutId = dntShim.setTimeout(() => reject(new Error("Test timeout")), timeoutMs);
32
- }),
33
- ]);
34
- }
35
- finally {
36
- clearTimeout(timeoutId);
37
- }
38
- const executionTime = Date.now() - startTime;
39
- const toolCalls = response.toolCalls.map((tc) => tc.name);
40
- const { passed, error } = await validateTestCase(testCase, response, toolCalls);
41
- return {
42
- name: testCase.name,
43
- passed,
44
- response,
45
- error,
46
- executionTime,
47
- toolCalls,
48
- };
49
- }
50
- catch (error) {
51
- return {
52
- name: testCase.name,
53
- passed: false,
54
- error: error instanceof Error ? error.message : String(error),
55
- executionTime: Date.now() - startTime,
56
- toolCalls: [],
57
- };
58
- }
59
- }
60
- async function validateTestCase(testCase, response, toolCalls) {
61
- const expected = testCase.expected;
62
- if (expected instanceof RegExp) {
63
- if (!expected.test(response.text)) {
64
- return {
65
- passed: false,
66
- error: `Output "${response.text}" does not match pattern ${expected}`,
67
- };
68
- }
69
- }
70
- else if (typeof expected === "string") {
71
- if (!response.text.includes(expected)) {
72
- return {
73
- passed: false,
74
- error: `Output does not contain expected text: "${expected}"`,
75
- };
76
- }
77
- }
78
- const expectedTools = testCase.expectToolCalls;
79
- if (expectedTools?.length) {
80
- const missingTools = expectedTools.filter((t) => !toolCalls.includes(t));
81
- if (missingTools.length) {
82
- return {
83
- passed: false,
84
- error: `Expected tool calls not found: ${missingTools.join(", ")}`,
85
- };
86
- }
87
- }
88
- if (testCase.validate) {
89
- try {
90
- const passed = await testCase.validate(response);
91
- if (!passed)
92
- return { passed: false, error: "Custom validation failed" };
93
- }
94
- catch (error) {
95
- return {
96
- passed: false,
97
- error: `Custom validation error: ${error instanceof Error ? error.message : String(error)}`,
98
- };
99
- }
100
- }
101
- return { passed: true };
102
- }
103
- /** Print test results helper. */
104
- export function printTestResults(suite) {
105
- console.log(`\n=== Test Suite: ${suite.name} ===\n`);
106
- const passed = suite.results.filter((r) => r.passed).length;
107
- const total = suite.results.length;
108
- for (const [index, result] of suite.results.entries()) {
109
- const icon = result.passed ? "✅" : "❌";
110
- console.log(`${icon} ${index + 1}. ${result.name}`);
111
- if (!result.passed && result.error)
112
- console.log(` Error: ${result.error}`);
113
- if (result.toolCalls.length)
114
- console.log(` Tools used: ${result.toolCalls.join(", ")}`);
115
- console.log(` Time: ${result.executionTime}ms\n`);
116
- }
117
- console.log(`Results: ${passed}/${total} passed`);
118
- console.log(`Total time: ${suite.totalTime}ms`);
119
- console.log(`Status: ${suite.passed ? "✅ PASSED" : "❌ FAILED"}\n`);
120
- }
121
- /** Assert contains helper. */
122
- export function assertContains(response, text) {
123
- return response.text.toLowerCase().includes(text.toLowerCase());
124
- }
125
- /** Assert tool called helper. */
126
- export function assertToolCalled(response, toolName) {
127
- return response.toolCalls.some((tc) => tc.name === toolName);
128
- }
129
- /** Assert completed helper. */
130
- export function assertCompleted(response) {
131
- return response.status === "completed";
132
- }
@@ -1,21 +0,0 @@
1
- import { type LiveEvalApiContext } from "../live-evals/api-client.js";
2
- import { createDurableRunCanaryRunner, type DurableRunCanaryCase, type DurableRunCanaryRunnerConfig } from "./runner.js";
3
- type EnvRecord = Record<string, string | undefined>;
4
- /** Input payload for durable run canary cli case factory. */
5
- export interface DurableRunCanaryCliCaseFactoryInput {
6
- context: LiveEvalApiContext;
7
- requestTimeoutMs: number;
8
- }
9
- /** Input payload for run durable run canary cli. */
10
- export interface RunDurableRunCanaryCliInput {
11
- env: EnvRecord;
12
- agentId: string;
13
- createCases: (input: DurableRunCanaryCliCaseFactoryInput) => DurableRunCanaryCase[];
14
- cwd?: string;
15
- log?: (message: string) => void;
16
- createRunner?: (config: DurableRunCanaryRunnerConfig) => ReturnType<typeof createDurableRunCanaryRunner>;
17
- }
18
- /** Run durable run canary cli. */
19
- export declare function runDurableRunCanaryCli(input: RunDurableRunCanaryCliInput): Promise<number>;
20
- export {};
21
- //# sourceMappingURL=cli-runner.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"cli-runner.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/durable-run-canaries/cli-runner.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,kBAAkB,EAAE,MAAM,6BAA6B,CAAC;AAEtE,OAAO,EACL,4BAA4B,EAC5B,KAAK,oBAAoB,EAEzB,KAAK,4BAA4B,EAClC,MAAM,aAAa,CAAC;AAErB,KAAK,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;AAEpD,6DAA6D;AAC7D,MAAM,WAAW,mCAAmC;IAClD,OAAO,EAAE,kBAAkB,CAAC;IAC5B,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,oDAAoD;AACpD,MAAM,WAAW,2BAA2B;IAC1C,GAAG,EAAE,SAAS,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,CAAC,KAAK,EAAE,mCAAmC,KAAK,oBAAoB,EAAE,CAAC;IACpF,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,YAAY,CAAC,EAAE,CACb,MAAM,EAAE,4BAA4B,KACjC,UAAU,CAAC,OAAO,4BAA4B,CAAC,CAAC;CACtD;AAcD,kCAAkC;AAClC,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,2BAA2B,GACjC,OAAO,CAAC,MAAM,CAAC,CAwEjB"}
@@ -1,66 +0,0 @@
1
- import { mkdir, writeTextFile } from "../../../platform/compat/fs.js";
2
- import { dirname, resolve } from "../../../platform/compat/path/index.js";
3
- import { cwd as getProcessCwd } from "../../../platform/compat/process.js";
4
- import { resolveDurableRunCanaryEnvironment } from "./environment.js";
5
- import { createDurableRunCanaryRunner, } from "./runner.js";
6
- function createTimestampedReportPath(input) {
7
- return resolve(input.cwd, ".omx/logs", input.directory, `${new Date().toISOString().replaceAll(":", "-").replaceAll(".", "-")}.json`);
8
- }
9
- /** Run durable run canary cli. */
10
- export async function runDurableRunCanaryCli(input) {
11
- const log = input.log ?? console.log;
12
- const cwd = input.cwd ?? getProcessCwd();
13
- const { apiUrl, authToken, projectId, requestTimeoutMs, keepSuccessfulEvidence } = resolveDurableRunCanaryEnvironment(input.env);
14
- const reportPath = input.env.DURABLE_CANARY_REPORT_PATH ??
15
- createTimestampedReportPath({ cwd, directory: "durable-run-staging-canaries" });
16
- if (!authToken) {
17
- throw new Error("Missing VERYFRONT_TOKEN");
18
- }
19
- if (!projectId) {
20
- throw new Error("Missing AG_UI_EVAL_PROJECT_ID");
21
- }
22
- const context = {
23
- apiUrl,
24
- authToken,
25
- projectId,
26
- };
27
- const createRunner = input.createRunner ?? createDurableRunCanaryRunner;
28
- const { runCase } = createRunner({
29
- apiUrl,
30
- authToken,
31
- agentId: input.agentId,
32
- projectId,
33
- requestTimeoutMs,
34
- keepSuccessfulEvidence,
35
- });
36
- const testCases = input.createCases({
37
- context,
38
- requestTimeoutMs,
39
- });
40
- log(`Durable run canaries -> ${apiUrl}`);
41
- log(`Project scope -> ${projectId}`);
42
- const results = [];
43
- for (const testCase of testCases) {
44
- log(`\n[run] ${testCase.label}`);
45
- const result = await runCase(testCase);
46
- results.push(result);
47
- log(`[${result.status}] ${result.id}: ${result.details}`);
48
- }
49
- const summary = {
50
- passed: results.filter((result) => result.status === "pass").length,
51
- failed: results.filter((result) => result.status === "fail").length,
52
- };
53
- await mkdir(dirname(reportPath), { recursive: true });
54
- await writeTextFile(reportPath, JSON.stringify({
55
- generatedAt: new Date().toISOString(),
56
- apiUrl,
57
- projectId,
58
- results,
59
- summary,
60
- }, null, 2));
61
- log("\nSummary");
62
- log(`passed: ${summary.passed}`);
63
- log(`failed: ${summary.failed}`);
64
- log(`report: ${reportPath}`);
65
- return summary.failed > 0 ? 1 : 0;
66
- }
@@ -1,14 +0,0 @@
1
- import { type AgentServiceConfigInput } from "../../service/config.js";
2
- /** Public API contract for durable run canary environment. */
3
- export interface DurableRunCanaryEnvironment {
4
- apiUrl: string;
5
- authToken: string;
6
- projectId: string;
7
- requestTimeoutMs: number;
8
- keepSuccessfulEvidence: boolean;
9
- }
10
- /** Default value for durable run canary timeout ms. */
11
- export declare const DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS = 240000;
12
- /** Resolves durable run canary environment. */
13
- export declare function resolveDurableRunCanaryEnvironment(env?: AgentServiceConfigInput): DurableRunCanaryEnvironment;
14
- //# sourceMappingURL=environment.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"environment.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/durable-run-canaries/environment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,uBAAuB,EAA2B,MAAM,yBAAyB,CAAC;AAEhG,8DAA8D;AAC9D,MAAM,WAAW,2BAA2B;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,EAAE,OAAO,CAAC;CACjC;AAED,uDAAuD;AACvD,eAAO,MAAM,qCAAqC,SAAU,CAAC;AAE7D,+CAA+C;AAC/C,wBAAgB,kCAAkC,CAChD,GAAG,GAAE,uBAA4B,GAChC,2BAA2B,CAY7B"}
@@ -1,15 +0,0 @@
1
- import { parseAgentServiceConfig } from "../../service/config.js";
2
- /** Default value for durable run canary timeout ms. */
3
- export const DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS = 240_000;
4
- /** Resolves durable run canary environment. */
5
- export function resolveDurableRunCanaryEnvironment(env = {}) {
6
- return {
7
- apiUrl: typeof env.VERYFRONT_API_URL === "string"
8
- ? env.VERYFRONT_API_URL
9
- : parseAgentServiceConfig(env).VERYFRONT_API_URL,
10
- authToken: typeof env.VERYFRONT_TOKEN === "string" ? env.VERYFRONT_TOKEN : "",
11
- projectId: typeof env.AG_UI_EVAL_PROJECT_ID === "string" ? env.AG_UI_EVAL_PROJECT_ID : "",
12
- requestTimeoutMs: Number(env.DURABLE_CANARY_TIMEOUT_MS ?? DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS),
13
- keepSuccessfulEvidence: env.DURABLE_CANARY_KEEP_SUCCESS === "1",
14
- };
15
- }
@@ -1,5 +0,0 @@
1
- export { type DurableRunCanaryCliCaseFactoryInput, runDurableRunCanaryCli, type RunDurableRunCanaryCliInput, } from "./cli-runner.js";
2
- export { DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS, type DurableRunCanaryEnvironment, resolveDurableRunCanaryEnvironment, } from "./environment.js";
3
- export { createDurableRunCanaryApiClient, createDurableRunCanaryRunner, type DurableRunCanaryApiClient, type DurableRunCanaryApiConfig, type DurableRunCanaryCase, type DurableRunCanaryCreateRootRunInput, type DurableRunCanaryMessage, type DurableRunCanaryPreparedCase, type DurableRunCanaryResult, type DurableRunCanaryRunnerConfig, durableRunCanaryRunnerInternals, type DurableRunCanaryRunSummary, type DurableRunCanarySendUserMessageInput, type DurableRunCanaryStartRunInput, getDurableRunCanaryMessageSchema, parseDurableRunCanaryRunSummary, } from "./runner.js";
4
- export { assertCompleted, assertNoMalformedCreateFileToolCalls, collectAssistantText, findAssistantMessage, stringifyUnknown, } from "./validation.js";
5
- //# sourceMappingURL=index.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/durable-run-canaries/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,mCAAmC,EACxC,sBAAsB,EACtB,KAAK,2BAA2B,GACjC,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,qCAAqC,EACrC,KAAK,2BAA2B,EAChC,kCAAkC,GACnC,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,+BAA+B,EAC/B,4BAA4B,EAC5B,KAAK,yBAAyB,EAC9B,KAAK,yBAAyB,EAC9B,KAAK,oBAAoB,EACzB,KAAK,kCAAkC,EACvC,KAAK,uBAAuB,EAC5B,KAAK,4BAA4B,EACjC,KAAK,sBAAsB,EAC3B,KAAK,4BAA4B,EACjC,+BAA+B,EAC/B,KAAK,0BAA0B,EAC/B,KAAK,oCAAoC,EACzC,KAAK,6BAA6B,EAClC,gCAAgC,EAChC,+BAA+B,GAChC,MAAM,aAAa,CAAC;AAErB,OAAO,EACL,eAAe,EACf,oCAAoC,EACpC,oBAAoB,EACpB,oBAAoB,EACpB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC"}
@@ -1,4 +0,0 @@
1
- export { runDurableRunCanaryCli, } from "./cli-runner.js";
2
- export { DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS, resolveDurableRunCanaryEnvironment, } from "./environment.js";
3
- export { createDurableRunCanaryApiClient, createDurableRunCanaryRunner, durableRunCanaryRunnerInternals, getDurableRunCanaryMessageSchema, parseDurableRunCanaryRunSummary, } from "./runner.js";
4
- export { assertCompleted, assertNoMalformedCreateFileToolCalls, collectAssistantText, findAssistantMessage, stringifyUnknown, } from "./validation.js";
@@ -1,118 +0,0 @@
1
- import type { InferSchema } from "../../../extensions/schema/index.js";
2
- /** Configuration used by durable run canary API. */
3
- export interface DurableRunCanaryApiConfig {
4
- apiUrl: string;
5
- authToken: string;
6
- agentId: string;
7
- projectId: string | null;
8
- branchId?: string | null;
9
- requestTimeoutMs: number;
10
- fetch?: (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
11
- }
12
- /** Input payload for durable run canary create root run. */
13
- export interface DurableRunCanaryCreateRootRunInput {
14
- conversationId: string;
15
- runId: string;
16
- }
17
- /** Input payload for durable run canary send user message. */
18
- export interface DurableRunCanarySendUserMessageInput {
19
- conversationId: string;
20
- prompt: string;
21
- }
22
- /** Input payload for durable run canary start run. */
23
- export interface DurableRunCanaryStartRunInput extends DurableRunCanaryCreateRootRunInput {
24
- messageId: string;
25
- prompt: string;
26
- userMessageId: string;
27
- }
28
- /** Zod schema for get durable run canary message. */
29
- export declare const getDurableRunCanaryMessageSchema: () => import("../../../internal-agents/schema.js").Schema<{
30
- id: string;
31
- role: "tool" | "user" | "assistant" | "system";
32
- parts: ({
33
- type: string;
34
- } & {} & Record<string, unknown>)[];
35
- } & {
36
- status?: string | undefined;
37
- } & Record<string, unknown>>;
38
- /** Message shape for durable run canary. */
39
- export type DurableRunCanaryMessage = InferSchema<ReturnType<typeof getDurableRunCanaryMessageSchema>>;
40
- /** Public API contract for durable run canary run summary. */
41
- export interface DurableRunCanaryRunSummary {
42
- runId: string;
43
- conversationId: string;
44
- messageId: string;
45
- agentId: string;
46
- status: string;
47
- latestEventId: number;
48
- latestExternalEventSequence: number | null;
49
- waitingToolCallId: string | null;
50
- waitingToolName: string | null;
51
- terminalErrorCode: string | null;
52
- terminalErrorMessage: string | null;
53
- startedAt: string | null;
54
- finishedAt: string | null;
55
- }
56
- /** Parses durable run canary run summary. */
57
- export declare function parseDurableRunCanaryRunSummary(value: unknown): DurableRunCanaryRunSummary;
58
- /** Public API contract for durable run canary API client. */
59
- export interface DurableRunCanaryApiClient {
60
- createDurableRootRun: (input: DurableRunCanaryCreateRootRunInput) => Promise<void>;
61
- getRunSummary: (input: DurableRunCanaryCreateRootRunInput) => Promise<DurableRunCanaryRunSummary>;
62
- listMessagesForCanary: (input: {
63
- conversationId: string;
64
- }) => Promise<DurableRunCanaryMessage[]>;
65
- sendUserMessageForCanary: (input: DurableRunCanarySendUserMessageInput) => Promise<DurableRunCanaryMessage>;
66
- startDurableRun: (input: DurableRunCanaryStartRunInput) => Promise<void>;
67
- }
68
- /** Create durable run canary API client. */
69
- export declare function createDurableRunCanaryApiClient(config: DurableRunCanaryApiConfig): DurableRunCanaryApiClient;
70
- /** Result returned from durable run canary. */
71
- export interface DurableRunCanaryResult {
72
- id: string;
73
- label: string;
74
- status: "pass" | "fail";
75
- details: string;
76
- durationMs: number;
77
- conversationId: string;
78
- runId: string;
79
- artifactPaths?: string[];
80
- }
81
- /** Public API contract for durable run canary prepared case. */
82
- export interface DurableRunCanaryPreparedCase {
83
- artifactPaths?: string[] | ((runId: string) => string[]);
84
- cleanup: (input?: {
85
- runId: string;
86
- }) => Promise<void>;
87
- conversationId: string;
88
- prompt: string;
89
- startSidecar?: () => Promise<(() => Promise<void>) | void>;
90
- title: string;
91
- validate: (input: {
92
- messages: DurableRunCanaryMessage[];
93
- run: DurableRunCanaryRunSummary;
94
- }) => Promise<void> | void;
95
- }
96
- /** Public API contract for durable run canary case. */
97
- export interface DurableRunCanaryCase {
98
- id: string;
99
- label: string;
100
- prepare: () => Promise<DurableRunCanaryPreparedCase>;
101
- }
102
- /** Configuration used by durable run canary runner. */
103
- export interface DurableRunCanaryRunnerConfig extends DurableRunCanaryApiConfig {
104
- keepSuccessfulEvidence: boolean;
105
- }
106
- declare function collectReferencedChildConversationIds(messages: DurableRunCanaryMessage[]): string[];
107
- declare function isTerminalRunStatus(status: string): boolean;
108
- /** Create durable run canary runner. */
109
- export declare function createDurableRunCanaryRunner(config: DurableRunCanaryRunnerConfig, apiClient?: DurableRunCanaryApiClient): {
110
- runCase: (testCase: DurableRunCanaryCase) => Promise<DurableRunCanaryResult>;
111
- };
112
- /** White-box helpers used by durable run canary tests. */
113
- export declare const durableRunCanaryRunnerInternals: {
114
- collectReferencedChildConversationIds: typeof collectReferencedChildConversationIds;
115
- isTerminalRunStatus: typeof isTerminalRunStatus;
116
- };
117
- export {};
118
- //# sourceMappingURL=runner.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/durable-run-canaries/runner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qCAAqC,CAAC;AAIvE,oDAAoD;AACpD,MAAM,WAAW,yBAAyB;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;CAClF;AAED,4DAA4D;AAC5D,MAAM,WAAW,kCAAkC;IACjD,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,8DAA8D;AAC9D,MAAM,WAAW,oCAAoC;IACnD,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,sDAAsD;AACtD,MAAM,WAAW,6BAA8B,SAAQ,kCAAkC;IACvF,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,qDAAqD;AACrD,eAAO,MAAM,gCAAgC;;;;;;;;4BAO5C,CAAC;AAEF,4CAA4C;AAC5C,MAAM,MAAM,uBAAuB,GAAG,WAAW,CAC/C,UAAU,CAAC,OAAO,gCAAgC,CAAC,CACpD,CAAC;AAEF,8DAA8D;AAC9D,MAAM,WAAW,0BAA0B;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AA4CD,6CAA6C;AAC7C,wBAAgB,+BAA+B,CAAC,KAAK,EAAE,OAAO,GAAG,0BAA0B,CAoC1F;AAkGD,6DAA6D;AAC7D,MAAM,WAAW,yBAAyB;IACxC,oBAAoB,EAAE,CAAC,KAAK,EAAE,kCAAkC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACnF,aAAa,EAAE,CAAC,KAAK,EAAE,kCAAkC,KAAK,OAAO,CAAC,0BAA0B,CAAC,CAAC;IAClG,qBAAqB,EAAE,CAAC,KAAK,EAAE;QAAE,cAAc,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,uBAAuB,EAAE,CAAC,CAAC;IACjG,wBAAwB,EAAE,CACxB,KAAK,EAAE,oCAAoC,KACxC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACtC,eAAe,EAAE,CAAC,KAAK,EAAE,6BAA6B,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1E;AAED,4CAA4C;AAC5C,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,yBAAyB,GAChC,yBAAyB,CAgF3B;AAED,+CAA+C;AAC/C,MAAM,WAAW,sBAAsB;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,gEAAgE;AAChE,MAAM,WAAW,4BAA4B;IAC3C,aAAa,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC,CAAC;IACzD,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAC3D,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,CAAC,KAAK,EAAE;QAChB,QAAQ,EAAE,uBAAuB,EAAE,CAAC;QACpC,GAAG,EAAE,0BAA0B,CAAC;KACjC,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CAC5B;AAED,uDAAuD;AACvD,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,OAAO,CAAC,4BAA4B,CAAC,CAAC;CACtD;AAED,uDAAuD;AACvD,MAAM,WAAW,4BAA6B,SAAQ,yBAAyB;IAC7E,sBAAsB,EAAE,OAAO,CAAC;CACjC;AA0DD,iBAAS,qCAAqC,CAAC,QAAQ,EAAE,uBAAuB,EAAE,GAAG,MAAM,EAAE,CAc5F;AAQD,iBAAS,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEpD;AA2CD,wCAAwC;AACxC,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,4BAA4B,EACpC,SAAS,GAAE,yBAAmE;wBAkB7C,oBAAoB,KAAG,OAAO,CAAC,sBAAsB,CAAC;EAqFxF;AAED,0DAA0D;AAC1D,eAAO,MAAM,+BAA+B;;;CAG3C,CAAC"}