pi-harness-runtime 1.1.440 → 1.1.441

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (349) hide show
  1. package/harness/e2e/auto-test-loop.ts +2 -0
  2. package/harness/e2e/browser-scrapers/base-browser-scraper.ts +2 -0
  3. package/harness/e2e/browser-scrapers/chrome-profile-manager.ts +2 -0
  4. package/harness/e2e/browser-scrapers/codex-browser-scraper.ts +2 -0
  5. package/harness/e2e/tools-detector.ts +2 -0
  6. package/harness/key-loader.ts +2 -0
  7. package/harness/langchain/checkpointer.ts +2 -0
  8. package/harness/langchain/cron-cli.ts +2 -0
  9. package/harness/langchain/graph.ts +2 -0
  10. package/harness/notification-center.ts +2 -0
  11. package/harness/secret-provider.ts +2 -0
  12. package/index.ts +2 -0
  13. package/package.json +7 -1
  14. package/packages/a2a-adapter/package.json +1 -1
  15. package/packages/agent-policy/dist/extension.d.ts.map +1 -1
  16. package/packages/agent-policy/dist/extension.js +2 -0
  17. package/packages/agent-policy/dist/extension.js.map +1 -1
  18. package/packages/agent-policy/package.json +1 -1
  19. package/packages/agent-policy/src/extension.ts +2 -0
  20. package/packages/architecture-generator/package.json +1 -1
  21. package/packages/auth/dist/minimax-browser-auth.d.ts.map +1 -1
  22. package/packages/auth/dist/minimax-browser-auth.js +23 -1
  23. package/packages/auth/dist/minimax-browser-auth.js.map +1 -1
  24. package/packages/auth/package.json +1 -1
  25. package/packages/auth/src/minimax-browser-auth.ts +30 -5
  26. package/packages/autonomous-refactor/package.json +1 -1
  27. package/packages/autonomous-runtime/dist/inbox.d.ts.map +1 -1
  28. package/packages/autonomous-runtime/dist/inbox.js +2 -0
  29. package/packages/autonomous-runtime/dist/inbox.js.map +1 -1
  30. package/packages/autonomous-runtime/package.json +1 -1
  31. package/packages/autonomous-runtime/src/inbox.ts +2 -0
  32. package/packages/cache-strategy/package.json +1 -1
  33. package/packages/capability-registry/package.json +1 -1
  34. package/packages/checkpoint/package.json +1 -1
  35. package/packages/cli-plugin-sdk/package.json +1 -1
  36. package/packages/clipboard/dist/src/clipboard.d.ts.map +1 -1
  37. package/packages/clipboard/dist/src/clipboard.js +2 -0
  38. package/packages/clipboard/dist/src/clipboard.js.map +1 -1
  39. package/packages/clipboard/dist/src/copy-sync.d.ts.map +1 -1
  40. package/packages/clipboard/dist/src/copy-sync.js +2 -0
  41. package/packages/clipboard/dist/src/copy-sync.js.map +1 -1
  42. package/packages/clipboard/dist/src/gist-relay.d.ts.map +1 -1
  43. package/packages/clipboard/dist/src/gist-relay.js +2 -0
  44. package/packages/clipboard/dist/src/gist-relay.js.map +1 -1
  45. package/packages/clipboard/package.json +1 -1
  46. package/packages/clipboard/src/clipboard.ts +2 -0
  47. package/packages/clipboard/src/copy-sync.ts +2 -0
  48. package/packages/clipboard/src/gist-relay.ts +2 -0
  49. package/packages/clipboard-plugin/package.json +1 -1
  50. package/packages/code-generation/package.json +1 -1
  51. package/packages/code-review/package.json +1 -1
  52. package/packages/codex-adapter/package.json +1 -1
  53. package/packages/codex-watcher/package.json +1 -1
  54. package/packages/config-capture/package.json +1 -1
  55. package/packages/connection-runtime/package.json +1 -1
  56. package/packages/context-compiler/package.json +1 -1
  57. package/packages/context-discovery/package.json +1 -1
  58. package/packages/context-manager/package.json +1 -1
  59. package/packages/contracts/dist/delegation-store.js +1 -4
  60. package/packages/contracts/dist/workspace-env.js +28 -1
  61. package/packages/contracts/package.json +1 -1
  62. package/packages/contracts/src/delegation-store.ts +9 -4
  63. package/packages/contracts/src/workspace-env.ts +29 -4
  64. package/packages/cookie-sanitizer/package.json +1 -1
  65. package/packages/cost-optimizer/package.json +1 -1
  66. package/packages/dependency-analyzer/package.json +1 -1
  67. package/packages/django-plugin/package.json +1 -1
  68. package/packages/doc-generator/package.json +1 -1
  69. package/packages/durable-review/package.json +1 -1
  70. package/packages/e2e-tester/package.json +1 -1
  71. package/packages/evaluation-engine/package.json +1 -1
  72. package/packages/evaluation-runner/package.json +1 -1
  73. package/packages/event-bus/package.json +1 -1
  74. package/packages/event-sink/package.json +1 -1
  75. package/packages/event-store/dist/src/store.d.ts.map +1 -1
  76. package/packages/event-store/dist/src/store.js +2 -0
  77. package/packages/event-store/dist/src/store.js.map +1 -1
  78. package/packages/event-store/package.json +1 -1
  79. package/packages/event-store/src/store.ts +3 -0
  80. package/packages/experience-replay/package.json +1 -1
  81. package/packages/feedback-collector/package.json +1 -1
  82. package/packages/file-copy-helper/package.json +1 -1
  83. package/packages/framework-detector/package.json +1 -1
  84. package/packages/framework-plugin-sdk/package.json +1 -1
  85. package/packages/frappe-plugin/package.json +1 -1
  86. package/packages/generic-web-plugin/dist/analyzer.d.ts.map +1 -1
  87. package/packages/generic-web-plugin/dist/analyzer.js +2 -0
  88. package/packages/generic-web-plugin/dist/analyzer.js.map +1 -1
  89. package/packages/generic-web-plugin/package.json +1 -1
  90. package/packages/generic-web-plugin/src/analyzer.ts +3 -0
  91. package/packages/health-monitor/package.json +1 -1
  92. package/packages/intent-analyzer/package.json +1 -1
  93. package/packages/jev-judge/package.json +1 -1
  94. package/packages/knowledge-graph/package.json +1 -1
  95. package/packages/knowledge-retrieval/dist/mcp-client.d.ts.map +1 -1
  96. package/packages/knowledge-retrieval/dist/mcp-client.js +0 -1
  97. package/packages/knowledge-retrieval/dist/mcp-client.js.map +1 -1
  98. package/packages/knowledge-retrieval/package.json +1 -4
  99. package/packages/knowledge-retrieval/src/mcp-client.ts +0 -1
  100. package/packages/laravel-plugin/package.json +1 -1
  101. package/packages/learning-engine/package.json +1 -1
  102. package/packages/logging/package.json +1 -1
  103. package/packages/mcp-adapter/dist/server.d.ts.map +1 -1
  104. package/packages/mcp-adapter/dist/server.js +2 -0
  105. package/packages/mcp-adapter/dist/server.js.map +1 -1
  106. package/packages/mcp-adapter/package.json +1 -1
  107. package/packages/mcp-adapter/src/server.ts +3 -0
  108. package/packages/memory-engine/package.json +1 -1
  109. package/packages/milestone-manager/package.json +1 -1
  110. package/packages/model-registry/package.json +1 -1
  111. package/packages/nextjs-plugin/package.json +1 -1
  112. package/packages/notification/package.json +1 -1
  113. package/packages/observability/package.json +1 -1
  114. package/packages/okf-indexer/package.json +1 -1
  115. package/packages/paperclip-integration/dist/index.d.ts.map +1 -1
  116. package/packages/paperclip-integration/dist/index.js +2 -0
  117. package/packages/paperclip-integration/dist/index.js.map +1 -1
  118. package/packages/paperclip-integration/package.json +1 -1
  119. package/packages/paperclip-integration/src/index.ts +2 -0
  120. package/packages/performance-optimizer/package.json +1 -1
  121. package/packages/permission-mode/package.json +1 -1
  122. package/packages/privilege-broker/dist/audit.d.ts.map +1 -1
  123. package/packages/privilege-broker/dist/audit.js +2 -0
  124. package/packages/privilege-broker/dist/audit.js.map +1 -1
  125. package/packages/privilege-broker/package.json +1 -1
  126. package/packages/privilege-broker/src/audit.ts +2 -0
  127. package/packages/project-analyzer/package.json +1 -1
  128. package/packages/project-bootstrap/package.json +1 -1
  129. package/packages/projection-engine/package.json +1 -1
  130. package/packages/prompt-compiler/dist/compiler.d.ts.map +1 -1
  131. package/packages/prompt-compiler/dist/compiler.js +2 -0
  132. package/packages/prompt-compiler/dist/compiler.js.map +1 -1
  133. package/packages/prompt-compiler/package.json +1 -1
  134. package/packages/prompt-compiler/src/compiler.ts +2 -0
  135. package/packages/prompt-versioning/package.json +1 -1
  136. package/packages/provider-adapter-sdk/package.json +1 -1
  137. package/packages/provider-router/package.json +1 -1
  138. package/packages/provider-selector/package.json +1 -1
  139. package/packages/providers/package.json +1 -1
  140. package/packages/qdrant-skills/package.json +1 -1
  141. package/packages/quota-manager/package.json +1 -1
  142. package/packages/rate-limiter/package.json +1 -1
  143. package/packages/react-vite-plugin/package.json +1 -1
  144. package/packages/relay/package.json +1 -1
  145. package/packages/release-manager/package.json +1 -1
  146. package/packages/requirement-compiler/package.json +1 -1
  147. package/packages/runtime/package.json +1 -1
  148. package/packages/scheduler/package.json +1 -1
  149. package/packages/scheduler-adapter/package.json +1 -1
  150. package/packages/session/package.json +1 -1
  151. package/packages/session-api/package.json +1 -1
  152. package/packages/session-export/package.json +1 -1
  153. package/packages/shared-context/package.json +1 -1
  154. package/packages/skill-mcp-client/dist/client.d.ts.map +1 -1
  155. package/packages/skill-mcp-client/dist/client.js +0 -1
  156. package/packages/skill-mcp-client/dist/client.js.map +1 -1
  157. package/packages/skill-mcp-client/package.json +1 -4
  158. package/packages/skill-mcp-client/src/client.ts +0 -1
  159. package/packages/skill-registry/package.json +1 -1
  160. package/packages/skills/dist/parser.d.ts.map +1 -1
  161. package/packages/skills/dist/parser.js +2 -0
  162. package/packages/skills/dist/parser.js.map +1 -1
  163. package/packages/skills/dist/skill-versioning.d.ts.map +1 -1
  164. package/packages/skills/dist/skill-versioning.js +2 -0
  165. package/packages/skills/dist/skill-versioning.js.map +1 -1
  166. package/packages/skills/package.json +1 -1
  167. package/packages/skills/src/parser.ts +3 -0
  168. package/packages/skills/src/skill-versioning.ts +2 -0
  169. package/packages/sprint-planner/package.json +1 -1
  170. package/packages/subscription-engine/package.json +1 -1
  171. package/packages/task-compiler/package.json +1 -1
  172. package/packages/tencentdb-memory/package.json +1 -1
  173. package/packages/tencentdb-sync/package.json +1 -1
  174. package/packages/test-data-generator/package.json +1 -1
  175. package/packages/test-generator/package.json +1 -1
  176. package/packages/todo-bd-sync/package.json +1 -1
  177. package/packages/token-estimation/package.json +1 -1
  178. package/packages/token-optimizer/package.json +1 -1
  179. package/packages/tooling-preference/package.json +1 -1
  180. package/packages/trajectory/package.json +1 -1
  181. package/packages/tui/package.json +1 -1
  182. package/packages/two-phase-review/package.json +1 -1
  183. package/packages/types/package.json +1 -1
  184. package/packages/voice-api/package.json +1 -1
  185. package/packages/webmcp-tester/dist/index.js +0 -2
  186. package/packages/webmcp-tester/package.json +1 -1
  187. package/packages/webmcp-tester/src/index.ts +0 -3
  188. package/packages/wiki/package.json +1 -1
  189. package/packages/workflow-events/package.json +1 -1
  190. package/packages/workspace-scanner/package.json +1 -1
  191. package/packages/worktree/package.json +1 -1
  192. package/packages/write-review/dist/trigger.d.ts.map +1 -1
  193. package/packages/write-review/dist/trigger.js +1 -2
  194. package/packages/write-review/dist/trigger.js.map +1 -1
  195. package/packages/write-review/package.json +1 -1
  196. package/packages/write-review/src/trigger.ts +1 -2
  197. package/scripts/lib/runtime-imports.ts +189 -3
  198. package/scripts/telegram-webhook-server.ts +2 -0
  199. package/scripts/verify-package-contents.ts +148 -1
  200. package/harness/e2e/tools-detector.test.ts +0 -155
  201. package/harness/glm-quota-countdown.test.ts +0 -323
  202. package/harness/langchain/agents.test.ts +0 -39
  203. package/harness/langchain/checkpointer.test.ts +0 -243
  204. package/harness/langchain/daemon.test.ts +0 -459
  205. package/harness/langchain/model-router.test.ts +0 -191
  206. package/harness/langchain/ping-pong-middleware.test.ts +0 -69
  207. package/harness/langchain/scoreboard.test.ts +0 -267
  208. package/harness/langchain/status-line.test.ts +0 -106
  209. package/harness/langchain/surge.test.ts +0 -467
  210. package/harness/langchain/widget.test.ts +0 -251
  211. package/harness/skill-sync.test.ts +0 -57
  212. package/harness/ssh-detach-interceptor.test.ts +0 -263
  213. package/harness/ssh-hang-recovery.test.ts +0 -86
  214. package/packages/a2a-adapter/test/a2a.test.ts +0 -151
  215. package/packages/agent-policy/test/interlock.test.ts +0 -101
  216. package/packages/agent-policy/test/policy-retention.test.ts +0 -380
  217. package/packages/agent-policy/test/policy.test.ts +0 -387
  218. package/packages/auth/test/auth.test.ts +0 -238
  219. package/packages/autonomous-refactor/test/refactor.test.ts +0 -202
  220. package/packages/autonomous-runtime/test/types.test.ts +0 -95
  221. package/packages/cache-strategy/test/cache.test.ts +0 -137
  222. package/packages/capability-registry/test/registry.test.ts +0 -204
  223. package/packages/checkpoint/test/engine.test.ts +0 -265
  224. package/packages/cli-plugin-sdk/test/cli.test.ts +0 -45
  225. package/packages/code-generation/test/generator.test.ts +0 -238
  226. package/packages/code-review/test/engine.test.ts +0 -302
  227. package/packages/codex-adapter/test/provider.test.ts +0 -142
  228. package/packages/codex-watcher/test/watch.test.ts +0 -453
  229. package/packages/context-compiler/test/context-compiler.test.ts +0 -746
  230. package/packages/context-manager/test/context.test.ts +0 -198
  231. package/packages/contracts/test/delegation.test.ts +0 -350
  232. package/packages/contracts/test/workspace-env.test.ts +0 -802
  233. package/packages/cookie-sanitizer/test/infer.test.ts +0 -110
  234. package/packages/cookie-sanitizer/test/normalize.test.ts +0 -113
  235. package/packages/cookie-sanitizer/test/parse.test.ts +0 -204
  236. package/packages/cookie-sanitizer/test/sync.test.ts +0 -237
  237. package/packages/cookie-sanitizer/test/watcher.test.ts +0 -27
  238. package/packages/cost-optimizer/test/cost-optimizer.test.ts +0 -438
  239. package/packages/dependency-analyzer/test/analyzer.test.ts +0 -49
  240. package/packages/django-plugin/test/analyzer.test.ts +0 -195
  241. package/packages/doc-generator/test/docs.test.ts +0 -168
  242. package/packages/durable-review/test/acceptance.test.ts +0 -357
  243. package/packages/durable-review/test/smoke.test.ts +0 -107
  244. package/packages/evaluation-engine/test/engine.test.ts +0 -277
  245. package/packages/evaluation-runner/test/runner.test.ts +0 -168
  246. package/packages/event-bus/test/bus.test.ts +0 -100
  247. package/packages/event-store/test/store.test.ts +0 -115
  248. package/packages/experience-replay/test/replay.test.ts +0 -267
  249. package/packages/feedback-collector/test/collector.test.ts +0 -142
  250. package/packages/framework-detector/test/detector.test.ts +0 -75
  251. package/packages/framework-plugin-sdk/test/manager.test.ts +0 -320
  252. package/packages/frappe-plugin/test/analyzer.test.ts +0 -250
  253. package/packages/generic-web-plugin/test/analyzer.test.ts +0 -134
  254. package/packages/health-monitor/test/health.test.ts +0 -231
  255. package/packages/intent-analyzer/test/analyzer.test.ts +0 -72
  256. package/packages/intent-analyzer/test/ping-pong.test.ts +0 -96
  257. package/packages/jev-judge/tests/auto-continue-context.test.ts +0 -280
  258. package/packages/jev-judge/tests/continuation-controller.test.ts +0 -1396
  259. package/packages/jev-judge/tests/jev-provider-chain.test.ts +0 -536
  260. package/packages/jev-judge/tests/outcome-application.test.ts +0 -132
  261. package/packages/jev-judge/tests/question-risk.test.ts +0 -129
  262. package/packages/jev-judge/tests/typesafe-provider.test.ts +0 -640
  263. package/packages/jev-judge/tests/validation-ledger.test.ts +0 -181
  264. package/packages/jev-judge/tests/validation-plan.test.ts +0 -170
  265. package/packages/jev-judge/tests/validation-tie-breaker.test.ts +0 -88
  266. package/packages/laravel-plugin/test/analyzer.test.ts +0 -220
  267. package/packages/learning-engine/test/engine.test.ts +0 -331
  268. package/packages/logging/test/buffer.test.ts +0 -44
  269. package/packages/logging/test/classification.test.ts +0 -137
  270. package/packages/logging/test/config.test.ts +0 -97
  271. package/packages/logging/test/fanout.test.ts +0 -186
  272. package/packages/logging/test/identity-transport.test.ts +0 -197
  273. package/packages/logging/test/incidents.test.ts +0 -404
  274. package/packages/logging/test/ingest-schema.test.ts +0 -184
  275. package/packages/logging/test/integration.test.ts +0 -140
  276. package/packages/logging/test/log-http.test.ts +0 -246
  277. package/packages/logging/test/redact.test.ts +0 -109
  278. package/packages/logging/test/sampler.test.ts +0 -57
  279. package/packages/logging/test/server.test.ts +0 -444
  280. package/packages/logging/test/shipper.test.ts +0 -152
  281. package/packages/logging/test/spool.test.ts +0 -96
  282. package/packages/mcp-adapter/test/server.test.ts +0 -91
  283. package/packages/memory-engine/test/engine.test.ts +0 -456
  284. package/packages/model-registry/test/registry.test.ts +0 -342
  285. package/packages/nextjs-plugin/test/analyzer.test.ts +0 -149
  286. package/packages/notification/test/approval-payload.test.ts +0 -269
  287. package/packages/notification/test/detail-format.test.ts +0 -106
  288. package/packages/notification/test/notification.test.ts +0 -375
  289. package/packages/notification/test/pending-question-lifecycle.test.ts +0 -195
  290. package/packages/notification/test/pending-question.test.ts +0 -331
  291. package/packages/notification/test/tooling-proposal-payload.test.ts +0 -212
  292. package/packages/observability/test/observability.test.ts +0 -1350
  293. package/packages/performance-optimizer/test/optimizer.test.ts +0 -267
  294. package/packages/project-analyzer/test/index.test.ts +0 -1719
  295. package/packages/prompt-compiler/test/prompt-compiler.test.ts +0 -813
  296. package/packages/prompt-versioning/test/versioning.test.ts +0 -112
  297. package/packages/provider-adapter-sdk/test/sdk.test.ts +0 -220
  298. package/packages/provider-router/test/provider-router.test.ts +0 -405
  299. package/packages/provider-selector/test/selector.test.ts +0 -192
  300. package/packages/providers/test/providers.test.ts +0 -225
  301. package/packages/qdrant-skills/test/collection-info.test.ts +0 -165
  302. package/packages/qdrant-skills/test/embedder-batch.test.ts +0 -184
  303. package/packages/qdrant-skills/test/embedder.test.ts +0 -97
  304. package/packages/quota-manager/test/quota.test.ts +0 -432
  305. package/packages/rate-limiter/test/limiter.test.ts +0 -128
  306. package/packages/react-vite-plugin/test/analyzer.test.ts +0 -139
  307. package/packages/relay/test/client.test.ts +0 -446
  308. package/packages/relay/test/relay-e2e.test.ts +0 -299
  309. package/packages/relay/test/relay-endpoint.test.ts +0 -278
  310. package/packages/release-manager/test/release.test.ts +0 -215
  311. package/packages/requirement-compiler/test/requirement-compiler.test.ts +0 -743
  312. package/packages/runtime/test/runtime-api.test.ts +0 -169
  313. package/packages/runtime/test/runtime.test.ts +0 -297
  314. package/packages/scheduler/test/scheduler.test.ts +0 -125
  315. package/packages/session/test/manager.test.ts +0 -233
  316. package/packages/shared-context/test/shared-context.test.ts +0 -114
  317. package/packages/skill-registry/test/registry.test.ts +0 -508
  318. package/packages/skills/test/init-skills.test.ts +0 -114
  319. package/packages/skills/test/remote-skills.test.ts +0 -360
  320. package/packages/skills/test/remote-source.test.ts +0 -248
  321. package/packages/skills/test/sanitize.test.ts +0 -250
  322. package/packages/skills/test/vector-indexing.test.ts +0 -205
  323. package/packages/skills/test/vector-schema.test.ts +0 -180
  324. package/packages/skills/test/vector-threshold.test.ts +0 -108
  325. package/packages/sprint-planner/test/planner.test.ts +0 -187
  326. package/packages/task-compiler/test/task-compiler.test.ts +0 -628
  327. package/packages/test-data-generator/test/generator.test.ts +0 -265
  328. package/packages/test-generator/test/generator.test.ts +0 -156
  329. package/packages/token-estimation/test/token-estimation.test.ts +0 -207
  330. package/packages/token-optimizer/test/optimizer.test.ts +0 -141
  331. package/packages/tooling-preference/tests/changed-files.test.ts +0 -124
  332. package/packages/tooling-preference/tests/detect.test.ts +0 -188
  333. package/packages/tooling-preference/tests/proposal.test.ts +0 -107
  334. package/packages/tooling-preference/tests/quality-check.test.ts +0 -272
  335. package/packages/tooling-preference/tests/registry.test.ts +0 -51
  336. package/packages/tooling-preference/tests/store.test.ts +0 -99
  337. package/packages/trajectory/test/store.test.ts +0 -250
  338. package/packages/tui/test/harness-footer.test.ts +0 -559
  339. package/packages/tui/test/harness-header.test.ts +0 -590
  340. package/packages/two-phase-review/test/classify.test.ts +0 -102
  341. package/packages/two-phase-review/test/helpers.ts +0 -72
  342. package/packages/two-phase-review/test/rules.test.ts +0 -150
  343. package/packages/two-phase-review/test/runtime.test.ts +0 -362
  344. package/packages/two-phase-review/test/scope.test.ts +0 -186
  345. package/packages/two-phase-review/test/signals.test.ts +0 -183
  346. package/packages/two-phase-review/test/state.test.ts +0 -201
  347. package/packages/wiki/test/basic.test.ts +0 -242
  348. package/packages/workspace-scanner/test/scanner.test.ts +0 -43
  349. package/packages/worktree/test/worktree.test.ts +0 -163
@@ -1,277 +0,0 @@
1
- /**
2
- * Evaluation Engine Tests (RFC-0057)
3
- */
4
-
5
- import { describe, it, expect, beforeEach } from "@jest/globals";
6
- import { type EvaluationEngine, createEvaluationEngine } from "../src/index.js";
7
- import type { EvaluationRequest } from "../src/types.js";
8
-
9
- describe("EvaluationEngine", () => {
10
- let engine: EvaluationEngine;
11
- let defaultTask: EvaluationRequest["task"];
12
- let defaultOutputs: EvaluationRequest["outputs"];
13
- let defaultTestResults: EvaluationRequest["testResults"];
14
- let defaultPolicyFindings: EvaluationRequest["policyFindings"];
15
-
16
- beforeEach(() => {
17
- engine = createEvaluationEngine();
18
- defaultTask = {
19
- id: "task-1",
20
- title: "Test Task",
21
- description: "A test task",
22
- acceptanceCriteria: ["Output file exists", "Tests pass"],
23
- requiredArtifacts: ["output.txt"],
24
- mandatoryStages: [],
25
- };
26
- defaultOutputs = [
27
- {
28
- id: "out-1",
29
- type: "file",
30
- path: "output.txt",
31
- description: "Generated output file",
32
- },
33
- ];
34
- defaultTestResults = [
35
- {
36
- id: "test-1",
37
- testFile: "test.spec.ts",
38
- status: "passed",
39
- durationMs: 100,
40
- },
41
- ];
42
- defaultPolicyFindings = [];
43
- });
44
-
45
- describe("evaluate", () => {
46
- it("passes when all criteria are met", () => {
47
- const request: EvaluationRequest = {
48
- jobId: "job-1",
49
- task: defaultTask,
50
- worktreePath: "/tmp/worktree",
51
- outputs: defaultOutputs,
52
- testResults: defaultTestResults,
53
- policyFindings: defaultPolicyFindings,
54
- };
55
-
56
- const result = engine.evaluate(request);
57
-
58
- expect(result.status).toBe("passed");
59
- expect(result.score).toBeGreaterThanOrEqual(70);
60
- expect(result.recommendedAction).toBe("continue");
61
- expect(result.evidence.length).toBeGreaterThan(0);
62
- });
63
-
64
- it("fails when required artifact is missing", () => {
65
- const request: EvaluationRequest = {
66
- jobId: "job-1",
67
- task: defaultTask,
68
- worktreePath: "/tmp/worktree",
69
- outputs: [], // No outputs
70
- testResults: defaultTestResults,
71
- policyFindings: defaultPolicyFindings,
72
- };
73
-
74
- const result = engine.evaluate(request);
75
-
76
- expect(result.status).toBe("failed");
77
- expect(result.findings.some((f) => f.message.includes("missing"))).toBe(
78
- true,
79
- );
80
- });
81
-
82
- it("fails on critical policy violation", () => {
83
- const request: EvaluationRequest = {
84
- jobId: "job-1",
85
- task: defaultTask,
86
- worktreePath: "/tmp/worktree",
87
- outputs: defaultOutputs,
88
- testResults: defaultTestResults,
89
- policyFindings: [
90
- {
91
- ruleId: "SEC-001",
92
- severity: "critical",
93
- message: "Hardcoded API key detected",
94
- filePath: "src/config.ts",
95
- line: 10,
96
- },
97
- ],
98
- };
99
-
100
- const result = engine.evaluate(request);
101
-
102
- expect(result.status).toBe("failed");
103
- expect(result.findings.some((f) => f.severity === "critical")).toBe(true);
104
- expect(result.recommendedAction).toBe("repair");
105
- });
106
-
107
- it("fails when tests fail", () => {
108
- const request: EvaluationRequest = {
109
- jobId: "job-1",
110
- task: defaultTask,
111
- worktreePath: "/tmp/worktree",
112
- outputs: defaultOutputs,
113
- testResults: [
114
- {
115
- id: "test-1",
116
- testFile: "test.spec.ts",
117
- status: "failed",
118
- errorMessage: "Assertion failed: expected 2 to equal 3",
119
- },
120
- ],
121
- policyFindings: defaultPolicyFindings,
122
- };
123
-
124
- const result = engine.evaluate(request);
125
-
126
- expect(result.status).toBe("failed");
127
- expect(result.findings.some((f) => f.message.includes("failed"))).toBe(
128
- true,
129
- );
130
- });
131
-
132
- it("returns needs_review for ambiguous criteria", () => {
133
- const request: EvaluationRequest = {
134
- jobId: "job-1",
135
- task: {
136
- ...defaultTask,
137
- acceptanceCriteria: ["Do something good", "Make it work well"],
138
- },
139
- worktreePath: "/tmp/worktree",
140
- outputs: [
141
- {
142
- id: "out-1",
143
- type: "file",
144
- path: "output.txt",
145
- description: "Some output",
146
- },
147
- ],
148
- testResults: [
149
- {
150
- id: "test-1",
151
- testFile: "test.spec.ts",
152
- status: "passed",
153
- },
154
- ],
155
- policyFindings: defaultPolicyFindings,
156
- };
157
-
158
- const result = engine.evaluate(request);
159
-
160
- // Should pass or needs_review depending on coverage
161
- expect(["passed", "needs_review"]).toContain(result.status);
162
- });
163
-
164
- it("includes evidence in result", () => {
165
- const request: EvaluationRequest = {
166
- jobId: "job-1",
167
- task: defaultTask,
168
- worktreePath: "/tmp/worktree",
169
- outputs: defaultOutputs,
170
- testResults: defaultTestResults,
171
- policyFindings: defaultPolicyFindings,
172
- };
173
-
174
- const result = engine.evaluate(request);
175
-
176
- expect(result.evidence).toBeDefined();
177
- expect(Array.isArray(result.evidence)).toBe(true);
178
- });
179
-
180
- it("maps acceptance criteria to evidence", () => {
181
- const request: EvaluationRequest = {
182
- jobId: "job-1",
183
- task: {
184
- ...defaultTask,
185
- acceptanceCriteria: ["Output file exists"],
186
- },
187
- worktreePath: "/tmp/worktree",
188
- outputs: defaultOutputs,
189
- testResults: defaultTestResults,
190
- policyFindings: defaultPolicyFindings,
191
- };
192
-
193
- const result = engine.evaluate(request);
194
-
195
- const acceptanceFinding = result.findings.find(
196
- (f) => f.dimension === "acceptance_coverage",
197
- );
198
-
199
- // Should either pass or have unmet criteria finding
200
- expect(result.status).toBe("passed");
201
- });
202
- });
203
-
204
- describe("state management", () => {
205
- it("starts in pending state", () => {
206
- const freshEngine = createEvaluationEngine();
207
- expect(freshEngine.getState()).toBe("pending");
208
- });
209
-
210
- it("transitions through states during evaluation", () => {
211
- const request: EvaluationRequest = {
212
- jobId: "job-1",
213
- task: defaultTask,
214
- worktreePath: "/tmp/worktree",
215
- outputs: defaultOutputs,
216
- testResults: defaultTestResults,
217
- policyFindings: defaultPolicyFindings,
218
- };
219
-
220
- engine.evaluate(request);
221
-
222
- expect(["passed", "failed", "needs_review"]).toContain(engine.getState());
223
- });
224
-
225
- it("resets state correctly", () => {
226
- const request: EvaluationRequest = {
227
- jobId: "job-1",
228
- task: defaultTask,
229
- worktreePath: "/tmp/worktree",
230
- outputs: defaultOutputs,
231
- testResults: defaultTestResults,
232
- policyFindings: defaultPolicyFindings,
233
- };
234
-
235
- engine.evaluate(request);
236
- engine.reset();
237
-
238
- expect(engine.getState()).toBe("pending");
239
- });
240
- });
241
-
242
- describe("scoring", () => {
243
- it("returns score between 0 and 100", () => {
244
- const request: EvaluationRequest = {
245
- jobId: "job-1",
246
- task: defaultTask,
247
- worktreePath: "/tmp/worktree",
248
- outputs: defaultOutputs,
249
- testResults: defaultTestResults,
250
- policyFindings: defaultPolicyFindings,
251
- };
252
-
253
- const result = engine.evaluate(request);
254
-
255
- expect(result.score).toBeGreaterThanOrEqual(0);
256
- expect(result.score).toBeLessThanOrEqual(100);
257
- });
258
-
259
- it("penalizes for missing tests", () => {
260
- const request: EvaluationRequest = {
261
- jobId: "job-1",
262
- task: defaultTask,
263
- worktreePath: "/tmp/worktree",
264
- outputs: defaultOutputs,
265
- testResults: [], // No tests
266
- policyFindings: defaultPolicyFindings,
267
- };
268
-
269
- const result = engine.evaluate(request);
270
-
271
- // Missing tests should lower score
272
- expect(
273
- result.findings.some((f) => f.message.includes("No test results")),
274
- ).toBe(true);
275
- });
276
- });
277
- });
@@ -1,168 +0,0 @@
1
- /**
2
- * Evaluation Runner Tests (RFC-0020)
3
- */
4
-
5
- import { describe, it, expect, beforeEach } from "bun:test";
6
- import {
7
- EvaluationRunner,
8
- exactMatchEvaluator,
9
- substringMatchEvaluator,
10
- lengthEvaluator,
11
- } from "../src/index.js";
12
- import type { EvaluationSuite } from "../src/index.js";
13
-
14
- describe("EvaluationRunner", () => {
15
- let runner: EvaluationRunner;
16
-
17
- beforeEach(() => { runner = new EvaluationRunner(); });
18
-
19
- it("registers and lists evaluators", () => {
20
- runner.registerEvaluator("custom", async () => ({ runId: "r1", metric: "custom", score: 0.5, passed: true }));
21
- expect(runner.listEvaluators()).toContain("custom");
22
- });
23
-
24
- it("runs suite and returns report", async () => {
25
- runner.registerEvaluator("exact", exactMatchEvaluator);
26
- const suite: EvaluationSuite = {
27
- id: "s1",
28
- name: "Basic",
29
- cases: [{
30
- id: "c1",
31
- name: "exact match",
32
- input: { prompt: "hello", expectedOutput: "world" },
33
- evaluators: ["exact"],
34
- }],
35
- thresholds: { "exact-match": 1 },
36
- };
37
- const report = await runner.runSuite(suite, async () => "world");
38
- expect(report.total).toBe(1);
39
- expect(report.passed).toBe(1);
40
- expect(report.results[0].score).toBe(1);
41
- });
42
-
43
- it("marks failed when output differs", async () => {
44
- runner.registerEvaluator("exact", exactMatchEvaluator);
45
- const suite: EvaluationSuite = {
46
- id: "s2",
47
- name: "Fail",
48
- cases: [{
49
- id: "c1",
50
- name: "differs",
51
- input: { prompt: "hello", expectedOutput: "world" },
52
- evaluators: ["exact"],
53
- }],
54
- thresholds: {},
55
- };
56
- const report = await runner.runSuite(suite, async () => "other");
57
- expect(report.failed).toBe(1);
58
- expect(report.passed).toBe(0);
59
- });
60
-
61
- it("handles timeout", async () => {
62
- runner = new EvaluationRunner({ timeoutMs: 10 });
63
- runner.registerEvaluator("exact", exactMatchEvaluator);
64
- const suite: EvaluationSuite = {
65
- id: "s3",
66
- name: "Timeout",
67
- cases: [{
68
- id: "c1",
69
- name: "slow",
70
- input: { prompt: "hello", expectedOutput: "world" },
71
- evaluators: ["exact"],
72
- }],
73
- thresholds: {},
74
- };
75
- const report = await runner.runSuite(suite, async () => new Promise((r) => setTimeout(() => r("world"), 200)));
76
- expect(report.failed).toBe(1);
77
- expect(report.results[0].details).toContain("timeout");
78
- });
79
-
80
- it("registers unknown evaluator gracefully", async () => {
81
- const suite: EvaluationSuite = {
82
- id: "s4",
83
- name: "Unknown",
84
- cases: [{
85
- id: "c1",
86
- name: "unknown",
87
- input: { prompt: "hello" },
88
- evaluators: ["not-registered"],
89
- }],
90
- thresholds: {},
91
- };
92
- const report = await runner.runSuite(suite, async () => "output");
93
- expect(report.total).toBe(1);
94
- expect(report.results[0].details).toContain("not registered");
95
- });
96
-
97
- it("calculates summary averages", async () => {
98
- runner.registerEvaluator("exact", exactMatchEvaluator);
99
- const suite: EvaluationSuite = {
100
- id: "s5",
101
- name: "Multi",
102
- cases: [
103
- { id: "c1", name: "a", input: { prompt: "a", expectedOutput: "x" }, evaluators: ["exact"] },
104
- { id: "c2", name: "b", input: { prompt: "b", expectedOutput: "y" }, evaluators: ["exact"] },
105
- ],
106
- thresholds: {},
107
- };
108
- const report = await runner.runSuite(suite, async () => "x");
109
- expect(report.summary["exact-match"]?.count).toBe(2);
110
- });
111
- });
112
-
113
- describe("exactMatchEvaluator", () => {
114
- it("scores 1 for exact match", async () => {
115
- const result = await exactMatchEvaluator(
116
- { prompt: "hello", expectedOutput: "world" },
117
- "world",
118
- );
119
- expect(result.score).toBe(1);
120
- expect(result.passed).toBe(true);
121
- });
122
-
123
- it("scores 0 for mismatch", async () => {
124
- const result = await exactMatchEvaluator(
125
- { prompt: "hello", expectedOutput: "world" },
126
- "other",
127
- );
128
- expect(result.score).toBe(0);
129
- expect(result.passed).toBe(false);
130
- });
131
- });
132
-
133
- describe("substringMatchEvaluator", () => {
134
- it("passes when expected is substring", async () => {
135
- const result = await substringMatchEvaluator(
136
- { prompt: "hello", expectedOutput: "world peace" },
137
- "hello world peace today",
138
- );
139
- expect(result.passed).toBe(true);
140
- expect(result.score).toBe(1);
141
- });
142
-
143
- it("fails when expected not found", async () => {
144
- const result = await substringMatchEvaluator(
145
- { prompt: "hello", expectedOutput: "xyz" },
146
- "abc def",
147
- );
148
- expect(result.passed).toBe(false);
149
- });
150
- });
151
-
152
- describe("lengthEvaluator", () => {
153
- it("passes when within range", async () => {
154
- const result = await lengthEvaluator(
155
- { prompt: "hello", metadata: { minLength: 5, maxLength: 20 } },
156
- "hello world",
157
- );
158
- expect(result.passed).toBe(true);
159
- });
160
-
161
- it("fails when too long", async () => {
162
- const result = await lengthEvaluator(
163
- { prompt: "hello", metadata: { minLength: 5, maxLength: 10 } },
164
- "this is a very long response",
165
- );
166
- expect(result.passed).toBe(false);
167
- });
168
- });
@@ -1,100 +0,0 @@
1
- /// <reference types="node" />
2
- import { describe, it } from "node:test";
3
- import { equal, deepEqual } from "node:assert";
4
- import { EventBus } from "../src/bus.js";
5
- import type { EventPayload } from "../src/types.js";
6
-
7
- describe("EventBus", () => {
8
- const bus = new EventBus();
9
-
10
- it("publish returns eventId", () => {
11
- const id = bus.publish("test.topic", { msg: "hello" });
12
- equal(typeof id, "string");
13
- equal(id.length > 0, true);
14
- });
15
-
16
- it("subscriber receives event", () => {
17
- // The value is held in a box: a variable that is only assigned in a
18
- // callback would be narrowed to null by the type checker.
19
- const received: { value: EventPayload<{ data: string }> | null } = {
20
- value: null,
21
- };
22
- bus.subscribe<{ data: string }>("test.1", (p) => {
23
- received.value = p;
24
- });
25
- bus.publish("test.1", { data: "hello" });
26
- // The subscriber receives the envelope of the event, not the raw data.
27
- equal(received.value?.topic, "test.1");
28
- equal(received.value?.data.data, "hello");
29
- equal(typeof received.value?.eventId, "string");
30
- equal(typeof received.value?.timestamp, "string");
31
- // A published event comes from the system unless a source is given.
32
- equal(received.value?.source, "system");
33
- });
34
-
35
- it("multiple subscribers all called", () => {
36
- const calls: number[] = [];
37
- bus.subscribe("test.2", () => {
38
- calls.push(1);
39
- });
40
- bus.subscribe("test.2", () => {
41
- calls.push(2);
42
- });
43
- bus.publish("test.2", { x: 1 });
44
- deepEqual(calls, [1, 2]);
45
- });
46
-
47
- it("unsubscribe stops notifications", () => {
48
- let called = false;
49
- const id = bus.subscribe("test.3", () => {
50
- called = true;
51
- });
52
- bus.unsubscribe(id);
53
- bus.publish("test.3", {});
54
- equal(called, false);
55
- });
56
-
57
- it("filter predicates work", () => {
58
- let called = false;
59
- bus.subscribe(
60
- "test.4",
61
- () => {
62
- called = true;
63
- },
64
- (d) => (d as { ok: boolean }).ok === true,
65
- );
66
- bus.publish("test.4", { ok: false });
67
- equal(called, false);
68
- bus.publish("test.4", { ok: true });
69
- equal(called, true);
70
- });
71
-
72
- it("priority ordering respected", () => {
73
- const order: number[] = [];
74
- bus.subscribe(
75
- "test.5",
76
- () => {
77
- order.push(1);
78
- },
79
- undefined,
80
- 1,
81
- );
82
- bus.subscribe(
83
- "test.5",
84
- () => {
85
- order.push(2);
86
- },
87
- undefined,
88
- 2,
89
- );
90
- bus.publish("test.5", {});
91
- deepEqual(order, [2, 1]); // higher priority first
92
- });
93
-
94
- it("unsubscribeAll(topic) removes all for topic", () => {
95
- bus.subscribe("test.6", () => {});
96
- bus.subscribe("test.6", () => {});
97
- bus.unsubscribeAll("test.6");
98
- equal(bus.getSubscribers("test.6").length, 0);
99
- });
100
- });
@@ -1,115 +0,0 @@
1
- import { describe, it, beforeEach, afterEach } from "node:test";
2
- import { equal } from "node:assert";
3
- import { join } from "node:path";
4
- import { mkdirSync, rmSync } from "node:fs";
5
- import { tmpdir } from "node:os";
6
- import { EventStore } from "../src/store.js";
7
-
8
- describe("EventStore", () => {
9
- const dir = join(tmpdir(), "event-store-test-" + Date.now());
10
- beforeEach(() => mkdirSync(dir, { recursive: true }));
11
- afterEach(() => rmSync(dir, { recursive: true, force: true }));
12
-
13
- const store = new EventStore({ sessionsDir: dir });
14
-
15
- it("append generates id and timestamp", async () => {
16
- const result = await store.append({
17
- sessionId: "test-1",
18
- type: "message",
19
- role: "user",
20
- content: "hello",
21
- });
22
- equal(typeof result.id, "string");
23
- equal(result.id.length > 0, true);
24
- equal(typeof result.timestamp, "string");
25
- });
26
-
27
- it("read returns appended events", async () => {
28
- await store.append({
29
- sessionId: "s1",
30
- type: "message",
31
- role: "user",
32
- content: "hello",
33
- });
34
- await store.append({
35
- sessionId: "s1",
36
- type: "message",
37
- role: "assistant",
38
- content: "hi",
39
- });
40
- const events = await store.read({ sessionId: "s1" });
41
- equal(events.length, 2);
42
- equal(events[0].content, "hello");
43
- });
44
-
45
- it("read filters by type", async () => {
46
- await store.append({
47
- sessionId: "s2",
48
- type: "message",
49
- role: "user",
50
- content: "a",
51
- });
52
- await store.append({
53
- sessionId: "s2",
54
- type: "tool_start",
55
- tool: "bash",
56
- content: "",
57
- });
58
- const events = await store.read({ sessionId: "s2", types: ["message"] });
59
- equal(events.length, 1);
60
- });
61
-
62
- it("search finds matching content", async () => {
63
- await store.append({
64
- sessionId: "s3",
65
- type: "message",
66
- role: "user",
67
- content: "fix the login bug",
68
- });
69
- await store.append({
70
- sessionId: "s3",
71
- type: "message",
72
- role: "user",
73
- content: "add dark mode",
74
- });
75
- const events = await store.search({ query: "login" });
76
- equal(events.length >= 1, true);
77
- equal(events[0].content?.includes("login"), true);
78
- });
79
-
80
- it("stats returns counts", async () => {
81
- await store.append({
82
- sessionId: "s4",
83
- type: "message",
84
- role: "user",
85
- content: "a",
86
- });
87
- await store.append({
88
- sessionId: "s4",
89
- type: "message",
90
- role: "user",
91
- content: "b",
92
- });
93
- const st = store.stats("s4");
94
- equal(st.totalEvents, 2);
95
- equal(st.sizeBytes > 0, true);
96
- });
97
-
98
- it("list returns session IDs", async () => {
99
- await store.append({
100
- sessionId: "alpha",
101
- type: "message",
102
- role: "user",
103
- content: "x",
104
- });
105
- await store.append({
106
- sessionId: "beta",
107
- type: "message",
108
- role: "user",
109
- content: "y",
110
- });
111
- const sessions = store.list();
112
- equal(sessions.includes("alpha"), true);
113
- equal(sessions.includes("beta"), true);
114
- });
115
- });