reeval 0.14.0 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (364) hide show
  1. package/README.md +15 -15
  2. package/THIRD_PARTY_NOTICES.md +69 -69
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-BloGVZ_a.js → adapter-CRKYPkRt.js} +37 -8
  5. package/dist/adapter-CRKYPkRt.js.map +1 -0
  6. package/dist/{agent-m-_t-IuW.js → agent-BgH4-kFc.js} +58 -16
  7. package/dist/agent-BgH4-kFc.js.map +1 -0
  8. package/dist/{app-files-C-jwhRDt.js → app-files-BMVBr079.js} +4 -4
  9. package/dist/app-files-BMVBr079.js.map +1 -0
  10. package/dist/{assemble-BDEpelpi.js → assemble-BVbI6NW-.js} +214 -128
  11. package/dist/assemble-BVbI6NW-.js.map +1 -0
  12. package/dist/assemble-SiHa4__F.js +1 -0
  13. package/dist/{attempt-jI0OWxlz.js → attempt-CpbZ9arV.js} +10 -10
  14. package/dist/{attempt-jI0OWxlz.js.map → attempt-CpbZ9arV.js.map} +1 -1
  15. package/dist/{match-Bz0nSdzN.js → attempt-host-CIxjs2Be.js} +104 -6
  16. package/dist/attempt-host-CIxjs2Be.js.map +1 -0
  17. package/dist/{capture-CVRwRTnb.js → capture-iQ6kYJV0.js} +2360 -1523
  18. package/dist/capture-iQ6kYJV0.js.map +1 -0
  19. package/dist/capture-px7Ey95_.js +1 -0
  20. package/dist/{cli-C_4gaJ50.js → cli-1YQU-66F.js} +153 -180
  21. package/dist/cli-1YQU-66F.js.map +1 -0
  22. package/dist/{cli-spec-CpnbATtI.js → cli-spec-CPWT1mAG.js} +10 -21
  23. package/dist/cli-spec-CPWT1mAG.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-D5hNjpEx.js → client-DMmOkbRA.js} +14 -7
  26. package/dist/client-DMmOkbRA.js.map +1 -0
  27. package/dist/{command-BXLUfw4h.js → command-8_Cg5v6b.js} +35 -22
  28. package/dist/command-8_Cg5v6b.js.map +1 -0
  29. package/dist/{command-I1euLLio.js → command-BlYCAqu-.js} +7 -7
  30. package/dist/{command-I1euLLio.js.map → command-BlYCAqu-.js.map} +1 -1
  31. package/dist/{validate-BhwqrVsm.js → command-C-HeGlap.js} +388 -60
  32. package/dist/command-C-HeGlap.js.map +1 -0
  33. package/dist/{command-Bxdy62dW.js → command-Cr1zjRvl.js} +32 -31
  34. package/dist/command-Cr1zjRvl.js.map +1 -0
  35. package/dist/{commands-p1S4nw_H.js → commands-Bj9NbmsO.js} +1 -1
  36. package/dist/{commands-jzigbNFb.js → commands-BvTNZ2-b.js} +181 -73
  37. package/dist/commands-BvTNZ2-b.js.map +1 -0
  38. package/dist/{config-BrdHA9bJ.js → config-CneBZB84.js} +883 -433
  39. package/dist/config-CneBZB84.js.map +1 -0
  40. package/dist/{contrasts-DRxMQQTC.js → contrasts-lhFsekco.js} +3 -3
  41. package/dist/{contrasts-DRxMQQTC.js.map → contrasts-lhFsekco.js.map} +1 -1
  42. package/dist/cost-C5RBTTJr.js +1 -0
  43. package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
  44. package/dist/cost-ChPa8-Fp.js.map +1 -0
  45. package/dist/db-BnXyamuj.js +2 -0
  46. package/dist/db-Cnh3jIv_.js +1530 -0
  47. package/dist/db-Cnh3jIv_.js.map +1 -0
  48. package/dist/debug-CAx3qcaP.js +1 -0
  49. package/dist/{debug-D24NOQH1.js → debug-CgG3T071.js} +5 -2
  50. package/dist/debug-CgG3T071.js.map +1 -0
  51. package/dist/{docker-BnRlTpZA.js → docker-BihzJwbV.js} +11 -6
  52. package/dist/docker-BihzJwbV.js.map +1 -0
  53. package/dist/{docker-D01slDF3.js → docker-Cqq03H72.js} +26 -8
  54. package/dist/docker-Cqq03H72.js.map +1 -0
  55. package/dist/{docker-DivfZl2u.js → docker-DCAtFMo9.js} +1 -1
  56. package/dist/{doctor-DQ2p0euG.js → doctor-Dl4vMeqt.js} +275 -37
  57. package/dist/doctor-Dl4vMeqt.js.map +1 -0
  58. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  59. package/dist/env-CARj5Axq.js.map +1 -0
  60. package/dist/{environment-CO0P0soF.js → environment-Cf5HMjib.js} +116 -297
  61. package/dist/environment-Cf5HMjib.js.map +1 -0
  62. package/dist/{errors-r025B0Zb.js → errors-CHebohjc.js} +10 -4
  63. package/dist/{errors-r025B0Zb.js.map → errors-CHebohjc.js.map} +1 -1
  64. package/dist/eval-source-CAE4y2Sr.js +146 -0
  65. package/dist/eval-source-CAE4y2Sr.js.map +1 -0
  66. package/dist/{evaluate-BUmsN7yG.js → evaluate-CUtgyr43.js} +682 -150
  67. package/dist/evaluate-CUtgyr43.js.map +1 -0
  68. package/dist/evaluate-D35T51AC.d.ts +45 -0
  69. package/dist/files-BEnUJW1e.js +1 -0
  70. package/dist/{files-B-CJeeEh.js → files-BRViE7m-.js} +200 -87
  71. package/dist/files-BRViE7m-.js.map +1 -0
  72. package/dist/{files-kUCrswI6.js → files-BSqA2sZE.js} +20 -9
  73. package/dist/files-BSqA2sZE.js.map +1 -0
  74. package/dist/files-C3tQtD26.js +1 -0
  75. package/dist/files-CNiJ_fZf.js +37 -0
  76. package/dist/files-CNiJ_fZf.js.map +1 -0
  77. package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
  78. package/dist/files-D1y8Pg4Z.js.map +1 -0
  79. package/dist/files-Dts7EY3H.js +1 -0
  80. package/dist/{pairs-CHfZNRT6.js → gate-BC6SdQZs.js} +58 -4
  81. package/dist/gate-BC6SdQZs.js.map +1 -0
  82. package/dist/gateway.d.ts +22 -9
  83. package/dist/gateway.js +30 -8
  84. package/dist/gateway.js.map +1 -1
  85. package/dist/{git-Bk8DSwHy.js → git-UNxT_3z0.js} +3 -3
  86. package/dist/{git-Bk8DSwHy.js.map → git-UNxT_3z0.js.map} +1 -1
  87. package/dist/harness-3NVyTSGK.js +392 -0
  88. package/dist/harness-3NVyTSGK.js.map +1 -0
  89. package/dist/harness.d.ts +12 -8
  90. package/dist/harness.js +1 -1
  91. package/dist/{images-DzLeFjLx.js → images-8A_LzVfw.js} +8 -8
  92. package/dist/{images-DzLeFjLx.js.map → images-8A_LzVfw.js.map} +1 -1
  93. package/dist/index.d.ts +99 -11
  94. package/dist/index.js +5 -5
  95. package/dist/init-okoALIah.js +1 -0
  96. package/dist/{init-SRalM5xF.js → init-sFBgODhu.js} +10 -11
  97. package/dist/init-sFBgODhu.js.map +1 -0
  98. package/dist/{invocations-DzyKSJBa.js → invocations-C9ejlOYl.js} +10 -6
  99. package/dist/invocations-C9ejlOYl.js.map +1 -0
  100. package/dist/{judge-yLvjC5A3.js → judge-Bg4v4e_K.js} +236 -46
  101. package/dist/judge-Bg4v4e_K.js.map +1 -0
  102. package/dist/judge.d.ts +5 -7
  103. package/dist/judge.js +4 -8
  104. package/dist/judge.js.map +1 -1
  105. package/dist/{live-DYd2QEyR.js → live-DeMvnDD5.js} +3 -3
  106. package/dist/{live-DYd2QEyR.js.map → live-DeMvnDD5.js.map} +1 -1
  107. package/dist/{matchers-D4eUqx4j.js → matchers-BKPuaXlj.js} +14 -13
  108. package/dist/matchers-BKPuaXlj.js.map +1 -0
  109. package/dist/{model-1ZLr9Dcv.js → model-2rBZrdZA.js} +29 -36
  110. package/dist/model-2rBZrdZA.js.map +1 -0
  111. package/dist/{model-B45lDUV1.js → model-3MJtpho8.js} +22 -13
  112. package/dist/model-3MJtpho8.js.map +1 -0
  113. package/dist/{model-CgnOpss4.js → model-Bz7-j71_.js} +245 -68
  114. package/dist/model-Bz7-j71_.js.map +1 -0
  115. package/dist/model-DkPwypN4.js +1 -0
  116. package/dist/model-bREXVXwT.js +1 -0
  117. package/dist/{node-BSq5qHnQ.js → node-fP-H2uxn.js} +72 -50
  118. package/dist/node-fP-H2uxn.js.map +1 -0
  119. package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
  120. package/dist/object-BkXGPKWY.js.map +1 -0
  121. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  122. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  123. package/dist/openrouter-models-Bo8PZ4Ew.js +34 -0
  124. package/dist/openrouter-models-Bo8PZ4Ew.js.map +1 -0
  125. package/dist/{patch-DWf-I_Q_.js → patch-6T9XAipm.js} +7 -9
  126. package/dist/patch-6T9XAipm.js.map +1 -0
  127. package/dist/patch-CquGAGX5.js +76 -0
  128. package/dist/patch-CquGAGX5.js.map +1 -0
  129. package/dist/{plan-BIwUzlB8.js → plan-Dw2BjafJ.js} +79 -35
  130. package/dist/plan-Dw2BjafJ.js.map +1 -0
  131. package/dist/{plugins-Dkp_H3pB.js → plugins-ngQEJBjJ.js} +274 -17
  132. package/dist/plugins-ngQEJBjJ.js.map +1 -0
  133. package/dist/{preflight-D9y-HvZQ.js → preflight-BYXkweNN.js} +1 -1
  134. package/dist/{preflight-DJCD_BDn.js → preflight-CDRpog9o.js} +3 -3
  135. package/dist/{preflight-DJCD_BDn.js.map → preflight-CDRpog9o.js.map} +1 -1
  136. package/dist/{prepare-BJG_pD54.js → prepare-B3btUlT6.js} +2 -2
  137. package/dist/{prepare-CIxCGG0M.js → prepare-PkARUGoF.js} +8 -8
  138. package/dist/{prepare-CIxCGG0M.js.map → prepare-PkARUGoF.js.map} +1 -1
  139. package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
  140. package/dist/prices-QaS-kjSQ.js.map +1 -0
  141. package/dist/{provenance-CfZpFpAh.js → provenance-7Hzg88f4.js} +4 -4
  142. package/dist/{provenance-CfZpFpAh.js.map → provenance-7Hzg88f4.js.map} +1 -1
  143. package/dist/{providers-D1Zxp2he.js → providers-DDNMbv1T.js} +1 -1
  144. package/dist/{providers-DFk_6W4K.js → providers-PKoJtGK0.js} +5 -5
  145. package/dist/{providers-DFk_6W4K.js.map → providers-PKoJtGK0.js.map} +1 -1
  146. package/dist/query-Bt6vLJ3m.js +1 -0
  147. package/dist/{query-CemclveB.js → query-nBXrgEwl.js} +3 -3
  148. package/dist/{query-CemclveB.js.map → query-nBXrgEwl.js.map} +1 -1
  149. package/dist/{questions-bAudIc04.js → questions-CkAwePPR.js} +110 -36
  150. package/dist/questions-CkAwePPR.js.map +1 -0
  151. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  152. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  153. package/dist/redact-DVIaahLv.js +658 -0
  154. package/dist/redact-DVIaahLv.js.map +1 -0
  155. package/dist/remarks-Bp6AwPgb.js +1 -0
  156. package/dist/{rules-VhftPOOT.js → remarks-Bt0zwiQB.js} +330 -66
  157. package/dist/remarks-Bt0zwiQB.js.map +1 -0
  158. package/dist/{header-BLPP6aCU.js → remarks-IFFe_oty.js} +35 -9
  159. package/dist/remarks-IFFe_oty.js.map +1 -0
  160. package/dist/{remarks-mujJv1Ey.js → remarks-eBcrIZRu.js} +10 -5
  161. package/dist/remarks-eBcrIZRu.js.map +1 -0
  162. package/dist/{remote-CKcEKja4.js → remote-DfJYiP-R.js} +39 -99
  163. package/dist/remote-DfJYiP-R.js.map +1 -0
  164. package/dist/resolve-CT4XZIfV.js +1 -0
  165. package/dist/{resolve-CBoKzQMP.js → resolve-D_cLa4tW.js} +46 -30
  166. package/dist/resolve-D_cLa4tW.js.map +1 -0
  167. package/dist/results-DrKJuIQF.js +63 -0
  168. package/dist/results-DrKJuIQF.js.map +1 -0
  169. package/dist/{resume-B9SOSnK9.js → resume-BmIM52MJ.js} +47 -17
  170. package/dist/resume-BmIM52MJ.js.map +1 -0
  171. package/dist/{runtime-BgQRyCLl.js → runtime-CZVpkr04.js} +139 -48
  172. package/dist/runtime-CZVpkr04.js.map +1 -0
  173. package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmi6IQML.js} +2 -2
  174. package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmi6IQML.js.map} +1 -1
  175. package/dist/sandbox.d.ts +1 -1
  176. package/dist/sandbox.js +1 -1
  177. package/dist/server-CKJOnIB0.js +1 -0
  178. package/dist/{server-TGW4WyAz.js → server-DS8BGhGU.js} +75 -33
  179. package/dist/server-DS8BGhGU.js.map +1 -0
  180. package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
  181. package/dist/shared-B_lvGqWz.js.map +1 -0
  182. package/dist/source-CgUqpumI.js +1 -0
  183. package/dist/source-DcXv4de-.js +59 -0
  184. package/dist/source-DcXv4de-.js.map +1 -0
  185. package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
  186. package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
  187. package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
  188. package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
  189. package/dist/{src-DjRI-IeE.js → src-A8HLtO0a.js} +135 -35
  190. package/dist/{src-DjRI-IeE.js.map → src-A8HLtO0a.js.map} +1 -1
  191. package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
  192. package/dist/src-BO8AYqJ5.js.map +1 -0
  193. package/dist/src-BTqJbvj6.js +2 -0
  194. package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
  195. package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BkiComGh.js +2 -0
  197. package/dist/src-BvgeZ4tS.js +2 -0
  198. package/dist/{src-8zoX_qrb.js → src-C0ySiej5.js} +572 -1067
  199. package/dist/src-C0ySiej5.js.map +1 -0
  200. package/dist/src-CgJrz5p0.js +3 -0
  201. package/dist/src-Cp-2Hdib.js +2 -0
  202. package/dist/{src-CyaDNG9r.js → src-DMo2eCOJ.js} +1 -1
  203. package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
  204. package/dist/src-DSZyg0Uq.js.map +1 -0
  205. package/dist/src-Dk2Na0b-.js +2 -0
  206. package/dist/src-OESRtvOi.js +2 -0
  207. package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
  208. package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
  209. package/dist/src-h5uDlk3m.js +1 -0
  210. package/dist/state-C97udoh3.js +1 -0
  211. package/dist/{state-B1t143wG.js → state-C_kdhp5v.js} +15 -15
  212. package/dist/{state-B1t143wG.js.map → state-C_kdhp5v.js.map} +1 -1
  213. package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
  214. package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
  215. package/dist/suite-ByT1lsdD.js +2 -0
  216. package/dist/{suite-WgT7m6Vs.js → suite-DieLXvCT.js} +61 -65
  217. package/dist/suite-DieLXvCT.js.map +1 -0
  218. package/dist/{summary-DIGEVr08.js → summary-CSzyBVsF.js} +113 -277
  219. package/dist/summary-CSzyBVsF.js.map +1 -0
  220. package/dist/tarball-CHcahyNK.js +258 -0
  221. package/dist/tarball-CHcahyNK.js.map +1 -0
  222. package/dist/testing.d.ts +426 -143
  223. package/dist/testing.js +12 -10
  224. package/dist/testing.js.map +1 -1
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
  228. package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
  229. package/dist/{transcript-BTxJpW4R.js → transcript-BJrKfgRW.js} +13 -5
  230. package/dist/transcript-BJrKfgRW.js.map +1 -0
  231. package/dist/types-C5vI3g1C.js.map +1 -1
  232. package/dist/{types-DXBbP9Mk.d.ts → types-DVvW66ci.d.ts} +569 -895
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +548 -121
  238. package/dist/view/index.js +9 -9
  239. package/dist/{views-sXpGE6a2.js → views-B9STsLB4.js} +9 -10
  240. package/dist/{views-sXpGE6a2.js.map → views-B9STsLB4.js.map} +1 -1
  241. package/dist/views-Ca-rTixp.js +2 -0
  242. package/dist/{worker-entry-Duo9oAeT.js → worker-entry-C5bLCv6s.js} +48 -20
  243. package/dist/worker-entry-C5bLCv6s.js.map +1 -0
  244. package/dist/workspace-8uHtBSve.js +1 -0
  245. package/dist/{workspace-CDI68wtX.js → workspace-CBuI9Uds.js} +5 -5
  246. package/dist/{workspace-CDI68wtX.js.map → workspace-CBuI9Uds.js.map} +1 -1
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +5 -1
  249. package/skills/eval-rubric/SKILL.md +46 -104
  250. package/dist/adapter-BloGVZ_a.js.map +0 -1
  251. package/dist/agent-m-_t-IuW.js.map +0 -1
  252. package/dist/app-files-C-jwhRDt.js.map +0 -1
  253. package/dist/assemble-BDEpelpi.js.map +0 -1
  254. package/dist/assemble-C5bfncHU.js +0 -1
  255. package/dist/capture-CVRwRTnb.js.map +0 -1
  256. package/dist/capture-CherbjzB.js +0 -1
  257. package/dist/cli-C_4gaJ50.js.map +0 -1
  258. package/dist/cli-spec-CpnbATtI.js.map +0 -1
  259. package/dist/client-D5hNjpEx.js.map +0 -1
  260. package/dist/command-BWAuwQNk.js +0 -91
  261. package/dist/command-BWAuwQNk.js.map +0 -1
  262. package/dist/command-BXLUfw4h.js.map +0 -1
  263. package/dist/command-Be6BSE2L.js +0 -335
  264. package/dist/command-Be6BSE2L.js.map +0 -1
  265. package/dist/command-Bxdy62dW.js.map +0 -1
  266. package/dist/commands-jzigbNFb.js.map +0 -1
  267. package/dist/config-BrdHA9bJ.js.map +0 -1
  268. package/dist/cost-D4elx933.js.map +0 -1
  269. package/dist/cost-DmnAod9x.js +0 -1
  270. package/dist/db-CAfksinX.js +0 -2
  271. package/dist/db-DO0p7xgP.js +0 -619
  272. package/dist/db-DO0p7xgP.js.map +0 -1
  273. package/dist/debug-D24NOQH1.js.map +0 -1
  274. package/dist/debug-D7bEv_UV.js +0 -1
  275. package/dist/docker-BnRlTpZA.js.map +0 -1
  276. package/dist/docker-D01slDF3.js.map +0 -1
  277. package/dist/doctor-DQ2p0euG.js.map +0 -1
  278. package/dist/env-BQqj93gu.js.map +0 -1
  279. package/dist/environment-CO0P0soF.js.map +0 -1
  280. package/dist/evaluate-BUmsN7yG.js.map +0 -1
  281. package/dist/evaluate-C_sgRA-V.d.ts +0 -134
  282. package/dist/facts-DzOUTreB.js +0 -906
  283. package/dist/facts-DzOUTreB.js.map +0 -1
  284. package/dist/files-B-CJeeEh.js.map +0 -1
  285. package/dist/files-BAt0i00X.js +0 -1
  286. package/dist/files-Bp5j5BqI.js +0 -1
  287. package/dist/files-C2sjmXhk.js +0 -1
  288. package/dist/files-C_gQNVB-.js +0 -65
  289. package/dist/files-C_gQNVB-.js.map +0 -1
  290. package/dist/files-Ca0d6Uq3.js.map +0 -1
  291. package/dist/files-kUCrswI6.js.map +0 -1
  292. package/dist/generate-DZ7vp69F.js +0 -1647
  293. package/dist/generate-DZ7vp69F.js.map +0 -1
  294. package/dist/harness-COVXSwJ6.js +0 -354
  295. package/dist/harness-COVXSwJ6.js.map +0 -1
  296. package/dist/header-BLPP6aCU.js.map +0 -1
  297. package/dist/init-RgCQXZhb.js +0 -1
  298. package/dist/init-SRalM5xF.js.map +0 -1
  299. package/dist/insights-TyAvz9aV.js +0 -68
  300. package/dist/insights-TyAvz9aV.js.map +0 -1
  301. package/dist/invocations-DzyKSJBa.js.map +0 -1
  302. package/dist/judge-yLvjC5A3.js.map +0 -1
  303. package/dist/match-Bz0nSdzN.js.map +0 -1
  304. package/dist/matchers-D4eUqx4j.js.map +0 -1
  305. package/dist/model-1ZLr9Dcv.js.map +0 -1
  306. package/dist/model-B45lDUV1.js.map +0 -1
  307. package/dist/model-BTZq8HIZ.js +0 -1
  308. package/dist/model-CgnOpss4.js.map +0 -1
  309. package/dist/model-Ch8kX87u.js +0 -1
  310. package/dist/node-BSq5qHnQ.js.map +0 -1
  311. package/dist/object-CLf2-tNq.js.map +0 -1
  312. package/dist/opencode-providers-klkqgzvb.js +0 -3276
  313. package/dist/opencode-providers-klkqgzvb.js.map +0 -1
  314. package/dist/pairs-CHfZNRT6.js.map +0 -1
  315. package/dist/patch-DWf-I_Q_.js.map +0 -1
  316. package/dist/plan-BIwUzlB8.js.map +0 -1
  317. package/dist/plugins-Dkp_H3pB.js.map +0 -1
  318. package/dist/prices-BcLEXvdY.js.map +0 -1
  319. package/dist/query-Cvbwl804.js +0 -1
  320. package/dist/questions-bAudIc04.js.map +0 -1
  321. package/dist/redact-CRGdfV2s.js +0 -240
  322. package/dist/redact-CRGdfV2s.js.map +0 -1
  323. package/dist/remarks-CXwSnSjR.js +0 -278
  324. package/dist/remarks-CXwSnSjR.js.map +0 -1
  325. package/dist/remarks-CyAu2DiB.js +0 -1
  326. package/dist/remarks-mujJv1Ey.js.map +0 -1
  327. package/dist/remote-CKcEKja4.js.map +0 -1
  328. package/dist/resolve-B5-uqmJ6.js +0 -1
  329. package/dist/resolve-CBoKzQMP.js.map +0 -1
  330. package/dist/results-CMgZEANy.js +0 -78
  331. package/dist/results-CMgZEANy.js.map +0 -1
  332. package/dist/resume-B9SOSnK9.js.map +0 -1
  333. package/dist/rules-VhftPOOT.js.map +0 -1
  334. package/dist/run-CwIEK16q.js +0 -88
  335. package/dist/run-CwIEK16q.js.map +0 -1
  336. package/dist/runtime-BgQRyCLl.js.map +0 -1
  337. package/dist/server-CHRSgKRp.js +0 -1
  338. package/dist/server-TGW4WyAz.js.map +0 -1
  339. package/dist/shared-BDUzStad.js.map +0 -1
  340. package/dist/source-B1WvBIQt.js +0 -40
  341. package/dist/source-B1WvBIQt.js.map +0 -1
  342. package/dist/source-DAzztZwU.js +0 -1
  343. package/dist/src-1jHbFaGW.js +0 -1
  344. package/dist/src-8zoX_qrb.js.map +0 -1
  345. package/dist/src-C_9wgowC.js +0 -2
  346. package/dist/src-ChhtI6BP.js.map +0 -1
  347. package/dist/src-D005Cbqz.js +0 -2
  348. package/dist/src-DIfRbp5a.js +0 -3
  349. package/dist/src-Dg8121Xy.js +0 -2
  350. package/dist/src-ElBqEN-S.js +0 -2
  351. package/dist/src-NSdQ9zI6.js.map +0 -1
  352. package/dist/src-YHGaUrGE.js +0 -2
  353. package/dist/src-mJVxw1Jm.js +0 -2
  354. package/dist/state-vznMh2wl.js +0 -1
  355. package/dist/suite-BsWGHzlG.js +0 -2
  356. package/dist/suite-WgT7m6Vs.js.map +0 -1
  357. package/dist/summary-DIGEVr08.js.map +0 -1
  358. package/dist/text-CrVy5ioL.js.map +0 -1
  359. package/dist/transcript-BTxJpW4R.js.map +0 -1
  360. package/dist/usage-DF3_p5pE.js +0 -1
  361. package/dist/validate-BhwqrVsm.js.map +0 -1
  362. package/dist/views-BW_C5eVH.js +0 -2
  363. package/dist/worker-entry-Duo9oAeT.js.map +0 -1
  364. package/dist/workspace-QZCacP8A.js +0 -1
@@ -1,1576 +1,2356 @@
1
- import { n as isOneOf } from "./object-CLf2-tNq.js";
1
+ import { n as isOneOf } from "./object-BkXGPKWY.js";
2
2
  import { n as UsageError } from "./errors-B6BSMs03.js";
3
- import { c as jsonValueSchema, i as durationSchema, n as dataProblem, p as parseDuration } from "./shared-BDUzStad.js";
4
- import { v as gatewayNameSchema } from "./harness-COVXSwJ6.js";
3
+ import { c as jsonValueSchema, h as parseDuration, i as durationSchema, m as TIMER_LIMIT, n as dataProblem, p as MAX_TIMER_MS } from "./shared-B_lvGqWz.js";
4
+ import { y as gatewayNameSchema } from "./harness-3NVyTSGK.js";
5
5
  import { l as sha256 } from "./ids-Bp6ut7ZD.js";
6
- import { Ct as judgeStateSchema, bt as judgeFields, v as gatewayScope, vt as captureWindowSchema, xt as judgeModelNameSchema, y as resolveModelGateway, yt as fakeJudgeDataSchema } from "./config-BrdHA9bJ.js";
7
- import { l as judgeRoute } from "./judge-yLvjC5A3.js";
8
- import { ft as shellQuote } from "./src-8zoX_qrb.js";
6
+ import { It as sharedInputFields, Lt as turnsSchema, Nt as judgeFields, Ot as captureWindowSchema, Pt as judgeModelNameSchema, at as awaitListening, b as resolveModelGateway, jt as fakeJudgeDataSchema, ot as parseProbe, st as probeScript, v as gatewayScope } from "./config-CneBZB84.js";
7
+ import { y as vertexSettingsIn } from "./vertex-DypYjx71.js";
8
+ import { m as redactSecrets } from "./redact-DVIaahLv.js";
9
+ import { a as plural, i as formatDuration, n as clip } from "./text-DTigcA64.js";
9
10
  import { s as cacheDir } from "./root-Dlff-zp8.js";
11
+ import { pt as shellQuote } from "./src-C0ySiej5.js";
10
12
  import { t as InfraError } from "./types-C5vI3g1C.js";
11
- import { i as formatDuration, n as clip } from "./text-CrVy5ioL.js";
12
- import { n as getRunContext, o as SUBJECT } from "./runtime-BgQRyCLl.js";
13
- import { i as debugTrace, r as debugLog } from "./attempt-jI0OWxlz.js";
14
- import { o as providerModel } from "./model-1ZLr9Dcv.js";
13
+ import { r as getRunContext, s as SUBJECT } from "./runtime-CZVpkr04.js";
14
+ import { i as debugTrace, r as debugLog } from "./attempt-CpbZ9arV.js";
15
+ import { r as openRouterFallbacks } from "./openrouter-models-Bo8PZ4Ew.js";
16
+ import { i as hostModel } from "./model-2rBZrdZA.js";
15
17
  import { t as addUsage } from "./usage-BNY1ABrU.js";
18
+ import { t as untraced } from "./docker-BihzJwbV.js";
16
19
  import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
17
20
  import { homedir, tmpdir } from "node:os";
18
21
  import { join, posix } from "node:path";
19
22
  import { z } from "zod";
20
- import { createServer } from "node:net";
23
+ import net, { createServer } from "node:net";
21
24
  import { createHash, randomUUID } from "node:crypto";
22
25
  import { setTimeout as setTimeout$1 } from "node:timers/promises";
23
26
  import { spawn } from "node:child_process";
24
- import { crc32, deflateSync, inflateRawSync } from "node:zlib";
27
+ import http from "node:http";
28
+ import { crc32, deflateSync, inflateRawSync, inflateSync } from "node:zlib";
25
29
  import { mkdir, writeFile } from "node:fs/promises";
26
30
  import { generateText, stepCountIs, tool } from "ai";
27
- //#region src/capture/fake-instructions.ts
28
- const FAKE_INSTRUCTIONS = Symbol.for("reeval.capture.fakeInstructions");
29
- function isWaitStep(step) {
30
- return typeof step === "object" && step !== null && !Array.isArray(step) && "wait" in step;
31
- }
32
- /** One list: action lists and `{ wait }` inlined. Returns branded `instructions`. */
33
- function fakeInstructions(steps) {
34
- if (!Array.isArray(steps)) throw new Error(`fakeInstructions() is an array of action lists and { wait } (got ${JSON.stringify(steps)})`);
35
- const instructions = [];
36
- let goals = 0;
37
- for (const [i, step] of steps.entries()) {
38
- if (isWaitStep(step)) {
39
- instructions.push({ wait: step.wait });
40
- continue;
41
- }
42
- if (!Array.isArray(step)) throw new Error(`fakeInstructions()[${i}] is an action list or { wait } (got ${JSON.stringify(step)})`);
43
- instructions.push(`fake[${goals}]`);
44
- goals++;
45
- }
46
- Object.defineProperty(instructions, FAKE_INSTRUCTIONS, {
47
- value: steps,
48
- enumerable: false
49
- });
50
- return instructions;
51
- }
52
- function fakeInstructionSteps(value) {
53
- if (typeof value === "object" && value !== null && FAKE_INSTRUCTIONS in value) return value[FAKE_INSTRUCTIONS];
54
- }
55
- //#endregion
56
- //#region src/capture/zip.ts
57
- const LOCAL = 67324752;
58
- /** Extracts a zip (store or deflate) into `files` keyed by posix path. */
59
- function unzip(bytes) {
60
- const buf = Buffer.from(bytes);
61
- const files = /* @__PURE__ */ new Map();
62
- let i = 0;
63
- while (i + 30 <= buf.length) {
64
- if (buf.readUInt32LE(i) !== LOCAL) break;
65
- const method = buf.readUInt16LE(i + 8);
66
- const comp = buf.readUInt32LE(i + 18);
67
- const nameLen = buf.readUInt16LE(i + 26);
68
- const extra = buf.readUInt16LE(i + 28);
69
- const name = buf.toString("utf8", i + 30, i + 30 + nameLen);
70
- const start = i + 30 + nameLen + extra;
71
- const raw = buf.subarray(start, start + comp);
72
- if (name.endsWith("/")) {
73
- i = start + comp;
74
- continue;
75
- }
76
- if (method === 0) files.set(name, Buffer.from(raw));
77
- else if (method === 8) files.set(name, Buffer.from(inflateRawSync(raw)));
78
- else throw new Error(`zip method ${method} is not supported (${name})`);
79
- i = start + comp;
80
- }
81
- return files;
82
- }
83
- //#endregion
84
- //#region src/capture/browser.ts
85
- const isMs = (wait) => typeof wait === "object" && wait !== null;
86
- /** The CDP event `'idle'` waits for; `'load'` is `Page.loadEventFired`. */
87
- function captureWaitLabel(wait) {
88
- return wait === "idle" ? "networkAlmostIdle" : "load";
89
- }
90
- function captureWaitTimeout(wait) {
91
- return /* @__PURE__ */ new Error(`c.capture() timed out waiting for ${captureWaitLabel(wait)}`);
92
- }
93
- function waitTimeout(wait, detail) {
94
- if (isMs(wait)) return /* @__PURE__ */ new Error(`c.capture() timed out${detail ? ` on ${detail}` : ""} waiting ${wait.ms}ms`);
95
- return captureWaitTimeout(wait);
96
- }
97
- async function waitForPage(cdp, wait, timeoutMs, nav) {
98
- if (isMs(wait)) {
99
- if (timeoutMs <= 0) throw waitTimeout(wait);
100
- const ms = Math.min(wait.ms, timeoutMs);
101
- await new Promise((resolve) => setTimeout(resolve, ms));
102
- return;
103
- }
104
- const page = listenForCaptureWait(cdp, wait);
105
- page.arm();
106
- await page.ready(nav, timeoutMs);
107
- }
108
- const clamp = (n, lo, hi) => Math.min(hi, Math.max(lo, n));
109
- /** Map a full-page click onto the live window, scrolling with the wheel if needed. */
110
- function mapPagePoint(x, y, window, page, scrollY) {
111
- const maxY = Math.max(0, (page?.height ?? window.height) - window.height);
112
- const target = y >= window.height || y < 0 ? clamp(y - window.height / 2, 0, maxY) : scrollY;
113
- const nextY = clamp(target, 0, maxY);
114
- return {
115
- x: clamp(x, 0, window.width - 1),
116
- y: clamp(y - nextY, 0, window.height - 1),
117
- scrollY: nextY,
118
- dy: nextY - scrollY
119
- };
120
- }
121
- async function dispatchInput(cdp, action, window, page, scrollY = 0) {
122
- if (action.type === "done") return scrollY;
123
- if (action.type === "type") {
124
- await cdp.send("Input.insertText", { text: action.text });
125
- return scrollY;
126
- }
127
- if (action.type === "key") {
128
- const key = action.key;
129
- const vk = key === "Tab" ? 9 : 13;
130
- await cdp.send("Input.dispatchKeyEvent", {
131
- type: "keyDown",
132
- key,
133
- code: key,
134
- windowsVirtualKeyCode: vk
135
- });
136
- await cdp.send("Input.dispatchKeyEvent", {
137
- type: "keyUp",
138
- key,
139
- code: key,
140
- windowsVirtualKeyCode: vk
141
- });
142
- return scrollY;
143
- }
144
- if (action.type === "scroll") {
145
- await cdp.send("Input.dispatchMouseEvent", {
146
- type: "mouseWheel",
147
- x: action.x,
148
- y: action.y,
149
- deltaX: 0,
150
- deltaY: action.dy
151
- });
152
- return scrollY + action.dy;
153
- }
154
- const mapped = mapPagePoint(action.type === "drag" ? action.x : action.x, action.type === "drag" ? action.y : action.y, window, page, scrollY);
155
- if (mapped.dy !== 0) await cdp.send("Input.dispatchMouseEvent", {
156
- type: "mouseWheel",
157
- x: window.width / 2,
158
- y: window.height / 2,
159
- deltaX: 0,
160
- deltaY: mapped.dy
161
- });
162
- const at = {
163
- x: mapped.x,
164
- y: mapped.y
165
- };
166
- if (action.type === "hover") {
167
- await cdp.send("Input.dispatchMouseEvent", {
168
- type: "mouseMoved",
169
- ...at
170
- });
171
- return mapped.scrollY;
172
- }
173
- if (action.type === "click") {
174
- await cdp.send("Input.dispatchMouseEvent", {
175
- type: "mouseMoved",
176
- ...at,
177
- button: "left"
178
- });
179
- await cdp.send("Input.dispatchMouseEvent", {
180
- type: "mousePressed",
181
- ...at,
182
- button: "left",
183
- clickCount: 1
184
- });
185
- await cdp.send("Input.dispatchMouseEvent", {
186
- type: "mouseReleased",
187
- ...at,
188
- button: "left",
189
- clickCount: 1
190
- });
191
- return mapped.scrollY;
192
- }
193
- const end = mapPagePoint(action.toX, action.toY, window, page, mapped.scrollY);
194
- if (end.dy !== 0) await cdp.send("Input.dispatchMouseEvent", {
195
- type: "mouseWheel",
196
- x: window.width / 2,
197
- y: window.height / 2,
198
- deltaX: 0,
199
- deltaY: end.dy
200
- });
201
- await cdp.send("Input.dispatchMouseEvent", {
202
- type: "mouseMoved",
203
- x: at.x,
204
- y: at.y,
205
- button: "left"
206
- });
207
- await cdp.send("Input.dispatchMouseEvent", {
208
- type: "mousePressed",
209
- x: at.x,
210
- y: at.y,
211
- button: "left",
212
- clickCount: 1
213
- });
214
- await cdp.send("Input.dispatchMouseEvent", {
215
- type: "mouseMoved",
216
- x: end.x,
217
- y: end.y,
218
- button: "left"
219
- });
220
- await cdp.send("Input.dispatchMouseEvent", {
221
- type: "mouseReleased",
222
- x: end.x,
223
- y: end.y,
224
- button: "left",
225
- clickCount: 1
226
- });
227
- return end.scrollY;
228
- }
229
- function lifecycleParams(msg) {
230
- const body = msg;
231
- return body.params ?? body;
232
- }
233
- /**
234
- * Arms after `Page.enable` so about:blank's events cannot resolve the wait.
235
- * Call `arm` immediately before `Page.navigate`.
236
- */
237
- function listenForCaptureWait(cdp, wait) {
238
- const method = wait === "idle" ? "Page.lifecycleEvent" : "Page.loadEventFired";
239
- const pending = [];
240
- let deliver;
241
- let started = false;
242
- cdp.on(method, (msg) => {
243
- if (!started) return;
244
- if (deliver) deliver(msg);
245
- else pending.push(msg);
246
- });
247
- const matches = (msg, loaderId) => {
248
- if (wait === "load") return true;
249
- const params = lifecycleParams(msg);
250
- if (params.name !== "networkAlmostIdle") return false;
251
- return !loaderId || params.loaderId === loaderId;
252
- };
253
- return {
254
- arm() {
255
- started = true;
256
- },
257
- ready(nav, timeoutMs) {
258
- return new Promise((resolve, reject) => {
259
- const timer = setTimeout(() => reject(captureWaitTimeout(wait)), timeoutMs);
260
- const finish = () => {
261
- clearTimeout(timer);
262
- resolve();
263
- };
264
- deliver = (msg) => {
265
- if (matches(msg, nav?.loaderId)) finish();
266
- };
267
- if (pending.some((msg) => matches(msg, nav?.loaderId))) finish();
268
- });
269
- }
270
- };
271
- }
272
- const QUIET_IN_FLIGHT = 2;
273
- function requestId(msg) {
274
- const body = msg;
275
- return (body.params ?? body).requestId;
276
- }
277
- /**
278
- * Counts the tab's requests from `Network.enable` on, for a journey's `{ wait: 'idle' }` beat.
279
- * Chromium emits `networkAlmostIdle` once per document, so after a click that only fetches it
280
- * never comes again. As in Chromium, every request that starts or ends restarts the quiet window.
281
- */
282
- function trackNetwork(cdp, quietMs = 500) {
283
- const inFlight = /* @__PURE__ */ new Set();
284
- let changed;
285
- cdp.on("Network.requestWillBeSent", (msg) => {
286
- const id = requestId(msg);
287
- if (id === void 0) return;
288
- inFlight.add(id);
289
- changed?.();
290
- });
291
- const end = (msg) => {
292
- const id = requestId(msg);
293
- if (id === void 0 || !inFlight.delete(id)) return;
294
- changed?.();
295
- };
296
- cdp.on("Network.loadingFinished", end);
297
- cdp.on("Network.loadingFailed", end);
298
- return { quiet(timeoutMs) {
299
- return new Promise((resolve, reject) => {
300
- let settle;
301
- const stop = () => {
302
- clearTimeout(timer);
303
- clearTimeout(settle);
304
- changed = void 0;
305
- };
306
- const timer = setTimeout(() => {
307
- stop();
308
- reject(captureWaitTimeout("idle"));
309
- }, timeoutMs);
310
- changed = () => {
311
- clearTimeout(settle);
312
- if (inFlight.size > QUIET_IN_FLIGHT) return;
313
- settle = setTimeout(() => {
314
- stop();
315
- resolve();
316
- }, quietMs);
317
- };
318
- changed();
319
- });
320
- } };
321
- }
322
- /** Pinned Chrome for Testing `chrome-headless-shell` build. */
323
- const CHROME_VERSION = "140.0.7339.185";
324
- const CHROME_NAME = "chrome-headless-shell";
325
- function chromePlatform() {
326
- if (process.platform === "darwin") return process.arch === "arm64" ? "mac-arm64" : "mac-x64";
327
- if (process.platform === "win32") return "win64";
328
- return "linux64";
329
- }
330
- function chromeUrl(platform) {
331
- return `https://storage.googleapis.com/chrome-for-testing-public/${CHROME_VERSION}/${platform}/${CHROME_NAME}-${platform}.zip`;
332
- }
333
- function binaryName() {
334
- return process.platform === "win32" ? `${CHROME_NAME}.exe` : CHROME_NAME;
335
- }
336
- async function downloadZip(url, onProgress, cacheLabel, signal) {
337
- const res = await fetch(url, { signal });
338
- if (!res.ok) throw new Error(`download failed: HTTP ${res.status}`);
339
- const total = Number(res.headers.get("content-length")) || null;
340
- const report = (received) => onProgress?.({
341
- name: CHROME_NAME,
342
- version: CHROME_VERSION,
343
- bytes: total,
344
- received,
345
- cache: cacheLabel
346
- });
347
- report(0);
348
- if (!res.body) return Buffer.from(await res.arrayBuffer());
349
- const reader = res.body.getReader();
350
- const chunks = [];
351
- let received = 0;
352
- let last = 0;
353
- for (;;) {
354
- const { done, value } = await reader.read();
355
- if (done) break;
356
- chunks.push(value);
357
- received += value.length;
358
- if (received - last >= 262144 || total && received === total) {
359
- report(received);
360
- last = received;
361
- }
362
- }
363
- if (received !== last) report(received);
364
- return Buffer.concat(chunks);
365
- }
366
- function extractBinary(zip, dest, platform) {
367
- const files = unzip(zip);
368
- const needle = `/${binaryName()}`;
369
- const entry = [...files.entries()].find(([name]) => name.endsWith(needle) || name === binaryName() || name.endsWith(`\\${binaryName()}`));
370
- if (!entry) {
371
- const names = [...files.keys()].slice(0, 8).join(", ");
372
- throw new Error(`zip has no ${binaryName()} (saw ${names || "nothing"})`);
373
- }
374
- const prefix = entry[0].slice(0, -binaryName().length);
375
- mkdirSync(dest, { recursive: true });
376
- for (const [name, data] of files) {
377
- const rel = prefix && name.startsWith(prefix) ? name.slice(prefix.length) : name;
378
- if (!rel || rel.endsWith("/")) continue;
379
- const target = join(dest, ...rel.split("/"));
380
- mkdirSync(join(target, ".."), { recursive: true });
381
- writeFileSync(target, data);
382
- }
383
- const bin = join(dest, binaryName());
384
- if (!existsSync(bin)) writeFileSync(bin, entry[1]);
385
- if (process.platform !== "win32") chmodSync(bin, 493);
386
- }
387
- /**
388
- * Returns the cached binary, downloading on a miss. `onProgress` runs only for a download.
389
- */
390
- async function ensureChrome(cacheRoot, options = { cacheLabel: cacheRoot }) {
391
- const platform = chromePlatform();
392
- const dest = join(cacheRoot, CHROME_NAME, CHROME_VERSION, platform);
393
- const bin = join(dest, binaryName());
394
- if (existsSync(bin)) return bin;
395
- mkdirSync(join(cacheRoot, CHROME_NAME, CHROME_VERSION), { recursive: true });
396
- const partial = `${dest}.${process.pid}.partial`;
397
- rmSync(partial, {
398
- recursive: true,
399
- force: true
400
- });
401
- try {
402
- extractBinary(await downloadZip(chromeUrl(platform), options.onProgress, options.cacheLabel, options.signal), partial, platform);
403
- try {
404
- renameSync(partial, dest);
405
- } catch (error) {
406
- rmSync(partial, {
407
- recursive: true,
408
- force: true
409
- });
410
- if (!existsSync(bin)) throw error;
411
- }
412
- } catch (error) {
413
- rmSync(partial, {
414
- recursive: true,
415
- force: true
416
- });
417
- if (existsSync(bin)) return bin;
418
- throw error;
419
- }
420
- return bin;
421
- }
422
- var Cdp = class {
423
- #ws;
424
- #next = 0;
425
- #pending = /* @__PURE__ */ new Map();
426
- #events = /* @__PURE__ */ new Map();
427
- constructor(ws) {
428
- this.#ws = ws;
429
- ws.addEventListener("message", (ev) => {
430
- const msg = JSON.parse(String(ev.data));
431
- if (msg.id !== void 0) {
432
- const p = this.#pending.get(msg.id);
433
- if (!p) return;
434
- this.#pending.delete(msg.id);
435
- if (msg.error) p.reject(new Error(msg.error.message));
436
- else p.resolve(msg.result);
437
- return;
438
- }
439
- if (msg.method) for (const fn of this.#events.get(msg.method) ?? []) fn(msg);
31
+ //#region src/contract/attempt.ts
32
+ const nullableNumber = z.number().nullable();
33
+ const tokenUsageSchema = z.strictObject({
34
+ input: z.number(),
35
+ output: z.number(),
36
+ cacheRead: z.number(),
37
+ cacheWrite: z.number(),
38
+ reasoning: nullableNumber
39
+ });
40
+ const fileChangeSchema = z.strictObject({
41
+ path: z.string(),
42
+ status: z.enum([
43
+ "added",
44
+ "modified",
45
+ "deleted"
46
+ ])
47
+ });
48
+ const diffSchema = z.strictObject({
49
+ patch: z.string(),
50
+ files: z.array(fileChangeSchema)
51
+ });
52
+ const turnStatusSchema = z.enum([
53
+ "completed",
54
+ "aborted",
55
+ "error"
56
+ ]);
57
+ const webEventTypeSchema = z.enum(["web-search", "web-fetch"]);
58
+ const at = {
59
+ ts: z.string(),
60
+ atMs: z.number()
61
+ };
62
+ const transcriptEventSchema = z.discriminatedUnion("type", [
63
+ z.strictObject({
64
+ ...at,
65
+ type: z.enum(["text", "reasoning"]),
66
+ text: z.string(),
67
+ durationMs: nullableNumber
68
+ }),
69
+ z.strictObject({
70
+ ...at,
71
+ type: z.literal("file-edit"),
72
+ toolCallId: z.string(),
73
+ path: z.string(),
74
+ kind: z.enum([
75
+ "create",
76
+ "modify",
77
+ "write",
78
+ "delete"
79
+ ]),
80
+ added: nullableNumber,
81
+ removed: nullableNumber,
82
+ durationMs: nullableNumber,
83
+ stalled: z.boolean().optional()
84
+ }),
85
+ z.strictObject({
86
+ ...at,
87
+ type: z.literal("web-search"),
88
+ toolCallId: z.string(),
89
+ query: z.string(),
90
+ references: nullableNumber,
91
+ ok: z.boolean().optional(),
92
+ durationMs: nullableNumber,
93
+ stalled: z.boolean().optional()
94
+ }),
95
+ z.strictObject({
96
+ ...at,
97
+ type: z.literal("web-fetch"),
98
+ toolCallId: z.string(),
99
+ url: z.string(),
100
+ ok: z.boolean(),
101
+ durationMs: nullableNumber,
102
+ stalled: z.boolean().optional()
103
+ }),
104
+ z.strictObject({
105
+ ...at,
106
+ type: z.literal("tool"),
107
+ toolCallId: z.string(),
108
+ tool: z.string(),
109
+ input: z.unknown(),
110
+ output: z.unknown(),
111
+ ok: z.boolean(),
112
+ durationMs: nullableNumber,
113
+ stalled: z.boolean().optional()
114
+ }),
115
+ z.strictObject({
116
+ ...at,
117
+ type: z.literal("error"),
118
+ message: z.string()
119
+ }),
120
+ z.strictObject({
121
+ ...at,
122
+ type: z.literal("retry"),
123
+ message: z.string()
124
+ })
125
+ ]);
126
+ const turnDataSchema = z.strictObject({
127
+ prompt: z.string(),
128
+ text: z.string(),
129
+ status: turnStatusSchema,
130
+ startedAt: z.string(),
131
+ durationMs: z.number(),
132
+ transcript: z.array(transcriptEventSchema),
133
+ usage: tokenUsageSchema.nullable(),
134
+ model: z.string().nullable(),
135
+ unreported: z.array(webEventTypeSchema),
136
+ diff: diffSchema
137
+ });
138
+ const execResultSchema = z.strictObject({
139
+ exitCode: z.number().int(),
140
+ stdout: z.string(),
141
+ stderr: z.string()
142
+ });
143
+ const execRetrySchema = z.strictObject({
144
+ retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional(),
145
+ interval: z.number().min(0, "must be 0 or more milliseconds").max(MAX_TIMER_MS, `must be ${TIMER_LIMIT}`).optional()
146
+ });
147
+ /** A background command has no exit for `retries` to wait on. */
148
+ const noBackgroundRetry = (options, ctx) => {
149
+ if (!options.background) return;
150
+ for (const key of ["retries", "interval"]) {
151
+ if (options[key] === void 0) continue;
152
+ ctx.addIssue({
153
+ code: "custom",
154
+ path: [key],
155
+ message: "can't be used with background: true, whose exit reeval doesn't wait for: drop one of them"
440
156
  });
441
157
  }
442
- on(method, fn) {
443
- const list = this.#events.get(method) ?? [];
444
- list.push(fn);
445
- this.#events.set(method, list);
446
- }
447
- send(method, params) {
448
- const id = ++this.#next;
449
- this.#ws.send(JSON.stringify({
450
- id,
451
- method,
452
- params
453
- }));
454
- return new Promise((resolve, reject) => this.#pending.set(id, {
455
- resolve,
456
- reject
457
- }));
458
- }
459
- close() {
460
- this.#ws.close();
461
- }
462
158
  };
463
- function waitForPortFile(file, timeoutMs) {
464
- const deadline = Date.now() + timeoutMs;
465
- return new Promise((resolve, reject) => {
466
- const tick = () => {
467
- if (existsSync(file)) {
468
- const line = readFileSync(file, "utf8").split(/\r?\n/)[0];
469
- const port = Number(line);
470
- if (port > 0) {
471
- resolve(port);
472
- return;
473
- }
474
- }
475
- if (Date.now() > deadline) {
476
- reject(/* @__PURE__ */ new Error("chrome-headless-shell did not open a DevTools port"));
477
- return;
478
- }
479
- setTimeout(tick, 50);
480
- };
481
- tick();
482
- });
483
- }
484
- function pickPageDebuggerUrl(targets) {
485
- return targets.find((t) => t.webSocketDebuggerUrl && t.type !== "browser")?.webSocketDebuggerUrl;
486
- }
487
- async function pageDebuggerUrl(port, timeoutMs) {
488
- const deadline = Date.now() + timeoutMs;
489
- const listPages = async () => {
490
- const res = await fetch(`http://127.0.0.1:${port}/json/list`);
491
- if (!res.ok) return void 0;
492
- return pickPageDebuggerUrl(await res.json());
493
- };
494
- for (;;) {
495
- const url = await listPages();
496
- if (url) return url;
497
- if (Date.now() > deadline) break;
498
- await new Promise((r) => setTimeout(r, 50));
499
- }
500
- const version = await (await fetch(`http://127.0.0.1:${port}/json/version`)).json();
501
- if (!version.webSocketDebuggerUrl) throw new Error("chrome-headless-shell reported no DevTools URL");
502
- const browser = await cdpConnect(version.webSocketDebuggerUrl, timeoutMs);
503
- try {
504
- await browser.send("Target.createTarget", { url: "about:blank" });
505
- } finally {
506
- browser.close();
507
- }
508
- const created = await listPages();
509
- if (!created) throw new Error("chrome-headless-shell reported no page target");
510
- return created;
159
+ const execOptionsSchema = z.strictObject({
160
+ background: z.boolean().optional(),
161
+ ...execRetrySchema.shape
162
+ }).superRefine(noBackgroundRetry);
163
+ /** What's wrong with `c.exec()`'s options, or undefined. */
164
+ const execOptionsProblem = (value) => dataProblem(execOptionsSchema, value, "c.exec()", () => Object.keys(execOptionsSchema.shape));
165
+ const assertionRecordSchema = z.strictObject({
166
+ label: z.string(),
167
+ labelled: z.boolean(),
168
+ matcher: z.string(),
169
+ status: z.enum([
170
+ "pass",
171
+ "fail",
172
+ "error"
173
+ ]),
174
+ soft: z.boolean(),
175
+ at: z.string().optional(),
176
+ message: z.string().optional(),
177
+ durationMs: z.number().optional()
178
+ });
179
+ const turnIndex = z.number().int("must be a turn index, from 0").min(0, "must be a turn index, from 0").describe("The turn's index, from 0.");
180
+ const captureSchema = z.strictObject({
181
+ name: z.string(),
182
+ port: z.number(),
183
+ path: z.string(),
184
+ viewport: z.union([captureWindowSchema, z.literal("full-page")]),
185
+ file: z.strictObject({
186
+ path: z.string(),
187
+ sha256: z.string(),
188
+ bytes: z.number()
189
+ })
190
+ });
191
+ const judgeArtifactRefSchema = z.strictObject({
192
+ name: z.string(),
193
+ path: z.string(),
194
+ sha256: z.string(),
195
+ bytes: z.number(),
196
+ mediaType: z.literal("image/png")
197
+ });
198
+ const verdictFields = {
199
+ threshold: z.number(),
200
+ question: z.string(),
201
+ model: z.string(),
202
+ gateway: gatewayNameSchema.optional(),
203
+ artifacts: z.array(judgeArtifactRefSchema).optional()
204
+ };
205
+ const verdictSchema = z.strictObject({
206
+ verdict: z.enum(["pass", "fail"]),
207
+ score: z.number(),
208
+ ...verdictFields,
209
+ resolvedModel: z.string().optional(),
210
+ upstream: z.string().optional(),
211
+ costUsd: z.number().optional()
212
+ });
213
+ const judgeFailureSchema = z.strictObject({
214
+ verdict: z.literal("error"),
215
+ score: z.null(),
216
+ ...verdictFields,
217
+ error: z.string()
218
+ });
219
+ /** A pattern, for the matchers that take a string or a `RegExp`. */
220
+ const patternSchema = z.strictObject({
221
+ regexp: z.string().describe("The pattern's source, as `RegExp.prototype.source` gives it."),
222
+ flags: z.string().regex(/^[dgimsuvy]*$/, "must use only the flags d, g, i, m, s, u, v and y").optional().describe("Its flags, e.g. `i`.")
223
+ }).describe("A regular expression, where a matcher takes a string or a `RegExp`.");
224
+ /** A pattern in the runner's own language, which reeval never compiles. */
225
+ function hasAnsweredPattern(value) {
226
+ if (Array.isArray(value)) return value.some(hasAnsweredPattern);
227
+ if (typeof value !== "object" || value === null) return false;
228
+ return "matchedBy" in value || Object.values(value).some(hasAnsweredPattern);
511
229
  }
512
- async function cdpConnect(url, timeoutMs) {
513
- const ws = new WebSocket(url);
514
- await new Promise((resolve, reject) => {
515
- const t = setTimeout(() => reject(/* @__PURE__ */ new Error("CDP websocket timed out")), timeoutMs);
516
- ws.addEventListener("open", () => {
517
- clearTimeout(t);
518
- resolve();
230
+ const answeredPatternSchema = z.strictObject({
231
+ matchedBy: z.string().regex(/\S/).describe("The pattern as the runner's language writes it, e.g. `re.compile('npm t')`."),
232
+ answers: z.record(z.string(), z.boolean()).describe("Whether it matches each string the runner has been asked about.")
233
+ }).describe("A pattern the runner matches in its own language. reeval runs the matcher with it, and when the matcher tests a string not in `answers` replies with those strings (`untested`) instead of a record; the runner matches them and makes the call again with them in `answers`.");
234
+ const isPatternLike = (value) => typeof value === "object" && value !== null && !Array.isArray(value) && "regexp" in value;
235
+ const matcherArgSchema = z.union([
236
+ patternSchema,
237
+ answeredPatternSchema,
238
+ jsonValueSchema
239
+ ]).superRefine((value, ctx) => {
240
+ if (!isPatternLike(value)) return;
241
+ for (const issue of patternSchema.safeParse(value).error?.issues ?? []) ctx.addIssue(issue);
242
+ }).describe("One argument to a matcher: a JSON value, a regular expression as `{ regexp, flags? }`, or a pattern in the runner's own language as `{ matchedBy, answers }`.");
243
+ const matcherSubjectSchema = z.union([
244
+ z.strictObject({ turn: turnIndex }),
245
+ z.strictObject({ attempt: z.literal(true).describe("The attempt itself, `c`.") }),
246
+ z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
247
+ z.strictObject({
248
+ exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
249
+ result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
250
+ }),
251
+ z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
252
+ ]).describe("What `expect` was given: a turn `c.send` returned, the attempt (for its files and latest turn), a value, such as an `ExecResult`, or a command or file reeval reads for it.");
253
+ const nonEmpty = z.string().regex(/\S/, "must not be empty");
254
+ const CAPTURE_NAME = new RegExp(`^[a-z0-9][a-z0-9_-]{0,99}$`);
255
+ const CAPTURE_NAME_RULE = `must be 1 to 100 of a-z, 0-9, _ and -, starting with a letter or digit`;
256
+ const portSchema = z.number().int().min(1, "a port from 1 to 65535").max(65535, "a port from 1 to 65535");
257
+ const millisSchema = z.number().positive("more than zero milliseconds").max(MAX_TIMER_MS, TIMER_LIMIT);
258
+ const captureWaitSchema = z.enum(["load", "idle"]);
259
+ const waitSchema = z.union([captureWaitSchema, z.strictObject({ ms: millisSchema })]);
260
+ const captureInstructionSchema = z.union([nonEmpty, z.strictObject({ wait: waitSchema })]);
261
+ const captureCallDataSchema = z.strictObject({
262
+ name: z.string().regex(CAPTURE_NAME, CAPTURE_NAME_RULE).optional(),
263
+ path: z.string().optional(),
264
+ viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
265
+ timeout: durationSchema.optional(),
266
+ wait: waitSchema.optional(),
267
+ instructions: z.array(captureInstructionSchema).optional().describe("Journey: goal strings and `{ wait }` beats, run by the vision model."),
268
+ model: z.string().optional()
269
+ }).describe("`c.capture()`'s camera and path as data. Default `path` is `/`; default `name` is derived from the path. A test's `fakeInstructions` journey is TypeScript-only.");
270
+ const handleId = nonEmpty.describe("A background process, as an exec or serve reply named it.");
271
+ const judgeOnSchema = z.union([z.strictObject({ turn: turnIndex }), z.strictObject({ value: jsonValueSchema.describe("Graded as the output.") })]);
272
+ const judgeCallDataSchema = z.strictObject({
273
+ model: z.union([judgeModelNameSchema, fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge with a numeric score."),
274
+ ...judgeFields,
275
+ ...sharedInputFields,
276
+ contents: z.array(nonEmpty).optional(),
277
+ turns: turnsSchema.optional(),
278
+ on: judgeOnSchema.optional().describe("The turn `task`, `output`, `transcript` and `diff` read instead of the latest: an earlier turn, or a value graded as the output."),
279
+ artifacts: z.union([z.literal(true), z.array(z.union([nonEmpty, captureSchema]))]).optional().describe("Captures this call sends the judge as image file parts: a capture `c.capture` returned, or its name, or `true` for every capture so far. The provider that cannot take files fails the call.")
280
+ }).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score, the inputs it shows the judge (at least one turned on) and `on`.");
281
+ const callSiteSchema = z.strictObject({
282
+ file: nonEmpty.describe("The eval file: absolute, or relative to the project root."),
283
+ line: z.number().int().min(1).describe("From 1.")
284
+ }).describe("Where the eval made the call, which the record names beside the check's outcome.");
285
+ const attemptCallSchema = z.discriminatedUnion("op", [
286
+ z.strictObject({
287
+ op: z.literal("send"),
288
+ prompt: z.string().describe("The turn's prompt.")
289
+ }).describe("`c.send(prompt)`: the next turn of the agent session. Replies with a turn."),
290
+ z.strictObject({
291
+ op: z.literal("exec"),
292
+ command: nonEmpty.describe("A shell command, run in the workspace."),
293
+ background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
294
+ ...execRetrySchema.shape
295
+ }).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
296
+ z.strictObject({
297
+ op: z.literal("serve"),
298
+ command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
299
+ port: portSchema.optional().describe("The container port. Omit it and reeval allocates one.")
300
+ }).describe("`c.serve(command, port?)`: a background process, once it listens, and its port."),
301
+ z.strictObject({
302
+ op: z.literal("kill"),
303
+ handle: handleId
304
+ }).describe("`handle.kill()`: stops a background process. Killing one that's gone is fine."),
305
+ z.strictObject({
306
+ op: z.literal("capture"),
307
+ target: z.union([z.strictObject({ port: portSchema }), z.strictObject({ handle: handleId })]).describe("A container port, or a serve handle by its id."),
308
+ options: captureCallDataSchema.optional()
309
+ }).describe("`c.capture(target, options)`: screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`. Replies with the capture."),
310
+ z.strictObject({
311
+ op: z.literal("readFile"),
312
+ path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory.")
313
+ }).describe("`c.readFile(path)`. Replies with its contents, or null when there is no such file."),
314
+ z.strictObject({
315
+ op: z.literal("writeFile"),
316
+ path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory."),
317
+ contents: z.string().describe("The file's contents, in `encoding`."),
318
+ encoding: z.enum(["utf8", "base64"]).optional().describe("How `contents` is encoded; base64 carries bytes. Default `utf8`.")
319
+ }).describe("`c.writeFile(path, contents)`, creating its directories."),
320
+ z.strictObject({
321
+ op: z.literal("expect"),
322
+ subject: matcherSubjectSchema,
323
+ matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a matcher name such as toHaveFile, without \"not.\" (set not: true instead)").describe("The matcher's name, e.g. `toHaveFile` or `toBe`, without `not.`."),
324
+ args: z.array(matcherArgSchema).describe("The matcher's arguments."),
325
+ not: z.boolean().describe("Called as `expect(…).not`."),
326
+ soft: z.boolean().describe("Called as `expect`: a failure fails the attempt at its end and the body carries on. False for `expect.require`, whose failure the runner raises, ending the body."),
327
+ label: z.string().optional().describe("`expect`'s second argument."),
328
+ at: callSiteSchema.optional(),
329
+ poll: z.strictObject({
330
+ interval: millisSchema.optional().describe("Milliseconds between tries. Default 50."),
331
+ timeout: millisSchema.optional().describe("Milliseconds before it fails with the last error. Default 1000.")
332
+ }).optional().describe("`expect.poll`: reeval re-reads the subject and retries the matcher until it passes, recording only the final outcome. The subject must be one reeval can re-read: the attempt, a command or a file.")
333
+ }).superRefine((call, ctx) => {
334
+ if (call.poll && "value" in call.subject) ctx.addIssue({
335
+ code: "custom",
336
+ path: ["subject"],
337
+ message: "can't be polled: reeval can't re-read a value. Poll the attempt, an exec or a readFile"
519
338
  });
520
- ws.addEventListener("error", () => {
521
- clearTimeout(t);
522
- reject(/* @__PURE__ */ new Error("CDP websocket failed"));
339
+ if (call.poll && "exec" in call.subject && call.subject.result) ctx.addIssue({
340
+ code: "custom",
341
+ path: ["subject", "result"],
342
+ message: "can't be polled: drop result so reeval runs the command on each try"
523
343
  });
524
- });
525
- return new Cdp(ws);
526
- }
527
- /**
528
- * Drops a chrome-headless-shell profile. Chrome can still write after SIGKILL,
529
- * so `ENOTEMPTY` / `EBUSY` must not replace a PNG already in hand.
530
- */
531
- function removeChromeProfile(dir, rm = rmSync) {
532
- try {
533
- rm(dir, {
534
- recursive: true,
535
- force: true,
536
- maxRetries: 5,
537
- retryDelay: 20
344
+ if (call.poll && call.args.some(hasAnsweredPattern)) ctx.addIssue({
345
+ code: "custom",
346
+ path: ["args"],
347
+ message: "can't be polled with a { matchedBy } pattern, whose answers can't follow the subject from try to try: poll with a string, or check once"
538
348
  });
539
- } catch {
540
- scheduleChromeProfileSweep(dir, rm);
541
- }
542
- }
543
- function scheduleChromeProfileSweep(dir, rm) {
544
- setTimeout(() => {
545
- try {
546
- rm(dir, {
547
- recursive: true,
548
- force: true,
549
- maxRetries: 5,
550
- retryDelay: 50
551
- });
552
- } catch {}
553
- }, 250).unref();
554
- }
555
- function waitForExit(child, timeoutMs) {
556
- if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
557
- return new Promise((resolve) => {
558
- const finish = () => {
559
- clearTimeout(timer);
560
- child.off("exit", finish);
561
- resolve();
562
- };
563
- const timer = setTimeout(finish, timeoutMs);
564
- child.once("exit", finish);
565
- });
566
- }
567
- /** Kills the shell and drops its profile. Never throws: the PNG is already in hand. */
568
- async function disposeChrome(child, userData) {
569
- try {
570
- if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
571
- await waitForExit(child, 1e3);
572
- } catch {}
573
- removeChromeProfile(userData);
574
- }
575
- /** One Chrome process, one tab. `fn` must not keep the session after it returns. */
576
- async function withChrome(bin, window, timeoutMs, fn) {
577
- const userData = mkdtempSync(join(tmpdir(), `reeval-chrome-${process.pid}-`));
578
- const child = spawn(bin, [
579
- "--headless",
580
- "--disable-gpu",
581
- "--no-sandbox",
582
- "--disable-dev-shm-usage",
583
- "--hide-scrollbars",
584
- "--remote-allow-origins=*",
585
- `--window-size=${window.width},${window.height}`,
586
- `--user-data-dir=${userData}`,
587
- "--remote-debugging-port=0",
588
- "about:blank"
589
- ], { stdio: [
590
- "ignore",
591
- "ignore",
592
- "pipe"
593
- ] });
594
- try {
595
- const cdp = await cdpConnect(await pageDebuggerUrl(await waitForPortFile(join(userData, "DevToolsActivePort"), timeoutMs), timeoutMs), timeoutMs);
596
- try {
597
- await cdp.send("Page.enable");
598
- await cdp.send("Page.setLifecycleEventsEnabled", { enabled: true });
599
- const network = trackNetwork(cdp);
600
- await cdp.send("Network.enable");
601
- await cdp.send("Emulation.setDeviceMetricsOverride", {
602
- width: window.width,
603
- height: window.height,
604
- deviceScaleFactor: 1,
605
- mobile: false
606
- });
607
- let scrollY = 0;
608
- return await fn({
609
- cdp,
610
- window,
611
- async navigate(url) {
612
- scrollY = 0;
613
- const nav = await cdp.send("Page.navigate", { url });
614
- if (nav?.errorText) throw new Error(`c.capture() navigation failed: ${nav.errorText}`);
615
- return nav;
616
- },
617
- wait(wait, waitMs, nav) {
618
- if (wait === "idle" && !nav) return network.quiet(waitMs);
619
- return waitForPage(cdp, wait, waitMs, nav);
620
- },
621
- async screenshot(fullPage) {
622
- const shot = await cdp.send("Page.captureScreenshot", {
623
- format: "png",
624
- captureBeyondViewport: fullPage
625
- });
626
- if (!shot.data) throw new Error("Page.captureScreenshot returned no data");
627
- return Buffer.from(shot.data, "base64");
628
- },
629
- async dispatch(action, page) {
630
- scrollY = await dispatchInput(cdp, action, window, page, scrollY);
631
- }
632
- });
633
- } finally {
634
- cdp.close();
349
+ }).describe("A matcher call: `expect(subject, label).not?.matcher(...args)`. Replies with its record."),
350
+ z.strictObject({
351
+ op: z.literal("record"),
352
+ matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a name such as assert").describe("What made the check in the eval's language, e.g. `assert`."),
353
+ subject: z.string().optional().describe("What it checked, as the eval wrote it (`result.ok`): names the check when it has no label."),
354
+ label: z.string().optional().describe("The check's own name, the same whether it passes or fails."),
355
+ status: z.enum(["pass", "fail"]),
356
+ message: z.string().optional().describe("Why it failed."),
357
+ soft: z.boolean().optional().describe("The body carried on after a failure, as after an `expect`. Default false: the check raised, as Python's `assert` does."),
358
+ at: callSiteSchema.optional()
359
+ }).describe("A check the eval's own language made, such as Python's `assert`: recorded as a matcher call is, so it's named, counted and compared the same. Replies with its record."),
360
+ z.strictObject({
361
+ op: z.literal("judge"),
362
+ question: nonEmpty.describe("A yes/no question about the inputs `options` turns on."),
363
+ options: judgeCallDataSchema
364
+ }).describe("`c.judge(question, options)`. Replies with its verdict, or a judge failure when the provider failed.")
365
+ ]).describe("One call an eval's body makes on its attempt. The sandbox and the agent session start on the first call that needs them; there's no call to start them.");
366
+ const attemptErrorSchema = z.looseObject({
367
+ name: z.string().describe("The error's class, e.g. `UsageError` or `AssertionError`."),
368
+ message: z.string()
369
+ }).describe("Why a call failed, in reeval's words.");
370
+ const id = nonEmpty.describe("Pairs a reply with its request.");
371
+ const attemptRequestSchema = z.strictObject({
372
+ id,
373
+ call: attemptCallSchema
374
+ }).describe("A call, numbered so its reply can find it.");
375
+ z.union([
376
+ z.discriminatedUnion("op", [
377
+ z.strictObject({
378
+ id,
379
+ ok: z.literal(true),
380
+ op: z.literal("send"),
381
+ turn: turnDataSchema
382
+ }),
383
+ z.strictObject({
384
+ id,
385
+ ok: z.literal(true),
386
+ op: z.literal("exec"),
387
+ result: z.union([execResultSchema, z.strictObject({ handle: handleId }).describe("A background process, still running.")])
388
+ }),
389
+ z.strictObject({
390
+ id,
391
+ ok: z.literal(true),
392
+ op: z.literal("serve"),
393
+ handle: handleId,
394
+ port: z.number().describe("The container port it listens on.")
395
+ }),
396
+ z.strictObject({
397
+ id,
398
+ ok: z.literal(true),
399
+ op: z.literal("kill")
400
+ }),
401
+ z.strictObject({
402
+ id,
403
+ ok: z.literal(true),
404
+ op: z.literal("capture"),
405
+ capture: captureSchema
406
+ }),
407
+ z.strictObject({
408
+ id,
409
+ ok: z.literal(true),
410
+ op: z.literal("readFile"),
411
+ contents: z.string().nullable().describe("Null when there is no such file.")
412
+ }),
413
+ z.strictObject({
414
+ id,
415
+ ok: z.literal(true),
416
+ op: z.literal("writeFile")
417
+ }),
418
+ z.strictObject({
419
+ id,
420
+ ok: z.literal(true),
421
+ op: z.literal("expect"),
422
+ assertion: assertionRecordSchema
423
+ }),
424
+ z.strictObject({
425
+ id,
426
+ ok: z.literal(true),
427
+ op: z.literal("record"),
428
+ assertion: assertionRecordSchema
429
+ }),
430
+ z.strictObject({
431
+ id,
432
+ ok: z.literal(true),
433
+ op: z.literal("judge"),
434
+ verdict: z.union([verdictSchema, judgeFailureSchema])
435
+ })
436
+ ]),
437
+ z.strictObject({
438
+ id,
439
+ ok: z.literal(true),
440
+ op: z.literal("expect"),
441
+ untested: z.record(z.string(), z.array(z.string())).describe("The strings the matcher tested that a `{ matchedBy, answers }` argument has no answer for, by its `matchedBy`. Nothing was recorded: answer them and make the call again.")
442
+ }),
443
+ z.strictObject({
444
+ id,
445
+ ok: z.literal(false),
446
+ op: z.enum([
447
+ "send",
448
+ "exec",
449
+ "readFile",
450
+ "writeFile",
451
+ "expect",
452
+ "record",
453
+ "judge",
454
+ "serve",
455
+ "kill",
456
+ "capture"
457
+ ]),
458
+ error: attemptErrorSchema
459
+ })
460
+ ]).describe("reeval's reply to a call: its result, or the error it threw. A failed matcher is a result (its record says `fail`), not an error, unless the matcher couldn't run.");
461
+ const cellRoleSchema = z.enum(["control", "variant"]);
462
+ const cellSchema = z.strictObject({
463
+ key: z.string(),
464
+ harness: z.string(),
465
+ environment: z.string(),
466
+ sandbox: z.string(),
467
+ role: cellRoleSchema,
468
+ variant: z.string().optional()
469
+ });
470
+ z.strictObject({
471
+ type: z.literal("start"),
472
+ attempt: z.strictObject({
473
+ id: nonEmpty,
474
+ cell: cellSchema,
475
+ n: z.number().int().min(1, "a run number, from 1").describe("The run number, from 1.")
476
+ }).describe("`c.attempt`: which cell of the matrix this attempt is, and its run number."),
477
+ file: nonEmpty.describe("The eval file, relative to the project."),
478
+ body: nonEmpty.describe("The eval's `body`, as its declaration named it.")
479
+ }).describe("reeval to the runner: run this eval's body for this attempt. Nothing has started yet: the sandbox and the agent session start on the body's first call that needs them, so its own code runs first. The runner sends its calls, then `done`.");
480
+ z.strictObject({
481
+ type: z.literal("abort"),
482
+ reason: z.enum(["timeout", "canceled"]).describe("`timeout`: the eval's `timeout` ran out. `canceled`: the run was stopped."),
483
+ message: z.string().describe("reeval's sentence, e.g. `Eval timed out after 600s`.")
484
+ }).describe("reeval to the runner: the attempt was aborted, as `c.signal` aborts. Calls in flight reply with errors; the runner should stop the body and send `done`.");
485
+ z.strictObject({
486
+ type: z.literal("done"),
487
+ error: attemptErrorSchema.optional().describe("What the body threw, if it threw. A failed `expect.require` is one.")
488
+ }).describe("The runner to reeval: the eval's body has finished. The attempt's checks and record follow.");
489
+ const CALL_KEYS = Object.fromEntries(attemptCallSchema.options.map((o) => [o.shape.op.value, Object.keys(o.shape)]));
490
+ const OTHER_KEYS = [
491
+ "id",
492
+ "call",
493
+ "turn",
494
+ "attempt",
495
+ "value",
496
+ "regexp",
497
+ "flags",
498
+ "matchedBy",
499
+ "answers",
500
+ ...Object.keys(judgeCallDataSchema.shape),
501
+ ...Object.keys(captureSchema.shape),
502
+ ...Object.keys(captureCallDataSchema.shape),
503
+ "port",
504
+ "handle",
505
+ "exec",
506
+ "result",
507
+ "readFile",
508
+ "interval",
509
+ "timeout",
510
+ "ms"
511
+ ];
512
+ /** What's wrong with an attempt request a runner sent, by the field's path, or undefined. */
513
+ const attemptRequestProblem = (value) => dataProblem(attemptRequestSchema, value, "The attempt request", (object) => typeof object.op === "string" ? CALL_KEYS[object.op] ?? [] : OTHER_KEYS);
514
+ //#endregion
515
+ //#region src/capture/fake-instructions.ts
516
+ const FAKE_INSTRUCTIONS = Symbol.for("reeval.capture.fakeInstructions");
517
+ function isWaitStep(step) {
518
+ return typeof step === "object" && step !== null && !Array.isArray(step) && "wait" in step;
519
+ }
520
+ /** One list: action lists and `{ wait }` inlined. Returns branded `instructions`. */
521
+ function fakeInstructions(steps) {
522
+ if (!Array.isArray(steps)) throw new Error(`fakeInstructions() is an array of action lists and { wait } (got ${JSON.stringify(steps)})`);
523
+ const instructions = [];
524
+ let goals = 0;
525
+ for (const [i, step] of steps.entries()) {
526
+ if (isWaitStep(step)) {
527
+ instructions.push({ wait: step.wait });
528
+ continue;
635
529
  }
636
- } finally {
637
- await disposeChrome(child, userData);
530
+ if (!Array.isArray(step)) throw new Error(`fakeInstructions()[${i}] is an action list or { wait } (got ${JSON.stringify(step)})`);
531
+ instructions.push(`fake[${goals}]`);
532
+ goals++;
638
533
  }
534
+ Object.defineProperty(instructions, FAKE_INSTRUCTIONS, {
535
+ value: steps,
536
+ enumerable: false
537
+ });
538
+ return instructions;
639
539
  }
640
- /** Arm the wait, then navigate, then wait. `arm` is before `Page.navigate`. */
641
- async function navigateAndWait(session, url, wait, timeoutMs) {
642
- if (isMs(wait)) {
643
- const nav = await session.navigate(url);
644
- await session.wait(wait, timeoutMs, nav);
645
- return;
646
- }
647
- const page = listenForCaptureWait(session.cdp, wait);
648
- page.arm();
649
- const nav = await session.navigate(url);
650
- await page.ready(nav, timeoutMs);
540
+ function fakeInstructionSteps(value) {
541
+ if (typeof value === "object" && value !== null && FAKE_INSTRUCTIONS in value) return value[FAKE_INSTRUCTIONS];
651
542
  }
652
543
  //#endregion
653
- //#region src/capture/poll.ts
654
- /** Any HTTP response, including 404, means something accepted the connection. */
655
- async function pollHttp(url, timeoutMs, signal) {
656
- const deadline = Date.now() + timeoutMs;
657
- let last;
658
- while (Date.now() < deadline) {
659
- signal?.throwIfAborted();
660
- const left = Math.max(1, deadline - Date.now());
661
- try {
662
- await fetch(url, { signal: AbortSignal.any([AbortSignal.timeout(Math.min(1e3, left)), ...signal ? [signal] : []]) });
663
- return;
664
- } catch (error) {
665
- last = error;
666
- if (signal?.aborted) throw error;
667
- await new Promise((r) => setTimeout(r, 100));
544
+ //#region src/capture/zip.ts
545
+ const LOCAL = 67324752;
546
+ /** Extracts a zip (store or deflate) into `files` keyed by posix path. */
547
+ function unzip(bytes) {
548
+ const buf = Buffer.from(bytes);
549
+ const files = /* @__PURE__ */ new Map();
550
+ let i = 0;
551
+ while (i + 30 <= buf.length) {
552
+ if (buf.readUInt32LE(i) !== LOCAL) break;
553
+ const method = buf.readUInt16LE(i + 8);
554
+ const comp = buf.readUInt32LE(i + 18);
555
+ const nameLen = buf.readUInt16LE(i + 26);
556
+ const extra = buf.readUInt16LE(i + 28);
557
+ const name = buf.toString("utf8", i + 30, i + 30 + nameLen);
558
+ const start = i + 30 + nameLen + extra;
559
+ const raw = buf.subarray(start, start + comp);
560
+ if (name.endsWith("/")) {
561
+ i = start + comp;
562
+ continue;
668
563
  }
564
+ if (method === 0) files.set(name, Buffer.from(raw));
565
+ else if (method === 8) files.set(name, Buffer.from(inflateRawSync(raw)));
566
+ else throw new Error(`zip method ${method} is not supported (${name})`);
567
+ i = start + comp;
669
568
  }
670
- throw last instanceof Error ? last : /* @__PURE__ */ new Error("poll timed out");
569
+ return files;
671
570
  }
672
571
  //#endregion
673
- //#region src/capture/stub.ts
674
- const PNG_SIG = Buffer.from([
675
- 137,
676
- 80,
677
- 78,
678
- 71,
679
- 13,
680
- 10,
681
- 26,
682
- 10
683
- ]);
684
- function chunk(type, data) {
685
- const body = Buffer.concat([Buffer.from(type, "ascii"), data]);
686
- const out = Buffer.alloc(12 + data.length);
687
- out.writeUInt32BE(data.length, 0);
688
- body.copy(out, 4);
689
- out.writeUInt32BE(crc32(body), 8 + data.length);
690
- return out;
572
+ //#region src/capture/browser.ts
573
+ const isMs = (wait) => typeof wait === "object" && wait !== null;
574
+ /** The CDP event `'idle'` waits for; `'load'` is `Page.loadEventFired`. */
575
+ function captureWaitLabel(wait) {
576
+ return wait === "idle" ? "networkAlmostIdle" : "load";
691
577
  }
692
- /** A valid 1×1 PNG whose bytes are unique for `(name, port, path, viewport)`. */
693
- function stubPng(name, port, path, viewport) {
694
- const digest = createHash("sha256").update(`${name}\0${port}\0${path}\0${viewport === "full-page" ? "full-page" : `${viewport.width}x${viewport.height}`}`).digest();
695
- const ihdr = Buffer.alloc(13);
696
- ihdr.writeUInt32BE(1, 0);
697
- ihdr.writeUInt32BE(1, 4);
698
- ihdr[8] = 8;
699
- ihdr[9] = 2;
700
- const scanline = Buffer.from([
701
- 0,
702
- digest[0] ?? 0,
703
- digest[1] ?? 0,
704
- digest[2] ?? 0
705
- ]);
706
- const idat = deflateSync(scanline);
707
- const text = Buffer.concat([Buffer.from("Comment\0", "ascii"), Buffer.from(digest.toString("hex"), "ascii")]);
708
- return Buffer.concat([
709
- PNG_SIG,
710
- chunk("IHDR", ihdr),
711
- chunk("tEXt", text),
712
- chunk("IDAT", idat),
713
- chunk("IEND", Buffer.alloc(0))
714
- ]);
578
+ function captureWaitTimeout(wait) {
579
+ return /* @__PURE__ */ new Error(`c.capture() timed out waiting for ${captureWaitLabel(wait)}`);
715
580
  }
716
- /** PNG IHDR width and height; throws if the bytes are not a PNG. */
717
- function pngSize(bytes) {
718
- if (bytes.length < 24 || Buffer.from(bytes.subarray(0, 8)).compare(PNG_SIG) !== 0) throw new Error("not a PNG");
719
- const view = Buffer.from(bytes.subarray(16, 24));
581
+ function waitTimeout(wait, detail) {
582
+ if (isMs(wait)) return /* @__PURE__ */ new Error(`c.capture() timed out${detail ? ` on ${detail}` : ""} waiting ${wait.ms}ms`);
583
+ return captureWaitTimeout(wait);
584
+ }
585
+ async function waitForPage(cdp, wait, timeoutMs, nav) {
586
+ if (isMs(wait)) {
587
+ if (timeoutMs <= 0) throw waitTimeout(wait);
588
+ const ms = Math.min(wait.ms, timeoutMs);
589
+ await new Promise((resolve) => setTimeout(resolve, ms));
590
+ return;
591
+ }
592
+ const page = listenForCaptureWait(cdp, wait);
593
+ page.arm();
594
+ await page.ready(nav, timeoutMs);
595
+ }
596
+ const clamp = (n, lo, hi) => Math.min(hi, Math.max(lo, n));
597
+ /** Map a full-page click onto the live window, scrolling with the wheel if needed. */
598
+ function mapPagePoint(x, y, window, page, scrollY) {
599
+ const maxY = Math.max(0, (page?.height ?? window.height) - window.height);
600
+ const target = y >= window.height || y < 0 ? clamp(y - window.height / 2, 0, maxY) : scrollY;
601
+ const nextY = clamp(target, 0, maxY);
720
602
  return {
721
- width: view.readUInt32BE(0),
722
- height: view.readUInt32BE(4)
603
+ x: clamp(x, 0, window.width - 1),
604
+ y: clamp(y - nextY, 0, window.height - 1),
605
+ scrollY: nextY,
606
+ dy: nextY - scrollY
723
607
  };
724
608
  }
725
- //#endregion
726
- //#region src/capture/index.ts
727
- /** Default CSS pixels when the call and cascade omit `viewport`. */
728
- const DEFAULT_VIEWPORT = {
729
- width: 1280,
730
- height: 720
609
+ const KEYS = {
610
+ Tab: {
611
+ key: "Tab",
612
+ code: "Tab",
613
+ windowsVirtualKeyCode: 9
614
+ },
615
+ Enter: {
616
+ key: "Enter",
617
+ code: "Enter",
618
+ windowsVirtualKeyCode: 13,
619
+ text: "\r"
620
+ },
621
+ Escape: {
622
+ key: "Escape",
623
+ code: "Escape",
624
+ windowsVirtualKeyCode: 27
625
+ },
626
+ Backspace: {
627
+ key: "Backspace",
628
+ code: "Backspace",
629
+ windowsVirtualKeyCode: 8
630
+ },
631
+ Space: {
632
+ key: " ",
633
+ code: "Space",
634
+ windowsVirtualKeyCode: 32,
635
+ text: " "
636
+ },
637
+ ArrowUp: {
638
+ key: "ArrowUp",
639
+ code: "ArrowUp",
640
+ windowsVirtualKeyCode: 38
641
+ },
642
+ ArrowDown: {
643
+ key: "ArrowDown",
644
+ code: "ArrowDown",
645
+ windowsVirtualKeyCode: 40
646
+ },
647
+ ArrowLeft: {
648
+ key: "ArrowLeft",
649
+ code: "ArrowLeft",
650
+ windowsVirtualKeyCode: 37
651
+ },
652
+ ArrowRight: {
653
+ key: "ArrowRight",
654
+ code: "ArrowRight",
655
+ windowsVirtualKeyCode: 39
656
+ }
731
657
  };
732
- const CAPTURE_NAME = /^[a-z0-9_-]+$/;
733
- const CAPTURE_WAITS = ["load", "idle"];
734
- /** `'load'` when the call omits `wait`. `'load'`, `'idle'` or `{ ms }`. */
735
- function resolveWait(wait) {
736
- if (wait === void 0) return "load";
737
- if (isOneOf(CAPTURE_WAITS, wait)) return wait;
738
- if (typeof wait === "object" && wait !== null && Number.isFinite(wait.ms) && wait.ms >= 0 && Object.keys(wait).every((k) => k === "ms")) return { ms: wait.ms };
739
- throw new Error(`c.capture() wait is 'load', 'idle' or { ms } (got ${JSON.stringify(wait)})`);
658
+ /** `CaptureKey` names, for the driver's tool schema. */
659
+ const CAPTURE_KEYS = Object.keys(KEYS);
660
+ async function pressKey(cdp, event) {
661
+ await cdp.send("Input.dispatchKeyEvent", {
662
+ type: "keyDown",
663
+ ...event
664
+ });
665
+ const { text: _text, commands: _commands, ...up } = event;
666
+ await cdp.send("Input.dispatchKeyEvent", {
667
+ type: "keyUp",
668
+ ...up
669
+ });
740
670
  }
741
- function parseInstructions(value) {
742
- if (value === void 0) return [];
743
- if (!Array.isArray(value)) throw new Error(`c.capture() instructions is an array of strings and { wait } (got ${JSON.stringify(value)})`);
744
- return value.map((step, i) => {
745
- if (typeof step === "string") {
746
- if (!step.trim()) throw new Error(`c.capture() instructions[${i}] is an empty string`);
747
- return step;
671
+ async function dispatchInput(cdp, action, window, page, scrollY = 0) {
672
+ if (action.type === "done") return scrollY;
673
+ if (action.type === "type") {
674
+ if (action.replace) {
675
+ await pressKey(cdp, {
676
+ key: "a",
677
+ code: "KeyA",
678
+ windowsVirtualKeyCode: 65,
679
+ modifiers: 2,
680
+ commands: ["selectAll"]
681
+ });
682
+ if (!action.text) await pressKey(cdp, KEYS.Backspace);
748
683
  }
749
- if (typeof step === "object" && step !== null && "wait" in step) return { wait: resolveWait(step.wait) };
750
- throw new Error(`c.capture() instructions[${i}] is a string or { wait } (got ${JSON.stringify(step)})`);
684
+ if (action.text) await cdp.send("Input.insertText", { text: action.text });
685
+ return scrollY;
686
+ }
687
+ if (action.type === "key") {
688
+ const event = KEYS[action.key];
689
+ if (!event) throw new Error(`c.capture() can't press ${JSON.stringify(action.key)}: use one of ${CAPTURE_KEYS.join(", ")}`);
690
+ await pressKey(cdp, event);
691
+ return scrollY;
692
+ }
693
+ if (action.type === "scroll") {
694
+ await cdp.send("Input.dispatchMouseEvent", {
695
+ type: "mouseWheel",
696
+ x: action.x,
697
+ y: action.y,
698
+ deltaX: 0,
699
+ deltaY: action.dy
700
+ });
701
+ return scrollY + action.dy;
702
+ }
703
+ const mapped = mapPagePoint(action.type === "drag" ? action.x : action.x, action.type === "drag" ? action.y : action.y, window, page, scrollY);
704
+ if (mapped.dy !== 0) await cdp.send("Input.dispatchMouseEvent", {
705
+ type: "mouseWheel",
706
+ x: window.width / 2,
707
+ y: window.height / 2,
708
+ deltaX: 0,
709
+ deltaY: mapped.dy
710
+ });
711
+ const at = {
712
+ x: mapped.x,
713
+ y: mapped.y
714
+ };
715
+ if (action.type === "hover") {
716
+ await cdp.send("Input.dispatchMouseEvent", {
717
+ type: "mouseMoved",
718
+ ...at
719
+ });
720
+ return mapped.scrollY;
721
+ }
722
+ if (action.type === "click") {
723
+ await cdp.send("Input.dispatchMouseEvent", {
724
+ type: "mouseMoved",
725
+ ...at,
726
+ button: "left"
727
+ });
728
+ await cdp.send("Input.dispatchMouseEvent", {
729
+ type: "mousePressed",
730
+ ...at,
731
+ button: "left",
732
+ clickCount: 1
733
+ });
734
+ await cdp.send("Input.dispatchMouseEvent", {
735
+ type: "mouseReleased",
736
+ ...at,
737
+ button: "left",
738
+ clickCount: 1
739
+ });
740
+ return mapped.scrollY;
741
+ }
742
+ const end = mapPagePoint(action.toX, action.toY, window, page, mapped.scrollY);
743
+ if (end.dy !== 0) await cdp.send("Input.dispatchMouseEvent", {
744
+ type: "mouseWheel",
745
+ x: window.width / 2,
746
+ y: window.height / 2,
747
+ deltaX: 0,
748
+ deltaY: end.dy
749
+ });
750
+ await cdp.send("Input.dispatchMouseEvent", {
751
+ type: "mouseMoved",
752
+ x: at.x,
753
+ y: at.y,
754
+ button: "left"
755
+ });
756
+ await cdp.send("Input.dispatchMouseEvent", {
757
+ type: "mousePressed",
758
+ x: at.x,
759
+ y: at.y,
760
+ button: "left",
761
+ clickCount: 1
762
+ });
763
+ await cdp.send("Input.dispatchMouseEvent", {
764
+ type: "mouseMoved",
765
+ x: end.x,
766
+ y: end.y,
767
+ button: "left"
768
+ });
769
+ await cdp.send("Input.dispatchMouseEvent", {
770
+ type: "mouseReleased",
771
+ x: end.x,
772
+ y: end.y,
773
+ button: "left",
774
+ clickCount: 1
751
775
  });
776
+ return end.scrollY;
752
777
  }
753
- function instructionGoals(steps) {
754
- return steps.filter((s) => typeof s === "string");
778
+ function lifecycleParams(msg) {
779
+ const body = msg;
780
+ return body.params ?? body;
755
781
  }
756
- /** Live window and whether the saved PNG uses `captureBeyondViewport`. */
757
- function resolveCaptureViewport(viewport) {
758
- const asked = viewport ?? DEFAULT_VIEWPORT;
759
- if (asked === "full-page") return {
760
- viewport: "full-page",
761
- window: DEFAULT_VIEWPORT,
762
- fullPage: true
782
+ /**
783
+ * Arms after `Page.enable` so about:blank's events cannot resolve the wait.
784
+ * Call `arm` immediately before `Page.navigate`.
785
+ */
786
+ function listenForCaptureWait(cdp, wait) {
787
+ const method = wait === "idle" ? "Page.lifecycleEvent" : "Page.loadEventFired";
788
+ const pending = [];
789
+ let deliver;
790
+ let started = false;
791
+ cdp.on(method, (msg) => {
792
+ if (!started) return;
793
+ if (deliver) deliver(msg);
794
+ else pending.push(msg);
795
+ });
796
+ const matches = (msg, loaderId) => {
797
+ if (wait === "load") return true;
798
+ const params = lifecycleParams(msg);
799
+ if (params.name !== "networkAlmostIdle") return false;
800
+ return !loaderId || params.loaderId === loaderId;
763
801
  };
764
802
  return {
765
- viewport: asked,
766
- window: asked,
767
- fullPage: false
803
+ arm() {
804
+ started = true;
805
+ },
806
+ ready(nav, timeoutMs) {
807
+ return new Promise((resolve, reject) => {
808
+ const timer = setTimeout(() => reject(captureWaitTimeout(wait)), timeoutMs);
809
+ const finish = () => {
810
+ clearTimeout(timer);
811
+ resolve();
812
+ };
813
+ deliver = (msg) => {
814
+ if (matches(msg, nav?.loaderId)) finish();
815
+ };
816
+ if (pending.some((msg) => matches(msg, nav?.loaderId))) finish();
817
+ });
818
+ }
768
819
  };
769
820
  }
770
- /** `~/.cache/reeval` when that's the resolved dir; otherwise the absolute path. */
771
- function cacheLabel(dir = cacheDir()) {
772
- const home = homedir();
773
- if (dir.replaceAll("\\", "/") === join(home, ".cache", "reeval").replaceAll("\\", "/")) return "~/.cache/reeval";
774
- return dir;
821
+ const QUIET_IN_FLIGHT = 2;
822
+ const FRESH_REQUEST_MS = 2e3;
823
+ /** How long a screenshot waits, in tries, for a page that has no layout yet. */
824
+ const SCREENSHOT_TRIES = 20;
825
+ const SCREENSHOT_RETRY_MS = 100;
826
+ function requestId(msg) {
827
+ const body = msg;
828
+ return (body.params ?? body).requestId;
775
829
  }
776
830
  /**
777
- * Env is named first when both refuse. `undefined` means the host binary is allowed.
831
+ * Counts the tab's requests from `Network.enable` on, for a journey's `{ wait: 'idle' }` beat.
832
+ * Chromium emits `networkAlmostIdle` once per document, so after a click that only fetches it
833
+ * never comes again. As in Chromium, every request that starts or ends restarts the quiet window.
778
834
  */
779
- function refusedBrowser(capture) {
780
- if (process.env.REEVAL_CAPTURE_BROWSER === "0") return "c.capture() is refused: REEVAL_CAPTURE_BROWSER=0";
781
- if (capture.browser === false) return "c.capture() is refused: capture.browser is false";
835
+ function trackNetwork(cdp, quietMs = 500) {
836
+ const inFlight = /* @__PURE__ */ new Map();
837
+ let changed;
838
+ cdp.on("Network.requestWillBeSent", (msg) => {
839
+ const id = requestId(msg);
840
+ if (id === void 0) return;
841
+ inFlight.set(id, Date.now());
842
+ changed?.();
843
+ });
844
+ const end = (msg) => {
845
+ const id = requestId(msg);
846
+ if (id === void 0 || !inFlight.delete(id)) return;
847
+ changed?.();
848
+ };
849
+ cdp.on("Network.loadingFinished", end);
850
+ cdp.on("Network.loadingFailed", end);
851
+ return {
852
+ /** A request started in the last `FRESH_REQUEST_MS` is still in flight; a long poll is not fresh. */
853
+ fresh() {
854
+ const since = Date.now() - FRESH_REQUEST_MS;
855
+ for (const started of inFlight.values()) if (started >= since) return true;
856
+ return false;
857
+ },
858
+ quiet(timeoutMs) {
859
+ return new Promise((resolve, reject) => {
860
+ let settle;
861
+ const stop = () => {
862
+ clearTimeout(timer);
863
+ clearTimeout(settle);
864
+ changed = void 0;
865
+ };
866
+ const timer = setTimeout(() => {
867
+ stop();
868
+ reject(captureWaitTimeout("idle"));
869
+ }, timeoutMs);
870
+ changed = () => {
871
+ clearTimeout(settle);
872
+ if (inFlight.size > QUIET_IN_FLIGHT) return;
873
+ settle = setTimeout(() => {
874
+ stop();
875
+ resolve();
876
+ }, quietMs);
877
+ };
878
+ changed();
879
+ });
880
+ }
881
+ };
782
882
  }
783
- function resolveCaptureTarget(target) {
784
- if (typeof target === "number") {
785
- if (!(Number.isInteger(target) && target >= 1 && target <= 65535)) throw new Error(`c.capture() port is a container port from 1 to 65535 (got ${String(target)})`);
786
- return target;
883
+ /** Pinned Chrome for Testing `chrome-headless-shell` build. */
884
+ const CHROME_VERSION = "140.0.7339.185";
885
+ const CHROME_NAME = "chrome-headless-shell";
886
+ function chromePlatform() {
887
+ if (process.platform === "darwin") return process.arch === "arm64" ? "mac-arm64" : "mac-x64";
888
+ if (process.platform === "win32") return "win64";
889
+ return "linux64";
890
+ }
891
+ function chromeUrl(platform) {
892
+ return `https://storage.googleapis.com/chrome-for-testing-public/${CHROME_VERSION}/${platform}/${CHROME_NAME}-${platform}.zip`;
893
+ }
894
+ function binaryName() {
895
+ return process.platform === "win32" ? `${CHROME_NAME}.exe` : CHROME_NAME;
896
+ }
897
+ async function downloadZip(url, onProgress, cacheLabel, signal) {
898
+ const res = await fetch(url, { signal });
899
+ if (!res.ok) throw new Error(`download failed: HTTP ${res.status}`);
900
+ const total = Number(res.headers.get("content-length")) || null;
901
+ const report = (received) => onProgress?.({
902
+ name: CHROME_NAME,
903
+ version: CHROME_VERSION,
904
+ bytes: total,
905
+ received,
906
+ cache: cacheLabel
907
+ });
908
+ report(0);
909
+ if (!res.body) return Buffer.from(await res.arrayBuffer());
910
+ const reader = res.body.getReader();
911
+ const chunks = [];
912
+ let received = 0;
913
+ let last = 0;
914
+ for (;;) {
915
+ const { done, value } = await reader.read();
916
+ if (done) break;
917
+ chunks.push(value);
918
+ received += value.length;
919
+ if (received - last >= 262144 || total && received === total) {
920
+ report(received);
921
+ last = received;
922
+ }
787
923
  }
788
- if (target !== null && typeof target === "object" && typeof target.port === "number" && typeof target.kill === "function") return resolveCaptureTarget(target.port);
789
- throw new Error("c.capture() needs a port or a ServeHandle as the first argument");
924
+ if (received !== last) report(received);
925
+ return Buffer.concat(chunks);
790
926
  }
791
- /** Artifact name from a path: `/login` → `login`, `/foo/bar` → `foo-bar`, `/` → `root`. */
792
- function nameFromPath(path) {
793
- return (path.split(/[?#]/, 1)[0] ?? "").replaceAll("\\", "/").split("/").filter(Boolean).join("-").toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "") || "root";
927
+ function extractBinary(zip, dest, platform) {
928
+ const files = unzip(zip);
929
+ const needle = `/${binaryName()}`;
930
+ const entry = [...files.entries()].find(([name]) => name.endsWith(needle) || name === binaryName() || name.endsWith(`\\${binaryName()}`));
931
+ if (!entry) {
932
+ const names = [...files.keys()].slice(0, 8).join(", ");
933
+ throw new Error(`zip has no ${binaryName()} (saw ${names || "nothing"})`);
934
+ }
935
+ const prefix = entry[0].slice(0, -binaryName().length);
936
+ mkdirSync(dest, { recursive: true });
937
+ for (const [name, data] of files) {
938
+ const rel = prefix && name.startsWith(prefix) ? name.slice(prefix.length) : name;
939
+ if (!rel || rel.endsWith("/")) continue;
940
+ const target = join(dest, ...rel.split("/"));
941
+ mkdirSync(join(target, ".."), { recursive: true });
942
+ writeFileSync(target, data);
943
+ }
944
+ const bin = join(dest, binaryName());
945
+ if (!existsSync(bin)) writeFileSync(bin, entry[1]);
946
+ if (process.platform !== "win32") chmodSync(bin, 493);
794
947
  }
795
- function resolveCaptureCall(target, call, cascade) {
796
- if (call && Object.hasOwn(call, "port")) throw new Error("c.capture() takes the port as the first argument, not in the options");
797
- if (call && Object.hasOwn(call, "use")) throw new Error("c.capture() has no use. Set model, or pass fakeInstructions from reeval/testing as instructions.");
798
- if (call && Object.hasOwn(call, "fake")) throw new Error("c.capture() has no fake. Tests pass fakeInstructions(...) as instructions.");
799
- if (call?.model !== void 0) checkCallModel(call.model);
800
- const port = resolveCaptureTarget(target);
801
- const path = call?.path || "/";
802
- const name = call?.name ?? nameFromPath(path);
803
- checkCaptureName(name);
804
- const asked = resolveCaptureViewport(call?.viewport ?? cascade.viewport);
805
- return {
806
- port,
807
- name,
808
- path,
809
- viewport: asked.viewport,
810
- window: asked.window,
811
- fullPage: asked.fullPage,
812
- timeoutMs: parseDuration(call?.timeout ?? cascade.timeout ?? "60s"),
813
- wait: resolveWait(call?.wait),
814
- instructions: parseInstructions(call?.instructions),
815
- model: call?.model ?? cascade.model,
816
- fake: fakeScriptFrom(call?.instructions)
817
- };
948
+ /**
949
+ * Returns the cached binary, downloading on a miss. `onProgress` runs only for a download.
950
+ */
951
+ async function ensureChrome(cacheRoot, options = { cacheLabel: cacheRoot }) {
952
+ const platform = chromePlatform();
953
+ const dest = join(cacheRoot, CHROME_NAME, CHROME_VERSION, platform);
954
+ const bin = join(dest, binaryName());
955
+ if (existsSync(bin)) return bin;
956
+ mkdirSync(join(cacheRoot, CHROME_NAME, CHROME_VERSION), { recursive: true });
957
+ const partial = `${dest}.${process.pid}.partial`;
958
+ rmSync(partial, {
959
+ recursive: true,
960
+ force: true
961
+ });
962
+ try {
963
+ extractBinary(await downloadZip(chromeUrl(platform), options.onProgress, options.cacheLabel, options.signal), partial, platform);
964
+ try {
965
+ renameSync(partial, dest);
966
+ } catch (error) {
967
+ rmSync(partial, {
968
+ recursive: true,
969
+ force: true
970
+ });
971
+ if (!existsSync(bin)) throw error;
972
+ }
973
+ } catch (error) {
974
+ rmSync(partial, {
975
+ recursive: true,
976
+ force: true
977
+ });
978
+ if (existsSync(bin)) return bin;
979
+ throw error;
980
+ }
981
+ return bin;
818
982
  }
819
- function fakeScriptFrom(value) {
820
- const steps = fakeInstructionSteps(value);
821
- if (!steps) return void 0;
822
- const script = [];
823
- for (const step of steps) {
824
- if (isWaitStep(step)) continue;
825
- script.push(step);
983
+ var Cdp = class {
984
+ #ws;
985
+ #next = 0;
986
+ #pending = /* @__PURE__ */ new Map();
987
+ #events = /* @__PURE__ */ new Map();
988
+ constructor(ws) {
989
+ this.#ws = ws;
990
+ ws.addEventListener("message", (ev) => {
991
+ const msg = JSON.parse(String(ev.data));
992
+ if (msg.id !== void 0) {
993
+ const p = this.#pending.get(msg.id);
994
+ if (!p) return;
995
+ this.#pending.delete(msg.id);
996
+ if (msg.error) p.reject(new Error(msg.error.message));
997
+ else p.resolve(msg.result);
998
+ return;
999
+ }
1000
+ if (msg.method) for (const fn of this.#events.get(msg.method) ?? []) fn(msg);
1001
+ });
826
1002
  }
827
- return script;
828
- }
829
- function checkCallModel(model) {
830
- if (!model.includes("/")) throw new Error(`c.capture() model is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${JSON.stringify(model)})`);
831
- if (model.startsWith("typesafe-ai/")) throw new Error(`c.capture() model "${model}" is an evaluation model: it scores yes/no questions and can't drive a capture. Use a language model that accepts images, e.g. 'google/gemini-3.8-flash'.`);
832
- }
833
- function checkCaptureName(name) {
834
- if (!CAPTURE_NAME.test(name)) throw new Error(`c.capture() name must match [a-z0-9_-]+ (got ${JSON.stringify(name)})`);
835
- }
836
- /** A free host port. Closed before the promise resolves, so the caller can bind it. */
837
- function allocatePort() {
1003
+ on(method, fn) {
1004
+ const list = this.#events.get(method) ?? [];
1005
+ list.push(fn);
1006
+ this.#events.set(method, list);
1007
+ }
1008
+ send(method, params) {
1009
+ const id = ++this.#next;
1010
+ this.#ws.send(JSON.stringify({
1011
+ id,
1012
+ method,
1013
+ params
1014
+ }));
1015
+ return new Promise((resolve, reject) => this.#pending.set(id, {
1016
+ resolve,
1017
+ reject
1018
+ }));
1019
+ }
1020
+ close() {
1021
+ this.#ws.close();
1022
+ }
1023
+ };
1024
+ function waitForPortFile(file, timeoutMs) {
1025
+ const deadline = Date.now() + timeoutMs;
838
1026
  return new Promise((resolve, reject) => {
839
- const server = createServer();
840
- server.once("error", reject);
841
- server.listen(0, "127.0.0.1", () => {
842
- const addr = server.address();
843
- const port = typeof addr === "object" && addr ? addr.port : 0;
844
- server.close((error) => error ? reject(error) : resolve(port));
845
- });
1027
+ const tick = () => {
1028
+ if (existsSync(file)) {
1029
+ const line = readFileSync(file, "utf8").split(/\r?\n/)[0];
1030
+ const port = Number(line);
1031
+ if (port > 0) {
1032
+ resolve(port);
1033
+ return;
1034
+ }
1035
+ }
1036
+ if (Date.now() > deadline) {
1037
+ reject(/* @__PURE__ */ new Error("chrome-headless-shell did not open a DevTools port"));
1038
+ return;
1039
+ }
1040
+ setTimeout(tick, 50);
1041
+ };
1042
+ tick();
846
1043
  });
847
1044
  }
848
- /** Origin plus a path (`/login`), without a doubled slash. */
849
- function pageUrl(origin, path) {
850
- const base = origin.endsWith("/") ? origin : `${origin}/`;
851
- const rel = path.startsWith("/") ? path.slice(1) : path;
852
- return new URL(rel, base).href;
853
- }
854
- /** Internal settle between moves; not an authored wait. */
855
- const SETTLE_MS = 80;
856
- function remainingOrThrow(remaining, detail) {
857
- const left = remaining();
858
- if (left <= 0) throw new Error(`c.capture() timed out on ${detail}`);
859
- return left;
860
- }
861
- async function settle() {
862
- await new Promise((resolve) => setTimeout(resolve, SETTLE_MS));
1045
+ function pickPageDebuggerUrl(targets) {
1046
+ return targets.find((t) => t.webSocketDebuggerUrl && t.type !== "browser")?.webSocketDebuggerUrl;
863
1047
  }
864
- function asUsage(value) {
865
- return {
866
- input: typeof value.inputTokens === "number" ? value.inputTokens : value.inputTokens?.total ?? 0,
867
- output: typeof value.outputTokens === "number" ? value.outputTokens : value.outputTokens?.total ?? 0,
868
- cacheRead: 0,
869
- cacheWrite: 0,
870
- reasoning: null
1048
+ async function pageDebuggerUrl(port, timeoutMs) {
1049
+ const deadline = Date.now() + timeoutMs;
1050
+ const listPages = async () => {
1051
+ const res = await fetch(`http://127.0.0.1:${port}/json/list`);
1052
+ if (!res.ok) return void 0;
1053
+ return pickPageDebuggerUrl(await res.json());
871
1054
  };
1055
+ for (;;) {
1056
+ const url = await listPages();
1057
+ if (url) return url;
1058
+ if (Date.now() > deadline) break;
1059
+ await new Promise((r) => setTimeout(r, 50));
1060
+ }
1061
+ const version = await (await fetch(`http://127.0.0.1:${port}/json/version`)).json();
1062
+ if (!version.webSocketDebuggerUrl) throw new Error("chrome-headless-shell reported no DevTools URL");
1063
+ const browser = await cdpConnect(version.webSocketDebuggerUrl, timeoutMs);
1064
+ try {
1065
+ await browser.send("Target.createTarget", { url: "about:blank" });
1066
+ } finally {
1067
+ browser.close();
1068
+ }
1069
+ const created = await listPages();
1070
+ if (!created) throw new Error("chrome-headless-shell reported no page target");
1071
+ return created;
872
1072
  }
873
- function actionFromTool(name, input) {
874
- if (name === "done") return { type: "done" };
875
- if (name === "type") return {
876
- type: "type",
877
- text: String(input.text ?? "")
878
- };
879
- if (name === "key") return {
880
- type: "key",
881
- key: input.key === "Tab" ? "Tab" : "Enter"
882
- };
883
- if (name === "scroll") return {
884
- type: "scroll",
885
- x: Number(input.x),
886
- y: Number(input.y),
887
- dy: Number(input.dy)
888
- };
889
- if (name === "drag") return {
890
- type: "drag",
891
- x: Number(input.x),
892
- y: Number(input.y),
893
- toX: Number(input.toX),
894
- toY: Number(input.toY)
895
- };
896
- if (name === "hover") return {
897
- type: "hover",
898
- x: Number(input.x),
899
- y: Number(input.y)
900
- };
901
- return {
902
- type: "click",
903
- x: Number(input.x),
904
- y: Number(input.y)
905
- };
1073
+ async function cdpConnect(url, timeoutMs) {
1074
+ const ws = new WebSocket(url);
1075
+ await new Promise((resolve, reject) => {
1076
+ const t = setTimeout(() => reject(/* @__PURE__ */ new Error("CDP websocket timed out")), timeoutMs);
1077
+ ws.addEventListener("open", () => {
1078
+ clearTimeout(t);
1079
+ resolve();
1080
+ });
1081
+ ws.addEventListener("error", () => {
1082
+ clearTimeout(t);
1083
+ reject(/* @__PURE__ */ new Error("CDP websocket failed"));
1084
+ });
1085
+ });
1086
+ return new Cdp(ws);
906
1087
  }
907
1088
  /**
908
- * Gemini models answer a point normalised to 0–1000 across the image they were shown, whatever
909
- * the prompt asks, so their journeys are prompted in that scale and their points scaled back.
1089
+ * Drops a chrome-headless-shell profile. Chrome can still write after SIGKILL,
1090
+ * so `ENOTEMPTY` / `EBUSY` must not replace a PNG already in hand.
910
1091
  */
911
- const normalisesPoints = (model) => model.slice(model.lastIndexOf("/") + 1).toLowerCase().startsWith("gemini-");
912
- const NORMALISED = 1e3;
913
- /** Scales an action's 0–1000 points to CSS pixels on a `page`-sized screenshot. */
914
- function scalePoints(action, page) {
915
- const x = (n) => Math.round(n * page.width / NORMALISED);
916
- const y = (n) => Math.round(n * page.height / NORMALISED);
917
- switch (action.type) {
918
- case "click":
919
- case "hover":
920
- case "scroll": return {
921
- ...action,
922
- x: x(action.x),
923
- y: y(action.y)
924
- };
925
- case "drag": return {
926
- ...action,
927
- x: x(action.x),
928
- y: y(action.y),
929
- toX: x(action.toX),
930
- toY: y(action.toY)
931
- };
932
- default: return action;
933
- }
934
- }
935
- const systemPrompt = (normalised) => `You drive a web page from screenshots. Call one tool. ${normalised ? `Coordinates are normalised to 0-${NORMALISED} on the full-page image: x and toX across its width, y and toY down its height. A scroll's dy is in CSS pixels.` : "Coordinates are CSS pixels on the full-page image."} Call done when the goal is met. Do not mention CDP.`;
936
- const useTools = {
937
- click: tool({
938
- description: "Click at (x, y) on the screenshot.",
939
- inputSchema: z.object({
940
- x: z.number(),
941
- y: z.number()
942
- })
943
- }),
944
- type: tool({
945
- description: "Insert text into the focused field. Click the field first.",
946
- inputSchema: z.object({ text: z.string() })
947
- }),
948
- key: tool({
949
- description: "Press Tab or Enter.",
950
- inputSchema: z.object({ key: z.enum(["Tab", "Enter"]) })
951
- }),
952
- scroll: tool({
953
- description: "Wheel-scroll a constrained scroller at (x, y).",
954
- inputSchema: z.object({
955
- x: z.number(),
956
- y: z.number(),
957
- dy: z.number()
958
- })
959
- }),
960
- drag: tool({
961
- description: "Drag from (x, y) to (toX, toY).",
962
- inputSchema: z.object({
963
- x: z.number(),
964
- y: z.number(),
965
- toX: z.number(),
966
- toY: z.number()
967
- })
968
- }),
969
- hover: tool({
970
- description: "Move the pointer to (x, y).",
971
- inputSchema: z.object({
972
- x: z.number(),
973
- y: z.number()
974
- })
975
- }),
976
- done: tool({
977
- description: "The goal is finished. Leave the loop.",
978
- inputSchema: z.object({})
979
- })
980
- };
981
- async function runFakeScript(session, script, goal, remaining) {
982
- let moves = 0;
983
- for (const action of script) {
984
- remainingOrThrow(remaining, JSON.stringify(goal));
985
- if (action.type === "done") return moves;
986
- if (moves >= 32) throw new Error(`c.capture() instruction ${JSON.stringify(goal)} did not finish in 32 moves`);
987
- const page = pngSize(await session.screenshot(true));
988
- debugTrace("check", "capture-use", {
989
- goal,
990
- action,
991
- page
992
- }, `capture use ${action.type}`);
993
- await session.dispatch(action, page);
994
- moves++;
995
- await settle();
1092
+ function removeChromeProfile(dir, rm = rmSync) {
1093
+ try {
1094
+ rm(dir, {
1095
+ recursive: true,
1096
+ force: true,
1097
+ maxRetries: 5,
1098
+ retryDelay: 20
1099
+ });
1100
+ } catch {
1101
+ scheduleChromeProfileSweep(dir, rm);
996
1102
  }
997
- while (remaining() > 0) await settle();
998
- throw new Error(`c.capture() timed out on ${JSON.stringify(goal)}`);
999
1103
  }
1000
- async function runModelGoal(session, goal, modelName, remaining, loadModel) {
1001
- const model = await loadModel(modelName);
1002
- const normalised = normalisesPoints(modelName);
1003
- let moves = 0;
1004
- let usage = null;
1005
- while (moves < 32) {
1006
- remainingOrThrow(remaining, JSON.stringify(goal));
1007
- const ephemeral = await session.screenshot(true);
1008
- const page = pngSize(ephemeral);
1009
- debugTrace("check", "capture-shot", {
1010
- goal,
1011
- bytes: ephemeral.length,
1012
- page
1013
- }, false);
1014
- let chosen;
1015
- const result = await generateText({
1016
- model,
1017
- system: systemPrompt(normalised),
1018
- messages: [{
1019
- role: "user",
1020
- content: [{
1021
- type: "text",
1022
- text: goal
1023
- }, {
1024
- type: "image",
1025
- image: ephemeral
1026
- }]
1027
- }],
1028
- tools: useTools,
1029
- toolChoice: "required",
1030
- stopWhen: stepCountIs(1),
1031
- abortSignal: AbortSignal.timeout(remainingOrThrow(remaining, JSON.stringify(goal)))
1032
- });
1033
- usage = addUsage(usage, asUsage(result.usage ?? {}));
1034
- const call = result.toolCalls?.[0];
1035
- if (call) chosen = actionFromTool(call.toolName, call.input ?? {});
1036
- if (chosen && normalised) chosen = scalePoints(chosen, page);
1037
- if (!chosen) throw new Error(`c.capture() instruction ${JSON.stringify(goal)} returned no action`);
1038
- debugLog("check", `capture use ${chosen.type} on ${JSON.stringify(goal)}`);
1039
- debugTrace("check", "capture-use", {
1040
- goal,
1041
- action: chosen,
1042
- page
1043
- }, `capture use ${chosen.type}`);
1044
- if (chosen.type === "done") return {
1045
- moves,
1046
- usage
1104
+ function scheduleChromeProfileSweep(dir, rm) {
1105
+ setTimeout(() => {
1106
+ try {
1107
+ rm(dir, {
1108
+ recursive: true,
1109
+ force: true,
1110
+ maxRetries: 5,
1111
+ retryDelay: 50
1112
+ });
1113
+ } catch {}
1114
+ }, 250).unref();
1115
+ }
1116
+ function waitForExit(child, timeoutMs) {
1117
+ if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
1118
+ return new Promise((resolve) => {
1119
+ const finish = () => {
1120
+ clearTimeout(timer);
1121
+ child.off("exit", finish);
1122
+ resolve();
1047
1123
  };
1048
- await session.dispatch(chosen, page);
1049
- moves++;
1050
- await settle();
1051
- }
1052
- throw new Error(`c.capture() instruction ${JSON.stringify(goal)} did not finish in 32 moves`);
1124
+ const timer = setTimeout(finish, timeoutMs);
1125
+ child.once("exit", finish);
1126
+ });
1127
+ }
1128
+ /** Kills the shell and drops its profile. Never throws: the PNG is already in hand. */
1129
+ async function disposeChrome(child, userData) {
1130
+ try {
1131
+ if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
1132
+ await waitForExit(child, 1e3);
1133
+ } catch {}
1134
+ removeChromeProfile(userData);
1053
1135
  }
1054
1136
  /**
1055
- * Top-level wait, then the instruction loop, then the saved PNG.
1056
- * Throws before `screenshot` when the budget or move cap is hit.
1137
+ * Chrome's flags for `proxy`: every request goes to it, loopback ones included, which Chrome
1138
+ * otherwise never proxies.
1057
1139
  */
1058
- async function runJourney(session, url, options) {
1059
- const { wait, timeoutMs, fullPage, instructions, use, remaining } = options;
1060
- remainingOrThrow(remaining, "navigate");
1061
- await navigateAndWait(session, url, wait, Math.min(timeoutMs, remaining()));
1062
- let moves = 0;
1063
- let usage = null;
1064
- let stringIndex = 0;
1065
- for (const step of instructions) {
1066
- if (typeof step !== "string") {
1067
- remainingOrThrow(remaining, "wait");
1068
- await session.wait(step.wait, remaining(), void 0);
1069
- continue;
1070
- }
1071
- remainingOrThrow(remaining, JSON.stringify(step));
1072
- debugLog("check", `capture instruction ${JSON.stringify(step)}`);
1073
- if (use && "fake" in use) {
1074
- const script = use.fake[stringIndex] ?? [];
1075
- moves += await runFakeScript(session, script, step, remaining);
1076
- } else if (use && "model" in use) {
1077
- if (!options.loadModel) throw new Error("c.capture() instructions need a model loader");
1078
- const ran = await runModelGoal(session, step, use.model, remaining, options.loadModel);
1079
- moves += ran.moves;
1080
- usage = addUsage(usage, ran.usage);
1140
+ function proxyArgs(proxy) {
1141
+ return proxy ? [`--proxy-server=http://${proxy}`, "--proxy-bypass-list=<-loopback>"] : [];
1142
+ }
1143
+ /** One Chrome process, one tab. `fn` must not keep the session after it returns. */
1144
+ async function withChrome(bin, window, timeoutMs, options, fn) {
1145
+ const userData = mkdtempSync(join(tmpdir(), `reeval-chrome-${process.pid}-`));
1146
+ const child = spawn(bin, [
1147
+ "--headless",
1148
+ "--disable-gpu",
1149
+ "--no-sandbox",
1150
+ "--disable-dev-shm-usage",
1151
+ "--hide-scrollbars",
1152
+ "--remote-allow-origins=*",
1153
+ `--window-size=${window.width},${window.height}`,
1154
+ `--user-data-dir=${userData}`,
1155
+ "--remote-debugging-port=0",
1156
+ ...proxyArgs(options.proxy),
1157
+ "about:blank"
1158
+ ], { stdio: [
1159
+ "ignore",
1160
+ "ignore",
1161
+ "pipe"
1162
+ ] });
1163
+ try {
1164
+ const cdp = await cdpConnect(await pageDebuggerUrl(await waitForPortFile(join(userData, "DevToolsActivePort"), timeoutMs), timeoutMs), timeoutMs);
1165
+ try {
1166
+ await cdp.send("Page.enable");
1167
+ await cdp.send("Page.setLifecycleEventsEnabled", { enabled: true });
1168
+ const network = trackNetwork(cdp);
1169
+ const loading = /* @__PURE__ */ new Set();
1170
+ const frame = (msg) => {
1171
+ const body = msg;
1172
+ return (body.params ?? body).frameId ?? "";
1173
+ };
1174
+ cdp.on("Page.frameStartedLoading", (msg) => loading.add(frame(msg)));
1175
+ cdp.on("Page.frameStoppedLoading", (msg) => loading.delete(frame(msg)));
1176
+ cdp.on("Page.frameDetached", (msg) => loading.delete(frame(msg)));
1177
+ const opened = [];
1178
+ cdp.on("Page.javascriptDialogOpening", (msg) => {
1179
+ const body = msg;
1180
+ const { type = "dialog", message = "", defaultPrompt = "" } = body.params ?? body;
1181
+ opened.push(`${type} ${JSON.stringify(message.slice(0, 60))}`);
1182
+ cdp.send("Page.handleJavaScriptDialog", {
1183
+ accept: true,
1184
+ promptText: defaultPrompt
1185
+ }).catch(() => {});
1186
+ });
1187
+ await cdp.send("Network.enable");
1188
+ await cdp.send("Emulation.setDeviceMetricsOverride", {
1189
+ width: window.width,
1190
+ height: window.height,
1191
+ deviceScaleFactor: 1,
1192
+ mobile: false
1193
+ });
1194
+ let scrollY = 0;
1195
+ return await fn({
1196
+ cdp,
1197
+ window,
1198
+ async navigate(url) {
1199
+ scrollY = 0;
1200
+ const nav = await cdp.send("Page.navigate", { url });
1201
+ if (nav?.errorText) throw new Error(`c.capture() navigation failed: ${nav.errorText}`);
1202
+ return nav;
1203
+ },
1204
+ wait(wait, waitMs, nav) {
1205
+ if (wait === "idle" && !nav) return network.quiet(waitMs);
1206
+ return waitForPage(cdp, wait, waitMs, nav);
1207
+ },
1208
+ async screenshot(fullPage) {
1209
+ for (let tries = 1;; tries++) try {
1210
+ const shot = await cdp.send("Page.captureScreenshot", {
1211
+ format: "png",
1212
+ captureBeyondViewport: fullPage
1213
+ });
1214
+ if (!shot.data) throw new Error("Page.captureScreenshot returned no data");
1215
+ return Buffer.from(shot.data, "base64");
1216
+ } catch (error) {
1217
+ if (!/Cannot take screenshot with 0 (width|height)/.test(String(error)) || tries >= SCREENSHOT_TRIES) throw error;
1218
+ await new Promise((resolve) => setTimeout(resolve, SCREENSHOT_RETRY_MS));
1219
+ }
1220
+ },
1221
+ async dispatch(action, page) {
1222
+ scrollY = await dispatchInput(cdp, action, window, page, scrollY);
1223
+ },
1224
+ busy() {
1225
+ return loading.size > 0 || network.fresh();
1226
+ },
1227
+ dialogs() {
1228
+ return opened.splice(0);
1229
+ }
1230
+ });
1231
+ } finally {
1232
+ cdp.close();
1081
1233
  }
1082
- stringIndex++;
1083
- await settle();
1234
+ } finally {
1235
+ await disposeChrome(child, userData);
1084
1236
  }
1085
- remainingOrThrow(remaining, "screenshot");
1086
- return {
1087
- png: await session.screenshot(fullPage),
1088
- moves,
1089
- model: use && "model" in use ? use.model : use && "fake" in use ? "fake" : void 0,
1090
- usage
1091
- };
1092
- }
1093
- function captureUseSpend(result) {
1094
- if (!result.model) return void 0;
1095
- return {
1096
- model: result.model,
1097
- moves: result.moves,
1098
- usage: result.usage
1099
- };
1100
1237
  }
1101
- //#endregion
1102
- //#region src/contract/attempt.ts
1103
- const nullableNumber = z.number().nullable();
1104
- const tokenUsageSchema = z.strictObject({
1105
- input: z.number(),
1106
- output: z.number(),
1107
- cacheRead: z.number(),
1108
- cacheWrite: z.number(),
1109
- reasoning: nullableNumber
1110
- }).describe("The tokens one turn used, as its harness reported them. `input` excludes cache reads and writes (so the prompt was `input + cacheRead + cacheWrite`); `reasoning` is part of `output`.");
1111
- const fileChangeSchema = z.strictObject({
1112
- path: z.string(),
1113
- status: z.enum([
1114
- "added",
1115
- "modified",
1116
- "deleted"
1117
- ])
1118
- });
1119
- const diffSchema = z.strictObject({
1120
- patch: z.string(),
1121
- files: z.array(fileChangeSchema)
1122
- });
1123
- const turnStatusSchema = z.enum([
1124
- "completed",
1125
- "aborted",
1126
- "error"
1127
- ]);
1128
- const webEventTypeSchema = z.enum(["web-search", "web-fetch"]).describe("The transcript events a harness may be unable to report.");
1129
- const at = {
1130
- ts: z.string(),
1131
- atMs: z.number()
1132
- };
1133
- const transcriptEventSchema = z.discriminatedUnion("type", [
1134
- z.strictObject({
1135
- ...at,
1136
- type: z.enum(["text", "reasoning"]),
1137
- text: z.string(),
1138
- durationMs: nullableNumber
1139
- }),
1140
- z.strictObject({
1141
- ...at,
1142
- type: z.literal("file-edit"),
1143
- toolCallId: z.string(),
1144
- path: z.string(),
1145
- kind: z.enum([
1146
- "create",
1147
- "modify",
1148
- "delete"
1149
- ]),
1150
- added: nullableNumber.describe("Line counts, or null when the harness didn't send the content (e.g. deletes)."),
1151
- removed: nullableNumber,
1152
- durationMs: nullableNumber,
1153
- stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
1154
- }),
1155
- z.strictObject({
1156
- ...at,
1157
- type: z.literal("web-search"),
1158
- toolCallId: z.string(),
1159
- query: z.string(),
1160
- references: nullableNumber.describe("How many sources the harness says it used; Cursor sends no URLs or titles."),
1161
- ok: z.boolean().optional().describe("Whether the search succeeded. Absent means it did."),
1162
- durationMs: nullableNumber,
1163
- stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
1164
- }),
1165
- z.strictObject({
1166
- ...at,
1167
- type: z.literal("web-fetch"),
1168
- toolCallId: z.string(),
1169
- url: z.string(),
1170
- ok: z.boolean(),
1171
- durationMs: nullableNumber,
1172
- stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
1173
- }),
1174
- z.strictObject({
1175
- ...at,
1176
- type: z.literal("tool"),
1177
- toolCallId: z.string(),
1178
- tool: z.string(),
1179
- input: z.unknown(),
1180
- output: z.unknown(),
1181
- ok: z.boolean(),
1182
- durationMs: nullableNumber,
1183
- stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
1184
- }),
1185
- z.strictObject({
1186
- ...at,
1187
- type: z.literal("error"),
1188
- message: z.string()
1189
- }),
1190
- z.strictObject({
1191
- ...at,
1192
- type: z.literal("retry"),
1193
- message: z.string()
1194
- })
1195
- ]).describe("Normalised agent events. Each tool call is one event, merged with its result; `durationMs` is wall time until the result arrived (Cursor's own `completed_at_ms` can be earlier). Null when the harness never reported the end. Long shell or edit calls set `stalled: true`.");
1196
- const turnDataSchema = z.strictObject({
1197
- prompt: z.string(),
1198
- text: z.string().describe("The agent's final message for this turn."),
1199
- status: turnStatusSchema,
1200
- startedAt: z.string(),
1201
- durationMs: z.number(),
1202
- transcript: z.array(transcriptEventSchema),
1203
- usage: tokenUsageSchema.nullable().describe("Tokens the turn used; null when the harness reports none."),
1204
- model: z.string().nullable().describe("The model the harness says served the turn, e.g. `composer-2.5[fast=false]`; null when it reports none. It can differ from the requested one (an alias, or parameters it filled in)."),
1205
- unreported: z.array(webEventTypeSchema).describe("Event types the harness couldn't report for this turn: the agent may have searched or fetched with no event in `transcript`. `toHaveSearched()` and `toHaveFetched()` refuse such a turn. Usually `[]`; a Cursor turn whose web calls the adapter couldn't recover (its SDK drops them from the stream) is `['web-search', 'web-fetch']`."),
1206
- diff: diffSchema.describe("Workspace changes since the eval started, as of the end of this turn.")
1207
- }).describe("A turn as data, as a record or another language's runner carries it: `Turn` without its methods.");
1208
- const execResultSchema = z.strictObject({
1209
- exitCode: z.number().int(),
1210
- stdout: z.string(),
1211
- stderr: z.string()
1212
- });
1213
- const execRetrySchema = z.strictObject({
1214
- retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional().describe("Extra tries after a non-zero exit, a whole number from 0. Default 0."),
1215
- interval: z.number().min(0, "must be 0 or more milliseconds").optional().describe("Milliseconds to wait before each extra try. Default 1000.")
1216
- }).describe("How `c.exec` reruns a command that exits non-zero, for a check that can fail on a transient.");
1217
- /** A background command has no exit for `retries` to wait on. */
1218
- const noBackgroundRetry = (options, ctx) => {
1219
- if (!options.background) return;
1220
- for (const key of ["retries", "interval"]) {
1221
- if (options[key] === void 0) continue;
1222
- ctx.addIssue({
1223
- code: "custom",
1224
- path: [key],
1225
- message: "can't be used with background: true, whose exit reeval doesn't wait for: drop one of them"
1226
- });
1227
- }
1228
- };
1229
- const execOptionsSchema = z.strictObject({
1230
- background: z.boolean().optional(),
1231
- ...execRetrySchema.shape
1232
- }).superRefine(noBackgroundRetry);
1233
- /** What's wrong with `c.exec()`'s options, or undefined. */
1234
- const execOptionsProblem = (value) => dataProblem(execOptionsSchema, value, "c.exec()", () => Object.keys(execOptionsSchema.shape));
1235
- const assertionRecordSchema = z.strictObject({
1236
- label: z.string().describe("`expect`'s second argument, or `<matcher> <first argument>` (or the command `c.exec` ran, or the judge's question), cut at 60 characters; ` #2`, ` #3` on repeats within the attempt."),
1237
- labelled: z.boolean().describe("The eval gave the label."),
1238
- matcher: z.string().describe("`toHaveFile`, `not.toBe`, `resolves.toBe`, `poll.toBe`."),
1239
- status: z.enum([
1240
- "pass",
1241
- "fail",
1242
- "error"
1243
- ]).describe("`error`: the matcher couldn't answer (a subject it can't read, an event the harness doesn't report)."),
1244
- soft: z.boolean(),
1245
- message: z.string().optional().describe("Why it failed or couldn't answer, cut at 1 KB."),
1246
- durationMs: z.number().optional().describe("Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`).")
1247
- }).describe("One `expect` or `expect.soft` matcher call an attempt made.");
1248
- const turnIndex = z.number().int("must be a turn index, from 0").min(0, "must be a turn index, from 0").describe("The turn's index, from 0.");
1249
- const captureSchema = z.strictObject({
1250
- name: z.string(),
1251
- port: z.number().describe("Container port that was tunneled."),
1252
- path: z.string().describe("Path on the app, as passed in."),
1253
- viewport: z.union([captureWindowSchema, z.literal("full-page")]).describe("What the author asked, including `'full-page'`."),
1254
- file: z.strictObject({
1255
- path: z.string(),
1256
- sha256: z.string(),
1257
- bytes: z.number()
1258
- }).describe("`artifacts/<name>.png` under the attempt directory.")
1259
- }).describe("What `c.capture()` returns: the PNG is also in the attempt record under `artifacts/`.");
1260
- const judgeArtifactRefSchema = z.strictObject({
1261
- name: z.string(),
1262
- path: z.string().describe("`artifacts/<name>.png` under the attempt directory."),
1263
- sha256: z.string(),
1264
- bytes: z.number(),
1265
- mediaType: z.literal("image/png")
1266
- }).describe("A capture this judge call sent as a file part. Bytes live in `artifacts/<name>.png`.");
1267
- const verdictFields = {
1268
- threshold: z.number(),
1269
- question: z.string(),
1270
- model: z.string(),
1271
- gateway: gatewayNameSchema.optional().describe("The gateway from the config's `gateway` list the call went through; unset without one."),
1272
- artifacts: z.array(judgeArtifactRefSchema).optional().describe("Captures this call sent the judge as file parts, when any.")
1238
+ /** Arm the wait, then navigate, then wait. `arm` is before `Page.navigate`. */
1239
+ async function navigateAndWait(session, url, wait, timeoutMs) {
1240
+ if (isMs(wait)) {
1241
+ const nav = await session.navigate(url);
1242
+ await session.wait(wait, timeoutMs, nav);
1243
+ return;
1244
+ }
1245
+ const page = listenForCaptureWait(session.cdp, wait);
1246
+ page.arm();
1247
+ const nav = await session.navigate(url);
1248
+ await page.ready(nav, timeoutMs);
1249
+ }
1250
+ //#endregion
1251
+ //#region src/capture/poll.ts
1252
+ /** A dev server still starting (Nuxt's loading page) or a proxy whose upstream isn't up yet. */
1253
+ const STARTING = /* @__PURE__ */ new Set([
1254
+ 502,
1255
+ 503,
1256
+ 504
1257
+ ]);
1258
+ /**
1259
+ * Any HTTP response, including 404, means something accepted the connection. A 502, 503 or 504
1260
+ * is polled past until the deadline, and then taken as it is.
1261
+ */
1262
+ async function pollHttp(url, timeoutMs, signal) {
1263
+ const deadline = Date.now() + timeoutMs;
1264
+ let last;
1265
+ let answered = false;
1266
+ while (Date.now() < deadline) {
1267
+ signal?.throwIfAborted();
1268
+ const left = Math.max(1, deadline - Date.now());
1269
+ try {
1270
+ const res = await fetch(url, { signal: AbortSignal.any([AbortSignal.timeout(Math.min(1e3, left)), ...signal ? [signal] : []]) });
1271
+ await res.body?.cancel();
1272
+ if (!STARTING.has(res.status)) return;
1273
+ answered = true;
1274
+ } catch (error) {
1275
+ last = error;
1276
+ if (signal?.aborted) throw error;
1277
+ }
1278
+ await new Promise((r) => setTimeout(r, 100));
1279
+ }
1280
+ if (answered) return;
1281
+ throw last instanceof Error ? last : /* @__PURE__ */ new Error("poll timed out");
1282
+ }
1283
+ //#endregion
1284
+ //#region src/capture/stub.ts
1285
+ const PNG_SIG = Buffer.from([
1286
+ 137,
1287
+ 80,
1288
+ 78,
1289
+ 71,
1290
+ 13,
1291
+ 10,
1292
+ 26,
1293
+ 10
1294
+ ]);
1295
+ function chunk(type, data) {
1296
+ const body = Buffer.concat([Buffer.from(type, "ascii"), data]);
1297
+ const out = Buffer.alloc(12 + data.length);
1298
+ out.writeUInt32BE(data.length, 0);
1299
+ body.copy(out, 4);
1300
+ out.writeUInt32BE(crc32(body), 8 + data.length);
1301
+ return out;
1302
+ }
1303
+ /** A valid 1×1 PNG whose bytes are unique for `(name, port, path, viewport)`. */
1304
+ function stubPng(name, port, path, viewport) {
1305
+ const digest = createHash("sha256").update(`${name}\0${port}\0${path}\0${viewport === "full-page" ? "full-page" : `${viewport.width}x${viewport.height}`}`).digest();
1306
+ const ihdr = Buffer.alloc(13);
1307
+ ihdr.writeUInt32BE(1, 0);
1308
+ ihdr.writeUInt32BE(1, 4);
1309
+ ihdr[8] = 8;
1310
+ ihdr[9] = 2;
1311
+ const scanline = Buffer.from([
1312
+ 0,
1313
+ digest[0] ?? 0,
1314
+ digest[1] ?? 0,
1315
+ digest[2] ?? 0
1316
+ ]);
1317
+ const idat = deflateSync(scanline);
1318
+ const text = Buffer.concat([Buffer.from("Comment\0", "ascii"), Buffer.from(digest.toString("hex"), "ascii")]);
1319
+ return Buffer.concat([
1320
+ PNG_SIG,
1321
+ chunk("IHDR", ihdr),
1322
+ chunk("tEXt", text),
1323
+ chunk("IDAT", idat),
1324
+ chunk("IEND", Buffer.alloc(0))
1325
+ ]);
1326
+ }
1327
+ /** PNG IHDR width and height; throws if the bytes are not a PNG. */
1328
+ function pngSize(bytes) {
1329
+ if (bytes.length < 24 || Buffer.from(bytes.subarray(0, 8)).compare(PNG_SIG) !== 0) throw new Error("not a PNG");
1330
+ const view = Buffer.from(bytes.subarray(16, 24));
1331
+ return {
1332
+ width: view.readUInt32BE(0),
1333
+ height: view.readUInt32BE(4)
1334
+ };
1335
+ }
1336
+ //#endregion
1337
+ //#region src/capture/index.ts
1338
+ /** Default CSS pixels when the call and cascade omit `viewport`. */
1339
+ const DEFAULT_VIEWPORT = {
1340
+ width: 1280,
1341
+ height: 720
1273
1342
  };
1274
- const verdictSchema = z.strictObject({
1275
- verdict: z.enum(["pass", "fail"]),
1276
- score: z.number().describe("The judge's probability that the answer is yes, from 0 to 1."),
1277
- ...verdictFields,
1278
- resolvedModel: z.string().optional().describe("The model id the provider says served the call, when it reports one.")
1279
- });
1280
- const judgeFailureSchema = z.strictObject({
1281
- verdict: z.literal("error"),
1282
- score: z.null(),
1283
- ...verdictFields,
1284
- error: z.string()
1285
- }).describe("What `c.judge()` returns when the call failed (the provider's error, after any retries): the attempt fails as a judge error once the eval's body has finished, so the judges and checks after it still run.");
1286
- /** A pattern, for the matchers that take a string or a `RegExp`. */
1287
- const patternSchema = z.strictObject({
1288
- regexp: z.string().describe("The pattern's source, as `RegExp.prototype.source` gives it."),
1289
- flags: z.string().regex(/^[dgimsuvy]*$/, "must use only the flags d, g, i, m, s, u, v and y").optional().describe("Its flags, e.g. `i`.")
1290
- }).describe("A regular expression, where a matcher takes a string or a `RegExp`.");
1291
- /** A pattern in the runner's own language, which reeval never compiles. */
1292
- function hasAnsweredPattern(value) {
1293
- if (Array.isArray(value)) return value.some(hasAnsweredPattern);
1294
- if (typeof value !== "object" || value === null) return false;
1295
- return "matchedBy" in value || Object.values(value).some(hasAnsweredPattern);
1343
+ const CAPTURE_WAITS = ["load", "idle"];
1344
+ /** `'load'` when the call omits `wait`. `'load'`, `'idle'` or `{ ms }`. */
1345
+ function resolveWait(wait) {
1346
+ if (wait === void 0) return "load";
1347
+ if (isOneOf(CAPTURE_WAITS, wait)) return wait;
1348
+ if (typeof wait === "object" && wait !== null && Number.isFinite(wait.ms) && wait.ms >= 0 && Object.keys(wait).every((k) => k === "ms")) return { ms: wait.ms };
1349
+ throw new Error(`c.capture() wait is 'load', 'idle' or { ms } (got ${JSON.stringify(wait)})`);
1296
1350
  }
1297
- const answeredPatternSchema = z.strictObject({
1298
- matchedBy: z.string().regex(/\S/).describe("The pattern as the runner's language writes it, e.g. `re.compile('npm t')`."),
1299
- answers: z.record(z.string(), z.boolean()).describe("Whether it matches each string the runner has been asked about.")
1300
- }).describe("A pattern the runner matches in its own language. reeval runs the matcher with it, and when the matcher tests a string not in `answers` replies with those strings (`untested`) instead of a record; the runner matches them and makes the call again with them in `answers`.");
1301
- const isPatternLike = (value) => typeof value === "object" && value !== null && !Array.isArray(value) && "regexp" in value;
1302
- const matcherArgSchema = z.union([
1303
- patternSchema,
1304
- answeredPatternSchema,
1305
- jsonValueSchema
1306
- ]).superRefine((value, ctx) => {
1307
- if (!isPatternLike(value)) return;
1308
- for (const issue of patternSchema.safeParse(value).error?.issues ?? []) ctx.addIssue(issue);
1309
- }).describe("One argument to a matcher: a JSON value, a regular expression as `{ regexp, flags? }`, or a pattern in the runner's own language as `{ matchedBy, answers }`.");
1310
- const matcherSubjectSchema = z.union([
1311
- z.strictObject({ turn: turnIndex }),
1312
- z.strictObject({ attempt: z.literal(true).describe("The attempt itself, `c`.") }),
1313
- z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
1314
- z.strictObject({
1315
- exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
1316
- result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
1317
- }),
1318
- z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
1319
- ]).describe("What `expect` was given: a turn `c.send` returned, the attempt (for its files and latest turn), a value, such as an `ExecResult`, or a command or file reeval reads for it.");
1320
- const nonEmpty = z.string().regex(/\S/, "must not be empty");
1321
- const portSchema = z.number().int().min(1, "a port from 1 to 65535").max(65535, "a port from 1 to 65535");
1322
- const millisSchema = z.number().positive("more than zero milliseconds");
1323
- const captureWaitSchema = z.enum(["load", "idle"]).describe("Chromium look-ready events. `'load'` is `window` load (`Page.loadEventFired`). `'idle'` is `Page.lifecycleEvent` `networkAlmostIdle`.");
1324
- const waitSchema = z.union([captureWaitSchema, z.strictObject({ ms: millisSchema })]).describe("Same shape at top-level `wait` and `{ wait }` between instructions.");
1325
- const captureInstructionSchema = z.union([nonEmpty, z.strictObject({ wait: waitSchema })]);
1326
- const captureCallDataSchema = z.strictObject({
1327
- name: nonEmpty.optional(),
1328
- path: z.string().optional(),
1329
- viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
1330
- timeout: durationSchema.optional(),
1331
- wait: waitSchema.optional().describe("Look-ready once, before the first model shot or the saved PNG. Default `'load'`."),
1332
- instructions: z.array(captureInstructionSchema).optional().describe("Journey: goal strings and `{ wait }` beats, run by the vision model."),
1333
- model: z.string().optional().describe("Overrides cascaded `capture.model` for this call.")
1334
- }).describe("`c.capture()`'s camera and path as data. Default `path` is `/`; default `name` is derived from the path. A test's `fakeInstructions` journey is TypeScript-only.");
1335
- const handleId = nonEmpty.describe("A background process, as an exec or serve reply named it.");
1336
- const judgeOnSchema = z.union([z.strictObject({ turn: turnIndex }), z.strictObject({ value: jsonValueSchema.describe("Graded as the output.") })]);
1337
- const judgeCallDataSchema = z.strictObject({
1338
- model: z.union([judgeModelNameSchema, fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge with a numeric score."),
1339
- ...judgeFields,
1340
- on: judgeOnSchema.optional().describe("What to grade instead of the latest turn: an earlier turn, or a value."),
1341
- artifacts: z.union([z.literal(true), z.array(z.union([nonEmpty, captureSchema]))]).optional().describe("Captures this call sends the judge as image file parts: a capture `c.capture` returned, or its name, or `true` for every capture so far. The provider that cannot take files fails the call.")
1342
- }).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score, plus the call-only `on` and `artifacts`.");
1343
- const attemptCallSchema = z.discriminatedUnion("op", [
1344
- z.strictObject({
1345
- op: z.literal("send"),
1346
- prompt: z.string().describe("The turn's prompt.")
1347
- }).describe("`c.send(prompt)`: the next turn of the agent session. Replies with a turn."),
1348
- z.strictObject({
1349
- op: z.literal("exec"),
1350
- command: nonEmpty.describe("A shell command, run in the workspace."),
1351
- background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
1352
- ...execRetrySchema.shape
1353
- }).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
1354
- z.strictObject({
1355
- op: z.literal("serve"),
1356
- command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
1357
- port: portSchema.optional().describe("The container port. Omit it and reeval allocates one.")
1358
- }).describe("`c.serve(command, port?)`: a background process and its port."),
1359
- z.strictObject({
1360
- op: z.literal("kill"),
1361
- handle: handleId
1362
- }).describe("`handle.kill()`: stops a background process. Killing one that's gone is fine."),
1363
- z.strictObject({
1364
- op: z.literal("capture"),
1365
- target: z.union([z.strictObject({ port: portSchema }), z.strictObject({ handle: handleId })]).describe("A container port, or a serve handle by its id."),
1366
- options: captureCallDataSchema.optional()
1367
- }).describe("`c.capture(target, options)`: screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`. Replies with the capture."),
1368
- z.strictObject({
1369
- op: z.literal("readFile"),
1370
- path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory.")
1371
- }).describe("`c.readFile(path)`. Replies with its contents, or null when there is no such file."),
1372
- z.strictObject({
1373
- op: z.literal("writeFile"),
1374
- path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory."),
1375
- contents: z.string().describe("The file's contents, in `encoding`."),
1376
- encoding: z.enum(["utf8", "base64"]).optional().describe("How `contents` is encoded; base64 carries bytes. Default `utf8`.")
1377
- }).describe("`c.writeFile(path, contents)`, creating its directories."),
1378
- z.strictObject({
1379
- op: z.literal("expect"),
1380
- subject: matcherSubjectSchema,
1381
- matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a matcher name such as toHaveFile, without \"not.\" (set not: true instead)").describe("The matcher's name, e.g. `toHaveFile` or `toBe`, without `not.`."),
1382
- args: z.array(matcherArgSchema).describe("The matcher's arguments."),
1383
- not: z.boolean().describe("Called as `expect(…).not`."),
1384
- soft: z.boolean().describe("Called as `expect.soft`: a failure fails the attempt at its end."),
1385
- label: z.string().optional().describe("`expect`'s second argument."),
1386
- poll: z.strictObject({
1387
- interval: millisSchema.optional().describe("Milliseconds between tries. Default 50."),
1388
- timeout: millisSchema.optional().describe("Milliseconds before it fails with the last error. Default 1000.")
1389
- }).optional().describe("`expect.poll`: reeval re-reads the subject and retries the matcher until it passes, recording only the final outcome. The subject must be one reeval can re-read: the attempt, a command or a file.")
1390
- }).superRefine((call, ctx) => {
1391
- if (call.poll && "value" in call.subject) ctx.addIssue({
1392
- code: "custom",
1393
- path: ["subject"],
1394
- message: "can't be polled: reeval can't re-read a value. Poll the attempt, an exec or a readFile"
1351
+ function parseInstructions(value) {
1352
+ if (value === void 0) return [];
1353
+ if (!Array.isArray(value)) throw new Error(`c.capture() instructions is an array of strings and { wait } (got ${JSON.stringify(value)})`);
1354
+ return value.map((step, i) => {
1355
+ if (typeof step === "string") {
1356
+ if (!step.trim()) throw new Error(`c.capture() instructions[${i}] is an empty string`);
1357
+ return step;
1358
+ }
1359
+ if (typeof step === "object" && step !== null && "wait" in step) return { wait: resolveWait(step.wait) };
1360
+ throw new Error(`c.capture() instructions[${i}] is a string or { wait } (got ${JSON.stringify(step)})`);
1361
+ });
1362
+ }
1363
+ function instructionGoals(steps) {
1364
+ return steps.filter((s) => typeof s === "string");
1365
+ }
1366
+ /** Live window and whether the saved PNG uses `captureBeyondViewport`. */
1367
+ function resolveCaptureViewport(viewport) {
1368
+ const asked = viewport ?? DEFAULT_VIEWPORT;
1369
+ if (asked === "full-page") return {
1370
+ viewport: "full-page",
1371
+ window: DEFAULT_VIEWPORT,
1372
+ fullPage: true
1373
+ };
1374
+ return {
1375
+ viewport: asked,
1376
+ window: asked,
1377
+ fullPage: false
1378
+ };
1379
+ }
1380
+ /** `~/.cache/reeval` when that's the resolved dir; otherwise the absolute path. */
1381
+ function cacheLabel(dir = cacheDir()) {
1382
+ const home = homedir();
1383
+ if (dir.replaceAll("\\", "/") === join(home, ".cache", "reeval").replaceAll("\\", "/")) return "~/.cache/reeval";
1384
+ return dir;
1385
+ }
1386
+ /**
1387
+ * Env is named first when both refuse. `undefined` means the host binary is allowed.
1388
+ */
1389
+ function refusedBrowser(capture) {
1390
+ if (process.env.REEVAL_CAPTURE_BROWSER === "0") return "c.capture() is refused: REEVAL_CAPTURE_BROWSER=0";
1391
+ if (capture.browser === false) return "c.capture() is refused: capture.browser is false";
1392
+ }
1393
+ function resolveCaptureTarget(target) {
1394
+ if (typeof target === "number") {
1395
+ if (!(Number.isInteger(target) && target >= 1 && target <= 65535)) throw new Error(`c.capture() port is a container port from 1 to 65535 (got ${String(target)})`);
1396
+ return target;
1397
+ }
1398
+ if (target !== null && typeof target === "object" && typeof target.port === "number" && typeof target.kill === "function") return resolveCaptureTarget(target.port);
1399
+ throw new Error("c.capture() needs a port or a ServeHandle as the first argument");
1400
+ }
1401
+ /**
1402
+ * Artifact name from a path: `/login` → `login`, `/foo/bar` → `foo-bar`, `/` → `root`. Always
1403
+ * a valid capture name: leading `_` and `-` go, and it's cut at the name's length limit.
1404
+ */
1405
+ function nameFromPath(path) {
1406
+ return (path.split(/[?#]/, 1)[0] ?? "").replaceAll("\\", "/").split("/").filter(Boolean).join("-").toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^[_-]+/, "").slice(0, 100).replace(/-+$/, "") || "root";
1407
+ }
1408
+ function resolveCaptureCall(target, call, cascade) {
1409
+ if (call && Object.hasOwn(call, "port")) throw new Error("c.capture() takes the port as the first argument, not in the options");
1410
+ if (call && Object.hasOwn(call, "use")) throw new Error("c.capture() has no use. Set model, or pass fakeInstructions from reeval/testing as instructions.");
1411
+ if (call && Object.hasOwn(call, "fake")) throw new Error("c.capture() has no fake. Tests pass fakeInstructions(...) as instructions.");
1412
+ if (call?.model !== void 0) checkCallModel(call.model);
1413
+ const port = resolveCaptureTarget(target);
1414
+ const path = call?.path || "/";
1415
+ const name = call?.name ?? nameFromPath(path);
1416
+ checkCaptureName(name);
1417
+ const asked = resolveCaptureViewport(call?.viewport ?? cascade.viewport);
1418
+ return {
1419
+ port,
1420
+ name,
1421
+ path,
1422
+ viewport: asked.viewport,
1423
+ window: asked.window,
1424
+ fullPage: asked.fullPage,
1425
+ timeoutMs: parseDuration(call?.timeout ?? cascade.timeout ?? "60s"),
1426
+ wait: resolveWait(call?.wait),
1427
+ instructions: parseInstructions(call?.instructions),
1428
+ model: call?.model ?? cascade.model,
1429
+ fake: fakeScriptFrom(call?.instructions)
1430
+ };
1431
+ }
1432
+ function fakeScriptFrom(value) {
1433
+ const steps = fakeInstructionSteps(value);
1434
+ if (!steps) return void 0;
1435
+ const script = [];
1436
+ for (const step of steps) {
1437
+ if (isWaitStep(step)) continue;
1438
+ script.push(step);
1439
+ }
1440
+ return script;
1441
+ }
1442
+ function checkCallModel(model) {
1443
+ if (!model.includes("/")) throw new Error(`c.capture() model is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${JSON.stringify(model)})`);
1444
+ if (model.startsWith("typesafe-ai/")) throw new Error(`c.capture() model "${model}" is an evaluation model: it scores yes/no questions and can't drive a capture. Use a language model that accepts images, e.g. 'google/gemini-3.8-flash'.`);
1445
+ }
1446
+ function checkCaptureName(name) {
1447
+ if (!CAPTURE_NAME.test(name)) throw new Error(`c.capture() name ${CAPTURE_NAME_RULE} (got ${JSON.stringify(name)})`);
1448
+ }
1449
+ /** A free host port. Closed before the promise resolves, so the caller can bind it. */
1450
+ function allocatePort() {
1451
+ return new Promise((resolve, reject) => {
1452
+ const server = createServer();
1453
+ server.once("error", reject);
1454
+ server.listen(0, "127.0.0.1", () => {
1455
+ const addr = server.address();
1456
+ const port = typeof addr === "object" && addr ? addr.port : 0;
1457
+ server.close((error) => error ? reject(error) : resolve(port));
1395
1458
  });
1396
- if (call.poll && "exec" in call.subject && call.subject.result) ctx.addIssue({
1397
- code: "custom",
1398
- path: ["subject", "result"],
1399
- message: "can't be polled: drop result so reeval runs the command on each try"
1459
+ });
1460
+ }
1461
+ /** Origin plus a path (`/login`), without a doubled slash. */
1462
+ function pageUrl(origin, path) {
1463
+ const base = origin.endsWith("/") ? origin : `${origin}/`;
1464
+ const rel = path.startsWith("/") ? path.slice(1) : path;
1465
+ return new URL(rel, base).href;
1466
+ }
1467
+ //#endregion
1468
+ //#region src/capture/loopback.ts
1469
+ /**
1470
+ * `localhost`, `*.localhost`, `127.0.0.0/8`, `::1`, their IPv4-mapped IPv6 form and the
1471
+ * unspecified addresses, with or without a trailing dot: the names a page means the sandbox by.
1472
+ */
1473
+ function isLoopbackHost(host) {
1474
+ const h = host.replace(/^\[|\]$/g, "").replace(/\.$/, "").toLowerCase();
1475
+ if (h === "localhost" || h.endsWith(".localhost")) return true;
1476
+ if (net.isIPv4(h)) return h.startsWith("127.") || h === "0.0.0.0";
1477
+ if (!net.isIPv6(h)) return false;
1478
+ if (h === "::1" || h === "::") return true;
1479
+ const mapped = /^::ffff:(?:(\d+\.\d+\.\d+\.\d+)|([0-9a-f]{1,4}):[0-9a-f]{1,4})$/.exec(h);
1480
+ if (!mapped) return false;
1481
+ return mapped[1] ? mapped[1].startsWith("127.") : Number.parseInt(mapped[2] ?? "", 16) >> 8 === 127;
1482
+ }
1483
+ function authority(value) {
1484
+ if (!URL.canParse(`http://${value}`)) return void 0;
1485
+ const url = new URL(`http://${value}`);
1486
+ return {
1487
+ host: url.hostname.replace(/^\[|\]$/g, ""),
1488
+ port: Number(url.port) || 80
1489
+ };
1490
+ }
1491
+ /** `rawHeaders` without the ones meant for the proxy, in the flat form `http.request` takes. */
1492
+ function forwarded(raw) {
1493
+ const out = [];
1494
+ for (let i = 0; i < raw.length; i += 2) {
1495
+ const name = raw[i] ?? "";
1496
+ if (!/^proxy-/i.test(name)) out.push(name, raw[i + 1] ?? "");
1497
+ }
1498
+ return out;
1499
+ }
1500
+ /**
1501
+ * The capture browser's HTTP proxy, on the host's loopback. Chrome sends it every request
1502
+ * (`--proxy-bypass-list=<-loopback>`): one for a loopback address goes into the sandbox, to that
1503
+ * port through its tunnel, opened on first use; everything else goes out as it would without a
1504
+ * proxy, so the page reaches the internet as before.
1505
+ *
1506
+ * A plain-HTTP request that gets no response (nothing listens on that sandbox port) drops the
1507
+ * page's connection rather than answering, so the page's fetch fails as a refused one does. A
1508
+ * CONNECT to such a port is answered and then closed: the tunnel accepts before the sandbox side
1509
+ * connects, and Chrome fails that request too.
1510
+ */
1511
+ async function startLoopbackProxy(options) {
1512
+ const direct = new Set(options.direct);
1513
+ const reached = [];
1514
+ const tunnels = /* @__PURE__ */ new Map();
1515
+ const live = /* @__PURE__ */ new Set();
1516
+ let closed = false;
1517
+ const track = (s) => {
1518
+ live.add(s);
1519
+ s.once("close", () => live.delete(s));
1520
+ return s;
1521
+ };
1522
+ const answered = (route) => {
1523
+ if (route.sandbox !== void 0 && !reached.includes(route.sandbox)) reached.push(route.sandbox);
1524
+ };
1525
+ const resolve = async (to) => {
1526
+ if (!isLoopbackHost(to.host) || direct.has(to.port)) return to;
1527
+ let open = tunnels.get(to.port);
1528
+ if (!open) {
1529
+ open = options.tunnel(to.port).then((url) => {
1530
+ const at = new URL(url);
1531
+ direct.add(Number(at.port));
1532
+ return {
1533
+ host: at.hostname,
1534
+ port: Number(at.port)
1535
+ };
1536
+ });
1537
+ open.catch(() => tunnels.delete(to.port));
1538
+ tunnels.set(to.port, open);
1539
+ }
1540
+ return {
1541
+ ...await open,
1542
+ sandbox: to.port
1543
+ };
1544
+ };
1545
+ const connect = async (to) => {
1546
+ const route = await resolve(to);
1547
+ if (closed) throw new Error("the capture proxy closed");
1548
+ return new Promise((ok, fail) => {
1549
+ const socket = track(net.connect({
1550
+ host: route.host,
1551
+ port: route.port,
1552
+ allowHalfOpen: true
1553
+ }));
1554
+ socket.once("connect", () => {
1555
+ socket.off("error", fail);
1556
+ ok({
1557
+ socket,
1558
+ route
1559
+ });
1560
+ });
1561
+ socket.once("error", fail);
1400
1562
  });
1401
- if (call.poll && call.args.some(hasAnsweredPattern)) ctx.addIssue({
1402
- code: "custom",
1403
- path: ["args"],
1404
- message: "can't be polled with a { matchedBy } pattern, whose answers can't follow the subject from try to try: poll with a string, or check once"
1563
+ };
1564
+ /** Joins the page's connection to the upstream one, unless the page left while it opened. */
1565
+ const join = (client, socket, route) => {
1566
+ if (closed || client.destroyed) {
1567
+ socket.destroy();
1568
+ return false;
1569
+ }
1570
+ socket.once("data", () => answered(route));
1571
+ socket.on("error", () => client.destroy());
1572
+ client.on("error", () => socket.destroy());
1573
+ client.once("close", () => socket.destroy());
1574
+ socket.once("close", () => client.destroy());
1575
+ client.pipe(socket);
1576
+ socket.pipe(client);
1577
+ return true;
1578
+ };
1579
+ const server = http.createServer((req, res) => {
1580
+ const url = URL.canParse(req.url ?? "") ? new URL(req.url ?? "") : void 0;
1581
+ const to = url && authority(url.host);
1582
+ if (!url || !to || url.protocol !== "http:") {
1583
+ res.writeHead(400).end();
1584
+ return;
1585
+ }
1586
+ const drop = () => req.socket.destroy();
1587
+ resolve(to).then((route) => {
1588
+ if (closed || res.destroyed) return drop();
1589
+ const upstream = http.request({
1590
+ host: route.host,
1591
+ port: route.port,
1592
+ method: req.method,
1593
+ path: `${url.pathname}${url.search}`,
1594
+ headers: forwarded(req.rawHeaders),
1595
+ agent: false
1596
+ }, (up) => {
1597
+ answered(route);
1598
+ res.writeHead(up.statusCode ?? 502, up.statusMessage, up.rawHeaders);
1599
+ up.pipe(res);
1600
+ });
1601
+ track(upstream);
1602
+ res.once("close", () => upstream.destroy());
1603
+ upstream.on("error", drop);
1604
+ req.pipe(upstream);
1605
+ }).catch(drop);
1606
+ });
1607
+ server.on("connection", track);
1608
+ server.on("connect", (req, client, early) => {
1609
+ client.on("error", () => client.destroy());
1610
+ const to = authority(req.url ?? "");
1611
+ if (!to) {
1612
+ client.end("HTTP/1.1 400 Bad Request\r\n\r\n");
1613
+ return;
1614
+ }
1615
+ connect(to).then(({ socket, route }) => {
1616
+ if (!join(client, socket, route)) return;
1617
+ client.write("HTTP/1.1 200 Connection Established\r\n\r\n");
1618
+ if (early.length) socket.write(early);
1619
+ }).catch(() => {
1620
+ if (!client.destroyed) client.end("HTTP/1.1 502 Bad Gateway\r\n\r\n");
1405
1621
  });
1406
- }).describe("A matcher call: `expect(subject, label).not?.matcher(...args)`. Replies with its record."),
1407
- z.strictObject({
1408
- op: z.literal("record"),
1409
- matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a name such as assert").describe("What made the check in the eval's language, e.g. `assert`."),
1410
- subject: z.string().optional().describe("What it checked, as the eval wrote it (`result.ok`): names the check when it has no label."),
1411
- label: z.string().optional().describe("The check's own name, the same whether it passes or fails."),
1412
- status: z.enum(["pass", "fail"]),
1413
- message: z.string().optional().describe("Why it failed."),
1414
- soft: z.boolean().optional().describe("A failure fails the attempt at its end, as `expect.soft`. Default false.")
1415
- }).describe("A check the eval's own language made, such as Python's `assert`: recorded as a matcher call is, so it's named, counted and compared the same. Replies with its record."),
1416
- z.strictObject({
1417
- op: z.literal("judge"),
1418
- question: nonEmpty.describe("A yes/no question about the graded turn or value."),
1419
- options: judgeCallDataSchema.optional()
1420
- }).describe("`c.judge(question, options)`. Replies with its verdict, or a judge failure when the provider failed.")
1421
- ]).describe("One call an eval's body makes on its attempt. The sandbox and the agent session start on the first call that needs them; there's no call to start them.");
1422
- const attemptErrorSchema = z.looseObject({
1423
- name: z.string().describe("The error's class, e.g. `UsageError` or `AssertionError`."),
1424
- message: z.string()
1425
- }).describe("Why a call failed, in reeval's words.");
1426
- const id = nonEmpty.describe("Pairs a reply with its request.");
1427
- const attemptRequestSchema = z.strictObject({
1428
- id,
1429
- call: attemptCallSchema
1430
- }).describe("A call, numbered so its reply can find it.");
1431
- z.union([
1432
- z.discriminatedUnion("op", [
1433
- z.strictObject({
1434
- id,
1435
- ok: z.literal(true),
1436
- op: z.literal("send"),
1437
- turn: turnDataSchema
1438
- }),
1439
- z.strictObject({
1440
- id,
1441
- ok: z.literal(true),
1442
- op: z.literal("exec"),
1443
- result: z.union([execResultSchema, z.strictObject({ handle: handleId }).describe("A background process, still running.")])
1444
- }),
1445
- z.strictObject({
1446
- id,
1447
- ok: z.literal(true),
1448
- op: z.literal("serve"),
1449
- handle: handleId,
1450
- port: z.number().describe("The container port it was given.")
1451
- }),
1452
- z.strictObject({
1453
- id,
1454
- ok: z.literal(true),
1455
- op: z.literal("kill")
1456
- }),
1457
- z.strictObject({
1458
- id,
1459
- ok: z.literal(true),
1460
- op: z.literal("capture"),
1461
- capture: captureSchema
1462
- }),
1463
- z.strictObject({
1464
- id,
1465
- ok: z.literal(true),
1466
- op: z.literal("readFile"),
1467
- contents: z.string().nullable().describe("Null when there is no such file.")
1468
- }),
1469
- z.strictObject({
1470
- id,
1471
- ok: z.literal(true),
1472
- op: z.literal("writeFile")
1473
- }),
1474
- z.strictObject({
1475
- id,
1476
- ok: z.literal(true),
1477
- op: z.literal("expect"),
1478
- assertion: assertionRecordSchema
1479
- }),
1480
- z.strictObject({
1481
- id,
1482
- ok: z.literal(true),
1483
- op: z.literal("record"),
1484
- assertion: assertionRecordSchema
1485
- }),
1486
- z.strictObject({
1487
- id,
1488
- ok: z.literal(true),
1489
- op: z.literal("judge"),
1490
- verdict: z.union([verdictSchema, judgeFailureSchema])
1622
+ });
1623
+ server.on("upgrade", (req, client, early) => {
1624
+ client.on("error", () => client.destroy());
1625
+ const url = URL.canParse(req.url ?? "") ? new URL(req.url ?? "") : void 0;
1626
+ const to = url && authority(url.host);
1627
+ if (!url || !to) {
1628
+ client.end("HTTP/1.1 400 Bad Request\r\n\r\n");
1629
+ return;
1630
+ }
1631
+ const headers = forwarded(req.rawHeaders);
1632
+ const lines = [`${req.method} ${url.pathname}${url.search} HTTP/${req.httpVersion}`];
1633
+ for (let i = 0; i < headers.length; i += 2) lines.push(`${headers[i]}: ${headers[i + 1]}`);
1634
+ connect(to).then(({ socket, route }) => {
1635
+ if (!join(client, socket, route)) return;
1636
+ socket.write(`${lines.join("\r\n")}\r\n\r\n`);
1637
+ if (early.length) socket.write(early);
1638
+ }).catch(() => {
1639
+ if (!client.destroyed) client.end("HTTP/1.1 502 Bad Gateway\r\n\r\n");
1640
+ });
1641
+ });
1642
+ await new Promise((ok, fail) => {
1643
+ server.once("error", fail);
1644
+ server.listen(0, "127.0.0.1", () => ok());
1645
+ });
1646
+ const { port } = server.address();
1647
+ return {
1648
+ address: `127.0.0.1:${port}`,
1649
+ reached,
1650
+ close: () => new Promise((ok) => {
1651
+ closed = true;
1652
+ for (const s of live) s.destroy();
1653
+ server.close(() => ok());
1654
+ })
1655
+ };
1656
+ }
1657
+ //#endregion
1658
+ //#region src/capture/png.ts
1659
+ const CHANNELS = {
1660
+ 0: 1,
1661
+ 2: 3,
1662
+ 4: 2,
1663
+ 6: 4
1664
+ };
1665
+ /** An 8-bit, non-interlaced PNG's unfiltered pixels, or undefined for any other. */
1666
+ function pixels(png) {
1667
+ if (png.length < 33 || png.readUInt32BE(12) !== 1229472850) return void 0;
1668
+ const width = png.readUInt32BE(16);
1669
+ const height = png.readUInt32BE(20);
1670
+ const channels = CHANNELS[png[25] ?? -1];
1671
+ if (png[24] !== 8 || png[28] !== 0 || !channels) return void 0;
1672
+ const idat = [];
1673
+ for (let at = 8; at + 8 <= png.length;) {
1674
+ const length = png.readUInt32BE(at);
1675
+ const type = png.toString("latin1", at + 4, at + 8);
1676
+ if (type === "IDAT") idat.push(png.subarray(at + 8, at + 8 + length));
1677
+ if (type === "IEND") break;
1678
+ at += 12 + length;
1679
+ }
1680
+ const raw = inflateSync(Buffer.concat(idat));
1681
+ const stride = width * channels;
1682
+ const data = Buffer.alloc(stride * height);
1683
+ for (let y = 0; y < height; y++) {
1684
+ const filter = raw[y * (stride + 1)];
1685
+ const line = raw.subarray(y * (stride + 1) + 1, (y + 1) * (stride + 1));
1686
+ const row = y * stride;
1687
+ for (let x = 0; x < stride; x++) {
1688
+ const a = x >= channels ? data[row + x - channels] ?? 0 : 0;
1689
+ const b = y > 0 ? data[row - stride + x] ?? 0 : 0;
1690
+ const c = x >= channels && y > 0 ? data[row - stride + x - channels] ?? 0 : 0;
1691
+ let value = line[x] ?? 0;
1692
+ if (filter === 1) value += a;
1693
+ else if (filter === 2) value += b;
1694
+ else if (filter === 3) value += a + b >> 1;
1695
+ else if (filter === 4) {
1696
+ const p = a + b - c;
1697
+ const pa = Math.abs(p - a);
1698
+ const pb = Math.abs(p - b);
1699
+ const pc = Math.abs(p - c);
1700
+ value += pa <= pb && pa <= pc ? a : pb <= pc ? b : c;
1701
+ }
1702
+ data[row + x] = value & 255;
1703
+ }
1704
+ }
1705
+ return {
1706
+ width,
1707
+ height,
1708
+ channels,
1709
+ data
1710
+ };
1711
+ }
1712
+ /** The diff samples every `SAMPLE`th pixel across and down. */
1713
+ const SAMPLE = 4;
1714
+ /** A sampled pixel differs when a channel moves more than this, so antialiasing doesn't count. */
1715
+ const CHANNEL_DELTA = 32;
1716
+ /** The share of sampled pixels that must differ: a caret at any font size is below it. */
1717
+ const VIEW_CHANGE = .002;
1718
+ /**
1719
+ * Whether `after` looks different from `before`, two viewport shots: more than `VIEW_CHANGE` of
1720
+ * the sampled pixels differ, or the size does. Bytes decide when either PNG is one this can't read.
1721
+ */
1722
+ function pageChanged(before, after) {
1723
+ if (before.equals(after)) return false;
1724
+ let a;
1725
+ let b;
1726
+ try {
1727
+ a = pixels(before);
1728
+ b = pixels(after);
1729
+ } catch {
1730
+ return true;
1731
+ }
1732
+ if (!a || !b) return true;
1733
+ if (a.width !== b.width || a.height !== b.height || a.channels !== b.channels) return true;
1734
+ const stride = a.width * a.channels;
1735
+ let sampled = 0;
1736
+ let differ = 0;
1737
+ for (let y = 0; y < a.height; y += SAMPLE) for (let x = 0; x < a.width; x += SAMPLE) {
1738
+ sampled++;
1739
+ const at = y * stride + x * a.channels;
1740
+ for (let c = 0; c < a.channels; c++) if (Math.abs((a.data[at + c] ?? 0) - (b.data[at + c] ?? 0)) > CHANNEL_DELTA) {
1741
+ differ++;
1742
+ break;
1743
+ }
1744
+ }
1745
+ return differ > sampled * VIEW_CHANGE;
1746
+ }
1747
+ //#endregion
1748
+ //#region src/capture/page-state.ts
1749
+ const FIELDS = [
1750
+ "url",
1751
+ "focus",
1752
+ "value",
1753
+ "form",
1754
+ "text",
1755
+ "nodes",
1756
+ "view"
1757
+ ];
1758
+ /** Page text in the prompt is capped: the page controls it. */
1759
+ const TEXT_CAP = 60;
1760
+ const PROBE = `(() => {
1761
+ const hash = (s) => { let h = 2166136261; for (let i = 0; i < s.length; i++) { h ^= s.charCodeAt(i); h = Math.imul(h, 16777619) } return (h >>> 0).toString(36) }
1762
+ const clip = (s) => String(s ?? '').replace(/\\s+/g, ' ').trim().slice(0, ${TEXT_CAP})
1763
+ let a = document.activeElement
1764
+ while (a && a.shadowRoot && a.shadowRoot.activeElement) a = a.shadowRoot.activeElement
1765
+ if (a === document.body || a === document.documentElement) a = null
1766
+ const tag = a ? a.tagName.toLowerCase() : ''
1767
+ const type = a && tag === 'input' ? (a.type || 'text') : ''
1768
+ const label = a ? clip(a.getAttribute('aria-label') || (a.labels && a.labels[0] && a.labels[0].innerText) || a.getAttribute('placeholder') || a.getAttribute('name') || a.id || (tag === 'button' || tag === 'a' ? a.innerText : '')) : ''
1769
+ const toggles = type === 'checkbox' || type === 'radio'
1770
+ const value = !a ? null
1771
+ : toggles ? (a.checked ? 'checked' : 'unchecked')
1772
+ : tag === 'select' ? clip(a.selectedOptions && a.selectedOptions[0] ? a.selectedOptions[0].text : '')
1773
+ : type === 'password' ? a.value.length + ' characters'
1774
+ : tag === 'input' || tag === 'textarea' ? clip(a.value)
1775
+ : a.isContentEditable ? clip(a.innerText)
1776
+ : null
1777
+ const buttons = ['button', 'submit', 'reset', 'image', 'file', 'range', 'color', 'hidden']
1778
+ const editable = !!a && ((tag === 'input' && !toggles && !buttons.includes(type)) || tag === 'textarea' || a.isContentEditable)
1779
+ const controls = Array.from(document.querySelectorAll('input,select,textarea'))
1780
+ .map((e) => (e.type === 'checkbox' || e.type === 'radio' ? (e.checked ? '1' : '0') : e.value))
1781
+ return {
1782
+ url: location.href,
1783
+ focus: a ? tag + (type ? '[' + type + ']' : '') + (label ? ' ' + JSON.stringify(label) : '') : null,
1784
+ value,
1785
+ editable,
1786
+ form: hash(controls.join('\\u0001')),
1787
+ text: hash(document.body ? document.body.innerText : ''),
1788
+ nodes: document.getElementsByTagName('*').length,
1789
+ }
1790
+ })()`;
1791
+ /** Reads the page. A field it can't read is left out. */
1792
+ async function probePage(session) {
1793
+ const read = await session.cdp.send("Runtime.evaluate", {
1794
+ expression: PROBE,
1795
+ returnByValue: true
1796
+ }).catch(() => void 0);
1797
+ const view = await session.screenshot(false).catch(() => void 0);
1798
+ return {
1799
+ ...read?.result?.value ?? {},
1800
+ ...view ? { view } : {}
1801
+ };
1802
+ }
1803
+ /** The fields that differ between two states, ignoring those `volatile` names. */
1804
+ function differing(before, after, volatile = /* @__PURE__ */ new Set()) {
1805
+ return FIELDS.filter((field) => {
1806
+ if (volatile.has(field)) return false;
1807
+ const a = before[field];
1808
+ const b = after[field];
1809
+ if (a === void 0 || b === void 0) return a !== b;
1810
+ if (field === "view") return pageChanged(a, b);
1811
+ return a !== b;
1812
+ });
1813
+ }
1814
+ /** The page's path, without its query or fragment, and capped. */
1815
+ function pathOf(url, origin) {
1816
+ if (!url) return void 0;
1817
+ const local = url.startsWith(`${origin}/`) ? url.slice(origin.length) : url;
1818
+ const cut = local.search(/[?#]/);
1819
+ const path = cut === -1 ? local : `${local.slice(0, cut)}${local[cut] === "?" ? "?…" : ""}`;
1820
+ return path.length > TEXT_CAP ? `${path.slice(0, TEXT_CAP)}…` : path;
1821
+ }
1822
+ /** What changed between two states, in the words the model reads. */
1823
+ function describeChange(after, fields, origin) {
1824
+ const notes = [];
1825
+ const has = (f) => fields.includes(f);
1826
+ if (has("url")) notes.push(`now at ${pathOf(after.url, origin) ?? "a page reeval could not read"}`);
1827
+ if (has("focus")) notes.push(after.focus ? `focus moved to ${after.focus}` : "nothing is focused now");
1828
+ if (after.focus && (has("value") || has("focus") && after.value)) notes.push(`${after.focus} holds ${JSON.stringify(after.value ?? "")}`);
1829
+ else if (has("form")) notes.push("a form control changed");
1830
+ if (has("text") || has("nodes")) notes.push("the page's content changed");
1831
+ else if (has("view") && !notes.length) notes.push("the page looks different");
1832
+ return notes;
1833
+ }
1834
+ /**
1835
+ * Waits for the page to stop changing after a move: two reads `SETTLE_POLL_MS` apart that agree,
1836
+ * with no navigation or fresh request in flight. At `SETTLE_CAP_MS` it gives up and adds the
1837
+ * fields still changing to `volatile`, so a clock or an animation doesn't hold up every move.
1838
+ */
1839
+ async function settledState(session, volatile, left) {
1840
+ const start = Date.now();
1841
+ const sleep = () => new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.min(100, left()))));
1842
+ let prev = await probePage(session);
1843
+ for (;;) {
1844
+ if (left() <= 0) return prev;
1845
+ await sleep();
1846
+ const next = await probePage(session);
1847
+ const moving = differing(prev, next, volatile);
1848
+ if (!moving.length && !session.busy()) return next;
1849
+ if (Date.now() - start >= 3e3 || left() <= 0) {
1850
+ for (const field of moving) if (field !== "url" && field !== "focus") volatile.add(field);
1851
+ return next;
1852
+ }
1853
+ prev = next;
1854
+ }
1855
+ }
1856
+ /** Points within this many CSS pixels of each other are the same move. */
1857
+ const SAME_POINT = 8;
1858
+ /** Model turns in a row thrown away because the page changed while the model chose. */
1859
+ const STALE_ASKS = 2;
1860
+ function timedOut(clock) {
1861
+ const { step } = clock;
1862
+ const where = step ? ` (instruction ${step.index} of ${step.total}, after ${plural(step.moves, "move")})` : "";
1863
+ return /* @__PURE__ */ new Error(`c.capture() timed out on ${clock.detail}${where}: it used all of its ${formatDuration(clock.budgetMs)} timeout. Raise timeout on the call or capture.timeout.`);
1864
+ }
1865
+ function check(clock) {
1866
+ const left = clock.remaining();
1867
+ if (left <= 0) throw timedOut(clock);
1868
+ return left;
1869
+ }
1870
+ /** `work()`, or the timeout once the budget runs out first: a page can hold any CDP call open. */
1871
+ function race(clock, work) {
1872
+ const left = clock.remaining();
1873
+ if (left <= 0) return Promise.reject(timedOut(clock));
1874
+ let timer;
1875
+ const out = new Promise((_, reject) => {
1876
+ timer = setTimeout(() => reject(timedOut(clock)), left);
1877
+ });
1878
+ return Promise.race([work(), out]).finally(() => clearTimeout(timer));
1879
+ }
1880
+ /** `session` with every call raced against the clock, and its own waits' timeouts named as the budget's. */
1881
+ function onClock(session, clock) {
1882
+ const own = (error) => {
1883
+ throw /c\.capture\(\) timed out/.test(String(error)) ? timedOut(clock) : error;
1884
+ };
1885
+ return {
1886
+ window: session.window,
1887
+ cdp: {
1888
+ on: (method, fn) => session.cdp.on(method, fn),
1889
+ send: (method, params) => race(clock, () => session.cdp.send(method, params))
1890
+ },
1891
+ navigate: (url) => race(clock, () => session.navigate(url)),
1892
+ wait: (wait, ms, nav) => race(clock, () => session.wait(wait, ms, nav).catch(own)),
1893
+ screenshot: (fullPage) => race(clock, () => session.screenshot(fullPage)),
1894
+ dispatch: (action, page) => race(clock, () => session.dispatch(action, page)),
1895
+ busy: () => session.busy(),
1896
+ dialogs: () => session.dialogs()
1897
+ };
1898
+ }
1899
+ function asUsage(value) {
1900
+ return {
1901
+ input: typeof value.inputTokens === "number" ? value.inputTokens : value.inputTokens?.total ?? 0,
1902
+ output: typeof value.outputTokens === "number" ? value.outputTokens : value.outputTokens?.total ?? 0,
1903
+ cacheRead: 0,
1904
+ cacheWrite: 0,
1905
+ reasoning: null
1906
+ };
1907
+ }
1908
+ const finite = (value) => {
1909
+ const n = typeof value === "number" ? value : typeof value === "string" ? Number(value) : NaN;
1910
+ return Number.isFinite(n) ? n : void 0;
1911
+ };
1912
+ /** The point at `x`/`y`, or both packed into one as `"217,163"`, which models sometimes send. */
1913
+ function point(input, x, y) {
1914
+ const packed = typeof input[x] === "string" ? input[x].split(",") : [];
1915
+ const px = finite(packed.length === 2 ? packed[0] : input[x]);
1916
+ const py = finite(packed.length === 2 ? packed[1] : input[y]);
1917
+ return px === void 0 || py === void 0 ? void 0 : [px, py];
1918
+ }
1919
+ /** The action a tool call asks for, or why it can't be made. */
1920
+ function actionFromTool(name, input) {
1921
+ if (name === "done") return { type: "done" };
1922
+ if (name === "type") {
1923
+ if (typeof input.text !== "string") return { refused: "it had no text" };
1924
+ return {
1925
+ type: "type",
1926
+ text: input.text,
1927
+ ...input.replace === true ? { replace: true } : {}
1928
+ };
1929
+ }
1930
+ if (name === "key") {
1931
+ const key = CAPTURE_KEYS.find((k) => k === input.key);
1932
+ if (!key) return { refused: `the key is not one of ${CAPTURE_KEYS.join(", ")}` };
1933
+ return {
1934
+ type: "key",
1935
+ key
1936
+ };
1937
+ }
1938
+ const at = point(input, "x", "y");
1939
+ if (!at) return { refused: "it had no numeric x and y" };
1940
+ const [x, y] = at;
1941
+ if (name === "scroll") {
1942
+ const dy = finite(input.dy);
1943
+ return dy === void 0 ? { refused: "it had no numeric dy" } : {
1944
+ type: "scroll",
1945
+ x,
1946
+ y,
1947
+ dy
1948
+ };
1949
+ }
1950
+ if (name === "drag") {
1951
+ const to = point(input, "toX", "toY");
1952
+ if (!to) return { refused: "it had no numeric toX and toY" };
1953
+ return {
1954
+ type: "drag",
1955
+ x,
1956
+ y,
1957
+ toX: to[0],
1958
+ toY: to[1]
1959
+ };
1960
+ }
1961
+ if (name === "hover") return {
1962
+ type: "hover",
1963
+ x,
1964
+ y
1965
+ };
1966
+ return {
1967
+ type: "click",
1968
+ x,
1969
+ y
1970
+ };
1971
+ }
1972
+ /**
1973
+ * Gemini models answer a point normalized to 0–1000 across the image they were shown, whatever
1974
+ * the prompt asks, so their journeys are prompted in that scale and their points scaled back.
1975
+ */
1976
+ const normalizesPoints = (model) => model.slice(model.lastIndexOf("/") + 1).toLowerCase().startsWith("gemini-");
1977
+ const NORMALIZED = 1e3;
1978
+ /** Scales an action's 0–1000 points to CSS pixels on a `page`-sized screenshot. */
1979
+ function scalePoints(action, page) {
1980
+ const x = (n) => Math.round(n * page.width / NORMALIZED);
1981
+ const y = (n) => Math.round(n * page.height / NORMALIZED);
1982
+ switch (action.type) {
1983
+ case "click":
1984
+ case "hover":
1985
+ case "scroll": return {
1986
+ ...action,
1987
+ x: x(action.x),
1988
+ y: y(action.y)
1989
+ };
1990
+ case "drag": return {
1991
+ ...action,
1992
+ x: x(action.x),
1993
+ y: y(action.y),
1994
+ toX: x(action.toX),
1995
+ toY: y(action.toY)
1996
+ };
1997
+ default: return action;
1998
+ }
1999
+ }
2000
+ const systemPrompt = (normalized) => `You drive a web page from screenshots. Call one tool. ${normalized ? `Coordinates are normalized to 0-${NORMALIZED} on the full-page image: x and toX across its width, y and toY down its height. A scroll's dy is in CSS pixels.` : "Coordinates are CSS pixels on the full-page image."} Each turn you get the goal, what has focus, your earlier moves with what each did, and a fresh screenshot. Click a field before typing into it, and do not repeat a move that already worked. A dropdown (select) never opens in the screenshot: click it once, press ArrowDown or ArrowUp until it shows the option you want, then press Enter to choose it. Call done when the goal is met. Do not mention CDP.`;
2001
+ const useTools = {
2002
+ click: tool({
2003
+ description: "Click at (x, y) on the screenshot.",
2004
+ inputSchema: z.object({
2005
+ x: z.number(),
2006
+ y: z.number()
1491
2007
  })
1492
- ]),
1493
- z.strictObject({
1494
- id,
1495
- ok: z.literal(true),
1496
- op: z.literal("expect"),
1497
- untested: z.record(z.string(), z.array(z.string())).describe("The strings the matcher tested that a `{ matchedBy, answers }` argument has no answer for, by its `matchedBy`. Nothing was recorded: answer them and make the call again.")
1498
2008
  }),
1499
- z.strictObject({
1500
- id,
1501
- ok: z.literal(false),
1502
- op: z.enum([
1503
- "send",
1504
- "exec",
1505
- "readFile",
1506
- "writeFile",
1507
- "expect",
1508
- "record",
1509
- "judge",
1510
- "serve",
1511
- "kill",
1512
- "capture"
1513
- ]),
1514
- error: attemptErrorSchema
2009
+ type: tool({
2010
+ description: "Insert text into the focused field. Click the field first. Set replace to true to replace what the field already holds instead of adding to it.",
2011
+ inputSchema: z.object({
2012
+ text: z.string(),
2013
+ replace: z.boolean().optional()
2014
+ })
2015
+ }),
2016
+ key: tool({
2017
+ description: "Press a key in the focused element. In a dropdown (select), click it, press ArrowDown or ArrowUp to the option, then Enter to choose it. Space toggles a focused checkbox.",
2018
+ inputSchema: z.object({ key: z.enum(CAPTURE_KEYS) })
2019
+ }),
2020
+ scroll: tool({
2021
+ description: "Wheel-scroll a constrained scroller at (x, y).",
2022
+ inputSchema: z.object({
2023
+ x: z.number(),
2024
+ y: z.number(),
2025
+ dy: z.number()
2026
+ })
2027
+ }),
2028
+ drag: tool({
2029
+ description: "Drag from (x, y) to (toX, toY).",
2030
+ inputSchema: z.object({
2031
+ x: z.number(),
2032
+ y: z.number(),
2033
+ toX: z.number(),
2034
+ toY: z.number()
2035
+ })
2036
+ }),
2037
+ hover: tool({
2038
+ description: "Move the pointer to (x, y).",
2039
+ inputSchema: z.object({
2040
+ x: z.number(),
2041
+ y: z.number()
2042
+ })
2043
+ }),
2044
+ done: tool({
2045
+ description: "The goal is finished. Leave the loop.",
2046
+ inputSchema: z.object({})
1515
2047
  })
1516
- ]).describe("reeval's reply to a call: its result, or the error it threw. A failed matcher is a result (its record says `fail`), not an error, unless the matcher couldn't run.");
1517
- const cellRoleSchema = z.enum(["control", "variant"]);
1518
- const cellSchema = z.strictObject({
1519
- key: z.string(),
1520
- harness: z.string(),
1521
- environment: z.string().describe("The environment's own name, also on a variant cell (`with-house-greeting`, with no prefix)."),
1522
- sandbox: z.string(),
1523
- role: cellRoleSchema,
1524
- variant: z.string().optional().describe("The variant's name, for variant cells.")
1525
- }).describe("One cell of an eval: a harness, an environment (`-` when none is set) and that environment's image (`sandbox` id: tag, dockerfile and install). With `compare`, the control is `baseline` (or the cascaded environment, or an empty `baseline`) and each variant is that environment on its own, including its sandbox.");
1526
- z.strictObject({
1527
- type: z.literal("start"),
1528
- attempt: z.strictObject({
1529
- id: nonEmpty,
1530
- cell: cellSchema,
1531
- n: z.number().int().min(1, "a run number, from 1").describe("The run number, from 1.")
1532
- }).describe("`c.attempt`: which cell of the matrix this attempt is, and its run number."),
1533
- file: nonEmpty.describe("The eval file, relative to the project."),
1534
- body: nonEmpty.describe("The eval's `body`, as its declaration named it.")
1535
- }).describe("reeval to the runner: run this eval's body for this attempt. Nothing has started yet: the sandbox and the agent session start on the body's first call that needs them, so its own code runs first. The runner sends its calls, then `done`.");
1536
- z.strictObject({
1537
- type: z.literal("abort"),
1538
- reason: z.enum(["timeout", "cancelled"]).describe("`timeout`: the eval's `timeout` ran out. `cancelled`: the run was stopped."),
1539
- message: z.string().describe("reeval's sentence, e.g. `Eval timed out after 600s`.")
1540
- }).describe("reeval to the runner: the attempt was aborted, as `c.signal` aborts. Calls in flight reply with errors; the runner should stop the body and send `done`.");
1541
- z.strictObject({
1542
- type: z.literal("done"),
1543
- error: attemptErrorSchema.optional().describe("What the body threw, if it threw. A failed `expect` (not soft) is one.")
1544
- }).describe("The runner to reeval: the eval's body has finished. The attempt's checks and record follow.");
1545
- const CALL_KEYS = Object.fromEntries(attemptCallSchema.options.map((o) => [o.shape.op.value, Object.keys(o.shape)]));
1546
- const OTHER_KEYS = [
1547
- "id",
1548
- "call",
1549
- "turn",
1550
- "attempt",
1551
- "value",
1552
- "regexp",
1553
- "flags",
1554
- "matchedBy",
1555
- "answers",
1556
- ...Object.keys(judgeCallDataSchema.shape),
1557
- ...Object.keys(judgeStateSchema.shape),
1558
- ...Object.keys(captureSchema.shape),
1559
- ...Object.keys(captureCallDataSchema.shape),
1560
- "port",
1561
- "handle",
1562
- "exec",
1563
- "result",
1564
- "readFile",
1565
- "interval",
1566
- "timeout",
1567
- "ms"
1568
- ];
1569
- /** What's wrong with an attempt request a runner sent, by the field's path, or undefined. */
1570
- const attemptRequestProblem = (value) => dataProblem(attemptRequestSchema, value, "The attempt request", (object) => typeof object.op === "string" ? CALL_KEYS[object.op] ?? [] : OTHER_KEYS);
2048
+ };
2049
+ async function runFakeScript(session, script, step, clock, volatile) {
2050
+ const goal = step.goal;
2051
+ let moves = 0;
2052
+ for (const action of script) {
2053
+ clock.step = {
2054
+ ...step,
2055
+ moves
2056
+ };
2057
+ check(clock);
2058
+ if (action.type === "done") return moves;
2059
+ if (moves >= 32) throw unfinished(step);
2060
+ const page = pngSize(await session.screenshot(true));
2061
+ debugTrace("check", "capture-use", {
2062
+ goal,
2063
+ action,
2064
+ page
2065
+ }, `capture use ${action.type}`);
2066
+ await session.dispatch(action, page);
2067
+ moves++;
2068
+ await settledState(session, volatile, clock.remaining);
2069
+ }
2070
+ clock.step = {
2071
+ ...step,
2072
+ moves
2073
+ };
2074
+ while (clock.remaining() > 0) await new Promise((resolve) => setTimeout(resolve, Math.min(100, clock.remaining())));
2075
+ throw timedOut(clock);
2076
+ }
2077
+ const unfinished = (step) => /* @__PURE__ */ new Error(`c.capture() instruction ${step.index} of ${step.total} ${JSON.stringify(step.goal)} did not finish in 32 moves`);
2078
+ const grid = (n) => Math.round(n / SAME_POINT);
2079
+ function moveKey(action) {
2080
+ switch (action.type) {
2081
+ case "click":
2082
+ case "hover": return JSON.stringify([
2083
+ action.type,
2084
+ grid(action.x),
2085
+ grid(action.y)
2086
+ ]);
2087
+ case "scroll": return JSON.stringify([
2088
+ action.type,
2089
+ grid(action.x),
2090
+ grid(action.y),
2091
+ action.dy
2092
+ ]);
2093
+ case "drag": return JSON.stringify([
2094
+ action.type,
2095
+ grid(action.x),
2096
+ grid(action.y),
2097
+ grid(action.toX),
2098
+ grid(action.toY)
2099
+ ]);
2100
+ default: return JSON.stringify(action);
2101
+ }
2102
+ }
2103
+ /** Sets what `move` did, from its page before and `after`. */
2104
+ function settleMove(move, after, origin, volatile) {
2105
+ move.after = after;
2106
+ const fields = move.before ? differing(move.before, after, volatile) : [];
2107
+ const notes = move.before ? describeChange(after, fields, origin) : [];
2108
+ for (const dialog of move.dialogs) notes.push(`the page opened ${dialog}, which reeval accepted`);
2109
+ move.changed = fields.length > 0 || move.dialogs.length > 0;
2110
+ move.effect = notes.length ? notes.join("; ") : "nothing changed";
2111
+ }
2112
+ /** The same move made, unchanged, at the end of `moves`: a refused move ends the run. */
2113
+ function stuckMoves(moves) {
2114
+ const last = moves.at(-1);
2115
+ let n = 0;
2116
+ for (let i = moves.length - 1; i >= 0; i--) {
2117
+ const move = moves[i];
2118
+ if (!move || move.refused || move.key !== last?.key || move.changed !== false) break;
2119
+ n++;
2120
+ }
2121
+ return n;
2122
+ }
2123
+ /** What the model reads about the page now: where it is and what has focus. */
2124
+ function stateLines(now, origin, volatile) {
2125
+ const lines = [];
2126
+ const path = pathOf(now.url, origin);
2127
+ if (path) lines.push(`The page is at ${path}.`);
2128
+ if (now.focus === null) lines.push("Nothing is focused.");
2129
+ else if (now.focus) {
2130
+ lines.push(`Focused: ${now.focus}${now.value ? `, holding ${JSON.stringify(now.value)}` : ""}.`);
2131
+ if (now.focus.startsWith("select")) lines.push("A dropdown never opens in the screenshot: press ArrowDown or ArrowUp to the option, then Enter; it holds the new option only after Enter.");
2132
+ }
2133
+ if (volatile.has("text") || volatile.has("nodes") || volatile.has("view")) lines.push("Parts of the page keep changing on their own, so the effects below leave them out.");
2134
+ return lines;
2135
+ }
2136
+ /** The goal, where the journey is, the page now, and the moves so far with what each did. */
2137
+ function goalPrompt(step, earlier, now, moves, origin, volatile = /* @__PURE__ */ new Set()) {
2138
+ const lines = [`Goal: ${step.goal}`];
2139
+ if (step.total > 1) lines.push(`This is instruction ${step.index} of ${step.total}.${earlier.length ? ` Already done: ${earlier.map((g) => JSON.stringify(g)).join(", ")}.` : ""}`);
2140
+ lines.push(...stateLines(now, origin, volatile));
2141
+ if (!moves.length) {
2142
+ lines.push("You have made no moves for this goal yet.");
2143
+ return lines.join("\n");
2144
+ }
2145
+ lines.push("Your moves for this goal so far, oldest first. The screenshot shows the page now.");
2146
+ moves.forEach((m, i) => {
2147
+ const effect = m.refused ? `not made: ${m.refused}` : m.effect ?? "";
2148
+ lines.push(`${i + 1}. ${m.said}${effect ? ` → ${effect}` : ""}`);
2149
+ });
2150
+ if (stuckMoves(moves) >= 2) lines.push("Your last moves repeated the same action and nothing changed. Do not repeat it: if the goal is met, call done; otherwise try a different action.");
2151
+ return lines.join("\n");
2152
+ }
2153
+ async function runModelGoal(session, { step, earlier, origin, modelName, clock, volatile, pristine, loadModel }) {
2154
+ const { goal } = step;
2155
+ const model = await loadModel(modelName);
2156
+ const normalized = normalizesPoints(modelName);
2157
+ const made = [];
2158
+ let dispatched = 0;
2159
+ let usage = null;
2160
+ let now = await probePage(session);
2161
+ let stale = 0;
2162
+ while (made.length < 32) {
2163
+ clock.step = {
2164
+ ...step,
2165
+ moves: dispatched
2166
+ };
2167
+ check(clock);
2168
+ const last = made.at(-1);
2169
+ if (last?.after) {
2170
+ const late = await probePage(session);
2171
+ last.dialogs.push(...session.dialogs());
2172
+ if (differing(last.after, late, volatile).length || last.dialogs.length) settleMove(last, late, origin, volatile);
2173
+ now = late;
2174
+ }
2175
+ if (stuckMoves(made) >= 5) throw new Error(`c.capture() instruction ${step.index} of ${step.total} ${JSON.stringify(goal)} repeated ${last?.said} 5 times and nothing changed`);
2176
+ const ephemeral = await session.screenshot(true);
2177
+ const page = pngSize(ephemeral);
2178
+ debugTrace("check", "capture-shot", {
2179
+ goal,
2180
+ bytes: ephemeral.length,
2181
+ page,
2182
+ url: now.url
2183
+ }, false);
2184
+ const signal = AbortSignal.timeout(check(clock));
2185
+ const asked = Date.now();
2186
+ const result = await race(clock, () => generateText({
2187
+ model,
2188
+ system: systemPrompt(normalized),
2189
+ messages: [{
2190
+ role: "user",
2191
+ content: [{
2192
+ type: "text",
2193
+ text: goalPrompt(step, earlier, now, made, origin, volatile)
2194
+ }, {
2195
+ type: "file",
2196
+ mediaType: "image/png",
2197
+ data: ephemeral
2198
+ }]
2199
+ }],
2200
+ tools: useTools,
2201
+ toolChoice: "required",
2202
+ stopWhen: stepCountIs(1),
2203
+ abortSignal: signal
2204
+ })).catch((error) => {
2205
+ throw signal.aborted ? timedOut(clock) : error;
2206
+ });
2207
+ usage = addUsage(usage, asUsage(result.usage ?? {}));
2208
+ const call = result.toolCalls?.[0];
2209
+ if (!call) throw new Error(`c.capture() instruction ${JSON.stringify(goal)} returned no action`);
2210
+ const input = call.input ?? {};
2211
+ const said = `${call.toolName} ${JSON.stringify(input)}`;
2212
+ const fresh = await probePage(session);
2213
+ const opened = session.dialogs();
2214
+ const moved = differing(now, fresh, volatile);
2215
+ if (!last) {
2216
+ if (pristine) {
2217
+ for (const f of moved) if (f !== "url" && f !== "focus") volatile.add(f);
2218
+ }
2219
+ } else if ((moved.length || opened.length) && stale < STALE_ASKS) {
2220
+ stale++;
2221
+ if (!last.refused) {
2222
+ last.dialogs.push(...opened);
2223
+ settleMove(last, fresh, origin, volatile);
2224
+ }
2225
+ now = fresh;
2226
+ debugLog("check", `capture use ${said} on ${JSON.stringify(goal)} not made: the page changed while the model chose`);
2227
+ continue;
2228
+ }
2229
+ stale = 0;
2230
+ let asks = actionFromTool(call.toolName, input);
2231
+ if (!("refused" in asks) && asks.type === "type" && now.editable === false) asks = { refused: "nothing that takes text is focused; click a field first" };
2232
+ const logged = !("refused" in asks) && asks.type === "type" && now.focus?.includes("[password]") ? `type ${JSON.stringify({
2233
+ ...input,
2234
+ text: "•".repeat(String(input.text ?? "").length)
2235
+ })}` : said;
2236
+ const latency = formatDuration(Date.now() - asked);
2237
+ if ("refused" in asks) {
2238
+ debugLog("check", `capture use ${logged} on ${JSON.stringify(goal)} not made: ${asks.refused} · ${latency}`);
2239
+ made.push({
2240
+ said,
2241
+ key: said,
2242
+ refused: asks.refused,
2243
+ dialogs: []
2244
+ });
2245
+ continue;
2246
+ }
2247
+ const chosen = normalized ? scalePoints(asks, page) : asks;
2248
+ debugLog("check", `capture use ${logged} on ${JSON.stringify(goal)} · ${latency}`);
2249
+ debugTrace("check", "capture-use", {
2250
+ goal,
2251
+ action: chosen,
2252
+ page
2253
+ }, `capture use ${chosen.type}`);
2254
+ if (chosen.type === "done") return {
2255
+ moves: dispatched,
2256
+ usage
2257
+ };
2258
+ const move = {
2259
+ said,
2260
+ key: moveKey(asks),
2261
+ before: now,
2262
+ dialogs: []
2263
+ };
2264
+ made.push(move);
2265
+ await session.dispatch(chosen, page);
2266
+ dispatched++;
2267
+ now = await settledState(session, volatile, clock.remaining);
2268
+ move.dialogs.push(...session.dialogs());
2269
+ settleMove(move, now, origin, volatile);
2270
+ }
2271
+ throw unfinished(step);
2272
+ }
2273
+ /**
2274
+ * Top-level wait, then the instruction loop, then the saved PNG.
2275
+ * Throws before `screenshot` when the budget or move cap is hit.
2276
+ */
2277
+ async function runJourney(raw, url, options) {
2278
+ const { wait, timeoutMs, fullPage, instructions, use, remaining } = options;
2279
+ const clock = {
2280
+ budgetMs: options.budgetMs ?? timeoutMs,
2281
+ remaining,
2282
+ detail: wait === "idle" ? "loading the page with wait: \"idle\"" : "loading the page"
2283
+ };
2284
+ const session = onClock(raw, clock);
2285
+ const origin = new URL(url).origin;
2286
+ const volatile = /* @__PURE__ */ new Set();
2287
+ check(clock);
2288
+ await navigateAndWait(session, url, wait, Math.min(timeoutMs, remaining())).catch((error) => {
2289
+ throw /c\.capture\(\) timed out/.test(String(error)) ? timedOut(clock) : error;
2290
+ });
2291
+ const goals = instructions.filter((s) => typeof s === "string");
2292
+ let moves = 0;
2293
+ let usage = null;
2294
+ let stringIndex = 0;
2295
+ for (const instruction of instructions) {
2296
+ if (typeof instruction !== "string") {
2297
+ clock.detail = `{ wait: ${JSON.stringify(instruction.wait)} }`;
2298
+ clock.step = void 0;
2299
+ check(clock);
2300
+ await session.wait(instruction.wait, remaining(), void 0);
2301
+ continue;
2302
+ }
2303
+ const step = {
2304
+ goal: instruction,
2305
+ index: stringIndex + 1,
2306
+ total: goals.length,
2307
+ moves: 0
2308
+ };
2309
+ clock.detail = JSON.stringify(instruction);
2310
+ clock.step = step;
2311
+ check(clock);
2312
+ debugLog("check", `capture instruction ${step.index} of ${step.total} ${JSON.stringify(instruction)}`);
2313
+ if (use && "fake" in use) {
2314
+ const script = use.fake[stringIndex] ?? [];
2315
+ moves += await runFakeScript(session, script, step, clock, volatile);
2316
+ } else if (use && "model" in use) {
2317
+ if (!options.loadModel) throw new Error("c.capture() instructions need a model loader");
2318
+ const ran = await runModelGoal(session, {
2319
+ step,
2320
+ earlier: goals.slice(0, stringIndex),
2321
+ origin,
2322
+ modelName: use.model,
2323
+ clock,
2324
+ volatile,
2325
+ pristine: moves === 0,
2326
+ loadModel: options.loadModel
2327
+ });
2328
+ moves += ran.moves;
2329
+ usage = addUsage(usage, ran.usage);
2330
+ }
2331
+ stringIndex++;
2332
+ }
2333
+ clock.detail = "the final screenshot";
2334
+ clock.step = void 0;
2335
+ check(clock);
2336
+ return {
2337
+ png: await session.screenshot(fullPage),
2338
+ moves,
2339
+ model: use && "model" in use ? use.model : use && "fake" in use ? "fake" : void 0,
2340
+ usage
2341
+ };
2342
+ }
2343
+ function captureUseSpend(result) {
2344
+ if (!result.model) return void 0;
2345
+ return {
2346
+ model: result.model,
2347
+ moves: result.moves,
2348
+ usage: result.usage
2349
+ };
2350
+ }
1571
2351
  //#endregion
1572
2352
  //#region src/diff.ts
1573
- const GIT = "git -c user.name=reeval -c user.email=reeval@localhost -c core.hooksPath=/dev/null -c core.logAllRefUpdates=false -c gc.auto=0";
2353
+ const GIT = "git -c user.name=reeval -c user.email=reeval@localhost -c core.hooksPath=/dev/null -c core.logAllRefUpdates=false -c gc.auto=0 -c core.quotePath=false";
1574
2354
  const snapshotEnv = (sandbox) => `export GIT_DIR=${shellQuote(`${sandbox.stateDir}/workspace.git`)} GIT_WORK_TREE=${shellQuote(sandbox.workdir)} GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null`;
1575
2355
  /** Points HEAD at a commit of `$tree` with no parent, then drops every object nothing reaches. */
1576
2356
  const RESET_HEAD = `c=$(${GIT} commit-tree -m base "$tree") && ${GIT} update-ref HEAD "$c" && ${GIT} prune --expire=now`;
@@ -1598,20 +2378,22 @@ async function acceptIntoSnapshot(sandbox, path, signal) {
1598
2378
  async function diffSinceSnapshot(sandbox, signal) {
1599
2379
  const env = `${snapshotEnv(sandbox)} && ${GIT} add -A && `;
1600
2380
  const patch = await sandbox.execOwn(`${env}${GIT} diff --cached --binary HEAD`, { signal });
1601
- const names = await sandbox.execOwn(`${env}${GIT} diff --cached --name-status --no-renames HEAD`, { signal });
2381
+ const names = await sandbox.execOwn(`${env}${GIT} diff --cached --name-status --no-renames -z HEAD`, { signal });
1602
2382
  if (patch.exitCode !== 0 || names.exitCode !== 0) throw new InfraError(`Failed to diff the workspace: ${patch.stderr || names.stderr}`);
1603
2383
  const status = {
1604
2384
  A: "added",
1605
2385
  M: "modified",
1606
2386
  D: "deleted"
1607
2387
  };
1608
- const files = names.stdout.split("\n").filter(Boolean).map((line) => {
1609
- const [code = "M", ...rest] = line.split(" ");
1610
- return {
1611
- path: rest.join(" "),
2388
+ const fields = names.stdout.split("\0");
2389
+ const files = [];
2390
+ for (let i = 0; i + 1 < fields.length; i += 2) {
2391
+ const code = fields[i];
2392
+ files.push({
2393
+ path: fields[i + 1],
1612
2394
  status: status[code[0]] ?? "modified"
1613
- };
1614
- });
2395
+ });
2396
+ }
1615
2397
  return {
1616
2398
  patch: patch.stdout,
1617
2399
  files
@@ -1656,7 +2438,7 @@ function createChecks(a) {
1656
2438
  durationMs: Math.round(performance.now() - began)
1657
2439
  });
1658
2440
  if (r.exitCode === 0 || tries.length > retries) return r;
1659
- debugLog("check", `$ ${clip(command, 160)} → exit ${r.exitCode} on try ${tries.length} of ${retries + 1}, again in ${formatDuration(interval)}`);
2441
+ debugLog("check", `$ ${clip(redactSecrets(command), 160)} → exit ${r.exitCode} on try ${tries.length} of ${retries + 1}, again in ${formatDuration(interval)}`);
1660
2442
  await setTimeout$1(interval, void 0, { signal: a.signal });
1661
2443
  }
1662
2444
  });
@@ -1672,13 +2454,13 @@ function createChecks(a) {
1672
2454
  ...retried
1673
2455
  });
1674
2456
  const of = retries > 0 ? ` on try ${tries.length} of ${retries + 1}` : "";
1675
- debugLog("check", `$ ${clip(command, 160)} → exit ${r.exitCode}${of} · ${formatDuration(durationMs)}`);
2457
+ debugLog("check", `$ ${clip(redactSecrets(command), 160)} → exit ${r.exitCode}${of} · ${formatDuration(durationMs)}`);
1676
2458
  debugTrace("check", "output", {
1677
2459
  command,
1678
2460
  ...r,
1679
2461
  durationMs,
1680
2462
  ...retried
1681
- }, `check output: ${clip(command, 80)}`);
2463
+ }, `check output: ${clip(redactSecrets(command), 80)}`);
1682
2464
  a.report({
1683
2465
  kind: "check-end",
1684
2466
  command,
@@ -1689,12 +2471,12 @@ function createChecks(a) {
1689
2471
  tries: retries + 1
1690
2472
  } : {}
1691
2473
  });
1692
- return r;
2474
+ return Object.defineProperty(r, SUBJECT, { value: command });
1693
2475
  };
1694
2476
  function exec(command, options) {
1695
2477
  const problem = options === void 0 ? void 0 : execOptionsProblem(options);
1696
2478
  if (problem) return Promise.reject(new UsageError(problem));
1697
- if (options?.background) return import("./capture-CherbjzB.js").then(({ startBackground }) => startBackground(a, command, "c.exec({ background: true })"));
2479
+ if (options?.background) return import("./capture-px7Ey95_.js").then(({ startBackground }) => startBackground(a, command, "c.exec({ background: true })"));
1698
2480
  const run = execForeground(command, options?.retries ?? 0, options?.interval ?? 1e3);
1699
2481
  return Object.defineProperty(run, SUBJECT, { value: command });
1700
2482
  }
@@ -1718,7 +2500,7 @@ function createChecks(a) {
1718
2500
  }
1719
2501
  //#endregion
1720
2502
  //#region src/attempt/capture.ts
1721
- const BG_LOG = "/tmp/reeval-bg.log";
2503
+ const BG_LOG = "/tmp/reeval-bg";
1722
2504
  const BG_TAG = "REEVAL_BACKGROUND";
1723
2505
  const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
1724
2506
  const languageModel = (model) => typeof model === "string" && !isEvaluationModel(model) ? model : void 0;
@@ -1768,10 +2550,14 @@ kill -9 -${group} 2>/dev/null`;
1768
2550
  }
1769
2551
  /** Starts `command` without waiting for exit. Teardown kills it if the eval didn't. */
1770
2552
  async function startBackground(a, command, who, extraEnv = {}) {
2553
+ return (await launch(a, command, who, extraEnv)).handle;
2554
+ }
2555
+ async function launch(a, command, who, extraEnv) {
1771
2556
  const box = await sandboxFor(a, who);
1772
2557
  const docker = a.spec.sandbox.kind === "docker";
1773
- const wrapped = `${docker ? "bash" : "sh"} -c ${shellQuote(command)} >${BG_LOG} 2>&1 & echo $$ $!`;
1774
2558
  const tag = randomUUID();
2559
+ const log = `${BG_LOG}-${tag.slice(0, 8)}.log`;
2560
+ const wrapped = `${docker ? "bash" : "sh"} -c ${shellQuote(command)} >${log} 2>&1 & echo $$ $!`;
1775
2561
  const r = await asCheck(a, () => box.exec(wrapped, {
1776
2562
  signal: a.signal,
1777
2563
  env: {
@@ -1787,16 +2573,53 @@ async function startBackground(a, command, who, extraEnv = {}) {
1787
2573
  const handle = { async kill() {
1788
2574
  if (killed) return;
1789
2575
  killed = true;
1790
- await box.exec(`${tree}; kill -9 ${pid} 2>/dev/null; true`).catch(() => {});
2576
+ await box.exec(`${tree}; kill -9 ${pid} 2>/dev/null; rm -f ${log}; true`).catch(() => {});
1791
2577
  } };
1792
2578
  a.backgrounds.push(handle);
1793
- return handle;
2579
+ return {
2580
+ handle,
2581
+ pid,
2582
+ log
2583
+ };
1794
2584
  }
2585
+ /**
2586
+ * Starts `command` with `PORT` set and returns once it listens: on `PORT` (or `port`), else on
2587
+ * the one port that wasn't listening before it started, which it records.
2588
+ */
1795
2589
  async function serveCommand(a, command, port) {
1796
- const bound = port ?? await allocatePort();
2590
+ const want = port ?? await allocatePort();
2591
+ const box = await sandboxFor(a, "c.serve()");
2592
+ const tables = a.spec.sandbox.kind !== "fake";
2593
+ const look = (pid) => untraced(() => box.exec(probeScript(pid, tables), { signal: a.signal })).then((r) => parseProbe(r.stdout));
2594
+ const first = await look();
2595
+ const before = new Set(first.proc ? first.listeners.map((l) => l.port) : []);
2596
+ const { handle, pid, log } = await launch(a, command, "c.serve()", { PORT: String(want) });
2597
+ const found = await awaitListening(command, {
2598
+ want,
2599
+ given: port !== void 0,
2600
+ before,
2601
+ signal: a.signal,
2602
+ probe: () => look(pid),
2603
+ output: async () => (await box.exec(`tail -c 65536 ${log} 2>/dev/null`)).stdout,
2604
+ ...tables ? { hostListening: async () => "" } : {}
2605
+ }).catch(async (error) => {
2606
+ await handle.kill();
2607
+ throw error;
2608
+ });
2609
+ const serve = {
2610
+ command,
2611
+ port: found.port,
2612
+ portEnv: want,
2613
+ detected: found.detected,
2614
+ address: found.address,
2615
+ readyMs: found.readyMs
2616
+ };
2617
+ a.record.serves = [...a.record.serves ?? [], serve];
2618
+ const at = found.address ? `${found.address} ` : "";
2619
+ debugLog("check", found.detected ? `c.serve ${command}: listening on ${at}port ${found.port}, not PORT ${want}; using ${found.port}` : `c.serve ${command}: listening on ${at}port ${found.port} after ${found.readyMs}ms`);
1797
2620
  return {
1798
- ...await startBackground(a, command, "c.serve()", { PORT: String(bound) }),
1799
- port: bound
2621
+ ...handle,
2622
+ port: found.port
1800
2623
  };
1801
2624
  }
1802
2625
  async function capturePage(a, target, call) {
@@ -1805,8 +2628,9 @@ async function capturePage(a, target, call) {
1805
2628
  const resolved = resolveCaptureCall(target, call, a.spec.capture);
1806
2629
  if (a.captureNames.has(resolved.name)) throw new Error(`c.capture() reused the name "${resolved.name}"`);
1807
2630
  const box = await sandboxFor(a, "c.capture()");
1808
- if (!box.getPortEndpoint) throw new Error("c.capture() needs a sandbox that can tunnel a port");
1809
- const { url: origin } = await box.getPortEndpoint({
2631
+ const tunnel = box.getPortEndpoint?.bind(box);
2632
+ if (!tunnel) throw new Error("c.capture() needs a sandbox that can tunnel a port");
2633
+ const { url: origin } = await tunnel({
1810
2634
  port: resolved.port,
1811
2635
  protocol: "http"
1812
2636
  });
@@ -1826,7 +2650,10 @@ async function capturePage(a, target, call) {
1826
2650
  use
1827
2651
  }, remaining, (n) => {
1828
2652
  moves = n;
1829
- });
2653
+ }, (port) => tunnel({
2654
+ port,
2655
+ protocol: "http"
2656
+ }).then((t) => t.url));
1830
2657
  const digest = sha256(png);
1831
2658
  const file = {
1832
2659
  path: `artifacts/${resolved.name}.png`,
@@ -1862,30 +2689,37 @@ async function capturePage(a, target, call) {
1862
2689
  file
1863
2690
  };
1864
2691
  }
1865
- async function shot(a, url, resolved, remaining, onMoves) {
2692
+ async function shot(a, url, resolved, remaining, onMoves, tunnel) {
1866
2693
  const cache = cacheDir();
1867
- return withChrome(await ensureChrome(cache, {
2694
+ const bin = await ensureChrome(cache, {
1868
2695
  cacheLabel: cacheLabel(cache),
1869
2696
  signal: a.signal,
1870
2697
  onProgress: (p) => a.report({
1871
2698
  kind: "browser-download",
1872
2699
  ...p
1873
2700
  })
1874
- }), resolved.window, remaining(), async (session) => {
2701
+ });
2702
+ const proxy = await startLoopbackProxy({
2703
+ tunnel,
2704
+ direct: [Number(new URL(url).port)]
2705
+ });
2706
+ const browser = { proxy: proxy.address };
2707
+ return withChrome(bin, resolved.window, remaining(), browser, async (session) => {
1875
2708
  const result = await runJourney(session, url, {
1876
2709
  wait: resolved.wait,
1877
2710
  timeoutMs: remaining(),
2711
+ budgetMs: resolved.timeoutMs,
1878
2712
  fullPage: resolved.fullPage,
1879
2713
  instructions: resolved.instructions,
1880
2714
  use: resolved.use,
1881
2715
  remaining,
1882
2716
  loadModel: async (name) => {
1883
2717
  const { config, profile } = getRunContext();
1884
- const gateway = resolveModelGateway(name, void 0, gatewayScope(config, profile), "capture model");
1885
- const route = judgeRoute(name, process.env, "capture model", gateway);
1886
- return providerModel({
1887
- name,
1888
- route: route.kind === "direct" ? "direct" : "gateway"
2718
+ const scope = gatewayScope(config, profile);
2719
+ const gateway = resolveModelGateway(name, void 0, scope, "capture model");
2720
+ return hostModel(name, gateway, "capture model", {
2721
+ fallbacks: openRouterFallbacks(scope.gateways),
2722
+ ...gateway === "vertex" ? { vertex: vertexSettingsIn(scope.gateways) } : {}
1889
2723
  });
1890
2724
  }
1891
2725
  });
@@ -1900,6 +2734,9 @@ async function shot(a, url, resolved, remaining, onMoves) {
1900
2734
  } : spend;
1901
2735
  }
1902
2736
  return result.png;
2737
+ }).finally(async () => {
2738
+ await proxy.close();
2739
+ if (proxy.reached.length) debugLog("check", `c.capture ${resolved.name}: sandbox port ${proxy.reached.join(", ")} answered the page through its tunnel`);
1903
2740
  });
1904
2741
  }
1905
2742
  /** Writes PNG sidecars next to the staging eval JSON so redact never sees the bytes. */
@@ -1910,6 +2747,6 @@ async function writeCaptureSidecars(stagingDir, recordFile, blobs) {
1910
2747
  for (const [name, png] of blobs) await writeFile(join(dir, `${name}.png`), png);
1911
2748
  }
1912
2749
  //#endregion
1913
- export { writeCaptureSidecars as a, sandboxFor as c, workspaceFiles as d, attemptRequestProblem as f, startBackground as i, diffSinceSnapshot as l, msysTreeKill as n, asCheck as o, fakeInstructions as p, serveCommand as r, createChecks as s, capturePage as t, snapshot as u };
2750
+ export { writeCaptureSidecars as a, sandboxFor as c, workspaceFiles as d, fakeInstructions as f, startBackground as i, diffSinceSnapshot as l, msysTreeKill as n, asCheck as o, attemptRequestProblem as p, serveCommand as r, createChecks as s, capturePage as t, snapshot as u };
1914
2751
 
1915
- //# sourceMappingURL=capture-CVRwRTnb.js.map
2752
+ //# sourceMappingURL=capture-iQ6kYJV0.js.map