reeval 0.14.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (347) hide show
  1. package/README.md +3 -3
  2. package/THIRD_PARTY_NOTICES.md +69 -69
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-BloGVZ_a.js → adapter-CbqbMR7y.js} +37 -8
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-m-_t-IuW.js → agent-Zgn-kiqq.js} +57 -15
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{app-files-C-jwhRDt.js → app-files-ChbvRx3P.js} +4 -4
  9. package/dist/app-files-ChbvRx3P.js.map +1 -0
  10. package/dist/{assemble-BDEpelpi.js → assemble-SKQGW_QF.js} +136 -116
  11. package/dist/assemble-SKQGW_QF.js.map +1 -0
  12. package/dist/assemble-XBWuK3Ed.js +1 -0
  13. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  14. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  15. package/dist/{match-Bz0nSdzN.js → attempt-host-BOp8JklD.js} +96 -5
  16. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  17. package/dist/{capture-CVRwRTnb.js → capture-BM_3100g.js} +1218 -1207
  18. package/dist/capture-BM_3100g.js.map +1 -0
  19. package/dist/capture-DSK8O5bA.js +1 -0
  20. package/dist/{cli-C_4gaJ50.js → cli-PdYrcoKM.js} +106 -170
  21. package/dist/cli-PdYrcoKM.js.map +1 -0
  22. package/dist/{cli-spec-CpnbATtI.js → cli-spec-D0qyrPZi.js} +8 -20
  23. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-D5hNjpEx.js → client-Ct6s6OnF.js} +2 -2
  26. package/dist/{client-D5hNjpEx.js.map → client-Ct6s6OnF.js.map} +1 -1
  27. package/dist/{command-BXLUfw4h.js → command-4pJYVCT4.js} +33 -20
  28. package/dist/command-4pJYVCT4.js.map +1 -0
  29. package/dist/{validate-BhwqrVsm.js → command-9xm5ImqI.js} +387 -59
  30. package/dist/command-9xm5ImqI.js.map +1 -0
  31. package/dist/{command-Bxdy62dW.js → command-BqBxLgTc.js} +31 -30
  32. package/dist/command-BqBxLgTc.js.map +1 -0
  33. package/dist/{command-I1euLLio.js → command-DbSMcxZL.js} +7 -7
  34. package/dist/{command-I1euLLio.js.map → command-DbSMcxZL.js.map} +1 -1
  35. package/dist/{commands-jzigbNFb.js → commands-BU6YJCz9.js} +175 -72
  36. package/dist/commands-BU6YJCz9.js.map +1 -0
  37. package/dist/{commands-p1S4nw_H.js → commands-Bi2SEQtr.js} +1 -1
  38. package/dist/{config-BrdHA9bJ.js → config-k6m7XbAz.js} +499 -271
  39. package/dist/config-k6m7XbAz.js.map +1 -0
  40. package/dist/{contrasts-DRxMQQTC.js → contrasts-DLZF4dGF.js} +3 -3
  41. package/dist/{contrasts-DRxMQQTC.js.map → contrasts-DLZF4dGF.js.map} +1 -1
  42. package/dist/cost-C5RBTTJr.js +1 -0
  43. package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
  44. package/dist/cost-ChPa8-Fp.js.map +1 -0
  45. package/dist/db-CpsifVln.js +1530 -0
  46. package/dist/db-CpsifVln.js.map +1 -0
  47. package/dist/db-DkD-F-X-.js +2 -0
  48. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  49. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  50. package/dist/debug-DeM5pswK.js +1 -0
  51. package/dist/{docker-BnRlTpZA.js → docker-51WvhN_X.js} +5 -4
  52. package/dist/docker-51WvhN_X.js.map +1 -0
  53. package/dist/{docker-D01slDF3.js → docker-B4-rx4dA.js} +26 -8
  54. package/dist/docker-B4-rx4dA.js.map +1 -0
  55. package/dist/{docker-DivfZl2u.js → docker-YFj4H1Cr.js} +1 -1
  56. package/dist/{doctor-DQ2p0euG.js → doctor-BuQnfz4e.js} +275 -37
  57. package/dist/doctor-BuQnfz4e.js.map +1 -0
  58. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  59. package/dist/env-CARj5Axq.js.map +1 -0
  60. package/dist/{environment-CO0P0soF.js → environment-BOFgzFDg.js} +115 -296
  61. package/dist/environment-BOFgzFDg.js.map +1 -0
  62. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  63. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  64. package/dist/eval-source-Dje3rERU.js +145 -0
  65. package/dist/eval-source-Dje3rERU.js.map +1 -0
  66. package/dist/{evaluate-BUmsN7yG.js → evaluate-BPP9s3s6.js} +367 -130
  67. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  68. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  69. package/dist/{files-B-CJeeEh.js → files-Bi461Be2.js} +197 -91
  70. package/dist/files-Bi461Be2.js.map +1 -0
  71. package/dist/files-C2h2Cw4F.js +1 -0
  72. package/dist/files-Clipx4OW.js +37 -0
  73. package/dist/files-Clipx4OW.js.map +1 -0
  74. package/dist/files-DmUrRmmL.js +1 -0
  75. package/dist/{files-kUCrswI6.js → files-H24pz0hL.js} +20 -9
  76. package/dist/files-H24pz0hL.js.map +1 -0
  77. package/dist/gateway.d.ts +22 -9
  78. package/dist/gateway.js +30 -8
  79. package/dist/gateway.js.map +1 -1
  80. package/dist/{git-Bk8DSwHy.js → git-CsVZkNNL.js} +3 -3
  81. package/dist/{git-Bk8DSwHy.js.map → git-CsVZkNNL.js.map} +1 -1
  82. package/dist/harness-3NVyTSGK.js +392 -0
  83. package/dist/harness-3NVyTSGK.js.map +1 -0
  84. package/dist/harness.d.ts +12 -8
  85. package/dist/harness.js +1 -1
  86. package/dist/{images-DzLeFjLx.js → images-4mwhNkvh.js} +8 -8
  87. package/dist/{images-DzLeFjLx.js.map → images-4mwhNkvh.js.map} +1 -1
  88. package/dist/index.d.ts +51 -10
  89. package/dist/index.js +5 -5
  90. package/dist/init-BDEOopK7.js +1 -0
  91. package/dist/{init-SRalM5xF.js → init-cOk3D44b.js} +10 -11
  92. package/dist/init-cOk3D44b.js.map +1 -0
  93. package/dist/{invocations-DzyKSJBa.js → invocations-DW-lTk2f.js} +10 -6
  94. package/dist/invocations-DW-lTk2f.js.map +1 -0
  95. package/dist/{judge-yLvjC5A3.js → judge-CKdgEXN7.js} +236 -46
  96. package/dist/judge-CKdgEXN7.js.map +1 -0
  97. package/dist/judge.d.ts +5 -7
  98. package/dist/judge.js +4 -8
  99. package/dist/judge.js.map +1 -1
  100. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  101. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  102. package/dist/{matchers-D4eUqx4j.js → matchers-DT2TaGix.js} +2 -2
  103. package/dist/{matchers-D4eUqx4j.js.map → matchers-DT2TaGix.js.map} +1 -1
  104. package/dist/{model-1ZLr9Dcv.js → model-Bq3G0OG1.js} +29 -36
  105. package/dist/model-Bq3G0OG1.js.map +1 -0
  106. package/dist/{model-CgnOpss4.js → model-CTnyvh7H.js} +292 -68
  107. package/dist/model-CTnyvh7H.js.map +1 -0
  108. package/dist/model-D-SWrLZG.js +1 -0
  109. package/dist/model-g3XnzUv0.js +1 -0
  110. package/dist/{model-B45lDUV1.js → model-vRiUycWh.js} +22 -13
  111. package/dist/model-vRiUycWh.js.map +1 -0
  112. package/dist/{node-BSq5qHnQ.js → node-BVbbnFtv.js} +72 -50
  113. package/dist/node-BVbbnFtv.js.map +1 -0
  114. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  115. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  116. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  117. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  118. package/dist/{pairs-CHfZNRT6.js → pairs-puYi-zj2.js} +11 -4
  119. package/dist/pairs-puYi-zj2.js.map +1 -0
  120. package/dist/patch-CquGAGX5.js +76 -0
  121. package/dist/patch-CquGAGX5.js.map +1 -0
  122. package/dist/{patch-DWf-I_Q_.js → patch-DKJiC5Sq.js} +5 -7
  123. package/dist/patch-DKJiC5Sq.js.map +1 -0
  124. package/dist/{plan-BIwUzlB8.js → plan-DdWkM6SW.js} +64 -29
  125. package/dist/plan-DdWkM6SW.js.map +1 -0
  126. package/dist/{plugins-Dkp_H3pB.js → plugins-D-CP6mMd.js} +253 -17
  127. package/dist/plugins-D-CP6mMd.js.map +1 -0
  128. package/dist/{preflight-DJCD_BDn.js → preflight-9ixuXzsx.js} +3 -3
  129. package/dist/{preflight-DJCD_BDn.js.map → preflight-9ixuXzsx.js.map} +1 -1
  130. package/dist/{preflight-D9y-HvZQ.js → preflight-DTiVv0gY.js} +1 -1
  131. package/dist/{prepare-CIxCGG0M.js → prepare-BfyikQit.js} +8 -8
  132. package/dist/{prepare-CIxCGG0M.js.map → prepare-BfyikQit.js.map} +1 -1
  133. package/dist/{prepare-BJG_pD54.js → prepare-tmBBf4W8.js} +2 -2
  134. package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
  135. package/dist/prices-QaS-kjSQ.js.map +1 -0
  136. package/dist/{provenance-CfZpFpAh.js → provenance-BizwUn3h.js} +4 -4
  137. package/dist/{provenance-CfZpFpAh.js.map → provenance-BizwUn3h.js.map} +1 -1
  138. package/dist/{providers-DFk_6W4K.js → providers-B6b9Tnyi.js} +5 -5
  139. package/dist/{providers-DFk_6W4K.js.map → providers-B6b9Tnyi.js.map} +1 -1
  140. package/dist/{providers-D1Zxp2he.js → providers-DkOupEBl.js} +1 -1
  141. package/dist/query-Clm2mNSm.js +1 -0
  142. package/dist/{query-CemclveB.js → query-DyM-VExS.js} +3 -3
  143. package/dist/{query-CemclveB.js.map → query-DyM-VExS.js.map} +1 -1
  144. package/dist/{questions-bAudIc04.js → questions-BRhU8MzB.js} +100 -25
  145. package/dist/questions-BRhU8MzB.js.map +1 -0
  146. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  147. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  148. package/dist/redact-DVIaahLv.js +658 -0
  149. package/dist/redact-DVIaahLv.js.map +1 -0
  150. package/dist/remarks-BwJ8pcTQ.js +1 -0
  151. package/dist/{rules-VhftPOOT.js → remarks-C7i2LMRb.js} +296 -66
  152. package/dist/remarks-C7i2LMRb.js.map +1 -0
  153. package/dist/{header-BLPP6aCU.js → remarks-D_3qhBWU.js} +34 -9
  154. package/dist/remarks-D_3qhBWU.js.map +1 -0
  155. package/dist/{remarks-mujJv1Ey.js → remarks-dA8LyoPc.js} +9 -5
  156. package/dist/remarks-dA8LyoPc.js.map +1 -0
  157. package/dist/{remote-CKcEKja4.js → remote-DFIzcx_6.js} +39 -99
  158. package/dist/remote-DFIzcx_6.js.map +1 -0
  159. package/dist/resolve--rQZ8u6Z.js +1 -0
  160. package/dist/{resolve-CBoKzQMP.js → resolve-DWSJpKHX.js} +46 -30
  161. package/dist/resolve-DWSJpKHX.js.map +1 -0
  162. package/dist/{results-CMgZEANy.js → results-BC3pcG_Z.js} +15 -30
  163. package/dist/results-BC3pcG_Z.js.map +1 -0
  164. package/dist/{resume-B9SOSnK9.js → resume-j-rcIryi.js} +47 -17
  165. package/dist/resume-j-rcIryi.js.map +1 -0
  166. package/dist/{runtime-BgQRyCLl.js → runtime-d56Gg3J5.js} +30 -13
  167. package/dist/runtime-d56Gg3J5.js.map +1 -0
  168. package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmln11gU.js} +2 -2
  169. package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmln11gU.js.map} +1 -1
  170. package/dist/sandbox.d.ts +1 -1
  171. package/dist/sandbox.js +1 -1
  172. package/dist/{server-TGW4WyAz.js → server-BOoH10J8.js} +51 -33
  173. package/dist/server-BOoH10J8.js.map +1 -0
  174. package/dist/server-DpuVAbrO.js +1 -0
  175. package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
  176. package/dist/shared-B_lvGqWz.js.map +1 -0
  177. package/dist/source-BFd6f0-R.js +1 -0
  178. package/dist/{source-B1WvBIQt.js → source-wD04-rRG.js} +15 -10
  179. package/dist/source-wD04-rRG.js.map +1 -0
  180. package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
  181. package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
  182. package/dist/{src-CyaDNG9r.js → src-6_zz7oJs.js} +1 -1
  183. package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
  184. package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
  185. package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
  186. package/dist/src-BO8AYqJ5.js.map +1 -0
  187. package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
  188. package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
  189. package/dist/src-BYPMM1x7.js +2 -0
  190. package/dist/src-Bp4aN3rA.js +2 -0
  191. package/dist/src-C7upnZSd.js +2 -0
  192. package/dist/{src-8zoX_qrb.js → src-D91Hd6hT.js} +546 -1060
  193. package/dist/src-D91Hd6hT.js.map +1 -0
  194. package/dist/src-DH-9IO7f.js +1 -0
  195. package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
  196. package/dist/src-DSZyg0Uq.js.map +1 -0
  197. package/dist/{src-DjRI-IeE.js → src-DgHuStEi.js} +134 -34
  198. package/dist/{src-DjRI-IeE.js.map → src-DgHuStEi.js.map} +1 -1
  199. package/dist/src-HSxBazcg.js +3 -0
  200. package/dist/src-LD1BAJFa.js +2 -0
  201. package/dist/src-dlX-Lmxg.js +2 -0
  202. package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
  203. package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
  204. package/dist/src-kYeaJJrl.js +2 -0
  205. package/dist/state-B5PLm5qY.js +1 -0
  206. package/dist/{state-B1t143wG.js → state-DrJ687ut.js} +15 -15
  207. package/dist/{state-B1t143wG.js.map → state-DrJ687ut.js.map} +1 -1
  208. package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
  209. package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
  210. package/dist/suite-22Kt9Grx.js +2 -0
  211. package/dist/{suite-WgT7m6Vs.js → suite-Co8cwelb.js} +48 -63
  212. package/dist/suite-Co8cwelb.js.map +1 -0
  213. package/dist/{summary-DIGEVr08.js → summary-DLFHVoux.js} +80 -270
  214. package/dist/summary-DLFHVoux.js.map +1 -0
  215. package/dist/testing.d.ts +417 -142
  216. package/dist/testing.js +12 -10
  217. package/dist/testing.js.map +1 -1
  218. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  219. package/dist/text-DTigcA64.js.map +1 -0
  220. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  221. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  222. package/dist/types-C5vI3g1C.js.map +1 -1
  223. package/dist/{types-DXBbP9Mk.d.ts → types-coMWHaU2.d.ts} +253 -733
  224. package/dist/vertex-Bv7vhMgs.js +1 -0
  225. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  226. package/dist/vertex-DypYjx71.js +456 -0
  227. package/dist/vertex-DypYjx71.js.map +1 -0
  228. package/dist/view/index.d.ts +536 -121
  229. package/dist/view/index.js +9 -9
  230. package/dist/{views-sXpGE6a2.js → views-BFO1BLKu.js} +9 -10
  231. package/dist/{views-sXpGE6a2.js.map → views-BFO1BLKu.js.map} +1 -1
  232. package/dist/views-BOf8qEL8.js +2 -0
  233. package/dist/{worker-entry-Duo9oAeT.js → worker-entry-CJtdsKBP.js} +19 -12
  234. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  235. package/dist/{workspace-CDI68wtX.js → workspace-BRxjfa8N.js} +5 -5
  236. package/dist/{workspace-CDI68wtX.js.map → workspace-BRxjfa8N.js.map} +1 -1
  237. package/dist/workspace-DDHzkMjm.js +1 -0
  238. package/docker/seccomp-default.json +1 -0
  239. package/package.json +5 -1
  240. package/skills/eval-rubric/SKILL.md +35 -104
  241. package/dist/adapter-BloGVZ_a.js.map +0 -1
  242. package/dist/agent-m-_t-IuW.js.map +0 -1
  243. package/dist/app-files-C-jwhRDt.js.map +0 -1
  244. package/dist/assemble-BDEpelpi.js.map +0 -1
  245. package/dist/assemble-C5bfncHU.js +0 -1
  246. package/dist/capture-CVRwRTnb.js.map +0 -1
  247. package/dist/capture-CherbjzB.js +0 -1
  248. package/dist/cli-C_4gaJ50.js.map +0 -1
  249. package/dist/cli-spec-CpnbATtI.js.map +0 -1
  250. package/dist/command-BWAuwQNk.js +0 -91
  251. package/dist/command-BWAuwQNk.js.map +0 -1
  252. package/dist/command-BXLUfw4h.js.map +0 -1
  253. package/dist/command-Be6BSE2L.js +0 -335
  254. package/dist/command-Be6BSE2L.js.map +0 -1
  255. package/dist/command-Bxdy62dW.js.map +0 -1
  256. package/dist/commands-jzigbNFb.js.map +0 -1
  257. package/dist/config-BrdHA9bJ.js.map +0 -1
  258. package/dist/cost-D4elx933.js.map +0 -1
  259. package/dist/cost-DmnAod9x.js +0 -1
  260. package/dist/db-CAfksinX.js +0 -2
  261. package/dist/db-DO0p7xgP.js +0 -619
  262. package/dist/db-DO0p7xgP.js.map +0 -1
  263. package/dist/debug-D7bEv_UV.js +0 -1
  264. package/dist/docker-BnRlTpZA.js.map +0 -1
  265. package/dist/docker-D01slDF3.js.map +0 -1
  266. package/dist/doctor-DQ2p0euG.js.map +0 -1
  267. package/dist/env-BQqj93gu.js.map +0 -1
  268. package/dist/environment-CO0P0soF.js.map +0 -1
  269. package/dist/evaluate-BUmsN7yG.js.map +0 -1
  270. package/dist/evaluate-C_sgRA-V.d.ts +0 -134
  271. package/dist/facts-DzOUTreB.js +0 -906
  272. package/dist/facts-DzOUTreB.js.map +0 -1
  273. package/dist/files-B-CJeeEh.js.map +0 -1
  274. package/dist/files-Bp5j5BqI.js +0 -1
  275. package/dist/files-C2sjmXhk.js +0 -1
  276. package/dist/files-C_gQNVB-.js +0 -65
  277. package/dist/files-C_gQNVB-.js.map +0 -1
  278. package/dist/files-kUCrswI6.js.map +0 -1
  279. package/dist/generate-DZ7vp69F.js +0 -1647
  280. package/dist/generate-DZ7vp69F.js.map +0 -1
  281. package/dist/harness-COVXSwJ6.js +0 -354
  282. package/dist/harness-COVXSwJ6.js.map +0 -1
  283. package/dist/header-BLPP6aCU.js.map +0 -1
  284. package/dist/init-RgCQXZhb.js +0 -1
  285. package/dist/init-SRalM5xF.js.map +0 -1
  286. package/dist/insights-TyAvz9aV.js +0 -68
  287. package/dist/insights-TyAvz9aV.js.map +0 -1
  288. package/dist/invocations-DzyKSJBa.js.map +0 -1
  289. package/dist/judge-yLvjC5A3.js.map +0 -1
  290. package/dist/match-Bz0nSdzN.js.map +0 -1
  291. package/dist/model-1ZLr9Dcv.js.map +0 -1
  292. package/dist/model-B45lDUV1.js.map +0 -1
  293. package/dist/model-BTZq8HIZ.js +0 -1
  294. package/dist/model-CgnOpss4.js.map +0 -1
  295. package/dist/model-Ch8kX87u.js +0 -1
  296. package/dist/node-BSq5qHnQ.js.map +0 -1
  297. package/dist/opencode-providers-klkqgzvb.js +0 -3276
  298. package/dist/opencode-providers-klkqgzvb.js.map +0 -1
  299. package/dist/pairs-CHfZNRT6.js.map +0 -1
  300. package/dist/patch-DWf-I_Q_.js.map +0 -1
  301. package/dist/plan-BIwUzlB8.js.map +0 -1
  302. package/dist/plugins-Dkp_H3pB.js.map +0 -1
  303. package/dist/prices-BcLEXvdY.js.map +0 -1
  304. package/dist/query-Cvbwl804.js +0 -1
  305. package/dist/questions-bAudIc04.js.map +0 -1
  306. package/dist/redact-CRGdfV2s.js +0 -240
  307. package/dist/redact-CRGdfV2s.js.map +0 -1
  308. package/dist/remarks-CXwSnSjR.js +0 -278
  309. package/dist/remarks-CXwSnSjR.js.map +0 -1
  310. package/dist/remarks-CyAu2DiB.js +0 -1
  311. package/dist/remarks-mujJv1Ey.js.map +0 -1
  312. package/dist/remote-CKcEKja4.js.map +0 -1
  313. package/dist/resolve-B5-uqmJ6.js +0 -1
  314. package/dist/resolve-CBoKzQMP.js.map +0 -1
  315. package/dist/results-CMgZEANy.js.map +0 -1
  316. package/dist/resume-B9SOSnK9.js.map +0 -1
  317. package/dist/rules-VhftPOOT.js.map +0 -1
  318. package/dist/run-CwIEK16q.js +0 -88
  319. package/dist/run-CwIEK16q.js.map +0 -1
  320. package/dist/runtime-BgQRyCLl.js.map +0 -1
  321. package/dist/server-CHRSgKRp.js +0 -1
  322. package/dist/server-TGW4WyAz.js.map +0 -1
  323. package/dist/shared-BDUzStad.js.map +0 -1
  324. package/dist/source-B1WvBIQt.js.map +0 -1
  325. package/dist/source-DAzztZwU.js +0 -1
  326. package/dist/src-1jHbFaGW.js +0 -1
  327. package/dist/src-8zoX_qrb.js.map +0 -1
  328. package/dist/src-C_9wgowC.js +0 -2
  329. package/dist/src-ChhtI6BP.js.map +0 -1
  330. package/dist/src-D005Cbqz.js +0 -2
  331. package/dist/src-DIfRbp5a.js +0 -3
  332. package/dist/src-Dg8121Xy.js +0 -2
  333. package/dist/src-ElBqEN-S.js +0 -2
  334. package/dist/src-NSdQ9zI6.js.map +0 -1
  335. package/dist/src-YHGaUrGE.js +0 -2
  336. package/dist/src-mJVxw1Jm.js +0 -2
  337. package/dist/state-vznMh2wl.js +0 -1
  338. package/dist/suite-BsWGHzlG.js +0 -2
  339. package/dist/suite-WgT7m6Vs.js.map +0 -1
  340. package/dist/summary-DIGEVr08.js.map +0 -1
  341. package/dist/text-CrVy5ioL.js.map +0 -1
  342. package/dist/transcript-BTxJpW4R.js.map +0 -1
  343. package/dist/usage-DF3_p5pE.js +0 -1
  344. package/dist/validate-BhwqrVsm.js.map +0 -1
  345. package/dist/views-BW_C5eVH.js +0 -2
  346. package/dist/worker-entry-Duo9oAeT.js.map +0 -1
  347. package/dist/workspace-QZCacP8A.js +0 -1
@@ -1,17 +1,19 @@
1
1
  import { n as isOneOf } from "./object-CLf2-tNq.js";
2
2
  import { n as UsageError } from "./errors-B6BSMs03.js";
3
- import { c as jsonValueSchema, i as durationSchema, n as dataProblem, p as parseDuration } from "./shared-BDUzStad.js";
4
- import { v as gatewayNameSchema } from "./harness-COVXSwJ6.js";
3
+ import { c as jsonValueSchema, h as parseDuration, i as durationSchema, m as TIMER_LIMIT, n as dataProblem, p as MAX_TIMER_MS } from "./shared-B_lvGqWz.js";
4
+ import { y as gatewayNameSchema } from "./harness-3NVyTSGK.js";
5
5
  import { l as sha256 } from "./ids-Bp6ut7ZD.js";
6
- import { Ct as judgeStateSchema, bt as judgeFields, v as gatewayScope, vt as captureWindowSchema, xt as judgeModelNameSchema, y as resolveModelGateway, yt as fakeJudgeDataSchema } from "./config-BrdHA9bJ.js";
7
- import { l as judgeRoute } from "./judge-yLvjC5A3.js";
8
- import { ft as shellQuote } from "./src-8zoX_qrb.js";
6
+ import { Ct as captureWindowSchema, Et as judgeModelNameSchema, Ot as judgeStateSchema, Tt as judgeFields, b as resolveModelGateway, v as gatewayScope, wt as fakeJudgeDataSchema } from "./config-k6m7XbAz.js";
7
+ import { y as vertexSettingsIn } from "./vertex-DypYjx71.js";
8
+ import { m as redactSecrets } from "./redact-DVIaahLv.js";
9
+ import { i as formatDuration, n as clip } from "./text-DTigcA64.js";
9
10
  import { s as cacheDir } from "./root-Dlff-zp8.js";
11
+ import { ft as shellQuote } from "./src-D91Hd6hT.js";
10
12
  import { t as InfraError } from "./types-C5vI3g1C.js";
11
- import { i as formatDuration, n as clip } from "./text-CrVy5ioL.js";
12
- import { n as getRunContext, o as SUBJECT } from "./runtime-BgQRyCLl.js";
13
- import { i as debugTrace, r as debugLog } from "./attempt-jI0OWxlz.js";
14
- import { o as providerModel } from "./model-1ZLr9Dcv.js";
13
+ import { r as getRunContext, s as SUBJECT } from "./runtime-d56Gg3J5.js";
14
+ import { i as debugTrace, r as debugLog } from "./attempt-Ba2QerOK.js";
15
+ import { r as openRouterFallbacks } from "./openrouter-models-CWRwLX-G.js";
16
+ import { i as hostModel } from "./model-Bq3G0OG1.js";
15
17
  import { t as addUsage } from "./usage-BNY1ABrU.js";
16
18
  import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
17
19
  import { homedir, tmpdir } from "node:os";
@@ -24,555 +26,1030 @@ import { spawn } from "node:child_process";
24
26
  import { crc32, deflateSync, inflateRawSync } from "node:zlib";
25
27
  import { mkdir, writeFile } from "node:fs/promises";
26
28
  import { generateText, stepCountIs, tool } from "ai";
27
- //#region src/capture/fake-instructions.ts
28
- const FAKE_INSTRUCTIONS = Symbol.for("reeval.capture.fakeInstructions");
29
- function isWaitStep(step) {
30
- return typeof step === "object" && step !== null && !Array.isArray(step) && "wait" in step;
31
- }
32
- /** One list: action lists and `{ wait }` inlined. Returns branded `instructions`. */
33
- function fakeInstructions(steps) {
34
- if (!Array.isArray(steps)) throw new Error(`fakeInstructions() is an array of action lists and { wait } (got ${JSON.stringify(steps)})`);
35
- const instructions = [];
36
- let goals = 0;
37
- for (const [i, step] of steps.entries()) {
38
- if (isWaitStep(step)) {
39
- instructions.push({ wait: step.wait });
40
- continue;
41
- }
42
- if (!Array.isArray(step)) throw new Error(`fakeInstructions()[${i}] is an action list or { wait } (got ${JSON.stringify(step)})`);
43
- instructions.push(`fake[${goals}]`);
44
- goals++;
45
- }
46
- Object.defineProperty(instructions, FAKE_INSTRUCTIONS, {
47
- value: steps,
48
- enumerable: false
49
- });
50
- return instructions;
51
- }
52
- function fakeInstructionSteps(value) {
53
- if (typeof value === "object" && value !== null && FAKE_INSTRUCTIONS in value) return value[FAKE_INSTRUCTIONS];
54
- }
55
- //#endregion
56
- //#region src/capture/zip.ts
57
- const LOCAL = 67324752;
58
- /** Extracts a zip (store or deflate) into `files` keyed by posix path. */
59
- function unzip(bytes) {
60
- const buf = Buffer.from(bytes);
61
- const files = /* @__PURE__ */ new Map();
62
- let i = 0;
63
- while (i + 30 <= buf.length) {
64
- if (buf.readUInt32LE(i) !== LOCAL) break;
65
- const method = buf.readUInt16LE(i + 8);
66
- const comp = buf.readUInt32LE(i + 18);
67
- const nameLen = buf.readUInt16LE(i + 26);
68
- const extra = buf.readUInt16LE(i + 28);
69
- const name = buf.toString("utf8", i + 30, i + 30 + nameLen);
70
- const start = i + 30 + nameLen + extra;
71
- const raw = buf.subarray(start, start + comp);
72
- if (name.endsWith("/")) {
73
- i = start + comp;
74
- continue;
75
- }
76
- if (method === 0) files.set(name, Buffer.from(raw));
77
- else if (method === 8) files.set(name, Buffer.from(inflateRawSync(raw)));
78
- else throw new Error(`zip method ${method} is not supported (${name})`);
79
- i = start + comp;
80
- }
81
- return files;
82
- }
83
- //#endregion
84
- //#region src/capture/browser.ts
85
- const isMs = (wait) => typeof wait === "object" && wait !== null;
86
- /** The CDP event `'idle'` waits for; `'load'` is `Page.loadEventFired`. */
87
- function captureWaitLabel(wait) {
88
- return wait === "idle" ? "networkAlmostIdle" : "load";
89
- }
90
- function captureWaitTimeout(wait) {
91
- return /* @__PURE__ */ new Error(`c.capture() timed out waiting for ${captureWaitLabel(wait)}`);
92
- }
93
- function waitTimeout(wait, detail) {
94
- if (isMs(wait)) return /* @__PURE__ */ new Error(`c.capture() timed out${detail ? ` on ${detail}` : ""} waiting ${wait.ms}ms`);
95
- return captureWaitTimeout(wait);
96
- }
97
- async function waitForPage(cdp, wait, timeoutMs, nav) {
98
- if (isMs(wait)) {
99
- if (timeoutMs <= 0) throw waitTimeout(wait);
100
- const ms = Math.min(wait.ms, timeoutMs);
101
- await new Promise((resolve) => setTimeout(resolve, ms));
102
- return;
103
- }
104
- const page = listenForCaptureWait(cdp, wait);
105
- page.arm();
106
- await page.ready(nav, timeoutMs);
107
- }
108
- const clamp = (n, lo, hi) => Math.min(hi, Math.max(lo, n));
109
- /** Map a full-page click onto the live window, scrolling with the wheel if needed. */
110
- function mapPagePoint(x, y, window, page, scrollY) {
111
- const maxY = Math.max(0, (page?.height ?? window.height) - window.height);
112
- const target = y >= window.height || y < 0 ? clamp(y - window.height / 2, 0, maxY) : scrollY;
113
- const nextY = clamp(target, 0, maxY);
114
- return {
115
- x: clamp(x, 0, window.width - 1),
116
- y: clamp(y - nextY, 0, window.height - 1),
117
- scrollY: nextY,
118
- dy: nextY - scrollY
119
- };
120
- }
121
- async function dispatchInput(cdp, action, window, page, scrollY = 0) {
122
- if (action.type === "done") return scrollY;
123
- if (action.type === "type") {
124
- await cdp.send("Input.insertText", { text: action.text });
125
- return scrollY;
126
- }
127
- if (action.type === "key") {
128
- const key = action.key;
129
- const vk = key === "Tab" ? 9 : 13;
130
- await cdp.send("Input.dispatchKeyEvent", {
131
- type: "keyDown",
132
- key,
133
- code: key,
134
- windowsVirtualKeyCode: vk
135
- });
136
- await cdp.send("Input.dispatchKeyEvent", {
137
- type: "keyUp",
138
- key,
139
- code: key,
140
- windowsVirtualKeyCode: vk
141
- });
142
- return scrollY;
143
- }
144
- if (action.type === "scroll") {
145
- await cdp.send("Input.dispatchMouseEvent", {
146
- type: "mouseWheel",
147
- x: action.x,
148
- y: action.y,
149
- deltaX: 0,
150
- deltaY: action.dy
151
- });
152
- return scrollY + action.dy;
153
- }
154
- const mapped = mapPagePoint(action.type === "drag" ? action.x : action.x, action.type === "drag" ? action.y : action.y, window, page, scrollY);
155
- if (mapped.dy !== 0) await cdp.send("Input.dispatchMouseEvent", {
156
- type: "mouseWheel",
157
- x: window.width / 2,
158
- y: window.height / 2,
159
- deltaX: 0,
160
- deltaY: mapped.dy
161
- });
162
- const at = {
163
- x: mapped.x,
164
- y: mapped.y
165
- };
166
- if (action.type === "hover") {
167
- await cdp.send("Input.dispatchMouseEvent", {
168
- type: "mouseMoved",
169
- ...at
29
+ //#region src/contract/attempt.ts
30
+ const nullableNumber = z.number().nullable();
31
+ const tokenUsageSchema = z.strictObject({
32
+ input: z.number(),
33
+ output: z.number(),
34
+ cacheRead: z.number(),
35
+ cacheWrite: z.number(),
36
+ reasoning: nullableNumber
37
+ });
38
+ const fileChangeSchema = z.strictObject({
39
+ path: z.string(),
40
+ status: z.enum([
41
+ "added",
42
+ "modified",
43
+ "deleted"
44
+ ])
45
+ });
46
+ const diffSchema = z.strictObject({
47
+ patch: z.string(),
48
+ files: z.array(fileChangeSchema)
49
+ });
50
+ const turnStatusSchema = z.enum([
51
+ "completed",
52
+ "aborted",
53
+ "error"
54
+ ]);
55
+ const webEventTypeSchema = z.enum(["web-search", "web-fetch"]);
56
+ const at = {
57
+ ts: z.string(),
58
+ atMs: z.number()
59
+ };
60
+ const transcriptEventSchema = z.discriminatedUnion("type", [
61
+ z.strictObject({
62
+ ...at,
63
+ type: z.enum(["text", "reasoning"]),
64
+ text: z.string(),
65
+ durationMs: nullableNumber
66
+ }),
67
+ z.strictObject({
68
+ ...at,
69
+ type: z.literal("file-edit"),
70
+ toolCallId: z.string(),
71
+ path: z.string(),
72
+ kind: z.enum([
73
+ "create",
74
+ "modify",
75
+ "write",
76
+ "delete"
77
+ ]),
78
+ added: nullableNumber,
79
+ removed: nullableNumber,
80
+ durationMs: nullableNumber,
81
+ stalled: z.boolean().optional()
82
+ }),
83
+ z.strictObject({
84
+ ...at,
85
+ type: z.literal("web-search"),
86
+ toolCallId: z.string(),
87
+ query: z.string(),
88
+ references: nullableNumber,
89
+ ok: z.boolean().optional(),
90
+ durationMs: nullableNumber,
91
+ stalled: z.boolean().optional()
92
+ }),
93
+ z.strictObject({
94
+ ...at,
95
+ type: z.literal("web-fetch"),
96
+ toolCallId: z.string(),
97
+ url: z.string(),
98
+ ok: z.boolean(),
99
+ durationMs: nullableNumber,
100
+ stalled: z.boolean().optional()
101
+ }),
102
+ z.strictObject({
103
+ ...at,
104
+ type: z.literal("tool"),
105
+ toolCallId: z.string(),
106
+ tool: z.string(),
107
+ input: z.unknown(),
108
+ output: z.unknown(),
109
+ ok: z.boolean(),
110
+ durationMs: nullableNumber,
111
+ stalled: z.boolean().optional()
112
+ }),
113
+ z.strictObject({
114
+ ...at,
115
+ type: z.literal("error"),
116
+ message: z.string()
117
+ }),
118
+ z.strictObject({
119
+ ...at,
120
+ type: z.literal("retry"),
121
+ message: z.string()
122
+ })
123
+ ]);
124
+ const turnDataSchema = z.strictObject({
125
+ prompt: z.string(),
126
+ text: z.string(),
127
+ status: turnStatusSchema,
128
+ startedAt: z.string(),
129
+ durationMs: z.number(),
130
+ transcript: z.array(transcriptEventSchema),
131
+ usage: tokenUsageSchema.nullable(),
132
+ model: z.string().nullable(),
133
+ unreported: z.array(webEventTypeSchema),
134
+ diff: diffSchema
135
+ });
136
+ const execResultSchema = z.strictObject({
137
+ exitCode: z.number().int(),
138
+ stdout: z.string(),
139
+ stderr: z.string()
140
+ });
141
+ const execRetrySchema = z.strictObject({
142
+ retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional(),
143
+ interval: z.number().min(0, "must be 0 or more milliseconds").max(MAX_TIMER_MS, `must be ${TIMER_LIMIT}`).optional()
144
+ });
145
+ /** A background command has no exit for `retries` to wait on. */
146
+ const noBackgroundRetry = (options, ctx) => {
147
+ if (!options.background) return;
148
+ for (const key of ["retries", "interval"]) {
149
+ if (options[key] === void 0) continue;
150
+ ctx.addIssue({
151
+ code: "custom",
152
+ path: [key],
153
+ message: "can't be used with background: true, whose exit reeval doesn't wait for: drop one of them"
170
154
  });
171
- return mapped.scrollY;
172
155
  }
173
- if (action.type === "click") {
174
- await cdp.send("Input.dispatchMouseEvent", {
175
- type: "mouseMoved",
176
- ...at,
177
- button: "left"
156
+ };
157
+ const execOptionsSchema = z.strictObject({
158
+ background: z.boolean().optional(),
159
+ ...execRetrySchema.shape
160
+ }).superRefine(noBackgroundRetry);
161
+ /** What's wrong with `c.exec()`'s options, or undefined. */
162
+ const execOptionsProblem = (value) => dataProblem(execOptionsSchema, value, "c.exec()", () => Object.keys(execOptionsSchema.shape));
163
+ const assertionRecordSchema = z.strictObject({
164
+ label: z.string(),
165
+ labelled: z.boolean(),
166
+ matcher: z.string(),
167
+ status: z.enum([
168
+ "pass",
169
+ "fail",
170
+ "error"
171
+ ]),
172
+ soft: z.boolean(),
173
+ message: z.string().optional(),
174
+ durationMs: z.number().optional()
175
+ });
176
+ const turnIndex = z.number().int("must be a turn index, from 0").min(0, "must be a turn index, from 0").describe("The turn's index, from 0.");
177
+ const captureSchema = z.strictObject({
178
+ name: z.string(),
179
+ port: z.number(),
180
+ path: z.string(),
181
+ viewport: z.union([captureWindowSchema, z.literal("full-page")]),
182
+ file: z.strictObject({
183
+ path: z.string(),
184
+ sha256: z.string(),
185
+ bytes: z.number()
186
+ })
187
+ });
188
+ const judgeArtifactRefSchema = z.strictObject({
189
+ name: z.string(),
190
+ path: z.string(),
191
+ sha256: z.string(),
192
+ bytes: z.number(),
193
+ mediaType: z.literal("image/png")
194
+ });
195
+ const verdictFields = {
196
+ threshold: z.number(),
197
+ question: z.string(),
198
+ model: z.string(),
199
+ gateway: gatewayNameSchema.optional(),
200
+ artifacts: z.array(judgeArtifactRefSchema).optional()
201
+ };
202
+ const verdictSchema = z.strictObject({
203
+ verdict: z.enum(["pass", "fail"]),
204
+ score: z.number(),
205
+ ...verdictFields,
206
+ resolvedModel: z.string().optional(),
207
+ upstream: z.string().optional(),
208
+ costUsd: z.number().optional()
209
+ });
210
+ const judgeFailureSchema = z.strictObject({
211
+ verdict: z.literal("error"),
212
+ score: z.null(),
213
+ ...verdictFields,
214
+ error: z.string()
215
+ });
216
+ /** A pattern, for the matchers that take a string or a `RegExp`. */
217
+ const patternSchema = z.strictObject({
218
+ regexp: z.string().describe("The pattern's source, as `RegExp.prototype.source` gives it."),
219
+ flags: z.string().regex(/^[dgimsuvy]*$/, "must use only the flags d, g, i, m, s, u, v and y").optional().describe("Its flags, e.g. `i`.")
220
+ }).describe("A regular expression, where a matcher takes a string or a `RegExp`.");
221
+ /** A pattern in the runner's own language, which reeval never compiles. */
222
+ function hasAnsweredPattern(value) {
223
+ if (Array.isArray(value)) return value.some(hasAnsweredPattern);
224
+ if (typeof value !== "object" || value === null) return false;
225
+ return "matchedBy" in value || Object.values(value).some(hasAnsweredPattern);
226
+ }
227
+ const answeredPatternSchema = z.strictObject({
228
+ matchedBy: z.string().regex(/\S/).describe("The pattern as the runner's language writes it, e.g. `re.compile('npm t')`."),
229
+ answers: z.record(z.string(), z.boolean()).describe("Whether it matches each string the runner has been asked about.")
230
+ }).describe("A pattern the runner matches in its own language. reeval runs the matcher with it, and when the matcher tests a string not in `answers` replies with those strings (`untested`) instead of a record; the runner matches them and makes the call again with them in `answers`.");
231
+ const isPatternLike = (value) => typeof value === "object" && value !== null && !Array.isArray(value) && "regexp" in value;
232
+ const matcherArgSchema = z.union([
233
+ patternSchema,
234
+ answeredPatternSchema,
235
+ jsonValueSchema
236
+ ]).superRefine((value, ctx) => {
237
+ if (!isPatternLike(value)) return;
238
+ for (const issue of patternSchema.safeParse(value).error?.issues ?? []) ctx.addIssue(issue);
239
+ }).describe("One argument to a matcher: a JSON value, a regular expression as `{ regexp, flags? }`, or a pattern in the runner's own language as `{ matchedBy, answers }`.");
240
+ const matcherSubjectSchema = z.union([
241
+ z.strictObject({ turn: turnIndex }),
242
+ z.strictObject({ attempt: z.literal(true).describe("The attempt itself, `c`.") }),
243
+ z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
244
+ z.strictObject({
245
+ exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
246
+ result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
247
+ }),
248
+ z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
249
+ ]).describe("What `expect` was given: a turn `c.send` returned, the attempt (for its files and latest turn), a value, such as an `ExecResult`, or a command or file reeval reads for it.");
250
+ const nonEmpty = z.string().regex(/\S/, "must not be empty");
251
+ const CAPTURE_NAME = new RegExp(`^[a-z0-9][a-z0-9_-]{0,99}$`);
252
+ const CAPTURE_NAME_RULE = `must be 1 to 100 of a-z, 0-9, _ and -, starting with a letter or digit`;
253
+ const portSchema = z.number().int().min(1, "a port from 1 to 65535").max(65535, "a port from 1 to 65535");
254
+ const millisSchema = z.number().positive("more than zero milliseconds").max(MAX_TIMER_MS, TIMER_LIMIT);
255
+ const captureWaitSchema = z.enum(["load", "idle"]);
256
+ const waitSchema = z.union([captureWaitSchema, z.strictObject({ ms: millisSchema })]);
257
+ const captureInstructionSchema = z.union([nonEmpty, z.strictObject({ wait: waitSchema })]);
258
+ const captureCallDataSchema = z.strictObject({
259
+ name: z.string().regex(CAPTURE_NAME, CAPTURE_NAME_RULE).optional(),
260
+ path: z.string().optional(),
261
+ viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
262
+ timeout: durationSchema.optional(),
263
+ wait: waitSchema.optional(),
264
+ instructions: z.array(captureInstructionSchema).optional().describe("Journey: goal strings and `{ wait }` beats, run by the vision model."),
265
+ model: z.string().optional()
266
+ }).describe("`c.capture()`'s camera and path as data. Default `path` is `/`; default `name` is derived from the path. A test's `fakeInstructions` journey is TypeScript-only.");
267
+ const handleId = nonEmpty.describe("A background process, as an exec or serve reply named it.");
268
+ const judgeOnSchema = z.union([z.strictObject({ turn: turnIndex }), z.strictObject({ value: jsonValueSchema.describe("Graded as the output.") })]);
269
+ const judgeCallDataSchema = z.strictObject({
270
+ model: z.union([judgeModelNameSchema, fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge with a numeric score."),
271
+ ...judgeFields,
272
+ on: judgeOnSchema.optional().describe("What to grade instead of the latest turn: an earlier turn, or a value."),
273
+ artifacts: z.union([z.literal(true), z.array(z.union([nonEmpty, captureSchema]))]).optional().describe("Captures this call sends the judge as image file parts: a capture `c.capture` returned, or its name, or `true` for every capture so far. The provider that cannot take files fails the call.")
274
+ }).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score, plus the call-only `on` and `artifacts`.");
275
+ const attemptCallSchema = z.discriminatedUnion("op", [
276
+ z.strictObject({
277
+ op: z.literal("send"),
278
+ prompt: z.string().describe("The turn's prompt.")
279
+ }).describe("`c.send(prompt)`: the next turn of the agent session. Replies with a turn."),
280
+ z.strictObject({
281
+ op: z.literal("exec"),
282
+ command: nonEmpty.describe("A shell command, run in the workspace."),
283
+ background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
284
+ ...execRetrySchema.shape
285
+ }).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
286
+ z.strictObject({
287
+ op: z.literal("serve"),
288
+ command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
289
+ port: portSchema.optional().describe("The container port. Omit it and reeval allocates one.")
290
+ }).describe("`c.serve(command, port?)`: a background process and its port."),
291
+ z.strictObject({
292
+ op: z.literal("kill"),
293
+ handle: handleId
294
+ }).describe("`handle.kill()`: stops a background process. Killing one that's gone is fine."),
295
+ z.strictObject({
296
+ op: z.literal("capture"),
297
+ target: z.union([z.strictObject({ port: portSchema }), z.strictObject({ handle: handleId })]).describe("A container port, or a serve handle by its id."),
298
+ options: captureCallDataSchema.optional()
299
+ }).describe("`c.capture(target, options)`: screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`. Replies with the capture."),
300
+ z.strictObject({
301
+ op: z.literal("readFile"),
302
+ path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory.")
303
+ }).describe("`c.readFile(path)`. Replies with its contents, or null when there is no such file."),
304
+ z.strictObject({
305
+ op: z.literal("writeFile"),
306
+ path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory."),
307
+ contents: z.string().describe("The file's contents, in `encoding`."),
308
+ encoding: z.enum(["utf8", "base64"]).optional().describe("How `contents` is encoded; base64 carries bytes. Default `utf8`.")
309
+ }).describe("`c.writeFile(path, contents)`, creating its directories."),
310
+ z.strictObject({
311
+ op: z.literal("expect"),
312
+ subject: matcherSubjectSchema,
313
+ matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a matcher name such as toHaveFile, without \"not.\" (set not: true instead)").describe("The matcher's name, e.g. `toHaveFile` or `toBe`, without `not.`."),
314
+ args: z.array(matcherArgSchema).describe("The matcher's arguments."),
315
+ not: z.boolean().describe("Called as `expect(…).not`."),
316
+ soft: z.boolean().describe("Called as `expect.soft`: a failure fails the attempt at its end."),
317
+ label: z.string().optional().describe("`expect`'s second argument."),
318
+ poll: z.strictObject({
319
+ interval: millisSchema.optional().describe("Milliseconds between tries. Default 50."),
320
+ timeout: millisSchema.optional().describe("Milliseconds before it fails with the last error. Default 1000.")
321
+ }).optional().describe("`expect.poll`: reeval re-reads the subject and retries the matcher until it passes, recording only the final outcome. The subject must be one reeval can re-read: the attempt, a command or a file.")
322
+ }).superRefine((call, ctx) => {
323
+ if (call.poll && "value" in call.subject) ctx.addIssue({
324
+ code: "custom",
325
+ path: ["subject"],
326
+ message: "can't be polled: reeval can't re-read a value. Poll the attempt, an exec or a readFile"
178
327
  });
179
- await cdp.send("Input.dispatchMouseEvent", {
180
- type: "mousePressed",
181
- ...at,
182
- button: "left",
183
- clickCount: 1
328
+ if (call.poll && "exec" in call.subject && call.subject.result) ctx.addIssue({
329
+ code: "custom",
330
+ path: ["subject", "result"],
331
+ message: "can't be polled: drop result so reeval runs the command on each try"
184
332
  });
185
- await cdp.send("Input.dispatchMouseEvent", {
186
- type: "mouseReleased",
187
- ...at,
188
- button: "left",
189
- clickCount: 1
333
+ if (call.poll && call.args.some(hasAnsweredPattern)) ctx.addIssue({
334
+ code: "custom",
335
+ path: ["args"],
336
+ message: "can't be polled with a { matchedBy } pattern, whose answers can't follow the subject from try to try: poll with a string, or check once"
190
337
  });
191
- return mapped.scrollY;
192
- }
193
- const end = mapPagePoint(action.toX, action.toY, window, page, mapped.scrollY);
194
- if (end.dy !== 0) await cdp.send("Input.dispatchMouseEvent", {
195
- type: "mouseWheel",
196
- x: window.width / 2,
197
- y: window.height / 2,
198
- deltaX: 0,
199
- deltaY: end.dy
200
- });
201
- await cdp.send("Input.dispatchMouseEvent", {
202
- type: "mouseMoved",
203
- x: at.x,
204
- y: at.y,
205
- button: "left"
206
- });
207
- await cdp.send("Input.dispatchMouseEvent", {
208
- type: "mousePressed",
209
- x: at.x,
210
- y: at.y,
211
- button: "left",
212
- clickCount: 1
213
- });
214
- await cdp.send("Input.dispatchMouseEvent", {
215
- type: "mouseMoved",
216
- x: end.x,
217
- y: end.y,
218
- button: "left"
219
- });
220
- await cdp.send("Input.dispatchMouseEvent", {
221
- type: "mouseReleased",
222
- x: end.x,
223
- y: end.y,
224
- button: "left",
225
- clickCount: 1
226
- });
227
- return end.scrollY;
228
- }
229
- function lifecycleParams(msg) {
230
- const body = msg;
231
- return body.params ?? body;
232
- }
233
- /**
234
- * Arms after `Page.enable` so about:blank's events cannot resolve the wait.
235
- * Call `arm` immediately before `Page.navigate`.
236
- */
237
- function listenForCaptureWait(cdp, wait) {
238
- const method = wait === "idle" ? "Page.lifecycleEvent" : "Page.loadEventFired";
239
- const pending = [];
240
- let deliver;
241
- let started = false;
242
- cdp.on(method, (msg) => {
243
- if (!started) return;
244
- if (deliver) deliver(msg);
245
- else pending.push(msg);
246
- });
247
- const matches = (msg, loaderId) => {
248
- if (wait === "load") return true;
249
- const params = lifecycleParams(msg);
250
- if (params.name !== "networkAlmostIdle") return false;
251
- return !loaderId || params.loaderId === loaderId;
252
- };
253
- return {
254
- arm() {
255
- started = true;
256
- },
257
- ready(nav, timeoutMs) {
258
- return new Promise((resolve, reject) => {
259
- const timer = setTimeout(() => reject(captureWaitTimeout(wait)), timeoutMs);
260
- const finish = () => {
261
- clearTimeout(timer);
262
- resolve();
263
- };
264
- deliver = (msg) => {
265
- if (matches(msg, nav?.loaderId)) finish();
266
- };
267
- if (pending.some((msg) => matches(msg, nav?.loaderId))) finish();
268
- });
269
- }
270
- };
271
- }
272
- const QUIET_IN_FLIGHT = 2;
273
- function requestId(msg) {
274
- const body = msg;
275
- return (body.params ?? body).requestId;
276
- }
277
- /**
278
- * Counts the tab's requests from `Network.enable` on, for a journey's `{ wait: 'idle' }` beat.
279
- * Chromium emits `networkAlmostIdle` once per document, so after a click that only fetches it
280
- * never comes again. As in Chromium, every request that starts or ends restarts the quiet window.
281
- */
282
- function trackNetwork(cdp, quietMs = 500) {
283
- const inFlight = /* @__PURE__ */ new Set();
284
- let changed;
285
- cdp.on("Network.requestWillBeSent", (msg) => {
286
- const id = requestId(msg);
287
- if (id === void 0) return;
288
- inFlight.add(id);
289
- changed?.();
290
- });
291
- const end = (msg) => {
292
- const id = requestId(msg);
293
- if (id === void 0 || !inFlight.delete(id)) return;
294
- changed?.();
295
- };
296
- cdp.on("Network.loadingFinished", end);
297
- cdp.on("Network.loadingFailed", end);
298
- return { quiet(timeoutMs) {
299
- return new Promise((resolve, reject) => {
300
- let settle;
301
- const stop = () => {
302
- clearTimeout(timer);
303
- clearTimeout(settle);
304
- changed = void 0;
305
- };
306
- const timer = setTimeout(() => {
307
- stop();
308
- reject(captureWaitTimeout("idle"));
309
- }, timeoutMs);
310
- changed = () => {
311
- clearTimeout(settle);
312
- if (inFlight.size > QUIET_IN_FLIGHT) return;
313
- settle = setTimeout(() => {
314
- stop();
315
- resolve();
316
- }, quietMs);
317
- };
318
- changed();
319
- });
320
- } };
321
- }
322
- /** Pinned Chrome for Testing `chrome-headless-shell` build. */
323
- const CHROME_VERSION = "140.0.7339.185";
324
- const CHROME_NAME = "chrome-headless-shell";
325
- function chromePlatform() {
326
- if (process.platform === "darwin") return process.arch === "arm64" ? "mac-arm64" : "mac-x64";
327
- if (process.platform === "win32") return "win64";
328
- return "linux64";
338
+ }).describe("A matcher call: `expect(subject, label).not?.matcher(...args)`. Replies with its record."),
339
+ z.strictObject({
340
+ op: z.literal("record"),
341
+ matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a name such as assert").describe("What made the check in the eval's language, e.g. `assert`."),
342
+ subject: z.string().optional().describe("What it checked, as the eval wrote it (`result.ok`): names the check when it has no label."),
343
+ label: z.string().optional().describe("The check's own name, the same whether it passes or fails."),
344
+ status: z.enum(["pass", "fail"]),
345
+ message: z.string().optional().describe("Why it failed."),
346
+ soft: z.boolean().optional().describe("A failure fails the attempt at its end, as `expect.soft`. Default false.")
347
+ }).describe("A check the eval's own language made, such as Python's `assert`: recorded as a matcher call is, so it's named, counted and compared the same. Replies with its record."),
348
+ z.strictObject({
349
+ op: z.literal("judge"),
350
+ question: nonEmpty.describe("A yes/no question about the graded turn or value."),
351
+ options: judgeCallDataSchema.optional()
352
+ }).describe("`c.judge(question, options)`. Replies with its verdict, or a judge failure when the provider failed.")
353
+ ]).describe("One call an eval's body makes on its attempt. The sandbox and the agent session start on the first call that needs them; there's no call to start them.");
354
+ const attemptErrorSchema = z.looseObject({
355
+ name: z.string().describe("The error's class, e.g. `UsageError` or `AssertionError`."),
356
+ message: z.string()
357
+ }).describe("Why a call failed, in reeval's words.");
358
+ const id = nonEmpty.describe("Pairs a reply with its request.");
359
+ const attemptRequestSchema = z.strictObject({
360
+ id,
361
+ call: attemptCallSchema
362
+ }).describe("A call, numbered so its reply can find it.");
363
+ z.union([
364
+ z.discriminatedUnion("op", [
365
+ z.strictObject({
366
+ id,
367
+ ok: z.literal(true),
368
+ op: z.literal("send"),
369
+ turn: turnDataSchema
370
+ }),
371
+ z.strictObject({
372
+ id,
373
+ ok: z.literal(true),
374
+ op: z.literal("exec"),
375
+ result: z.union([execResultSchema, z.strictObject({ handle: handleId }).describe("A background process, still running.")])
376
+ }),
377
+ z.strictObject({
378
+ id,
379
+ ok: z.literal(true),
380
+ op: z.literal("serve"),
381
+ handle: handleId,
382
+ port: z.number().describe("The container port it was given.")
383
+ }),
384
+ z.strictObject({
385
+ id,
386
+ ok: z.literal(true),
387
+ op: z.literal("kill")
388
+ }),
389
+ z.strictObject({
390
+ id,
391
+ ok: z.literal(true),
392
+ op: z.literal("capture"),
393
+ capture: captureSchema
394
+ }),
395
+ z.strictObject({
396
+ id,
397
+ ok: z.literal(true),
398
+ op: z.literal("readFile"),
399
+ contents: z.string().nullable().describe("Null when there is no such file.")
400
+ }),
401
+ z.strictObject({
402
+ id,
403
+ ok: z.literal(true),
404
+ op: z.literal("writeFile")
405
+ }),
406
+ z.strictObject({
407
+ id,
408
+ ok: z.literal(true),
409
+ op: z.literal("expect"),
410
+ assertion: assertionRecordSchema
411
+ }),
412
+ z.strictObject({
413
+ id,
414
+ ok: z.literal(true),
415
+ op: z.literal("record"),
416
+ assertion: assertionRecordSchema
417
+ }),
418
+ z.strictObject({
419
+ id,
420
+ ok: z.literal(true),
421
+ op: z.literal("judge"),
422
+ verdict: z.union([verdictSchema, judgeFailureSchema])
423
+ })
424
+ ]),
425
+ z.strictObject({
426
+ id,
427
+ ok: z.literal(true),
428
+ op: z.literal("expect"),
429
+ untested: z.record(z.string(), z.array(z.string())).describe("The strings the matcher tested that a `{ matchedBy, answers }` argument has no answer for, by its `matchedBy`. Nothing was recorded: answer them and make the call again.")
430
+ }),
431
+ z.strictObject({
432
+ id,
433
+ ok: z.literal(false),
434
+ op: z.enum([
435
+ "send",
436
+ "exec",
437
+ "readFile",
438
+ "writeFile",
439
+ "expect",
440
+ "record",
441
+ "judge",
442
+ "serve",
443
+ "kill",
444
+ "capture"
445
+ ]),
446
+ error: attemptErrorSchema
447
+ })
448
+ ]).describe("reeval's reply to a call: its result, or the error it threw. A failed matcher is a result (its record says `fail`), not an error, unless the matcher couldn't run.");
449
+ const cellRoleSchema = z.enum(["control", "variant"]);
450
+ const cellSchema = z.strictObject({
451
+ key: z.string(),
452
+ harness: z.string(),
453
+ environment: z.string(),
454
+ sandbox: z.string(),
455
+ role: cellRoleSchema,
456
+ variant: z.string().optional()
457
+ });
458
+ z.strictObject({
459
+ type: z.literal("start"),
460
+ attempt: z.strictObject({
461
+ id: nonEmpty,
462
+ cell: cellSchema,
463
+ n: z.number().int().min(1, "a run number, from 1").describe("The run number, from 1.")
464
+ }).describe("`c.attempt`: which cell of the matrix this attempt is, and its run number."),
465
+ file: nonEmpty.describe("The eval file, relative to the project."),
466
+ body: nonEmpty.describe("The eval's `body`, as its declaration named it.")
467
+ }).describe("reeval to the runner: run this eval's body for this attempt. Nothing has started yet: the sandbox and the agent session start on the body's first call that needs them, so its own code runs first. The runner sends its calls, then `done`.");
468
+ z.strictObject({
469
+ type: z.literal("abort"),
470
+ reason: z.enum(["timeout", "canceled"]).describe("`timeout`: the eval's `timeout` ran out. `canceled`: the run was stopped."),
471
+ message: z.string().describe("reeval's sentence, e.g. `Eval timed out after 600s`.")
472
+ }).describe("reeval to the runner: the attempt was aborted, as `c.signal` aborts. Calls in flight reply with errors; the runner should stop the body and send `done`.");
473
+ z.strictObject({
474
+ type: z.literal("done"),
475
+ error: attemptErrorSchema.optional().describe("What the body threw, if it threw. A failed `expect` (not soft) is one.")
476
+ }).describe("The runner to reeval: the eval's body has finished. The attempt's checks and record follow.");
477
+ const CALL_KEYS = Object.fromEntries(attemptCallSchema.options.map((o) => [o.shape.op.value, Object.keys(o.shape)]));
478
+ const OTHER_KEYS = [
479
+ "id",
480
+ "call",
481
+ "turn",
482
+ "attempt",
483
+ "value",
484
+ "regexp",
485
+ "flags",
486
+ "matchedBy",
487
+ "answers",
488
+ ...Object.keys(judgeCallDataSchema.shape),
489
+ ...Object.keys(judgeStateSchema.shape),
490
+ ...Object.keys(captureSchema.shape),
491
+ ...Object.keys(captureCallDataSchema.shape),
492
+ "port",
493
+ "handle",
494
+ "exec",
495
+ "result",
496
+ "readFile",
497
+ "interval",
498
+ "timeout",
499
+ "ms"
500
+ ];
501
+ /** What's wrong with an attempt request a runner sent, by the field's path, or undefined. */
502
+ const attemptRequestProblem = (value) => dataProblem(attemptRequestSchema, value, "The attempt request", (object) => typeof object.op === "string" ? CALL_KEYS[object.op] ?? [] : OTHER_KEYS);
503
+ //#endregion
504
+ //#region src/capture/fake-instructions.ts
505
+ const FAKE_INSTRUCTIONS = Symbol.for("reeval.capture.fakeInstructions");
506
+ function isWaitStep(step) {
507
+ return typeof step === "object" && step !== null && !Array.isArray(step) && "wait" in step;
329
508
  }
330
- function chromeUrl(platform) {
331
- return `https://storage.googleapis.com/chrome-for-testing-public/${CHROME_VERSION}/${platform}/${CHROME_NAME}-${platform}.zip`;
509
+ /** One list: action lists and `{ wait }` inlined. Returns branded `instructions`. */
510
+ function fakeInstructions(steps) {
511
+ if (!Array.isArray(steps)) throw new Error(`fakeInstructions() is an array of action lists and { wait } (got ${JSON.stringify(steps)})`);
512
+ const instructions = [];
513
+ let goals = 0;
514
+ for (const [i, step] of steps.entries()) {
515
+ if (isWaitStep(step)) {
516
+ instructions.push({ wait: step.wait });
517
+ continue;
518
+ }
519
+ if (!Array.isArray(step)) throw new Error(`fakeInstructions()[${i}] is an action list or { wait } (got ${JSON.stringify(step)})`);
520
+ instructions.push(`fake[${goals}]`);
521
+ goals++;
522
+ }
523
+ Object.defineProperty(instructions, FAKE_INSTRUCTIONS, {
524
+ value: steps,
525
+ enumerable: false
526
+ });
527
+ return instructions;
332
528
  }
333
- function binaryName() {
334
- return process.platform === "win32" ? `${CHROME_NAME}.exe` : CHROME_NAME;
529
+ function fakeInstructionSteps(value) {
530
+ if (typeof value === "object" && value !== null && FAKE_INSTRUCTIONS in value) return value[FAKE_INSTRUCTIONS];
335
531
  }
336
- async function downloadZip(url, onProgress, cacheLabel, signal) {
337
- const res = await fetch(url, { signal });
338
- if (!res.ok) throw new Error(`download failed: HTTP ${res.status}`);
339
- const total = Number(res.headers.get("content-length")) || null;
340
- const report = (received) => onProgress?.({
341
- name: CHROME_NAME,
342
- version: CHROME_VERSION,
343
- bytes: total,
344
- received,
345
- cache: cacheLabel
346
- });
347
- report(0);
348
- if (!res.body) return Buffer.from(await res.arrayBuffer());
349
- const reader = res.body.getReader();
350
- const chunks = [];
351
- let received = 0;
352
- let last = 0;
353
- for (;;) {
354
- const { done, value } = await reader.read();
355
- if (done) break;
356
- chunks.push(value);
357
- received += value.length;
358
- if (received - last >= 262144 || total && received === total) {
359
- report(received);
360
- last = received;
532
+ //#endregion
533
+ //#region src/capture/zip.ts
534
+ const LOCAL = 67324752;
535
+ /** Extracts a zip (store or deflate) into `files` keyed by posix path. */
536
+ function unzip(bytes) {
537
+ const buf = Buffer.from(bytes);
538
+ const files = /* @__PURE__ */ new Map();
539
+ let i = 0;
540
+ while (i + 30 <= buf.length) {
541
+ if (buf.readUInt32LE(i) !== LOCAL) break;
542
+ const method = buf.readUInt16LE(i + 8);
543
+ const comp = buf.readUInt32LE(i + 18);
544
+ const nameLen = buf.readUInt16LE(i + 26);
545
+ const extra = buf.readUInt16LE(i + 28);
546
+ const name = buf.toString("utf8", i + 30, i + 30 + nameLen);
547
+ const start = i + 30 + nameLen + extra;
548
+ const raw = buf.subarray(start, start + comp);
549
+ if (name.endsWith("/")) {
550
+ i = start + comp;
551
+ continue;
361
552
  }
553
+ if (method === 0) files.set(name, Buffer.from(raw));
554
+ else if (method === 8) files.set(name, Buffer.from(inflateRawSync(raw)));
555
+ else throw new Error(`zip method ${method} is not supported (${name})`);
556
+ i = start + comp;
362
557
  }
363
- if (received !== last) report(received);
364
- return Buffer.concat(chunks);
558
+ return files;
365
559
  }
366
- function extractBinary(zip, dest, platform) {
367
- const files = unzip(zip);
368
- const needle = `/${binaryName()}`;
369
- const entry = [...files.entries()].find(([name]) => name.endsWith(needle) || name === binaryName() || name.endsWith(`\\${binaryName()}`));
370
- if (!entry) {
371
- const names = [...files.keys()].slice(0, 8).join(", ");
372
- throw new Error(`zip has no ${binaryName()} (saw ${names || "nothing"})`);
373
- }
374
- const prefix = entry[0].slice(0, -binaryName().length);
375
- mkdirSync(dest, { recursive: true });
376
- for (const [name, data] of files) {
377
- const rel = prefix && name.startsWith(prefix) ? name.slice(prefix.length) : name;
378
- if (!rel || rel.endsWith("/")) continue;
379
- const target = join(dest, ...rel.split("/"));
380
- mkdirSync(join(target, ".."), { recursive: true });
381
- writeFileSync(target, data);
382
- }
383
- const bin = join(dest, binaryName());
384
- if (!existsSync(bin)) writeFileSync(bin, entry[1]);
385
- if (process.platform !== "win32") chmodSync(bin, 493);
560
+ //#endregion
561
+ //#region src/capture/browser.ts
562
+ const isMs = (wait) => typeof wait === "object" && wait !== null;
563
+ /** The CDP event `'idle'` waits for; `'load'` is `Page.loadEventFired`. */
564
+ function captureWaitLabel(wait) {
565
+ return wait === "idle" ? "networkAlmostIdle" : "load";
386
566
  }
387
- /**
388
- * Returns the cached binary, downloading on a miss. `onProgress` runs only for a download.
389
- */
390
- async function ensureChrome(cacheRoot, options = { cacheLabel: cacheRoot }) {
391
- const platform = chromePlatform();
392
- const dest = join(cacheRoot, CHROME_NAME, CHROME_VERSION, platform);
393
- const bin = join(dest, binaryName());
394
- if (existsSync(bin)) return bin;
395
- mkdirSync(join(cacheRoot, CHROME_NAME, CHROME_VERSION), { recursive: true });
396
- const partial = `${dest}.${process.pid}.partial`;
397
- rmSync(partial, {
398
- recursive: true,
399
- force: true
400
- });
401
- try {
402
- extractBinary(await downloadZip(chromeUrl(platform), options.onProgress, options.cacheLabel, options.signal), partial, platform);
403
- try {
404
- renameSync(partial, dest);
405
- } catch (error) {
406
- rmSync(partial, {
407
- recursive: true,
408
- force: true
409
- });
410
- if (!existsSync(bin)) throw error;
411
- }
412
- } catch (error) {
413
- rmSync(partial, {
414
- recursive: true,
415
- force: true
416
- });
417
- if (existsSync(bin)) return bin;
418
- throw error;
419
- }
420
- return bin;
567
+ function captureWaitTimeout(wait) {
568
+ return /* @__PURE__ */ new Error(`c.capture() timed out waiting for ${captureWaitLabel(wait)}`);
421
569
  }
422
- var Cdp = class {
423
- #ws;
424
- #next = 0;
425
- #pending = /* @__PURE__ */ new Map();
426
- #events = /* @__PURE__ */ new Map();
427
- constructor(ws) {
428
- this.#ws = ws;
429
- ws.addEventListener("message", (ev) => {
430
- const msg = JSON.parse(String(ev.data));
431
- if (msg.id !== void 0) {
432
- const p = this.#pending.get(msg.id);
433
- if (!p) return;
434
- this.#pending.delete(msg.id);
435
- if (msg.error) p.reject(new Error(msg.error.message));
436
- else p.resolve(msg.result);
437
- return;
438
- }
439
- if (msg.method) for (const fn of this.#events.get(msg.method) ?? []) fn(msg);
440
- });
570
+ function waitTimeout(wait, detail) {
571
+ if (isMs(wait)) return /* @__PURE__ */ new Error(`c.capture() timed out${detail ? ` on ${detail}` : ""} waiting ${wait.ms}ms`);
572
+ return captureWaitTimeout(wait);
573
+ }
574
+ async function waitForPage(cdp, wait, timeoutMs, nav) {
575
+ if (isMs(wait)) {
576
+ if (timeoutMs <= 0) throw waitTimeout(wait);
577
+ const ms = Math.min(wait.ms, timeoutMs);
578
+ await new Promise((resolve) => setTimeout(resolve, ms));
579
+ return;
441
580
  }
442
- on(method, fn) {
443
- const list = this.#events.get(method) ?? [];
444
- list.push(fn);
445
- this.#events.set(method, list);
581
+ const page = listenForCaptureWait(cdp, wait);
582
+ page.arm();
583
+ await page.ready(nav, timeoutMs);
584
+ }
585
+ const clamp = (n, lo, hi) => Math.min(hi, Math.max(lo, n));
586
+ /** Map a full-page click onto the live window, scrolling with the wheel if needed. */
587
+ function mapPagePoint(x, y, window, page, scrollY) {
588
+ const maxY = Math.max(0, (page?.height ?? window.height) - window.height);
589
+ const target = y >= window.height || y < 0 ? clamp(y - window.height / 2, 0, maxY) : scrollY;
590
+ const nextY = clamp(target, 0, maxY);
591
+ return {
592
+ x: clamp(x, 0, window.width - 1),
593
+ y: clamp(y - nextY, 0, window.height - 1),
594
+ scrollY: nextY,
595
+ dy: nextY - scrollY
596
+ };
597
+ }
598
+ async function dispatchInput(cdp, action, window, page, scrollY = 0) {
599
+ if (action.type === "done") return scrollY;
600
+ if (action.type === "type") {
601
+ await cdp.send("Input.insertText", { text: action.text });
602
+ return scrollY;
446
603
  }
447
- send(method, params) {
448
- const id = ++this.#next;
449
- this.#ws.send(JSON.stringify({
450
- id,
451
- method,
452
- params
453
- }));
454
- return new Promise((resolve, reject) => this.#pending.set(id, {
455
- resolve,
456
- reject
457
- }));
604
+ if (action.type === "key") {
605
+ const key = action.key;
606
+ const vk = key === "Tab" ? 9 : 13;
607
+ await cdp.send("Input.dispatchKeyEvent", {
608
+ type: "keyDown",
609
+ key,
610
+ code: key,
611
+ windowsVirtualKeyCode: vk
612
+ });
613
+ await cdp.send("Input.dispatchKeyEvent", {
614
+ type: "keyUp",
615
+ key,
616
+ code: key,
617
+ windowsVirtualKeyCode: vk
618
+ });
619
+ return scrollY;
458
620
  }
459
- close() {
460
- this.#ws.close();
621
+ if (action.type === "scroll") {
622
+ await cdp.send("Input.dispatchMouseEvent", {
623
+ type: "mouseWheel",
624
+ x: action.x,
625
+ y: action.y,
626
+ deltaX: 0,
627
+ deltaY: action.dy
628
+ });
629
+ return scrollY + action.dy;
461
630
  }
462
- };
463
- function waitForPortFile(file, timeoutMs) {
464
- const deadline = Date.now() + timeoutMs;
465
- return new Promise((resolve, reject) => {
466
- const tick = () => {
467
- if (existsSync(file)) {
468
- const line = readFileSync(file, "utf8").split(/\r?\n/)[0];
469
- const port = Number(line);
470
- if (port > 0) {
471
- resolve(port);
472
- return;
473
- }
474
- }
475
- if (Date.now() > deadline) {
476
- reject(/* @__PURE__ */ new Error("chrome-headless-shell did not open a DevTools port"));
477
- return;
478
- }
479
- setTimeout(tick, 50);
480
- };
481
- tick();
631
+ const mapped = mapPagePoint(action.type === "drag" ? action.x : action.x, action.type === "drag" ? action.y : action.y, window, page, scrollY);
632
+ if (mapped.dy !== 0) await cdp.send("Input.dispatchMouseEvent", {
633
+ type: "mouseWheel",
634
+ x: window.width / 2,
635
+ y: window.height / 2,
636
+ deltaX: 0,
637
+ deltaY: mapped.dy
482
638
  });
483
- }
484
- function pickPageDebuggerUrl(targets) {
485
- return targets.find((t) => t.webSocketDebuggerUrl && t.type !== "browser")?.webSocketDebuggerUrl;
486
- }
487
- async function pageDebuggerUrl(port, timeoutMs) {
488
- const deadline = Date.now() + timeoutMs;
489
- const listPages = async () => {
490
- const res = await fetch(`http://127.0.0.1:${port}/json/list`);
491
- if (!res.ok) return void 0;
492
- return pickPageDebuggerUrl(await res.json());
639
+ const at = {
640
+ x: mapped.x,
641
+ y: mapped.y
493
642
  };
494
- for (;;) {
495
- const url = await listPages();
496
- if (url) return url;
497
- if (Date.now() > deadline) break;
498
- await new Promise((r) => setTimeout(r, 50));
499
- }
500
- const version = await (await fetch(`http://127.0.0.1:${port}/json/version`)).json();
501
- if (!version.webSocketDebuggerUrl) throw new Error("chrome-headless-shell reported no DevTools URL");
502
- const browser = await cdpConnect(version.webSocketDebuggerUrl, timeoutMs);
503
- try {
504
- await browser.send("Target.createTarget", { url: "about:blank" });
505
- } finally {
506
- browser.close();
643
+ if (action.type === "hover") {
644
+ await cdp.send("Input.dispatchMouseEvent", {
645
+ type: "mouseMoved",
646
+ ...at
647
+ });
648
+ return mapped.scrollY;
507
649
  }
508
- const created = await listPages();
509
- if (!created) throw new Error("chrome-headless-shell reported no page target");
510
- return created;
511
- }
512
- async function cdpConnect(url, timeoutMs) {
513
- const ws = new WebSocket(url);
514
- await new Promise((resolve, reject) => {
515
- const t = setTimeout(() => reject(/* @__PURE__ */ new Error("CDP websocket timed out")), timeoutMs);
516
- ws.addEventListener("open", () => {
517
- clearTimeout(t);
518
- resolve();
650
+ if (action.type === "click") {
651
+ await cdp.send("Input.dispatchMouseEvent", {
652
+ type: "mouseMoved",
653
+ ...at,
654
+ button: "left"
519
655
  });
520
- ws.addEventListener("error", () => {
521
- clearTimeout(t);
522
- reject(/* @__PURE__ */ new Error("CDP websocket failed"));
656
+ await cdp.send("Input.dispatchMouseEvent", {
657
+ type: "mousePressed",
658
+ ...at,
659
+ button: "left",
660
+ clickCount: 1
661
+ });
662
+ await cdp.send("Input.dispatchMouseEvent", {
663
+ type: "mouseReleased",
664
+ ...at,
665
+ button: "left",
666
+ clickCount: 1
523
667
  });
668
+ return mapped.scrollY;
669
+ }
670
+ const end = mapPagePoint(action.toX, action.toY, window, page, mapped.scrollY);
671
+ if (end.dy !== 0) await cdp.send("Input.dispatchMouseEvent", {
672
+ type: "mouseWheel",
673
+ x: window.width / 2,
674
+ y: window.height / 2,
675
+ deltaX: 0,
676
+ deltaY: end.dy
524
677
  });
525
- return new Cdp(ws);
678
+ await cdp.send("Input.dispatchMouseEvent", {
679
+ type: "mouseMoved",
680
+ x: at.x,
681
+ y: at.y,
682
+ button: "left"
683
+ });
684
+ await cdp.send("Input.dispatchMouseEvent", {
685
+ type: "mousePressed",
686
+ x: at.x,
687
+ y: at.y,
688
+ button: "left",
689
+ clickCount: 1
690
+ });
691
+ await cdp.send("Input.dispatchMouseEvent", {
692
+ type: "mouseMoved",
693
+ x: end.x,
694
+ y: end.y,
695
+ button: "left"
696
+ });
697
+ await cdp.send("Input.dispatchMouseEvent", {
698
+ type: "mouseReleased",
699
+ x: end.x,
700
+ y: end.y,
701
+ button: "left",
702
+ clickCount: 1
703
+ });
704
+ return end.scrollY;
705
+ }
706
+ function lifecycleParams(msg) {
707
+ const body = msg;
708
+ return body.params ?? body;
526
709
  }
527
710
  /**
528
- * Drops a chrome-headless-shell profile. Chrome can still write after SIGKILL,
529
- * so `ENOTEMPTY` / `EBUSY` must not replace a PNG already in hand.
711
+ * Arms after `Page.enable` so about:blank's events cannot resolve the wait.
712
+ * Call `arm` immediately before `Page.navigate`.
530
713
  */
531
- function removeChromeProfile(dir, rm = rmSync) {
532
- try {
533
- rm(dir, {
534
- recursive: true,
535
- force: true,
536
- maxRetries: 5,
537
- retryDelay: 20
714
+ function listenForCaptureWait(cdp, wait) {
715
+ const method = wait === "idle" ? "Page.lifecycleEvent" : "Page.loadEventFired";
716
+ const pending = [];
717
+ let deliver;
718
+ let started = false;
719
+ cdp.on(method, (msg) => {
720
+ if (!started) return;
721
+ if (deliver) deliver(msg);
722
+ else pending.push(msg);
723
+ });
724
+ const matches = (msg, loaderId) => {
725
+ if (wait === "load") return true;
726
+ const params = lifecycleParams(msg);
727
+ if (params.name !== "networkAlmostIdle") return false;
728
+ return !loaderId || params.loaderId === loaderId;
729
+ };
730
+ return {
731
+ arm() {
732
+ started = true;
733
+ },
734
+ ready(nav, timeoutMs) {
735
+ return new Promise((resolve, reject) => {
736
+ const timer = setTimeout(() => reject(captureWaitTimeout(wait)), timeoutMs);
737
+ const finish = () => {
738
+ clearTimeout(timer);
739
+ resolve();
740
+ };
741
+ deliver = (msg) => {
742
+ if (matches(msg, nav?.loaderId)) finish();
743
+ };
744
+ if (pending.some((msg) => matches(msg, nav?.loaderId))) finish();
745
+ });
746
+ }
747
+ };
748
+ }
749
+ const QUIET_IN_FLIGHT = 2;
750
+ function requestId(msg) {
751
+ const body = msg;
752
+ return (body.params ?? body).requestId;
753
+ }
754
+ /**
755
+ * Counts the tab's requests from `Network.enable` on, for a journey's `{ wait: 'idle' }` beat.
756
+ * Chromium emits `networkAlmostIdle` once per document, so after a click that only fetches it
757
+ * never comes again. As in Chromium, every request that starts or ends restarts the quiet window.
758
+ */
759
+ function trackNetwork(cdp, quietMs = 500) {
760
+ const inFlight = /* @__PURE__ */ new Set();
761
+ let changed;
762
+ cdp.on("Network.requestWillBeSent", (msg) => {
763
+ const id = requestId(msg);
764
+ if (id === void 0) return;
765
+ inFlight.add(id);
766
+ changed?.();
767
+ });
768
+ const end = (msg) => {
769
+ const id = requestId(msg);
770
+ if (id === void 0 || !inFlight.delete(id)) return;
771
+ changed?.();
772
+ };
773
+ cdp.on("Network.loadingFinished", end);
774
+ cdp.on("Network.loadingFailed", end);
775
+ return { quiet(timeoutMs) {
776
+ return new Promise((resolve, reject) => {
777
+ let settle;
778
+ const stop = () => {
779
+ clearTimeout(timer);
780
+ clearTimeout(settle);
781
+ changed = void 0;
782
+ };
783
+ const timer = setTimeout(() => {
784
+ stop();
785
+ reject(captureWaitTimeout("idle"));
786
+ }, timeoutMs);
787
+ changed = () => {
788
+ clearTimeout(settle);
789
+ if (inFlight.size > QUIET_IN_FLIGHT) return;
790
+ settle = setTimeout(() => {
791
+ stop();
792
+ resolve();
793
+ }, quietMs);
794
+ };
795
+ changed();
538
796
  });
539
- } catch {
540
- scheduleChromeProfileSweep(dir, rm);
541
- }
797
+ } };
542
798
  }
543
- function scheduleChromeProfileSweep(dir, rm) {
544
- setTimeout(() => {
545
- try {
546
- rm(dir, {
547
- recursive: true,
548
- force: true,
549
- maxRetries: 5,
550
- retryDelay: 50
551
- });
552
- } catch {}
553
- }, 250).unref();
799
+ /** Pinned Chrome for Testing `chrome-headless-shell` build. */
800
+ const CHROME_VERSION = "140.0.7339.185";
801
+ const CHROME_NAME = "chrome-headless-shell";
802
+ function chromePlatform() {
803
+ if (process.platform === "darwin") return process.arch === "arm64" ? "mac-arm64" : "mac-x64";
804
+ if (process.platform === "win32") return "win64";
805
+ return "linux64";
554
806
  }
555
- function waitForExit(child, timeoutMs) {
556
- if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
557
- return new Promise((resolve) => {
558
- const finish = () => {
559
- clearTimeout(timer);
560
- child.off("exit", finish);
561
- resolve();
562
- };
563
- const timer = setTimeout(finish, timeoutMs);
564
- child.once("exit", finish);
807
+ function chromeUrl(platform) {
808
+ return `https://storage.googleapis.com/chrome-for-testing-public/${CHROME_VERSION}/${platform}/${CHROME_NAME}-${platform}.zip`;
809
+ }
810
+ function binaryName() {
811
+ return process.platform === "win32" ? `${CHROME_NAME}.exe` : CHROME_NAME;
812
+ }
813
+ async function downloadZip(url, onProgress, cacheLabel, signal) {
814
+ const res = await fetch(url, { signal });
815
+ if (!res.ok) throw new Error(`download failed: HTTP ${res.status}`);
816
+ const total = Number(res.headers.get("content-length")) || null;
817
+ const report = (received) => onProgress?.({
818
+ name: CHROME_NAME,
819
+ version: CHROME_VERSION,
820
+ bytes: total,
821
+ received,
822
+ cache: cacheLabel
565
823
  });
824
+ report(0);
825
+ if (!res.body) return Buffer.from(await res.arrayBuffer());
826
+ const reader = res.body.getReader();
827
+ const chunks = [];
828
+ let received = 0;
829
+ let last = 0;
830
+ for (;;) {
831
+ const { done, value } = await reader.read();
832
+ if (done) break;
833
+ chunks.push(value);
834
+ received += value.length;
835
+ if (received - last >= 262144 || total && received === total) {
836
+ report(received);
837
+ last = received;
838
+ }
839
+ }
840
+ if (received !== last) report(received);
841
+ return Buffer.concat(chunks);
566
842
  }
567
- /** Kills the shell and drops its profile. Never throws: the PNG is already in hand. */
568
- async function disposeChrome(child, userData) {
569
- try {
570
- if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
571
- await waitForExit(child, 1e3);
572
- } catch {}
573
- removeChromeProfile(userData);
843
+ function extractBinary(zip, dest, platform) {
844
+ const files = unzip(zip);
845
+ const needle = `/${binaryName()}`;
846
+ const entry = [...files.entries()].find(([name]) => name.endsWith(needle) || name === binaryName() || name.endsWith(`\\${binaryName()}`));
847
+ if (!entry) {
848
+ const names = [...files.keys()].slice(0, 8).join(", ");
849
+ throw new Error(`zip has no ${binaryName()} (saw ${names || "nothing"})`);
850
+ }
851
+ const prefix = entry[0].slice(0, -binaryName().length);
852
+ mkdirSync(dest, { recursive: true });
853
+ for (const [name, data] of files) {
854
+ const rel = prefix && name.startsWith(prefix) ? name.slice(prefix.length) : name;
855
+ if (!rel || rel.endsWith("/")) continue;
856
+ const target = join(dest, ...rel.split("/"));
857
+ mkdirSync(join(target, ".."), { recursive: true });
858
+ writeFileSync(target, data);
859
+ }
860
+ const bin = join(dest, binaryName());
861
+ if (!existsSync(bin)) writeFileSync(bin, entry[1]);
862
+ if (process.platform !== "win32") chmodSync(bin, 493);
574
863
  }
575
- /** One Chrome process, one tab. `fn` must not keep the session after it returns. */
864
+ /**
865
+ * Returns the cached binary, downloading on a miss. `onProgress` runs only for a download.
866
+ */
867
+ async function ensureChrome(cacheRoot, options = { cacheLabel: cacheRoot }) {
868
+ const platform = chromePlatform();
869
+ const dest = join(cacheRoot, CHROME_NAME, CHROME_VERSION, platform);
870
+ const bin = join(dest, binaryName());
871
+ if (existsSync(bin)) return bin;
872
+ mkdirSync(join(cacheRoot, CHROME_NAME, CHROME_VERSION), { recursive: true });
873
+ const partial = `${dest}.${process.pid}.partial`;
874
+ rmSync(partial, {
875
+ recursive: true,
876
+ force: true
877
+ });
878
+ try {
879
+ extractBinary(await downloadZip(chromeUrl(platform), options.onProgress, options.cacheLabel, options.signal), partial, platform);
880
+ try {
881
+ renameSync(partial, dest);
882
+ } catch (error) {
883
+ rmSync(partial, {
884
+ recursive: true,
885
+ force: true
886
+ });
887
+ if (!existsSync(bin)) throw error;
888
+ }
889
+ } catch (error) {
890
+ rmSync(partial, {
891
+ recursive: true,
892
+ force: true
893
+ });
894
+ if (existsSync(bin)) return bin;
895
+ throw error;
896
+ }
897
+ return bin;
898
+ }
899
+ var Cdp = class {
900
+ #ws;
901
+ #next = 0;
902
+ #pending = /* @__PURE__ */ new Map();
903
+ #events = /* @__PURE__ */ new Map();
904
+ constructor(ws) {
905
+ this.#ws = ws;
906
+ ws.addEventListener("message", (ev) => {
907
+ const msg = JSON.parse(String(ev.data));
908
+ if (msg.id !== void 0) {
909
+ const p = this.#pending.get(msg.id);
910
+ if (!p) return;
911
+ this.#pending.delete(msg.id);
912
+ if (msg.error) p.reject(new Error(msg.error.message));
913
+ else p.resolve(msg.result);
914
+ return;
915
+ }
916
+ if (msg.method) for (const fn of this.#events.get(msg.method) ?? []) fn(msg);
917
+ });
918
+ }
919
+ on(method, fn) {
920
+ const list = this.#events.get(method) ?? [];
921
+ list.push(fn);
922
+ this.#events.set(method, list);
923
+ }
924
+ send(method, params) {
925
+ const id = ++this.#next;
926
+ this.#ws.send(JSON.stringify({
927
+ id,
928
+ method,
929
+ params
930
+ }));
931
+ return new Promise((resolve, reject) => this.#pending.set(id, {
932
+ resolve,
933
+ reject
934
+ }));
935
+ }
936
+ close() {
937
+ this.#ws.close();
938
+ }
939
+ };
940
+ function waitForPortFile(file, timeoutMs) {
941
+ const deadline = Date.now() + timeoutMs;
942
+ return new Promise((resolve, reject) => {
943
+ const tick = () => {
944
+ if (existsSync(file)) {
945
+ const line = readFileSync(file, "utf8").split(/\r?\n/)[0];
946
+ const port = Number(line);
947
+ if (port > 0) {
948
+ resolve(port);
949
+ return;
950
+ }
951
+ }
952
+ if (Date.now() > deadline) {
953
+ reject(/* @__PURE__ */ new Error("chrome-headless-shell did not open a DevTools port"));
954
+ return;
955
+ }
956
+ setTimeout(tick, 50);
957
+ };
958
+ tick();
959
+ });
960
+ }
961
+ function pickPageDebuggerUrl(targets) {
962
+ return targets.find((t) => t.webSocketDebuggerUrl && t.type !== "browser")?.webSocketDebuggerUrl;
963
+ }
964
+ async function pageDebuggerUrl(port, timeoutMs) {
965
+ const deadline = Date.now() + timeoutMs;
966
+ const listPages = async () => {
967
+ const res = await fetch(`http://127.0.0.1:${port}/json/list`);
968
+ if (!res.ok) return void 0;
969
+ return pickPageDebuggerUrl(await res.json());
970
+ };
971
+ for (;;) {
972
+ const url = await listPages();
973
+ if (url) return url;
974
+ if (Date.now() > deadline) break;
975
+ await new Promise((r) => setTimeout(r, 50));
976
+ }
977
+ const version = await (await fetch(`http://127.0.0.1:${port}/json/version`)).json();
978
+ if (!version.webSocketDebuggerUrl) throw new Error("chrome-headless-shell reported no DevTools URL");
979
+ const browser = await cdpConnect(version.webSocketDebuggerUrl, timeoutMs);
980
+ try {
981
+ await browser.send("Target.createTarget", { url: "about:blank" });
982
+ } finally {
983
+ browser.close();
984
+ }
985
+ const created = await listPages();
986
+ if (!created) throw new Error("chrome-headless-shell reported no page target");
987
+ return created;
988
+ }
989
+ async function cdpConnect(url, timeoutMs) {
990
+ const ws = new WebSocket(url);
991
+ await new Promise((resolve, reject) => {
992
+ const t = setTimeout(() => reject(/* @__PURE__ */ new Error("CDP websocket timed out")), timeoutMs);
993
+ ws.addEventListener("open", () => {
994
+ clearTimeout(t);
995
+ resolve();
996
+ });
997
+ ws.addEventListener("error", () => {
998
+ clearTimeout(t);
999
+ reject(/* @__PURE__ */ new Error("CDP websocket failed"));
1000
+ });
1001
+ });
1002
+ return new Cdp(ws);
1003
+ }
1004
+ /**
1005
+ * Drops a chrome-headless-shell profile. Chrome can still write after SIGKILL,
1006
+ * so `ENOTEMPTY` / `EBUSY` must not replace a PNG already in hand.
1007
+ */
1008
+ function removeChromeProfile(dir, rm = rmSync) {
1009
+ try {
1010
+ rm(dir, {
1011
+ recursive: true,
1012
+ force: true,
1013
+ maxRetries: 5,
1014
+ retryDelay: 20
1015
+ });
1016
+ } catch {
1017
+ scheduleChromeProfileSweep(dir, rm);
1018
+ }
1019
+ }
1020
+ function scheduleChromeProfileSweep(dir, rm) {
1021
+ setTimeout(() => {
1022
+ try {
1023
+ rm(dir, {
1024
+ recursive: true,
1025
+ force: true,
1026
+ maxRetries: 5,
1027
+ retryDelay: 50
1028
+ });
1029
+ } catch {}
1030
+ }, 250).unref();
1031
+ }
1032
+ function waitForExit(child, timeoutMs) {
1033
+ if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
1034
+ return new Promise((resolve) => {
1035
+ const finish = () => {
1036
+ clearTimeout(timer);
1037
+ child.off("exit", finish);
1038
+ resolve();
1039
+ };
1040
+ const timer = setTimeout(finish, timeoutMs);
1041
+ child.once("exit", finish);
1042
+ });
1043
+ }
1044
+ /** Kills the shell and drops its profile. Never throws: the PNG is already in hand. */
1045
+ async function disposeChrome(child, userData) {
1046
+ try {
1047
+ if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
1048
+ await waitForExit(child, 1e3);
1049
+ } catch {}
1050
+ removeChromeProfile(userData);
1051
+ }
1052
+ /** One Chrome process, one tab. `fn` must not keep the session after it returns. */
576
1053
  async function withChrome(bin, window, timeoutMs, fn) {
577
1054
  const userData = mkdtempSync(join(tmpdir(), `reeval-chrome-${process.pid}-`));
578
1055
  const child = spawn(bin, [
@@ -729,7 +1206,6 @@ const DEFAULT_VIEWPORT = {
729
1206
  width: 1280,
730
1207
  height: 720
731
1208
  };
732
- const CAPTURE_NAME = /^[a-z0-9_-]+$/;
733
1209
  const CAPTURE_WAITS = ["load", "idle"];
734
1210
  /** `'load'` when the call omits `wait`. `'load'`, `'idle'` or `{ ms }`. */
735
1211
  function resolveWait(wait) {
@@ -788,9 +1264,12 @@ function resolveCaptureTarget(target) {
788
1264
  if (target !== null && typeof target === "object" && typeof target.port === "number" && typeof target.kill === "function") return resolveCaptureTarget(target.port);
789
1265
  throw new Error("c.capture() needs a port or a ServeHandle as the first argument");
790
1266
  }
791
- /** Artifact name from a path: `/login` → `login`, `/foo/bar` → `foo-bar`, `/` → `root`. */
1267
+ /**
1268
+ * Artifact name from a path: `/login` → `login`, `/foo/bar` → `foo-bar`, `/` → `root`. Always
1269
+ * a valid capture name: leading `_` and `-` go, and it's cut at the name's length limit.
1270
+ */
792
1271
  function nameFromPath(path) {
793
- return (path.split(/[?#]/, 1)[0] ?? "").replaceAll("\\", "/").split("/").filter(Boolean).join("-").toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "") || "root";
1272
+ return (path.split(/[?#]/, 1)[0] ?? "").replaceAll("\\", "/").split("/").filter(Boolean).join("-").toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^[_-]+/, "").slice(0, 100).replace(/-+$/, "") || "root";
794
1273
  }
795
1274
  function resolveCaptureCall(target, call, cascade) {
796
1275
  if (call && Object.hasOwn(call, "port")) throw new Error("c.capture() takes the port as the first argument, not in the options");
@@ -831,7 +1310,7 @@ function checkCallModel(model) {
831
1310
  if (model.startsWith("typesafe-ai/")) throw new Error(`c.capture() model "${model}" is an evaluation model: it scores yes/no questions and can't drive a capture. Use a language model that accepts images, e.g. 'google/gemini-3.8-flash'.`);
832
1311
  }
833
1312
  function checkCaptureName(name) {
834
- if (!CAPTURE_NAME.test(name)) throw new Error(`c.capture() name must match [a-z0-9_-]+ (got ${JSON.stringify(name)})`);
1313
+ if (!CAPTURE_NAME.test(name)) throw new Error(`c.capture() name ${CAPTURE_NAME_RULE} (got ${JSON.stringify(name)})`);
835
1314
  }
836
1315
  /** A free host port. Closed before the promise resolves, so the caller can bind it. */
837
1316
  function allocatePort() {
@@ -905,672 +1384,202 @@ function actionFromTool(name, input) {
905
1384
  };
906
1385
  }
907
1386
  /**
908
- * Gemini models answer a point normalised to 0–1000 across the image they were shown, whatever
1387
+ * Gemini models answer a point normalized to 0–1000 across the image they were shown, whatever
909
1388
  * the prompt asks, so their journeys are prompted in that scale and their points scaled back.
910
1389
  */
911
- const normalisesPoints = (model) => model.slice(model.lastIndexOf("/") + 1).toLowerCase().startsWith("gemini-");
912
- const NORMALISED = 1e3;
913
- /** Scales an action's 0–1000 points to CSS pixels on a `page`-sized screenshot. */
914
- function scalePoints(action, page) {
915
- const x = (n) => Math.round(n * page.width / NORMALISED);
916
- const y = (n) => Math.round(n * page.height / NORMALISED);
917
- switch (action.type) {
918
- case "click":
919
- case "hover":
920
- case "scroll": return {
921
- ...action,
922
- x: x(action.x),
923
- y: y(action.y)
924
- };
925
- case "drag": return {
926
- ...action,
927
- x: x(action.x),
928
- y: y(action.y),
929
- toX: x(action.toX),
930
- toY: y(action.toY)
931
- };
932
- default: return action;
933
- }
934
- }
935
- const systemPrompt = (normalised) => `You drive a web page from screenshots. Call one tool. ${normalised ? `Coordinates are normalised to 0-${NORMALISED} on the full-page image: x and toX across its width, y and toY down its height. A scroll's dy is in CSS pixels.` : "Coordinates are CSS pixels on the full-page image."} Call done when the goal is met. Do not mention CDP.`;
936
- const useTools = {
937
- click: tool({
938
- description: "Click at (x, y) on the screenshot.",
939
- inputSchema: z.object({
940
- x: z.number(),
941
- y: z.number()
942
- })
943
- }),
944
- type: tool({
945
- description: "Insert text into the focused field. Click the field first.",
946
- inputSchema: z.object({ text: z.string() })
947
- }),
948
- key: tool({
949
- description: "Press Tab or Enter.",
950
- inputSchema: z.object({ key: z.enum(["Tab", "Enter"]) })
951
- }),
952
- scroll: tool({
953
- description: "Wheel-scroll a constrained scroller at (x, y).",
954
- inputSchema: z.object({
955
- x: z.number(),
956
- y: z.number(),
957
- dy: z.number()
958
- })
959
- }),
960
- drag: tool({
961
- description: "Drag from (x, y) to (toX, toY).",
962
- inputSchema: z.object({
963
- x: z.number(),
964
- y: z.number(),
965
- toX: z.number(),
966
- toY: z.number()
967
- })
968
- }),
969
- hover: tool({
970
- description: "Move the pointer to (x, y).",
971
- inputSchema: z.object({
972
- x: z.number(),
973
- y: z.number()
974
- })
975
- }),
976
- done: tool({
977
- description: "The goal is finished. Leave the loop.",
978
- inputSchema: z.object({})
979
- })
980
- };
981
- async function runFakeScript(session, script, goal, remaining) {
982
- let moves = 0;
983
- for (const action of script) {
984
- remainingOrThrow(remaining, JSON.stringify(goal));
985
- if (action.type === "done") return moves;
986
- if (moves >= 32) throw new Error(`c.capture() instruction ${JSON.stringify(goal)} did not finish in 32 moves`);
987
- const page = pngSize(await session.screenshot(true));
988
- debugTrace("check", "capture-use", {
989
- goal,
990
- action,
991
- page
992
- }, `capture use ${action.type}`);
993
- await session.dispatch(action, page);
994
- moves++;
995
- await settle();
996
- }
997
- while (remaining() > 0) await settle();
998
- throw new Error(`c.capture() timed out on ${JSON.stringify(goal)}`);
999
- }
1000
- async function runModelGoal(session, goal, modelName, remaining, loadModel) {
1001
- const model = await loadModel(modelName);
1002
- const normalised = normalisesPoints(modelName);
1003
- let moves = 0;
1004
- let usage = null;
1005
- while (moves < 32) {
1006
- remainingOrThrow(remaining, JSON.stringify(goal));
1007
- const ephemeral = await session.screenshot(true);
1008
- const page = pngSize(ephemeral);
1009
- debugTrace("check", "capture-shot", {
1010
- goal,
1011
- bytes: ephemeral.length,
1012
- page
1013
- }, false);
1014
- let chosen;
1015
- const result = await generateText({
1016
- model,
1017
- system: systemPrompt(normalised),
1018
- messages: [{
1019
- role: "user",
1020
- content: [{
1021
- type: "text",
1022
- text: goal
1023
- }, {
1024
- type: "image",
1025
- image: ephemeral
1026
- }]
1027
- }],
1028
- tools: useTools,
1029
- toolChoice: "required",
1030
- stopWhen: stepCountIs(1),
1031
- abortSignal: AbortSignal.timeout(remainingOrThrow(remaining, JSON.stringify(goal)))
1032
- });
1033
- usage = addUsage(usage, asUsage(result.usage ?? {}));
1034
- const call = result.toolCalls?.[0];
1035
- if (call) chosen = actionFromTool(call.toolName, call.input ?? {});
1036
- if (chosen && normalised) chosen = scalePoints(chosen, page);
1037
- if (!chosen) throw new Error(`c.capture() instruction ${JSON.stringify(goal)} returned no action`);
1038
- debugLog("check", `capture use ${chosen.type} on ${JSON.stringify(goal)}`);
1039
- debugTrace("check", "capture-use", {
1040
- goal,
1041
- action: chosen,
1042
- page
1043
- }, `capture use ${chosen.type}`);
1044
- if (chosen.type === "done") return {
1045
- moves,
1046
- usage
1047
- };
1048
- await session.dispatch(chosen, page);
1049
- moves++;
1050
- await settle();
1051
- }
1052
- throw new Error(`c.capture() instruction ${JSON.stringify(goal)} did not finish in 32 moves`);
1053
- }
1054
- /**
1055
- * Top-level wait, then the instruction loop, then the saved PNG.
1056
- * Throws before `screenshot` when the budget or move cap is hit.
1057
- */
1058
- async function runJourney(session, url, options) {
1059
- const { wait, timeoutMs, fullPage, instructions, use, remaining } = options;
1060
- remainingOrThrow(remaining, "navigate");
1061
- await navigateAndWait(session, url, wait, Math.min(timeoutMs, remaining()));
1062
- let moves = 0;
1063
- let usage = null;
1064
- let stringIndex = 0;
1065
- for (const step of instructions) {
1066
- if (typeof step !== "string") {
1067
- remainingOrThrow(remaining, "wait");
1068
- await session.wait(step.wait, remaining(), void 0);
1069
- continue;
1070
- }
1071
- remainingOrThrow(remaining, JSON.stringify(step));
1072
- debugLog("check", `capture instruction ${JSON.stringify(step)}`);
1073
- if (use && "fake" in use) {
1074
- const script = use.fake[stringIndex] ?? [];
1075
- moves += await runFakeScript(session, script, step, remaining);
1076
- } else if (use && "model" in use) {
1077
- if (!options.loadModel) throw new Error("c.capture() instructions need a model loader");
1078
- const ran = await runModelGoal(session, step, use.model, remaining, options.loadModel);
1079
- moves += ran.moves;
1080
- usage = addUsage(usage, ran.usage);
1081
- }
1082
- stringIndex++;
1083
- await settle();
1084
- }
1085
- remainingOrThrow(remaining, "screenshot");
1086
- return {
1087
- png: await session.screenshot(fullPage),
1088
- moves,
1089
- model: use && "model" in use ? use.model : use && "fake" in use ? "fake" : void 0,
1090
- usage
1091
- };
1092
- }
1093
- function captureUseSpend(result) {
1094
- if (!result.model) return void 0;
1095
- return {
1096
- model: result.model,
1097
- moves: result.moves,
1098
- usage: result.usage
1099
- };
1100
- }
1101
- //#endregion
1102
- //#region src/contract/attempt.ts
1103
- const nullableNumber = z.number().nullable();
1104
- const tokenUsageSchema = z.strictObject({
1105
- input: z.number(),
1106
- output: z.number(),
1107
- cacheRead: z.number(),
1108
- cacheWrite: z.number(),
1109
- reasoning: nullableNumber
1110
- }).describe("The tokens one turn used, as its harness reported them. `input` excludes cache reads and writes (so the prompt was `input + cacheRead + cacheWrite`); `reasoning` is part of `output`.");
1111
- const fileChangeSchema = z.strictObject({
1112
- path: z.string(),
1113
- status: z.enum([
1114
- "added",
1115
- "modified",
1116
- "deleted"
1117
- ])
1118
- });
1119
- const diffSchema = z.strictObject({
1120
- patch: z.string(),
1121
- files: z.array(fileChangeSchema)
1122
- });
1123
- const turnStatusSchema = z.enum([
1124
- "completed",
1125
- "aborted",
1126
- "error"
1127
- ]);
1128
- const webEventTypeSchema = z.enum(["web-search", "web-fetch"]).describe("The transcript events a harness may be unable to report.");
1129
- const at = {
1130
- ts: z.string(),
1131
- atMs: z.number()
1132
- };
1133
- const transcriptEventSchema = z.discriminatedUnion("type", [
1134
- z.strictObject({
1135
- ...at,
1136
- type: z.enum(["text", "reasoning"]),
1137
- text: z.string(),
1138
- durationMs: nullableNumber
1139
- }),
1140
- z.strictObject({
1141
- ...at,
1142
- type: z.literal("file-edit"),
1143
- toolCallId: z.string(),
1144
- path: z.string(),
1145
- kind: z.enum([
1146
- "create",
1147
- "modify",
1148
- "delete"
1149
- ]),
1150
- added: nullableNumber.describe("Line counts, or null when the harness didn't send the content (e.g. deletes)."),
1151
- removed: nullableNumber,
1152
- durationMs: nullableNumber,
1153
- stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
1154
- }),
1155
- z.strictObject({
1156
- ...at,
1157
- type: z.literal("web-search"),
1158
- toolCallId: z.string(),
1159
- query: z.string(),
1160
- references: nullableNumber.describe("How many sources the harness says it used; Cursor sends no URLs or titles."),
1161
- ok: z.boolean().optional().describe("Whether the search succeeded. Absent means it did."),
1162
- durationMs: nullableNumber,
1163
- stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
1164
- }),
1165
- z.strictObject({
1166
- ...at,
1167
- type: z.literal("web-fetch"),
1168
- toolCallId: z.string(),
1169
- url: z.string(),
1170
- ok: z.boolean(),
1171
- durationMs: nullableNumber,
1172
- stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
1173
- }),
1174
- z.strictObject({
1175
- ...at,
1176
- type: z.literal("tool"),
1177
- toolCallId: z.string(),
1178
- tool: z.string(),
1179
- input: z.unknown(),
1180
- output: z.unknown(),
1181
- ok: z.boolean(),
1182
- durationMs: nullableNumber,
1183
- stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
1184
- }),
1185
- z.strictObject({
1186
- ...at,
1187
- type: z.literal("error"),
1188
- message: z.string()
1189
- }),
1190
- z.strictObject({
1191
- ...at,
1192
- type: z.literal("retry"),
1193
- message: z.string()
1194
- })
1195
- ]).describe("Normalised agent events. Each tool call is one event, merged with its result; `durationMs` is wall time until the result arrived (Cursor's own `completed_at_ms` can be earlier). Null when the harness never reported the end. Long shell or edit calls set `stalled: true`.");
1196
- const turnDataSchema = z.strictObject({
1197
- prompt: z.string(),
1198
- text: z.string().describe("The agent's final message for this turn."),
1199
- status: turnStatusSchema,
1200
- startedAt: z.string(),
1201
- durationMs: z.number(),
1202
- transcript: z.array(transcriptEventSchema),
1203
- usage: tokenUsageSchema.nullable().describe("Tokens the turn used; null when the harness reports none."),
1204
- model: z.string().nullable().describe("The model the harness says served the turn, e.g. `composer-2.5[fast=false]`; null when it reports none. It can differ from the requested one (an alias, or parameters it filled in)."),
1205
- unreported: z.array(webEventTypeSchema).describe("Event types the harness couldn't report for this turn: the agent may have searched or fetched with no event in `transcript`. `toHaveSearched()` and `toHaveFetched()` refuse such a turn. Usually `[]`; a Cursor turn whose web calls the adapter couldn't recover (its SDK drops them from the stream) is `['web-search', 'web-fetch']`."),
1206
- diff: diffSchema.describe("Workspace changes since the eval started, as of the end of this turn.")
1207
- }).describe("A turn as data, as a record or another language's runner carries it: `Turn` without its methods.");
1208
- const execResultSchema = z.strictObject({
1209
- exitCode: z.number().int(),
1210
- stdout: z.string(),
1211
- stderr: z.string()
1212
- });
1213
- const execRetrySchema = z.strictObject({
1214
- retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional().describe("Extra tries after a non-zero exit, a whole number from 0. Default 0."),
1215
- interval: z.number().min(0, "must be 0 or more milliseconds").optional().describe("Milliseconds to wait before each extra try. Default 1000.")
1216
- }).describe("How `c.exec` reruns a command that exits non-zero, for a check that can fail on a transient.");
1217
- /** A background command has no exit for `retries` to wait on. */
1218
- const noBackgroundRetry = (options, ctx) => {
1219
- if (!options.background) return;
1220
- for (const key of ["retries", "interval"]) {
1221
- if (options[key] === void 0) continue;
1222
- ctx.addIssue({
1223
- code: "custom",
1224
- path: [key],
1225
- message: "can't be used with background: true, whose exit reeval doesn't wait for: drop one of them"
1226
- });
1227
- }
1228
- };
1229
- const execOptionsSchema = z.strictObject({
1230
- background: z.boolean().optional(),
1231
- ...execRetrySchema.shape
1232
- }).superRefine(noBackgroundRetry);
1233
- /** What's wrong with `c.exec()`'s options, or undefined. */
1234
- const execOptionsProblem = (value) => dataProblem(execOptionsSchema, value, "c.exec()", () => Object.keys(execOptionsSchema.shape));
1235
- const assertionRecordSchema = z.strictObject({
1236
- label: z.string().describe("`expect`'s second argument, or `<matcher> <first argument>` (or the command `c.exec` ran, or the judge's question), cut at 60 characters; ` #2`, ` #3` on repeats within the attempt."),
1237
- labelled: z.boolean().describe("The eval gave the label."),
1238
- matcher: z.string().describe("`toHaveFile`, `not.toBe`, `resolves.toBe`, `poll.toBe`."),
1239
- status: z.enum([
1240
- "pass",
1241
- "fail",
1242
- "error"
1243
- ]).describe("`error`: the matcher couldn't answer (a subject it can't read, an event the harness doesn't report)."),
1244
- soft: z.boolean(),
1245
- message: z.string().optional().describe("Why it failed or couldn't answer, cut at 1 KB."),
1246
- durationMs: z.number().optional().describe("Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`).")
1247
- }).describe("One `expect` or `expect.soft` matcher call an attempt made.");
1248
- const turnIndex = z.number().int("must be a turn index, from 0").min(0, "must be a turn index, from 0").describe("The turn's index, from 0.");
1249
- const captureSchema = z.strictObject({
1250
- name: z.string(),
1251
- port: z.number().describe("Container port that was tunneled."),
1252
- path: z.string().describe("Path on the app, as passed in."),
1253
- viewport: z.union([captureWindowSchema, z.literal("full-page")]).describe("What the author asked, including `'full-page'`."),
1254
- file: z.strictObject({
1255
- path: z.string(),
1256
- sha256: z.string(),
1257
- bytes: z.number()
1258
- }).describe("`artifacts/<name>.png` under the attempt directory.")
1259
- }).describe("What `c.capture()` returns: the PNG is also in the attempt record under `artifacts/`.");
1260
- const judgeArtifactRefSchema = z.strictObject({
1261
- name: z.string(),
1262
- path: z.string().describe("`artifacts/<name>.png` under the attempt directory."),
1263
- sha256: z.string(),
1264
- bytes: z.number(),
1265
- mediaType: z.literal("image/png")
1266
- }).describe("A capture this judge call sent as a file part. Bytes live in `artifacts/<name>.png`.");
1267
- const verdictFields = {
1268
- threshold: z.number(),
1269
- question: z.string(),
1270
- model: z.string(),
1271
- gateway: gatewayNameSchema.optional().describe("The gateway from the config's `gateway` list the call went through; unset without one."),
1272
- artifacts: z.array(judgeArtifactRefSchema).optional().describe("Captures this call sent the judge as file parts, when any.")
1273
- };
1274
- const verdictSchema = z.strictObject({
1275
- verdict: z.enum(["pass", "fail"]),
1276
- score: z.number().describe("The judge's probability that the answer is yes, from 0 to 1."),
1277
- ...verdictFields,
1278
- resolvedModel: z.string().optional().describe("The model id the provider says served the call, when it reports one.")
1279
- });
1280
- const judgeFailureSchema = z.strictObject({
1281
- verdict: z.literal("error"),
1282
- score: z.null(),
1283
- ...verdictFields,
1284
- error: z.string()
1285
- }).describe("What `c.judge()` returns when the call failed (the provider's error, after any retries): the attempt fails as a judge error once the eval's body has finished, so the judges and checks after it still run.");
1286
- /** A pattern, for the matchers that take a string or a `RegExp`. */
1287
- const patternSchema = z.strictObject({
1288
- regexp: z.string().describe("The pattern's source, as `RegExp.prototype.source` gives it."),
1289
- flags: z.string().regex(/^[dgimsuvy]*$/, "must use only the flags d, g, i, m, s, u, v and y").optional().describe("Its flags, e.g. `i`.")
1290
- }).describe("A regular expression, where a matcher takes a string or a `RegExp`.");
1291
- /** A pattern in the runner's own language, which reeval never compiles. */
1292
- function hasAnsweredPattern(value) {
1293
- if (Array.isArray(value)) return value.some(hasAnsweredPattern);
1294
- if (typeof value !== "object" || value === null) return false;
1295
- return "matchedBy" in value || Object.values(value).some(hasAnsweredPattern);
1296
- }
1297
- const answeredPatternSchema = z.strictObject({
1298
- matchedBy: z.string().regex(/\S/).describe("The pattern as the runner's language writes it, e.g. `re.compile('npm t')`."),
1299
- answers: z.record(z.string(), z.boolean()).describe("Whether it matches each string the runner has been asked about.")
1300
- }).describe("A pattern the runner matches in its own language. reeval runs the matcher with it, and when the matcher tests a string not in `answers` replies with those strings (`untested`) instead of a record; the runner matches them and makes the call again with them in `answers`.");
1301
- const isPatternLike = (value) => typeof value === "object" && value !== null && !Array.isArray(value) && "regexp" in value;
1302
- const matcherArgSchema = z.union([
1303
- patternSchema,
1304
- answeredPatternSchema,
1305
- jsonValueSchema
1306
- ]).superRefine((value, ctx) => {
1307
- if (!isPatternLike(value)) return;
1308
- for (const issue of patternSchema.safeParse(value).error?.issues ?? []) ctx.addIssue(issue);
1309
- }).describe("One argument to a matcher: a JSON value, a regular expression as `{ regexp, flags? }`, or a pattern in the runner's own language as `{ matchedBy, answers }`.");
1310
- const matcherSubjectSchema = z.union([
1311
- z.strictObject({ turn: turnIndex }),
1312
- z.strictObject({ attempt: z.literal(true).describe("The attempt itself, `c`.") }),
1313
- z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
1314
- z.strictObject({
1315
- exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
1316
- result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
1317
- }),
1318
- z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
1319
- ]).describe("What `expect` was given: a turn `c.send` returned, the attempt (for its files and latest turn), a value, such as an `ExecResult`, or a command or file reeval reads for it.");
1320
- const nonEmpty = z.string().regex(/\S/, "must not be empty");
1321
- const portSchema = z.number().int().min(1, "a port from 1 to 65535").max(65535, "a port from 1 to 65535");
1322
- const millisSchema = z.number().positive("more than zero milliseconds");
1323
- const captureWaitSchema = z.enum(["load", "idle"]).describe("Chromium look-ready events. `'load'` is `window` load (`Page.loadEventFired`). `'idle'` is `Page.lifecycleEvent` `networkAlmostIdle`.");
1324
- const waitSchema = z.union([captureWaitSchema, z.strictObject({ ms: millisSchema })]).describe("Same shape at top-level `wait` and `{ wait }` between instructions.");
1325
- const captureInstructionSchema = z.union([nonEmpty, z.strictObject({ wait: waitSchema })]);
1326
- const captureCallDataSchema = z.strictObject({
1327
- name: nonEmpty.optional(),
1328
- path: z.string().optional(),
1329
- viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
1330
- timeout: durationSchema.optional(),
1331
- wait: waitSchema.optional().describe("Look-ready once, before the first model shot or the saved PNG. Default `'load'`."),
1332
- instructions: z.array(captureInstructionSchema).optional().describe("Journey: goal strings and `{ wait }` beats, run by the vision model."),
1333
- model: z.string().optional().describe("Overrides cascaded `capture.model` for this call.")
1334
- }).describe("`c.capture()`'s camera and path as data. Default `path` is `/`; default `name` is derived from the path. A test's `fakeInstructions` journey is TypeScript-only.");
1335
- const handleId = nonEmpty.describe("A background process, as an exec or serve reply named it.");
1336
- const judgeOnSchema = z.union([z.strictObject({ turn: turnIndex }), z.strictObject({ value: jsonValueSchema.describe("Graded as the output.") })]);
1337
- const judgeCallDataSchema = z.strictObject({
1338
- model: z.union([judgeModelNameSchema, fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge with a numeric score."),
1339
- ...judgeFields,
1340
- on: judgeOnSchema.optional().describe("What to grade instead of the latest turn: an earlier turn, or a value."),
1341
- artifacts: z.union([z.literal(true), z.array(z.union([nonEmpty, captureSchema]))]).optional().describe("Captures this call sends the judge as image file parts: a capture `c.capture` returned, or its name, or `true` for every capture so far. The provider that cannot take files fails the call.")
1342
- }).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score, plus the call-only `on` and `artifacts`.");
1343
- const attemptCallSchema = z.discriminatedUnion("op", [
1344
- z.strictObject({
1345
- op: z.literal("send"),
1346
- prompt: z.string().describe("The turn's prompt.")
1347
- }).describe("`c.send(prompt)`: the next turn of the agent session. Replies with a turn."),
1348
- z.strictObject({
1349
- op: z.literal("exec"),
1350
- command: nonEmpty.describe("A shell command, run in the workspace."),
1351
- background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
1352
- ...execRetrySchema.shape
1353
- }).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
1354
- z.strictObject({
1355
- op: z.literal("serve"),
1356
- command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
1357
- port: portSchema.optional().describe("The container port. Omit it and reeval allocates one.")
1358
- }).describe("`c.serve(command, port?)`: a background process and its port."),
1359
- z.strictObject({
1360
- op: z.literal("kill"),
1361
- handle: handleId
1362
- }).describe("`handle.kill()`: stops a background process. Killing one that's gone is fine."),
1363
- z.strictObject({
1364
- op: z.literal("capture"),
1365
- target: z.union([z.strictObject({ port: portSchema }), z.strictObject({ handle: handleId })]).describe("A container port, or a serve handle by its id."),
1366
- options: captureCallDataSchema.optional()
1367
- }).describe("`c.capture(target, options)`: screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`. Replies with the capture."),
1368
- z.strictObject({
1369
- op: z.literal("readFile"),
1370
- path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory.")
1371
- }).describe("`c.readFile(path)`. Replies with its contents, or null when there is no such file."),
1372
- z.strictObject({
1373
- op: z.literal("writeFile"),
1374
- path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory."),
1375
- contents: z.string().describe("The file's contents, in `encoding`."),
1376
- encoding: z.enum(["utf8", "base64"]).optional().describe("How `contents` is encoded; base64 carries bytes. Default `utf8`.")
1377
- }).describe("`c.writeFile(path, contents)`, creating its directories."),
1378
- z.strictObject({
1379
- op: z.literal("expect"),
1380
- subject: matcherSubjectSchema,
1381
- matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a matcher name such as toHaveFile, without \"not.\" (set not: true instead)").describe("The matcher's name, e.g. `toHaveFile` or `toBe`, without `not.`."),
1382
- args: z.array(matcherArgSchema).describe("The matcher's arguments."),
1383
- not: z.boolean().describe("Called as `expect(…).not`."),
1384
- soft: z.boolean().describe("Called as `expect.soft`: a failure fails the attempt at its end."),
1385
- label: z.string().optional().describe("`expect`'s second argument."),
1386
- poll: z.strictObject({
1387
- interval: millisSchema.optional().describe("Milliseconds between tries. Default 50."),
1388
- timeout: millisSchema.optional().describe("Milliseconds before it fails with the last error. Default 1000.")
1389
- }).optional().describe("`expect.poll`: reeval re-reads the subject and retries the matcher until it passes, recording only the final outcome. The subject must be one reeval can re-read: the attempt, a command or a file.")
1390
- }).superRefine((call, ctx) => {
1391
- if (call.poll && "value" in call.subject) ctx.addIssue({
1392
- code: "custom",
1393
- path: ["subject"],
1394
- message: "can't be polled: reeval can't re-read a value. Poll the attempt, an exec or a readFile"
1395
- });
1396
- if (call.poll && "exec" in call.subject && call.subject.result) ctx.addIssue({
1397
- code: "custom",
1398
- path: ["subject", "result"],
1399
- message: "can't be polled: drop result so reeval runs the command on each try"
1400
- });
1401
- if (call.poll && call.args.some(hasAnsweredPattern)) ctx.addIssue({
1402
- code: "custom",
1403
- path: ["args"],
1404
- message: "can't be polled with a { matchedBy } pattern, whose answers can't follow the subject from try to try: poll with a string, or check once"
1405
- });
1406
- }).describe("A matcher call: `expect(subject, label).not?.matcher(...args)`. Replies with its record."),
1407
- z.strictObject({
1408
- op: z.literal("record"),
1409
- matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a name such as assert").describe("What made the check in the eval's language, e.g. `assert`."),
1410
- subject: z.string().optional().describe("What it checked, as the eval wrote it (`result.ok`): names the check when it has no label."),
1411
- label: z.string().optional().describe("The check's own name, the same whether it passes or fails."),
1412
- status: z.enum(["pass", "fail"]),
1413
- message: z.string().optional().describe("Why it failed."),
1414
- soft: z.boolean().optional().describe("A failure fails the attempt at its end, as `expect.soft`. Default false.")
1415
- }).describe("A check the eval's own language made, such as Python's `assert`: recorded as a matcher call is, so it's named, counted and compared the same. Replies with its record."),
1416
- z.strictObject({
1417
- op: z.literal("judge"),
1418
- question: nonEmpty.describe("A yes/no question about the graded turn or value."),
1419
- options: judgeCallDataSchema.optional()
1420
- }).describe("`c.judge(question, options)`. Replies with its verdict, or a judge failure when the provider failed.")
1421
- ]).describe("One call an eval's body makes on its attempt. The sandbox and the agent session start on the first call that needs them; there's no call to start them.");
1422
- const attemptErrorSchema = z.looseObject({
1423
- name: z.string().describe("The error's class, e.g. `UsageError` or `AssertionError`."),
1424
- message: z.string()
1425
- }).describe("Why a call failed, in reeval's words.");
1426
- const id = nonEmpty.describe("Pairs a reply with its request.");
1427
- const attemptRequestSchema = z.strictObject({
1428
- id,
1429
- call: attemptCallSchema
1430
- }).describe("A call, numbered so its reply can find it.");
1431
- z.union([
1432
- z.discriminatedUnion("op", [
1433
- z.strictObject({
1434
- id,
1435
- ok: z.literal(true),
1436
- op: z.literal("send"),
1437
- turn: turnDataSchema
1438
- }),
1439
- z.strictObject({
1440
- id,
1441
- ok: z.literal(true),
1442
- op: z.literal("exec"),
1443
- result: z.union([execResultSchema, z.strictObject({ handle: handleId }).describe("A background process, still running.")])
1444
- }),
1445
- z.strictObject({
1446
- id,
1447
- ok: z.literal(true),
1448
- op: z.literal("serve"),
1449
- handle: handleId,
1450
- port: z.number().describe("The container port it was given.")
1451
- }),
1452
- z.strictObject({
1453
- id,
1454
- ok: z.literal(true),
1455
- op: z.literal("kill")
1456
- }),
1457
- z.strictObject({
1458
- id,
1459
- ok: z.literal(true),
1460
- op: z.literal("capture"),
1461
- capture: captureSchema
1462
- }),
1463
- z.strictObject({
1464
- id,
1465
- ok: z.literal(true),
1466
- op: z.literal("readFile"),
1467
- contents: z.string().nullable().describe("Null when there is no such file.")
1468
- }),
1469
- z.strictObject({
1470
- id,
1471
- ok: z.literal(true),
1472
- op: z.literal("writeFile")
1473
- }),
1474
- z.strictObject({
1475
- id,
1476
- ok: z.literal(true),
1477
- op: z.literal("expect"),
1478
- assertion: assertionRecordSchema
1479
- }),
1480
- z.strictObject({
1481
- id,
1482
- ok: z.literal(true),
1483
- op: z.literal("record"),
1484
- assertion: assertionRecordSchema
1485
- }),
1486
- z.strictObject({
1487
- id,
1488
- ok: z.literal(true),
1489
- op: z.literal("judge"),
1490
- verdict: z.union([verdictSchema, judgeFailureSchema])
1390
+ const normalizesPoints = (model) => model.slice(model.lastIndexOf("/") + 1).toLowerCase().startsWith("gemini-");
1391
+ const NORMALIZED = 1e3;
1392
+ /** Scales an action's 0–1000 points to CSS pixels on a `page`-sized screenshot. */
1393
+ function scalePoints(action, page) {
1394
+ const x = (n) => Math.round(n * page.width / NORMALIZED);
1395
+ const y = (n) => Math.round(n * page.height / NORMALIZED);
1396
+ switch (action.type) {
1397
+ case "click":
1398
+ case "hover":
1399
+ case "scroll": return {
1400
+ ...action,
1401
+ x: x(action.x),
1402
+ y: y(action.y)
1403
+ };
1404
+ case "drag": return {
1405
+ ...action,
1406
+ x: x(action.x),
1407
+ y: y(action.y),
1408
+ toX: x(action.toX),
1409
+ toY: y(action.toY)
1410
+ };
1411
+ default: return action;
1412
+ }
1413
+ }
1414
+ const systemPrompt = (normalized) => `You drive a web page from screenshots. Call one tool. ${normalized ? `Coordinates are normalized to 0-${NORMALIZED} on the full-page image: x and toX across its width, y and toY down its height. A scroll's dy is in CSS pixels.` : "Coordinates are CSS pixels on the full-page image."} Call done when the goal is met. Do not mention CDP.`;
1415
+ const useTools = {
1416
+ click: tool({
1417
+ description: "Click at (x, y) on the screenshot.",
1418
+ inputSchema: z.object({
1419
+ x: z.number(),
1420
+ y: z.number()
1491
1421
  })
1492
- ]),
1493
- z.strictObject({
1494
- id,
1495
- ok: z.literal(true),
1496
- op: z.literal("expect"),
1497
- untested: z.record(z.string(), z.array(z.string())).describe("The strings the matcher tested that a `{ matchedBy, answers }` argument has no answer for, by its `matchedBy`. Nothing was recorded: answer them and make the call again.")
1498
1422
  }),
1499
- z.strictObject({
1500
- id,
1501
- ok: z.literal(false),
1502
- op: z.enum([
1503
- "send",
1504
- "exec",
1505
- "readFile",
1506
- "writeFile",
1507
- "expect",
1508
- "record",
1509
- "judge",
1510
- "serve",
1511
- "kill",
1512
- "capture"
1513
- ]),
1514
- error: attemptErrorSchema
1423
+ type: tool({
1424
+ description: "Insert text into the focused field. Click the field first.",
1425
+ inputSchema: z.object({ text: z.string() })
1426
+ }),
1427
+ key: tool({
1428
+ description: "Press Tab or Enter.",
1429
+ inputSchema: z.object({ key: z.enum(["Tab", "Enter"]) })
1430
+ }),
1431
+ scroll: tool({
1432
+ description: "Wheel-scroll a constrained scroller at (x, y).",
1433
+ inputSchema: z.object({
1434
+ x: z.number(),
1435
+ y: z.number(),
1436
+ dy: z.number()
1437
+ })
1438
+ }),
1439
+ drag: tool({
1440
+ description: "Drag from (x, y) to (toX, toY).",
1441
+ inputSchema: z.object({
1442
+ x: z.number(),
1443
+ y: z.number(),
1444
+ toX: z.number(),
1445
+ toY: z.number()
1446
+ })
1447
+ }),
1448
+ hover: tool({
1449
+ description: "Move the pointer to (x, y).",
1450
+ inputSchema: z.object({
1451
+ x: z.number(),
1452
+ y: z.number()
1453
+ })
1454
+ }),
1455
+ done: tool({
1456
+ description: "The goal is finished. Leave the loop.",
1457
+ inputSchema: z.object({})
1515
1458
  })
1516
- ]).describe("reeval's reply to a call: its result, or the error it threw. A failed matcher is a result (its record says `fail`), not an error, unless the matcher couldn't run.");
1517
- const cellRoleSchema = z.enum(["control", "variant"]);
1518
- const cellSchema = z.strictObject({
1519
- key: z.string(),
1520
- harness: z.string(),
1521
- environment: z.string().describe("The environment's own name, also on a variant cell (`with-house-greeting`, with no prefix)."),
1522
- sandbox: z.string(),
1523
- role: cellRoleSchema,
1524
- variant: z.string().optional().describe("The variant's name, for variant cells.")
1525
- }).describe("One cell of an eval: a harness, an environment (`-` when none is set) and that environment's image (`sandbox` id: tag, dockerfile and install). With `compare`, the control is `baseline` (or the cascaded environment, or an empty `baseline`) and each variant is that environment on its own, including its sandbox.");
1526
- z.strictObject({
1527
- type: z.literal("start"),
1528
- attempt: z.strictObject({
1529
- id: nonEmpty,
1530
- cell: cellSchema,
1531
- n: z.number().int().min(1, "a run number, from 1").describe("The run number, from 1.")
1532
- }).describe("`c.attempt`: which cell of the matrix this attempt is, and its run number."),
1533
- file: nonEmpty.describe("The eval file, relative to the project."),
1534
- body: nonEmpty.describe("The eval's `body`, as its declaration named it.")
1535
- }).describe("reeval to the runner: run this eval's body for this attempt. Nothing has started yet: the sandbox and the agent session start on the body's first call that needs them, so its own code runs first. The runner sends its calls, then `done`.");
1536
- z.strictObject({
1537
- type: z.literal("abort"),
1538
- reason: z.enum(["timeout", "cancelled"]).describe("`timeout`: the eval's `timeout` ran out. `cancelled`: the run was stopped."),
1539
- message: z.string().describe("reeval's sentence, e.g. `Eval timed out after 600s`.")
1540
- }).describe("reeval to the runner: the attempt was aborted, as `c.signal` aborts. Calls in flight reply with errors; the runner should stop the body and send `done`.");
1541
- z.strictObject({
1542
- type: z.literal("done"),
1543
- error: attemptErrorSchema.optional().describe("What the body threw, if it threw. A failed `expect` (not soft) is one.")
1544
- }).describe("The runner to reeval: the eval's body has finished. The attempt's checks and record follow.");
1545
- const CALL_KEYS = Object.fromEntries(attemptCallSchema.options.map((o) => [o.shape.op.value, Object.keys(o.shape)]));
1546
- const OTHER_KEYS = [
1547
- "id",
1548
- "call",
1549
- "turn",
1550
- "attempt",
1551
- "value",
1552
- "regexp",
1553
- "flags",
1554
- "matchedBy",
1555
- "answers",
1556
- ...Object.keys(judgeCallDataSchema.shape),
1557
- ...Object.keys(judgeStateSchema.shape),
1558
- ...Object.keys(captureSchema.shape),
1559
- ...Object.keys(captureCallDataSchema.shape),
1560
- "port",
1561
- "handle",
1562
- "exec",
1563
- "result",
1564
- "readFile",
1565
- "interval",
1566
- "timeout",
1567
- "ms"
1568
- ];
1569
- /** What's wrong with an attempt request a runner sent, by the field's path, or undefined. */
1570
- const attemptRequestProblem = (value) => dataProblem(attemptRequestSchema, value, "The attempt request", (object) => typeof object.op === "string" ? CALL_KEYS[object.op] ?? [] : OTHER_KEYS);
1459
+ };
1460
+ async function runFakeScript(session, script, goal, remaining) {
1461
+ let moves = 0;
1462
+ for (const action of script) {
1463
+ remainingOrThrow(remaining, JSON.stringify(goal));
1464
+ if (action.type === "done") return moves;
1465
+ if (moves >= 32) throw new Error(`c.capture() instruction ${JSON.stringify(goal)} did not finish in 32 moves`);
1466
+ const page = pngSize(await session.screenshot(true));
1467
+ debugTrace("check", "capture-use", {
1468
+ goal,
1469
+ action,
1470
+ page
1471
+ }, `capture use ${action.type}`);
1472
+ await session.dispatch(action, page);
1473
+ moves++;
1474
+ await settle();
1475
+ }
1476
+ while (remaining() > 0) await settle();
1477
+ throw new Error(`c.capture() timed out on ${JSON.stringify(goal)}`);
1478
+ }
1479
+ async function runModelGoal(session, goal, modelName, remaining, loadModel) {
1480
+ const model = await loadModel(modelName);
1481
+ const normalized = normalizesPoints(modelName);
1482
+ let moves = 0;
1483
+ let usage = null;
1484
+ while (moves < 32) {
1485
+ remainingOrThrow(remaining, JSON.stringify(goal));
1486
+ const ephemeral = await session.screenshot(true);
1487
+ const page = pngSize(ephemeral);
1488
+ debugTrace("check", "capture-shot", {
1489
+ goal,
1490
+ bytes: ephemeral.length,
1491
+ page
1492
+ }, false);
1493
+ let chosen;
1494
+ const result = await generateText({
1495
+ model,
1496
+ system: systemPrompt(normalized),
1497
+ messages: [{
1498
+ role: "user",
1499
+ content: [{
1500
+ type: "text",
1501
+ text: goal
1502
+ }, {
1503
+ type: "image",
1504
+ image: ephemeral
1505
+ }]
1506
+ }],
1507
+ tools: useTools,
1508
+ toolChoice: "required",
1509
+ stopWhen: stepCountIs(1),
1510
+ abortSignal: AbortSignal.timeout(remainingOrThrow(remaining, JSON.stringify(goal)))
1511
+ });
1512
+ usage = addUsage(usage, asUsage(result.usage ?? {}));
1513
+ const call = result.toolCalls?.[0];
1514
+ if (call) chosen = actionFromTool(call.toolName, call.input ?? {});
1515
+ if (chosen && normalized) chosen = scalePoints(chosen, page);
1516
+ if (!chosen) throw new Error(`c.capture() instruction ${JSON.stringify(goal)} returned no action`);
1517
+ debugLog("check", `capture use ${chosen.type} on ${JSON.stringify(goal)}`);
1518
+ debugTrace("check", "capture-use", {
1519
+ goal,
1520
+ action: chosen,
1521
+ page
1522
+ }, `capture use ${chosen.type}`);
1523
+ if (chosen.type === "done") return {
1524
+ moves,
1525
+ usage
1526
+ };
1527
+ await session.dispatch(chosen, page);
1528
+ moves++;
1529
+ await settle();
1530
+ }
1531
+ throw new Error(`c.capture() instruction ${JSON.stringify(goal)} did not finish in 32 moves`);
1532
+ }
1533
+ /**
1534
+ * Top-level wait, then the instruction loop, then the saved PNG.
1535
+ * Throws before `screenshot` when the budget or move cap is hit.
1536
+ */
1537
+ async function runJourney(session, url, options) {
1538
+ const { wait, timeoutMs, fullPage, instructions, use, remaining } = options;
1539
+ remainingOrThrow(remaining, "navigate");
1540
+ await navigateAndWait(session, url, wait, Math.min(timeoutMs, remaining()));
1541
+ let moves = 0;
1542
+ let usage = null;
1543
+ let stringIndex = 0;
1544
+ for (const step of instructions) {
1545
+ if (typeof step !== "string") {
1546
+ remainingOrThrow(remaining, "wait");
1547
+ await session.wait(step.wait, remaining(), void 0);
1548
+ continue;
1549
+ }
1550
+ remainingOrThrow(remaining, JSON.stringify(step));
1551
+ debugLog("check", `capture instruction ${JSON.stringify(step)}`);
1552
+ if (use && "fake" in use) {
1553
+ const script = use.fake[stringIndex] ?? [];
1554
+ moves += await runFakeScript(session, script, step, remaining);
1555
+ } else if (use && "model" in use) {
1556
+ if (!options.loadModel) throw new Error("c.capture() instructions need a model loader");
1557
+ const ran = await runModelGoal(session, step, use.model, remaining, options.loadModel);
1558
+ moves += ran.moves;
1559
+ usage = addUsage(usage, ran.usage);
1560
+ }
1561
+ stringIndex++;
1562
+ await settle();
1563
+ }
1564
+ remainingOrThrow(remaining, "screenshot");
1565
+ return {
1566
+ png: await session.screenshot(fullPage),
1567
+ moves,
1568
+ model: use && "model" in use ? use.model : use && "fake" in use ? "fake" : void 0,
1569
+ usage
1570
+ };
1571
+ }
1572
+ function captureUseSpend(result) {
1573
+ if (!result.model) return void 0;
1574
+ return {
1575
+ model: result.model,
1576
+ moves: result.moves,
1577
+ usage: result.usage
1578
+ };
1579
+ }
1571
1580
  //#endregion
1572
1581
  //#region src/diff.ts
1573
- const GIT = "git -c user.name=reeval -c user.email=reeval@localhost -c core.hooksPath=/dev/null -c core.logAllRefUpdates=false -c gc.auto=0";
1582
+ const GIT = "git -c user.name=reeval -c user.email=reeval@localhost -c core.hooksPath=/dev/null -c core.logAllRefUpdates=false -c gc.auto=0 -c core.quotePath=false";
1574
1583
  const snapshotEnv = (sandbox) => `export GIT_DIR=${shellQuote(`${sandbox.stateDir}/workspace.git`)} GIT_WORK_TREE=${shellQuote(sandbox.workdir)} GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null`;
1575
1584
  /** Points HEAD at a commit of `$tree` with no parent, then drops every object nothing reaches. */
1576
1585
  const RESET_HEAD = `c=$(${GIT} commit-tree -m base "$tree") && ${GIT} update-ref HEAD "$c" && ${GIT} prune --expire=now`;
@@ -1598,20 +1607,22 @@ async function acceptIntoSnapshot(sandbox, path, signal) {
1598
1607
  async function diffSinceSnapshot(sandbox, signal) {
1599
1608
  const env = `${snapshotEnv(sandbox)} && ${GIT} add -A && `;
1600
1609
  const patch = await sandbox.execOwn(`${env}${GIT} diff --cached --binary HEAD`, { signal });
1601
- const names = await sandbox.execOwn(`${env}${GIT} diff --cached --name-status --no-renames HEAD`, { signal });
1610
+ const names = await sandbox.execOwn(`${env}${GIT} diff --cached --name-status --no-renames -z HEAD`, { signal });
1602
1611
  if (patch.exitCode !== 0 || names.exitCode !== 0) throw new InfraError(`Failed to diff the workspace: ${patch.stderr || names.stderr}`);
1603
1612
  const status = {
1604
1613
  A: "added",
1605
1614
  M: "modified",
1606
1615
  D: "deleted"
1607
1616
  };
1608
- const files = names.stdout.split("\n").filter(Boolean).map((line) => {
1609
- const [code = "M", ...rest] = line.split(" ");
1610
- return {
1611
- path: rest.join(" "),
1617
+ const fields = names.stdout.split("\0");
1618
+ const files = [];
1619
+ for (let i = 0; i + 1 < fields.length; i += 2) {
1620
+ const code = fields[i];
1621
+ files.push({
1622
+ path: fields[i + 1],
1612
1623
  status: status[code[0]] ?? "modified"
1613
- };
1614
- });
1624
+ });
1625
+ }
1615
1626
  return {
1616
1627
  patch: patch.stdout,
1617
1628
  files
@@ -1656,7 +1667,7 @@ function createChecks(a) {
1656
1667
  durationMs: Math.round(performance.now() - began)
1657
1668
  });
1658
1669
  if (r.exitCode === 0 || tries.length > retries) return r;
1659
- debugLog("check", `$ ${clip(command, 160)} → exit ${r.exitCode} on try ${tries.length} of ${retries + 1}, again in ${formatDuration(interval)}`);
1670
+ debugLog("check", `$ ${clip(redactSecrets(command), 160)} → exit ${r.exitCode} on try ${tries.length} of ${retries + 1}, again in ${formatDuration(interval)}`);
1660
1671
  await setTimeout$1(interval, void 0, { signal: a.signal });
1661
1672
  }
1662
1673
  });
@@ -1672,13 +1683,13 @@ function createChecks(a) {
1672
1683
  ...retried
1673
1684
  });
1674
1685
  const of = retries > 0 ? ` on try ${tries.length} of ${retries + 1}` : "";
1675
- debugLog("check", `$ ${clip(command, 160)} → exit ${r.exitCode}${of} · ${formatDuration(durationMs)}`);
1686
+ debugLog("check", `$ ${clip(redactSecrets(command), 160)} → exit ${r.exitCode}${of} · ${formatDuration(durationMs)}`);
1676
1687
  debugTrace("check", "output", {
1677
1688
  command,
1678
1689
  ...r,
1679
1690
  durationMs,
1680
1691
  ...retried
1681
- }, `check output: ${clip(command, 80)}`);
1692
+ }, `check output: ${clip(redactSecrets(command), 80)}`);
1682
1693
  a.report({
1683
1694
  kind: "check-end",
1684
1695
  command,
@@ -1694,7 +1705,7 @@ function createChecks(a) {
1694
1705
  function exec(command, options) {
1695
1706
  const problem = options === void 0 ? void 0 : execOptionsProblem(options);
1696
1707
  if (problem) return Promise.reject(new UsageError(problem));
1697
- if (options?.background) return import("./capture-CherbjzB.js").then(({ startBackground }) => startBackground(a, command, "c.exec({ background: true })"));
1708
+ if (options?.background) return import("./capture-DSK8O5bA.js").then(({ startBackground }) => startBackground(a, command, "c.exec({ background: true })"));
1698
1709
  const run = execForeground(command, options?.retries ?? 0, options?.interval ?? 1e3);
1699
1710
  return Object.defineProperty(run, SUBJECT, { value: command });
1700
1711
  }
@@ -1881,11 +1892,11 @@ async function shot(a, url, resolved, remaining, onMoves) {
1881
1892
  remaining,
1882
1893
  loadModel: async (name) => {
1883
1894
  const { config, profile } = getRunContext();
1884
- const gateway = resolveModelGateway(name, void 0, gatewayScope(config, profile), "capture model");
1885
- const route = judgeRoute(name, process.env, "capture model", gateway);
1886
- return providerModel({
1887
- name,
1888
- route: route.kind === "direct" ? "direct" : "gateway"
1895
+ const scope = gatewayScope(config, profile);
1896
+ const gateway = resolveModelGateway(name, void 0, scope, "capture model");
1897
+ return hostModel(name, gateway, "capture model", {
1898
+ fallbacks: openRouterFallbacks(scope.gateways),
1899
+ ...gateway === "vertex" ? { vertex: vertexSettingsIn(scope.gateways) } : {}
1889
1900
  });
1890
1901
  }
1891
1902
  });
@@ -1910,6 +1921,6 @@ async function writeCaptureSidecars(stagingDir, recordFile, blobs) {
1910
1921
  for (const [name, png] of blobs) await writeFile(join(dir, `${name}.png`), png);
1911
1922
  }
1912
1923
  //#endregion
1913
- export { writeCaptureSidecars as a, sandboxFor as c, workspaceFiles as d, attemptRequestProblem as f, startBackground as i, diffSinceSnapshot as l, msysTreeKill as n, asCheck as o, fakeInstructions as p, serveCommand as r, createChecks as s, capturePage as t, snapshot as u };
1924
+ export { writeCaptureSidecars as a, sandboxFor as c, workspaceFiles as d, fakeInstructions as f, startBackground as i, diffSinceSnapshot as l, msysTreeKill as n, asCheck as o, attemptRequestProblem as p, serveCommand as r, createChecks as s, capturePage as t, snapshot as u };
1914
1925
 
1915
- //# sourceMappingURL=capture-CVRwRTnb.js.map
1926
+ //# sourceMappingURL=capture-BM_3100g.js.map