reeval 0.12.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (331) hide show
  1. package/README.md +117 -259
  2. package/THIRD_PARTY_NOTICES.md +121 -121
  3. package/assets/package.json +1 -1
  4. package/assets/pnpm-lock.yaml +41 -30
  5. package/dist/{adapter-DigeVFWC.js → adapter-BloGVZ_a.js} +20 -8
  6. package/dist/adapter-BloGVZ_a.js.map +1 -0
  7. package/dist/{agent-CatUEo5h.js → agent-m-_t-IuW.js} +54 -26
  8. package/dist/{agent-CatUEo5h.js.map → agent-m-_t-IuW.js.map} +1 -1
  9. package/dist/{analysis-Bxub9LMT.js → analysis-C2-aEbr7.js} +4 -4
  10. package/dist/{analysis-Bxub9LMT.js.map → analysis-C2-aEbr7.js.map} +1 -1
  11. package/dist/app-files-C-jwhRDt.js +11 -0
  12. package/dist/app-files-C-jwhRDt.js.map +1 -0
  13. package/dist/{assemble-CF063ms6.js → assemble-BDEpelpi.js} +95 -38
  14. package/dist/assemble-BDEpelpi.js.map +1 -0
  15. package/dist/assemble-C5bfncHU.js +1 -0
  16. package/dist/{attempt-CGPV3BZY.js → attempt-jI0OWxlz.js} +2 -2
  17. package/dist/{attempt-CGPV3BZY.js.map → attempt-jI0OWxlz.js.map} +1 -1
  18. package/dist/{capture-BiD7lVaM.js → capture-CVRwRTnb.js} +574 -38
  19. package/dist/capture-CVRwRTnb.js.map +1 -0
  20. package/dist/capture-CherbjzB.js +1 -0
  21. package/dist/{cli-B5zQCR2r.js → cli-C_4gaJ50.js} +145 -142
  22. package/dist/cli-C_4gaJ50.js.map +1 -0
  23. package/dist/{cli-spec-BXIBUlaK.js → cli-spec-CpnbATtI.js} +2 -2
  24. package/dist/{cli-spec-BXIBUlaK.js.map → cli-spec-CpnbATtI.js.map} +1 -1
  25. package/dist/cli.js +2 -2
  26. package/dist/{assert-EUIoRdIt.js → client-D5hNjpEx.js} +105 -2
  27. package/dist/client-D5hNjpEx.js.map +1 -0
  28. package/dist/{command-C94CQNh8.js → command-BWAuwQNk.js} +12 -12
  29. package/dist/{command-C94CQNh8.js.map → command-BWAuwQNk.js.map} +1 -1
  30. package/dist/{command-CaZgCTIw.js → command-BXLUfw4h.js} +14 -14
  31. package/dist/command-BXLUfw4h.js.map +1 -0
  32. package/dist/{command-CVtgyqlM.js → command-Be6BSE2L.js} +12 -12
  33. package/dist/{command-CVtgyqlM.js.map → command-Be6BSE2L.js.map} +1 -1
  34. package/dist/{command-BMHXHqMT.js → command-Bxdy62dW.js} +15 -14
  35. package/dist/command-Bxdy62dW.js.map +1 -0
  36. package/dist/{command-DTgBZR0k.js → command-I1euLLio.js} +35 -15
  37. package/dist/command-I1euLLio.js.map +1 -0
  38. package/dist/{commands-CjpGBUWr.js → commands-jzigbNFb.js} +23 -14
  39. package/dist/commands-jzigbNFb.js.map +1 -0
  40. package/dist/{commands-BXzIDB3X.js → commands-p1S4nw_H.js} +1 -1
  41. package/dist/config-BrdHA9bJ.js +3549 -0
  42. package/dist/config-BrdHA9bJ.js.map +1 -0
  43. package/dist/{contrasts-CcoMLC_y.js → contrasts-DRxMQQTC.js} +3 -2
  44. package/dist/contrasts-DRxMQQTC.js.map +1 -0
  45. package/dist/cost-D4elx933.js +359 -0
  46. package/dist/cost-D4elx933.js.map +1 -0
  47. package/dist/cost-DmnAod9x.js +1 -0
  48. package/dist/db-CAfksinX.js +2 -0
  49. package/dist/{db-BeqRiWLl.js → db-DO0p7xgP.js} +25 -8
  50. package/dist/db-DO0p7xgP.js.map +1 -0
  51. package/dist/{dist-CMjp3qqF.js → dist-CYrBwyHA.js} +2 -2
  52. package/dist/{dist-CMjp3qqF.js.map → dist-CYrBwyHA.js.map} +1 -1
  53. package/dist/{docker-BbTlMmYi.js → docker-BnRlTpZA.js} +3 -3
  54. package/dist/{docker-BbTlMmYi.js.map → docker-BnRlTpZA.js.map} +1 -1
  55. package/dist/{docker-CCXdbREk.js → docker-D01slDF3.js} +29 -28
  56. package/dist/docker-D01slDF3.js.map +1 -0
  57. package/dist/{docker-DC69dKxB.js → docker-DivfZl2u.js} +1 -1
  58. package/dist/{doctor-CqttrZcs.js → doctor-DQ2p0euG.js} +136 -38
  59. package/dist/doctor-DQ2p0euG.js.map +1 -0
  60. package/dist/{environment-BBz1Yes9.js → environment-CO0P0soF.js} +31 -11
  61. package/dist/environment-CO0P0soF.js.map +1 -0
  62. package/dist/{evaluate-CXQ1o-pd.js → evaluate-BUmsN7yG.js} +68 -29
  63. package/dist/evaluate-BUmsN7yG.js.map +1 -0
  64. package/dist/{evaluate-yjZKUexW.d.ts → evaluate-C_sgRA-V.d.ts} +45 -3
  65. package/dist/{facts-DtENlf9s.js → facts-DzOUTreB.js} +13 -5
  66. package/dist/facts-DzOUTreB.js.map +1 -0
  67. package/dist/{files-B-eUpCZ1.js → files-B-CJeeEh.js} +14 -9
  68. package/dist/{files-B-eUpCZ1.js.map → files-B-CJeeEh.js.map} +1 -1
  69. package/dist/files-Bp5j5BqI.js +1 -0
  70. package/dist/files-C2sjmXhk.js +1 -0
  71. package/dist/{files-BsQbYz2i.js → files-C_gQNVB-.js} +4 -4
  72. package/dist/{files-BsQbYz2i.js.map → files-C_gQNVB-.js.map} +1 -1
  73. package/dist/{files-COHQLEHE.js → files-kUCrswI6.js} +3 -3
  74. package/dist/{files-COHQLEHE.js.map → files-kUCrswI6.js.map} +1 -1
  75. package/dist/gateway.d.ts +17 -0
  76. package/dist/gateway.js +17 -0
  77. package/dist/gateway.js.map +1 -0
  78. package/dist/{generate-Cs2TMH_k.js → generate-DZ7vp69F.js} +18 -18
  79. package/dist/generate-DZ7vp69F.js.map +1 -0
  80. package/dist/{git-DhykwgKF.js → git-Bk8DSwHy.js} +3 -3
  81. package/dist/{git-DhykwgKF.js.map → git-Bk8DSwHy.js.map} +1 -1
  82. package/dist/{harness-BAjw1fcr.js → harness-COVXSwJ6.js} +46 -15
  83. package/dist/harness-COVXSwJ6.js.map +1 -0
  84. package/dist/harness.d.ts +26 -18
  85. package/dist/harness.js +1 -1
  86. package/dist/{header-kSmWatcO.js → header-BLPP6aCU.js} +2 -2
  87. package/dist/{header-kSmWatcO.js.map → header-BLPP6aCU.js.map} +1 -1
  88. package/dist/{images-BmJRuTpm.js → images-DzLeFjLx.js} +8 -8
  89. package/dist/{images-BmJRuTpm.js.map → images-DzLeFjLx.js.map} +1 -1
  90. package/dist/index.d.ts +3 -45
  91. package/dist/index.js +5 -6
  92. package/dist/init-RgCQXZhb.js +1 -0
  93. package/dist/{init-Bwc9byQr.js → init-SRalM5xF.js} +6 -8
  94. package/dist/init-SRalM5xF.js.map +1 -0
  95. package/dist/{insights-B2Xmhz9o.js → insights-TyAvz9aV.js} +5 -1
  96. package/dist/{insights-B2Xmhz9o.js.map → insights-TyAvz9aV.js.map} +1 -1
  97. package/dist/{invocations-CQTbP092.js → invocations-DzyKSJBa.js} +3 -3
  98. package/dist/{invocations-CQTbP092.js.map → invocations-DzyKSJBa.js.map} +1 -1
  99. package/dist/judge-yLvjC5A3.js +385 -0
  100. package/dist/judge-yLvjC5A3.js.map +1 -0
  101. package/dist/judge.d.ts +7 -6
  102. package/dist/judge.js +7 -5
  103. package/dist/judge.js.map +1 -1
  104. package/dist/match-Bz0nSdzN.js +204 -0
  105. package/dist/match-Bz0nSdzN.js.map +1 -0
  106. package/dist/{matchers-Ddmc2z7H.js → matchers-D4eUqx4j.js} +2 -2
  107. package/dist/{matchers-Ddmc2z7H.js.map → matchers-D4eUqx4j.js.map} +1 -1
  108. package/dist/{model-B-mel-Gv.js → model-1ZLr9Dcv.js} +59 -35
  109. package/dist/model-1ZLr9Dcv.js.map +1 -0
  110. package/dist/model-B45lDUV1.js +68 -0
  111. package/dist/model-B45lDUV1.js.map +1 -0
  112. package/dist/model-BTZq8HIZ.js +1 -0
  113. package/dist/{model-CWlAqmbU.js → model-CgnOpss4.js} +58 -316
  114. package/dist/model-CgnOpss4.js.map +1 -0
  115. package/dist/model-Ch8kX87u.js +1 -0
  116. package/dist/{node-D0mtjRAO.js → node-BSq5qHnQ.js} +12 -5
  117. package/dist/node-BSq5qHnQ.js.map +1 -0
  118. package/dist/{opencode-providers-J9colP0-.js → opencode-providers-klkqgzvb.js} +72 -62
  119. package/dist/opencode-providers-klkqgzvb.js.map +1 -0
  120. package/dist/{pairs-DP6XOJby.js → pairs-CHfZNRT6.js} +8 -7
  121. package/dist/pairs-CHfZNRT6.js.map +1 -0
  122. package/dist/{patch-D4oluWzH.js → patch-DWf-I_Q_.js} +2 -2
  123. package/dist/{patch-D4oluWzH.js.map → patch-DWf-I_Q_.js.map} +1 -1
  124. package/dist/{plan-OkwUcJQ9.js → plan-BIwUzlB8.js} +63 -26
  125. package/dist/plan-BIwUzlB8.js.map +1 -0
  126. package/dist/{plugins-CUWMnSNb.js → plugins-Dkp_H3pB.js} +5 -5
  127. package/dist/{plugins-CUWMnSNb.js.map → plugins-Dkp_H3pB.js.map} +1 -1
  128. package/dist/{preflight-B6pY89bF.js → preflight-D9y-HvZQ.js} +1 -1
  129. package/dist/{preflight-Bjm7_ylE.js → preflight-DJCD_BDn.js} +3 -3
  130. package/dist/{preflight-Bjm7_ylE.js.map → preflight-DJCD_BDn.js.map} +1 -1
  131. package/dist/{prepare-DbdgtZKe.js → prepare-BJG_pD54.js} +2 -2
  132. package/dist/{prepare-CQmK3SBn.js → prepare-CIxCGG0M.js} +8 -8
  133. package/dist/{prepare-CQmK3SBn.js.map → prepare-CIxCGG0M.js.map} +1 -1
  134. package/dist/prices-BcLEXvdY.js +250 -0
  135. package/dist/prices-BcLEXvdY.js.map +1 -0
  136. package/dist/{provenance-B3Y17oCC.js → provenance-CfZpFpAh.js} +3 -3
  137. package/dist/{provenance-B3Y17oCC.js.map → provenance-CfZpFpAh.js.map} +1 -1
  138. package/dist/{providers-CXN2RFz-.js → providers-D1Zxp2he.js} +1 -1
  139. package/dist/{providers-eoniqK6w.js → providers-DFk_6W4K.js} +5 -5
  140. package/dist/{providers-eoniqK6w.js.map → providers-DFk_6W4K.js.map} +1 -1
  141. package/dist/{query-CLRZmWBT.js → query-CemclveB.js} +2 -2
  142. package/dist/{query-CLRZmWBT.js.map → query-CemclveB.js.map} +1 -1
  143. package/dist/query-Cvbwl804.js +1 -0
  144. package/dist/{questions-1XOulevV.js → questions-bAudIc04.js} +309 -47
  145. package/dist/questions-bAudIc04.js.map +1 -0
  146. package/dist/{redact-DJ9JCShu.js → redact-CRGdfV2s.js} +4 -2
  147. package/dist/redact-CRGdfV2s.js.map +1 -0
  148. package/dist/{remarks-DlJCqWRs.js → remarks-CXwSnSjR.js} +9 -9
  149. package/dist/{remarks-DlJCqWRs.js.map → remarks-CXwSnSjR.js.map} +1 -1
  150. package/dist/remarks-CyAu2DiB.js +1 -0
  151. package/dist/{remarks-1521zwfW.js → remarks-mujJv1Ey.js} +5 -3
  152. package/dist/remarks-mujJv1Ey.js.map +1 -0
  153. package/dist/{remote-cE2VWfkJ.js → remote-CKcEKja4.js} +129 -174
  154. package/dist/remote-CKcEKja4.js.map +1 -0
  155. package/dist/resolve-B5-uqmJ6.js +1 -0
  156. package/dist/{resolve-C4zeccCH.js → resolve-CBoKzQMP.js} +9 -10
  157. package/dist/resolve-CBoKzQMP.js.map +1 -0
  158. package/dist/{results-Drtt0uFl.js → results-CMgZEANy.js} +8 -8
  159. package/dist/{results-Drtt0uFl.js.map → results-CMgZEANy.js.map} +1 -1
  160. package/dist/{resume-BEFazUfy.js → resume-B9SOSnK9.js} +3 -3
  161. package/dist/{resume-BEFazUfy.js.map → resume-B9SOSnK9.js.map} +1 -1
  162. package/dist/{root-DICDHCtp.js → root-Dlff-zp8.js} +5 -5
  163. package/dist/root-Dlff-zp8.js.map +1 -0
  164. package/dist/{rules-BFzQqPIm.js → rules-VhftPOOT.js} +53 -4
  165. package/dist/rules-VhftPOOT.js.map +1 -0
  166. package/dist/{run-BeMy92Us.js → run-CwIEK16q.js} +12 -12
  167. package/dist/run-CwIEK16q.js.map +1 -0
  168. package/dist/{runtime-PX_oXiCS.js → runtime-BgQRyCLl.js} +14 -2
  169. package/dist/runtime-BgQRyCLl.js.map +1 -0
  170. package/dist/{sandbox-CQaJj7n9.js → sandbox-BOYZ0hlo.js} +2 -2
  171. package/dist/{sandbox-CQaJj7n9.js.map → sandbox-BOYZ0hlo.js.map} +1 -1
  172. package/dist/sandbox.d.ts +2 -1
  173. package/dist/sandbox.js +1 -1
  174. package/dist/server-CHRSgKRp.js +1 -0
  175. package/dist/{server-xK0les4f.js → server-TGW4WyAz.js} +26 -27
  176. package/dist/{server-xK0les4f.js.map → server-TGW4WyAz.js.map} +1 -1
  177. package/dist/{shared-CA61BuPJ.js → shared-BDUzStad.js} +2 -2
  178. package/dist/shared-BDUzStad.js.map +1 -0
  179. package/dist/shared-CHvO7eLi.d.ts +1 -0
  180. package/dist/{source-Q1hEKbCs.js → source-B1WvBIQt.js} +2 -2
  181. package/dist/{source-Q1hEKbCs.js.map → source-B1WvBIQt.js.map} +1 -1
  182. package/dist/source-DAzztZwU.js +1 -0
  183. package/dist/src-1jHbFaGW.js +1 -0
  184. package/dist/{src-LekQrr_S.js → src-8zoX_qrb.js} +614 -171
  185. package/dist/src-8zoX_qrb.js.map +1 -0
  186. package/dist/{src-C0CeKiLU.js → src-BYLM3hEz.js} +4 -3
  187. package/dist/src-BYLM3hEz.js.map +1 -0
  188. package/dist/{src-CFp86GtV.js → src-CHIwOy0o.js} +87 -55
  189. package/dist/{src-CFp86GtV.js.map → src-CHIwOy0o.js.map} +1 -1
  190. package/dist/src-C_9wgowC.js +2 -0
  191. package/dist/{src-C7_1SqDP.js → src-ChhtI6BP.js} +2 -2
  192. package/dist/{src-C7_1SqDP.js.map → src-ChhtI6BP.js.map} +1 -1
  193. package/dist/src-CyaDNG9r.js +2 -0
  194. package/dist/src-D005Cbqz.js +2 -0
  195. package/dist/src-DIfRbp5a.js +3 -0
  196. package/dist/{src-BhmaorOU.js → src-DKPVX2Us.js} +43 -17
  197. package/dist/{src-BhmaorOU.js.map → src-DKPVX2Us.js.map} +1 -1
  198. package/dist/src-Dg8121Xy.js +2 -0
  199. package/dist/{src-cbCb-MPy.js → src-DjF_M8_f.js} +44 -20
  200. package/dist/{src-cbCb-MPy.js.map → src-DjF_M8_f.js.map} +1 -1
  201. package/dist/{src-DIL0jST0.js → src-DjRI-IeE.js} +42 -14
  202. package/dist/src-DjRI-IeE.js.map +1 -0
  203. package/dist/{src-DiLMkIr9.js → src-DmUgeTlE.js} +47 -17
  204. package/dist/src-DmUgeTlE.js.map +1 -0
  205. package/dist/src-ElBqEN-S.js +2 -0
  206. package/dist/{src-qGoFkuJF.js → src-NSdQ9zI6.js} +88 -23
  207. package/dist/src-NSdQ9zI6.js.map +1 -0
  208. package/dist/src-YHGaUrGE.js +2 -0
  209. package/dist/src-mJVxw1Jm.js +2 -0
  210. package/dist/{state-BmZ8QVZ8.js → state-B1t143wG.js} +22 -20
  211. package/dist/{state-BmZ8QVZ8.js.map → state-B1t143wG.js.map} +1 -1
  212. package/dist/state-vznMh2wl.js +1 -0
  213. package/dist/stats-DSy5HfOv.js +315 -0
  214. package/dist/stats-DSy5HfOv.js.map +1 -0
  215. package/dist/suite-BsWGHzlG.js +2 -0
  216. package/dist/{suite-C8sw2pox.js → suite-WgT7m6Vs.js} +76 -13
  217. package/dist/suite-WgT7m6Vs.js.map +1 -0
  218. package/dist/{summary-0oQNGskv.js → summary-DIGEVr08.js} +58 -17
  219. package/dist/summary-DIGEVr08.js.map +1 -0
  220. package/dist/testing.d.ts +174 -2
  221. package/dist/testing.js +14 -3
  222. package/dist/testing.js.map +1 -0
  223. package/dist/types-C5vI3g1C.js.map +1 -1
  224. package/dist/{types-GraZtfHN.d.ts → types-DXBbP9Mk.d.ts} +302 -72
  225. package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
  226. package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
  227. package/dist/usage-DF3_p5pE.js +1 -0
  228. package/dist/{validate-BdoXUuL_.js → validate-BhwqrVsm.js} +48 -16
  229. package/dist/validate-BhwqrVsm.js.map +1 -0
  230. package/dist/view/index.d.ts +143 -91
  231. package/dist/view/index.js +7 -6
  232. package/dist/views-BW_C5eVH.js +2 -0
  233. package/dist/{views-BoUoX4_A.js → views-sXpGE6a2.js} +10 -8
  234. package/dist/views-sXpGE6a2.js.map +1 -0
  235. package/dist/{worker-entry-D0-GErmU.js → worker-entry-Duo9oAeT.js} +8 -8
  236. package/dist/{worker-entry-D0-GErmU.js.map → worker-entry-Duo9oAeT.js.map} +1 -1
  237. package/dist/{workspace-tjZFkjxC.js → workspace-CDI68wtX.js} +45 -11
  238. package/dist/workspace-CDI68wtX.js.map +1 -0
  239. package/dist/workspace-QZCacP8A.js +1 -0
  240. package/package.json +8 -2
  241. package/skills/eval-rubric/SKILL.md +9 -0
  242. package/dist/adapter-DigeVFWC.js.map +0 -1
  243. package/dist/assemble-CF063ms6.js.map +0 -1
  244. package/dist/assemble-CJoD_Rzr.js +0 -1
  245. package/dist/assert-EUIoRdIt.js.map +0 -1
  246. package/dist/capture-BiD7lVaM.js.map +0 -1
  247. package/dist/capture-Dp3q7K7Y.js +0 -1
  248. package/dist/cli-B5zQCR2r.js.map +0 -1
  249. package/dist/client-fcwgwOAD.js +0 -106
  250. package/dist/client-fcwgwOAD.js.map +0 -1
  251. package/dist/command-BMHXHqMT.js.map +0 -1
  252. package/dist/command-CaZgCTIw.js.map +0 -1
  253. package/dist/command-DTgBZR0k.js.map +0 -1
  254. package/dist/commands-CjpGBUWr.js.map +0 -1
  255. package/dist/config-Dv5MG7Rp.js +0 -753
  256. package/dist/config-Dv5MG7Rp.js.map +0 -1
  257. package/dist/contrasts-CcoMLC_y.js.map +0 -1
  258. package/dist/db-BeqRiWLl.js.map +0 -1
  259. package/dist/db-CJDRVQal.js +0 -2
  260. package/dist/docker-CCXdbREk.js.map +0 -1
  261. package/dist/doctor-CqttrZcs.js.map +0 -1
  262. package/dist/egress-BfK_O9ce.js +0 -1241
  263. package/dist/egress-BfK_O9ce.js.map +0 -1
  264. package/dist/environment-BBz1Yes9.js.map +0 -1
  265. package/dist/evaluate-CXQ1o-pd.js.map +0 -1
  266. package/dist/facts-DtENlf9s.js.map +0 -1
  267. package/dist/files-DqCBW7i8.js +0 -1
  268. package/dist/files-Ha-jsQXf.js +0 -1
  269. package/dist/generate-Cs2TMH_k.js.map +0 -1
  270. package/dist/harness-BAjw1fcr.js.map +0 -1
  271. package/dist/init-Bwc9byQr.js.map +0 -1
  272. package/dist/init-D7Y1ny1z.js +0 -1
  273. package/dist/judge-Dt6KW6_O.js +0 -153
  274. package/dist/judge-Dt6KW6_O.js.map +0 -1
  275. package/dist/model-B-mel-Gv.js.map +0 -1
  276. package/dist/model-BGKYVpoj.js +0 -1
  277. package/dist/model-C7tSwT7f.js +0 -44
  278. package/dist/model-C7tSwT7f.js.map +0 -1
  279. package/dist/model-CWlAqmbU.js.map +0 -1
  280. package/dist/model-DHY-dfoO.js +0 -1
  281. package/dist/node-D0mtjRAO.js.map +0 -1
  282. package/dist/opencode-providers-J9colP0-.js.map +0 -1
  283. package/dist/options-CPUW97_7.js +0 -1116
  284. package/dist/options-CPUW97_7.js.map +0 -1
  285. package/dist/pairs-DP6XOJby.js.map +0 -1
  286. package/dist/plan-OkwUcJQ9.js.map +0 -1
  287. package/dist/query-CmBDYt0_.js +0 -1
  288. package/dist/questions-1XOulevV.js.map +0 -1
  289. package/dist/redact-DJ9JCShu.js.map +0 -1
  290. package/dist/remarks-1521zwfW.js.map +0 -1
  291. package/dist/remarks-BhAV1H4V.js +0 -1
  292. package/dist/remote-cE2VWfkJ.js.map +0 -1
  293. package/dist/resolve-C4zeccCH.js.map +0 -1
  294. package/dist/resolve-D30yZYmB.js +0 -1
  295. package/dist/root-DICDHCtp.js.map +0 -1
  296. package/dist/rules-BFzQqPIm.js.map +0 -1
  297. package/dist/run-BeMy92Us.js.map +0 -1
  298. package/dist/runtime-PX_oXiCS.js.map +0 -1
  299. package/dist/server-V4-2w0l8.js +0 -1
  300. package/dist/shared-CA61BuPJ.js.map +0 -1
  301. package/dist/source-BDow_oW8.js +0 -1
  302. package/dist/src-B9L9yGKI.js +0 -1
  303. package/dist/src-Bcah6ZkT.js +0 -3
  304. package/dist/src-C0CeKiLU.js.map +0 -1
  305. package/dist/src-C0xowZ3C.js +0 -2
  306. package/dist/src-CHIfIKUu.js +0 -2
  307. package/dist/src-DIL0jST0.js.map +0 -1
  308. package/dist/src-DLtyz4w5.js +0 -2
  309. package/dist/src-DPIjmReZ.js +0 -2
  310. package/dist/src-DXNh5mTr.js +0 -2
  311. package/dist/src-DiLMkIr9.js.map +0 -1
  312. package/dist/src-LekQrr_S.js.map +0 -1
  313. package/dist/src-bCnsA6Vp.js +0 -2
  314. package/dist/src-qGoFkuJF.js.map +0 -1
  315. package/dist/src-ynBVosfX.js +0 -2
  316. package/dist/state-BQR_BCm4.js +0 -1
  317. package/dist/state-CeR6oz0k.js +0 -66
  318. package/dist/state-CeR6oz0k.js.map +0 -1
  319. package/dist/studio-app/assets/index-CnYJijyI.js +0 -58
  320. package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
  321. package/dist/studio-app/favicon.svg +0 -1
  322. package/dist/studio-app/index.html +0 -15
  323. package/dist/suite-BaHPmAGy.js +0 -1
  324. package/dist/suite-C8sw2pox.js.map +0 -1
  325. package/dist/summary-0oQNGskv.js.map +0 -1
  326. package/dist/usage-BDo2q_yu.js +0 -1
  327. package/dist/validate-BdoXUuL_.js.map +0 -1
  328. package/dist/views-BoUoX4_A.js.map +0 -1
  329. package/dist/views-noVQNCUP.js +0 -2
  330. package/dist/workspace-B90YDU_a.js +0 -1
  331. package/dist/workspace-tjZFkjxC.js.map +0 -1
@@ -1,25 +1,29 @@
1
1
  import { n as isOneOf } from "./object-CLf2-tNq.js";
2
- import { p as parseDuration } from "./shared-CA61BuPJ.js";
2
+ import { n as UsageError } from "./errors-B6BSMs03.js";
3
+ import { c as jsonValueSchema, i as durationSchema, n as dataProblem, p as parseDuration } from "./shared-BDUzStad.js";
4
+ import { v as gatewayNameSchema } from "./harness-COVXSwJ6.js";
3
5
  import { l as sha256 } from "./ids-Bp6ut7ZD.js";
4
- import { l as judgeRoute } from "./judge-Dt6KW6_O.js";
5
- import { ut as shellQuote } from "./src-LekQrr_S.js";
6
- import { s as cacheDir } from "./root-DICDHCtp.js";
6
+ import { Ct as judgeStateSchema, bt as judgeFields, v as gatewayScope, vt as captureWindowSchema, xt as judgeModelNameSchema, y as resolveModelGateway, yt as fakeJudgeDataSchema } from "./config-BrdHA9bJ.js";
7
+ import { l as judgeRoute } from "./judge-yLvjC5A3.js";
8
+ import { ft as shellQuote } from "./src-8zoX_qrb.js";
9
+ import { s as cacheDir } from "./root-Dlff-zp8.js";
7
10
  import { t as InfraError } from "./types-C5vI3g1C.js";
8
- import { n as getRunContext, o as SUBJECT } from "./runtime-PX_oXiCS.js";
9
- import { i as debugTrace, r as debugLog } from "./attempt-CGPV3BZY.js";
10
11
  import { i as formatDuration, n as clip } from "./text-CrVy5ioL.js";
11
- import { o as providerModel } from "./model-B-mel-Gv.js";
12
- import { t as addUsage } from "./usage-CleNKkly.js";
12
+ import { n as getRunContext, o as SUBJECT } from "./runtime-BgQRyCLl.js";
13
+ import { i as debugTrace, r as debugLog } from "./attempt-jI0OWxlz.js";
14
+ import { o as providerModel } from "./model-1ZLr9Dcv.js";
15
+ import { t as addUsage } from "./usage-BNY1ABrU.js";
13
16
  import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
14
17
  import { homedir, tmpdir } from "node:os";
15
18
  import { join, posix } from "node:path";
16
19
  import { z } from "zod";
17
20
  import { createServer } from "node:net";
18
21
  import { createHash, randomUUID } from "node:crypto";
22
+ import { setTimeout as setTimeout$1 } from "node:timers/promises";
19
23
  import { spawn } from "node:child_process";
24
+ import { crc32, deflateSync, inflateRawSync } from "node:zlib";
20
25
  import { mkdir, writeFile } from "node:fs/promises";
21
26
  import { generateText, stepCountIs, tool } from "ai";
22
- import { crc32, deflateSync, inflateRawSync } from "node:zlib";
23
27
  //#region src/capture/fake-instructions.ts
24
28
  const FAKE_INSTRUCTIONS = Symbol.for("reeval.capture.fakeInstructions");
25
29
  function isWaitStep(step) {
@@ -900,9 +904,38 @@ function actionFromTool(name, input) {
900
904
  y: Number(input.y)
901
905
  };
902
906
  }
907
+ /**
908
+ * Gemini models answer a point normalised to 0–1000 across the image they were shown, whatever
909
+ * the prompt asks, so their journeys are prompted in that scale and their points scaled back.
910
+ */
911
+ const normalisesPoints = (model) => model.slice(model.lastIndexOf("/") + 1).toLowerCase().startsWith("gemini-");
912
+ const NORMALISED = 1e3;
913
+ /** Scales an action's 0–1000 points to CSS pixels on a `page`-sized screenshot. */
914
+ function scalePoints(action, page) {
915
+ const x = (n) => Math.round(n * page.width / NORMALISED);
916
+ const y = (n) => Math.round(n * page.height / NORMALISED);
917
+ switch (action.type) {
918
+ case "click":
919
+ case "hover":
920
+ case "scroll": return {
921
+ ...action,
922
+ x: x(action.x),
923
+ y: y(action.y)
924
+ };
925
+ case "drag": return {
926
+ ...action,
927
+ x: x(action.x),
928
+ y: y(action.y),
929
+ toX: x(action.toX),
930
+ toY: y(action.toY)
931
+ };
932
+ default: return action;
933
+ }
934
+ }
935
+ const systemPrompt = (normalised) => `You drive a web page from screenshots. Call one tool. ${normalised ? `Coordinates are normalised to 0-${NORMALISED} on the full-page image: x and toX across its width, y and toY down its height. A scroll's dy is in CSS pixels.` : "Coordinates are CSS pixels on the full-page image."} Call done when the goal is met. Do not mention CDP.`;
903
936
  const useTools = {
904
937
  click: tool({
905
- description: "Click at CSS pixels on the screenshot.",
938
+ description: "Click at (x, y) on the screenshot.",
906
939
  inputSchema: z.object({
907
940
  x: z.number(),
908
941
  y: z.number()
@@ -966,6 +999,7 @@ async function runFakeScript(session, script, goal, remaining) {
966
999
  }
967
1000
  async function runModelGoal(session, goal, modelName, remaining, loadModel) {
968
1001
  const model = await loadModel(modelName);
1002
+ const normalised = normalisesPoints(modelName);
969
1003
  let moves = 0;
970
1004
  let usage = null;
971
1005
  while (moves < 32) {
@@ -980,7 +1014,7 @@ async function runModelGoal(session, goal, modelName, remaining, loadModel) {
980
1014
  let chosen;
981
1015
  const result = await generateText({
982
1016
  model,
983
- system: "You drive a web page from screenshots. Call one tool. Coordinates are CSS pixels on the full-page image. Call done when the goal is met. Do not mention CDP.",
1017
+ system: systemPrompt(normalised),
984
1018
  messages: [{
985
1019
  role: "user",
986
1020
  content: [{
@@ -999,6 +1033,7 @@ async function runModelGoal(session, goal, modelName, remaining, loadModel) {
999
1033
  usage = addUsage(usage, asUsage(result.usage ?? {}));
1000
1034
  const call = result.toolCalls?.[0];
1001
1035
  if (call) chosen = actionFromTool(call.toolName, call.input ?? {});
1036
+ if (chosen && normalised) chosen = scalePoints(chosen, page);
1002
1037
  if (!chosen) throw new Error(`c.capture() instruction ${JSON.stringify(goal)} returned no action`);
1003
1038
  debugLog("check", `capture use ${chosen.type} on ${JSON.stringify(goal)}`);
1004
1039
  debugTrace("check", "capture-use", {
@@ -1064,9 +1099,479 @@ function captureUseSpend(result) {
1064
1099
  };
1065
1100
  }
1066
1101
  //#endregion
1102
+ //#region src/contract/attempt.ts
1103
+ const nullableNumber = z.number().nullable();
1104
+ const tokenUsageSchema = z.strictObject({
1105
+ input: z.number(),
1106
+ output: z.number(),
1107
+ cacheRead: z.number(),
1108
+ cacheWrite: z.number(),
1109
+ reasoning: nullableNumber
1110
+ }).describe("The tokens one turn used, as its harness reported them. `input` excludes cache reads and writes (so the prompt was `input + cacheRead + cacheWrite`); `reasoning` is part of `output`.");
1111
+ const fileChangeSchema = z.strictObject({
1112
+ path: z.string(),
1113
+ status: z.enum([
1114
+ "added",
1115
+ "modified",
1116
+ "deleted"
1117
+ ])
1118
+ });
1119
+ const diffSchema = z.strictObject({
1120
+ patch: z.string(),
1121
+ files: z.array(fileChangeSchema)
1122
+ });
1123
+ const turnStatusSchema = z.enum([
1124
+ "completed",
1125
+ "aborted",
1126
+ "error"
1127
+ ]);
1128
+ const webEventTypeSchema = z.enum(["web-search", "web-fetch"]).describe("The transcript events a harness may be unable to report.");
1129
+ const at = {
1130
+ ts: z.string(),
1131
+ atMs: z.number()
1132
+ };
1133
+ const transcriptEventSchema = z.discriminatedUnion("type", [
1134
+ z.strictObject({
1135
+ ...at,
1136
+ type: z.enum(["text", "reasoning"]),
1137
+ text: z.string(),
1138
+ durationMs: nullableNumber
1139
+ }),
1140
+ z.strictObject({
1141
+ ...at,
1142
+ type: z.literal("file-edit"),
1143
+ toolCallId: z.string(),
1144
+ path: z.string(),
1145
+ kind: z.enum([
1146
+ "create",
1147
+ "modify",
1148
+ "delete"
1149
+ ]),
1150
+ added: nullableNumber.describe("Line counts, or null when the harness didn't send the content (e.g. deletes)."),
1151
+ removed: nullableNumber,
1152
+ durationMs: nullableNumber,
1153
+ stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
1154
+ }),
1155
+ z.strictObject({
1156
+ ...at,
1157
+ type: z.literal("web-search"),
1158
+ toolCallId: z.string(),
1159
+ query: z.string(),
1160
+ references: nullableNumber.describe("How many sources the harness says it used; Cursor sends no URLs or titles."),
1161
+ ok: z.boolean().optional().describe("Whether the search succeeded. Absent means it did."),
1162
+ durationMs: nullableNumber,
1163
+ stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
1164
+ }),
1165
+ z.strictObject({
1166
+ ...at,
1167
+ type: z.literal("web-fetch"),
1168
+ toolCallId: z.string(),
1169
+ url: z.string(),
1170
+ ok: z.boolean(),
1171
+ durationMs: nullableNumber,
1172
+ stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
1173
+ }),
1174
+ z.strictObject({
1175
+ ...at,
1176
+ type: z.literal("tool"),
1177
+ toolCallId: z.string(),
1178
+ tool: z.string(),
1179
+ input: z.unknown(),
1180
+ output: z.unknown(),
1181
+ ok: z.boolean(),
1182
+ durationMs: nullableNumber,
1183
+ stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
1184
+ }),
1185
+ z.strictObject({
1186
+ ...at,
1187
+ type: z.literal("error"),
1188
+ message: z.string()
1189
+ }),
1190
+ z.strictObject({
1191
+ ...at,
1192
+ type: z.literal("retry"),
1193
+ message: z.string()
1194
+ })
1195
+ ]).describe("Normalised agent events. Each tool call is one event, merged with its result; `durationMs` is wall time until the result arrived (Cursor's own `completed_at_ms` can be earlier). Null when the harness never reported the end. Long shell or edit calls set `stalled: true`.");
1196
+ const turnDataSchema = z.strictObject({
1197
+ prompt: z.string(),
1198
+ text: z.string().describe("The agent's final message for this turn."),
1199
+ status: turnStatusSchema,
1200
+ startedAt: z.string(),
1201
+ durationMs: z.number(),
1202
+ transcript: z.array(transcriptEventSchema),
1203
+ usage: tokenUsageSchema.nullable().describe("Tokens the turn used; null when the harness reports none."),
1204
+ model: z.string().nullable().describe("The model the harness says served the turn, e.g. `composer-2.5[fast=false]`; null when it reports none. It can differ from the requested one (an alias, or parameters it filled in)."),
1205
+ unreported: z.array(webEventTypeSchema).describe("Event types the harness couldn't report for this turn: the agent may have searched or fetched with no event in `transcript`. `toHaveSearched()` and `toHaveFetched()` refuse such a turn. Usually `[]`; a Cursor turn whose web calls the adapter couldn't recover (its SDK drops them from the stream) is `['web-search', 'web-fetch']`."),
1206
+ diff: diffSchema.describe("Workspace changes since the eval started, as of the end of this turn.")
1207
+ }).describe("A turn as data, as a record or another language's runner carries it: `Turn` without its methods.");
1208
+ const execResultSchema = z.strictObject({
1209
+ exitCode: z.number().int(),
1210
+ stdout: z.string(),
1211
+ stderr: z.string()
1212
+ });
1213
+ const execRetrySchema = z.strictObject({
1214
+ retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional().describe("Extra tries after a non-zero exit, a whole number from 0. Default 0."),
1215
+ interval: z.number().min(0, "must be 0 or more milliseconds").optional().describe("Milliseconds to wait before each extra try. Default 1000.")
1216
+ }).describe("How `c.exec` reruns a command that exits non-zero, for a check that can fail on a transient.");
1217
+ /** A background command has no exit for `retries` to wait on. */
1218
+ const noBackgroundRetry = (options, ctx) => {
1219
+ if (!options.background) return;
1220
+ for (const key of ["retries", "interval"]) {
1221
+ if (options[key] === void 0) continue;
1222
+ ctx.addIssue({
1223
+ code: "custom",
1224
+ path: [key],
1225
+ message: "can't be used with background: true, whose exit reeval doesn't wait for: drop one of them"
1226
+ });
1227
+ }
1228
+ };
1229
+ const execOptionsSchema = z.strictObject({
1230
+ background: z.boolean().optional(),
1231
+ ...execRetrySchema.shape
1232
+ }).superRefine(noBackgroundRetry);
1233
+ /** What's wrong with `c.exec()`'s options, or undefined. */
1234
+ const execOptionsProblem = (value) => dataProblem(execOptionsSchema, value, "c.exec()", () => Object.keys(execOptionsSchema.shape));
1235
+ const assertionRecordSchema = z.strictObject({
1236
+ label: z.string().describe("`expect`'s second argument, or `<matcher> <first argument>` (or the command `c.exec` ran, or the judge's question), cut at 60 characters; ` #2`, ` #3` on repeats within the attempt."),
1237
+ labelled: z.boolean().describe("The eval gave the label."),
1238
+ matcher: z.string().describe("`toHaveFile`, `not.toBe`, `resolves.toBe`, `poll.toBe`."),
1239
+ status: z.enum([
1240
+ "pass",
1241
+ "fail",
1242
+ "error"
1243
+ ]).describe("`error`: the matcher couldn't answer (a subject it can't read, an event the harness doesn't report)."),
1244
+ soft: z.boolean(),
1245
+ message: z.string().optional().describe("Why it failed or couldn't answer, cut at 1 KB."),
1246
+ durationMs: z.number().optional().describe("Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`).")
1247
+ }).describe("One `expect` or `expect.soft` matcher call an attempt made.");
1248
+ const turnIndex = z.number().int("must be a turn index, from 0").min(0, "must be a turn index, from 0").describe("The turn's index, from 0.");
1249
+ const captureSchema = z.strictObject({
1250
+ name: z.string(),
1251
+ port: z.number().describe("Container port that was tunneled."),
1252
+ path: z.string().describe("Path on the app, as passed in."),
1253
+ viewport: z.union([captureWindowSchema, z.literal("full-page")]).describe("What the author asked, including `'full-page'`."),
1254
+ file: z.strictObject({
1255
+ path: z.string(),
1256
+ sha256: z.string(),
1257
+ bytes: z.number()
1258
+ }).describe("`artifacts/<name>.png` under the attempt directory.")
1259
+ }).describe("What `c.capture()` returns: the PNG is also in the attempt record under `artifacts/`.");
1260
+ const judgeArtifactRefSchema = z.strictObject({
1261
+ name: z.string(),
1262
+ path: z.string().describe("`artifacts/<name>.png` under the attempt directory."),
1263
+ sha256: z.string(),
1264
+ bytes: z.number(),
1265
+ mediaType: z.literal("image/png")
1266
+ }).describe("A capture this judge call sent as a file part. Bytes live in `artifacts/<name>.png`.");
1267
+ const verdictFields = {
1268
+ threshold: z.number(),
1269
+ question: z.string(),
1270
+ model: z.string(),
1271
+ gateway: gatewayNameSchema.optional().describe("The gateway from the config's `gateway` list the call went through; unset without one."),
1272
+ artifacts: z.array(judgeArtifactRefSchema).optional().describe("Captures this call sent the judge as file parts, when any.")
1273
+ };
1274
+ const verdictSchema = z.strictObject({
1275
+ verdict: z.enum(["pass", "fail"]),
1276
+ score: z.number().describe("The judge's probability that the answer is yes, from 0 to 1."),
1277
+ ...verdictFields,
1278
+ resolvedModel: z.string().optional().describe("The model id the provider says served the call, when it reports one.")
1279
+ });
1280
+ const judgeFailureSchema = z.strictObject({
1281
+ verdict: z.literal("error"),
1282
+ score: z.null(),
1283
+ ...verdictFields,
1284
+ error: z.string()
1285
+ }).describe("What `c.judge()` returns when the call failed (the provider's error, after any retries): the attempt fails as a judge error once the eval's body has finished, so the judges and checks after it still run.");
1286
+ /** A pattern, for the matchers that take a string or a `RegExp`. */
1287
+ const patternSchema = z.strictObject({
1288
+ regexp: z.string().describe("The pattern's source, as `RegExp.prototype.source` gives it."),
1289
+ flags: z.string().regex(/^[dgimsuvy]*$/, "must use only the flags d, g, i, m, s, u, v and y").optional().describe("Its flags, e.g. `i`.")
1290
+ }).describe("A regular expression, where a matcher takes a string or a `RegExp`.");
1291
+ /** A pattern in the runner's own language, which reeval never compiles. */
1292
+ function hasAnsweredPattern(value) {
1293
+ if (Array.isArray(value)) return value.some(hasAnsweredPattern);
1294
+ if (typeof value !== "object" || value === null) return false;
1295
+ return "matchedBy" in value || Object.values(value).some(hasAnsweredPattern);
1296
+ }
1297
+ const answeredPatternSchema = z.strictObject({
1298
+ matchedBy: z.string().regex(/\S/).describe("The pattern as the runner's language writes it, e.g. `re.compile('npm t')`."),
1299
+ answers: z.record(z.string(), z.boolean()).describe("Whether it matches each string the runner has been asked about.")
1300
+ }).describe("A pattern the runner matches in its own language. reeval runs the matcher with it, and when the matcher tests a string not in `answers` replies with those strings (`untested`) instead of a record; the runner matches them and makes the call again with them in `answers`.");
1301
+ const isPatternLike = (value) => typeof value === "object" && value !== null && !Array.isArray(value) && "regexp" in value;
1302
+ const matcherArgSchema = z.union([
1303
+ patternSchema,
1304
+ answeredPatternSchema,
1305
+ jsonValueSchema
1306
+ ]).superRefine((value, ctx) => {
1307
+ if (!isPatternLike(value)) return;
1308
+ for (const issue of patternSchema.safeParse(value).error?.issues ?? []) ctx.addIssue(issue);
1309
+ }).describe("One argument to a matcher: a JSON value, a regular expression as `{ regexp, flags? }`, or a pattern in the runner's own language as `{ matchedBy, answers }`.");
1310
+ const matcherSubjectSchema = z.union([
1311
+ z.strictObject({ turn: turnIndex }),
1312
+ z.strictObject({ attempt: z.literal(true).describe("The attempt itself, `c`.") }),
1313
+ z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
1314
+ z.strictObject({
1315
+ exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
1316
+ result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
1317
+ }),
1318
+ z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
1319
+ ]).describe("What `expect` was given: a turn `c.send` returned, the attempt (for its files and latest turn), a value, such as an `ExecResult`, or a command or file reeval reads for it.");
1320
+ const nonEmpty = z.string().regex(/\S/, "must not be empty");
1321
+ const portSchema = z.number().int().min(1, "a port from 1 to 65535").max(65535, "a port from 1 to 65535");
1322
+ const millisSchema = z.number().positive("more than zero milliseconds");
1323
+ const captureWaitSchema = z.enum(["load", "idle"]).describe("Chromium look-ready events. `'load'` is `window` load (`Page.loadEventFired`). `'idle'` is `Page.lifecycleEvent` `networkAlmostIdle`.");
1324
+ const waitSchema = z.union([captureWaitSchema, z.strictObject({ ms: millisSchema })]).describe("Same shape at top-level `wait` and `{ wait }` between instructions.");
1325
+ const captureInstructionSchema = z.union([nonEmpty, z.strictObject({ wait: waitSchema })]);
1326
+ const captureCallDataSchema = z.strictObject({
1327
+ name: nonEmpty.optional(),
1328
+ path: z.string().optional(),
1329
+ viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
1330
+ timeout: durationSchema.optional(),
1331
+ wait: waitSchema.optional().describe("Look-ready once, before the first model shot or the saved PNG. Default `'load'`."),
1332
+ instructions: z.array(captureInstructionSchema).optional().describe("Journey: goal strings and `{ wait }` beats, run by the vision model."),
1333
+ model: z.string().optional().describe("Overrides cascaded `capture.model` for this call.")
1334
+ }).describe("`c.capture()`'s camera and path as data. Default `path` is `/`; default `name` is derived from the path. A test's `fakeInstructions` journey is TypeScript-only.");
1335
+ const handleId = nonEmpty.describe("A background process, as an exec or serve reply named it.");
1336
+ const judgeOnSchema = z.union([z.strictObject({ turn: turnIndex }), z.strictObject({ value: jsonValueSchema.describe("Graded as the output.") })]);
1337
+ const judgeCallDataSchema = z.strictObject({
1338
+ model: z.union([judgeModelNameSchema, fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge with a numeric score."),
1339
+ ...judgeFields,
1340
+ on: judgeOnSchema.optional().describe("What to grade instead of the latest turn: an earlier turn, or a value."),
1341
+ artifacts: z.union([z.literal(true), z.array(z.union([nonEmpty, captureSchema]))]).optional().describe("Captures this call sends the judge as image file parts: a capture `c.capture` returned, or its name, or `true` for every capture so far. The provider that cannot take files fails the call.")
1342
+ }).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score, plus the call-only `on` and `artifacts`.");
1343
+ const attemptCallSchema = z.discriminatedUnion("op", [
1344
+ z.strictObject({
1345
+ op: z.literal("send"),
1346
+ prompt: z.string().describe("The turn's prompt.")
1347
+ }).describe("`c.send(prompt)`: the next turn of the agent session. Replies with a turn."),
1348
+ z.strictObject({
1349
+ op: z.literal("exec"),
1350
+ command: nonEmpty.describe("A shell command, run in the workspace."),
1351
+ background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
1352
+ ...execRetrySchema.shape
1353
+ }).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
1354
+ z.strictObject({
1355
+ op: z.literal("serve"),
1356
+ command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
1357
+ port: portSchema.optional().describe("The container port. Omit it and reeval allocates one.")
1358
+ }).describe("`c.serve(command, port?)`: a background process and its port."),
1359
+ z.strictObject({
1360
+ op: z.literal("kill"),
1361
+ handle: handleId
1362
+ }).describe("`handle.kill()`: stops a background process. Killing one that's gone is fine."),
1363
+ z.strictObject({
1364
+ op: z.literal("capture"),
1365
+ target: z.union([z.strictObject({ port: portSchema }), z.strictObject({ handle: handleId })]).describe("A container port, or a serve handle by its id."),
1366
+ options: captureCallDataSchema.optional()
1367
+ }).describe("`c.capture(target, options)`: screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`. Replies with the capture."),
1368
+ z.strictObject({
1369
+ op: z.literal("readFile"),
1370
+ path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory.")
1371
+ }).describe("`c.readFile(path)`. Replies with its contents, or null when there is no such file."),
1372
+ z.strictObject({
1373
+ op: z.literal("writeFile"),
1374
+ path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory."),
1375
+ contents: z.string().describe("The file's contents, in `encoding`."),
1376
+ encoding: z.enum(["utf8", "base64"]).optional().describe("How `contents` is encoded; base64 carries bytes. Default `utf8`.")
1377
+ }).describe("`c.writeFile(path, contents)`, creating its directories."),
1378
+ z.strictObject({
1379
+ op: z.literal("expect"),
1380
+ subject: matcherSubjectSchema,
1381
+ matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a matcher name such as toHaveFile, without \"not.\" (set not: true instead)").describe("The matcher's name, e.g. `toHaveFile` or `toBe`, without `not.`."),
1382
+ args: z.array(matcherArgSchema).describe("The matcher's arguments."),
1383
+ not: z.boolean().describe("Called as `expect(…).not`."),
1384
+ soft: z.boolean().describe("Called as `expect.soft`: a failure fails the attempt at its end."),
1385
+ label: z.string().optional().describe("`expect`'s second argument."),
1386
+ poll: z.strictObject({
1387
+ interval: millisSchema.optional().describe("Milliseconds between tries. Default 50."),
1388
+ timeout: millisSchema.optional().describe("Milliseconds before it fails with the last error. Default 1000.")
1389
+ }).optional().describe("`expect.poll`: reeval re-reads the subject and retries the matcher until it passes, recording only the final outcome. The subject must be one reeval can re-read: the attempt, a command or a file.")
1390
+ }).superRefine((call, ctx) => {
1391
+ if (call.poll && "value" in call.subject) ctx.addIssue({
1392
+ code: "custom",
1393
+ path: ["subject"],
1394
+ message: "can't be polled: reeval can't re-read a value. Poll the attempt, an exec or a readFile"
1395
+ });
1396
+ if (call.poll && "exec" in call.subject && call.subject.result) ctx.addIssue({
1397
+ code: "custom",
1398
+ path: ["subject", "result"],
1399
+ message: "can't be polled: drop result so reeval runs the command on each try"
1400
+ });
1401
+ if (call.poll && call.args.some(hasAnsweredPattern)) ctx.addIssue({
1402
+ code: "custom",
1403
+ path: ["args"],
1404
+ message: "can't be polled with a { matchedBy } pattern, whose answers can't follow the subject from try to try: poll with a string, or check once"
1405
+ });
1406
+ }).describe("A matcher call: `expect(subject, label).not?.matcher(...args)`. Replies with its record."),
1407
+ z.strictObject({
1408
+ op: z.literal("record"),
1409
+ matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a name such as assert").describe("What made the check in the eval's language, e.g. `assert`."),
1410
+ subject: z.string().optional().describe("What it checked, as the eval wrote it (`result.ok`): names the check when it has no label."),
1411
+ label: z.string().optional().describe("The check's own name, the same whether it passes or fails."),
1412
+ status: z.enum(["pass", "fail"]),
1413
+ message: z.string().optional().describe("Why it failed."),
1414
+ soft: z.boolean().optional().describe("A failure fails the attempt at its end, as `expect.soft`. Default false.")
1415
+ }).describe("A check the eval's own language made, such as Python's `assert`: recorded as a matcher call is, so it's named, counted and compared the same. Replies with its record."),
1416
+ z.strictObject({
1417
+ op: z.literal("judge"),
1418
+ question: nonEmpty.describe("A yes/no question about the graded turn or value."),
1419
+ options: judgeCallDataSchema.optional()
1420
+ }).describe("`c.judge(question, options)`. Replies with its verdict, or a judge failure when the provider failed.")
1421
+ ]).describe("One call an eval's body makes on its attempt. The sandbox and the agent session start on the first call that needs them; there's no call to start them.");
1422
+ const attemptErrorSchema = z.looseObject({
1423
+ name: z.string().describe("The error's class, e.g. `UsageError` or `AssertionError`."),
1424
+ message: z.string()
1425
+ }).describe("Why a call failed, in reeval's words.");
1426
+ const id = nonEmpty.describe("Pairs a reply with its request.");
1427
+ const attemptRequestSchema = z.strictObject({
1428
+ id,
1429
+ call: attemptCallSchema
1430
+ }).describe("A call, numbered so its reply can find it.");
1431
+ z.union([
1432
+ z.discriminatedUnion("op", [
1433
+ z.strictObject({
1434
+ id,
1435
+ ok: z.literal(true),
1436
+ op: z.literal("send"),
1437
+ turn: turnDataSchema
1438
+ }),
1439
+ z.strictObject({
1440
+ id,
1441
+ ok: z.literal(true),
1442
+ op: z.literal("exec"),
1443
+ result: z.union([execResultSchema, z.strictObject({ handle: handleId }).describe("A background process, still running.")])
1444
+ }),
1445
+ z.strictObject({
1446
+ id,
1447
+ ok: z.literal(true),
1448
+ op: z.literal("serve"),
1449
+ handle: handleId,
1450
+ port: z.number().describe("The container port it was given.")
1451
+ }),
1452
+ z.strictObject({
1453
+ id,
1454
+ ok: z.literal(true),
1455
+ op: z.literal("kill")
1456
+ }),
1457
+ z.strictObject({
1458
+ id,
1459
+ ok: z.literal(true),
1460
+ op: z.literal("capture"),
1461
+ capture: captureSchema
1462
+ }),
1463
+ z.strictObject({
1464
+ id,
1465
+ ok: z.literal(true),
1466
+ op: z.literal("readFile"),
1467
+ contents: z.string().nullable().describe("Null when there is no such file.")
1468
+ }),
1469
+ z.strictObject({
1470
+ id,
1471
+ ok: z.literal(true),
1472
+ op: z.literal("writeFile")
1473
+ }),
1474
+ z.strictObject({
1475
+ id,
1476
+ ok: z.literal(true),
1477
+ op: z.literal("expect"),
1478
+ assertion: assertionRecordSchema
1479
+ }),
1480
+ z.strictObject({
1481
+ id,
1482
+ ok: z.literal(true),
1483
+ op: z.literal("record"),
1484
+ assertion: assertionRecordSchema
1485
+ }),
1486
+ z.strictObject({
1487
+ id,
1488
+ ok: z.literal(true),
1489
+ op: z.literal("judge"),
1490
+ verdict: z.union([verdictSchema, judgeFailureSchema])
1491
+ })
1492
+ ]),
1493
+ z.strictObject({
1494
+ id,
1495
+ ok: z.literal(true),
1496
+ op: z.literal("expect"),
1497
+ untested: z.record(z.string(), z.array(z.string())).describe("The strings the matcher tested that a `{ matchedBy, answers }` argument has no answer for, by its `matchedBy`. Nothing was recorded: answer them and make the call again.")
1498
+ }),
1499
+ z.strictObject({
1500
+ id,
1501
+ ok: z.literal(false),
1502
+ op: z.enum([
1503
+ "send",
1504
+ "exec",
1505
+ "readFile",
1506
+ "writeFile",
1507
+ "expect",
1508
+ "record",
1509
+ "judge",
1510
+ "serve",
1511
+ "kill",
1512
+ "capture"
1513
+ ]),
1514
+ error: attemptErrorSchema
1515
+ })
1516
+ ]).describe("reeval's reply to a call: its result, or the error it threw. A failed matcher is a result (its record says `fail`), not an error, unless the matcher couldn't run.");
1517
+ const cellRoleSchema = z.enum(["control", "variant"]);
1518
+ const cellSchema = z.strictObject({
1519
+ key: z.string(),
1520
+ harness: z.string(),
1521
+ environment: z.string().describe("The environment's own name, also on a variant cell (`with-house-greeting`, with no prefix)."),
1522
+ sandbox: z.string(),
1523
+ role: cellRoleSchema,
1524
+ variant: z.string().optional().describe("The variant's name, for variant cells.")
1525
+ }).describe("One cell of an eval: a harness, an environment (`-` when none is set) and that environment's image (`sandbox` id: tag, dockerfile and install). With `compare`, the control is `baseline` (or the cascaded environment, or an empty `baseline`) and each variant is that environment on its own, including its sandbox.");
1526
+ z.strictObject({
1527
+ type: z.literal("start"),
1528
+ attempt: z.strictObject({
1529
+ id: nonEmpty,
1530
+ cell: cellSchema,
1531
+ n: z.number().int().min(1, "a run number, from 1").describe("The run number, from 1.")
1532
+ }).describe("`c.attempt`: which cell of the matrix this attempt is, and its run number."),
1533
+ file: nonEmpty.describe("The eval file, relative to the project."),
1534
+ body: nonEmpty.describe("The eval's `body`, as its declaration named it.")
1535
+ }).describe("reeval to the runner: run this eval's body for this attempt. Nothing has started yet: the sandbox and the agent session start on the body's first call that needs them, so its own code runs first. The runner sends its calls, then `done`.");
1536
+ z.strictObject({
1537
+ type: z.literal("abort"),
1538
+ reason: z.enum(["timeout", "cancelled"]).describe("`timeout`: the eval's `timeout` ran out. `cancelled`: the run was stopped."),
1539
+ message: z.string().describe("reeval's sentence, e.g. `Eval timed out after 600s`.")
1540
+ }).describe("reeval to the runner: the attempt was aborted, as `c.signal` aborts. Calls in flight reply with errors; the runner should stop the body and send `done`.");
1541
+ z.strictObject({
1542
+ type: z.literal("done"),
1543
+ error: attemptErrorSchema.optional().describe("What the body threw, if it threw. A failed `expect` (not soft) is one.")
1544
+ }).describe("The runner to reeval: the eval's body has finished. The attempt's checks and record follow.");
1545
+ const CALL_KEYS = Object.fromEntries(attemptCallSchema.options.map((o) => [o.shape.op.value, Object.keys(o.shape)]));
1546
+ const OTHER_KEYS = [
1547
+ "id",
1548
+ "call",
1549
+ "turn",
1550
+ "attempt",
1551
+ "value",
1552
+ "regexp",
1553
+ "flags",
1554
+ "matchedBy",
1555
+ "answers",
1556
+ ...Object.keys(judgeCallDataSchema.shape),
1557
+ ...Object.keys(judgeStateSchema.shape),
1558
+ ...Object.keys(captureSchema.shape),
1559
+ ...Object.keys(captureCallDataSchema.shape),
1560
+ "port",
1561
+ "handle",
1562
+ "exec",
1563
+ "result",
1564
+ "readFile",
1565
+ "interval",
1566
+ "timeout",
1567
+ "ms"
1568
+ ];
1569
+ /** What's wrong with an attempt request a runner sent, by the field's path, or undefined. */
1570
+ const attemptRequestProblem = (value) => dataProblem(attemptRequestSchema, value, "The attempt request", (object) => typeof object.op === "string" ? CALL_KEYS[object.op] ?? [] : OTHER_KEYS);
1571
+ //#endregion
1067
1572
  //#region src/diff.ts
1068
1573
  const GIT = "git -c user.name=reeval -c user.email=reeval@localhost -c core.hooksPath=/dev/null -c core.logAllRefUpdates=false -c gc.auto=0";
1069
- const snapshotEnv = (sandbox) => `export GIT_DIR=${shellQuote(`${sandbox.stateDir}/workspace.git`)} GIT_WORK_TREE="$PWD" GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null`;
1574
+ const snapshotEnv = (sandbox) => `export GIT_DIR=${shellQuote(`${sandbox.stateDir}/workspace.git`)} GIT_WORK_TREE=${shellQuote(sandbox.workdir)} GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null`;
1070
1575
  /** Points HEAD at a commit of `$tree` with no parent, then drops every object nothing reaches. */
1071
1576
  const RESET_HEAD = `c=$(${GIT} commit-tree -m base "$tree") && ${GIT} update-ref HEAD "$c" && ${GIT} prune --expire=now`;
1072
1577
  /**
@@ -1125,45 +1630,73 @@ const asCheck = (a, fn) => a.phases ? a.phases.checks(fn) : fn();
1125
1630
  /** The attempt's sandbox, started (with its session) by the first call that needs it. */
1126
1631
  const sandboxFor = async (a, who) => (await a.start(who))[0];
1127
1632
  /**
1128
- * `c.exec` runs with the checks' variables and secrets, each run recorded and
1129
- * timed; `c.readFile` reads the sandbox as is; `c.writeFile` places a file the diff leaves out.
1633
+ * `c.exec` runs with the checks' variables and secrets, each run recorded and timed, and runs
1634
+ * again on a non-zero exit while it has `retries` left; `c.readFile` reads the sandbox as is;
1635
+ * `c.writeFile` places a file the diff leaves out.
1130
1636
  */
1131
1637
  function createChecks(a) {
1132
1638
  const checkEnv = () => a.env?.checkVars ?? {};
1133
- const execForeground = async (command) => {
1639
+ const execForeground = async (command, retries, interval) => {
1134
1640
  const box = await sandboxFor(a, "c.exec()");
1135
1641
  const start = performance.now();
1642
+ const tries = [];
1136
1643
  a.report({
1137
1644
  kind: "check-start",
1138
1645
  command
1139
1646
  });
1140
- const r = await a.timed("checksMs", () => asCheck(a, () => box.exec(command, {
1141
- signal: a.signal,
1142
- env: checkEnv()
1143
- })));
1647
+ const r = await a.timed("checksMs", async () => {
1648
+ for (;;) {
1649
+ const began = performance.now();
1650
+ const r = await asCheck(a, () => box.exec(command, {
1651
+ signal: a.signal,
1652
+ env: checkEnv()
1653
+ }));
1654
+ tries.push({
1655
+ ...r,
1656
+ durationMs: Math.round(performance.now() - began)
1657
+ });
1658
+ if (r.exitCode === 0 || tries.length > retries) return r;
1659
+ debugLog("check", `$ ${clip(command, 160)} → exit ${r.exitCode} on try ${tries.length} of ${retries + 1}, again in ${formatDuration(interval)}`);
1660
+ await setTimeout$1(interval, void 0, { signal: a.signal });
1661
+ }
1662
+ });
1144
1663
  const durationMs = Math.round(performance.now() - start);
1664
+ const retried = retries > 0 ? {
1665
+ retries,
1666
+ tries
1667
+ } : {};
1145
1668
  a.record.checks.push({
1146
1669
  command,
1147
1670
  ...r,
1148
- durationMs
1671
+ durationMs,
1672
+ ...retried
1149
1673
  });
1150
- debugLog("check", `$ ${clip(command, 160)} → exit ${r.exitCode} · ${formatDuration(durationMs)}`);
1674
+ const of = retries > 0 ? ` on try ${tries.length} of ${retries + 1}` : "";
1675
+ debugLog("check", `$ ${clip(command, 160)} → exit ${r.exitCode}${of} · ${formatDuration(durationMs)}`);
1151
1676
  debugTrace("check", "output", {
1152
1677
  command,
1153
1678
  ...r,
1154
- durationMs
1679
+ durationMs,
1680
+ ...retried
1155
1681
  }, `check output: ${clip(command, 80)}`);
1156
1682
  a.report({
1157
1683
  kind: "check-end",
1158
1684
  command,
1159
1685
  exitCode: r.exitCode,
1160
- ms: durationMs
1686
+ ms: durationMs,
1687
+ ...retries > 0 ? {
1688
+ try: tries.length,
1689
+ tries: retries + 1
1690
+ } : {}
1161
1691
  });
1162
1692
  return r;
1163
1693
  };
1164
1694
  function exec(command, options) {
1165
- if (options?.background) return import("./capture-Dp3q7K7Y.js").then(({ startBackground }) => startBackground(a, command, "c.exec({ background: true })"));
1166
- return Object.defineProperty(execForeground(command), SUBJECT, { value: command });
1695
+ const problem = options === void 0 ? void 0 : execOptionsProblem(options);
1696
+ if (problem) return Promise.reject(new UsageError(problem));
1697
+ if (options?.background) return import("./capture-CherbjzB.js").then(({ startBackground }) => startBackground(a, command, "c.exec({ background: true })"));
1698
+ const run = execForeground(command, options?.retries ?? 0, options?.interval ?? 1e3);
1699
+ return Object.defineProperty(run, SUBJECT, { value: command });
1167
1700
  }
1168
1701
  const readFile = async (path) => (await sandboxFor(a, "c.readFile()")).readFile(path);
1169
1702
  const writeFile = async (path, contents) => {
@@ -1188,25 +1721,26 @@ function createChecks(a) {
1188
1721
  const BG_LOG = "/tmp/reeval-bg.log";
1189
1722
  const BG_TAG = "REEVAL_BACKGROUND";
1190
1723
  const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
1191
- function defaultCaptureModel() {
1724
+ const languageModel = (model) => typeof model === "string" && !isEvaluationModel(model) ? model : void 0;
1725
+ /**
1726
+ * `analysis.model`, else the attempt's judge (a profile's, group's or eval's over the config's),
1727
+ * whichever is first a language model.
1728
+ */
1729
+ function defaultCaptureModel(a) {
1192
1730
  try {
1193
- const { config } = getRunContext();
1194
- const analysis = config.analysis.model;
1195
- if (typeof analysis === "string" && !isEvaluationModel(analysis)) return analysis;
1196
- const judge = config.judge.model;
1197
- if (typeof judge === "string" && !isEvaluationModel(judge)) return judge;
1731
+ return languageModel(getRunContext().config.analysis.model) ?? languageModel(a.spec.judge.model);
1198
1732
  } catch {
1199
1733
  return;
1200
1734
  }
1201
1735
  }
1202
- function resolveUse(resolved, fakeSandbox) {
1736
+ function resolveUse(a, resolved, fakeSandbox) {
1203
1737
  if (resolved.fake) return { fake: resolved.fake };
1204
1738
  if (resolved.model) return { model: resolved.model };
1205
1739
  if (!resolved.instructions.length) return void 0;
1206
- const fallback = defaultCaptureModel();
1740
+ const fallback = defaultCaptureModel(a);
1207
1741
  if (fallback) return { model: fallback };
1208
1742
  if (fakeSandbox) return void 0;
1209
- throw new Error("c.capture() instructions need a vision model. Set capture.model, pass model on the call, or analysis.model to a language model that accepts images. Tests pass fakeInstructions from reeval/testing.");
1743
+ throw new Error("c.capture() instructions need a vision model. Set capture.model, pass model on the call, or set analysis.model or the judge model to a language model that accepts images. Tests pass fakeInstructions from reeval/testing.");
1210
1744
  }
1211
1745
  /**
1212
1746
  * Stops `pid` and its descendants under Git for Windows' MSYS. Native children (node.exe) are
@@ -1282,7 +1816,7 @@ async function capturePage(a, target, call) {
1282
1816
  throw new Error(`nothing listening on port ${resolved.port}`);
1283
1817
  }
1284
1818
  const fake = a.spec.sandbox.kind === "fake";
1285
- const use = resolveUse(resolved, fake);
1819
+ const use = resolveUse(a, resolved, fake);
1286
1820
  a.captureNames.add(resolved.name);
1287
1821
  const started = Date.now();
1288
1822
  const remaining = () => Math.max(0, resolved.timeoutMs - (Date.now() - started));
@@ -1346,7 +1880,9 @@ async function shot(a, url, resolved, remaining, onMoves) {
1346
1880
  use: resolved.use,
1347
1881
  remaining,
1348
1882
  loadModel: async (name) => {
1349
- const route = judgeRoute(name);
1883
+ const { config, profile } = getRunContext();
1884
+ const gateway = resolveModelGateway(name, void 0, gatewayScope(config, profile), "capture model");
1885
+ const route = judgeRoute(name, process.env, "capture model", gateway);
1350
1886
  return providerModel({
1351
1887
  name,
1352
1888
  route: route.kind === "direct" ? "direct" : "gateway"
@@ -1374,6 +1910,6 @@ async function writeCaptureSidecars(stagingDir, recordFile, blobs) {
1374
1910
  for (const [name, png] of blobs) await writeFile(join(dir, `${name}.png`), png);
1375
1911
  }
1376
1912
  //#endregion
1377
- export { writeCaptureSidecars as a, sandboxFor as c, workspaceFiles as d, fakeInstructions as f, startBackground as i, diffSinceSnapshot as l, msysTreeKill as n, asCheck as o, serveCommand as r, createChecks as s, capturePage as t, snapshot as u };
1913
+ export { writeCaptureSidecars as a, sandboxFor as c, workspaceFiles as d, attemptRequestProblem as f, startBackground as i, diffSinceSnapshot as l, msysTreeKill as n, asCheck as o, fakeInstructions as p, serveCommand as r, createChecks as s, capturePage as t, snapshot as u };
1378
1914
 
1379
- //# sourceMappingURL=capture-BiD7lVaM.js.map
1915
+ //# sourceMappingURL=capture-CVRwRTnb.js.map