reeval 0.14.0 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (364) hide show
  1. package/README.md +15 -15
  2. package/THIRD_PARTY_NOTICES.md +69 -69
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-BloGVZ_a.js → adapter-CRKYPkRt.js} +37 -8
  5. package/dist/adapter-CRKYPkRt.js.map +1 -0
  6. package/dist/{agent-m-_t-IuW.js → agent-BgH4-kFc.js} +58 -16
  7. package/dist/agent-BgH4-kFc.js.map +1 -0
  8. package/dist/{app-files-C-jwhRDt.js → app-files-BMVBr079.js} +4 -4
  9. package/dist/app-files-BMVBr079.js.map +1 -0
  10. package/dist/{assemble-BDEpelpi.js → assemble-BVbI6NW-.js} +214 -128
  11. package/dist/assemble-BVbI6NW-.js.map +1 -0
  12. package/dist/assemble-SiHa4__F.js +1 -0
  13. package/dist/{attempt-jI0OWxlz.js → attempt-CpbZ9arV.js} +10 -10
  14. package/dist/{attempt-jI0OWxlz.js.map → attempt-CpbZ9arV.js.map} +1 -1
  15. package/dist/{match-Bz0nSdzN.js → attempt-host-CIxjs2Be.js} +104 -6
  16. package/dist/attempt-host-CIxjs2Be.js.map +1 -0
  17. package/dist/{capture-CVRwRTnb.js → capture-iQ6kYJV0.js} +2360 -1523
  18. package/dist/capture-iQ6kYJV0.js.map +1 -0
  19. package/dist/capture-px7Ey95_.js +1 -0
  20. package/dist/{cli-C_4gaJ50.js → cli-1YQU-66F.js} +153 -180
  21. package/dist/cli-1YQU-66F.js.map +1 -0
  22. package/dist/{cli-spec-CpnbATtI.js → cli-spec-CPWT1mAG.js} +10 -21
  23. package/dist/cli-spec-CPWT1mAG.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-D5hNjpEx.js → client-DMmOkbRA.js} +14 -7
  26. package/dist/client-DMmOkbRA.js.map +1 -0
  27. package/dist/{command-BXLUfw4h.js → command-8_Cg5v6b.js} +35 -22
  28. package/dist/command-8_Cg5v6b.js.map +1 -0
  29. package/dist/{command-I1euLLio.js → command-BlYCAqu-.js} +7 -7
  30. package/dist/{command-I1euLLio.js.map → command-BlYCAqu-.js.map} +1 -1
  31. package/dist/{validate-BhwqrVsm.js → command-C-HeGlap.js} +388 -60
  32. package/dist/command-C-HeGlap.js.map +1 -0
  33. package/dist/{command-Bxdy62dW.js → command-Cr1zjRvl.js} +32 -31
  34. package/dist/command-Cr1zjRvl.js.map +1 -0
  35. package/dist/{commands-p1S4nw_H.js → commands-Bj9NbmsO.js} +1 -1
  36. package/dist/{commands-jzigbNFb.js → commands-BvTNZ2-b.js} +181 -73
  37. package/dist/commands-BvTNZ2-b.js.map +1 -0
  38. package/dist/{config-BrdHA9bJ.js → config-CneBZB84.js} +883 -433
  39. package/dist/config-CneBZB84.js.map +1 -0
  40. package/dist/{contrasts-DRxMQQTC.js → contrasts-lhFsekco.js} +3 -3
  41. package/dist/{contrasts-DRxMQQTC.js.map → contrasts-lhFsekco.js.map} +1 -1
  42. package/dist/cost-C5RBTTJr.js +1 -0
  43. package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
  44. package/dist/cost-ChPa8-Fp.js.map +1 -0
  45. package/dist/db-BnXyamuj.js +2 -0
  46. package/dist/db-Cnh3jIv_.js +1530 -0
  47. package/dist/db-Cnh3jIv_.js.map +1 -0
  48. package/dist/debug-CAx3qcaP.js +1 -0
  49. package/dist/{debug-D24NOQH1.js → debug-CgG3T071.js} +5 -2
  50. package/dist/debug-CgG3T071.js.map +1 -0
  51. package/dist/{docker-BnRlTpZA.js → docker-BihzJwbV.js} +11 -6
  52. package/dist/docker-BihzJwbV.js.map +1 -0
  53. package/dist/{docker-D01slDF3.js → docker-Cqq03H72.js} +26 -8
  54. package/dist/docker-Cqq03H72.js.map +1 -0
  55. package/dist/{docker-DivfZl2u.js → docker-DCAtFMo9.js} +1 -1
  56. package/dist/{doctor-DQ2p0euG.js → doctor-Dl4vMeqt.js} +275 -37
  57. package/dist/doctor-Dl4vMeqt.js.map +1 -0
  58. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  59. package/dist/env-CARj5Axq.js.map +1 -0
  60. package/dist/{environment-CO0P0soF.js → environment-Cf5HMjib.js} +116 -297
  61. package/dist/environment-Cf5HMjib.js.map +1 -0
  62. package/dist/{errors-r025B0Zb.js → errors-CHebohjc.js} +10 -4
  63. package/dist/{errors-r025B0Zb.js.map → errors-CHebohjc.js.map} +1 -1
  64. package/dist/eval-source-CAE4y2Sr.js +146 -0
  65. package/dist/eval-source-CAE4y2Sr.js.map +1 -0
  66. package/dist/{evaluate-BUmsN7yG.js → evaluate-CUtgyr43.js} +682 -150
  67. package/dist/evaluate-CUtgyr43.js.map +1 -0
  68. package/dist/evaluate-D35T51AC.d.ts +45 -0
  69. package/dist/files-BEnUJW1e.js +1 -0
  70. package/dist/{files-B-CJeeEh.js → files-BRViE7m-.js} +200 -87
  71. package/dist/files-BRViE7m-.js.map +1 -0
  72. package/dist/{files-kUCrswI6.js → files-BSqA2sZE.js} +20 -9
  73. package/dist/files-BSqA2sZE.js.map +1 -0
  74. package/dist/files-C3tQtD26.js +1 -0
  75. package/dist/files-CNiJ_fZf.js +37 -0
  76. package/dist/files-CNiJ_fZf.js.map +1 -0
  77. package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
  78. package/dist/files-D1y8Pg4Z.js.map +1 -0
  79. package/dist/files-Dts7EY3H.js +1 -0
  80. package/dist/{pairs-CHfZNRT6.js → gate-BC6SdQZs.js} +58 -4
  81. package/dist/gate-BC6SdQZs.js.map +1 -0
  82. package/dist/gateway.d.ts +22 -9
  83. package/dist/gateway.js +30 -8
  84. package/dist/gateway.js.map +1 -1
  85. package/dist/{git-Bk8DSwHy.js → git-UNxT_3z0.js} +3 -3
  86. package/dist/{git-Bk8DSwHy.js.map → git-UNxT_3z0.js.map} +1 -1
  87. package/dist/harness-3NVyTSGK.js +392 -0
  88. package/dist/harness-3NVyTSGK.js.map +1 -0
  89. package/dist/harness.d.ts +12 -8
  90. package/dist/harness.js +1 -1
  91. package/dist/{images-DzLeFjLx.js → images-8A_LzVfw.js} +8 -8
  92. package/dist/{images-DzLeFjLx.js.map → images-8A_LzVfw.js.map} +1 -1
  93. package/dist/index.d.ts +99 -11
  94. package/dist/index.js +5 -5
  95. package/dist/init-okoALIah.js +1 -0
  96. package/dist/{init-SRalM5xF.js → init-sFBgODhu.js} +10 -11
  97. package/dist/init-sFBgODhu.js.map +1 -0
  98. package/dist/{invocations-DzyKSJBa.js → invocations-C9ejlOYl.js} +10 -6
  99. package/dist/invocations-C9ejlOYl.js.map +1 -0
  100. package/dist/{judge-yLvjC5A3.js → judge-Bg4v4e_K.js} +236 -46
  101. package/dist/judge-Bg4v4e_K.js.map +1 -0
  102. package/dist/judge.d.ts +5 -7
  103. package/dist/judge.js +4 -8
  104. package/dist/judge.js.map +1 -1
  105. package/dist/{live-DYd2QEyR.js → live-DeMvnDD5.js} +3 -3
  106. package/dist/{live-DYd2QEyR.js.map → live-DeMvnDD5.js.map} +1 -1
  107. package/dist/{matchers-D4eUqx4j.js → matchers-BKPuaXlj.js} +14 -13
  108. package/dist/matchers-BKPuaXlj.js.map +1 -0
  109. package/dist/{model-1ZLr9Dcv.js → model-2rBZrdZA.js} +29 -36
  110. package/dist/model-2rBZrdZA.js.map +1 -0
  111. package/dist/{model-B45lDUV1.js → model-3MJtpho8.js} +22 -13
  112. package/dist/model-3MJtpho8.js.map +1 -0
  113. package/dist/{model-CgnOpss4.js → model-Bz7-j71_.js} +245 -68
  114. package/dist/model-Bz7-j71_.js.map +1 -0
  115. package/dist/model-DkPwypN4.js +1 -0
  116. package/dist/model-bREXVXwT.js +1 -0
  117. package/dist/{node-BSq5qHnQ.js → node-fP-H2uxn.js} +72 -50
  118. package/dist/node-fP-H2uxn.js.map +1 -0
  119. package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
  120. package/dist/object-BkXGPKWY.js.map +1 -0
  121. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  122. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  123. package/dist/openrouter-models-Bo8PZ4Ew.js +34 -0
  124. package/dist/openrouter-models-Bo8PZ4Ew.js.map +1 -0
  125. package/dist/{patch-DWf-I_Q_.js → patch-6T9XAipm.js} +7 -9
  126. package/dist/patch-6T9XAipm.js.map +1 -0
  127. package/dist/patch-CquGAGX5.js +76 -0
  128. package/dist/patch-CquGAGX5.js.map +1 -0
  129. package/dist/{plan-BIwUzlB8.js → plan-Dw2BjafJ.js} +79 -35
  130. package/dist/plan-Dw2BjafJ.js.map +1 -0
  131. package/dist/{plugins-Dkp_H3pB.js → plugins-ngQEJBjJ.js} +274 -17
  132. package/dist/plugins-ngQEJBjJ.js.map +1 -0
  133. package/dist/{preflight-D9y-HvZQ.js → preflight-BYXkweNN.js} +1 -1
  134. package/dist/{preflight-DJCD_BDn.js → preflight-CDRpog9o.js} +3 -3
  135. package/dist/{preflight-DJCD_BDn.js.map → preflight-CDRpog9o.js.map} +1 -1
  136. package/dist/{prepare-BJG_pD54.js → prepare-B3btUlT6.js} +2 -2
  137. package/dist/{prepare-CIxCGG0M.js → prepare-PkARUGoF.js} +8 -8
  138. package/dist/{prepare-CIxCGG0M.js.map → prepare-PkARUGoF.js.map} +1 -1
  139. package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
  140. package/dist/prices-QaS-kjSQ.js.map +1 -0
  141. package/dist/{provenance-CfZpFpAh.js → provenance-7Hzg88f4.js} +4 -4
  142. package/dist/{provenance-CfZpFpAh.js.map → provenance-7Hzg88f4.js.map} +1 -1
  143. package/dist/{providers-D1Zxp2he.js → providers-DDNMbv1T.js} +1 -1
  144. package/dist/{providers-DFk_6W4K.js → providers-PKoJtGK0.js} +5 -5
  145. package/dist/{providers-DFk_6W4K.js.map → providers-PKoJtGK0.js.map} +1 -1
  146. package/dist/query-Bt6vLJ3m.js +1 -0
  147. package/dist/{query-CemclveB.js → query-nBXrgEwl.js} +3 -3
  148. package/dist/{query-CemclveB.js.map → query-nBXrgEwl.js.map} +1 -1
  149. package/dist/{questions-bAudIc04.js → questions-CkAwePPR.js} +110 -36
  150. package/dist/questions-CkAwePPR.js.map +1 -0
  151. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  152. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  153. package/dist/redact-DVIaahLv.js +658 -0
  154. package/dist/redact-DVIaahLv.js.map +1 -0
  155. package/dist/remarks-Bp6AwPgb.js +1 -0
  156. package/dist/{rules-VhftPOOT.js → remarks-Bt0zwiQB.js} +330 -66
  157. package/dist/remarks-Bt0zwiQB.js.map +1 -0
  158. package/dist/{header-BLPP6aCU.js → remarks-IFFe_oty.js} +35 -9
  159. package/dist/remarks-IFFe_oty.js.map +1 -0
  160. package/dist/{remarks-mujJv1Ey.js → remarks-eBcrIZRu.js} +10 -5
  161. package/dist/remarks-eBcrIZRu.js.map +1 -0
  162. package/dist/{remote-CKcEKja4.js → remote-DfJYiP-R.js} +39 -99
  163. package/dist/remote-DfJYiP-R.js.map +1 -0
  164. package/dist/resolve-CT4XZIfV.js +1 -0
  165. package/dist/{resolve-CBoKzQMP.js → resolve-D_cLa4tW.js} +46 -30
  166. package/dist/resolve-D_cLa4tW.js.map +1 -0
  167. package/dist/results-DrKJuIQF.js +63 -0
  168. package/dist/results-DrKJuIQF.js.map +1 -0
  169. package/dist/{resume-B9SOSnK9.js → resume-BmIM52MJ.js} +47 -17
  170. package/dist/resume-BmIM52MJ.js.map +1 -0
  171. package/dist/{runtime-BgQRyCLl.js → runtime-CZVpkr04.js} +139 -48
  172. package/dist/runtime-CZVpkr04.js.map +1 -0
  173. package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmi6IQML.js} +2 -2
  174. package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmi6IQML.js.map} +1 -1
  175. package/dist/sandbox.d.ts +1 -1
  176. package/dist/sandbox.js +1 -1
  177. package/dist/server-CKJOnIB0.js +1 -0
  178. package/dist/{server-TGW4WyAz.js → server-DS8BGhGU.js} +75 -33
  179. package/dist/server-DS8BGhGU.js.map +1 -0
  180. package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
  181. package/dist/shared-B_lvGqWz.js.map +1 -0
  182. package/dist/source-CgUqpumI.js +1 -0
  183. package/dist/source-DcXv4de-.js +59 -0
  184. package/dist/source-DcXv4de-.js.map +1 -0
  185. package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
  186. package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
  187. package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
  188. package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
  189. package/dist/{src-DjRI-IeE.js → src-A8HLtO0a.js} +135 -35
  190. package/dist/{src-DjRI-IeE.js.map → src-A8HLtO0a.js.map} +1 -1
  191. package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
  192. package/dist/src-BO8AYqJ5.js.map +1 -0
  193. package/dist/src-BTqJbvj6.js +2 -0
  194. package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
  195. package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BkiComGh.js +2 -0
  197. package/dist/src-BvgeZ4tS.js +2 -0
  198. package/dist/{src-8zoX_qrb.js → src-C0ySiej5.js} +572 -1067
  199. package/dist/src-C0ySiej5.js.map +1 -0
  200. package/dist/src-CgJrz5p0.js +3 -0
  201. package/dist/src-Cp-2Hdib.js +2 -0
  202. package/dist/{src-CyaDNG9r.js → src-DMo2eCOJ.js} +1 -1
  203. package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
  204. package/dist/src-DSZyg0Uq.js.map +1 -0
  205. package/dist/src-Dk2Na0b-.js +2 -0
  206. package/dist/src-OESRtvOi.js +2 -0
  207. package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
  208. package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
  209. package/dist/src-h5uDlk3m.js +1 -0
  210. package/dist/state-C97udoh3.js +1 -0
  211. package/dist/{state-B1t143wG.js → state-C_kdhp5v.js} +15 -15
  212. package/dist/{state-B1t143wG.js.map → state-C_kdhp5v.js.map} +1 -1
  213. package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
  214. package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
  215. package/dist/suite-ByT1lsdD.js +2 -0
  216. package/dist/{suite-WgT7m6Vs.js → suite-DieLXvCT.js} +61 -65
  217. package/dist/suite-DieLXvCT.js.map +1 -0
  218. package/dist/{summary-DIGEVr08.js → summary-CSzyBVsF.js} +113 -277
  219. package/dist/summary-CSzyBVsF.js.map +1 -0
  220. package/dist/tarball-CHcahyNK.js +258 -0
  221. package/dist/tarball-CHcahyNK.js.map +1 -0
  222. package/dist/testing.d.ts +426 -143
  223. package/dist/testing.js +12 -10
  224. package/dist/testing.js.map +1 -1
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
  228. package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
  229. package/dist/{transcript-BTxJpW4R.js → transcript-BJrKfgRW.js} +13 -5
  230. package/dist/transcript-BJrKfgRW.js.map +1 -0
  231. package/dist/types-C5vI3g1C.js.map +1 -1
  232. package/dist/{types-DXBbP9Mk.d.ts → types-DVvW66ci.d.ts} +569 -895
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +548 -121
  238. package/dist/view/index.js +9 -9
  239. package/dist/{views-sXpGE6a2.js → views-B9STsLB4.js} +9 -10
  240. package/dist/{views-sXpGE6a2.js.map → views-B9STsLB4.js.map} +1 -1
  241. package/dist/views-Ca-rTixp.js +2 -0
  242. package/dist/{worker-entry-Duo9oAeT.js → worker-entry-C5bLCv6s.js} +48 -20
  243. package/dist/worker-entry-C5bLCv6s.js.map +1 -0
  244. package/dist/workspace-8uHtBSve.js +1 -0
  245. package/dist/{workspace-CDI68wtX.js → workspace-CBuI9Uds.js} +5 -5
  246. package/dist/{workspace-CDI68wtX.js.map → workspace-CBuI9Uds.js.map} +1 -1
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +5 -1
  249. package/skills/eval-rubric/SKILL.md +46 -104
  250. package/dist/adapter-BloGVZ_a.js.map +0 -1
  251. package/dist/agent-m-_t-IuW.js.map +0 -1
  252. package/dist/app-files-C-jwhRDt.js.map +0 -1
  253. package/dist/assemble-BDEpelpi.js.map +0 -1
  254. package/dist/assemble-C5bfncHU.js +0 -1
  255. package/dist/capture-CVRwRTnb.js.map +0 -1
  256. package/dist/capture-CherbjzB.js +0 -1
  257. package/dist/cli-C_4gaJ50.js.map +0 -1
  258. package/dist/cli-spec-CpnbATtI.js.map +0 -1
  259. package/dist/client-D5hNjpEx.js.map +0 -1
  260. package/dist/command-BWAuwQNk.js +0 -91
  261. package/dist/command-BWAuwQNk.js.map +0 -1
  262. package/dist/command-BXLUfw4h.js.map +0 -1
  263. package/dist/command-Be6BSE2L.js +0 -335
  264. package/dist/command-Be6BSE2L.js.map +0 -1
  265. package/dist/command-Bxdy62dW.js.map +0 -1
  266. package/dist/commands-jzigbNFb.js.map +0 -1
  267. package/dist/config-BrdHA9bJ.js.map +0 -1
  268. package/dist/cost-D4elx933.js.map +0 -1
  269. package/dist/cost-DmnAod9x.js +0 -1
  270. package/dist/db-CAfksinX.js +0 -2
  271. package/dist/db-DO0p7xgP.js +0 -619
  272. package/dist/db-DO0p7xgP.js.map +0 -1
  273. package/dist/debug-D24NOQH1.js.map +0 -1
  274. package/dist/debug-D7bEv_UV.js +0 -1
  275. package/dist/docker-BnRlTpZA.js.map +0 -1
  276. package/dist/docker-D01slDF3.js.map +0 -1
  277. package/dist/doctor-DQ2p0euG.js.map +0 -1
  278. package/dist/env-BQqj93gu.js.map +0 -1
  279. package/dist/environment-CO0P0soF.js.map +0 -1
  280. package/dist/evaluate-BUmsN7yG.js.map +0 -1
  281. package/dist/evaluate-C_sgRA-V.d.ts +0 -134
  282. package/dist/facts-DzOUTreB.js +0 -906
  283. package/dist/facts-DzOUTreB.js.map +0 -1
  284. package/dist/files-B-CJeeEh.js.map +0 -1
  285. package/dist/files-BAt0i00X.js +0 -1
  286. package/dist/files-Bp5j5BqI.js +0 -1
  287. package/dist/files-C2sjmXhk.js +0 -1
  288. package/dist/files-C_gQNVB-.js +0 -65
  289. package/dist/files-C_gQNVB-.js.map +0 -1
  290. package/dist/files-Ca0d6Uq3.js.map +0 -1
  291. package/dist/files-kUCrswI6.js.map +0 -1
  292. package/dist/generate-DZ7vp69F.js +0 -1647
  293. package/dist/generate-DZ7vp69F.js.map +0 -1
  294. package/dist/harness-COVXSwJ6.js +0 -354
  295. package/dist/harness-COVXSwJ6.js.map +0 -1
  296. package/dist/header-BLPP6aCU.js.map +0 -1
  297. package/dist/init-RgCQXZhb.js +0 -1
  298. package/dist/init-SRalM5xF.js.map +0 -1
  299. package/dist/insights-TyAvz9aV.js +0 -68
  300. package/dist/insights-TyAvz9aV.js.map +0 -1
  301. package/dist/invocations-DzyKSJBa.js.map +0 -1
  302. package/dist/judge-yLvjC5A3.js.map +0 -1
  303. package/dist/match-Bz0nSdzN.js.map +0 -1
  304. package/dist/matchers-D4eUqx4j.js.map +0 -1
  305. package/dist/model-1ZLr9Dcv.js.map +0 -1
  306. package/dist/model-B45lDUV1.js.map +0 -1
  307. package/dist/model-BTZq8HIZ.js +0 -1
  308. package/dist/model-CgnOpss4.js.map +0 -1
  309. package/dist/model-Ch8kX87u.js +0 -1
  310. package/dist/node-BSq5qHnQ.js.map +0 -1
  311. package/dist/object-CLf2-tNq.js.map +0 -1
  312. package/dist/opencode-providers-klkqgzvb.js +0 -3276
  313. package/dist/opencode-providers-klkqgzvb.js.map +0 -1
  314. package/dist/pairs-CHfZNRT6.js.map +0 -1
  315. package/dist/patch-DWf-I_Q_.js.map +0 -1
  316. package/dist/plan-BIwUzlB8.js.map +0 -1
  317. package/dist/plugins-Dkp_H3pB.js.map +0 -1
  318. package/dist/prices-BcLEXvdY.js.map +0 -1
  319. package/dist/query-Cvbwl804.js +0 -1
  320. package/dist/questions-bAudIc04.js.map +0 -1
  321. package/dist/redact-CRGdfV2s.js +0 -240
  322. package/dist/redact-CRGdfV2s.js.map +0 -1
  323. package/dist/remarks-CXwSnSjR.js +0 -278
  324. package/dist/remarks-CXwSnSjR.js.map +0 -1
  325. package/dist/remarks-CyAu2DiB.js +0 -1
  326. package/dist/remarks-mujJv1Ey.js.map +0 -1
  327. package/dist/remote-CKcEKja4.js.map +0 -1
  328. package/dist/resolve-B5-uqmJ6.js +0 -1
  329. package/dist/resolve-CBoKzQMP.js.map +0 -1
  330. package/dist/results-CMgZEANy.js +0 -78
  331. package/dist/results-CMgZEANy.js.map +0 -1
  332. package/dist/resume-B9SOSnK9.js.map +0 -1
  333. package/dist/rules-VhftPOOT.js.map +0 -1
  334. package/dist/run-CwIEK16q.js +0 -88
  335. package/dist/run-CwIEK16q.js.map +0 -1
  336. package/dist/runtime-BgQRyCLl.js.map +0 -1
  337. package/dist/server-CHRSgKRp.js +0 -1
  338. package/dist/server-TGW4WyAz.js.map +0 -1
  339. package/dist/shared-BDUzStad.js.map +0 -1
  340. package/dist/source-B1WvBIQt.js +0 -40
  341. package/dist/source-B1WvBIQt.js.map +0 -1
  342. package/dist/source-DAzztZwU.js +0 -1
  343. package/dist/src-1jHbFaGW.js +0 -1
  344. package/dist/src-8zoX_qrb.js.map +0 -1
  345. package/dist/src-C_9wgowC.js +0 -2
  346. package/dist/src-ChhtI6BP.js.map +0 -1
  347. package/dist/src-D005Cbqz.js +0 -2
  348. package/dist/src-DIfRbp5a.js +0 -3
  349. package/dist/src-Dg8121Xy.js +0 -2
  350. package/dist/src-ElBqEN-S.js +0 -2
  351. package/dist/src-NSdQ9zI6.js.map +0 -1
  352. package/dist/src-YHGaUrGE.js +0 -2
  353. package/dist/src-mJVxw1Jm.js +0 -2
  354. package/dist/state-vznMh2wl.js +0 -1
  355. package/dist/suite-BsWGHzlG.js +0 -2
  356. package/dist/suite-WgT7m6Vs.js.map +0 -1
  357. package/dist/summary-DIGEVr08.js.map +0 -1
  358. package/dist/text-CrVy5ioL.js.map +0 -1
  359. package/dist/transcript-BTxJpW4R.js.map +0 -1
  360. package/dist/usage-DF3_p5pE.js +0 -1
  361. package/dist/validate-BhwqrVsm.js.map +0 -1
  362. package/dist/views-BW_C5eVH.js +0 -2
  363. package/dist/worker-entry-Duo9oAeT.js.map +0 -1
  364. package/dist/workspace-QZCacP8A.js +0 -1
@@ -1,36 +1,47 @@
1
- import { t as isObjectLike } from "./object-CLf2-tNq.js";
2
- import { r as withLease } from "./client-D5hNjpEx.js";
1
+ import { t as isObjectLike } from "./object-BkXGPKWY.js";
2
+ import { r as withLease } from "./client-DMmOkbRA.js";
3
3
  import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-B6BSMs03.js";
4
- import { n as dataProblem, p as parseDuration } from "./shared-BDUzStad.js";
5
- import { N as workspaceOf, O as sandboxPaths, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-CO0P0soF.js";
4
+ import { h as parseDuration, n as dataProblem, p as MAX_TIMER_MS } from "./shared-B_lvGqWz.js";
5
+ import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-Cf5HMjib.js";
6
+ import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-ngQEJBjJ.js";
6
7
  import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
7
- import { A as matchesTags, B as mergeJudge, C as brokeredIn, D as expand, Dt as filePartsOf, E as evalPlan, Et as decideState, F as sandboxProvider, I as sandboxSetBy, L as secretVarsIn, N as refuseIllegalCompare, O as harnessId, Ot as resolveJudgeArtifacts, P as sandboxId, Q as createFakeSandbox, R as unbrokered, St as judgeProblem, T as checkOptions, Tt as badJudgeArtifacts, V as resolveStateOptions, W as allowedHosts, X as unenforceable, Y as isDirect, _t as writePluginFiles, ct as agentMcp, dt as claudeMcpServers, gt as refusedPlugins, ht as refusedMcp, mt as layoutOf, pt as expectedBlockedFor, q as egressLabel, ut as brokeringFor, v as gatewayScope, w as cascade, wt as artifactRefsOf, x as resolveGateway, y as resolveModelGateway, z as unbrokeredError } from "./config-BrdHA9bJ.js";
8
- import { c as judgeModelName, d as runJudge, m as isApiError, w as vercelCredentialProblem } from "./judge-yLvjC5A3.js";
9
- import { s as redactSecrets } from "./redact-CRGdfV2s.js";
10
- import { t as inCI } from "./env-BQqj93gu.js";
11
- import { ft as shellQuote, t as MCP_TIMER } from "./src-8zoX_qrb.js";
12
- import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-Dkp_H3pB.js";
8
+ import { $ as egressLabel, A as harnessId, At as NO_JUDGE_INPUT, B as sandboxId, Bt as decideState, D as checkOptions, Dt as writePluginFiles, Et as refusedPlugins, F as plainHeaders, Ft as judgeProblem, G as unbrokeredError, H as sandboxSetBy, Ht as resolveJudgeArtifacts, K as mergeJudge, M as majorityOf, Mt as inputsOn, N as matchesTags, O as evalPlan, P as plainHeaderError, R as refuseIllegalCompare, Rt as artifactRefsOf, S as resolveGateway, St as expectedBlockedFor, T as cascade, Tt as refusedMcp, U as secretVarsIn, V as sandboxProvider, Vt as filePartsOf, W as unbrokered, X as allowedHosts, _ as fellBackToDirect, _t as agentMcp, b as resolveModelGateway, bt as claudeMcpServers, it as createFakeSandbox, k as expand, kt as INPUT_KEYS, nt as unenforceable, q as resolveStateOptions, tt as isDirect, v as gatewayScope, w as brokeredIn, wt as layoutOf, yt as brokeringFor, z as runsOf, zt as badJudgeArtifacts } from "./config-CneBZB84.js";
9
+ import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-Bg4v4e_K.js";
10
+ import { v as vertexSettings } from "./vertex-DypYjx71.js";
11
+ import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets, o as findSecrets } from "./redact-DVIaahLv.js";
12
+ import { a as plural, i as formatDuration, n as clip, s as withoutBom, t as capMiddle } from "./text-DTigcA64.js";
13
+ import { t as inCI } from "./env-CARj5Axq.js";
14
+ import { pt as shellQuote, t as MCP_TIMER, ut as OPAQUE_OUTPUT } from "./src-C0ySiej5.js";
13
15
  import { t as InfraError } from "./types-C5vI3g1C.js";
14
- import { i as formatDuration, n as clip, t as capMiddle } from "./text-CrVy5ioL.js";
15
- import { i as registerTest, n as getRunContext, o as SUBJECT, r as registerGroup, t as collectingFile } from "./runtime-BgQRyCLl.js";
16
- import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-jI0OWxlz.js";
17
- import { t as attemptDebugDir } from "./files-Ca0d6Uq3.js";
18
- import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-CVRwRTnb.js";
16
+ import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-CZVpkr04.js";
17
+ import { c as formatBytes } from "./debug-CgG3T071.js";
18
+ import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CpbZ9arV.js";
19
+ import { t as attemptDebugDir } from "./files-D1y8Pg4Z.js";
20
+ import { r as openRouterFallbacks } from "./openrouter-models-Bo8PZ4Ew.js";
21
+ import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-iQ6kYJV0.js";
19
22
  import { t as contextTaker } from "./context-Dgn48BqT.js";
20
- import { a as skillFiles, i as resolveEnvironment } from "./resolve-CBoKzQMP.js";
23
+ import { a as skillFiles, i as resolveEnvironment } from "./resolve-D_cLa4tW.js";
21
24
  import { t as PrepareError } from "./provider-CmAYTQQs.js";
22
- import { i as preparedWorkspace } from "./workspace-CDI68wtX.js";
23
- import { n as judgeDiff } from "./patch-DWf-I_Q_.js";
24
- import { t as transcriptForJudge } from "./transcript-BTxJpW4R.js";
25
+ import { i as preparedWorkspace } from "./workspace-CBuI9Uds.js";
26
+ import { t as patchPath } from "./patch-CquGAGX5.js";
27
+ import { n as judgeDiff } from "./patch-6T9XAipm.js";
28
+ import { t as transcriptForJudge } from "./transcript-BJrKfgRW.js";
29
+ import { i as writeTar, n as readTar, t as extractTar } from "./tarball-CHcahyNK.js";
30
+ import { i as optionsHash, n as caseHash, t as bodyHash } from "./eval-source-CAE4y2Sr.js";
25
31
  import { dirname, join } from "node:path";
26
32
  import { parse, stringify } from "yaml";
27
33
  import { z } from "zod";
28
34
  import { setTimeout as setTimeout$1 } from "node:timers/promises";
29
- import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
35
+ import { gunzipSync, gzipSync } from "node:zlib";
36
+ import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
30
37
  //#region src/attempt/assertions.ts
31
38
  const MESSAGE_LIMIT = 1024;
32
39
  const LABEL_LIMIT = 60;
33
- const clipped = (s, limit) => s.length > limit ? `${s.slice(0, limit)}…` : s;
40
+ /** Masked first, so a secret the limit would cut through is masked whole. */
41
+ const clipped = (text, limit) => {
42
+ const s = redactSecrets(text);
43
+ return s.length > limit ? `${s.slice(0, limit)}…` : s;
44
+ };
34
45
  const text = (error) => error instanceof Error ? error.message : String(error);
35
46
  const shown = (value) => {
36
47
  if (typeof value === "string") return value;
@@ -44,31 +55,52 @@ const shown = (value) => {
44
55
  const shownSubject = (subject) => typeof subject === "string" && subject ? subject.replace(/\s+/g, " ").trim() : void 0;
45
56
  /**
46
57
  * The records of an attempt's matcher calls, in call order. Without a label, one is
47
- * `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3`.
58
+ * `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3`, skipping
59
+ * any label already recorded, so each record's label is unique (a given `build #2` after two
60
+ * `build`s is `build #2 #2`).
48
61
  */
49
62
  function assertionRecords(raw) {
50
63
  const seen = /* @__PURE__ */ new Map();
64
+ const used = /* @__PURE__ */ new Set();
51
65
  return raw.map((r) => {
52
66
  const named = r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
53
67
  const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
54
- const n = (seen.get(base) ?? 0) + 1;
68
+ let n = seen.get(base) ?? 0;
69
+ let label;
70
+ do {
71
+ n += 1;
72
+ label = n > 1 ? `${base} #${n}` : base;
73
+ } while (used.has(label));
55
74
  seen.set(base, n);
75
+ used.add(label);
56
76
  return {
57
- label: n > 1 ? `${base} #${n}` : base,
77
+ label,
58
78
  labelled: r.label !== void 0,
59
79
  matcher: r.matcher,
60
80
  status: r.status,
61
81
  soft: r.soft,
82
+ ...r.at ? { at: r.at } : {},
62
83
  ...r.status === "pass" ? {} : { message: clipped(text(r.error), MESSAGE_LIMIT) },
63
84
  ...r.durationMs === void 0 ? {} : { durationMs: r.durationMs }
64
85
  };
65
86
  });
66
87
  }
67
- /** What fails the attempt at its end for its `expect.soft` calls: the first that didn't pass. */
68
- function softFailure(raw) {
69
- const failed = raw.find((r) => r.soft && r.status !== "pass");
88
+ const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
89
+ /**
90
+ * What fails the attempt at its end for the checks that failed without ending the body: the first
91
+ * that didn't pass, an `expect` as what it failed with, an `expect.require` the eval caught with
92
+ * `CAUGHT_HINT` after its message.
93
+ */
94
+ function checkFailure(raw) {
95
+ const failed = raw.find((r) => r.status !== "pass");
70
96
  if (!failed) return void 0;
71
- return failed.error instanceof Error ? failed.error : new Error(text(failed.error));
97
+ const error = failed.error instanceof Error ? failed.error : new Error(text(failed.error));
98
+ if (failed.soft) return error;
99
+ const caught = Object.setPrototypeOf(Object.assign(/* @__PURE__ */ new Error(), error), Object.getPrototypeOf(error));
100
+ caught.name = error.name;
101
+ caught.message = `${error.message} ${CAUGHT_HINT}`;
102
+ caught.stack = error.stack?.replace(error.message, caught.message);
103
+ return caught;
72
104
  }
73
105
  //#endregion
74
106
  //#region src/attempt/debug.ts
@@ -120,7 +152,7 @@ function traceEnvironment(debug, spec, env) {
120
152
  const unset = r.unset.length ? ` · unset (optional) ${r.unset.join(", ")}` : "";
121
153
  debug.log("env", `vars ${r.vars.join(", ") || "none"} · secrets ${r.secrets.join(", ") || "none"}${unset}`);
122
154
  for (const s of env.substitutions) debug.log("broker", `${s.name} is a placeholder in the sandbox · swapped in on HTTPS to ${s.hosts.join(", ")} in the ${s.in.join(", ")} · for ${s.phase === "checks" ? "checks and setup only" : "the agent and checks"}`);
123
- const brokering = brokeringFor(spec.harness);
155
+ const brokering = brokeringFor(spec.harness, spec.vertex);
124
156
  if (brokering) {
125
157
  const endpoints = brokering.endpoints.map(endpointLabel).join(", ");
126
158
  debug.log("broker", `${brokering.credential} stays on the host · the gateway swaps it in on ${endpoints}`);
@@ -406,10 +438,10 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
406
438
  }
407
439
  //#endregion
408
440
  //#region src/harness/index.ts
409
- async function createHarness(descriptor) {
441
+ async function createHarness(descriptor, vertex) {
410
442
  if (descriptor.kind === "fake") return createFakeHarness(descriptor);
411
- const { createAgentHarness } = await import("./agent-m-_t-IuW.js");
412
- return createAgentHarness(descriptor);
443
+ const { createAgentHarness } = await import("./agent-BgH4-kFc.js");
444
+ return createAgentHarness(descriptor, vertex);
413
445
  }
414
446
  /** Where each harness's agent looks for repo skills, relative to the workspace. */
415
447
  const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
@@ -428,9 +460,9 @@ async function prepareEnvironment(a, root, runId, where, signal) {
428
460
  });
429
461
  a.env = env;
430
462
  record.environment = env.record;
431
- const { secrets, brokered } = env.record;
463
+ const { vars, secrets, brokered } = env.record;
432
464
  record.env = {
433
- vars: Object.keys(env.vars).filter((n) => !secrets.includes(n)).sort(),
465
+ vars,
434
466
  secrets,
435
467
  brokered
436
468
  };
@@ -438,8 +470,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
438
470
  if (debug) traceEnvironment(debug, spec, env);
439
471
  record.image = null;
440
472
  if (spec.sandbox.kind === "docker") {
441
- const { preparedFor } = await import("./preflight-D9y-HvZQ.js");
442
- const { attemptImage, dockerProvider, preparedImage } = await import("./providers-D1Zxp2he.js");
473
+ const { preparedFor } = await import("./preflight-BYXkweNN.js");
474
+ const { attemptImage, dockerProvider, preparedImage } = await import("./providers-DDNMbv1T.js");
443
475
  signal.throwIfAborted();
444
476
  const provider = dockerProvider({ engine: spec.sandbox.engine ?? "docker" });
445
477
  const builds = builtPlugins(spec.environment.layers);
@@ -472,7 +504,7 @@ const CHARS_PER_TOKEN = 4;
472
504
  const tokensOf = (chars) => Math.ceil(chars / CHARS_PER_TOKEN);
473
505
  const k = (tokens) => tokens < 1e3 ? String(tokens) : `${Math.round(tokens / 1e3)}k`;
474
506
  const diffItems = (patch) => patch.split(/(?=^diff --git )/m).map((part) => ({
475
- name: /^diff --git a\/.+ b\/(.+)$/m.exec(part)?.[1] ?? "",
507
+ name: patchPath(part) ?? "",
476
508
  chars: part.length
477
509
  })).filter((item) => item.name);
478
510
  /** The state's parts by size, each with its biggest files where it has any. */
@@ -508,16 +540,16 @@ function describeSize(state, question) {
508
540
  return `${p.name} ${k(tokensOf(p.chars))}${top.length ? `, of which ${top.join(", ")}` : ""}`;
509
541
  }).join("; ");
510
542
  }
511
- /** `Narrow state.diff, state.transcript or state.output.`: the state's parts, largest first. */
543
+ /** `Narrow diff, transcript or output.`: the inputs that were on, largest first. */
512
544
  function stateHint(state, question) {
513
- const keys = partsOf(state, question).filter((p) => p.name !== "question").map((p) => `state.${p.name}`);
545
+ const keys = partsOf(state, question).filter((p) => p.name !== "question").map((p) => p.name);
514
546
  if (!keys.length) return "";
515
547
  const last = keys.pop();
516
548
  return ` Narrow ${keys.length ? `${keys.join(", ")} or ${last}` : last}.`;
517
549
  }
518
550
  /**
519
551
  * A provider's error, as the provider worded it, then what the judge's input held: its estimated
520
- * size and biggest parts and files, and the `state` keys that were on, largest first. reeval knows
552
+ * size and biggest parts and files, and the inputs that were on, largest first. reeval knows
521
553
  * no model's limit, so this follows every provider error.
522
554
  */
523
555
  const withInputSize = (message, state, question) => `${message}\nThe judge's input was ~${k(inputTokens(state, question))} tokens (estimated): ${describeSize(state, question)}.${stateHint(state, question)}`;
@@ -557,7 +589,7 @@ const workspacePath = (path) => path.replace(/^\.\//, "");
557
589
  /** A `contents` entry that names one file rather than matching many. */
558
590
  const isExactPath = (entry) => !entry.startsWith("!") && !/[*?[\]{}]/.test(entry);
559
591
  /**
560
- * The judge's state for what the `state` option chose: the graded turn's task and output (or every turn's),
592
+ * The judge's state for the inputs the call turned on: the graded turn's task and output (or every turn's),
561
593
  * its changed files and diff as `judgeDiff` shows them, files whole, `c.exec` results and the
562
594
  * transcript.
563
595
  */
@@ -605,7 +637,7 @@ function checksShown(which, ran) {
605
637
  if (which === false) return void 0;
606
638
  if (which !== true) {
607
639
  const missing = which.find((command) => !ran.some((r) => r.command === command));
608
- if (missing !== void 0) throw new Error(`c.judge() state.checks names a command this attempt hasn't run: ${JSON.stringify(missing)}`);
640
+ if (missing !== void 0) throw new Error(`c.judge() checks names a command this attempt hasn't run: ${JSON.stringify(missing)}`);
609
641
  }
610
642
  return ran.filter((r) => which === true || which.includes(r.command)).map(({ command, exitCode, stdout, stderr }) => ({
611
643
  command,
@@ -615,29 +647,38 @@ function checksShown(which, ran) {
615
647
  }));
616
648
  }
617
649
  /**
650
+ * Where `order` goes among `calls`, which are in ascending order: calls finish in any order, but
651
+ * what they record stays in the order they were made.
652
+ */
653
+ function placeFor(calls, order) {
654
+ const after = calls.findIndex((c) => c > order);
655
+ return after < 0 ? calls.length : after;
656
+ }
657
+ /**
618
658
  * Grades a turn (the last by default) or a value against a question, under
619
659
  * the judge provider's concurrency lease. Every call's input is recorded, including calls that
620
660
  * fail. A call that fails returns a `JudgeFailure` and fails the attempt as a judge error after the eval's body, so the judges and
621
661
  * checks after it still run.
622
662
  */
623
663
  function judgeCaller(a, readFile) {
624
- const ask = async (question, { on, ...call } = {}) => {
625
- if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
664
+ const ask = async (order, question, { on, ...call }) => {
626
665
  const problem = judgeProblem(call, { at: "call" });
627
666
  if (problem) throw new Error(problem);
628
667
  const badArts = badJudgeArtifacts(call.artifacts);
629
668
  if (badArts) throw new Error(`c.judge() artifacts ${badArts}`);
630
669
  const subject = on && "turn" in on ? a.turns[on.turn] : a.turns.at(-1);
631
670
  if (!subject) throw new Error(on && "turn" in on ? `c.judge() on names turn ${on.turn}, but this attempt has ${a.turns.length}` : "c.judge() has no turn to grade: call c.send() first");
632
- const options = mergeJudge(a.spec.judge, call);
671
+ const wanted = resolveStateOptions(call);
672
+ if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives a value to grade as the output, but output is off. Pass output: true.");
673
+ const { task: _task, output: _output, files: _files, diff: _diff, contents: _contents, transcript: _transcript, turns: _turns, checks: _checks, artifacts, ...callJudge } = call;
674
+ const options = mergeJudge(a.spec.judge, callJudge);
633
675
  const { model } = options;
634
676
  if (model === void 0) {
635
- a.judgeErrored = true;
636
- throw new Error(noJudgeModel(getRunContext().config.file));
677
+ const message = noJudgeModel(getRunContext().config.file);
678
+ a.judgeThrown.add(message);
679
+ throw new Error(message);
637
680
  }
638
- const wanted = resolveStateOptions(options.state);
639
- if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives the output, but state.output is false");
640
- const { state: _state, maxConcurrent, ...judgeOptions } = options;
681
+ const { maxConcurrent, ...judgeOptions } = options;
641
682
  const output = on && "value" in on ? on.value : subject.text;
642
683
  const { draft, dropped } = await draftState({
643
684
  turns: a.turns,
@@ -649,8 +690,9 @@ function judgeCaller(a, readFile) {
649
690
  listFiles: async () => workspaceFiles(await sandboxFor(a, "c.judge()"), a.signal)
650
691
  });
651
692
  const name = judgeModelName(model);
652
- const state = stateOf(draft);
653
- const resolved = resolveJudgeArtifacts(call.artifacts, a.artifactBlobs);
693
+ const state = redactValue(stateOf(draft));
694
+ const resolved = resolveJudgeArtifacts(artifacts, a.artifactBlobs);
695
+ if (artifacts === true && !resolved.length && inputsOn(call, INPUT_KEYS).length === 1) throw new Error("c.judge() artifacts: true shows the judge nothing yet: this attempt has no captures. Call c.capture() first, or turn on another input.");
654
696
  const refs = artifactRefsOf(resolved);
655
697
  const decide = decideState(state, filePartsOf(resolved));
656
698
  const input = {
@@ -661,7 +703,9 @@ function judgeCaller(a, readFile) {
661
703
  ...refs.length ? { artifacts: refs } : {},
662
704
  ...dropped.length ? { dropped } : {}
663
705
  };
664
- a.record.judgeInputs?.push(input);
706
+ const place = placeFor(a.judgeInputCalls, order);
707
+ a.judgeInputCalls.splice(place, 0, order);
708
+ a.record.judgeInputs?.splice(place, 0, input);
665
709
  const start = performance.now();
666
710
  const ms = () => Math.round(performance.now() - start);
667
711
  const atLeast = options.threshold ?? .8;
@@ -669,12 +713,14 @@ function judgeCaller(a, readFile) {
669
713
  kind: "judge-start",
670
714
  question
671
715
  });
672
- debugLog("judge", `asking ${name}: ${JSON.stringify(clip(question, 160))}`);
716
+ debugLog("judge", `asking ${name}: ${JSON.stringify(clip(redactSecrets(question), 160))}`);
673
717
  if (dropped.length) debugLog("judge", `left out of the judge's input: ${dropped.join(" · ")}`);
674
718
  debugTrace("judge", "prompt", input, "judge prompt");
675
719
  const failed = (message, retries) => {
676
- a.judgeErrored = true;
677
- a.judgeErrors.push(`c.judge(${JSON.stringify(clip(question, 80))}) failed: ${message}`);
720
+ a.judgeErrors.push({
721
+ call: order,
722
+ message: `c.judge(${JSON.stringify(clip(redactSecrets(question), 80))}) failed: ${message}`
723
+ });
678
724
  input.error = message;
679
725
  if (retries?.length) input.retries = retries;
680
726
  debugLog("judge", `${name} · ${formatDuration(ms())} · error: ${message}`);
@@ -703,11 +749,13 @@ function judgeCaller(a, readFile) {
703
749
  const provider = name.split("/")[0];
704
750
  try {
705
751
  const { config, profile } = getRunContext();
706
- const gateway = resolveModelGateway(model, judgeOptions.gateway, gatewayScope(config, profile), "judge model");
752
+ const scope = gatewayScope(config, profile);
753
+ const gateway = resolveModelGateway(model, judgeOptions.gateway, scope, "judge model");
707
754
  const { gateway: _pin, ...unpinned } = judgeOptions;
755
+ const entry = gateway && scope.gateways.find((g) => g.kind === gateway);
708
756
  const routed = gateway ? {
709
757
  ...unpinned,
710
- gateway
758
+ gateway: entry || gateway
711
759
  } : unpinned;
712
760
  const { verdict, retries } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
713
761
  ...routed,
@@ -718,10 +766,12 @@ function judgeCaller(a, readFile) {
718
766
  ...verdict,
719
767
  artifacts: refs
720
768
  } : verdict;
721
- a.record.judgements.push(retries.length ? {
769
+ const { judgements: judgments } = a.record;
770
+ judgments.splice(placeFor(judgments.map((j) => j.call ?? 0), order), 0, {
722
771
  ...recorded,
723
- retries
724
- } : recorded);
772
+ ...retries.length ? { retries } : {},
773
+ call: order
774
+ });
725
775
  const { score } = verdict;
726
776
  debugLog("judge", `${verdict.resolvedModel ?? verdict.model} · ${formatDuration(ms())} · score ${score.toFixed(2)} ${score >= verdict.threshold ? "≥" : "<"} ${verdict.threshold} · ${verdict.verdict}${retries.length ? ` · ${retries.length} retries` : ""} · reason: P(yes) ${score.toFixed(2)} is ${score >= verdict.threshold ? "at or above" : "below"} the ${verdict.threshold} threshold`);
727
777
  debugTrace("judge", "response", {
@@ -738,21 +788,22 @@ function judgeCaller(a, readFile) {
738
788
  });
739
789
  return recorded;
740
790
  } catch (error) {
741
- if (a.signal.aborted) {
742
- a.judgeErrored = true;
743
- throw error;
744
- }
791
+ if (a.signal.aborted) throw error;
745
792
  const { retries } = error;
746
793
  const message = error instanceof Error ? error.message : String(error);
747
794
  return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
748
795
  }
749
796
  };
750
- return async (question, { on, ...options } = {}) => {
797
+ return async (question, given) => {
798
+ if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
799
+ if (given === null || typeof given !== "object" || Array.isArray(given)) throw new Error(judgeProblem(given, { at: "call" }) ?? NO_JUDGE_INPUT);
800
+ const { on, ...options } = given;
801
+ const order = a.judgeCalls++;
751
802
  const call = on === void 0 ? options : {
752
803
  ...options,
753
804
  on: judgeOn(a.turns, on)
754
805
  };
755
- return Object.defineProperty(await ask(question, call), SUBJECT, { value: question });
806
+ return Object.defineProperty(await ask(order, question, call), SUBJECT, { value: question });
756
807
  };
757
808
  }
758
809
  //#endregion
@@ -762,7 +813,7 @@ async function createSandbox(descriptor, options) {
762
813
  switch (descriptor.kind) {
763
814
  case "fake": return await createFakeSandbox(descriptor, options.env);
764
815
  case "docker": {
765
- const { createDockerSandbox } = await import("./docker-DivfZl2u.js");
816
+ const { createDockerSandbox } = await import("./docker-DCAtFMo9.js");
766
817
  return await createDockerSandbox(descriptor, options);
767
818
  }
768
819
  }
@@ -781,7 +832,7 @@ function classify(error, signal) {
781
832
  message: "The eval timed out"
782
833
  } : {
783
834
  kind: "aborted",
784
- message: "The run was cancelled"
835
+ message: "The run was canceled"
785
836
  };
786
837
  if (error instanceof InfraError) return {
787
838
  kind: "infra",
@@ -792,17 +843,39 @@ function classify(error, signal) {
792
843
  message
793
844
  };
794
845
  }
846
+ const messageOf = (error) => error instanceof Error ? error.message : String(error);
847
+ const judged = (r) => /(^|\.)toPass$/.test(r.matcher);
848
+ /**
849
+ * Whether `error` is a judge's: one a judge threw, or what a `toPass()` or `.not.toPass()` call
850
+ * failed with. A failing verdict the eval didn't assert, or asserted with `.not.toPass()`, isn't.
851
+ * By message too, since another language's runner hands back its own copy of the error.
852
+ */
853
+ function isJudges(a, error, raw) {
854
+ const message = messageOf(error);
855
+ if (a.judgeThrown.has(message)) return true;
856
+ return raw.some((r) => r.status !== "pass" && judged(r) && (r.error === error || messageOf(r.error) === message));
857
+ }
858
+ /**
859
+ * The error that fails the attempt: a judge's gives way to the first other check that didn't hold
860
+ * (an `expect`, or an `expect.require` the eval caught), whichever came first, so the attempt
861
+ * fails as that check.
862
+ */
863
+ function attemptError(a, error, raw) {
864
+ if (!isJudges(a, error, raw)) return error;
865
+ const check = raw.find((r) => r.status !== "pass" && !judged(r));
866
+ return check ? checkFailure([check]) : error;
867
+ }
795
868
  /**
796
869
  * Why the attempt failed, into the record: the eval's own timeout, a cancel, an infra error, or an
797
- * assertion, which is the judge's when a judge call failed or gave a failing verdict.
870
+ * assertion, which is the judge's when `error`, from `attemptError`, is a judge's.
798
871
  */
799
- function recordFailure(a, error, timedOut) {
872
+ function recordFailure(a, error, timedOut, raw) {
800
873
  const { record } = a;
801
874
  record.error = timedOut ? {
802
875
  kind: "timeout",
803
876
  message: "The eval timed out"
804
877
  } : classify(error, a.signal);
805
- record.error.message = capMiddle(record.error.message);
878
+ record.error.message = capMiddle(redactSecrets(record.error.message));
806
879
  if (error instanceof PrepareError && !timedOut) {
807
880
  const { class: kind, stage, ref, cause, log, setBy } = error.failure;
808
881
  record.error.prepare = {
@@ -814,8 +887,7 @@ function recordFailure(a, error, timedOut) {
814
887
  setBy: setBy ?? []
815
888
  };
816
889
  }
817
- const judgeFailed = a.judgeErrored || record.judgements.some((j) => j.verdict !== "pass");
818
- if (record.error.kind === "assertion" && judgeFailed) record.error.kind = "judge";
890
+ if (record.error.kind === "assertion" && isJudges(a, error, raw)) record.error.kind = "judge";
819
891
  record.outcome = record.error.kind === "infra" ? "infra_error" : "fail";
820
892
  }
821
893
  /**
@@ -842,11 +914,11 @@ async function recordOom(a, sandbox) {
842
914
  /**
843
915
  * The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
844
916
  * Refused hosts the harness always tries (in a session like this one) are `expectedBlocked`,
845
- * summarised in one `-v` line.
917
+ * summarized in one `-v` line.
846
918
  */
847
919
  function recordEgress(a, sandbox) {
848
920
  const traffic = sandbox?.traffic?.();
849
- const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0);
921
+ const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0, a.spec.vertex);
850
922
  const refused = traffic?.blocked ?? [];
851
923
  const isExpected = (h) => !!expected?.hosts.includes(h.host);
852
924
  const expectedBlocked = refused.filter(isExpected);
@@ -866,6 +938,24 @@ function recordEgress(a, sandbox) {
866
938
  a.record.servedModels = sandbox?.servedModels?.() ?? null;
867
939
  const asked = sandbox?.askedModels?.() ?? [];
868
940
  if (asked.length) a.record.askedModels = asked;
941
+ const upstreams = sandbox?.upstreams?.() ?? [];
942
+ if (upstreams.length) a.record.upstreams = upstreams;
943
+ const cost = sandbox?.reportedCostUsd?.();
944
+ if (cost !== void 0) a.record.reportedCostUsd = cost;
945
+ }
946
+ /**
947
+ * The record as its staging file holds it: redacted, with a secret split across a turn's raw
948
+ * stream parts masked first, since masking each part on its own leaves pieces of it to rebuild.
949
+ */
950
+ function stagedRecordJson(record) {
951
+ const turns = record.turns.map((turn) => turn.raw ? {
952
+ ...turn,
953
+ raw: redactSplitSecrets(turn.raw.map((part) => JSON.stringify(part))).map((line) => JSON.parse(line))
954
+ } : turn);
955
+ return redactJson({
956
+ ...record,
957
+ turns
958
+ }, 2);
869
959
  }
870
960
  /** Writes the attempt's record, redacted, and returns what the runner reports. */
871
961
  async function writeRecord(a, stagingDir, id) {
@@ -874,7 +964,7 @@ async function writeRecord(a, stagingDir, id) {
874
964
  await mkdir(join(stagingDir, "evals"), { recursive: true });
875
965
  await writeCaptureSidecars(stagingDir, recordFileName(attemptKey(id, spec.cell.key, spec.n)), a.artifactBlobs);
876
966
  const tmp = `${recordPath}.${process.pid}.tmp`;
877
- await writeFile(tmp, redactSecrets(JSON.stringify(record, null, 2)));
967
+ await writeFile(tmp, stagedRecordJson(record));
878
968
  await rename(tmp, recordPath);
879
969
  return {
880
970
  outcome: record.outcome,
@@ -886,17 +976,28 @@ async function writeRecord(a, stagingDir, id) {
886
976
  question: clip(j.question, 200)
887
977
  })),
888
978
  turnErrors: turnErrors(record.turns.map((t) => t.transcript)),
979
+ ...failedChecks(record),
889
980
  ...record.error?.prepare ? { prepare: {
890
981
  ref: record.error.prepare.ref,
891
982
  stage: record.error.prepare.stage
892
983
  } } : {}
893
984
  };
894
985
  }
986
+ /** The record's checks that didn't pass, for the terminal; unset when all did. */
987
+ function failedChecks(record) {
988
+ const failed = (record.assertions ?? []).filter((x) => x.status !== "pass");
989
+ if (!failed.length) return {};
990
+ return { failedChecks: failed.map((x) => ({
991
+ label: x.label,
992
+ ...x.at ? { at: x.at } : {},
993
+ message: clip(x.message?.split("\n")[0] ?? "", 500)
994
+ })) };
995
+ }
895
996
  /** `turn N error: <message>` for each turn with an error event. */
896
997
  function turnErrors(transcripts) {
897
998
  return transcripts.flatMap((events, i) => {
898
999
  const found = events.find((e) => e.type === "error");
899
- return found?.type === "error" ? [`turn ${i + 1} error: ${redactSecrets(clip(found.message.trim(), 500))}`] : [];
1000
+ return found?.type === "error" ? [`turn ${i + 1} error: ${clip(redactSecrets(found.message.trim()), 500)}`] : [];
900
1001
  });
901
1002
  }
902
1003
  //#endregion
@@ -928,7 +1029,7 @@ function hookHandlers(plugin, text, hooks) {
928
1029
  const what = `the ${plugin.name} plugin's hooks/hooks.json`;
929
1030
  let raw;
930
1031
  try {
931
- raw = JSON.parse(text);
1032
+ raw = JSON.parse(withoutBom(text));
932
1033
  } catch {
933
1034
  throw new InfraError(`${what} isn't valid JSON`);
934
1035
  }
@@ -989,7 +1090,7 @@ function unpackPlugins(plugins, unpack, hooksFile = null) {
989
1090
  if (!hooks) {
990
1091
  let own = {};
991
1092
  try {
992
- own = hooksFile === null ? {} : JSON.parse(hooksFile);
1093
+ own = hooksFile === null ? {} : JSON.parse(withoutBom(hooksFile));
993
1094
  } catch {
994
1095
  throw new InfraError(`The workspace's ${unpack.hooks.file} isn't valid JSON, so reeval can't add the plugins' hooks to it`);
995
1096
  }
@@ -1108,7 +1209,7 @@ const timedOutServer = (entries) => entries.find((e) => e.timeout)?.server;
1108
1209
  //#region src/attempt/state.ts
1109
1210
  /** Extra time an attempt gets, outside its eval timeout, when its image may need building. */
1110
1211
  const SETUP_BUDGET_MS = 12e5;
1111
- function createAttempt(spec, signal, progress, start) {
1212
+ function createAttempt(spec, signal, progress, start, snapshotDir) {
1112
1213
  const record = {
1113
1214
  startedAt: (/* @__PURE__ */ new Date()).toISOString(),
1114
1215
  outcome: "pass",
@@ -1157,11 +1258,14 @@ function createAttempt(spec, signal, progress, start) {
1157
1258
  }
1158
1259
  },
1159
1260
  signal,
1160
- judgeErrored: false,
1261
+ judgeThrown: /* @__PURE__ */ new Set(),
1161
1262
  judgeErrors: [],
1263
+ judgeCalls: 0,
1264
+ judgeInputCalls: [],
1162
1265
  captureNames: /* @__PURE__ */ new Set(),
1163
1266
  backgrounds: [],
1164
- artifactBlobs: /* @__PURE__ */ new Map()
1267
+ artifactBlobs: /* @__PURE__ */ new Map(),
1268
+ snapshotDir
1165
1269
  };
1166
1270
  return a;
1167
1271
  }
@@ -1206,7 +1310,7 @@ async function runSetup(a, box, env) {
1206
1310
  const steps = env.setup;
1207
1311
  record.setup = [];
1208
1312
  for (const [i, step] of steps.entries()) {
1209
- const what = `setup step ${i + 1}/${steps.length} "${clip(step.run, 80)}" in environment ${spec.environment.name}`;
1313
+ const what = `setup step ${i + 1}/${steps.length} "${clip(redactSecrets(step.run), 80)}" in environment ${spec.environment.name}`;
1210
1314
  const start = performance.now();
1211
1315
  const stepSignal = AbortSignal.any([a.signal, AbortSignal.timeout(step.timeoutMs)]);
1212
1316
  const r = await asCheck(a, () => box.exec(step.run, {
@@ -1226,7 +1330,7 @@ async function runSetup(a, box, env) {
1226
1330
  durationMs,
1227
1331
  output
1228
1332
  });
1229
- debugLog("env", `setup ${i + 1}/${steps.length} $ ${clip(step.run, 120)} → ${r ? `exit ${r.exitCode}` : `timed out after ${formatDuration(step.timeoutMs)}`} · ${formatDuration(durationMs)}${step.cwd ? ` · in ${step.cwd}` : ""}`);
1333
+ debugLog("env", `setup ${i + 1}/${steps.length} $ ${clip(redactSecrets(step.run), 120)} → ${r ? `exit ${r.exitCode}` : `timed out after ${formatDuration(step.timeoutMs)}`} · ${formatDuration(durationMs)}${step.cwd ? ` · in ${step.cwd}` : ""}`);
1230
1334
  debugTrace("env", "setup", {
1231
1335
  run: step.run,
1232
1336
  cwd: step.cwd,
@@ -1239,6 +1343,54 @@ async function runSetup(a, box, env) {
1239
1343
  throw new InfraError(r ? `${what} exited ${r.exitCode} after ${formatDuration(durationMs)} (see setup.log):\n${tailOf(r.stderr || r.stdout)}` : `${what} timed out after ${formatDuration(step.timeoutMs)} (see setup.log)`);
1240
1344
  }
1241
1345
  }
1346
+ /** The `to: 'checks'` secrets the setup steps got as their real value: those without `hosts`. */
1347
+ function unbrokeredChecks(env) {
1348
+ const brokered = new Set(env.substitutions.filter((s) => s.phase === "checks").map((s) => s.name));
1349
+ return Object.keys(env.checkVars).filter((name) => !brokered.has(name));
1350
+ }
1351
+ /** How many of the processes setup left running the record keeps, and how much of each command line. */
1352
+ const SETUP_PROCESSES = 20;
1353
+ const SETUP_COMMAND_CHARS = 200;
1354
+ /**
1355
+ * The processes the setup steps left running with an unbrokered `to: 'checks'` secret in their
1356
+ * environment, by the secret's variable name: a copy under another name or in a process's
1357
+ * arguments isn't found. The listing runs as the agent's user, with no checks secrets of its own,
1358
+ * so it sees the environments the agent could read, and prints names, never values. Docker only:
1359
+ * the fake sandbox runs on the host, with no separation to report on.
1360
+ */
1361
+ async function leftRunning(a, box, env) {
1362
+ const names = unbrokeredChecks(env);
1363
+ if (a.spec.sandbox.kind !== "docker" || !env.setup.length || !names.length) return [];
1364
+ const r = await box.exec([
1365
+ "for t in tr sed sort; do command -v \"$t\" >/dev/null || { echo \"no $t in the image\" >&2; exit 1; }; done",
1366
+ "for p in /proc/[0-9]*; do",
1367
+ " [ \"$p\" = \"/proc/$$\" ] && continue",
1368
+ ` n=$(tr '\\0' '\\n' < "$p/environ" 2>/dev/null | sed -n -E 's/^(${names.join("|")})=.*/\\1/p' | sort -u | tr '\\n' ' ')`,
1369
+ " [ -n \"$n\" ] || continue",
1370
+ ` c=$(tr '\\0\\t\\n' ' ' < "$p/cmdline" 2>/dev/null)`,
1371
+ ` [ -n "$c" ] || c=$(tr '\\t\\n' ' ' < "$p/comm" 2>/dev/null)`,
1372
+ ` printf '%s\\t%s\\n' "$n" "$c"`,
1373
+ "done",
1374
+ "true"
1375
+ ].join("\n"), { signal: a.signal });
1376
+ if (r.exitCode !== 0) {
1377
+ debugLog("env", `couldn't list the processes setup left running: ${tailOf(r.stderr)}`);
1378
+ return [];
1379
+ }
1380
+ const known = new Set(names);
1381
+ const found = r.stdout.split("\n").flatMap((line) => {
1382
+ const tab = line.indexOf(" ");
1383
+ if (tab < 0) return [];
1384
+ const secrets = line.slice(0, tab).split(" ").filter((n) => known.has(n));
1385
+ const command = clip(redactSecrets(line.slice(tab + 1).trim()), SETUP_COMMAND_CHARS);
1386
+ return secrets.length ? [{
1387
+ command,
1388
+ secrets
1389
+ }] : [];
1390
+ });
1391
+ if (found.length > SETUP_PROCESSES) debugLog("env", `setup left ${found.length} processes running with a checks secret; the record keeps the first ${SETUP_PROCESSES}`);
1392
+ return found.slice(0, SETUP_PROCESSES);
1393
+ }
1242
1394
  const WORKSPACE_TAR = ".reeval-workspace.tar";
1243
1395
  /** Unpacks the git workspace's tarball into the (empty) workspace. */
1244
1396
  async function copyWorkspace(box, workspace, signal) {
@@ -1434,8 +1586,8 @@ async function startSession(a, runId) {
1434
1586
  signal: a.signal,
1435
1587
  env: env.vars,
1436
1588
  image: a.image,
1437
- brokering: brokeringFor(spec.harness),
1438
- expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0)?.hosts,
1589
+ brokering: brokeringFor(spec.harness, spec.vertex),
1590
+ expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0, spec.vertex)?.hosts,
1439
1591
  substitutions: env.substitutions,
1440
1592
  egress: spec.egress
1441
1593
  });
@@ -1473,6 +1625,11 @@ async function startSession(a, runId) {
1473
1625
  for (const skill of env.skills) debugLog("env", `skill ${skill.name} → the harness adapter places it (~/${layout.harnessSkillDir}/${skill.name}/ for ${spec.harness.kind === "fake" ? `the fake ${spec.harness.layout ?? "cursor"} layout` : spec.harness.kind})`);
1474
1626
  for (const [name, server] of Object.entries(env.mcp)) debugLog("env", `mcp ${name} (${server.transport}${server.transport === "stdio" ? `: ${server.command} ${server.args.join(" ")}` : `: ${server.url}`}) → the harness adapter writes its MCP config`);
1475
1627
  await runSetup(a, box, env);
1628
+ const running = await leftRunning(a, box, env);
1629
+ if (running.length) {
1630
+ a.record.setupProcesses = running;
1631
+ debugLog("env", `setup left ${running.length} process${running.length === 1 ? "" : "es"} running with ${[...new Set(running.flatMap((p) => p.secrets))].join(", ")} (to: 'checks', no hosts) in the environment: ${running.map((p) => clip(p.command, 80)).join("; ")}`);
1632
+ }
1476
1633
  await snapshot(box, a.signal);
1477
1634
  debugLog("env", "workspace snapshot taken: the diff starts here");
1478
1635
  const timed = await timeMcpServers(box, env.mcp, a.signal);
@@ -1481,7 +1638,7 @@ async function startSession(a, runId) {
1481
1638
  debugLog("env", "stdio MCP servers start through reeval's MCP timer");
1482
1639
  } else if (timed.untimed) debugLog("env", `stdio MCP servers start untimed: ${timed.untimed}`);
1483
1640
  if (layout.pluginPlacement === "namespaced") await writePluginMcp(box, env, plugins, timed.servers);
1484
- const harness = await createHarness(spec.harness);
1641
+ const harness = await createHarness(spec.harness, spec.vertex);
1485
1642
  const harnessStart = performance.now();
1486
1643
  debugLog("harness", `starting ${harnessId(spec.harness)}: bootstrap and session`);
1487
1644
  const session = await startHarness(a, harness, box, {
@@ -1501,6 +1658,237 @@ async function startSession(a, runId) {
1501
1658
  return [box, session];
1502
1659
  }
1503
1660
  //#endregion
1661
+ //#region src/attempt/snapshot.ts
1662
+ /** The most a snapshot directory's files may add up to; a larger one isn't taken. */
1663
+ const SNAPSHOT_DIR_LIMIT = 50 * 2 ** 20;
1664
+ /** The most a compressed snapshot may be. */
1665
+ const SNAPSHOT_TAR_LIMIT = 10 * 2 ** 20;
1666
+ /** The most a compressed snapshot may be uncompressed, which the host holds in memory to mask. */
1667
+ const SNAPSHOT_RAW_LIMIT = 200 * 2 ** 20;
1668
+ /** How long one snapshot may take before the attempt goes on without it. */
1669
+ const SNAPSHOT_TIMEOUT_MS = 12e4;
1670
+ /** Installed dependencies, caches and git's objects: left out by directory name, at any depth. */
1671
+ const EXCLUDED_DIRS = [
1672
+ ".git",
1673
+ "node_modules",
1674
+ ".pnpm-store",
1675
+ ".venv",
1676
+ "venv",
1677
+ ".tox",
1678
+ "__pycache__",
1679
+ ".pytest_cache",
1680
+ ".mypy_cache",
1681
+ ".ruff_cache",
1682
+ ".next",
1683
+ ".nuxt",
1684
+ ".svelte-kit",
1685
+ ".turbo",
1686
+ ".parcel-cache",
1687
+ ".cache",
1688
+ ".dart_tool",
1689
+ ".gradle"
1690
+ ];
1691
+ const LISTED = 50;
1692
+ const SIZE_MARK = "--reeval-size--";
1693
+ const excluded = (path) => path.split("/").some((part) => EXCLUDED_DIRS.includes(part));
1694
+ const utf8 = new TextDecoder("utf-8", { fatal: true });
1695
+ function asText(data) {
1696
+ if (data.subarray(0, 8192).includes(0)) return null;
1697
+ try {
1698
+ return utf8.decode(data);
1699
+ } catch {
1700
+ return null;
1701
+ }
1702
+ }
1703
+ /** The sandbox's archive, with the excluded directories gone and secret-looking values masked. */
1704
+ function cleanArchive(tar) {
1705
+ const entries = [];
1706
+ const masked = [];
1707
+ const dropped = [];
1708
+ const left = /* @__PURE__ */ new Set();
1709
+ for (const e of readTar(tar)) {
1710
+ if (excluded(e.path)) {
1711
+ const parts = e.path.split("/");
1712
+ const at = parts.findIndex((p) => EXCLUDED_DIRS.includes(p));
1713
+ left.add(redactSecrets(`${parts.slice(0, at + 1).join("/")}/`));
1714
+ continue;
1715
+ }
1716
+ if (findSecrets(e.path).length || e.type === "symlink" && findSecrets(e.target).length) {
1717
+ dropped.push(redactSecrets(e.path));
1718
+ continue;
1719
+ }
1720
+ if (e.type !== "file") {
1721
+ entries.push(e);
1722
+ continue;
1723
+ }
1724
+ const text = asText(e.data);
1725
+ if (text === null) {
1726
+ if (findSecrets(e.data.toString("latin1")).length) dropped.push(e.path);
1727
+ else entries.push(e);
1728
+ continue;
1729
+ }
1730
+ const clean = redactSecrets(text);
1731
+ if (clean === text) entries.push(e);
1732
+ else {
1733
+ masked.push(e.path);
1734
+ entries.push({
1735
+ ...e,
1736
+ data: Buffer.from(clean)
1737
+ });
1738
+ }
1739
+ }
1740
+ return {
1741
+ entries,
1742
+ masked,
1743
+ dropped,
1744
+ excluded: [...left]
1745
+ };
1746
+ }
1747
+ const tarCommand = (workdir) => {
1748
+ const flags = EXCLUDED_DIRS.map((d) => `--exclude=${shellQuote(d)}`).join(" ");
1749
+ return `cd ${shellQuote(workdir)} && COPYFILE_DISABLE=1 tar -cf - ${flags} . 2>/dev/null`;
1750
+ };
1751
+ const GZIP = "if command -v gzip >/dev/null 2>&1; then gzip -c; else cat; fi";
1752
+ const NEEDS_TAR = "command -v tar >/dev/null 2>&1 || { echo 'the sandbox has no tar' >&2; exit 127; }";
1753
+ const listed = (paths) => paths.slice(0, LISTED);
1754
+ const LIMITS = {
1755
+ dir: SNAPSHOT_DIR_LIMIT,
1756
+ tar: SNAPSHOT_TAR_LIMIT,
1757
+ raw: SNAPSHOT_RAW_LIMIT
1758
+ };
1759
+ /**
1760
+ * Archives the sandbox's workspace and returns what to record and the masked entries to write,
1761
+ * or why none: over its form's cap, or an archive the sandbox couldn't make.
1762
+ */
1763
+ async function archiveWorkspace(box, signal, compress, limits = LIMITS) {
1764
+ const none = (cause, message, dirs = []) => ({
1765
+ taken: {
1766
+ bytes: 0,
1767
+ files: 0,
1768
+ excluded: listed(dirs),
1769
+ masked: [],
1770
+ dropped: [],
1771
+ skipped: {
1772
+ cause,
1773
+ message
1774
+ }
1775
+ },
1776
+ entries: null
1777
+ });
1778
+ const names = EXCLUDED_DIRS.map((d) => `-name ${shellQuote(d)}`).join(" -o ");
1779
+ const tar = tarCommand(box.workdir);
1780
+ const sized = await box.execOwn(`${NEEDS_TAR}; cd ${shellQuote(box.workdir)} && { find . -type d \\( ${names} \\) -prune -print 2>/dev/null | head -n ${LISTED}; echo ${SIZE_MARK}; ${tar} | wc -c; echo ${SIZE_MARK}; ${tar} | { ${GZIP}; } | wc -c; }`, { signal });
1781
+ const [list = "", raw = "", size = ""] = sized.stdout.split(`${SIZE_MARK}\n`);
1782
+ const dirs = list.split("\n").filter(Boolean).map((p) => redactSecrets(`${p.replace(/^\.\//, "")}/`));
1783
+ const rawBytes = Number(raw.trim());
1784
+ const gzBytes = Number(size.trim());
1785
+ if (sized.exitCode !== 0 || !(rawBytes > 0) || !(gzBytes > 0)) return none("error", `couldn't archive the workspace: ${(sized.stderr || sized.stdout).trim()}`);
1786
+ const without = dirs.length ? dirs.join(", ") : "its dependency and cache directories";
1787
+ const rawLimit = compress ? limits.raw : limits.dir;
1788
+ if (rawBytes > rawLimit) return none("size", `the workspace is ${formatBytes(rawBytes)} without ${without}, over the ${formatBytes(rawLimit)} cap${compress ? "" : ` (compressed, the cap is ${formatBytes(limits.tar)})`}`, dirs);
1789
+ if (compress && gzBytes > limits.tar) return none("size", `the workspace is ${formatBytes(gzBytes)} compressed without ${without}, over the ${formatBytes(limits.tar)} cap`, dirs);
1790
+ const r = await box.execOwn(`${NEEDS_TAR}; echo ${OPAQUE_OUTPUT}; ${tar} | { ${GZIP}; } | base64`, { signal });
1791
+ const encoded = r.stdout.startsWith(`--opaque-output--\n`) ? r.stdout.slice(OPAQUE_OUTPUT.length + 1) : "";
1792
+ const fetched = Buffer.from(encoded, "base64");
1793
+ if (r.exitCode !== 0 || fetched.length === 0) return none("error", `couldn't archive the workspace: ${r.stderr.trim() || "no output"}`);
1794
+ const clean = cleanArchive(fetched[0] === 31 && fetched[1] === 139 ? gunzipSync(fetched, { maxOutputLength: rawLimit + 2 ** 20 }) : fetched);
1795
+ const all = [.../* @__PURE__ */ new Set([...dirs, ...clean.excluded])].sort();
1796
+ return {
1797
+ taken: {
1798
+ bytes: 0,
1799
+ files: clean.entries.filter((e) => e.type !== "dir").length,
1800
+ excluded: listed(all),
1801
+ masked: listed(clean.masked),
1802
+ dropped: listed(clean.dropped),
1803
+ skipped: null
1804
+ },
1805
+ entries: clean.entries
1806
+ };
1807
+ }
1808
+ /** Where an attempt's snapshots wait, beside its staging record, until the run is assembled. */
1809
+ const snapshotSidecar = (stagingDir, key) => join(stagingDir, "evals", recordFileName(key).replace(/\.json$/, ".snapshots"));
1810
+ /**
1811
+ * Takes snapshot `name` of the workspace into the attempt's sidecar and adds it to the record;
1812
+ * never fails the attempt. Adds nothing when the attempt's signal stopped it.
1813
+ */
1814
+ async function takeSnapshot(a, box, name, phase, limits = LIMITS, timeoutMs = SNAPSHOT_TIMEOUT_MS) {
1815
+ const { compress } = a.spec.snapshot;
1816
+ const form = compress ? "tar.gz" : "dir";
1817
+ const dir = a.snapshotDir;
1818
+ const target = join(dir, compress ? `${name}.tar.gz` : name);
1819
+ const timeout = AbortSignal.timeout(timeoutMs);
1820
+ const add = (taken) => {
1821
+ a.record.snapshots ??= [];
1822
+ a.record.snapshots.push({
1823
+ name,
1824
+ form,
1825
+ path: null,
1826
+ ...taken
1827
+ });
1828
+ };
1829
+ const failed = (message) => {
1830
+ debugLog(phase, `no ${name} snapshot: ${message}`);
1831
+ add({
1832
+ bytes: 0,
1833
+ files: 0,
1834
+ excluded: [],
1835
+ masked: [],
1836
+ dropped: [],
1837
+ skipped: {
1838
+ cause: "error",
1839
+ message: `couldn't archive the workspace: ${message}`
1840
+ }
1841
+ });
1842
+ };
1843
+ try {
1844
+ const { taken, entries } = await archiveWorkspace(box, AbortSignal.any([a.signal, timeout]), compress, limits);
1845
+ if (!entries) {
1846
+ debugLog(phase, `no ${name} snapshot: ${taken.skipped?.message}`);
1847
+ add(taken);
1848
+ return;
1849
+ }
1850
+ await rm(target, {
1851
+ recursive: true,
1852
+ force: true
1853
+ });
1854
+ await mkdir(dir, { recursive: true });
1855
+ if (compress) {
1856
+ const archive = gzipSync(writeTar(entries));
1857
+ if (archive.length > limits.tar) {
1858
+ const message = `the workspace is ${formatBytes(archive.length)} compressed, over the ${formatBytes(limits.tar)} cap`;
1859
+ debugLog(phase, `no ${name} snapshot: ${message}`);
1860
+ add({
1861
+ ...taken,
1862
+ files: 0,
1863
+ skipped: {
1864
+ cause: "size",
1865
+ message
1866
+ }
1867
+ });
1868
+ return;
1869
+ }
1870
+ await writeFile(target, archive);
1871
+ taken.bytes = archive.length;
1872
+ } else {
1873
+ const skipped = await extractTar(entries, target);
1874
+ taken.bytes = entries.reduce((n, e) => n + (e.type === "file" ? e.data.length : 0), 0);
1875
+ taken.dropped = listed([...taken.dropped, ...skipped]);
1876
+ }
1877
+ const masked = taken.masked.length ? ` · masked ${taken.masked.length}` : "";
1878
+ const left = taken.excluded.length ? ` · left out ${taken.excluded.join(", ")}` : "";
1879
+ debugLog(phase, `${name} snapshot: ${taken.files} files, ${formatBytes(taken.bytes)} (${form})${masked}${left}`);
1880
+ add(taken);
1881
+ } catch (error) {
1882
+ await rm(target, {
1883
+ recursive: true,
1884
+ force: true
1885
+ }).catch(() => {});
1886
+ if (a.signal.aborted) return;
1887
+ if (timeout.aborted) return failed(`it took over ${Math.round(timeoutMs / 1e3)}s`);
1888
+ failed(error instanceof Error ? error.message : String(error));
1889
+ }
1890
+ }
1891
+ //#endregion
1504
1892
  //#region src/attempt/turns.ts
1505
1893
  /** How long a turn gets to stop once its attempt is aborted, before reeval stops waiting for it. */
1506
1894
  const TURN_STOP_GRACE_MS = 3e3;
@@ -1514,7 +1902,7 @@ function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
1514
1902
  const cutoff = new Promise((resolve) => {
1515
1903
  onAbort = () => {
1516
1904
  timer = setTimeout(() => {
1517
- const message = `The agent didn't stop within ${grace / 1e3}s of being cancelled, so reeval stopped waiting for its turn`;
1905
+ const message = `The agent didn't stop within ${grace / 1e3}s of being canceled, so reeval stopped waiting for its turn`;
1518
1906
  const at = Date.now() - Date.parse(startedAt);
1519
1907
  resolve({
1520
1908
  text: "",
@@ -1610,7 +1998,7 @@ function turnSender(a, takeContext, checks) {
1610
1998
  kind: "turn-start",
1611
1999
  turn: turnNumber
1612
2000
  });
1613
- debugLog("harness", `turn ${turnNumber} started: ${JSON.stringify(clip(prompt, 160))}`);
2001
+ debugLog("harness", `turn ${turnNumber} started: ${JSON.stringify(clip(redactSecrets(prompt), 160))}`);
1614
2002
  debugTrace("harness", "prompt", {
1615
2003
  turn: turnNumber,
1616
2004
  prompt
@@ -1671,10 +2059,9 @@ function turnSender(a, takeContext, checks) {
1671
2059
  ms: result.durationMs,
1672
2060
  tools: count("tool", "web-search", "web-fetch"),
1673
2061
  fileEdits: count("file-edit"),
1674
- ...failure?.type === "error" ? { error: redactSecrets(clip(failure.message.trim(), 300)) } : {}
2062
+ ...failure?.type === "error" ? { error: clip(redactSecrets(failure.message.trim()), 300) } : {}
1675
2063
  });
1676
- const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
1677
- debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(String(failure.message), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
2064
+ debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(redactSecrets(String(failure.message)), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
1678
2065
  for (const s of entry.mcpStartup ?? []) debugLog("harness", `turn ${turnNumber} mcp ${s.server} ${s.ms === null ? "didn't answer initialize" : `answered initialize in ${formatDuration(s.ms)}`}`);
1679
2066
  if (timedOut) {
1680
2067
  const error = new InfraError(timedOut);
@@ -1684,6 +2071,7 @@ function turnSender(a, takeContext, checks) {
1684
2071
  a.signal.throwIfAborted();
1685
2072
  const diff = await a.timed("diffMs", () => diffSinceSnapshot(box, a.signal));
1686
2073
  a.record.diff = diff;
2074
+ if (a.spec.snapshot.when === "turns") await takeSnapshot(a, box, `turn-${turnNumber}`, "harness");
1687
2075
  const data = {
1688
2076
  prompt,
1689
2077
  ...result,
@@ -1727,7 +2115,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
1727
2115
  } finally {
1728
2116
  clock?.resume();
1729
2117
  }
1730
- });
2118
+ }, snapshotSidecar(stagingDir, attemptKey(id, spec.cell.key, spec.n)));
1731
2119
  const destroy = () => void a.sandbox?.destroy();
1732
2120
  testSignal.addEventListener("abort", destroy, { once: true });
1733
2121
  const takeContext = contextTaker(a);
@@ -1768,13 +2156,19 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
1768
2156
  throw a.startFailure ? a.startFailure.error : error;
1769
2157
  });
1770
2158
  a.signal.throwIfAborted();
1771
- const missed = await unawaited() ?? softFailure(assertions()) ?? expectError();
2159
+ const missed = await unawaited() ?? checkFailure(assertions()) ?? expectError();
1772
2160
  if (a.startFailure) throw a.startFailure.error;
1773
2161
  if (missed) throw missed;
1774
- const [only, ...more] = a.judgeErrors;
1775
- if (only) throw new Error(more.length ? `${more.length + 1} judge calls failed:\n${a.judgeErrors.join("\n")}` : only);
1776
- } catch (error) {
1777
- recordFailure(a, error, timedOut());
2162
+ const judgeErrors = a.judgeErrors.toSorted((x, y) => x.call - y.call).map((e) => e.message);
2163
+ const [only, ...more] = judgeErrors;
2164
+ if (only) {
2165
+ const message = more.length ? `${more.length + 1} judge calls failed:\n${judgeErrors.join("\n")}` : only;
2166
+ a.judgeThrown.add(message);
2167
+ throw new Error(message);
2168
+ }
2169
+ } catch (thrown) {
2170
+ const error = attemptError(a, thrown, assertions());
2171
+ recordFailure(a, error, timedOut(), assertions());
1778
2172
  failure = { error: timedOut() ? new TimeoutError(`Eval timed out after ${spec.timeoutMs / 1e3}s`) : error };
1779
2173
  } finally {
1780
2174
  if (!a.signal.aborted) await unawaited().catch(() => {});
@@ -1790,13 +2184,14 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
1790
2184
  if (a.session) debug.log("teardown", "harness session closed");
1791
2185
  recordEgress(a, sandbox);
1792
2186
  oom = a.signal.aborted ? void 0 : await recordOom(a, sandbox);
2187
+ if (sandbox && spec.snapshot.when && !a.signal.aborted) await takeSnapshot(a, sandbox, "final", "teardown");
1793
2188
  await sandbox?.destroy();
1794
2189
  if (sandbox) debug.log("teardown", `sandbox destroyed (${formatDuration(performance.now() - teardown)} for teardown)`);
1795
2190
  const { record } = a;
1796
2191
  const checked = assertionRecords(assertions());
1797
2192
  if (checked.length) record.assertions = checked;
1798
2193
  for (const x of checked) {
1799
- const why = x.message ? ` · ${clip(x.message.split("\n")[0] ?? "", 160)}` : "";
2194
+ const why = x.message ? ` · ${clip(redactSecrets(x.message.split("\n")[0] ?? ""), 160)}` : "";
1800
2195
  debug.log("check", `assert ${x.status === "pass" ? "✓" : "✗"} ${x.label} (${x.matcher}, ${x.status})${why}`);
1801
2196
  }
1802
2197
  const error = record.error ? ` · ${record.error.kind}: ${record.error.message}` : "";
@@ -1811,6 +2206,30 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
1811
2206
  if (failure) throw failure.error;
1812
2207
  }
1813
2208
  //#endregion
2209
+ //#region src/best-of.ts
2210
+ /** The side an attempt that settled counts on: its record's outcome, else the engine's state. */
2211
+ function sideOf({ state, meta }) {
2212
+ const outcome = meta?.outcome ?? (state === "passed" ? "pass" : state === "failed" ? "fail" : null);
2213
+ return outcome === "pass" || outcome === "fail" ? outcome : null;
2214
+ }
2215
+ /** Whether a best-of cell still needs another attempt after these. */
2216
+ function undecided(sides, bestOf) {
2217
+ const majority = majorityOf(bestOf);
2218
+ const pass = sides.filter((s) => s === "pass").length;
2219
+ const fail = sides.filter((s) => s === "fail").length;
2220
+ return pass < majority && fail < majority;
2221
+ }
2222
+ /** One best-of cell: the `ready` and `settled` hooks of its attempt `n`, from 1 to `bestOf`. */
2223
+ function bestOfCell(bestOf) {
2224
+ const sides = [];
2225
+ const settle = [];
2226
+ for (let n = 1; n <= bestOf; n++) sides.push(new Promise((resolve) => settle.push(resolve)));
2227
+ return (n) => ({
2228
+ ...n > majorityOf(bestOf) ? { ready: async () => undecided(await Promise.all(sides.slice(0, n - 1)), bestOf) } : {},
2229
+ settled: (settled) => settle[n - 1]?.(sideOf(settled))
2230
+ });
2231
+ }
2232
+ //#endregion
1814
2233
  //#region src/contract/declaration.ts
1815
2234
  /** A node's options: any object, until the cascade checks it. Its JSON Schema is the type's. */
1816
2235
  const evalOptionsDataSchema = z.record(z.string(), z.unknown());
@@ -1818,39 +2237,45 @@ const declaredModeSchema = z.enum([
1818
2237
  "run",
1819
2238
  "skip",
1820
2239
  "only"
1821
- ]).describe("How a group or eval was declared: `group`/`evaluate`, `.skip` or `.only`.");
2240
+ ]);
1822
2241
  const declaredFamilySchema = z.strictObject({
1823
- name: z.string().describe("The name template passed to `evaluate.each`."),
1824
- case: z.number().int().min(0).describe("The case's index, from 0."),
1825
- cases: z.number().int().min(2).describe("How many cases the list has.")
1826
- }).describe("Where a case of a pooled `evaluate.each` sits in its list.");
2242
+ name: z.string(),
2243
+ case: z.number().int().min(0),
2244
+ cases: z.number().int().min(2)
2245
+ });
1827
2246
  const declaredTodoSchema = z.strictObject({
1828
2247
  type: z.literal("todo"),
1829
- name: z.string().describe("The name passed to `.todo`.")
1830
- }).describe("`evaluate.todo` or `group.todo`.");
2248
+ name: z.string()
2249
+ });
2250
+ const hash = () => z.string().regex(/^[0-9a-f]{64}$/, "must be a sha256 hash: 64 lowercase hex digits");
2251
+ const declaredSourceSchema = z.strictObject({
2252
+ body: hash(),
2253
+ case: hash().optional()
2254
+ });
1831
2255
  const declaredEvalSchema = z.strictObject({
1832
2256
  type: z.literal("eval"),
1833
- name: z.string().describe("The eval's name, as passed to `evaluate` (a case's, for `evaluate.each`)."),
2257
+ name: z.string(),
1834
2258
  mode: declaredModeSchema,
1835
- options: evalOptionsDataSchema.describe("The eval's own options, before the cascade."),
1836
- body: z.string().describe("Names the eval's body to whoever runs it."),
1837
- family: declaredFamilySchema.optional().describe("Set on each case of a pooled `evaluate.each`: its template, index and the list's length.")
1838
- }).describe("An `evaluate`: one eval, planned into a test per attempt.");
2259
+ options: evalOptionsDataSchema,
2260
+ body: z.string(),
2261
+ source: declaredSourceSchema,
2262
+ family: declaredFamilySchema.optional()
2263
+ });
1839
2264
  const declaredGroupSchema = z.strictObject({
1840
2265
  type: z.literal("group"),
1841
- name: z.string().describe("The group's name, as passed to `group`."),
2266
+ name: z.string(),
1842
2267
  mode: declaredModeSchema,
1843
- options: evalOptionsDataSchema.describe("The group's own options, before the cascade."),
2268
+ options: evalOptionsDataSchema,
1844
2269
  get children() {
1845
2270
  return z.array(declaredNodeSchema);
1846
2271
  }
1847
- }).describe("A `group`: options its evals and groups inherit, and its children in declaration order.");
2272
+ });
1848
2273
  const declaredNodeSchema = z.discriminatedUnion("type", [
1849
2274
  declaredEvalSchema,
1850
2275
  declaredGroupSchema,
1851
2276
  declaredTodoSchema
1852
- ]).describe("One node of the tree: a group, an eval or a todo.");
1853
- const declarationTreeSchema = z.strictObject({ children: z.array(declaredNodeSchema) }).describe("An eval file's top-level groups, evals and todos, in declaration order.");
2277
+ ]);
2278
+ const declarationTreeSchema = z.strictObject({ children: z.array(declaredNodeSchema) });
1854
2279
  const NODE_SCHEMAS = {
1855
2280
  eval: declaredEvalSchema,
1856
2281
  group: declaredGroupSchema,
@@ -1866,12 +2291,14 @@ const declarationProblem = (children) => dataProblem(declarationTreeSchema, { ch
1866
2291
  /** Lets the attempt's own eval timer fire before the engine's backstop, so the message and cleanup are its own. */
1867
2292
  const GRACE_MS = 5e3;
1868
2293
  function configLevel(config) {
2294
+ const { runs, bestOf } = config.runs !== void 0 ? runsOf(config.runs) : { runs: void 0 };
1869
2295
  return {
1870
2296
  harness: config.harness,
1871
2297
  sandbox: config.sandbox,
1872
2298
  environment: config.environment,
1873
2299
  compare: void 0,
1874
- runs: config.runs,
2300
+ runs,
2301
+ bestOf,
1875
2302
  gate: config.gate,
1876
2303
  env: config.env,
1877
2304
  egress: config.egress !== void 0 ? { inherited: config.egress } : {},
@@ -1880,19 +2307,17 @@ function configLevel(config) {
1880
2307
  question: config.question,
1881
2308
  questions: [],
1882
2309
  timeout: void 0,
2310
+ snapshot: config.snapshot,
1883
2311
  tags: [],
2312
+ revision: void 0,
1884
2313
  pin: [],
1885
2314
  sources: {}
1886
2315
  };
1887
2316
  }
1888
- /**
1889
- * The tests an eval registers, or `null` when the run's tags leave it out (it isn't counted in
1890
- * `order`). Throws a UsageError for options the plan refuses.
1891
- */
2317
+ /** The tests an eval registers. Throws a UsageError for options the plan refuses. */
1892
2318
  function planEval(declaration, ctx) {
1893
- const { name, mode, groups: enclosing, options: own } = declaration;
1894
- const groups = enclosing.map((g) => g.name);
1895
- const where = `evaluate("${[...groups, name].join(" › ")}")`;
2319
+ const { name, groups: enclosing, options: own } = declaration;
2320
+ const where = `evaluate("${[...enclosing.map((g) => g.name), name].join(" › ")}")`;
1896
2321
  checkOptions(own, where);
1897
2322
  const levels = [...enclosing.map((g) => ({
1898
2323
  options: g.options,
@@ -1904,25 +2329,54 @@ function planEval(declaration, ctx) {
1904
2329
  const resolvedEval = cascade(configLevel(ctx.config), levels, where, {
1905
2330
  profile: ctx.profile,
1906
2331
  runs: ctx.runs,
1907
- skipped: mode === "skip" || mode === "todo"
2332
+ skipped: declaration.mode === "skip" || declaration.mode === "todo",
2333
+ pooled: declaration.family !== void 0
1908
2334
  });
2335
+ const { harnesses } = resolvedEval;
2336
+ if (!matchesTags(resolvedEval.resolved.tags, ctx.tags)) return {
2337
+ tests: null,
2338
+ harnesses
2339
+ };
2340
+ return {
2341
+ tests: planTests(declaration, ctx, where, resolvedEval),
2342
+ harnesses
2343
+ };
2344
+ }
2345
+ function planTests(declaration, ctx, where, resolvedEval) {
2346
+ const { name, mode, groups: enclosing } = declaration;
2347
+ const groups = enclosing.map((g) => g.name);
1909
2348
  const { resolved } = resolvedEval;
1910
- if (!matchesTags(resolved.tags, ctx.tags)) return null;
2349
+ const source = {
2350
+ ...declaration.source,
2351
+ options: optionsHash([...enclosing.map((g) => g.options), declaration.options])
2352
+ };
1911
2353
  const timeoutMs = resolved.timeout === void 0 ? ctx.config.timeoutMs : parseDuration(resolved.timeout);
1912
2354
  const gateways = ctx.profile?.gateway ?? ctx.config.gateway;
1913
2355
  const gatewaySet = ctx.config.gatewaySet || ctx.profile?.gateway !== void 0;
1914
2356
  const routes = /* @__PURE__ */ new Map();
1915
2357
  const gatewayNotes = [];
2358
+ let vertex;
1916
2359
  const routed = resolvedEval.skip ? resolved.harness : resolved.harness.map((harness) => {
1917
2360
  const route = resolveGateway(gateways, harness, where);
1918
2361
  const hid = harnessId(harness);
1919
2362
  if (gatewaySet || harness.gateway !== void 0) {
1920
2363
  routes.set(hid, route);
1921
- gatewayNotes.push(`gateway: ${hid} → ${route.name}`);
2364
+ gatewayNotes.push(`gateway: ${hid} → ${route.name}${route.name === "direct" && route.skipped.length ? " (fallback)" : ""}`);
1922
2365
  }
1923
2366
  if (route.name === "direct") return harness;
1924
- const problem = harness.kind === "fake" ? void 0 : vercelCredentialProblem();
1925
- if (problem) throw new UsageError(`${where} on ${hid}: ${problem}`);
2367
+ if (harness.kind !== "fake") try {
2368
+ if (route.name === "vercel") {
2369
+ const problem = vercelCredentialProblem();
2370
+ if (problem) throw new UsageError(problem);
2371
+ }
2372
+ if (route.name === "vertex") {
2373
+ const entry = gateways.find((g) => g.kind === "vertex");
2374
+ vertex ??= vertexSettings(entry?.kind === "vertex" ? entry : {});
2375
+ }
2376
+ } catch (error) {
2377
+ if (error instanceof UsageError) throw new UsageError(`${where} on ${hid}: ${error.message}`);
2378
+ throw error;
2379
+ }
1926
2380
  return {
1927
2381
  ...harness,
1928
2382
  gateway: route.name
@@ -1931,9 +2385,16 @@ function planEval(declaration, ctx) {
1931
2385
  const plannedAttempts = resolvedEval.skip ? [] : expand({
1932
2386
  ...resolved,
1933
2387
  harness: routed
1934
- });
2388
+ }, vertex);
2389
+ let judge;
1935
2390
  if (plannedAttempts.length) try {
1936
- resolveModelGateway(resolved.judge.model, resolved.judge.gateway, { gateways }, "the judge model");
2391
+ const gateway = resolveModelGateway(resolved.judge.model, resolved.judge.gateway, { gateways }, "the judge model");
2392
+ const { model } = resolved.judge;
2393
+ if (typeof model === "string") judge = {
2394
+ model,
2395
+ ...gateway ? { gateway } : {}
2396
+ };
2397
+ if (gatewaySet && fellBackToDirect(gateway, resolved.judge.gateway, { gateways })) gatewayNotes.push(`gateway: judge ${model} → direct (fallback)`);
1937
2398
  } catch (error) {
1938
2399
  if (error instanceof UsageError) throw new UsageError(`${where}: ${error.message}`);
1939
2400
  throw error;
@@ -1941,6 +2402,8 @@ function planEval(declaration, ctx) {
1941
2402
  if (ctx.config.secrets.unbrokered === "error") {
1942
2403
  const [first] = unbrokered(resolved.env, plannedAttempts);
1943
2404
  if (first) throw new UsageError(unbrokeredError(first, where, ctx.config.file));
2405
+ const [plain] = plainHeaders(plannedAttempts);
2406
+ if (plain) throw new UsageError(plainHeaderError(plain, where, ctx.config.file));
1944
2407
  }
1945
2408
  const pluginNotes = [];
1946
2409
  const pluginLabels = /* @__PURE__ */ new Map();
@@ -1966,6 +2429,7 @@ function planEval(declaration, ctx) {
1966
2429
  configFile: ctx.config.file,
1967
2430
  file: path
1968
2431
  });
2432
+ if (gatewayNotes.length && openRouterFallbacks(gateways)) gatewayNotes.push("gateway: openrouter() fallbacks on: reeval's own calls may fail over to another upstream provider; the harnesses' calls take OpenRouter's own routing");
1969
2433
  plan.notes.push(...gatewayNotes, ...pluginNotes);
1970
2434
  const setByOf = (planned) => ({
1971
2435
  sandbox: sandboxSetBy(planned, resolved, ctx.config.file),
@@ -1978,6 +2442,8 @@ function planEval(declaration, ctx) {
1978
2442
  name,
1979
2443
  groups,
1980
2444
  tags: resolved.tags,
2445
+ ...resolved.revision !== void 0 ? { revision: resolved.revision } : {},
2446
+ source,
1981
2447
  cell: first.cell,
1982
2448
  n: 1,
1983
2449
  label: "",
@@ -2014,6 +2480,8 @@ function planEval(declaration, ctx) {
2014
2480
  name,
2015
2481
  groups,
2016
2482
  tags: resolved.tags,
2483
+ ...resolved.revision !== void 0 ? { revision: resolved.revision } : {},
2484
+ source,
2017
2485
  cell: planned.cell,
2018
2486
  n: planned.n,
2019
2487
  label: planned.label,
@@ -2022,6 +2490,7 @@ function planEval(declaration, ctx) {
2022
2490
  setBy,
2023
2491
  gate: planned.gate,
2024
2492
  ...planned.minLift !== void 0 ? { minLift: planned.minLift } : {},
2493
+ ...planned.bestOf !== void 0 ? { bestOf: planned.bestOf } : {},
2025
2494
  brokered: brokeredIn(resolved.env, planned),
2026
2495
  secretVars: secretVarsIn(resolved.env, planned),
2027
2496
  egress: planned.egress,
@@ -2035,13 +2504,14 @@ function planEval(declaration, ctx) {
2035
2504
  } : plan,
2036
2505
  ...skip ? { skip } : {},
2037
2506
  ...family ? { family } : {},
2038
- ...routes.has(harnessId(planned.harness)) ? { gateway: routes.get(harnessId(planned.harness)) } : {}
2507
+ ...routes.has(harnessId(planned.harness)) ? { gateway: routes.get(harnessId(planned.harness)) } : {},
2508
+ ...judge ? { judge } : {}
2039
2509
  };
2040
2510
  return {
2041
2511
  name: `${name} [${attemptId(planned.cell.key, planned.n)}]`,
2042
2512
  mode: skip ? "skip" : mode,
2043
2513
  attempt,
2044
- timeout: skip ? void 0 : timeoutMs + setupAllowance(planned) + GRACE_MS,
2514
+ timeout: skip ? void 0 : Math.min(timeoutMs + setupAllowance(planned) + GRACE_MS, MAX_TIMER_MS),
2045
2515
  gate: skip ? void 0 : {
2046
2516
  limits: attemptLimits(planned, ctx.concurrency),
2047
2517
  rank: {
@@ -2060,6 +2530,7 @@ function planEval(declaration, ctx) {
2060
2530
  env: resolved.env,
2061
2531
  judge: resolved.judge,
2062
2532
  capture: resolved.capture,
2533
+ snapshot: resolved.snapshot,
2063
2534
  sources: {
2064
2535
  ...resolved.sources,
2065
2536
  sandbox: setBy.sandbox
@@ -2221,16 +2692,54 @@ function registerTodo(name) {
2221
2692
  const tags = [...new Set(groupStack.flatMap((g) => g.options.tags ?? []))];
2222
2693
  if (matchesTags(tags, getRunContext().tags)) registerTest(name, { mode: "todo" }, async () => {});
2223
2694
  }
2224
- /** The evals declared so far, by file and name path, so a second one with the same id is refused. */
2225
- const declared = /* @__PURE__ */ new Set();
2226
- function checkUnique(where, groups, name, hint) {
2227
- const key = `${collectingFile() ?? ""}\0${[...groups, name].join("\0")}`;
2228
- if (declared.has(key)) throw new UsageError(`${where} is declared twice in this file. Each eval needs its own name, or its attempts are merged into one eval.${hint}`);
2229
- declared.add(key);
2695
+ /**
2696
+ * The evals declared so far, by id, with their name paths and whether each is a family's case,
2697
+ * so a second id is refused.
2698
+ */
2699
+ const declared = /* @__PURE__ */ new Map();
2700
+ /** The pooled families declared so far, by id: their name path, length and the cases seen. */
2701
+ const families = /* @__PURE__ */ new Map();
2702
+ /** Two name paths with one id differ only where a name holds the ` › ` the id joins them with. */
2703
+ const separatorHint = (path, earlier) => path.length === earlier.length ? "" : ` " › " joins the group and eval names in an eval's id, so a name containing it can be a grouped eval's id.`;
2704
+ /** A run's facts name an eval and a family alike by id, so an eval can't take a family's. */
2705
+ const namedLikeFamily = (eval_, family) => new UsageError(`evaluate("${eval_.join(" › ")}") has the same id as the pooled evaluate.each("${family.join(" › ")}") in this file: a family's id is its name template. Give the eval or the template another name.${separatorHint(eval_, family)}`);
2706
+ function checkUnique(where, groups, name, hint, inFamily) {
2707
+ const path = [...groups, name];
2708
+ const id = evalId(collectingFile() ?? "", path);
2709
+ const earlier = declared.get(id);
2710
+ if (earlier) throw new UsageError(`${where} is declared twice in this file. Each eval needs its own name, or its attempts are merged into one eval.${hint}${separatorHint(path, earlier.path)}`);
2711
+ const family = inFamily ? void 0 : families.get(id);
2712
+ if (family) throw namedLikeFamily(path, family.path);
2713
+ declared.set(id, {
2714
+ path,
2715
+ inFamily
2716
+ });
2717
+ }
2718
+ /**
2719
+ * A family's cases pool by its id, the groups and the template, so each case may be declared
2720
+ * once and every case must agree on the list's length.
2721
+ */
2722
+ function checkFamily(groups, family) {
2723
+ const path = [...groups, family.name];
2724
+ const id = evalId(collectingFile() ?? "", path);
2725
+ const earlier = families.get(id);
2726
+ if (!earlier) {
2727
+ const plain = declared.get(id);
2728
+ if (plain && !plain.inFamily) throw namedLikeFamily(plain.path, path);
2729
+ families.set(id, {
2730
+ path,
2731
+ cases: family.cases,
2732
+ seen: /* @__PURE__ */ new Set([family.case])
2733
+ });
2734
+ return;
2735
+ }
2736
+ if (earlier.path.length !== path.length || earlier.cases !== family.cases || earlier.seen.has(family.case)) throw new UsageError(`evaluate.each("${path.join(" › ")}") is declared twice in this file. Each pooled evaluate.each needs its own name, or their cases are pooled into one family. Give each its own name template, or pass evaluate.each(cases, { pool: false }) to keep the cases apart.${separatorHint(path, earlier.path)}`);
2737
+ earlier.seen.add(family.case);
2230
2738
  }
2231
- function register(mode, name, own, fn, hint = "", family) {
2739
+ function register(mode, name, own, fn, source, hint = "", family) {
2232
2740
  const groups = groupStack.map((g) => g.name);
2233
- checkUnique(`evaluate("${[...groups, name].join(" › ")}")`, groups, name, hint);
2741
+ if (family) checkFamily(groups, family);
2742
+ checkUnique(`evaluate("${[...groups, name].join(" › ")}")`, groups, name, hint, !!family);
2234
2743
  const body = `b${bodies.size + 1}`;
2235
2744
  bodies.set(body, fn);
2236
2745
  declare({
@@ -2239,9 +2748,10 @@ function register(mode, name, own, fn, hint = "", family) {
2239
2748
  mode,
2240
2749
  options: data(own),
2241
2750
  body,
2751
+ source,
2242
2752
  ...family ? { family } : {}
2243
2753
  });
2244
- const tests = planEval({
2754
+ const { tests, harnesses } = planEval({
2245
2755
  name,
2246
2756
  mode,
2247
2757
  groups: groupStack.map(({ name, options }) => ({
@@ -2249,22 +2759,38 @@ function register(mode, name, own, fn, hint = "", family) {
2249
2759
  options
2250
2760
  })),
2251
2761
  options: own,
2762
+ source,
2252
2763
  ...family ? { family } : {}
2253
2764
  }, {
2254
2765
  ...getRunContext(),
2255
2766
  file: collectingFile(),
2256
2767
  order: registered
2257
2768
  });
2769
+ declareHarnesses(harnesses);
2258
2770
  if (!tests) return;
2259
2771
  registered++;
2260
- for (const { name: testName, spec, ...options } of tests) registerTest(testName, options, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
2772
+ const bestOfCells = /* @__PURE__ */ new Map();
2773
+ for (const { name: testName, spec, ...options } of tests) {
2774
+ const { cell, n, bestOf } = options.attempt;
2775
+ let hooks = {};
2776
+ if (bestOf !== void 0) {
2777
+ const of = bestOfCells.get(cell.key) ?? bestOfCell(bestOf);
2778
+ bestOfCells.set(cell.key, of);
2779
+ hooks = of(n);
2780
+ }
2781
+ registerTest(testName, {
2782
+ ...options,
2783
+ ...hooks
2784
+ }, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
2785
+ }
2261
2786
  }
2262
2787
  /** Evals registered so far in this worker, for the scheduler's tie-break. */
2263
2788
  let registered = 0;
2264
- const defineEval = (mode) => (name, a, b) => {
2789
+ const defineEval = (mode, declared) => (name, a, b) => {
2265
2790
  checkOnly(mode, "evaluate");
2266
2791
  const [own, fn] = splitArgs("evaluate", name, a, b);
2267
- register(mode, name, withPaths(own, `evaluate("${name}")`), fn);
2792
+ const where = `evaluate("${name}")`;
2793
+ register(mode, name, withPaths(own, where), fn, declared ?? { body: bodyHash(fn, where, collectingFile()) });
2268
2794
  };
2269
2795
  /** `$key` is a property of the case, `%s` the case itself, `%#` its index. */
2270
2796
  function formatName(template, item, index) {
@@ -2290,13 +2816,19 @@ const evalEach = (cases, options) => (name, a, b) => {
2290
2816
  const own = withPaths(written, `evaluate.each("${name}")`);
2291
2817
  const hint = ` For evaluate.each, put $key, %s or %# in the name ("${name}") so each case's name differs.`;
2292
2818
  const pooled = pool && cases.length >= 2;
2819
+ const file = collectingFile();
2820
+ const body = bodyHash(fn, `evaluate.each("${name}")`, file);
2293
2821
  cases.forEach((item, i) => {
2294
2822
  const family = pooled ? {
2295
2823
  name,
2296
2824
  case: i,
2297
2825
  cases: cases.length
2298
2826
  } : void 0;
2299
- register("run", formatName(name, item, i), own, (c) => fn(c, item), hint, family);
2827
+ const source = {
2828
+ body,
2829
+ case: caseHash(item, `evaluate.each("${name}") case ${i}`, file)
2830
+ };
2831
+ register("run", formatName(name, item, i), own, (c) => fn(c, item), source, hint, family);
2300
2832
  });
2301
2833
  };
2302
2834
  const evaluate = Object.assign(defineEval("run"), {
@@ -2326,8 +2858,8 @@ function declareNodes(nodes, bodyFor, base) {
2326
2858
  } else if (node.family) {
2327
2859
  const options = optionsFrom(node.options, `evaluate("${node.name}")`, base);
2328
2860
  checkOnly(node.mode, "evaluate");
2329
- register(node.mode, node.name, options, bodyFor(node.body), "", node.family);
2330
- } else defineEval(node.mode)(node.name, optionsFrom(node.options, `evaluate("${node.name}")`, base), bodyFor(node.body));
2861
+ register(node.mode, node.name, options, bodyFor(node.body), node.source, "", node.family);
2862
+ } else defineEval(node.mode, node.source)(node.name, optionsFrom(node.options, `evaluate("${node.name}")`, base), bodyFor(node.body));
2331
2863
  }
2332
2864
  /**
2333
2865
  * Options from their data forms, with host paths absolute against `base`: environments become
@@ -2364,4 +2896,4 @@ function optionsFrom(written, where, base) {
2364
2896
  //#endregion
2365
2897
  export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
2366
2898
 
2367
- //# sourceMappingURL=evaluate-BUmsN7yG.js.map
2899
+ //# sourceMappingURL=evaluate-CUtgyr43.js.map