reeval 0.14.0 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (364) hide show
  1. package/README.md +15 -15
  2. package/THIRD_PARTY_NOTICES.md +69 -69
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-BloGVZ_a.js → adapter-CRKYPkRt.js} +37 -8
  5. package/dist/adapter-CRKYPkRt.js.map +1 -0
  6. package/dist/{agent-m-_t-IuW.js → agent-BgH4-kFc.js} +58 -16
  7. package/dist/agent-BgH4-kFc.js.map +1 -0
  8. package/dist/{app-files-C-jwhRDt.js → app-files-BMVBr079.js} +4 -4
  9. package/dist/app-files-BMVBr079.js.map +1 -0
  10. package/dist/{assemble-BDEpelpi.js → assemble-BVbI6NW-.js} +214 -128
  11. package/dist/assemble-BVbI6NW-.js.map +1 -0
  12. package/dist/assemble-SiHa4__F.js +1 -0
  13. package/dist/{attempt-jI0OWxlz.js → attempt-CpbZ9arV.js} +10 -10
  14. package/dist/{attempt-jI0OWxlz.js.map → attempt-CpbZ9arV.js.map} +1 -1
  15. package/dist/{match-Bz0nSdzN.js → attempt-host-CIxjs2Be.js} +104 -6
  16. package/dist/attempt-host-CIxjs2Be.js.map +1 -0
  17. package/dist/{capture-CVRwRTnb.js → capture-iQ6kYJV0.js} +2360 -1523
  18. package/dist/capture-iQ6kYJV0.js.map +1 -0
  19. package/dist/capture-px7Ey95_.js +1 -0
  20. package/dist/{cli-C_4gaJ50.js → cli-1YQU-66F.js} +153 -180
  21. package/dist/cli-1YQU-66F.js.map +1 -0
  22. package/dist/{cli-spec-CpnbATtI.js → cli-spec-CPWT1mAG.js} +10 -21
  23. package/dist/cli-spec-CPWT1mAG.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-D5hNjpEx.js → client-DMmOkbRA.js} +14 -7
  26. package/dist/client-DMmOkbRA.js.map +1 -0
  27. package/dist/{command-BXLUfw4h.js → command-8_Cg5v6b.js} +35 -22
  28. package/dist/command-8_Cg5v6b.js.map +1 -0
  29. package/dist/{command-I1euLLio.js → command-BlYCAqu-.js} +7 -7
  30. package/dist/{command-I1euLLio.js.map → command-BlYCAqu-.js.map} +1 -1
  31. package/dist/{validate-BhwqrVsm.js → command-C-HeGlap.js} +388 -60
  32. package/dist/command-C-HeGlap.js.map +1 -0
  33. package/dist/{command-Bxdy62dW.js → command-Cr1zjRvl.js} +32 -31
  34. package/dist/command-Cr1zjRvl.js.map +1 -0
  35. package/dist/{commands-p1S4nw_H.js → commands-Bj9NbmsO.js} +1 -1
  36. package/dist/{commands-jzigbNFb.js → commands-BvTNZ2-b.js} +181 -73
  37. package/dist/commands-BvTNZ2-b.js.map +1 -0
  38. package/dist/{config-BrdHA9bJ.js → config-CneBZB84.js} +883 -433
  39. package/dist/config-CneBZB84.js.map +1 -0
  40. package/dist/{contrasts-DRxMQQTC.js → contrasts-lhFsekco.js} +3 -3
  41. package/dist/{contrasts-DRxMQQTC.js.map → contrasts-lhFsekco.js.map} +1 -1
  42. package/dist/cost-C5RBTTJr.js +1 -0
  43. package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
  44. package/dist/cost-ChPa8-Fp.js.map +1 -0
  45. package/dist/db-BnXyamuj.js +2 -0
  46. package/dist/db-Cnh3jIv_.js +1530 -0
  47. package/dist/db-Cnh3jIv_.js.map +1 -0
  48. package/dist/debug-CAx3qcaP.js +1 -0
  49. package/dist/{debug-D24NOQH1.js → debug-CgG3T071.js} +5 -2
  50. package/dist/debug-CgG3T071.js.map +1 -0
  51. package/dist/{docker-BnRlTpZA.js → docker-BihzJwbV.js} +11 -6
  52. package/dist/docker-BihzJwbV.js.map +1 -0
  53. package/dist/{docker-D01slDF3.js → docker-Cqq03H72.js} +26 -8
  54. package/dist/docker-Cqq03H72.js.map +1 -0
  55. package/dist/{docker-DivfZl2u.js → docker-DCAtFMo9.js} +1 -1
  56. package/dist/{doctor-DQ2p0euG.js → doctor-Dl4vMeqt.js} +275 -37
  57. package/dist/doctor-Dl4vMeqt.js.map +1 -0
  58. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  59. package/dist/env-CARj5Axq.js.map +1 -0
  60. package/dist/{environment-CO0P0soF.js → environment-Cf5HMjib.js} +116 -297
  61. package/dist/environment-Cf5HMjib.js.map +1 -0
  62. package/dist/{errors-r025B0Zb.js → errors-CHebohjc.js} +10 -4
  63. package/dist/{errors-r025B0Zb.js.map → errors-CHebohjc.js.map} +1 -1
  64. package/dist/eval-source-CAE4y2Sr.js +146 -0
  65. package/dist/eval-source-CAE4y2Sr.js.map +1 -0
  66. package/dist/{evaluate-BUmsN7yG.js → evaluate-CUtgyr43.js} +682 -150
  67. package/dist/evaluate-CUtgyr43.js.map +1 -0
  68. package/dist/evaluate-D35T51AC.d.ts +45 -0
  69. package/dist/files-BEnUJW1e.js +1 -0
  70. package/dist/{files-B-CJeeEh.js → files-BRViE7m-.js} +200 -87
  71. package/dist/files-BRViE7m-.js.map +1 -0
  72. package/dist/{files-kUCrswI6.js → files-BSqA2sZE.js} +20 -9
  73. package/dist/files-BSqA2sZE.js.map +1 -0
  74. package/dist/files-C3tQtD26.js +1 -0
  75. package/dist/files-CNiJ_fZf.js +37 -0
  76. package/dist/files-CNiJ_fZf.js.map +1 -0
  77. package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
  78. package/dist/files-D1y8Pg4Z.js.map +1 -0
  79. package/dist/files-Dts7EY3H.js +1 -0
  80. package/dist/{pairs-CHfZNRT6.js → gate-BC6SdQZs.js} +58 -4
  81. package/dist/gate-BC6SdQZs.js.map +1 -0
  82. package/dist/gateway.d.ts +22 -9
  83. package/dist/gateway.js +30 -8
  84. package/dist/gateway.js.map +1 -1
  85. package/dist/{git-Bk8DSwHy.js → git-UNxT_3z0.js} +3 -3
  86. package/dist/{git-Bk8DSwHy.js.map → git-UNxT_3z0.js.map} +1 -1
  87. package/dist/harness-3NVyTSGK.js +392 -0
  88. package/dist/harness-3NVyTSGK.js.map +1 -0
  89. package/dist/harness.d.ts +12 -8
  90. package/dist/harness.js +1 -1
  91. package/dist/{images-DzLeFjLx.js → images-8A_LzVfw.js} +8 -8
  92. package/dist/{images-DzLeFjLx.js.map → images-8A_LzVfw.js.map} +1 -1
  93. package/dist/index.d.ts +99 -11
  94. package/dist/index.js +5 -5
  95. package/dist/init-okoALIah.js +1 -0
  96. package/dist/{init-SRalM5xF.js → init-sFBgODhu.js} +10 -11
  97. package/dist/init-sFBgODhu.js.map +1 -0
  98. package/dist/{invocations-DzyKSJBa.js → invocations-C9ejlOYl.js} +10 -6
  99. package/dist/invocations-C9ejlOYl.js.map +1 -0
  100. package/dist/{judge-yLvjC5A3.js → judge-Bg4v4e_K.js} +236 -46
  101. package/dist/judge-Bg4v4e_K.js.map +1 -0
  102. package/dist/judge.d.ts +5 -7
  103. package/dist/judge.js +4 -8
  104. package/dist/judge.js.map +1 -1
  105. package/dist/{live-DYd2QEyR.js → live-DeMvnDD5.js} +3 -3
  106. package/dist/{live-DYd2QEyR.js.map → live-DeMvnDD5.js.map} +1 -1
  107. package/dist/{matchers-D4eUqx4j.js → matchers-BKPuaXlj.js} +14 -13
  108. package/dist/matchers-BKPuaXlj.js.map +1 -0
  109. package/dist/{model-1ZLr9Dcv.js → model-2rBZrdZA.js} +29 -36
  110. package/dist/model-2rBZrdZA.js.map +1 -0
  111. package/dist/{model-B45lDUV1.js → model-3MJtpho8.js} +22 -13
  112. package/dist/model-3MJtpho8.js.map +1 -0
  113. package/dist/{model-CgnOpss4.js → model-Bz7-j71_.js} +245 -68
  114. package/dist/model-Bz7-j71_.js.map +1 -0
  115. package/dist/model-DkPwypN4.js +1 -0
  116. package/dist/model-bREXVXwT.js +1 -0
  117. package/dist/{node-BSq5qHnQ.js → node-fP-H2uxn.js} +72 -50
  118. package/dist/node-fP-H2uxn.js.map +1 -0
  119. package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
  120. package/dist/object-BkXGPKWY.js.map +1 -0
  121. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  122. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  123. package/dist/openrouter-models-Bo8PZ4Ew.js +34 -0
  124. package/dist/openrouter-models-Bo8PZ4Ew.js.map +1 -0
  125. package/dist/{patch-DWf-I_Q_.js → patch-6T9XAipm.js} +7 -9
  126. package/dist/patch-6T9XAipm.js.map +1 -0
  127. package/dist/patch-CquGAGX5.js +76 -0
  128. package/dist/patch-CquGAGX5.js.map +1 -0
  129. package/dist/{plan-BIwUzlB8.js → plan-Dw2BjafJ.js} +79 -35
  130. package/dist/plan-Dw2BjafJ.js.map +1 -0
  131. package/dist/{plugins-Dkp_H3pB.js → plugins-ngQEJBjJ.js} +274 -17
  132. package/dist/plugins-ngQEJBjJ.js.map +1 -0
  133. package/dist/{preflight-D9y-HvZQ.js → preflight-BYXkweNN.js} +1 -1
  134. package/dist/{preflight-DJCD_BDn.js → preflight-CDRpog9o.js} +3 -3
  135. package/dist/{preflight-DJCD_BDn.js.map → preflight-CDRpog9o.js.map} +1 -1
  136. package/dist/{prepare-BJG_pD54.js → prepare-B3btUlT6.js} +2 -2
  137. package/dist/{prepare-CIxCGG0M.js → prepare-PkARUGoF.js} +8 -8
  138. package/dist/{prepare-CIxCGG0M.js.map → prepare-PkARUGoF.js.map} +1 -1
  139. package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
  140. package/dist/prices-QaS-kjSQ.js.map +1 -0
  141. package/dist/{provenance-CfZpFpAh.js → provenance-7Hzg88f4.js} +4 -4
  142. package/dist/{provenance-CfZpFpAh.js.map → provenance-7Hzg88f4.js.map} +1 -1
  143. package/dist/{providers-D1Zxp2he.js → providers-DDNMbv1T.js} +1 -1
  144. package/dist/{providers-DFk_6W4K.js → providers-PKoJtGK0.js} +5 -5
  145. package/dist/{providers-DFk_6W4K.js.map → providers-PKoJtGK0.js.map} +1 -1
  146. package/dist/query-Bt6vLJ3m.js +1 -0
  147. package/dist/{query-CemclveB.js → query-nBXrgEwl.js} +3 -3
  148. package/dist/{query-CemclveB.js.map → query-nBXrgEwl.js.map} +1 -1
  149. package/dist/{questions-bAudIc04.js → questions-CkAwePPR.js} +110 -36
  150. package/dist/questions-CkAwePPR.js.map +1 -0
  151. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  152. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  153. package/dist/redact-DVIaahLv.js +658 -0
  154. package/dist/redact-DVIaahLv.js.map +1 -0
  155. package/dist/remarks-Bp6AwPgb.js +1 -0
  156. package/dist/{rules-VhftPOOT.js → remarks-Bt0zwiQB.js} +330 -66
  157. package/dist/remarks-Bt0zwiQB.js.map +1 -0
  158. package/dist/{header-BLPP6aCU.js → remarks-IFFe_oty.js} +35 -9
  159. package/dist/remarks-IFFe_oty.js.map +1 -0
  160. package/dist/{remarks-mujJv1Ey.js → remarks-eBcrIZRu.js} +10 -5
  161. package/dist/remarks-eBcrIZRu.js.map +1 -0
  162. package/dist/{remote-CKcEKja4.js → remote-DfJYiP-R.js} +39 -99
  163. package/dist/remote-DfJYiP-R.js.map +1 -0
  164. package/dist/resolve-CT4XZIfV.js +1 -0
  165. package/dist/{resolve-CBoKzQMP.js → resolve-D_cLa4tW.js} +46 -30
  166. package/dist/resolve-D_cLa4tW.js.map +1 -0
  167. package/dist/results-DrKJuIQF.js +63 -0
  168. package/dist/results-DrKJuIQF.js.map +1 -0
  169. package/dist/{resume-B9SOSnK9.js → resume-BmIM52MJ.js} +47 -17
  170. package/dist/resume-BmIM52MJ.js.map +1 -0
  171. package/dist/{runtime-BgQRyCLl.js → runtime-CZVpkr04.js} +139 -48
  172. package/dist/runtime-CZVpkr04.js.map +1 -0
  173. package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmi6IQML.js} +2 -2
  174. package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmi6IQML.js.map} +1 -1
  175. package/dist/sandbox.d.ts +1 -1
  176. package/dist/sandbox.js +1 -1
  177. package/dist/server-CKJOnIB0.js +1 -0
  178. package/dist/{server-TGW4WyAz.js → server-DS8BGhGU.js} +75 -33
  179. package/dist/server-DS8BGhGU.js.map +1 -0
  180. package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
  181. package/dist/shared-B_lvGqWz.js.map +1 -0
  182. package/dist/source-CgUqpumI.js +1 -0
  183. package/dist/source-DcXv4de-.js +59 -0
  184. package/dist/source-DcXv4de-.js.map +1 -0
  185. package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
  186. package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
  187. package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
  188. package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
  189. package/dist/{src-DjRI-IeE.js → src-A8HLtO0a.js} +135 -35
  190. package/dist/{src-DjRI-IeE.js.map → src-A8HLtO0a.js.map} +1 -1
  191. package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
  192. package/dist/src-BO8AYqJ5.js.map +1 -0
  193. package/dist/src-BTqJbvj6.js +2 -0
  194. package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
  195. package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BkiComGh.js +2 -0
  197. package/dist/src-BvgeZ4tS.js +2 -0
  198. package/dist/{src-8zoX_qrb.js → src-C0ySiej5.js} +572 -1067
  199. package/dist/src-C0ySiej5.js.map +1 -0
  200. package/dist/src-CgJrz5p0.js +3 -0
  201. package/dist/src-Cp-2Hdib.js +2 -0
  202. package/dist/{src-CyaDNG9r.js → src-DMo2eCOJ.js} +1 -1
  203. package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
  204. package/dist/src-DSZyg0Uq.js.map +1 -0
  205. package/dist/src-Dk2Na0b-.js +2 -0
  206. package/dist/src-OESRtvOi.js +2 -0
  207. package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
  208. package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
  209. package/dist/src-h5uDlk3m.js +1 -0
  210. package/dist/state-C97udoh3.js +1 -0
  211. package/dist/{state-B1t143wG.js → state-C_kdhp5v.js} +15 -15
  212. package/dist/{state-B1t143wG.js.map → state-C_kdhp5v.js.map} +1 -1
  213. package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
  214. package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
  215. package/dist/suite-ByT1lsdD.js +2 -0
  216. package/dist/{suite-WgT7m6Vs.js → suite-DieLXvCT.js} +61 -65
  217. package/dist/suite-DieLXvCT.js.map +1 -0
  218. package/dist/{summary-DIGEVr08.js → summary-CSzyBVsF.js} +113 -277
  219. package/dist/summary-CSzyBVsF.js.map +1 -0
  220. package/dist/tarball-CHcahyNK.js +258 -0
  221. package/dist/tarball-CHcahyNK.js.map +1 -0
  222. package/dist/testing.d.ts +426 -143
  223. package/dist/testing.js +12 -10
  224. package/dist/testing.js.map +1 -1
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
  228. package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
  229. package/dist/{transcript-BTxJpW4R.js → transcript-BJrKfgRW.js} +13 -5
  230. package/dist/transcript-BJrKfgRW.js.map +1 -0
  231. package/dist/types-C5vI3g1C.js.map +1 -1
  232. package/dist/{types-DXBbP9Mk.d.ts → types-DVvW66ci.d.ts} +569 -895
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +548 -121
  238. package/dist/view/index.js +9 -9
  239. package/dist/{views-sXpGE6a2.js → views-B9STsLB4.js} +9 -10
  240. package/dist/{views-sXpGE6a2.js.map → views-B9STsLB4.js.map} +1 -1
  241. package/dist/views-Ca-rTixp.js +2 -0
  242. package/dist/{worker-entry-Duo9oAeT.js → worker-entry-C5bLCv6s.js} +48 -20
  243. package/dist/worker-entry-C5bLCv6s.js.map +1 -0
  244. package/dist/workspace-8uHtBSve.js +1 -0
  245. package/dist/{workspace-CDI68wtX.js → workspace-CBuI9Uds.js} +5 -5
  246. package/dist/{workspace-CDI68wtX.js.map → workspace-CBuI9Uds.js.map} +1 -1
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +5 -1
  249. package/skills/eval-rubric/SKILL.md +46 -104
  250. package/dist/adapter-BloGVZ_a.js.map +0 -1
  251. package/dist/agent-m-_t-IuW.js.map +0 -1
  252. package/dist/app-files-C-jwhRDt.js.map +0 -1
  253. package/dist/assemble-BDEpelpi.js.map +0 -1
  254. package/dist/assemble-C5bfncHU.js +0 -1
  255. package/dist/capture-CVRwRTnb.js.map +0 -1
  256. package/dist/capture-CherbjzB.js +0 -1
  257. package/dist/cli-C_4gaJ50.js.map +0 -1
  258. package/dist/cli-spec-CpnbATtI.js.map +0 -1
  259. package/dist/client-D5hNjpEx.js.map +0 -1
  260. package/dist/command-BWAuwQNk.js +0 -91
  261. package/dist/command-BWAuwQNk.js.map +0 -1
  262. package/dist/command-BXLUfw4h.js.map +0 -1
  263. package/dist/command-Be6BSE2L.js +0 -335
  264. package/dist/command-Be6BSE2L.js.map +0 -1
  265. package/dist/command-Bxdy62dW.js.map +0 -1
  266. package/dist/commands-jzigbNFb.js.map +0 -1
  267. package/dist/config-BrdHA9bJ.js.map +0 -1
  268. package/dist/cost-D4elx933.js.map +0 -1
  269. package/dist/cost-DmnAod9x.js +0 -1
  270. package/dist/db-CAfksinX.js +0 -2
  271. package/dist/db-DO0p7xgP.js +0 -619
  272. package/dist/db-DO0p7xgP.js.map +0 -1
  273. package/dist/debug-D24NOQH1.js.map +0 -1
  274. package/dist/debug-D7bEv_UV.js +0 -1
  275. package/dist/docker-BnRlTpZA.js.map +0 -1
  276. package/dist/docker-D01slDF3.js.map +0 -1
  277. package/dist/doctor-DQ2p0euG.js.map +0 -1
  278. package/dist/env-BQqj93gu.js.map +0 -1
  279. package/dist/environment-CO0P0soF.js.map +0 -1
  280. package/dist/evaluate-BUmsN7yG.js.map +0 -1
  281. package/dist/evaluate-C_sgRA-V.d.ts +0 -134
  282. package/dist/facts-DzOUTreB.js +0 -906
  283. package/dist/facts-DzOUTreB.js.map +0 -1
  284. package/dist/files-B-CJeeEh.js.map +0 -1
  285. package/dist/files-BAt0i00X.js +0 -1
  286. package/dist/files-Bp5j5BqI.js +0 -1
  287. package/dist/files-C2sjmXhk.js +0 -1
  288. package/dist/files-C_gQNVB-.js +0 -65
  289. package/dist/files-C_gQNVB-.js.map +0 -1
  290. package/dist/files-Ca0d6Uq3.js.map +0 -1
  291. package/dist/files-kUCrswI6.js.map +0 -1
  292. package/dist/generate-DZ7vp69F.js +0 -1647
  293. package/dist/generate-DZ7vp69F.js.map +0 -1
  294. package/dist/harness-COVXSwJ6.js +0 -354
  295. package/dist/harness-COVXSwJ6.js.map +0 -1
  296. package/dist/header-BLPP6aCU.js.map +0 -1
  297. package/dist/init-RgCQXZhb.js +0 -1
  298. package/dist/init-SRalM5xF.js.map +0 -1
  299. package/dist/insights-TyAvz9aV.js +0 -68
  300. package/dist/insights-TyAvz9aV.js.map +0 -1
  301. package/dist/invocations-DzyKSJBa.js.map +0 -1
  302. package/dist/judge-yLvjC5A3.js.map +0 -1
  303. package/dist/match-Bz0nSdzN.js.map +0 -1
  304. package/dist/matchers-D4eUqx4j.js.map +0 -1
  305. package/dist/model-1ZLr9Dcv.js.map +0 -1
  306. package/dist/model-B45lDUV1.js.map +0 -1
  307. package/dist/model-BTZq8HIZ.js +0 -1
  308. package/dist/model-CgnOpss4.js.map +0 -1
  309. package/dist/model-Ch8kX87u.js +0 -1
  310. package/dist/node-BSq5qHnQ.js.map +0 -1
  311. package/dist/object-CLf2-tNq.js.map +0 -1
  312. package/dist/opencode-providers-klkqgzvb.js +0 -3276
  313. package/dist/opencode-providers-klkqgzvb.js.map +0 -1
  314. package/dist/pairs-CHfZNRT6.js.map +0 -1
  315. package/dist/patch-DWf-I_Q_.js.map +0 -1
  316. package/dist/plan-BIwUzlB8.js.map +0 -1
  317. package/dist/plugins-Dkp_H3pB.js.map +0 -1
  318. package/dist/prices-BcLEXvdY.js.map +0 -1
  319. package/dist/query-Cvbwl804.js +0 -1
  320. package/dist/questions-bAudIc04.js.map +0 -1
  321. package/dist/redact-CRGdfV2s.js +0 -240
  322. package/dist/redact-CRGdfV2s.js.map +0 -1
  323. package/dist/remarks-CXwSnSjR.js +0 -278
  324. package/dist/remarks-CXwSnSjR.js.map +0 -1
  325. package/dist/remarks-CyAu2DiB.js +0 -1
  326. package/dist/remarks-mujJv1Ey.js.map +0 -1
  327. package/dist/remote-CKcEKja4.js.map +0 -1
  328. package/dist/resolve-B5-uqmJ6.js +0 -1
  329. package/dist/resolve-CBoKzQMP.js.map +0 -1
  330. package/dist/results-CMgZEANy.js +0 -78
  331. package/dist/results-CMgZEANy.js.map +0 -1
  332. package/dist/resume-B9SOSnK9.js.map +0 -1
  333. package/dist/rules-VhftPOOT.js.map +0 -1
  334. package/dist/run-CwIEK16q.js +0 -88
  335. package/dist/run-CwIEK16q.js.map +0 -1
  336. package/dist/runtime-BgQRyCLl.js.map +0 -1
  337. package/dist/server-CHRSgKRp.js +0 -1
  338. package/dist/server-TGW4WyAz.js.map +0 -1
  339. package/dist/shared-BDUzStad.js.map +0 -1
  340. package/dist/source-B1WvBIQt.js +0 -40
  341. package/dist/source-B1WvBIQt.js.map +0 -1
  342. package/dist/source-DAzztZwU.js +0 -1
  343. package/dist/src-1jHbFaGW.js +0 -1
  344. package/dist/src-8zoX_qrb.js.map +0 -1
  345. package/dist/src-C_9wgowC.js +0 -2
  346. package/dist/src-ChhtI6BP.js.map +0 -1
  347. package/dist/src-D005Cbqz.js +0 -2
  348. package/dist/src-DIfRbp5a.js +0 -3
  349. package/dist/src-Dg8121Xy.js +0 -2
  350. package/dist/src-ElBqEN-S.js +0 -2
  351. package/dist/src-NSdQ9zI6.js.map +0 -1
  352. package/dist/src-YHGaUrGE.js +0 -2
  353. package/dist/src-mJVxw1Jm.js +0 -2
  354. package/dist/state-vznMh2wl.js +0 -1
  355. package/dist/suite-BsWGHzlG.js +0 -2
  356. package/dist/suite-WgT7m6Vs.js.map +0 -1
  357. package/dist/summary-DIGEVr08.js.map +0 -1
  358. package/dist/text-CrVy5ioL.js.map +0 -1
  359. package/dist/transcript-BTxJpW4R.js.map +0 -1
  360. package/dist/usage-DF3_p5pE.js +0 -1
  361. package/dist/validate-BhwqrVsm.js.map +0 -1
  362. package/dist/views-BW_C5eVH.js +0 -2
  363. package/dist/worker-entry-Duo9oAeT.js.map +0 -1
  364. package/dist/workspace-QZCacP8A.js +0 -1
@@ -1,26 +1,29 @@
1
- import { n as isOneOf, t as isObjectLike } from "./object-CLf2-tNq.js";
1
+ import { n as isOneOf, t as isObjectLike } from "./object-BkXGPKWY.js";
2
2
  import { n as UsageError } from "./errors-B6BSMs03.js";
3
- import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, i as durationSchema, l as nonEmpty, o as genericProblem, p as parseDuration, s as got, t as closest, u as oneOf } from "./shared-BDUzStad.js";
4
- import { D as sandboxOf, E as sameWorkspaceOnly, F as sandboxDescriptorSchema, I as egressPolicySchema, L as envVarsSchema, M as workspaceLabel, N as workspaceOf, O as sandboxPaths, P as environmentDataSchema, R as gatePolicySchema, V as sandboxProblem, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, j as unbrokeredSecrets, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, w as plannedSecrets, y as isMovingRef, z as gateProblem } from "./environment-CO0P0soF.js";
5
- import { _ as gatewayEntryDataSchema, f as FACTORY, g as gatewayDescriptorSchema, h as GATEWAY_NAMES, m as harnessProblem, p as harnessDescriptorSchema, u as opencodeModelError, v as gatewayNameSchema } from "./harness-COVXSwJ6.js";
3
+ import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, h as parseDuration, i as durationSchema, l as nonEmpty, o as genericProblem, s as got, t as closest, u as oneOf } from "./shared-B_lvGqWz.js";
4
+ import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-Cf5HMjib.js";
5
+ import { B as gatePolicySchema, D as pluginServerName, J as snapshotOptionSchema, L as egressPolicySchema, V as gateProblem, Y as snapshotProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-ngQEJBjJ.js";
6
+ import { _ as gatewayDescriptorSchema, f as FACTORY, g as directOffPin, h as GATEWAY_NAMES, m as harnessProblem, p as harnessDescriptorSchema, u as opencodeModelError, v as gatewayEntryDataSchema, y as gatewayNameSchema } from "./harness-3NVyTSGK.js";
6
7
  import { a as fromCwd, l as sha256 } from "./ids-Bp6ut7ZD.js";
7
- import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-klkqgzvb.js";
8
- import { C as servedOnVercel, S as fxAskedModel, _ as VERCEL_HOST, a as checkModelString, b as VERCEL_ROUTES, g as FX_VERCEL_ENDPOINTS, h as FX_VERCEL_CALL, i as RECOMMENDED_JUDGE_MODEL, n as DIRECT_PROVIDERS, s as judgeCredentials, u as modelKeyOptions, v as VERCEL_KEY, x as VERCEL_UNPROVEN_EFFORT, y as VERCEL_REFUSALS } from "./judge-yLvjC5A3.js";
9
- import { r as badCharacterAt } from "./redact-CRGdfV2s.js";
10
- import { t as inCI } from "./env-BQqj93gu.js";
11
- import { F as DOCKER_EGRESS, I as HARNESS_HOSTS, L as PACKAGE_HOSTS, dt as run, ft as shellQuote } from "./src-8zoX_qrb.js";
8
+ import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-Bz0l7OZI.js";
9
+ import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-Bg4v4e_K.js";
10
+ import { a as VERTEX_ROUTES, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV, t as SANDBOX_ACCESS_TOKEN } from "./vertex-DypYjx71.js";
11
+ import { r as badCharacterAt } from "./redact-DVIaahLv.js";
12
+ import { s as withoutBom } from "./text-DTigcA64.js";
13
+ import { t as inCI } from "./env-CARj5Axq.js";
12
14
  import { u as ownFileURL } from "./root-Dlff-zp8.js";
13
- import { D as pluginServerName, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers } from "./plugins-Dkp_H3pB.js";
15
+ import { F as PACKAGE_HOSTS, N as DOCKER_EGRESS, P as HARNESS_HOSTS, ft as run, pt as shellQuote } from "./src-C0ySiej5.js";
14
16
  import { t as InfraError } from "./types-C5vI3g1C.js";
15
- import { r as pairFactors } from "./pairs-CHfZNRT6.js";
16
- import { a as reevalJiti, n as defaultIncludeGlobs } from "./adapter-BloGVZ_a.js";
17
- import { t as docker } from "./sandbox-BOYZ0hlo.js";
17
+ import { n as gateLabel, u as pairFactors } from "./gate-BC6SdQZs.js";
18
+ import "./stats-hfpuYsAM.js";
19
+ import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-CRKYPkRt.js";
20
+ import { t as docker } from "./sandbox-Cmi6IQML.js";
18
21
  import { existsSync, readFileSync } from "node:fs";
19
22
  import { availableParallelism, tmpdir, totalmem } from "node:os";
20
23
  import { dirname, join, resolve, win32 } from "node:path";
21
24
  import { parseDocument } from "yaml";
22
25
  import { z } from "zod";
23
- import { isIP } from "node:net";
26
+ import { connect, isIP } from "node:net";
24
27
  import { createHash, randomUUID } from "node:crypto";
25
28
  import { spawn } from "node:child_process";
26
29
  import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
@@ -81,48 +84,65 @@ function decideState(json, files) {
81
84
  }
82
85
  //#endregion
83
86
  //#region src/contract/judge.ts
84
- const judgeStateSchema = z.strictObject({
85
- task: z.boolean().optional().describe("The graded turn's prompt. Default true."),
86
- output: z.boolean().optional().describe("The graded turn's final message, or `on`'s value. Default true."),
87
- files: z.boolean().optional().describe("`status path` lines for the changed files, under `diff`'s patterns. Default true."),
88
- diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional().describe("The diff since the eval started. A list is gitignore-style patterns over workspace paths: the last matching pattern wins and `!` excludes. Default true."),
89
- contents: z.array(nonEmpty()).optional().describe("Workspace files shown whole, as they are now, changed or not: exact paths or gitignore-style patterns (`'**'`, `'src/**'`; the last match wins, `!` excludes) over the files git sees in the workspace. A missing exact path is `null`; a pattern that matches nothing adds nothing. These files leave the diff. Default none."),
90
- transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional().describe("The graded turn's transcript as text lines: `true` is messages and tool calls with tool output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. Default false."),
91
- turns: z.enum(["last", "all"]).optional().describe("`'all'`: every turn up to the graded one under `turns`, each with its task, output and (with `transcript`) transcript. Default `'last'`."),
92
- checks: z.union([z.boolean(), z.array(nonEmpty())]).optional().describe("`c.exec` results so far; a list shows only those commands, as passed to `c.exec`. Default false.")
93
- }).describe("What the judge is given: each key is a key of the state in `judge-input.json`. An unset key keeps its default, which is what the judge gets with no `state` option at all.");
87
+ /** The inputs every judge call and question shares; `c.judge()` adds `contents` and its own `artifacts`. */
88
+ const sharedInputFields = {
89
+ task: z.boolean().optional(),
90
+ output: z.boolean().optional(),
91
+ files: z.boolean().optional(),
92
+ diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
93
+ transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
94
+ checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
95
+ };
96
+ const turnsSchema = z.enum(["last", "all"]);
97
+ const { task, output, files, diff, transcript, checks } = sharedInputFields;
98
+ const judgeInputsSchema = z.strictObject({
99
+ task,
100
+ output,
101
+ files,
102
+ diff,
103
+ contents: z.array(nonEmpty()).optional(),
104
+ transcript,
105
+ checks,
106
+ artifacts: z.union([z.literal(true), z.array(z.custom())]).optional()
107
+ });
108
+ const judgeSubjectSchema = z.strictObject({
109
+ on: z.custom().optional(),
110
+ turns: turnsSchema.optional()
111
+ });
112
+ const INPUT_KEYS = Object.keys(judgeInputsSchema.shape);
113
+ /** True when an input's value turns it on: not `false`, unset or an empty list. */
114
+ const inputOn = (value) => value !== void 0 && value !== false && !(Array.isArray(value) && value.length === 0);
115
+ /** The input keys an options object turns on, in `keys`' order. */
116
+ const inputsOn = (value, keys) => keys.filter((k) => inputOn(value[k]));
117
+ /** `a`, `b` and `c` as prose: "a, b or c". */
118
+ const orList = (items) => items.length < 2 ? items[0] ?? "" : `${items.slice(0, -1).join(", ")} or ${items.at(-1)}`;
119
+ const NO_JUDGE_INPUT = `c.judge() shows the judge nothing: turn on at least one input, such as { diff: true } or { contents: ['src/**'] }. The inputs are ${orList(INPUT_KEYS)}.`;
94
120
  const judgeFields = {
95
- threshold: z.number().min(0, "must be between 0 and 1").max(1, "must be between 0 and 1").optional().describe("Minimum score (0–1) for a `pass` verdict."),
96
- providerOptions: z.record(z.string(), z.record(z.string(), jsonValueSchema)).optional().describe("AI SDK provider options for the judge's calls, keyed by provider."),
97
- maxConcurrent: z.number().int().min(1).optional().describe("The most judge calls to this model's provider at once, across the run."),
98
- state: judgeStateSchema.optional().describe("The judge's state: what the judge is given beside the question, not the workspace's or the attempt's state. Merged key by key down the cascade."),
99
- gateway: z.union([gatewayNameSchema, gatewayDescriptorSchema]).optional().describe("The gateway the judge's calls go through, by name or by its helper. It skips the config's `gateway` fallback: the plan refuses a gateway that isn't listed or can't serve the model.")
121
+ threshold: z.number().min(0, "must be between 0 and 1").max(1, "must be between 0 and 1").optional(),
122
+ providerOptions: z.record(z.string(), z.record(z.string(), jsonValueSchema)).optional(),
123
+ maxConcurrent: z.number().int().min(1).optional(),
124
+ gateway: z.union([gatewayNameSchema, gatewayDescriptorSchema]).optional()
100
125
  };
101
126
  const fakeJudgeDataSchema = z.strictObject({
102
127
  kind: z.literal("fake-judge"),
103
- score: z.number().min(0).max(1).describe("The score (0–1) every call gets.")
104
- }).describe("`fakeJudge()` as data: the score every call gets.");
128
+ score: z.number().min(0).max(1)
129
+ });
105
130
  const judgeModelNameSchema = z.string().regex(/\//, "must be a 'provider/model' string, e.g. 'google/gemini-3.8-flash'");
106
131
  const configJudgeDataSchema = z.strictObject({
107
- model: z.union([z.string(), fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge."),
132
+ model: z.union([z.string(), fakeJudgeDataSchema]).optional(),
108
133
  ...judgeFields
109
- }).describe("`ConfigJudgeOptions` as data: the model by name, or a fake judge with a fixed score.");
134
+ });
110
135
  const judgeOptionsSchema = z.strictObject({
111
136
  model: z.custom().optional(),
112
137
  ...judgeFields
113
138
  });
114
- const judgeCallSchema = judgeOptionsSchema.extend({
115
- on: z.custom().optional(),
116
- artifacts: z.union([z.literal(true), z.array(z.custom())]).optional()
117
- });
118
- const STATE_KEYS$1 = Object.keys(judgeStateSchema.shape);
119
- const JUDGE_HINTS = {
120
- see: "state",
121
- shows: "state",
122
- show: "state",
123
- input: "state"
124
- };
125
- const VALUES = {
139
+ const judgeCallSchema = judgeOptionsSchema.extend(judgeInputsSchema.shape).extend(judgeSubjectSchema.shape);
140
+ const CALL_ONLY_KEYS = [
141
+ ...INPUT_KEYS,
142
+ "on",
143
+ "turns"
144
+ ];
145
+ const VALUES$1 = {
126
146
  task: "true or false",
127
147
  output: "true or false",
128
148
  files: "true or false",
@@ -159,7 +179,10 @@ function judgeProblem(judge, site) {
159
179
  return `${key ? `${base}.${key}` : base} in ${site.at === "config" ? site.file : site.where}`;
160
180
  };
161
181
  const loc = site.at === "call" ? " in c.judge()" : site.at === "options" ? ` in ${site.where}` : ` in ${site.path === "judge" ? "" : `${site.path.replace(/\.judge$/, "")} in `}${site.file}`;
162
- if (typeof judge !== "object" || judge === null || Array.isArray(judge)) return `${field()} must be an object such as { threshold: 0.8 } (got ${shown(judge)})`;
182
+ if (typeof judge !== "object" || judge === null || Array.isArray(judge)) {
183
+ const example = site.at === "call" ? "{ diff: true }" : "{ threshold: 0.8 }";
184
+ return `${field()} must be an object such as ${example} (got ${shown(judge)})`;
185
+ }
163
186
  const value = judge;
164
187
  const model = value.model;
165
188
  if (site.at === "config" && typeof model === "string") {
@@ -168,7 +191,7 @@ function judgeProblem(judge, site) {
168
191
  }
169
192
  const schema = site.at === "call" ? judgeCallSchema : site.at === "config" ? configJudgeDataSchema : judgeOptionsSchema;
170
193
  const parsed = schema.safeParse(judge);
171
- if (parsed.success) return void 0;
194
+ if (parsed.success) return site.at === "call" && !inputsOn(value, INPUT_KEYS).length ? NO_JUDGE_INPUT : void 0;
172
195
  const numericFake = site.at === "config" && typeof model === "object" && model?.kind === "fake-judge" && typeof model.score === "number";
173
196
  const issues = parsed.error.issues.filter((i) => !(numericFake && i.path[0] === "model"));
174
197
  if (!issues.length) return void 0;
@@ -191,29 +214,22 @@ function judgeProblem(judge, site) {
191
214
  const name = issues.find((i) => i.code === "unrecognized_keys" && i.path.length === 0)?.keys[0];
192
215
  if (name === void 0) return void 0;
193
216
  const allowed = Object.keys(schema.shape);
194
- const hint = JUDGE_HINTS[name] ?? (STATE_KEYS$1.includes(name) ? `state.${name}` : nearest(name, allowed));
195
- const why = site.at !== "call" && (name === "on" || name === "artifacts") ? ` ${name} is per call: c.judge(question, { ${name} }).` : hint ? ` Did you mean "${hint}"?` : "";
217
+ const hint = nearest(name, allowed);
218
+ const why = site.at !== "call" && CALL_ONLY_KEYS.includes(name) ? ` ${name} is per call: c.judge(question, { ${name} }).` : hint ? ` Did you mean "${hint}"?` : "";
196
219
  return `Unknown judge option "${name}"${loc}.${why}`;
197
220
  };
198
- const stateProblem = () => {
199
- const issue = issues.find((i) => i.path[0] === "state");
221
+ const inputProblem = () => {
222
+ const issue = issues.find((i) => typeof i.path[0] === "string" && i.path[0] in VALUES$1);
200
223
  if (!issue) return void 0;
201
- const { state } = value;
202
- if (typeof state !== "object" || state === null || Array.isArray(state)) return `${field("state")} must be an object such as { transcript: 'tools' } (got ${shown(state)})`;
203
- if (issue.code === "unrecognized_keys" && issue.path.length === 1) {
204
- const key = issue.keys[0] ?? "";
205
- const hint = nearest(key, STATE_KEYS$1);
206
- return `Unknown judge.state key "${key}"${loc}.${hint ? ` Did you mean "${hint}"?` : ""}`;
207
- }
208
- const key = issue.path[1];
209
- return `${field(`state.${String(key)}`)} must be ${VALUES[key]} (got ${shown(state[key])})`;
224
+ const key = issue.path[0];
225
+ return `${field(key)} must be ${VALUES$1[key]} (got ${shown(value[key])})`;
210
226
  };
211
227
  const providerOptionsProblem = () => at("providerOptions") ? `${field("providerOptions")} must map each provider to an object of JSON values, e.g. { google: { thinkingConfig: { thinkingBudget: 0 } } } (got ${shown(value.providerOptions)})` : void 0;
212
228
  const artifactsProblem = () => at("artifacts") ? `c.judge() artifacts ${badJudgeArtifacts(value.artifacts)}` : void 0;
213
229
  const gatewayProblem = () => at("gateway") ? `${field("gateway")} names a gateway in the config's gateway list: ${GATEWAY_NAMES.map((g) => `'${g}'`).join(" or ")}, or is its helper (got ${shown(value.gateway)})` : void 0;
214
- if (site.at === "config") order.push(modelProblem, keyProblem, stateProblem, providerOptionsProblem, thresholdProblem, gatewayProblem);
215
- else if (site.at === "call") order.push(thresholdProblem, maxConcurrentProblem, keyProblem, stateProblem, providerOptionsProblem, artifactsProblem, gatewayProblem);
216
- else order.push(maxConcurrentProblem, thresholdProblem, keyProblem, stateProblem, providerOptionsProblem, gatewayProblem);
230
+ if (site.at === "config") order.push(modelProblem, keyProblem, providerOptionsProblem, thresholdProblem, gatewayProblem);
231
+ else if (site.at === "call") order.push(thresholdProblem, maxConcurrentProblem, keyProblem, inputProblem, providerOptionsProblem, artifactsProblem, gatewayProblem);
232
+ else order.push(maxConcurrentProblem, thresholdProblem, keyProblem, providerOptionsProblem, gatewayProblem);
217
233
  for (const check of order) {
218
234
  const problem = check();
219
235
  if (problem) return problem;
@@ -222,25 +238,18 @@ function judgeProblem(judge, site) {
222
238
  }
223
239
  //#endregion
224
240
  //#region src/contract/questions.ts
225
- const pairFactorSchema = z.enum(["environment", "harness"]).describe("A matrix factor a pairwise question varies; the others are held.");
241
+ const pairFactorSchema = z.enum(["environment", "harness"]);
226
242
  /** Evaluation models score yes/no questions only, so they can't answer a question. */
227
243
  const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
228
- const questionStateSchema = z.strictObject({
229
- task: z.boolean().optional().describe("The prompts the attempt was sent, once for both sides when they match. Default true."),
230
- output: z.boolean().optional().describe("The final message: the agent's own account of its work. Default false in a pair, true per attempt."),
231
- files: z.boolean().optional().describe("`status path` lines for the changed files, under `diff`'s patterns. Default false."),
232
- diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional().describe("The diff since the eval started. A list is gitignore-style patterns over workspace paths: the last matching pattern wins and `!` excludes. Default true."),
233
- transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional().describe("The transcript as text lines: `true` is messages and tool calls with tool output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each harness's tools, so a pair of two harnesses isn't blind. Default false in a pair, true per attempt."),
234
- turns: z.enum(["last", "all"]).optional().describe("`'all'`: every turn under `turns`, each with its task, output and (with `transcript`) transcript; `'last'`: the last turn only. Default `'all'`."),
235
- checks: z.union([z.boolean(), z.array(nonEmpty())]).optional().describe("`c.exec` results; a list shows only those commands, as passed to `c.exec`. Default false.")
236
- }).describe("What a question is shown of each attempt: the keys of `judge.state` but `contents` (the attempts' workspaces are gone when questions are asked). An unset key keeps its default: a pairwise question shows the task and the diff; one asked of each attempt also shows the final message and the transcript.");
237
244
  const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty()).min(1)]);
245
+ const questionInputFields = {
246
+ ...sharedInputFields,
247
+ artifacts: questionArtifactsSchema.optional()
248
+ };
249
+ z.strictObject(questionInputFields);
238
250
  const questionModelSchema = z.string().refine((m) => m.includes("/"), { params: { provider: true } }).refine((m) => !isEvaluationModel(m), { params: { evaluation: true } });
239
- const questionOptionsSchema = z.strictObject({
240
- model: questionModelSchema.optional().describe("`'provider/model'` that answers every question below it without its own `model`."),
241
- state: questionStateSchema.optional().describe("What it's shown of each attempt."),
242
- artifacts: questionArtifactsSchema.optional().describe("Each attempt's own captures, by name; `true` sends all of them. Default none.")
243
- }).describe("Defaults for every question: `model` and `artifacts` replaced, `state` merged key by key.");
251
+ const questionOptionsSchema = z.strictObject({ model: questionModelSchema.optional() });
252
+ const QUESTION_INPUT_KEYS = Object.keys(questionInputFields);
244
253
  /** The first item a list holds twice. */
245
254
  const twice = (items) => items.find((k, i) => items.indexOf(k) !== i);
246
255
  const unique = (list, ctx) => {
@@ -251,35 +260,38 @@ const unique = (list, ctx) => {
251
260
  };
252
261
  const acrossSchema = z.union([pairFactorSchema, z.array(pairFactorSchema).min(1).superRefine(unique)]);
253
262
  const choicesSchema = z.array(nonEmpty()).min(2).superRefine(unique);
254
- const questionObjectSchema = z.strictObject({
263
+ const questionSchema = z.strictObject({
255
264
  question: nonEmpty(),
256
- model: questionModelSchema.optional().describe("`'provider/model'` that answers it, over the cascaded `question.model`."),
257
- across: acrossSchema.optional().describe("The factor it varies when pairing, or both: `'environment'` holds the harness, `'harness'` holds the environment. Unset, it's asked of each attempt on its own."),
258
- choices: choicesSchema.optional().describe("The answers a question asked of each attempt picks from, with a short reason."),
259
- state: questionStateSchema.optional().describe("What it's shown of each attempt, over the cascaded `question.state`, key by key."),
260
- artifacts: questionArtifactsSchema.optional().describe("Each attempt's own captures, by name; `true` sends all of them. Replaces `question.artifacts`.")
265
+ model: questionModelSchema.optional(),
266
+ across: acrossSchema.optional(),
267
+ choices: choicesSchema.optional(),
268
+ turns: turnsSchema.optional(),
269
+ ...questionInputFields
261
270
  }).superRefine((q, ctx) => {
262
271
  if (q.across !== void 0 && q.choices !== void 0) ctx.addIssue({
263
272
  code: "custom",
264
273
  path: ["choices"],
265
274
  message: "with across"
266
275
  });
276
+ if (!inputsOn(q, QUESTION_INPUT_KEYS).length) ctx.addIssue({
277
+ code: "custom",
278
+ message: "no input"
279
+ });
267
280
  });
268
- const questionSchema = z.union([nonEmpty(), questionObjectSchema]).describe("A question a language model answers after the attempts: its text, or the text and its options. With `across`, it picks between two attempts that differ only in that factor, in both orders; without it, it's asked of each attempt on its own. Without a model, the cascaded `question.model` answers, else `analysis.model`, else the judge's when it's a language model. `state` and `artifacts` choose what it's shown of each attempt.");
269
- const questionText = (q) => typeof q === "string" ? q : q.question;
281
+ const questionText = (q) => q.question;
270
282
  const questionsSchema = z.array(questionSchema).superRefine((list, ctx) => {
271
283
  if (twice(list.map(questionText)) !== void 0) ctx.addIssue({
272
284
  code: "custom",
273
285
  message: "twice"
274
286
  });
275
287
  });
276
- const QUESTION_EXAMPLE = `questions: ['How did the agent find the failing test?']`;
288
+ const QUESTION_EXAMPLE = `questions: [{ question: 'How did the agent find the failing test?', transcript: true }]`;
277
289
  const ACROSS_HINT = `'environment' (vary the environment, same harness) or 'harness' (vary the harness, same environment)`;
278
290
  const ACROSS_MEAN = " Did you mean \"across\"? It is the factor a pairwise question varies: across: 'environment'.";
279
- const QUESTION_KEYS = Object.keys(questionObjectSchema.shape);
280
- const QUESTION_SHAPE = "{ question, model?, across?, choices?, state?, artifacts? }";
281
- const STATE_KEYS = Object.keys(questionStateSchema.shape);
282
- const STATE_VALUES = {
291
+ const QUESTION_KEYS = Object.keys(questionSchema.shape);
292
+ const INPUTS = orList(QUESTION_INPUT_KEYS);
293
+ const QUESTION_SHAPE = `{ question, model?, across?, choices?, turns? } with at least one of ${INPUTS}`;
294
+ const VALUES = {
283
295
  task: "true or false",
284
296
  output: "true or false",
285
297
  files: "true or false",
@@ -296,41 +308,31 @@ const describe$2 = (value) => {
296
308
  }
297
309
  return JSON.stringify(value) ?? String(value);
298
310
  };
299
- /**
300
- * What's wrong with a question's `state` or `artifacts`, as one sentence, or undefined. `name`
301
- * turns a key into how the sentence names it: `question.state` on the option,
302
- * `questions "…" state` on an entry.
303
- */
304
- function questionPartsProblem(value, name, where) {
305
- const { state, artifacts } = value;
306
- if (state !== void 0) {
307
- if (!isObjectLike(state) || Array.isArray(state)) return `${name("state")} in ${where} must be an object such as { transcript: 'tools' } (got ${describe$2(state)})`;
308
- for (const [key, v] of Object.entries(state)) {
309
- if (key === "contents") return `${name("state.contents")} in ${where} can't be set: questions are asked after the attempts, when their workspaces are gone. Narrow state.diff instead.`;
310
- const schema = questionStateSchema.shape[key];
311
- if (!schema) {
312
- const hint = closest(key, STATE_KEYS);
313
- return `Unknown ${name("state")} key "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""}`;
314
- }
315
- if (!schema.safeParse(v).success) return `${name(`state.${key}`)} in ${where} must be ${STATE_VALUES[key]} (got ${describe$2(v)})`;
311
+ /** What's wrong with an entry's inputs and `turns`, as one sentence, or undefined. */
312
+ function questionInputsProblem(value, at, where) {
313
+ for (const [key, v] of Object.entries(value)) {
314
+ if (key === "artifacts") {
315
+ if (!questionArtifactsSchema.safeParse(v).success) return `${at} artifacts in ${where} must be true or a non-empty list of capture names (got ${describe$2(v)})`;
316
+ continue;
316
317
  }
318
+ if (!(key in VALUES)) continue;
319
+ if (!(key === "turns" ? turnsSchema : sharedInputFields[key]).safeParse(v).success) return `${at} ${key} in ${where} must be ${VALUES[key]} (got ${describe$2(v)})`;
317
320
  }
318
- if (artifacts !== void 0 && !questionArtifactsSchema.safeParse(artifacts).success) return `${name("artifacts")} in ${where} must be true or a non-empty list of capture names (got ${describe$2(artifacts)})`;
321
+ if (!inputsOn(value, QUESTION_INPUT_KEYS).length) return `${at} in ${where} shows the model nothing: turn on at least one of ${INPUTS}, e.g. { question: '…', diff: true }.`;
319
322
  }
320
323
  /** What's wrong with the `question` option set in `where`, as one sentence, or undefined. */
321
324
  function questionOptionsProblem(value, where) {
322
- if (!isObjectLike(value) || Array.isArray(value)) return `question in ${where} is { model?, state?, artifacts? } (got ${describe$2(value)}), e.g. question: { model: 'google/gemini-3.8-flash' }`;
325
+ if (!isObjectLike(value) || Array.isArray(value)) return `question in ${where} is { model? } (got ${describe$2(value)}), e.g. question: { model: 'google/gemini-3.8-flash' }`;
323
326
  const keys = Object.keys(questionOptionsSchema.shape);
324
327
  for (const key of Object.keys(value)) {
325
328
  if (keys.includes(key)) continue;
326
- if (key === "across" || key === "choices") return `Unknown question option "${key}" in ${where}. ${key} goes on each questions entry, { question, ${key} }: it says what kind of question that entry is.`;
329
+ if (key === "across" || key === "choices" || key === "turns" || key in questionInputFields) return `Unknown question option "${key}" in ${where}. ${key} goes on each questions entry, { question, ${key} }: what a question is and what it's shown are its own.`;
327
330
  const hint = closest(key, keys);
328
- return `Unknown question option "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""} question takes model, state and artifacts.`;
331
+ return `Unknown question option "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""} question takes model.`;
329
332
  }
330
333
  const { model } = value;
331
334
  if (model !== void 0 && typeof model === "string" && isEvaluationModel(model)) return `question.model in ${where} names ${model}, an evaluation model: it scores yes/no questions and can't answer a question. Use a language model, e.g. 'google/gemini-3.8-flash'.`;
332
335
  if (model !== void 0 && !questionModelSchema.safeParse(model).success) return `question.model in ${where} is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${describe$2(model)})`;
333
- return questionPartsProblem(value, (key) => `question.${key}`, where);
334
336
  }
335
337
  /** What's wrong with an entry's `across`, or undefined. */
336
338
  function acrossProblem(question, across, where) {
@@ -349,11 +351,11 @@ function choicesProblem(question, choices, where) {
349
351
  if (blank !== void 0 && blank !== false) return `${at} has an answer that isn't a non-empty string (got ${describe$2(blank)}), e.g. choices: ['the test', 'the code']`;
350
352
  return `${at} is a list of at least two answers, e.g. choices: ['the test', 'the code'] (got ${describe$2(choices)})`;
351
353
  }
352
- /** One question's sentence: the union refuses it whole, so the branch it meant says why. */
354
+ /** One question's sentence. */
353
355
  function questionProblem(item, where) {
354
- if (typeof item === "string") return `questions in ${where} has an empty string. Ask something, e.g. ${QUESTION_EXAMPLE}.`;
355
- if (typeof item !== "object" || item === null || Array.isArray(item)) return `questions in ${where} takes strings or ${QUESTION_SHAPE} (got ${describe$2(item)}), e.g. ${QUESTION_EXAMPLE}.`;
356
+ if (typeof item !== "object" || item === null || Array.isArray(item)) return `questions in ${where} takes ${QUESTION_SHAPE} (got ${describe$2(item)}), e.g. ${QUESTION_EXAMPLE}.`;
356
357
  const unknown = Object.keys(item).filter((k) => !QUESTION_KEYS.includes(k));
358
+ if (unknown.includes("contents")) return `questions in ${where} can't set contents: questions are asked after the attempts, when their workspaces are gone. Narrow diff instead.`;
357
359
  if (unknown.length) {
358
360
  const hint = unknown.some((k) => k === "cut" || k === "cuts") ? ACROSS_MEAN : "";
359
361
  return `questions in ${where} takes ${QUESTION_SHAPE} (got ${unknown.map((k) => `"${k}"`).join(", ")}).${hint}`;
@@ -370,7 +372,7 @@ function questionProblem(item, where) {
370
372
  if (problem) return problem;
371
373
  }
372
374
  if (across !== void 0 && choices !== void 0) return `questions "${text}" in ${where} sets both across and choices. choices are the answers a question asked of each attempt picks from, and across makes it pick between two attempts instead: drop choices to vote between pairs, or drop across to ask each attempt.`;
373
- const parts = questionPartsProblem(item, (key) => `questions "${text}" ${key}`, where);
375
+ const parts = questionInputsProblem(item, `questions "${text}"`, where);
374
376
  if (parts) return parts;
375
377
  if (typeof model === "string" && isEvaluationModel(model)) return `questions "${text}" in ${where} names ${model}, an evaluation model: it scores yes/no questions and can't answer a question. Use a language model, e.g. 'google/gemini-3.8-flash'.`;
376
378
  return `questions "${text}" in ${where}: model is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${describe$2(model)})`;
@@ -386,28 +388,30 @@ function questionsProblem(value, where) {
386
388
  //#endregion
387
389
  //#region src/contract/config.ts
388
390
  const count = () => z.number().int().min(1);
391
+ /** `runs`: a count, or `{ bestOf }` with an odd count of 3 or more. */
392
+ const runs = () => z.union([count(), z.strictObject({ bestOf: z.number().int().min(3).refine((n) => n % 2 === 1, "must be odd") })]);
389
393
  const captureWindowSchema = z.strictObject({
390
- width: count().describe("In CSS pixels."),
391
- height: count().describe("In CSS pixels.")
392
- }).describe("Live tab size for `c.capture()`. Default 1280×720 when the call and cascade omit `viewport`.");
394
+ width: count(),
395
+ height: count()
396
+ });
393
397
  const captureOptionsSchema = z.strictObject({
394
- viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional().describe("The window and saved PNG size, or `'full-page'`. Default 1280×720."),
395
- timeout: durationSchema.optional().describe("How long to poll before failing. Default `'60s'`."),
396
- browser: z.boolean().optional().describe("`false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws."),
397
- model: z.string().includes("/").optional().describe("Vision model for `instructions`. Cascades like `question`. Default is `analysis.model`, else the attempt's judge model, whichever is first a language model; never Jev.")
398
- }).describe("Camera defaults for `c.capture()`. Cascades config → profile defaults → group → eval, shallow-merged. `viewport` on a level replaces the whole value. `port` is not an option: pass it as the first argument to `c.capture`.");
398
+ viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
399
+ timeout: durationSchema.optional(),
400
+ browser: z.boolean().optional(),
401
+ model: z.string().includes("/").optional()
402
+ });
399
403
  const harnessList = z.union([harnessDescriptorSchema, z.array(harnessDescriptorSchema).min(1)]);
400
404
  const budgetSchema = z.strictObject({
401
- attempts: count().optional().describe("The most attempts the plan may have."),
402
- duration: durationSchema.optional().describe("How long the run may go on, from the first attempt's start.")
403
- }).describe("Run-level caps. `attempts` is checked against the plan; `duration` while running.");
405
+ attempts: count().optional(),
406
+ duration: durationSchema.optional()
407
+ });
404
408
  const historyConfigSchema = z.strictObject({
405
- dir: z.string().optional().describe("Committed history folder (default: `.history` in the directory the evals live in)."),
409
+ dir: z.string().optional(),
406
410
  artifacts: z.enum([
407
411
  "all",
408
412
  "failures",
409
413
  "none"
410
- ]).optional().describe("Artifacts recorded next to each run: every eval's, failing evals' only, or none.")
414
+ ]).optional()
411
415
  });
412
416
  const recordMode = z.enum([
413
417
  "manual",
@@ -415,32 +419,33 @@ const recordMode = z.enum([
415
419
  "always"
416
420
  ]);
417
421
  const profileDefaultsDataSchema = z.strictObject({
418
- harness: harnessList.optional().describe("The agents every eval runs on, unless a group or eval replaces them."),
419
- sandbox: sandboxDescriptorSchema.optional().describe("The default machine when the resolved environment omits `sandbox`."),
420
- environment: environmentDataSchema.optional().describe("What every eval runs in when it names no environment."),
421
- runs: count().optional().describe("Attempts per cell."),
422
- gate: gatePolicySchema.optional().describe("How each cell's attempts decide its outcome."),
423
- env: envVarsSchema.optional().describe("Variables and secrets for the sandbox, merged by name over the config's."),
424
- egress: egressPolicySchema.optional().describe("Where the sandbox may connect; replaces the config's policy."),
425
- judge: configJudgeDataSchema.optional().describe("Defaults for `c.judge()`, merged field by field over the config's."),
426
- timeout: durationSchema.optional().describe("Per attempt."),
427
- capture: captureOptionsSchema.optional().describe("Screenshot defaults for `c.capture()`, shallow-merged over the config's."),
428
- question: questionOptionsSchema.optional().describe("Defaults for every question, merged over the config's.")
429
- }).describe("`ProfileDefaults` as data: the environment without `extend`, and the judge as data.");
422
+ harness: harnessList.optional(),
423
+ sandbox: sandboxDescriptorSchema.optional(),
424
+ environment: environmentDataSchema.optional(),
425
+ runs: runs().optional(),
426
+ gate: gatePolicySchema.optional(),
427
+ env: envVarsSchema.optional(),
428
+ egress: egressPolicySchema.optional(),
429
+ judge: configJudgeDataSchema.optional(),
430
+ timeout: durationSchema.optional(),
431
+ capture: captureOptionsSchema.optional(),
432
+ question: questionOptionsSchema.optional(),
433
+ snapshot: snapshotOptionSchema.optional()
434
+ });
430
435
  const profileLimitSchema = z.strictObject({
431
- runs: count().optional().describe("The most runs per cell."),
432
- compare: z.union([z.literal(false), z.strictObject({ maxVariants: z.number().int().min(0) })]).optional().describe("`false` runs the control only; `{ maxVariants }` keeps the first n variants."),
433
- harness: z.array(z.string()).optional().describe("Harness ids to allow; evals left with none are skipped.")
434
- }).describe("Over every eval, and it can only narrow; `pin` on a group or eval skips it.");
436
+ runs: count().optional(),
437
+ compare: z.union([z.literal(false), z.strictObject({ maxVariants: z.number().int().min(0) })]).optional(),
438
+ harness: z.array(z.string()).optional()
439
+ });
435
440
  const profileDataSchema = z.strictObject({
436
- defaults: profileDefaultsDataSchema.optional().describe("What evals get when they don't say otherwise: under groups and evals."),
437
- limit: profileLimitSchema.optional().describe("Caps over every eval, which only narrow; `pin` on a group or eval skips them."),
438
- judge: configJudgeDataSchema.optional().describe("Judge settings for this profile, the same as `defaults.judge`, which wins field by field."),
439
- budget: budgetSchema.optional().describe("Merged over the config's `budget`."),
440
- record: recordMode.optional().describe("When this profile's runs are copied into the history folder."),
441
- tags: z.array(z.string()).optional().describe("Tag filters for this profile, like `--tag` (`'!slow'` excludes)."),
442
- gateway: z.array(gatewayEntryDataSchema).min(1).optional().describe("Replaces the config's `gateway` list for this profile's runs.")
443
- }).describe("`Profile` as data: its defaults and judge as data.");
441
+ defaults: profileDefaultsDataSchema.optional(),
442
+ limit: profileLimitSchema.optional(),
443
+ judge: configJudgeDataSchema.optional(),
444
+ budget: budgetSchema.optional(),
445
+ record: recordMode.optional(),
446
+ tags: z.array(z.string()).optional(),
447
+ gateway: z.array(gatewayEntryDataSchema).min(1).optional()
448
+ });
444
449
  const fakeVote = z.enum([
445
450
  "A",
446
451
  "B",
@@ -449,7 +454,7 @@ const fakeVote = z.enum([
449
454
  const failure = z.strictObject({ error: z.string() });
450
455
  const fakeAnalysisDataSchema = z.strictObject({
451
456
  kind: z.literal("fake-analysis"),
452
- output: z.unknown().optional().describe("The model's structured answer."),
457
+ output: z.unknown().optional(),
453
458
  vote: z.union([
454
459
  fakeVote,
455
460
  z.strictObject({
@@ -457,7 +462,7 @@ const fakeAnalysisDataSchema = z.strictObject({
457
462
  reason: z.string().optional()
458
463
  }),
459
464
  failure
460
- ]).optional().describe("How it answers pairwise questions: a fixed pick, or `{ error }` to fail every call."),
465
+ ]).optional(),
461
466
  answer: z.union([
462
467
  z.string(),
463
468
  z.strictObject({
@@ -465,46 +470,45 @@ const fakeAnalysisDataSchema = z.strictObject({
465
470
  text: z.string().optional()
466
471
  }),
467
472
  failure
468
- ]).optional().describe("How it answers questions asked of each attempt: a fixed answer, or `{ error }` to fail every call.")
469
- }).describe("`fakeAnalysis()` as data: canned answers, without the functions of the digest or pair.");
473
+ ]).optional()
474
+ });
470
475
  const analysisConfigDataSchema = z.strictObject({
471
- model: z.union([z.string(), fakeAnalysisDataSchema]).optional().describe("`'provider/model'`, e.g. `'google/gemini-3.8-flash'`. Insights use it only; `reeval analyze` and questions fall back to the judge's when it's a language model."),
472
- insights: z.boolean().optional().describe("`reeval run` classifies each eval and explains its result with `model` once the attempts finish (default true when `model` is set; `--no-insights` turns it off for one run)."),
473
- maxTokens: z.number().int().min(1).optional().describe("Tokens (in and out) that one run's insights may use, over every classification, insight and headline call (default 250000). Past it, the evals left print without insights; the run's outcome and exit code never change."),
474
- instructions: z.string().optional().describe("Added to the prompt, e.g. \"Fewer tool calls count as a win. The skill under test is reeval-good.\""),
475
- maxInputTokens: z.number().int().min(1e3).optional().describe("The digest's cap; evidence is dropped, lowest priority first, to fit (default 20000)."),
476
- maxOutputTokens: z.number().int().min(500).optional().describe("The most tokens the model may answer with (default 8000). reeval has no prices, so tokens are the cap.")
477
- }).describe("`AnalysisConfig` as data: the model by name, or a fake analysis.");
476
+ model: z.union([z.string(), fakeAnalysisDataSchema]).optional(),
477
+ instructions: z.string().optional(),
478
+ maxInputTokens: z.number().int().min(1e3).optional(),
479
+ maxOutputTokens: z.number().int().min(500).optional()
480
+ });
478
481
  const reevalConfigDataSchema = z.strictObject({
479
- include: z.array(z.string()).optional().describe("Globs of eval files, relative to the config's directory; default every `.eval.ts` under `evals`."),
480
- harness: harnessList.optional().describe("The agents every eval runs on, unless a group or eval replaces them; default Cursor."),
481
- sandbox: sandboxDescriptorSchema.optional().describe("The default machine when the resolved environment omits `sandbox`. Not an environment: it does not invent skills, files or a name. Implicit `docker()` when nobody sets one."),
482
- environment: environmentDataSchema.optional().describe("What every eval runs in when it names no environment, and a baseline-less compare's control."),
483
- runs: count().optional().describe("Attempts per cell; default 1."),
484
- gate: gatePolicySchema.optional().describe("How each cell's attempts decide its outcome; default `'all'`."),
485
- env: envVarsSchema.optional().describe("Variables and secrets for the sandbox, merged by name down the cascade."),
486
- egress: egressPolicySchema.optional().describe("Where the sandbox may connect; default `'proxy'`."),
487
- judge: configJudgeDataSchema.optional().describe("Defaults for `c.judge()`, merged field by field down the cascade."),
488
- timeout: durationSchema.optional().describe("Per attempt; default `'15m'`."),
489
- capture: captureOptionsSchema.optional().describe("Screenshot defaults for `c.capture()`. `REEVAL_CAPTURE_BROWSER=0` refuses the host binary when `c.capture()` runs; it does not change this object."),
490
- question: questionOptionsSchema.optional().describe("Defaults for every question: the model that answers it and what it's shown of each attempt. Merged down the cascade."),
491
- concurrency: count().optional().describe("How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it."),
492
- record: recordMode.optional().describe("When runs are copied into the history folder: `reeval record` only, in CI, or on every run."),
493
- history: historyConfigSchema.optional().describe("The history folder, and which attempts keep their artifacts there."),
494
- budget: budgetSchema.optional().describe("Caps on a run; a profile's `budget` is merged over it."),
495
- profiles: z.record(z.string(), profileDataSchema).optional().describe("Named presets of `defaults`, `limit`, `judge`, `budget`, `record` and `tags`."),
482
+ include: z.array(z.string()).optional(),
483
+ harness: harnessList.optional(),
484
+ sandbox: sandboxDescriptorSchema.optional(),
485
+ environment: environmentDataSchema.optional(),
486
+ runs: runs().optional(),
487
+ gate: gatePolicySchema.optional(),
488
+ env: envVarsSchema.optional(),
489
+ egress: egressPolicySchema.optional(),
490
+ judge: configJudgeDataSchema.optional(),
491
+ timeout: durationSchema.optional(),
492
+ capture: captureOptionsSchema.optional(),
493
+ question: questionOptionsSchema.optional(),
494
+ snapshot: snapshotOptionSchema.optional(),
495
+ concurrency: count().optional(),
496
+ record: recordMode.optional(),
497
+ history: historyConfigSchema.optional(),
498
+ budget: budgetSchema.optional(),
499
+ profiles: z.record(z.string(), profileDataSchema).optional(),
496
500
  defaultProfile: z.strictObject({
497
501
  ci: z.string().optional(),
498
502
  local: z.string().optional()
499
- }).optional().describe("The profile when neither `--profile` nor `REEVAL_PROFILE` names one: `ci` when `CI` is set."),
500
- secrets: z.strictObject({ unbrokered: z.enum(["warn", "error"]).optional() }).optional().describe("`unbrokered: 'error'` refuses Docker secrets without `hosts` instead of warning (default `'warn'`)."),
503
+ }).optional(),
504
+ secrets: z.strictObject({ unbrokered: z.enum(["warn", "error"]).optional() }).optional(),
501
505
  debug: z.strictObject({
502
506
  keep: count().optional(),
503
507
  traceBufferMb: z.number().positive().optional()
504
- }).optional().describe("Local debug data in `.reeval/debug`: how many runs to keep (default 20), and how many MB of raw trace each attempt buffers for trace.ndjson (default 8; the oldest entries go first)."),
505
- analysis: analysisConfigDataSchema.optional().describe("Insights during `reeval run`, `reeval analyze`, and questions without a model."),
506
- gateway: z.array(gatewayEntryDataSchema).min(1).optional().describe("The gateways model calls may go through, in the order you prefer them: each harness takes the first that can serve it. Default `[direct]`, each provider's own key.")
507
- }).describe("`ReevalConfig` in its data forms, as `reeval.yaml` or another language carries it: the environment without `extend`, judge and analysis models by name or as fakes with fixed answers, and profiles the same way. AI SDK model objects and fake score functions are TypeScript only.");
508
+ }).optional(),
509
+ analysis: analysisConfigDataSchema.optional(),
510
+ gateway: z.array(gatewayEntryDataSchema).min(1).optional()
511
+ });
508
512
  /** The object schema an issue's path leads to through `schema`, for "Did you mean". */
509
513
  function keysAt(schema, path, value) {
510
514
  const def = schema.def;
@@ -573,6 +577,15 @@ function configDataProblem(value, file) {
573
577
  }, v)}`;
574
578
  }
575
579
  //#endregion
580
+ //#region src/gateway/routes.ts
581
+ /** Every gateway that brokers a key, by name. */
582
+ const ROUTING_GATEWAYS = {
583
+ vercel: VERCEL_GATEWAY,
584
+ openrouter: OPENROUTER_GATEWAY
585
+ };
586
+ /** Whether a harness's `gateway` names one of {@link ROUTING_GATEWAYS}. */
587
+ const isRoutingGateway = (name) => name !== void 0 && Object.hasOwn(ROUTING_GATEWAYS, name);
588
+ //#endregion
576
589
  //#region src/harness/agents.ts
577
590
  const WEB_EVENTS = {
578
591
  webSearch: "web-search",
@@ -710,12 +723,40 @@ function antigravityMcpServers(servers) {
710
723
  }]));
711
724
  }
712
725
  /**
726
+ * A `{env:…}` or `{file:…}` OpenCode would substitute. It runs `\{env:[^}]+\}`, then
727
+ * `\{file:[^}]+\}`, over its whole config text with no escape, so a reference left open at the end
728
+ * of a value runs on into the JSON after it, and only `{env:}` and `{file:}` are left alone.
729
+ */
730
+ const OPENCODE_REFERENCE = /\{(?:env|file):(?!\})/;
731
+ const fieldKey = (key) => /^[A-Za-z_$][\w$]*$/.test(key) ? `.${key}` : `[${JSON.stringify(key)}]`;
732
+ /**
733
+ * The first part of a server OpenCode would substitute, as what the server wouldn't get: its name,
734
+ * a value as `mcp.<name>.<field>`, or an env or header name. Names are in the same config text as
735
+ * values, so they're substituted too; a secret's value is never named.
736
+ */
737
+ function openCodeReference(name, s) {
738
+ if (OPENCODE_REFERENCE.test(name)) return "its name";
739
+ const server = `mcp${fieldKey(name)}`;
740
+ const [map, kind, values] = s.transport === "stdio" ? [
741
+ "env",
742
+ "env",
743
+ s.env ?? {}
744
+ ] : [
745
+ "headers",
746
+ "header",
747
+ s.headers ?? {}
748
+ ];
749
+ return [...s.transport === "stdio" ? [[`${server}.command`, s.command], ...s.args.map((a, i) => [`${server}.args[${i}]`, a])] : [[`${server}.url`, s.url]], ...Object.entries(values).flatMap(([k, v]) => [[`the ${kind} name ${k} in ${server}.${map}`, k], [`${server}.${map}${fieldKey(k)}`, v]])].find(([, text]) => OPENCODE_REFERENCE.test(text))?.[0];
750
+ }
751
+ /**
713
752
  * OpenCode's config shape (its `mcp.servers`), which the adapter sets `codemode: false` on.
714
- * OpenCode has no SSE transport.
753
+ * OpenCode has no SSE transport, and refuses a value it would substitute.
715
754
  */
716
755
  function opencodeMcpServers(servers) {
717
756
  return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
718
757
  if (s.transport === "sse") throw new InfraError(`OpenCode has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
758
+ const field = openCodeReference(name, s);
759
+ if (field) throw new InfraError(`OpenCode substitutes {env:…} and {file:…} anywhere in its config, with no escape, so the ${name} server wouldn't get ${field} as written: take the reference out, or leave OpenCode out of this cell`);
719
760
  return s.transport === "stdio" ? [name, {
720
761
  type: "local",
721
762
  command: [s.command, ...s.args],
@@ -975,7 +1016,7 @@ async function writePluginFiles(sandbox, layout, plugins) {
975
1016
  if (existing?.trim()) {
976
1017
  let base;
977
1018
  try {
978
- base = JSON.parse(existing);
1019
+ base = JSON.parse(withoutBom(existing));
979
1020
  } catch {}
980
1021
  if (!isObject(base)) throw new InfraError(`${f.path} isn't a JSON object, so reeval can't enable the session's plugins in it: write it as one in files, or leave it out`);
981
1022
  const ours = JSON.parse(f.content);
@@ -1087,7 +1128,7 @@ const AGENTS = {
1087
1128
  },
1088
1129
  unreported: cursorUnreported,
1089
1130
  mcpServers: cursorMcpServers,
1090
- adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-CyaDNG9r.js")).createCursorSdk({
1131
+ adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-DMo2eCOJ.js")).createCursorSdk({
1091
1132
  auth,
1092
1133
  ...mcpServers ? { mcpServers } : {},
1093
1134
  ...descriptor.kind === "cursor" && descriptor.params ? { modelParams: descriptor.params } : {},
@@ -1124,13 +1165,14 @@ const AGENTS = {
1124
1165
  }
1125
1166
  ],
1126
1167
  mcpServers: claudeMcpServers,
1127
- adapter: async ({ auth, mcpServers, descriptor }) => (await import("./dist-CYrBwyHA.js")).createClaudeCode({
1168
+ adapter: async ({ auth, env, mcpServers, descriptor }) => (await import("./dist-CYrBwyHA.js")).createClaudeCode({
1128
1169
  auth,
1129
1170
  ...mcpServers ? { mcpServers } : {},
1130
1171
  ...descriptor.kind === "claude-code" && descriptor.effort ? { effort: descriptor.effort } : {},
1131
1172
  env: {
1132
1173
  IS_SANDBOX: "1",
1133
- CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1"
1174
+ CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",
1175
+ ...env
1134
1176
  }
1135
1177
  })
1136
1178
  },
@@ -1153,8 +1195,9 @@ const AGENTS = {
1153
1195
  },
1154
1196
  servedModel: geminiServedModel,
1155
1197
  mcpServers: antigravityMcpServers,
1156
- adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-ElBqEN-S.js")).createAntigravity({
1198
+ adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-OESRtvOi.js")).createAntigravity({
1157
1199
  auth,
1200
+ ...vertex ? { vertex } : {},
1158
1201
  ...mcpServers ? { mcpServers } : {},
1159
1202
  ...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
1160
1203
  name,
@@ -1179,10 +1222,10 @@ const AGENTS = {
1179
1222
  servedModel: geminiServedModel,
1180
1223
  forModel: opencodeForModel,
1181
1224
  mcpServers: opencodeMcpServers,
1182
- adapter: async ({ auth, mcpServers, descriptor, model }) => (await import("./src-DIfRbp5a.js")).createOpenCode({
1225
+ adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-CgJrz5p0.js")).createOpenCode({
1183
1226
  auth,
1184
1227
  model,
1185
- vars: opencodeVars(model),
1228
+ ...vertex ? { vertex } : { vars: opencodeVars(model) },
1186
1229
  ...descriptor.kind === "opencode" && descriptor.effort ? { effort: descriptor.effort } : {},
1187
1230
  ...mcpServers ? { mcpServers } : {},
1188
1231
  requireCredentialBrokering: true
@@ -1197,14 +1240,18 @@ const AGENTS = {
1197
1240
  endpoints: GOOSE_ENDPOINTS,
1198
1241
  expectedBlocked: {
1199
1242
  hosts: ["models.dev"],
1200
- why: "Goose's model catalogue refresh, which no setting turns off"
1243
+ why: "Goose's model catalog refresh, which no setting turns off"
1201
1244
  },
1202
1245
  servedModel: geminiServedModel,
1203
1246
  mcpServers: gooseMcpServers,
1204
- adapter: async ({ auth, mcpServers, descriptor, model }) => (await import("./src-Dg8121Xy.js")).createGoose({
1247
+ adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-BTqJbvj6.js")).createGoose({
1248
+ ...vertex ? { vertex } : {},
1205
1249
  ...auth["AI_GATEWAY_API_KEY"] ? {
1206
1250
  auth,
1207
- provider: (await import("./src-Dg8121Xy.js")).AI_GATEWAY_PROVIDER
1251
+ provider: (await import("./src-BTqJbvj6.js")).AI_GATEWAY_PROVIDER
1252
+ } : auth["OPENROUTER_API_KEY"] ? {
1253
+ auth,
1254
+ provider: (await import("./src-BTqJbvj6.js")).OPENROUTER_PROVIDER
1208
1255
  } : { auth: { GOOGLE_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" } },
1209
1256
  model,
1210
1257
  ...descriptor.kind === "goose" && descriptor.effort ? { effort: descriptor.effort } : {},
@@ -1225,8 +1272,17 @@ const AGENTS = {
1225
1272
  endpoints: GEMINI_ENDPOINTS,
1226
1273
  servedModel: geminiServedModel,
1227
1274
  mcpServers: piMcpServers,
1228
- adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-mJVxw1Jm.js")).createPi({
1229
- auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
1275
+ adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-BkiComGh.js")).createPi({
1276
+ ...auth["AI_GATEWAY_API_KEY"] ? {
1277
+ auth,
1278
+ gateway: (await import("./src-BkiComGh.js")).PI_AI_GATEWAY
1279
+ } : auth["OPENROUTER_API_KEY"] ? {
1280
+ auth,
1281
+ gateway: (await import("./src-BkiComGh.js")).PI_OPENROUTER
1282
+ } : {
1283
+ auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
1284
+ ...vertex ? { vertex } : {}
1285
+ },
1230
1286
  ...descriptor.kind === "pi" && descriptor.effort ? { effort: descriptor.effort } : {},
1231
1287
  ...mcpServers ? { mcpServers } : {},
1232
1288
  requireCredentialBrokering: true
@@ -1255,9 +1311,9 @@ const AGENTS = {
1255
1311
  },
1256
1312
  servedModel: openaiServedModel,
1257
1313
  mcpServers: codexMcpServers,
1258
- adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-D005Cbqz.js")).createCodex({
1314
+ adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-BvgeZ4tS.js")).createCodex({
1259
1315
  auth,
1260
- ...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-D005Cbqz.js")).AI_GATEWAY_PROVIDER } : {},
1316
+ ...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-BvgeZ4tS.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-BvgeZ4tS.js")).OPENROUTER_PROVIDER } : {},
1261
1317
  ...descriptor.kind === "codex" && descriptor.effort ? { effort: descriptor.effort } : {},
1262
1318
  ...mcpServers ? { mcpServers } : {},
1263
1319
  ...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
@@ -1275,9 +1331,15 @@ const AGENTS = {
1275
1331
  ...fxForModel("provider/model"),
1276
1332
  forModel: fxForModel,
1277
1333
  mcpServers: acpMcpServers,
1278
- adapter: async ({ auth, mcpServers, model }) => (await import("./src-YHGaUrGE.js")).createFx({
1334
+ adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-Dk2Na0b-.js")).createFx({
1279
1335
  auth,
1280
1336
  model,
1337
+ ...auth["OPENROUTER_API_KEY"] ? { provider: "openrouter" } : {},
1338
+ ...env?.FX_VERTEX_PROJECT ? { vertex: {
1339
+ project: env.FX_VERTEX_PROJECT,
1340
+ location: env.FX_VERTEX_LOCATION ?? ""
1341
+ } } : {},
1342
+ ...descriptor.kind === "fx" && descriptor.effort ? { effort: descriptor.effort } : {},
1281
1343
  ...mcpServers ? { mcpServers } : {},
1282
1344
  requireCredentialBrokering: true
1283
1345
  })
@@ -1293,18 +1355,26 @@ const AGENTS = {
1293
1355
  modelResponses: XAI_MODEL_RESPONSES,
1294
1356
  servedModel: xaiServedModel,
1295
1357
  mcpServers: acpMcpServers,
1296
- adapter: async ({ auth, mcpServers, descriptor, plugins, model }) => (await import("./src-C_9wgowC.js")).createGrokBuild({
1297
- auth,
1298
- ...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-C_9wgowC.js")).AI_GATEWAY_PROVIDER } : {},
1299
- model,
1300
- ...plugins.length ? {
1301
- pluginDirs: plugins.map((p) => p.dir),
1302
- pluginMcpServers: plugins.flatMap((p) => p.servers.map((s) => s.slice(p.name.length + 1)))
1303
- } : {},
1304
- ...descriptor.kind === "grok-build" && descriptor.effort ? { effort: descriptor.effort } : {},
1305
- ...mcpServers ? { mcpServers } : {},
1306
- requireCredentialBrokering: true
1307
- })
1358
+ adapter: async ({ auth, mcpServers, descriptor, plugins, model, vertexChat }) => {
1359
+ const grok = await import("./src-Cp-2Hdib.js");
1360
+ return grok.createGrokBuild({
1361
+ ...vertexChat ? {
1362
+ auth: { [grok.VERTEX_TOKEN_ENV]: SANDBOX_ACCESS_TOKEN },
1363
+ provider: grok.vertexProvider(vertexChat.project, vertexChat.region)
1364
+ } : {
1365
+ auth,
1366
+ ...auth["AI_GATEWAY_API_KEY"] ? { provider: grok.AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: grok.OPENROUTER_PROVIDER } : {}
1367
+ },
1368
+ model,
1369
+ ...plugins.length ? {
1370
+ pluginDirs: plugins.map((p) => p.dir),
1371
+ pluginMcpServers: plugins.flatMap((p) => p.servers.map((s) => s.slice(p.name.length + 1)))
1372
+ } : {},
1373
+ ...descriptor.kind === "grok-build" && descriptor.effort ? { effort: descriptor.effort } : {},
1374
+ ...mcpServers ? { mcpServers } : {},
1375
+ requireCredentialBrokering: true
1376
+ });
1377
+ }
1308
1378
  }
1309
1379
  };
1310
1380
  /** The harness of a config that names none: Cursor, on Cursor's own model. */
@@ -1316,20 +1386,31 @@ const DEFAULT_HARNESS = {
1316
1386
  const HARNESS_EXAMPLE = "cursor({ model: 'composer-2.5' })";
1317
1387
  const isAgentKind = (kind) => typeof kind === "string" && Object.hasOwn(AGENTS, kind);
1318
1388
  /**
1389
+ * A server as the plan knows it: its names, literal values and each `secret.template`'s text. A
1390
+ * host secret's value is read when the attempt starts, so the plan has it as `''`.
1391
+ */
1392
+ function plannedMcpServer(s) {
1393
+ const literal = (values) => Object.fromEntries(Object.entries(values).map(([k, v]) => [k, typeof v === "string" ? v : v.kind === "secret-template" ? v.template : ""]));
1394
+ return s.transport === "stdio" ? {
1395
+ transport: "stdio",
1396
+ command: s.command,
1397
+ args: s.args,
1398
+ env: literal(s.env)
1399
+ } : {
1400
+ transport: s.transport,
1401
+ url: s.url,
1402
+ headers: literal(s.headers)
1403
+ };
1404
+ }
1405
+ /**
1319
1406
  * Why a harness's agent can't take the environment's MCP servers (Goose, OpenCode, pi and Codex
1320
- * take no `mcp.sse`), in the agent's own words, so the plan refuses them before any attempt
1321
- * starts; undefined when it can.
1407
+ * take no `mcp.sse`; OpenCode would substitute a `{env:…}` or `{file:…}` in a name or value), in the
1408
+ * agent's own words, so the plan refuses them before any attempt starts; undefined when it can.
1322
1409
  */
1323
1410
  function refusedMcp(harness, layers) {
1324
1411
  if (!isAgentKind(harness.kind)) return void 0;
1325
1412
  const servers = /* @__PURE__ */ new Map();
1326
- for (const { spec } of layers) for (const [name, server] of Object.entries(spec.mcp ?? {})) servers.set(name, server.transport === "stdio" ? {
1327
- ...server,
1328
- env: {}
1329
- } : {
1330
- ...server,
1331
- headers: {}
1332
- });
1413
+ for (const { spec } of layers) for (const [name, server] of Object.entries(spec.mcp ?? {})) servers.set(name, plannedMcpServer(server));
1333
1414
  if (servers.size === 0) return void 0;
1334
1415
  try {
1335
1416
  AGENTS[harness.kind].mcpServers(Object.fromEntries(servers));
@@ -1392,13 +1473,7 @@ function refusedPlugins(harness, plugins, own, unread = []) {
1392
1473
  const fix = rest.length ? `use only: [${quoted(rest)}], or leave` : "leave";
1393
1474
  return event ? `${name} can't run the ${p.name} plugin's ${event} hooks (hooks/hooks.json): it runs ${runs?.join(", ")} hooks; ${fix} the plugin out of this cell` : `${name} can't run the ${p.name} plugin's ${type} hooks (hooks/hooks.json): it runs only command hooks; ${fix} the plugin out of this cell`;
1394
1475
  }
1395
- const servers = Object.fromEntries(plugins.flatMap((p) => Object.entries(p.mcp).map(([server, s]) => [pluginServerName(p.name, server), s.transport === "stdio" ? {
1396
- ...s,
1397
- env: {}
1398
- } : {
1399
- ...s,
1400
- headers: {}
1401
- }])));
1476
+ const servers = Object.fromEntries(plugins.flatMap((p) => Object.entries(p.mcp).map(([server, s]) => [pluginServerName(p.name, server), plannedMcpServer(s)])));
1402
1477
  if (agent && Object.keys(servers).length) {
1403
1478
  const alongside = agent.pluginPlacement === "flat" ? Object.fromEntries(own.mcp.map((n) => [n, {
1404
1479
  transport: "stdio",
@@ -1472,12 +1547,25 @@ function refusedPlugins(harness, plugins, own, unread = []) {
1472
1547
  ruleOwner.set(file, what);
1473
1548
  }
1474
1549
  }
1475
- /** A real agent's catalog entry, with what its model's provider decides. */
1476
- function agentFor(h) {
1550
+ /**
1551
+ * A real agent's catalog entry, with what its model's provider decides, or its gateway's route:
1552
+ * on Vertex, the attempt's `vertex()` settings decide the project and region.
1553
+ */
1554
+ function agentFor(h, vertex) {
1477
1555
  const agent = AGENTS[h.kind];
1478
- if (h.gateway === "vercel") {
1479
- const route = VERCEL_ROUTES[h.kind];
1480
- if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on Vercel AI Gateway`);
1556
+ if (h.gateway === "vertex") {
1557
+ const route = VERTEX_ROUTES[h.kind];
1558
+ if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on Vertex AI`);
1559
+ if (!vertex) throw new InfraError(`${agent.name} on Vertex AI has no vertex() settings`);
1560
+ return {
1561
+ ...agent,
1562
+ ...route(vertex)
1563
+ };
1564
+ }
1565
+ if (isRoutingGateway(h.gateway)) {
1566
+ const gateway = ROUTING_GATEWAYS[h.gateway];
1567
+ const route = gateway.routes[h.kind];
1568
+ if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on ${gateway.name}`);
1481
1569
  return {
1482
1570
  ...agent,
1483
1571
  ...route(process.env)
@@ -1489,9 +1577,12 @@ function agentFor(h) {
1489
1577
  } : agent;
1490
1578
  }
1491
1579
  /** The catalog entry for a harness's agent; undefined for the fake harness. */
1492
- const agentOf = (h) => h.kind === "fake" ? void 0 : agentFor(h);
1493
- /** The host variable a harness authenticates with; undefined for the fake harness. */
1494
- const credentialOf = (h) => agentOf(h)?.credential;
1580
+ const agentOf = (h, vertex) => h.kind === "fake" ? void 0 : agentFor(h, vertex);
1581
+ /**
1582
+ * The host variable a harness authenticates with; undefined for the fake harness, and on Vertex,
1583
+ * whose token comes from Application Default Credentials.
1584
+ */
1585
+ const credentialOf = (h) => h.kind === "fake" || h.gateway === "vertex" ? void 0 : agentFor(h).credential;
1495
1586
  /** The fake agent's own layout. */
1496
1587
  const FAKE_LAYOUT = {
1497
1588
  rulesFile: "AGENTS.md",
@@ -1506,10 +1597,10 @@ function layoutOf(descriptor) {
1506
1597
  }
1507
1598
  /**
1508
1599
  * The hosts a harness's agent is known to try in a session with or without plugins; a fake agent
1509
- * with `layout` expects that agent's.
1600
+ * with `layout` expects that agent's. On Vertex, the attempt's `vertex()` settings pick the route.
1510
1601
  */
1511
- function expectedBlockedFor(descriptor, plugins) {
1512
- const expected = descriptor.kind !== "fake" ? agentFor(descriptor).expectedBlocked : descriptor.layout ? AGENTS[descriptor.layout].expectedBlocked : void 0;
1602
+ function expectedBlockedFor(descriptor, plugins, vertex) {
1603
+ const expected = descriptor.kind !== "fake" ? agentFor(descriptor, vertex).expectedBlocked : descriptor.layout ? AGENTS[descriptor.layout].expectedBlocked : void 0;
1513
1604
  return expected?.withPlugins && !plugins ? void 0 : expected;
1514
1605
  }
1515
1606
  /**
@@ -1527,18 +1618,27 @@ const REFUSED_KEY = /\bHTTP 401\b|\b401\b.{0,40}\bunauthori[sz]ed\b|\bunauthori[
1527
1618
  */
1528
1619
  function authHint(agent, message, refused = false) {
1529
1620
  if (!refused && !REFUSED_KEY.test(message)) return void 0;
1621
+ if (agent.token) return `The API rejected the access token reeval minted from ${agent.credential}: check gcloud auth application-default login (or GOOGLE_APPLICATION_CREDENTIALS) and the project's Vertex AI access`;
1530
1622
  return `The API rejected ${agent.credential}: set a valid key in the environment or .env.local`;
1531
1623
  }
1532
- /** The agent's credential from the host, as the adapter's explicit `auth`. Fails closed. */
1624
+ /**
1625
+ * The agent's credential from the host, as the adapter's explicit `auth`. Fails closed. Empty for
1626
+ * a brokered token, which the adapter never sees.
1627
+ */
1533
1628
  function agentAuth(agent, env = process.env) {
1629
+ if (agent.token) return {};
1534
1630
  const value = env[agent.credential];
1535
1631
  if (!value) throw new InfraError(`${agent.credential} isn't set, so the ${agent.name} harness can't authenticate. Set it in the environment or .env.local.`);
1536
- return { [agent.authAs ?? agent.credential]: value };
1632
+ return {
1633
+ ...agent.authEnv,
1634
+ [agent.authAs ?? agent.credential]: value
1635
+ };
1537
1636
  }
1538
- function brokeringFor(h) {
1539
- const agent = agentOf(h);
1637
+ function brokeringFor(h, vertex) {
1638
+ const agent = agentOf(h, vertex);
1540
1639
  if (!agent) return void 0;
1541
- const { credential, endpoints, tokens, servedModel, askedModel, modelResponses } = agent;
1640
+ const { credential, endpoints, tokens, servedModel, askedModel, modelResponses, upstream } = agent;
1641
+ const { localResponses } = agent;
1542
1642
  return {
1543
1643
  credential,
1544
1644
  endpoints,
@@ -1548,7 +1648,9 @@ function brokeringFor(h) {
1548
1648
  } } : {},
1549
1649
  ...servedModel ? { servedModel } : {},
1550
1650
  ...askedModel ? { askedModel } : {},
1551
- ...modelResponses ? { modelResponses } : {}
1651
+ ...modelResponses ? { modelResponses } : {},
1652
+ ...upstream ? { upstream } : {},
1653
+ ...localResponses?.length ? { localResponses } : {}
1552
1654
  };
1553
1655
  }
1554
1656
  //#endregion
@@ -1605,11 +1707,10 @@ function resolvedEffortOf(models) {
1605
1707
  /** The error a per-attempt answer carries when the run stopped before a model answered it. */
1606
1708
  const STOPPED_FIRST = "the run stopped before it was answered";
1607
1709
  /**
1608
- * What each side of a pairwise question shows with no `state`: the task and the diff, every
1609
- * turn's prompt. The final message stays out, since a better-written account of the work would
1610
- * win a side-by-side pick on its wording; the changed files repeat what the diff shows.
1710
+ * What a recorded pairwise answer without `stateOptions` showed each side: the task and the diff,
1711
+ * every turn's prompt.
1611
1712
  */
1612
- const DEFAULT_QUESTION_STATE = {
1713
+ const UNRECORDED_QUESTION_STATE = {
1613
1714
  task: true,
1614
1715
  output: false,
1615
1716
  files: false,
@@ -1618,62 +1719,39 @@ const DEFAULT_QUESTION_STATE = {
1618
1719
  turns: "all",
1619
1720
  checks: false
1620
1721
  };
1722
+ const KEYS = Object.keys(UNRECORDED_QUESTION_STATE);
1621
1723
  /**
1622
- * What a question asked of each attempt on its own shows with no `state`: the task, the diff,
1623
- * the final message and the transcript. Nothing is weighed against another side, so how the
1624
- * agent went about the work is the point.
1724
+ * Every key set, in the record's key order, so equal selections serialize the same: an input the
1725
+ * question didn't turn on is off, and `turns` is `'all'` unless it says `'last'`.
1625
1726
  */
1626
- const DEFAULT_ATTEMPT_QUESTION_STATE = {
1627
- ...DEFAULT_QUESTION_STATE,
1628
- output: true,
1629
- transcript: true
1630
- };
1631
- const KEYS = Object.keys(DEFAULT_QUESTION_STATE);
1632
- /** A pairwise question's defaults when `pairwise`, a per-attempt question's otherwise. */
1633
- const defaultQuestionState = (pairwise) => pairwise ? DEFAULT_QUESTION_STATE : DEFAULT_ATTEMPT_QUESTION_STATE;
1634
- /** Every key set, in `DEFAULT_QUESTION_STATE`'s order, so equal selections serialise the same. */
1635
- function resolveQuestionState(state, pairwise) {
1636
- const defaults = defaultQuestionState(pairwise);
1637
- const out = {};
1638
- for (const key of KEYS) out[key] = state?.[key] ?? defaults[key];
1639
- return out;
1727
+ function resolveQuestionState(state) {
1728
+ return {
1729
+ task: state?.task ?? false,
1730
+ output: state?.output ?? false,
1731
+ files: state?.files ?? false,
1732
+ diff: state?.diff ?? false,
1733
+ transcript: state?.transcript ?? false,
1734
+ turns: state?.turns ?? "all",
1735
+ checks: state?.checks ?? false
1736
+ };
1640
1737
  }
1641
- /** `inner` over `outer`: `state` key by key, `model` and `artifacts` replaced. */
1738
+ /** `inner` over `outer`, field by field. */
1642
1739
  function mergeQuestion(outer, inner) {
1643
- if (!inner) return outer;
1644
- const out = {
1740
+ return inner ? {
1645
1741
  ...outer,
1646
1742
  ...inner
1647
- };
1648
- if (outer.state || inner.state) out.state = {
1649
- ...outer.state,
1650
- ...inner.state
1651
- };
1652
- return out;
1743
+ } : outer;
1653
1744
  }
1654
- /** A planned question with the cascaded `question` defaults under its own `model`, `state` and `artifacts`. */
1745
+ /** A planned question under the cascaded `question` defaults: its own `model`, else theirs. */
1655
1746
  function withQuestionDefaults(q, defaults) {
1656
- const { state, artifacts } = mergeQuestion(defaults, {
1657
- ...q.state ? { state: q.state } : {},
1658
- ...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
1659
- });
1660
- const { state: _state, artifacts: _artifacts, ...rest } = q;
1661
1747
  const model = q.model ?? defaults.model;
1662
- return {
1663
- ...rest,
1664
- ...model !== void 0 ? { model } : {},
1665
- ...state && Object.keys(state).length ? { state } : {},
1666
- ...artifacts !== void 0 ? { artifacts } : {}
1667
- };
1748
+ return model !== void 0 ? {
1749
+ ...q,
1750
+ model
1751
+ } : q;
1668
1752
  }
1669
1753
  const isOff = (value) => value === false || Array.isArray(value) && !value.length;
1670
- /** True when the resolved state shows nothing and no captures are sent. */
1671
- const showsNothing = (q) => {
1672
- if (q.artifacts !== void 0) return false;
1673
- const s = resolveQuestionState(q.state, q.across !== void 0);
1674
- return KEYS.every((k) => k === "turns" || isOff(s[k]));
1675
- };
1676
- /** The state keys a question shows, in order, plus `artifacts` when it sends captures. */
1754
+ /** The inputs a question shows, in order, plus `artifacts` when it sends captures. */
1677
1755
  function questionShows(state, artifacts) {
1678
1756
  const on = KEYS.filter((k) => k !== "turns" && !isOff(state[k]));
1679
1757
  return artifacts === void 0 ? on : [...on, "artifacts"];
@@ -1739,6 +1817,189 @@ function compareProblem(value, where, environment) {
1739
1817
  if (!isObjectLike(value) || !Array.isArray(value.variants)) return `compare in ${where} takes { variants, baseline? } (got ${describe$1(value)}). Use compare: { variants: [a, b], baseline?, gate?, minLift?, harness? }.`;
1740
1818
  return refusal(compareSchema(environment), value, explain(where), { order: ORDER });
1741
1819
  }
1820
+ const POLL_MS = 200;
1821
+ /**
1822
+ * Prints `alive` while `pid` runs, then the sandbox's TCP tables, or `noproc` without them
1823
+ * (or without `tables`: the fake sandbox shares the host's, where other processes bind too).
1824
+ */
1825
+ function probeScript(pid, tables = true) {
1826
+ return `${pid ? `kill -0 ${pid} 2>/dev/null && echo alive; ` : ""}${tables ? "if [ -r /proc/net/tcp ]; then cat /proc/net/tcp /proc/net/tcp6 2>/dev/null; else echo noproc; fi" : "echo noproc"}; exit 0`;
1827
+ }
1828
+ /** `0100007F` → `127.0.0.1`; 32 hex digits → an IPv6 address. Each 32-bit word is host-endian. */
1829
+ function procAddress(hex) {
1830
+ const bytes = (hex.match(/.{8}/g) ?? []).flatMap((w) => (w.match(/../g) ?? []).reverse().map((b) => Number.parseInt(b, 16)));
1831
+ if (bytes.length === 4) return bytes.join(".");
1832
+ if (bytes.slice(0, 10).every((b) => b === 0) && bytes[10] === 255 && bytes[11] === 255) return bytes.slice(12).join(".");
1833
+ const groups = [];
1834
+ for (let i = 0; i < 16; i += 2) groups.push(((bytes[i] ?? 0) << 8 | (bytes[i + 1] ?? 0)).toString(16));
1835
+ const full = groups.join(":");
1836
+ const longest = (full.match(/(^|:)0(:0)+(:|$)/g) ?? []).reduce((a, b) => b.length > a.length ? b : a, "");
1837
+ return longest ? full.replace(longest, "::") : full;
1838
+ }
1839
+ /** A listener the tunnel reaches: on loopback or every address, IPv4 or IPv6. */
1840
+ const reachable = (address) => address === "0.0.0.0" || address === "::" || address === "::1" || address.startsWith("127.");
1841
+ function parseProbe(stdout) {
1842
+ const lines = stdout.split("\n");
1843
+ const alive = lines.includes("alive");
1844
+ if (lines.includes("noproc")) return {
1845
+ proc: false,
1846
+ alive
1847
+ };
1848
+ const listeners = [];
1849
+ for (const l of lines) {
1850
+ const m = /^\s*\d+:\s+([0-9A-F]+):([0-9A-F]{4})\s+\S+\s+0A\s/i.exec(l);
1851
+ const address = m ? procAddress(m[1]) : "";
1852
+ if (m && reachable(address)) listeners.push({
1853
+ port: Number.parseInt(m[2], 16),
1854
+ address
1855
+ });
1856
+ }
1857
+ return {
1858
+ proc: true,
1859
+ alive,
1860
+ listeners
1861
+ };
1862
+ }
1863
+ /** The ports listening now that weren't before the command started, ascending. */
1864
+ const newPorts = (listeners, before) => [...new Set(listeners.map((l) => l.port).filter((p) => !before.has(p)))].sort((a, b) => a - b);
1865
+ /** The address to record for `port`: IPv4 first, since that's where the tunnel connects first. */
1866
+ const addressFor = (listeners, port) => {
1867
+ const all = listeners.filter((l) => l.port === port).map((l) => l.address);
1868
+ return all.find((a) => !a.includes(":")) ?? all.find((a) => a === "::") ?? all[0] ?? "";
1869
+ };
1870
+ /** `want` is taken whoever listens on it: reeval allocated it, or the eval named it. */
1871
+ function decide(s) {
1872
+ const { listeners, alive } = s.probe;
1873
+ const ports = newPorts(listeners, s.before);
1874
+ if (listeners.some((l) => l.port === s.want)) return {
1875
+ kind: "port",
1876
+ port: s.want,
1877
+ address: addressFor(listeners, s.want)
1878
+ };
1879
+ const late = s.elapsedMs >= s.timeoutMs;
1880
+ if (!s.given && ports.length && (s.stableMs >= 3e3 || late || !alive)) {
1881
+ if (ports.length === 1) {
1882
+ const port = ports[0];
1883
+ return {
1884
+ kind: "port",
1885
+ port,
1886
+ address: addressFor(listeners, port)
1887
+ };
1888
+ }
1889
+ return {
1890
+ kind: "several",
1891
+ ports
1892
+ };
1893
+ }
1894
+ if (!alive) return { kind: "exited" };
1895
+ return late ? {
1896
+ kind: "timeout",
1897
+ ports
1898
+ } : { kind: "wait" };
1899
+ }
1900
+ const plain = (output) => output.replace(/\x1b\[[0-9;?]*[A-Za-z]/g, "");
1901
+ /** The URLs `output` prints on one of `ports`, as Vite's `Local: http://localhost:5173/`. */
1902
+ function printedUrls(output, ports) {
1903
+ const found = /* @__PURE__ */ new Map();
1904
+ for (const m of plain(output).matchAll(/https?:\/\/(?:\[[0-9a-f:]+\]|[^\s/:[\]]+):(\d+)[^\s'"]*/gi)) {
1905
+ const port = Number(m[1]);
1906
+ if (ports.includes(port) && !found.has(m[0])) found.set(m[0], port);
1907
+ }
1908
+ return [...found].map(([url, port]) => ({
1909
+ url,
1910
+ port
1911
+ }));
1912
+ }
1913
+ /** Whether something on the host accepts a TCP connection on `port`, IPv4 or IPv6 loopback. */
1914
+ async function hostListening(port) {
1915
+ for (const host of ["127.0.0.1", "::1"]) if (await new Promise((resolve) => {
1916
+ const socket = connect({
1917
+ port,
1918
+ host
1919
+ });
1920
+ socket.once("connect", () => {
1921
+ socket.destroy();
1922
+ resolve(true);
1923
+ });
1924
+ socket.once("error", () => resolve(false));
1925
+ })) return host;
1926
+ }
1927
+ const sleep = (ms, signal) => new Promise((resolve) => {
1928
+ const done = () => {
1929
+ clearTimeout(timer);
1930
+ signal?.removeEventListener("abort", done);
1931
+ resolve();
1932
+ };
1933
+ const timer = setTimeout(done, ms);
1934
+ signal?.addEventListener("abort", done, { once: true });
1935
+ });
1936
+ const listOf = (items) => items.length > 1 ? `${items.slice(0, -1).join(", ")} and ${items.at(-1)}` : String(items[0]);
1937
+ /**
1938
+ * Polls until something listens on `want`, or, without a given port, until the ports that are
1939
+ * new since the command started have settled on one. Throws, with the command's last output,
1940
+ * when they settle on several, it exits, or nothing listens in time.
1941
+ */
1942
+ async function awaitListening(command, w) {
1943
+ const timeoutMs = w.timeoutMs ?? 12e4;
1944
+ const started = Date.now();
1945
+ let last = "";
1946
+ let changed = started;
1947
+ const output = async () => plain(await w.output().catch(() => ""));
1948
+ const fail = async (why, out) => {
1949
+ const tail = (out ?? await output()).trimEnd().split("\n").slice(-20).join("\n").trim();
1950
+ return /* @__PURE__ */ new Error(`c.serve(${JSON.stringify(command)}) ${why}${tail ? `. Its last output:\n${tail}` : ""}`);
1951
+ };
1952
+ for (;;) {
1953
+ w.signal?.throwIfAborted();
1954
+ const probe = await w.probe();
1955
+ const now = Date.now();
1956
+ if (!probe.proc) {
1957
+ const address = await (w.hostListening ?? hostListening)(w.want);
1958
+ if (address !== void 0) return {
1959
+ port: w.want,
1960
+ address,
1961
+ detected: false,
1962
+ readyMs: now - started
1963
+ };
1964
+ if (!probe.alive) throw await fail(`exited before it listened on ${w.want}`);
1965
+ if (now - started >= timeoutMs) throw await fail(`didn't listen on ${w.want} within ${timeoutMs / 1e3}s. This sandbox can't list the ports it listens on, so it must listen on PORT or the port passed`);
1966
+ } else {
1967
+ const key = newPorts(probe.listeners, w.before).join(",");
1968
+ if (key !== last) {
1969
+ last = key;
1970
+ changed = now;
1971
+ }
1972
+ const d = decide({
1973
+ want: w.want,
1974
+ given: w.given,
1975
+ probe,
1976
+ before: w.before,
1977
+ stableMs: now - changed,
1978
+ elapsedMs: now - started,
1979
+ timeoutMs
1980
+ });
1981
+ if (d.kind === "port") return {
1982
+ port: d.port,
1983
+ address: d.address,
1984
+ detected: d.port !== w.want,
1985
+ readyMs: now - started
1986
+ };
1987
+ if (d.kind === "several") {
1988
+ const out = await output();
1989
+ const urls = printedUrls(out, d.ports);
1990
+ const printed = urls.length ? ` It printed ${listOf(urls.map((u) => u.url))}.` : "";
1991
+ const hint = urls[0]?.port ?? d.ports[0];
1992
+ throw await fail(`listens on ports ${listOf(d.ports)}, and not on PORT (${w.want}).${printed} Pass the one it serves on: c.serve(${JSON.stringify(command)}, ${hint})`, out);
1993
+ }
1994
+ if (d.kind === "exited") throw await fail("exited before it listened on a port");
1995
+ if (d.kind === "timeout") {
1996
+ const on = d.ports.length ? `; ${d.ports.length > 1 ? "ports" : "port"} ${listOf(d.ports)} started listening` : "";
1997
+ throw await fail(`didn't listen on ${w.given ? `the port passed (${w.want})` : "a port"} within ${timeoutMs / 1e3}s${on}`);
1998
+ }
1999
+ }
2000
+ await sleep(POLL_MS, w.signal);
2001
+ }
2002
+ }
1742
2003
  //#endregion
1743
2004
  //#region src/sandbox/fake.ts
1744
2005
  /** Commands run on the host's own network, so the fake sandbox can neither restrict nor see it. */
@@ -1883,7 +2144,7 @@ async function createFakeSandbox(descriptor, env = {}) {
1883
2144
  },
1884
2145
  home: base,
1885
2146
  readFile: (path) => readFile(resolve(path), "utf8").catch(() => null),
1886
- getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}://127.0.0.1:${port}` }),
2147
+ getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}://${await hostListening(port) === "::1" ? "[::1]" : "127.0.0.1"}:${port}` }),
1887
2148
  async writeFile(path, content) {
1888
2149
  const target = resolve(path);
1889
2150
  await mkdir(dirname(target), { recursive: true });
@@ -1951,13 +2212,10 @@ function resolveOutput(input) {
1951
2212
  const { flags, env } = input;
1952
2213
  const ci = isCI(env);
1953
2214
  const githubActions = set(env, "GITHUB_ACTIONS");
1954
- const ui = pick("--ui", flags.ui ?? env.REEVAL_UI, [
1955
- "live",
1956
- "plain",
1957
- "agent"
1958
- ]);
1959
- const requested = ui && ui !== "auto" ? ui : input.agent ? "agent" : ci || !input.isTTY ? "plain" : "live";
1960
- const mode = requested === "live" ? "plain" : requested;
2215
+ const ui = pick("--ui", flags.ui ?? env.REEVAL_UI, ["plain", "agent"]);
2216
+ const named = ui !== void 0 && ui !== "auto";
2217
+ const mode = named ? ui : input.agent ? "agent" : "plain";
2218
+ const statusLine = !named && mode === "plain" && !ci && input.isTTY;
1961
2219
  const order = pick("--log-order", flags.logOrder ?? env.REEVAL_LOG_ORDER, ["stream", "grouped"]);
1962
2220
  const logOrder = order && order !== "auto" ? order : ci ? "grouped" : "stream";
1963
2221
  const prefix = pick("--log-prefix", flags.logPrefix ?? env.REEVAL_LOG_PREFIX, ["task", "none"]);
@@ -1968,8 +2226,8 @@ function resolveOutput(input) {
1968
2226
  "none"
1969
2227
  ]);
1970
2228
  return {
1971
- requested,
1972
2229
  mode,
2230
+ statusLine,
1973
2231
  logOrder,
1974
2232
  logPrefix,
1975
2233
  outputLogs: logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only",
@@ -2003,8 +2261,8 @@ const isDirect = (policy) => policy === "allow-all";
2003
2261
  * The hosts the harness needs: a real agent's own API (its model's provider's, for OpenCode), or
2004
2262
  * a fake agent's with `layout` (it behaves like that agent). A plain fake agent talks to nothing.
2005
2263
  */
2006
- function harnessHosts(harness) {
2007
- if (harness.kind !== "fake" && isAgentKind(harness.kind)) return agentFor(harness).hosts ?? HARNESS_HOSTS[harness.kind];
2264
+ function harnessHosts(harness, vertex) {
2265
+ if (harness.kind !== "fake" && isAgentKind(harness.kind)) return agentFor(harness, vertex).hosts ?? HARNESS_HOSTS[harness.kind];
2008
2266
  return harness.kind === "fake" && harness.layout ? HARNESS_HOSTS[harness.layout] : [];
2009
2267
  }
2010
2268
  /** MCP `http`/`sse` servers' hosts, local plugins' included: the agent connects to them from the sandbox. */
@@ -2020,7 +2278,7 @@ function mcpServerHosts(layers) {
2020
2278
  * harness's, the brokered secrets' `hosts` and the MCP servers'. The cascade is config and
2021
2279
  * profile defaults (`inherited`), then the environment's layers, then groups and the eval (`own`).
2022
2280
  */
2023
- function plannedEgress(egress, harness, env, layers) {
2281
+ function plannedEgress(egress, harness, env, layers, vertex) {
2024
2282
  const fromLayers = layers.findLast((l) => l.spec.egress !== void 0)?.spec.egress;
2025
2283
  const policy = egress.own ?? fromLayers ?? egress.inherited ?? "proxy";
2026
2284
  if (!isRestricted(policy)) return {
@@ -2029,7 +2287,7 @@ function plannedEgress(egress, harness, env, layers) {
2029
2287
  };
2030
2288
  const own = policy === "deny-all" ? [] : policy.allow;
2031
2289
  const auto = [
2032
- ...harnessHosts(harness),
2290
+ ...harnessHosts(harness, vertex),
2033
2291
  ...plannedSecrets(env, layers).flatMap((s) => s.hosts ?? []),
2034
2292
  ...mcpServerHosts(layers)
2035
2293
  ];
@@ -2104,34 +2362,28 @@ function egressLabel({ policy, auto }) {
2104
2362
  }
2105
2363
  //#endregion
2106
2364
  //#region src/judge/state-options.ts
2107
- const DEFAULT_STATE = {
2108
- task: true,
2109
- output: true,
2110
- files: true,
2111
- diff: true,
2112
- contents: [],
2113
- transcript: false,
2114
- turns: "last",
2115
- checks: false
2116
- };
2117
- /** Every key set, in `STATE_KEYS` order, so equal selections serialise the same. */
2118
- function resolveStateOptions(state) {
2119
- const out = {};
2120
- for (const key of STATE_KEYS$1) out[key] = state?.[key] ?? DEFAULT_STATE[key];
2121
- return out;
2365
+ /**
2366
+ * What one call turned on, every key set (`false` when off), in the record's key order, so equal
2367
+ * selections serialize the same.
2368
+ */
2369
+ function resolveStateOptions(call) {
2370
+ return {
2371
+ task: call.task ?? false,
2372
+ output: call.output ?? false,
2373
+ files: call.files ?? false,
2374
+ diff: call.diff ?? false,
2375
+ contents: call.contents ?? [],
2376
+ transcript: call.transcript ?? false,
2377
+ turns: call.turns ?? "last",
2378
+ checks: call.checks ?? false
2379
+ };
2122
2380
  }
2123
- /** `inner` over `outer`, field by field, and `state` key by key. */
2381
+ /** `inner` over `outer`, field by field. */
2124
2382
  function mergeJudge(outer, inner) {
2125
- if (!inner) return outer;
2126
- const out = {
2383
+ return inner ? {
2127
2384
  ...outer,
2128
2385
  ...inner
2129
- };
2130
- if (outer.state || inner.state) out.state = {
2131
- ...outer.state,
2132
- ...inner.state
2133
- };
2134
- return out;
2386
+ } : outer;
2135
2387
  }
2136
2388
  const KNOWN = [...[
2137
2389
  "harness",
@@ -2147,7 +2399,9 @@ const KNOWN = [...[
2147
2399
  "question",
2148
2400
  "questions",
2149
2401
  "timeout",
2402
+ "snapshot",
2150
2403
  "tags",
2404
+ "revision",
2151
2405
  "pin"
2152
2406
  ]];
2153
2407
  const SUGGEST = {
@@ -2160,15 +2414,35 @@ const SUGGEST = {
2160
2414
  repeats: "runs",
2161
2415
  iterations: "runs",
2162
2416
  trials: "runs",
2417
+ bestOf: "runs",
2163
2418
  secrets: "env",
2164
2419
  vars: "env",
2165
2420
  network: "egress",
2166
2421
  networkPolicy: "egress",
2167
2422
  tag: "tags",
2423
+ version: "revision",
2424
+ rev: "revision",
2168
2425
  pins: "pin",
2426
+ workspace: "snapshot",
2427
+ snapshots: "snapshot",
2169
2428
  passRate: "gate",
2170
2429
  threshold: "gate"
2171
2430
  };
2431
+ /**
2432
+ * `snapshot` over the levels above: the shorthand sets `when`, the object sets the fields it
2433
+ * names, and `compress` carries through either.
2434
+ */
2435
+ function mergeSnapshot(outer, inner) {
2436
+ if (inner === void 0) return outer;
2437
+ if (typeof inner !== "object") return {
2438
+ ...outer,
2439
+ when: inner
2440
+ };
2441
+ return {
2442
+ when: inner.when ?? outer.when,
2443
+ compress: inner.compress ?? outer.compress
2444
+ };
2445
+ }
2172
2446
  /** Throws unless `value` is `'all'` or `{ passRate }` with a rate in (0, 1]. */
2173
2447
  function checkGate(value, where) {
2174
2448
  const problem = gateProblem(value, where);
@@ -2235,18 +2509,45 @@ function refuseIllegalCompare(options, where) {
2235
2509
  const baseline = options.compare !== null && typeof options.compare === "object" && "baseline" in options.compare ? options.compare.baseline : void 0;
2236
2510
  if (options.environment !== void 0 && baseline !== void 0) throw new UsageError(`environment and compare.baseline can't both be set in ${where}. compare: { baseline, variants } names every arm, so drop environment, or drop baseline to make environment the control.`);
2237
2511
  }
2512
+ const isRevision = (value) => typeof value === "number" && Number.isSafeInteger(value) || typeof value === "string" && value.trim() !== "";
2513
+ /** `runs: 3` or `runs: { bestOf: 5 }`, refused in the words of whichever it was meant to be. */
2514
+ function checkRuns(runs, where) {
2515
+ if (typeof runs === "number" && Number.isInteger(runs) && runs >= 1) return;
2516
+ if (typeof runs === "number") throw new UsageError(`runs in ${where} must be a whole number of at least 1 (got ${JSON.stringify(runs)})`);
2517
+ if (!isObjectLike(runs)) throw new UsageError(`runs in ${where} must be a whole number of at least 1, or { bestOf: n } (got ${JSON.stringify(runs)})`);
2518
+ const extra = Object.keys(runs).find((k) => k !== "bestOf");
2519
+ if (extra !== void 0) throw new UsageError(`runs in ${where} takes { bestOf: n } only (got the key "${extra}"), e.g. runs: { bestOf: 3 }`);
2520
+ const { bestOf } = runs;
2521
+ if (!(Number.isInteger(bestOf) && bestOf >= 3 && bestOf % 2 === 1)) throw new UsageError(`runs.bestOf in ${where} must be an odd whole number of 3 or more, so one side always has a majority, e.g. runs: { bestOf: 3 } (got ${JSON.stringify(bestOf)})`);
2522
+ }
2523
+ /** The most attempts `runs` plans a cell, and its `bestOf` when it has one. */
2524
+ const runsOf = (runs) => typeof runs === "number" ? { runs } : {
2525
+ runs: runs.bestOf,
2526
+ bestOf: runs.bestOf
2527
+ };
2528
+ /**
2529
+ * A best-of cell's gate: half or more of its counted attempts pass, which is its majority once it
2530
+ * has one (an infra error counts on neither side).
2531
+ */
2532
+ const MAJORITY_GATE = { passRate: .5 };
2533
+ /** How many of a best-of cell's attempts always run: the majority. */
2534
+ const majorityOf = (bestOf) => (bestOf + 1) / 2;
2238
2535
  /** Throws on keys that aren't options, and on values of the wrong shape, suggesting a fix. */
2239
2536
  function checkOptions(options, where, allowed = KNOWN) {
2240
2537
  for (const key of Object.keys(options)) {
2241
2538
  if (allowed.includes(key)) continue;
2242
2539
  const near = SUGGEST[key] ?? closest(key, allowed);
2243
- const hint = key === "pool" ? " pool is an option of evaluate.each, beside the cases: evaluate.each(cases, { pool: false })." : near ? ` Did you mean "${near}"?` : "";
2540
+ const value = options[key];
2541
+ const hint = key === "pool" ? " pool is an option of evaluate.each, beside the cases: evaluate.each(cases, { pool: false })." : key === "workspace" && typeof value !== "boolean" ? " workspace is an environment's starting tree: defineEnvironment(name, { workspace: git(…) })." : near ? ` Did you mean "${near}"?` : "";
2244
2542
  throw new UsageError(`Unknown option "${key}" in ${where}.${hint}`);
2245
2543
  }
2246
2544
  const o = options;
2247
- if (o.runs !== void 0 && !(Number.isInteger(o.runs) && o.runs >= 1)) throw new UsageError(`runs in ${where} must be a whole number of at least 1 (got ${JSON.stringify(o.runs)})`);
2545
+ if (o.runs !== void 0) checkRuns(o.runs, where);
2248
2546
  if (o.gate !== void 0) checkGate(o.gate, where);
2249
2547
  if (o.tags !== void 0 && (!Array.isArray(o.tags) || o.tags.some((t) => typeof t !== "string"))) throw new UsageError(`tags in ${where} must be a list of strings, e.g. tags: ['smoke'] (got ${JSON.stringify(o.tags)})`);
2548
+ if (o.revision !== void 0 && !isRevision(o.revision)) throw new UsageError(`revision in ${where} must be a whole number or a non-empty string, e.g. revision: 2 (got ${JSON.stringify(o.revision)})`);
2549
+ const snapshotIssue = snapshotProblem(o.snapshot, where);
2550
+ if (snapshotIssue) throw new UsageError(snapshotIssue);
2250
2551
  if (o.env !== void 0) checkEnv(o.env, where, UsageError);
2251
2552
  if (o.egress !== void 0) checkEgress(o.egress, where, UsageError);
2252
2553
  const judgeIssue = o.judge !== void 0 && judgeProblem(o.judge, {
@@ -2377,17 +2678,18 @@ function checkUnique(list, where) {
2377
2678
  seen.add(key);
2378
2679
  }
2379
2680
  }
2380
- const plannedQuestions = (value) => value.map((q, i) => typeof q === "string" ? {
2381
- n: i + 1,
2382
- question: q
2383
- } : {
2384
- n: i + 1,
2385
- question: q.question,
2386
- ...q.model !== void 0 ? { model: q.model } : {},
2387
- ...q.across !== void 0 ? { across: q.across } : {},
2388
- ...q.choices ? { choices: q.choices } : {},
2389
- ...q.state ? { state: q.state } : {},
2390
- ...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
2681
+ const plannedQuestions = (value) => value.map((q, i) => {
2682
+ const { question, model, across, choices, artifacts, ...rest } = q;
2683
+ const state = Object.fromEntries(Object.entries(rest).filter(([, v]) => v !== void 0));
2684
+ return {
2685
+ n: i + 1,
2686
+ question,
2687
+ ...model !== void 0 ? { model } : {},
2688
+ ...across !== void 0 ? { across } : {},
2689
+ ...choices ? { choices } : {},
2690
+ state,
2691
+ ...artifacts !== void 0 ? { artifacts } : {}
2692
+ };
2391
2693
  });
2392
2694
  /** Whether the eval's matrix varies each factor a pairwise question can vary. */
2393
2695
  const variesBy = (r) => ({
@@ -2412,6 +2714,12 @@ function normalizeCompare(value) {
2412
2714
  of: value.variants.length
2413
2715
  };
2414
2716
  }
2717
+ /**
2718
+ * Runs per cell when no level sets `runs`: a compare with variants runs `EFFECT_MIN_COUNTED` a
2719
+ * side, the fewest whose lift can reach a verdict; anything else, a pooled family's cases
2720
+ * included, runs once.
2721
+ */
2722
+ const COMPARE_DEFAULT_RUNS = 4;
2415
2723
  function applyLevel(out, options, where, egress = "own") {
2416
2724
  for (const [key, value] of Object.entries(options)) if (value !== void 0) out.sources = {
2417
2725
  ...out.sources,
@@ -2422,9 +2730,15 @@ function applyLevel(out, options, where, egress = "own") {
2422
2730
  if (options.sandbox !== void 0) out.sandbox = options.sandbox;
2423
2731
  if (options.environment !== void 0) out.environment = options.environment;
2424
2732
  if (options.compare !== void 0) out.compare = normalizeCompare(options.compare);
2425
- if (options.runs !== void 0) out.runs = options.runs;
2733
+ if (options.runs !== void 0) {
2734
+ const { runs, bestOf } = runsOf(options.runs);
2735
+ out.runs = runs;
2736
+ out.bestOf = bestOf;
2737
+ }
2426
2738
  if (options.gate !== void 0) out.gate = options.gate;
2427
2739
  if (options.timeout !== void 0) out.timeout = options.timeout;
2740
+ out.snapshot = mergeSnapshot(out.snapshot, options.snapshot);
2741
+ if (options.revision !== void 0) out.revision = options.revision;
2428
2742
  Object.assign(out.env, options.env);
2429
2743
  out.judge = mergeJudge(out.judge, options.judge);
2430
2744
  if (options.capture !== void 0) {
@@ -2470,11 +2784,7 @@ function cascade(base, levels, where, options = {}) {
2470
2784
  baseline: out.environment,
2471
2785
  from: "environment"
2472
2786
  };
2473
- if (out.questions.length) {
2474
- out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
2475
- const blank = out.questions.find(showsNothing);
2476
- if (blank) throw new UsageError(`questions "${blank.question}" in ${where} shows the model nothing: every state key is off and it sends no artifacts. Turn on a state key, such as state: { diff: true }, or send artifacts.`);
2477
- }
2787
+ if (out.questions.length) out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
2478
2788
  checkUnique(out.harness, where);
2479
2789
  const harnessFromProfile = profile?.defaults.harness !== void 0 && !levels.some((l) => l.options.harness);
2480
2790
  const authored = variesBy(out);
@@ -2500,23 +2810,30 @@ function cascade(base, levels, where, options = {}) {
2500
2810
  const dupe = names.find((n, i) => names.indexOf(n) !== i);
2501
2811
  if (dupe) throw new UsageError(`compare in ${where} lists two variants named ${dupe}. Give each variant its own name.`);
2502
2812
  let skip;
2813
+ const harnesses = out.harness.map(harnessId);
2503
2814
  const limit = profile?.limit;
2504
2815
  if (profile && limit) {
2505
2816
  const tag = `profile ${profile.name}`;
2506
2817
  const pinned = (key) => out.pin.includes(key);
2507
- if (limit.runs !== void 0 && options.runs === void 0 && !pinned("runs") && out.runs > limit.runs) {
2508
- notes.push(`runs ${out.runs} → ${limit.runs} (${tag})`);
2509
- out.runs = limit.runs;
2510
- }
2818
+ let comparedLeft = true;
2511
2819
  if (limit.harness && !pinned("harness")) {
2512
2820
  const kept = out.harness.filter((h) => limit.harness?.includes(harnessId(h)));
2513
2821
  if (kept.length === 0) skip = `no harness left: ${tag} allows ${limit.harness.join(", ") || "none"}`;
2514
2822
  else if (kept.length < out.harness.length) {
2515
2823
  notes.push(`harnesses ${out.harness.length} → ${kept.length} (${tag})`);
2516
2824
  out.harness = kept;
2825
+ const scope = out.compare?.harness;
2826
+ if (out.compare?.variants.length && scope && !kept.some((h) => scope.includes(harnessId(h)))) {
2827
+ notes.push(`compare off: no compared harness left (${tag})`);
2828
+ out.compare = {
2829
+ ...out.compare,
2830
+ variants: []
2831
+ };
2832
+ comparedLeft = false;
2833
+ }
2517
2834
  }
2518
2835
  }
2519
- if (out.compare && limit.compare !== void 0 && !pinned("compare")) {
2836
+ if (out.compare && comparedLeft && limit.compare !== void 0 && !pinned("compare")) {
2520
2837
  if (limit.compare === false) {
2521
2838
  notes.push(`compare off (${tag})`);
2522
2839
  out.compare = {
@@ -2531,7 +2848,18 @@ function cascade(base, levels, where, options = {}) {
2531
2848
  };
2532
2849
  }
2533
2850
  }
2534
- const pins = out.pin.filter((k) => k === "compare" ? limit.compare !== void 0 && out.compare : limit[k] !== void 0);
2851
+ defaultRuns(out, notes, options);
2852
+ if (limit.runs !== void 0 && options.runs === void 0 && !pinned("runs") && (out.runs ?? 1) > limit.runs) {
2853
+ const was = out.bestOf !== void 0 ? `best of ${out.bestOf}` : String(out.runs);
2854
+ notes.push(`runs ${was} → ${limit.runs} (${tag})`);
2855
+ out.runs = limit.runs;
2856
+ out.bestOf = void 0;
2857
+ out.sources = {
2858
+ ...out.sources,
2859
+ runs: `profiles.${profile.name}.limit`
2860
+ };
2861
+ }
2862
+ const pins = out.pin.filter((k) => k === "compare" ? limit.compare !== void 0 && out.compare && comparedLeft : limit[k] !== void 0);
2535
2863
  if (pins.length) notes.push(`pinned: ${pins.join(", ")}`);
2536
2864
  }
2537
2865
  if (profile) {
@@ -2554,13 +2882,59 @@ function cascade(base, levels, where, options = {}) {
2554
2882
  if (narrowed) notes.push(`${plural(narrowed)} narrowed (profile ${profile.name})`);
2555
2883
  out.questions = kept;
2556
2884
  }
2557
- if (options.runs !== void 0) out.runs = options.runs;
2885
+ if (options.runs !== void 0) out.bestOf = void 0;
2886
+ if (out.bestOf !== void 0 && out.compare?.variants.length) {
2887
+ const from = out.sources?.runs ?? "the config";
2888
+ const set = from === where ? "" : ` from ${from}`;
2889
+ throw new UsageError(`${where} compares environments, so it can't take runs: { bestOf: ${out.bestOf} }${set}: a lift needs the same number of attempts on each side, and best-of stops each cell on its own. Set a fixed count on it, e.g. runs: 4.`);
2890
+ }
2891
+ if (out.bestOf !== void 0) {
2892
+ if (out.gate !== void 0 && gateLabel(out.gate) !== gateLabel(MAJORITY_GATE)) notes.push(`gate ${gateLabel(out.gate)} → majority (runs.bestOf)`);
2893
+ out.gate = MAJORITY_GATE;
2894
+ }
2895
+ defaultRuns(out, notes, options);
2558
2896
  return {
2559
- resolved: out,
2897
+ resolved: {
2898
+ ...out,
2899
+ runs: options.runs ?? out.runs ?? 1
2900
+ },
2560
2901
  notes,
2902
+ harnesses,
2561
2903
  ...skip ? { skip } : {}
2562
2904
  };
2563
2905
  }
2906
+ /** Gives `out` its default runs when no level set them and `--runs` doesn't replace them. */
2907
+ function defaultRuns(out, notes, options) {
2908
+ if (out.runs !== void 0 || options.runs !== void 0) return;
2909
+ if (!out.compare?.variants.length || options.pooled) {
2910
+ out.runs = 1;
2911
+ return;
2912
+ }
2913
+ out.runs = COMPARE_DEFAULT_RUNS;
2914
+ out.sources = {
2915
+ ...out.sources,
2916
+ runs: `compare default: ${COMPARE_DEFAULT_RUNS} a side`
2917
+ };
2918
+ notes.push(`runs ${COMPARE_DEFAULT_RUNS} (compare default)`);
2919
+ }
2920
+ /**
2921
+ * Refuses a profile's `limit.harness` id that no collected eval runs on: it keeps nothing, and a
2922
+ * limit that keeps nothing skips every eval. `harnesses` are the ids every eval in the loaded
2923
+ * files runs on before the limit, whatever the tag and name filters pick. When a path filter
2924
+ * loaded only some files (`partial`), an id may name a harness of the others, so only a limit
2925
+ * none of whose ids match is refused.
2926
+ */
2927
+ function checkLimitHarness(profile, configFile, collected) {
2928
+ const ids = profile?.limit.harness;
2929
+ if (!profile || !ids?.length) return;
2930
+ const unknown = ids.filter((id) => !collected.harnesses.includes(id));
2931
+ if (!unknown.length || collected.partial && unknown.length < ids.length) return;
2932
+ const listed = unknown.map((id) => JSON.stringify(id)).join(", ");
2933
+ const have = [...collected.harnesses].sort();
2934
+ const evals = collected.partial ? "eval in the selected files" : "eval";
2935
+ const the = collected.partial ? "the evals in the selected files" : "the evals";
2936
+ throw new UsageError(`profiles.${profile.name}.limit.harness in ${configFile} lists ${listed}, which no ${evals} runs on. Harness ids ${the} run on: ${have.join(", ") || "none"}.`);
2937
+ }
2564
2938
  /**
2565
2939
  * The sandbox an attempt runs in: the resolved environment's (including an `.extend()`
2566
2940
  * parent's), else the first variant's when this is an omitted-baseline empty control, else
@@ -2594,8 +2968,11 @@ function sandboxSetBy(planned, resolved, configFile) {
2594
2968
  }
2595
2969
  return resolved.sources?.environment ?? configFile;
2596
2970
  }
2597
- /** Per harness: the control, then each variant it compares on; then run numbers. */
2598
- function expand(resolved) {
2971
+ /**
2972
+ * Per harness: the control, then each variant it compares on; then run numbers. `vertex` is the
2973
+ * run's `vertex()` settings, for each harness that resolved to Vertex AI.
2974
+ */
2975
+ function expand(resolved, vertex) {
2599
2976
  const { compare } = resolved;
2600
2977
  const control = applied(compare ? compare.baseline : resolved.environment);
2601
2978
  const policy = resolved.gate ?? "all";
@@ -2603,6 +2980,7 @@ function expand(resolved) {
2603
2980
  const attempts = [];
2604
2981
  for (const harness of resolved.harness) {
2605
2982
  const hid = harnessId(harness);
2983
+ const onVertex = harness.gateway === "vertex" && harness.kind !== "fake" ? vertex : void 0;
2606
2984
  const cells = [(() => {
2607
2985
  const sandbox = attemptSandbox(resolved, control);
2608
2986
  const sid = sandboxId(sandbox);
@@ -2646,11 +3024,13 @@ function expand(resolved) {
2646
3024
  harness,
2647
3025
  sandbox,
2648
3026
  environment: env,
2649
- egress: plannedEgress(resolved.egress, harness, resolved.env, env.layers),
3027
+ egress: plannedEgress(resolved.egress, harness, resolved.env, env.layers, onVertex),
2650
3028
  gate,
2651
3029
  ...cell.role === "variant" && compare?.minLift !== void 0 ? { minLift: compare.minLift } : {},
3030
+ ...resolved.bestOf !== void 0 ? { bestOf: resolved.bestOf } : {},
2652
3031
  n,
2653
- label
3032
+ label,
3033
+ ...onVertex ? { vertex: onVertex } : {}
2654
3034
  });
2655
3035
  }
2656
3036
  }
@@ -2698,9 +3078,18 @@ function unbrokeredOnFake(env, attempts) {
2698
3078
  }
2699
3079
  /** The host variables an attempt's secrets read, brokered or not. */
2700
3080
  const secretVarsIn = (env, attempt) => [...new Set(plannedSecrets(env, attempt.environment.layers).map((s) => s.from))];
2701
- const brokerHint = (s) => /^mcp\.[^.]+\.headers\./.test(s.label) ? "give the MCP server an https:// URL (reeval brokers its header secrets to its own host, over HTTPS only)" : `add the hosts it's sent to, e.g. secret.fromEnv('${s.from}', { hosts: ['api.example.com'] })`;
2702
- const unbrokeredWarning = (s) => s.checksOnly ? `${s.label} reaches the checks (c.exec, setup steps) as its real value; the agent never gets it. To give the checks only a placeholder, ${brokerHint(s)}.` : `${s.label} is in the sandbox as its real value. To give the agent only a placeholder, ${brokerHint(s)}.`;
3081
+ const brokerHint = (s) => /^mcp\.[^.]+\.headers\./.test(s.label) ? "give the MCP server an https:// URL (reeval brokers its header secrets over HTTPS only, to its own host unless they set hosts)" : `add the hosts it's sent to, e.g. secret.fromEnv('${s.from}', { hosts: ['api.example.com'] })`;
3082
+ const unbrokeredWarning = (s) => s.checksOnly ? `${s.label} reaches the checks (c.exec, setup steps) as its real value. It's never in the agent's own environment, but a process a setup step leaves running holds it while the agent works. To give the checks only a placeholder, ${brokerHint(s)}.` : `${s.label} is in the sandbox as its real value. To give the agent only a placeholder, ${brokerHint(s)}.`;
2703
3083
  const unbrokeredError = (s, where, configFile) => `${s.label} in ${where} isn't brokered, and secrets.unbrokered is 'error' in ${configFile}. To broker it, ${brokerHint(s)}.`;
3084
+ /** The attempts' MCP headers written as plain strings that look like credentials, once each. */
3085
+ function plainHeaders(attempts) {
3086
+ const out = /* @__PURE__ */ new Map();
3087
+ for (const a of attempts) for (const h of plainCredentialHeaders(a.environment.layers)) out.set(h.label, h);
3088
+ return [...out.values()];
3089
+ }
3090
+ const templateHint = (h) => `read it from a host variable with secret.template, e.g. secret.template('Bearer \${VAR}')${h.https ? "" : " and give the MCP server an https:// URL"}`;
3091
+ const plainHeaderWarning = (h) => `${h.label} is a plain string, so it's in the sandbox as its real value. To give the agent only a placeholder, ${templateHint(h)}.`;
3092
+ const plainHeaderError = (h, where, configFile) => `${h.label} in ${where} is a plain string, and secrets.unbrokered is 'error' in ${configFile}. To broker it, ${templateHint(h)}.`;
2704
3093
  const levelOf = (where) => where.startsWith("evaluate(") ? "eval" : where.startsWith("group(") ? "group" : where.startsWith("profiles.") ? "profile" : "config";
2705
3094
  /**
2706
3095
  * A warning per privilege an attempt's Docker sandbox adds, naming the level that set the
@@ -2767,6 +3156,7 @@ function evalPlan(r, attempts, origin) {
2767
3156
  } } : {},
2768
3157
  ...!skip && resolved.questions.length ? { questions: resolved.questions } : {},
2769
3158
  runs: resolved.runs,
3159
+ ...resolved.bestOf !== void 0 ? { bestOf: resolved.bestOf } : {},
2770
3160
  ...resolved.gate !== void 0 ? { gate: resolved.gate } : {},
2771
3161
  attempts: skip ? 0 : attempts.length,
2772
3162
  notes,
@@ -2774,7 +3164,8 @@ function evalPlan(r, attempts, origin) {
2774
3164
  warnings: [
2775
3165
  ...environmentWarnings(attempts),
2776
3166
  ...skip ? [] : questionWarnings(resolved),
2777
- ...unbrokered(r.resolved.env, attempts).filter(reachesSandbox).map(unbrokeredWarning)
3167
+ ...unbrokered(r.resolved.env, attempts).filter(reachesSandbox).map(unbrokeredWarning),
3168
+ ...plainHeaders(attempts).map(plainHeaderWarning)
2778
3169
  ],
2779
3170
  sandboxWarnings: skip ? [] : sandboxWarnings(attempts, resolved, origin),
2780
3171
  brokered: [...new Set(attempts.flatMap((a) => brokeredIn(r.resolved.env, a)))],
@@ -2789,7 +3180,7 @@ function evalPlan(r, attempts, origin) {
2789
3180
  /** A pairwise question that shows each side's transcript, asked of two harnesses: tool names unblind it. */
2790
3181
  function questionWarnings(resolved) {
2791
3182
  if (resolved.harness.length < 2) return [];
2792
- return resolved.questions.filter((q) => pairFactors(q.across).includes("harness") && q.state?.transcript).map((q) => `question "${q.question}" shows state.transcript, which names each harness's tools, so its harness pairs aren't blind. Ask it with across: 'environment', or leave the transcript out.`);
3183
+ return resolved.questions.filter((q) => pairFactors(q.across).includes("harness") && q.state?.transcript).map((q) => `question "${q.question}" shows the transcript, which names each harness's tools, so its harness pairs aren't blind. Ask it with across: 'environment', or leave the transcript out.`);
2793
3184
  }
2794
3185
  function captureNote(c) {
2795
3186
  const parts = [];
@@ -2806,23 +3197,47 @@ function matchesTags(tags, filters) {
2806
3197
  }
2807
3198
  //#endregion
2808
3199
  //#region src/gateway/resolve.ts
2809
- /** The config's list when it sets none: each provider's own key, as reeval has always run. */
3200
+ /** The config's list when it sets none: each provider's own key. */
2810
3201
  const DEFAULT_GATEWAYS = [{ kind: "direct" }];
2811
3202
  /** Every gateway reeval can route through, by name. */
2812
3203
  const GATEWAYS = {
2813
3204
  direct: { cantServe: (h) => {
2814
- if (h.kind === "fx" && h.model.includes("/")) return `fx runs a gateway id such as ${JSON.stringify(h.model)} only through Vercel AI Gateway`;
3205
+ const fxEffort = fxEffortRefusal(h, "direct()");
3206
+ if (fxEffort) return fxEffort;
3207
+ if (h.kind === "fx" && h.model.includes("/")) return `fx runs a gateway id such as ${JSON.stringify(h.model)} only through Vercel AI Gateway or OpenRouter`;
2815
3208
  if (h.kind === "opencode") {
2816
3209
  const resolved = resolveOpenCodeProvider(h.model, process.env);
2817
3210
  if ("error" in resolved) return resolved.error;
2818
3211
  }
2819
3212
  } },
2820
- vercel: { cantServe: (h) => {
2821
- if (h.kind !== "fake" && !VERCEL_ROUTES[h.kind]) return VERCEL_REFUSALS[h.kind] ?? `reeval doesn't run ${agentName(h)} on Vercel AI Gateway yet`;
2822
- const effort = harnessEffort(h);
2823
- if (effort !== void 0 && h.kind !== "fake" && VERCEL_UNPROVEN_EFFORT.includes(h.kind)) return `reeval doesn't send ${agentName(h)}'s effort (${JSON.stringify(effort)}) through Vercel AI Gateway yet: nothing checks the model gets it as its own setting, so drop effort or run ${agentName(h)} on another gateway`;
3213
+ vercel: routingEntry("vercel"),
3214
+ openrouter: routingEntry("openrouter"),
3215
+ vertex: { cantServe: (h) => {
3216
+ if (h.kind === "fake") return void 0;
3217
+ if (!VERTEX_ROUTES[h.kind]) return VERTEX_REFUSALS[h.kind] ?? `reeval doesn't run ${agentName(h)} on Vertex AI yet`;
3218
+ return vertexModelRefusalFor(h);
2824
3219
  } }
2825
3220
  };
3221
+ /**
3222
+ * Why fx can't take `effort` on a gateway: fx sends none itself, and reeval adds it as
3223
+ * `reasoning_effort` only to its calls on Vertex AI's Chat Completions endpoint.
3224
+ */
3225
+ function fxEffortRefusal(h, gateway) {
3226
+ const effort = harnessEffort(h);
3227
+ if (h.kind !== "fx" || effort === void 0) return void 0;
3228
+ return `fx sends no effort of its own, and reeval adds fx's effort (${JSON.stringify(effort)}) only to its calls on Vertex AI, not on ${gateway}: drop effort or run fx on vertex()`;
3229
+ }
3230
+ /** A routing gateway's entry: its route table decides whom it serves, and with what effort. */
3231
+ function routingEntry(name) {
3232
+ const gateway = ROUTING_GATEWAYS[name];
3233
+ return { cantServe: (h) => {
3234
+ const fxEffort = fxEffortRefusal(h, gateway.name);
3235
+ if (fxEffort) return fxEffort;
3236
+ if (h.kind !== "fake" && !gateway.routes[h.kind]) return gateway.refusals[h.kind] ?? `reeval doesn't run ${agentName(h)} on ${gateway.name} yet`;
3237
+ const effort = harnessEffort(h);
3238
+ if (effort !== void 0 && h.kind !== "fake" && gateway.unprovenEffort[h.kind]?.(h.model)) return `reeval doesn't send ${agentName(h)}'s effort (${JSON.stringify(effort)}) to ${h.model} through ${gateway.name} yet: nothing checks the model gets it as its own setting, so drop effort or run ${agentName(h)} on another gateway`;
3239
+ } };
3240
+ }
2826
3241
  /** The `reeval/gateway` helper that makes the gateway `name`, as messages name it. */
2827
3242
  const gatewayHelper = (name) => `${name}()`;
2828
3243
  /** The agent's name, for messages. */
@@ -2837,7 +3252,7 @@ function resolveGateway(gateways, harness, where, catalog = GATEWAYS) {
2837
3252
  const why = (name) => catalog[name]?.cantServe(harness);
2838
3253
  const pin = harness.gateway;
2839
3254
  if (pin !== void 0) {
2840
- if (!gateways.some((g) => g.kind === pin)) throw new UsageError(`${on}: it pins gateway '${pin}', which isn't in the gateway list (${listed}). Add ${gatewayHelper(pin)} to gateway, or pin one that's there.`);
3255
+ if (!gateways.some((g) => g.kind === pin)) throw new UsageError(`${on}: it pins gateway '${pin}', ${notListed(pin, listed)}`);
2841
3256
  const refused = why(pin);
2842
3257
  if (refused) throw new UsageError(`${on}: it pins gateway '${pin}', which can't serve it (${refused}). A pin never falls back; pin another gateway, or remove the pin to take the first in the list that can.`);
2843
3258
  return {
@@ -2859,10 +3274,13 @@ function resolveGateway(gateways, harness, where, catalog = GATEWAYS) {
2859
3274
  }
2860
3275
  throw new UsageError(`${on}: no gateway in the list can serve it (${skipped.map((s) => `${s.gateway}: ${s.why}`).join("; ")}). ${addHint(gateways, harness, catalog)}`);
2861
3276
  }
2862
- /** What to add to the list so it serves `harness`: `direct()` when it can, else the first that can. */
3277
+ /**
3278
+ * What to change so the list serves `harness`: a list without `direct` has it turned off, so
3279
+ * turning it back on when it can serve, else adding the first gateway that can.
3280
+ */
2863
3281
  function addHint(gateways, harness, catalog) {
2864
3282
  const serves = Object.keys(catalog).filter((name) => !gateways.some((g) => g.kind === name)).filter((name) => !catalog[name]?.cantServe(harness));
2865
- if (serves.includes("direct")) return `Add direct() to the end of gateway to use ${agentName(harness)}'s own key.`;
3283
+ if (serves.includes("direct")) return `Drop direct({ enabled: false }) from gateway to fall back to ${agentName(harness)}'s own key.`;
2866
3284
  const [first] = serves;
2867
3285
  return first ? `Add ${gatewayHelper(first)} to gateway.` : `No gateway reeval knows can serve it.`;
2868
3286
  }
@@ -2881,30 +3299,40 @@ function routedHarness(harness, gateways) {
2881
3299
  return harness;
2882
3300
  }
2883
3301
  }
2884
- /** `gateway: claude-code:… → direct (vercel can't serve it: …)`, for the plan's hints. */
2885
- const fallbackHint = (harness, resolved) => `gateway: ${harness} → ${resolved.name} (${resolved.skipped.map((s) => `${s.gateway} can't serve it: ${s.why}`).join("; ")})`;
3302
+ /**
3303
+ * Why a pin names a gateway the list doesn't have: a list without `direct` has it turned off.
3304
+ */
3305
+ const notListed = (pin, listed) => pin === "direct" ? `which the gateway list (${listed}) turns off with direct({ enabled: false }). Drop that, or pin one that's there.` : `which isn't in the gateway list (${listed}). Add ${gatewayHelper(pin)} to gateway, or pin one that's there.`;
3306
+ /**
3307
+ * `gateway: cursor:… falls back to direct (vercel can't serve it: …)`, or `→ vercel (…)` past
3308
+ * another gateway, for the plan's hints.
3309
+ */
3310
+ const fallbackHint = (harness, resolved) => `gateway: ${harness} ${resolved.name === "direct" ? "falls back to" : "→"} ${resolved.name} (${resolved.skipped.map((s) => `${s.gateway} can't serve it: ${s.why}`).join("; ")})`;
2886
3311
  //#endregion
2887
3312
  //#region src/gateway/models.ts
2888
3313
  const providerOf = (model) => model.slice(0, model.indexOf("/"));
2889
3314
  /** Why each gateway can't serve a model by name, or undefined when it can. */
2890
3315
  const MODEL_GATEWAYS = {
2891
3316
  direct: (model) => DIRECT_PROVIDERS[providerOf(model)] ? void 0 : `reeval calls only ${Object.keys(DIRECT_PROVIDERS).map((p) => `${p}/*`).join(", ")} models on their own keys`,
2892
- vercel: () => void 0
3317
+ vercel: () => void 0,
3318
+ openrouter: () => void 0,
3319
+ vertex: vertexModelRefusal
2893
3320
  };
2894
3321
  const gatewayScope = (config, profile) => ({ gateways: profile?.gateway ?? config.gateway });
2895
3322
  /**
2896
- * The gateway `model` goes through: its pin, or the first in the list that can serve it. An unset
2897
- * list is `[direct()]`, so a model takes its provider's own key unless the config lists another
2898
- * gateway. A model object or a fake calls through itself, and is undefined. Throws a UsageError
3323
+ * The gateway `model` goes through: its pin, or the first in the list that can serve it. A list
3324
+ * ends with `direct()` unless it's turned off, so a model no other gateway serves takes its
3325
+ * provider's own key. A model object or a fake calls through itself, and is undefined. Throws a UsageError
2899
3326
  * naming each gateway tried.
2900
3327
  */
2901
3328
  function resolveModelGateway(model, pin, scope, what) {
2902
3329
  if (typeof model !== "string" || !model.includes("/")) return void 0;
2903
3330
  const on = `${what} "${model}"`;
2904
3331
  const listed = scope.gateways.map((g) => g.kind);
3332
+ if (typeof pin === "object" && pin.kind === "direct" && pin.enabled === false) throw new UsageError(directOffPin(on));
2905
3333
  const name = typeof pin === "object" ? pin.kind : pin;
2906
3334
  if (name !== void 0) {
2907
- if (!listed.includes(name)) throw new UsageError(`${on} pins gateway '${name}', which isn't in the gateway list (${listed.join(", ")}). Add ${gatewayHelper(name)} to gateway, or pin one that's there.`);
3335
+ if (!listed.includes(name)) throw new UsageError(`${on} pins gateway '${name}', ${notListed(name, listed.join(", "))}`);
2908
3336
  const refused = MODEL_GATEWAYS[name](model);
2909
3337
  if (refused) throw new UsageError(`${on} pins gateway '${name}', which can't serve it (${refused}). A pin never falls back; pin another gateway, or remove the pin to take the first in the list that can.`);
2910
3338
  return name;
@@ -2916,9 +3344,11 @@ function resolveModelGateway(model, pin, scope, what) {
2916
3344
  skipped.push(`${kind}: ${refused}`);
2917
3345
  }
2918
3346
  const other = GATEWAY_NAMES.find((n) => !listed.includes(n) && !MODEL_GATEWAYS[n](model));
2919
- const judge = other === "vercel" && what.includes("judge") ? ` There a judge must be a decision model, such as '${RECOMMENDED_JUDGE_MODEL}'.` : "";
2920
- throw new UsageError(`${on}: no gateway in the list can serve it (${skipped.join("; ")}).${other ? ` Add ${gatewayHelper(other)} to gateway.${judge}` : ""}`);
3347
+ const judge = other && other !== "direct" && what.includes("judge") ? ` There a judge must be a decision model, such as '${other === "openrouter" ? OPENROUTER_JUDGE_MODEL : RECOMMENDED_JUDGE_MODEL}'.` : "";
3348
+ throw new UsageError(`${on}: no gateway in the list can serve it (${skipped.join("; ")}).${other === "direct" ? ` Drop direct({ enabled: false }) from gateway to fall back to its provider's own key.` : other ? ` Add ${gatewayHelper(other)} to gateway.${judge}` : ""}`);
2921
3349
  }
3350
+ /** Whether an unpinned model resolved to `direct` past the list's other entries, for the plan and doctor. */
3351
+ const fellBackToDirect = (gateway, pin, scope) => gateway === "direct" && pin === void 0 && scope.gateways[0]?.kind !== "direct";
2922
3352
  /** As `resolveModelGateway`, but undefined where it would throw, for checks the plan already makes. */
2923
3353
  function modelGatewayOrNone(model, pin, scope) {
2924
3354
  try {
@@ -2943,6 +3373,7 @@ const CONFIG_KEYS = [
2943
3373
  "capture",
2944
3374
  "question",
2945
3375
  "timeout",
3376
+ "snapshot",
2946
3377
  "concurrency",
2947
3378
  "record",
2948
3379
  "history",
@@ -2973,7 +3404,8 @@ const DEFAULTS_KEYS = [
2973
3404
  "judge",
2974
3405
  "capture",
2975
3406
  "question",
2976
- "timeout"
3407
+ "timeout",
3408
+ "snapshot"
2977
3409
  ];
2978
3410
  const LIMIT_KEYS = [
2979
3411
  "runs",
@@ -3133,7 +3565,7 @@ function configFromData(data) {
3133
3565
  * Resolves a config in its data form (a parsed `reeval.yaml`, or `ReevalConfigData` from another
3134
3566
  * language) whose relative host paths resolve against `root`, its directory: `resolveConfig`'s
3135
3567
  * checks name a mistake in their own words first, and the config contract refuses anything they
3136
- * let through, such as a `null` or a misspelt nested key.
3568
+ * let through, such as a `null` or a misspelled nested key.
3137
3569
  */
3138
3570
  function resolveConfigData(input, root, file) {
3139
3571
  if (!isRecord(input)) throw new UsageError(`${file} must be a mapping of config keys, like harness: and judge: (got ${JSON.stringify(input)})`);
@@ -3237,12 +3669,13 @@ function checkProfile(name, profile, file) {
3237
3669
  };
3238
3670
  return p;
3239
3671
  }
3240
- const GATEWAY_EXAMPLE = "gateway: [direct()]";
3672
+ const GATEWAY_EXAMPLE = "gateway: [vercel()]";
3241
3673
  const isGatewayName = (v) => typeof v === "string" && GATEWAY_NAMES.includes(v);
3242
3674
  /**
3243
3675
  * A `gateway` list as descriptors: each entry is a helper's descriptor, or as data its name or a
3244
- * one-key map from its name to its settings. Refuses an empty list, an unknown entry and a
3245
- * gateway listed twice.
3676
+ * one-key map from its name to its settings. `direct()` ends the list unless it's listed in
3677
+ * another place, and `direct({ enabled: false })` leaves it out, so the resolved list is the one
3678
+ * every route walks. Refuses an empty list, an unknown entry and a gateway listed twice.
3246
3679
  */
3247
3680
  function checkGateways(value, where, file) {
3248
3681
  if (!Array.isArray(value) || value.length === 0) throw new UsageError(`${where} in ${file} must be a list of gateways in the order you prefer them, e.g. ${GATEWAY_EXAMPLE} (got ${JSON.stringify(value)})`);
@@ -3273,10 +3706,20 @@ function checkGateways(value, where, file) {
3273
3706
  const kind = isRecord(descriptor) ? descriptor.kind : void 0;
3274
3707
  throw new UsageError(isGatewayName(kind) ? `${at} isn't a valid ${gatewayHelper(kind)}: ${parsed.error.issues[0]?.message ?? "unknown settings"} (got ${JSON.stringify(entry)})` : `${at} must be a gateway, e.g. direct() from 'reeval/gateway' (got ${describeValue(entry)})`);
3275
3708
  }
3276
- if (out.some((g) => g.kind === parsed.data.kind)) throw new UsageError(`${at} lists ${parsed.data.kind} a second time. Each gateway is listed once, where you prefer it.`);
3709
+ const twice = out.find((g) => g.kind === parsed.data.kind);
3710
+ if (twice?.kind === "direct" && parsed.data.kind === "direct") {
3711
+ if (twice.enabled === false !== (parsed.data.enabled === false)) throw new UsageError(`${at} lists direct() and direct({ enabled: false }). direct() places the fallback to each provider's own key and direct({ enabled: false }) turns it off: keep one.`);
3712
+ }
3713
+ if (twice) throw new UsageError(`${at} lists ${parsed.data.kind} a second time. Each gateway is listed once, where you prefer it.`);
3277
3714
  out.push(parsed.data);
3278
3715
  });
3279
- return out;
3716
+ const direct = out.find((g) => g.kind === "direct");
3717
+ if (direct?.enabled === false) {
3718
+ const rest = out.filter((g) => g.kind !== "direct");
3719
+ if (!rest.length) throw new UsageError(`${where} in ${file} turns direct() off and lists no other gateway, so no model call has one. Add a gateway such as vercel(), or drop direct({ enabled: false }).`);
3720
+ return rest;
3721
+ }
3722
+ return direct ? out.map((g) => g.kind === "direct" ? { kind: "direct" } : g) : [...out, { kind: "direct" }];
3280
3723
  }
3281
3724
  /** `--profile`, then `REEVAL_PROFILE`, then `defaultProfile.ci` in CI (or `.local`), then none. */
3282
3725
  function selectProfile(config, flag, env = process.env) {
@@ -3318,7 +3761,7 @@ function selectProfile(config, flag, env = process.env) {
3318
3761
  function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3319
3762
  const user = withHostPaths(input, root, file);
3320
3763
  if ("compare" in user) throw new UsageError(`compare can't be set in ${file}: it would multiply every eval. Put it on the group or eval that asks the question.`);
3321
- if ("questions" in user) throw new UsageError(`questions can't be set in ${file}: each is about one eval's task. Put them on the group or eval they ask about; question: { model, state, artifacts } in ${file} sets every question's defaults.`);
3764
+ if ("questions" in user) throw new UsageError(`questions can't be set in ${file}: each is about one eval's task. Put them on the group or eval they ask about; question: { model } in ${file} sets every question's model.`);
3322
3765
  checkOptions(user, file, CONFIG_KEYS);
3323
3766
  const include = user.include ?? defaultIncludeGlobs();
3324
3767
  if (!Array.isArray(include) || include.some((p) => typeof p !== "string")) throw new UsageError(`include in ${file} must be a list of glob patterns, e.g. include: ['evals/**/*.eval.ts'] (got ${JSON.stringify(include)})`);
@@ -3352,7 +3795,7 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3352
3795
  harness,
3353
3796
  sandbox: user.sandbox ?? docker(),
3354
3797
  environment: user.environment,
3355
- runs: user.runs ?? 1,
3798
+ runs: user.runs,
3356
3799
  gate: user.gate,
3357
3800
  env: user.env ?? {},
3358
3801
  ...user.egress !== void 0 ? { egress: user.egress } : {},
@@ -3363,6 +3806,10 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3363
3806
  capture: user.capture ?? {},
3364
3807
  question: user.question ?? {},
3365
3808
  timeoutMs: parseDuration(user.timeout ?? "15m"),
3809
+ snapshot: mergeSnapshot({
3810
+ when: true,
3811
+ compress: false
3812
+ }, user.snapshot),
3366
3813
  concurrency: user.concurrency === void 0 ? void 0 : checkConcurrency(user.concurrency, `concurrency in ${file}`),
3367
3814
  record,
3368
3815
  history: {
@@ -3378,35 +3825,27 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3378
3825
  }
3379
3826
  const ANALYSIS_DEFAULTS = {
3380
3827
  maxInputTokens: 2e4,
3381
- maxOutputTokens: 8e3,
3382
- insights: true,
3383
- maxTokens: 25e4
3828
+ maxOutputTokens: 8e3
3384
3829
  };
3385
3830
  function checkAnalysis(analysis, file) {
3386
3831
  if (analysis === void 0) return { ...ANALYSIS_DEFAULTS };
3387
3832
  if (typeof analysis !== "object" || analysis === null) throw new UsageError(`analysis in ${file} must be an object like { model: 'google/gemini-3.8-flash' } (got ${JSON.stringify(analysis)})`);
3388
3833
  unknownKey(analysis, [
3389
3834
  "model",
3390
- "insights",
3391
- "maxTokens",
3392
3835
  "instructions",
3393
3836
  "maxInputTokens",
3394
3837
  "maxOutputTokens"
3395
3838
  ], "analysis", file);
3396
- const { model, insights = ANALYSIS_DEFAULTS.insights, maxTokens = ANALYSIS_DEFAULTS.maxTokens, instructions, maxInputTokens = ANALYSIS_DEFAULTS.maxInputTokens, maxOutputTokens = ANALYSIS_DEFAULTS.maxOutputTokens } = analysis;
3839
+ const { model, instructions, maxInputTokens = ANALYSIS_DEFAULTS.maxInputTokens, maxOutputTokens = ANALYSIS_DEFAULTS.maxOutputTokens } = analysis;
3397
3840
  if (typeof model === "string") {
3398
3841
  const problem = checkModelString(model, `analysis.model in ${file}`, [`google/${model}`]);
3399
3842
  if (problem) throw new UsageError(problem);
3400
3843
  } else if (model !== void 0 && !(isObjectLike(model) && model.kind === "fake-analysis")) throw new UsageError(`analysis.model in ${file} must be a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${JSON.stringify(model)})`);
3401
- if (typeof insights !== "boolean") throw new UsageError(`analysis.insights in ${file} must be true or false (got ${JSON.stringify(insights)})`);
3402
- if (!(Number.isInteger(maxTokens) && maxTokens >= 1)) throw new UsageError(`analysis.maxTokens in ${file} must be a whole number of tokens of at least 1, e.g. 250000 (got ${JSON.stringify(maxTokens)})`);
3403
3844
  if (instructions !== void 0 && typeof instructions !== "string") throw new UsageError(`analysis.instructions in ${file} must be a string (got ${JSON.stringify(instructions)})`);
3404
3845
  if (!(Number.isInteger(maxInputTokens) && maxInputTokens >= 1e3)) throw new UsageError(`analysis.maxInputTokens in ${file} must be a whole number of at least 1000 (got ${maxInputTokens})`);
3405
3846
  if (!(Number.isInteger(maxOutputTokens) && maxOutputTokens >= 500)) throw new UsageError(`analysis.maxOutputTokens in ${file} must be a whole number of at least 500 (got ${maxOutputTokens})`);
3406
3847
  return {
3407
3848
  ...analysis,
3408
- insights,
3409
- maxTokens,
3410
3849
  maxInputTokens,
3411
3850
  maxOutputTokens
3412
3851
  };
@@ -3482,8 +3921,6 @@ function secretSources(env) {
3482
3921
  }
3483
3922
  /** The model that answers what each eval asks: `analysis.model` only, never the judge's. */
3484
3923
  const analysisModelOf = (config) => config.analysis.model;
3485
- /** Whether `analysis.model` is set, so a model can answer what each eval asks. */
3486
- const hasAnalysisModel = (config) => analysisModelOf(config) !== void 0;
3487
3924
  /** The run's judge model: the profile's when it sets one, else the config's. */
3488
3925
  function runJudgeModel(config, profile) {
3489
3926
  return profile ? profileJudge(profile).model ?? config.judge.model : config.judge.model;
@@ -3494,30 +3931,43 @@ function runModelGateway(config, profile, model, judge = false) {
3494
3931
  }
3495
3932
  /**
3496
3933
  * Names the environment variables the run's providers need but don't have: the harnesses it
3497
- * plans (default: the config's), the run's judge, the models that answer questions,
3498
- * and required config and profile secrets. Insights' model isn't among them: without its key a run
3499
- * prints no insights, and never stops for it.
3934
+ * plans (default: the config's), the run's judge and the judges its attempts resolve, the models
3935
+ * that answer questions, and required config and profile secrets.
3500
3936
  */
3501
3937
  function missingCredentials(config, env = process.env, options = {}) {
3502
3938
  const missing = [];
3503
3939
  const keys = new Set((options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []));
3504
3940
  for (const key of keys) if (!env[key]) missing.push(key);
3505
3941
  const judge = runJudgeModel(config, options.profile);
3506
- const models = [...judge !== void 0 ? [{
3507
- model: judge,
3508
- what: "the judge model",
3509
- judge: true
3510
- }] : [], ...(options.questionModels ?? []).map((model) => ({
3511
- model,
3512
- what: "questions",
3513
- judge: false
3514
- }))];
3942
+ const models = [
3943
+ ...typeof judge === "string" ? [{
3944
+ model: judge,
3945
+ gateway: runModelGateway(config, options.profile, judge, true),
3946
+ what: "the judge model"
3947
+ }] : [],
3948
+ ...(options.judges ?? []).map((j) => ({
3949
+ ...j,
3950
+ what: "the judge model"
3951
+ })),
3952
+ ...(options.questionModels ?? []).map((model) => ({
3953
+ model,
3954
+ gateway: runModelGateway(config, options.profile, model),
3955
+ what: "questions"
3956
+ }))
3957
+ ];
3515
3958
  const named = /* @__PURE__ */ new Set();
3516
- for (const { model, what, judge: isJudge } of models) {
3517
- if (typeof model !== "string" || named.has(model)) continue;
3518
- named.add(model);
3519
- const gateway = runModelGateway(config, options.profile, model, isJudge);
3520
- if (!judgeCredentials(model, gateway).some((k) => env[k])) missing.push(`${modelKeyOptions(model, gateway)} for ${what}, "${model}"`);
3959
+ for (const { model, gateway, what } of models) {
3960
+ if (gateway === "vertex") {
3961
+ const gateways = gatewayScope(config, options.profile).gateways;
3962
+ const line = `${VERTEX_PROJECT_ENV} (for vertex()) for ${what}, "${model}"`;
3963
+ if (!gateways.some((g) => g.kind === "vertex" && g.project) && !env["GOOGLE_CLOUD_PROJECT"] && !missing.includes(line)) missing.push(line);
3964
+ continue;
3965
+ }
3966
+ const keys = judgeCredentials(model, gateway);
3967
+ const once = `${keys.join(" ")}\0${model}`;
3968
+ if (named.has(once)) continue;
3969
+ named.add(once);
3970
+ if (!keys.some((k) => env[k])) missing.push(`${modelKeyOptions(model, gateway)} for ${what}, "${model}"`);
3521
3971
  }
3522
3972
  const vars = {
3523
3973
  ...config.env,
@@ -3527,16 +3977,16 @@ function missingCredentials(config, env = process.env, options = {}) {
3527
3977
  return missing;
3528
3978
  }
3529
3979
  /**
3530
- * Keys the run sends in request headers (the agents' credentials, the run's judge's and the
3531
- * questions' models') whose value has
3532
- * whitespace or a control character in it, with where the first one is: a key pasted from a
3533
- * wrapped terminal line picks up a line break, and the agent then hangs on a request it can't send
3534
- * rather than failing to authenticate.
3980
+ * Keys the run sends in request headers (the agents' credentials, the run's and attempts' judges'
3981
+ * and the questions' models') whose value has whitespace or a control character in it, with where
3982
+ * the first one is: a key pasted from a wrapped terminal line picks up a line break, and the agent
3983
+ * then hangs on a request it can't send rather than failing to authenticate.
3535
3984
  */
3536
3985
  function malformedCredentials(config, env = process.env, options = {}) {
3537
3986
  const names = (options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []);
3538
3987
  const judge = runJudgeModel(config, options.profile);
3539
3988
  if (judge !== void 0) names.push(...judgeCredentials(judge, runModelGateway(config, options.profile, judge, true)));
3989
+ for (const { model, gateway } of options.judges ?? []) names.push(...judgeCredentials(model, gateway));
3540
3990
  for (const model of options.questionModels ?? []) names.push(...judgeCredentials(model, runModelGateway(config, options.profile, model)));
3541
3991
  return [...new Set(names)].flatMap((name) => {
3542
3992
  const at = badCharacterAt(env[name] ?? "");
@@ -3544,6 +3994,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
3544
3994
  });
3545
3995
  }
3546
3996
  //#endregion
3547
- export { DEFAULT_QUESTION_STATE as $, matchesTags as A, mergeJudge as B, brokeredIn as C, judgeStateSchema as Ct, expand as D, filePartsOf as Dt, evalPlan as E, decideState as Et, sandboxProvider as F, ciRunner as G, DEFAULT_EGRESS as H, sandboxSetBy as I, gatewayPolicy as J, directOnCloud as K, secretVarsIn as L, profileJudge as M, refuseIllegalCompare as N, harnessId as O, resolveJudgeArtifacts as Ot, sandboxId as P, createFakeSandbox as Q, unbrokered as R, routedHarness as S, judgeProblem as St, checkOptions as T, badJudgeArtifacts as Tt, DEFAULT_EGRESS_ALLOW as U, resolveStateOptions as V, allowedHosts as W, unenforceable as X, isDirect as Y, resolveOutput as Z, selectProfile as _, writePluginFiles as _t, configPathFor as a, servedEffortOf as at, fallbackHint as b, judgeFields as bt, findConfig as c, agentMcp as ct, loadConfig as d, claudeMcpServers as dt, STOPPED_FIRST as et, malformedCredentials as f, credentialOf as ft, runJudgeModel as g, refusedPlugins as gt, resolveConcurrency as h, refusedMcp as ht, configLabel as i, resolvedEffortOf as it, plannedSandboxes as j, levelOf as k, hasAnalysisModel as l, authHint as lt, requireConfig as m, layoutOf as mt, analysisModelOf as n, resolveQuestionState as nt, defaultConcurrency as o, agentAuth as ot, missingCredentials as p, expectedBlockedFor as pt, egressLabel as q, configFilesPhrase as r, harnessEffort as rt, defineConfig as s, agentFor as st, CONFIG_FILES as t, questionShows as tt, hostCapacity as u, brokeringFor as ut, gatewayScope as v, captureWindowSchema as vt, cascade as w, artifactRefsOf as wt, resolveGateway as x, judgeModelNameSchema as xt, resolveModelGateway as y, fakeJudgeDataSchema as yt, unbrokeredError as z };
3997
+ export { egressLabel as $, harnessId as A, NO_JUDGE_INPUT as At, sandboxId as B, decideState as Bt, routedHarness as C, isAgentKind as Ct, checkOptions as D, writePluginFiles as Dt, checkLimitHarness as E, refusedPlugins as Et, plainHeaders as F, judgeProblem as Ft, unbrokeredError as G, sandboxSetBy as H, resolveJudgeArtifacts as Ht, plannedSandboxes as I, sharedInputFields as It, DEFAULT_EGRESS as J, mergeJudge as K, profileJudge as L, turnsSchema as Lt, majorityOf as M, inputsOn as Mt, matchesTags as N, judgeFields as Nt, evalPlan as O, captureWindowSchema as Ot, plainHeaderError as P, judgeModelNameSchema as Pt, directOnCloud as Q, refuseIllegalCompare as R, artifactRefsOf as Rt, resolveGateway as S, expectedBlockedFor as St, cascade as T, refusedMcp as Tt, secretVarsIn as U, sandboxProvider as V, filePartsOf as Vt, unbrokered as W, allowedHosts as X, DEFAULT_EGRESS_ALLOW as Y, ciRunner as Z, fellBackToDirect as _, agentMcp as _t, configPathFor as a, awaitListening as at, resolveModelGateway as b, claudeMcpServers as bt, findConfig as c, STOPPED_FIRST as ct, malformedCredentials as d, resolveQuestionState as dt, gatewayPolicy as et, missingCredentials as f, harnessEffort as ft, selectProfile as g, agentFor as gt, runJudgeModel as h, agentAuth as ht, configLabel as i, createFakeSandbox as it, levelOf as j, fakeJudgeDataSchema as jt, expand as k, INPUT_KEYS as kt, hostCapacity as l, UNRECORDED_QUESTION_STATE as lt, resolveConcurrency as m, servedEffortOf as mt, analysisModelOf as n, unenforceable as nt, defaultConcurrency as o, parseProbe as ot, requireConfig as p, resolvedEffortOf as pt, resolveStateOptions as q, configFilesPhrase as r, resolveOutput as rt, defineConfig as s, probeScript as st, CONFIG_FILES as t, isDirect as tt, loadConfig as u, questionShows as ut, gatewayScope as v, authHint as vt, brokeredIn as w, layoutOf as wt, fallbackHint as x, credentialOf as xt, modelGatewayOrNone as y, brokeringFor as yt, runsOf as z, badJudgeArtifacts as zt };
3548
3998
 
3549
- //# sourceMappingURL=config-BrdHA9bJ.js.map
3999
+ //# sourceMappingURL=config-CneBZB84.js.map