reeval 0.14.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (347) hide show
  1. package/README.md +3 -3
  2. package/THIRD_PARTY_NOTICES.md +69 -69
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-BloGVZ_a.js → adapter-CbqbMR7y.js} +37 -8
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-m-_t-IuW.js → agent-Zgn-kiqq.js} +57 -15
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{app-files-C-jwhRDt.js → app-files-ChbvRx3P.js} +4 -4
  9. package/dist/app-files-ChbvRx3P.js.map +1 -0
  10. package/dist/{assemble-BDEpelpi.js → assemble-SKQGW_QF.js} +136 -116
  11. package/dist/assemble-SKQGW_QF.js.map +1 -0
  12. package/dist/assemble-XBWuK3Ed.js +1 -0
  13. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  14. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  15. package/dist/{match-Bz0nSdzN.js → attempt-host-BOp8JklD.js} +96 -5
  16. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  17. package/dist/{capture-CVRwRTnb.js → capture-BM_3100g.js} +1218 -1207
  18. package/dist/capture-BM_3100g.js.map +1 -0
  19. package/dist/capture-DSK8O5bA.js +1 -0
  20. package/dist/{cli-C_4gaJ50.js → cli-PdYrcoKM.js} +106 -170
  21. package/dist/cli-PdYrcoKM.js.map +1 -0
  22. package/dist/{cli-spec-CpnbATtI.js → cli-spec-D0qyrPZi.js} +8 -20
  23. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-D5hNjpEx.js → client-Ct6s6OnF.js} +2 -2
  26. package/dist/{client-D5hNjpEx.js.map → client-Ct6s6OnF.js.map} +1 -1
  27. package/dist/{command-BXLUfw4h.js → command-4pJYVCT4.js} +33 -20
  28. package/dist/command-4pJYVCT4.js.map +1 -0
  29. package/dist/{validate-BhwqrVsm.js → command-9xm5ImqI.js} +387 -59
  30. package/dist/command-9xm5ImqI.js.map +1 -0
  31. package/dist/{command-Bxdy62dW.js → command-BqBxLgTc.js} +31 -30
  32. package/dist/command-BqBxLgTc.js.map +1 -0
  33. package/dist/{command-I1euLLio.js → command-DbSMcxZL.js} +7 -7
  34. package/dist/{command-I1euLLio.js.map → command-DbSMcxZL.js.map} +1 -1
  35. package/dist/{commands-jzigbNFb.js → commands-BU6YJCz9.js} +175 -72
  36. package/dist/commands-BU6YJCz9.js.map +1 -0
  37. package/dist/{commands-p1S4nw_H.js → commands-Bi2SEQtr.js} +1 -1
  38. package/dist/{config-BrdHA9bJ.js → config-k6m7XbAz.js} +499 -271
  39. package/dist/config-k6m7XbAz.js.map +1 -0
  40. package/dist/{contrasts-DRxMQQTC.js → contrasts-DLZF4dGF.js} +3 -3
  41. package/dist/{contrasts-DRxMQQTC.js.map → contrasts-DLZF4dGF.js.map} +1 -1
  42. package/dist/cost-C5RBTTJr.js +1 -0
  43. package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
  44. package/dist/cost-ChPa8-Fp.js.map +1 -0
  45. package/dist/db-CpsifVln.js +1530 -0
  46. package/dist/db-CpsifVln.js.map +1 -0
  47. package/dist/db-DkD-F-X-.js +2 -0
  48. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  49. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  50. package/dist/debug-DeM5pswK.js +1 -0
  51. package/dist/{docker-BnRlTpZA.js → docker-51WvhN_X.js} +5 -4
  52. package/dist/docker-51WvhN_X.js.map +1 -0
  53. package/dist/{docker-D01slDF3.js → docker-B4-rx4dA.js} +26 -8
  54. package/dist/docker-B4-rx4dA.js.map +1 -0
  55. package/dist/{docker-DivfZl2u.js → docker-YFj4H1Cr.js} +1 -1
  56. package/dist/{doctor-DQ2p0euG.js → doctor-BuQnfz4e.js} +275 -37
  57. package/dist/doctor-BuQnfz4e.js.map +1 -0
  58. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  59. package/dist/env-CARj5Axq.js.map +1 -0
  60. package/dist/{environment-CO0P0soF.js → environment-BOFgzFDg.js} +115 -296
  61. package/dist/environment-BOFgzFDg.js.map +1 -0
  62. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  63. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  64. package/dist/eval-source-Dje3rERU.js +145 -0
  65. package/dist/eval-source-Dje3rERU.js.map +1 -0
  66. package/dist/{evaluate-BUmsN7yG.js → evaluate-BPP9s3s6.js} +367 -130
  67. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  68. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  69. package/dist/{files-B-CJeeEh.js → files-Bi461Be2.js} +197 -91
  70. package/dist/files-Bi461Be2.js.map +1 -0
  71. package/dist/files-C2h2Cw4F.js +1 -0
  72. package/dist/files-Clipx4OW.js +37 -0
  73. package/dist/files-Clipx4OW.js.map +1 -0
  74. package/dist/files-DmUrRmmL.js +1 -0
  75. package/dist/{files-kUCrswI6.js → files-H24pz0hL.js} +20 -9
  76. package/dist/files-H24pz0hL.js.map +1 -0
  77. package/dist/gateway.d.ts +22 -9
  78. package/dist/gateway.js +30 -8
  79. package/dist/gateway.js.map +1 -1
  80. package/dist/{git-Bk8DSwHy.js → git-CsVZkNNL.js} +3 -3
  81. package/dist/{git-Bk8DSwHy.js.map → git-CsVZkNNL.js.map} +1 -1
  82. package/dist/harness-3NVyTSGK.js +392 -0
  83. package/dist/harness-3NVyTSGK.js.map +1 -0
  84. package/dist/harness.d.ts +12 -8
  85. package/dist/harness.js +1 -1
  86. package/dist/{images-DzLeFjLx.js → images-4mwhNkvh.js} +8 -8
  87. package/dist/{images-DzLeFjLx.js.map → images-4mwhNkvh.js.map} +1 -1
  88. package/dist/index.d.ts +51 -10
  89. package/dist/index.js +5 -5
  90. package/dist/init-BDEOopK7.js +1 -0
  91. package/dist/{init-SRalM5xF.js → init-cOk3D44b.js} +10 -11
  92. package/dist/init-cOk3D44b.js.map +1 -0
  93. package/dist/{invocations-DzyKSJBa.js → invocations-DW-lTk2f.js} +10 -6
  94. package/dist/invocations-DW-lTk2f.js.map +1 -0
  95. package/dist/{judge-yLvjC5A3.js → judge-CKdgEXN7.js} +236 -46
  96. package/dist/judge-CKdgEXN7.js.map +1 -0
  97. package/dist/judge.d.ts +5 -7
  98. package/dist/judge.js +4 -8
  99. package/dist/judge.js.map +1 -1
  100. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  101. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  102. package/dist/{matchers-D4eUqx4j.js → matchers-DT2TaGix.js} +2 -2
  103. package/dist/{matchers-D4eUqx4j.js.map → matchers-DT2TaGix.js.map} +1 -1
  104. package/dist/{model-1ZLr9Dcv.js → model-Bq3G0OG1.js} +29 -36
  105. package/dist/model-Bq3G0OG1.js.map +1 -0
  106. package/dist/{model-CgnOpss4.js → model-CTnyvh7H.js} +292 -68
  107. package/dist/model-CTnyvh7H.js.map +1 -0
  108. package/dist/model-D-SWrLZG.js +1 -0
  109. package/dist/model-g3XnzUv0.js +1 -0
  110. package/dist/{model-B45lDUV1.js → model-vRiUycWh.js} +22 -13
  111. package/dist/model-vRiUycWh.js.map +1 -0
  112. package/dist/{node-BSq5qHnQ.js → node-BVbbnFtv.js} +72 -50
  113. package/dist/node-BVbbnFtv.js.map +1 -0
  114. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  115. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  116. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  117. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  118. package/dist/{pairs-CHfZNRT6.js → pairs-puYi-zj2.js} +11 -4
  119. package/dist/pairs-puYi-zj2.js.map +1 -0
  120. package/dist/patch-CquGAGX5.js +76 -0
  121. package/dist/patch-CquGAGX5.js.map +1 -0
  122. package/dist/{patch-DWf-I_Q_.js → patch-DKJiC5Sq.js} +5 -7
  123. package/dist/patch-DKJiC5Sq.js.map +1 -0
  124. package/dist/{plan-BIwUzlB8.js → plan-DdWkM6SW.js} +64 -29
  125. package/dist/plan-DdWkM6SW.js.map +1 -0
  126. package/dist/{plugins-Dkp_H3pB.js → plugins-D-CP6mMd.js} +253 -17
  127. package/dist/plugins-D-CP6mMd.js.map +1 -0
  128. package/dist/{preflight-DJCD_BDn.js → preflight-9ixuXzsx.js} +3 -3
  129. package/dist/{preflight-DJCD_BDn.js.map → preflight-9ixuXzsx.js.map} +1 -1
  130. package/dist/{preflight-D9y-HvZQ.js → preflight-DTiVv0gY.js} +1 -1
  131. package/dist/{prepare-CIxCGG0M.js → prepare-BfyikQit.js} +8 -8
  132. package/dist/{prepare-CIxCGG0M.js.map → prepare-BfyikQit.js.map} +1 -1
  133. package/dist/{prepare-BJG_pD54.js → prepare-tmBBf4W8.js} +2 -2
  134. package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
  135. package/dist/prices-QaS-kjSQ.js.map +1 -0
  136. package/dist/{provenance-CfZpFpAh.js → provenance-BizwUn3h.js} +4 -4
  137. package/dist/{provenance-CfZpFpAh.js.map → provenance-BizwUn3h.js.map} +1 -1
  138. package/dist/{providers-DFk_6W4K.js → providers-B6b9Tnyi.js} +5 -5
  139. package/dist/{providers-DFk_6W4K.js.map → providers-B6b9Tnyi.js.map} +1 -1
  140. package/dist/{providers-D1Zxp2he.js → providers-DkOupEBl.js} +1 -1
  141. package/dist/query-Clm2mNSm.js +1 -0
  142. package/dist/{query-CemclveB.js → query-DyM-VExS.js} +3 -3
  143. package/dist/{query-CemclveB.js.map → query-DyM-VExS.js.map} +1 -1
  144. package/dist/{questions-bAudIc04.js → questions-BRhU8MzB.js} +100 -25
  145. package/dist/questions-BRhU8MzB.js.map +1 -0
  146. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  147. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  148. package/dist/redact-DVIaahLv.js +658 -0
  149. package/dist/redact-DVIaahLv.js.map +1 -0
  150. package/dist/remarks-BwJ8pcTQ.js +1 -0
  151. package/dist/{rules-VhftPOOT.js → remarks-C7i2LMRb.js} +296 -66
  152. package/dist/remarks-C7i2LMRb.js.map +1 -0
  153. package/dist/{header-BLPP6aCU.js → remarks-D_3qhBWU.js} +34 -9
  154. package/dist/remarks-D_3qhBWU.js.map +1 -0
  155. package/dist/{remarks-mujJv1Ey.js → remarks-dA8LyoPc.js} +9 -5
  156. package/dist/remarks-dA8LyoPc.js.map +1 -0
  157. package/dist/{remote-CKcEKja4.js → remote-DFIzcx_6.js} +39 -99
  158. package/dist/remote-DFIzcx_6.js.map +1 -0
  159. package/dist/resolve--rQZ8u6Z.js +1 -0
  160. package/dist/{resolve-CBoKzQMP.js → resolve-DWSJpKHX.js} +46 -30
  161. package/dist/resolve-DWSJpKHX.js.map +1 -0
  162. package/dist/{results-CMgZEANy.js → results-BC3pcG_Z.js} +15 -30
  163. package/dist/results-BC3pcG_Z.js.map +1 -0
  164. package/dist/{resume-B9SOSnK9.js → resume-j-rcIryi.js} +47 -17
  165. package/dist/resume-j-rcIryi.js.map +1 -0
  166. package/dist/{runtime-BgQRyCLl.js → runtime-d56Gg3J5.js} +30 -13
  167. package/dist/runtime-d56Gg3J5.js.map +1 -0
  168. package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmln11gU.js} +2 -2
  169. package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmln11gU.js.map} +1 -1
  170. package/dist/sandbox.d.ts +1 -1
  171. package/dist/sandbox.js +1 -1
  172. package/dist/{server-TGW4WyAz.js → server-BOoH10J8.js} +51 -33
  173. package/dist/server-BOoH10J8.js.map +1 -0
  174. package/dist/server-DpuVAbrO.js +1 -0
  175. package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
  176. package/dist/shared-B_lvGqWz.js.map +1 -0
  177. package/dist/source-BFd6f0-R.js +1 -0
  178. package/dist/{source-B1WvBIQt.js → source-wD04-rRG.js} +15 -10
  179. package/dist/source-wD04-rRG.js.map +1 -0
  180. package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
  181. package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
  182. package/dist/{src-CyaDNG9r.js → src-6_zz7oJs.js} +1 -1
  183. package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
  184. package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
  185. package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
  186. package/dist/src-BO8AYqJ5.js.map +1 -0
  187. package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
  188. package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
  189. package/dist/src-BYPMM1x7.js +2 -0
  190. package/dist/src-Bp4aN3rA.js +2 -0
  191. package/dist/src-C7upnZSd.js +2 -0
  192. package/dist/{src-8zoX_qrb.js → src-D91Hd6hT.js} +546 -1060
  193. package/dist/src-D91Hd6hT.js.map +1 -0
  194. package/dist/src-DH-9IO7f.js +1 -0
  195. package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
  196. package/dist/src-DSZyg0Uq.js.map +1 -0
  197. package/dist/{src-DjRI-IeE.js → src-DgHuStEi.js} +134 -34
  198. package/dist/{src-DjRI-IeE.js.map → src-DgHuStEi.js.map} +1 -1
  199. package/dist/src-HSxBazcg.js +3 -0
  200. package/dist/src-LD1BAJFa.js +2 -0
  201. package/dist/src-dlX-Lmxg.js +2 -0
  202. package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
  203. package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
  204. package/dist/src-kYeaJJrl.js +2 -0
  205. package/dist/state-B5PLm5qY.js +1 -0
  206. package/dist/{state-B1t143wG.js → state-DrJ687ut.js} +15 -15
  207. package/dist/{state-B1t143wG.js.map → state-DrJ687ut.js.map} +1 -1
  208. package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
  209. package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
  210. package/dist/suite-22Kt9Grx.js +2 -0
  211. package/dist/{suite-WgT7m6Vs.js → suite-Co8cwelb.js} +48 -63
  212. package/dist/suite-Co8cwelb.js.map +1 -0
  213. package/dist/{summary-DIGEVr08.js → summary-DLFHVoux.js} +80 -270
  214. package/dist/summary-DLFHVoux.js.map +1 -0
  215. package/dist/testing.d.ts +417 -142
  216. package/dist/testing.js +12 -10
  217. package/dist/testing.js.map +1 -1
  218. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  219. package/dist/text-DTigcA64.js.map +1 -0
  220. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  221. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  222. package/dist/types-C5vI3g1C.js.map +1 -1
  223. package/dist/{types-DXBbP9Mk.d.ts → types-coMWHaU2.d.ts} +253 -733
  224. package/dist/vertex-Bv7vhMgs.js +1 -0
  225. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  226. package/dist/vertex-DypYjx71.js +456 -0
  227. package/dist/vertex-DypYjx71.js.map +1 -0
  228. package/dist/view/index.d.ts +536 -121
  229. package/dist/view/index.js +9 -9
  230. package/dist/{views-sXpGE6a2.js → views-BFO1BLKu.js} +9 -10
  231. package/dist/{views-sXpGE6a2.js.map → views-BFO1BLKu.js.map} +1 -1
  232. package/dist/views-BOf8qEL8.js +2 -0
  233. package/dist/{worker-entry-Duo9oAeT.js → worker-entry-CJtdsKBP.js} +19 -12
  234. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  235. package/dist/{workspace-CDI68wtX.js → workspace-BRxjfa8N.js} +5 -5
  236. package/dist/{workspace-CDI68wtX.js.map → workspace-BRxjfa8N.js.map} +1 -1
  237. package/dist/workspace-DDHzkMjm.js +1 -0
  238. package/docker/seccomp-default.json +1 -0
  239. package/package.json +5 -1
  240. package/skills/eval-rubric/SKILL.md +35 -104
  241. package/dist/adapter-BloGVZ_a.js.map +0 -1
  242. package/dist/agent-m-_t-IuW.js.map +0 -1
  243. package/dist/app-files-C-jwhRDt.js.map +0 -1
  244. package/dist/assemble-BDEpelpi.js.map +0 -1
  245. package/dist/assemble-C5bfncHU.js +0 -1
  246. package/dist/capture-CVRwRTnb.js.map +0 -1
  247. package/dist/capture-CherbjzB.js +0 -1
  248. package/dist/cli-C_4gaJ50.js.map +0 -1
  249. package/dist/cli-spec-CpnbATtI.js.map +0 -1
  250. package/dist/command-BWAuwQNk.js +0 -91
  251. package/dist/command-BWAuwQNk.js.map +0 -1
  252. package/dist/command-BXLUfw4h.js.map +0 -1
  253. package/dist/command-Be6BSE2L.js +0 -335
  254. package/dist/command-Be6BSE2L.js.map +0 -1
  255. package/dist/command-Bxdy62dW.js.map +0 -1
  256. package/dist/commands-jzigbNFb.js.map +0 -1
  257. package/dist/config-BrdHA9bJ.js.map +0 -1
  258. package/dist/cost-D4elx933.js.map +0 -1
  259. package/dist/cost-DmnAod9x.js +0 -1
  260. package/dist/db-CAfksinX.js +0 -2
  261. package/dist/db-DO0p7xgP.js +0 -619
  262. package/dist/db-DO0p7xgP.js.map +0 -1
  263. package/dist/debug-D7bEv_UV.js +0 -1
  264. package/dist/docker-BnRlTpZA.js.map +0 -1
  265. package/dist/docker-D01slDF3.js.map +0 -1
  266. package/dist/doctor-DQ2p0euG.js.map +0 -1
  267. package/dist/env-BQqj93gu.js.map +0 -1
  268. package/dist/environment-CO0P0soF.js.map +0 -1
  269. package/dist/evaluate-BUmsN7yG.js.map +0 -1
  270. package/dist/evaluate-C_sgRA-V.d.ts +0 -134
  271. package/dist/facts-DzOUTreB.js +0 -906
  272. package/dist/facts-DzOUTreB.js.map +0 -1
  273. package/dist/files-B-CJeeEh.js.map +0 -1
  274. package/dist/files-Bp5j5BqI.js +0 -1
  275. package/dist/files-C2sjmXhk.js +0 -1
  276. package/dist/files-C_gQNVB-.js +0 -65
  277. package/dist/files-C_gQNVB-.js.map +0 -1
  278. package/dist/files-kUCrswI6.js.map +0 -1
  279. package/dist/generate-DZ7vp69F.js +0 -1647
  280. package/dist/generate-DZ7vp69F.js.map +0 -1
  281. package/dist/harness-COVXSwJ6.js +0 -354
  282. package/dist/harness-COVXSwJ6.js.map +0 -1
  283. package/dist/header-BLPP6aCU.js.map +0 -1
  284. package/dist/init-RgCQXZhb.js +0 -1
  285. package/dist/init-SRalM5xF.js.map +0 -1
  286. package/dist/insights-TyAvz9aV.js +0 -68
  287. package/dist/insights-TyAvz9aV.js.map +0 -1
  288. package/dist/invocations-DzyKSJBa.js.map +0 -1
  289. package/dist/judge-yLvjC5A3.js.map +0 -1
  290. package/dist/match-Bz0nSdzN.js.map +0 -1
  291. package/dist/model-1ZLr9Dcv.js.map +0 -1
  292. package/dist/model-B45lDUV1.js.map +0 -1
  293. package/dist/model-BTZq8HIZ.js +0 -1
  294. package/dist/model-CgnOpss4.js.map +0 -1
  295. package/dist/model-Ch8kX87u.js +0 -1
  296. package/dist/node-BSq5qHnQ.js.map +0 -1
  297. package/dist/opencode-providers-klkqgzvb.js +0 -3276
  298. package/dist/opencode-providers-klkqgzvb.js.map +0 -1
  299. package/dist/pairs-CHfZNRT6.js.map +0 -1
  300. package/dist/patch-DWf-I_Q_.js.map +0 -1
  301. package/dist/plan-BIwUzlB8.js.map +0 -1
  302. package/dist/plugins-Dkp_H3pB.js.map +0 -1
  303. package/dist/prices-BcLEXvdY.js.map +0 -1
  304. package/dist/query-Cvbwl804.js +0 -1
  305. package/dist/questions-bAudIc04.js.map +0 -1
  306. package/dist/redact-CRGdfV2s.js +0 -240
  307. package/dist/redact-CRGdfV2s.js.map +0 -1
  308. package/dist/remarks-CXwSnSjR.js +0 -278
  309. package/dist/remarks-CXwSnSjR.js.map +0 -1
  310. package/dist/remarks-CyAu2DiB.js +0 -1
  311. package/dist/remarks-mujJv1Ey.js.map +0 -1
  312. package/dist/remote-CKcEKja4.js.map +0 -1
  313. package/dist/resolve-B5-uqmJ6.js +0 -1
  314. package/dist/resolve-CBoKzQMP.js.map +0 -1
  315. package/dist/results-CMgZEANy.js.map +0 -1
  316. package/dist/resume-B9SOSnK9.js.map +0 -1
  317. package/dist/rules-VhftPOOT.js.map +0 -1
  318. package/dist/run-CwIEK16q.js +0 -88
  319. package/dist/run-CwIEK16q.js.map +0 -1
  320. package/dist/runtime-BgQRyCLl.js.map +0 -1
  321. package/dist/server-CHRSgKRp.js +0 -1
  322. package/dist/server-TGW4WyAz.js.map +0 -1
  323. package/dist/shared-BDUzStad.js.map +0 -1
  324. package/dist/source-B1WvBIQt.js.map +0 -1
  325. package/dist/source-DAzztZwU.js +0 -1
  326. package/dist/src-1jHbFaGW.js +0 -1
  327. package/dist/src-8zoX_qrb.js.map +0 -1
  328. package/dist/src-C_9wgowC.js +0 -2
  329. package/dist/src-ChhtI6BP.js.map +0 -1
  330. package/dist/src-D005Cbqz.js +0 -2
  331. package/dist/src-DIfRbp5a.js +0 -3
  332. package/dist/src-Dg8121Xy.js +0 -2
  333. package/dist/src-ElBqEN-S.js +0 -2
  334. package/dist/src-NSdQ9zI6.js.map +0 -1
  335. package/dist/src-YHGaUrGE.js +0 -2
  336. package/dist/src-mJVxw1Jm.js +0 -2
  337. package/dist/state-vznMh2wl.js +0 -1
  338. package/dist/suite-BsWGHzlG.js +0 -2
  339. package/dist/suite-WgT7m6Vs.js.map +0 -1
  340. package/dist/summary-DIGEVr08.js.map +0 -1
  341. package/dist/text-CrVy5ioL.js.map +0 -1
  342. package/dist/transcript-BTxJpW4R.js.map +0 -1
  343. package/dist/usage-DF3_p5pE.js +0 -1
  344. package/dist/validate-BhwqrVsm.js.map +0 -1
  345. package/dist/views-BW_C5eVH.js +0 -2
  346. package/dist/worker-entry-Duo9oAeT.js.map +0 -1
  347. package/dist/workspace-QZCacP8A.js +0 -1
@@ -1,20 +1,23 @@
1
1
  import { n as isOneOf, t as isObjectLike } from "./object-CLf2-tNq.js";
2
2
  import { n as UsageError } from "./errors-B6BSMs03.js";
3
- import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, i as durationSchema, l as nonEmpty, o as genericProblem, p as parseDuration, s as got, t as closest, u as oneOf } from "./shared-BDUzStad.js";
4
- import { D as sandboxOf, E as sameWorkspaceOnly, F as sandboxDescriptorSchema, I as egressPolicySchema, L as envVarsSchema, M as workspaceLabel, N as workspaceOf, O as sandboxPaths, P as environmentDataSchema, R as gatePolicySchema, V as sandboxProblem, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, j as unbrokeredSecrets, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, w as plannedSecrets, y as isMovingRef, z as gateProblem } from "./environment-CO0P0soF.js";
5
- import { _ as gatewayEntryDataSchema, f as FACTORY, g as gatewayDescriptorSchema, h as GATEWAY_NAMES, m as harnessProblem, p as harnessDescriptorSchema, u as opencodeModelError, v as gatewayNameSchema } from "./harness-COVXSwJ6.js";
3
+ import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, h as parseDuration, i as durationSchema, l as nonEmpty, o as genericProblem, s as got, t as closest, u as oneOf } from "./shared-B_lvGqWz.js";
4
+ import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-BOFgzFDg.js";
5
+ import { B as gatePolicySchema, D as pluginServerName, L as egressPolicySchema, V as gateProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-D-CP6mMd.js";
6
+ import { _ as gatewayDescriptorSchema, f as FACTORY, g as directOffPin, h as GATEWAY_NAMES, m as harnessProblem, p as harnessDescriptorSchema, u as opencodeModelError, v as gatewayEntryDataSchema, y as gatewayNameSchema } from "./harness-3NVyTSGK.js";
6
7
  import { a as fromCwd, l as sha256 } from "./ids-Bp6ut7ZD.js";
7
- import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-klkqgzvb.js";
8
- import { C as servedOnVercel, S as fxAskedModel, _ as VERCEL_HOST, a as checkModelString, b as VERCEL_ROUTES, g as FX_VERCEL_ENDPOINTS, h as FX_VERCEL_CALL, i as RECOMMENDED_JUDGE_MODEL, n as DIRECT_PROVIDERS, s as judgeCredentials, u as modelKeyOptions, v as VERCEL_KEY, x as VERCEL_UNPROVEN_EFFORT, y as VERCEL_REFUSALS } from "./judge-yLvjC5A3.js";
9
- import { r as badCharacterAt } from "./redact-CRGdfV2s.js";
10
- import { t as inCI } from "./env-BQqj93gu.js";
11
- import { F as DOCKER_EGRESS, I as HARNESS_HOSTS, L as PACKAGE_HOSTS, dt as run, ft as shellQuote } from "./src-8zoX_qrb.js";
8
+ import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-Bz0l7OZI.js";
9
+ import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-CKdgEXN7.js";
10
+ import { a as VERTEX_ROUTES, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV, t as SANDBOX_ACCESS_TOKEN } from "./vertex-DypYjx71.js";
11
+ import { r as badCharacterAt } from "./redact-DVIaahLv.js";
12
+ import { s as withoutBom } from "./text-DTigcA64.js";
13
+ import { t as inCI } from "./env-CARj5Axq.js";
12
14
  import { u as ownFileURL } from "./root-Dlff-zp8.js";
13
- import { D as pluginServerName, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers } from "./plugins-Dkp_H3pB.js";
15
+ import { F as PACKAGE_HOSTS, N as DOCKER_EGRESS, P as HARNESS_HOSTS, dt as run, ft as shellQuote } from "./src-D91Hd6hT.js";
14
16
  import { t as InfraError } from "./types-C5vI3g1C.js";
15
- import { r as pairFactors } from "./pairs-CHfZNRT6.js";
16
- import { a as reevalJiti, n as defaultIncludeGlobs } from "./adapter-BloGVZ_a.js";
17
- import { t as docker } from "./sandbox-BOYZ0hlo.js";
17
+ import { i as pairFactors } from "./pairs-puYi-zj2.js";
18
+ import "./stats-hfpuYsAM.js";
19
+ import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-CbqbMR7y.js";
20
+ import { t as docker } from "./sandbox-Cmln11gU.js";
18
21
  import { existsSync, readFileSync } from "node:fs";
19
22
  import { availableParallelism, tmpdir, totalmem } from "node:os";
20
23
  import { dirname, join, resolve, win32 } from "node:path";
@@ -82,31 +85,31 @@ function decideState(json, files) {
82
85
  //#endregion
83
86
  //#region src/contract/judge.ts
84
87
  const judgeStateSchema = z.strictObject({
85
- task: z.boolean().optional().describe("The graded turn's prompt. Default true."),
86
- output: z.boolean().optional().describe("The graded turn's final message, or `on`'s value. Default true."),
87
- files: z.boolean().optional().describe("`status path` lines for the changed files, under `diff`'s patterns. Default true."),
88
- diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional().describe("The diff since the eval started. A list is gitignore-style patterns over workspace paths: the last matching pattern wins and `!` excludes. Default true."),
89
- contents: z.array(nonEmpty()).optional().describe("Workspace files shown whole, as they are now, changed or not: exact paths or gitignore-style patterns (`'**'`, `'src/**'`; the last match wins, `!` excludes) over the files git sees in the workspace. A missing exact path is `null`; a pattern that matches nothing adds nothing. These files leave the diff. Default none."),
90
- transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional().describe("The graded turn's transcript as text lines: `true` is messages and tool calls with tool output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. Default false."),
91
- turns: z.enum(["last", "all"]).optional().describe("`'all'`: every turn up to the graded one under `turns`, each with its task, output and (with `transcript`) transcript. Default `'last'`."),
92
- checks: z.union([z.boolean(), z.array(nonEmpty())]).optional().describe("`c.exec` results so far; a list shows only those commands, as passed to `c.exec`. Default false.")
93
- }).describe("What the judge is given: each key is a key of the state in `judge-input.json`. An unset key keeps its default, which is what the judge gets with no `state` option at all.");
88
+ task: z.boolean().optional(),
89
+ output: z.boolean().optional(),
90
+ files: z.boolean().optional(),
91
+ diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
92
+ contents: z.array(nonEmpty()).optional(),
93
+ transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
94
+ turns: z.enum(["last", "all"]).optional(),
95
+ checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
96
+ });
94
97
  const judgeFields = {
95
- threshold: z.number().min(0, "must be between 0 and 1").max(1, "must be between 0 and 1").optional().describe("Minimum score (0–1) for a `pass` verdict."),
96
- providerOptions: z.record(z.string(), z.record(z.string(), jsonValueSchema)).optional().describe("AI SDK provider options for the judge's calls, keyed by provider."),
97
- maxConcurrent: z.number().int().min(1).optional().describe("The most judge calls to this model's provider at once, across the run."),
98
- state: judgeStateSchema.optional().describe("The judge's state: what the judge is given beside the question, not the workspace's or the attempt's state. Merged key by key down the cascade."),
99
- gateway: z.union([gatewayNameSchema, gatewayDescriptorSchema]).optional().describe("The gateway the judge's calls go through, by name or by its helper. It skips the config's `gateway` fallback: the plan refuses a gateway that isn't listed or can't serve the model.")
98
+ threshold: z.number().min(0, "must be between 0 and 1").max(1, "must be between 0 and 1").optional(),
99
+ providerOptions: z.record(z.string(), z.record(z.string(), jsonValueSchema)).optional(),
100
+ maxConcurrent: z.number().int().min(1).optional(),
101
+ state: judgeStateSchema.optional(),
102
+ gateway: z.union([gatewayNameSchema, gatewayDescriptorSchema]).optional()
100
103
  };
101
104
  const fakeJudgeDataSchema = z.strictObject({
102
105
  kind: z.literal("fake-judge"),
103
- score: z.number().min(0).max(1).describe("The score (0–1) every call gets.")
104
- }).describe("`fakeJudge()` as data: the score every call gets.");
106
+ score: z.number().min(0).max(1)
107
+ });
105
108
  const judgeModelNameSchema = z.string().regex(/\//, "must be a 'provider/model' string, e.g. 'google/gemini-3.8-flash'");
106
109
  const configJudgeDataSchema = z.strictObject({
107
- model: z.union([z.string(), fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge."),
110
+ model: z.union([z.string(), fakeJudgeDataSchema]).optional(),
108
111
  ...judgeFields
109
- }).describe("`ConfigJudgeOptions` as data: the model by name, or a fake judge with a fixed score.");
112
+ });
110
113
  const judgeOptionsSchema = z.strictObject({
111
114
  model: z.custom().optional(),
112
115
  ...judgeFields
@@ -222,25 +225,25 @@ function judgeProblem(judge, site) {
222
225
  }
223
226
  //#endregion
224
227
  //#region src/contract/questions.ts
225
- const pairFactorSchema = z.enum(["environment", "harness"]).describe("A matrix factor a pairwise question varies; the others are held.");
228
+ const pairFactorSchema = z.enum(["environment", "harness"]);
226
229
  /** Evaluation models score yes/no questions only, so they can't answer a question. */
227
230
  const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
228
231
  const questionStateSchema = z.strictObject({
229
- task: z.boolean().optional().describe("The prompts the attempt was sent, once for both sides when they match. Default true."),
230
- output: z.boolean().optional().describe("The final message: the agent's own account of its work. Default false in a pair, true per attempt."),
231
- files: z.boolean().optional().describe("`status path` lines for the changed files, under `diff`'s patterns. Default false."),
232
- diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional().describe("The diff since the eval started. A list is gitignore-style patterns over workspace paths: the last matching pattern wins and `!` excludes. Default true."),
233
- transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional().describe("The transcript as text lines: `true` is messages and tool calls with tool output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each harness's tools, so a pair of two harnesses isn't blind. Default false in a pair, true per attempt."),
234
- turns: z.enum(["last", "all"]).optional().describe("`'all'`: every turn under `turns`, each with its task, output and (with `transcript`) transcript; `'last'`: the last turn only. Default `'all'`."),
235
- checks: z.union([z.boolean(), z.array(nonEmpty())]).optional().describe("`c.exec` results; a list shows only those commands, as passed to `c.exec`. Default false.")
236
- }).describe("What a question is shown of each attempt: the keys of `judge.state` but `contents` (the attempts' workspaces are gone when questions are asked). An unset key keeps its default: a pairwise question shows the task and the diff; one asked of each attempt also shows the final message and the transcript.");
232
+ task: z.boolean().optional(),
233
+ output: z.boolean().optional(),
234
+ files: z.boolean().optional(),
235
+ diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
236
+ transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
237
+ turns: z.enum(["last", "all"]).optional(),
238
+ checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
239
+ });
237
240
  const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty()).min(1)]);
238
241
  const questionModelSchema = z.string().refine((m) => m.includes("/"), { params: { provider: true } }).refine((m) => !isEvaluationModel(m), { params: { evaluation: true } });
239
242
  const questionOptionsSchema = z.strictObject({
240
- model: questionModelSchema.optional().describe("`'provider/model'` that answers every question below it without its own `model`."),
241
- state: questionStateSchema.optional().describe("What it's shown of each attempt."),
242
- artifacts: questionArtifactsSchema.optional().describe("Each attempt's own captures, by name; `true` sends all of them. Default none.")
243
- }).describe("Defaults for every question: `model` and `artifacts` replaced, `state` merged key by key.");
243
+ model: questionModelSchema.optional(),
244
+ state: questionStateSchema.optional(),
245
+ artifacts: questionArtifactsSchema.optional()
246
+ });
244
247
  /** The first item a list holds twice. */
245
248
  const twice = (items) => items.find((k, i) => items.indexOf(k) !== i);
246
249
  const unique = (list, ctx) => {
@@ -253,11 +256,11 @@ const acrossSchema = z.union([pairFactorSchema, z.array(pairFactorSchema).min(1)
253
256
  const choicesSchema = z.array(nonEmpty()).min(2).superRefine(unique);
254
257
  const questionObjectSchema = z.strictObject({
255
258
  question: nonEmpty(),
256
- model: questionModelSchema.optional().describe("`'provider/model'` that answers it, over the cascaded `question.model`."),
257
- across: acrossSchema.optional().describe("The factor it varies when pairing, or both: `'environment'` holds the harness, `'harness'` holds the environment. Unset, it's asked of each attempt on its own."),
258
- choices: choicesSchema.optional().describe("The answers a question asked of each attempt picks from, with a short reason."),
259
- state: questionStateSchema.optional().describe("What it's shown of each attempt, over the cascaded `question.state`, key by key."),
260
- artifacts: questionArtifactsSchema.optional().describe("Each attempt's own captures, by name; `true` sends all of them. Replaces `question.artifacts`.")
259
+ model: questionModelSchema.optional(),
260
+ across: acrossSchema.optional(),
261
+ choices: choicesSchema.optional(),
262
+ state: questionStateSchema.optional(),
263
+ artifacts: questionArtifactsSchema.optional()
261
264
  }).superRefine((q, ctx) => {
262
265
  if (q.across !== void 0 && q.choices !== void 0) ctx.addIssue({
263
266
  code: "custom",
@@ -265,7 +268,7 @@ const questionObjectSchema = z.strictObject({
265
268
  message: "with across"
266
269
  });
267
270
  });
268
- const questionSchema = z.union([nonEmpty(), questionObjectSchema]).describe("A question a language model answers after the attempts: its text, or the text and its options. With `across`, it picks between two attempts that differ only in that factor, in both orders; without it, it's asked of each attempt on its own. Without a model, the cascaded `question.model` answers, else `analysis.model`, else the judge's when it's a language model. `state` and `artifacts` choose what it's shown of each attempt.");
271
+ const questionSchema = z.union([nonEmpty(), questionObjectSchema]);
269
272
  const questionText = (q) => typeof q === "string" ? q : q.question;
270
273
  const questionsSchema = z.array(questionSchema).superRefine((list, ctx) => {
271
274
  if (twice(list.map(questionText)) !== void 0) ctx.addIssue({
@@ -387,27 +390,27 @@ function questionsProblem(value, where) {
387
390
  //#region src/contract/config.ts
388
391
  const count = () => z.number().int().min(1);
389
392
  const captureWindowSchema = z.strictObject({
390
- width: count().describe("In CSS pixels."),
391
- height: count().describe("In CSS pixels.")
392
- }).describe("Live tab size for `c.capture()`. Default 1280×720 when the call and cascade omit `viewport`.");
393
+ width: count(),
394
+ height: count()
395
+ });
393
396
  const captureOptionsSchema = z.strictObject({
394
- viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional().describe("The window and saved PNG size, or `'full-page'`. Default 1280×720."),
395
- timeout: durationSchema.optional().describe("How long to poll before failing. Default `'60s'`."),
396
- browser: z.boolean().optional().describe("`false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws."),
397
- model: z.string().includes("/").optional().describe("Vision model for `instructions`. Cascades like `question`. Default is `analysis.model`, else the attempt's judge model, whichever is first a language model; never Jev.")
398
- }).describe("Camera defaults for `c.capture()`. Cascades config → profile defaults → group → eval, shallow-merged. `viewport` on a level replaces the whole value. `port` is not an option: pass it as the first argument to `c.capture`.");
397
+ viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
398
+ timeout: durationSchema.optional(),
399
+ browser: z.boolean().optional(),
400
+ model: z.string().includes("/").optional()
401
+ });
399
402
  const harnessList = z.union([harnessDescriptorSchema, z.array(harnessDescriptorSchema).min(1)]);
400
403
  const budgetSchema = z.strictObject({
401
- attempts: count().optional().describe("The most attempts the plan may have."),
402
- duration: durationSchema.optional().describe("How long the run may go on, from the first attempt's start.")
403
- }).describe("Run-level caps. `attempts` is checked against the plan; `duration` while running.");
404
+ attempts: count().optional(),
405
+ duration: durationSchema.optional()
406
+ });
404
407
  const historyConfigSchema = z.strictObject({
405
- dir: z.string().optional().describe("Committed history folder (default: `.history` in the directory the evals live in)."),
408
+ dir: z.string().optional(),
406
409
  artifacts: z.enum([
407
410
  "all",
408
411
  "failures",
409
412
  "none"
410
- ]).optional().describe("Artifacts recorded next to each run: every eval's, failing evals' only, or none.")
413
+ ]).optional()
411
414
  });
412
415
  const recordMode = z.enum([
413
416
  "manual",
@@ -415,32 +418,32 @@ const recordMode = z.enum([
415
418
  "always"
416
419
  ]);
417
420
  const profileDefaultsDataSchema = z.strictObject({
418
- harness: harnessList.optional().describe("The agents every eval runs on, unless a group or eval replaces them."),
419
- sandbox: sandboxDescriptorSchema.optional().describe("The default machine when the resolved environment omits `sandbox`."),
420
- environment: environmentDataSchema.optional().describe("What every eval runs in when it names no environment."),
421
- runs: count().optional().describe("Attempts per cell."),
422
- gate: gatePolicySchema.optional().describe("How each cell's attempts decide its outcome."),
423
- env: envVarsSchema.optional().describe("Variables and secrets for the sandbox, merged by name over the config's."),
424
- egress: egressPolicySchema.optional().describe("Where the sandbox may connect; replaces the config's policy."),
425
- judge: configJudgeDataSchema.optional().describe("Defaults for `c.judge()`, merged field by field over the config's."),
426
- timeout: durationSchema.optional().describe("Per attempt."),
427
- capture: captureOptionsSchema.optional().describe("Screenshot defaults for `c.capture()`, shallow-merged over the config's."),
428
- question: questionOptionsSchema.optional().describe("Defaults for every question, merged over the config's.")
429
- }).describe("`ProfileDefaults` as data: the environment without `extend`, and the judge as data.");
421
+ harness: harnessList.optional(),
422
+ sandbox: sandboxDescriptorSchema.optional(),
423
+ environment: environmentDataSchema.optional(),
424
+ runs: count().optional(),
425
+ gate: gatePolicySchema.optional(),
426
+ env: envVarsSchema.optional(),
427
+ egress: egressPolicySchema.optional(),
428
+ judge: configJudgeDataSchema.optional(),
429
+ timeout: durationSchema.optional(),
430
+ capture: captureOptionsSchema.optional(),
431
+ question: questionOptionsSchema.optional()
432
+ });
430
433
  const profileLimitSchema = z.strictObject({
431
- runs: count().optional().describe("The most runs per cell."),
432
- compare: z.union([z.literal(false), z.strictObject({ maxVariants: z.number().int().min(0) })]).optional().describe("`false` runs the control only; `{ maxVariants }` keeps the first n variants."),
433
- harness: z.array(z.string()).optional().describe("Harness ids to allow; evals left with none are skipped.")
434
- }).describe("Over every eval, and it can only narrow; `pin` on a group or eval skips it.");
434
+ runs: count().optional(),
435
+ compare: z.union([z.literal(false), z.strictObject({ maxVariants: z.number().int().min(0) })]).optional(),
436
+ harness: z.array(z.string()).optional()
437
+ });
435
438
  const profileDataSchema = z.strictObject({
436
- defaults: profileDefaultsDataSchema.optional().describe("What evals get when they don't say otherwise: under groups and evals."),
437
- limit: profileLimitSchema.optional().describe("Caps over every eval, which only narrow; `pin` on a group or eval skips them."),
438
- judge: configJudgeDataSchema.optional().describe("Judge settings for this profile, the same as `defaults.judge`, which wins field by field."),
439
- budget: budgetSchema.optional().describe("Merged over the config's `budget`."),
440
- record: recordMode.optional().describe("When this profile's runs are copied into the history folder."),
441
- tags: z.array(z.string()).optional().describe("Tag filters for this profile, like `--tag` (`'!slow'` excludes)."),
442
- gateway: z.array(gatewayEntryDataSchema).min(1).optional().describe("Replaces the config's `gateway` list for this profile's runs.")
443
- }).describe("`Profile` as data: its defaults and judge as data.");
439
+ defaults: profileDefaultsDataSchema.optional(),
440
+ limit: profileLimitSchema.optional(),
441
+ judge: configJudgeDataSchema.optional(),
442
+ budget: budgetSchema.optional(),
443
+ record: recordMode.optional(),
444
+ tags: z.array(z.string()).optional(),
445
+ gateway: z.array(gatewayEntryDataSchema).min(1).optional()
446
+ });
444
447
  const fakeVote = z.enum([
445
448
  "A",
446
449
  "B",
@@ -449,7 +452,7 @@ const fakeVote = z.enum([
449
452
  const failure = z.strictObject({ error: z.string() });
450
453
  const fakeAnalysisDataSchema = z.strictObject({
451
454
  kind: z.literal("fake-analysis"),
452
- output: z.unknown().optional().describe("The model's structured answer."),
455
+ output: z.unknown().optional(),
453
456
  vote: z.union([
454
457
  fakeVote,
455
458
  z.strictObject({
@@ -457,7 +460,7 @@ const fakeAnalysisDataSchema = z.strictObject({
457
460
  reason: z.string().optional()
458
461
  }),
459
462
  failure
460
- ]).optional().describe("How it answers pairwise questions: a fixed pick, or `{ error }` to fail every call."),
463
+ ]).optional(),
461
464
  answer: z.union([
462
465
  z.string(),
463
466
  z.strictObject({
@@ -465,46 +468,44 @@ const fakeAnalysisDataSchema = z.strictObject({
465
468
  text: z.string().optional()
466
469
  }),
467
470
  failure
468
- ]).optional().describe("How it answers questions asked of each attempt: a fixed answer, or `{ error }` to fail every call.")
469
- }).describe("`fakeAnalysis()` as data: canned answers, without the functions of the digest or pair.");
471
+ ]).optional()
472
+ });
470
473
  const analysisConfigDataSchema = z.strictObject({
471
- model: z.union([z.string(), fakeAnalysisDataSchema]).optional().describe("`'provider/model'`, e.g. `'google/gemini-3.8-flash'`. Insights use it only; `reeval analyze` and questions fall back to the judge's when it's a language model."),
472
- insights: z.boolean().optional().describe("`reeval run` classifies each eval and explains its result with `model` once the attempts finish (default true when `model` is set; `--no-insights` turns it off for one run)."),
473
- maxTokens: z.number().int().min(1).optional().describe("Tokens (in and out) that one run's insights may use, over every classification, insight and headline call (default 250000). Past it, the evals left print without insights; the run's outcome and exit code never change."),
474
- instructions: z.string().optional().describe("Added to the prompt, e.g. \"Fewer tool calls count as a win. The skill under test is reeval-good.\""),
475
- maxInputTokens: z.number().int().min(1e3).optional().describe("The digest's cap; evidence is dropped, lowest priority first, to fit (default 20000)."),
476
- maxOutputTokens: z.number().int().min(500).optional().describe("The most tokens the model may answer with (default 8000). reeval has no prices, so tokens are the cap.")
477
- }).describe("`AnalysisConfig` as data: the model by name, or a fake analysis.");
474
+ model: z.union([z.string(), fakeAnalysisDataSchema]).optional(),
475
+ instructions: z.string().optional(),
476
+ maxInputTokens: z.number().int().min(1e3).optional(),
477
+ maxOutputTokens: z.number().int().min(500).optional()
478
+ });
478
479
  const reevalConfigDataSchema = z.strictObject({
479
- include: z.array(z.string()).optional().describe("Globs of eval files, relative to the config's directory; default every `.eval.ts` under `evals`."),
480
- harness: harnessList.optional().describe("The agents every eval runs on, unless a group or eval replaces them; default Cursor."),
481
- sandbox: sandboxDescriptorSchema.optional().describe("The default machine when the resolved environment omits `sandbox`. Not an environment: it does not invent skills, files or a name. Implicit `docker()` when nobody sets one."),
482
- environment: environmentDataSchema.optional().describe("What every eval runs in when it names no environment, and a baseline-less compare's control."),
483
- runs: count().optional().describe("Attempts per cell; default 1."),
484
- gate: gatePolicySchema.optional().describe("How each cell's attempts decide its outcome; default `'all'`."),
485
- env: envVarsSchema.optional().describe("Variables and secrets for the sandbox, merged by name down the cascade."),
486
- egress: egressPolicySchema.optional().describe("Where the sandbox may connect; default `'proxy'`."),
487
- judge: configJudgeDataSchema.optional().describe("Defaults for `c.judge()`, merged field by field down the cascade."),
488
- timeout: durationSchema.optional().describe("Per attempt; default `'15m'`."),
489
- capture: captureOptionsSchema.optional().describe("Screenshot defaults for `c.capture()`. `REEVAL_CAPTURE_BROWSER=0` refuses the host binary when `c.capture()` runs; it does not change this object."),
490
- question: questionOptionsSchema.optional().describe("Defaults for every question: the model that answers it and what it's shown of each attempt. Merged down the cascade."),
491
- concurrency: count().optional().describe("How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it."),
492
- record: recordMode.optional().describe("When runs are copied into the history folder: `reeval record` only, in CI, or on every run."),
493
- history: historyConfigSchema.optional().describe("The history folder, and which attempts keep their artifacts there."),
494
- budget: budgetSchema.optional().describe("Caps on a run; a profile's `budget` is merged over it."),
495
- profiles: z.record(z.string(), profileDataSchema).optional().describe("Named presets of `defaults`, `limit`, `judge`, `budget`, `record` and `tags`."),
480
+ include: z.array(z.string()).optional(),
481
+ harness: harnessList.optional(),
482
+ sandbox: sandboxDescriptorSchema.optional(),
483
+ environment: environmentDataSchema.optional(),
484
+ runs: count().optional(),
485
+ gate: gatePolicySchema.optional(),
486
+ env: envVarsSchema.optional(),
487
+ egress: egressPolicySchema.optional(),
488
+ judge: configJudgeDataSchema.optional(),
489
+ timeout: durationSchema.optional(),
490
+ capture: captureOptionsSchema.optional(),
491
+ question: questionOptionsSchema.optional(),
492
+ concurrency: count().optional(),
493
+ record: recordMode.optional(),
494
+ history: historyConfigSchema.optional(),
495
+ budget: budgetSchema.optional(),
496
+ profiles: z.record(z.string(), profileDataSchema).optional(),
496
497
  defaultProfile: z.strictObject({
497
498
  ci: z.string().optional(),
498
499
  local: z.string().optional()
499
- }).optional().describe("The profile when neither `--profile` nor `REEVAL_PROFILE` names one: `ci` when `CI` is set."),
500
- secrets: z.strictObject({ unbrokered: z.enum(["warn", "error"]).optional() }).optional().describe("`unbrokered: 'error'` refuses Docker secrets without `hosts` instead of warning (default `'warn'`)."),
500
+ }).optional(),
501
+ secrets: z.strictObject({ unbrokered: z.enum(["warn", "error"]).optional() }).optional(),
501
502
  debug: z.strictObject({
502
503
  keep: count().optional(),
503
504
  traceBufferMb: z.number().positive().optional()
504
- }).optional().describe("Local debug data in `.reeval/debug`: how many runs to keep (default 20), and how many MB of raw trace each attempt buffers for trace.ndjson (default 8; the oldest entries go first)."),
505
- analysis: analysisConfigDataSchema.optional().describe("Insights during `reeval run`, `reeval analyze`, and questions without a model."),
506
- gateway: z.array(gatewayEntryDataSchema).min(1).optional().describe("The gateways model calls may go through, in the order you prefer them: each harness takes the first that can serve it. Default `[direct]`, each provider's own key.")
507
- }).describe("`ReevalConfig` in its data forms, as `reeval.yaml` or another language carries it: the environment without `extend`, judge and analysis models by name or as fakes with fixed answers, and profiles the same way. AI SDK model objects and fake score functions are TypeScript only.");
505
+ }).optional(),
506
+ analysis: analysisConfigDataSchema.optional(),
507
+ gateway: z.array(gatewayEntryDataSchema).min(1).optional()
508
+ });
508
509
  /** The object schema an issue's path leads to through `schema`, for "Did you mean". */
509
510
  function keysAt(schema, path, value) {
510
511
  const def = schema.def;
@@ -573,6 +574,15 @@ function configDataProblem(value, file) {
573
574
  }, v)}`;
574
575
  }
575
576
  //#endregion
577
+ //#region src/gateway/routes.ts
578
+ /** Every gateway that brokers a key, by name. */
579
+ const ROUTING_GATEWAYS = {
580
+ vercel: VERCEL_GATEWAY,
581
+ openrouter: OPENROUTER_GATEWAY
582
+ };
583
+ /** Whether a harness's `gateway` names one of {@link ROUTING_GATEWAYS}. */
584
+ const isRoutingGateway = (name) => name !== void 0 && Object.hasOwn(ROUTING_GATEWAYS, name);
585
+ //#endregion
576
586
  //#region src/harness/agents.ts
577
587
  const WEB_EVENTS = {
578
588
  webSearch: "web-search",
@@ -710,12 +720,40 @@ function antigravityMcpServers(servers) {
710
720
  }]));
711
721
  }
712
722
  /**
723
+ * A `{env:…}` or `{file:…}` OpenCode would substitute. It runs `\{env:[^}]+\}`, then
724
+ * `\{file:[^}]+\}`, over its whole config text with no escape, so a reference left open at the end
725
+ * of a value runs on into the JSON after it, and only `{env:}` and `{file:}` are left alone.
726
+ */
727
+ const OPENCODE_REFERENCE = /\{(?:env|file):(?!\})/;
728
+ const fieldKey = (key) => /^[A-Za-z_$][\w$]*$/.test(key) ? `.${key}` : `[${JSON.stringify(key)}]`;
729
+ /**
730
+ * The first part of a server OpenCode would substitute, as what the server wouldn't get: its name,
731
+ * a value as `mcp.<name>.<field>`, or an env or header name. Names are in the same config text as
732
+ * values, so they're substituted too; a secret's value is never named.
733
+ */
734
+ function openCodeReference(name, s) {
735
+ if (OPENCODE_REFERENCE.test(name)) return "its name";
736
+ const server = `mcp${fieldKey(name)}`;
737
+ const [map, kind, values] = s.transport === "stdio" ? [
738
+ "env",
739
+ "env",
740
+ s.env ?? {}
741
+ ] : [
742
+ "headers",
743
+ "header",
744
+ s.headers ?? {}
745
+ ];
746
+ return [...s.transport === "stdio" ? [[`${server}.command`, s.command], ...s.args.map((a, i) => [`${server}.args[${i}]`, a])] : [[`${server}.url`, s.url]], ...Object.entries(values).flatMap(([k, v]) => [[`the ${kind} name ${k} in ${server}.${map}`, k], [`${server}.${map}${fieldKey(k)}`, v]])].find(([, text]) => OPENCODE_REFERENCE.test(text))?.[0];
747
+ }
748
+ /**
713
749
  * OpenCode's config shape (its `mcp.servers`), which the adapter sets `codemode: false` on.
714
- * OpenCode has no SSE transport.
750
+ * OpenCode has no SSE transport, and refuses a value it would substitute.
715
751
  */
716
752
  function opencodeMcpServers(servers) {
717
753
  return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
718
754
  if (s.transport === "sse") throw new InfraError(`OpenCode has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
755
+ const field = openCodeReference(name, s);
756
+ if (field) throw new InfraError(`OpenCode substitutes {env:…} and {file:…} anywhere in its config, with no escape, so the ${name} server wouldn't get ${field} as written: take the reference out, or leave OpenCode out of this cell`);
719
757
  return s.transport === "stdio" ? [name, {
720
758
  type: "local",
721
759
  command: [s.command, ...s.args],
@@ -975,7 +1013,7 @@ async function writePluginFiles(sandbox, layout, plugins) {
975
1013
  if (existing?.trim()) {
976
1014
  let base;
977
1015
  try {
978
- base = JSON.parse(existing);
1016
+ base = JSON.parse(withoutBom(existing));
979
1017
  } catch {}
980
1018
  if (!isObject(base)) throw new InfraError(`${f.path} isn't a JSON object, so reeval can't enable the session's plugins in it: write it as one in files, or leave it out`);
981
1019
  const ours = JSON.parse(f.content);
@@ -1087,7 +1125,7 @@ const AGENTS = {
1087
1125
  },
1088
1126
  unreported: cursorUnreported,
1089
1127
  mcpServers: cursorMcpServers,
1090
- adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-CyaDNG9r.js")).createCursorSdk({
1128
+ adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-6_zz7oJs.js")).createCursorSdk({
1091
1129
  auth,
1092
1130
  ...mcpServers ? { mcpServers } : {},
1093
1131
  ...descriptor.kind === "cursor" && descriptor.params ? { modelParams: descriptor.params } : {},
@@ -1124,13 +1162,14 @@ const AGENTS = {
1124
1162
  }
1125
1163
  ],
1126
1164
  mcpServers: claudeMcpServers,
1127
- adapter: async ({ auth, mcpServers, descriptor }) => (await import("./dist-CYrBwyHA.js")).createClaudeCode({
1165
+ adapter: async ({ auth, env, mcpServers, descriptor }) => (await import("./dist-CYrBwyHA.js")).createClaudeCode({
1128
1166
  auth,
1129
1167
  ...mcpServers ? { mcpServers } : {},
1130
1168
  ...descriptor.kind === "claude-code" && descriptor.effort ? { effort: descriptor.effort } : {},
1131
1169
  env: {
1132
1170
  IS_SANDBOX: "1",
1133
- CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1"
1171
+ CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",
1172
+ ...env
1134
1173
  }
1135
1174
  })
1136
1175
  },
@@ -1153,8 +1192,9 @@ const AGENTS = {
1153
1192
  },
1154
1193
  servedModel: geminiServedModel,
1155
1194
  mcpServers: antigravityMcpServers,
1156
- adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-ElBqEN-S.js")).createAntigravity({
1195
+ adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-C7upnZSd.js")).createAntigravity({
1157
1196
  auth,
1197
+ ...vertex ? { vertex } : {},
1158
1198
  ...mcpServers ? { mcpServers } : {},
1159
1199
  ...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
1160
1200
  name,
@@ -1179,10 +1219,10 @@ const AGENTS = {
1179
1219
  servedModel: geminiServedModel,
1180
1220
  forModel: opencodeForModel,
1181
1221
  mcpServers: opencodeMcpServers,
1182
- adapter: async ({ auth, mcpServers, descriptor, model }) => (await import("./src-DIfRbp5a.js")).createOpenCode({
1222
+ adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-HSxBazcg.js")).createOpenCode({
1183
1223
  auth,
1184
1224
  model,
1185
- vars: opencodeVars(model),
1225
+ ...vertex ? { vertex } : { vars: opencodeVars(model) },
1186
1226
  ...descriptor.kind === "opencode" && descriptor.effort ? { effort: descriptor.effort } : {},
1187
1227
  ...mcpServers ? { mcpServers } : {},
1188
1228
  requireCredentialBrokering: true
@@ -1197,14 +1237,18 @@ const AGENTS = {
1197
1237
  endpoints: GOOSE_ENDPOINTS,
1198
1238
  expectedBlocked: {
1199
1239
  hosts: ["models.dev"],
1200
- why: "Goose's model catalogue refresh, which no setting turns off"
1240
+ why: "Goose's model catalog refresh, which no setting turns off"
1201
1241
  },
1202
1242
  servedModel: geminiServedModel,
1203
1243
  mcpServers: gooseMcpServers,
1204
- adapter: async ({ auth, mcpServers, descriptor, model }) => (await import("./src-Dg8121Xy.js")).createGoose({
1244
+ adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-kYeaJJrl.js")).createGoose({
1245
+ ...vertex ? { vertex } : {},
1205
1246
  ...auth["AI_GATEWAY_API_KEY"] ? {
1206
1247
  auth,
1207
- provider: (await import("./src-Dg8121Xy.js")).AI_GATEWAY_PROVIDER
1248
+ provider: (await import("./src-kYeaJJrl.js")).AI_GATEWAY_PROVIDER
1249
+ } : auth["OPENROUTER_API_KEY"] ? {
1250
+ auth,
1251
+ provider: (await import("./src-kYeaJJrl.js")).OPENROUTER_PROVIDER
1208
1252
  } : { auth: { GOOGLE_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" } },
1209
1253
  model,
1210
1254
  ...descriptor.kind === "goose" && descriptor.effort ? { effort: descriptor.effort } : {},
@@ -1225,8 +1269,17 @@ const AGENTS = {
1225
1269
  endpoints: GEMINI_ENDPOINTS,
1226
1270
  servedModel: geminiServedModel,
1227
1271
  mcpServers: piMcpServers,
1228
- adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-mJVxw1Jm.js")).createPi({
1229
- auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
1272
+ adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-Bp4aN3rA.js")).createPi({
1273
+ ...auth["AI_GATEWAY_API_KEY"] ? {
1274
+ auth,
1275
+ gateway: (await import("./src-Bp4aN3rA.js")).PI_AI_GATEWAY
1276
+ } : auth["OPENROUTER_API_KEY"] ? {
1277
+ auth,
1278
+ gateway: (await import("./src-Bp4aN3rA.js")).PI_OPENROUTER
1279
+ } : {
1280
+ auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
1281
+ ...vertex ? { vertex } : {}
1282
+ },
1230
1283
  ...descriptor.kind === "pi" && descriptor.effort ? { effort: descriptor.effort } : {},
1231
1284
  ...mcpServers ? { mcpServers } : {},
1232
1285
  requireCredentialBrokering: true
@@ -1255,9 +1308,9 @@ const AGENTS = {
1255
1308
  },
1256
1309
  servedModel: openaiServedModel,
1257
1310
  mcpServers: codexMcpServers,
1258
- adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-D005Cbqz.js")).createCodex({
1311
+ adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-dlX-Lmxg.js")).createCodex({
1259
1312
  auth,
1260
- ...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-D005Cbqz.js")).AI_GATEWAY_PROVIDER } : {},
1313
+ ...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-dlX-Lmxg.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-dlX-Lmxg.js")).OPENROUTER_PROVIDER } : {},
1261
1314
  ...descriptor.kind === "codex" && descriptor.effort ? { effort: descriptor.effort } : {},
1262
1315
  ...mcpServers ? { mcpServers } : {},
1263
1316
  ...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
@@ -1275,9 +1328,15 @@ const AGENTS = {
1275
1328
  ...fxForModel("provider/model"),
1276
1329
  forModel: fxForModel,
1277
1330
  mcpServers: acpMcpServers,
1278
- adapter: async ({ auth, mcpServers, model }) => (await import("./src-YHGaUrGE.js")).createFx({
1331
+ adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-LD1BAJFa.js")).createFx({
1279
1332
  auth,
1280
1333
  model,
1334
+ ...auth["OPENROUTER_API_KEY"] ? { provider: "openrouter" } : {},
1335
+ ...env?.FX_VERTEX_PROJECT ? { vertex: {
1336
+ project: env.FX_VERTEX_PROJECT,
1337
+ location: env.FX_VERTEX_LOCATION ?? ""
1338
+ } } : {},
1339
+ ...descriptor.kind === "fx" && descriptor.effort ? { effort: descriptor.effort } : {},
1281
1340
  ...mcpServers ? { mcpServers } : {},
1282
1341
  requireCredentialBrokering: true
1283
1342
  })
@@ -1293,18 +1352,26 @@ const AGENTS = {
1293
1352
  modelResponses: XAI_MODEL_RESPONSES,
1294
1353
  servedModel: xaiServedModel,
1295
1354
  mcpServers: acpMcpServers,
1296
- adapter: async ({ auth, mcpServers, descriptor, plugins, model }) => (await import("./src-C_9wgowC.js")).createGrokBuild({
1297
- auth,
1298
- ...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-C_9wgowC.js")).AI_GATEWAY_PROVIDER } : {},
1299
- model,
1300
- ...plugins.length ? {
1301
- pluginDirs: plugins.map((p) => p.dir),
1302
- pluginMcpServers: plugins.flatMap((p) => p.servers.map((s) => s.slice(p.name.length + 1)))
1303
- } : {},
1304
- ...descriptor.kind === "grok-build" && descriptor.effort ? { effort: descriptor.effort } : {},
1305
- ...mcpServers ? { mcpServers } : {},
1306
- requireCredentialBrokering: true
1307
- })
1355
+ adapter: async ({ auth, mcpServers, descriptor, plugins, model, vertexChat }) => {
1356
+ const grok = await import("./src-BYPMM1x7.js");
1357
+ return grok.createGrokBuild({
1358
+ ...vertexChat ? {
1359
+ auth: { [grok.VERTEX_TOKEN_ENV]: SANDBOX_ACCESS_TOKEN },
1360
+ provider: grok.vertexProvider(vertexChat.project, vertexChat.region)
1361
+ } : {
1362
+ auth,
1363
+ ...auth["AI_GATEWAY_API_KEY"] ? { provider: grok.AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: grok.OPENROUTER_PROVIDER } : {}
1364
+ },
1365
+ model,
1366
+ ...plugins.length ? {
1367
+ pluginDirs: plugins.map((p) => p.dir),
1368
+ pluginMcpServers: plugins.flatMap((p) => p.servers.map((s) => s.slice(p.name.length + 1)))
1369
+ } : {},
1370
+ ...descriptor.kind === "grok-build" && descriptor.effort ? { effort: descriptor.effort } : {},
1371
+ ...mcpServers ? { mcpServers } : {},
1372
+ requireCredentialBrokering: true
1373
+ });
1374
+ }
1308
1375
  }
1309
1376
  };
1310
1377
  /** The harness of a config that names none: Cursor, on Cursor's own model. */
@@ -1316,20 +1383,31 @@ const DEFAULT_HARNESS = {
1316
1383
  const HARNESS_EXAMPLE = "cursor({ model: 'composer-2.5' })";
1317
1384
  const isAgentKind = (kind) => typeof kind === "string" && Object.hasOwn(AGENTS, kind);
1318
1385
  /**
1386
+ * A server as the plan knows it: its names, literal values and each `secret.template`'s text. A
1387
+ * host secret's value is read when the attempt starts, so the plan has it as `''`.
1388
+ */
1389
+ function plannedMcpServer(s) {
1390
+ const literal = (values) => Object.fromEntries(Object.entries(values).map(([k, v]) => [k, typeof v === "string" ? v : v.kind === "secret-template" ? v.template : ""]));
1391
+ return s.transport === "stdio" ? {
1392
+ transport: "stdio",
1393
+ command: s.command,
1394
+ args: s.args,
1395
+ env: literal(s.env)
1396
+ } : {
1397
+ transport: s.transport,
1398
+ url: s.url,
1399
+ headers: literal(s.headers)
1400
+ };
1401
+ }
1402
+ /**
1319
1403
  * Why a harness's agent can't take the environment's MCP servers (Goose, OpenCode, pi and Codex
1320
- * take no `mcp.sse`), in the agent's own words, so the plan refuses them before any attempt
1321
- * starts; undefined when it can.
1404
+ * take no `mcp.sse`; OpenCode would substitute a `{env:…}` or `{file:…}` in a name or value), in the
1405
+ * agent's own words, so the plan refuses them before any attempt starts; undefined when it can.
1322
1406
  */
1323
1407
  function refusedMcp(harness, layers) {
1324
1408
  if (!isAgentKind(harness.kind)) return void 0;
1325
1409
  const servers = /* @__PURE__ */ new Map();
1326
- for (const { spec } of layers) for (const [name, server] of Object.entries(spec.mcp ?? {})) servers.set(name, server.transport === "stdio" ? {
1327
- ...server,
1328
- env: {}
1329
- } : {
1330
- ...server,
1331
- headers: {}
1332
- });
1410
+ for (const { spec } of layers) for (const [name, server] of Object.entries(spec.mcp ?? {})) servers.set(name, plannedMcpServer(server));
1333
1411
  if (servers.size === 0) return void 0;
1334
1412
  try {
1335
1413
  AGENTS[harness.kind].mcpServers(Object.fromEntries(servers));
@@ -1392,13 +1470,7 @@ function refusedPlugins(harness, plugins, own, unread = []) {
1392
1470
  const fix = rest.length ? `use only: [${quoted(rest)}], or leave` : "leave";
1393
1471
  return event ? `${name} can't run the ${p.name} plugin's ${event} hooks (hooks/hooks.json): it runs ${runs?.join(", ")} hooks; ${fix} the plugin out of this cell` : `${name} can't run the ${p.name} plugin's ${type} hooks (hooks/hooks.json): it runs only command hooks; ${fix} the plugin out of this cell`;
1394
1472
  }
1395
- const servers = Object.fromEntries(plugins.flatMap((p) => Object.entries(p.mcp).map(([server, s]) => [pluginServerName(p.name, server), s.transport === "stdio" ? {
1396
- ...s,
1397
- env: {}
1398
- } : {
1399
- ...s,
1400
- headers: {}
1401
- }])));
1473
+ const servers = Object.fromEntries(plugins.flatMap((p) => Object.entries(p.mcp).map(([server, s]) => [pluginServerName(p.name, server), plannedMcpServer(s)])));
1402
1474
  if (agent && Object.keys(servers).length) {
1403
1475
  const alongside = agent.pluginPlacement === "flat" ? Object.fromEntries(own.mcp.map((n) => [n, {
1404
1476
  transport: "stdio",
@@ -1472,12 +1544,25 @@ function refusedPlugins(harness, plugins, own, unread = []) {
1472
1544
  ruleOwner.set(file, what);
1473
1545
  }
1474
1546
  }
1475
- /** A real agent's catalog entry, with what its model's provider decides. */
1476
- function agentFor(h) {
1547
+ /**
1548
+ * A real agent's catalog entry, with what its model's provider decides, or its gateway's route:
1549
+ * on Vertex, the attempt's `vertex()` settings decide the project and region.
1550
+ */
1551
+ function agentFor(h, vertex) {
1477
1552
  const agent = AGENTS[h.kind];
1478
- if (h.gateway === "vercel") {
1479
- const route = VERCEL_ROUTES[h.kind];
1480
- if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on Vercel AI Gateway`);
1553
+ if (h.gateway === "vertex") {
1554
+ const route = VERTEX_ROUTES[h.kind];
1555
+ if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on Vertex AI`);
1556
+ if (!vertex) throw new InfraError(`${agent.name} on Vertex AI has no vertex() settings`);
1557
+ return {
1558
+ ...agent,
1559
+ ...route(vertex)
1560
+ };
1561
+ }
1562
+ if (isRoutingGateway(h.gateway)) {
1563
+ const gateway = ROUTING_GATEWAYS[h.gateway];
1564
+ const route = gateway.routes[h.kind];
1565
+ if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on ${gateway.name}`);
1481
1566
  return {
1482
1567
  ...agent,
1483
1568
  ...route(process.env)
@@ -1489,9 +1574,12 @@ function agentFor(h) {
1489
1574
  } : agent;
1490
1575
  }
1491
1576
  /** The catalog entry for a harness's agent; undefined for the fake harness. */
1492
- const agentOf = (h) => h.kind === "fake" ? void 0 : agentFor(h);
1493
- /** The host variable a harness authenticates with; undefined for the fake harness. */
1494
- const credentialOf = (h) => agentOf(h)?.credential;
1577
+ const agentOf = (h, vertex) => h.kind === "fake" ? void 0 : agentFor(h, vertex);
1578
+ /**
1579
+ * The host variable a harness authenticates with; undefined for the fake harness, and on Vertex,
1580
+ * whose token comes from Application Default Credentials.
1581
+ */
1582
+ const credentialOf = (h) => h.kind === "fake" || h.gateway === "vertex" ? void 0 : agentFor(h).credential;
1495
1583
  /** The fake agent's own layout. */
1496
1584
  const FAKE_LAYOUT = {
1497
1585
  rulesFile: "AGENTS.md",
@@ -1506,10 +1594,10 @@ function layoutOf(descriptor) {
1506
1594
  }
1507
1595
  /**
1508
1596
  * The hosts a harness's agent is known to try in a session with or without plugins; a fake agent
1509
- * with `layout` expects that agent's.
1597
+ * with `layout` expects that agent's. On Vertex, the attempt's `vertex()` settings pick the route.
1510
1598
  */
1511
- function expectedBlockedFor(descriptor, plugins) {
1512
- const expected = descriptor.kind !== "fake" ? agentFor(descriptor).expectedBlocked : descriptor.layout ? AGENTS[descriptor.layout].expectedBlocked : void 0;
1599
+ function expectedBlockedFor(descriptor, plugins, vertex) {
1600
+ const expected = descriptor.kind !== "fake" ? agentFor(descriptor, vertex).expectedBlocked : descriptor.layout ? AGENTS[descriptor.layout].expectedBlocked : void 0;
1513
1601
  return expected?.withPlugins && !plugins ? void 0 : expected;
1514
1602
  }
1515
1603
  /**
@@ -1527,18 +1615,27 @@ const REFUSED_KEY = /\bHTTP 401\b|\b401\b.{0,40}\bunauthori[sz]ed\b|\bunauthori[
1527
1615
  */
1528
1616
  function authHint(agent, message, refused = false) {
1529
1617
  if (!refused && !REFUSED_KEY.test(message)) return void 0;
1618
+ if (agent.token) return `The API rejected the access token reeval minted from ${agent.credential}: check gcloud auth application-default login (or GOOGLE_APPLICATION_CREDENTIALS) and the project's Vertex AI access`;
1530
1619
  return `The API rejected ${agent.credential}: set a valid key in the environment or .env.local`;
1531
1620
  }
1532
- /** The agent's credential from the host, as the adapter's explicit `auth`. Fails closed. */
1621
+ /**
1622
+ * The agent's credential from the host, as the adapter's explicit `auth`. Fails closed. Empty for
1623
+ * a brokered token, which the adapter never sees.
1624
+ */
1533
1625
  function agentAuth(agent, env = process.env) {
1626
+ if (agent.token) return {};
1534
1627
  const value = env[agent.credential];
1535
1628
  if (!value) throw new InfraError(`${agent.credential} isn't set, so the ${agent.name} harness can't authenticate. Set it in the environment or .env.local.`);
1536
- return { [agent.authAs ?? agent.credential]: value };
1629
+ return {
1630
+ ...agent.authEnv,
1631
+ [agent.authAs ?? agent.credential]: value
1632
+ };
1537
1633
  }
1538
- function brokeringFor(h) {
1539
- const agent = agentOf(h);
1634
+ function brokeringFor(h, vertex) {
1635
+ const agent = agentOf(h, vertex);
1540
1636
  if (!agent) return void 0;
1541
- const { credential, endpoints, tokens, servedModel, askedModel, modelResponses } = agent;
1637
+ const { credential, endpoints, tokens, servedModel, askedModel, modelResponses, upstream } = agent;
1638
+ const { localResponses } = agent;
1542
1639
  return {
1543
1640
  credential,
1544
1641
  endpoints,
@@ -1548,7 +1645,9 @@ function brokeringFor(h) {
1548
1645
  } } : {},
1549
1646
  ...servedModel ? { servedModel } : {},
1550
1647
  ...askedModel ? { askedModel } : {},
1551
- ...modelResponses ? { modelResponses } : {}
1648
+ ...modelResponses ? { modelResponses } : {},
1649
+ ...upstream ? { upstream } : {},
1650
+ ...localResponses?.length ? { localResponses } : {}
1552
1651
  };
1553
1652
  }
1554
1653
  //#endregion
@@ -1631,7 +1730,7 @@ const DEFAULT_ATTEMPT_QUESTION_STATE = {
1631
1730
  const KEYS = Object.keys(DEFAULT_QUESTION_STATE);
1632
1731
  /** A pairwise question's defaults when `pairwise`, a per-attempt question's otherwise. */
1633
1732
  const defaultQuestionState = (pairwise) => pairwise ? DEFAULT_QUESTION_STATE : DEFAULT_ATTEMPT_QUESTION_STATE;
1634
- /** Every key set, in `DEFAULT_QUESTION_STATE`'s order, so equal selections serialise the same. */
1733
+ /** Every key set, in `DEFAULT_QUESTION_STATE`'s order, so equal selections serialize the same. */
1635
1734
  function resolveQuestionState(state, pairwise) {
1636
1735
  const defaults = defaultQuestionState(pairwise);
1637
1736
  const out = {};
@@ -1951,13 +2050,10 @@ function resolveOutput(input) {
1951
2050
  const { flags, env } = input;
1952
2051
  const ci = isCI(env);
1953
2052
  const githubActions = set(env, "GITHUB_ACTIONS");
1954
- const ui = pick("--ui", flags.ui ?? env.REEVAL_UI, [
1955
- "live",
1956
- "plain",
1957
- "agent"
1958
- ]);
1959
- const requested = ui && ui !== "auto" ? ui : input.agent ? "agent" : ci || !input.isTTY ? "plain" : "live";
1960
- const mode = requested === "live" ? "plain" : requested;
2053
+ const ui = pick("--ui", flags.ui ?? env.REEVAL_UI, ["plain", "agent"]);
2054
+ const named = ui !== void 0 && ui !== "auto";
2055
+ const mode = named ? ui : input.agent ? "agent" : "plain";
2056
+ const statusLine = !named && mode === "plain" && !ci && input.isTTY;
1961
2057
  const order = pick("--log-order", flags.logOrder ?? env.REEVAL_LOG_ORDER, ["stream", "grouped"]);
1962
2058
  const logOrder = order && order !== "auto" ? order : ci ? "grouped" : "stream";
1963
2059
  const prefix = pick("--log-prefix", flags.logPrefix ?? env.REEVAL_LOG_PREFIX, ["task", "none"]);
@@ -1968,8 +2064,8 @@ function resolveOutput(input) {
1968
2064
  "none"
1969
2065
  ]);
1970
2066
  return {
1971
- requested,
1972
2067
  mode,
2068
+ statusLine,
1973
2069
  logOrder,
1974
2070
  logPrefix,
1975
2071
  outputLogs: logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only",
@@ -2003,8 +2099,8 @@ const isDirect = (policy) => policy === "allow-all";
2003
2099
  * The hosts the harness needs: a real agent's own API (its model's provider's, for OpenCode), or
2004
2100
  * a fake agent's with `layout` (it behaves like that agent). A plain fake agent talks to nothing.
2005
2101
  */
2006
- function harnessHosts(harness) {
2007
- if (harness.kind !== "fake" && isAgentKind(harness.kind)) return agentFor(harness).hosts ?? HARNESS_HOSTS[harness.kind];
2102
+ function harnessHosts(harness, vertex) {
2103
+ if (harness.kind !== "fake" && isAgentKind(harness.kind)) return agentFor(harness, vertex).hosts ?? HARNESS_HOSTS[harness.kind];
2008
2104
  return harness.kind === "fake" && harness.layout ? HARNESS_HOSTS[harness.layout] : [];
2009
2105
  }
2010
2106
  /** MCP `http`/`sse` servers' hosts, local plugins' included: the agent connects to them from the sandbox. */
@@ -2020,7 +2116,7 @@ function mcpServerHosts(layers) {
2020
2116
  * harness's, the brokered secrets' `hosts` and the MCP servers'. The cascade is config and
2021
2117
  * profile defaults (`inherited`), then the environment's layers, then groups and the eval (`own`).
2022
2118
  */
2023
- function plannedEgress(egress, harness, env, layers) {
2119
+ function plannedEgress(egress, harness, env, layers, vertex) {
2024
2120
  const fromLayers = layers.findLast((l) => l.spec.egress !== void 0)?.spec.egress;
2025
2121
  const policy = egress.own ?? fromLayers ?? egress.inherited ?? "proxy";
2026
2122
  if (!isRestricted(policy)) return {
@@ -2029,7 +2125,7 @@ function plannedEgress(egress, harness, env, layers) {
2029
2125
  };
2030
2126
  const own = policy === "deny-all" ? [] : policy.allow;
2031
2127
  const auto = [
2032
- ...harnessHosts(harness),
2128
+ ...harnessHosts(harness, vertex),
2033
2129
  ...plannedSecrets(env, layers).flatMap((s) => s.hosts ?? []),
2034
2130
  ...mcpServerHosts(layers)
2035
2131
  ];
@@ -2114,7 +2210,7 @@ const DEFAULT_STATE = {
2114
2210
  turns: "last",
2115
2211
  checks: false
2116
2212
  };
2117
- /** Every key set, in `STATE_KEYS` order, so equal selections serialise the same. */
2213
+ /** Every key set, in `STATE_KEYS` order, so equal selections serialize the same. */
2118
2214
  function resolveStateOptions(state) {
2119
2215
  const out = {};
2120
2216
  for (const key of STATE_KEYS$1) out[key] = state?.[key] ?? DEFAULT_STATE[key];
@@ -2148,6 +2244,7 @@ const KNOWN = [...[
2148
2244
  "questions",
2149
2245
  "timeout",
2150
2246
  "tags",
2247
+ "revision",
2151
2248
  "pin"
2152
2249
  ]];
2153
2250
  const SUGGEST = {
@@ -2165,6 +2262,8 @@ const SUGGEST = {
2165
2262
  network: "egress",
2166
2263
  networkPolicy: "egress",
2167
2264
  tag: "tags",
2265
+ version: "revision",
2266
+ rev: "revision",
2168
2267
  pins: "pin",
2169
2268
  passRate: "gate",
2170
2269
  threshold: "gate"
@@ -2235,6 +2334,7 @@ function refuseIllegalCompare(options, where) {
2235
2334
  const baseline = options.compare !== null && typeof options.compare === "object" && "baseline" in options.compare ? options.compare.baseline : void 0;
2236
2335
  if (options.environment !== void 0 && baseline !== void 0) throw new UsageError(`environment and compare.baseline can't both be set in ${where}. compare: { baseline, variants } names every arm, so drop environment, or drop baseline to make environment the control.`);
2237
2336
  }
2337
+ const isRevision = (value) => typeof value === "number" && Number.isSafeInteger(value) || typeof value === "string" && value.trim() !== "";
2238
2338
  /** Throws on keys that aren't options, and on values of the wrong shape, suggesting a fix. */
2239
2339
  function checkOptions(options, where, allowed = KNOWN) {
2240
2340
  for (const key of Object.keys(options)) {
@@ -2247,6 +2347,7 @@ function checkOptions(options, where, allowed = KNOWN) {
2247
2347
  if (o.runs !== void 0 && !(Number.isInteger(o.runs) && o.runs >= 1)) throw new UsageError(`runs in ${where} must be a whole number of at least 1 (got ${JSON.stringify(o.runs)})`);
2248
2348
  if (o.gate !== void 0) checkGate(o.gate, where);
2249
2349
  if (o.tags !== void 0 && (!Array.isArray(o.tags) || o.tags.some((t) => typeof t !== "string"))) throw new UsageError(`tags in ${where} must be a list of strings, e.g. tags: ['smoke'] (got ${JSON.stringify(o.tags)})`);
2350
+ if (o.revision !== void 0 && !isRevision(o.revision)) throw new UsageError(`revision in ${where} must be a whole number or a non-empty string, e.g. revision: 2 (got ${JSON.stringify(o.revision)})`);
2250
2351
  if (o.env !== void 0) checkEnv(o.env, where, UsageError);
2251
2352
  if (o.egress !== void 0) checkEgress(o.egress, where, UsageError);
2252
2353
  const judgeIssue = o.judge !== void 0 && judgeProblem(o.judge, {
@@ -2412,6 +2513,12 @@ function normalizeCompare(value) {
2412
2513
  of: value.variants.length
2413
2514
  };
2414
2515
  }
2516
+ /**
2517
+ * Runs per cell when no level sets `runs`: a compare with variants runs `EFFECT_MIN_COUNTED` a
2518
+ * side, the fewest whose lift can reach a verdict; anything else, a pooled family's cases
2519
+ * included, runs once.
2520
+ */
2521
+ const COMPARE_DEFAULT_RUNS = 4;
2415
2522
  function applyLevel(out, options, where, egress = "own") {
2416
2523
  for (const [key, value] of Object.entries(options)) if (value !== void 0) out.sources = {
2417
2524
  ...out.sources,
@@ -2425,6 +2532,7 @@ function applyLevel(out, options, where, egress = "own") {
2425
2532
  if (options.runs !== void 0) out.runs = options.runs;
2426
2533
  if (options.gate !== void 0) out.gate = options.gate;
2427
2534
  if (options.timeout !== void 0) out.timeout = options.timeout;
2535
+ if (options.revision !== void 0) out.revision = options.revision;
2428
2536
  Object.assign(out.env, options.env);
2429
2537
  out.judge = mergeJudge(out.judge, options.judge);
2430
2538
  if (options.capture !== void 0) {
@@ -2500,23 +2608,30 @@ function cascade(base, levels, where, options = {}) {
2500
2608
  const dupe = names.find((n, i) => names.indexOf(n) !== i);
2501
2609
  if (dupe) throw new UsageError(`compare in ${where} lists two variants named ${dupe}. Give each variant its own name.`);
2502
2610
  let skip;
2611
+ const harnesses = out.harness.map(harnessId);
2503
2612
  const limit = profile?.limit;
2504
2613
  if (profile && limit) {
2505
2614
  const tag = `profile ${profile.name}`;
2506
2615
  const pinned = (key) => out.pin.includes(key);
2507
- if (limit.runs !== void 0 && options.runs === void 0 && !pinned("runs") && out.runs > limit.runs) {
2508
- notes.push(`runs ${out.runs} → ${limit.runs} (${tag})`);
2509
- out.runs = limit.runs;
2510
- }
2616
+ let comparedLeft = true;
2511
2617
  if (limit.harness && !pinned("harness")) {
2512
2618
  const kept = out.harness.filter((h) => limit.harness?.includes(harnessId(h)));
2513
2619
  if (kept.length === 0) skip = `no harness left: ${tag} allows ${limit.harness.join(", ") || "none"}`;
2514
2620
  else if (kept.length < out.harness.length) {
2515
2621
  notes.push(`harnesses ${out.harness.length} → ${kept.length} (${tag})`);
2516
2622
  out.harness = kept;
2623
+ const scope = out.compare?.harness;
2624
+ if (out.compare?.variants.length && scope && !kept.some((h) => scope.includes(harnessId(h)))) {
2625
+ notes.push(`compare off: no compared harness left (${tag})`);
2626
+ out.compare = {
2627
+ ...out.compare,
2628
+ variants: []
2629
+ };
2630
+ comparedLeft = false;
2631
+ }
2517
2632
  }
2518
2633
  }
2519
- if (out.compare && limit.compare !== void 0 && !pinned("compare")) {
2634
+ if (out.compare && comparedLeft && limit.compare !== void 0 && !pinned("compare")) {
2520
2635
  if (limit.compare === false) {
2521
2636
  notes.push(`compare off (${tag})`);
2522
2637
  out.compare = {
@@ -2531,7 +2646,16 @@ function cascade(base, levels, where, options = {}) {
2531
2646
  };
2532
2647
  }
2533
2648
  }
2534
- const pins = out.pin.filter((k) => k === "compare" ? limit.compare !== void 0 && out.compare : limit[k] !== void 0);
2649
+ defaultRuns(out, notes, options);
2650
+ if (limit.runs !== void 0 && options.runs === void 0 && !pinned("runs") && (out.runs ?? 1) > limit.runs) {
2651
+ notes.push(`runs ${out.runs} → ${limit.runs} (${tag})`);
2652
+ out.runs = limit.runs;
2653
+ out.sources = {
2654
+ ...out.sources,
2655
+ runs: `profiles.${profile.name}.limit`
2656
+ };
2657
+ }
2658
+ const pins = out.pin.filter((k) => k === "compare" ? limit.compare !== void 0 && out.compare && comparedLeft : limit[k] !== void 0);
2535
2659
  if (pins.length) notes.push(`pinned: ${pins.join(", ")}`);
2536
2660
  }
2537
2661
  if (profile) {
@@ -2554,13 +2678,49 @@ function cascade(base, levels, where, options = {}) {
2554
2678
  if (narrowed) notes.push(`${plural(narrowed)} narrowed (profile ${profile.name})`);
2555
2679
  out.questions = kept;
2556
2680
  }
2557
- if (options.runs !== void 0) out.runs = options.runs;
2681
+ defaultRuns(out, notes, options);
2558
2682
  return {
2559
- resolved: out,
2683
+ resolved: {
2684
+ ...out,
2685
+ runs: options.runs ?? out.runs ?? 1
2686
+ },
2560
2687
  notes,
2688
+ harnesses,
2561
2689
  ...skip ? { skip } : {}
2562
2690
  };
2563
2691
  }
2692
+ /** Gives `out` its default runs when no level set them and `--runs` doesn't replace them. */
2693
+ function defaultRuns(out, notes, options) {
2694
+ if (out.runs !== void 0 || options.runs !== void 0) return;
2695
+ if (!out.compare?.variants.length || options.pooled) {
2696
+ out.runs = 1;
2697
+ return;
2698
+ }
2699
+ out.runs = COMPARE_DEFAULT_RUNS;
2700
+ out.sources = {
2701
+ ...out.sources,
2702
+ runs: `compare default: ${COMPARE_DEFAULT_RUNS} a side`
2703
+ };
2704
+ notes.push(`runs ${COMPARE_DEFAULT_RUNS} (compare default)`);
2705
+ }
2706
+ /**
2707
+ * Refuses a profile's `limit.harness` id that no collected eval runs on: it keeps nothing, and a
2708
+ * limit that keeps nothing skips every eval. `harnesses` are the ids every eval in the loaded
2709
+ * files runs on before the limit, whatever the tag and name filters pick. When a path filter
2710
+ * loaded only some files (`partial`), an id may name a harness of the others, so only a limit
2711
+ * none of whose ids match is refused.
2712
+ */
2713
+ function checkLimitHarness(profile, configFile, collected) {
2714
+ const ids = profile?.limit.harness;
2715
+ if (!profile || !ids?.length) return;
2716
+ const unknown = ids.filter((id) => !collected.harnesses.includes(id));
2717
+ if (!unknown.length || collected.partial && unknown.length < ids.length) return;
2718
+ const listed = unknown.map((id) => JSON.stringify(id)).join(", ");
2719
+ const have = [...collected.harnesses].sort();
2720
+ const evals = collected.partial ? "eval in the selected files" : "eval";
2721
+ const the = collected.partial ? "the evals in the selected files" : "the evals";
2722
+ throw new UsageError(`profiles.${profile.name}.limit.harness in ${configFile} lists ${listed}, which no ${evals} runs on. Harness ids ${the} run on: ${have.join(", ") || "none"}.`);
2723
+ }
2564
2724
  /**
2565
2725
  * The sandbox an attempt runs in: the resolved environment's (including an `.extend()`
2566
2726
  * parent's), else the first variant's when this is an omitted-baseline empty control, else
@@ -2594,8 +2754,11 @@ function sandboxSetBy(planned, resolved, configFile) {
2594
2754
  }
2595
2755
  return resolved.sources?.environment ?? configFile;
2596
2756
  }
2597
- /** Per harness: the control, then each variant it compares on; then run numbers. */
2598
- function expand(resolved) {
2757
+ /**
2758
+ * Per harness: the control, then each variant it compares on; then run numbers. `vertex` is the
2759
+ * run's `vertex()` settings, for each harness that resolved to Vertex AI.
2760
+ */
2761
+ function expand(resolved, vertex) {
2599
2762
  const { compare } = resolved;
2600
2763
  const control = applied(compare ? compare.baseline : resolved.environment);
2601
2764
  const policy = resolved.gate ?? "all";
@@ -2603,6 +2766,7 @@ function expand(resolved) {
2603
2766
  const attempts = [];
2604
2767
  for (const harness of resolved.harness) {
2605
2768
  const hid = harnessId(harness);
2769
+ const onVertex = harness.gateway === "vertex" && harness.kind !== "fake" ? vertex : void 0;
2606
2770
  const cells = [(() => {
2607
2771
  const sandbox = attemptSandbox(resolved, control);
2608
2772
  const sid = sandboxId(sandbox);
@@ -2646,11 +2810,12 @@ function expand(resolved) {
2646
2810
  harness,
2647
2811
  sandbox,
2648
2812
  environment: env,
2649
- egress: plannedEgress(resolved.egress, harness, resolved.env, env.layers),
2813
+ egress: plannedEgress(resolved.egress, harness, resolved.env, env.layers, onVertex),
2650
2814
  gate,
2651
2815
  ...cell.role === "variant" && compare?.minLift !== void 0 ? { minLift: compare.minLift } : {},
2652
2816
  n,
2653
- label
2817
+ label,
2818
+ ...onVertex ? { vertex: onVertex } : {}
2654
2819
  });
2655
2820
  }
2656
2821
  }
@@ -2698,9 +2863,18 @@ function unbrokeredOnFake(env, attempts) {
2698
2863
  }
2699
2864
  /** The host variables an attempt's secrets read, brokered or not. */
2700
2865
  const secretVarsIn = (env, attempt) => [...new Set(plannedSecrets(env, attempt.environment.layers).map((s) => s.from))];
2701
- const brokerHint = (s) => /^mcp\.[^.]+\.headers\./.test(s.label) ? "give the MCP server an https:// URL (reeval brokers its header secrets to its own host, over HTTPS only)" : `add the hosts it's sent to, e.g. secret.fromEnv('${s.from}', { hosts: ['api.example.com'] })`;
2702
- const unbrokeredWarning = (s) => s.checksOnly ? `${s.label} reaches the checks (c.exec, setup steps) as its real value; the agent never gets it. To give the checks only a placeholder, ${brokerHint(s)}.` : `${s.label} is in the sandbox as its real value. To give the agent only a placeholder, ${brokerHint(s)}.`;
2866
+ const brokerHint = (s) => /^mcp\.[^.]+\.headers\./.test(s.label) ? "give the MCP server an https:// URL (reeval brokers its header secrets over HTTPS only, to its own host unless they set hosts)" : `add the hosts it's sent to, e.g. secret.fromEnv('${s.from}', { hosts: ['api.example.com'] })`;
2867
+ const unbrokeredWarning = (s) => s.checksOnly ? `${s.label} reaches the checks (c.exec, setup steps) as its real value. It's never in the agent's own environment, but a process a setup step leaves running holds it while the agent works. To give the checks only a placeholder, ${brokerHint(s)}.` : `${s.label} is in the sandbox as its real value. To give the agent only a placeholder, ${brokerHint(s)}.`;
2703
2868
  const unbrokeredError = (s, where, configFile) => `${s.label} in ${where} isn't brokered, and secrets.unbrokered is 'error' in ${configFile}. To broker it, ${brokerHint(s)}.`;
2869
+ /** The attempts' MCP headers written as plain strings that look like credentials, once each. */
2870
+ function plainHeaders(attempts) {
2871
+ const out = /* @__PURE__ */ new Map();
2872
+ for (const a of attempts) for (const h of plainCredentialHeaders(a.environment.layers)) out.set(h.label, h);
2873
+ return [...out.values()];
2874
+ }
2875
+ const templateHint = (h) => `read it from a host variable with secret.template, e.g. secret.template('Bearer \${VAR}')${h.https ? "" : " and give the MCP server an https:// URL"}`;
2876
+ const plainHeaderWarning = (h) => `${h.label} is a plain string, so it's in the sandbox as its real value. To give the agent only a placeholder, ${templateHint(h)}.`;
2877
+ const plainHeaderError = (h, where, configFile) => `${h.label} in ${where} is a plain string, and secrets.unbrokered is 'error' in ${configFile}. To broker it, ${templateHint(h)}.`;
2704
2878
  const levelOf = (where) => where.startsWith("evaluate(") ? "eval" : where.startsWith("group(") ? "group" : where.startsWith("profiles.") ? "profile" : "config";
2705
2879
  /**
2706
2880
  * A warning per privilege an attempt's Docker sandbox adds, naming the level that set the
@@ -2774,7 +2948,8 @@ function evalPlan(r, attempts, origin) {
2774
2948
  warnings: [
2775
2949
  ...environmentWarnings(attempts),
2776
2950
  ...skip ? [] : questionWarnings(resolved),
2777
- ...unbrokered(r.resolved.env, attempts).filter(reachesSandbox).map(unbrokeredWarning)
2951
+ ...unbrokered(r.resolved.env, attempts).filter(reachesSandbox).map(unbrokeredWarning),
2952
+ ...plainHeaders(attempts).map(plainHeaderWarning)
2778
2953
  ],
2779
2954
  sandboxWarnings: skip ? [] : sandboxWarnings(attempts, resolved, origin),
2780
2955
  brokered: [...new Set(attempts.flatMap((a) => brokeredIn(r.resolved.env, a)))],
@@ -2806,23 +2981,47 @@ function matchesTags(tags, filters) {
2806
2981
  }
2807
2982
  //#endregion
2808
2983
  //#region src/gateway/resolve.ts
2809
- /** The config's list when it sets none: each provider's own key, as reeval has always run. */
2984
+ /** The config's list when it sets none: each provider's own key. */
2810
2985
  const DEFAULT_GATEWAYS = [{ kind: "direct" }];
2811
2986
  /** Every gateway reeval can route through, by name. */
2812
2987
  const GATEWAYS = {
2813
2988
  direct: { cantServe: (h) => {
2814
- if (h.kind === "fx" && h.model.includes("/")) return `fx runs a gateway id such as ${JSON.stringify(h.model)} only through Vercel AI Gateway`;
2989
+ const fxEffort = fxEffortRefusal(h, "direct()");
2990
+ if (fxEffort) return fxEffort;
2991
+ if (h.kind === "fx" && h.model.includes("/")) return `fx runs a gateway id such as ${JSON.stringify(h.model)} only through Vercel AI Gateway or OpenRouter`;
2815
2992
  if (h.kind === "opencode") {
2816
2993
  const resolved = resolveOpenCodeProvider(h.model, process.env);
2817
2994
  if ("error" in resolved) return resolved.error;
2818
2995
  }
2819
2996
  } },
2820
- vercel: { cantServe: (h) => {
2821
- if (h.kind !== "fake" && !VERCEL_ROUTES[h.kind]) return VERCEL_REFUSALS[h.kind] ?? `reeval doesn't run ${agentName(h)} on Vercel AI Gateway yet`;
2822
- const effort = harnessEffort(h);
2823
- if (effort !== void 0 && h.kind !== "fake" && VERCEL_UNPROVEN_EFFORT.includes(h.kind)) return `reeval doesn't send ${agentName(h)}'s effort (${JSON.stringify(effort)}) through Vercel AI Gateway yet: nothing checks the model gets it as its own setting, so drop effort or run ${agentName(h)} on another gateway`;
2997
+ vercel: routingEntry("vercel"),
2998
+ openrouter: routingEntry("openrouter"),
2999
+ vertex: { cantServe: (h) => {
3000
+ if (h.kind === "fake") return void 0;
3001
+ if (!VERTEX_ROUTES[h.kind]) return VERTEX_REFUSALS[h.kind] ?? `reeval doesn't run ${agentName(h)} on Vertex AI yet`;
3002
+ return vertexModelRefusalFor(h);
2824
3003
  } }
2825
3004
  };
3005
+ /**
3006
+ * Why fx can't take `effort` on a gateway: fx sends none itself, and reeval adds it as
3007
+ * `reasoning_effort` only to its calls on Vertex AI's Chat Completions endpoint.
3008
+ */
3009
+ function fxEffortRefusal(h, gateway) {
3010
+ const effort = harnessEffort(h);
3011
+ if (h.kind !== "fx" || effort === void 0) return void 0;
3012
+ return `fx sends no effort of its own, and reeval adds fx's effort (${JSON.stringify(effort)}) only to its calls on Vertex AI, not on ${gateway}: drop effort or run fx on vertex()`;
3013
+ }
3014
+ /** A routing gateway's entry: its route table decides whom it serves, and with what effort. */
3015
+ function routingEntry(name) {
3016
+ const gateway = ROUTING_GATEWAYS[name];
3017
+ return { cantServe: (h) => {
3018
+ const fxEffort = fxEffortRefusal(h, gateway.name);
3019
+ if (fxEffort) return fxEffort;
3020
+ if (h.kind !== "fake" && !gateway.routes[h.kind]) return gateway.refusals[h.kind] ?? `reeval doesn't run ${agentName(h)} on ${gateway.name} yet`;
3021
+ const effort = harnessEffort(h);
3022
+ if (effort !== void 0 && h.kind !== "fake" && gateway.unprovenEffort[h.kind]?.(h.model)) return `reeval doesn't send ${agentName(h)}'s effort (${JSON.stringify(effort)}) to ${h.model} through ${gateway.name} yet: nothing checks the model gets it as its own setting, so drop effort or run ${agentName(h)} on another gateway`;
3023
+ } };
3024
+ }
2826
3025
  /** The `reeval/gateway` helper that makes the gateway `name`, as messages name it. */
2827
3026
  const gatewayHelper = (name) => `${name}()`;
2828
3027
  /** The agent's name, for messages. */
@@ -2837,7 +3036,7 @@ function resolveGateway(gateways, harness, where, catalog = GATEWAYS) {
2837
3036
  const why = (name) => catalog[name]?.cantServe(harness);
2838
3037
  const pin = harness.gateway;
2839
3038
  if (pin !== void 0) {
2840
- if (!gateways.some((g) => g.kind === pin)) throw new UsageError(`${on}: it pins gateway '${pin}', which isn't in the gateway list (${listed}). Add ${gatewayHelper(pin)} to gateway, or pin one that's there.`);
3039
+ if (!gateways.some((g) => g.kind === pin)) throw new UsageError(`${on}: it pins gateway '${pin}', ${notListed(pin, listed)}`);
2841
3040
  const refused = why(pin);
2842
3041
  if (refused) throw new UsageError(`${on}: it pins gateway '${pin}', which can't serve it (${refused}). A pin never falls back; pin another gateway, or remove the pin to take the first in the list that can.`);
2843
3042
  return {
@@ -2859,10 +3058,13 @@ function resolveGateway(gateways, harness, where, catalog = GATEWAYS) {
2859
3058
  }
2860
3059
  throw new UsageError(`${on}: no gateway in the list can serve it (${skipped.map((s) => `${s.gateway}: ${s.why}`).join("; ")}). ${addHint(gateways, harness, catalog)}`);
2861
3060
  }
2862
- /** What to add to the list so it serves `harness`: `direct()` when it can, else the first that can. */
3061
+ /**
3062
+ * What to change so the list serves `harness`: a list without `direct` has it turned off, so
3063
+ * turning it back on when it can serve, else adding the first gateway that can.
3064
+ */
2863
3065
  function addHint(gateways, harness, catalog) {
2864
3066
  const serves = Object.keys(catalog).filter((name) => !gateways.some((g) => g.kind === name)).filter((name) => !catalog[name]?.cantServe(harness));
2865
- if (serves.includes("direct")) return `Add direct() to the end of gateway to use ${agentName(harness)}'s own key.`;
3067
+ if (serves.includes("direct")) return `Drop direct({ enabled: false }) from gateway to fall back to ${agentName(harness)}'s own key.`;
2866
3068
  const [first] = serves;
2867
3069
  return first ? `Add ${gatewayHelper(first)} to gateway.` : `No gateway reeval knows can serve it.`;
2868
3070
  }
@@ -2881,30 +3083,40 @@ function routedHarness(harness, gateways) {
2881
3083
  return harness;
2882
3084
  }
2883
3085
  }
2884
- /** `gateway: claude-code:… → direct (vercel can't serve it: …)`, for the plan's hints. */
2885
- const fallbackHint = (harness, resolved) => `gateway: ${harness} → ${resolved.name} (${resolved.skipped.map((s) => `${s.gateway} can't serve it: ${s.why}`).join("; ")})`;
3086
+ /**
3087
+ * Why a pin names a gateway the list doesn't have: a list without `direct` has it turned off.
3088
+ */
3089
+ const notListed = (pin, listed) => pin === "direct" ? `which the gateway list (${listed}) turns off with direct({ enabled: false }). Drop that, or pin one that's there.` : `which isn't in the gateway list (${listed}). Add ${gatewayHelper(pin)} to gateway, or pin one that's there.`;
3090
+ /**
3091
+ * `gateway: cursor:… falls back to direct (vercel can't serve it: …)`, or `→ vercel (…)` past
3092
+ * another gateway, for the plan's hints.
3093
+ */
3094
+ const fallbackHint = (harness, resolved) => `gateway: ${harness} ${resolved.name === "direct" ? "falls back to" : "→"} ${resolved.name} (${resolved.skipped.map((s) => `${s.gateway} can't serve it: ${s.why}`).join("; ")})`;
2886
3095
  //#endregion
2887
3096
  //#region src/gateway/models.ts
2888
3097
  const providerOf = (model) => model.slice(0, model.indexOf("/"));
2889
3098
  /** Why each gateway can't serve a model by name, or undefined when it can. */
2890
3099
  const MODEL_GATEWAYS = {
2891
3100
  direct: (model) => DIRECT_PROVIDERS[providerOf(model)] ? void 0 : `reeval calls only ${Object.keys(DIRECT_PROVIDERS).map((p) => `${p}/*`).join(", ")} models on their own keys`,
2892
- vercel: () => void 0
3101
+ vercel: () => void 0,
3102
+ openrouter: () => void 0,
3103
+ vertex: vertexModelRefusal
2893
3104
  };
2894
3105
  const gatewayScope = (config, profile) => ({ gateways: profile?.gateway ?? config.gateway });
2895
3106
  /**
2896
- * The gateway `model` goes through: its pin, or the first in the list that can serve it. An unset
2897
- * list is `[direct()]`, so a model takes its provider's own key unless the config lists another
2898
- * gateway. A model object or a fake calls through itself, and is undefined. Throws a UsageError
3107
+ * The gateway `model` goes through: its pin, or the first in the list that can serve it. A list
3108
+ * ends with `direct()` unless it's turned off, so a model no other gateway serves takes its
3109
+ * provider's own key. A model object or a fake calls through itself, and is undefined. Throws a UsageError
2899
3110
  * naming each gateway tried.
2900
3111
  */
2901
3112
  function resolveModelGateway(model, pin, scope, what) {
2902
3113
  if (typeof model !== "string" || !model.includes("/")) return void 0;
2903
3114
  const on = `${what} "${model}"`;
2904
3115
  const listed = scope.gateways.map((g) => g.kind);
3116
+ if (typeof pin === "object" && pin.kind === "direct" && pin.enabled === false) throw new UsageError(directOffPin(on));
2905
3117
  const name = typeof pin === "object" ? pin.kind : pin;
2906
3118
  if (name !== void 0) {
2907
- if (!listed.includes(name)) throw new UsageError(`${on} pins gateway '${name}', which isn't in the gateway list (${listed.join(", ")}). Add ${gatewayHelper(name)} to gateway, or pin one that's there.`);
3119
+ if (!listed.includes(name)) throw new UsageError(`${on} pins gateway '${name}', ${notListed(name, listed.join(", "))}`);
2908
3120
  const refused = MODEL_GATEWAYS[name](model);
2909
3121
  if (refused) throw new UsageError(`${on} pins gateway '${name}', which can't serve it (${refused}). A pin never falls back; pin another gateway, or remove the pin to take the first in the list that can.`);
2910
3122
  return name;
@@ -2916,9 +3128,11 @@ function resolveModelGateway(model, pin, scope, what) {
2916
3128
  skipped.push(`${kind}: ${refused}`);
2917
3129
  }
2918
3130
  const other = GATEWAY_NAMES.find((n) => !listed.includes(n) && !MODEL_GATEWAYS[n](model));
2919
- const judge = other === "vercel" && what.includes("judge") ? ` There a judge must be a decision model, such as '${RECOMMENDED_JUDGE_MODEL}'.` : "";
2920
- throw new UsageError(`${on}: no gateway in the list can serve it (${skipped.join("; ")}).${other ? ` Add ${gatewayHelper(other)} to gateway.${judge}` : ""}`);
3131
+ const judge = other && other !== "direct" && what.includes("judge") ? ` There a judge must be a decision model, such as '${other === "openrouter" ? OPENROUTER_JUDGE_MODEL : RECOMMENDED_JUDGE_MODEL}'.` : "";
3132
+ throw new UsageError(`${on}: no gateway in the list can serve it (${skipped.join("; ")}).${other === "direct" ? ` Drop direct({ enabled: false }) from gateway to fall back to its provider's own key.` : other ? ` Add ${gatewayHelper(other)} to gateway.${judge}` : ""}`);
2921
3133
  }
3134
+ /** Whether an unpinned model resolved to `direct` past the list's other entries, for the plan and doctor. */
3135
+ const fellBackToDirect = (gateway, pin, scope) => gateway === "direct" && pin === void 0 && scope.gateways[0]?.kind !== "direct";
2922
3136
  /** As `resolveModelGateway`, but undefined where it would throw, for checks the plan already makes. */
2923
3137
  function modelGatewayOrNone(model, pin, scope) {
2924
3138
  try {
@@ -3133,7 +3347,7 @@ function configFromData(data) {
3133
3347
  * Resolves a config in its data form (a parsed `reeval.yaml`, or `ReevalConfigData` from another
3134
3348
  * language) whose relative host paths resolve against `root`, its directory: `resolveConfig`'s
3135
3349
  * checks name a mistake in their own words first, and the config contract refuses anything they
3136
- * let through, such as a `null` or a misspelt nested key.
3350
+ * let through, such as a `null` or a misspelled nested key.
3137
3351
  */
3138
3352
  function resolveConfigData(input, root, file) {
3139
3353
  if (!isRecord(input)) throw new UsageError(`${file} must be a mapping of config keys, like harness: and judge: (got ${JSON.stringify(input)})`);
@@ -3237,12 +3451,13 @@ function checkProfile(name, profile, file) {
3237
3451
  };
3238
3452
  return p;
3239
3453
  }
3240
- const GATEWAY_EXAMPLE = "gateway: [direct()]";
3454
+ const GATEWAY_EXAMPLE = "gateway: [vercel()]";
3241
3455
  const isGatewayName = (v) => typeof v === "string" && GATEWAY_NAMES.includes(v);
3242
3456
  /**
3243
3457
  * A `gateway` list as descriptors: each entry is a helper's descriptor, or as data its name or a
3244
- * one-key map from its name to its settings. Refuses an empty list, an unknown entry and a
3245
- * gateway listed twice.
3458
+ * one-key map from its name to its settings. `direct()` ends the list unless it's listed in
3459
+ * another place, and `direct({ enabled: false })` leaves it out, so the resolved list is the one
3460
+ * every route walks. Refuses an empty list, an unknown entry and a gateway listed twice.
3246
3461
  */
3247
3462
  function checkGateways(value, where, file) {
3248
3463
  if (!Array.isArray(value) || value.length === 0) throw new UsageError(`${where} in ${file} must be a list of gateways in the order you prefer them, e.g. ${GATEWAY_EXAMPLE} (got ${JSON.stringify(value)})`);
@@ -3273,10 +3488,20 @@ function checkGateways(value, where, file) {
3273
3488
  const kind = isRecord(descriptor) ? descriptor.kind : void 0;
3274
3489
  throw new UsageError(isGatewayName(kind) ? `${at} isn't a valid ${gatewayHelper(kind)}: ${parsed.error.issues[0]?.message ?? "unknown settings"} (got ${JSON.stringify(entry)})` : `${at} must be a gateway, e.g. direct() from 'reeval/gateway' (got ${describeValue(entry)})`);
3275
3490
  }
3276
- if (out.some((g) => g.kind === parsed.data.kind)) throw new UsageError(`${at} lists ${parsed.data.kind} a second time. Each gateway is listed once, where you prefer it.`);
3491
+ const twice = out.find((g) => g.kind === parsed.data.kind);
3492
+ if (twice?.kind === "direct" && parsed.data.kind === "direct") {
3493
+ if (twice.enabled === false !== (parsed.data.enabled === false)) throw new UsageError(`${at} lists direct() and direct({ enabled: false }). direct() places the fallback to each provider's own key and direct({ enabled: false }) turns it off: keep one.`);
3494
+ }
3495
+ if (twice) throw new UsageError(`${at} lists ${parsed.data.kind} a second time. Each gateway is listed once, where you prefer it.`);
3277
3496
  out.push(parsed.data);
3278
3497
  });
3279
- return out;
3498
+ const direct = out.find((g) => g.kind === "direct");
3499
+ if (direct?.enabled === false) {
3500
+ const rest = out.filter((g) => g.kind !== "direct");
3501
+ if (!rest.length) throw new UsageError(`${where} in ${file} turns direct() off and lists no other gateway, so no model call has one. Add a gateway such as vercel(), or drop direct({ enabled: false }).`);
3502
+ return rest;
3503
+ }
3504
+ return direct ? out.map((g) => g.kind === "direct" ? { kind: "direct" } : g) : [...out, { kind: "direct" }];
3280
3505
  }
3281
3506
  /** `--profile`, then `REEVAL_PROFILE`, then `defaultProfile.ci` in CI (or `.local`), then none. */
3282
3507
  function selectProfile(config, flag, env = process.env) {
@@ -3352,7 +3577,7 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3352
3577
  harness,
3353
3578
  sandbox: user.sandbox ?? docker(),
3354
3579
  environment: user.environment,
3355
- runs: user.runs ?? 1,
3580
+ runs: user.runs,
3356
3581
  gate: user.gate,
3357
3582
  env: user.env ?? {},
3358
3583
  ...user.egress !== void 0 ? { egress: user.egress } : {},
@@ -3378,35 +3603,27 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3378
3603
  }
3379
3604
  const ANALYSIS_DEFAULTS = {
3380
3605
  maxInputTokens: 2e4,
3381
- maxOutputTokens: 8e3,
3382
- insights: true,
3383
- maxTokens: 25e4
3606
+ maxOutputTokens: 8e3
3384
3607
  };
3385
3608
  function checkAnalysis(analysis, file) {
3386
3609
  if (analysis === void 0) return { ...ANALYSIS_DEFAULTS };
3387
3610
  if (typeof analysis !== "object" || analysis === null) throw new UsageError(`analysis in ${file} must be an object like { model: 'google/gemini-3.8-flash' } (got ${JSON.stringify(analysis)})`);
3388
3611
  unknownKey(analysis, [
3389
3612
  "model",
3390
- "insights",
3391
- "maxTokens",
3392
3613
  "instructions",
3393
3614
  "maxInputTokens",
3394
3615
  "maxOutputTokens"
3395
3616
  ], "analysis", file);
3396
- const { model, insights = ANALYSIS_DEFAULTS.insights, maxTokens = ANALYSIS_DEFAULTS.maxTokens, instructions, maxInputTokens = ANALYSIS_DEFAULTS.maxInputTokens, maxOutputTokens = ANALYSIS_DEFAULTS.maxOutputTokens } = analysis;
3617
+ const { model, instructions, maxInputTokens = ANALYSIS_DEFAULTS.maxInputTokens, maxOutputTokens = ANALYSIS_DEFAULTS.maxOutputTokens } = analysis;
3397
3618
  if (typeof model === "string") {
3398
3619
  const problem = checkModelString(model, `analysis.model in ${file}`, [`google/${model}`]);
3399
3620
  if (problem) throw new UsageError(problem);
3400
3621
  } else if (model !== void 0 && !(isObjectLike(model) && model.kind === "fake-analysis")) throw new UsageError(`analysis.model in ${file} must be a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${JSON.stringify(model)})`);
3401
- if (typeof insights !== "boolean") throw new UsageError(`analysis.insights in ${file} must be true or false (got ${JSON.stringify(insights)})`);
3402
- if (!(Number.isInteger(maxTokens) && maxTokens >= 1)) throw new UsageError(`analysis.maxTokens in ${file} must be a whole number of tokens of at least 1, e.g. 250000 (got ${JSON.stringify(maxTokens)})`);
3403
3622
  if (instructions !== void 0 && typeof instructions !== "string") throw new UsageError(`analysis.instructions in ${file} must be a string (got ${JSON.stringify(instructions)})`);
3404
3623
  if (!(Number.isInteger(maxInputTokens) && maxInputTokens >= 1e3)) throw new UsageError(`analysis.maxInputTokens in ${file} must be a whole number of at least 1000 (got ${maxInputTokens})`);
3405
3624
  if (!(Number.isInteger(maxOutputTokens) && maxOutputTokens >= 500)) throw new UsageError(`analysis.maxOutputTokens in ${file} must be a whole number of at least 500 (got ${maxOutputTokens})`);
3406
3625
  return {
3407
3626
  ...analysis,
3408
- insights,
3409
- maxTokens,
3410
3627
  maxInputTokens,
3411
3628
  maxOutputTokens
3412
3629
  };
@@ -3482,8 +3699,6 @@ function secretSources(env) {
3482
3699
  }
3483
3700
  /** The model that answers what each eval asks: `analysis.model` only, never the judge's. */
3484
3701
  const analysisModelOf = (config) => config.analysis.model;
3485
- /** Whether `analysis.model` is set, so a model can answer what each eval asks. */
3486
- const hasAnalysisModel = (config) => analysisModelOf(config) !== void 0;
3487
3702
  /** The run's judge model: the profile's when it sets one, else the config's. */
3488
3703
  function runJudgeModel(config, profile) {
3489
3704
  return profile ? profileJudge(profile).model ?? config.judge.model : config.judge.model;
@@ -3494,30 +3709,43 @@ function runModelGateway(config, profile, model, judge = false) {
3494
3709
  }
3495
3710
  /**
3496
3711
  * Names the environment variables the run's providers need but don't have: the harnesses it
3497
- * plans (default: the config's), the run's judge, the models that answer questions,
3498
- * and required config and profile secrets. Insights' model isn't among them: without its key a run
3499
- * prints no insights, and never stops for it.
3712
+ * plans (default: the config's), the run's judge and the judges its attempts resolve, the models
3713
+ * that answer questions, and required config and profile secrets.
3500
3714
  */
3501
3715
  function missingCredentials(config, env = process.env, options = {}) {
3502
3716
  const missing = [];
3503
3717
  const keys = new Set((options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []));
3504
3718
  for (const key of keys) if (!env[key]) missing.push(key);
3505
3719
  const judge = runJudgeModel(config, options.profile);
3506
- const models = [...judge !== void 0 ? [{
3507
- model: judge,
3508
- what: "the judge model",
3509
- judge: true
3510
- }] : [], ...(options.questionModels ?? []).map((model) => ({
3511
- model,
3512
- what: "questions",
3513
- judge: false
3514
- }))];
3720
+ const models = [
3721
+ ...typeof judge === "string" ? [{
3722
+ model: judge,
3723
+ gateway: runModelGateway(config, options.profile, judge, true),
3724
+ what: "the judge model"
3725
+ }] : [],
3726
+ ...(options.judges ?? []).map((j) => ({
3727
+ ...j,
3728
+ what: "the judge model"
3729
+ })),
3730
+ ...(options.questionModels ?? []).map((model) => ({
3731
+ model,
3732
+ gateway: runModelGateway(config, options.profile, model),
3733
+ what: "questions"
3734
+ }))
3735
+ ];
3515
3736
  const named = /* @__PURE__ */ new Set();
3516
- for (const { model, what, judge: isJudge } of models) {
3517
- if (typeof model !== "string" || named.has(model)) continue;
3518
- named.add(model);
3519
- const gateway = runModelGateway(config, options.profile, model, isJudge);
3520
- if (!judgeCredentials(model, gateway).some((k) => env[k])) missing.push(`${modelKeyOptions(model, gateway)} for ${what}, "${model}"`);
3737
+ for (const { model, gateway, what } of models) {
3738
+ if (gateway === "vertex") {
3739
+ const gateways = gatewayScope(config, options.profile).gateways;
3740
+ const line = `${VERTEX_PROJECT_ENV} (for vertex()) for ${what}, "${model}"`;
3741
+ if (!gateways.some((g) => g.kind === "vertex" && g.project) && !env["GOOGLE_CLOUD_PROJECT"] && !missing.includes(line)) missing.push(line);
3742
+ continue;
3743
+ }
3744
+ const keys = judgeCredentials(model, gateway);
3745
+ const once = `${keys.join(" ")}\0${model}`;
3746
+ if (named.has(once)) continue;
3747
+ named.add(once);
3748
+ if (!keys.some((k) => env[k])) missing.push(`${modelKeyOptions(model, gateway)} for ${what}, "${model}"`);
3521
3749
  }
3522
3750
  const vars = {
3523
3751
  ...config.env,
@@ -3527,16 +3755,16 @@ function missingCredentials(config, env = process.env, options = {}) {
3527
3755
  return missing;
3528
3756
  }
3529
3757
  /**
3530
- * Keys the run sends in request headers (the agents' credentials, the run's judge's and the
3531
- * questions' models') whose value has
3532
- * whitespace or a control character in it, with where the first one is: a key pasted from a
3533
- * wrapped terminal line picks up a line break, and the agent then hangs on a request it can't send
3534
- * rather than failing to authenticate.
3758
+ * Keys the run sends in request headers (the agents' credentials, the run's and attempts' judges'
3759
+ * and the questions' models') whose value has whitespace or a control character in it, with where
3760
+ * the first one is: a key pasted from a wrapped terminal line picks up a line break, and the agent
3761
+ * then hangs on a request it can't send rather than failing to authenticate.
3535
3762
  */
3536
3763
  function malformedCredentials(config, env = process.env, options = {}) {
3537
3764
  const names = (options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []);
3538
3765
  const judge = runJudgeModel(config, options.profile);
3539
3766
  if (judge !== void 0) names.push(...judgeCredentials(judge, runModelGateway(config, options.profile, judge, true)));
3767
+ for (const { model, gateway } of options.judges ?? []) names.push(...judgeCredentials(model, gateway));
3540
3768
  for (const model of options.questionModels ?? []) names.push(...judgeCredentials(model, runModelGateway(config, options.profile, model)));
3541
3769
  return [...new Set(names)].flatMap((name) => {
3542
3770
  const at = badCharacterAt(env[name] ?? "");
@@ -3544,6 +3772,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
3544
3772
  });
3545
3773
  }
3546
3774
  //#endregion
3547
- export { DEFAULT_QUESTION_STATE as $, matchesTags as A, mergeJudge as B, brokeredIn as C, judgeStateSchema as Ct, expand as D, filePartsOf as Dt, evalPlan as E, decideState as Et, sandboxProvider as F, ciRunner as G, DEFAULT_EGRESS as H, sandboxSetBy as I, gatewayPolicy as J, directOnCloud as K, secretVarsIn as L, profileJudge as M, refuseIllegalCompare as N, harnessId as O, resolveJudgeArtifacts as Ot, sandboxId as P, createFakeSandbox as Q, unbrokered as R, routedHarness as S, judgeProblem as St, checkOptions as T, badJudgeArtifacts as Tt, DEFAULT_EGRESS_ALLOW as U, resolveStateOptions as V, allowedHosts as W, unenforceable as X, isDirect as Y, resolveOutput as Z, selectProfile as _, writePluginFiles as _t, configPathFor as a, servedEffortOf as at, fallbackHint as b, judgeFields as bt, findConfig as c, agentMcp as ct, loadConfig as d, claudeMcpServers as dt, STOPPED_FIRST as et, malformedCredentials as f, credentialOf as ft, runJudgeModel as g, refusedPlugins as gt, resolveConcurrency as h, refusedMcp as ht, configLabel as i, resolvedEffortOf as it, plannedSandboxes as j, levelOf as k, hasAnalysisModel as l, authHint as lt, requireConfig as m, layoutOf as mt, analysisModelOf as n, resolveQuestionState as nt, defaultConcurrency as o, agentAuth as ot, missingCredentials as p, expectedBlockedFor as pt, egressLabel as q, configFilesPhrase as r, harnessEffort as rt, defineConfig as s, agentFor as st, CONFIG_FILES as t, questionShows as tt, hostCapacity as u, brokeringFor as ut, gatewayScope as v, captureWindowSchema as vt, cascade as w, artifactRefsOf as wt, resolveGateway as x, judgeModelNameSchema as xt, resolveModelGateway as y, fakeJudgeDataSchema as yt, unbrokeredError as z };
3775
+ export { isDirect as $, harnessId as A, badJudgeArtifacts as At, sandboxSetBy as B, routedHarness as C, captureWindowSchema as Ct, checkOptions as D, judgeProblem as Dt, checkLimitHarness as E, judgeModelNameSchema as Et, plannedSandboxes as F, resolveStateOptions as G, unbrokered as H, profileJudge as I, allowedHosts as J, DEFAULT_EGRESS as K, refuseIllegalCompare as L, matchesTags as M, filePartsOf as Mt, plainHeaderError as N, resolveJudgeArtifacts as Nt, evalPlan as O, judgeStateSchema as Ot, plainHeaders as P, gatewayPolicy as Q, sandboxId as R, resolveGateway as S, writePluginFiles as St, cascade as T, judgeFields as Tt, unbrokeredError as U, secretVarsIn as V, mergeJudge as W, directOnCloud as X, ciRunner as Y, egressLabel as Z, fellBackToDirect as _, expectedBlockedFor as _t, configPathFor as a, questionShows as at, resolveModelGateway as b, refusedMcp as bt, findConfig as c, resolvedEffortOf as ct, malformedCredentials as d, agentFor as dt, unenforceable as et, missingCredentials as f, agentMcp as ft, selectProfile as g, credentialOf as gt, runJudgeModel as h, claudeMcpServers as ht, configLabel as i, STOPPED_FIRST as it, levelOf as j, decideState as jt, expand as k, artifactRefsOf as kt, hostCapacity as l, servedEffortOf as lt, resolveConcurrency as m, brokeringFor as mt, analysisModelOf as n, createFakeSandbox as nt, defaultConcurrency as o, resolveQuestionState as ot, requireConfig as p, authHint as pt, DEFAULT_EGRESS_ALLOW as q, configFilesPhrase as r, DEFAULT_QUESTION_STATE as rt, defineConfig as s, harnessEffort as st, CONFIG_FILES as t, resolveOutput as tt, loadConfig as u, agentAuth as ut, gatewayScope as v, isAgentKind as vt, brokeredIn as w, fakeJudgeDataSchema as wt, fallbackHint as x, refusedPlugins as xt, modelGatewayOrNone as y, layoutOf as yt, sandboxProvider as z };
3548
3776
 
3549
- //# sourceMappingURL=config-BrdHA9bJ.js.map
3777
+ //# sourceMappingURL=config-k6m7XbAz.js.map