reeval 0.18.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (380) hide show
  1. package/README.md +1 -1
  2. package/THIRD_PARTY_NOTICES.md +5 -5
  3. package/dist/{adapter-CptK032x.js → adapter-C60z3wLq.js} +5 -4
  4. package/dist/adapter-C60z3wLq.js.map +1 -0
  5. package/dist/{agent-B63OzUX9.js → agent-BmsMXZXx.js} +30 -10
  6. package/dist/{agent-B63OzUX9.js.map → agent-BmsMXZXx.js.map} +1 -1
  7. package/dist/agents-DXuFxtGC.js +1151 -0
  8. package/dist/agents-DXuFxtGC.js.map +1 -0
  9. package/dist/{app-files-sgqwfjtn.js → app-files-DT5mJyV4.js} +3 -3
  10. package/dist/app-files-DT5mJyV4.js.map +1 -0
  11. package/dist/assemble-CE4pHKIa.js +1 -0
  12. package/dist/{assemble-Me7Whp-y.js → assemble-DM2PNU4R.js} +152 -129
  13. package/dist/assemble-DM2PNU4R.js.map +1 -0
  14. package/dist/{attempt-Dp7Wtm8M.js → attempt-CAdlwlNu.js} +2 -2
  15. package/dist/{attempt-Dp7Wtm8M.js.map → attempt-CAdlwlNu.js.map} +1 -1
  16. package/dist/{attempt-host-Cik0SoIn.js → attempt-host-7VviAJrl.js} +6 -6
  17. package/dist/{attempt-host-Cik0SoIn.js.map → attempt-host-7VviAJrl.js.map} +1 -1
  18. package/dist/boat-BjLqFJIZ.js +108 -0
  19. package/dist/boat-BjLqFJIZ.js.map +1 -0
  20. package/dist/boat-DSNgZt-t.js +322 -0
  21. package/dist/boat-DSNgZt-t.js.map +1 -0
  22. package/dist/bridge/index.mjs +401 -79
  23. package/dist/capture-BzotD2hs.js +1 -0
  24. package/dist/{capture-C3RmjPph.js → capture-Ccf0Hmyx.js} +27 -25
  25. package/dist/capture-Ccf0Hmyx.js.map +1 -0
  26. package/dist/cases-Dy0mifal.js +1 -0
  27. package/dist/cases-z9E6DCd5.js +639 -0
  28. package/dist/cases-z9E6DCd5.js.map +1 -0
  29. package/dist/{cli-BBOlXKPG.js → cli-DTuLSlmW.js} +442 -707
  30. package/dist/cli-DTuLSlmW.js.map +1 -0
  31. package/dist/{cli-spec-DZ0bnP0_.js → cli-spec-Dsr79Qy1.js} +38 -11
  32. package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
  33. package/dist/cli.js +2 -2
  34. package/dist/{client-tZ2WBDxf.js → client-KM4C5r5_.js} +93 -3
  35. package/dist/client-KM4C5r5_.js.map +1 -0
  36. package/dist/{command-DhiwsZl1.js → command-C2C7P5qo.js} +9 -9
  37. package/dist/{command-DhiwsZl1.js.map → command-C2C7P5qo.js.map} +1 -1
  38. package/dist/{command-CFQ9c1hk.js → command-D8oMB2fa.js} +16 -16
  39. package/dist/command-D8oMB2fa.js.map +1 -0
  40. package/dist/{command-fO0BFsTq.js → command-DSL7I-SI.js} +37 -37
  41. package/dist/command-DSL7I-SI.js.map +1 -0
  42. package/dist/{command-qGS802uX.js → command-Ff0T-1Hi.js} +21 -80
  43. package/dist/command-Ff0T-1Hi.js.map +1 -0
  44. package/dist/{commands-xYTMNTPv.js → commands-Bk9NG1s_.js} +1 -1
  45. package/dist/{commands-CC60TOT5.js → commands-xAOi_VEk.js} +186 -85
  46. package/dist/commands-xAOi_VEk.js.map +1 -0
  47. package/dist/{config-C2UtS9vy.js → config-SvK8pjEK.js} +261 -1708
  48. package/dist/config-SvK8pjEK.js.map +1 -0
  49. package/dist/{context-BajfiH0S.js → context-BdgCao9Q.js} +3 -2
  50. package/dist/context-BdgCao9Q.js.map +1 -0
  51. package/dist/{cost-BuVDamui.js → cost-BdRgySQl.js} +2 -2
  52. package/dist/{cost-BuVDamui.js.map → cost-BdRgySQl.js.map} +1 -1
  53. package/dist/cost-Cr26Alhq.js +1 -0
  54. package/dist/{cursor-catalog-BTK_apL-.js → cursor-catalog-DEzan4IT.js} +3 -3
  55. package/dist/{cursor-catalog-BTK_apL-.js.map → cursor-catalog-DEzan4IT.js.map} +1 -1
  56. package/dist/{db-C5SZ0510.js → db-BzOJ4Js4.js} +130 -84
  57. package/dist/db-BzOJ4Js4.js.map +1 -0
  58. package/dist/db-CQRZ3URX.js +2 -0
  59. package/dist/debug-CF4p4Lp_.js +1 -0
  60. package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
  61. package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
  62. package/dist/{default-effort-FEidNAIi.js → default-effort-BBDPbJ4D.js} +1 -1
  63. package/dist/{default-effort-BVmx7r7L.js → default-effort-BDKIxJyC.js} +4 -4
  64. package/dist/{default-effort-BVmx7r7L.js.map → default-effort-BDKIxJyC.js.map} +1 -1
  65. package/dist/{dist-ByYk_wAs.js → dist-Db7ifchA.js} +37 -8
  66. package/dist/dist-Db7ifchA.js.map +1 -0
  67. package/dist/{docker-BlSopHpZ.js → docker--mukta3L.js} +1 -1
  68. package/dist/{docker-B2F8-PGC.js → docker-DFrjoNs-.js} +3 -3
  69. package/dist/{docker-B2F8-PGC.js.map → docker-DFrjoNs-.js.map} +1 -1
  70. package/dist/{docker-Dto8ROCq.js → docker-DG_uFgn2.js} +109 -77
  71. package/dist/docker-DG_uFgn2.js.map +1 -0
  72. package/dist/{doctor-BcvfKpBX.js → doctor-DWK4ttex.js} +84 -27
  73. package/dist/doctor-DWK4ttex.js.map +1 -0
  74. package/dist/{environment-CC1RV21n.js → environment-cPMCOk-h.js} +79 -39
  75. package/dist/environment-cPMCOk-h.js.map +1 -0
  76. package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
  77. package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
  78. package/dist/{eval-source-CzxdgFBW.js → eval-source-C9b1h8L6.js} +10 -10
  79. package/dist/{eval-source-CzxdgFBW.js.map → eval-source-C9b1h8L6.js.map} +1 -1
  80. package/dist/{evaluate-NTNJ8T9H.js → evaluate-B12Tosvq.js} +377 -142
  81. package/dist/evaluate-B12Tosvq.js.map +1 -0
  82. package/dist/{evaluate-DToSB2bt.d.ts → evaluate-COdzf7Hl.d.ts} +17 -26
  83. package/dist/{files-aAzoGNCB.js → files-BOBdC0uI.js} +5 -5
  84. package/dist/{files-aAzoGNCB.js.map → files-BOBdC0uI.js.map} +1 -1
  85. package/dist/files-C1Rk9FYA.js +1 -0
  86. package/dist/files-DMQ9-Vv5.js +1 -0
  87. package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
  88. package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
  89. package/dist/files-Di0xDbCc.js +1 -0
  90. package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
  91. package/dist/files-DlQshsks.js.map +1 -0
  92. package/dist/{files-DsXULUi5.js → files-F5EG0WPT.js} +5 -5
  93. package/dist/{files-DsXULUi5.js.map → files-F5EG0WPT.js.map} +1 -1
  94. package/dist/files-G5jf8wBp.js +1 -0
  95. package/dist/{files-CZXrkPja.js → files-s28a6czq.js} +54 -13
  96. package/dist/{files-CZXrkPja.js.map → files-s28a6czq.js.map} +1 -1
  97. package/dist/files-tiNWe8Ih.js +64 -0
  98. package/dist/files-tiNWe8Ih.js.map +1 -0
  99. package/dist/gate-CgYdaprG.js +49 -0
  100. package/dist/gate-CgYdaprG.js.map +1 -0
  101. package/dist/gateway.d.ts +1 -1
  102. package/dist/{git-C1MB5bjr.js → git-BpoqEgDV.js} +2 -2
  103. package/dist/{git-C1MB5bjr.js.map → git-BpoqEgDV.js.map} +1 -1
  104. package/dist/{harness-DGsRZDSo.js → harness-C2berMiM.js} +3 -3
  105. package/dist/{harness-DGsRZDSo.js.map → harness-C2berMiM.js.map} +1 -1
  106. package/dist/{harness-DYSC4zMh.js → harness-DXv2gnaT.js} +4 -2
  107. package/dist/harness-DXv2gnaT.js.map +1 -0
  108. package/dist/harness.d.ts +1 -1
  109. package/dist/harness.js +1 -1
  110. package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
  111. package/dist/ids-C_advdJA.js.map +1 -0
  112. package/dist/{images-CNayXCp2.js → images-D2jFze2v.js} +98 -10
  113. package/dist/images-D2jFze2v.js.map +1 -0
  114. package/dist/index.d.ts +23 -3
  115. package/dist/index.js +5 -5
  116. package/dist/{init-D5HtmyMI.js → init-Bi1Q3lRi.js} +7 -6
  117. package/dist/init-Bi1Q3lRi.js.map +1 -0
  118. package/dist/init-CUw65o36.js +1 -0
  119. package/dist/{ink-list-Ca-5dunO.js → ink-list-BpfidQ1h.js} +121 -31
  120. package/dist/ink-list-BpfidQ1h.js.map +1 -0
  121. package/dist/{invocations-DrGXF9dT.js → invocations-CxGNOlh0.js} +3 -3
  122. package/dist/{invocations-DrGXF9dT.js.map → invocations-CxGNOlh0.js.map} +1 -1
  123. package/dist/judge-COMXT1Hx.js +230 -0
  124. package/dist/judge-COMXT1Hx.js.map +1 -0
  125. package/dist/judge.d.ts +1 -1
  126. package/dist/judge.js +1 -1
  127. package/dist/{judgments-m-mb0zv8.js → judgments-CzgpsTqv.js} +2 -2
  128. package/dist/{judgments-m-mb0zv8.js.map → judgments-CzgpsTqv.js.map} +1 -1
  129. package/dist/kinds-Bh2I35id.js +698 -0
  130. package/dist/kinds-Bh2I35id.js.map +1 -0
  131. package/dist/{summary-Uzc9osfV.js → layout-Cm85tQYk.js} +39 -150
  132. package/dist/layout-Cm85tQYk.js.map +1 -0
  133. package/dist/list-DuThsE28.js +1348 -0
  134. package/dist/list-DuThsE28.js.map +1 -0
  135. package/dist/{live-B71o2TkX.js → live-CgYbO4Te.js} +6 -4
  136. package/dist/{live-B71o2TkX.js.map → live-CgYbO4Te.js.map} +1 -1
  137. package/dist/{live-uRURFDMZ.js → live-DL0se4Zd.js} +7 -7
  138. package/dist/{live-uRURFDMZ.js.map → live-DL0se4Zd.js.map} +1 -1
  139. package/dist/live-DZLmj0iG.js +1 -0
  140. package/dist/{matchers-Cs8aDqdU.js → matchers-CXLn-1jm.js} +3 -2
  141. package/dist/{matchers-Cs8aDqdU.js.map → matchers-CXLn-1jm.js.map} +1 -1
  142. package/dist/{model-CXVVB4RK.js → model-CkNVAxob.js} +1 -1
  143. package/dist/model-Cv4GuTsZ.js +1 -0
  144. package/dist/{model-1L19Zukh.js → model-CwwkrCb7.js} +5 -5
  145. package/dist/{model-1L19Zukh.js.map → model-CwwkrCb7.js.map} +1 -1
  146. package/dist/{model-Ce9l9_rF.js → model-V3mMXp6x.js} +203 -313
  147. package/dist/model-V3mMXp6x.js.map +1 -0
  148. package/dist/{model-BxBWFOZn.js → model-cgt5uvu8.js} +5 -5
  149. package/dist/{model-BxBWFOZn.js.map → model-cgt5uvu8.js.map} +1 -1
  150. package/dist/{node-DhIRSrrK.js → node-D7pAU--0.js} +14 -10
  151. package/dist/node-D7pAU--0.js.map +1 -0
  152. package/dist/{opencode-providers-Co7plCea.js → opencode-providers-BGiXt8_B.js} +104 -35
  153. package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
  154. package/dist/{openrouter-models-DNvLRYM7.js → openrouter-models-Cw1z8-sq.js} +2 -2
  155. package/dist/{openrouter-models-DNvLRYM7.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
  156. package/dist/options-DPUmk7v_.js +1 -0
  157. package/dist/{gate-Cp_pQoEe.js → pairs-BGb134Vu.js} +2 -48
  158. package/dist/pairs-BGb134Vu.js.map +1 -0
  159. package/dist/{patch-ug5HpMOj.js → patch-DAi1nCuH.js} +2 -2
  160. package/dist/{patch-ug5HpMOj.js.map → patch-DAi1nCuH.js.map} +1 -1
  161. package/dist/plain-BYUf_fUV.js +267 -0
  162. package/dist/plain-BYUf_fUV.js.map +1 -0
  163. package/dist/{plan-DqnzfDH1.js → plan-CsPilzpk.js} +110 -39
  164. package/dist/plan-CsPilzpk.js.map +1 -0
  165. package/dist/{plugins-B2KQ0y3P.js → plugins-BfnhazOv.js} +10 -267
  166. package/dist/plugins-BfnhazOv.js.map +1 -0
  167. package/dist/{preflight-DyzmvK7i.js → preflight-BixaE4BA.js} +16 -9
  168. package/dist/preflight-BixaE4BA.js.map +1 -0
  169. package/dist/preflight-Bn3ianye.js +1 -0
  170. package/dist/{preflight-DVXEvzFU.js → preflight-D1C-iH0U.js} +1 -1
  171. package/dist/preflight-GkpgrBxP.js +68 -0
  172. package/dist/preflight-GkpgrBxP.js.map +1 -0
  173. package/dist/{prepare-BcSRa7-3.js → prepare-C--rM_wO.js} +2 -2
  174. package/dist/{prepare-21WMQZyt.js → prepare-DQEDXoDp.js} +9 -9
  175. package/dist/{prepare-21WMQZyt.js.map → prepare-DQEDXoDp.js.map} +1 -1
  176. package/dist/{prices-DYJA0tJN.js → prices-5m3SzMZ3.js} +2 -2
  177. package/dist/{prices-DYJA0tJN.js.map → prices-5m3SzMZ3.js.map} +1 -1
  178. package/dist/{provenance-iRxjp4Pu.js → provenance-BXyYwUC8.js} +4 -3
  179. package/dist/provenance-BXyYwUC8.js.map +1 -0
  180. package/dist/provider-BnUoz4KB.js +1 -0
  181. package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
  182. package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
  183. package/dist/{providers-vXMXuwn-.js → providers-1p4HKYIp.js} +152 -9
  184. package/dist/providers-1p4HKYIp.js.map +1 -0
  185. package/dist/providers-B-ijCktf.js +1 -0
  186. package/dist/query-BYhyf0LR.js +1 -0
  187. package/dist/{query-BhaKQ07P.js → query-DhhU9ADI.js} +2 -2
  188. package/dist/{query-BhaKQ07P.js.map → query-DhhU9ADI.js.map} +1 -1
  189. package/dist/{questions-BzdLdV3C.js → questions-CMAq4pr8.js} +9 -9
  190. package/dist/{questions-BzdLdV3C.js.map → questions-CMAq4pr8.js.map} +1 -1
  191. package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
  192. package/dist/record-6IUzwP9K.js.map +1 -0
  193. package/dist/{remarks-CauU408t.js → remarks-BNABzemt.js} +115 -35
  194. package/dist/remarks-BNABzemt.js.map +1 -0
  195. package/dist/{remarks-NMgPd30U.js → remarks-DPCONFxZ.js} +71 -50
  196. package/dist/remarks-DPCONFxZ.js.map +1 -0
  197. package/dist/{remote-Dwy3fyo3.js → remote-U7fNt0UP.js} +6 -6
  198. package/dist/{remote-Dwy3fyo3.js.map → remote-U7fNt0UP.js.map} +1 -1
  199. package/dist/resolve-C14Pmtej.js +1 -0
  200. package/dist/{resolve-B1zMnPwK.js → resolve-Cx3a9UQ2.js} +10 -9
  201. package/dist/resolve-Cx3a9UQ2.js.map +1 -0
  202. package/dist/{results-CfC0id_u.js → results-CQKXmHym.js} +6 -6
  203. package/dist/{results-CfC0id_u.js.map → results-CQKXmHym.js.map} +1 -1
  204. package/dist/{resume-Dkhrpjow.js → resume-POpZa1ge.js} +169 -27
  205. package/dist/resume-POpZa1ge.js.map +1 -0
  206. package/dist/retry-CmG809Mp.js +105 -0
  207. package/dist/retry-CmG809Mp.js.map +1 -0
  208. package/dist/{runtime-DcvQcr36.js → runtime-Xm9oeC7o.js} +26 -7
  209. package/dist/runtime-Xm9oeC7o.js.map +1 -0
  210. package/dist/sandbox/boat.d.ts +26 -0
  211. package/dist/sandbox/boat.js +37 -0
  212. package/dist/sandbox/boat.js.map +1 -0
  213. package/dist/{sandbox-8zCrtuCf.js → sandbox-DrlL4yqb.js} +31 -3
  214. package/dist/sandbox-DrlL4yqb.js.map +1 -0
  215. package/dist/sandbox.d.ts +20 -2
  216. package/dist/sandbox.js +1 -1
  217. package/dist/{server-Dplq8g0r.js → server-C00zd6zl.js} +9 -9
  218. package/dist/{server-Dplq8g0r.js.map → server-C00zd6zl.js.map} +1 -1
  219. package/dist/server-C1UJcpUh.js +1 -0
  220. package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
  221. package/dist/shared-Cfi45Bvy.js.map +1 -0
  222. package/dist/slug-IHh3f770.js +136 -0
  223. package/dist/slug-IHh3f770.js.map +1 -0
  224. package/dist/{source-NWHqn5-o.js → source-DDgr48Gn.js} +2 -2
  225. package/dist/{source-NWHqn5-o.js.map → source-DDgr48Gn.js.map} +1 -1
  226. package/dist/source-DJHmQv_j.js +1 -0
  227. package/dist/src-1dBuXzs4.js +3 -0
  228. package/dist/{src-CUpoVVb8.js → src-BKpioayC.js} +2 -2
  229. package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
  230. package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
  231. package/dist/src-BY1cCaoJ.js +2 -0
  232. package/dist/src-BbZYaQ8b.js +2 -0
  233. package/dist/src-CDFH2vVN.js +2 -0
  234. package/dist/src-CMW5R9Q0.js +1 -0
  235. package/dist/{src-BQXdf-Gw.js → src-CV_7DkPa.js} +47 -45
  236. package/dist/{src-BQXdf-Gw.js.map → src-CV_7DkPa.js.map} +1 -1
  237. package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
  238. package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
  239. package/dist/{src-B9oE_mNS.js → src-CrV2WA2i.js} +2 -2
  240. package/dist/src-CsFaawcZ.js +2 -0
  241. package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
  242. package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
  243. package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
  244. package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
  245. package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
  246. package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
  247. package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
  248. package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
  249. package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
  250. package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
  251. package/dist/src-wKwznPgt.js +2 -0
  252. package/dist/{src-BQ6BiXoc.js → src-y0nOeRRE.js} +610 -72
  253. package/dist/src-y0nOeRRE.js.map +1 -0
  254. package/dist/{state-DDCGUhva.js → state-328JpnHA.js} +17 -17
  255. package/dist/{state-DDCGUhva.js.map → state-328JpnHA.js.map} +1 -1
  256. package/dist/state-Ca55Gk_O.js +1 -0
  257. package/dist/suite-CKBa6MKt.js +2 -0
  258. package/dist/{suite-M8_7xio7.js → suite-COglUhPd.js} +70 -92
  259. package/dist/suite-COglUhPd.js.map +1 -0
  260. package/dist/summary-DDy9DfyJ.js +148 -0
  261. package/dist/summary-DDy9DfyJ.js.map +1 -0
  262. package/dist/tart-cli-C-GsbVsQ.js +1 -0
  263. package/dist/tart-cli-CU0nFS-L.js +55 -0
  264. package/dist/tart-cli-CU0nFS-L.js.map +1 -0
  265. package/dist/tart-dWoJvWHc.js +586 -0
  266. package/dist/tart-dWoJvWHc.js.map +1 -0
  267. package/dist/tart-images-DZ_Mbh-g.js +233 -0
  268. package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
  269. package/dist/tart-ref-7Qv3cm3B.js +1 -0
  270. package/dist/tart-ref-DYZijjrU.js +55 -0
  271. package/dist/tart-ref-DYZijjrU.js.map +1 -0
  272. package/dist/testing.d.ts +2 -2
  273. package/dist/testing.js +4 -4
  274. package/dist/trace-Dx2NBE1O.js +311 -0
  275. package/dist/trace-Dx2NBE1O.js.map +1 -0
  276. package/dist/traits-DKnYkqeK.js +66 -0
  277. package/dist/traits-DKnYkqeK.js.map +1 -0
  278. package/dist/{transcript-Cg4SeqRX.js → transcript-CCf07UmU.js} +2 -2
  279. package/dist/{transcript-Cg4SeqRX.js.map → transcript-CCf07UmU.js.map} +1 -1
  280. package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
  281. package/dist/types-C2mLpph3.js.map +1 -0
  282. package/dist/{types-DGvexkcp.d.ts → types-DBwFsGem.d.ts} +352 -76
  283. package/dist/{judge-DYOF69Oe.js → vercel-CGlSBv63.js} +3 -229
  284. package/dist/vercel-CGlSBv63.js.map +1 -0
  285. package/dist/{vertex-BLQMayc1.js → vertex-Eb2wqahk.js} +2 -2
  286. package/dist/{vertex-BLQMayc1.js.map → vertex-Eb2wqahk.js.map} +1 -1
  287. package/dist/vertex-O88TCwgY.js +1 -0
  288. package/dist/view/index.d.ts +101 -58
  289. package/dist/view/index.js +9 -8
  290. package/dist/views-B0Yt_Q1S.js +2 -0
  291. package/dist/{views-BISrsIjw.js → views-BROacjBv.js} +7 -6
  292. package/dist/views-BROacjBv.js.map +1 -0
  293. package/dist/{worker-entry-DJD_nHkS.js → worker-entry-Smb4bNDc.js} +11 -8
  294. package/dist/{worker-entry-DJD_nHkS.js.map → worker-entry-Smb4bNDc.js.map} +1 -1
  295. package/dist/workspace-C0dDP75l.js +1 -0
  296. package/dist/{workspace-BO1VhUR9.js → workspace-DNDTEk8v.js} +6 -6
  297. package/dist/{workspace-BO1VhUR9.js.map → workspace-DNDTEk8v.js.map} +1 -1
  298. package/package.json +24 -11
  299. package/skills/eval-rubric/SKILL.md +15 -7
  300. package/dist/adapter-CptK032x.js.map +0 -1
  301. package/dist/app-files-sgqwfjtn.js.map +0 -1
  302. package/dist/assemble-CxBHkavM.js +0 -1
  303. package/dist/assemble-Me7Whp-y.js.map +0 -1
  304. package/dist/capture-C3RmjPph.js.map +0 -1
  305. package/dist/capture-YPwxp_iK.js +0 -1
  306. package/dist/cli-BBOlXKPG.js.map +0 -1
  307. package/dist/cli-spec-DZ0bnP0_.js.map +0 -1
  308. package/dist/client-tZ2WBDxf.js.map +0 -1
  309. package/dist/command-CFQ9c1hk.js.map +0 -1
  310. package/dist/command-fO0BFsTq.js.map +0 -1
  311. package/dist/command-qGS802uX.js.map +0 -1
  312. package/dist/commands-CC60TOT5.js.map +0 -1
  313. package/dist/config-C2UtS9vy.js.map +0 -1
  314. package/dist/context-BajfiH0S.js.map +0 -1
  315. package/dist/contrasts-BjOg7wyr.js +0 -101
  316. package/dist/contrasts-BjOg7wyr.js.map +0 -1
  317. package/dist/cost-BJU0MAji.js +0 -1
  318. package/dist/db-C5SZ0510.js.map +0 -1
  319. package/dist/db-puzLzX4k.js +0 -2
  320. package/dist/debug-BnXV7JbX.js +0 -1
  321. package/dist/dist-ByYk_wAs.js.map +0 -1
  322. package/dist/docker-Dto8ROCq.js.map +0 -1
  323. package/dist/doctor-BcvfKpBX.js.map +0 -1
  324. package/dist/environment-CC1RV21n.js.map +0 -1
  325. package/dist/evaluate-NTNJ8T9H.js.map +0 -1
  326. package/dist/files-COW5JBJM.js +0 -1
  327. package/dist/files-DFtuRQJx.js +0 -1
  328. package/dist/files-oOYNR7ZH.js +0 -1
  329. package/dist/files-p-uTdOjA.js.map +0 -1
  330. package/dist/gate-Cp_pQoEe.js.map +0 -1
  331. package/dist/harness-DYSC4zMh.js.map +0 -1
  332. package/dist/ids-DuK6xl4w.js.map +0 -1
  333. package/dist/images-CNayXCp2.js.map +0 -1
  334. package/dist/init-8o6W3_fn.js +0 -1
  335. package/dist/init-D5HtmyMI.js.map +0 -1
  336. package/dist/ink-list-Ca-5dunO.js.map +0 -1
  337. package/dist/judge-DYOF69Oe.js.map +0 -1
  338. package/dist/list-CM_YSJXq.js +0 -374
  339. package/dist/list-CM_YSJXq.js.map +0 -1
  340. package/dist/live-DQc8jr6o.js +0 -1
  341. package/dist/model-Ce9l9_rF.js.map +0 -1
  342. package/dist/model-Cxzo4cjw.js +0 -1
  343. package/dist/node-DhIRSrrK.js.map +0 -1
  344. package/dist/opencode-providers-Co7plCea.js.map +0 -1
  345. package/dist/plan-DqnzfDH1.js.map +0 -1
  346. package/dist/plugins-B2KQ0y3P.js.map +0 -1
  347. package/dist/preflight-DyzmvK7i.js.map +0 -1
  348. package/dist/provenance-iRxjp4Pu.js.map +0 -1
  349. package/dist/provider-DCdTHQpk.js +0 -1
  350. package/dist/providers-D0ob_tra.js +0 -1
  351. package/dist/providers-vXMXuwn-.js.map +0 -1
  352. package/dist/query-CvgUsvep.js +0 -1
  353. package/dist/record-DhzwkSXc.js.map +0 -1
  354. package/dist/remarks-CauU408t.js.map +0 -1
  355. package/dist/remarks-NMgPd30U.js.map +0 -1
  356. package/dist/resolve-B1zMnPwK.js.map +0 -1
  357. package/dist/resolve-BIIeVT12.js +0 -1
  358. package/dist/resume-Dkhrpjow.js.map +0 -1
  359. package/dist/runtime-DcvQcr36.js.map +0 -1
  360. package/dist/sandbox-8zCrtuCf.js.map +0 -1
  361. package/dist/server-0tuj8VSa.js +0 -1
  362. package/dist/shared-ogqPJ0uo.js.map +0 -1
  363. package/dist/source-llwLewOU.js +0 -1
  364. package/dist/src-BQ6BiXoc.js.map +0 -1
  365. package/dist/src-BXnv0ViU.js +0 -2
  366. package/dist/src-BiS8YNN_.js +0 -3
  367. package/dist/src-BkH4GOxu.js +0 -1
  368. package/dist/src-C-tH5ebd.js +0 -2
  369. package/dist/src-Cg1bFs9g.js +0 -2
  370. package/dist/src-DV_UorZv.js +0 -2
  371. package/dist/src-XxUpr2sa.js +0 -2
  372. package/dist/state-DZbAO95a.js +0 -1
  373. package/dist/suite-M8_7xio7.js.map +0 -1
  374. package/dist/suite-u2KJnGpY.js +0 -2
  375. package/dist/summary-Uzc9osfV.js.map +0 -1
  376. package/dist/types-C4OkYhZg.js.map +0 -1
  377. package/dist/vertex-h_peaZTN.js +0 -1
  378. package/dist/views-BISrsIjw.js.map +0 -1
  379. package/dist/views-Bfab8fM9.js +0 -2
  380. package/dist/workspace-B8EOCuxl.js +0 -1
@@ -1,33 +1,33 @@
1
- import { n as isOneOf, t as isObjectLike } from "./object-CeQ8dgUC.js";
2
1
  import { n as UsageError } from "./errors-ttMTnIYX.js";
3
- import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, h as parseDuration, i as durationSchema, l as nonEmpty$1, m as TIMER_LIMIT, n as dataProblem, o as genericProblem, p as MAX_TIMER_MS, s as got, t as closest, u as oneOf } from "./shared-ogqPJ0uo.js";
4
- import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-CC1RV21n.js";
5
- import { B as gatePolicySchema, D as pluginServerName, J as snapshotOptionSchema, L as egressPolicySchema, V as gateProblem, Y as snapshotProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-B2KQ0y3P.js";
6
- import { c as GATEWAY_NAMES, d as gatewayEntryDataSchema, f as gatewayNameSchema, i as claudeCodeTakesEffort, l as directOffPin, o as harnessDescriptorSchema, r as claudeCodeModelName, s as harnessProblem, t as FACTORY, u as gatewayDescriptorSchema } from "./harness-DYSC4zMh.js";
7
- import { a as fromCwd, l as sha256 } from "./ids-DuK6xl4w.js";
8
- import { a as providerEndpoints, c as providerServedModel, n as openCodeEffortBody, o as providerHosts, r as openCodeEffortRefusal, s as providerModelResponses, u as resolveOpenCodeProvider } from "./opencode-providers-Co7plCea.js";
9
- import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-DYOF69Oe.js";
10
- import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV, t as SANDBOX_ACCESS_TOKEN } from "./vertex-BLQMayc1.js";
2
+ import { a as fieldPath, c as jsonValueSchema, d as refusal, f as tokenUsageSchema, g as parseDuration, h as TIMER_LIMIT, i as durationSchema, l as nonEmpty$1, m as MAX_TIMER_MS, n as dataProblem, o as genericProblem, p as valueAt, s as got, t as closest, u as oneOf } from "./shared-Cfi45Bvy.js";
3
+ import { n as isOneOf, t as isObjectLike } from "./object-CeQ8dgUC.js";
4
+ import { _ as snapshotProblem, a as egressPolicySchema, c as gatePolicySchema, g as snapshotOptionSchema, l as gateProblem, s as envVarsSchema } from "./plain-BYUf_fUV.js";
5
+ import { a as isSandboxKind, d as sandboxKind, f as suggestedSandbox, i as idPart, n as capabilitiesOf, o as maxConcurrentProblem, t as SANDBOX_KINDS, u as sandboxHelpers } from "./kinds-Bh2I35id.js";
6
+ import { C as plainCredentialHeaders, D as sandboxOf, E as sameWorkspaceOnly, F as environmentDataSchema, I as sandboxDescriptorSchema, L as traitsSchema, M as unbrokeredSecrets, N as workspaceLabel, O as sandboxPaths, P as workspaceOf, _ as isMovingRef, a as checkEgress, c as environmentDataProblem, h as isEnvironment, j as traitsOf, l as environmentFrom, m as helpersFor, o as checkEnv, r as applied, u as environmentPaths, v as isSecret, w as plannedSecrets, z as sandboxProblem } from "./environment-cPMCOk-h.js";
7
+ import { c as GATEWAY_NAMES, d as gatewayEntryDataSchema, f as gatewayNameSchema, i as claudeCodeTakesEffort, l as directOffPin, o as harnessDescriptorSchema, r as claudeCodeModelName, s as harnessProblem, t as FACTORY, u as gatewayDescriptorSchema } from "./harness-DXv2gnaT.js";
8
+ import { c as sha256, i as fromCwd } from "./ids-C_advdJA.js";
9
+ import { n as openCodeEffortBody, r as openCodeEffortRefusal, u as resolveOpenCodeProvider } from "./opencode-providers-BGiXt8_B.js";
10
+ import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV } from "./vertex-Eb2wqahk.js";
11
11
  import { _ as redactValue, h as redactSecrets, r as badCharacterAt, t as REDACTED } from "./redact-BhjtnjrU.js";
12
- import { o as withoutBom } from "./text-B8-W3Cvn.js";
12
+ import { a as RECOMMENDED_JUDGE_MODEL, c as judgeCredentials, d as modelKeyOptions, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL } from "./judge-COMXT1Hx.js";
13
13
  import { t as inCI } from "./env-8x1O8I1M.js";
14
14
  import { u as ownFileURL } from "./root-Co0jhOT5.js";
15
- import { F as HARNESS_HOSTS, I as PACKAGE_HOSTS, P as DOCKER_EGRESS, mt as shellQuote, pt as run } from "./src-BQ6BiXoc.js";
16
- import { t as InfraError } from "./types-C4OkYhZg.js";
17
- import { n as gateLabel, u as pairFactors } from "./gate-Cp_pQoEe.js";
18
- import { u as opencodeModelError } from "./harness-DGsRZDSo.js";
15
+ import { B as HARNESS_HOSTS, V as PACKAGE_HOSTS } from "./src-y0nOeRRE.js";
16
+ import { y as localPluginServers } from "./plugins-BfnhazOv.js";
17
+ import { a as agentFor, b as isRoutingGateway, f as isAgentKind, n as DEFAULT_HARNESS, r as HARNESS_EXAMPLE, t as AGENTS, u as credentialOf, y as ROUTING_GATEWAYS } from "./agents-DXuFxtGC.js";
18
+ import { i as pairFactors } from "./pairs-BGb134Vu.js";
19
+ import { u as opencodeModelError } from "./harness-C2berMiM.js";
20
+ import { r as tartTagWarning } from "./tart-ref-DYZijjrU.js";
19
21
  import "./stats-C18W4xPX.js";
20
- import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-CptK032x.js";
21
- import { t as docker } from "./sandbox-8zCrtuCf.js";
22
+ import { n as gateLabel } from "./gate-CgYdaprG.js";
23
+ import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-C60z3wLq.js";
24
+ import { t as docker } from "./sandbox-DrlL4yqb.js";
25
+ import { z } from "zod";
22
26
  import { existsSync, readFileSync } from "node:fs";
23
- import { availableParallelism, tmpdir, totalmem } from "node:os";
24
- import { dirname, join, resolve, win32 } from "node:path";
27
+ import { availableParallelism, totalmem } from "node:os";
28
+ import { dirname, join, resolve } from "node:path";
25
29
  import { parseDocument } from "yaml";
26
- import { z } from "zod";
27
- import { connect, isIP } from "node:net";
28
- import { createHash, randomUUID } from "node:crypto";
29
- import { spawn } from "node:child_process";
30
- import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
30
+ import { isIP } from "node:net";
31
31
  // src/judge/artifacts.ts
32
32
  /** A capture handle: a name plus the file the attempt wrote. */
33
33
  function isCaptureHandle(value) {
@@ -403,6 +403,7 @@ const captureOptionsSchema = z.strictObject({
403
403
  model: z.string().includes("/").optional()
404
404
  });
405
405
  const harnessList = z.union([harnessDescriptorSchema, z.array(harnessDescriptorSchema).min(1)]);
406
+ const environmentList = z.union([environmentDataSchema, z.array(environmentDataSchema).min(1)]);
406
407
  const budgetSchema = z.strictObject({
407
408
  attempts: count().optional(),
408
409
  duration: durationSchema.optional()
@@ -421,9 +422,9 @@ const recordMode = z.enum([
421
422
  "always"
422
423
  ]);
423
424
  const profileDefaultsDataSchema = z.strictObject({
424
- harness: harnessList.optional(),
425
+ harnesses: harnessList.optional(),
425
426
  sandbox: sandboxDescriptorSchema.optional(),
426
- environment: environmentDataSchema.optional(),
427
+ environments: environmentList.optional(),
427
428
  runs: runs().optional(),
428
429
  gate: gatePolicySchema.optional(),
429
430
  env: envVarsSchema.optional(),
@@ -437,7 +438,8 @@ const profileDefaultsDataSchema = z.strictObject({
437
438
  const profileLimitSchema = z.strictObject({
438
439
  runs: count().optional(),
439
440
  compare: z.union([z.literal(false), z.strictObject({ maxVariants: z.number().int().min(0) })]).optional(),
440
- harness: z.array(z.string()).optional()
441
+ harnesses: z.array(z.string()).optional(),
442
+ traits: z.record(z.string(), z.union([z.string(), z.array(z.string()).min(1)])).optional()
441
443
  });
442
444
  const profileDataSchema = z.strictObject({
443
445
  defaults: profileDefaultsDataSchema.optional(),
@@ -482,9 +484,9 @@ const analysisConfigDataSchema = z.strictObject({
482
484
  });
483
485
  const reevalConfigDataSchema = z.strictObject({
484
486
  include: z.array(z.string()).optional(),
485
- harness: harnessList.optional(),
487
+ harnesses: harnessList.optional(),
486
488
  sandbox: sandboxDescriptorSchema.optional(),
487
- environment: environmentDataSchema.optional(),
489
+ environments: environmentList.optional(),
488
490
  runs: runs().optional(),
489
491
  gate: gatePolicySchema.optional(),
490
492
  env: envVarsSchema.optional(),
@@ -564,6 +566,7 @@ function configDataProblem(value, file) {
564
566
  }
565
567
  if (issue.code === "invalid_union") {
566
568
  const values = "options" in issue ? issue.options : void 0;
569
+ if (!values && issue.path.at(-1) === "environments") return `${at} must be an environment or a list of them (got ${got(v)})`;
567
570
  return values ? `${at} is ${oneOf(values)} (got ${got(v)})` : `${at} matches none of its forms (got ${got(v)})`;
568
571
  }
569
572
  if (issue.code === "invalid_type" && v === void 0) return `${at} is missing`;
@@ -578,1126 +581,6 @@ function configDataProblem(value, file) {
578
581
  path: []
579
582
  }, v)}`;
580
583
  }
581
- // src/gateway/routes.ts
582
- /** Every gateway that brokers a key, by name. */
583
- const ROUTING_GATEWAYS = {
584
- vercel: VERCEL_GATEWAY,
585
- openrouter: OPENROUTER_GATEWAY
586
- };
587
- /** Whether a harness's `gateway` names one of {@link ROUTING_GATEWAYS}. */
588
- const isRoutingGateway = (name) => name !== void 0 && Object.hasOwn(ROUTING_GATEWAYS, name);
589
- // src/harness/agents.ts
590
- const WEB_EVENTS = {
591
- webSearch: "web-search",
592
- webFetch: "web-fetch"
593
- };
594
- /** `harnessMetadata.cursorSdk.unreportedTools` as transcript event types. */
595
- function cursorUnreported(harnessMetadata) {
596
- const tools = harnessMetadata?.cursorSdk?.unreportedTools;
597
- return Array.isArray(tools) ? tools.flatMap((t) => WEB_EVENTS[t] ? [WEB_EVENTS[t]] : []) : [];
598
- }
599
- /** A Gemini API key's only request: `POST /v1beta/models/<model>:streamGenerateContent`. */
600
- const GEMINI_ENDPOINTS = [{
601
- host: "generativelanguage.googleapis.com",
602
- path: { regex: "^/v1beta/models/[^/]+:streamGenerateContent$" },
603
- method: ["POST"]
604
- }];
605
- /** Antigravity's: its model calls, and the `:generateContent` call its `search_web` tool makes. */
606
- const ANTIGRAVITY_ENDPOINTS = [{
607
- host: "generativelanguage.googleapis.com",
608
- path: { regex: "^/v1beta/models/[\\w.-]+:(?:streamGenerateContent|generateContent)$" },
609
- method: ["POST"]
610
- }];
611
- /** Goose's: the model call, and the model list it fetches with the key at the start of a turn. */
612
- const GOOSE_ENDPOINTS = [...GEMINI_ENDPOINTS, {
613
- host: "generativelanguage.googleapis.com",
614
- path: { regex: "^/v1beta/models$" },
615
- method: ["GET"]
616
- }];
617
- /**
618
- * What Grok Build sends the key on: the model list and the key's status as a session starts, and
619
- * its model calls (Chat Completions, and the Responses API for a model or tool that takes it).
620
- */
621
- const XAI_ENDPOINTS = [
622
- {
623
- host: "api.x.ai",
624
- path: { exact: "/v1/models" },
625
- method: ["GET"]
626
- },
627
- {
628
- host: "api.x.ai",
629
- path: { exact: "/v1/api-key" },
630
- method: ["GET"]
631
- },
632
- {
633
- host: "api.x.ai",
634
- path: { exact: "/v1/chat/completions" },
635
- method: ["POST"]
636
- },
637
- {
638
- host: "api.x.ai",
639
- path: { exact: "/v1/responses" },
640
- method: ["POST"]
641
- }
642
- ];
643
- /** xAI's model calls, whose responses name the model that served them. */
644
- const XAI_MODEL_RESPONSES = XAI_ENDPOINTS.filter((e) => e.path.exact === "/v1/chat/completions" || e.path.exact === "/v1/responses");
645
- /** The model an xAI response says served it. */
646
- const xaiServedModel = ({ host, responseModel }) => host === "api.x.ai" ? responseModel : void 0;
647
- /** The model a Gemini API request names in its URL. */
648
- const geminiServedModel = ({ host, path }) => host === "generativelanguage.googleapis.com" ? /^\/v1beta\/models\/([^/:]+):streamGenerateContent$/.exec(path)?.[1] : void 0;
649
- /** Cursor's key exchange: the only request the Cursor SDK sends the API key on. */
650
- const CURSOR_EXCHANGE = {
651
- host: "api2.cursor.sh",
652
- path: { exact: "/auth/exchange_user_api_key" },
653
- method: ["POST"]
654
- };
655
- /** Codex's model calls: the Responses API, whose response names the served model. */
656
- const OPENAI_MODEL_ENDPOINTS = [{
657
- host: "api.openai.com",
658
- path: { exact: "/v1/responses" },
659
- method: ["POST"]
660
- }];
661
- /** What Codex sends the key on: its model calls, and its web search tool's own endpoint. */
662
- const OPENAI_ENDPOINTS = [...OPENAI_MODEL_ENDPOINTS, {
663
- host: "api.openai.com",
664
- path: { exact: "/v1/alpha/search" },
665
- method: ["POST"]
666
- }];
667
- /**
668
- * The model the Responses API says served a request, from its response (a dated snapshot for some
669
- * models), and the reasoning effort Codex resolved for it, e.g. `gpt-5.6-luna[effort=medium]`.
670
- */
671
- const openaiServedModel = ({ host, path, responseModel, responseEffort }) => host === "api.openai.com" && path === "/v1/responses" && responseModel ? `${responseModel}${responseEffort ? `[effort=${responseEffort}]` : ""}` : void 0;
672
- /**
673
- * OpenCode on the model's provider (`anthropic/claude-haiku-4-5` is Anthropic's): its key, the
674
- * requests OpenCode sends it on, its hosts plus Exa's for web search, and the served model from
675
- * the URL (Gemini's API, Bedrock's) or the response body. Bedrock's hosts are its `AWS_REGION`'s.
676
- */
677
- function opencodeForModel(model, env = process.env) {
678
- const resolved = resolveOpenCodeProvider(model, env);
679
- if ("error" in resolved) throw new InfraError(`opencode(): ${resolved.error}`);
680
- const { provider } = resolved;
681
- const modelResponses = providerModelResponses(provider);
682
- return {
683
- credential: provider.env,
684
- endpoints: providerEndpoints(provider),
685
- ...modelResponses.length ? { modelResponses } : {},
686
- servedModel: (request) => providerServedModel(provider, request),
687
- hosts: [...providerHosts(provider), "mcp.exa.ai"]
688
- };
689
- }
690
- /**
691
- * The variables besides its key that OpenCode's provider takes from the host, as they are:
692
- * Bedrock's `AWS_REGION` (default `us-east-1`); none for most providers.
693
- */
694
- function opencodeVars(model, env = process.env) {
695
- const resolved = resolveOpenCodeProvider(model, env);
696
- if ("error" in resolved) throw new InfraError(`opencode(): ${resolved.error}`);
697
- return resolved.vars;
698
- }
699
- /** Codex's `config.toml` `[mcp_servers]` shape. */
700
- function codexMcpServers(servers) {
701
- return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
702
- if (s.transport === "sse") throw new InfraError(`Codex has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
703
- return s.transport === "stdio" ? [name, {
704
- command: s.command,
705
- args: s.args,
706
- env: s.env ?? {}
707
- }] : [name, {
708
- url: s.url,
709
- http_headers: s.headers ?? {}
710
- }];
711
- }));
712
- }
713
- /** The `.mcp.json` shape. */
714
- function claudeMcpServers(servers) {
715
- return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, {
716
- type: "stdio",
717
- command: s.command,
718
- args: s.args,
719
- env: s.env
720
- }] : [name, {
721
- type: s.transport,
722
- url: s.url,
723
- headers: s.headers
724
- }]));
725
- }
726
- /** The Antigravity CLI's `mcp_config.json` shape: remote servers take `serverUrl`, not `url`. */
727
- function antigravityMcpServers(servers) {
728
- return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, {
729
- command: s.command,
730
- args: s.args,
731
- env: s.env
732
- }] : [name, {
733
- serverUrl: s.url,
734
- headers: s.headers
735
- }]));
736
- }
737
- /**
738
- * A `{env:…}` or `{file:…}` OpenCode would substitute. It runs `\{env:[^}]+\}`, then
739
- * `\{file:[^}]+\}`, over its whole config text with no escape, so a reference left open at the end
740
- * of a value runs on into the JSON after it, and only `{env:}` and `{file:}` are left alone.
741
- */
742
- const OPENCODE_REFERENCE = /\{(?:env|file):(?!\})/;
743
- const fieldKey = (key) => /^[A-Za-z_$][\w$]*$/.test(key) ? `.${key}` : `[${JSON.stringify(key)}]`;
744
- /**
745
- * The first part of a server OpenCode would substitute, as what the server wouldn't get: its name,
746
- * a value as `mcp.<name>.<field>`, or an env or header name. Names are in the same config text as
747
- * values, so they're substituted too; a secret's value is never named.
748
- */
749
- function openCodeReference(name, s) {
750
- if (OPENCODE_REFERENCE.test(name)) return "its name";
751
- const server = `mcp${fieldKey(name)}`;
752
- const [map, kind, values] = s.transport === "stdio" ? [
753
- "env",
754
- "env",
755
- s.env ?? {}
756
- ] : [
757
- "headers",
758
- "header",
759
- s.headers ?? {}
760
- ];
761
- return [...s.transport === "stdio" ? [[`${server}.command`, s.command], ...s.args.map((a, i) => [`${server}.args[${i}]`, a])] : [[`${server}.url`, s.url]], ...Object.entries(values).flatMap(([k, v]) => [[`the ${kind} name ${k} in ${server}.${map}`, k], [`${server}.${map}${fieldKey(k)}`, v]])].find(([, text]) => OPENCODE_REFERENCE.test(text))?.[0];
762
- }
763
- /**
764
- * OpenCode's config shape (its `mcp.servers`), which the adapter sets `codemode: false` on.
765
- * OpenCode has no SSE transport, and refuses a value it would substitute.
766
- */
767
- function opencodeMcpServers(servers) {
768
- return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
769
- if (s.transport === "sse") throw new InfraError(`OpenCode has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
770
- const field = openCodeReference(name, s);
771
- if (field) throw new InfraError(`OpenCode substitutes {env:…} and {file:…} anywhere in its config, with no escape, so the ${name} server wouldn't get ${field} as written: take the reference out, or leave OpenCode out of this cell`);
772
- return s.transport === "stdio" ? [name, {
773
- type: "local",
774
- command: [s.command, ...s.args],
775
- environment: s.env ?? {}
776
- }] : [name, {
777
- type: "remote",
778
- url: s.url,
779
- headers: s.headers ?? {},
780
- oauth: false
781
- }];
782
- }));
783
- }
784
- /**
785
- * The key Goose keeps an extension by (its `name_to_key`): whitespace dropped, anything but ASCII
786
- * letters, digits, `_` and `-` made `_`, lower-cased. Of two extensions with one key it keeps one.
787
- */
788
- const gooseKey = (name) => name.replace(/\s/g, "").replace(/[^A-Za-z0-9_-]/g, "_").toLowerCase();
789
- /** Goose's extension shape (its `config.yaml`); it has no SSE transport. */
790
- function gooseMcpServers(servers) {
791
- const byKey = /* @__PURE__ */ new Map();
792
- for (const name of Object.keys(servers)) {
793
- const key = gooseKey(name);
794
- const other = byKey.get(key);
795
- if (other !== void 0) throw new InfraError(`Goose keys the ${other} and ${name} MCP servers both as ${key}, and keeps only one: rename one`);
796
- byKey.set(key, name);
797
- }
798
- return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
799
- if (s.transport === "sse") throw new InfraError(`Goose has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
800
- return s.transport === "stdio" ? [name, {
801
- type: "stdio",
802
- cmd: s.command,
803
- args: s.args,
804
- envs: s.env ?? {}
805
- }] : [name, {
806
- type: "streamable_http",
807
- uri: s.url,
808
- headers: s.headers ?? {}
809
- }];
810
- }));
811
- }
812
- /**
813
- * pi's `mcp.json` shape, which the adapter escapes and sets `"exposure": "direct"` on. pi
814
- * skips a server whose name isn't letters, digits, `_` and `-`, so the plan refuses it instead.
815
- * A plugin's server, `<plugin>:<server>`, is named `<plugin>_<server>`, so its tools are
816
- * `mcp__<plugin>_<server>__<tool>`.
817
- */
818
- function piMcpServers(servers) {
819
- const named = /* @__PURE__ */ new Map();
820
- return Object.fromEntries(Object.entries(servers).map(([key, s]) => {
821
- const name = /^[a-z0-9-]+:[^:]+$/.test(key) ? key.replace(":", "_") : key;
822
- if (!/^[A-Za-z0-9_-]+$/.test(name)) throw new InfraError(`pi takes MCP server names of letters, digits, "_" and "-" only, so it can't use the ${key} server: rename it`);
823
- const other = named.get(name);
824
- if (other) throw new InfraError(`pi names the ${key} MCP server ${name}, as it does the ${other} server, and keeps only one: rename one`);
825
- named.set(name, key);
826
- if (s.transport === "sse") throw new InfraError(`pi has no SSE MCP transport, so it can't use the ${key} server (mcp.sse): use mcp.http or mcp.stdio`);
827
- return s.transport === "stdio" ? [name, {
828
- command: s.command,
829
- args: s.args,
830
- env: s.env ?? {}
831
- }] : [name, {
832
- url: s.url,
833
- headers: s.headers ?? {}
834
- }];
835
- }));
836
- }
837
- /**
838
- * ACP's `McpServer` shape, which fx and Grok Build take on `session/new`: `env` and `headers` as
839
- * name-value lists.
840
- */
841
- function acpMcpServers(servers) {
842
- const pairs = (values = {}) => Object.entries(values).map(([name, value]) => ({
843
- name,
844
- value
845
- }));
846
- return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, s.command.startsWith("/") ? {
847
- command: s.command,
848
- args: s.args,
849
- env: pairs(s.env)
850
- } : {
851
- command: "/usr/bin/env",
852
- args: [s.command, ...s.args],
853
- env: pairs(s.env)
854
- }] : [name, {
855
- type: s.transport,
856
- url: s.url,
857
- headers: pairs(s.headers)
858
- }]));
859
- }
860
- /**
861
- * fx on the model's provider: OpenAI's Chat Completions API for a bare OpenAI id, and Vercel AI
862
- * Gateway for a gateway id, which only `vercel()` in the gateway list serves. Each response's
863
- * first `model` names the model that served it.
864
- */
865
- function fxForModel(model) {
866
- if (!model.includes("/")) {
867
- const endpoints = [{
868
- host: "api.openai.com",
869
- path: { exact: "/v1/chat/completions" },
870
- method: ["POST"]
871
- }];
872
- return {
873
- credential: "OPENAI_API_KEY",
874
- endpoints,
875
- modelResponses: endpoints,
876
- servedModel: ({ host, responseModel }) => host === "api.openai.com" ? responseModel : void 0,
877
- hosts: ["api.openai.com"]
878
- };
879
- }
880
- return {
881
- credential: VERCEL_KEY,
882
- endpoints: FX_VERCEL_ENDPOINTS,
883
- modelResponses: [FX_VERCEL_CALL],
884
- servedModel: servedOnVercel,
885
- askedModel: fxAskedModel,
886
- hosts: [VERCEL_HOST]
887
- };
888
- }
889
- /** The Cursor SDK's `McpServerConfig` shape. */
890
- function cursorMcpServers(servers) {
891
- return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, {
892
- type: "stdio",
893
- command: s.command,
894
- args: s.args,
895
- env: s.env
896
- }] : [name, {
897
- type: s.transport,
898
- url: s.url,
899
- headers: s.headers
900
- }]));
901
- }
902
- /** Claude Code's hook events as Cursor names them, as the Cursor SDK converts Claude Code's own. */
903
- const CURSOR_HOOK_EVENTS = {
904
- SessionStart: "sessionStart",
905
- SessionEnd: "sessionEnd",
906
- UserPromptSubmit: "beforeSubmitPrompt",
907
- PreToolUse: "preToolUse",
908
- PostToolUse: "postToolUse",
909
- SubagentStop: "subagentStop",
910
- PreCompact: "preCompact",
911
- Stop: "stop"
912
- };
913
- /** Claude Code's tool names as Cursor's hooks match them; null for one Cursor has no tool for. */
914
- const CURSOR_HOOK_TOOLS = {
915
- Bash: "Shell",
916
- Read: "Read",
917
- Write: "Write",
918
- Edit: "Write",
919
- Glob: null,
920
- Grep: "Grep",
921
- WebFetch: "WebFetch",
922
- WebSearch: "WebSearch",
923
- Task: "Task"
924
- };
925
- /**
926
- * A Claude Code hook group's tool matcher as Cursor's, on the tool events only: each name mapped,
927
- * `mcp__<server>__<tool>` as `MCP:<tool>`; null when it names only tools Cursor lacks.
928
- */
929
- function cursorHookMatcher(event, matcher) {
930
- if (event !== "PreToolUse" && event !== "PostToolUse") return void 0;
931
- if (!matcher || matcher === "*") return void 0;
932
- const out = [];
933
- for (const raw of matcher.split("|")) {
934
- const name = raw.trim();
935
- const parts = name.split("__");
936
- const tool = name.startsWith("mcp__") && parts.length >= 3 ? `MCP:${parts.slice(2).join("__")}` : CURSOR_HOOK_TOOLS[name] === void 0 ? name : CURSOR_HOOK_TOOLS[name];
937
- if (tool && !out.includes(tool)) out.push(tool);
938
- }
939
- return out.length ? out.join("|") : null;
940
- }
941
- /** Where Cursor reads what reeval unpacks from a plugin, in the workspace (settingSources `project`). */
942
- const CURSOR_PLUGIN_UNPACK = {
943
- rules: ".cursor/rules",
944
- agents: ".cursor/agents",
945
- commands: ".cursor/commands",
946
- hooks: {
947
- file: ".cursor/hooks.json",
948
- events: CURSOR_HOOK_EVENTS,
949
- matcher: cursorHookMatcher
950
- }
951
- };
952
- /**
953
- * A plugin subagent's or command's frontmatter as OpenCode 2 takes it: the description, and a
954
- * subagent's `mode`. OpenCode skips a file whose frontmatter its schema refuses, without saying,
955
- * and Claude Code's `tools: Read, Grep`, `model: sonnet` and `allowed-tools` are such keys.
956
- */
957
- function opencodeFrontmatter(part, front) {
958
- const d = front.description;
959
- const description = typeof d === "string" || typeof d === "number" || typeof d === "boolean" ? String(d) : void 0;
960
- return {
961
- ...description ? { description } : {},
962
- ...part === "agents" ? { mode: "subagent" } : {}
963
- };
964
- }
965
- /** Where OpenCode reads plugin subagents and commands, in the workspace's config directory. */
966
- const OPENCODE_PLUGIN_UNPACK = {
967
- agents: ".opencode/agents",
968
- commands: ".opencode/commands",
969
- frontmatter: opencodeFrontmatter
970
- };
971
- /**
972
- * Where pi reads plugin commands, as prompt templates (`/<name>`), in the workspace, which its
973
- * `--approve` trusts. Its frontmatter takes `description` and `argument-hint`, and ignores the
974
- * rest of Claude Code's.
975
- */
976
- const PI_PLUGIN_UNPACK = { commands: ".pi/prompts" };
977
- /**
978
- * What Claude Code's plugin format bundles, which Grok Build reads too: every part but rules,
979
- * which neither loads from a plugin.
980
- */
981
- const CLAUDE_FORMAT_PARTS = PLUGIN_PARTS.filter((p) => p !== "rules");
982
- /** The marketplace Claude Code's settings name for the session's plugins. */
983
- const CLAUDE_PLUGIN_MARKETPLACE = "reeval";
984
- /**
985
- * Claude Code's settings for the session's plugins: a `directory` marketplace at the directory
986
- * they were copied to, listing each, and each enabled as `<plugin>@reeval`. Claude Code installs
987
- * and loads them as it starts, since the bridge leaves every settings source on.
988
- */
989
- function claudePluginFiles(plugins) {
990
- const [first] = plugins;
991
- if (!first) return [];
992
- const json = (value) => `${JSON.stringify(value, null, 2)}\n`;
993
- const marketplace = {
994
- name: CLAUDE_PLUGIN_MARKETPLACE,
995
- owner: { name: "reeval" },
996
- plugins: plugins.map((p) => ({
997
- name: p.name,
998
- source: `./${p.name}`
999
- }))
1000
- };
1001
- const settings = {
1002
- extraKnownMarketplaces: { [CLAUDE_PLUGIN_MARKETPLACE]: { source: {
1003
- source: "directory",
1004
- path: first.dir.slice(0, first.dir.lastIndexOf("/"))
1005
- } } },
1006
- enabledPlugins: Object.fromEntries(plugins.map((p) => [`${p.name}@${CLAUDE_PLUGIN_MARKETPLACE}`, true]))
1007
- };
1008
- return [{
1009
- path: `${PLUGIN_DIR}/.claude-plugin/marketplace.json`,
1010
- content: json(marketplace)
1011
- }, {
1012
- path: "~/.claude/settings.json",
1013
- content: json(settings),
1014
- merge: true
1015
- }];
1016
- }
1017
- const isObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
1018
- /**
1019
- * Writes the layout's plugin files into the sandbox and returns their paths. A `merge` file's keys
1020
- * go into the JSON object already at its path (an object-valued key merged one level down), so
1021
- * settings an environment's `files` wrote there stay.
1022
- */
1023
- async function writePluginFiles(sandbox, layout, plugins) {
1024
- const written = [];
1025
- for (const f of layout.pluginFiles?.(plugins) ?? []) {
1026
- const existing = f.merge ? await sandbox.readFile(f.path) : null;
1027
- let content = f.content;
1028
- if (existing?.trim()) {
1029
- let base;
1030
- try {
1031
- base = JSON.parse(withoutBom(existing));
1032
- } catch {}
1033
- if (!isObject(base)) throw new InfraError(`${f.path} isn't a JSON object, so reeval can't enable the session's plugins in it: write it as one in files, or leave it out`);
1034
- const ours = JSON.parse(f.content);
1035
- const merged = { ...base };
1036
- for (const [k, v] of Object.entries(ours)) {
1037
- const was = base[k];
1038
- merged[k] = isObject(v) && isObject(was) ? {
1039
- ...was,
1040
- ...v
1041
- } : v;
1042
- }
1043
- content = `${JSON.stringify(merged, null, 2)}\n`;
1044
- }
1045
- await sandbox.writeFile(f.path, content);
1046
- written.push(f.path);
1047
- }
1048
- return written;
1049
- }
1050
- /**
1051
- * The MCP servers the agent is given: all of them, less the plugins' own where it takes plugin
1052
- * items namespaced, since it loads those from the plugin and would otherwise start each twice.
1053
- */
1054
- function agentMcp(layout, mcp, plugins) {
1055
- if (layout.pluginPlacement !== "namespaced") return mcp;
1056
- const own = new Set(plugins.flatMap((p) => p.servers));
1057
- return Object.fromEntries(Object.entries(mcp).filter(([name]) => !own.has(name)));
1058
- }
1059
- /** Where each agent with an image reads rules and skills. */
1060
- const LAYOUTS = {
1061
- cursor: {
1062
- rulesFile: "AGENTS.md",
1063
- workspaceSkillDir: ".cursor/skills",
1064
- harnessSkillDir: ".agents/skills",
1065
- pluginPlacement: "flat",
1066
- pluginUnpack: CURSOR_PLUGIN_UNPACK,
1067
- pluginHookEvents: Object.keys(CURSOR_HOOK_EVENTS)
1068
- },
1069
- "claude-code": {
1070
- rulesFile: "CLAUDE.md",
1071
- workspaceSkillDir: ".claude/skills",
1072
- harnessSkillDir: ".claude/skills",
1073
- pluginPlacement: "namespaced",
1074
- pluginFiles: claudePluginFiles
1075
- },
1076
- antigravity: {
1077
- rulesFile: "AGENTS.md",
1078
- workspaceSkillDir: ".agents/skills",
1079
- harnessSkillDir: ".gemini/config/skills",
1080
- pluginPlacement: "namespaced",
1081
- pluginHookEvents: [
1082
- "SessionStart",
1083
- "PreToolUse",
1084
- "PostToolUse",
1085
- "Stop"
1086
- ]
1087
- },
1088
- opencode: {
1089
- rulesFile: "AGENTS.md",
1090
- workspaceSkillDir: ".opencode/skills",
1091
- harnessSkillDir: ".config/opencode/skills",
1092
- pluginPlacement: "flat",
1093
- pluginUnpack: OPENCODE_PLUGIN_UNPACK
1094
- },
1095
- goose: {
1096
- rulesFile: "AGENTS.md",
1097
- workspaceSkillDir: ".goose/skills",
1098
- harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/goose/home/.config/goose/skills",
1099
- pluginPlacement: "flat"
1100
- },
1101
- pi: {
1102
- rulesFile: "AGENTS.md",
1103
- workspaceSkillDir: ".pi/skills",
1104
- harnessSkillDir: ".pi/agent/skills",
1105
- pluginPlacement: "flat",
1106
- pluginUnpack: PI_PLUGIN_UNPACK
1107
- },
1108
- codex: {
1109
- rulesFile: "AGENTS.md",
1110
- workspaceSkillDir: ".agents/skills",
1111
- harnessSkillDir: ".agents/skills",
1112
- pluginPlacement: "namespaced"
1113
- },
1114
- fx: {
1115
- rulesFile: "AGENTS.md",
1116
- workspaceSkillDir: ".fx/skills",
1117
- harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/fx/home/.fx/skills",
1118
- pluginPlacement: "flat"
1119
- },
1120
- "grok-build": {
1121
- rulesFile: "AGENTS.md",
1122
- workspaceSkillDir: ".grok/skills",
1123
- harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/grok-build/home/.grok/skills",
1124
- pluginPlacement: "namespaced"
1125
- }
1126
- };
1127
- /** Every real agent reeval can drive, by harness kind. */
1128
- const AGENTS = {
1129
- cursor: {
1130
- ...LAYOUTS.cursor,
1131
- pluginParts: PLUGIN_PARTS,
1132
- name: "Cursor",
1133
- factory: "cursor",
1134
- reasoning: "streamed",
1135
- reasoningTokens: false,
1136
- credential: "CURSOR_API_KEY",
1137
- endpoints: [CURSOR_EXCHANGE],
1138
- tokens: {
1139
- endpoint: CURSOR_EXCHANGE,
1140
- fields: ["accessToken", "refreshToken"],
1141
- hosts: ["cursor.sh", "*.cursor.sh"]
1142
- },
1143
- unreported: cursorUnreported,
1144
- mcpServers: cursorMcpServers,
1145
- adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-BXnv0ViU.js")).createCursorSdk({
1146
- auth,
1147
- ...mcpServers ? { mcpServers } : {},
1148
- ...descriptor.kind === "cursor" && descriptor.params ? { modelParams: descriptor.params } : {},
1149
- ...descriptor.kind === "cursor" && descriptor.effort ? { effort: descriptor.effort } : {},
1150
- requireCredentialBrokering: true
1151
- })
1152
- },
1153
- "claude-code": {
1154
- ...LAYOUTS["claude-code"],
1155
- pluginParts: CLAUDE_FORMAT_PARTS,
1156
- name: "Claude Code",
1157
- factory: "claudeCode",
1158
- reasoning: "streamed",
1159
- reasoningTokens: false,
1160
- subagentCalls: true,
1161
- credential: "ANTHROPIC_API_KEY",
1162
- endpoints: [
1163
- {
1164
- host: "api.anthropic.com",
1165
- path: { exact: "/v1/messages" },
1166
- method: ["POST"]
1167
- },
1168
- {
1169
- host: "api.anthropic.com",
1170
- path: { exact: "/v1/messages/count_tokens" },
1171
- method: ["POST"]
1172
- },
1173
- {
1174
- host: "api.anthropic.com",
1175
- path: { exact: "/api/claude_code/settings" },
1176
- method: ["GET"]
1177
- },
1178
- {
1179
- host: "api.anthropic.com",
1180
- path: { exact: "/api/claude_code/policy_limits" },
1181
- method: ["GET"]
1182
- }
1183
- ],
1184
- mcpServers: claudeMcpServers,
1185
- adapter: async ({ auth, env, mcpServers, descriptor }) => (await import("./dist-ByYk_wAs.js")).createClaudeCode({
1186
- auth,
1187
- ...mcpServers ? { mcpServers } : {},
1188
- ...descriptor.kind === "claude-code" && descriptor.effort ? { effort: descriptor.effort } : {},
1189
- env: {
1190
- IS_SANDBOX: "1",
1191
- CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",
1192
- ...env
1193
- }
1194
- })
1195
- },
1196
- antigravity: {
1197
- ...LAYOUTS.antigravity,
1198
- pluginParts: PLUGIN_PARTS,
1199
- pluginManifestsFirst: [
1200
- "plugin.json",
1201
- ".antigravity-plugin/plugin.json",
1202
- ".gemini-plugin/plugin.json",
1203
- "gemini-extension.json"
1204
- ],
1205
- name: "Antigravity",
1206
- factory: "antigravity",
1207
- reasoning: "none",
1208
- reasoningTokens: true,
1209
- credential: "GEMINI_API_KEY",
1210
- endpoints: ANTIGRAVITY_ENDPOINTS,
1211
- expectedBlocked: {
1212
- hosts: ["play.googleapis.com", "antigravity-unleash.goog"],
1213
- why: "Antigravity's telemetry and feature flags, which its CLI sends even with telemetry off"
1214
- },
1215
- servedModel: geminiServedModel,
1216
- mcpServers: antigravityMcpServers,
1217
- adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-XxUpr2sa.js")).createAntigravity({
1218
- auth,
1219
- ...vertex ? { vertex } : {},
1220
- ...mcpServers ? { mcpServers } : {},
1221
- ...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
1222
- name,
1223
- dir
1224
- })) } : {},
1225
- ...descriptor.kind === "antigravity" && descriptor.effort ? { effort: descriptor.effort } : {},
1226
- requireCredentialBrokering: true
1227
- })
1228
- },
1229
- opencode: {
1230
- ...LAYOUTS.opencode,
1231
- pluginParts: [
1232
- "skills",
1233
- "agents",
1234
- "commands",
1235
- "mcp"
1236
- ],
1237
- name: "OpenCode",
1238
- factory: "opencode",
1239
- configuredModelOnStart: true,
1240
- reasoning: "streamed",
1241
- reasoningTokens: "provider",
1242
- credential: "GOOGLE_GENERATIVE_AI_API_KEY",
1243
- endpoints: GEMINI_ENDPOINTS,
1244
- servedModel: geminiServedModel,
1245
- forModel: opencodeForModel,
1246
- mcpServers: opencodeMcpServers,
1247
- adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-BiS8YNN_.js")).createOpenCode({
1248
- auth,
1249
- model,
1250
- ...vertex ? { vertex } : { vars: opencodeVars(model) },
1251
- ...descriptor.kind === "opencode" && descriptor.effort ? { effort: descriptor.effort } : {},
1252
- ...mcpServers ? { mcpServers } : {},
1253
- requireCredentialBrokering: true
1254
- })
1255
- },
1256
- goose: {
1257
- ...LAYOUTS.goose,
1258
- pluginParts: ["skills", "mcp"],
1259
- name: "Goose",
1260
- factory: "goose",
1261
- configuredModelOnStart: true,
1262
- reasoning: "streamed",
1263
- reasoningTokens: false,
1264
- credential: "GOOGLE_GENERATIVE_AI_API_KEY",
1265
- endpoints: GOOSE_ENDPOINTS,
1266
- expectedBlocked: {
1267
- hosts: ["models.dev"],
1268
- why: "Goose's model catalog refresh, which no setting turns off"
1269
- },
1270
- servedModel: geminiServedModel,
1271
- mcpServers: gooseMcpServers,
1272
- adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-C-tH5ebd.js")).createGoose({
1273
- ...vertex ? { vertex } : {},
1274
- ...auth["AI_GATEWAY_API_KEY"] ? {
1275
- auth,
1276
- provider: (await import("./src-C-tH5ebd.js")).AI_GATEWAY_PROVIDER
1277
- } : auth["OPENROUTER_API_KEY"] ? {
1278
- auth,
1279
- provider: (await import("./src-C-tH5ebd.js")).OPENROUTER_PROVIDER
1280
- } : { auth: { GOOGLE_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" } },
1281
- model,
1282
- ...descriptor.kind === "goose" && descriptor.effort ? { effort: descriptor.effort } : {},
1283
- ...mcpServers ? { mcpServers } : {},
1284
- requireCredentialBrokering: true
1285
- })
1286
- },
1287
- pi: {
1288
- ...LAYOUTS.pi,
1289
- pluginParts: [
1290
- "skills",
1291
- "commands",
1292
- "mcp"
1293
- ],
1294
- name: "pi",
1295
- factory: "pi",
1296
- reasoning: "streamed",
1297
- reasoningTokens: true,
1298
- credential: "GOOGLE_GENERATIVE_AI_API_KEY",
1299
- endpoints: GEMINI_ENDPOINTS,
1300
- servedModel: geminiServedModel,
1301
- mcpServers: piMcpServers,
1302
- adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-B9oE_mNS.js")).createPi({
1303
- ...auth["AI_GATEWAY_API_KEY"] ? {
1304
- auth,
1305
- gateway: (await import("./src-B9oE_mNS.js")).PI_AI_GATEWAY
1306
- } : auth["OPENROUTER_API_KEY"] ? {
1307
- auth,
1308
- gateway: (await import("./src-B9oE_mNS.js")).PI_OPENROUTER
1309
- } : {
1310
- auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
1311
- ...vertex ? { vertex } : {}
1312
- },
1313
- ...descriptor.kind === "pi" && descriptor.effort ? { effort: descriptor.effort } : {},
1314
- ...mcpServers ? { mcpServers } : {},
1315
- requireCredentialBrokering: true
1316
- })
1317
- },
1318
- codex: {
1319
- ...LAYOUTS.codex,
1320
- pluginParts: ["skills", "mcp"],
1321
- pluginApps: "they need a ChatGPT login",
1322
- pluginManifestsFirst: ["plugin.json", ".codex-plugin/plugin.json"],
1323
- pluginRootManifestNeedsSchema: true,
1324
- pluginServersByName: true,
1325
- name: "Codex",
1326
- factory: "codex",
1327
- reasoning: "whole",
1328
- reasoningTokens: true,
1329
- credential: "OPENAI_API_KEY",
1330
- endpoints: OPENAI_ENDPOINTS,
1331
- modelResponses: OPENAI_MODEL_ENDPOINTS,
1332
- expectedBlocked: {
1333
- hosts: [
1334
- "github.com",
1335
- "api.github.com",
1336
- "chatgpt.com"
1337
- ],
1338
- why: "Codex's plugin sync, which it runs on every turn once plugins are on",
1339
- withPlugins: true
1340
- },
1341
- servedModel: openaiServedModel,
1342
- mcpServers: codexMcpServers,
1343
- adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-Cg1bFs9g.js")).createCodex({
1344
- auth,
1345
- ...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-Cg1bFs9g.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-Cg1bFs9g.js")).OPENROUTER_PROVIDER } : {},
1346
- ...descriptor.kind === "codex" && descriptor.effort ? { effort: descriptor.effort } : {},
1347
- ...mcpServers ? { mcpServers } : {},
1348
- ...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
1349
- name,
1350
- dir
1351
- })) } : {},
1352
- requireCredentialBrokering: true
1353
- })
1354
- },
1355
- fx: {
1356
- ...LAYOUTS.fx,
1357
- pluginParts: ["skills", "mcp"],
1358
- name: "fx",
1359
- factory: "fx",
1360
- configuredModelOnStart: true,
1361
- reasoning: "streamed",
1362
- reasoningTokens: "provider",
1363
- ...fxForModel("provider/model"),
1364
- forModel: fxForModel,
1365
- mcpServers: acpMcpServers,
1366
- adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-DV_UorZv.js")).createFx({
1367
- auth,
1368
- model,
1369
- ...auth["OPENROUTER_API_KEY"] ? { provider: "openrouter" } : {},
1370
- ...env?.FX_VERTEX_PROJECT ? { vertex: {
1371
- project: env.FX_VERTEX_PROJECT,
1372
- location: env.FX_VERTEX_LOCATION ?? ""
1373
- } } : {},
1374
- ...descriptor.kind === "fx" && descriptor.effort ? { effort: descriptor.effort } : {},
1375
- ...mcpServers ? { mcpServers } : {},
1376
- requireCredentialBrokering: true
1377
- })
1378
- },
1379
- "grok-build": {
1380
- ...LAYOUTS["grok-build"],
1381
- pluginParts: CLAUDE_FORMAT_PARTS,
1382
- pluginServersByName: true,
1383
- name: "Grok Build",
1384
- factory: "grokBuild",
1385
- configuredModelOnStart: true,
1386
- reasoning: "streamed",
1387
- reasoningTokens: true,
1388
- credential: "XAI_API_KEY",
1389
- endpoints: XAI_ENDPOINTS,
1390
- modelResponses: XAI_MODEL_RESPONSES,
1391
- servedModel: xaiServedModel,
1392
- mcpServers: acpMcpServers,
1393
- adapter: async ({ auth, mcpServers, descriptor, plugins, model, vertexChat }) => {
1394
- const grok = await import("./src-CUpoVVb8.js");
1395
- return grok.createGrokBuild({
1396
- ...vertexChat ? {
1397
- auth: { [grok.VERTEX_TOKEN_ENV]: SANDBOX_ACCESS_TOKEN },
1398
- provider: grok.vertexProvider(vertexChat.project, vertexChat.region)
1399
- } : {
1400
- auth,
1401
- ...auth["AI_GATEWAY_API_KEY"] ? { provider: grok.AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: grok.OPENROUTER_PROVIDER } : {}
1402
- },
1403
- model,
1404
- ...plugins.length ? {
1405
- pluginDirs: plugins.map((p) => p.dir),
1406
- pluginMcpServers: plugins.flatMap((p) => p.servers.map((s) => s.slice(p.name.length + 1)))
1407
- } : {},
1408
- ...descriptor.kind === "grok-build" && descriptor.effort ? { effort: descriptor.effort } : {},
1409
- ...mcpServers ? { mcpServers } : {},
1410
- requireCredentialBrokering: true
1411
- });
1412
- }
1413
- }
1414
- };
1415
- /** The harness of a config that names none: Cursor, on Cursor's own model. */
1416
- const DEFAULT_HARNESS = {
1417
- kind: "cursor",
1418
- model: "composer-2.5"
1419
- };
1420
- /** A harness to name in messages that ask for one. */
1421
- const HARNESS_EXAMPLE = "cursor({ model: 'composer-2.5' })";
1422
- const isAgentKind = (kind) => typeof kind === "string" && Object.hasOwn(AGENTS, kind);
1423
- /**
1424
- * A server as the plan knows it: its names, literal values and each `secret.template`'s text. A
1425
- * host secret's value is read when the attempt starts, so the plan has it as `''`.
1426
- */
1427
- function plannedMcpServer(s) {
1428
- const literal = (values) => Object.fromEntries(Object.entries(values).map(([k, v]) => [k, typeof v === "string" ? v : v.kind === "secret-template" ? v.template : ""]));
1429
- return s.transport === "stdio" ? {
1430
- transport: "stdio",
1431
- command: s.command,
1432
- args: s.args,
1433
- env: literal(s.env)
1434
- } : {
1435
- transport: s.transport,
1436
- url: s.url,
1437
- headers: literal(s.headers)
1438
- };
1439
- }
1440
- /**
1441
- * Why a harness's agent can't take the environment's MCP servers (Goose, OpenCode, pi and Codex
1442
- * take no `mcp.sse`; OpenCode would substitute a `{env:…}` or `{file:…}` in a name or value), in the
1443
- * agent's own words, so the plan refuses them before any attempt starts; undefined when it can.
1444
- */
1445
- function refusedMcp(harness, layers) {
1446
- if (!isAgentKind(harness.kind)) return void 0;
1447
- const servers = /* @__PURE__ */ new Map();
1448
- for (const { spec } of layers) for (const [name, server] of Object.entries(spec.mcp ?? {})) servers.set(name, plannedMcpServer(server));
1449
- if (servers.size === 0) return void 0;
1450
- try {
1451
- AGENTS[harness.kind].mcpServers(Object.fromEntries(servers));
1452
- return;
1453
- } catch (error) {
1454
- if (error instanceof InfraError) return error.message;
1455
- throw error;
1456
- }
1457
- }
1458
- /** The first host variable a plugin MCP server reads (`${VAR}`), as `VAR`; undefined for none. */
1459
- function hostVariable(s) {
1460
- const values = Object.entries(s.transport === "stdio" ? s.env : s.headers);
1461
- for (const [key, v] of values) {
1462
- if (typeof v === "string") continue;
1463
- if (v.kind === "secret") return v.from ?? key;
1464
- return /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/.exec(v.template)?.[1] ?? key;
1465
- }
1466
- }
1467
- const quoted = (parts) => parts.map((p) => `'${p}'`).join(", ");
1468
- /**
1469
- * Why a harness's agent can't take the plugins, in the agent's own words: a part it doesn't
1470
- * load, apps it can't take (`pluginApps`), a hook event or type it doesn't run, a plugin MCP
1471
- * server it refuses (named `<plugin>:<server>`) or whose `<plugin>:<server>` key `mcp` already has,
1472
- * or, where it takes plugin items flat, a skill or MCP server whose name another already has, or a
1473
- * subagent, command or rule another unpacks to the same file. `own` names the
1474
- * environment's own skills and MCP servers, and `unread` the plugins not fetched yet (git and
1475
- * npm ones, at plan time) by their call. Undefined when it can take them.
1476
- */
1477
- function refusedPlugins(harness, plugins, own, unread = []) {
1478
- if (plugins.length === 0 && unread.length === 0) return void 0;
1479
- const agent = isAgentKind(harness.kind) ? AGENTS[harness.kind] : void 0;
1480
- const mimics = harness.kind === "fake" && harness.layout ? AGENTS[harness.layout] : void 0;
1481
- const loads = (agent ?? mimics)?.pluginParts ?? PLUGIN_PARTS;
1482
- const name = agent?.name ?? (mimics ? `The fake agent with ${mimics.name}'s layout` : "The fake agent");
1483
- if (loads.length === 0) return `${name} can't load plugins yet: leave ${plugins[0] ? `the ${plugins[0].name} plugin` : unread[0]} out of this cell`;
1484
- for (const p of plugins) {
1485
- const refused = p.kept.filter((part) => !loads.includes(part));
1486
- if (refused.length === 0) continue;
1487
- const can = p.kept.filter((part) => loads.includes(part));
1488
- const what = refused.map((part) => `${part} (${PART_PATHS[part]})`).join(" and ");
1489
- const fix = can.length ? `use only: [${quoted(can)}], or leave` : "leave";
1490
- return `${name} can't load the ${p.name} plugin's ${what}: ${fix} the plugin out of this cell`;
1491
- }
1492
- const reads = agent ?? mimics;
1493
- const first = reads?.pluginManifestsFirst ?? [];
1494
- for (const p of plugins) {
1495
- const other = p.manifests.find((m) => first.includes(m) && (m !== "plugin.json" || !reads?.pluginRootManifestNeedsSchema || p.rootSchema));
1496
- if (other) return `${name} reads the ${p.name} plugin's ${other} in place of its ${PLUGIN_MANIFEST}, so reeval can't hold it to the parts it places: remove ${other} from the plugin, or leave the plugin out of this cell`;
1497
- }
1498
- const apps = (agent ?? mimics)?.pluginApps;
1499
- const withApps = apps ? plugins.find((p) => p.apps) : void 0;
1500
- if (withApps) return `${name} can't load the ${withApps.name} plugin's apps: ${apps}`;
1501
- const layout = layoutOf(harness);
1502
- const runs = layout.pluginHookEvents;
1503
- for (const p of runs ? plugins : []) {
1504
- const event = p.hooks.events.find((e) => !runs?.includes(e));
1505
- const type = p.hooks.otherTypes[0];
1506
- if (!event && !type) continue;
1507
- const rest = p.kept.filter((part) => part !== "hooks");
1508
- const fix = rest.length ? `use only: [${quoted(rest)}], or leave` : "leave";
1509
- return event ? `${name} can't run the ${p.name} plugin's ${event} hooks (hooks/hooks.json): it runs ${runs?.join(", ")} hooks; ${fix} the plugin out of this cell` : `${name} can't run the ${p.name} plugin's ${type} hooks (hooks/hooks.json): it runs only command hooks; ${fix} the plugin out of this cell`;
1510
- }
1511
- const servers = Object.fromEntries(plugins.flatMap((p) => Object.entries(p.mcp).map(([server, s]) => [pluginServerName(p.name, server), plannedMcpServer(s)])));
1512
- if (agent && Object.keys(servers).length) {
1513
- const alongside = agent.pluginPlacement === "flat" ? Object.fromEntries(own.mcp.map((n) => [n, {
1514
- transport: "stdio",
1515
- command: "true",
1516
- args: [],
1517
- env: {}
1518
- }])) : {};
1519
- try {
1520
- agent.mcpServers({
1521
- ...alongside,
1522
- ...servers
1523
- });
1524
- } catch (error) {
1525
- if (error instanceof InfraError) return error.message;
1526
- throw error;
1527
- }
1528
- }
1529
- for (const p of plugins) for (const server of Object.keys(p.mcp)) {
1530
- const key = pluginServerName(p.name, server);
1531
- if (own.mcp.includes(key)) return `the ${p.name} plugin's MCP server "${server}" would join mcp as ${key}, which mcp already names: rename mcp["${key}"]`;
1532
- }
1533
- if (layout.pluginPlacement === "namespaced") {
1534
- for (const p of plugins) {
1535
- if (!p.kept.includes("mcp")) continue;
1536
- for (const [server, s] of Object.entries(p.mcp)) {
1537
- const secret = hostVariable(s);
1538
- if (!secret) continue;
1539
- const rest = p.kept.filter((part) => part !== "mcp");
1540
- return `${name} loads the ${p.name} plugin's MCP server "${server}" itself, so reeval can't give it ${secret} from the host: use only: [${quoted(rest)}] and declare the server in mcp, where reeval fills it in`;
1541
- }
1542
- }
1543
- if (!(agent ?? mimics)?.pluginServersByName) return void 0;
1544
- const byName = new Map(own.mcp.map((n) => [n, `mcp.${n}`]));
1545
- for (const p of plugins) for (const server of Object.keys(p.mcp)) {
1546
- const other = byName.get(server);
1547
- if (other) return `${name} names the ${p.name} plugin's MCP server "${server}" by its own name, as it does ${other}, and keeps only one: rename one, or leave mcp out of the plugin with only`;
1548
- byName.set(server, `the ${p.name} plugin's MCP server "${server}"`);
1549
- }
1550
- return;
1551
- }
1552
- const serverOwner = new Map(own.mcp.map((n) => [n, `mcp.${n}`]));
1553
- for (const p of plugins) for (const server of Object.keys(p.mcp)) {
1554
- const other = serverOwner.get(server);
1555
- if (other) return `the ${p.name} plugin's MCP server "${server}" clashes with ${other}`;
1556
- serverOwner.set(server, `the ${p.name} plugin's MCP server "${server}"`);
1557
- }
1558
- const skillOwner = new Map(own.skills.map((s) => [s.name, `${s.call}'s "${s.name}"`]));
1559
- for (const p of plugins) for (const { name: skill } of p.skills) {
1560
- const other = skillOwner.get(skill);
1561
- if (other) return `the ${p.name} plugin's skill "${skill}" and ${other} would both be placed at ~/${layout.harnessSkillDir}/${skill}: rename one, or narrow it with only or exclude`;
1562
- skillOwner.set(skill, `the ${p.name} plugin's skill "${skill}"`);
1563
- }
1564
- for (const part of ["agents", "commands"]) {
1565
- const dir = layout.pluginUnpack?.[part];
1566
- if (!dir) continue;
1567
- const owner = /* @__PURE__ */ new Map();
1568
- for (const p of plugins) for (const item of p[part]) {
1569
- const other = owner.get(item);
1570
- const what = part === "agents" ? "subagent" : "command";
1571
- if (other) return `the ${p.name} plugin's ${what} "${item}" and the ${other} plugin's would both be placed at ${dir}/${item}.md: rename one, or narrow one with only`;
1572
- owner.set(item, p.name);
1573
- }
1574
- }
1575
- const rules = layout.pluginUnpack?.rules;
1576
- const ruleOwner = /* @__PURE__ */ new Map();
1577
- for (const p of rules ? plugins : []) for (const rule of p.rules) {
1578
- const file = `${p.name}-${rule.replace(/\.mdc?$/, "")}.mdc`;
1579
- const what = `the ${p.name} plugin's rule "${rule}"`;
1580
- const other = ruleOwner.get(file);
1581
- if (other) return `${what} and ${other} would both be placed at ${rules}/${file}: rename one, or narrow one with only`;
1582
- ruleOwner.set(file, what);
1583
- }
1584
- }
1585
- /**
1586
- * A real agent's catalog entry, with what its model's provider decides, or its gateway's route:
1587
- * on Vertex, the attempt's `vertex()` settings decide the project and region.
1588
- */
1589
- function agentFor(h, vertex) {
1590
- const agent = AGENTS[h.kind];
1591
- if (h.gateway === "vertex") {
1592
- const route = VERTEX_ROUTES[h.kind];
1593
- if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on Vertex AI`);
1594
- if (!vertex) throw new InfraError(`${agent.name} on Vertex AI has no vertex() settings`);
1595
- return {
1596
- ...agent,
1597
- ...route(vertex)
1598
- };
1599
- }
1600
- if (isRoutingGateway(h.gateway)) {
1601
- const gateway = ROUTING_GATEWAYS[h.gateway];
1602
- const route = gateway.routes[h.kind];
1603
- if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on ${gateway.name}`);
1604
- return {
1605
- ...agent,
1606
- ...route(process.env)
1607
- };
1608
- }
1609
- return agent.forModel ? {
1610
- ...agent,
1611
- ...agent.forModel(h.model)
1612
- } : agent;
1613
- }
1614
- /** The catalog entry for a harness's agent; undefined for the fake harness. */
1615
- const agentOf = (h, vertex) => h.kind === "fake" ? void 0 : agentFor(h, vertex);
1616
- /**
1617
- * The host variable a harness authenticates with; undefined for the fake harness, and on Vertex,
1618
- * whose token comes from Application Default Credentials.
1619
- */
1620
- const credentialOf = (h) => h.kind === "fake" || h.gateway === "vertex" ? void 0 : agentFor(h).credential;
1621
- /** The fake agent's own layout. */
1622
- const FAKE_LAYOUT = {
1623
- rulesFile: "AGENTS.md",
1624
- workspaceSkillDir: ".agents/skills",
1625
- harnessSkillDir: ".agents/skills",
1626
- pluginPlacement: "flat"
1627
- };
1628
- /** Where a harness's agent reads rules and skills; a fake one with `layout` mimics that agent. */
1629
- function layoutOf(descriptor) {
1630
- if (descriptor.kind !== "fake") return LAYOUTS[descriptor.kind];
1631
- return descriptor.layout ? LAYOUTS[descriptor.layout] : FAKE_LAYOUT;
1632
- }
1633
- /**
1634
- * The hosts a harness's agent is known to try in a session with or without plugins; a fake agent
1635
- * with `layout` expects that agent's. On Vertex, the attempt's `vertex()` settings pick the route.
1636
- */
1637
- function expectedBlockedFor(descriptor, plugins, vertex) {
1638
- const expected = descriptor.kind !== "fake" ? agentFor(descriptor, vertex).expectedBlocked : descriptor.layout ? AGENTS[descriptor.layout].expectedBlocked : void 0;
1639
- return expected?.withPlugins && !plugins ? void 0 : expected;
1640
- }
1641
- /**
1642
- * How the APIs behind the harnesses say they refused a key: an HTTP 401, Anthropic's and OpenAI's
1643
- * wording, Google's (which answers 400), and OpenCode's for any provider's 401 or 403. A bare 403 isn't here: the gateway and MCP servers send
1644
- * those for reasons that have nothing to do with the harness's key. Grok Build checks its key as
1645
- * it starts and, when xAI refuses it (a 400 or 401, not an outage), offers no `xai.api_key`
1646
- * sign-in, which is all `@ai-sdk/harness-acp` reports.
1647
- */
1648
- const REFUSED_KEY = /\bHTTP 401\b|\b401\b.{0,40}\bunauthori[sz]ed\b|\bunauthori[sz]ed\b.{0,40}\b401\b|\b(?:invalid|incorrect) (?:x-)?api[ _-]?key\b|\bapi key not valid\b|\bAPI_KEY_INVALID\b|\bauthentication_error\b|\bprovider authentication required\b|\bauthentication method "xai\.api_key" is not advertised\b/i;
1649
- /**
1650
- * What to do when the harness's API refused its key, naming the variable; else undefined. The
1651
- * error's wording says so for some CLIs; `refused` is the egress record's word for every one: a
1652
- * brokered request the API answered with 401, whatever the CLI made of it.
1653
- */
1654
- function authHint(agent, message, refused = false) {
1655
- if (!refused && !REFUSED_KEY.test(message)) return void 0;
1656
- if (agent.token) return `The API rejected the access token reeval minted from ${agent.credential}: check gcloud auth application-default login (or GOOGLE_APPLICATION_CREDENTIALS) and the project's Vertex AI access`;
1657
- return keyRejected(agent.credential);
1658
- }
1659
- /** What to do when the API refused the key in `credential`, naming it. */
1660
- const keyRejected = (credential) => `The API rejected ${credential}: set a valid key in the environment or .env.local`;
1661
- /**
1662
- * The agent's credential from the host, as the adapter's explicit `auth`. Fails closed. Empty for
1663
- * a brokered token, which the adapter never sees.
1664
- */
1665
- function agentAuth(agent, env = process.env) {
1666
- if (agent.token) return {};
1667
- const value = env[agent.credential];
1668
- if (!value) throw new InfraError(`${agent.credential} isn't set, so the ${agent.name} harness can't authenticate. Set it in the environment or .env.local.`);
1669
- return {
1670
- ...agent.authEnv,
1671
- [agent.authAs ?? agent.credential]: value
1672
- };
1673
- }
1674
- function brokeringFor(h, vertex) {
1675
- if (h.kind === "fake") {
1676
- if (!h.brokers) return void 0;
1677
- const { credential, endpoints } = AGENTS[h.layout ?? "cursor"];
1678
- return {
1679
- credential,
1680
- endpoints
1681
- };
1682
- }
1683
- const agent = agentOf(h, vertex);
1684
- if (!agent) return void 0;
1685
- const { credential, endpoints, tokens, servedModel, askedModel, modelResponses, upstream } = agent;
1686
- const { localResponses } = agent;
1687
- return {
1688
- credential,
1689
- endpoints,
1690
- ...tokens ? { tokens: {
1691
- name: credential,
1692
- ...tokens
1693
- } } : {},
1694
- ...servedModel ? { servedModel } : {},
1695
- ...askedModel ? { askedModel } : {},
1696
- ...modelResponses ? { modelResponses } : {},
1697
- ...upstream ? { upstream } : {},
1698
- ...localResponses?.length ? { localResponses } : {}
1699
- };
1700
- }
1701
584
  // src/harness/effort.ts
1702
585
  /**
1703
586
  * The reasoning effort a harness asked for. Same user-facing name on every factory; the CLI's
@@ -1749,13 +632,6 @@ function resolvedEffortOf(models) {
1749
632
  }
1750
633
  // src/contract/attempt.ts
1751
634
  const nullableNumber = z.number().nullable();
1752
- const tokenUsageSchema = z.strictObject({
1753
- input: z.number(),
1754
- output: z.number(),
1755
- cacheRead: z.number(),
1756
- cacheWrite: z.number(),
1757
- reasoning: nullableNumber
1758
- });
1759
635
  const fileChangeSchema = z.strictObject({
1760
636
  path: z.string(),
1761
637
  status: z.enum([
@@ -2229,7 +1105,8 @@ const cellSchema = z.strictObject({
2229
1105
  environment: z.string(),
2230
1106
  sandbox: z.string(),
2231
1107
  role: cellRoleSchema,
2232
- variant: z.string().optional()
1108
+ variant: z.string().optional(),
1109
+ traits: traitsSchema.optional()
2233
1110
  });
2234
1111
  z.strictObject({
2235
1112
  type: z.literal("start"),
@@ -2444,15 +1321,15 @@ function questionShows(state, artifacts) {
2444
1321
  const compareGateSchema = z.enum([
2445
1322
  "variants",
2446
1323
  "all",
2447
- "none"
1324
+ "observe"
2448
1325
  ]);
2449
1326
  /** The compare object, with environments checked by `environment`. */
2450
1327
  const compareSchema = (environment) => z.strictObject({
2451
- baseline: environment.optional(),
1328
+ control: environment,
2452
1329
  variants: z.array(environment).min(1),
2453
1330
  gate: compareGateSchema.optional(),
2454
1331
  minLift: z.number().min(-1).max(1).optional(),
2455
- harness: z.array(z.union([z.string(), harnessDescriptorSchema])).optional()
1332
+ harnesses: z.array(z.union([z.string(), harnessDescriptorSchema])).optional()
2456
1333
  });
2457
1334
  compareSchema(z.custom());
2458
1335
  const describe$1 = (value) => {
@@ -2470,14 +1347,17 @@ function explain(where) {
2470
1347
  const [key, index] = issue.path;
2471
1348
  const v = spec[key];
2472
1349
  switch (key) {
2473
- case "baseline": return `compare.baseline in ${where} must be an environment from defineEnvironment (got ${describe$1(v)}). It is the control; variants are not layered on it or on the eval's environment. Leave it out to use the eval's environment as the control.`;
1350
+ case "control":
1351
+ if (v === void 0) return noControl(where);
1352
+ if (Array.isArray(v)) return `compare.control in ${where} takes one environment (got ${v.map((e) => e?.name ?? describe$1(e)).join(", ")}). For a grid without a compare, use environments.`;
1353
+ return `compare.control in ${where} must be an environment from defineEnvironment (got ${describe$1(v)}). It is what each variant is measured against; variants are not layered on it.`;
2474
1354
  case "gate": return `compare.gate in ${where} is ${oneOf(compareGateSchema.options)} (got ${describe$1(v)})`;
2475
1355
  case "minLift": return `compare.minLift in ${where} is points from -1 to 1, e.g. minLift: 0.1 (got ${describe$1(v)})`;
2476
- case "harness": {
1356
+ case "harnesses": {
2477
1357
  const item = index === void 0 ? void 0 : v[index];
2478
1358
  const kind = item?.kind;
2479
1359
  const problem = kind && kind in FACTORY ? harnessProblem(item) : void 0;
2480
- return problem ? `compare.harness in ${where}: ${problem}` : `compare.harness in ${where} must be a list of harness ids, e.g. harness: ['cursor:composer-2.5'] (got ${describe$1(index === void 0 ? v : item)})`;
1360
+ return problem ? `compare.harnesses in ${where}: ${problem}` : `compare.harnesses in ${where} must be a list of harness ids, e.g. harnesses: ['cursor:composer-2.5'] (got ${describe$1(index === void 0 ? v : item)})`;
2481
1361
  }
2482
1362
  default: return index === void 0 ? `compare has no variants in ${where}. Give at least one environment.` : `compare.variants in ${where} must be environments from defineEnvironment (got ${describe$1(v[index])}). Variants share the eval's prompt and checks; if a variant needs different checks, write a separate eval.`;
2483
1363
  }
@@ -2485,393 +1365,23 @@ function explain(where) {
2485
1365
  }
2486
1366
  /** The order the hand-written checks ran in, so the first refusal stays the same. */
2487
1367
  const ORDER = [
2488
- "baseline",
1368
+ "control",
2489
1369
  "gate",
2490
1370
  "minLift",
2491
- "harness",
1371
+ "harnesses",
2492
1372
  "variants"
2493
1373
  ];
1374
+ const noControl = (where) => `compare in ${where} has no control. Name it: compare: { control: app, variants: [...] }.`;
2494
1375
  /**
2495
1376
  * What's wrong with a compare object whose environments `environment` checks, or undefined.
2496
1377
  * `where` names the group or eval it's set on.
2497
1378
  */
2498
1379
  function compareProblem(value, where, environment) {
2499
- if (Array.isArray(value)) return `compare in ${where} must be an object, not a list. Use compare: { baseline, variants: [...] }.`;
2500
- if (!isObjectLike(value) || !Array.isArray(value.variants)) return `compare in ${where} takes { variants, baseline? } (got ${describe$1(value)}). Use compare: { variants: [a, b], baseline?, gate?, minLift?, harness? }.`;
1380
+ if (Array.isArray(value)) return `compare in ${where} must be an object, not a list. Use compare: { control, variants: [...] }.`;
1381
+ if (isObjectLike(value) && "baseline" in value) return `Unknown compare option "baseline" in ${where}. Did you mean "control"?`;
1382
+ if (!isObjectLike(value) || !Array.isArray(value.variants)) return `compare in ${where} takes { control, variants } (got ${describe$1(value)}). Use compare: { control: a, variants: [b, c], gate?, minLift?, harnesses? }.`;
2501
1383
  return refusal(compareSchema(environment), value, explain(where), { order: ORDER });
2502
1384
  }
2503
- const POLL_MS = 200;
2504
- /**
2505
- * Prints `alive` while `pid` runs, then the sandbox's TCP tables, or `noproc` without them
2506
- * (or without `tables`: the fake sandbox shares the host's, where other processes bind too).
2507
- */
2508
- function probeScript(pid, tables = true) {
2509
- return `${pid ? `kill -0 ${pid} 2>/dev/null && echo alive; ` : ""}${tables ? "if [ -r /proc/net/tcp ]; then cat /proc/net/tcp /proc/net/tcp6 2>/dev/null; else echo noproc; fi" : "echo noproc"}; exit 0`;
2510
- }
2511
- /** `0100007F` → `127.0.0.1`; 32 hex digits → an IPv6 address. Each 32-bit word is host-endian. */
2512
- function procAddress(hex) {
2513
- const bytes = (hex.match(/.{8}/g) ?? []).flatMap((w) => (w.match(/../g) ?? []).reverse().map((b) => Number.parseInt(b, 16)));
2514
- if (bytes.length === 4) return bytes.join(".");
2515
- if (bytes.slice(0, 10).every((b) => b === 0) && bytes[10] === 255 && bytes[11] === 255) return bytes.slice(12).join(".");
2516
- const groups = [];
2517
- for (let i = 0; i < 16; i += 2) groups.push(((bytes[i] ?? 0) << 8 | (bytes[i + 1] ?? 0)).toString(16));
2518
- const full = groups.join(":");
2519
- const longest = (full.match(/(^|:)0(:0)+(:|$)/g) ?? []).reduce((a, b) => b.length > a.length ? b : a, "");
2520
- return longest ? full.replace(longest, "::") : full;
2521
- }
2522
- /** A listener the tunnel reaches: on loopback or every address, IPv4 or IPv6. */
2523
- const reachable = (address) => address === "0.0.0.0" || address === "::" || address === "::1" || address.startsWith("127.");
2524
- function parseProbe(stdout) {
2525
- const lines = stdout.split("\n");
2526
- const alive = lines.includes("alive");
2527
- if (lines.includes("noproc")) return {
2528
- proc: false,
2529
- alive
2530
- };
2531
- const listeners = [];
2532
- for (const l of lines) {
2533
- const m = /^\s*\d+:\s+([0-9A-F]+):([0-9A-F]{4})\s+\S+\s+0A\s/i.exec(l);
2534
- const address = m ? procAddress(m[1]) : "";
2535
- if (m && reachable(address)) listeners.push({
2536
- port: Number.parseInt(m[2], 16),
2537
- address
2538
- });
2539
- }
2540
- return {
2541
- proc: true,
2542
- alive,
2543
- listeners
2544
- };
2545
- }
2546
- /** The ports listening now that weren't before the command started, ascending. */
2547
- const newPorts = (listeners, before) => [...new Set(listeners.map((l) => l.port).filter((p) => !before.has(p)))].sort((a, b) => a - b);
2548
- /** The address to record for `port`: IPv4 first, since that's where the tunnel connects first. */
2549
- const addressFor = (listeners, port) => {
2550
- const all = listeners.filter((l) => l.port === port).map((l) => l.address);
2551
- return all.find((a) => !a.includes(":")) ?? all.find((a) => a === "::") ?? all[0] ?? "";
2552
- };
2553
- /** `want` is taken whoever listens on it: `servePort` found it free when the command started. */
2554
- function decide(s) {
2555
- const { listeners, alive } = s.probe;
2556
- const ports = newPorts(listeners, s.before);
2557
- if (listeners.some((l) => l.port === s.want)) return {
2558
- kind: "port",
2559
- port: s.want,
2560
- address: addressFor(listeners, s.want)
2561
- };
2562
- const late = s.elapsedMs >= s.timeoutMs;
2563
- if (!s.given && ports.length && (s.stableMs >= 3e3 || late || !alive)) {
2564
- if (ports.length === 1) {
2565
- const port = ports[0];
2566
- return {
2567
- kind: "port",
2568
- port,
2569
- address: addressFor(listeners, port)
2570
- };
2571
- }
2572
- return {
2573
- kind: "several",
2574
- ports
2575
- };
2576
- }
2577
- if (!alive) return { kind: "exited" };
2578
- return late ? {
2579
- kind: "timeout",
2580
- ports
2581
- } : { kind: "wait" };
2582
- }
2583
- const plain = (output) => output.replace(/\x1b\[[0-9;?]*[A-Za-z]/g, "");
2584
- /** The URLs `output` prints on one of `ports`, as Vite's `Local: http://localhost:5173/`. */
2585
- function printedUrls(output, ports) {
2586
- const found = /* @__PURE__ */ new Map();
2587
- for (const m of plain(output).matchAll(/https?:\/\/(?:\[[0-9a-f:]+\]|[^\s/:[\]]+):(\d+)[^\s'"]*/gi)) {
2588
- const port = Number(m[1]);
2589
- if (ports.includes(port) && !found.has(m[0])) found.set(m[0], port);
2590
- }
2591
- return [...found].map(([url, port]) => ({
2592
- url,
2593
- port
2594
- }));
2595
- }
2596
- /** Whether something on the host accepts a TCP connection on `port`, IPv4 or IPv6 loopback. */
2597
- async function hostListening(port) {
2598
- for (const host of ["127.0.0.1", "::1"]) if (await new Promise((resolve) => {
2599
- const socket = connect({
2600
- port,
2601
- host
2602
- });
2603
- socket.once("connect", () => {
2604
- socket.destroy();
2605
- resolve(true);
2606
- });
2607
- socket.once("error", () => resolve(false));
2608
- })) return host;
2609
- }
2610
- /** How many times `c.serve` allocates `PORT` again when the sandbox already listens on it. */
2611
- const ALLOCATE_TRIES = 5;
2612
- /**
2613
- * The port `c.serve` sets as `PORT` and waits on: `port`, or one from `allocate` that nothing
2614
- * listens on yet. Throws when something already listens on `port`, since that server, not the
2615
- * command's, would answer the checks: c.serve only stops what it started.
2616
- */
2617
- async function servePort(command, port, allocate, taken) {
2618
- if (port === void 0) {
2619
- for (let i = 0; i < ALLOCATE_TRIES; i++) {
2620
- const want = await allocate();
2621
- if (!await taken(want)) return want;
2622
- }
2623
- throw new Error(`c.serve(${JSON.stringify(command)}) found every port it allocated for PORT already in use in the sandbox`);
2624
- }
2625
- if (await taken(port)) throw new Error(`c.serve(${JSON.stringify(command)}, ${port}): port ${port} was already in use before c.serve started (a server the agent left running?), so the checks would reach that server, not this command's. Stop it first, or drop the port to serve on PORT`);
2626
- return port;
2627
- }
2628
- const sleep = (ms, signal) => new Promise((resolve) => {
2629
- const done = () => {
2630
- clearTimeout(timer);
2631
- signal?.removeEventListener("abort", done);
2632
- resolve();
2633
- };
2634
- const timer = setTimeout(done, ms);
2635
- signal?.addEventListener("abort", done, { once: true });
2636
- });
2637
- const listOf = (items) => items.length > 1 ? `${items.slice(0, -1).join(", ")} and ${items.at(-1)}` : String(items[0]);
2638
- /**
2639
- * Polls until something listens on `want`, or, without a given port, until the ports that are
2640
- * new since the command started have settled on one. Throws, with the command's last output,
2641
- * when they settle on several, it exits, or nothing listens in time.
2642
- */
2643
- async function awaitListening(command, w) {
2644
- const timeoutMs = w.timeoutMs ?? 12e4;
2645
- const started = Date.now();
2646
- let last = "";
2647
- let changed = started;
2648
- const output = async () => plain(await w.output().catch(() => ""));
2649
- const fail = async (why, out) => {
2650
- const tail = (out ?? await output()).trimEnd().split("\n").slice(-20).join("\n").trim();
2651
- return /* @__PURE__ */ new Error(`c.serve(${JSON.stringify(command)}) ${why}${tail ? `. Its last output:\n${tail}` : ""}`);
2652
- };
2653
- for (;;) {
2654
- w.signal?.throwIfAborted();
2655
- const probe = await w.probe();
2656
- const now = Date.now();
2657
- if (!probe.proc) {
2658
- const address = await (w.hostListening ?? hostListening)(w.want);
2659
- if (address !== void 0) return {
2660
- port: w.want,
2661
- address,
2662
- detected: false,
2663
- readyMs: now - started
2664
- };
2665
- if (!probe.alive) throw await fail(`exited before it listened on ${w.want}`);
2666
- if (now - started >= timeoutMs) throw await fail(`didn't listen on ${w.want} within ${timeoutMs / 1e3}s. This sandbox can't list the ports it listens on, so it must listen on PORT or the port passed`);
2667
- } else {
2668
- const key = newPorts(probe.listeners, w.before).join(",");
2669
- if (key !== last) {
2670
- last = key;
2671
- changed = now;
2672
- }
2673
- const d = decide({
2674
- want: w.want,
2675
- given: w.given,
2676
- probe,
2677
- before: w.before,
2678
- stableMs: now - changed,
2679
- elapsedMs: now - started,
2680
- timeoutMs
2681
- });
2682
- if (d.kind === "port") return {
2683
- port: d.port,
2684
- address: d.address,
2685
- detected: d.port !== w.want,
2686
- readyMs: now - started
2687
- };
2688
- if (d.kind === "several") {
2689
- const out = await output();
2690
- const urls = printedUrls(out, d.ports);
2691
- const printed = urls.length ? ` It printed ${listOf(urls.map((u) => u.url))}.` : "";
2692
- const hint = urls[0]?.port ?? d.ports[0];
2693
- throw await fail(`listens on ports ${listOf(d.ports)}, and not on PORT (${w.want}).${printed} Pass the one it serves on: c.serve(${JSON.stringify(command)}, ${hint})`, out);
2694
- }
2695
- if (d.kind === "exited") throw await fail("exited before it listened on a port");
2696
- if (d.kind === "timeout") {
2697
- const on = d.ports.length ? `; ${d.ports.length > 1 ? "ports" : "port"} ${listOf(d.ports)} started listening` : "";
2698
- throw await fail(`didn't listen on ${w.given ? `the port passed (${w.want})` : "a port"} within ${timeoutMs / 1e3}s${on}`);
2699
- }
2700
- }
2701
- await sleep(POLL_MS, w.signal);
2702
- }
2703
- }
2704
- // src/sandbox/fake.ts
2705
- /** Commands run on the host's own network, so the fake sandbox can neither restrict nor see it. */
2706
- const FAKE_EGRESS = {
2707
- enforce: "none",
2708
- record: false
2709
- };
2710
- /**
2711
- * The POSIX shell commands run in: `sh`, which Windows lacks, so there the `sh.exe` on PATH or
2712
- * Git for Windows'.
2713
- */
2714
- function posixShell(platform = process.platform, env = process.env, exists = existsSync) {
2715
- if (platform !== "win32") return "sh";
2716
- const onPath = (env.PATH ?? env.Path ?? "").split(win32.delimiter).filter(Boolean);
2717
- const git = [
2718
- env.ProgramFiles,
2719
- env["ProgramFiles(x86)"],
2720
- env.LOCALAPPDATA && win32.join(env.LOCALAPPDATA, "Programs")
2721
- ].filter((dir) => !!dir).map((dir) => win32.join(dir, "Git", "bin"));
2722
- const found = [...onPath, ...git].map((dir) => win32.join(dir, "sh.exe")).find(exists);
2723
- if (found) return found;
2724
- throw new UsageError("The fake sandbox runs commands with a POSIX sh, and there's none on this machine. Install Git for Windows (https://git-scm.com/download/win), which brings sh.exe, or run the evals in docker().");
2725
- }
2726
- const EXEC_TAG = "REEVAL_FAKE_EXEC";
2727
- const KILL_TAGGED = `for pass in 1 2 3 4 5; do
2728
- found=
2729
- for f in $(grep -lzx "${EXEC_TAG}=$1" /proc/[0-9]*/environ 2>/dev/null); do
2730
- p=\${f#/proc/}; kill -9 "\${p%/environ}" 2>/dev/null; found=1
2731
- done
2732
- [ -n "$found" ] || break
2733
- done`;
2734
- /**
2735
- * Stops a command's MSYS processes on Windows. `taskkill /t` doesn't reach them: Git's `sh.exe`
2736
- * starts bash, whose fork and exec leave each new process's Windows parent gone, so a timed-out
2737
- * `sleep 30` would run on, holding the command's pipes and working directory.
2738
- */
2739
- function killTagged(sh, tag) {
2740
- runSh(sh, KILL_TAGGED, tag);
2741
- }
2742
- /**
2743
- * Kills host processes whose cwd is the sandbox, including native `node.exe` `c.serve` starts.
2744
- * Match only the unique `reeval-sandbox-*` folder name: Git Bash `pwd` and node.exe's
2745
- * `/proc/cwd` disagree (`/tmp/...` vs `D:\\Users\\...\\Temp\\...`), and an empty
2746
- * `"$prefix"/*` is `/*`, which SIGKILLs the Windows runner (exit 2304).
2747
- */
2748
- const KILL_UNDER = `cd "$1" || exit 0
2749
- base=$(pwd -P 2>/dev/null || pwd)
2750
- name=\${base##*/}
2751
- case "$name" in
2752
- reeval-sandbox-?*) ;;
2753
- *) exit 0 ;;
2754
- esac
2755
- for f in /proc/[0-9]*/cwd; do
2756
- t=$(readlink "$f" 2>/dev/null) || continue
2757
- case "$t" in
2758
- *"$name"*) ;;
2759
- *) continue ;;
2760
- esac
2761
- p=\${f#/proc/}; p=\${p%/cwd}
2762
- [ -n "$p" ] || continue
2763
- kill -9 "$p" 2>/dev/null
2764
- wp=$(tr -d '\\r\\n' < /proc/$p/winpid 2>/dev/null)
2765
- [ -n "$wp" ] && taskkill /F /PID "$wp" >/dev/null 2>&1
2766
- done`;
2767
- function runSh(sh, script, arg) {
2768
- return new Promise((resolve) => {
2769
- const child = spawn(sh, [
2770
- "-c",
2771
- script,
2772
- "sh",
2773
- arg
2774
- ], {
2775
- stdio: "ignore",
2776
- windowsHide: true
2777
- });
2778
- const done = () => resolve();
2779
- child.on("error", done);
2780
- child.on("close", done);
2781
- });
2782
- }
2783
- /** Windows refuses to delete a directory a live process still has as cwd; retry after kills. */
2784
- async function removeTree(path) {
2785
- const deadline = Date.now() + 8e3;
2786
- for (;;) {
2787
- try {
2788
- await rm(path, {
2789
- recursive: true,
2790
- force: true
2791
- });
2792
- if (!existsSync(path)) return;
2793
- } catch {}
2794
- if (Date.now() >= deadline) {
2795
- await rm(path, {
2796
- recursive: true,
2797
- force: true
2798
- }).catch(() => {});
2799
- return;
2800
- }
2801
- await new Promise((r) => setTimeout(r, 50));
2802
- }
2803
- }
2804
- /**
2805
- * A temp directory on the host. Commands run unisolated on the host shell, so this is
2806
- * only meant for tests and for the fake harness.
2807
- */
2808
- async function createFakeSandbox(descriptor, env = {}) {
2809
- const sh = posixShell();
2810
- const parent = descriptor.dir ?? tmpdir();
2811
- await mkdir(parent, { recursive: true });
2812
- const base = await mkdtemp(join(parent, "reeval-sandbox-"));
2813
- const workdir = join(base, "work");
2814
- await mkdir(workdir);
2815
- const stateDir = `${base}.state`;
2816
- await mkdir(stateDir);
2817
- let destroyed;
2818
- const resolve = (path) => path.startsWith("~/") ? join(base, path.slice(2)) : join(workdir, path);
2819
- return {
2820
- workdir,
2821
- stateDir,
2822
- execOwn: (command, { signal } = {}) => run(sh, ["-c", command], {
2823
- cwd: workdir,
2824
- env: {
2825
- HOME: base,
2826
- ...env
2827
- },
2828
- signal
2829
- }),
2830
- exec: (command, { cwd, env: extra, signal } = {}) => {
2831
- const tag = process.platform === "win32" && signal ? randomUUID() : void 0;
2832
- const stop = () => killTagged(sh, tag);
2833
- if (tag) signal?.addEventListener("abort", stop, { once: true });
2834
- return run(sh, ["-c", cwd ? `cd ${shellQuote(cwd)} && ${command}` : command], {
2835
- cwd: workdir,
2836
- env: {
2837
- HOME: base,
2838
- ...env,
2839
- ...extra,
2840
- ...tag ? { [EXEC_TAG]: tag } : {}
2841
- },
2842
- signal
2843
- }).finally(() => signal?.removeEventListener("abort", stop));
2844
- },
2845
- home: base,
2846
- readFile: (path) => readFile(resolve(path), "utf8").catch(() => null),
2847
- getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}://${await hostListening(port) === "::1" ? "[::1]" : "127.0.0.1"}:${port}` }),
2848
- async writeFile(path, content) {
2849
- const target = resolve(path);
2850
- await mkdir(dirname(target), { recursive: true });
2851
- await writeFile(target, content);
2852
- },
2853
- destroy: () => {
2854
- destroyed ??= (async () => {
2855
- await rm(stateDir, {
2856
- recursive: true,
2857
- force: true
2858
- }).catch(() => {});
2859
- if (process.platform === "win32") {
2860
- if (existsSync(base)) await runSh(sh, KILL_UNDER, base);
2861
- await removeTree(base);
2862
- return;
2863
- }
2864
- await rm(base, {
2865
- recursive: true,
2866
- force: true
2867
- }).catch(() => {});
2868
- })().finally(() => {
2869
- destroyed = void 0;
2870
- });
2871
- return destroyed;
2872
- }
2873
- };
2874
- }
2875
1385
  // src/ui/mode.ts
2876
1386
  const set = (env, name) => {
2877
1387
  const value = env[name];
@@ -2944,11 +1454,7 @@ function resolveOutput(input) {
2944
1454
  * `egress: { allow: [...DEFAULT_EGRESS_ALLOW, 'api.example.com'] }`.
2945
1455
  */
2946
1456
  const DEFAULT_EGRESS_ALLOW = PACKAGE_HOSTS;
2947
- /** What each sandbox kind can do about egress, as its provider declares it. */
2948
- const SANDBOX_EGRESS = {
2949
- docker: DOCKER_EGRESS,
2950
- fake: FAKE_EGRESS
2951
- };
1457
+ Object.fromEntries(Object.entries(SANDBOX_KINDS).map(([kind, k]) => [kind, k.capabilities.egress]));
2952
1458
  /** The policy an attempt runs under when nothing sets one. */
2953
1459
  const DEFAULT_EGRESS = "proxy";
2954
1460
  /** Whether the policy allows only some hosts: `deny-all` and allowlists. */
@@ -3031,13 +1537,16 @@ const LEVELS = [
3031
1537
  ];
3032
1538
  /** Why the sandbox can't enforce the attempt's policy, or undefined when it can. */
3033
1539
  function unenforceable(planned, sandbox, where) {
1540
+ const capabilities = capabilitiesOf(sandbox);
1541
+ const can = capabilities.egress.enforce;
1542
+ if (planned.policy === "proxy" && capabilities.isolated && can === "none") return `${where} runs under egress 'proxy', where the egress gateway is the only way out, but the ${sandbox.kind} sandbox enforces no egress policy: its traffic that ignores the proxy goes out directly. Set egress: 'allow-all' to run it there, or run it in ${suggestedSandbox((c) => c.isolated && c.egress.enforce !== "none")}.`;
3034
1543
  const needs = egressNeeds(planned);
3035
- const can = SANDBOX_EGRESS[sandbox.kind].enforce;
3036
1544
  if (LEVELS.indexOf(can) >= LEVELS.indexOf(needs)) return void 0;
3037
1545
  const hosts = allowedHosts(planned)?.join(", ");
3038
1546
  const what = needs !== "allowlist" ? "turning egress off" : planned.policy === "deny-all" ? `a host allowlist for its automatic hosts (${hosts})` : `a host allowlist (${hosts})`;
3039
1547
  const enforces = can === "none" ? "enforces no egress policy" : `only enforces ${can === "all-or-nothing" ? "'allow-all' or 'deny-all'" : can}`;
3040
- const hint = sandbox.kind === "fake" ? " (its commands run on the host). Run it in docker(), or set egress: 'allow-all'" : "";
1548
+ const enforcing = suggestedSandbox((c) => LEVELS.indexOf(c.egress.enforce) >= LEVELS.indexOf(needs));
1549
+ const hint = capabilities.isolated ? "" : ` (its commands run on the host). ${enforcing ? `Run it in ${enforcing}, or set` : "Set"} egress: 'allow-all'`;
3041
1550
  return `${where} runs under egress ${describeEgress(planned.policy)}, which needs ${what}, but the ${sandbox.kind} sandbox ${enforces}${hint}.`;
3042
1551
  }
3043
1552
  const describeEgress = (policy) => typeof policy === "string" ? `'${policy}'` : `{ allow: [${policy.allow.map((h) => `'${h}'`).join(", ")}] }`;
@@ -3061,10 +1570,12 @@ function egressLabel({ policy, auto }) {
3061
1570
  const own = policy === "deny-all" ? "deny-all" : policy.allow.length ? `allow ${policy.allow.join(", ")}` : "allow none";
3062
1571
  return auto.length ? `${own} + auto: ${auto.join(", ")}` : own;
3063
1572
  }
3064
- const KNOWN = [...[
3065
- "harness",
1573
+ // src/options.ts
1574
+ /** Every eval/group option key `checkOptions` accepts. The feature matrix holds each to an inventory entry. */
1575
+ const EVAL_OPTION_KEYS = [
1576
+ "harnesses",
3066
1577
  "sandbox",
3067
- "environment",
1578
+ "environments",
3068
1579
  "compare",
3069
1580
  "runs",
3070
1581
  "gate",
@@ -3079,12 +1590,14 @@ const KNOWN = [...[
3079
1590
  "tags",
3080
1591
  "revision",
3081
1592
  "pin"
3082
- ]];
1593
+ ];
1594
+ const KNOWN = [...EVAL_OPTION_KEYS];
3083
1595
  const SUGGEST = {
3084
- harnesses: "harness",
1596
+ harness: "harnesses",
3085
1597
  sandboxes: "sandbox",
3086
- environments: "environment",
3087
- baseline: "environment",
1598
+ environment: "environments",
1599
+ baseline: "compare",
1600
+ control: "compare",
3088
1601
  variants: "compare",
3089
1602
  repeat: "runs",
3090
1603
  repeats: "runs",
@@ -3119,47 +1632,27 @@ function mergeSnapshot(outer, inner) {
3119
1632
  compress: inner.compress ?? outer.compress
3120
1633
  };
3121
1634
  }
3122
- /** Throws unless `value` is `'all'` or `{ passRate }` with a rate in (0, 1]. */
1635
+ /** Throws unless `value` is `'all'`, `'observe'` or `{ passRate }` with a rate in (0, 1]. */
3123
1636
  function checkGate(value, where) {
3124
1637
  const problem = gateProblem(value, where);
3125
1638
  if (problem) throw new UsageError(problem);
3126
1639
  }
3127
1640
  const PINNABLE = [
3128
- "harness",
1641
+ "harnesses",
3129
1642
  "runs",
3130
1643
  "compare"
3131
1644
  ];
3132
1645
  const ID = /^[A-Za-z0-9._:@+\-=[\]]+$/;
3133
- const sanitize = (text) => (text.split("/").at(-1) ?? text).replace(/[^A-Za-z0-9._:@-]+/g, "-");
3134
1646
  function harnessId(h) {
3135
1647
  if (h.id) return h.id;
3136
1648
  const effort = harnessEffort(h);
3137
- const suffix = effort ? `[effort=${sanitize(effort)}]` : "";
1649
+ const suffix = effort ? `[effort=${idPart(effort)}]` : "";
3138
1650
  if (h.kind === "fake") return `fake${suffix}`;
3139
- return `${h.model ? `${h.kind}:${sanitize(h.model)}` : h.kind}${suffix}`;
3140
- }
3141
- function installId(install) {
3142
- if (!install?.length) return "";
3143
- const key = JSON.stringify(install.map((step) => typeof step === "string" ? step : {
3144
- run: step.run,
3145
- timeout: step.timeout
3146
- }));
3147
- return `+${createHash("sha256").update(key).digest("hex").slice(0, 8)}`;
1651
+ return `${h.model ? `${h.kind}:${idPart(h.model)}` : h.kind}${suffix}`;
3148
1652
  }
3149
1653
  /** The provider a sandbox runs on: its kind, or `podman` for a Docker sandbox on Podman. */
3150
- const sandboxProvider = (s) => s.kind === "docker" && s.engine === "podman" ? "podman" : s.kind;
3151
- function sandboxId(s) {
3152
- if (s.id) return s.id;
3153
- if (s.kind === "fake") return "fake";
3154
- const install = s.kind === "docker" ? installId(s.install) : "";
3155
- const engine = s.engine ?? "docker";
3156
- if (s.dockerfile) {
3157
- const [file = "", dir = ""] = s.dockerfile.split(/[\\/]/).reverse();
3158
- return `${engine}:${sanitize(file === "Dockerfile" ? dir || file : file)}${install}`;
3159
- }
3160
- const image = s.image ?? s.from;
3161
- return image ? `${engine}:${sanitize(image)}${install}` : `${engine}${install}`;
3162
- }
1654
+ const sandboxProvider = (s) => sandboxKind(s).provider(s);
1655
+ const sandboxId = (s) => s.id ?? sandboxKind(s).id(s);
3163
1656
  /** The planned attempts' sandboxes by id, once each, or `fallback`'s when none is planned. */
3164
1657
  function plannedSandboxIds(planned, fallback) {
3165
1658
  const ids = new Set(planned.map(sandboxId));
@@ -3181,11 +1674,14 @@ const describe = (value) => {
3181
1674
  }
3182
1675
  return JSON.stringify(value) ?? String(value);
3183
1676
  };
3184
- /** `compare: false` and `environment` beside `compare.baseline` are never legal options. */
1677
+ /** `compare: false` and `environments` beside `compare` are never legal options. */
3185
1678
  function refuseIllegalCompare(options, where) {
3186
- if (options.compare === false) throw new UsageError(`compare: false can't be set in ${where}. A profile turns compare off with limit.compare: false, which runs the baseline and strips the variants. On a group or eval, omit compare or pass compare: { baseline, variants }.`);
3187
- const baseline = options.compare !== null && typeof options.compare === "object" && "baseline" in options.compare ? options.compare.baseline : void 0;
3188
- if (options.environment !== void 0 && baseline !== void 0) throw new UsageError(`environment and compare.baseline can't both be set in ${where}. compare: { baseline, variants } names every arm, so drop environment, or drop baseline to make environment the control.`);
1679
+ if (options.compare === false) throw new UsageError(`compare: false can't be set in ${where}. A profile turns compare off with limit.compare: false, which runs the control and strips the variants. On a group or eval, omit compare or pass compare: { control, variants }.`);
1680
+ if (options.environments !== void 0 && options.compare !== void 0) {
1681
+ const first = Array.isArray(options.environments) ? options.environments[0] : options.environments;
1682
+ const name = isObjectLike(first) && typeof first.name === "string" ? first.name : "app";
1683
+ throw new UsageError(`${where} sets both environments and compare. With compare, the control is compare.control: move it there, as compare: { control: ${name}, variants: [...] }.`);
1684
+ }
3189
1685
  }
3190
1686
  const isRevision = (value) => typeof value === "number" && Number.isSafeInteger(value) || typeof value === "string" && value.trim() !== "";
3191
1687
  /** `runs: 3` or `runs: { bestOf: 5 }`, refused in the words of whichever it was meant to be. */
@@ -3216,7 +1712,7 @@ function checkOptions(options, where, allowed = KNOWN) {
3216
1712
  if (allowed.includes(key)) continue;
3217
1713
  const near = SUGGEST[key] ?? closest(key, allowed);
3218
1714
  const value = options[key];
3219
- const hint = key === "pool" ? " pool is an option of evaluate.each, beside the cases: evaluate.each(cases, { pool: false })." : key === "workspace" && typeof value !== "boolean" ? " workspace is an environment's starting tree: defineEnvironment(name, { workspace: git(…) })." : near ? ` Did you mean "${near}"?` : "";
1715
+ const hint = key === "cases" || key === "caseKey" ? ` ${key} belongs to one eval: evaluate(name, { cases: [...] }, async (c, item) => { … }).` : key === "where" ? " where is an option of evaluate() only, beside cases: evaluate(name, { cases: [...], where: ({ case, environment }) => … }, …)." : key === "workspace" && typeof value !== "boolean" ? " workspace is an environment's starting tree: defineEnvironment(name, { workspace: git(…) })." : near ? ` Did you mean "${near}"?` : "";
3220
1716
  throw new UsageError(`Unknown option "${key}" in ${where}.${hint}`);
3221
1717
  }
3222
1718
  const o = options;
@@ -3234,32 +1730,26 @@ function checkOptions(options, where, allowed = KNOWN) {
3234
1730
  });
3235
1731
  if (judgeIssue) throw new UsageError(judgeIssue);
3236
1732
  if (o.sandbox !== void 0) {
3237
- const kind = o.sandbox.kind;
3238
- if (kind !== "docker" && kind !== "fake") throw new UsageError(`sandbox in ${where} must come from docker(), podman() or fakeSandbox() (got ${describe(o.sandbox)})`);
3239
- const max = o.sandbox.maxConcurrent;
3240
- if (max !== void 0 && !(Number.isInteger(max) && max >= 1)) throw new UsageError(`sandbox.maxConcurrent in ${where} must be a whole number of at least 1 (got ${String(max)})`);
1733
+ if (!isSandboxKind(o.sandbox.kind)) throw new UsageError(`sandbox in ${where} must come from ${sandboxHelpers()} (got ${describe(o.sandbox)})`);
1734
+ const limit = maxConcurrentProblem(o.sandbox, where);
1735
+ if (limit) throw new UsageError(limit);
3241
1736
  const problem = sandboxProblem(o.sandbox);
3242
1737
  if (problem) throw new UsageError(`sandbox in ${where}: ${problem}`);
3243
1738
  }
3244
- if (o.environment !== void 0) {
3245
- if (Array.isArray(o.environment)) throw new UsageError(`environment takes one environment, not a list (${where}). To compare environments, use compare: { baseline, variants: [a, b] }. baseline is the control; variants are not layered on it.`);
3246
- if (!isEnvironment(o.environment)) throw new UsageError(`environment in ${where} must come from defineEnvironment(…) (got ${describe(o.environment)})`);
3247
- const problem = environmentDataProblem(o.environment, "environment", where);
3248
- if (problem) throw new UsageError(problem);
3249
- }
1739
+ if (o.environments !== void 0) checkEnvironments(o.environments, where);
3250
1740
  refuseIllegalCompare(options, where);
3251
1741
  if (o.compare !== void 0) {
3252
1742
  const problem = compareProblem(o.compare, where, z.custom(isEnvironment));
3253
1743
  if (problem) throw new UsageError(problem);
3254
- const { baseline, variants } = o.compare;
3255
- for (const [label, env] of [["compare.baseline", baseline], ...variants.map((v, i) => [`compare.variants[${i}]`, v])]) {
1744
+ const { control, variants } = o.compare;
1745
+ for (const [label, env] of [["compare.control", control], ...variants.map((v, i) => [`compare.variants[${i}]`, v])]) {
3256
1746
  const bad = env && environmentDataProblem(env, label, where);
3257
1747
  if (bad) throw new UsageError(bad);
3258
1748
  }
3259
1749
  }
3260
1750
  if (o.pin !== void 0) {
3261
1751
  const bad = Array.isArray(o.pin) ? o.pin.filter((k) => !PINNABLE.includes(k)) : [o.pin];
3262
- if (bad.length) throw new UsageError(`pin in ${where} lists harness, runs and/or compare (got ${bad.map((k) => JSON.stringify(k)).join(", ")})`);
1752
+ if (bad.length) throw new UsageError(`pin in ${where} lists harnesses, runs and/or compare (got ${bad.map((k) => JSON.stringify(k)).join(", ")})`);
3263
1753
  }
3264
1754
  if (o.capture !== void 0) checkCapture(o.capture, where);
3265
1755
  const questionIssue = o.question !== void 0 && questionOptionsProblem(o.question, where);
@@ -3309,42 +1799,82 @@ function checkCaptureModel(value, where) {
3309
1799
  if (typeof value !== "string" || !value.includes("/")) throw new UsageError(`capture.model in ${where} is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${JSON.stringify(value)})`);
3310
1800
  if (isEvaluationModel(value)) throw new UsageError(`capture.model "${value}" in ${where} is an evaluation model: it scores yes/no questions and can't drive a capture. Use a language model that accepts images, e.g. 'google/gemini-3.8-flash'.`);
3311
1801
  }
1802
+ /** Each environment `environments` names, as one, a list or an edit, comes from defineEnvironment. */
1803
+ function checkEnvironments(value, where) {
1804
+ if (typeof value === "function") return;
1805
+ let list;
1806
+ let label = "environments";
1807
+ if (isEnvironmentEdit(value)) {
1808
+ if ("add" in value && "only" in value) throw new UsageError(`environments in ${where} takes { add } or { only }, not both. Use add to append to the inherited environments, only to keep some of them.`);
1809
+ if ("only" in value) {
1810
+ const names = [value.only].flat();
1811
+ if (!names.length || names.some((n) => typeof n !== "string")) throw new UsageError(`environments { only } in ${where} lists environment names, e.g. { only: ['app'] } (got ${describe(value.only)})`);
1812
+ return;
1813
+ }
1814
+ list = [value.add].flat();
1815
+ label = "environments.add";
1816
+ } else list = [value].flat();
1817
+ if (list.length === 0) throw new UsageError(`${label} is an empty list in ${where}. Give at least one, or leave environments out.`);
1818
+ list.forEach((env, i) => {
1819
+ const at = Array.isArray(value) || label !== "environments" ? `${label}[${i}]` : label;
1820
+ if (!isEnvironment(env)) throw new UsageError(`${at} in ${where} must come from defineEnvironment(…) (got ${describe(env)})`);
1821
+ const problem = environmentDataProblem(env, at, where);
1822
+ if (problem) throw new UsageError(problem);
1823
+ });
1824
+ }
1825
+ const isEnvironmentEdit = (v) => isObjectLike(v) && !Array.isArray(v) && !isEnvironment(v) && ("add" in v || "only" in v);
1826
+ function applyEnvironments(inherited, value, where) {
1827
+ if (value === void 0) return inherited;
1828
+ const list = [typeof value === "function" ? value(inherited) : isEnvironmentEdit(value) ? editEnvironments(inherited, value, where) : value].flat();
1829
+ if (list.length === 0) throw new UsageError(`environments is an empty list in ${where}. Give at least one, or leave environments out.`);
1830
+ if (typeof value === "function") checkEnvironments(list, where);
1831
+ return list;
1832
+ }
1833
+ /** `{ add }` appends to the inherited environments; `{ only }` keeps those with these names. */
1834
+ function editEnvironments(inherited, edit, where) {
1835
+ if ("add" in edit) return [...inherited, ...[edit.add].flat()];
1836
+ const names = [edit.only].flat();
1837
+ const have = inherited.map((e) => e.name);
1838
+ const missing = names.filter((n) => !have.includes(n));
1839
+ if (missing.length) throw new UsageError(`environments { only: ${missing.map((n) => JSON.stringify(n)).join(", ")} } in ${where} names no inherited environment (inherited: ${have.join(", ") || "none"})`);
1840
+ return inherited.filter((e) => names.includes(e.name));
1841
+ }
3312
1842
  function applyHarness(inherited, value, where) {
3313
1843
  if (value === void 0) return inherited;
3314
1844
  const next = typeof value === "function" ? value(inherited) : isHarnessEdit(value) ? editHarness(inherited, value, where) : value;
3315
1845
  const list = Array.isArray(next) ? next : [next];
3316
- if (list.length === 0) throw new UsageError(`harness is an empty list in ${where}. Give at least one, e.g. harness: ${HARNESS_EXAMPLE}`);
1846
+ if (list.length === 0) throw new UsageError(`harnesses is an empty list in ${where}. Give at least one, e.g. harnesses: ${HARNESS_EXAMPLE}`);
3317
1847
  checkHarnesses(list, where);
3318
1848
  return list;
3319
1849
  }
3320
1850
  const isHarnessEdit = (v) => typeof v === "object" && v !== null && !Array.isArray(v) && !("kind" in v) && ("add" in v || "only" in v);
3321
1851
  /** `{ add }` appends to the inherited harnesses; `{ only }` keeps those with these ids. */
3322
1852
  function editHarness(inherited, edit, where) {
3323
- if ("add" in edit && "only" in edit) throw new UsageError(`harness in ${where} takes { add } or { only }, not both. Use add to append to the inherited harnesses, only to keep some of them.`);
1853
+ if ("add" in edit && "only" in edit) throw new UsageError(`harnesses in ${where} takes { add } or { only }, not both. Use add to append to the inherited harnesses, only to keep some of them.`);
3324
1854
  if ("add" in edit) return [...inherited, ...Array.isArray(edit.add) ? edit.add : [edit.add]];
3325
1855
  const ids = Array.isArray(edit.only) ? edit.only : [edit.only];
3326
1856
  const have = inherited.map(harnessId);
3327
1857
  const missing = ids.filter((id) => typeof id !== "string" || !have.includes(id));
3328
- if (missing.length) throw new UsageError(`harness { only: ${missing.map((id) => JSON.stringify(id)).join(", ")} } in ${where} names no inherited harness (inherited: ${have.join(", ")})`);
1858
+ if (missing.length) throw new UsageError(`harnesses { only: ${missing.map((id) => JSON.stringify(id)).join(", ")} } in ${where} names no inherited harness (inherited: ${have.join(", ")})`);
3329
1859
  return inherited.filter((h) => ids.includes(harnessId(h)));
3330
1860
  }
3331
1861
  /** Each harness is one, with a valid `maxConcurrent`. */
3332
1862
  function checkHarnesses(list, where) {
3333
1863
  for (const item of list) {
3334
- if (!isHarness(item)) throw new UsageError(`harness in ${where} must be like ${HARNESS_EXAMPLE} (got ${describe(item)})`);
1864
+ if (!isHarness(item)) throw new UsageError(`harnesses in ${where} must be like ${HARNESS_EXAMPLE} (got ${describe(item)})`);
3335
1865
  if (item.kind === "opencode") {
3336
1866
  const why = opencodeModelError(item.model);
3337
1867
  if (why) throw new UsageError(`opencode() in ${where}: ${why}`);
3338
1868
  }
3339
- checkMaxConcurrent(item.maxConcurrent, `harness.maxConcurrent in ${where}`);
1869
+ checkMaxConcurrent(item.maxConcurrent, `harnesses.maxConcurrent in ${where}`);
3340
1870
  const { startTimeout } = item;
3341
1871
  if (startTimeout !== void 0) try {
3342
1872
  parseDuration(startTimeout);
3343
1873
  } catch (error) {
3344
- throw new UsageError(`harness.startTimeout in ${where}: ${error.message}`);
1874
+ throw new UsageError(`harnesses.startTimeout in ${where}: ${error.message}`);
3345
1875
  }
3346
1876
  const problem = harnessProblem(item);
3347
- if (problem) throw new UsageError(`harness in ${where}: ${problem}`);
1877
+ if (problem) throw new UsageError(`harnesses in ${where}: ${problem}`);
3348
1878
  }
3349
1879
  }
3350
1880
  function checkUnique(list, where) {
@@ -3371,24 +1901,21 @@ const plannedQuestions = (value) => value.map((q, i) => {
3371
1901
  });
3372
1902
  /** Whether the eval's matrix varies each factor a pairwise question can vary. */
3373
1903
  const variesBy = (r) => ({
3374
- environment: (r.compare?.variants.length ?? 0) > 0,
1904
+ environment: r.compare ? r.compare.variants.length > 0 : r.environments.length > 1,
3375
1905
  harness: r.harness.length > 1
3376
1906
  });
3377
1907
  const FACTOR_FIX = {
3378
- environment: (_, where) => `varies the environment (across: 'environment'), but ${where} runs one environment, so it has no pairs to ask. Add compare: { variants: [...] }`,
3379
- harness: (r, where) => `varies the harness (across: 'harness'), but ${where} runs one harness (${r.harness.map(harnessId).join(", ")}), so it has no pairs to ask. Run it on two or more, harness: [a, b]`
1908
+ environment: (_, where) => `varies the environment (across: 'environment'), but ${where} runs one environment, so it has no pairs to ask. Run it in two or more, environments: [a, b], or compare: { control, variants: [...] }`,
1909
+ harness: (r, where) => `varies the harness (across: 'harness'), but ${where} runs one harness (${r.harness.map(harnessId).join(", ")}), so it has no pairs to ask. Run it on two or more, harnesses: [a, b]`
3380
1910
  };
3381
1911
  function normalizeCompare(value) {
3382
1912
  if (value === void 0) return void 0;
3383
1913
  return {
3384
- ...value.baseline ? { baseline: value.baseline } : {
3385
- baseline: emptyEnvironment(EMPTY_BASELINE),
3386
- from: "empty"
3387
- },
1914
+ control: value.control,
3388
1915
  variants: value.variants,
3389
1916
  gate: value.gate ?? "variants",
3390
1917
  ...value.minLift !== void 0 ? { minLift: value.minLift } : {},
3391
- ...value.harness ? { harness: value.harness.map((h) => typeof h === "string" ? h : harnessId(h)) } : {},
1918
+ ...value.harnesses ? { harnesses: value.harnesses.map((h) => typeof h === "string" ? h : harnessId(h)) } : {},
3392
1919
  of: value.variants.length
3393
1920
  };
3394
1921
  }
@@ -3403,9 +1930,9 @@ function applyLevel(out, options, where, egress = "own") {
3403
1930
  [key]: where
3404
1931
  };
3405
1932
  if (options.egress !== void 0) out.egress[egress] = options.egress;
3406
- out.harness = applyHarness(out.harness, options.harness, where);
1933
+ out.harness = applyHarness(out.harness, options.harnesses, where);
3407
1934
  if (options.sandbox !== void 0) out.sandbox = options.sandbox;
3408
- if (options.environment !== void 0) out.environment = options.environment;
1935
+ out.environments = applyEnvironments(out.environments, options.environments, where);
3409
1936
  if (options.compare !== void 0) out.compare = normalizeCompare(options.compare);
3410
1937
  if (options.runs !== void 0) {
3411
1938
  const { runs, bestOf } = runsOf(options.runs);
@@ -3446,6 +1973,7 @@ function cascade(base, levels, where, options = {}) {
3446
1973
  capture: { ...base.capture },
3447
1974
  question: { ...base.question },
3448
1975
  questions: [...base.questions],
1976
+ environments: [...base.environments],
3449
1977
  tags: [...base.tags],
3450
1978
  pin: [...base.pin]
3451
1979
  };
@@ -3455,28 +1983,33 @@ function cascade(base, levels, where, options = {}) {
3455
1983
  ...profile.defaults,
3456
1984
  judge: profileJudge(profile)
3457
1985
  }, `profiles.${profile.name}.defaults`, "inherited");
3458
- for (const { options: own, where: at } of levels) applyLevel(out, own, at);
3459
- if (out.compare?.from && out.environment) out.compare = {
3460
- ...out.compare,
3461
- baseline: out.environment,
3462
- from: "environment"
3463
- };
1986
+ let compareAt;
1987
+ for (const { options: own, where: at } of levels) {
1988
+ if (compareAt && own.environments !== void 0) throw new UsageError(`${at} sets environments, but ${compareAt} sets compare, which runs its control in their place. Set compare: { control, variants: [...] } on ${at} instead, or move ${at} out of ${compareAt}.`);
1989
+ applyLevel(out, own, at);
1990
+ if (own.compare !== void 0) compareAt = at;
1991
+ }
3464
1992
  if (out.questions.length) out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
3465
1993
  checkUnique(out.harness, where);
3466
- const harnessFromProfile = profile?.defaults.harness !== void 0 && !levels.some((l) => l.options.harness);
1994
+ if (!out.compare) {
1995
+ const names = out.environments.map((e) => e.name);
1996
+ const dupe = names.find((n, i) => names.indexOf(n) !== i);
1997
+ if (dupe) throw new UsageError(`environments in ${where} lists two environments named ${dupe}. Give each its own name.`);
1998
+ }
1999
+ const harnessFromProfile = profile?.defaults.harnesses !== void 0 && !levels.some((l) => l.options.harnesses);
3467
2000
  const authored = variesBy(out);
3468
2001
  if (harnessFromProfile) authored.harness ||= base.harness.length > 1;
3469
2002
  for (const q of options.skipped ? [] : out.questions) for (const factor of pairFactors(q.across)) {
3470
2003
  if (authored[factor]) continue;
3471
2004
  throw new UsageError(`questions "${q.question}" in ${where} ${FACTOR_FIX[factor](out, where)}, or leave out across to ask each attempt on its own.`);
3472
2005
  }
3473
- if (profile?.defaults.harness !== void 0 && levels.some((l) => l.options.harness)) notes.push("harness set by the eval");
2006
+ if (profile?.defaults.harnesses !== void 0 && levels.some((l) => l.options.harnesses)) notes.push("harnesses set by the eval");
3474
2007
  const ids = out.harness.map(harnessId);
3475
- const missing = (out.compare?.harness)?.filter((id) => !ids.includes(id)) ?? [];
3476
- if (missing.length) throw new UsageError(`compare.harness lists ${missing.join(", ")}, which ${where} doesn't run (it runs ${ids.join(", ")}).`);
2008
+ const missing = (out.compare?.harnesses)?.filter((id) => !ids.includes(id)) ?? [];
2009
+ if (missing.length) throw new UsageError(`compare.harnesses lists ${missing.join(", ")}, which ${where} doesn't run (it runs ${ids.join(", ")}).`);
3477
2010
  for (const v of out.compare?.variants ?? []) {
3478
- if (v.name === out.compare?.baseline.name) throw new UsageError(out.compare.from === "empty" ? `compare in ${where} lists a variant named ${v.name}, the name of the empty control a compare without a baseline or environment runs. Rename the variant, or set compare.baseline.` : out.compare.from === "environment" ? `compare in ${where} lists ${v.name}, which is the control: compare has no baseline, so the eval's environment is. List only the other environments.` : `compare in ${where} lists ${v.name}, which is the baseline. List only the other environments.`);
3479
- const control = out.compare?.baseline;
2011
+ if (v.name === out.compare?.control.name) throw new UsageError(`compare in ${where} lists ${v.name}, which is the control. List only the other environments.`);
2012
+ const control = out.compare?.control;
3480
2013
  const shared = control && sameWorkspaceOnly(control, v);
3481
2014
  if (control && shared) {
3482
2015
  const at = shared.ref ? `at ref "${shared.ref}"` : "with no ref (the repository HEAD)";
@@ -3493,13 +2026,13 @@ function cascade(base, levels, where, options = {}) {
3493
2026
  const tag = `profile ${profile.name}`;
3494
2027
  const pinned = (key) => out.pin.includes(key);
3495
2028
  let comparedLeft = true;
3496
- if (limit.harness && !pinned("harness")) {
3497
- const kept = out.harness.filter((h) => limit.harness?.includes(harnessId(h)));
3498
- if (kept.length === 0) skip = `no harness left: ${tag} allows ${limit.harness.join(", ") || "none"}`;
2029
+ if (limit.harnesses && !pinned("harnesses")) {
2030
+ const kept = out.harness.filter((h) => limit.harnesses?.includes(harnessId(h)));
2031
+ if (kept.length === 0) skip = `no harness left: ${tag} allows ${limit.harnesses.join(", ") || "none"}`;
3499
2032
  else if (kept.length < out.harness.length) {
3500
2033
  notes.push(`harnesses ${out.harness.length} → ${kept.length} (${tag})`);
3501
2034
  out.harness = kept;
3502
- const scope = out.compare?.harness;
2035
+ const scope = out.compare?.harnesses;
3503
2036
  if (out.compare?.variants.length && scope && !kept.some((h) => scope.includes(harnessId(h)))) {
3504
2037
  notes.push(`compare off: no compared harness left (${tag})`);
3505
2038
  out.compare = {
@@ -3565,6 +2098,8 @@ function cascade(base, levels, where, options = {}) {
3565
2098
  const set = from === where ? "" : ` from ${from}`;
3566
2099
  throw new UsageError(`${where} compares environments, so it can't take runs: { bestOf: ${out.bestOf} }${set}: a lift needs the same number of attempts on each side, and best-of stops each cell on its own. Set a fixed count on it, e.g. runs: 4.`);
3567
2100
  }
2101
+ if (out.gate === "observe" && out.compare?.minLift !== void 0 && out.compare.variants.length) throw new UsageError(`${where} has gate: 'observe', so its compare can't take minLift: ${out.compare.minLift}: minLift gates the lift, and nothing in it gates. Drop minLift, or gate the eval.`);
2102
+ if (out.gate === "observe" && out.bestOf !== void 0) throw new UsageError(`${where} has gate: 'observe', so it can't take runs: { bestOf: ${out.bestOf} }: best-of decides each cell by its majority. Set a fixed count, e.g. runs: 3.`);
3568
2103
  if (out.bestOf !== void 0) {
3569
2104
  if (out.gate !== void 0 && gateLabel(out.gate) !== gateLabel(MAJORITY_GATE)) notes.push(`gate ${gateLabel(out.gate)} → majority (runs.bestOf)`);
3570
2105
  out.gate = MAJORITY_GATE;
@@ -3595,14 +2130,14 @@ function defaultRuns(out, notes, options) {
3595
2130
  notes.push(`runs ${COMPARE_DEFAULT_RUNS} (compare default)`);
3596
2131
  }
3597
2132
  /**
3598
- * Refuses a profile's `limit.harness` id that no collected eval runs on: it keeps nothing, and a
2133
+ * Refuses a profile's `limit.harnesses` id that no collected eval runs on: it keeps nothing, and a
3599
2134
  * limit that keeps nothing skips every eval. `harnesses` are the ids every eval in the loaded
3600
2135
  * files runs on before the limit, whatever the tag and name filters pick. When a path filter
3601
2136
  * loaded only some files (`partial`), an id may name a harness of the others, so only a limit
3602
2137
  * none of whose ids match is refused.
3603
2138
  */
3604
2139
  function checkLimitHarness(profile, configFile, collected) {
3605
- const ids = profile?.limit.harness;
2140
+ const ids = profile?.limit.harnesses;
3606
2141
  if (!profile || !ids?.length) return;
3607
2142
  const unknown = ids.filter((id) => !collected.harnesses.includes(id));
3608
2143
  if (!unknown.length || collected.partial && unknown.length < ids.length) return;
@@ -3610,71 +2145,59 @@ function checkLimitHarness(profile, configFile, collected) {
3610
2145
  const have = [...collected.harnesses].sort();
3611
2146
  const evals = collected.partial ? "eval in the selected files" : "eval";
3612
2147
  const the = collected.partial ? "the evals in the selected files" : "the evals";
3613
- throw new UsageError(`profiles.${profile.name}.limit.harness in ${configFile} lists ${listed}, which no ${evals} runs on. Harness ids ${the} run on: ${have.join(", ") || "none"}.`);
2148
+ throw new UsageError(`profiles.${profile.name}.limit.harnesses in ${configFile} lists ${listed}, which no ${evals} runs on. Harness ids ${the} run on: ${have.join(", ") || "none"}.`);
3614
2149
  }
3615
2150
  /**
3616
2151
  * The sandbox an attempt runs in: the resolved environment's (including an `.extend()`
3617
- * parent's), else the first variant's when this is an omitted-baseline empty control, else
3618
- * the cascaded option (`defineConfig` / profile defaults / group / eval), else implicit Docker.
3619
- * A named environment is a total override: the config environment's sandbox is not inherited.
2152
+ * parent's), else the cascaded option (`defineConfig` / profile defaults / group / eval), else
2153
+ * implicit Docker. A named environment is a total override: the config environment's sandbox is
2154
+ * not inherited.
3620
2155
  */
3621
- const attemptSandbox = (resolved, env) => {
3622
- const own = env.layers.findLast((l) => l.spec.sandbox)?.spec.sandbox;
3623
- if (own) return own;
3624
- const variant = resolved.compare?.from === "empty" && env.name === "baseline" ? resolved.compare.variants[0] : void 0;
3625
- return sandboxOf(variant?.layers ?? [], resolved.sandbox);
3626
- };
2156
+ const attemptSandbox = (resolved, env) => env.layers.findLast((l) => l.spec.sandbox)?.spec.sandbox ?? sandboxOf([], resolved.sandbox);
3627
2157
  /**
3628
2158
  * Which level set this attempt's sandbox: the cascade source of the environment that named one
3629
- * (an `.extend()` child still names that environment), else the first variant when this is an
3630
- * omitted-baseline empty control, else the cascaded `sandbox` option, else the config (implicit
3631
- * Docker).
2159
+ * (an `.extend()` child still names that environment), else the cascaded `sandbox` option, else
2160
+ * the config (implicit Docker).
3632
2161
  */
3633
2162
  function sandboxSetBy(planned, resolved, configFile) {
3634
- if (!planned.environment.layers.some((l) => l.spec.sandbox)) {
3635
- if (resolved.compare?.from === "empty" && planned.cell.role === "control" && resolved.compare.variants[0]?.layers.some((l) => l.spec.sandbox)) return resolved.sources?.compare ?? configFile;
3636
- return resolved.sources?.sandbox ?? configFile;
3637
- }
3638
- if (resolved.compare) {
3639
- if (planned.cell.role === "control") {
3640
- if (resolved.compare.from === "environment") return resolved.sources?.environment ?? configFile;
3641
- if (resolved.compare.from === "empty") return configFile;
3642
- return resolved.sources?.compare ?? configFile;
3643
- }
3644
- return resolved.sources?.compare ?? configFile;
3645
- }
3646
- return resolved.sources?.environment ?? configFile;
2163
+ if (!planned.environment.layers.some((l) => l.spec.sandbox)) return resolved.sources?.sandbox ?? configFile;
2164
+ return (resolved.compare ? resolved.sources?.compare : resolved.sources?.environments) ?? configFile;
3647
2165
  }
3648
2166
  /**
3649
2167
  * Per harness: the control, then each variant it compares on; then run numbers. `vertex` is the
3650
2168
  * run's `vertex()` settings, for each harness that resolved to Vertex AI.
3651
2169
  */
2170
+ const cellTraits = (env) => {
2171
+ const traits = traitsOf(env.layers);
2172
+ return Object.keys(traits).length ? { traits } : {};
2173
+ };
3652
2174
  function expand(resolved, vertex) {
3653
2175
  const { compare } = resolved;
3654
- const control = applied(compare ? compare.baseline : resolved.environment);
3655
- const policy = resolved.gate ?? "all";
2176
+ const controls = (compare ? [compare.control] : resolved.environments.length ? resolved.environments : [void 0]).map((e) => applied(e));
2177
+ const policy = resolved.gate === "observe" ? "none" : resolved.gate ?? "all";
3656
2178
  const gateIf = (gated) => gated ? policy : "none";
3657
2179
  const attempts = [];
3658
2180
  for (const harness of resolved.harness) {
3659
2181
  const hid = harnessId(harness);
3660
2182
  const onVertex = harness.gateway === "vertex" && harness.kind !== "fake" ? vertex : void 0;
3661
- const cells = [(() => {
3662
- const sandbox = attemptSandbox(resolved, control);
2183
+ const cells = controls.map((env) => {
2184
+ const sandbox = attemptSandbox(resolved, env);
3663
2185
  const sid = sandboxId(sandbox);
3664
2186
  return {
3665
- env: control,
2187
+ env,
3666
2188
  sandbox,
3667
2189
  cell: {
3668
- key: cellKey(hid, control.name, sid),
2190
+ key: cellKey(hid, env.name, sid),
3669
2191
  harness: hid,
3670
- environment: control.name,
2192
+ environment: env.name,
3671
2193
  sandbox: sid,
3672
- role: "control"
2194
+ role: "control",
2195
+ ...cellTraits(env)
3673
2196
  },
3674
2197
  gate: gateIf(!compare || compare.variants.length === 0 || compare.gate === "all")
3675
2198
  };
3676
- })()];
3677
- if (compare && (!compare.harness || compare.harness.includes(hid))) for (const v of compare.variants) {
2199
+ });
2200
+ if (compare && (!compare.harnesses || compare.harnesses.includes(hid))) for (const v of compare.variants) {
3678
2201
  const env = applied(v);
3679
2202
  const sandbox = attemptSandbox(resolved, env);
3680
2203
  const sid = sandboxId(sandbox);
@@ -3687,13 +2210,14 @@ function expand(resolved, vertex) {
3687
2210
  environment: env.name,
3688
2211
  sandbox: sid,
3689
2212
  role: "variant",
3690
- variant: v.name
2213
+ variant: v.name,
2214
+ ...cellTraits(env)
3691
2215
  },
3692
- gate: gateIf(compare.gate !== "none")
2216
+ gate: gateIf(compare.gate !== "observe")
3693
2217
  });
3694
2218
  }
3695
2219
  for (const { env, sandbox, cell, gate } of cells) {
3696
- const axes = [resolved.harness.length > 1 ? hid : "", compare ? cell.role === "control" ? "control" : env.name : ""].filter(Boolean);
2220
+ const axes = [resolved.harness.length > 1 ? hid : "", compare ? cell.role === "control" ? "control" : env.name : controls.length > 1 ? env.name : ""].filter(Boolean);
3697
2221
  for (let n = 1; n <= resolved.runs; n++) {
3698
2222
  const label = [axes.join(" · "), resolved.runs > 1 ? `#${n}` : ""].filter(Boolean).join(" ");
3699
2223
  attempts.push({
@@ -3730,14 +2254,16 @@ function environmentWarnings(attempts) {
3730
2254
  }
3731
2255
  if (s.source === "registry" && !s.version) out.add(`skill.registry(${s.id}) has no version: results aren't comparable over time. Pin one (the resolved commit is recorded either way).`);
3732
2256
  }
2257
+ const tag = a.sandbox.kind === "tart" ? tartTagWarning(a.sandbox.image) : null;
2258
+ if (tag) out.add(tag);
3733
2259
  }
3734
2260
  return [...out];
3735
2261
  }
3736
- /** Secrets a Docker attempt would get as real values, because they have no `hosts`. */
2262
+ /** Secrets an isolated sandbox's attempt would get as real values, because they have no `hosts`. */
3737
2263
  function unbrokered(env, attempts) {
3738
2264
  const out = /* @__PURE__ */ new Map();
3739
2265
  for (const a of attempts) {
3740
- if (a.sandbox.kind !== "docker") continue;
2266
+ if (!capabilitiesOf(a.sandbox).isolated) continue;
3741
2267
  for (const s of unbrokeredSecrets(env, a.environment.layers)) out.set(s.label, s);
3742
2268
  }
3743
2269
  return [...out.values()];
@@ -3746,12 +2272,21 @@ function unbrokered(env, attempts) {
3746
2272
  const reachesSandbox = (s) => !s.optional || Boolean(process.env[s.from]);
3747
2273
  /** `GEMINI_API_KEY → generativelanguage.googleapis.com` for each secret with `hosts`. */
3748
2274
  const withHosts = (env, attempt) => [...new Set(plannedSecrets(env, attempt.environment.layers).flatMap((s) => s.hosts ? [`${s.label} → ${s.hosts.join(", ")}`] : []))];
3749
- /** The secrets an attempt brokers: only Docker's egress gateway swaps a placeholder for its value. */
3750
- const brokeredIn = (env, attempt) => attempt.sandbox.kind === "docker" ? withHosts(env, attempt) : [];
3751
- /** The plan's note for secrets with `hosts` on the fake sandbox, which only ever has their placeholder. */
3752
- function unbrokeredOnFake(env, attempts) {
3753
- const names = [...new Set(attempts.flatMap((a) => a.sandbox.kind === "fake" ? withHosts(env, a) : []))];
3754
- return names.length ? [`not brokered (fake sandbox, placeholder only): ${names.join("; ")}`] : [];
2275
+ /** The secrets an attempt brokers: only a sandbox that brokers swaps a placeholder for its value. */
2276
+ const brokeredIn = (env, attempt) => capabilitiesOf(attempt.sandbox).brokers ? withHosts(env, attempt) : [];
2277
+ /** The plan's note for secrets with `hosts` on a sandbox that can't broker them, which only ever has their placeholder. */
2278
+ function unbrokeredHere(env, attempts) {
2279
+ const byProvider = /* @__PURE__ */ new Map();
2280
+ for (const a of attempts) {
2281
+ if (capabilitiesOf(a.sandbox).brokers) continue;
2282
+ const names = withHosts(env, a);
2283
+ if (!names.length) continue;
2284
+ const provider = sandboxProvider(a.sandbox);
2285
+ const seen = byProvider.get(provider) ?? /* @__PURE__ */ new Set();
2286
+ for (const name of names) seen.add(name);
2287
+ byProvider.set(provider, seen);
2288
+ }
2289
+ return [...byProvider].map(([provider, names]) => `not brokered (${provider} sandbox, placeholder only): ${[...names].join("; ")}`);
3755
2290
  }
3756
2291
  /** The host variables an attempt's secrets read, brokered or not. */
3757
2292
  const secretVarsIn = (env, attempt) => [...new Set(plannedSecrets(env, attempt.environment.layers).map((s) => s.from))];
@@ -3776,11 +2311,12 @@ function sandboxWarnings(attempts, resolved, origin) {
3776
2311
  const out = [];
3777
2312
  const seen = /* @__PURE__ */ new Set();
3778
2313
  for (const a of attempts) {
3779
- if (a.sandbox.kind !== "docker") continue;
2314
+ const privileges = sandboxKind(a.sandbox).privileges(a.sandbox);
2315
+ if (!privileges.root && !privileges.capabilities.length) continue;
3780
2316
  const where = sandboxSetBy(a, resolved, origin.configFile);
3781
2317
  const level = levelOf(where);
3782
2318
  const file = level === "eval" || level === "group" ? origin.file : null;
3783
- if (a.sandbox.root) {
2319
+ if (privileges.root) {
3784
2320
  const message = `running as root in the sandbox: ${where} (docker({ root: true }))`;
3785
2321
  const key = `${file}\0${message}`;
3786
2322
  if (!seen.has(key)) {
@@ -3794,8 +2330,8 @@ function sandboxWarnings(attempts, resolved, origin) {
3794
2330
  });
3795
2331
  }
3796
2332
  }
3797
- if (a.sandbox.capabilities?.length) {
3798
- const message = `extra capabilities in the sandbox: ${where} (docker({ capabilities: [${a.sandbox.capabilities.map((c) => `'${c}'`).join(", ")}] }))`;
2333
+ if (privileges.capabilities.length) {
2334
+ const message = `extra capabilities in the sandbox: ${where} (docker({ capabilities: [${privileges.capabilities.map((c) => `'${c}'`).join(", ")}] }))`;
3799
2335
  const key = `${file}\0${message}`;
3800
2336
  if (!seen.has(key)) {
3801
2337
  seen.add(key);
@@ -3804,7 +2340,7 @@ function sandboxWarnings(attempts, resolved, origin) {
3804
2340
  level,
3805
2341
  where,
3806
2342
  file,
3807
- capabilities: [...a.sandbox.capabilities],
2343
+ capabilities: [...privileges.capabilities],
3808
2344
  message
3809
2345
  });
3810
2346
  }
@@ -3819,7 +2355,7 @@ function evalPlan(r, attempts, origin) {
3819
2355
  const notes = [
3820
2356
  ...r.notes,
3821
2357
  ...capture ? [capture] : [],
3822
- ...unbrokeredOnFake(resolved.env, attempts)
2358
+ ...unbrokeredHere(resolved.env, attempts)
3823
2359
  ];
3824
2360
  return {
3825
2361
  harnesses: skip ? 0 : resolved.harness.length,
@@ -3827,10 +2363,7 @@ function evalPlan(r, attempts, origin) {
3827
2363
  run: resolved.compare.variants.length,
3828
2364
  of: resolved.compare.of
3829
2365
  } : null,
3830
- ...resolved.compare ? { compare: {
3831
- baseline: resolved.compare.baseline.name,
3832
- ...resolved.compare.from ? { from: resolved.compare.from } : {}
3833
- } } : {},
2366
+ ...resolved.compare ? { compare: { baseline: resolved.compare.control.name } } : {},
3834
2367
  ...!skip && resolved.questions.length ? { questions: resolved.questions } : {},
3835
2368
  runs: resolved.runs,
3836
2369
  ...resolved.bestOf !== void 0 ? { bestOf: resolved.bestOf } : {},
@@ -4065,9 +2598,9 @@ function modelGatewayOrNone(model, pin, scope) {
4065
2598
  const CONFIG_KEYS = [
4066
2599
  "include",
4067
2600
  "gateway",
4068
- "harness",
2601
+ "harnesses",
4069
2602
  "sandbox",
4070
- "environment",
2603
+ "environments",
4071
2604
  "runs",
4072
2605
  "gate",
4073
2606
  "env",
@@ -4097,9 +2630,9 @@ const PROFILE_KEYS = [
4097
2630
  "gateway"
4098
2631
  ];
4099
2632
  const DEFAULTS_KEYS = [
4100
- "harness",
2633
+ "harnesses",
4101
2634
  "sandbox",
4102
- "environment",
2635
+ "environments",
4103
2636
  "runs",
4104
2637
  "gate",
4105
2638
  "env",
@@ -4113,7 +2646,8 @@ const DEFAULTS_KEYS = [
4113
2646
  const LIMIT_KEYS = [
4114
2647
  "runs",
4115
2648
  "compare",
4116
- "harness"
2649
+ "harnesses",
2650
+ "traits"
4117
2651
  ];
4118
2652
  /**
4119
2653
  * The config: a plain object, or a callback that destructures the helpers it needs, e.g.
@@ -4207,7 +2741,7 @@ function parseYamlConfig(text, file) {
4207
2741
  return doc.toJS() ?? {};
4208
2742
  }
4209
2743
  const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
4210
- /** A config's Dockerfile that isn't there, named by its field. */
2744
+ /** A config's host file (a Dockerfile, an SSH key) that isn't there, named by its field. */
4211
2745
  const missingIn = (file) => (field, full) => {
4212
2746
  throw new UsageError(`${field} in ${file}: ${full} doesn't exist. Paths in ${file} are relative to its directory.`);
4213
2747
  };
@@ -4223,7 +2757,10 @@ function withHostPaths(config, root, file, { environments = false } = {}) {
4223
2757
  if (!isRecord(value)) return value;
4224
2758
  const out = { ...value };
4225
2759
  if ("sandbox" in value) out.sandbox = sandboxPaths(value.sandbox, root, `${prefix}sandbox`, missing);
4226
- if (environments && "environment" in value) out.environment = environmentPaths(value.environment, root, `${prefix}environment`, missing);
2760
+ if (environments && "environments" in value) {
2761
+ const field = `${prefix}environments`;
2762
+ out.environments = Array.isArray(value.environments) ? value.environments.map((e, i) => environmentPaths(e, root, `${field}[${i}]`, missing)) : environmentPaths(value.environments, root, field, missing);
2763
+ }
4227
2764
  return out;
4228
2765
  };
4229
2766
  const out = options(config, "");
@@ -4242,10 +2779,15 @@ function withHostPaths(config, root, file, { environments = false } = {}) {
4242
2779
  function configFromData(data) {
4243
2780
  const withEnvironment = (options) => {
4244
2781
  if (!isObjectLike(options)) return options;
4245
- const { environment } = options;
4246
- return isEnvironment(environment) ? {
2782
+ const { environments } = options;
2783
+ const from = (e) => isEnvironment(e) ? environmentFrom(e) : e;
2784
+ if (Array.isArray(environments)) return {
4247
2785
  ...options,
4248
- environment: environmentFrom(environment)
2786
+ environments: environments.map(from)
2787
+ };
2788
+ return isEnvironment(environments) ? {
2789
+ ...options,
2790
+ environments: from(environments)
4249
2791
  } : options;
4250
2792
  };
4251
2793
  const config = withEnvironment(data);
@@ -4271,10 +2813,10 @@ function configFromData(data) {
4271
2813
  * let through, such as a `null` or a misspelled nested key.
4272
2814
  */
4273
2815
  function resolveConfigData(input, root, file) {
4274
- if (!isRecord(input)) throw new UsageError(`${file} must be a mapping of config keys, like harness: and judge: (got ${JSON.stringify(input)})`);
2816
+ if (!isRecord(input)) throw new UsageError(`${file} must be a mapping of config keys, like harnesses: and judge: (got ${JSON.stringify(input)})`);
4275
2817
  const data = withHostPaths(input, root, file, { environments: true });
4276
- const { environment, profiles } = data;
4277
- if ([environment, ...Object.values(typeof profiles === "object" && profiles ? profiles : {}).map((p) => p?.defaults?.environment)].some((e) => e !== void 0 && !isEnvironment(e))) {
2818
+ const { environments: environment, profiles } = data;
2819
+ if ([environment, ...Object.values(typeof profiles === "object" && profiles ? profiles : {}).map((p) => p?.defaults?.environments)].flat().some((e) => e !== void 0 && !isEnvironment(e))) {
4278
2820
  const problem = configDataProblem(data, file);
4279
2821
  if (problem) throw new UsageError(problem);
4280
2822
  }
@@ -4354,12 +2896,18 @@ function checkProfile(name, profile, file) {
4354
2896
  }
4355
2897
  if (p.limit !== void 0) {
4356
2898
  unknownKey(p.limit, LIMIT_KEYS, `${where}.limit`, file);
4357
- const { runs, compare, harness } = p.limit;
2899
+ const { runs, compare, harnesses: harness, traits } = p.limit;
4358
2900
  if (runs !== void 0 && !(Number.isInteger(runs) && runs >= 1)) throw new UsageError(`${where}.limit.runs in ${file} must be a whole number of at least 1 (got ${JSON.stringify(runs)})`);
4359
2901
  if (compare !== void 0 && compare !== false && !(Number.isInteger(compare?.maxVariants) && compare.maxVariants >= 0)) throw new UsageError(`${where}.limit.compare in ${file} must be false or { maxVariants: n }, e.g. { maxVariants: 1 } (got ${JSON.stringify(compare)})`);
4360
2902
  if (harness !== void 0) {
4361
2903
  const bad = Array.isArray(harness) ? harness.find((h) => typeof h !== "string") : harness;
4362
- if (bad !== void 0) throw new UsageError(`${where}.limit.harness in ${file} can only list harness ids to allow (got ${describeValue(bad)}). To add harnesses, use ${where}.defaults.harness.`);
2904
+ if (bad !== void 0) throw new UsageError(`${where}.limit.harnesses in ${file} can only list harness ids to allow (got ${describeValue(bad)}). To add harnesses, use ${where}.defaults.harnesses.`);
2905
+ }
2906
+ if (traits !== void 0) {
2907
+ const ok = (v) => typeof v === "string" || Array.isArray(v) && v.length > 0 && v.every((s) => typeof s === "string");
2908
+ const entries = typeof traits === "object" && traits !== null && !Array.isArray(traits) ? Object.entries(traits) : null;
2909
+ const bad = entries?.find(([, v]) => !ok(v));
2910
+ if (!entries || bad) throw new UsageError(`${where}.limit.traits in ${file} must map each trait key to a value or a list of values a cell's environment has, e.g. { lang: 'ts' } or { lang: ['ts', 'py'] } (got ${bad ? `${bad[0]}: ${JSON.stringify(bad[1])}` : JSON.stringify(traits)})`);
4363
2911
  }
4364
2912
  }
4365
2913
  if (p.tags !== void 0 && (!Array.isArray(p.tags) || p.tags.some((t) => typeof t !== "string"))) throw new UsageError(`${where}.tags in ${file} must be a list of tag filters, e.g. tags: ['smoke'] (got ${JSON.stringify(p.tags)})`);
@@ -4473,8 +3021,8 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
4473
3021
  const artifacts = user.history?.artifacts ?? "all";
4474
3022
  if (!ARTIFACTS.includes(artifacts)) throw new UsageError(`history.artifacts in ${file} is ${oneOf(ARTIFACTS)} (got ${JSON.stringify(artifacts)})`);
4475
3023
  const historyDir = resolve(root, user.history?.dir ?? join(evalRoot(include), ".history"));
4476
- const harness = user.harness === void 0 ? [DEFAULT_HARNESS] : [user.harness].flat();
4477
- if (harness.length === 0) throw new UsageError(`harness is an empty list in ${file}. Give at least one, e.g. harness: ${HARNESS_EXAMPLE}`);
3024
+ const harness = user.harnesses === void 0 ? [DEFAULT_HARNESS] : [user.harnesses].flat();
3025
+ if (harness.length === 0) throw new UsageError(`harnesses is an empty list in ${file}. Give at least one, e.g. harnesses: ${HARNESS_EXAMPLE}`);
4478
3026
  checkHarnesses(harness, file);
4479
3027
  const profiles = {};
4480
3028
  if (user.profiles !== void 0) {
@@ -4497,7 +3045,7 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
4497
3045
  include,
4498
3046
  harness,
4499
3047
  sandbox: user.sandbox ?? docker(),
4500
- environment: user.environment,
3048
+ environments: user.environments === void 0 ? [] : [user.environments].flat(),
4501
3049
  runs: user.runs,
4502
3050
  gate: user.gate,
4503
3051
  env: user.env ?? {},
@@ -4639,6 +3187,11 @@ function runModelGateway(config, profile, model, judge = false) {
4639
3187
  */
4640
3188
  function missingCredentials(config, env = process.env, options = {}) {
4641
3189
  const missing = [];
3190
+ for (const s of options.sandboxes ?? []) {
3191
+ if (s.kind !== "boat" || env[s.apiKey.from]) continue;
3192
+ const line = `${s.apiKey.from} (for boat())`;
3193
+ if (!missing.includes(line)) missing.push(line);
3194
+ }
4642
3195
  const keys = new Set((options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []));
4643
3196
  for (const key of keys) if (!env[key]) missing.push(key);
4644
3197
  const judge = runJudgeModel(config, options.profile);
@@ -4696,6 +3249,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
4696
3249
  return at ? [`${name} (character ${at})`] : [];
4697
3250
  });
4698
3251
  }
4699
- export { directOnCloud as $, evalPlan as A, agentMcp as At, profileJudge as B, refusedPlugins as Bt, openCodeModelOn as C, execOptionsProblem as Ct, cascade as D, AGENTS as Dt, brokeredIn as E, servedEffortOf as Et, matchesTags as F, expectedBlockedFor as Ft, sandboxSetBy as G, judgeProblem as Gt, runsOf as H, INPUT_KEYS as Ht, plainHeaderError as I, isAgentKind as It, unbrokeredError as J, decideState as Jt, secretVarsIn as K, artifactRefsOf as Kt, plainHeaders as L, keyRejected as Lt, harnessId as M, brokeringFor as Mt, levelOf as N, claudeMcpServers as Nt, checkLimitHarness as O, agentAuth as Ot, majorityOf as P, credentialOf as Pt, ciRunner as Q, plannedSandboxIds as R, layoutOf as Rt, fallbackHint as S, attemptRequestProblem as St, routedHarness as T, resolvedEffortOf as Tt, sandboxId as U, NO_JUDGE_INPUT as Ut, refuseIllegalCompare as V, writePluginFiles as Vt, sandboxProvider as W, inputsOn as Wt, DEFAULT_EGRESS_ALLOW as X, resolveJudgeArtifacts as Xt, DEFAULT_EGRESS as Y, filePartsOf as Yt, allowedHosts as Z, fellBackToDirect as _, observedHere as _t, configPathFor as a, createFakeSandbox as at, resolveModelGateway as b, CAPTURE_NAME as bt, findConfig as c, parseProbe as ct, malformedCredentials as d, STOPPED_FIRST as dt, egressLabel as et, missingCredentials as f, UNRECORDED_QUESTION_STATE as ft, selectProfile as g, resolveStateOptions as gt, runJudgeModel as h, mergeJudge as ht, configLabel as i, resolveOutput as it, expand as j, authHint as jt, checkOptions as k, agentFor as kt, hostCapacity as l, probeScript as lt, resolveConcurrency as m, resolveQuestionState as mt, analysisModelOf as n, isDirect as nt, defaultConcurrency as o, awaitListening as ot, requireConfig as p, questionShows as pt, unbrokered as q, badJudgeArtifacts as qt, configFilesPhrase as r, unenforceable as rt, defineConfig as s, hostListening as st, CONFIG_FILES as t, gatewayPolicy as tt, loadConfig as u, servePort as ut, gatewayScope as v, observedName as vt, resolveGateway as w, harnessEffort as wt, effortRefusedOn as x, CAPTURE_NAME_RULE as xt, modelGatewayOrNone as y, observer as yt, plannedSandboxes as z, refusedMcp as zt };
3252
+ export { runsOf as $, brokeredIn as A, execOptionsProblem as At, expand as B, decideState as Bt, openCodeModelOn as C, resolveStateOptions as Ct, EVAL_OPTION_KEYS as D, CAPTURE_NAME as Dt, COMPARE_DEFAULT_RUNS as E, observer as Et, checkHarnesses as F, NO_JUDGE_INPUT as Ft, mergeSnapshot as G, levelOf as H, resolveJudgeArtifacts as Ht, checkLimitHarness as I, inputsOn as It, plainHeaders as J, plainHeaderError as K, checkOptions as L, judgeProblem as Lt, cellKey as M, resolvedEffortOf as Mt, checkCapture as N, servedEffortOf as Nt, MAJORITY_GATE as O, CAPTURE_NAME_RULE as Ot, checkGate as P, INPUT_KEYS as Pt, refuseIllegalCompare as Q, environmentWarnings as R, artifactRefsOf as Rt, fallbackHint as S, mergeJudge as St, routedHarness as T, observedName as Tt, majorityOf as U, harnessId as V, filePartsOf as Vt, matchesTags as W, plannedSandboxes as X, plannedSandboxIds as Y, profileJudge as Z, fellBackToDirect as _, resolveOutput as _t, configPathFor as a, unbrokered as at, resolveModelGateway as b, questionShows as bt, findConfig as c, DEFAULT_EGRESS as ct, malformedCredentials as d, ciRunner as dt, sandboxId as et, missingCredentials as f, directOnCloud as ft, selectProfile as g, unenforceable as gt, runJudgeModel as h, isDirect as ht, configLabel as i, secretVarsIn as it, cascade as j, harnessEffort as jt, attemptSandbox as k, attemptRequestProblem as kt, hostCapacity as l, DEFAULT_EGRESS_ALLOW as lt, resolveConcurrency as m, gatewayPolicy as mt, analysisModelOf as n, sandboxSetBy as nt, defaultConcurrency as o, unbrokeredError as ot, requireConfig as p, egressLabel as pt, plainHeaderWarning as q, configFilesPhrase as r, sandboxWarnings as rt, defineConfig as s, unbrokeredWarning as st, CONFIG_FILES as t, sandboxProvider as tt, loadConfig as u, allowedHosts as ut, gatewayScope as v, STOPPED_FIRST as vt, resolveGateway as w, observedHere as wt, effortRefusedOn as x, resolveQuestionState as xt, modelGatewayOrNone as y, UNRECORDED_QUESTION_STATE as yt, evalPlan as z, badJudgeArtifacts as zt };
4700
3253
 
4701
- //# sourceMappingURL=config-C2UtS9vy.js.map
3254
+ //# sourceMappingURL=config-SvK8pjEK.js.map