reeval 0.17.2 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (403) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +5 -5
  3. package/dist/{adapter-DyvDj4ZN.js → adapter-C8xPuXYq.js} +5 -4
  4. package/dist/adapter-C8xPuXYq.js.map +1 -0
  5. package/dist/{agent-v036GEgG.js → agent-CBoLndM7.js} +30 -10
  6. package/dist/{agent-v036GEgG.js.map → agent-CBoLndM7.js.map} +1 -1
  7. package/dist/agents-C6zhkxp3.js +1151 -0
  8. package/dist/agents-C6zhkxp3.js.map +1 -0
  9. package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
  10. package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
  11. package/dist/{app-files-hpRT5lj_.js → app-files-DT5mJyV4.js} +4 -4
  12. package/dist/app-files-DT5mJyV4.js.map +1 -0
  13. package/dist/assemble-UPCRrtOz.js +1 -0
  14. package/dist/{assemble-Dlh6Xv3U.js → assemble-t6h018An.js} +194 -307
  15. package/dist/assemble-t6h018An.js.map +1 -0
  16. package/dist/{attempt-CtKGvaQR.js → attempt-CAdlwlNu.js} +3 -3
  17. package/dist/{attempt-CtKGvaQR.js.map → attempt-CAdlwlNu.js.map} +1 -1
  18. package/dist/{attempt-host-DErH-DYV.js → attempt-host-CIm1-11m.js} +18 -10
  19. package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-CIm1-11m.js.map} +1 -1
  20. package/dist/boat-BjLqFJIZ.js +108 -0
  21. package/dist/boat-BjLqFJIZ.js.map +1 -0
  22. package/dist/boat-CJCtzfB1.js +322 -0
  23. package/dist/boat-CJCtzfB1.js.map +1 -0
  24. package/dist/bridge/index.mjs +300 -78
  25. package/dist/capture-DoxFUN3J.js +1 -0
  26. package/dist/{capture-Bdcx9Gap.js → capture-iBzFiC0j.js} +406 -625
  27. package/dist/capture-iBzFiC0j.js.map +1 -0
  28. package/dist/cases-Dy0mifal.js +1 -0
  29. package/dist/cases-z9E6DCd5.js +639 -0
  30. package/dist/cases-z9E6DCd5.js.map +1 -0
  31. package/dist/{cli-CGd51thd.js → cli-20Fpvs5H.js} +929 -576
  32. package/dist/cli-20Fpvs5H.js.map +1 -0
  33. package/dist/{cli-spec-ChkleEVm.js → cli-spec-Dsr79Qy1.js} +46 -18
  34. package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
  35. package/dist/cli.js +2 -2
  36. package/dist/{client-CF-DOjOU.js → client-KM4C5r5_.js} +111 -14
  37. package/dist/client-KM4C5r5_.js.map +1 -0
  38. package/dist/{command-CsfcwMyc.js → command-B5OyHpOu.js} +24 -81
  39. package/dist/command-B5OyHpOu.js.map +1 -0
  40. package/dist/{command-CtTdCF3B.js → command-BLNVDiIg.js} +10 -10
  41. package/dist/{command-CtTdCF3B.js.map → command-BLNVDiIg.js.map} +1 -1
  42. package/dist/{command-CyIb9BSC.js → command-CRh-cb6Z.js} +120 -55
  43. package/dist/command-CRh-cb6Z.js.map +1 -0
  44. package/dist/{command-TLmY-rYp.js → command-aUJvENwy.js} +18 -18
  45. package/dist/command-aUJvENwy.js.map +1 -0
  46. package/dist/{commands-C54S45d7.js → commands-CWy-WndG.js} +239 -92
  47. package/dist/commands-CWy-WndG.js.map +1 -0
  48. package/dist/{commands-jO9Jj35r.js → commands-CfdBX-mc.js} +1 -1
  49. package/dist/{config-ImY6AlRE.js → config-B9mB1kF5.js} +924 -1729
  50. package/dist/config-B9mB1kF5.js.map +1 -0
  51. package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
  52. package/dist/context-Bkv3ndjq.js.map +1 -0
  53. package/dist/{cost-e4IppZpJ.js → cost-BdRgySQl.js} +4 -4
  54. package/dist/{cost-e4IppZpJ.js.map → cost-BdRgySQl.js.map} +1 -1
  55. package/dist/cost-Cr26Alhq.js +1 -0
  56. package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BSn9xjmo.js} +3 -3
  57. package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
  58. package/dist/db-P2fXw4nx.js +2 -0
  59. package/dist/{db-sHGpaLo9.js → db-zgC1tvKB.js} +178 -74
  60. package/dist/db-zgC1tvKB.js.map +1 -0
  61. package/dist/debug-CF4p4Lp_.js +1 -0
  62. package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
  63. package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
  64. package/dist/{default-effort-fpdbYVKm.js → default-effort-BAy8ixd1.js} +1 -1
  65. package/dist/{default-effort-BZRa-SQP.js → default-effort-DGhQmu95.js} +8 -5
  66. package/dist/default-effort-DGhQmu95.js.map +1 -0
  67. package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
  68. package/dist/dist-CAFD1yfv.js.map +1 -0
  69. package/dist/{docker-CWuSRj7a.js → docker-Cqw-CyWz.js} +4 -4
  70. package/dist/{docker-CWuSRj7a.js.map → docker-Cqw-CyWz.js.map} +1 -1
  71. package/dist/{docker--2ytOHcu.js → docker-DAMnm3L9.js} +103 -73
  72. package/dist/docker-DAMnm3L9.js.map +1 -0
  73. package/dist/{docker-CIzHzwlC.js → docker-Dr4XeDc8.js} +1 -1
  74. package/dist/{doctor-eE65YpEv.js → doctor-B2PohneB.js} +86 -29
  75. package/dist/doctor-B2PohneB.js.map +1 -0
  76. package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
  77. package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
  78. package/dist/{environment-Cy3FVLR5.js → environment-DB8o3hGi.js} +80 -40
  79. package/dist/environment-DB8o3hGi.js.map +1 -0
  80. package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
  81. package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
  82. package/dist/{eval-source-BTsTRiBW.js → eval-source-D3sbAzt-.js} +23 -15
  83. package/dist/eval-source-D3sbAzt-.js.map +1 -0
  84. package/dist/evaluate-COdzf7Hl.d.ts +92 -0
  85. package/dist/{evaluate-CFFdyYZX.js → evaluate-CwKeN_fo.js} +472 -159
  86. package/dist/evaluate-CwKeN_fo.js.map +1 -0
  87. package/dist/files-B4KQUiM7.js +1 -0
  88. package/dist/files-BEwfjD8J.js +64 -0
  89. package/dist/files-BEwfjD8J.js.map +1 -0
  90. package/dist/{files-CQU08uh4.js → files-BRzV7aPh.js} +5 -5
  91. package/dist/{files-CQU08uh4.js.map → files-BRzV7aPh.js.map} +1 -1
  92. package/dist/{files-CIX-pLl4.js → files-BiFU16BW.js} +6 -6
  93. package/dist/{files-CIX-pLl4.js.map → files-BiFU16BW.js.map} +1 -1
  94. package/dist/files-C1Rk9FYA.js +1 -0
  95. package/dist/files-C3mCC8Mn.js +1 -0
  96. package/dist/{files-tnMgu7-X.js → files-Cl7BQ5gE.js} +55 -14
  97. package/dist/{files-tnMgu7-X.js.map → files-Cl7BQ5gE.js.map} +1 -1
  98. package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
  99. package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
  100. package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
  101. package/dist/files-DlQshsks.js.map +1 -0
  102. package/dist/files-DnlA1A7N.js +1 -0
  103. package/dist/gate-CgYdaprG.js +49 -0
  104. package/dist/gate-CgYdaprG.js.map +1 -0
  105. package/dist/gateway.d.ts +1 -1
  106. package/dist/{git-BWVkG4Yl.js → git-fDiJ8UIV.js} +3 -3
  107. package/dist/{git-BWVkG4Yl.js.map → git-fDiJ8UIV.js.map} +1 -1
  108. package/dist/harness-C2berMiM.js +105 -0
  109. package/dist/harness-C2berMiM.js.map +1 -0
  110. package/dist/{harness-Ch2P81sx.js → harness-DXv2gnaT.js} +68 -111
  111. package/dist/harness-DXv2gnaT.js.map +1 -0
  112. package/dist/harness.d.ts +1 -1
  113. package/dist/harness.js +1 -1
  114. package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
  115. package/dist/ids-C_advdJA.js.map +1 -0
  116. package/dist/{images-BVZHDf_z.js → images-BNgcR77K.js} +98 -10
  117. package/dist/images-BNgcR77K.js.map +1 -0
  118. package/dist/index.d.ts +23 -3
  119. package/dist/index.js +5 -5
  120. package/dist/init-4RPeD-K-.js +1 -0
  121. package/dist/{init-BWnMbUnc.js → init-DbuZVRrW.js} +8 -7
  122. package/dist/init-DbuZVRrW.js.map +1 -0
  123. package/dist/ink-list-BpfidQ1h.js +274 -0
  124. package/dist/ink-list-BpfidQ1h.js.map +1 -0
  125. package/dist/{invocations-BGRs1r_C.js → invocations-CxGNOlh0.js} +4 -4
  126. package/dist/{invocations-BGRs1r_C.js.map → invocations-CxGNOlh0.js.map} +1 -1
  127. package/dist/judge-COMXT1Hx.js +230 -0
  128. package/dist/judge-COMXT1Hx.js.map +1 -0
  129. package/dist/judge.d.ts +1 -1
  130. package/dist/judge.js +1 -1
  131. package/dist/judgments-CzgpsTqv.js +246 -0
  132. package/dist/judgments-CzgpsTqv.js.map +1 -0
  133. package/dist/kinds-C1oY3qyC.js +697 -0
  134. package/dist/kinds-C1oY3qyC.js.map +1 -0
  135. package/dist/{summary-zP3cCXDN.js → layout-Cm85tQYk.js} +77 -154
  136. package/dist/layout-Cm85tQYk.js.map +1 -0
  137. package/dist/list-DuThsE28.js +1348 -0
  138. package/dist/list-DuThsE28.js.map +1 -0
  139. package/dist/live-CJCK11nh.js +1 -0
  140. package/dist/{live-d_5a5iUv.js → live-CgYbO4Te.js} +24 -4
  141. package/dist/live-CgYbO4Te.js.map +1 -0
  142. package/dist/{live-ColieOai.js → live-CnGrRM6j.js} +8 -8
  143. package/dist/{live-ColieOai.js.map → live-CnGrRM6j.js.map} +1 -1
  144. package/dist/{matchers-B4ZtlJyP.js → matchers-CXLn-1jm.js} +4 -3
  145. package/dist/{matchers-B4ZtlJyP.js.map → matchers-CXLn-1jm.js.map} +1 -1
  146. package/dist/{model-DOXnWH65.js → model-047Dcd-a.js} +1 -1
  147. package/dist/model-Cv4GuTsZ.js +1 -0
  148. package/dist/{model-CFSjV-HJ.js → model-CwwkrCb7.js} +5 -5
  149. package/dist/{model-CFSjV-HJ.js.map → model-CwwkrCb7.js.map} +1 -1
  150. package/dist/{model-DBpdRXtO.js → model-DJjLw-0r.js} +43 -19
  151. package/dist/{model-DBpdRXtO.js.map → model-DJjLw-0r.js.map} +1 -1
  152. package/dist/{model-DOiCiMCD.js → model-V3mMXp6x.js} +213 -559
  153. package/dist/model-V3mMXp6x.js.map +1 -0
  154. package/dist/{node-CwP1G178.js → node-D7pAU--0.js} +34 -25
  155. package/dist/node-D7pAU--0.js.map +1 -0
  156. package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-BGiXt8_B.js} +107 -49
  157. package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
  158. package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-Cw1z8-sq.js} +2 -2
  159. package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
  160. package/dist/options-CHubslQf.js +1 -0
  161. package/dist/{gate-BoDKeuJu.js → pairs-BGb134Vu.js} +3 -49
  162. package/dist/pairs-BGb134Vu.js.map +1 -0
  163. package/dist/{patch-DvLr8wCZ.js → patch-DH4e_Gkd.js} +3 -3
  164. package/dist/{patch-DvLr8wCZ.js.map → patch-DH4e_Gkd.js.map} +1 -1
  165. package/dist/plain-BYUf_fUV.js +267 -0
  166. package/dist/plain-BYUf_fUV.js.map +1 -0
  167. package/dist/{plan-CIfOFhth.js → plan-Cgq9NCEX.js} +117 -45
  168. package/dist/plan-Cgq9NCEX.js.map +1 -0
  169. package/dist/{plugins-BEs2Am3L.js → plugins-BkmPxDiY.js} +11 -268
  170. package/dist/plugins-BkmPxDiY.js.map +1 -0
  171. package/dist/preflight-4gWSy0Pe.js +1 -0
  172. package/dist/{preflight-50Qw5e5K.js → preflight-BElE_q0u.js} +16 -9
  173. package/dist/preflight-BElE_q0u.js.map +1 -0
  174. package/dist/preflight-CtlloeOQ.js +68 -0
  175. package/dist/preflight-CtlloeOQ.js.map +1 -0
  176. package/dist/{preflight-BAIRsTpl.js → preflight-R-InZCQb.js} +1 -1
  177. package/dist/{prepare-8XhzhLHX.js → prepare-C7f5mPl5.js} +9 -9
  178. package/dist/{prepare-8XhzhLHX.js.map → prepare-C7f5mPl5.js.map} +1 -1
  179. package/dist/{prepare-8nvfDFQx.js → prepare-CzkAdbXm.js} +2 -2
  180. package/dist/{prices-C4BRuSe8.js → prices-5m3SzMZ3.js} +3 -3
  181. package/dist/{prices-C4BRuSe8.js.map → prices-5m3SzMZ3.js.map} +1 -1
  182. package/dist/{provenance-Cc8fFE7Z.js → provenance-zsc4--dP.js} +6 -5
  183. package/dist/provenance-zsc4--dP.js.map +1 -0
  184. package/dist/provider-BnUoz4KB.js +1 -0
  185. package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
  186. package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
  187. package/dist/{providers-DZSTlC9R.js → providers-B8xYMeah.js} +152 -9
  188. package/dist/providers-B8xYMeah.js.map +1 -0
  189. package/dist/providers-CVu8e0xS.js +1 -0
  190. package/dist/{query-DXpxaaTH.js → query-Bb3SYICk.js} +3 -3
  191. package/dist/{query-DXpxaaTH.js.map → query-Bb3SYICk.js.map} +1 -1
  192. package/dist/query-DIfkUcPh.js +1 -0
  193. package/dist/{questions-C_Y-oXW6.js → questions-CWlR9ffu.js} +41 -11
  194. package/dist/questions-CWlR9ffu.js.map +1 -0
  195. package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
  196. package/dist/record-6IUzwP9K.js.map +1 -0
  197. package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
  198. package/dist/redact-BhjtnjrU.js.map +1 -0
  199. package/dist/remarks-CAvTSq16.js +1 -0
  200. package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
  201. package/dist/remarks-D9Lcd1fF.js.map +1 -0
  202. package/dist/remarks-DPCONFxZ.js +138 -0
  203. package/dist/remarks-DPCONFxZ.js.map +1 -0
  204. package/dist/{remarks-E4rpi8Ym.js → remarks-vtkPmQb9.js} +327 -41
  205. package/dist/remarks-vtkPmQb9.js.map +1 -0
  206. package/dist/{remote-DZ00A8V_.js → remote-BgYWqogI.js} +6 -6
  207. package/dist/{remote-DZ00A8V_.js.map → remote-BgYWqogI.js.map} +1 -1
  208. package/dist/{resolve-B_JlQ0La.js → resolve-Do0ZaYH6.js} +11 -10
  209. package/dist/resolve-Do0ZaYH6.js.map +1 -0
  210. package/dist/resolve-amw_2Odz.js +1 -0
  211. package/dist/{results-BMJHaQbi.js → results-JyuM64gy.js} +7 -7
  212. package/dist/{results-BMJHaQbi.js.map → results-JyuM64gy.js.map} +1 -1
  213. package/dist/resume-AW5oXzUc.js +691 -0
  214. package/dist/resume-AW5oXzUc.js.map +1 -0
  215. package/dist/retry-ChzISxmy.js +105 -0
  216. package/dist/retry-ChzISxmy.js.map +1 -0
  217. package/dist/{runtime-CSZ1hjlN.js → runtime-BpW1f1Mx.js} +26 -7
  218. package/dist/runtime-BpW1f1Mx.js.map +1 -0
  219. package/dist/sandbox/boat.d.ts +26 -0
  220. package/dist/sandbox/boat.js +37 -0
  221. package/dist/sandbox/boat.js.map +1 -0
  222. package/dist/{sandbox-B5W8lLL1.js → sandbox-kSgr7vo5.js} +31 -3
  223. package/dist/sandbox-kSgr7vo5.js.map +1 -0
  224. package/dist/sandbox.d.ts +20 -2
  225. package/dist/sandbox.js +1 -1
  226. package/dist/server-2_gUnVvf.js +1 -0
  227. package/dist/{server-BfN6q2lH.js → server-Cn8VqO9S.js} +10 -10
  228. package/dist/{server-BfN6q2lH.js.map → server-Cn8VqO9S.js.map} +1 -1
  229. package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
  230. package/dist/shared-Cfi45Bvy.js.map +1 -0
  231. package/dist/slug-IHh3f770.js +136 -0
  232. package/dist/slug-IHh3f770.js.map +1 -0
  233. package/dist/source-CY3etKzw.js +1 -0
  234. package/dist/{source-BD6Bg0Tr.js → source-CYOhR98B.js} +2 -2
  235. package/dist/{source-BD6Bg0Tr.js.map → source-CYOhR98B.js.map} +1 -1
  236. package/dist/src-6Zzyg4Qd.js +2 -0
  237. package/dist/{src-BC2WXULN.js → src-BDhcPtnI.js} +2 -2
  238. package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
  239. package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
  240. package/dist/src-BRMsk9nh.js +2 -0
  241. package/dist/{src-Dp-fWOfG.js → src-CV_7DkPa.js} +47 -45
  242. package/dist/{src-Dp-fWOfG.js.map → src-CV_7DkPa.js.map} +1 -1
  243. package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
  244. package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
  245. package/dist/{src-Cpy9MNS3.js → src-Ck2r_EMT.js} +617 -74
  246. package/dist/src-Ck2r_EMT.js.map +1 -0
  247. package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
  248. package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
  249. package/dist/src-DB7N8TYJ.js +2 -0
  250. package/dist/src-DPDaAYwh.js +2 -0
  251. package/dist/{src-Cx0HJIJ-.js → src-Dbk19BiP.js} +2 -2
  252. package/dist/src-DtodMT59.js +1 -0
  253. package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
  254. package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
  255. package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
  256. package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
  257. package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
  258. package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
  259. package/dist/src-T-OeuJLr.js +2 -0
  260. package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
  261. package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
  262. package/dist/src-zntEDlnx.js +3 -0
  263. package/dist/{state-CdSjXIfE.js → state-BOLt8pJg.js} +17 -17
  264. package/dist/{state-CdSjXIfE.js.map → state-BOLt8pJg.js.map} +1 -1
  265. package/dist/state-kimtIhBP.js +1 -0
  266. package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
  267. package/dist/stats-C18W4xPX.js.map +1 -0
  268. package/dist/suite-CKBa6MKt.js +2 -0
  269. package/dist/suite-COglUhPd.js +1272 -0
  270. package/dist/suite-COglUhPd.js.map +1 -0
  271. package/dist/summary-DU5zaibx.js +148 -0
  272. package/dist/summary-DU5zaibx.js.map +1 -0
  273. package/dist/tart-BTVXp8rz.js +373 -0
  274. package/dist/tart-BTVXp8rz.js.map +1 -0
  275. package/dist/tart-cli-BIC9HqPg.js +37 -0
  276. package/dist/tart-cli-BIC9HqPg.js.map +1 -0
  277. package/dist/tart-cli-BMrkmBjI.js +1 -0
  278. package/dist/tart-images-DZ_Mbh-g.js +233 -0
  279. package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
  280. package/dist/tart-ref-7Qv3cm3B.js +1 -0
  281. package/dist/tart-ref-DYZijjrU.js +55 -0
  282. package/dist/tart-ref-DYZijjrU.js.map +1 -0
  283. package/dist/testing.d.ts +47 -37
  284. package/dist/testing.js +4 -4
  285. package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
  286. package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
  287. package/dist/trace-Dx2NBE1O.js +311 -0
  288. package/dist/trace-Dx2NBE1O.js.map +1 -0
  289. package/dist/traits-DKnYkqeK.js +66 -0
  290. package/dist/traits-DKnYkqeK.js.map +1 -0
  291. package/dist/{transcript-C_IpYMRk.js → transcript-CCf07UmU.js} +3 -3
  292. package/dist/{transcript-C_IpYMRk.js.map → transcript-CCf07UmU.js.map} +1 -1
  293. package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
  294. package/dist/types-C2mLpph3.js.map +1 -0
  295. package/dist/{types-BHR1lGPh.d.ts → types-DBwFsGem.d.ts} +420 -136
  296. package/dist/{judge-ByEUlzOM.js → vercel-CGlSBv63.js} +3 -229
  297. package/dist/vercel-CGlSBv63.js.map +1 -0
  298. package/dist/{vertex-6uSpZ5L8.js → vertex-Eb2wqahk.js} +5 -2
  299. package/dist/vertex-Eb2wqahk.js.map +1 -0
  300. package/dist/vertex-O88TCwgY.js +1 -0
  301. package/dist/view/index.d.ts +238 -90
  302. package/dist/view/index.js +11 -9
  303. package/dist/{views-CKoN5NAx.js → views-DAIiLiV3.js} +8 -7
  304. package/dist/views-DAIiLiV3.js.map +1 -0
  305. package/dist/views-DtRhj2Tn.js +2 -0
  306. package/dist/{worker-entry-D5dB8cgT.js → worker-entry-BIROTPX1.js} +11 -8
  307. package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-BIROTPX1.js.map} +1 -1
  308. package/dist/workspace-CcQRT31C.js +1 -0
  309. package/dist/{workspace-Cho30qlM.js → workspace-uuE51x35.js} +7 -7
  310. package/dist/{workspace-Cho30qlM.js.map → workspace-uuE51x35.js.map} +1 -1
  311. package/package.json +26 -11
  312. package/skills/eval-rubric/SKILL.md +31 -4
  313. package/dist/adapter-DyvDj4ZN.js.map +0 -1
  314. package/dist/app-files-hpRT5lj_.js.map +0 -1
  315. package/dist/assemble-DTXwCL3m.js +0 -1
  316. package/dist/assemble-Dlh6Xv3U.js.map +0 -1
  317. package/dist/capture-BT2fAmpZ.js +0 -1
  318. package/dist/capture-Bdcx9Gap.js.map +0 -1
  319. package/dist/cli-CGd51thd.js.map +0 -1
  320. package/dist/cli-spec-ChkleEVm.js.map +0 -1
  321. package/dist/client-CF-DOjOU.js.map +0 -1
  322. package/dist/command-CsfcwMyc.js.map +0 -1
  323. package/dist/command-CyIb9BSC.js.map +0 -1
  324. package/dist/command-TLmY-rYp.js.map +0 -1
  325. package/dist/commands-C54S45d7.js.map +0 -1
  326. package/dist/config-ImY6AlRE.js.map +0 -1
  327. package/dist/context-BajfiH0S.js.map +0 -1
  328. package/dist/contrasts-DtSX8E_V.js +0 -101
  329. package/dist/contrasts-DtSX8E_V.js.map +0 -1
  330. package/dist/cost-CMzY9VXv.js +0 -1
  331. package/dist/db-ZIk6S2CD.js +0 -2
  332. package/dist/db-sHGpaLo9.js.map +0 -1
  333. package/dist/debug-BnXV7JbX.js +0 -1
  334. package/dist/default-effort-BZRa-SQP.js.map +0 -1
  335. package/dist/dist-ByYk_wAs.js.map +0 -1
  336. package/dist/docker--2ytOHcu.js.map +0 -1
  337. package/dist/doctor-eE65YpEv.js.map +0 -1
  338. package/dist/environment-Cy3FVLR5.js.map +0 -1
  339. package/dist/eval-source-BTsTRiBW.js.map +0 -1
  340. package/dist/evaluate-CFFdyYZX.js.map +0 -1
  341. package/dist/evaluate-DnGOaGQc.d.ts +0 -43
  342. package/dist/files-B3d-8Fe0.js +0 -1
  343. package/dist/files-COW5JBJM.js +0 -1
  344. package/dist/files-Do4Rp3yZ.js +0 -1
  345. package/dist/files-p-uTdOjA.js.map +0 -1
  346. package/dist/gate-BoDKeuJu.js.map +0 -1
  347. package/dist/harness-Ch2P81sx.js.map +0 -1
  348. package/dist/ids-DuK6xl4w.js.map +0 -1
  349. package/dist/images-BVZHDf_z.js.map +0 -1
  350. package/dist/init-BWnMbUnc.js.map +0 -1
  351. package/dist/init-s1jN-PCI.js +0 -1
  352. package/dist/judge-ByEUlzOM.js.map +0 -1
  353. package/dist/live-DmJix4c8.js +0 -1
  354. package/dist/live-d_5a5iUv.js.map +0 -1
  355. package/dist/model-8uZBYqV5.js +0 -1
  356. package/dist/model-DOiCiMCD.js.map +0 -1
  357. package/dist/node-CwP1G178.js.map +0 -1
  358. package/dist/opencode-providers-yNKyXD49.js.map +0 -1
  359. package/dist/plan-CIfOFhth.js.map +0 -1
  360. package/dist/plugins-BEs2Am3L.js.map +0 -1
  361. package/dist/preflight-50Qw5e5K.js.map +0 -1
  362. package/dist/provenance-Cc8fFE7Z.js.map +0 -1
  363. package/dist/provider-DCdTHQpk.js +0 -1
  364. package/dist/providers-CP_bBtYm.js +0 -1
  365. package/dist/providers-DZSTlC9R.js.map +0 -1
  366. package/dist/query-kFPuqBv1.js +0 -1
  367. package/dist/questions-C_Y-oXW6.js.map +0 -1
  368. package/dist/record-DhzwkSXc.js.map +0 -1
  369. package/dist/redact-BEEGc37e.js.map +0 -1
  370. package/dist/remarks-0EXMh6AD.js.map +0 -1
  371. package/dist/remarks-BL18Id09.js +0 -1
  372. package/dist/remarks-DeZZT8PK.js +0 -116
  373. package/dist/remarks-DeZZT8PK.js.map +0 -1
  374. package/dist/remarks-E4rpi8Ym.js.map +0 -1
  375. package/dist/resolve-BJSay360.js +0 -1
  376. package/dist/resolve-B_JlQ0La.js.map +0 -1
  377. package/dist/resume-CexsWuGK.js +0 -434
  378. package/dist/resume-CexsWuGK.js.map +0 -1
  379. package/dist/runtime-CSZ1hjlN.js.map +0 -1
  380. package/dist/sandbox-B5W8lLL1.js.map +0 -1
  381. package/dist/server-SBzrf35T.js +0 -1
  382. package/dist/shared-ogqPJ0uo.js.map +0 -1
  383. package/dist/source-Dn1uqPon.js +0 -1
  384. package/dist/src-BKDZO3ra.js +0 -2
  385. package/dist/src-BxrHipV0.js +0 -3
  386. package/dist/src-C9elednc.js +0 -2
  387. package/dist/src-CM7xdJFA.js +0 -2
  388. package/dist/src-CWEGknZo.js +0 -2
  389. package/dist/src-Cpy9MNS3.js.map +0 -1
  390. package/dist/src-DVaAKCsH.js +0 -1
  391. package/dist/src-DurP--_D.js +0 -2
  392. package/dist/state-Cicc9Oa5.js +0 -1
  393. package/dist/stats-D-Gqwqt0.js.map +0 -1
  394. package/dist/suite-BYJ2Q0S-.js +0 -2
  395. package/dist/suite-BtHPBFYL.js +0 -544
  396. package/dist/suite-BtHPBFYL.js.map +0 -1
  397. package/dist/summary-zP3cCXDN.js.map +0 -1
  398. package/dist/types-C4OkYhZg.js.map +0 -1
  399. package/dist/vertex-6uSpZ5L8.js.map +0 -1
  400. package/dist/vertex-BaAz44YN.js +0 -1
  401. package/dist/views-CKoN5NAx.js.map +0 -1
  402. package/dist/views-CmyX255j.js +0 -2
  403. package/dist/workspace-Dw9DEZox.js +0 -1
@@ -1,32 +1,33 @@
1
- import { n as isOneOf, t as isObjectLike } from "./object-CeQ8dgUC.js";
2
1
  import { n as UsageError } from "./errors-ttMTnIYX.js";
3
- import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, h as parseDuration, i as durationSchema, l as nonEmpty, o as genericProblem, s as got, t as closest, u as oneOf } from "./shared-ogqPJ0uo.js";
4
- import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-Cy3FVLR5.js";
5
- import { B as gatePolicySchema, D as pluginServerName, J as snapshotOptionSchema, L as egressPolicySchema, V as gateProblem, Y as snapshotProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-BEs2Am3L.js";
6
- import { _ as GATEWAY_NAMES, b as gatewayEntryDataSchema, f as FACTORY, g as harnessProblem, h as harnessDescriptorSchema, u as opencodeModelError, v as directOffPin, x as gatewayNameSchema, y as gatewayDescriptorSchema } from "./harness-Ch2P81sx.js";
7
- import { a as fromCwd, l as sha256 } from "./ids-DuK6xl4w.js";
8
- import { a as providerEndpoints, c as providerServedModel, n as openCodeEffortBody, o as providerHosts, r as openCodeEffortRefusal, s as providerModelResponses, u as resolveOpenCodeProvider } from "./opencode-providers-yNKyXD49.js";
9
- import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-ByEUlzOM.js";
10
- import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV, t as SANDBOX_ACCESS_TOKEN } from "./vertex-6uSpZ5L8.js";
11
- import { r as badCharacterAt } from "./redact-BEEGc37e.js";
12
- import { s as withoutBom } from "./text-D--Q5KsA.js";
13
- import { t as inCI } from "./env-T86lAqUD.js";
2
+ import { a as fieldPath, c as jsonValueSchema, d as refusal, f as tokenUsageSchema, g as parseDuration, h as TIMER_LIMIT, i as durationSchema, l as nonEmpty$1, m as MAX_TIMER_MS, n as dataProblem, o as genericProblem, p as valueAt, s as got, t as closest, u as oneOf } from "./shared-Cfi45Bvy.js";
3
+ import { n as isOneOf, t as isObjectLike } from "./object-CeQ8dgUC.js";
4
+ import { _ as snapshotProblem, a as egressPolicySchema, c as gatePolicySchema, g as snapshotOptionSchema, l as gateProblem, s as envVarsSchema } from "./plain-BYUf_fUV.js";
5
+ import { a as isSandboxKind, d as sandboxKind, f as suggestedSandbox, i as idPart, n as capabilitiesOf, o as maxConcurrentProblem, t as SANDBOX_KINDS, u as sandboxHelpers } from "./kinds-C1oY3qyC.js";
6
+ import { C as plainCredentialHeaders, D as sandboxOf, E as sameWorkspaceOnly, F as environmentDataSchema, I as sandboxDescriptorSchema, L as traitsSchema, M as unbrokeredSecrets, N as workspaceLabel, O as sandboxPaths, P as workspaceOf, _ as isMovingRef, a as checkEgress, c as environmentDataProblem, h as isEnvironment, j as traitsOf, l as environmentFrom, m as helpersFor, o as checkEnv, r as applied, u as environmentPaths, v as isSecret, w as plannedSecrets, z as sandboxProblem } from "./environment-DB8o3hGi.js";
7
+ import { c as GATEWAY_NAMES, d as gatewayEntryDataSchema, f as gatewayNameSchema, i as claudeCodeTakesEffort, l as directOffPin, o as harnessDescriptorSchema, r as claudeCodeModelName, s as harnessProblem, t as FACTORY, u as gatewayDescriptorSchema } from "./harness-DXv2gnaT.js";
8
+ import { c as sha256, i as fromCwd } from "./ids-C_advdJA.js";
9
+ import { n as openCodeEffortBody, r as openCodeEffortRefusal, u as resolveOpenCodeProvider } from "./opencode-providers-BGiXt8_B.js";
10
+ import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV } from "./vertex-Eb2wqahk.js";
11
+ import { _ as redactValue, h as redactSecrets, r as badCharacterAt, t as REDACTED } from "./redact-BhjtnjrU.js";
12
+ import { a as RECOMMENDED_JUDGE_MODEL, c as judgeCredentials, d as modelKeyOptions, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL } from "./judge-COMXT1Hx.js";
13
+ import { t as inCI } from "./env-8x1O8I1M.js";
14
14
  import { u as ownFileURL } from "./root-Co0jhOT5.js";
15
- import { F as HARNESS_HOSTS, I as PACKAGE_HOSTS, P as DOCKER_EGRESS, mt as shellQuote, pt as run } from "./src-Cpy9MNS3.js";
16
- import { t as InfraError } from "./types-C4OkYhZg.js";
17
- import { n as gateLabel, u as pairFactors } from "./gate-BoDKeuJu.js";
18
- import "./stats-D-Gqwqt0.js";
19
- import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-DyvDj4ZN.js";
20
- import { t as docker } from "./sandbox-B5W8lLL1.js";
15
+ import { B as PACKAGE_HOSTS, z as HARNESS_HOSTS } from "./src-Ck2r_EMT.js";
16
+ import { y as localPluginServers } from "./plugins-BkmPxDiY.js";
17
+ import { a as agentFor, b as isRoutingGateway, f as isAgentKind, n as DEFAULT_HARNESS, r as HARNESS_EXAMPLE, t as AGENTS, u as credentialOf, y as ROUTING_GATEWAYS } from "./agents-C6zhkxp3.js";
18
+ import { i as pairFactors } from "./pairs-BGb134Vu.js";
19
+ import { u as opencodeModelError } from "./harness-C2berMiM.js";
20
+ import { r as tartTagWarning } from "./tart-ref-DYZijjrU.js";
21
+ import "./stats-C18W4xPX.js";
22
+ import { n as gateLabel } from "./gate-CgYdaprG.js";
23
+ import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-C8xPuXYq.js";
24
+ import { t as docker } from "./sandbox-kSgr7vo5.js";
25
+ import { z } from "zod";
21
26
  import { existsSync, readFileSync } from "node:fs";
22
- import { availableParallelism, tmpdir, totalmem } from "node:os";
23
- import { dirname, join, resolve, win32 } from "node:path";
27
+ import { availableParallelism, totalmem } from "node:os";
28
+ import { dirname, join, resolve } from "node:path";
24
29
  import { parseDocument } from "yaml";
25
- import { z } from "zod";
26
- import { connect, isIP } from "node:net";
27
- import { createHash, randomUUID } from "node:crypto";
28
- import { spawn } from "node:child_process";
29
- import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
30
+ import { isIP } from "node:net";
30
31
  // src/judge/artifacts.ts
31
32
  /** A capture handle: a name plus the file the attempt wrote. */
32
33
  function isCaptureHandle(value) {
@@ -88,20 +89,22 @@ const sharedInputFields = {
88
89
  task: z.boolean().optional(),
89
90
  output: z.boolean().optional(),
90
91
  files: z.boolean().optional(),
91
- diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
92
+ diff: z.union([z.boolean(), z.array(nonEmpty$1()).min(1)]).optional(),
92
93
  transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
93
- checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
94
+ checks: z.union([z.boolean(), z.array(nonEmpty$1())]).optional(),
95
+ observations: z.union([z.boolean(), z.array(nonEmpty$1())]).optional()
94
96
  };
95
97
  const turnsSchema = z.enum(["last", "all"]);
96
- const { task, output, files, diff, transcript, checks } = sharedInputFields;
98
+ const { task, output, files, diff, transcript, checks, observations } = sharedInputFields;
97
99
  const judgeInputsSchema = z.strictObject({
98
100
  task,
99
101
  output,
100
102
  files,
101
103
  diff,
102
- contents: z.array(nonEmpty()).optional(),
104
+ contents: z.array(nonEmpty$1()).optional(),
103
105
  transcript,
104
106
  checks,
107
+ observations,
105
108
  artifacts: z.union([z.literal(true), z.array(z.custom())]).optional()
106
109
  });
107
110
  const judgeSubjectSchema = z.strictObject({
@@ -149,7 +152,8 @@ const VALUES$1 = {
149
152
  contents: "a list of paths or patterns",
150
153
  transcript: "true, false, 'tools' or 'full'",
151
154
  turns: "'last' or 'all'",
152
- checks: "true, false or a list of commands"
155
+ checks: "true, false or a list of commands",
156
+ observations: "true, false or a list of names"
153
157
  };
154
158
  function distance(a, b) {
155
159
  const row = Array.from({ length: b.length + 1 }, (_, i) => i);
@@ -239,7 +243,7 @@ function judgeProblem(judge, site) {
239
243
  const pairFactorSchema = z.enum(["environment", "harness"]);
240
244
  /** Evaluation models score yes/no questions only, so they can't answer a question. */
241
245
  const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
242
- const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty()).min(1)]);
246
+ const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty$1()).min(1)]);
243
247
  const questionInputFields = {
244
248
  ...sharedInputFields,
245
249
  artifacts: questionArtifactsSchema.optional()
@@ -257,9 +261,9 @@ const unique = (list, ctx) => {
257
261
  });
258
262
  };
259
263
  const acrossSchema = z.union([pairFactorSchema, z.array(pairFactorSchema).min(1).superRefine(unique)]);
260
- const choicesSchema = z.array(nonEmpty()).min(2).superRefine(unique);
264
+ const choicesSchema = z.array(nonEmpty$1()).min(2).superRefine(unique);
261
265
  const questionSchema = z.strictObject({
262
- question: nonEmpty(),
266
+ question: nonEmpty$1(),
263
267
  model: questionModelSchema.optional(),
264
268
  across: acrossSchema.optional(),
265
269
  choices: choicesSchema.optional(),
@@ -296,7 +300,8 @@ const VALUES = {
296
300
  diff: "true, false or a non-empty list of patterns",
297
301
  transcript: "true, false, 'tools' or 'full'",
298
302
  turns: "'last' or 'all'",
299
- checks: "true, false or a list of commands"
303
+ checks: "true, false or a list of commands",
304
+ observations: "true, false or a list of names"
300
305
  };
301
306
  const describe$2 = (value) => {
302
307
  if (Array.isArray(value)) return `a list of ${value.length}`;
@@ -344,8 +349,8 @@ function acrossProblem(question, across, where) {
344
349
  function choicesProblem(question, choices, where) {
345
350
  if (choicesSchema.safeParse(choices).success) return void 0;
346
351
  const at = `questions "${question}" choices in ${where}`;
347
- if (Array.isArray(choices) && choices.length >= 2 && choices.every((c) => nonEmpty().safeParse(c).success)) return `${at} lists "${twice(choices)}" twice. List each answer once.`;
348
- const blank = Array.isArray(choices) && choices.find((c) => !nonEmpty().safeParse(c).success);
352
+ if (Array.isArray(choices) && choices.length >= 2 && choices.every((c) => nonEmpty$1().safeParse(c).success)) return `${at} lists "${twice(choices)}" twice. List each answer once.`;
353
+ const blank = Array.isArray(choices) && choices.find((c) => !nonEmpty$1().safeParse(c).success);
349
354
  if (blank !== void 0 && blank !== false) return `${at} has an answer that isn't a non-empty string (got ${describe$2(blank)}), e.g. choices: ['the test', 'the code']`;
350
355
  return `${at} is a list of at least two answers, e.g. choices: ['the test', 'the code'] (got ${describe$2(choices)})`;
351
356
  }
@@ -359,7 +364,7 @@ function questionProblem(item, where) {
359
364
  return `questions in ${where} takes ${QUESTION_SHAPE} (got ${unknown.map((k) => `"${k}"`).join(", ")}).${hint}`;
360
365
  }
361
366
  const { question, model, across, choices } = item;
362
- if (!nonEmpty().safeParse(question).success) return `questions in ${where} has an entry without its question${question === void 0 ? "" : ` (got ${describe$2(question)})`}, e.g. { question: 'Which is cleaner?', across: 'environment' }.`;
367
+ if (!nonEmpty$1().safeParse(question).success) return `questions in ${where} has an entry without its question${question === void 0 ? "" : ` (got ${describe$2(question)})`}, e.g. { question: 'Which is cleaner?', across: 'environment' }.`;
363
368
  const text = question;
364
369
  if (across !== void 0) {
365
370
  const problem = acrossProblem(text, across, where);
@@ -398,6 +403,7 @@ const captureOptionsSchema = z.strictObject({
398
403
  model: z.string().includes("/").optional()
399
404
  });
400
405
  const harnessList = z.union([harnessDescriptorSchema, z.array(harnessDescriptorSchema).min(1)]);
406
+ const environmentList = z.union([environmentDataSchema, z.array(environmentDataSchema).min(1)]);
401
407
  const budgetSchema = z.strictObject({
402
408
  attempts: count().optional(),
403
409
  duration: durationSchema.optional()
@@ -416,9 +422,9 @@ const recordMode = z.enum([
416
422
  "always"
417
423
  ]);
418
424
  const profileDefaultsDataSchema = z.strictObject({
419
- harness: harnessList.optional(),
425
+ harnesses: harnessList.optional(),
420
426
  sandbox: sandboxDescriptorSchema.optional(),
421
- environment: environmentDataSchema.optional(),
427
+ environments: environmentList.optional(),
422
428
  runs: runs().optional(),
423
429
  gate: gatePolicySchema.optional(),
424
430
  env: envVarsSchema.optional(),
@@ -432,7 +438,8 @@ const profileDefaultsDataSchema = z.strictObject({
432
438
  const profileLimitSchema = z.strictObject({
433
439
  runs: count().optional(),
434
440
  compare: z.union([z.literal(false), z.strictObject({ maxVariants: z.number().int().min(0) })]).optional(),
435
- harness: z.array(z.string()).optional()
441
+ harnesses: z.array(z.string()).optional(),
442
+ traits: z.record(z.string(), z.union([z.string(), z.array(z.string()).min(1)])).optional()
436
443
  });
437
444
  const profileDataSchema = z.strictObject({
438
445
  defaults: profileDefaultsDataSchema.optional(),
@@ -477,9 +484,9 @@ const analysisConfigDataSchema = z.strictObject({
477
484
  });
478
485
  const reevalConfigDataSchema = z.strictObject({
479
486
  include: z.array(z.string()).optional(),
480
- harness: harnessList.optional(),
487
+ harnesses: harnessList.optional(),
481
488
  sandbox: sandboxDescriptorSchema.optional(),
482
- environment: environmentDataSchema.optional(),
489
+ environments: environmentList.optional(),
483
490
  runs: runs().optional(),
484
491
  gate: gatePolicySchema.optional(),
485
492
  env: envVarsSchema.optional(),
@@ -559,6 +566,7 @@ function configDataProblem(value, file) {
559
566
  }
560
567
  if (issue.code === "invalid_union") {
561
568
  const values = "options" in issue ? issue.options : void 0;
569
+ if (!values && issue.path.at(-1) === "environments") return `${at} must be an environment or a list of them (got ${got(v)})`;
562
570
  return values ? `${at} is ${oneOf(values)} (got ${got(v)})` : `${at} matches none of its forms (got ${got(v)})`;
563
571
  }
564
572
  if (issue.code === "invalid_type" && v === void 0) return `${at} is missing`;
@@ -573,1126 +581,6 @@ function configDataProblem(value, file) {
573
581
  path: []
574
582
  }, v)}`;
575
583
  }
576
- // src/gateway/routes.ts
577
- /** Every gateway that brokers a key, by name. */
578
- const ROUTING_GATEWAYS = {
579
- vercel: VERCEL_GATEWAY,
580
- openrouter: OPENROUTER_GATEWAY
581
- };
582
- /** Whether a harness's `gateway` names one of {@link ROUTING_GATEWAYS}. */
583
- const isRoutingGateway = (name) => name !== void 0 && Object.hasOwn(ROUTING_GATEWAYS, name);
584
- // src/harness/agents.ts
585
- const WEB_EVENTS = {
586
- webSearch: "web-search",
587
- webFetch: "web-fetch"
588
- };
589
- /** `harnessMetadata.cursorSdk.unreportedTools` as transcript event types. */
590
- function cursorUnreported(harnessMetadata) {
591
- const tools = harnessMetadata?.cursorSdk?.unreportedTools;
592
- return Array.isArray(tools) ? tools.flatMap((t) => WEB_EVENTS[t] ? [WEB_EVENTS[t]] : []) : [];
593
- }
594
- /** A Gemini API key's only request: `POST /v1beta/models/<model>:streamGenerateContent`. */
595
- const GEMINI_ENDPOINTS = [{
596
- host: "generativelanguage.googleapis.com",
597
- path: { regex: "^/v1beta/models/[^/]+:streamGenerateContent$" },
598
- method: ["POST"]
599
- }];
600
- /** Antigravity's: its model calls, and the `:generateContent` call its `search_web` tool makes. */
601
- const ANTIGRAVITY_ENDPOINTS = [{
602
- host: "generativelanguage.googleapis.com",
603
- path: { regex: "^/v1beta/models/[\\w.-]+:(?:streamGenerateContent|generateContent)$" },
604
- method: ["POST"]
605
- }];
606
- /** Goose's: the model call, and the model list it fetches with the key at the start of a turn. */
607
- const GOOSE_ENDPOINTS = [...GEMINI_ENDPOINTS, {
608
- host: "generativelanguage.googleapis.com",
609
- path: { regex: "^/v1beta/models$" },
610
- method: ["GET"]
611
- }];
612
- /**
613
- * What Grok Build sends the key on: the model list and the key's status as a session starts, and
614
- * its model calls (Chat Completions, and the Responses API for a model or tool that takes it).
615
- */
616
- const XAI_ENDPOINTS = [
617
- {
618
- host: "api.x.ai",
619
- path: { exact: "/v1/models" },
620
- method: ["GET"]
621
- },
622
- {
623
- host: "api.x.ai",
624
- path: { exact: "/v1/api-key" },
625
- method: ["GET"]
626
- },
627
- {
628
- host: "api.x.ai",
629
- path: { exact: "/v1/chat/completions" },
630
- method: ["POST"]
631
- },
632
- {
633
- host: "api.x.ai",
634
- path: { exact: "/v1/responses" },
635
- method: ["POST"]
636
- }
637
- ];
638
- /** xAI's model calls, whose responses name the model that served them. */
639
- const XAI_MODEL_RESPONSES = XAI_ENDPOINTS.filter((e) => e.path.exact === "/v1/chat/completions" || e.path.exact === "/v1/responses");
640
- /** The model an xAI response says served it. */
641
- const xaiServedModel = ({ host, responseModel }) => host === "api.x.ai" ? responseModel : void 0;
642
- /** The model a Gemini API request names in its URL. */
643
- const geminiServedModel = ({ host, path }) => host === "generativelanguage.googleapis.com" ? /^\/v1beta\/models\/([^/:]+):streamGenerateContent$/.exec(path)?.[1] : void 0;
644
- /** Cursor's key exchange: the only request the Cursor SDK sends the API key on. */
645
- const CURSOR_EXCHANGE = {
646
- host: "api2.cursor.sh",
647
- path: { exact: "/auth/exchange_user_api_key" },
648
- method: ["POST"]
649
- };
650
- /** Codex's model calls: the Responses API, whose response names the served model. */
651
- const OPENAI_MODEL_ENDPOINTS = [{
652
- host: "api.openai.com",
653
- path: { exact: "/v1/responses" },
654
- method: ["POST"]
655
- }];
656
- /** What Codex sends the key on: its model calls, and its web search tool's own endpoint. */
657
- const OPENAI_ENDPOINTS = [...OPENAI_MODEL_ENDPOINTS, {
658
- host: "api.openai.com",
659
- path: { exact: "/v1/alpha/search" },
660
- method: ["POST"]
661
- }];
662
- /**
663
- * The model the Responses API says served a request, from its response (a dated snapshot for some
664
- * models), and the reasoning effort Codex resolved for it, e.g. `gpt-5.6-luna[effort=medium]`.
665
- */
666
- const openaiServedModel = ({ host, path, responseModel, responseEffort }) => host === "api.openai.com" && path === "/v1/responses" && responseModel ? `${responseModel}${responseEffort ? `[effort=${responseEffort}]` : ""}` : void 0;
667
- /**
668
- * OpenCode on the model's provider (`anthropic/claude-haiku-4-5` is Anthropic's): its key, the
669
- * requests OpenCode sends it on, its hosts plus Exa's for web search, and the served model from
670
- * the URL (Gemini's API, Bedrock's) or the response body. Bedrock's hosts are its `AWS_REGION`'s.
671
- */
672
- function opencodeForModel(model, env = process.env) {
673
- const resolved = resolveOpenCodeProvider(model, env);
674
- if ("error" in resolved) throw new InfraError(`opencode(): ${resolved.error}`);
675
- const { provider } = resolved;
676
- const modelResponses = providerModelResponses(provider);
677
- return {
678
- credential: provider.env,
679
- endpoints: providerEndpoints(provider),
680
- ...modelResponses.length ? { modelResponses } : {},
681
- servedModel: (request) => providerServedModel(provider, request),
682
- hosts: [...providerHosts(provider), "mcp.exa.ai"]
683
- };
684
- }
685
- /**
686
- * The variables besides its key that OpenCode's provider takes from the host, as they are:
687
- * Bedrock's `AWS_REGION` (default `us-east-1`); none for most providers.
688
- */
689
- function opencodeVars(model, env = process.env) {
690
- const resolved = resolveOpenCodeProvider(model, env);
691
- if ("error" in resolved) throw new InfraError(`opencode(): ${resolved.error}`);
692
- return resolved.vars;
693
- }
694
- /** Codex's `config.toml` `[mcp_servers]` shape. */
695
- function codexMcpServers(servers) {
696
- return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
697
- if (s.transport === "sse") throw new InfraError(`Codex has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
698
- return s.transport === "stdio" ? [name, {
699
- command: s.command,
700
- args: s.args,
701
- env: s.env ?? {}
702
- }] : [name, {
703
- url: s.url,
704
- http_headers: s.headers ?? {}
705
- }];
706
- }));
707
- }
708
- /** The `.mcp.json` shape. */
709
- function claudeMcpServers(servers) {
710
- return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, {
711
- type: "stdio",
712
- command: s.command,
713
- args: s.args,
714
- env: s.env
715
- }] : [name, {
716
- type: s.transport,
717
- url: s.url,
718
- headers: s.headers
719
- }]));
720
- }
721
- /** The Antigravity CLI's `mcp_config.json` shape: remote servers take `serverUrl`, not `url`. */
722
- function antigravityMcpServers(servers) {
723
- return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, {
724
- command: s.command,
725
- args: s.args,
726
- env: s.env
727
- }] : [name, {
728
- serverUrl: s.url,
729
- headers: s.headers
730
- }]));
731
- }
732
- /**
733
- * A `{env:…}` or `{file:…}` OpenCode would substitute. It runs `\{env:[^}]+\}`, then
734
- * `\{file:[^}]+\}`, over its whole config text with no escape, so a reference left open at the end
735
- * of a value runs on into the JSON after it, and only `{env:}` and `{file:}` are left alone.
736
- */
737
- const OPENCODE_REFERENCE = /\{(?:env|file):(?!\})/;
738
- const fieldKey = (key) => /^[A-Za-z_$][\w$]*$/.test(key) ? `.${key}` : `[${JSON.stringify(key)}]`;
739
- /**
740
- * The first part of a server OpenCode would substitute, as what the server wouldn't get: its name,
741
- * a value as `mcp.<name>.<field>`, or an env or header name. Names are in the same config text as
742
- * values, so they're substituted too; a secret's value is never named.
743
- */
744
- function openCodeReference(name, s) {
745
- if (OPENCODE_REFERENCE.test(name)) return "its name";
746
- const server = `mcp${fieldKey(name)}`;
747
- const [map, kind, values] = s.transport === "stdio" ? [
748
- "env",
749
- "env",
750
- s.env ?? {}
751
- ] : [
752
- "headers",
753
- "header",
754
- s.headers ?? {}
755
- ];
756
- return [...s.transport === "stdio" ? [[`${server}.command`, s.command], ...s.args.map((a, i) => [`${server}.args[${i}]`, a])] : [[`${server}.url`, s.url]], ...Object.entries(values).flatMap(([k, v]) => [[`the ${kind} name ${k} in ${server}.${map}`, k], [`${server}.${map}${fieldKey(k)}`, v]])].find(([, text]) => OPENCODE_REFERENCE.test(text))?.[0];
757
- }
758
- /**
759
- * OpenCode's config shape (its `mcp.servers`), which the adapter sets `codemode: false` on.
760
- * OpenCode has no SSE transport, and refuses a value it would substitute.
761
- */
762
- function opencodeMcpServers(servers) {
763
- return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
764
- if (s.transport === "sse") throw new InfraError(`OpenCode has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
765
- const field = openCodeReference(name, s);
766
- if (field) throw new InfraError(`OpenCode substitutes {env:…} and {file:…} anywhere in its config, with no escape, so the ${name} server wouldn't get ${field} as written: take the reference out, or leave OpenCode out of this cell`);
767
- return s.transport === "stdio" ? [name, {
768
- type: "local",
769
- command: [s.command, ...s.args],
770
- environment: s.env ?? {}
771
- }] : [name, {
772
- type: "remote",
773
- url: s.url,
774
- headers: s.headers ?? {},
775
- oauth: false
776
- }];
777
- }));
778
- }
779
- /**
780
- * The key Goose keeps an extension by (its `name_to_key`): whitespace dropped, anything but ASCII
781
- * letters, digits, `_` and `-` made `_`, lower-cased. Of two extensions with one key it keeps one.
782
- */
783
- const gooseKey = (name) => name.replace(/\s/g, "").replace(/[^A-Za-z0-9_-]/g, "_").toLowerCase();
784
- /** Goose's extension shape (its `config.yaml`); it has no SSE transport. */
785
- function gooseMcpServers(servers) {
786
- const byKey = /* @__PURE__ */ new Map();
787
- for (const name of Object.keys(servers)) {
788
- const key = gooseKey(name);
789
- const other = byKey.get(key);
790
- if (other !== void 0) throw new InfraError(`Goose keys the ${other} and ${name} MCP servers both as ${key}, and keeps only one: rename one`);
791
- byKey.set(key, name);
792
- }
793
- return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
794
- if (s.transport === "sse") throw new InfraError(`Goose has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
795
- return s.transport === "stdio" ? [name, {
796
- type: "stdio",
797
- cmd: s.command,
798
- args: s.args,
799
- envs: s.env ?? {}
800
- }] : [name, {
801
- type: "streamable_http",
802
- uri: s.url,
803
- headers: s.headers ?? {}
804
- }];
805
- }));
806
- }
807
- /**
808
- * pi's `mcp.json` shape, which the adapter escapes and sets `"exposure": "direct"` on. pi
809
- * skips a server whose name isn't letters, digits, `_` and `-`, so the plan refuses it instead.
810
- * A plugin's server, `<plugin>:<server>`, is named `<plugin>_<server>`, so its tools are
811
- * `mcp__<plugin>_<server>__<tool>`.
812
- */
813
- function piMcpServers(servers) {
814
- const named = /* @__PURE__ */ new Map();
815
- return Object.fromEntries(Object.entries(servers).map(([key, s]) => {
816
- const name = /^[a-z0-9-]+:[^:]+$/.test(key) ? key.replace(":", "_") : key;
817
- if (!/^[A-Za-z0-9_-]+$/.test(name)) throw new InfraError(`pi takes MCP server names of letters, digits, "_" and "-" only, so it can't use the ${key} server: rename it`);
818
- const other = named.get(name);
819
- if (other) throw new InfraError(`pi names the ${key} MCP server ${name}, as it does the ${other} server, and keeps only one: rename one`);
820
- named.set(name, key);
821
- if (s.transport === "sse") throw new InfraError(`pi has no SSE MCP transport, so it can't use the ${key} server (mcp.sse): use mcp.http or mcp.stdio`);
822
- return s.transport === "stdio" ? [name, {
823
- command: s.command,
824
- args: s.args,
825
- env: s.env ?? {}
826
- }] : [name, {
827
- url: s.url,
828
- headers: s.headers ?? {}
829
- }];
830
- }));
831
- }
832
- /**
833
- * ACP's `McpServer` shape, which fx and Grok Build take on `session/new`: `env` and `headers` as
834
- * name-value lists.
835
- */
836
- function acpMcpServers(servers) {
837
- const pairs = (values = {}) => Object.entries(values).map(([name, value]) => ({
838
- name,
839
- value
840
- }));
841
- return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, s.command.startsWith("/") ? {
842
- command: s.command,
843
- args: s.args,
844
- env: pairs(s.env)
845
- } : {
846
- command: "/usr/bin/env",
847
- args: [s.command, ...s.args],
848
- env: pairs(s.env)
849
- }] : [name, {
850
- type: s.transport,
851
- url: s.url,
852
- headers: pairs(s.headers)
853
- }]));
854
- }
855
- /**
856
- * fx on the model's provider: OpenAI's Chat Completions API for a bare OpenAI id, and Vercel AI
857
- * Gateway for a gateway id, which only `vercel()` in the gateway list serves. Each response's
858
- * first `model` names the model that served it.
859
- */
860
- function fxForModel(model) {
861
- if (!model.includes("/")) {
862
- const endpoints = [{
863
- host: "api.openai.com",
864
- path: { exact: "/v1/chat/completions" },
865
- method: ["POST"]
866
- }];
867
- return {
868
- credential: "OPENAI_API_KEY",
869
- endpoints,
870
- modelResponses: endpoints,
871
- servedModel: ({ host, responseModel }) => host === "api.openai.com" ? responseModel : void 0,
872
- hosts: ["api.openai.com"]
873
- };
874
- }
875
- return {
876
- credential: VERCEL_KEY,
877
- endpoints: FX_VERCEL_ENDPOINTS,
878
- modelResponses: [FX_VERCEL_CALL],
879
- servedModel: servedOnVercel,
880
- askedModel: fxAskedModel,
881
- hosts: [VERCEL_HOST]
882
- };
883
- }
884
- /** The Cursor SDK's `McpServerConfig` shape. */
885
- function cursorMcpServers(servers) {
886
- return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, {
887
- type: "stdio",
888
- command: s.command,
889
- args: s.args,
890
- env: s.env
891
- }] : [name, {
892
- type: s.transport,
893
- url: s.url,
894
- headers: s.headers
895
- }]));
896
- }
897
- /** Claude Code's hook events as Cursor names them, as the Cursor SDK converts Claude Code's own. */
898
- const CURSOR_HOOK_EVENTS = {
899
- SessionStart: "sessionStart",
900
- SessionEnd: "sessionEnd",
901
- UserPromptSubmit: "beforeSubmitPrompt",
902
- PreToolUse: "preToolUse",
903
- PostToolUse: "postToolUse",
904
- SubagentStop: "subagentStop",
905
- PreCompact: "preCompact",
906
- Stop: "stop"
907
- };
908
- /** Claude Code's tool names as Cursor's hooks match them; null for one Cursor has no tool for. */
909
- const CURSOR_HOOK_TOOLS = {
910
- Bash: "Shell",
911
- Read: "Read",
912
- Write: "Write",
913
- Edit: "Write",
914
- Glob: null,
915
- Grep: "Grep",
916
- WebFetch: "WebFetch",
917
- WebSearch: "WebSearch",
918
- Task: "Task"
919
- };
920
- /**
921
- * A Claude Code hook group's tool matcher as Cursor's, on the tool events only: each name mapped,
922
- * `mcp__<server>__<tool>` as `MCP:<tool>`; null when it names only tools Cursor lacks.
923
- */
924
- function cursorHookMatcher(event, matcher) {
925
- if (event !== "PreToolUse" && event !== "PostToolUse") return void 0;
926
- if (!matcher || matcher === "*") return void 0;
927
- const out = [];
928
- for (const raw of matcher.split("|")) {
929
- const name = raw.trim();
930
- const parts = name.split("__");
931
- const tool = name.startsWith("mcp__") && parts.length >= 3 ? `MCP:${parts.slice(2).join("__")}` : CURSOR_HOOK_TOOLS[name] === void 0 ? name : CURSOR_HOOK_TOOLS[name];
932
- if (tool && !out.includes(tool)) out.push(tool);
933
- }
934
- return out.length ? out.join("|") : null;
935
- }
936
- /** Where Cursor reads what reeval unpacks from a plugin, in the workspace (settingSources `project`). */
937
- const CURSOR_PLUGIN_UNPACK = {
938
- rules: ".cursor/rules",
939
- agents: ".cursor/agents",
940
- commands: ".cursor/commands",
941
- hooks: {
942
- file: ".cursor/hooks.json",
943
- events: CURSOR_HOOK_EVENTS,
944
- matcher: cursorHookMatcher
945
- }
946
- };
947
- /**
948
- * A plugin subagent's or command's frontmatter as OpenCode 2 takes it: the description, and a
949
- * subagent's `mode`. OpenCode skips a file whose frontmatter its schema refuses, without saying,
950
- * and Claude Code's `tools: Read, Grep`, `model: sonnet` and `allowed-tools` are such keys.
951
- */
952
- function opencodeFrontmatter(part, front) {
953
- const d = front.description;
954
- const description = typeof d === "string" || typeof d === "number" || typeof d === "boolean" ? String(d) : void 0;
955
- return {
956
- ...description ? { description } : {},
957
- ...part === "agents" ? { mode: "subagent" } : {}
958
- };
959
- }
960
- /** Where OpenCode reads plugin subagents and commands, in the workspace's config directory. */
961
- const OPENCODE_PLUGIN_UNPACK = {
962
- agents: ".opencode/agents",
963
- commands: ".opencode/commands",
964
- frontmatter: opencodeFrontmatter
965
- };
966
- /**
967
- * Where pi reads plugin commands, as prompt templates (`/<name>`), in the workspace, which its
968
- * `--approve` trusts. Its frontmatter takes `description` and `argument-hint`, and ignores the
969
- * rest of Claude Code's.
970
- */
971
- const PI_PLUGIN_UNPACK = { commands: ".pi/prompts" };
972
- /**
973
- * What Claude Code's plugin format bundles, which Grok Build reads too: every part but rules,
974
- * which neither loads from a plugin.
975
- */
976
- const CLAUDE_FORMAT_PARTS = PLUGIN_PARTS.filter((p) => p !== "rules");
977
- /** The marketplace Claude Code's settings name for the session's plugins. */
978
- const CLAUDE_PLUGIN_MARKETPLACE = "reeval";
979
- /**
980
- * Claude Code's settings for the session's plugins: a `directory` marketplace at the directory
981
- * they were copied to, listing each, and each enabled as `<plugin>@reeval`. Claude Code installs
982
- * and loads them as it starts, since the bridge leaves every settings source on.
983
- */
984
- function claudePluginFiles(plugins) {
985
- const [first] = plugins;
986
- if (!first) return [];
987
- const json = (value) => `${JSON.stringify(value, null, 2)}\n`;
988
- const marketplace = {
989
- name: CLAUDE_PLUGIN_MARKETPLACE,
990
- owner: { name: "reeval" },
991
- plugins: plugins.map((p) => ({
992
- name: p.name,
993
- source: `./${p.name}`
994
- }))
995
- };
996
- const settings = {
997
- extraKnownMarketplaces: { [CLAUDE_PLUGIN_MARKETPLACE]: { source: {
998
- source: "directory",
999
- path: first.dir.slice(0, first.dir.lastIndexOf("/"))
1000
- } } },
1001
- enabledPlugins: Object.fromEntries(plugins.map((p) => [`${p.name}@${CLAUDE_PLUGIN_MARKETPLACE}`, true]))
1002
- };
1003
- return [{
1004
- path: `${PLUGIN_DIR}/.claude-plugin/marketplace.json`,
1005
- content: json(marketplace)
1006
- }, {
1007
- path: "~/.claude/settings.json",
1008
- content: json(settings),
1009
- merge: true
1010
- }];
1011
- }
1012
- const isObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
1013
- /**
1014
- * Writes the layout's plugin files into the sandbox and returns their paths. A `merge` file's keys
1015
- * go into the JSON object already at its path (an object-valued key merged one level down), so
1016
- * settings an environment's `files` wrote there stay.
1017
- */
1018
- async function writePluginFiles(sandbox, layout, plugins) {
1019
- const written = [];
1020
- for (const f of layout.pluginFiles?.(plugins) ?? []) {
1021
- const existing = f.merge ? await sandbox.readFile(f.path) : null;
1022
- let content = f.content;
1023
- if (existing?.trim()) {
1024
- let base;
1025
- try {
1026
- base = JSON.parse(withoutBom(existing));
1027
- } catch {}
1028
- if (!isObject(base)) throw new InfraError(`${f.path} isn't a JSON object, so reeval can't enable the session's plugins in it: write it as one in files, or leave it out`);
1029
- const ours = JSON.parse(f.content);
1030
- const merged = { ...base };
1031
- for (const [k, v] of Object.entries(ours)) {
1032
- const was = base[k];
1033
- merged[k] = isObject(v) && isObject(was) ? {
1034
- ...was,
1035
- ...v
1036
- } : v;
1037
- }
1038
- content = `${JSON.stringify(merged, null, 2)}\n`;
1039
- }
1040
- await sandbox.writeFile(f.path, content);
1041
- written.push(f.path);
1042
- }
1043
- return written;
1044
- }
1045
- /**
1046
- * The MCP servers the agent is given: all of them, less the plugins' own where it takes plugin
1047
- * items namespaced, since it loads those from the plugin and would otherwise start each twice.
1048
- */
1049
- function agentMcp(layout, mcp, plugins) {
1050
- if (layout.pluginPlacement !== "namespaced") return mcp;
1051
- const own = new Set(plugins.flatMap((p) => p.servers));
1052
- return Object.fromEntries(Object.entries(mcp).filter(([name]) => !own.has(name)));
1053
- }
1054
- /** Where each agent with an image reads rules and skills. */
1055
- const LAYOUTS = {
1056
- cursor: {
1057
- rulesFile: "AGENTS.md",
1058
- workspaceSkillDir: ".cursor/skills",
1059
- harnessSkillDir: ".agents/skills",
1060
- pluginPlacement: "flat",
1061
- pluginUnpack: CURSOR_PLUGIN_UNPACK,
1062
- pluginHookEvents: Object.keys(CURSOR_HOOK_EVENTS)
1063
- },
1064
- "claude-code": {
1065
- rulesFile: "CLAUDE.md",
1066
- workspaceSkillDir: ".claude/skills",
1067
- harnessSkillDir: ".claude/skills",
1068
- pluginPlacement: "namespaced",
1069
- pluginFiles: claudePluginFiles
1070
- },
1071
- antigravity: {
1072
- rulesFile: "AGENTS.md",
1073
- workspaceSkillDir: ".agents/skills",
1074
- harnessSkillDir: ".gemini/config/skills",
1075
- pluginPlacement: "namespaced",
1076
- pluginHookEvents: [
1077
- "SessionStart",
1078
- "PreToolUse",
1079
- "PostToolUse",
1080
- "Stop"
1081
- ]
1082
- },
1083
- opencode: {
1084
- rulesFile: "AGENTS.md",
1085
- workspaceSkillDir: ".opencode/skills",
1086
- harnessSkillDir: ".config/opencode/skills",
1087
- pluginPlacement: "flat",
1088
- pluginUnpack: OPENCODE_PLUGIN_UNPACK
1089
- },
1090
- goose: {
1091
- rulesFile: "AGENTS.md",
1092
- workspaceSkillDir: ".goose/skills",
1093
- harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/goose/home/.config/goose/skills",
1094
- pluginPlacement: "flat"
1095
- },
1096
- pi: {
1097
- rulesFile: "AGENTS.md",
1098
- workspaceSkillDir: ".pi/skills",
1099
- harnessSkillDir: ".pi/agent/skills",
1100
- pluginPlacement: "flat",
1101
- pluginUnpack: PI_PLUGIN_UNPACK
1102
- },
1103
- codex: {
1104
- rulesFile: "AGENTS.md",
1105
- workspaceSkillDir: ".agents/skills",
1106
- harnessSkillDir: ".agents/skills",
1107
- pluginPlacement: "namespaced"
1108
- },
1109
- fx: {
1110
- rulesFile: "AGENTS.md",
1111
- workspaceSkillDir: ".fx/skills",
1112
- harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/fx/home/.fx/skills",
1113
- pluginPlacement: "flat"
1114
- },
1115
- "grok-build": {
1116
- rulesFile: "AGENTS.md",
1117
- workspaceSkillDir: ".grok/skills",
1118
- harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/grok-build/home/.grok/skills",
1119
- pluginPlacement: "namespaced"
1120
- }
1121
- };
1122
- /** Every real agent reeval can drive, by harness kind. */
1123
- const AGENTS = {
1124
- cursor: {
1125
- ...LAYOUTS.cursor,
1126
- pluginParts: PLUGIN_PARTS,
1127
- name: "Cursor",
1128
- factory: "cursor",
1129
- reasoning: "streamed",
1130
- reasoningTokens: false,
1131
- credential: "CURSOR_API_KEY",
1132
- endpoints: [CURSOR_EXCHANGE],
1133
- tokens: {
1134
- endpoint: CURSOR_EXCHANGE,
1135
- fields: ["accessToken", "refreshToken"],
1136
- hosts: ["cursor.sh", "*.cursor.sh"]
1137
- },
1138
- unreported: cursorUnreported,
1139
- mcpServers: cursorMcpServers,
1140
- adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-DurP--_D.js")).createCursorSdk({
1141
- auth,
1142
- ...mcpServers ? { mcpServers } : {},
1143
- ...descriptor.kind === "cursor" && descriptor.params ? { modelParams: descriptor.params } : {},
1144
- ...descriptor.kind === "cursor" && descriptor.effort ? { effort: descriptor.effort } : {},
1145
- requireCredentialBrokering: true
1146
- })
1147
- },
1148
- "claude-code": {
1149
- ...LAYOUTS["claude-code"],
1150
- pluginParts: CLAUDE_FORMAT_PARTS,
1151
- name: "Claude Code",
1152
- factory: "claudeCode",
1153
- reasoning: "streamed",
1154
- reasoningTokens: false,
1155
- subagentCalls: true,
1156
- credential: "ANTHROPIC_API_KEY",
1157
- endpoints: [
1158
- {
1159
- host: "api.anthropic.com",
1160
- path: { exact: "/v1/messages" },
1161
- method: ["POST"]
1162
- },
1163
- {
1164
- host: "api.anthropic.com",
1165
- path: { exact: "/v1/messages/count_tokens" },
1166
- method: ["POST"]
1167
- },
1168
- {
1169
- host: "api.anthropic.com",
1170
- path: { exact: "/api/claude_code/settings" },
1171
- method: ["GET"]
1172
- },
1173
- {
1174
- host: "api.anthropic.com",
1175
- path: { exact: "/api/claude_code/policy_limits" },
1176
- method: ["GET"]
1177
- }
1178
- ],
1179
- mcpServers: claudeMcpServers,
1180
- adapter: async ({ auth, env, mcpServers, descriptor }) => (await import("./dist-ByYk_wAs.js")).createClaudeCode({
1181
- auth,
1182
- ...mcpServers ? { mcpServers } : {},
1183
- ...descriptor.kind === "claude-code" && descriptor.effort ? { effort: descriptor.effort } : {},
1184
- env: {
1185
- IS_SANDBOX: "1",
1186
- CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",
1187
- ...env
1188
- }
1189
- })
1190
- },
1191
- antigravity: {
1192
- ...LAYOUTS.antigravity,
1193
- pluginParts: PLUGIN_PARTS,
1194
- pluginManifestsFirst: [
1195
- "plugin.json",
1196
- ".antigravity-plugin/plugin.json",
1197
- ".gemini-plugin/plugin.json",
1198
- "gemini-extension.json"
1199
- ],
1200
- name: "Antigravity",
1201
- factory: "antigravity",
1202
- reasoning: "none",
1203
- reasoningTokens: true,
1204
- credential: "GEMINI_API_KEY",
1205
- endpoints: ANTIGRAVITY_ENDPOINTS,
1206
- expectedBlocked: {
1207
- hosts: ["play.googleapis.com", "antigravity-unleash.goog"],
1208
- why: "Antigravity's telemetry and feature flags, which its CLI sends even with telemetry off"
1209
- },
1210
- servedModel: geminiServedModel,
1211
- mcpServers: antigravityMcpServers,
1212
- adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-CWEGknZo.js")).createAntigravity({
1213
- auth,
1214
- ...vertex ? { vertex } : {},
1215
- ...mcpServers ? { mcpServers } : {},
1216
- ...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
1217
- name,
1218
- dir
1219
- })) } : {},
1220
- ...descriptor.kind === "antigravity" && descriptor.effort ? { effort: descriptor.effort } : {},
1221
- requireCredentialBrokering: true
1222
- })
1223
- },
1224
- opencode: {
1225
- ...LAYOUTS.opencode,
1226
- pluginParts: [
1227
- "skills",
1228
- "agents",
1229
- "commands",
1230
- "mcp"
1231
- ],
1232
- name: "OpenCode",
1233
- factory: "opencode",
1234
- configuredModelOnStart: true,
1235
- reasoning: "streamed",
1236
- reasoningTokens: "provider",
1237
- credential: "GOOGLE_GENERATIVE_AI_API_KEY",
1238
- endpoints: GEMINI_ENDPOINTS,
1239
- servedModel: geminiServedModel,
1240
- forModel: opencodeForModel,
1241
- mcpServers: opencodeMcpServers,
1242
- adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-BxrHipV0.js")).createOpenCode({
1243
- auth,
1244
- model,
1245
- ...vertex ? { vertex } : { vars: opencodeVars(model) },
1246
- ...descriptor.kind === "opencode" && descriptor.effort ? { effort: descriptor.effort } : {},
1247
- ...mcpServers ? { mcpServers } : {},
1248
- requireCredentialBrokering: true
1249
- })
1250
- },
1251
- goose: {
1252
- ...LAYOUTS.goose,
1253
- pluginParts: ["skills", "mcp"],
1254
- name: "Goose",
1255
- factory: "goose",
1256
- configuredModelOnStart: true,
1257
- reasoning: "streamed",
1258
- reasoningTokens: false,
1259
- credential: "GOOGLE_GENERATIVE_AI_API_KEY",
1260
- endpoints: GOOSE_ENDPOINTS,
1261
- expectedBlocked: {
1262
- hosts: ["models.dev"],
1263
- why: "Goose's model catalog refresh, which no setting turns off"
1264
- },
1265
- servedModel: geminiServedModel,
1266
- mcpServers: gooseMcpServers,
1267
- adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-CM7xdJFA.js")).createGoose({
1268
- ...vertex ? { vertex } : {},
1269
- ...auth["AI_GATEWAY_API_KEY"] ? {
1270
- auth,
1271
- provider: (await import("./src-CM7xdJFA.js")).AI_GATEWAY_PROVIDER
1272
- } : auth["OPENROUTER_API_KEY"] ? {
1273
- auth,
1274
- provider: (await import("./src-CM7xdJFA.js")).OPENROUTER_PROVIDER
1275
- } : { auth: { GOOGLE_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" } },
1276
- model,
1277
- ...descriptor.kind === "goose" && descriptor.effort ? { effort: descriptor.effort } : {},
1278
- ...mcpServers ? { mcpServers } : {},
1279
- requireCredentialBrokering: true
1280
- })
1281
- },
1282
- pi: {
1283
- ...LAYOUTS.pi,
1284
- pluginParts: [
1285
- "skills",
1286
- "commands",
1287
- "mcp"
1288
- ],
1289
- name: "pi",
1290
- factory: "pi",
1291
- reasoning: "streamed",
1292
- reasoningTokens: true,
1293
- credential: "GOOGLE_GENERATIVE_AI_API_KEY",
1294
- endpoints: GEMINI_ENDPOINTS,
1295
- servedModel: geminiServedModel,
1296
- mcpServers: piMcpServers,
1297
- adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-BC2WXULN.js")).createPi({
1298
- ...auth["AI_GATEWAY_API_KEY"] ? {
1299
- auth,
1300
- gateway: (await import("./src-BC2WXULN.js")).PI_AI_GATEWAY
1301
- } : auth["OPENROUTER_API_KEY"] ? {
1302
- auth,
1303
- gateway: (await import("./src-BC2WXULN.js")).PI_OPENROUTER
1304
- } : {
1305
- auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
1306
- ...vertex ? { vertex } : {}
1307
- },
1308
- ...descriptor.kind === "pi" && descriptor.effort ? { effort: descriptor.effort } : {},
1309
- ...mcpServers ? { mcpServers } : {},
1310
- requireCredentialBrokering: true
1311
- })
1312
- },
1313
- codex: {
1314
- ...LAYOUTS.codex,
1315
- pluginParts: ["skills", "mcp"],
1316
- pluginApps: "they need a ChatGPT login",
1317
- pluginManifestsFirst: ["plugin.json", ".codex-plugin/plugin.json"],
1318
- pluginRootManifestNeedsSchema: true,
1319
- pluginServersByName: true,
1320
- name: "Codex",
1321
- factory: "codex",
1322
- reasoning: "whole",
1323
- reasoningTokens: true,
1324
- credential: "OPENAI_API_KEY",
1325
- endpoints: OPENAI_ENDPOINTS,
1326
- modelResponses: OPENAI_MODEL_ENDPOINTS,
1327
- expectedBlocked: {
1328
- hosts: [
1329
- "github.com",
1330
- "api.github.com",
1331
- "chatgpt.com"
1332
- ],
1333
- why: "Codex's plugin sync, which it runs on every turn once plugins are on",
1334
- withPlugins: true
1335
- },
1336
- servedModel: openaiServedModel,
1337
- mcpServers: codexMcpServers,
1338
- adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-C9elednc.js")).createCodex({
1339
- auth,
1340
- ...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-C9elednc.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-C9elednc.js")).OPENROUTER_PROVIDER } : {},
1341
- ...descriptor.kind === "codex" && descriptor.effort ? { effort: descriptor.effort } : {},
1342
- ...mcpServers ? { mcpServers } : {},
1343
- ...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
1344
- name,
1345
- dir
1346
- })) } : {},
1347
- requireCredentialBrokering: true
1348
- })
1349
- },
1350
- fx: {
1351
- ...LAYOUTS.fx,
1352
- pluginParts: ["skills", "mcp"],
1353
- name: "fx",
1354
- factory: "fx",
1355
- configuredModelOnStart: true,
1356
- reasoning: "streamed",
1357
- reasoningTokens: "provider",
1358
- ...fxForModel("provider/model"),
1359
- forModel: fxForModel,
1360
- mcpServers: acpMcpServers,
1361
- adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-BKDZO3ra.js")).createFx({
1362
- auth,
1363
- model,
1364
- ...auth["OPENROUTER_API_KEY"] ? { provider: "openrouter" } : {},
1365
- ...env?.FX_VERTEX_PROJECT ? { vertex: {
1366
- project: env.FX_VERTEX_PROJECT,
1367
- location: env.FX_VERTEX_LOCATION ?? ""
1368
- } } : {},
1369
- ...descriptor.kind === "fx" && descriptor.effort ? { effort: descriptor.effort } : {},
1370
- ...mcpServers ? { mcpServers } : {},
1371
- requireCredentialBrokering: true
1372
- })
1373
- },
1374
- "grok-build": {
1375
- ...LAYOUTS["grok-build"],
1376
- pluginParts: CLAUDE_FORMAT_PARTS,
1377
- pluginServersByName: true,
1378
- name: "Grok Build",
1379
- factory: "grokBuild",
1380
- configuredModelOnStart: true,
1381
- reasoning: "streamed",
1382
- reasoningTokens: true,
1383
- credential: "XAI_API_KEY",
1384
- endpoints: XAI_ENDPOINTS,
1385
- modelResponses: XAI_MODEL_RESPONSES,
1386
- servedModel: xaiServedModel,
1387
- mcpServers: acpMcpServers,
1388
- adapter: async ({ auth, mcpServers, descriptor, plugins, model, vertexChat }) => {
1389
- const grok = await import("./src-Cx0HJIJ-.js");
1390
- return grok.createGrokBuild({
1391
- ...vertexChat ? {
1392
- auth: { [grok.VERTEX_TOKEN_ENV]: SANDBOX_ACCESS_TOKEN },
1393
- provider: grok.vertexProvider(vertexChat.project, vertexChat.region)
1394
- } : {
1395
- auth,
1396
- ...auth["AI_GATEWAY_API_KEY"] ? { provider: grok.AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: grok.OPENROUTER_PROVIDER } : {}
1397
- },
1398
- model,
1399
- ...plugins.length ? {
1400
- pluginDirs: plugins.map((p) => p.dir),
1401
- pluginMcpServers: plugins.flatMap((p) => p.servers.map((s) => s.slice(p.name.length + 1)))
1402
- } : {},
1403
- ...descriptor.kind === "grok-build" && descriptor.effort ? { effort: descriptor.effort } : {},
1404
- ...mcpServers ? { mcpServers } : {},
1405
- requireCredentialBrokering: true
1406
- });
1407
- }
1408
- }
1409
- };
1410
- /** The harness of a config that names none: Cursor, on Cursor's own model. */
1411
- const DEFAULT_HARNESS = {
1412
- kind: "cursor",
1413
- model: "composer-2.5"
1414
- };
1415
- /** A harness to name in messages that ask for one. */
1416
- const HARNESS_EXAMPLE = "cursor({ model: 'composer-2.5' })";
1417
- const isAgentKind = (kind) => typeof kind === "string" && Object.hasOwn(AGENTS, kind);
1418
- /**
1419
- * A server as the plan knows it: its names, literal values and each `secret.template`'s text. A
1420
- * host secret's value is read when the attempt starts, so the plan has it as `''`.
1421
- */
1422
- function plannedMcpServer(s) {
1423
- const literal = (values) => Object.fromEntries(Object.entries(values).map(([k, v]) => [k, typeof v === "string" ? v : v.kind === "secret-template" ? v.template : ""]));
1424
- return s.transport === "stdio" ? {
1425
- transport: "stdio",
1426
- command: s.command,
1427
- args: s.args,
1428
- env: literal(s.env)
1429
- } : {
1430
- transport: s.transport,
1431
- url: s.url,
1432
- headers: literal(s.headers)
1433
- };
1434
- }
1435
- /**
1436
- * Why a harness's agent can't take the environment's MCP servers (Goose, OpenCode, pi and Codex
1437
- * take no `mcp.sse`; OpenCode would substitute a `{env:…}` or `{file:…}` in a name or value), in the
1438
- * agent's own words, so the plan refuses them before any attempt starts; undefined when it can.
1439
- */
1440
- function refusedMcp(harness, layers) {
1441
- if (!isAgentKind(harness.kind)) return void 0;
1442
- const servers = /* @__PURE__ */ new Map();
1443
- for (const { spec } of layers) for (const [name, server] of Object.entries(spec.mcp ?? {})) servers.set(name, plannedMcpServer(server));
1444
- if (servers.size === 0) return void 0;
1445
- try {
1446
- AGENTS[harness.kind].mcpServers(Object.fromEntries(servers));
1447
- return;
1448
- } catch (error) {
1449
- if (error instanceof InfraError) return error.message;
1450
- throw error;
1451
- }
1452
- }
1453
- /** The first host variable a plugin MCP server reads (`${VAR}`), as `VAR`; undefined for none. */
1454
- function hostVariable(s) {
1455
- const values = Object.entries(s.transport === "stdio" ? s.env : s.headers);
1456
- for (const [key, v] of values) {
1457
- if (typeof v === "string") continue;
1458
- if (v.kind === "secret") return v.from ?? key;
1459
- return /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/.exec(v.template)?.[1] ?? key;
1460
- }
1461
- }
1462
- const quoted = (parts) => parts.map((p) => `'${p}'`).join(", ");
1463
- /**
1464
- * Why a harness's agent can't take the plugins, in the agent's own words: a part it doesn't
1465
- * load, apps it can't take (`pluginApps`), a hook event or type it doesn't run, a plugin MCP
1466
- * server it refuses (named `<plugin>:<server>`) or whose `<plugin>:<server>` key `mcp` already has,
1467
- * or, where it takes plugin items flat, a skill or MCP server whose name another already has, or a
1468
- * subagent, command or rule another unpacks to the same file. `own` names the
1469
- * environment's own skills and MCP servers, and `unread` the plugins not fetched yet (git and
1470
- * npm ones, at plan time) by their call. Undefined when it can take them.
1471
- */
1472
- function refusedPlugins(harness, plugins, own, unread = []) {
1473
- if (plugins.length === 0 && unread.length === 0) return void 0;
1474
- const agent = isAgentKind(harness.kind) ? AGENTS[harness.kind] : void 0;
1475
- const mimics = harness.kind === "fake" && harness.layout ? AGENTS[harness.layout] : void 0;
1476
- const loads = (agent ?? mimics)?.pluginParts ?? PLUGIN_PARTS;
1477
- const name = agent?.name ?? (mimics ? `The fake agent with ${mimics.name}'s layout` : "The fake agent");
1478
- if (loads.length === 0) return `${name} can't load plugins yet: leave ${plugins[0] ? `the ${plugins[0].name} plugin` : unread[0]} out of this cell`;
1479
- for (const p of plugins) {
1480
- const refused = p.kept.filter((part) => !loads.includes(part));
1481
- if (refused.length === 0) continue;
1482
- const can = p.kept.filter((part) => loads.includes(part));
1483
- const what = refused.map((part) => `${part} (${PART_PATHS[part]})`).join(" and ");
1484
- const fix = can.length ? `use only: [${quoted(can)}], or leave` : "leave";
1485
- return `${name} can't load the ${p.name} plugin's ${what}: ${fix} the plugin out of this cell`;
1486
- }
1487
- const reads = agent ?? mimics;
1488
- const first = reads?.pluginManifestsFirst ?? [];
1489
- for (const p of plugins) {
1490
- const other = p.manifests.find((m) => first.includes(m) && (m !== "plugin.json" || !reads?.pluginRootManifestNeedsSchema || p.rootSchema));
1491
- if (other) return `${name} reads the ${p.name} plugin's ${other} in place of its ${PLUGIN_MANIFEST}, so reeval can't hold it to the parts it places: remove ${other} from the plugin, or leave the plugin out of this cell`;
1492
- }
1493
- const apps = (agent ?? mimics)?.pluginApps;
1494
- const withApps = apps ? plugins.find((p) => p.apps) : void 0;
1495
- if (withApps) return `${name} can't load the ${withApps.name} plugin's apps: ${apps}`;
1496
- const layout = layoutOf(harness);
1497
- const runs = layout.pluginHookEvents;
1498
- for (const p of runs ? plugins : []) {
1499
- const event = p.hooks.events.find((e) => !runs?.includes(e));
1500
- const type = p.hooks.otherTypes[0];
1501
- if (!event && !type) continue;
1502
- const rest = p.kept.filter((part) => part !== "hooks");
1503
- const fix = rest.length ? `use only: [${quoted(rest)}], or leave` : "leave";
1504
- return event ? `${name} can't run the ${p.name} plugin's ${event} hooks (hooks/hooks.json): it runs ${runs?.join(", ")} hooks; ${fix} the plugin out of this cell` : `${name} can't run the ${p.name} plugin's ${type} hooks (hooks/hooks.json): it runs only command hooks; ${fix} the plugin out of this cell`;
1505
- }
1506
- const servers = Object.fromEntries(plugins.flatMap((p) => Object.entries(p.mcp).map(([server, s]) => [pluginServerName(p.name, server), plannedMcpServer(s)])));
1507
- if (agent && Object.keys(servers).length) {
1508
- const alongside = agent.pluginPlacement === "flat" ? Object.fromEntries(own.mcp.map((n) => [n, {
1509
- transport: "stdio",
1510
- command: "true",
1511
- args: [],
1512
- env: {}
1513
- }])) : {};
1514
- try {
1515
- agent.mcpServers({
1516
- ...alongside,
1517
- ...servers
1518
- });
1519
- } catch (error) {
1520
- if (error instanceof InfraError) return error.message;
1521
- throw error;
1522
- }
1523
- }
1524
- for (const p of plugins) for (const server of Object.keys(p.mcp)) {
1525
- const key = pluginServerName(p.name, server);
1526
- if (own.mcp.includes(key)) return `the ${p.name} plugin's MCP server "${server}" would join mcp as ${key}, which mcp already names: rename mcp["${key}"]`;
1527
- }
1528
- if (layout.pluginPlacement === "namespaced") {
1529
- for (const p of plugins) {
1530
- if (!p.kept.includes("mcp")) continue;
1531
- for (const [server, s] of Object.entries(p.mcp)) {
1532
- const secret = hostVariable(s);
1533
- if (!secret) continue;
1534
- const rest = p.kept.filter((part) => part !== "mcp");
1535
- return `${name} loads the ${p.name} plugin's MCP server "${server}" itself, so reeval can't give it ${secret} from the host: use only: [${quoted(rest)}] and declare the server in mcp, where reeval fills it in`;
1536
- }
1537
- }
1538
- if (!(agent ?? mimics)?.pluginServersByName) return void 0;
1539
- const byName = new Map(own.mcp.map((n) => [n, `mcp.${n}`]));
1540
- for (const p of plugins) for (const server of Object.keys(p.mcp)) {
1541
- const other = byName.get(server);
1542
- if (other) return `${name} names the ${p.name} plugin's MCP server "${server}" by its own name, as it does ${other}, and keeps only one: rename one, or leave mcp out of the plugin with only`;
1543
- byName.set(server, `the ${p.name} plugin's MCP server "${server}"`);
1544
- }
1545
- return;
1546
- }
1547
- const serverOwner = new Map(own.mcp.map((n) => [n, `mcp.${n}`]));
1548
- for (const p of plugins) for (const server of Object.keys(p.mcp)) {
1549
- const other = serverOwner.get(server);
1550
- if (other) return `the ${p.name} plugin's MCP server "${server}" clashes with ${other}`;
1551
- serverOwner.set(server, `the ${p.name} plugin's MCP server "${server}"`);
1552
- }
1553
- const skillOwner = new Map(own.skills.map((s) => [s.name, `${s.call}'s "${s.name}"`]));
1554
- for (const p of plugins) for (const { name: skill } of p.skills) {
1555
- const other = skillOwner.get(skill);
1556
- if (other) return `the ${p.name} plugin's skill "${skill}" and ${other} would both be placed at ~/${layout.harnessSkillDir}/${skill}: rename one, or narrow it with only or exclude`;
1557
- skillOwner.set(skill, `the ${p.name} plugin's skill "${skill}"`);
1558
- }
1559
- for (const part of ["agents", "commands"]) {
1560
- const dir = layout.pluginUnpack?.[part];
1561
- if (!dir) continue;
1562
- const owner = /* @__PURE__ */ new Map();
1563
- for (const p of plugins) for (const item of p[part]) {
1564
- const other = owner.get(item);
1565
- const what = part === "agents" ? "subagent" : "command";
1566
- if (other) return `the ${p.name} plugin's ${what} "${item}" and the ${other} plugin's would both be placed at ${dir}/${item}.md: rename one, or narrow one with only`;
1567
- owner.set(item, p.name);
1568
- }
1569
- }
1570
- const rules = layout.pluginUnpack?.rules;
1571
- const ruleOwner = /* @__PURE__ */ new Map();
1572
- for (const p of rules ? plugins : []) for (const rule of p.rules) {
1573
- const file = `${p.name}-${rule.replace(/\.mdc?$/, "")}.mdc`;
1574
- const what = `the ${p.name} plugin's rule "${rule}"`;
1575
- const other = ruleOwner.get(file);
1576
- if (other) return `${what} and ${other} would both be placed at ${rules}/${file}: rename one, or narrow one with only`;
1577
- ruleOwner.set(file, what);
1578
- }
1579
- }
1580
- /**
1581
- * A real agent's catalog entry, with what its model's provider decides, or its gateway's route:
1582
- * on Vertex, the attempt's `vertex()` settings decide the project and region.
1583
- */
1584
- function agentFor(h, vertex) {
1585
- const agent = AGENTS[h.kind];
1586
- if (h.gateway === "vertex") {
1587
- const route = VERTEX_ROUTES[h.kind];
1588
- if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on Vertex AI`);
1589
- if (!vertex) throw new InfraError(`${agent.name} on Vertex AI has no vertex() settings`);
1590
- return {
1591
- ...agent,
1592
- ...route(vertex)
1593
- };
1594
- }
1595
- if (isRoutingGateway(h.gateway)) {
1596
- const gateway = ROUTING_GATEWAYS[h.gateway];
1597
- const route = gateway.routes[h.kind];
1598
- if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on ${gateway.name}`);
1599
- return {
1600
- ...agent,
1601
- ...route(process.env)
1602
- };
1603
- }
1604
- return agent.forModel ? {
1605
- ...agent,
1606
- ...agent.forModel(h.model)
1607
- } : agent;
1608
- }
1609
- /** The catalog entry for a harness's agent; undefined for the fake harness. */
1610
- const agentOf = (h, vertex) => h.kind === "fake" ? void 0 : agentFor(h, vertex);
1611
- /**
1612
- * The host variable a harness authenticates with; undefined for the fake harness, and on Vertex,
1613
- * whose token comes from Application Default Credentials.
1614
- */
1615
- const credentialOf = (h) => h.kind === "fake" || h.gateway === "vertex" ? void 0 : agentFor(h).credential;
1616
- /** The fake agent's own layout. */
1617
- const FAKE_LAYOUT = {
1618
- rulesFile: "AGENTS.md",
1619
- workspaceSkillDir: ".agents/skills",
1620
- harnessSkillDir: ".agents/skills",
1621
- pluginPlacement: "flat"
1622
- };
1623
- /** Where a harness's agent reads rules and skills; a fake one with `layout` mimics that agent. */
1624
- function layoutOf(descriptor) {
1625
- if (descriptor.kind !== "fake") return LAYOUTS[descriptor.kind];
1626
- return descriptor.layout ? LAYOUTS[descriptor.layout] : FAKE_LAYOUT;
1627
- }
1628
- /**
1629
- * The hosts a harness's agent is known to try in a session with or without plugins; a fake agent
1630
- * with `layout` expects that agent's. On Vertex, the attempt's `vertex()` settings pick the route.
1631
- */
1632
- function expectedBlockedFor(descriptor, plugins, vertex) {
1633
- const expected = descriptor.kind !== "fake" ? agentFor(descriptor, vertex).expectedBlocked : descriptor.layout ? AGENTS[descriptor.layout].expectedBlocked : void 0;
1634
- return expected?.withPlugins && !plugins ? void 0 : expected;
1635
- }
1636
- /**
1637
- * How the APIs behind the harnesses say they refused a key: an HTTP 401, Anthropic's and OpenAI's
1638
- * wording, Google's (which answers 400), and OpenCode's for any provider's 401 or 403. A bare 403 isn't here: the gateway and MCP servers send
1639
- * those for reasons that have nothing to do with the harness's key. Grok Build checks its key as
1640
- * it starts and, when xAI refuses it (a 400 or 401, not an outage), offers no `xai.api_key`
1641
- * sign-in, which is all `@ai-sdk/harness-acp` reports.
1642
- */
1643
- const REFUSED_KEY = /\bHTTP 401\b|\b401\b.{0,40}\bunauthori[sz]ed\b|\bunauthori[sz]ed\b.{0,40}\b401\b|\b(?:invalid|incorrect) (?:x-)?api[ _-]?key\b|\bapi key not valid\b|\bAPI_KEY_INVALID\b|\bauthentication_error\b|\bprovider authentication required\b|\bauthentication method "xai\.api_key" is not advertised\b/i;
1644
- /**
1645
- * What to do when the harness's API refused its key, naming the variable; else undefined. The
1646
- * error's wording says so for some CLIs; `refused` is the egress record's word for every one: a
1647
- * brokered request the API answered with 401, whatever the CLI made of it.
1648
- */
1649
- function authHint(agent, message, refused = false) {
1650
- if (!refused && !REFUSED_KEY.test(message)) return void 0;
1651
- if (agent.token) return `The API rejected the access token reeval minted from ${agent.credential}: check gcloud auth application-default login (or GOOGLE_APPLICATION_CREDENTIALS) and the project's Vertex AI access`;
1652
- return keyRejected(agent.credential);
1653
- }
1654
- /** What to do when the API refused the key in `credential`, naming it. */
1655
- const keyRejected = (credential) => `The API rejected ${credential}: set a valid key in the environment or .env.local`;
1656
- /**
1657
- * The agent's credential from the host, as the adapter's explicit `auth`. Fails closed. Empty for
1658
- * a brokered token, which the adapter never sees.
1659
- */
1660
- function agentAuth(agent, env = process.env) {
1661
- if (agent.token) return {};
1662
- const value = env[agent.credential];
1663
- if (!value) throw new InfraError(`${agent.credential} isn't set, so the ${agent.name} harness can't authenticate. Set it in the environment or .env.local.`);
1664
- return {
1665
- ...agent.authEnv,
1666
- [agent.authAs ?? agent.credential]: value
1667
- };
1668
- }
1669
- function brokeringFor(h, vertex) {
1670
- if (h.kind === "fake") {
1671
- if (!h.brokers) return void 0;
1672
- const { credential, endpoints } = AGENTS[h.layout ?? "cursor"];
1673
- return {
1674
- credential,
1675
- endpoints
1676
- };
1677
- }
1678
- const agent = agentOf(h, vertex);
1679
- if (!agent) return void 0;
1680
- const { credential, endpoints, tokens, servedModel, askedModel, modelResponses, upstream } = agent;
1681
- const { localResponses } = agent;
1682
- return {
1683
- credential,
1684
- endpoints,
1685
- ...tokens ? { tokens: {
1686
- name: credential,
1687
- ...tokens
1688
- } } : {},
1689
- ...servedModel ? { servedModel } : {},
1690
- ...askedModel ? { askedModel } : {},
1691
- ...modelResponses ? { modelResponses } : {},
1692
- ...upstream ? { upstream } : {},
1693
- ...localResponses?.length ? { localResponses } : {}
1694
- };
1695
- }
1696
584
  // src/harness/effort.ts
1697
585
  /**
1698
586
  * The reasoning effort a harness asked for. Same user-facing name on every factory; the CLI's
@@ -1742,6 +630,638 @@ function resolvedEffortOf(models) {
1742
630
  }
1743
631
  return out;
1744
632
  }
633
+ // src/contract/attempt.ts
634
+ const nullableNumber = z.number().nullable();
635
+ const fileChangeSchema = z.strictObject({
636
+ path: z.string(),
637
+ status: z.enum([
638
+ "added",
639
+ "modified",
640
+ "deleted"
641
+ ])
642
+ });
643
+ const diffSchema = z.strictObject({
644
+ patch: z.string(),
645
+ files: z.array(fileChangeSchema)
646
+ });
647
+ const turnStatusSchema = z.enum([
648
+ "completed",
649
+ "aborted",
650
+ "error"
651
+ ]);
652
+ const webEventTypeSchema = z.enum(["web-search", "web-fetch"]);
653
+ const at$1 = {
654
+ ts: z.string(),
655
+ atMs: z.number()
656
+ };
657
+ const transcriptEventSchema = z.discriminatedUnion("type", [
658
+ z.strictObject({
659
+ ...at$1,
660
+ type: z.enum(["text", "reasoning"]),
661
+ text: z.string(),
662
+ durationMs: nullableNumber
663
+ }),
664
+ z.strictObject({
665
+ ...at$1,
666
+ type: z.literal("file-edit"),
667
+ toolCallId: z.string(),
668
+ path: z.string(),
669
+ kind: z.enum([
670
+ "create",
671
+ "modify",
672
+ "write",
673
+ "delete"
674
+ ]),
675
+ added: nullableNumber,
676
+ removed: nullableNumber,
677
+ durationMs: nullableNumber,
678
+ stalled: z.boolean().optional(),
679
+ subagent: z.string().optional()
680
+ }),
681
+ z.strictObject({
682
+ ...at$1,
683
+ type: z.literal("web-search"),
684
+ toolCallId: z.string(),
685
+ query: z.string(),
686
+ references: nullableNumber,
687
+ ok: z.boolean().nullable().optional(),
688
+ durationMs: nullableNumber,
689
+ stalled: z.boolean().optional(),
690
+ subagent: z.string().optional()
691
+ }),
692
+ z.strictObject({
693
+ ...at$1,
694
+ type: z.literal("web-fetch"),
695
+ toolCallId: z.string(),
696
+ url: z.string(),
697
+ ok: z.boolean().nullable(),
698
+ durationMs: nullableNumber,
699
+ stalled: z.boolean().optional(),
700
+ subagent: z.string().optional()
701
+ }),
702
+ z.strictObject({
703
+ ...at$1,
704
+ type: z.literal("tool"),
705
+ toolCallId: z.string(),
706
+ tool: z.string(),
707
+ input: z.unknown(),
708
+ output: z.unknown(),
709
+ ok: z.boolean().nullable(),
710
+ durationMs: nullableNumber,
711
+ stalled: z.boolean().optional(),
712
+ subagent: z.string().optional()
713
+ }),
714
+ z.strictObject({
715
+ ...at$1,
716
+ type: z.literal("error"),
717
+ message: z.string()
718
+ }),
719
+ z.strictObject({
720
+ ...at$1,
721
+ type: z.literal("retry"),
722
+ message: z.string()
723
+ })
724
+ ]);
725
+ const turnDataSchema = z.strictObject({
726
+ prompt: z.string(),
727
+ text: z.string(),
728
+ status: turnStatusSchema,
729
+ startedAt: z.string(),
730
+ durationMs: z.number(),
731
+ transcript: z.array(transcriptEventSchema),
732
+ usage: tokenUsageSchema.nullable(),
733
+ model: z.string().nullable(),
734
+ unreported: z.array(webEventTypeSchema),
735
+ diff: diffSchema
736
+ });
737
+ const execResultSchema = z.strictObject({
738
+ exitCode: z.number().int(),
739
+ stdout: z.string(),
740
+ stderr: z.string()
741
+ });
742
+ const execRetrySchema = z.strictObject({
743
+ retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional(),
744
+ interval: z.number().min(0, "must be 0 or more milliseconds").max(MAX_TIMER_MS, `must be ${TIMER_LIMIT}`).optional()
745
+ });
746
+ /** A background command has no exit for `retries` to wait on. */
747
+ const noBackgroundRetry = (options, ctx) => {
748
+ if (!options.background) return;
749
+ for (const key of ["retries", "interval"]) {
750
+ if (options[key] === void 0) continue;
751
+ ctx.addIssue({
752
+ code: "custom",
753
+ path: [key],
754
+ message: "can't be used with background: true, whose exit reeval doesn't wait for: drop one of them"
755
+ });
756
+ }
757
+ };
758
+ const execOptionsSchema = z.strictObject({
759
+ background: z.boolean().optional(),
760
+ ...execRetrySchema.shape
761
+ }).superRefine(noBackgroundRetry);
762
+ /** What's wrong with `c.exec()`'s options, or undefined. */
763
+ const execOptionsProblem = (value) => dataProblem(execOptionsSchema, value, "c.exec()", () => Object.keys(execOptionsSchema.shape));
764
+ const assertionRecordSchema = z.strictObject({
765
+ label: z.string(),
766
+ labelled: z.boolean(),
767
+ matcher: z.string(),
768
+ status: z.enum([
769
+ "pass",
770
+ "fail",
771
+ "error"
772
+ ]),
773
+ soft: z.boolean(),
774
+ at: z.string().optional(),
775
+ message: z.string().optional(),
776
+ durationMs: z.number().optional()
777
+ });
778
+ const turnIndex = z.number().int("must be a turn index, from 0").min(0, "must be a turn index, from 0").describe("The turn's index, from 0.");
779
+ const captureConsoleMessageSchema = z.strictObject({
780
+ level: z.enum([
781
+ "error",
782
+ "warning",
783
+ "info",
784
+ "log",
785
+ "debug"
786
+ ]),
787
+ text: z.string(),
788
+ source: z.string(),
789
+ url: z.string().optional()
790
+ });
791
+ const captureFailedRequestSchema = z.strictObject({
792
+ method: z.string(),
793
+ url: z.string(),
794
+ status: z.number().optional(),
795
+ error: z.string().optional()
796
+ });
797
+ const capturePageFields = {
798
+ status: z.number().optional(),
799
+ finalUrl: z.string(),
800
+ console: z.array(captureConsoleMessageSchema),
801
+ errors: z.array(z.string()),
802
+ failedRequests: z.array(captureFailedRequestSchema),
803
+ dropped: z.number().optional()
804
+ };
805
+ z.strictObject(capturePageFields);
806
+ const captureSchema = z.strictObject({
807
+ name: z.string(),
808
+ port: z.number(),
809
+ path: z.string(),
810
+ viewport: z.union([captureWindowSchema, z.literal("full-page")]),
811
+ file: z.strictObject({
812
+ path: z.string(),
813
+ sha256: z.string(),
814
+ bytes: z.number()
815
+ }),
816
+ ...capturePageFields
817
+ });
818
+ const judgeArtifactRefSchema = z.strictObject({
819
+ name: z.string(),
820
+ path: z.string(),
821
+ sha256: z.string(),
822
+ bytes: z.number(),
823
+ mediaType: z.literal("image/png")
824
+ });
825
+ const verdictFields = {
826
+ threshold: z.number(),
827
+ question: z.string(),
828
+ model: z.string(),
829
+ gateway: gatewayNameSchema.optional(),
830
+ artifacts: z.array(judgeArtifactRefSchema).optional()
831
+ };
832
+ const verdictSchema = z.strictObject({
833
+ verdict: z.enum(["pass", "fail"]),
834
+ score: z.number(),
835
+ ...verdictFields,
836
+ resolvedModel: z.string().optional(),
837
+ upstream: z.string().optional(),
838
+ costUsd: z.number().optional()
839
+ });
840
+ const judgeFailureSchema = z.strictObject({
841
+ verdict: z.literal("error"),
842
+ score: z.null(),
843
+ ...verdictFields,
844
+ error: z.string()
845
+ });
846
+ /** A pattern, for the matchers that take a string or a `RegExp`. */
847
+ const patternSchema = z.strictObject({
848
+ regexp: z.string().describe("The pattern's source, as `RegExp.prototype.source` gives it."),
849
+ flags: z.string().regex(/^[dgimsuvy]*$/, "must use only the flags d, g, i, m, s, u, v and y").optional().describe("Its flags, e.g. `i`.")
850
+ }).describe("A regular expression, where a matcher takes a string or a `RegExp`.");
851
+ /** A pattern in the runner's own language, which reeval never compiles. */
852
+ function hasAnsweredPattern(value) {
853
+ if (Array.isArray(value)) return value.some(hasAnsweredPattern);
854
+ if (typeof value !== "object" || value === null) return false;
855
+ return "matchedBy" in value || Object.values(value).some(hasAnsweredPattern);
856
+ }
857
+ const answeredPatternSchema = z.strictObject({
858
+ matchedBy: z.string().regex(/\S/).describe("The pattern as the runner's language writes it, e.g. `re.compile('npm t')`."),
859
+ answers: z.record(z.string(), z.boolean()).describe("Whether it matches each string the runner has been asked about.")
860
+ }).describe("A pattern the runner matches in its own language. reeval runs the matcher with it, and when the matcher tests a string not in `answers` replies with those strings (`untested`) instead of a record; the runner matches them and makes the call again with them in `answers`.");
861
+ const isPatternLike = (value) => typeof value === "object" && value !== null && !Array.isArray(value) && "regexp" in value;
862
+ const matcherArgSchema = z.union([
863
+ patternSchema,
864
+ answeredPatternSchema,
865
+ jsonValueSchema
866
+ ]).superRefine((value, ctx) => {
867
+ if (!isPatternLike(value)) return;
868
+ for (const issue of patternSchema.safeParse(value).error?.issues ?? []) ctx.addIssue(issue);
869
+ }).describe("One argument to a matcher: a JSON value, a regular expression as `{ regexp, flags? }`, or a pattern in the runner's own language as `{ matchedBy, answers }`.");
870
+ const matcherSubjectSchema = z.union([
871
+ z.strictObject({ turn: turnIndex }),
872
+ z.strictObject({ attempt: z.literal(true).describe("The attempt itself, `c`.") }),
873
+ z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
874
+ z.strictObject({
875
+ exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
876
+ result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
877
+ }),
878
+ z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
879
+ ]).describe("What `expect` was given: a turn `c.send` returned, the attempt (for its files and latest turn), a value, such as an `ExecResult`, or a command or file reeval reads for it.");
880
+ const nonEmpty = z.string().regex(/\S/, "must not be empty");
881
+ const CAPTURE_NAME = new RegExp(`^[a-z0-9][a-z0-9_-]{0,99}$`);
882
+ const CAPTURE_NAME_RULE = `must be 1 to 100 of a-z, 0-9, _ and -, starting with a letter or digit`;
883
+ const portSchema = z.number().int().min(1, "a port from 1 to 65535").max(65535, "a port from 1 to 65535");
884
+ const millisSchema = z.number().positive("more than zero milliseconds").max(MAX_TIMER_MS, TIMER_LIMIT);
885
+ const captureWaitSchema = z.enum(["load", "idle"]);
886
+ const waitSchema = z.union([captureWaitSchema, z.strictObject({ ms: millisSchema })]);
887
+ const captureInstructionSchema = z.union([nonEmpty, z.strictObject({ wait: waitSchema })]);
888
+ const captureCallDataSchema = z.strictObject({
889
+ name: z.string().regex(CAPTURE_NAME, CAPTURE_NAME_RULE).optional(),
890
+ path: z.string().optional(),
891
+ viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
892
+ timeout: durationSchema.optional(),
893
+ wait: waitSchema.optional(),
894
+ instructions: z.array(captureInstructionSchema).optional().describe("Journey: goal strings and `{ wait }` beats, run by the vision model."),
895
+ model: z.string().optional()
896
+ }).describe("`c.capture()`'s camera and path as data. Default `path` is `/`; default `name` is derived from the path. A test's `fakeInstructions` journey is TypeScript-only.");
897
+ const handleId = nonEmpty.describe("A background process, as an exec or serve reply named it.");
898
+ const judgeOnSchema = z.union([z.strictObject({ turn: turnIndex }), z.strictObject({ value: jsonValueSchema.describe("Graded as the output.") })]);
899
+ const judgeCallDataSchema = z.strictObject({
900
+ model: z.union([judgeModelNameSchema, fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge with a numeric score."),
901
+ ...judgeFields,
902
+ ...sharedInputFields,
903
+ contents: z.array(nonEmpty).optional(),
904
+ turns: turnsSchema.optional(),
905
+ on: judgeOnSchema.optional().describe("The turn `task`, `output`, `transcript` and `diff` read instead of the latest: an earlier turn, or a value graded as the output."),
906
+ artifacts: z.union([z.literal(true), z.array(z.union([nonEmpty, captureSchema]))]).optional().describe("Captures this call sends the judge as image file parts: a capture `c.capture` returned, or its name, or `true` for every capture so far. The provider that cannot take files fails the call.")
907
+ }).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score, the inputs it shows the judge (at least one turned on) and `on`.");
908
+ /** An observation's name: one a query, a tally line and a judge's input can carry as it is. */
909
+ const OBSERVE_NAME = /^[A-Za-z0-9_.:/-]{1,64}$/;
910
+ const callSiteSchema = z.strictObject({
911
+ file: nonEmpty.describe("The eval file: absolute, or relative to the project root."),
912
+ line: z.number().int().min(1).describe("From 1.")
913
+ }).describe("Where the eval made the call, which the record names beside the check's outcome.");
914
+ const attemptCallSchema = z.discriminatedUnion("op", [
915
+ z.strictObject({
916
+ op: z.literal("send"),
917
+ prompt: z.string().describe("The turn's prompt.")
918
+ }).describe("`c.send(prompt)`: the next turn of the agent session. Replies with a turn."),
919
+ z.strictObject({
920
+ op: z.literal("exec"),
921
+ command: nonEmpty.describe("A shell command, run in the workspace."),
922
+ background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
923
+ ...execRetrySchema.shape
924
+ }).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
925
+ z.strictObject({
926
+ op: z.literal("serve"),
927
+ command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
928
+ port: portSchema.optional().describe("The container port. Omit it and reeval allocates one.")
929
+ }).describe("`c.serve(command, port?)`: a background process, once it listens, and its port."),
930
+ z.strictObject({
931
+ op: z.literal("kill"),
932
+ handle: handleId
933
+ }).describe("`handle.kill()`: stops a background process. Killing one that's gone is fine."),
934
+ z.strictObject({
935
+ op: z.literal("capture"),
936
+ target: z.union([z.strictObject({ port: portSchema }), z.strictObject({ handle: handleId })]).describe("A container port, or a serve handle by its id."),
937
+ options: captureCallDataSchema.optional()
938
+ }).describe("`c.capture(target, options)`: screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`. Replies with the capture."),
939
+ z.strictObject({
940
+ op: z.literal("readFile"),
941
+ path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory.")
942
+ }).describe("`c.readFile(path)`. Replies with its contents, or null when there is no such file."),
943
+ z.strictObject({
944
+ op: z.literal("writeFile"),
945
+ path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory."),
946
+ contents: z.string().describe("The file's contents, in `encoding`."),
947
+ encoding: z.enum(["utf8", "base64"]).optional().describe("How `contents` is encoded; base64 carries bytes. Default `utf8`.")
948
+ }).describe("`c.writeFile(path, contents)`, creating its directories."),
949
+ z.strictObject({
950
+ op: z.literal("observe"),
951
+ name: z.string().regex(OBSERVE_NAME, "must be 1 to 64 letters, digits, _ . : / or -").describe("What the value is of, e.g. `installed`."),
952
+ value: jsonValueSchema.describe("Any JSON, at most 4 KB once serialized.")
953
+ }).describe("`c.observe(name, value)`: records a fact about the attempt that never affects its outcome. A name observed again keeps the last value. Fails after the body's `done`."),
954
+ z.strictObject({
955
+ op: z.literal("expect"),
956
+ subject: matcherSubjectSchema,
957
+ matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a matcher name such as toHaveFile, without \"not.\" (set not: true instead)").describe("The matcher's name, e.g. `toHaveFile` or `toBe`, without `not.`."),
958
+ args: z.array(matcherArgSchema).describe("The matcher's arguments."),
959
+ not: z.boolean().describe("Called as `expect(…).not`."),
960
+ soft: z.boolean().describe("Called as `expect`: a failure fails the attempt at its end and the body carries on. False for `expect.require`, whose failure the runner raises, ending the body."),
961
+ label: z.string().optional().describe("`expect`'s second argument."),
962
+ at: callSiteSchema.optional(),
963
+ poll: z.strictObject({
964
+ interval: millisSchema.optional().describe("Milliseconds between tries. Default 50."),
965
+ timeout: millisSchema.optional().describe("Milliseconds before it fails with the last error. Default 1000.")
966
+ }).optional().describe("`expect.poll`: reeval re-reads the subject and retries the matcher until it passes, recording only the final outcome. The subject must be one reeval can re-read: the attempt, a command or a file.")
967
+ }).superRefine((call, ctx) => {
968
+ if (call.poll && "value" in call.subject) ctx.addIssue({
969
+ code: "custom",
970
+ path: ["subject"],
971
+ message: "can't be polled: reeval can't re-read a value. Poll the attempt, an exec or a readFile"
972
+ });
973
+ if (call.poll && "exec" in call.subject && call.subject.result) ctx.addIssue({
974
+ code: "custom",
975
+ path: ["subject", "result"],
976
+ message: "can't be polled: drop result so reeval runs the command on each try"
977
+ });
978
+ if (call.poll && call.args.some(hasAnsweredPattern)) ctx.addIssue({
979
+ code: "custom",
980
+ path: ["args"],
981
+ message: "can't be polled with a { matchedBy } pattern, whose answers can't follow the subject from try to try: poll with a string, or check once"
982
+ });
983
+ }).describe("A matcher call: `expect(subject, label).not?.matcher(...args)`. Replies with its record."),
984
+ z.strictObject({
985
+ op: z.literal("record"),
986
+ matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a name such as assert").describe("What made the check in the eval's language, e.g. `assert`."),
987
+ subject: z.string().optional().describe("What it checked, as the eval wrote it (`result.ok`): names the check when it has no label."),
988
+ label: z.string().optional().describe("The check's own name, the same whether it passes or fails."),
989
+ status: z.enum(["pass", "fail"]),
990
+ message: z.string().optional().describe("Why it failed."),
991
+ soft: z.boolean().optional().describe("The body carried on after a failure, as after an `expect`. Default false: the check raised, as Python's `assert` does."),
992
+ at: callSiteSchema.optional()
993
+ }).describe("A check the eval's own language made, such as Python's `assert`: recorded as a matcher call is, so it's named, counted and compared the same. Replies with its record."),
994
+ z.strictObject({
995
+ op: z.literal("judge"),
996
+ question: nonEmpty.describe("A yes/no question about the inputs `options` turns on."),
997
+ options: judgeCallDataSchema
998
+ }).describe("`c.judge(question, options)`. Replies with its verdict, or a judge failure when the provider failed.")
999
+ ]).describe("One call an eval's body makes on its attempt. The sandbox and the agent session start on the first call that needs them; there's no call to start them.");
1000
+ const attemptErrorSchema = z.looseObject({
1001
+ name: z.string().describe("The error's class, e.g. `UsageError` or `AssertionError`."),
1002
+ message: z.string()
1003
+ }).describe("Why a call failed, in reeval's words.");
1004
+ const id = nonEmpty.describe("Pairs a reply with its request.");
1005
+ const attemptRequestSchema = z.strictObject({
1006
+ id,
1007
+ call: attemptCallSchema
1008
+ }).describe("A call, numbered so its reply can find it.");
1009
+ z.union([
1010
+ z.discriminatedUnion("op", [
1011
+ z.strictObject({
1012
+ id,
1013
+ ok: z.literal(true),
1014
+ op: z.literal("send"),
1015
+ turn: turnDataSchema
1016
+ }),
1017
+ z.strictObject({
1018
+ id,
1019
+ ok: z.literal(true),
1020
+ op: z.literal("exec"),
1021
+ result: z.union([execResultSchema, z.strictObject({ handle: handleId }).describe("A background process, still running.")])
1022
+ }),
1023
+ z.strictObject({
1024
+ id,
1025
+ ok: z.literal(true),
1026
+ op: z.literal("serve"),
1027
+ handle: handleId,
1028
+ port: z.number().describe("The container port it listens on.")
1029
+ }),
1030
+ z.strictObject({
1031
+ id,
1032
+ ok: z.literal(true),
1033
+ op: z.literal("kill")
1034
+ }),
1035
+ z.strictObject({
1036
+ id,
1037
+ ok: z.literal(true),
1038
+ op: z.literal("capture"),
1039
+ capture: captureSchema
1040
+ }),
1041
+ z.strictObject({
1042
+ id,
1043
+ ok: z.literal(true),
1044
+ op: z.literal("readFile"),
1045
+ contents: z.string().nullable().describe("Null when there is no such file.")
1046
+ }),
1047
+ z.strictObject({
1048
+ id,
1049
+ ok: z.literal(true),
1050
+ op: z.literal("writeFile")
1051
+ }),
1052
+ z.strictObject({
1053
+ id,
1054
+ ok: z.literal(true),
1055
+ op: z.literal("observe")
1056
+ }),
1057
+ z.strictObject({
1058
+ id,
1059
+ ok: z.literal(true),
1060
+ op: z.literal("expect"),
1061
+ assertion: assertionRecordSchema
1062
+ }),
1063
+ z.strictObject({
1064
+ id,
1065
+ ok: z.literal(true),
1066
+ op: z.literal("record"),
1067
+ assertion: assertionRecordSchema
1068
+ }),
1069
+ z.strictObject({
1070
+ id,
1071
+ ok: z.literal(true),
1072
+ op: z.literal("judge"),
1073
+ verdict: z.union([verdictSchema, judgeFailureSchema])
1074
+ })
1075
+ ]),
1076
+ z.strictObject({
1077
+ id,
1078
+ ok: z.literal(true),
1079
+ op: z.literal("expect"),
1080
+ untested: z.record(z.string(), z.array(z.string())).describe("The strings the matcher tested that a `{ matchedBy, answers }` argument has no answer for, by its `matchedBy`. Nothing was recorded: answer them and make the call again.")
1081
+ }),
1082
+ z.strictObject({
1083
+ id,
1084
+ ok: z.literal(false),
1085
+ op: z.enum([
1086
+ "send",
1087
+ "exec",
1088
+ "readFile",
1089
+ "writeFile",
1090
+ "observe",
1091
+ "expect",
1092
+ "record",
1093
+ "judge",
1094
+ "serve",
1095
+ "kill",
1096
+ "capture"
1097
+ ]),
1098
+ error: attemptErrorSchema
1099
+ })
1100
+ ]).describe("reeval's reply to a call: its result, or the error it threw. A failed matcher is a result (its record says `fail`), not an error, unless the matcher couldn't run.");
1101
+ const cellRoleSchema = z.enum(["control", "variant"]);
1102
+ const cellSchema = z.strictObject({
1103
+ key: z.string(),
1104
+ harness: z.string(),
1105
+ environment: z.string(),
1106
+ sandbox: z.string(),
1107
+ role: cellRoleSchema,
1108
+ variant: z.string().optional(),
1109
+ traits: traitsSchema.optional()
1110
+ });
1111
+ z.strictObject({
1112
+ type: z.literal("start"),
1113
+ attempt: z.strictObject({
1114
+ id: nonEmpty,
1115
+ cell: cellSchema,
1116
+ n: z.number().int().min(1, "a run number, from 1").describe("The run number, from 1.")
1117
+ }).describe("`c.attempt`: which cell of the matrix this attempt is, and its run number."),
1118
+ file: nonEmpty.describe("The eval file, relative to the project."),
1119
+ body: nonEmpty.describe("The eval's `body`, as its declaration named it.")
1120
+ }).describe("reeval to the runner: run this eval's body for this attempt. Nothing has started yet: the sandbox and the agent session start on the body's first call that needs them, so its own code runs first. The runner sends its calls, then `done`.");
1121
+ z.strictObject({
1122
+ type: z.literal("abort"),
1123
+ reason: z.enum(["timeout", "canceled"]).describe("`timeout`: the eval's `timeout` ran out. `canceled`: the run was stopped."),
1124
+ message: z.string().describe("reeval's sentence, e.g. `Eval timed out after 600s`.")
1125
+ }).describe("reeval to the runner: the attempt was aborted, as `c.signal` aborts. Calls in flight reply with errors; the runner should stop the body and send `done`.");
1126
+ z.strictObject({
1127
+ type: z.literal("done"),
1128
+ error: attemptErrorSchema.optional().describe("What the body threw, if it threw. A failed `expect.require` is one.")
1129
+ }).describe("The runner to reeval: the eval's body has finished. The attempt's checks and record follow.");
1130
+ const CALL_KEYS = Object.fromEntries(attemptCallSchema.options.map((o) => [o.shape.op.value, Object.keys(o.shape)]));
1131
+ const OTHER_KEYS = [
1132
+ "id",
1133
+ "call",
1134
+ "turn",
1135
+ "attempt",
1136
+ "value",
1137
+ "regexp",
1138
+ "flags",
1139
+ "matchedBy",
1140
+ "answers",
1141
+ ...Object.keys(judgeCallDataSchema.shape),
1142
+ ...Object.keys(captureSchema.shape),
1143
+ ...Object.keys(captureCallDataSchema.shape),
1144
+ "port",
1145
+ "handle",
1146
+ "exec",
1147
+ "result",
1148
+ "readFile",
1149
+ "interval",
1150
+ "timeout",
1151
+ "ms"
1152
+ ];
1153
+ /** What's wrong with an attempt request a runner sent, by the field's path, or undefined. */
1154
+ const attemptRequestProblem = (value) => dataProblem(attemptRequestSchema, value, "The attempt request", (object) => typeof object.op === "string" ? CALL_KEYS[object.op] ?? [] : OTHER_KEYS);
1155
+ // src/attempt/observe.ts
1156
+ /** The most JSON one value may serialize to, masked. */
1157
+ const MAX_OBSERVED_BYTES = 4096;
1158
+ /** The deepest a value may nest arrays and objects, so a check of it can't run out of stack. */
1159
+ const MAX_DEPTH = 64;
1160
+ const at = (path) => path ? ` at ${path}` : "";
1161
+ /** Why `value` isn't JSON, naming where, or undefined. */
1162
+ function notJson(value, path, seen, depth = 0) {
1163
+ if (value === null || typeof value === "string" || typeof value === "boolean") return void 0;
1164
+ if (typeof value === "number") return Number.isFinite(value) ? void 0 : `${value}${at(path)} isn't a finite number`;
1165
+ if (typeof value !== "object") return `a ${typeof value}${at(path)} isn't JSON`;
1166
+ if (seen.has(value)) return `the value refers to itself${at(path)}`;
1167
+ if (depth >= MAX_DEPTH) return `it nests deeper than ${MAX_DEPTH} levels${at(path)}`;
1168
+ const proto = Object.getPrototypeOf(value);
1169
+ if (!Array.isArray(value) && proto !== Object.prototype && proto !== null) return `a ${value.constructor?.name ?? "class instance"}${at(path)} isn't plain JSON`;
1170
+ seen.add(value);
1171
+ const entries = Array.isArray(value) ? Array.from(value, (v, i) => [`${path}[${i}]`, v]) : Object.entries(value).map(([k, v]) => [path ? `${path}.${k}` : k, v]);
1172
+ for (const [p, v] of entries) {
1173
+ const problem = notJson(v, p, seen, depth + 1);
1174
+ if (problem) return problem;
1175
+ }
1176
+ seen.delete(value);
1177
+ }
1178
+ /**
1179
+ * `value` with each number whose digits are a secret's masked too: `redactValue` leaves numbers
1180
+ * as they are, but an observed value goes to a judge's or question's model as it's recorded.
1181
+ */
1182
+ function maskNumbers(value) {
1183
+ if (typeof value === "number") return redactSecrets(String(value)) === String(value) ? value : REDACTED;
1184
+ if (Array.isArray(value)) return value.map(maskNumbers);
1185
+ if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, maskNumbers(v)]));
1186
+ return value;
1187
+ }
1188
+ /**
1189
+ * An observation's name as it's recorded and shown: masked as its value is. A name that holds a
1190
+ * secret is a value too, and a judge's or question's list of names is matched against this.
1191
+ */
1192
+ const observedName = (name) => redactSecrets(name);
1193
+ /** Each attempt's names as its body gave them, by the name they're recorded as. */
1194
+ const givenNames = /* @__PURE__ */ new WeakMap();
1195
+ /**
1196
+ * Whether this attempt observed `name` as given: its masked name alone may be another secret's.
1197
+ * An attempt that never called `c.observe` has observed nothing.
1198
+ */
1199
+ const observedHere = (a, name) => givenNames.get(a)?.get(observedName(name)) === name;
1200
+ /**
1201
+ * `c.observe`: keeps the last value per name, masked, with the turn it came after. Two names that
1202
+ * mask to the same name are refused rather than renamed: a name is the key that tallies, compare
1203
+ * and `attempt_values` group by across attempts, and a numbered stand-in would depend on the
1204
+ * order of each attempt's calls, joining values that don't belong together.
1205
+ */
1206
+ function observer(a) {
1207
+ const raw = /* @__PURE__ */ new Map();
1208
+ givenNames.set(a, raw);
1209
+ return (given, value) => {
1210
+ if (typeof given !== "string" || !OBSERVE_NAME.test(given)) throw new UsageError(`c.observe() needs a name of 1 to 64 letters, digits, _ . : / or - (got ${JSON.stringify(typeof given === "string" ? observedName(given) : given)})`);
1211
+ const name = observedName(given);
1212
+ const earlier = raw.get(name);
1213
+ if (earlier !== void 0 && earlier !== given) throw new UsageError(`c.observe("${name}") names a secret, and its masked name is another observation's: name the value without the secret`);
1214
+ if (a.ended) throw new UsageError(`c.observe("${name}") came after the eval's body returned`);
1215
+ if (a.signal.aborted) throw new UsageError(`c.observe("${name}") came after the attempt timed out or was canceled`);
1216
+ const problem = notJson(value, "", /* @__PURE__ */ new Set());
1217
+ if (problem) throw new UsageError(`c.observe("${name}") takes JSON: ${problem}`);
1218
+ const masked = maskNumbers(redactValue(value));
1219
+ const bytes = Buffer.byteLength(JSON.stringify(masked));
1220
+ if (bytes > 4096) throw new UsageError(`c.observe("${name}") takes at most ${MAX_OBSERVED_BYTES} bytes of JSON (got ${bytes}): observe a count or a summary instead`);
1221
+ a.record.observations ??= [];
1222
+ const list = a.record.observations;
1223
+ const same = list.find((o) => o.name === name);
1224
+ if (!same && list.length >= 50) throw new UsageError(`c.observe("${name}") is one name too many: an attempt observes at most 50`);
1225
+ raw.set(name, given);
1226
+ const observation = {
1227
+ name,
1228
+ value: masked,
1229
+ turn: a.turns.length
1230
+ };
1231
+ if (same) Object.assign(same, observation);
1232
+ else list.push(observation);
1233
+ };
1234
+ }
1235
+ // src/judge/state-options.ts
1236
+ /**
1237
+ * What one call turned on, every key set (`false` when off), in the record's key order, so equal
1238
+ * selections serialize the same.
1239
+ */
1240
+ function resolveStateOptions(call) {
1241
+ const observations = observedNames(call.observations);
1242
+ return {
1243
+ task: call.task ?? false,
1244
+ output: call.output ?? false,
1245
+ files: call.files ?? false,
1246
+ diff: call.diff ?? false,
1247
+ contents: call.contents ?? [],
1248
+ transcript: call.transcript ?? false,
1249
+ turns: call.turns ?? "last",
1250
+ checks: call.checks ?? false,
1251
+ ...observationsOn(observations) ? { observations } : {}
1252
+ };
1253
+ }
1254
+ /** A list of observation names as they're recorded, so a secret in one is masked. */
1255
+ const observedNames = (value) => Array.isArray(value) ? value.map(observedName) : value;
1256
+ /** `observations` when it turns the input on: `true` or a non-empty list. */
1257
+ const observationsOn = (value) => value === true || Array.isArray(value) && value.length > 0;
1258
+ /** `inner` over `outer`, field by field. */
1259
+ function mergeJudge(outer, inner) {
1260
+ return inner ? {
1261
+ ...outer,
1262
+ ...inner
1263
+ } : outer;
1264
+ }
1745
1265
  // src/compare/state.ts
1746
1266
  /** The error a per-attempt answer carries when the run stopped before a model answered it. */
1747
1267
  const STOPPED_FIRST = "the run stopped before it was answered";
@@ -1764,6 +1284,7 @@ const KEYS = Object.keys(UNRECORDED_QUESTION_STATE);
1764
1284
  * question didn't turn on is off, and `turns` is `'all'` unless it says `'last'`.
1765
1285
  */
1766
1286
  function resolveQuestionState(state) {
1287
+ const observations = observedNames(state?.observations);
1767
1288
  return {
1768
1289
  task: state?.task ?? false,
1769
1290
  output: state?.output ?? false,
@@ -1771,7 +1292,8 @@ function resolveQuestionState(state) {
1771
1292
  diff: state?.diff ?? false,
1772
1293
  transcript: state?.transcript ?? false,
1773
1294
  turns: state?.turns ?? "all",
1774
- checks: state?.checks ?? false
1295
+ checks: state?.checks ?? false,
1296
+ ...observationsOn(observations) ? { observations } : {}
1775
1297
  };
1776
1298
  }
1777
1299
  /** `inner` over `outer`, field by field. */
@@ -1792,22 +1314,22 @@ function withQuestionDefaults(q, defaults) {
1792
1314
  const isOff = (value) => value === false || Array.isArray(value) && !value.length;
1793
1315
  /** The inputs a question shows, in order, plus `artifacts` when it sends captures. */
1794
1316
  function questionShows(state, artifacts) {
1795
- const on = KEYS.filter((k) => k !== "turns" && !isOff(state[k]));
1317
+ const on = [...KEYS.filter((k) => k !== "turns" && !isOff(state[k])), ...state.observations ? ["observations"] : []];
1796
1318
  return artifacts === void 0 ? on : [...on, "artifacts"];
1797
1319
  }
1798
1320
  // src/contract/compare.ts
1799
1321
  const compareGateSchema = z.enum([
1800
1322
  "variants",
1801
1323
  "all",
1802
- "none"
1324
+ "observe"
1803
1325
  ]);
1804
1326
  /** The compare object, with environments checked by `environment`. */
1805
1327
  const compareSchema = (environment) => z.strictObject({
1806
- baseline: environment.optional(),
1328
+ control: environment,
1807
1329
  variants: z.array(environment).min(1),
1808
1330
  gate: compareGateSchema.optional(),
1809
1331
  minLift: z.number().min(-1).max(1).optional(),
1810
- harness: z.array(z.union([z.string(), harnessDescriptorSchema])).optional()
1332
+ harnesses: z.array(z.union([z.string(), harnessDescriptorSchema])).optional()
1811
1333
  });
1812
1334
  compareSchema(z.custom());
1813
1335
  const describe$1 = (value) => {
@@ -1825,14 +1347,17 @@ function explain(where) {
1825
1347
  const [key, index] = issue.path;
1826
1348
  const v = spec[key];
1827
1349
  switch (key) {
1828
- case "baseline": return `compare.baseline in ${where} must be an environment from defineEnvironment (got ${describe$1(v)}). It is the control; variants are not layered on it or on the eval's environment. Leave it out to use the eval's environment as the control.`;
1350
+ case "control":
1351
+ if (v === void 0) return noControl(where);
1352
+ if (Array.isArray(v)) return `compare.control in ${where} takes one environment (got ${v.map((e) => e?.name ?? describe$1(e)).join(", ")}). For a grid without a compare, use environments.`;
1353
+ return `compare.control in ${where} must be an environment from defineEnvironment (got ${describe$1(v)}). It is what each variant is measured against; variants are not layered on it.`;
1829
1354
  case "gate": return `compare.gate in ${where} is ${oneOf(compareGateSchema.options)} (got ${describe$1(v)})`;
1830
1355
  case "minLift": return `compare.minLift in ${where} is points from -1 to 1, e.g. minLift: 0.1 (got ${describe$1(v)})`;
1831
- case "harness": {
1356
+ case "harnesses": {
1832
1357
  const item = index === void 0 ? void 0 : v[index];
1833
1358
  const kind = item?.kind;
1834
1359
  const problem = kind && kind in FACTORY ? harnessProblem(item) : void 0;
1835
- return problem ? `compare.harness in ${where}: ${problem}` : `compare.harness in ${where} must be a list of harness ids, e.g. harness: ['cursor:composer-2.5'] (got ${describe$1(index === void 0 ? v : item)})`;
1360
+ return problem ? `compare.harnesses in ${where}: ${problem}` : `compare.harnesses in ${where} must be a list of harness ids, e.g. harnesses: ['cursor:composer-2.5'] (got ${describe$1(index === void 0 ? v : item)})`;
1836
1361
  }
1837
1362
  default: return index === void 0 ? `compare has no variants in ${where}. Give at least one environment.` : `compare.variants in ${where} must be environments from defineEnvironment (got ${describe$1(v[index])}). Variants share the eval's prompt and checks; if a variant needs different checks, write a separate eval.`;
1838
1363
  }
@@ -1840,375 +1365,23 @@ function explain(where) {
1840
1365
  }
1841
1366
  /** The order the hand-written checks ran in, so the first refusal stays the same. */
1842
1367
  const ORDER = [
1843
- "baseline",
1368
+ "control",
1844
1369
  "gate",
1845
1370
  "minLift",
1846
- "harness",
1371
+ "harnesses",
1847
1372
  "variants"
1848
1373
  ];
1374
+ const noControl = (where) => `compare in ${where} has no control. Name it: compare: { control: app, variants: [...] }.`;
1849
1375
  /**
1850
1376
  * What's wrong with a compare object whose environments `environment` checks, or undefined.
1851
1377
  * `where` names the group or eval it's set on.
1852
1378
  */
1853
1379
  function compareProblem(value, where, environment) {
1854
- if (Array.isArray(value)) return `compare in ${where} must be an object, not a list. Use compare: { baseline, variants: [...] }.`;
1855
- if (!isObjectLike(value) || !Array.isArray(value.variants)) return `compare in ${where} takes { variants, baseline? } (got ${describe$1(value)}). Use compare: { variants: [a, b], baseline?, gate?, minLift?, harness? }.`;
1380
+ if (Array.isArray(value)) return `compare in ${where} must be an object, not a list. Use compare: { control, variants: [...] }.`;
1381
+ if (isObjectLike(value) && "baseline" in value) return `Unknown compare option "baseline" in ${where}. Did you mean "control"?`;
1382
+ if (!isObjectLike(value) || !Array.isArray(value.variants)) return `compare in ${where} takes { control, variants } (got ${describe$1(value)}). Use compare: { control: a, variants: [b, c], gate?, minLift?, harnesses? }.`;
1856
1383
  return refusal(compareSchema(environment), value, explain(where), { order: ORDER });
1857
1384
  }
1858
- const POLL_MS = 200;
1859
- /**
1860
- * Prints `alive` while `pid` runs, then the sandbox's TCP tables, or `noproc` without them
1861
- * (or without `tables`: the fake sandbox shares the host's, where other processes bind too).
1862
- */
1863
- function probeScript(pid, tables = true) {
1864
- return `${pid ? `kill -0 ${pid} 2>/dev/null && echo alive; ` : ""}${tables ? "if [ -r /proc/net/tcp ]; then cat /proc/net/tcp /proc/net/tcp6 2>/dev/null; else echo noproc; fi" : "echo noproc"}; exit 0`;
1865
- }
1866
- /** `0100007F` → `127.0.0.1`; 32 hex digits → an IPv6 address. Each 32-bit word is host-endian. */
1867
- function procAddress(hex) {
1868
- const bytes = (hex.match(/.{8}/g) ?? []).flatMap((w) => (w.match(/../g) ?? []).reverse().map((b) => Number.parseInt(b, 16)));
1869
- if (bytes.length === 4) return bytes.join(".");
1870
- if (bytes.slice(0, 10).every((b) => b === 0) && bytes[10] === 255 && bytes[11] === 255) return bytes.slice(12).join(".");
1871
- const groups = [];
1872
- for (let i = 0; i < 16; i += 2) groups.push(((bytes[i] ?? 0) << 8 | (bytes[i + 1] ?? 0)).toString(16));
1873
- const full = groups.join(":");
1874
- const longest = (full.match(/(^|:)0(:0)+(:|$)/g) ?? []).reduce((a, b) => b.length > a.length ? b : a, "");
1875
- return longest ? full.replace(longest, "::") : full;
1876
- }
1877
- /** A listener the tunnel reaches: on loopback or every address, IPv4 or IPv6. */
1878
- const reachable = (address) => address === "0.0.0.0" || address === "::" || address === "::1" || address.startsWith("127.");
1879
- function parseProbe(stdout) {
1880
- const lines = stdout.split("\n");
1881
- const alive = lines.includes("alive");
1882
- if (lines.includes("noproc")) return {
1883
- proc: false,
1884
- alive
1885
- };
1886
- const listeners = [];
1887
- for (const l of lines) {
1888
- const m = /^\s*\d+:\s+([0-9A-F]+):([0-9A-F]{4})\s+\S+\s+0A\s/i.exec(l);
1889
- const address = m ? procAddress(m[1]) : "";
1890
- if (m && reachable(address)) listeners.push({
1891
- port: Number.parseInt(m[2], 16),
1892
- address
1893
- });
1894
- }
1895
- return {
1896
- proc: true,
1897
- alive,
1898
- listeners
1899
- };
1900
- }
1901
- /** The ports listening now that weren't before the command started, ascending. */
1902
- const newPorts = (listeners, before) => [...new Set(listeners.map((l) => l.port).filter((p) => !before.has(p)))].sort((a, b) => a - b);
1903
- /** The address to record for `port`: IPv4 first, since that's where the tunnel connects first. */
1904
- const addressFor = (listeners, port) => {
1905
- const all = listeners.filter((l) => l.port === port).map((l) => l.address);
1906
- return all.find((a) => !a.includes(":")) ?? all.find((a) => a === "::") ?? all[0] ?? "";
1907
- };
1908
- /** `want` is taken whoever listens on it: reeval allocated it, or the eval named it. */
1909
- function decide(s) {
1910
- const { listeners, alive } = s.probe;
1911
- const ports = newPorts(listeners, s.before);
1912
- if (listeners.some((l) => l.port === s.want)) return {
1913
- kind: "port",
1914
- port: s.want,
1915
- address: addressFor(listeners, s.want)
1916
- };
1917
- const late = s.elapsedMs >= s.timeoutMs;
1918
- if (!s.given && ports.length && (s.stableMs >= 3e3 || late || !alive)) {
1919
- if (ports.length === 1) {
1920
- const port = ports[0];
1921
- return {
1922
- kind: "port",
1923
- port,
1924
- address: addressFor(listeners, port)
1925
- };
1926
- }
1927
- return {
1928
- kind: "several",
1929
- ports
1930
- };
1931
- }
1932
- if (!alive) return { kind: "exited" };
1933
- return late ? {
1934
- kind: "timeout",
1935
- ports
1936
- } : { kind: "wait" };
1937
- }
1938
- const plain = (output) => output.replace(/\x1b\[[0-9;?]*[A-Za-z]/g, "");
1939
- /** The URLs `output` prints on one of `ports`, as Vite's `Local: http://localhost:5173/`. */
1940
- function printedUrls(output, ports) {
1941
- const found = /* @__PURE__ */ new Map();
1942
- for (const m of plain(output).matchAll(/https?:\/\/(?:\[[0-9a-f:]+\]|[^\s/:[\]]+):(\d+)[^\s'"]*/gi)) {
1943
- const port = Number(m[1]);
1944
- if (ports.includes(port) && !found.has(m[0])) found.set(m[0], port);
1945
- }
1946
- return [...found].map(([url, port]) => ({
1947
- url,
1948
- port
1949
- }));
1950
- }
1951
- /** Whether something on the host accepts a TCP connection on `port`, IPv4 or IPv6 loopback. */
1952
- async function hostListening(port) {
1953
- for (const host of ["127.0.0.1", "::1"]) if (await new Promise((resolve) => {
1954
- const socket = connect({
1955
- port,
1956
- host
1957
- });
1958
- socket.once("connect", () => {
1959
- socket.destroy();
1960
- resolve(true);
1961
- });
1962
- socket.once("error", () => resolve(false));
1963
- })) return host;
1964
- }
1965
- const sleep = (ms, signal) => new Promise((resolve) => {
1966
- const done = () => {
1967
- clearTimeout(timer);
1968
- signal?.removeEventListener("abort", done);
1969
- resolve();
1970
- };
1971
- const timer = setTimeout(done, ms);
1972
- signal?.addEventListener("abort", done, { once: true });
1973
- });
1974
- const listOf = (items) => items.length > 1 ? `${items.slice(0, -1).join(", ")} and ${items.at(-1)}` : String(items[0]);
1975
- /**
1976
- * Polls until something listens on `want`, or, without a given port, until the ports that are
1977
- * new since the command started have settled on one. Throws, with the command's last output,
1978
- * when they settle on several, it exits, or nothing listens in time.
1979
- */
1980
- async function awaitListening(command, w) {
1981
- const timeoutMs = w.timeoutMs ?? 12e4;
1982
- const started = Date.now();
1983
- let last = "";
1984
- let changed = started;
1985
- const output = async () => plain(await w.output().catch(() => ""));
1986
- const fail = async (why, out) => {
1987
- const tail = (out ?? await output()).trimEnd().split("\n").slice(-20).join("\n").trim();
1988
- return /* @__PURE__ */ new Error(`c.serve(${JSON.stringify(command)}) ${why}${tail ? `. Its last output:\n${tail}` : ""}`);
1989
- };
1990
- for (;;) {
1991
- w.signal?.throwIfAborted();
1992
- const probe = await w.probe();
1993
- const now = Date.now();
1994
- if (!probe.proc) {
1995
- const address = await (w.hostListening ?? hostListening)(w.want);
1996
- if (address !== void 0) return {
1997
- port: w.want,
1998
- address,
1999
- detected: false,
2000
- readyMs: now - started
2001
- };
2002
- if (!probe.alive) throw await fail(`exited before it listened on ${w.want}`);
2003
- if (now - started >= timeoutMs) throw await fail(`didn't listen on ${w.want} within ${timeoutMs / 1e3}s. This sandbox can't list the ports it listens on, so it must listen on PORT or the port passed`);
2004
- } else {
2005
- const key = newPorts(probe.listeners, w.before).join(",");
2006
- if (key !== last) {
2007
- last = key;
2008
- changed = now;
2009
- }
2010
- const d = decide({
2011
- want: w.want,
2012
- given: w.given,
2013
- probe,
2014
- before: w.before,
2015
- stableMs: now - changed,
2016
- elapsedMs: now - started,
2017
- timeoutMs
2018
- });
2019
- if (d.kind === "port") return {
2020
- port: d.port,
2021
- address: d.address,
2022
- detected: d.port !== w.want,
2023
- readyMs: now - started
2024
- };
2025
- if (d.kind === "several") {
2026
- const out = await output();
2027
- const urls = printedUrls(out, d.ports);
2028
- const printed = urls.length ? ` It printed ${listOf(urls.map((u) => u.url))}.` : "";
2029
- const hint = urls[0]?.port ?? d.ports[0];
2030
- throw await fail(`listens on ports ${listOf(d.ports)}, and not on PORT (${w.want}).${printed} Pass the one it serves on: c.serve(${JSON.stringify(command)}, ${hint})`, out);
2031
- }
2032
- if (d.kind === "exited") throw await fail("exited before it listened on a port");
2033
- if (d.kind === "timeout") {
2034
- const on = d.ports.length ? `; ${d.ports.length > 1 ? "ports" : "port"} ${listOf(d.ports)} started listening` : "";
2035
- throw await fail(`didn't listen on ${w.given ? `the port passed (${w.want})` : "a port"} within ${timeoutMs / 1e3}s${on}`);
2036
- }
2037
- }
2038
- await sleep(POLL_MS, w.signal);
2039
- }
2040
- }
2041
- // src/sandbox/fake.ts
2042
- /** Commands run on the host's own network, so the fake sandbox can neither restrict nor see it. */
2043
- const FAKE_EGRESS = {
2044
- enforce: "none",
2045
- record: false
2046
- };
2047
- /**
2048
- * The POSIX shell commands run in: `sh`, which Windows lacks, so there the `sh.exe` on PATH or
2049
- * Git for Windows'.
2050
- */
2051
- function posixShell(platform = process.platform, env = process.env, exists = existsSync) {
2052
- if (platform !== "win32") return "sh";
2053
- const onPath = (env.PATH ?? env.Path ?? "").split(win32.delimiter).filter(Boolean);
2054
- const git = [
2055
- env.ProgramFiles,
2056
- env["ProgramFiles(x86)"],
2057
- env.LOCALAPPDATA && win32.join(env.LOCALAPPDATA, "Programs")
2058
- ].filter((dir) => !!dir).map((dir) => win32.join(dir, "Git", "bin"));
2059
- const found = [...onPath, ...git].map((dir) => win32.join(dir, "sh.exe")).find(exists);
2060
- if (found) return found;
2061
- throw new UsageError("The fake sandbox runs commands with a POSIX sh, and there's none on this machine. Install Git for Windows (https://git-scm.com/download/win), which brings sh.exe, or run the evals in docker().");
2062
- }
2063
- const EXEC_TAG = "REEVAL_FAKE_EXEC";
2064
- const KILL_TAGGED = `for pass in 1 2 3 4 5; do
2065
- found=
2066
- for f in $(grep -lzx "${EXEC_TAG}=$1" /proc/[0-9]*/environ 2>/dev/null); do
2067
- p=\${f#/proc/}; kill -9 "\${p%/environ}" 2>/dev/null; found=1
2068
- done
2069
- [ -n "$found" ] || break
2070
- done`;
2071
- /**
2072
- * Stops a command's MSYS processes on Windows. `taskkill /t` doesn't reach them: Git's `sh.exe`
2073
- * starts bash, whose fork and exec leave each new process's Windows parent gone, so a timed-out
2074
- * `sleep 30` would run on, holding the command's pipes and working directory.
2075
- */
2076
- function killTagged(sh, tag) {
2077
- runSh(sh, KILL_TAGGED, tag);
2078
- }
2079
- /**
2080
- * Kills host processes whose cwd is the sandbox, including native `node.exe` `c.serve` starts.
2081
- * Match only the unique `reeval-sandbox-*` folder name: Git Bash `pwd` and node.exe's
2082
- * `/proc/cwd` disagree (`/tmp/...` vs `D:\\Users\\...\\Temp\\...`), and an empty
2083
- * `"$prefix"/*` is `/*`, which SIGKILLs the Windows runner (exit 2304).
2084
- */
2085
- const KILL_UNDER = `cd "$1" || exit 0
2086
- base=$(pwd -P 2>/dev/null || pwd)
2087
- name=\${base##*/}
2088
- case "$name" in
2089
- reeval-sandbox-?*) ;;
2090
- *) exit 0 ;;
2091
- esac
2092
- for f in /proc/[0-9]*/cwd; do
2093
- t=$(readlink "$f" 2>/dev/null) || continue
2094
- case "$t" in
2095
- *"$name"*) ;;
2096
- *) continue ;;
2097
- esac
2098
- p=\${f#/proc/}; p=\${p%/cwd}
2099
- [ -n "$p" ] || continue
2100
- kill -9 "$p" 2>/dev/null
2101
- wp=$(tr -d '\\r\\n' < /proc/$p/winpid 2>/dev/null)
2102
- [ -n "$wp" ] && taskkill /F /PID "$wp" >/dev/null 2>&1
2103
- done`;
2104
- function runSh(sh, script, arg) {
2105
- return new Promise((resolve) => {
2106
- const child = spawn(sh, [
2107
- "-c",
2108
- script,
2109
- "sh",
2110
- arg
2111
- ], {
2112
- stdio: "ignore",
2113
- windowsHide: true
2114
- });
2115
- const done = () => resolve();
2116
- child.on("error", done);
2117
- child.on("close", done);
2118
- });
2119
- }
2120
- /** Windows refuses to delete a directory a live process still has as cwd; retry after kills. */
2121
- async function removeTree(path) {
2122
- const deadline = Date.now() + 8e3;
2123
- for (;;) {
2124
- try {
2125
- await rm(path, {
2126
- recursive: true,
2127
- force: true
2128
- });
2129
- if (!existsSync(path)) return;
2130
- } catch {}
2131
- if (Date.now() >= deadline) {
2132
- await rm(path, {
2133
- recursive: true,
2134
- force: true
2135
- }).catch(() => {});
2136
- return;
2137
- }
2138
- await new Promise((r) => setTimeout(r, 50));
2139
- }
2140
- }
2141
- /**
2142
- * A temp directory on the host. Commands run unisolated on the host shell, so this is
2143
- * only meant for tests and for the fake harness.
2144
- */
2145
- async function createFakeSandbox(descriptor, env = {}) {
2146
- const sh = posixShell();
2147
- const parent = descriptor.dir ?? tmpdir();
2148
- await mkdir(parent, { recursive: true });
2149
- const base = await mkdtemp(join(parent, "reeval-sandbox-"));
2150
- const workdir = join(base, "work");
2151
- await mkdir(workdir);
2152
- const stateDir = `${base}.state`;
2153
- await mkdir(stateDir);
2154
- let destroyed;
2155
- const resolve = (path) => path.startsWith("~/") ? join(base, path.slice(2)) : join(workdir, path);
2156
- return {
2157
- workdir,
2158
- stateDir,
2159
- execOwn: (command, { signal } = {}) => run(sh, ["-c", command], {
2160
- cwd: workdir,
2161
- env: {
2162
- HOME: base,
2163
- ...env
2164
- },
2165
- signal
2166
- }),
2167
- exec: (command, { cwd, env: extra, signal } = {}) => {
2168
- const tag = process.platform === "win32" && signal ? randomUUID() : void 0;
2169
- const stop = () => killTagged(sh, tag);
2170
- if (tag) signal?.addEventListener("abort", stop, { once: true });
2171
- return run(sh, ["-c", cwd ? `cd ${shellQuote(cwd)} && ${command}` : command], {
2172
- cwd: workdir,
2173
- env: {
2174
- HOME: base,
2175
- ...env,
2176
- ...extra,
2177
- ...tag ? { [EXEC_TAG]: tag } : {}
2178
- },
2179
- signal
2180
- }).finally(() => signal?.removeEventListener("abort", stop));
2181
- },
2182
- home: base,
2183
- readFile: (path) => readFile(resolve(path), "utf8").catch(() => null),
2184
- getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}://${await hostListening(port) === "::1" ? "[::1]" : "127.0.0.1"}:${port}` }),
2185
- async writeFile(path, content) {
2186
- const target = resolve(path);
2187
- await mkdir(dirname(target), { recursive: true });
2188
- await writeFile(target, content);
2189
- },
2190
- destroy: () => {
2191
- destroyed ??= (async () => {
2192
- await rm(stateDir, {
2193
- recursive: true,
2194
- force: true
2195
- }).catch(() => {});
2196
- if (process.platform === "win32") {
2197
- if (existsSync(base)) await runSh(sh, KILL_UNDER, base);
2198
- await removeTree(base);
2199
- return;
2200
- }
2201
- await rm(base, {
2202
- recursive: true,
2203
- force: true
2204
- }).catch(() => {});
2205
- })().finally(() => {
2206
- destroyed = void 0;
2207
- });
2208
- return destroyed;
2209
- }
2210
- };
2211
- }
2212
1385
  // src/ui/mode.ts
2213
1386
  const set = (env, name) => {
2214
1387
  const value = env[name];
@@ -2261,16 +1434,16 @@ function resolveOutput(input) {
2261
1434
  "errors-only",
2262
1435
  "none"
2263
1436
  ]);
2264
- const outputLogs = logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only";
1437
+ const outputLogs = logs && logs !== "auto" ? logs : flags.verbose ? "full" : "none";
2265
1438
  const color = env.NO_COLOR ? false : env.FORCE_COLOR !== void 0 && env.FORCE_COLOR !== "" ? set(env, "FORCE_COLOR") : mode === "agent" ? false : input.isTTY || githubActions;
2266
1439
  return {
2267
1440
  mode,
2268
1441
  statusLine,
1442
+ interactive: statusLine && !!input.stdinTTY && outputLogs === "none",
2269
1443
  logOrder,
2270
1444
  logPrefix,
2271
1445
  outputLogs,
2272
1446
  color,
2273
- rich: statusLine && color,
2274
1447
  githubActions,
2275
1448
  heartbeatMs: mode === "agent" ? 0 : parseHeartbeat(env.REEVAL_HEARTBEAT)
2276
1449
  };
@@ -2281,11 +1454,7 @@ function resolveOutput(input) {
2281
1454
  * `egress: { allow: [...DEFAULT_EGRESS_ALLOW, 'api.example.com'] }`.
2282
1455
  */
2283
1456
  const DEFAULT_EGRESS_ALLOW = PACKAGE_HOSTS;
2284
- /** What each sandbox kind can do about egress, as its provider declares it. */
2285
- const SANDBOX_EGRESS = {
2286
- docker: DOCKER_EGRESS,
2287
- fake: FAKE_EGRESS
2288
- };
1457
+ Object.fromEntries(Object.entries(SANDBOX_KINDS).map(([kind, k]) => [kind, k.capabilities.egress]));
2289
1458
  /** The policy an attempt runs under when nothing sets one. */
2290
1459
  const DEFAULT_EGRESS = "proxy";
2291
1460
  /** Whether the policy allows only some hosts: `deny-all` and allowlists. */
@@ -2368,13 +1537,16 @@ const LEVELS = [
2368
1537
  ];
2369
1538
  /** Why the sandbox can't enforce the attempt's policy, or undefined when it can. */
2370
1539
  function unenforceable(planned, sandbox, where) {
1540
+ const capabilities = capabilitiesOf(sandbox);
1541
+ const can = capabilities.egress.enforce;
1542
+ if (planned.policy === "proxy" && capabilities.isolated && can === "none") return `${where} runs under egress 'proxy', where the egress gateway is the only way out, but the ${sandbox.kind} sandbox enforces no egress policy: its traffic that ignores the proxy goes out directly. Set egress: 'allow-all' to run it there, or run it in ${suggestedSandbox((c) => c.isolated && c.egress.enforce !== "none")}.`;
2371
1543
  const needs = egressNeeds(planned);
2372
- const can = SANDBOX_EGRESS[sandbox.kind].enforce;
2373
1544
  if (LEVELS.indexOf(can) >= LEVELS.indexOf(needs)) return void 0;
2374
1545
  const hosts = allowedHosts(planned)?.join(", ");
2375
1546
  const what = needs !== "allowlist" ? "turning egress off" : planned.policy === "deny-all" ? `a host allowlist for its automatic hosts (${hosts})` : `a host allowlist (${hosts})`;
2376
1547
  const enforces = can === "none" ? "enforces no egress policy" : `only enforces ${can === "all-or-nothing" ? "'allow-all' or 'deny-all'" : can}`;
2377
- const hint = sandbox.kind === "fake" ? " (its commands run on the host). Run it in docker(), or set egress: 'allow-all'" : "";
1548
+ const enforcing = suggestedSandbox((c) => LEVELS.indexOf(c.egress.enforce) >= LEVELS.indexOf(needs));
1549
+ const hint = capabilities.isolated ? "" : ` (its commands run on the host). ${enforcing ? `Run it in ${enforcing}, or set` : "Set"} egress: 'allow-all'`;
2378
1550
  return `${where} runs under egress ${describeEgress(planned.policy)}, which needs ${what}, but the ${sandbox.kind} sandbox ${enforces}${hint}.`;
2379
1551
  }
2380
1552
  const describeEgress = (policy) => typeof policy === "string" ? `'${policy}'` : `{ allow: [${policy.allow.map((h) => `'${h}'`).join(", ")}] }`;
@@ -2398,34 +1570,12 @@ function egressLabel({ policy, auto }) {
2398
1570
  const own = policy === "deny-all" ? "deny-all" : policy.allow.length ? `allow ${policy.allow.join(", ")}` : "allow none";
2399
1571
  return auto.length ? `${own} + auto: ${auto.join(", ")}` : own;
2400
1572
  }
2401
- // src/judge/state-options.ts
2402
- /**
2403
- * What one call turned on, every key set (`false` when off), in the record's key order, so equal
2404
- * selections serialize the same.
2405
- */
2406
- function resolveStateOptions(call) {
2407
- return {
2408
- task: call.task ?? false,
2409
- output: call.output ?? false,
2410
- files: call.files ?? false,
2411
- diff: call.diff ?? false,
2412
- contents: call.contents ?? [],
2413
- transcript: call.transcript ?? false,
2414
- turns: call.turns ?? "last",
2415
- checks: call.checks ?? false
2416
- };
2417
- }
2418
- /** `inner` over `outer`, field by field. */
2419
- function mergeJudge(outer, inner) {
2420
- return inner ? {
2421
- ...outer,
2422
- ...inner
2423
- } : outer;
2424
- }
2425
- const KNOWN = [...[
2426
- "harness",
1573
+ // src/options.ts
1574
+ /** Every eval/group option key `checkOptions` accepts. The feature matrix holds each to an inventory entry. */
1575
+ const EVAL_OPTION_KEYS = [
1576
+ "harnesses",
2427
1577
  "sandbox",
2428
- "environment",
1578
+ "environments",
2429
1579
  "compare",
2430
1580
  "runs",
2431
1581
  "gate",
@@ -2440,12 +1590,14 @@ const KNOWN = [...[
2440
1590
  "tags",
2441
1591
  "revision",
2442
1592
  "pin"
2443
- ]];
1593
+ ];
1594
+ const KNOWN = [...EVAL_OPTION_KEYS];
2444
1595
  const SUGGEST = {
2445
- harnesses: "harness",
1596
+ harness: "harnesses",
2446
1597
  sandboxes: "sandbox",
2447
- environments: "environment",
2448
- baseline: "environment",
1598
+ environment: "environments",
1599
+ baseline: "compare",
1600
+ control: "compare",
2449
1601
  variants: "compare",
2450
1602
  repeat: "runs",
2451
1603
  repeats: "runs",
@@ -2480,47 +1632,27 @@ function mergeSnapshot(outer, inner) {
2480
1632
  compress: inner.compress ?? outer.compress
2481
1633
  };
2482
1634
  }
2483
- /** Throws unless `value` is `'all'` or `{ passRate }` with a rate in (0, 1]. */
1635
+ /** Throws unless `value` is `'all'`, `'observe'` or `{ passRate }` with a rate in (0, 1]. */
2484
1636
  function checkGate(value, where) {
2485
1637
  const problem = gateProblem(value, where);
2486
1638
  if (problem) throw new UsageError(problem);
2487
1639
  }
2488
1640
  const PINNABLE = [
2489
- "harness",
1641
+ "harnesses",
2490
1642
  "runs",
2491
1643
  "compare"
2492
1644
  ];
2493
1645
  const ID = /^[A-Za-z0-9._:@+\-=[\]]+$/;
2494
- const sanitize = (text) => (text.split("/").at(-1) ?? text).replace(/[^A-Za-z0-9._:@-]+/g, "-");
2495
1646
  function harnessId(h) {
2496
1647
  if (h.id) return h.id;
2497
1648
  const effort = harnessEffort(h);
2498
- const suffix = effort ? `[effort=${sanitize(effort)}]` : "";
1649
+ const suffix = effort ? `[effort=${idPart(effort)}]` : "";
2499
1650
  if (h.kind === "fake") return `fake${suffix}`;
2500
- return `${h.model ? `${h.kind}:${sanitize(h.model)}` : h.kind}${suffix}`;
2501
- }
2502
- function installId(install) {
2503
- if (!install?.length) return "";
2504
- const key = JSON.stringify(install.map((step) => typeof step === "string" ? step : {
2505
- run: step.run,
2506
- timeout: step.timeout
2507
- }));
2508
- return `+${createHash("sha256").update(key).digest("hex").slice(0, 8)}`;
1651
+ return `${h.model ? `${h.kind}:${idPart(h.model)}` : h.kind}${suffix}`;
2509
1652
  }
2510
1653
  /** The provider a sandbox runs on: its kind, or `podman` for a Docker sandbox on Podman. */
2511
- const sandboxProvider = (s) => s.kind === "docker" && s.engine === "podman" ? "podman" : s.kind;
2512
- function sandboxId(s) {
2513
- if (s.id) return s.id;
2514
- if (s.kind === "fake") return "fake";
2515
- const install = s.kind === "docker" ? installId(s.install) : "";
2516
- const engine = s.engine ?? "docker";
2517
- if (s.dockerfile) {
2518
- const [file = "", dir = ""] = s.dockerfile.split(/[\\/]/).reverse();
2519
- return `${engine}:${sanitize(file === "Dockerfile" ? dir || file : file)}${install}`;
2520
- }
2521
- const image = s.image ?? s.from;
2522
- return image ? `${engine}:${sanitize(image)}${install}` : `${engine}${install}`;
2523
- }
1654
+ const sandboxProvider = (s) => sandboxKind(s).provider(s);
1655
+ const sandboxId = (s) => s.id ?? sandboxKind(s).id(s);
2524
1656
  /** The planned attempts' sandboxes by id, once each, or `fallback`'s when none is planned. */
2525
1657
  function plannedSandboxIds(planned, fallback) {
2526
1658
  const ids = new Set(planned.map(sandboxId));
@@ -2542,11 +1674,14 @@ const describe = (value) => {
2542
1674
  }
2543
1675
  return JSON.stringify(value) ?? String(value);
2544
1676
  };
2545
- /** `compare: false` and `environment` beside `compare.baseline` are never legal options. */
1677
+ /** `compare: false` and `environments` beside `compare` are never legal options. */
2546
1678
  function refuseIllegalCompare(options, where) {
2547
- if (options.compare === false) throw new UsageError(`compare: false can't be set in ${where}. A profile turns compare off with limit.compare: false, which runs the baseline and strips the variants. On a group or eval, omit compare or pass compare: { baseline, variants }.`);
2548
- const baseline = options.compare !== null && typeof options.compare === "object" && "baseline" in options.compare ? options.compare.baseline : void 0;
2549
- if (options.environment !== void 0 && baseline !== void 0) throw new UsageError(`environment and compare.baseline can't both be set in ${where}. compare: { baseline, variants } names every arm, so drop environment, or drop baseline to make environment the control.`);
1679
+ if (options.compare === false) throw new UsageError(`compare: false can't be set in ${where}. A profile turns compare off with limit.compare: false, which runs the control and strips the variants. On a group or eval, omit compare or pass compare: { control, variants }.`);
1680
+ if (options.environments !== void 0 && options.compare !== void 0) {
1681
+ const first = Array.isArray(options.environments) ? options.environments[0] : options.environments;
1682
+ const name = isObjectLike(first) && typeof first.name === "string" ? first.name : "app";
1683
+ throw new UsageError(`${where} sets both environments and compare. With compare, the control is compare.control: move it there, as compare: { control: ${name}, variants: [...] }.`);
1684
+ }
2550
1685
  }
2551
1686
  const isRevision = (value) => typeof value === "number" && Number.isSafeInteger(value) || typeof value === "string" && value.trim() !== "";
2552
1687
  /** `runs: 3` or `runs: { bestOf: 5 }`, refused in the words of whichever it was meant to be. */
@@ -2577,7 +1712,7 @@ function checkOptions(options, where, allowed = KNOWN) {
2577
1712
  if (allowed.includes(key)) continue;
2578
1713
  const near = SUGGEST[key] ?? closest(key, allowed);
2579
1714
  const value = options[key];
2580
- const hint = key === "pool" ? " pool is an option of evaluate.each, beside the cases: evaluate.each(cases, { pool: false })." : key === "workspace" && typeof value !== "boolean" ? " workspace is an environment's starting tree: defineEnvironment(name, { workspace: git(…) })." : near ? ` Did you mean "${near}"?` : "";
1715
+ const hint = key === "cases" || key === "caseKey" ? ` ${key} belongs to one eval: evaluate(name, { cases: [...] }, async (c, item) => { … }).` : key === "where" ? " where is an option of evaluate() only, beside cases: evaluate(name, { cases: [...], where: ({ case, environment }) => … }, …)." : key === "workspace" && typeof value !== "boolean" ? " workspace is an environment's starting tree: defineEnvironment(name, { workspace: git(…) })." : near ? ` Did you mean "${near}"?` : "";
2581
1716
  throw new UsageError(`Unknown option "${key}" in ${where}.${hint}`);
2582
1717
  }
2583
1718
  const o = options;
@@ -2595,32 +1730,26 @@ function checkOptions(options, where, allowed = KNOWN) {
2595
1730
  });
2596
1731
  if (judgeIssue) throw new UsageError(judgeIssue);
2597
1732
  if (o.sandbox !== void 0) {
2598
- const kind = o.sandbox.kind;
2599
- if (kind !== "docker" && kind !== "fake") throw new UsageError(`sandbox in ${where} must come from docker(), podman() or fakeSandbox() (got ${describe(o.sandbox)})`);
2600
- const max = o.sandbox.maxConcurrent;
2601
- if (max !== void 0 && !(Number.isInteger(max) && max >= 1)) throw new UsageError(`sandbox.maxConcurrent in ${where} must be a whole number of at least 1 (got ${String(max)})`);
1733
+ if (!isSandboxKind(o.sandbox.kind)) throw new UsageError(`sandbox in ${where} must come from ${sandboxHelpers()} (got ${describe(o.sandbox)})`);
1734
+ const limit = maxConcurrentProblem(o.sandbox, where);
1735
+ if (limit) throw new UsageError(limit);
2602
1736
  const problem = sandboxProblem(o.sandbox);
2603
1737
  if (problem) throw new UsageError(`sandbox in ${where}: ${problem}`);
2604
1738
  }
2605
- if (o.environment !== void 0) {
2606
- if (Array.isArray(o.environment)) throw new UsageError(`environment takes one environment, not a list (${where}). To compare environments, use compare: { baseline, variants: [a, b] }. baseline is the control; variants are not layered on it.`);
2607
- if (!isEnvironment(o.environment)) throw new UsageError(`environment in ${where} must come from defineEnvironment(…) (got ${describe(o.environment)})`);
2608
- const problem = environmentDataProblem(o.environment, "environment", where);
2609
- if (problem) throw new UsageError(problem);
2610
- }
1739
+ if (o.environments !== void 0) checkEnvironments(o.environments, where);
2611
1740
  refuseIllegalCompare(options, where);
2612
1741
  if (o.compare !== void 0) {
2613
1742
  const problem = compareProblem(o.compare, where, z.custom(isEnvironment));
2614
1743
  if (problem) throw new UsageError(problem);
2615
- const { baseline, variants } = o.compare;
2616
- for (const [label, env] of [["compare.baseline", baseline], ...variants.map((v, i) => [`compare.variants[${i}]`, v])]) {
1744
+ const { control, variants } = o.compare;
1745
+ for (const [label, env] of [["compare.control", control], ...variants.map((v, i) => [`compare.variants[${i}]`, v])]) {
2617
1746
  const bad = env && environmentDataProblem(env, label, where);
2618
1747
  if (bad) throw new UsageError(bad);
2619
1748
  }
2620
1749
  }
2621
1750
  if (o.pin !== void 0) {
2622
1751
  const bad = Array.isArray(o.pin) ? o.pin.filter((k) => !PINNABLE.includes(k)) : [o.pin];
2623
- if (bad.length) throw new UsageError(`pin in ${where} lists harness, runs and/or compare (got ${bad.map((k) => JSON.stringify(k)).join(", ")})`);
1752
+ if (bad.length) throw new UsageError(`pin in ${where} lists harnesses, runs and/or compare (got ${bad.map((k) => JSON.stringify(k)).join(", ")})`);
2624
1753
  }
2625
1754
  if (o.capture !== void 0) checkCapture(o.capture, where);
2626
1755
  const questionIssue = o.question !== void 0 && questionOptionsProblem(o.question, where);
@@ -2670,42 +1799,82 @@ function checkCaptureModel(value, where) {
2670
1799
  if (typeof value !== "string" || !value.includes("/")) throw new UsageError(`capture.model in ${where} is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${JSON.stringify(value)})`);
2671
1800
  if (isEvaluationModel(value)) throw new UsageError(`capture.model "${value}" in ${where} is an evaluation model: it scores yes/no questions and can't drive a capture. Use a language model that accepts images, e.g. 'google/gemini-3.8-flash'.`);
2672
1801
  }
1802
+ /** Each environment `environments` names, as one, a list or an edit, comes from defineEnvironment. */
1803
+ function checkEnvironments(value, where) {
1804
+ if (typeof value === "function") return;
1805
+ let list;
1806
+ let label = "environments";
1807
+ if (isEnvironmentEdit(value)) {
1808
+ if ("add" in value && "only" in value) throw new UsageError(`environments in ${where} takes { add } or { only }, not both. Use add to append to the inherited environments, only to keep some of them.`);
1809
+ if ("only" in value) {
1810
+ const names = [value.only].flat();
1811
+ if (!names.length || names.some((n) => typeof n !== "string")) throw new UsageError(`environments { only } in ${where} lists environment names, e.g. { only: ['app'] } (got ${describe(value.only)})`);
1812
+ return;
1813
+ }
1814
+ list = [value.add].flat();
1815
+ label = "environments.add";
1816
+ } else list = [value].flat();
1817
+ if (list.length === 0) throw new UsageError(`${label} is an empty list in ${where}. Give at least one, or leave environments out.`);
1818
+ list.forEach((env, i) => {
1819
+ const at = Array.isArray(value) || label !== "environments" ? `${label}[${i}]` : label;
1820
+ if (!isEnvironment(env)) throw new UsageError(`${at} in ${where} must come from defineEnvironment(…) (got ${describe(env)})`);
1821
+ const problem = environmentDataProblem(env, at, where);
1822
+ if (problem) throw new UsageError(problem);
1823
+ });
1824
+ }
1825
+ const isEnvironmentEdit = (v) => isObjectLike(v) && !Array.isArray(v) && !isEnvironment(v) && ("add" in v || "only" in v);
1826
+ function applyEnvironments(inherited, value, where) {
1827
+ if (value === void 0) return inherited;
1828
+ const list = [typeof value === "function" ? value(inherited) : isEnvironmentEdit(value) ? editEnvironments(inherited, value, where) : value].flat();
1829
+ if (list.length === 0) throw new UsageError(`environments is an empty list in ${where}. Give at least one, or leave environments out.`);
1830
+ if (typeof value === "function") checkEnvironments(list, where);
1831
+ return list;
1832
+ }
1833
+ /** `{ add }` appends to the inherited environments; `{ only }` keeps those with these names. */
1834
+ function editEnvironments(inherited, edit, where) {
1835
+ if ("add" in edit) return [...inherited, ...[edit.add].flat()];
1836
+ const names = [edit.only].flat();
1837
+ const have = inherited.map((e) => e.name);
1838
+ const missing = names.filter((n) => !have.includes(n));
1839
+ if (missing.length) throw new UsageError(`environments { only: ${missing.map((n) => JSON.stringify(n)).join(", ")} } in ${where} names no inherited environment (inherited: ${have.join(", ") || "none"})`);
1840
+ return inherited.filter((e) => names.includes(e.name));
1841
+ }
2673
1842
  function applyHarness(inherited, value, where) {
2674
1843
  if (value === void 0) return inherited;
2675
1844
  const next = typeof value === "function" ? value(inherited) : isHarnessEdit(value) ? editHarness(inherited, value, where) : value;
2676
1845
  const list = Array.isArray(next) ? next : [next];
2677
- if (list.length === 0) throw new UsageError(`harness is an empty list in ${where}. Give at least one, e.g. harness: ${HARNESS_EXAMPLE}`);
1846
+ if (list.length === 0) throw new UsageError(`harnesses is an empty list in ${where}. Give at least one, e.g. harnesses: ${HARNESS_EXAMPLE}`);
2678
1847
  checkHarnesses(list, where);
2679
1848
  return list;
2680
1849
  }
2681
1850
  const isHarnessEdit = (v) => typeof v === "object" && v !== null && !Array.isArray(v) && !("kind" in v) && ("add" in v || "only" in v);
2682
1851
  /** `{ add }` appends to the inherited harnesses; `{ only }` keeps those with these ids. */
2683
1852
  function editHarness(inherited, edit, where) {
2684
- if ("add" in edit && "only" in edit) throw new UsageError(`harness in ${where} takes { add } or { only }, not both. Use add to append to the inherited harnesses, only to keep some of them.`);
1853
+ if ("add" in edit && "only" in edit) throw new UsageError(`harnesses in ${where} takes { add } or { only }, not both. Use add to append to the inherited harnesses, only to keep some of them.`);
2685
1854
  if ("add" in edit) return [...inherited, ...Array.isArray(edit.add) ? edit.add : [edit.add]];
2686
1855
  const ids = Array.isArray(edit.only) ? edit.only : [edit.only];
2687
1856
  const have = inherited.map(harnessId);
2688
1857
  const missing = ids.filter((id) => typeof id !== "string" || !have.includes(id));
2689
- if (missing.length) throw new UsageError(`harness { only: ${missing.map((id) => JSON.stringify(id)).join(", ")} } in ${where} names no inherited harness (inherited: ${have.join(", ")})`);
1858
+ if (missing.length) throw new UsageError(`harnesses { only: ${missing.map((id) => JSON.stringify(id)).join(", ")} } in ${where} names no inherited harness (inherited: ${have.join(", ")})`);
2690
1859
  return inherited.filter((h) => ids.includes(harnessId(h)));
2691
1860
  }
2692
1861
  /** Each harness is one, with a valid `maxConcurrent`. */
2693
1862
  function checkHarnesses(list, where) {
2694
1863
  for (const item of list) {
2695
- if (!isHarness(item)) throw new UsageError(`harness in ${where} must be like ${HARNESS_EXAMPLE} (got ${describe(item)})`);
1864
+ if (!isHarness(item)) throw new UsageError(`harnesses in ${where} must be like ${HARNESS_EXAMPLE} (got ${describe(item)})`);
2696
1865
  if (item.kind === "opencode") {
2697
1866
  const why = opencodeModelError(item.model);
2698
1867
  if (why) throw new UsageError(`opencode() in ${where}: ${why}`);
2699
1868
  }
2700
- checkMaxConcurrent(item.maxConcurrent, `harness.maxConcurrent in ${where}`);
1869
+ checkMaxConcurrent(item.maxConcurrent, `harnesses.maxConcurrent in ${where}`);
2701
1870
  const { startTimeout } = item;
2702
1871
  if (startTimeout !== void 0) try {
2703
1872
  parseDuration(startTimeout);
2704
1873
  } catch (error) {
2705
- throw new UsageError(`harness.startTimeout in ${where}: ${error.message}`);
1874
+ throw new UsageError(`harnesses.startTimeout in ${where}: ${error.message}`);
2706
1875
  }
2707
1876
  const problem = harnessProblem(item);
2708
- if (problem) throw new UsageError(`harness in ${where}: ${problem}`);
1877
+ if (problem) throw new UsageError(`harnesses in ${where}: ${problem}`);
2709
1878
  }
2710
1879
  }
2711
1880
  function checkUnique(list, where) {
@@ -2732,31 +1901,27 @@ const plannedQuestions = (value) => value.map((q, i) => {
2732
1901
  });
2733
1902
  /** Whether the eval's matrix varies each factor a pairwise question can vary. */
2734
1903
  const variesBy = (r) => ({
2735
- environment: (r.compare?.variants.length ?? 0) > 0,
1904
+ environment: r.compare ? r.compare.variants.length > 0 : r.environments.length > 1,
2736
1905
  harness: r.harness.length > 1
2737
1906
  });
2738
1907
  const FACTOR_FIX = {
2739
- environment: (_, where) => `varies the environment (across: 'environment'), but ${where} runs one environment, so it has no pairs to ask. Add compare: { variants: [...] }`,
2740
- harness: (r, where) => `varies the harness (across: 'harness'), but ${where} runs one harness (${r.harness.map(harnessId).join(", ")}), so it has no pairs to ask. Run it on two or more, harness: [a, b]`
1908
+ environment: (_, where) => `varies the environment (across: 'environment'), but ${where} runs one environment, so it has no pairs to ask. Run it in two or more, environments: [a, b], or compare: { control, variants: [...] }`,
1909
+ harness: (r, where) => `varies the harness (across: 'harness'), but ${where} runs one harness (${r.harness.map(harnessId).join(", ")}), so it has no pairs to ask. Run it on two or more, harnesses: [a, b]`
2741
1910
  };
2742
1911
  function normalizeCompare(value) {
2743
1912
  if (value === void 0) return void 0;
2744
1913
  return {
2745
- ...value.baseline ? { baseline: value.baseline } : {
2746
- baseline: emptyEnvironment(EMPTY_BASELINE),
2747
- from: "empty"
2748
- },
1914
+ control: value.control,
2749
1915
  variants: value.variants,
2750
1916
  gate: value.gate ?? "variants",
2751
1917
  ...value.minLift !== void 0 ? { minLift: value.minLift } : {},
2752
- ...value.harness ? { harness: value.harness.map((h) => typeof h === "string" ? h : harnessId(h)) } : {},
1918
+ ...value.harnesses ? { harnesses: value.harnesses.map((h) => typeof h === "string" ? h : harnessId(h)) } : {},
2753
1919
  of: value.variants.length
2754
1920
  };
2755
1921
  }
2756
1922
  /**
2757
- * Runs per cell when no level sets `runs`: a compare with variants runs `EFFECT_MIN_COUNTED` a
2758
- * side, the fewest whose lift can reach a verdict; anything else, a pooled family's cases
2759
- * included, runs once.
1923
+ * Runs per cell when no level sets `runs`: a compare with variants runs `COMPARE_RUNS` a side;
1924
+ * anything else, a pooled family's cases included, runs once.
2760
1925
  */
2761
1926
  const COMPARE_DEFAULT_RUNS = 4;
2762
1927
  function applyLevel(out, options, where, egress = "own") {
@@ -2765,9 +1930,9 @@ function applyLevel(out, options, where, egress = "own") {
2765
1930
  [key]: where
2766
1931
  };
2767
1932
  if (options.egress !== void 0) out.egress[egress] = options.egress;
2768
- out.harness = applyHarness(out.harness, options.harness, where);
1933
+ out.harness = applyHarness(out.harness, options.harnesses, where);
2769
1934
  if (options.sandbox !== void 0) out.sandbox = options.sandbox;
2770
- if (options.environment !== void 0) out.environment = options.environment;
1935
+ out.environments = applyEnvironments(out.environments, options.environments, where);
2771
1936
  if (options.compare !== void 0) out.compare = normalizeCompare(options.compare);
2772
1937
  if (options.runs !== void 0) {
2773
1938
  const { runs, bestOf } = runsOf(options.runs);
@@ -2808,6 +1973,7 @@ function cascade(base, levels, where, options = {}) {
2808
1973
  capture: { ...base.capture },
2809
1974
  question: { ...base.question },
2810
1975
  questions: [...base.questions],
1976
+ environments: [...base.environments],
2811
1977
  tags: [...base.tags],
2812
1978
  pin: [...base.pin]
2813
1979
  };
@@ -2817,28 +1983,33 @@ function cascade(base, levels, where, options = {}) {
2817
1983
  ...profile.defaults,
2818
1984
  judge: profileJudge(profile)
2819
1985
  }, `profiles.${profile.name}.defaults`, "inherited");
2820
- for (const { options: own, where: at } of levels) applyLevel(out, own, at);
2821
- if (out.compare?.from && out.environment) out.compare = {
2822
- ...out.compare,
2823
- baseline: out.environment,
2824
- from: "environment"
2825
- };
1986
+ let compareAt;
1987
+ for (const { options: own, where: at } of levels) {
1988
+ if (compareAt && own.environments !== void 0) throw new UsageError(`${at} sets environments, but ${compareAt} sets compare, which runs its control in their place. Set compare: { control, variants: [...] } on ${at} instead, or move ${at} out of ${compareAt}.`);
1989
+ applyLevel(out, own, at);
1990
+ if (own.compare !== void 0) compareAt = at;
1991
+ }
2826
1992
  if (out.questions.length) out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
2827
1993
  checkUnique(out.harness, where);
2828
- const harnessFromProfile = profile?.defaults.harness !== void 0 && !levels.some((l) => l.options.harness);
1994
+ if (!out.compare) {
1995
+ const names = out.environments.map((e) => e.name);
1996
+ const dupe = names.find((n, i) => names.indexOf(n) !== i);
1997
+ if (dupe) throw new UsageError(`environments in ${where} lists two environments named ${dupe}. Give each its own name.`);
1998
+ }
1999
+ const harnessFromProfile = profile?.defaults.harnesses !== void 0 && !levels.some((l) => l.options.harnesses);
2829
2000
  const authored = variesBy(out);
2830
2001
  if (harnessFromProfile) authored.harness ||= base.harness.length > 1;
2831
2002
  for (const q of options.skipped ? [] : out.questions) for (const factor of pairFactors(q.across)) {
2832
2003
  if (authored[factor]) continue;
2833
2004
  throw new UsageError(`questions "${q.question}" in ${where} ${FACTOR_FIX[factor](out, where)}, or leave out across to ask each attempt on its own.`);
2834
2005
  }
2835
- if (profile?.defaults.harness !== void 0 && levels.some((l) => l.options.harness)) notes.push("harness set by the eval");
2006
+ if (profile?.defaults.harnesses !== void 0 && levels.some((l) => l.options.harnesses)) notes.push("harnesses set by the eval");
2836
2007
  const ids = out.harness.map(harnessId);
2837
- const missing = (out.compare?.harness)?.filter((id) => !ids.includes(id)) ?? [];
2838
- if (missing.length) throw new UsageError(`compare.harness lists ${missing.join(", ")}, which ${where} doesn't run (it runs ${ids.join(", ")}).`);
2008
+ const missing = (out.compare?.harnesses)?.filter((id) => !ids.includes(id)) ?? [];
2009
+ if (missing.length) throw new UsageError(`compare.harnesses lists ${missing.join(", ")}, which ${where} doesn't run (it runs ${ids.join(", ")}).`);
2839
2010
  for (const v of out.compare?.variants ?? []) {
2840
- if (v.name === out.compare?.baseline.name) throw new UsageError(out.compare.from === "empty" ? `compare in ${where} lists a variant named ${v.name}, the name of the empty control a compare without a baseline or environment runs. Rename the variant, or set compare.baseline.` : out.compare.from === "environment" ? `compare in ${where} lists ${v.name}, which is the control: compare has no baseline, so the eval's environment is. List only the other environments.` : `compare in ${where} lists ${v.name}, which is the baseline. List only the other environments.`);
2841
- const control = out.compare?.baseline;
2011
+ if (v.name === out.compare?.control.name) throw new UsageError(`compare in ${where} lists ${v.name}, which is the control. List only the other environments.`);
2012
+ const control = out.compare?.control;
2842
2013
  const shared = control && sameWorkspaceOnly(control, v);
2843
2014
  if (control && shared) {
2844
2015
  const at = shared.ref ? `at ref "${shared.ref}"` : "with no ref (the repository HEAD)";
@@ -2855,13 +2026,13 @@ function cascade(base, levels, where, options = {}) {
2855
2026
  const tag = `profile ${profile.name}`;
2856
2027
  const pinned = (key) => out.pin.includes(key);
2857
2028
  let comparedLeft = true;
2858
- if (limit.harness && !pinned("harness")) {
2859
- const kept = out.harness.filter((h) => limit.harness?.includes(harnessId(h)));
2860
- if (kept.length === 0) skip = `no harness left: ${tag} allows ${limit.harness.join(", ") || "none"}`;
2029
+ if (limit.harnesses && !pinned("harnesses")) {
2030
+ const kept = out.harness.filter((h) => limit.harnesses?.includes(harnessId(h)));
2031
+ if (kept.length === 0) skip = `no harness left: ${tag} allows ${limit.harnesses.join(", ") || "none"}`;
2861
2032
  else if (kept.length < out.harness.length) {
2862
2033
  notes.push(`harnesses ${out.harness.length} → ${kept.length} (${tag})`);
2863
2034
  out.harness = kept;
2864
- const scope = out.compare?.harness;
2035
+ const scope = out.compare?.harnesses;
2865
2036
  if (out.compare?.variants.length && scope && !kept.some((h) => scope.includes(harnessId(h)))) {
2866
2037
  notes.push(`compare off: no compared harness left (${tag})`);
2867
2038
  out.compare = {
@@ -2927,6 +2098,8 @@ function cascade(base, levels, where, options = {}) {
2927
2098
  const set = from === where ? "" : ` from ${from}`;
2928
2099
  throw new UsageError(`${where} compares environments, so it can't take runs: { bestOf: ${out.bestOf} }${set}: a lift needs the same number of attempts on each side, and best-of stops each cell on its own. Set a fixed count on it, e.g. runs: 4.`);
2929
2100
  }
2101
+ if (out.gate === "observe" && out.compare?.minLift !== void 0 && out.compare.variants.length) throw new UsageError(`${where} has gate: 'observe', so its compare can't take minLift: ${out.compare.minLift}: minLift gates the lift, and nothing in it gates. Drop minLift, or gate the eval.`);
2102
+ if (out.gate === "observe" && out.bestOf !== void 0) throw new UsageError(`${where} has gate: 'observe', so it can't take runs: { bestOf: ${out.bestOf} }: best-of decides each cell by its majority. Set a fixed count, e.g. runs: 3.`);
2930
2103
  if (out.bestOf !== void 0) {
2931
2104
  if (out.gate !== void 0 && gateLabel(out.gate) !== gateLabel(MAJORITY_GATE)) notes.push(`gate ${gateLabel(out.gate)} → majority (runs.bestOf)`);
2932
2105
  out.gate = MAJORITY_GATE;
@@ -2957,14 +2130,14 @@ function defaultRuns(out, notes, options) {
2957
2130
  notes.push(`runs ${COMPARE_DEFAULT_RUNS} (compare default)`);
2958
2131
  }
2959
2132
  /**
2960
- * Refuses a profile's `limit.harness` id that no collected eval runs on: it keeps nothing, and a
2133
+ * Refuses a profile's `limit.harnesses` id that no collected eval runs on: it keeps nothing, and a
2961
2134
  * limit that keeps nothing skips every eval. `harnesses` are the ids every eval in the loaded
2962
2135
  * files runs on before the limit, whatever the tag and name filters pick. When a path filter
2963
2136
  * loaded only some files (`partial`), an id may name a harness of the others, so only a limit
2964
2137
  * none of whose ids match is refused.
2965
2138
  */
2966
2139
  function checkLimitHarness(profile, configFile, collected) {
2967
- const ids = profile?.limit.harness;
2140
+ const ids = profile?.limit.harnesses;
2968
2141
  if (!profile || !ids?.length) return;
2969
2142
  const unknown = ids.filter((id) => !collected.harnesses.includes(id));
2970
2143
  if (!unknown.length || collected.partial && unknown.length < ids.length) return;
@@ -2972,71 +2145,59 @@ function checkLimitHarness(profile, configFile, collected) {
2972
2145
  const have = [...collected.harnesses].sort();
2973
2146
  const evals = collected.partial ? "eval in the selected files" : "eval";
2974
2147
  const the = collected.partial ? "the evals in the selected files" : "the evals";
2975
- throw new UsageError(`profiles.${profile.name}.limit.harness in ${configFile} lists ${listed}, which no ${evals} runs on. Harness ids ${the} run on: ${have.join(", ") || "none"}.`);
2148
+ throw new UsageError(`profiles.${profile.name}.limit.harnesses in ${configFile} lists ${listed}, which no ${evals} runs on. Harness ids ${the} run on: ${have.join(", ") || "none"}.`);
2976
2149
  }
2977
2150
  /**
2978
2151
  * The sandbox an attempt runs in: the resolved environment's (including an `.extend()`
2979
- * parent's), else the first variant's when this is an omitted-baseline empty control, else
2980
- * the cascaded option (`defineConfig` / profile defaults / group / eval), else implicit Docker.
2981
- * A named environment is a total override: the config environment's sandbox is not inherited.
2152
+ * parent's), else the cascaded option (`defineConfig` / profile defaults / group / eval), else
2153
+ * implicit Docker. A named environment is a total override: the config environment's sandbox is
2154
+ * not inherited.
2982
2155
  */
2983
- const attemptSandbox = (resolved, env) => {
2984
- const own = env.layers.findLast((l) => l.spec.sandbox)?.spec.sandbox;
2985
- if (own) return own;
2986
- const variant = resolved.compare?.from === "empty" && env.name === "baseline" ? resolved.compare.variants[0] : void 0;
2987
- return sandboxOf(variant?.layers ?? [], resolved.sandbox);
2988
- };
2156
+ const attemptSandbox = (resolved, env) => env.layers.findLast((l) => l.spec.sandbox)?.spec.sandbox ?? sandboxOf([], resolved.sandbox);
2989
2157
  /**
2990
2158
  * Which level set this attempt's sandbox: the cascade source of the environment that named one
2991
- * (an `.extend()` child still names that environment), else the first variant when this is an
2992
- * omitted-baseline empty control, else the cascaded `sandbox` option, else the config (implicit
2993
- * Docker).
2159
+ * (an `.extend()` child still names that environment), else the cascaded `sandbox` option, else
2160
+ * the config (implicit Docker).
2994
2161
  */
2995
2162
  function sandboxSetBy(planned, resolved, configFile) {
2996
- if (!planned.environment.layers.some((l) => l.spec.sandbox)) {
2997
- if (resolved.compare?.from === "empty" && planned.cell.role === "control" && resolved.compare.variants[0]?.layers.some((l) => l.spec.sandbox)) return resolved.sources?.compare ?? configFile;
2998
- return resolved.sources?.sandbox ?? configFile;
2999
- }
3000
- if (resolved.compare) {
3001
- if (planned.cell.role === "control") {
3002
- if (resolved.compare.from === "environment") return resolved.sources?.environment ?? configFile;
3003
- if (resolved.compare.from === "empty") return configFile;
3004
- return resolved.sources?.compare ?? configFile;
3005
- }
3006
- return resolved.sources?.compare ?? configFile;
3007
- }
3008
- return resolved.sources?.environment ?? configFile;
2163
+ if (!planned.environment.layers.some((l) => l.spec.sandbox)) return resolved.sources?.sandbox ?? configFile;
2164
+ return (resolved.compare ? resolved.sources?.compare : resolved.sources?.environments) ?? configFile;
3009
2165
  }
3010
2166
  /**
3011
2167
  * Per harness: the control, then each variant it compares on; then run numbers. `vertex` is the
3012
2168
  * run's `vertex()` settings, for each harness that resolved to Vertex AI.
3013
2169
  */
2170
+ const cellTraits = (env) => {
2171
+ const traits = traitsOf(env.layers);
2172
+ return Object.keys(traits).length ? { traits } : {};
2173
+ };
3014
2174
  function expand(resolved, vertex) {
3015
2175
  const { compare } = resolved;
3016
- const control = applied(compare ? compare.baseline : resolved.environment);
3017
- const policy = resolved.gate ?? "all";
2176
+ const controls = (compare ? [compare.control] : resolved.environments.length ? resolved.environments : [void 0]).map((e) => applied(e));
2177
+ const policy = resolved.gate === "observe" ? "none" : resolved.gate ?? "all";
3018
2178
  const gateIf = (gated) => gated ? policy : "none";
3019
2179
  const attempts = [];
3020
2180
  for (const harness of resolved.harness) {
3021
2181
  const hid = harnessId(harness);
3022
2182
  const onVertex = harness.gateway === "vertex" && harness.kind !== "fake" ? vertex : void 0;
3023
- const cells = [(() => {
3024
- const sandbox = attemptSandbox(resolved, control);
2183
+ const cells = controls.map((env) => {
2184
+ const sandbox = attemptSandbox(resolved, env);
3025
2185
  const sid = sandboxId(sandbox);
3026
2186
  return {
3027
- env: control,
2187
+ env,
3028
2188
  sandbox,
3029
2189
  cell: {
3030
- key: cellKey(hid, control.name, sid),
2190
+ key: cellKey(hid, env.name, sid),
3031
2191
  harness: hid,
3032
- environment: control.name,
2192
+ environment: env.name,
3033
2193
  sandbox: sid,
3034
- role: "control"
2194
+ role: "control",
2195
+ ...cellTraits(env)
3035
2196
  },
3036
2197
  gate: gateIf(!compare || compare.variants.length === 0 || compare.gate === "all")
3037
2198
  };
3038
- })()];
3039
- if (compare && (!compare.harness || compare.harness.includes(hid))) for (const v of compare.variants) {
2199
+ });
2200
+ if (compare && (!compare.harnesses || compare.harnesses.includes(hid))) for (const v of compare.variants) {
3040
2201
  const env = applied(v);
3041
2202
  const sandbox = attemptSandbox(resolved, env);
3042
2203
  const sid = sandboxId(sandbox);
@@ -3049,13 +2210,14 @@ function expand(resolved, vertex) {
3049
2210
  environment: env.name,
3050
2211
  sandbox: sid,
3051
2212
  role: "variant",
3052
- variant: v.name
2213
+ variant: v.name,
2214
+ ...cellTraits(env)
3053
2215
  },
3054
- gate: gateIf(compare.gate !== "none")
2216
+ gate: gateIf(compare.gate !== "observe")
3055
2217
  });
3056
2218
  }
3057
2219
  for (const { env, sandbox, cell, gate } of cells) {
3058
- const axes = [resolved.harness.length > 1 ? hid : "", compare ? cell.role === "control" ? "control" : env.name : ""].filter(Boolean);
2220
+ const axes = [resolved.harness.length > 1 ? hid : "", compare ? cell.role === "control" ? "control" : env.name : controls.length > 1 ? env.name : ""].filter(Boolean);
3059
2221
  for (let n = 1; n <= resolved.runs; n++) {
3060
2222
  const label = [axes.join(" · "), resolved.runs > 1 ? `#${n}` : ""].filter(Boolean).join(" ");
3061
2223
  attempts.push({
@@ -3092,14 +2254,16 @@ function environmentWarnings(attempts) {
3092
2254
  }
3093
2255
  if (s.source === "registry" && !s.version) out.add(`skill.registry(${s.id}) has no version: results aren't comparable over time. Pin one (the resolved commit is recorded either way).`);
3094
2256
  }
2257
+ const tag = a.sandbox.kind === "tart" ? tartTagWarning(a.sandbox.image) : null;
2258
+ if (tag) out.add(tag);
3095
2259
  }
3096
2260
  return [...out];
3097
2261
  }
3098
- /** Secrets a Docker attempt would get as real values, because they have no `hosts`. */
2262
+ /** Secrets an isolated sandbox's attempt would get as real values, because they have no `hosts`. */
3099
2263
  function unbrokered(env, attempts) {
3100
2264
  const out = /* @__PURE__ */ new Map();
3101
2265
  for (const a of attempts) {
3102
- if (a.sandbox.kind !== "docker") continue;
2266
+ if (!capabilitiesOf(a.sandbox).isolated) continue;
3103
2267
  for (const s of unbrokeredSecrets(env, a.environment.layers)) out.set(s.label, s);
3104
2268
  }
3105
2269
  return [...out.values()];
@@ -3108,12 +2272,21 @@ function unbrokered(env, attempts) {
3108
2272
  const reachesSandbox = (s) => !s.optional || Boolean(process.env[s.from]);
3109
2273
  /** `GEMINI_API_KEY → generativelanguage.googleapis.com` for each secret with `hosts`. */
3110
2274
  const withHosts = (env, attempt) => [...new Set(plannedSecrets(env, attempt.environment.layers).flatMap((s) => s.hosts ? [`${s.label} → ${s.hosts.join(", ")}`] : []))];
3111
- /** The secrets an attempt brokers: only Docker's egress gateway swaps a placeholder for its value. */
3112
- const brokeredIn = (env, attempt) => attempt.sandbox.kind === "docker" ? withHosts(env, attempt) : [];
3113
- /** The plan's note for secrets with `hosts` on the fake sandbox, which only ever has their placeholder. */
3114
- function unbrokeredOnFake(env, attempts) {
3115
- const names = [...new Set(attempts.flatMap((a) => a.sandbox.kind === "fake" ? withHosts(env, a) : []))];
3116
- return names.length ? [`not brokered (fake sandbox, placeholder only): ${names.join("; ")}`] : [];
2275
+ /** The secrets an attempt brokers: only a sandbox that brokers swaps a placeholder for its value. */
2276
+ const brokeredIn = (env, attempt) => capabilitiesOf(attempt.sandbox).brokers ? withHosts(env, attempt) : [];
2277
+ /** The plan's note for secrets with `hosts` on a sandbox that can't broker them, which only ever has their placeholder. */
2278
+ function unbrokeredHere(env, attempts) {
2279
+ const byProvider = /* @__PURE__ */ new Map();
2280
+ for (const a of attempts) {
2281
+ if (capabilitiesOf(a.sandbox).brokers) continue;
2282
+ const names = withHosts(env, a);
2283
+ if (!names.length) continue;
2284
+ const provider = sandboxProvider(a.sandbox);
2285
+ const seen = byProvider.get(provider) ?? /* @__PURE__ */ new Set();
2286
+ for (const name of names) seen.add(name);
2287
+ byProvider.set(provider, seen);
2288
+ }
2289
+ return [...byProvider].map(([provider, names]) => `not brokered (${provider} sandbox, placeholder only): ${[...names].join("; ")}`);
3117
2290
  }
3118
2291
  /** The host variables an attempt's secrets read, brokered or not. */
3119
2292
  const secretVarsIn = (env, attempt) => [...new Set(plannedSecrets(env, attempt.environment.layers).map((s) => s.from))];
@@ -3138,11 +2311,12 @@ function sandboxWarnings(attempts, resolved, origin) {
3138
2311
  const out = [];
3139
2312
  const seen = /* @__PURE__ */ new Set();
3140
2313
  for (const a of attempts) {
3141
- if (a.sandbox.kind !== "docker") continue;
2314
+ const privileges = sandboxKind(a.sandbox).privileges(a.sandbox);
2315
+ if (!privileges.root && !privileges.capabilities.length) continue;
3142
2316
  const where = sandboxSetBy(a, resolved, origin.configFile);
3143
2317
  const level = levelOf(where);
3144
2318
  const file = level === "eval" || level === "group" ? origin.file : null;
3145
- if (a.sandbox.root) {
2319
+ if (privileges.root) {
3146
2320
  const message = `running as root in the sandbox: ${where} (docker({ root: true }))`;
3147
2321
  const key = `${file}\0${message}`;
3148
2322
  if (!seen.has(key)) {
@@ -3156,8 +2330,8 @@ function sandboxWarnings(attempts, resolved, origin) {
3156
2330
  });
3157
2331
  }
3158
2332
  }
3159
- if (a.sandbox.capabilities?.length) {
3160
- const message = `extra capabilities in the sandbox: ${where} (docker({ capabilities: [${a.sandbox.capabilities.map((c) => `'${c}'`).join(", ")}] }))`;
2333
+ if (privileges.capabilities.length) {
2334
+ const message = `extra capabilities in the sandbox: ${where} (docker({ capabilities: [${privileges.capabilities.map((c) => `'${c}'`).join(", ")}] }))`;
3161
2335
  const key = `${file}\0${message}`;
3162
2336
  if (!seen.has(key)) {
3163
2337
  seen.add(key);
@@ -3166,7 +2340,7 @@ function sandboxWarnings(attempts, resolved, origin) {
3166
2340
  level,
3167
2341
  where,
3168
2342
  file,
3169
- capabilities: [...a.sandbox.capabilities],
2343
+ capabilities: [...privileges.capabilities],
3170
2344
  message
3171
2345
  });
3172
2346
  }
@@ -3181,7 +2355,7 @@ function evalPlan(r, attempts, origin) {
3181
2355
  const notes = [
3182
2356
  ...r.notes,
3183
2357
  ...capture ? [capture] : [],
3184
- ...unbrokeredOnFake(resolved.env, attempts)
2358
+ ...unbrokeredHere(resolved.env, attempts)
3185
2359
  ];
3186
2360
  return {
3187
2361
  harnesses: skip ? 0 : resolved.harness.length,
@@ -3189,10 +2363,7 @@ function evalPlan(r, attempts, origin) {
3189
2363
  run: resolved.compare.variants.length,
3190
2364
  of: resolved.compare.of
3191
2365
  } : null,
3192
- ...resolved.compare ? { compare: {
3193
- baseline: resolved.compare.baseline.name,
3194
- ...resolved.compare.from ? { from: resolved.compare.from } : {}
3195
- } } : {},
2366
+ ...resolved.compare ? { compare: { baseline: resolved.compare.control.name } } : {},
3196
2367
  ...!skip && resolved.questions.length ? { questions: resolved.questions } : {},
3197
2368
  runs: resolved.runs,
3198
2369
  ...resolved.bestOf !== void 0 ? { bestOf: resolved.bestOf } : {},
@@ -3257,7 +2428,11 @@ const GATEWAYS = {
3257
2428
  vertex: { cantServe: (h) => {
3258
2429
  if (h.kind === "fake") return void 0;
3259
2430
  if (!VERTEX_ROUTES[h.kind]) return VERTEX_REFUSALS[h.kind] ?? `reeval doesn't run ${agentName(h)} on Vertex AI yet`;
3260
- return vertexModelRefusalFor(h) ?? (h.kind === "opencode" ? openCodeEffortOn(h.model, "vertex", harnessEffort(h)) : void 0);
2431
+ const effort = harnessEffort(h);
2432
+ const refused = vertexModelRefusalFor(h);
2433
+ if (refused) return refused;
2434
+ if (h.kind === "claude-code" && effort !== void 0) return claudeCodeTakesEffort(h.model, "vertex") ? void 0 : `Claude Code sends no effort to ${claudeCodeModelName(h.model, "vertex")} on Vertex AI, where it sends one only to the models its catalog lists: drop effort, or run one of them, e.g. claude-sonnet-5-5`;
2435
+ return h.kind === "opencode" ? openCodeEffortOn(h.model, "vertex", effort) : void 0;
3261
2436
  } }
3262
2437
  };
3263
2438
  /** OpenCode's id for `model` on `gateway`, as that gateway's route writes it. */
@@ -3423,9 +2598,9 @@ function modelGatewayOrNone(model, pin, scope) {
3423
2598
  const CONFIG_KEYS = [
3424
2599
  "include",
3425
2600
  "gateway",
3426
- "harness",
2601
+ "harnesses",
3427
2602
  "sandbox",
3428
- "environment",
2603
+ "environments",
3429
2604
  "runs",
3430
2605
  "gate",
3431
2606
  "env",
@@ -3455,9 +2630,9 @@ const PROFILE_KEYS = [
3455
2630
  "gateway"
3456
2631
  ];
3457
2632
  const DEFAULTS_KEYS = [
3458
- "harness",
2633
+ "harnesses",
3459
2634
  "sandbox",
3460
- "environment",
2635
+ "environments",
3461
2636
  "runs",
3462
2637
  "gate",
3463
2638
  "env",
@@ -3471,7 +2646,8 @@ const DEFAULTS_KEYS = [
3471
2646
  const LIMIT_KEYS = [
3472
2647
  "runs",
3473
2648
  "compare",
3474
- "harness"
2649
+ "harnesses",
2650
+ "traits"
3475
2651
  ];
3476
2652
  /**
3477
2653
  * The config: a plain object, or a callback that destructures the helpers it needs, e.g.
@@ -3565,7 +2741,7 @@ function parseYamlConfig(text, file) {
3565
2741
  return doc.toJS() ?? {};
3566
2742
  }
3567
2743
  const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
3568
- /** A config's Dockerfile that isn't there, named by its field. */
2744
+ /** A config's host file (a Dockerfile, an SSH key) that isn't there, named by its field. */
3569
2745
  const missingIn = (file) => (field, full) => {
3570
2746
  throw new UsageError(`${field} in ${file}: ${full} doesn't exist. Paths in ${file} are relative to its directory.`);
3571
2747
  };
@@ -3581,7 +2757,10 @@ function withHostPaths(config, root, file, { environments = false } = {}) {
3581
2757
  if (!isRecord(value)) return value;
3582
2758
  const out = { ...value };
3583
2759
  if ("sandbox" in value) out.sandbox = sandboxPaths(value.sandbox, root, `${prefix}sandbox`, missing);
3584
- if (environments && "environment" in value) out.environment = environmentPaths(value.environment, root, `${prefix}environment`, missing);
2760
+ if (environments && "environments" in value) {
2761
+ const field = `${prefix}environments`;
2762
+ out.environments = Array.isArray(value.environments) ? value.environments.map((e, i) => environmentPaths(e, root, `${field}[${i}]`, missing)) : environmentPaths(value.environments, root, field, missing);
2763
+ }
3585
2764
  return out;
3586
2765
  };
3587
2766
  const out = options(config, "");
@@ -3600,10 +2779,15 @@ function withHostPaths(config, root, file, { environments = false } = {}) {
3600
2779
  function configFromData(data) {
3601
2780
  const withEnvironment = (options) => {
3602
2781
  if (!isObjectLike(options)) return options;
3603
- const { environment } = options;
3604
- return isEnvironment(environment) ? {
2782
+ const { environments } = options;
2783
+ const from = (e) => isEnvironment(e) ? environmentFrom(e) : e;
2784
+ if (Array.isArray(environments)) return {
2785
+ ...options,
2786
+ environments: environments.map(from)
2787
+ };
2788
+ return isEnvironment(environments) ? {
3605
2789
  ...options,
3606
- environment: environmentFrom(environment)
2790
+ environments: from(environments)
3607
2791
  } : options;
3608
2792
  };
3609
2793
  const config = withEnvironment(data);
@@ -3629,10 +2813,10 @@ function configFromData(data) {
3629
2813
  * let through, such as a `null` or a misspelled nested key.
3630
2814
  */
3631
2815
  function resolveConfigData(input, root, file) {
3632
- if (!isRecord(input)) throw new UsageError(`${file} must be a mapping of config keys, like harness: and judge: (got ${JSON.stringify(input)})`);
2816
+ if (!isRecord(input)) throw new UsageError(`${file} must be a mapping of config keys, like harnesses: and judge: (got ${JSON.stringify(input)})`);
3633
2817
  const data = withHostPaths(input, root, file, { environments: true });
3634
- const { environment, profiles } = data;
3635
- if ([environment, ...Object.values(typeof profiles === "object" && profiles ? profiles : {}).map((p) => p?.defaults?.environment)].some((e) => e !== void 0 && !isEnvironment(e))) {
2818
+ const { environments: environment, profiles } = data;
2819
+ if ([environment, ...Object.values(typeof profiles === "object" && profiles ? profiles : {}).map((p) => p?.defaults?.environments)].flat().some((e) => e !== void 0 && !isEnvironment(e))) {
3636
2820
  const problem = configDataProblem(data, file);
3637
2821
  if (problem) throw new UsageError(problem);
3638
2822
  }
@@ -3712,12 +2896,18 @@ function checkProfile(name, profile, file) {
3712
2896
  }
3713
2897
  if (p.limit !== void 0) {
3714
2898
  unknownKey(p.limit, LIMIT_KEYS, `${where}.limit`, file);
3715
- const { runs, compare, harness } = p.limit;
2899
+ const { runs, compare, harnesses: harness, traits } = p.limit;
3716
2900
  if (runs !== void 0 && !(Number.isInteger(runs) && runs >= 1)) throw new UsageError(`${where}.limit.runs in ${file} must be a whole number of at least 1 (got ${JSON.stringify(runs)})`);
3717
2901
  if (compare !== void 0 && compare !== false && !(Number.isInteger(compare?.maxVariants) && compare.maxVariants >= 0)) throw new UsageError(`${where}.limit.compare in ${file} must be false or { maxVariants: n }, e.g. { maxVariants: 1 } (got ${JSON.stringify(compare)})`);
3718
2902
  if (harness !== void 0) {
3719
2903
  const bad = Array.isArray(harness) ? harness.find((h) => typeof h !== "string") : harness;
3720
- if (bad !== void 0) throw new UsageError(`${where}.limit.harness in ${file} can only list harness ids to allow (got ${describeValue(bad)}). To add harnesses, use ${where}.defaults.harness.`);
2904
+ if (bad !== void 0) throw new UsageError(`${where}.limit.harnesses in ${file} can only list harness ids to allow (got ${describeValue(bad)}). To add harnesses, use ${where}.defaults.harnesses.`);
2905
+ }
2906
+ if (traits !== void 0) {
2907
+ const ok = (v) => typeof v === "string" || Array.isArray(v) && v.length > 0 && v.every((s) => typeof s === "string");
2908
+ const entries = typeof traits === "object" && traits !== null && !Array.isArray(traits) ? Object.entries(traits) : null;
2909
+ const bad = entries?.find(([, v]) => !ok(v));
2910
+ if (!entries || bad) throw new UsageError(`${where}.limit.traits in ${file} must map each trait key to a value or a list of values a cell's environment has, e.g. { lang: 'ts' } or { lang: ['ts', 'py'] } (got ${bad ? `${bad[0]}: ${JSON.stringify(bad[1])}` : JSON.stringify(traits)})`);
3721
2911
  }
3722
2912
  }
3723
2913
  if (p.tags !== void 0 && (!Array.isArray(p.tags) || p.tags.some((t) => typeof t !== "string"))) throw new UsageError(`${where}.tags in ${file} must be a list of tag filters, e.g. tags: ['smoke'] (got ${JSON.stringify(p.tags)})`);
@@ -3831,8 +3021,8 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3831
3021
  const artifacts = user.history?.artifacts ?? "all";
3832
3022
  if (!ARTIFACTS.includes(artifacts)) throw new UsageError(`history.artifacts in ${file} is ${oneOf(ARTIFACTS)} (got ${JSON.stringify(artifacts)})`);
3833
3023
  const historyDir = resolve(root, user.history?.dir ?? join(evalRoot(include), ".history"));
3834
- const harness = user.harness === void 0 ? [DEFAULT_HARNESS] : [user.harness].flat();
3835
- if (harness.length === 0) throw new UsageError(`harness is an empty list in ${file}. Give at least one, e.g. harness: ${HARNESS_EXAMPLE}`);
3024
+ const harness = user.harnesses === void 0 ? [DEFAULT_HARNESS] : [user.harnesses].flat();
3025
+ if (harness.length === 0) throw new UsageError(`harnesses is an empty list in ${file}. Give at least one, e.g. harnesses: ${HARNESS_EXAMPLE}`);
3836
3026
  checkHarnesses(harness, file);
3837
3027
  const profiles = {};
3838
3028
  if (user.profiles !== void 0) {
@@ -3855,7 +3045,7 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3855
3045
  include,
3856
3046
  harness,
3857
3047
  sandbox: user.sandbox ?? docker(),
3858
- environment: user.environment,
3048
+ environments: user.environments === void 0 ? [] : [user.environments].flat(),
3859
3049
  runs: user.runs,
3860
3050
  gate: user.gate,
3861
3051
  env: user.env ?? {},
@@ -3886,7 +3076,7 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3886
3076
  }
3887
3077
  const ANALYSIS_DEFAULTS = {
3888
3078
  maxInputTokens: 2e4,
3889
- maxOutputTokens: 8e3
3079
+ maxOutputTokens: 32e3
3890
3080
  };
3891
3081
  function checkAnalysis(analysis, file) {
3892
3082
  if (analysis === void 0) return { ...ANALYSIS_DEFAULTS };
@@ -3997,6 +3187,11 @@ function runModelGateway(config, profile, model, judge = false) {
3997
3187
  */
3998
3188
  function missingCredentials(config, env = process.env, options = {}) {
3999
3189
  const missing = [];
3190
+ for (const s of options.sandboxes ?? []) {
3191
+ if (s.kind !== "boat" || env[s.apiKey.from]) continue;
3192
+ const line = `${s.apiKey.from} (for boat())`;
3193
+ if (!missing.includes(line)) missing.push(line);
3194
+ }
4000
3195
  const keys = new Set((options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []));
4001
3196
  for (const key of keys) if (!env[key]) missing.push(key);
4002
3197
  const judge = runJudgeModel(config, options.profile);
@@ -4054,6 +3249,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
4054
3249
  return at ? [`${name} (character ${at})`] : [];
4055
3250
  });
4056
3251
  }
4057
- export { allowedHosts as $, evalPlan as A, refusedMcp as At, profileJudge as B, judgeProblem as Bt, openCodeModelOn as C, brokeringFor as Ct, cascade as D, isAgentKind as Dt, brokeredIn as E, expectedBlockedFor as Et, matchesTags as F, NO_JUDGE_INPUT as Ft, sandboxSetBy as G, decideState as Gt, runsOf as H, turnsSchema as Ht, plainHeaderError as I, fakeJudgeDataSchema as It, unbrokeredError as J, secretVarsIn as K, filePartsOf as Kt, plainHeaders as L, inputsOn as Lt, harnessId as M, writePluginFiles as Mt, levelOf as N, captureWindowSchema as Nt, checkLimitHarness as O, keyRejected as Ot, majorityOf as P, INPUT_KEYS as Pt, DEFAULT_EGRESS_ALLOW as Q, plannedSandboxIds as R, judgeFields as Rt, fallbackHint as S, authHint as St, routedHarness as T, credentialOf as Tt, sandboxId as U, artifactRefsOf as Ut, refuseIllegalCompare as V, sharedInputFields as Vt, sandboxProvider as W, badJudgeArtifacts as Wt, resolveStateOptions as X, mergeJudge as Y, DEFAULT_EGRESS as Z, fellBackToDirect as _, servedEffortOf as _t, configPathFor as a, unenforceable as at, resolveModelGateway as b, agentFor as bt, findConfig as c, awaitListening as ct, malformedCredentials as d, STOPPED_FIRST as dt, ciRunner as et, missingCredentials as f, UNRECORDED_QUESTION_STATE as ft, selectProfile as g, resolvedEffortOf as gt, runJudgeModel as h, harnessEffort as ht, configLabel as i, isDirect as it, expand as j, refusedPlugins as jt, checkOptions as k, layoutOf as kt, hostCapacity as l, parseProbe as lt, resolveConcurrency as m, resolveQuestionState as mt, analysisModelOf as n, egressLabel as nt, defaultConcurrency as o, resolveOutput as ot, requireConfig as p, questionShows as pt, unbrokered as q, resolveJudgeArtifacts as qt, configFilesPhrase as r, gatewayPolicy as rt, defineConfig as s, createFakeSandbox as st, CONFIG_FILES as t, directOnCloud as tt, loadConfig as u, probeScript as ut, gatewayScope as v, AGENTS as vt, resolveGateway as w, claudeMcpServers as wt, effortRefusedOn as x, agentMcp as xt, modelGatewayOrNone as y, agentAuth as yt, plannedSandboxes as z, judgeModelNameSchema as zt };
3252
+ export { runsOf as $, brokeredIn as A, execOptionsProblem as At, expand as B, decideState as Bt, openCodeModelOn as C, resolveStateOptions as Ct, EVAL_OPTION_KEYS as D, CAPTURE_NAME as Dt, COMPARE_DEFAULT_RUNS as E, observer as Et, checkHarnesses as F, NO_JUDGE_INPUT as Ft, mergeSnapshot as G, levelOf as H, resolveJudgeArtifacts as Ht, checkLimitHarness as I, inputsOn as It, plainHeaders as J, plainHeaderError as K, checkOptions as L, judgeProblem as Lt, cellKey as M, resolvedEffortOf as Mt, checkCapture as N, servedEffortOf as Nt, MAJORITY_GATE as O, CAPTURE_NAME_RULE as Ot, checkGate as P, INPUT_KEYS as Pt, refuseIllegalCompare as Q, environmentWarnings as R, artifactRefsOf as Rt, fallbackHint as S, mergeJudge as St, routedHarness as T, observedName as Tt, majorityOf as U, harnessId as V, filePartsOf as Vt, matchesTags as W, plannedSandboxes as X, plannedSandboxIds as Y, profileJudge as Z, fellBackToDirect as _, resolveOutput as _t, configPathFor as a, unbrokered as at, resolveModelGateway as b, questionShows as bt, findConfig as c, DEFAULT_EGRESS as ct, malformedCredentials as d, ciRunner as dt, sandboxId as et, missingCredentials as f, directOnCloud as ft, selectProfile as g, unenforceable as gt, runJudgeModel as h, isDirect as ht, configLabel as i, secretVarsIn as it, cascade as j, harnessEffort as jt, attemptSandbox as k, attemptRequestProblem as kt, hostCapacity as l, DEFAULT_EGRESS_ALLOW as lt, resolveConcurrency as m, gatewayPolicy as mt, analysisModelOf as n, sandboxSetBy as nt, defaultConcurrency as o, unbrokeredError as ot, requireConfig as p, egressLabel as pt, plainHeaderWarning as q, configFilesPhrase as r, sandboxWarnings as rt, defineConfig as s, unbrokeredWarning as st, CONFIG_FILES as t, sandboxProvider as tt, loadConfig as u, allowedHosts as ut, gatewayScope as v, STOPPED_FIRST as vt, resolveGateway as w, observedHere as wt, effortRefusedOn as x, resolveQuestionState as xt, modelGatewayOrNone as y, UNRECORDED_QUESTION_STATE as yt, evalPlan as z, badJudgeArtifacts as zt };
4058
3253
 
4059
- //# sourceMappingURL=config-ImY6AlRE.js.map
3254
+ //# sourceMappingURL=config-B9mB1kF5.js.map