reeval 0.18.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (380) hide show
  1. package/README.md +1 -1
  2. package/THIRD_PARTY_NOTICES.md +5 -5
  3. package/dist/{adapter-CptK032x.js → adapter-C60z3wLq.js} +5 -4
  4. package/dist/adapter-C60z3wLq.js.map +1 -0
  5. package/dist/{agent-B63OzUX9.js → agent-BmsMXZXx.js} +30 -10
  6. package/dist/{agent-B63OzUX9.js.map → agent-BmsMXZXx.js.map} +1 -1
  7. package/dist/agents-DXuFxtGC.js +1151 -0
  8. package/dist/agents-DXuFxtGC.js.map +1 -0
  9. package/dist/{app-files-sgqwfjtn.js → app-files-DT5mJyV4.js} +3 -3
  10. package/dist/app-files-DT5mJyV4.js.map +1 -0
  11. package/dist/assemble-CE4pHKIa.js +1 -0
  12. package/dist/{assemble-Me7Whp-y.js → assemble-DM2PNU4R.js} +152 -129
  13. package/dist/assemble-DM2PNU4R.js.map +1 -0
  14. package/dist/{attempt-Dp7Wtm8M.js → attempt-CAdlwlNu.js} +2 -2
  15. package/dist/{attempt-Dp7Wtm8M.js.map → attempt-CAdlwlNu.js.map} +1 -1
  16. package/dist/{attempt-host-Cik0SoIn.js → attempt-host-7VviAJrl.js} +6 -6
  17. package/dist/{attempt-host-Cik0SoIn.js.map → attempt-host-7VviAJrl.js.map} +1 -1
  18. package/dist/boat-BjLqFJIZ.js +108 -0
  19. package/dist/boat-BjLqFJIZ.js.map +1 -0
  20. package/dist/boat-DSNgZt-t.js +322 -0
  21. package/dist/boat-DSNgZt-t.js.map +1 -0
  22. package/dist/bridge/index.mjs +401 -79
  23. package/dist/capture-BzotD2hs.js +1 -0
  24. package/dist/{capture-C3RmjPph.js → capture-Ccf0Hmyx.js} +27 -25
  25. package/dist/capture-Ccf0Hmyx.js.map +1 -0
  26. package/dist/cases-Dy0mifal.js +1 -0
  27. package/dist/cases-z9E6DCd5.js +639 -0
  28. package/dist/cases-z9E6DCd5.js.map +1 -0
  29. package/dist/{cli-BBOlXKPG.js → cli-DTuLSlmW.js} +442 -707
  30. package/dist/cli-DTuLSlmW.js.map +1 -0
  31. package/dist/{cli-spec-DZ0bnP0_.js → cli-spec-Dsr79Qy1.js} +38 -11
  32. package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
  33. package/dist/cli.js +2 -2
  34. package/dist/{client-tZ2WBDxf.js → client-KM4C5r5_.js} +93 -3
  35. package/dist/client-KM4C5r5_.js.map +1 -0
  36. package/dist/{command-DhiwsZl1.js → command-C2C7P5qo.js} +9 -9
  37. package/dist/{command-DhiwsZl1.js.map → command-C2C7P5qo.js.map} +1 -1
  38. package/dist/{command-CFQ9c1hk.js → command-D8oMB2fa.js} +16 -16
  39. package/dist/command-D8oMB2fa.js.map +1 -0
  40. package/dist/{command-fO0BFsTq.js → command-DSL7I-SI.js} +37 -37
  41. package/dist/command-DSL7I-SI.js.map +1 -0
  42. package/dist/{command-qGS802uX.js → command-Ff0T-1Hi.js} +21 -80
  43. package/dist/command-Ff0T-1Hi.js.map +1 -0
  44. package/dist/{commands-xYTMNTPv.js → commands-Bk9NG1s_.js} +1 -1
  45. package/dist/{commands-CC60TOT5.js → commands-xAOi_VEk.js} +186 -85
  46. package/dist/commands-xAOi_VEk.js.map +1 -0
  47. package/dist/{config-C2UtS9vy.js → config-SvK8pjEK.js} +261 -1708
  48. package/dist/config-SvK8pjEK.js.map +1 -0
  49. package/dist/{context-BajfiH0S.js → context-BdgCao9Q.js} +3 -2
  50. package/dist/context-BdgCao9Q.js.map +1 -0
  51. package/dist/{cost-BuVDamui.js → cost-BdRgySQl.js} +2 -2
  52. package/dist/{cost-BuVDamui.js.map → cost-BdRgySQl.js.map} +1 -1
  53. package/dist/cost-Cr26Alhq.js +1 -0
  54. package/dist/{cursor-catalog-BTK_apL-.js → cursor-catalog-DEzan4IT.js} +3 -3
  55. package/dist/{cursor-catalog-BTK_apL-.js.map → cursor-catalog-DEzan4IT.js.map} +1 -1
  56. package/dist/{db-C5SZ0510.js → db-BzOJ4Js4.js} +130 -84
  57. package/dist/db-BzOJ4Js4.js.map +1 -0
  58. package/dist/db-CQRZ3URX.js +2 -0
  59. package/dist/debug-CF4p4Lp_.js +1 -0
  60. package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
  61. package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
  62. package/dist/{default-effort-FEidNAIi.js → default-effort-BBDPbJ4D.js} +1 -1
  63. package/dist/{default-effort-BVmx7r7L.js → default-effort-BDKIxJyC.js} +4 -4
  64. package/dist/{default-effort-BVmx7r7L.js.map → default-effort-BDKIxJyC.js.map} +1 -1
  65. package/dist/{dist-ByYk_wAs.js → dist-Db7ifchA.js} +37 -8
  66. package/dist/dist-Db7ifchA.js.map +1 -0
  67. package/dist/{docker-BlSopHpZ.js → docker--mukta3L.js} +1 -1
  68. package/dist/{docker-B2F8-PGC.js → docker-DFrjoNs-.js} +3 -3
  69. package/dist/{docker-B2F8-PGC.js.map → docker-DFrjoNs-.js.map} +1 -1
  70. package/dist/{docker-Dto8ROCq.js → docker-DG_uFgn2.js} +109 -77
  71. package/dist/docker-DG_uFgn2.js.map +1 -0
  72. package/dist/{doctor-BcvfKpBX.js → doctor-DWK4ttex.js} +84 -27
  73. package/dist/doctor-DWK4ttex.js.map +1 -0
  74. package/dist/{environment-CC1RV21n.js → environment-cPMCOk-h.js} +79 -39
  75. package/dist/environment-cPMCOk-h.js.map +1 -0
  76. package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
  77. package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
  78. package/dist/{eval-source-CzxdgFBW.js → eval-source-C9b1h8L6.js} +10 -10
  79. package/dist/{eval-source-CzxdgFBW.js.map → eval-source-C9b1h8L6.js.map} +1 -1
  80. package/dist/{evaluate-NTNJ8T9H.js → evaluate-B12Tosvq.js} +377 -142
  81. package/dist/evaluate-B12Tosvq.js.map +1 -0
  82. package/dist/{evaluate-DToSB2bt.d.ts → evaluate-COdzf7Hl.d.ts} +17 -26
  83. package/dist/{files-aAzoGNCB.js → files-BOBdC0uI.js} +5 -5
  84. package/dist/{files-aAzoGNCB.js.map → files-BOBdC0uI.js.map} +1 -1
  85. package/dist/files-C1Rk9FYA.js +1 -0
  86. package/dist/files-DMQ9-Vv5.js +1 -0
  87. package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
  88. package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
  89. package/dist/files-Di0xDbCc.js +1 -0
  90. package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
  91. package/dist/files-DlQshsks.js.map +1 -0
  92. package/dist/{files-DsXULUi5.js → files-F5EG0WPT.js} +5 -5
  93. package/dist/{files-DsXULUi5.js.map → files-F5EG0WPT.js.map} +1 -1
  94. package/dist/files-G5jf8wBp.js +1 -0
  95. package/dist/{files-CZXrkPja.js → files-s28a6czq.js} +54 -13
  96. package/dist/{files-CZXrkPja.js.map → files-s28a6czq.js.map} +1 -1
  97. package/dist/files-tiNWe8Ih.js +64 -0
  98. package/dist/files-tiNWe8Ih.js.map +1 -0
  99. package/dist/gate-CgYdaprG.js +49 -0
  100. package/dist/gate-CgYdaprG.js.map +1 -0
  101. package/dist/gateway.d.ts +1 -1
  102. package/dist/{git-C1MB5bjr.js → git-BpoqEgDV.js} +2 -2
  103. package/dist/{git-C1MB5bjr.js.map → git-BpoqEgDV.js.map} +1 -1
  104. package/dist/{harness-DGsRZDSo.js → harness-C2berMiM.js} +3 -3
  105. package/dist/{harness-DGsRZDSo.js.map → harness-C2berMiM.js.map} +1 -1
  106. package/dist/{harness-DYSC4zMh.js → harness-DXv2gnaT.js} +4 -2
  107. package/dist/harness-DXv2gnaT.js.map +1 -0
  108. package/dist/harness.d.ts +1 -1
  109. package/dist/harness.js +1 -1
  110. package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
  111. package/dist/ids-C_advdJA.js.map +1 -0
  112. package/dist/{images-CNayXCp2.js → images-D2jFze2v.js} +98 -10
  113. package/dist/images-D2jFze2v.js.map +1 -0
  114. package/dist/index.d.ts +23 -3
  115. package/dist/index.js +5 -5
  116. package/dist/{init-D5HtmyMI.js → init-Bi1Q3lRi.js} +7 -6
  117. package/dist/init-Bi1Q3lRi.js.map +1 -0
  118. package/dist/init-CUw65o36.js +1 -0
  119. package/dist/{ink-list-Ca-5dunO.js → ink-list-BpfidQ1h.js} +121 -31
  120. package/dist/ink-list-BpfidQ1h.js.map +1 -0
  121. package/dist/{invocations-DrGXF9dT.js → invocations-CxGNOlh0.js} +3 -3
  122. package/dist/{invocations-DrGXF9dT.js.map → invocations-CxGNOlh0.js.map} +1 -1
  123. package/dist/judge-COMXT1Hx.js +230 -0
  124. package/dist/judge-COMXT1Hx.js.map +1 -0
  125. package/dist/judge.d.ts +1 -1
  126. package/dist/judge.js +1 -1
  127. package/dist/{judgments-m-mb0zv8.js → judgments-CzgpsTqv.js} +2 -2
  128. package/dist/{judgments-m-mb0zv8.js.map → judgments-CzgpsTqv.js.map} +1 -1
  129. package/dist/kinds-Bh2I35id.js +698 -0
  130. package/dist/kinds-Bh2I35id.js.map +1 -0
  131. package/dist/{summary-Uzc9osfV.js → layout-Cm85tQYk.js} +39 -150
  132. package/dist/layout-Cm85tQYk.js.map +1 -0
  133. package/dist/list-DuThsE28.js +1348 -0
  134. package/dist/list-DuThsE28.js.map +1 -0
  135. package/dist/{live-B71o2TkX.js → live-CgYbO4Te.js} +6 -4
  136. package/dist/{live-B71o2TkX.js.map → live-CgYbO4Te.js.map} +1 -1
  137. package/dist/{live-uRURFDMZ.js → live-DL0se4Zd.js} +7 -7
  138. package/dist/{live-uRURFDMZ.js.map → live-DL0se4Zd.js.map} +1 -1
  139. package/dist/live-DZLmj0iG.js +1 -0
  140. package/dist/{matchers-Cs8aDqdU.js → matchers-CXLn-1jm.js} +3 -2
  141. package/dist/{matchers-Cs8aDqdU.js.map → matchers-CXLn-1jm.js.map} +1 -1
  142. package/dist/{model-CXVVB4RK.js → model-CkNVAxob.js} +1 -1
  143. package/dist/model-Cv4GuTsZ.js +1 -0
  144. package/dist/{model-1L19Zukh.js → model-CwwkrCb7.js} +5 -5
  145. package/dist/{model-1L19Zukh.js.map → model-CwwkrCb7.js.map} +1 -1
  146. package/dist/{model-Ce9l9_rF.js → model-V3mMXp6x.js} +203 -313
  147. package/dist/model-V3mMXp6x.js.map +1 -0
  148. package/dist/{model-BxBWFOZn.js → model-cgt5uvu8.js} +5 -5
  149. package/dist/{model-BxBWFOZn.js.map → model-cgt5uvu8.js.map} +1 -1
  150. package/dist/{node-DhIRSrrK.js → node-D7pAU--0.js} +14 -10
  151. package/dist/node-D7pAU--0.js.map +1 -0
  152. package/dist/{opencode-providers-Co7plCea.js → opencode-providers-BGiXt8_B.js} +104 -35
  153. package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
  154. package/dist/{openrouter-models-DNvLRYM7.js → openrouter-models-Cw1z8-sq.js} +2 -2
  155. package/dist/{openrouter-models-DNvLRYM7.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
  156. package/dist/options-DPUmk7v_.js +1 -0
  157. package/dist/{gate-Cp_pQoEe.js → pairs-BGb134Vu.js} +2 -48
  158. package/dist/pairs-BGb134Vu.js.map +1 -0
  159. package/dist/{patch-ug5HpMOj.js → patch-DAi1nCuH.js} +2 -2
  160. package/dist/{patch-ug5HpMOj.js.map → patch-DAi1nCuH.js.map} +1 -1
  161. package/dist/plain-BYUf_fUV.js +267 -0
  162. package/dist/plain-BYUf_fUV.js.map +1 -0
  163. package/dist/{plan-DqnzfDH1.js → plan-CsPilzpk.js} +110 -39
  164. package/dist/plan-CsPilzpk.js.map +1 -0
  165. package/dist/{plugins-B2KQ0y3P.js → plugins-BfnhazOv.js} +10 -267
  166. package/dist/plugins-BfnhazOv.js.map +1 -0
  167. package/dist/{preflight-DyzmvK7i.js → preflight-BixaE4BA.js} +16 -9
  168. package/dist/preflight-BixaE4BA.js.map +1 -0
  169. package/dist/preflight-Bn3ianye.js +1 -0
  170. package/dist/{preflight-DVXEvzFU.js → preflight-D1C-iH0U.js} +1 -1
  171. package/dist/preflight-GkpgrBxP.js +68 -0
  172. package/dist/preflight-GkpgrBxP.js.map +1 -0
  173. package/dist/{prepare-BcSRa7-3.js → prepare-C--rM_wO.js} +2 -2
  174. package/dist/{prepare-21WMQZyt.js → prepare-DQEDXoDp.js} +9 -9
  175. package/dist/{prepare-21WMQZyt.js.map → prepare-DQEDXoDp.js.map} +1 -1
  176. package/dist/{prices-DYJA0tJN.js → prices-5m3SzMZ3.js} +2 -2
  177. package/dist/{prices-DYJA0tJN.js.map → prices-5m3SzMZ3.js.map} +1 -1
  178. package/dist/{provenance-iRxjp4Pu.js → provenance-BXyYwUC8.js} +4 -3
  179. package/dist/provenance-BXyYwUC8.js.map +1 -0
  180. package/dist/provider-BnUoz4KB.js +1 -0
  181. package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
  182. package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
  183. package/dist/{providers-vXMXuwn-.js → providers-1p4HKYIp.js} +152 -9
  184. package/dist/providers-1p4HKYIp.js.map +1 -0
  185. package/dist/providers-B-ijCktf.js +1 -0
  186. package/dist/query-BYhyf0LR.js +1 -0
  187. package/dist/{query-BhaKQ07P.js → query-DhhU9ADI.js} +2 -2
  188. package/dist/{query-BhaKQ07P.js.map → query-DhhU9ADI.js.map} +1 -1
  189. package/dist/{questions-BzdLdV3C.js → questions-CMAq4pr8.js} +9 -9
  190. package/dist/{questions-BzdLdV3C.js.map → questions-CMAq4pr8.js.map} +1 -1
  191. package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
  192. package/dist/record-6IUzwP9K.js.map +1 -0
  193. package/dist/{remarks-CauU408t.js → remarks-BNABzemt.js} +115 -35
  194. package/dist/remarks-BNABzemt.js.map +1 -0
  195. package/dist/{remarks-NMgPd30U.js → remarks-DPCONFxZ.js} +71 -50
  196. package/dist/remarks-DPCONFxZ.js.map +1 -0
  197. package/dist/{remote-Dwy3fyo3.js → remote-U7fNt0UP.js} +6 -6
  198. package/dist/{remote-Dwy3fyo3.js.map → remote-U7fNt0UP.js.map} +1 -1
  199. package/dist/resolve-C14Pmtej.js +1 -0
  200. package/dist/{resolve-B1zMnPwK.js → resolve-Cx3a9UQ2.js} +10 -9
  201. package/dist/resolve-Cx3a9UQ2.js.map +1 -0
  202. package/dist/{results-CfC0id_u.js → results-CQKXmHym.js} +6 -6
  203. package/dist/{results-CfC0id_u.js.map → results-CQKXmHym.js.map} +1 -1
  204. package/dist/{resume-Dkhrpjow.js → resume-POpZa1ge.js} +169 -27
  205. package/dist/resume-POpZa1ge.js.map +1 -0
  206. package/dist/retry-CmG809Mp.js +105 -0
  207. package/dist/retry-CmG809Mp.js.map +1 -0
  208. package/dist/{runtime-DcvQcr36.js → runtime-Xm9oeC7o.js} +26 -7
  209. package/dist/runtime-Xm9oeC7o.js.map +1 -0
  210. package/dist/sandbox/boat.d.ts +26 -0
  211. package/dist/sandbox/boat.js +37 -0
  212. package/dist/sandbox/boat.js.map +1 -0
  213. package/dist/{sandbox-8zCrtuCf.js → sandbox-DrlL4yqb.js} +31 -3
  214. package/dist/sandbox-DrlL4yqb.js.map +1 -0
  215. package/dist/sandbox.d.ts +20 -2
  216. package/dist/sandbox.js +1 -1
  217. package/dist/{server-Dplq8g0r.js → server-C00zd6zl.js} +9 -9
  218. package/dist/{server-Dplq8g0r.js.map → server-C00zd6zl.js.map} +1 -1
  219. package/dist/server-C1UJcpUh.js +1 -0
  220. package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
  221. package/dist/shared-Cfi45Bvy.js.map +1 -0
  222. package/dist/slug-IHh3f770.js +136 -0
  223. package/dist/slug-IHh3f770.js.map +1 -0
  224. package/dist/{source-NWHqn5-o.js → source-DDgr48Gn.js} +2 -2
  225. package/dist/{source-NWHqn5-o.js.map → source-DDgr48Gn.js.map} +1 -1
  226. package/dist/source-DJHmQv_j.js +1 -0
  227. package/dist/src-1dBuXzs4.js +3 -0
  228. package/dist/{src-CUpoVVb8.js → src-BKpioayC.js} +2 -2
  229. package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
  230. package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
  231. package/dist/src-BY1cCaoJ.js +2 -0
  232. package/dist/src-BbZYaQ8b.js +2 -0
  233. package/dist/src-CDFH2vVN.js +2 -0
  234. package/dist/src-CMW5R9Q0.js +1 -0
  235. package/dist/{src-BQXdf-Gw.js → src-CV_7DkPa.js} +47 -45
  236. package/dist/{src-BQXdf-Gw.js.map → src-CV_7DkPa.js.map} +1 -1
  237. package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
  238. package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
  239. package/dist/{src-B9oE_mNS.js → src-CrV2WA2i.js} +2 -2
  240. package/dist/src-CsFaawcZ.js +2 -0
  241. package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
  242. package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
  243. package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
  244. package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
  245. package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
  246. package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
  247. package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
  248. package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
  249. package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
  250. package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
  251. package/dist/src-wKwznPgt.js +2 -0
  252. package/dist/{src-BQ6BiXoc.js → src-y0nOeRRE.js} +610 -72
  253. package/dist/src-y0nOeRRE.js.map +1 -0
  254. package/dist/{state-DDCGUhva.js → state-328JpnHA.js} +17 -17
  255. package/dist/{state-DDCGUhva.js.map → state-328JpnHA.js.map} +1 -1
  256. package/dist/state-Ca55Gk_O.js +1 -0
  257. package/dist/suite-CKBa6MKt.js +2 -0
  258. package/dist/{suite-M8_7xio7.js → suite-COglUhPd.js} +70 -92
  259. package/dist/suite-COglUhPd.js.map +1 -0
  260. package/dist/summary-DDy9DfyJ.js +148 -0
  261. package/dist/summary-DDy9DfyJ.js.map +1 -0
  262. package/dist/tart-cli-C-GsbVsQ.js +1 -0
  263. package/dist/tart-cli-CU0nFS-L.js +55 -0
  264. package/dist/tart-cli-CU0nFS-L.js.map +1 -0
  265. package/dist/tart-dWoJvWHc.js +586 -0
  266. package/dist/tart-dWoJvWHc.js.map +1 -0
  267. package/dist/tart-images-DZ_Mbh-g.js +233 -0
  268. package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
  269. package/dist/tart-ref-7Qv3cm3B.js +1 -0
  270. package/dist/tart-ref-DYZijjrU.js +55 -0
  271. package/dist/tart-ref-DYZijjrU.js.map +1 -0
  272. package/dist/testing.d.ts +2 -2
  273. package/dist/testing.js +4 -4
  274. package/dist/trace-Dx2NBE1O.js +311 -0
  275. package/dist/trace-Dx2NBE1O.js.map +1 -0
  276. package/dist/traits-DKnYkqeK.js +66 -0
  277. package/dist/traits-DKnYkqeK.js.map +1 -0
  278. package/dist/{transcript-Cg4SeqRX.js → transcript-CCf07UmU.js} +2 -2
  279. package/dist/{transcript-Cg4SeqRX.js.map → transcript-CCf07UmU.js.map} +1 -1
  280. package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
  281. package/dist/types-C2mLpph3.js.map +1 -0
  282. package/dist/{types-DGvexkcp.d.ts → types-DBwFsGem.d.ts} +352 -76
  283. package/dist/{judge-DYOF69Oe.js → vercel-CGlSBv63.js} +3 -229
  284. package/dist/vercel-CGlSBv63.js.map +1 -0
  285. package/dist/{vertex-BLQMayc1.js → vertex-Eb2wqahk.js} +2 -2
  286. package/dist/{vertex-BLQMayc1.js.map → vertex-Eb2wqahk.js.map} +1 -1
  287. package/dist/vertex-O88TCwgY.js +1 -0
  288. package/dist/view/index.d.ts +101 -58
  289. package/dist/view/index.js +9 -8
  290. package/dist/views-B0Yt_Q1S.js +2 -0
  291. package/dist/{views-BISrsIjw.js → views-BROacjBv.js} +7 -6
  292. package/dist/views-BROacjBv.js.map +1 -0
  293. package/dist/{worker-entry-DJD_nHkS.js → worker-entry-Smb4bNDc.js} +11 -8
  294. package/dist/{worker-entry-DJD_nHkS.js.map → worker-entry-Smb4bNDc.js.map} +1 -1
  295. package/dist/workspace-C0dDP75l.js +1 -0
  296. package/dist/{workspace-BO1VhUR9.js → workspace-DNDTEk8v.js} +6 -6
  297. package/dist/{workspace-BO1VhUR9.js.map → workspace-DNDTEk8v.js.map} +1 -1
  298. package/package.json +24 -11
  299. package/skills/eval-rubric/SKILL.md +15 -7
  300. package/dist/adapter-CptK032x.js.map +0 -1
  301. package/dist/app-files-sgqwfjtn.js.map +0 -1
  302. package/dist/assemble-CxBHkavM.js +0 -1
  303. package/dist/assemble-Me7Whp-y.js.map +0 -1
  304. package/dist/capture-C3RmjPph.js.map +0 -1
  305. package/dist/capture-YPwxp_iK.js +0 -1
  306. package/dist/cli-BBOlXKPG.js.map +0 -1
  307. package/dist/cli-spec-DZ0bnP0_.js.map +0 -1
  308. package/dist/client-tZ2WBDxf.js.map +0 -1
  309. package/dist/command-CFQ9c1hk.js.map +0 -1
  310. package/dist/command-fO0BFsTq.js.map +0 -1
  311. package/dist/command-qGS802uX.js.map +0 -1
  312. package/dist/commands-CC60TOT5.js.map +0 -1
  313. package/dist/config-C2UtS9vy.js.map +0 -1
  314. package/dist/context-BajfiH0S.js.map +0 -1
  315. package/dist/contrasts-BjOg7wyr.js +0 -101
  316. package/dist/contrasts-BjOg7wyr.js.map +0 -1
  317. package/dist/cost-BJU0MAji.js +0 -1
  318. package/dist/db-C5SZ0510.js.map +0 -1
  319. package/dist/db-puzLzX4k.js +0 -2
  320. package/dist/debug-BnXV7JbX.js +0 -1
  321. package/dist/dist-ByYk_wAs.js.map +0 -1
  322. package/dist/docker-Dto8ROCq.js.map +0 -1
  323. package/dist/doctor-BcvfKpBX.js.map +0 -1
  324. package/dist/environment-CC1RV21n.js.map +0 -1
  325. package/dist/evaluate-NTNJ8T9H.js.map +0 -1
  326. package/dist/files-COW5JBJM.js +0 -1
  327. package/dist/files-DFtuRQJx.js +0 -1
  328. package/dist/files-oOYNR7ZH.js +0 -1
  329. package/dist/files-p-uTdOjA.js.map +0 -1
  330. package/dist/gate-Cp_pQoEe.js.map +0 -1
  331. package/dist/harness-DYSC4zMh.js.map +0 -1
  332. package/dist/ids-DuK6xl4w.js.map +0 -1
  333. package/dist/images-CNayXCp2.js.map +0 -1
  334. package/dist/init-8o6W3_fn.js +0 -1
  335. package/dist/init-D5HtmyMI.js.map +0 -1
  336. package/dist/ink-list-Ca-5dunO.js.map +0 -1
  337. package/dist/judge-DYOF69Oe.js.map +0 -1
  338. package/dist/list-CM_YSJXq.js +0 -374
  339. package/dist/list-CM_YSJXq.js.map +0 -1
  340. package/dist/live-DQc8jr6o.js +0 -1
  341. package/dist/model-Ce9l9_rF.js.map +0 -1
  342. package/dist/model-Cxzo4cjw.js +0 -1
  343. package/dist/node-DhIRSrrK.js.map +0 -1
  344. package/dist/opencode-providers-Co7plCea.js.map +0 -1
  345. package/dist/plan-DqnzfDH1.js.map +0 -1
  346. package/dist/plugins-B2KQ0y3P.js.map +0 -1
  347. package/dist/preflight-DyzmvK7i.js.map +0 -1
  348. package/dist/provenance-iRxjp4Pu.js.map +0 -1
  349. package/dist/provider-DCdTHQpk.js +0 -1
  350. package/dist/providers-D0ob_tra.js +0 -1
  351. package/dist/providers-vXMXuwn-.js.map +0 -1
  352. package/dist/query-CvgUsvep.js +0 -1
  353. package/dist/record-DhzwkSXc.js.map +0 -1
  354. package/dist/remarks-CauU408t.js.map +0 -1
  355. package/dist/remarks-NMgPd30U.js.map +0 -1
  356. package/dist/resolve-B1zMnPwK.js.map +0 -1
  357. package/dist/resolve-BIIeVT12.js +0 -1
  358. package/dist/resume-Dkhrpjow.js.map +0 -1
  359. package/dist/runtime-DcvQcr36.js.map +0 -1
  360. package/dist/sandbox-8zCrtuCf.js.map +0 -1
  361. package/dist/server-0tuj8VSa.js +0 -1
  362. package/dist/shared-ogqPJ0uo.js.map +0 -1
  363. package/dist/source-llwLewOU.js +0 -1
  364. package/dist/src-BQ6BiXoc.js.map +0 -1
  365. package/dist/src-BXnv0ViU.js +0 -2
  366. package/dist/src-BiS8YNN_.js +0 -3
  367. package/dist/src-BkH4GOxu.js +0 -1
  368. package/dist/src-C-tH5ebd.js +0 -2
  369. package/dist/src-Cg1bFs9g.js +0 -2
  370. package/dist/src-DV_UorZv.js +0 -2
  371. package/dist/src-XxUpr2sa.js +0 -2
  372. package/dist/state-DZbAO95a.js +0 -1
  373. package/dist/suite-M8_7xio7.js.map +0 -1
  374. package/dist/suite-u2KJnGpY.js +0 -2
  375. package/dist/summary-Uzc9osfV.js.map +0 -1
  376. package/dist/types-C4OkYhZg.js.map +0 -1
  377. package/dist/vertex-h_peaZTN.js +0 -1
  378. package/dist/views-BISrsIjw.js.map +0 -1
  379. package/dist/views-Bfab8fM9.js +0 -2
  380. package/dist/workspace-B8EOCuxl.js +0 -1
@@ -1,38 +1,42 @@
1
- import { t as isObjectLike } from "./object-CeQ8dgUC.js";
2
- import { a as TURN_EDITED, r as withLease } from "./client-tZ2WBDxf.js";
3
1
  import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-ttMTnIYX.js";
4
- import { h as parseDuration, n as dataProblem, p as MAX_TIMER_MS } from "./shared-ogqPJ0uo.js";
5
- import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-CC1RV21n.js";
6
- import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-B2KQ0y3P.js";
7
- import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-DuK6xl4w.js";
8
- import { A as evalPlan, At as agentMcp, Bt as refusedPlugins, D as cascade, Dt as AGENTS, E as brokeredIn, F as matchesTags, Ft as expectedBlockedFor, G as sandboxSetBy, Gt as judgeProblem, H as runsOf, Ht as INPUT_KEYS, I as plainHeaderError, J as unbrokeredError, Jt as decideState, K as secretVarsIn, Kt as artifactRefsOf, L as plainHeaders, Lt as keyRejected, M as harnessId, Mt as brokeringFor, Nt as claudeMcpServers, P as majorityOf, Rt as layoutOf, U as sandboxId, Ut as NO_JUDGE_INPUT, V as refuseIllegalCompare, Vt as writePluginFiles, W as sandboxProvider, Wt as inputsOn, Xt as resolveJudgeArtifacts, Yt as filePartsOf, Z as allowedHosts, _ as fellBackToDirect, _t as observedHere, at as createFakeSandbox, b as resolveModelGateway, et as egressLabel, gt as resolveStateOptions, ht as mergeJudge, j as expand, k as checkOptions, nt as isDirect, q as unbrokered, qt as badJudgeArtifacts, rt as unenforceable, v as gatewayScope, vt as observedName, w as resolveGateway, yt as observer, zt as refusedMcp } from "./config-C2UtS9vy.js";
9
- import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-DYOF69Oe.js";
10
- import { v as vertexSettings } from "./vertex-BLQMayc1.js";
11
- import { _ as redactValue, d as maskedVariables, f as redactJson, g as redactSplitSecrets, h as redactSecrets, o as findSecrets } from "./redact-BhjtnjrU.js";
2
+ import { g as parseDuration, m as MAX_TIMER_MS, n as dataProblem, t as closest } from "./shared-Cfi45Bvy.js";
3
+ import { t as isObjectLike } from "./object-CeQ8dgUC.js";
12
4
  import { a as plural, i as formatDuration, n as clip, o as withoutBom, t as capMiddle } from "./text-B8-W3Cvn.js";
5
+ import { a as TURN_EDITED, f as limitsLabel, r as withLease } from "./client-KM4C5r5_.js";
6
+ import { d as sandboxKind, n as capabilitiesOf, s as providerLimit } from "./kinds-Bh2I35id.js";
7
+ import { O as sandboxPaths, P as workspaceOf, f as filesOf, i as callerDir, j as traitsOf, l as environmentFrom, p as globFilter, u as environmentPaths } from "./environment-cPMCOk-h.js";
8
+ import { t as evalSlug } from "./slug-IHh3f770.js";
9
+ import { n as attemptKey, o as recordFileName, r as evalId, s as relativeFile, t as attemptId } from "./ids-C_advdJA.js";
10
+ import { $ as runsOf, A as brokeredIn, B as expand, Bt as decideState, Ct as resolveStateOptions, Et as observer, Ft as NO_JUDGE_INPUT, Ht as resolveJudgeArtifacts, It as inputsOn, J as plainHeaders, K as plainHeaderError, L as checkOptions, Lt as judgeProblem, Pt as INPUT_KEYS, Q as refuseIllegalCompare, Rt as artifactRefsOf, St as mergeJudge, Tt as observedName, U as majorityOf, V as harnessId, Vt as filePartsOf, W as matchesTags, _ as fellBackToDirect, at as unbrokered, b as resolveModelGateway, et as sandboxId, gt as unenforceable, ht as isDirect, it as secretVarsIn, j as cascade, nt as sandboxSetBy, ot as unbrokeredError, pt as egressLabel, tt as sandboxProvider, ut as allowedHosts, v as gatewayScope, w as resolveGateway, wt as observedHere, z as evalPlan, zt as badJudgeArtifacts } from "./config-SvK8pjEK.js";
11
+ import { l as vercelCredentialProblem } from "./vercel-CGlSBv63.js";
12
+ import { v as vertexSettings } from "./vertex-Eb2wqahk.js";
13
+ import { _ as redactValue, d as maskedVariables, f as redactJson, g as redactSplitSecrets, h as redactSecrets, o as findSecrets } from "./redact-BhjtnjrU.js";
14
+ import { f as runJudge, h as isApiError, l as judgeModelName } from "./judge-COMXT1Hx.js";
13
15
  import { t as inCI } from "./env-8x1O8I1M.js";
14
- import { dt as OPAQUE_OUTPUT, mt as shellQuote, t as MCP_TIMER } from "./src-BQ6BiXoc.js";
15
- import { t as InfraError } from "./types-C4OkYhZg.js";
16
- import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-DcvQcr36.js";
17
- import { c as formatBytes } from "./debug-B5jDImcW.js";
18
- import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-Dp7Wtm8M.js";
19
- import { t as attemptDebugDir } from "./files-p-uTdOjA.js";
20
- import { r as openRouterFallbacks } from "./openrouter-models-DNvLRYM7.js";
21
- import { a as writeCaptureSidecars, c as sandboxFor, d as snapshot, f as workspaceBlobs, l as changedBetween, o as asCheck, p as workspaceFiles, r as serveCommand, s as createChecks, t as capturePage, u as diffSinceSnapshot } from "./capture-C3RmjPph.js";
22
- import { t as contextTaker } from "./context-BajfiH0S.js";
23
- import { a as skillFiles, i as resolveEnvironment } from "./resolve-B1zMnPwK.js";
24
- import { n as isDockerSandbox } from "./docker-Dto8ROCq.js";
25
- import { t as PrepareError } from "./provider-DZd8t_XW.js";
26
- import { i as preparedWorkspace } from "./workspace-BO1VhUR9.js";
16
+ import { bt as shellQuote, gt as OPAQUE_OUTPUT, n as MCP_TIMER } from "./src-y0nOeRRE.js";
17
+ import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-BfnhazOv.js";
18
+ import { _ as unverifiedHarness, c as brokeringFor, d as expectedBlockedFor, g as refusedPlugins, h as refusedMcp, l as claudeMcpServers, m as layoutOf, o as agentMcp, p as keyRejected, t as AGENTS, v as writePluginFiles } from "./agents-DXuFxtGC.js";
19
+ import { n as hasNetworkSession, t as InfraError } from "./types-C2mLpph3.js";
20
+ import { a as registerTest, i as registerGroup, n as declarePlanned, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-Xm9oeC7o.js";
21
+ import { c as formatBytes } from "./debug-CIXzB4wK.js";
22
+ import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CAdlwlNu.js";
23
+ import { t as attemptDebugDir } from "./files-DlQshsks.js";
24
+ import { r as openRouterFallbacks } from "./openrouter-models-Cw1z8-sq.js";
25
+ import { a as writeCaptureSidecars, c as sandboxFor, d as snapshot, f as workspaceBlobs, l as changedBetween, o as asCheck, p as workspaceFiles, r as serveCommand, s as createChecks, t as capturePage, u as diffSinceSnapshot } from "./capture-Ccf0Hmyx.js";
26
+ import { t as contextTaker } from "./context-BdgCao9Q.js";
27
+ import { a as skillFiles, i as resolveEnvironment } from "./resolve-Cx3a9UQ2.js";
28
+ import { t as PrepareError } from "./provider-BxzZq3Th.js";
29
+ import { i as preparedWorkspace } from "./workspace-DNDTEk8v.js";
27
30
  import { t as patchPath } from "./patch-C7yBztZj.js";
28
- import { n as judgeDiff } from "./patch-ug5HpMOj.js";
29
- import { t as transcriptForJudge } from "./transcript-Cg4SeqRX.js";
31
+ import { n as judgeDiff } from "./patch-DAi1nCuH.js";
32
+ import { t as transcriptForJudge } from "./transcript-CCf07UmU.js";
30
33
  import { i as writeTar, n as readTar, t as extractTar } from "./tarball-CESxtK4X.js";
31
- import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-CzxdgFBW.js";
32
- import { i as withDefaultEffort } from "./default-effort-BVmx7r7L.js";
34
+ import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-C9b1h8L6.js";
35
+ import { i as withDefaultEffort } from "./default-effort-BDKIxJyC.js";
36
+ import { a as traitFilters, i as pickTraits, n as controlTraitKeys, r as filtersLabel } from "./traits-DKnYkqeK.js";
37
+ import { z } from "zod";
33
38
  import { dirname, join } from "node:path";
34
39
  import { parse, stringify } from "yaml";
35
- import { z } from "zod";
36
40
  import { randomBytes } from "node:crypto";
37
41
  import { setTimeout as setTimeout$1 } from "node:timers/promises";
38
42
  import { gunzipSync, gzipSync } from "node:zlib";
@@ -66,7 +70,7 @@ function assertionRecords(raw) {
66
70
  const seen = /* @__PURE__ */ new Map();
67
71
  const used = /* @__PURE__ */ new Set();
68
72
  return raw.map((r) => {
69
- const named = r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
73
+ const named = r.matcher === "toStayWithin" ? limitsLabel(r.args[0]) : r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
70
74
  const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
71
75
  let n = seen.get(base) ?? 0;
72
76
  let label;
@@ -112,11 +116,11 @@ function tracePlan(debug, spec, ctx) {
112
116
  const from = (key, fallback = ctx.configFile ? `${config} or its default` : "default") => `(${spec.sources[key] ?? fallback})`;
113
117
  const profile = ctx.profile ? `profile ${ctx.profile.name} (${ctx.profile.reason})` : "no profile";
114
118
  debug.log("plan", `config ${config} · ${profile} · run ${ctx.runId} · attempt ${spec.cell.key}#${spec.n}`);
115
- debug.log("plan", `harness ${harnessId(spec.harness)} ${from("harness")}`);
119
+ debug.log("plan", `harness ${harnessId(spec.harness)} ${from("harnesses")}`);
116
120
  debug.log("plan", `sandbox ${sandboxId(spec.sandbox)} ${from("sandbox")}`);
117
121
  const env = spec.environment;
118
122
  const role = spec.cell.role === "variant" ? ` · variant ${spec.cell.variant}` : "";
119
- const envFrom = spec.cell.role === "variant" ? from("compare") : from("environment", "none set");
123
+ const envFrom = spec.cell.role === "variant" || spec.sources.compare ? from("compare") : from("environments", "none set");
120
124
  debug.log("plan", `environment ${env.name}${role} ${envFrom}`);
121
125
  debug.log("plan", `egress ${egressLabel(spec.egress) || "proxy"} ${from("egress")}`);
122
126
  const gate = typeof spec.gate === "string" ? spec.gate : `passRate ${spec.gate.passRate}`;
@@ -254,11 +258,12 @@ const createFakeHarness = (descriptor) => {
254
258
  text,
255
259
  status,
256
260
  startedAt: new Date(startedAt).toISOString(),
257
- durationMs: Date.now() - startedAt,
261
+ durationMs: descriptor.turnMs ?? Date.now() - startedAt,
258
262
  transcript,
259
- usage: null,
263
+ usage: descriptor.usage ?? null,
260
264
  model: descriptor.servedModel ?? null,
261
- unreported: []
265
+ unreported: [],
266
+ ...descriptor.turnMs === void 0 ? {} : { agentMs: descriptor.turnMs }
262
267
  });
263
268
  const write = async (path, content) => {
264
269
  const time = at();
@@ -386,7 +391,7 @@ async function brokerKey(descriptor, sandbox) {
386
391
  const { credential, endpoints } = AGENTS[descriptor.layout ?? "cursor"];
387
392
  const key = process.env[credential];
388
393
  if (!key) throw new InfraError(`${credential} isn't set, so the fake agent has no key to broker. Set it in the environment or .env.local.`);
389
- const network = isDockerSandbox(sandbox) ? sandbox.network : void 0;
394
+ const network = hasNetworkSession(sandbox) ? sandbox.network : void 0;
390
395
  if (!network?.setRequestTransformations) throw new InfraError("fakeHarness({ brokers: true }) needs docker(): this sandbox can't broker");
391
396
  const placeholder = `reeval_fake_${randomBytes(24).toString("base64url")}`;
392
397
  await network.setRequestTransformations([...new Set(endpoints.map((e) => e.host))].map((host) => ({
@@ -478,18 +483,19 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
478
483
  };
479
484
  }
480
485
  // src/harness/index.ts
481
- async function createHarness(descriptor, vertex) {
486
+ /** The attempt's harness, for the sandbox it runs in; an agent the sandbox can't run is an infra error. */
487
+ async function createHarness(descriptor, sandbox, vertex) {
482
488
  if (descriptor.kind === "fake") return createFakeHarness(descriptor);
483
- const { createAgentHarness } = await import("./agent-B63OzUX9.js");
484
- return createAgentHarness(descriptor, vertex);
489
+ const { createAgentHarness } = await import("./agent-BmsMXZXx.js");
490
+ return createAgentHarness(descriptor, sandbox, vertex);
485
491
  }
486
492
  /** Where each harness's agent looks for repo skills, relative to the workspace. */
487
493
  const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
488
494
  // src/attempt/environment.ts
489
495
  /**
490
496
  * Resolves the attempt's environment (files, skills, secrets and their placeholders)
491
- * and finds the image a Docker sandbox starts from, which preflight prepared. Runs on the test's
492
- * signal, before the eval timer starts.
497
+ * and finds the image its sandbox starts from, when its kind has one, which preflight prepared.
498
+ * Runs on the test's signal, before the eval timer starts.
493
499
  */
494
500
  async function prepareEnvironment(a, root, runId, where, signal) {
495
501
  const { spec, record } = a;
@@ -508,13 +514,14 @@ async function prepareEnvironment(a, root, runId, where, signal) {
508
514
  const debug = attemptDebug();
509
515
  if (debug) traceEnvironment(debug, spec, env);
510
516
  record.image = null;
511
- if (spec.sandbox.kind === "docker") {
512
- const { preparedFor } = await import("./preflight-DVXEvzFU.js");
513
- const { attemptImage, dockerProvider, preparedImage } = await import("./providers-D0ob_tra.js");
517
+ const kind = sandboxKind(spec.sandbox);
518
+ if (kind.capabilities.images) {
519
+ const { preparedFor } = await import("./preflight-D1C-iH0U.js");
520
+ const { attemptImage, createProviders, preparedImage } = await import("./providers-B-ijCktf.js");
514
521
  signal.throwIfAborted();
515
- const provider = dockerProvider({ engine: spec.sandbox.engine ?? "docker" });
522
+ const provider = createProviders()[kind.provider(spec.sandbox)];
516
523
  const builds = builtPlugins(spec.environment.layers);
517
- const prepared = preparedFor(root, runId, provider, spec.sandbox, spec.harness, builds);
524
+ const prepared = provider && preparedFor(root, runId, provider, spec.sandbox, spec.harness, builds);
518
525
  if (!prepared) throw new InfraError(`The sandbox image for ${where} wasn't prepared before the run started`);
519
526
  record.image = attemptImage(prepared);
520
527
  a.image = record.image.ref;
@@ -534,7 +541,16 @@ async function prepareEnvironment(a, root, runId, where, signal) {
534
541
  }
535
542
  const { ref, id, base, install, agentCli } = record.image;
536
543
  debugLog("image", `ready ${ref} (${id.replace(/^sha256:/, "").slice(0, 12)}) · prepared before the run${ref !== base ? ` · on ${base}` : ""}${agentCli ? ` · agent CLI ${agentCli}` : ""}${install.length ? ` · install: ${install.join(" && ")}` : ""}`);
537
- } else debugLog("image", `none: the ${spec.sandbox.kind} sandbox runs on the host`);
544
+ } else if (kind.capabilities.isolated) {
545
+ const { preparedFor } = await import("./preflight-D1C-iH0U.js");
546
+ const { createProviders, vmImage } = await import("./providers-B-ijCktf.js");
547
+ const provider = createProviders()[kind.provider(spec.sandbox)];
548
+ const prepared = provider && preparedFor(root, runId, provider, spec.sandbox, spec.harness);
549
+ const vm = prepared ? vmImage(prepared) : null;
550
+ if (vm) record.vmImage = vm;
551
+ const from = vm?.ref ?? ("image" in spec.sandbox ? spec.sandbox.image : void 0);
552
+ debugLog("image", `${from ?? "none recorded"}${vm?.digest ? ` (${vm.digest})` : ""} · ${kind.helpers[0]}() checked it before the run and starts each attempt from it`);
553
+ } else debugLog("image", kind.capabilities.isolated ? `none: the ${spec.sandbox.kind} sandbox starts from its provider's base` : `none: the ${spec.sandbox.kind} sandbox runs on the host`);
538
554
  }
539
555
  // src/judge/size.ts
540
556
  /** Roughly four characters a token, as `reeval analyze` counts: an estimate, not the model's. */
@@ -854,18 +870,12 @@ function judgeCaller(a, readFile) {
854
870
  }
855
871
  // src/sandbox/index.ts
856
872
  async function createSandbox(descriptor, options) {
873
+ const kind = sandboxKind(descriptor);
857
874
  try {
858
- switch (descriptor.kind) {
859
- case "fake": return await createFakeSandbox(descriptor, options.env);
860
- case "docker": {
861
- const { createDockerSandbox } = await import("./docker-BlSopHpZ.js");
862
- return await createDockerSandbox(descriptor, options);
863
- }
864
- }
875
+ return await kind.create(descriptor, options);
865
876
  } catch (error) {
866
877
  if (error instanceof InfraError || options.signal.aborted) throw error;
867
- const kind = descriptor.kind === "docker" ? descriptor.engine ?? "docker" : descriptor.kind;
868
- throw new InfraError(`Failed to create ${kind} sandbox: ${error.message}`);
878
+ throw new InfraError(`Failed to create ${kind.provider(descriptor)} sandbox: ${error.message}`);
869
879
  }
870
880
  }
871
881
  // src/attempt/record.ts
@@ -1323,7 +1333,7 @@ const startTimeoutMs = (harness) => parseDuration(harness.startTimeout ?? "90s")
1323
1333
  function setupAllowance(planned) {
1324
1334
  const setupMs = planned.environment.layers.flatMap((l) => l.spec.setup ?? []).reduce((ms, step) => ms + parseDuration((typeof step === "string" ? void 0 : step.timeout) ?? "5m"), 0);
1325
1335
  const { sandbox } = planned;
1326
- return setupMs + (sandbox.kind === "docker" && !!(sandbox.from || sandbox.dockerfile || sandbox.install?.length) ? SETUP_BUDGET_MS : 0) + 2 * startTimeoutMs(planned.harness);
1336
+ return setupMs + (sandboxKind(sandbox).buildsAtStart(sandbox) ? SETUP_BUDGET_MS : 0) + 2 * startTimeoutMs(planned.harness);
1327
1337
  }
1328
1338
  /** The eval's timer, paused while the sandbox and its setup steps start. */
1329
1339
  function evalTimer(ms) {
@@ -1405,7 +1415,7 @@ const SETUP_COMMAND_CHARS = 200;
1405
1415
  */
1406
1416
  async function leftRunning(a, box, env) {
1407
1417
  const names = unbrokeredChecks(env);
1408
- if (a.spec.sandbox.kind !== "docker" || !env.setup.length || !names.length) return [];
1418
+ if (!capabilitiesOf(a.spec.sandbox).processTable || !env.setup.length || !names.length) return [];
1409
1419
  const r = await box.exec([
1410
1420
  "for t in tr sed sort; do command -v \"$t\" >/dev/null || { echo \"no $t in the image\" >&2; exit 1; }; done",
1411
1421
  "for p in /proc/[0-9]*; do",
@@ -1641,7 +1651,7 @@ async function startSession(a, runId) {
1641
1651
  })
1642
1652
  });
1643
1653
  a.sandbox = box;
1644
- if (spec.sandbox.kind === "fake") debugLog("container", `fake sandbox at ${box.workdir}`);
1654
+ if (!capabilitiesOf(spec.sandbox).isolated) debugLog("container", `${sandboxKind(spec.sandbox).provider(spec.sandbox)} sandbox at ${box.workdir}`);
1645
1655
  a.phases = brokerPhases(box, env.substitutions);
1646
1656
  if (a.signal.aborted) box.destroy();
1647
1657
  a.signal.throwIfAborted();
@@ -1687,7 +1697,7 @@ async function startSession(a, runId) {
1687
1697
  debugLog("env", "stdio MCP servers start through reeval's MCP timer");
1688
1698
  } else if (timed.untimed) debugLog("env", `stdio MCP servers start untimed: ${timed.untimed}`);
1689
1699
  if (layout.pluginPlacement === "namespaced") await writePluginMcp(box, env, plugins, timed.servers);
1690
- const harness = await createHarness(spec.harness, spec.vertex);
1700
+ const harness = await createHarness(spec.harness, spec.sandbox, spec.vertex);
1691
1701
  const harnessStart = performance.now();
1692
1702
  debugLog("harness", `starting ${harnessId(spec.harness)}: bootstrap and session`);
1693
1703
  const session = await startHarness(a, harness, box, {
@@ -2070,7 +2080,8 @@ function turnSender(a, takeContext, checks) {
2070
2080
  durationMs: Date.now() - sendStarted
2071
2081
  };
2072
2082
  });
2073
- const { raw, rejectedKey, reasoningMs, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
2083
+ const timedBefore = a.record.timings.agentMs;
2084
+ const { raw, rejectedKey, reasoningMs, agentMs, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
2074
2085
  entry.reasoningMs = null;
2075
2086
  if (watch) entry.mcpStartup = mcpStartupOf((await watch.end()).entries);
2076
2087
  throw error;
@@ -2078,7 +2089,8 @@ function turnSender(a, takeContext, checks) {
2078
2089
  cancel();
2079
2090
  watch?.close();
2080
2091
  });
2081
- const sendWallMs = Date.now() - sendStarted;
2092
+ const sendWallMs = agentMs ?? Date.now() - sendStarted;
2093
+ if (agentMs !== void 0) a.record.timings.agentMs = timedBefore + agentMs;
2082
2094
  const startup = await watch?.end();
2083
2095
  const timedOut = startup?.timedOut && mcpTimeoutMessage(startup.timedOut);
2084
2096
  const seenAt = startup?.at ?? Date.now();
@@ -2291,6 +2303,37 @@ function bestOfCell(bestOf) {
2291
2303
  settled: (settled) => settle[n - 1]?.(sideOf(settled))
2292
2304
  });
2293
2305
  }
2306
+ /** Characters a case key can't hold: ids and paths split on them, and control characters. */
2307
+ const BAD_KEY = /[#/›\u0000-\u001f\u007f]/;
2308
+ /** The name a case's eval plans under: `adds a health check [nextjs]`. */
2309
+ const caseEvalName = (name, key) => `${name} [${key}]`;
2310
+ /** The key a case gets without `caseKey`: a string, a finite number, or an object's `id` or `name`. */
2311
+ function defaultKey(item) {
2312
+ if (typeof item === "string") return item;
2313
+ if (typeof item === "number" && Number.isFinite(item)) return String(item);
2314
+ if (isObjectLike(item) && !Array.isArray(item)) {
2315
+ if (typeof item.id === "string") return item.id;
2316
+ if (typeof item.name === "string") return item.name;
2317
+ }
2318
+ }
2319
+ /**
2320
+ * Each case's key, in list order: `caseKey(item)` when given, else its default. Trimmed, 1–80
2321
+ * characters, with no `#`, `/`, `›` or control characters, and unique within the eval.
2322
+ */
2323
+ function caseKeys(cases, caseKey, where) {
2324
+ const seen = /* @__PURE__ */ new Map();
2325
+ return cases.map((item, i) => {
2326
+ const raw = caseKey ? caseKey(item) : defaultKey(item);
2327
+ if (raw === void 0) throw new UsageError(`cases[${i}] in ${where} has no key: a case is its own key when it's a string or a number, or an object's id or name. Give it an id, or pass caseKey: (item) => item.slug.`);
2328
+ if (typeof raw !== "string") throw new UsageError(`caseKey in ${where} must return a string (got ${redactSecrets(JSON.stringify(raw) ?? String(raw))} for cases[${i}])`);
2329
+ const key = raw.trim();
2330
+ if (!key || key.length > 80 || BAD_KEY.test(key)) throw new UsageError(`cases[${i}] in ${where} has the key ${redactSecrets(JSON.stringify(raw))}: a case key is 1–80 characters, with no #, /, › or control characters.${caseKey ? "" : " Pass caseKey to give it another."}`);
2331
+ const earlier = seen.get(key);
2332
+ if (earlier !== void 0) throw new UsageError(`cases[${earlier}] and cases[${i}] in ${where} have the same key "${redactSecrets(key)}". Each case needs its own key${caseKey ? "" : ": give them distinct ids or names, or pass caseKey"}.`);
2333
+ seen.set(key, i);
2334
+ return key;
2335
+ });
2336
+ }
2294
2337
  // src/contract/declaration.ts
2295
2338
  /** A node's options: any object, until the cascade checks it. Its JSON Schema is the type's. */
2296
2339
  const evalOptionsDataSchema = z.record(z.string(), z.unknown());
@@ -2302,7 +2345,7 @@ const declaredModeSchema = z.enum([
2302
2345
  const declaredFamilySchema = z.strictObject({
2303
2346
  name: z.string(),
2304
2347
  case: z.number().int().min(0),
2305
- cases: z.number().int().min(2)
2348
+ cases: z.number().int().min(1)
2306
2349
  });
2307
2350
  const declaredTodoSchema = z.strictObject({
2308
2351
  type: z.literal("todo"),
@@ -2347,6 +2390,89 @@ const NODE_SCHEMAS = {
2347
2390
  * tree's `children`. Options are left to the cascade.
2348
2391
  */
2349
2392
  const declarationProblem = (children) => dataProblem(declarationTreeSchema, { children }, "The declaration tree", (node) => Object.keys(NODE_SCHEMAS[String(node.type)]?.shape ?? {}));
2393
+ // src/where.ts
2394
+ /** Keys a trait lookup can hit without naming a trait: JSON, promises and inspection. */
2395
+ const PASSTHROUGH = /* @__PURE__ */ new Set([
2396
+ "toJSON",
2397
+ "then",
2398
+ "constructor"
2399
+ ]);
2400
+ function pickPairs(w, attempts) {
2401
+ const controls = attempts.filter((a) => a.cell.role === "control");
2402
+ const known = [...new Set(controls.flatMap((a) => Object.keys(traitsOf(a.environment.layers))))];
2403
+ const decisions = /* @__PURE__ */ new Map();
2404
+ const decide = (control) => {
2405
+ const id = control.cell.key;
2406
+ const made = decisions.get(id);
2407
+ if (made) return made;
2408
+ const env = control.environment;
2409
+ const traits = traitsOf(env.layers);
2410
+ let readHarness = false;
2411
+ const view = new Proxy(traits, { get(target, key, receiver) {
2412
+ if (typeof key === "string" && !(key in target) && !PASSTHROUGH.has(key)) {
2413
+ if (!known.includes(key)) {
2414
+ const near = closest(key, known);
2415
+ throw new UsageError(`${w.label}'s where reads environment.traits.${key}, which no environment defines (${known.length ? `they define: ${known.join(", ")}` : "none defines any"}).${near ? ` Did you mean "${near}"?` : ""}`);
2416
+ }
2417
+ }
2418
+ return Reflect.get(target, key, receiver);
2419
+ } });
2420
+ const pair = {
2421
+ case: w.item,
2422
+ environment: Object.freeze({
2423
+ name: env.name,
2424
+ traits: view
2425
+ }),
2426
+ get harness() {
2427
+ readHarness = true;
2428
+ return control.cell.harness;
2429
+ }
2430
+ };
2431
+ let result;
2432
+ try {
2433
+ result = w.where(pair);
2434
+ } catch (error) {
2435
+ if (error instanceof UsageError) throw error;
2436
+ const why = error instanceof Error ? error.message : String(error);
2437
+ throw new UsageError(`${w.label}'s where threw for case ${w.key} on ${env.name}: ${why}`);
2438
+ }
2439
+ if (typeof result !== "boolean") throw new UsageError(`${w.label}'s where returned ${JSON.stringify(result) ?? String(result)} for case ${w.key} on ${env.name}; it must return true or false.`);
2440
+ const decided = {
2441
+ keep: result,
2442
+ readHarness
2443
+ };
2444
+ decisions.set(id, decided);
2445
+ return decided;
2446
+ };
2447
+ const controlOf = (a) => a.cell.role === "control" ? a : controls.find((c) => c.cell.harness === a.cell.harness);
2448
+ const cells = [];
2449
+ const keptCells = [];
2450
+ const excluded = [];
2451
+ return {
2452
+ kept: attempts.filter((a) => {
2453
+ const { keep, readHarness } = decide(controlOf(a));
2454
+ const first = !cells.some((c) => c.key === a.cell.key);
2455
+ if (first) cells.push({
2456
+ key: a.cell.key,
2457
+ environment: a.cell.environment
2458
+ });
2459
+ if (keep) {
2460
+ if (first) keptCells.push(a.cell.key);
2461
+ return true;
2462
+ }
2463
+ const pair = {
2464
+ case: w.key,
2465
+ environment: a.cell.environment,
2466
+ ...readHarness ? { harness: a.cell.harness } : {}
2467
+ };
2468
+ if (!excluded.some((e) => e.environment === pair.environment && e.harness === pair.harness)) excluded.push(pair);
2469
+ return false;
2470
+ }),
2471
+ excluded,
2472
+ cells,
2473
+ keptCells
2474
+ };
2475
+ }
2350
2476
  // src/plan-eval.ts
2351
2477
  /** Lets the attempt's own eval timer fire before the engine's backstop, so the message and cleanup are its own. */
2352
2478
  const GRACE_MS = 5e3;
@@ -2355,7 +2481,7 @@ function configLevel(config) {
2355
2481
  return {
2356
2482
  harness: config.harness,
2357
2483
  sandbox: config.sandbox,
2358
- environment: config.environment,
2484
+ environments: config.environments,
2359
2485
  compare: void 0,
2360
2486
  runs,
2361
2487
  bestOf,
@@ -2390,19 +2516,80 @@ function planEval(declaration, ctx) {
2390
2516
  profile: ctx.profile,
2391
2517
  runs: ctx.runs,
2392
2518
  skipped: declaration.mode === "skip" || declaration.mode === "todo",
2393
- pooled: declaration.family !== void 0
2519
+ pooled: (declaration.family?.cases ?? 0) >= 2
2394
2520
  });
2395
2521
  const { harnesses } = resolvedEval;
2396
2522
  if (!matchesTags(resolvedEval.resolved.tags, ctx.tags)) return {
2397
2523
  tests: null,
2398
- harnesses
2524
+ harnesses,
2525
+ traits: []
2526
+ };
2527
+ const { resolved } = resolvedEval;
2528
+ const declared = expand({
2529
+ ...resolved,
2530
+ harness: resolved.harness.map((h) => withDefaultEffort(h, ctx.cursorEfforts))
2531
+ });
2532
+ const traits = controlTraitKeys(declared);
2533
+ const filters = traitFilters(ctx.traits, ctx.profile);
2534
+ const traitSkip = filters.length > 0 && !resolvedEval.skip && !pickTraits(declared, filters).kept.length;
2535
+ const narrowed = traitSkip ? {
2536
+ ...resolvedEval,
2537
+ skip: noCellLeft(declared, filters)
2538
+ } : resolvedEval;
2539
+ const picks = [];
2540
+ if (traitSkip && declaration.where) picks.push(pickPairs(declaration.where, declared));
2541
+ let traitExcluded = [];
2542
+ const pick = (attempts) => {
2543
+ let kept = attempts;
2544
+ if (declaration.where) {
2545
+ const picked = pickPairs(declaration.where, attempts);
2546
+ picks.push(picked);
2547
+ kept = picked.kept;
2548
+ }
2549
+ if (!filters.length) return kept;
2550
+ const byTraits = pickTraits(kept, filters);
2551
+ traitExcluded = byTraits.excluded;
2552
+ return byTraits.kept;
2553
+ };
2554
+ const leftOut = () => `${filtersLabel(filters)} ${filters.length === 1 ? "leaves" : "leave"} out ${traitExcluded.join(", ")}`;
2555
+ let tests = planTests(declaration, ctx, where, narrowed, pick);
2556
+ const allLeftOut = !tests.length && traitExcluded.length > 0;
2557
+ if (allLeftOut) {
2558
+ const skip = `no cell left: ${leftOut()}`;
2559
+ tests = planTests(declaration, ctx, where, {
2560
+ ...narrowed,
2561
+ skip
2562
+ }, pick);
2563
+ }
2564
+ if (traitExcluded.length) {
2565
+ for (const t of tests) t.attempt.plan.traitExcluded = traitExcluded;
2566
+ const notes = new Set(tests.map((t) => t.attempt.plan.notes));
2567
+ if (!allLeftOut) for (const list of notes) list.push(leftOut());
2568
+ }
2569
+ const [picked] = picks;
2570
+ if (!picked) return {
2571
+ tests,
2572
+ harnesses,
2573
+ traits
2399
2574
  };
2575
+ const { excluded, cells, keptCells } = picked;
2400
2576
  return {
2401
- tests: planTests(declaration, ctx, where, resolvedEval),
2402
- harnesses
2577
+ tests,
2578
+ harnesses,
2579
+ traits,
2580
+ picked: {
2581
+ excluded,
2582
+ cells,
2583
+ keptCells
2584
+ }
2403
2585
  };
2404
2586
  }
2405
- function planTests(declaration, ctx, where, resolvedEval) {
2587
+ /** Why an eval none of whose control environments the trait filters match doesn't run. */
2588
+ function noCellLeft(attempts, filters) {
2589
+ const envs = [...new Set(attempts.filter((a) => a.cell.role === "control").map((a) => a.cell.environment))];
2590
+ return `no cell left: ${filtersLabel(filters)} ${filters.length === 1 ? "matches" : "match"} none of ${envs.join(", ")}`;
2591
+ }
2592
+ function planTests(declaration, ctx, where, resolvedEval, pick) {
2406
2593
  const { name, mode, groups: enclosing } = declaration;
2407
2594
  const groups = enclosing.map((g) => g.name);
2408
2595
  const { resolved } = resolvedEval;
@@ -2438,10 +2625,10 @@ function planTests(declaration, ctx, where, resolvedEval) {
2438
2625
  gateway: route.name
2439
2626
  };
2440
2627
  }).map((harness) => withDefaultEffort(harness, ctx.cursorEfforts));
2441
- const plannedAttempts = resolvedEval.skip ? [] : expand({
2628
+ const plannedAttempts = resolvedEval.skip ? [] : pick(expand({
2442
2629
  ...resolved,
2443
2630
  harness: routed
2444
- }, vertex);
2631
+ }, vertex));
2445
2632
  const { caseJson, ...declared } = declaration.source;
2446
2633
  const source = {
2447
2634
  ...declared,
@@ -2473,6 +2660,8 @@ function planTests(declaration, ctx, where, resolvedEval) {
2473
2660
  const why = unenforceable(planned.egress, planned.sandbox, where);
2474
2661
  if (why) throw new UsageError(why);
2475
2662
  const on = `${where} on ${harnessId(planned.harness)}`;
2663
+ const unverified = unverifiedHarness(planned.harness, planned.sandbox);
2664
+ if (unverified) throw new UsageError(`${on}: ${unverified}`);
2476
2665
  const mcp = refusedMcp(planned.harness, planned.environment.layers);
2477
2666
  if (mcp) throw new UsageError(`${on}: ${mcp}`);
2478
2667
  const plugins = planPlugins(planned, ctx.config.root);
@@ -2486,7 +2675,7 @@ function planTests(declaration, ctx, where, resolvedEval) {
2486
2675
  id: evalId(rankFile, [...groups, declaration.family.name]),
2487
2676
  ...declaration.family
2488
2677
  };
2489
- const harnessBy = resolved.sources?.harness ?? ctx.config.file;
2678
+ const harnessBy = resolved.sources?.harnesses ?? ctx.config.file;
2490
2679
  const plan = evalPlan(resolvedEval, plannedAttempts, {
2491
2680
  configFile: ctx.config.file,
2492
2681
  file: path
@@ -2668,7 +2857,8 @@ function planPlugins(planned, root) {
2668
2857
  function attemptLimits(planned, concurrency) {
2669
2858
  const limits = { attempts: concurrency };
2670
2859
  const { sandbox, harness } = planned;
2671
- if (sandbox.maxConcurrent) limits[`sandbox:${sandboxProvider(sandbox)}`] = sandbox.maxConcurrent;
2860
+ const cap = providerLimit(sandbox);
2861
+ if (cap) limits[`sandbox:${sandboxProvider(sandbox)}`] = cap;
2672
2862
  if (harness.maxConcurrent) limits[`harness:${harness.kind}`] = harness.maxConcurrent;
2673
2863
  return limits;
2674
2864
  }
@@ -2703,7 +2893,7 @@ function withPaths(options, where, base = callerDir() ?? process.cwd()) {
2703
2893
  }
2704
2894
  /** `(name, fn)`, `(name, options, fn)` as in Vitest, or `(name, fn, options)`. */
2705
2895
  function splitArgs(what, name, a, b) {
2706
- if (typeof a === "function" && typeof b === "function") throw new UsageError(`${what}("${name}") got two functions. Options must be a plain object; helpers such as secret are only available in defineConfig and defineEnvironment callbacks. Put the secret in an environment (environment: …) or in the config's env.`);
2896
+ if (typeof a === "function" && typeof b === "function") throw new UsageError(`${what}("${name}") got two functions. Options must be a plain object; helpers such as secret are only available in defineConfig and defineEnvironment callbacks. Put the secret in an environment (environments: …) or in the config's env.`);
2707
2897
  if (typeof a === "function") return [b ?? {}, a];
2708
2898
  if (typeof b === "function") return [a ?? {}, b];
2709
2899
  throw new UsageError(what === "group" ? `group("${name}") has no function. Pass the group's body, e.g. group("${name}", () => { … }).` : `${what}("${name}") has no function. Pass the eval's body, e.g. ${what}("${name}", async (c) => { … }).`);
@@ -2804,7 +2994,7 @@ const families = /* @__PURE__ */ new Map();
2804
2994
  /** Two name paths with one id differ only where a name holds the ` › ` the id joins them with. */
2805
2995
  const separatorHint = (path, earlier) => path.length === earlier.length ? "" : ` " › " joins the group and eval names in an eval's id, so a name containing it can be a grouped eval's id.`;
2806
2996
  /** A run's facts name an eval and a family alike by id, so an eval can't take a family's. */
2807
- const namedLikeFamily = (eval_, family) => new UsageError(`evaluate("${eval_.join(" › ")}") has the same id as the pooled evaluate.each("${family.join(" › ")}") in this file: a family's id is its name template. Give the eval or the template another name.${separatorHint(eval_, family)}`);
2997
+ const namedLikeFamily = (eval_, family) => new UsageError(`evaluate("${eval_.join(" › ")}") has the same id as evaluate("${family.join(" › ")}") with cases in this file. Give one of them another name.${separatorHint(eval_, family)}`);
2808
2998
  function checkUnique(where, groups, name, hint, inFamily) {
2809
2999
  const path = [...groups, name];
2810
3000
  const id = evalId(collectingFile() ?? "", path);
@@ -2835,11 +3025,14 @@ function checkFamily(groups, family) {
2835
3025
  });
2836
3026
  return;
2837
3027
  }
2838
- if (earlier.path.length !== path.length || earlier.cases !== family.cases || earlier.seen.has(family.case)) throw new UsageError(`evaluate.each("${path.join(" › ")}") is declared twice in this file. Each pooled evaluate.each needs its own name, or their cases are pooled into one family. Give each its own name template, or pass evaluate.each(cases, { pool: false }) to keep the cases apart.${separatorHint(path, earlier.path)}`);
3028
+ if (earlier.path.length !== path.length || earlier.cases !== family.cases || earlier.seen.has(family.case)) throw new UsageError(`evaluate("${path.join(" › ")}") with cases is declared twice in this file. Each eval needs its own name, or their cases are pooled into one. Give each its own name.${separatorHint(path, earlier.path)}`);
2839
3029
  earlier.seen.add(family.case);
2840
3030
  }
2841
3031
  function register(mode, given, own, fn, source, hint = "", givenFamily) {
2842
- const name = shownName("eval", given, hint && " For evaluate.each, name each case by %# or by a field that holds no secret.");
3032
+ prepare(mode, given, own, fn, source, hint, givenFamily).commit();
3033
+ }
3034
+ function prepare(mode, given, own, fn, source, hint = "", givenFamily, where) {
3035
+ const name = shownName("eval", given, givenFamily ? " Give each case a key that holds no secret." : "");
2843
3036
  const family = givenFamily && {
2844
3037
  ...givenFamily,
2845
3038
  name: redactSecrets(givenFamily.name)
@@ -2858,7 +3051,7 @@ function register(mode, given, own, fn, source, hint = "", givenFamily) {
2858
3051
  source,
2859
3052
  ...family ? { family } : {}
2860
3053
  });
2861
- const { tests, harnesses } = planEval({
3054
+ const planned = planEval({
2862
3055
  name,
2863
3056
  mode,
2864
3057
  groups: groupStack.map(({ name, options }) => ({
@@ -2867,86 +3060,130 @@ function register(mode, given, own, fn, source, hint = "", givenFamily) {
2867
3060
  })),
2868
3061
  options: own,
2869
3062
  source,
2870
- ...family ? { family } : {}
3063
+ ...family ? { family } : {},
3064
+ ...where ? { where } : {}
2871
3065
  }, {
2872
3066
  ...getRunContext(),
2873
3067
  file: collectingFile(),
2874
3068
  order: registered
2875
3069
  });
2876
- declareHarnesses(harnesses);
2877
- if (!tests) return;
2878
- const note = maskedNote([...groupStack.map((g) => g.given), given]);
2879
- if (note) for (const notes of new Set(tests.map((t) => t.attempt.plan.notes))) notes.push(note);
2880
- registered++;
2881
- const bestOfCells = /* @__PURE__ */ new Map();
2882
- for (const { name: testName, spec, ...options } of tests) {
2883
- const { cell, n, bestOf } = options.attempt;
2884
- let hooks = {};
2885
- if (bestOf !== void 0) {
2886
- const of = bestOfCells.get(cell.key) ?? bestOfCell(bestOf);
2887
- bestOfCells.set(cell.key, of);
2888
- hooks = of(n);
2889
- }
2890
- registerTest(testName, {
2891
- ...options,
2892
- ...hooks
2893
- }, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
3070
+ if (planned.tests) {
3071
+ const note = maskedNote([...groupStack.map((g) => g.given), given]);
3072
+ if (note) for (const notes of new Set(planned.tests.map((t) => t.attempt.plan.notes))) notes.push(note);
3073
+ registered++;
2894
3074
  }
3075
+ const commit = () => {
3076
+ declarePlanned(planned.harnesses, planned.traits);
3077
+ if (!planned.tests) return;
3078
+ const bestOfCells = /* @__PURE__ */ new Map();
3079
+ for (const { name: testName, spec, ...options } of planned.tests) {
3080
+ const { cell, n, bestOf } = options.attempt;
3081
+ let hooks = {};
3082
+ if (bestOf !== void 0) {
3083
+ const of = bestOfCells.get(cell.key) ?? bestOfCell(bestOf);
3084
+ bestOfCells.set(cell.key, of);
3085
+ hooks = of(n);
3086
+ }
3087
+ registerTest(testName, {
3088
+ ...options,
3089
+ ...hooks
3090
+ }, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
3091
+ }
3092
+ };
3093
+ return {
3094
+ planned,
3095
+ commit
3096
+ };
2895
3097
  }
2896
3098
  /** Evals registered so far in this worker, for the scheduler's tie-break. */
2897
3099
  let registered = 0;
2898
3100
  const defineEval = (mode, declared) => (name, a, b) => {
2899
3101
  checkOnly(mode, "evaluate");
2900
3102
  const shown = redactSecrets(name);
2901
- const [own, fn] = splitArgs("evaluate", shown, a, b);
3103
+ const [written, fn] = splitArgs("evaluate", shown, a, b);
2902
3104
  const where = `evaluate("${shown}")`;
2903
- register(mode, name, withPaths(own, where), fn, declared ?? { body: bodyHash(fn, where, collectingFile()) });
3105
+ if (isObjectLike(written) && "where" in written && !("cases" in written)) throw new UsageError(`${where} sets where without cases. where picks case-and-environment pairs; to drop an environment, leave it out of environments.`);
3106
+ if (isObjectLike(written) && ("cases" in written || "caseKey" in written)) return registerCases(mode, name, written, fn, where);
3107
+ register(mode, name, withPaths(written, where), fn, declared ?? { body: bodyHash(fn, where, collectingFile()) });
2904
3108
  };
2905
- /** `$key` is a property of the case, `%s` the case itself, `%#` its index. */
2906
- function formatName(template, item, index) {
2907
- return template.replace(/%#/g, String(index)).replace(/%s/g, typeof item === "string" ? item : JSON.stringify(item)).replace(/\$([A-Za-z_][A-Za-z0-9_]*)/g, (whole, key) => isObjectLike(item) && key in item ? String(item[key]) : whole);
2908
- }
2909
- function checkEachOptions(options) {
2910
- if (options === void 0) return {};
2911
- if (isObjectLike(options) && Object.keys(options).every((k) => k === "pool")) {
2912
- const { pool } = options;
2913
- if (pool === void 0) return {};
2914
- if (typeof pool === "boolean") return { pool };
2915
- }
2916
- throw new UsageError(`evaluate.each(cases, options) takes { pool?: boolean } (got ${JSON.stringify(options)}). Eval options such as compare go after the name: evaluate.each(cases)(name, { compare }, fn).`);
2917
- }
2918
3109
  /**
2919
- * One eval per case, pooled into one suite:
2920
- * `evaluate.each(cases)('adds $a and $b', async (c, { a, b }) => …)`.
3110
+ * An eval with `cases`: each case plans as an entry of its own named for the eval, which the
3111
+ * record folds back into the eval. Two or more cases pool into one pass rate and, with
3112
+ * `compare`, one lift paired by case.
2921
3113
  */
2922
- const evalEach = (cases, options) => (name, a, b) => {
2923
- const { pool = true } = checkEachOptions(options);
2924
- if (!Array.isArray(cases)) throw new UsageError(`evaluate.each takes an array of cases (got ${typeof cases}), e.g. evaluate.each([{ n: 1 }, { n: 2 }])("case $n", async (c, item) => { … }).`);
2925
- const template = redactSecrets(name);
2926
- const [written, fn] = splitArgs("evaluate.each", template, a, b);
2927
- const own = withPaths(written, `evaluate.each("${template}")`);
2928
- const hint = ` For evaluate.each, put $key, %s or %# in the name ("${template}") so each case's name differs.`;
2929
- const pooled = pool && cases.length >= 2;
3114
+ function registerCases(mode, name, written, fn, where) {
3115
+ const { cases, caseKey, where: pick, ...rest } = written;
3116
+ if (!Array.isArray(cases)) throw new UsageError(cases === void 0 ? `caseKey in ${where} names each case, but there are no cases. Add cases: [...], or drop caseKey.` : `cases in ${where} must be a list (got ${typeof cases}), e.g. cases: ['users', 'posts'].`);
3117
+ if (cases.length === 0) throw new UsageError(`cases in ${where} is an empty list. Give at least one case.`);
3118
+ if (caseKey !== void 0 && typeof caseKey !== "function") throw new UsageError(`caseKey in ${where} must be a function from a case to its key, e.g. caseKey: (item) => item.slug.`);
3119
+ if (pick !== void 0 && typeof pick !== "function") throw new UsageError(`where in ${where} must be a function from a case and an environment to true or false, e.g. where: ({ case: item, environment }) => item.stacks.includes(environment.traits.stack).`);
3120
+ if (isObjectLike(rest.runs) && "bestOf" in rest.runs) throw new UsageError(`${where} has cases, so it can't take runs: { bestOf }: best-of stops each cell on its own, and the cases pool over the runs. Set a fixed count, e.g. runs: 3.`);
3121
+ const keys = caseKeys(cases, caseKey, where);
3122
+ const shownKeys = keys.map((k) => redactSecrets(k));
3123
+ const own = withPaths(rest, where);
2930
3124
  const file = collectingFile();
2931
- const body = bodyHash(fn, `evaluate.each("${template}")`, file);
2932
- cases.forEach((item, i) => {
2933
- const family = pooled ? {
3125
+ const body = bodyHash(fn, where, file);
3126
+ const prepared = cases.map((item, i) => {
3127
+ const key = shownKeys[i];
3128
+ const family = {
2934
3129
  name,
2935
3130
  case: i,
2936
3131
  cases: cases.length
2937
- } : void 0;
3132
+ };
2938
3133
  const source = {
2939
3134
  body,
2940
- caseJson: caseJson(item, `evaluate.each("${template}") case ${i}`, file)
3135
+ caseJson: caseJson(item, `${where} case "${key}"`, file)
2941
3136
  };
2942
- register("run", formatName(name, item, i), own, (c) => fn(c, item), source, hint, family);
3137
+ const run = (c) => fn(c, item);
3138
+ const picks = pick ? {
3139
+ label: where,
3140
+ key,
3141
+ item,
3142
+ where: pick
3143
+ } : void 0;
3144
+ return prepare(mode, caseEvalName(name, keys[i]), own, run, source, "", family, picks);
2943
3145
  });
2944
- };
3146
+ if (pick) applyWhere(where, shownKeys, prepared);
3147
+ for (const p of prepared) p.commit();
3148
+ }
3149
+ /**
3150
+ * The checks `where` needs across an eval's cases, then the pairs it left out: each case's own on
3151
+ * its plan, and every case's in the eval's notes. A cell that no case runs on is refused, and so
3152
+ * is an eval it leaves nothing to run.
3153
+ */
3154
+ function applyWhere(where, keys, prepared) {
3155
+ const picks = prepared.flatMap((p) => p.planned.picked ? [p.planned.picked] : []);
3156
+ if (!picks.length) return;
3157
+ const excluded = picks.flatMap((p) => p.excluded);
3158
+ const kept = new Set(picks.flatMap((p) => p.keptCells));
3159
+ if (!kept.size) throw new UsageError(`${where}'s where excludes all ${excluded.length} case-and-environment pairs, so it has nothing to run.`);
3160
+ const empty = picks.flatMap((p) => p.cells).find((c) => !kept.has(c.key));
3161
+ if (empty) throw new UsageError(`${where}'s where leaves cell ${empty.key} with no cases. Leave ${empty.environment} out of environments, or widen where.`);
3162
+ if (!excluded.length) return;
3163
+ const byEnv = /* @__PURE__ */ new Map();
3164
+ for (const e of excluded) {
3165
+ const on = e.harness ? `${e.environment} for ${e.harness}` : e.environment;
3166
+ const list = byEnv.get(on) ?? [];
3167
+ if (!list.includes(e.case)) list.push(e.case);
3168
+ byEnv.set(on, list);
3169
+ }
3170
+ const order = (k) => keys.indexOf(k);
3171
+ const notes = [`where excludes ${excluded.length} ${excluded.length === 1 ? "pair" : "pairs"}`, ...[...byEnv].map(([on, list]) => `excluded on ${on}: ${list.sort((a, b) => order(a) - order(b)).join(", ")}`)];
3172
+ const plans = /* @__PURE__ */ new Set();
3173
+ for (const p of prepared) {
3174
+ const own = p.planned.picked?.excluded ?? [];
3175
+ for (const t of p.planned.tests ?? []) {
3176
+ plans.add(t.attempt.plan);
3177
+ if (own.length) t.attempt.plan.excluded = own;
3178
+ if (t.attempt.family) t.attempt.plan.familyExcluded = excluded;
3179
+ }
3180
+ }
3181
+ for (const list of new Set([...plans].map((p) => p.notes))) list.push(...notes);
3182
+ }
2945
3183
  const evaluate = Object.assign(defineEval("run"), {
2946
3184
  skip: defineEval("skip"),
2947
3185
  only: defineEval("only"),
2948
- todo: registerTodo,
2949
- each: evalEach
3186
+ todo: registerTodo
2950
3187
  });
2951
3188
  /**
2952
3189
  * Declares a tree's groups and evals as the calls that made it would, in the file being
@@ -2978,32 +3215,30 @@ function declareNodes(nodes, bodyFor, base) {
2978
3215
  */
2979
3216
  function optionsFrom(written, where, base) {
2980
3217
  refuseIllegalCompare(written, where);
2981
- const { environment, compare, ...rest } = withPaths(written, where, base);
3218
+ const { environments, compare, ...rest } = withPaths(written, where, base);
2982
3219
  const missing = missingIn(where);
2983
3220
  const asEnv = (env, field) => environmentFrom(environmentPaths(env, base, field, missing));
2984
- const own = environment !== void 0 ? { environment: asEnv(environment, "environment") } : {};
3221
+ const asEnvs = (value, field) => {
3222
+ if (Array.isArray(value)) return value.map((e, i) => asEnv(e, `${field}[${i}]`));
3223
+ if ("only" in value) return value;
3224
+ if ("add" in value) return { add: [value.add].flat().map((e, i) => asEnv(e, `${field}.add[${i}]`)) };
3225
+ return asEnv(value, field);
3226
+ };
3227
+ const own = environments !== void 0 ? { environments: asEnvs(environments, "environments") } : {};
2985
3228
  if (compare === void 0) return {
2986
3229
  ...rest,
2987
3230
  ...own
2988
3231
  };
2989
3232
  const variants = compare.variants.map((v, i) => asEnv(v, `compare.variants[${i}]`));
2990
- if (compare.baseline !== void 0) return {
2991
- ...rest,
2992
- compare: {
2993
- ...compare,
2994
- baseline: asEnv(compare.baseline, "compare.baseline"),
2995
- variants
2996
- }
2997
- };
2998
3233
  return {
2999
3234
  ...rest,
3000
- ...own,
3001
3235
  compare: {
3002
3236
  ...compare,
3237
+ control: asEnv(compare.control, "compare.control"),
3003
3238
  variants
3004
3239
  }
3005
3240
  };
3006
3241
  }
3007
3242
  export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
3008
3243
 
3009
- //# sourceMappingURL=evaluate-NTNJ8T9H.js.map
3244
+ //# sourceMappingURL=evaluate-B12Tosvq.js.map