reeval 0.17.2 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (403) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +5 -5
  3. package/dist/{adapter-DyvDj4ZN.js → adapter-C8xPuXYq.js} +5 -4
  4. package/dist/adapter-C8xPuXYq.js.map +1 -0
  5. package/dist/{agent-v036GEgG.js → agent-CBoLndM7.js} +30 -10
  6. package/dist/{agent-v036GEgG.js.map → agent-CBoLndM7.js.map} +1 -1
  7. package/dist/agents-C6zhkxp3.js +1151 -0
  8. package/dist/agents-C6zhkxp3.js.map +1 -0
  9. package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
  10. package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
  11. package/dist/{app-files-hpRT5lj_.js → app-files-DT5mJyV4.js} +4 -4
  12. package/dist/app-files-DT5mJyV4.js.map +1 -0
  13. package/dist/assemble-UPCRrtOz.js +1 -0
  14. package/dist/{assemble-Dlh6Xv3U.js → assemble-t6h018An.js} +194 -307
  15. package/dist/assemble-t6h018An.js.map +1 -0
  16. package/dist/{attempt-CtKGvaQR.js → attempt-CAdlwlNu.js} +3 -3
  17. package/dist/{attempt-CtKGvaQR.js.map → attempt-CAdlwlNu.js.map} +1 -1
  18. package/dist/{attempt-host-DErH-DYV.js → attempt-host-CIm1-11m.js} +18 -10
  19. package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-CIm1-11m.js.map} +1 -1
  20. package/dist/boat-BjLqFJIZ.js +108 -0
  21. package/dist/boat-BjLqFJIZ.js.map +1 -0
  22. package/dist/boat-CJCtzfB1.js +322 -0
  23. package/dist/boat-CJCtzfB1.js.map +1 -0
  24. package/dist/bridge/index.mjs +300 -78
  25. package/dist/capture-DoxFUN3J.js +1 -0
  26. package/dist/{capture-Bdcx9Gap.js → capture-iBzFiC0j.js} +406 -625
  27. package/dist/capture-iBzFiC0j.js.map +1 -0
  28. package/dist/cases-Dy0mifal.js +1 -0
  29. package/dist/cases-z9E6DCd5.js +639 -0
  30. package/dist/cases-z9E6DCd5.js.map +1 -0
  31. package/dist/{cli-CGd51thd.js → cli-20Fpvs5H.js} +929 -576
  32. package/dist/cli-20Fpvs5H.js.map +1 -0
  33. package/dist/{cli-spec-ChkleEVm.js → cli-spec-Dsr79Qy1.js} +46 -18
  34. package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
  35. package/dist/cli.js +2 -2
  36. package/dist/{client-CF-DOjOU.js → client-KM4C5r5_.js} +111 -14
  37. package/dist/client-KM4C5r5_.js.map +1 -0
  38. package/dist/{command-CsfcwMyc.js → command-B5OyHpOu.js} +24 -81
  39. package/dist/command-B5OyHpOu.js.map +1 -0
  40. package/dist/{command-CtTdCF3B.js → command-BLNVDiIg.js} +10 -10
  41. package/dist/{command-CtTdCF3B.js.map → command-BLNVDiIg.js.map} +1 -1
  42. package/dist/{command-CyIb9BSC.js → command-CRh-cb6Z.js} +120 -55
  43. package/dist/command-CRh-cb6Z.js.map +1 -0
  44. package/dist/{command-TLmY-rYp.js → command-aUJvENwy.js} +18 -18
  45. package/dist/command-aUJvENwy.js.map +1 -0
  46. package/dist/{commands-C54S45d7.js → commands-CWy-WndG.js} +239 -92
  47. package/dist/commands-CWy-WndG.js.map +1 -0
  48. package/dist/{commands-jO9Jj35r.js → commands-CfdBX-mc.js} +1 -1
  49. package/dist/{config-ImY6AlRE.js → config-B9mB1kF5.js} +924 -1729
  50. package/dist/config-B9mB1kF5.js.map +1 -0
  51. package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
  52. package/dist/context-Bkv3ndjq.js.map +1 -0
  53. package/dist/{cost-e4IppZpJ.js → cost-BdRgySQl.js} +4 -4
  54. package/dist/{cost-e4IppZpJ.js.map → cost-BdRgySQl.js.map} +1 -1
  55. package/dist/cost-Cr26Alhq.js +1 -0
  56. package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BSn9xjmo.js} +3 -3
  57. package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
  58. package/dist/db-P2fXw4nx.js +2 -0
  59. package/dist/{db-sHGpaLo9.js → db-zgC1tvKB.js} +178 -74
  60. package/dist/db-zgC1tvKB.js.map +1 -0
  61. package/dist/debug-CF4p4Lp_.js +1 -0
  62. package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
  63. package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
  64. package/dist/{default-effort-fpdbYVKm.js → default-effort-BAy8ixd1.js} +1 -1
  65. package/dist/{default-effort-BZRa-SQP.js → default-effort-DGhQmu95.js} +8 -5
  66. package/dist/default-effort-DGhQmu95.js.map +1 -0
  67. package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
  68. package/dist/dist-CAFD1yfv.js.map +1 -0
  69. package/dist/{docker-CWuSRj7a.js → docker-Cqw-CyWz.js} +4 -4
  70. package/dist/{docker-CWuSRj7a.js.map → docker-Cqw-CyWz.js.map} +1 -1
  71. package/dist/{docker--2ytOHcu.js → docker-DAMnm3L9.js} +103 -73
  72. package/dist/docker-DAMnm3L9.js.map +1 -0
  73. package/dist/{docker-CIzHzwlC.js → docker-Dr4XeDc8.js} +1 -1
  74. package/dist/{doctor-eE65YpEv.js → doctor-B2PohneB.js} +86 -29
  75. package/dist/doctor-B2PohneB.js.map +1 -0
  76. package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
  77. package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
  78. package/dist/{environment-Cy3FVLR5.js → environment-DB8o3hGi.js} +80 -40
  79. package/dist/environment-DB8o3hGi.js.map +1 -0
  80. package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
  81. package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
  82. package/dist/{eval-source-BTsTRiBW.js → eval-source-D3sbAzt-.js} +23 -15
  83. package/dist/eval-source-D3sbAzt-.js.map +1 -0
  84. package/dist/evaluate-COdzf7Hl.d.ts +92 -0
  85. package/dist/{evaluate-CFFdyYZX.js → evaluate-CwKeN_fo.js} +472 -159
  86. package/dist/evaluate-CwKeN_fo.js.map +1 -0
  87. package/dist/files-B4KQUiM7.js +1 -0
  88. package/dist/files-BEwfjD8J.js +64 -0
  89. package/dist/files-BEwfjD8J.js.map +1 -0
  90. package/dist/{files-CQU08uh4.js → files-BRzV7aPh.js} +5 -5
  91. package/dist/{files-CQU08uh4.js.map → files-BRzV7aPh.js.map} +1 -1
  92. package/dist/{files-CIX-pLl4.js → files-BiFU16BW.js} +6 -6
  93. package/dist/{files-CIX-pLl4.js.map → files-BiFU16BW.js.map} +1 -1
  94. package/dist/files-C1Rk9FYA.js +1 -0
  95. package/dist/files-C3mCC8Mn.js +1 -0
  96. package/dist/{files-tnMgu7-X.js → files-Cl7BQ5gE.js} +55 -14
  97. package/dist/{files-tnMgu7-X.js.map → files-Cl7BQ5gE.js.map} +1 -1
  98. package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
  99. package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
  100. package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
  101. package/dist/files-DlQshsks.js.map +1 -0
  102. package/dist/files-DnlA1A7N.js +1 -0
  103. package/dist/gate-CgYdaprG.js +49 -0
  104. package/dist/gate-CgYdaprG.js.map +1 -0
  105. package/dist/gateway.d.ts +1 -1
  106. package/dist/{git-BWVkG4Yl.js → git-fDiJ8UIV.js} +3 -3
  107. package/dist/{git-BWVkG4Yl.js.map → git-fDiJ8UIV.js.map} +1 -1
  108. package/dist/harness-C2berMiM.js +105 -0
  109. package/dist/harness-C2berMiM.js.map +1 -0
  110. package/dist/{harness-Ch2P81sx.js → harness-DXv2gnaT.js} +68 -111
  111. package/dist/harness-DXv2gnaT.js.map +1 -0
  112. package/dist/harness.d.ts +1 -1
  113. package/dist/harness.js +1 -1
  114. package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
  115. package/dist/ids-C_advdJA.js.map +1 -0
  116. package/dist/{images-BVZHDf_z.js → images-BNgcR77K.js} +98 -10
  117. package/dist/images-BNgcR77K.js.map +1 -0
  118. package/dist/index.d.ts +23 -3
  119. package/dist/index.js +5 -5
  120. package/dist/init-4RPeD-K-.js +1 -0
  121. package/dist/{init-BWnMbUnc.js → init-DbuZVRrW.js} +8 -7
  122. package/dist/init-DbuZVRrW.js.map +1 -0
  123. package/dist/ink-list-BpfidQ1h.js +274 -0
  124. package/dist/ink-list-BpfidQ1h.js.map +1 -0
  125. package/dist/{invocations-BGRs1r_C.js → invocations-CxGNOlh0.js} +4 -4
  126. package/dist/{invocations-BGRs1r_C.js.map → invocations-CxGNOlh0.js.map} +1 -1
  127. package/dist/judge-COMXT1Hx.js +230 -0
  128. package/dist/judge-COMXT1Hx.js.map +1 -0
  129. package/dist/judge.d.ts +1 -1
  130. package/dist/judge.js +1 -1
  131. package/dist/judgments-CzgpsTqv.js +246 -0
  132. package/dist/judgments-CzgpsTqv.js.map +1 -0
  133. package/dist/kinds-C1oY3qyC.js +697 -0
  134. package/dist/kinds-C1oY3qyC.js.map +1 -0
  135. package/dist/{summary-zP3cCXDN.js → layout-Cm85tQYk.js} +77 -154
  136. package/dist/layout-Cm85tQYk.js.map +1 -0
  137. package/dist/list-DuThsE28.js +1348 -0
  138. package/dist/list-DuThsE28.js.map +1 -0
  139. package/dist/live-CJCK11nh.js +1 -0
  140. package/dist/{live-d_5a5iUv.js → live-CgYbO4Te.js} +24 -4
  141. package/dist/live-CgYbO4Te.js.map +1 -0
  142. package/dist/{live-ColieOai.js → live-CnGrRM6j.js} +8 -8
  143. package/dist/{live-ColieOai.js.map → live-CnGrRM6j.js.map} +1 -1
  144. package/dist/{matchers-B4ZtlJyP.js → matchers-CXLn-1jm.js} +4 -3
  145. package/dist/{matchers-B4ZtlJyP.js.map → matchers-CXLn-1jm.js.map} +1 -1
  146. package/dist/{model-DOXnWH65.js → model-047Dcd-a.js} +1 -1
  147. package/dist/model-Cv4GuTsZ.js +1 -0
  148. package/dist/{model-CFSjV-HJ.js → model-CwwkrCb7.js} +5 -5
  149. package/dist/{model-CFSjV-HJ.js.map → model-CwwkrCb7.js.map} +1 -1
  150. package/dist/{model-DBpdRXtO.js → model-DJjLw-0r.js} +43 -19
  151. package/dist/{model-DBpdRXtO.js.map → model-DJjLw-0r.js.map} +1 -1
  152. package/dist/{model-DOiCiMCD.js → model-V3mMXp6x.js} +213 -559
  153. package/dist/model-V3mMXp6x.js.map +1 -0
  154. package/dist/{node-CwP1G178.js → node-D7pAU--0.js} +34 -25
  155. package/dist/node-D7pAU--0.js.map +1 -0
  156. package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-BGiXt8_B.js} +107 -49
  157. package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
  158. package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-Cw1z8-sq.js} +2 -2
  159. package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
  160. package/dist/options-CHubslQf.js +1 -0
  161. package/dist/{gate-BoDKeuJu.js → pairs-BGb134Vu.js} +3 -49
  162. package/dist/pairs-BGb134Vu.js.map +1 -0
  163. package/dist/{patch-DvLr8wCZ.js → patch-DH4e_Gkd.js} +3 -3
  164. package/dist/{patch-DvLr8wCZ.js.map → patch-DH4e_Gkd.js.map} +1 -1
  165. package/dist/plain-BYUf_fUV.js +267 -0
  166. package/dist/plain-BYUf_fUV.js.map +1 -0
  167. package/dist/{plan-CIfOFhth.js → plan-Cgq9NCEX.js} +117 -45
  168. package/dist/plan-Cgq9NCEX.js.map +1 -0
  169. package/dist/{plugins-BEs2Am3L.js → plugins-BkmPxDiY.js} +11 -268
  170. package/dist/plugins-BkmPxDiY.js.map +1 -0
  171. package/dist/preflight-4gWSy0Pe.js +1 -0
  172. package/dist/{preflight-50Qw5e5K.js → preflight-BElE_q0u.js} +16 -9
  173. package/dist/preflight-BElE_q0u.js.map +1 -0
  174. package/dist/preflight-CtlloeOQ.js +68 -0
  175. package/dist/preflight-CtlloeOQ.js.map +1 -0
  176. package/dist/{preflight-BAIRsTpl.js → preflight-R-InZCQb.js} +1 -1
  177. package/dist/{prepare-8XhzhLHX.js → prepare-C7f5mPl5.js} +9 -9
  178. package/dist/{prepare-8XhzhLHX.js.map → prepare-C7f5mPl5.js.map} +1 -1
  179. package/dist/{prepare-8nvfDFQx.js → prepare-CzkAdbXm.js} +2 -2
  180. package/dist/{prices-C4BRuSe8.js → prices-5m3SzMZ3.js} +3 -3
  181. package/dist/{prices-C4BRuSe8.js.map → prices-5m3SzMZ3.js.map} +1 -1
  182. package/dist/{provenance-Cc8fFE7Z.js → provenance-zsc4--dP.js} +6 -5
  183. package/dist/provenance-zsc4--dP.js.map +1 -0
  184. package/dist/provider-BnUoz4KB.js +1 -0
  185. package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
  186. package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
  187. package/dist/{providers-DZSTlC9R.js → providers-B8xYMeah.js} +152 -9
  188. package/dist/providers-B8xYMeah.js.map +1 -0
  189. package/dist/providers-CVu8e0xS.js +1 -0
  190. package/dist/{query-DXpxaaTH.js → query-Bb3SYICk.js} +3 -3
  191. package/dist/{query-DXpxaaTH.js.map → query-Bb3SYICk.js.map} +1 -1
  192. package/dist/query-DIfkUcPh.js +1 -0
  193. package/dist/{questions-C_Y-oXW6.js → questions-CWlR9ffu.js} +41 -11
  194. package/dist/questions-CWlR9ffu.js.map +1 -0
  195. package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
  196. package/dist/record-6IUzwP9K.js.map +1 -0
  197. package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
  198. package/dist/redact-BhjtnjrU.js.map +1 -0
  199. package/dist/remarks-CAvTSq16.js +1 -0
  200. package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
  201. package/dist/remarks-D9Lcd1fF.js.map +1 -0
  202. package/dist/remarks-DPCONFxZ.js +138 -0
  203. package/dist/remarks-DPCONFxZ.js.map +1 -0
  204. package/dist/{remarks-E4rpi8Ym.js → remarks-vtkPmQb9.js} +327 -41
  205. package/dist/remarks-vtkPmQb9.js.map +1 -0
  206. package/dist/{remote-DZ00A8V_.js → remote-BgYWqogI.js} +6 -6
  207. package/dist/{remote-DZ00A8V_.js.map → remote-BgYWqogI.js.map} +1 -1
  208. package/dist/{resolve-B_JlQ0La.js → resolve-Do0ZaYH6.js} +11 -10
  209. package/dist/resolve-Do0ZaYH6.js.map +1 -0
  210. package/dist/resolve-amw_2Odz.js +1 -0
  211. package/dist/{results-BMJHaQbi.js → results-JyuM64gy.js} +7 -7
  212. package/dist/{results-BMJHaQbi.js.map → results-JyuM64gy.js.map} +1 -1
  213. package/dist/resume-AW5oXzUc.js +691 -0
  214. package/dist/resume-AW5oXzUc.js.map +1 -0
  215. package/dist/retry-ChzISxmy.js +105 -0
  216. package/dist/retry-ChzISxmy.js.map +1 -0
  217. package/dist/{runtime-CSZ1hjlN.js → runtime-BpW1f1Mx.js} +26 -7
  218. package/dist/runtime-BpW1f1Mx.js.map +1 -0
  219. package/dist/sandbox/boat.d.ts +26 -0
  220. package/dist/sandbox/boat.js +37 -0
  221. package/dist/sandbox/boat.js.map +1 -0
  222. package/dist/{sandbox-B5W8lLL1.js → sandbox-kSgr7vo5.js} +31 -3
  223. package/dist/sandbox-kSgr7vo5.js.map +1 -0
  224. package/dist/sandbox.d.ts +20 -2
  225. package/dist/sandbox.js +1 -1
  226. package/dist/server-2_gUnVvf.js +1 -0
  227. package/dist/{server-BfN6q2lH.js → server-Cn8VqO9S.js} +10 -10
  228. package/dist/{server-BfN6q2lH.js.map → server-Cn8VqO9S.js.map} +1 -1
  229. package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
  230. package/dist/shared-Cfi45Bvy.js.map +1 -0
  231. package/dist/slug-IHh3f770.js +136 -0
  232. package/dist/slug-IHh3f770.js.map +1 -0
  233. package/dist/source-CY3etKzw.js +1 -0
  234. package/dist/{source-BD6Bg0Tr.js → source-CYOhR98B.js} +2 -2
  235. package/dist/{source-BD6Bg0Tr.js.map → source-CYOhR98B.js.map} +1 -1
  236. package/dist/src-6Zzyg4Qd.js +2 -0
  237. package/dist/{src-BC2WXULN.js → src-BDhcPtnI.js} +2 -2
  238. package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
  239. package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
  240. package/dist/src-BRMsk9nh.js +2 -0
  241. package/dist/{src-Dp-fWOfG.js → src-CV_7DkPa.js} +47 -45
  242. package/dist/{src-Dp-fWOfG.js.map → src-CV_7DkPa.js.map} +1 -1
  243. package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
  244. package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
  245. package/dist/{src-Cpy9MNS3.js → src-Ck2r_EMT.js} +617 -74
  246. package/dist/src-Ck2r_EMT.js.map +1 -0
  247. package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
  248. package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
  249. package/dist/src-DB7N8TYJ.js +2 -0
  250. package/dist/src-DPDaAYwh.js +2 -0
  251. package/dist/{src-Cx0HJIJ-.js → src-Dbk19BiP.js} +2 -2
  252. package/dist/src-DtodMT59.js +1 -0
  253. package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
  254. package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
  255. package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
  256. package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
  257. package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
  258. package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
  259. package/dist/src-T-OeuJLr.js +2 -0
  260. package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
  261. package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
  262. package/dist/src-zntEDlnx.js +3 -0
  263. package/dist/{state-CdSjXIfE.js → state-BOLt8pJg.js} +17 -17
  264. package/dist/{state-CdSjXIfE.js.map → state-BOLt8pJg.js.map} +1 -1
  265. package/dist/state-kimtIhBP.js +1 -0
  266. package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
  267. package/dist/stats-C18W4xPX.js.map +1 -0
  268. package/dist/suite-CKBa6MKt.js +2 -0
  269. package/dist/suite-COglUhPd.js +1272 -0
  270. package/dist/suite-COglUhPd.js.map +1 -0
  271. package/dist/summary-DU5zaibx.js +148 -0
  272. package/dist/summary-DU5zaibx.js.map +1 -0
  273. package/dist/tart-BTVXp8rz.js +373 -0
  274. package/dist/tart-BTVXp8rz.js.map +1 -0
  275. package/dist/tart-cli-BIC9HqPg.js +37 -0
  276. package/dist/tart-cli-BIC9HqPg.js.map +1 -0
  277. package/dist/tart-cli-BMrkmBjI.js +1 -0
  278. package/dist/tart-images-DZ_Mbh-g.js +233 -0
  279. package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
  280. package/dist/tart-ref-7Qv3cm3B.js +1 -0
  281. package/dist/tart-ref-DYZijjrU.js +55 -0
  282. package/dist/tart-ref-DYZijjrU.js.map +1 -0
  283. package/dist/testing.d.ts +47 -37
  284. package/dist/testing.js +4 -4
  285. package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
  286. package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
  287. package/dist/trace-Dx2NBE1O.js +311 -0
  288. package/dist/trace-Dx2NBE1O.js.map +1 -0
  289. package/dist/traits-DKnYkqeK.js +66 -0
  290. package/dist/traits-DKnYkqeK.js.map +1 -0
  291. package/dist/{transcript-C_IpYMRk.js → transcript-CCf07UmU.js} +3 -3
  292. package/dist/{transcript-C_IpYMRk.js.map → transcript-CCf07UmU.js.map} +1 -1
  293. package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
  294. package/dist/types-C2mLpph3.js.map +1 -0
  295. package/dist/{types-BHR1lGPh.d.ts → types-DBwFsGem.d.ts} +420 -136
  296. package/dist/{judge-ByEUlzOM.js → vercel-CGlSBv63.js} +3 -229
  297. package/dist/vercel-CGlSBv63.js.map +1 -0
  298. package/dist/{vertex-6uSpZ5L8.js → vertex-Eb2wqahk.js} +5 -2
  299. package/dist/vertex-Eb2wqahk.js.map +1 -0
  300. package/dist/vertex-O88TCwgY.js +1 -0
  301. package/dist/view/index.d.ts +238 -90
  302. package/dist/view/index.js +11 -9
  303. package/dist/{views-CKoN5NAx.js → views-DAIiLiV3.js} +8 -7
  304. package/dist/views-DAIiLiV3.js.map +1 -0
  305. package/dist/views-DtRhj2Tn.js +2 -0
  306. package/dist/{worker-entry-D5dB8cgT.js → worker-entry-BIROTPX1.js} +11 -8
  307. package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-BIROTPX1.js.map} +1 -1
  308. package/dist/workspace-CcQRT31C.js +1 -0
  309. package/dist/{workspace-Cho30qlM.js → workspace-uuE51x35.js} +7 -7
  310. package/dist/{workspace-Cho30qlM.js.map → workspace-uuE51x35.js.map} +1 -1
  311. package/package.json +26 -11
  312. package/skills/eval-rubric/SKILL.md +31 -4
  313. package/dist/adapter-DyvDj4ZN.js.map +0 -1
  314. package/dist/app-files-hpRT5lj_.js.map +0 -1
  315. package/dist/assemble-DTXwCL3m.js +0 -1
  316. package/dist/assemble-Dlh6Xv3U.js.map +0 -1
  317. package/dist/capture-BT2fAmpZ.js +0 -1
  318. package/dist/capture-Bdcx9Gap.js.map +0 -1
  319. package/dist/cli-CGd51thd.js.map +0 -1
  320. package/dist/cli-spec-ChkleEVm.js.map +0 -1
  321. package/dist/client-CF-DOjOU.js.map +0 -1
  322. package/dist/command-CsfcwMyc.js.map +0 -1
  323. package/dist/command-CyIb9BSC.js.map +0 -1
  324. package/dist/command-TLmY-rYp.js.map +0 -1
  325. package/dist/commands-C54S45d7.js.map +0 -1
  326. package/dist/config-ImY6AlRE.js.map +0 -1
  327. package/dist/context-BajfiH0S.js.map +0 -1
  328. package/dist/contrasts-DtSX8E_V.js +0 -101
  329. package/dist/contrasts-DtSX8E_V.js.map +0 -1
  330. package/dist/cost-CMzY9VXv.js +0 -1
  331. package/dist/db-ZIk6S2CD.js +0 -2
  332. package/dist/db-sHGpaLo9.js.map +0 -1
  333. package/dist/debug-BnXV7JbX.js +0 -1
  334. package/dist/default-effort-BZRa-SQP.js.map +0 -1
  335. package/dist/dist-ByYk_wAs.js.map +0 -1
  336. package/dist/docker--2ytOHcu.js.map +0 -1
  337. package/dist/doctor-eE65YpEv.js.map +0 -1
  338. package/dist/environment-Cy3FVLR5.js.map +0 -1
  339. package/dist/eval-source-BTsTRiBW.js.map +0 -1
  340. package/dist/evaluate-CFFdyYZX.js.map +0 -1
  341. package/dist/evaluate-DnGOaGQc.d.ts +0 -43
  342. package/dist/files-B3d-8Fe0.js +0 -1
  343. package/dist/files-COW5JBJM.js +0 -1
  344. package/dist/files-Do4Rp3yZ.js +0 -1
  345. package/dist/files-p-uTdOjA.js.map +0 -1
  346. package/dist/gate-BoDKeuJu.js.map +0 -1
  347. package/dist/harness-Ch2P81sx.js.map +0 -1
  348. package/dist/ids-DuK6xl4w.js.map +0 -1
  349. package/dist/images-BVZHDf_z.js.map +0 -1
  350. package/dist/init-BWnMbUnc.js.map +0 -1
  351. package/dist/init-s1jN-PCI.js +0 -1
  352. package/dist/judge-ByEUlzOM.js.map +0 -1
  353. package/dist/live-DmJix4c8.js +0 -1
  354. package/dist/live-d_5a5iUv.js.map +0 -1
  355. package/dist/model-8uZBYqV5.js +0 -1
  356. package/dist/model-DOiCiMCD.js.map +0 -1
  357. package/dist/node-CwP1G178.js.map +0 -1
  358. package/dist/opencode-providers-yNKyXD49.js.map +0 -1
  359. package/dist/plan-CIfOFhth.js.map +0 -1
  360. package/dist/plugins-BEs2Am3L.js.map +0 -1
  361. package/dist/preflight-50Qw5e5K.js.map +0 -1
  362. package/dist/provenance-Cc8fFE7Z.js.map +0 -1
  363. package/dist/provider-DCdTHQpk.js +0 -1
  364. package/dist/providers-CP_bBtYm.js +0 -1
  365. package/dist/providers-DZSTlC9R.js.map +0 -1
  366. package/dist/query-kFPuqBv1.js +0 -1
  367. package/dist/questions-C_Y-oXW6.js.map +0 -1
  368. package/dist/record-DhzwkSXc.js.map +0 -1
  369. package/dist/redact-BEEGc37e.js.map +0 -1
  370. package/dist/remarks-0EXMh6AD.js.map +0 -1
  371. package/dist/remarks-BL18Id09.js +0 -1
  372. package/dist/remarks-DeZZT8PK.js +0 -116
  373. package/dist/remarks-DeZZT8PK.js.map +0 -1
  374. package/dist/remarks-E4rpi8Ym.js.map +0 -1
  375. package/dist/resolve-BJSay360.js +0 -1
  376. package/dist/resolve-B_JlQ0La.js.map +0 -1
  377. package/dist/resume-CexsWuGK.js +0 -434
  378. package/dist/resume-CexsWuGK.js.map +0 -1
  379. package/dist/runtime-CSZ1hjlN.js.map +0 -1
  380. package/dist/sandbox-B5W8lLL1.js.map +0 -1
  381. package/dist/server-SBzrf35T.js +0 -1
  382. package/dist/shared-ogqPJ0uo.js.map +0 -1
  383. package/dist/source-Dn1uqPon.js +0 -1
  384. package/dist/src-BKDZO3ra.js +0 -2
  385. package/dist/src-BxrHipV0.js +0 -3
  386. package/dist/src-C9elednc.js +0 -2
  387. package/dist/src-CM7xdJFA.js +0 -2
  388. package/dist/src-CWEGknZo.js +0 -2
  389. package/dist/src-Cpy9MNS3.js.map +0 -1
  390. package/dist/src-DVaAKCsH.js +0 -1
  391. package/dist/src-DurP--_D.js +0 -2
  392. package/dist/state-Cicc9Oa5.js +0 -1
  393. package/dist/stats-D-Gqwqt0.js.map +0 -1
  394. package/dist/suite-BYJ2Q0S-.js +0 -2
  395. package/dist/suite-BtHPBFYL.js +0 -544
  396. package/dist/suite-BtHPBFYL.js.map +0 -1
  397. package/dist/summary-zP3cCXDN.js.map +0 -1
  398. package/dist/types-C4OkYhZg.js.map +0 -1
  399. package/dist/vertex-6uSpZ5L8.js.map +0 -1
  400. package/dist/vertex-BaAz44YN.js +0 -1
  401. package/dist/views-CKoN5NAx.js.map +0 -1
  402. package/dist/views-CmyX255j.js +0 -2
  403. package/dist/workspace-Dw9DEZox.js +0 -1
@@ -1,38 +1,42 @@
1
- import { t as isObjectLike } from "./object-CeQ8dgUC.js";
2
- import { r as withLease } from "./client-CF-DOjOU.js";
3
1
  import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-ttMTnIYX.js";
4
- import { h as parseDuration, n as dataProblem, p as MAX_TIMER_MS } from "./shared-ogqPJ0uo.js";
5
- import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-Cy3FVLR5.js";
6
- import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-BEs2Am3L.js";
7
- import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-DuK6xl4w.js";
8
- import { $ as allowedHosts, A as evalPlan, At as refusedMcp, Bt as judgeProblem, Ct as brokeringFor, D as cascade, E as brokeredIn, Et as expectedBlockedFor, F as matchesTags, Ft as NO_JUDGE_INPUT, G as sandboxSetBy, Gt as decideState, H as runsOf, I as plainHeaderError, J as unbrokeredError, K as secretVarsIn, Kt as filePartsOf, L as plainHeaders, Lt as inputsOn, M as harnessId, Mt as writePluginFiles, Ot as keyRejected, P as majorityOf, Pt as INPUT_KEYS, U as sandboxId, Ut as artifactRefsOf, V as refuseIllegalCompare, W as sandboxProvider, Wt as badJudgeArtifacts, X as resolveStateOptions, Y as mergeJudge, _ as fellBackToDirect, at as unenforceable, b as resolveModelGateway, it as isDirect, j as expand, jt as refusedPlugins, k as checkOptions, kt as layoutOf, nt as egressLabel, q as unbrokered, qt as resolveJudgeArtifacts, st as createFakeSandbox, v as gatewayScope, vt as AGENTS, w as resolveGateway, wt as claudeMcpServers, xt as agentMcp } from "./config-ImY6AlRE.js";
9
- import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-ByEUlzOM.js";
10
- import { v as vertexSettings } from "./vertex-6uSpZ5L8.js";
11
- import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets, o as findSecrets } from "./redact-BEEGc37e.js";
12
- import { a as plural, i as formatDuration, n as clip, s as withoutBom, t as capMiddle } from "./text-D--Q5KsA.js";
13
- import { t as inCI } from "./env-T86lAqUD.js";
14
- import { dt as OPAQUE_OUTPUT, mt as shellQuote, t as MCP_TIMER } from "./src-Cpy9MNS3.js";
15
- import { t as InfraError } from "./types-C4OkYhZg.js";
16
- import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-CSZ1hjlN.js";
17
- import { c as formatBytes } from "./debug-B5jDImcW.js";
18
- import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CtKGvaQR.js";
19
- import { t as attemptDebugDir } from "./files-p-uTdOjA.js";
20
- import { r as openRouterFallbacks } from "./openrouter-models-C8nzPKG2.js";
21
- import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-Bdcx9Gap.js";
22
- import { t as contextTaker } from "./context-BajfiH0S.js";
23
- import { a as skillFiles, i as resolveEnvironment } from "./resolve-B_JlQ0La.js";
24
- import { n as isDockerSandbox } from "./docker--2ytOHcu.js";
25
- import { t as PrepareError } from "./provider-DZd8t_XW.js";
26
- import { i as preparedWorkspace } from "./workspace-Cho30qlM.js";
2
+ import { g as parseDuration, m as MAX_TIMER_MS, n as dataProblem, t as closest } from "./shared-Cfi45Bvy.js";
3
+ import { t as isObjectLike } from "./object-CeQ8dgUC.js";
4
+ import { a as plural, i as formatDuration, n as clip, o as withoutBom, t as capMiddle } from "./text-B8-W3Cvn.js";
5
+ import { a as TURN_EDITED, f as limitsLabel, r as withLease } from "./client-KM4C5r5_.js";
6
+ import { d as sandboxKind, n as capabilitiesOf, s as providerLimit } from "./kinds-C1oY3qyC.js";
7
+ import { O as sandboxPaths, P as workspaceOf, f as filesOf, i as callerDir, j as traitsOf, l as environmentFrom, p as globFilter, u as environmentPaths } from "./environment-DB8o3hGi.js";
8
+ import { t as evalSlug } from "./slug-IHh3f770.js";
9
+ import { n as attemptKey, o as recordFileName, r as evalId, s as relativeFile, t as attemptId } from "./ids-C_advdJA.js";
10
+ import { $ as runsOf, A as brokeredIn, B as expand, Bt as decideState, Ct as resolveStateOptions, Et as observer, Ft as NO_JUDGE_INPUT, Ht as resolveJudgeArtifacts, It as inputsOn, J as plainHeaders, K as plainHeaderError, L as checkOptions, Lt as judgeProblem, Pt as INPUT_KEYS, Q as refuseIllegalCompare, Rt as artifactRefsOf, St as mergeJudge, Tt as observedName, U as majorityOf, V as harnessId, Vt as filePartsOf, W as matchesTags, _ as fellBackToDirect, at as unbrokered, b as resolveModelGateway, et as sandboxId, gt as unenforceable, ht as isDirect, it as secretVarsIn, j as cascade, nt as sandboxSetBy, ot as unbrokeredError, pt as egressLabel, tt as sandboxProvider, ut as allowedHosts, v as gatewayScope, w as resolveGateway, wt as observedHere, z as evalPlan, zt as badJudgeArtifacts } from "./config-B9mB1kF5.js";
11
+ import { l as vercelCredentialProblem } from "./vercel-CGlSBv63.js";
12
+ import { v as vertexSettings } from "./vertex-Eb2wqahk.js";
13
+ import { _ as redactValue, d as maskedVariables, f as redactJson, g as redactSplitSecrets, h as redactSecrets, o as findSecrets } from "./redact-BhjtnjrU.js";
14
+ import { f as runJudge, h as isApiError, l as judgeModelName } from "./judge-COMXT1Hx.js";
15
+ import { t as inCI } from "./env-8x1O8I1M.js";
16
+ import { ht as OPAQUE_OUTPUT, n as MCP_TIMER, yt as shellQuote } from "./src-Ck2r_EMT.js";
17
+ import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-BkmPxDiY.js";
18
+ import { _ as unverifiedHarness, c as brokeringFor, d as expectedBlockedFor, g as refusedPlugins, h as refusedMcp, l as claudeMcpServers, m as layoutOf, o as agentMcp, p as keyRejected, t as AGENTS, v as writePluginFiles } from "./agents-C6zhkxp3.js";
19
+ import { n as hasNetworkSession, t as InfraError } from "./types-C2mLpph3.js";
20
+ import { a as registerTest, i as registerGroup, n as declarePlanned, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-BpW1f1Mx.js";
21
+ import { c as formatBytes } from "./debug-CIXzB4wK.js";
22
+ import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CAdlwlNu.js";
23
+ import { t as attemptDebugDir } from "./files-DlQshsks.js";
24
+ import { r as openRouterFallbacks } from "./openrouter-models-Cw1z8-sq.js";
25
+ import { a as writeCaptureSidecars, c as sandboxFor, d as snapshot, f as workspaceBlobs, l as changedBetween, o as asCheck, p as workspaceFiles, r as serveCommand, s as createChecks, t as capturePage, u as diffSinceSnapshot } from "./capture-iBzFiC0j.js";
26
+ import { t as contextTaker } from "./context-Bkv3ndjq.js";
27
+ import { a as skillFiles, i as resolveEnvironment } from "./resolve-Do0ZaYH6.js";
28
+ import { t as PrepareError } from "./provider-BxzZq3Th.js";
29
+ import { i as preparedWorkspace } from "./workspace-uuE51x35.js";
27
30
  import { t as patchPath } from "./patch-C7yBztZj.js";
28
- import { n as judgeDiff } from "./patch-DvLr8wCZ.js";
29
- import { t as transcriptForJudge } from "./transcript-C_IpYMRk.js";
31
+ import { n as judgeDiff } from "./patch-DH4e_Gkd.js";
32
+ import { t as transcriptForJudge } from "./transcript-CCf07UmU.js";
30
33
  import { i as writeTar, n as readTar, t as extractTar } from "./tarball-CESxtK4X.js";
31
- import { i as optionsHash, n as caseHash, t as bodyHash } from "./eval-source-BTsTRiBW.js";
32
- import { i as withDefaultEffort } from "./default-effort-BZRa-SQP.js";
34
+ import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-D3sbAzt-.js";
35
+ import { i as withDefaultEffort } from "./default-effort-DGhQmu95.js";
36
+ import { a as traitFilters, i as pickTraits, n as controlTraitKeys, r as filtersLabel } from "./traits-DKnYkqeK.js";
37
+ import { z } from "zod";
33
38
  import { dirname, join } from "node:path";
34
39
  import { parse, stringify } from "yaml";
35
- import { z } from "zod";
36
40
  import { randomBytes } from "node:crypto";
37
41
  import { setTimeout as setTimeout$1 } from "node:timers/promises";
38
42
  import { gunzipSync, gzipSync } from "node:zlib";
@@ -66,7 +70,7 @@ function assertionRecords(raw) {
66
70
  const seen = /* @__PURE__ */ new Map();
67
71
  const used = /* @__PURE__ */ new Set();
68
72
  return raw.map((r) => {
69
- const named = r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
73
+ const named = r.matcher === "toStayWithin" ? limitsLabel(r.args[0]) : r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
70
74
  const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
71
75
  let n = seen.get(base) ?? 0;
72
76
  let label;
@@ -112,11 +116,11 @@ function tracePlan(debug, spec, ctx) {
112
116
  const from = (key, fallback = ctx.configFile ? `${config} or its default` : "default") => `(${spec.sources[key] ?? fallback})`;
113
117
  const profile = ctx.profile ? `profile ${ctx.profile.name} (${ctx.profile.reason})` : "no profile";
114
118
  debug.log("plan", `config ${config} · ${profile} · run ${ctx.runId} · attempt ${spec.cell.key}#${spec.n}`);
115
- debug.log("plan", `harness ${harnessId(spec.harness)} ${from("harness")}`);
119
+ debug.log("plan", `harness ${harnessId(spec.harness)} ${from("harnesses")}`);
116
120
  debug.log("plan", `sandbox ${sandboxId(spec.sandbox)} ${from("sandbox")}`);
117
121
  const env = spec.environment;
118
122
  const role = spec.cell.role === "variant" ? ` · variant ${spec.cell.variant}` : "";
119
- const envFrom = spec.cell.role === "variant" ? from("compare") : from("environment", "none set");
123
+ const envFrom = spec.cell.role === "variant" || spec.sources.compare ? from("compare") : from("environments", "none set");
120
124
  debug.log("plan", `environment ${env.name}${role} ${envFrom}`);
121
125
  debug.log("plan", `egress ${egressLabel(spec.egress) || "proxy"} ${from("egress")}`);
122
126
  const gate = typeof spec.gate === "string" ? spec.gate : `passRate ${spec.gate.passRate}`;
@@ -254,11 +258,12 @@ const createFakeHarness = (descriptor) => {
254
258
  text,
255
259
  status,
256
260
  startedAt: new Date(startedAt).toISOString(),
257
- durationMs: Date.now() - startedAt,
261
+ durationMs: descriptor.turnMs ?? Date.now() - startedAt,
258
262
  transcript,
259
- usage: null,
263
+ usage: descriptor.usage ?? null,
260
264
  model: descriptor.servedModel ?? null,
261
- unreported: []
265
+ unreported: [],
266
+ ...descriptor.turnMs === void 0 ? {} : { agentMs: descriptor.turnMs }
262
267
  });
263
268
  const write = async (path, content) => {
264
269
  const time = at();
@@ -386,7 +391,7 @@ async function brokerKey(descriptor, sandbox) {
386
391
  const { credential, endpoints } = AGENTS[descriptor.layout ?? "cursor"];
387
392
  const key = process.env[credential];
388
393
  if (!key) throw new InfraError(`${credential} isn't set, so the fake agent has no key to broker. Set it in the environment or .env.local.`);
389
- const network = isDockerSandbox(sandbox) ? sandbox.network : void 0;
394
+ const network = hasNetworkSession(sandbox) ? sandbox.network : void 0;
390
395
  if (!network?.setRequestTransformations) throw new InfraError("fakeHarness({ brokers: true }) needs docker(): this sandbox can't broker");
391
396
  const placeholder = `reeval_fake_${randomBytes(24).toString("base64url")}`;
392
397
  await network.setRequestTransformations([...new Set(endpoints.map((e) => e.host))].map((host) => ({
@@ -478,18 +483,19 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
478
483
  };
479
484
  }
480
485
  // src/harness/index.ts
481
- async function createHarness(descriptor, vertex) {
486
+ /** The attempt's harness, for the sandbox it runs in; an agent the sandbox can't run is an infra error. */
487
+ async function createHarness(descriptor, sandbox, vertex) {
482
488
  if (descriptor.kind === "fake") return createFakeHarness(descriptor);
483
- const { createAgentHarness } = await import("./agent-v036GEgG.js");
484
- return createAgentHarness(descriptor, vertex);
489
+ const { createAgentHarness } = await import("./agent-CBoLndM7.js");
490
+ return createAgentHarness(descriptor, sandbox, vertex);
485
491
  }
486
492
  /** Where each harness's agent looks for repo skills, relative to the workspace. */
487
493
  const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
488
494
  // src/attempt/environment.ts
489
495
  /**
490
496
  * Resolves the attempt's environment (files, skills, secrets and their placeholders)
491
- * and finds the image a Docker sandbox starts from, which preflight prepared. Runs on the test's
492
- * signal, before the eval timer starts.
497
+ * and finds the image its sandbox starts from, when its kind has one, which preflight prepared.
498
+ * Runs on the test's signal, before the eval timer starts.
493
499
  */
494
500
  async function prepareEnvironment(a, root, runId, where, signal) {
495
501
  const { spec, record } = a;
@@ -508,13 +514,14 @@ async function prepareEnvironment(a, root, runId, where, signal) {
508
514
  const debug = attemptDebug();
509
515
  if (debug) traceEnvironment(debug, spec, env);
510
516
  record.image = null;
511
- if (spec.sandbox.kind === "docker") {
512
- const { preparedFor } = await import("./preflight-BAIRsTpl.js");
513
- const { attemptImage, dockerProvider, preparedImage } = await import("./providers-CP_bBtYm.js");
517
+ const kind = sandboxKind(spec.sandbox);
518
+ if (kind.capabilities.images) {
519
+ const { preparedFor } = await import("./preflight-R-InZCQb.js");
520
+ const { attemptImage, createProviders, preparedImage } = await import("./providers-CVu8e0xS.js");
514
521
  signal.throwIfAborted();
515
- const provider = dockerProvider({ engine: spec.sandbox.engine ?? "docker" });
522
+ const provider = createProviders()[kind.provider(spec.sandbox)];
516
523
  const builds = builtPlugins(spec.environment.layers);
517
- const prepared = preparedFor(root, runId, provider, spec.sandbox, spec.harness, builds);
524
+ const prepared = provider && preparedFor(root, runId, provider, spec.sandbox, spec.harness, builds);
518
525
  if (!prepared) throw new InfraError(`The sandbox image for ${where} wasn't prepared before the run started`);
519
526
  record.image = attemptImage(prepared);
520
527
  a.image = record.image.ref;
@@ -534,7 +541,16 @@ async function prepareEnvironment(a, root, runId, where, signal) {
534
541
  }
535
542
  const { ref, id, base, install, agentCli } = record.image;
536
543
  debugLog("image", `ready ${ref} (${id.replace(/^sha256:/, "").slice(0, 12)}) · prepared before the run${ref !== base ? ` · on ${base}` : ""}${agentCli ? ` · agent CLI ${agentCli}` : ""}${install.length ? ` · install: ${install.join(" && ")}` : ""}`);
537
- } else debugLog("image", `none: the ${spec.sandbox.kind} sandbox runs on the host`);
544
+ } else if (kind.capabilities.isolated) {
545
+ const { preparedFor } = await import("./preflight-R-InZCQb.js");
546
+ const { createProviders, vmImage } = await import("./providers-CVu8e0xS.js");
547
+ const provider = createProviders()[kind.provider(spec.sandbox)];
548
+ const prepared = provider && preparedFor(root, runId, provider, spec.sandbox, spec.harness);
549
+ const vm = prepared ? vmImage(prepared) : null;
550
+ if (vm) record.vmImage = vm;
551
+ const from = vm?.ref ?? ("image" in spec.sandbox ? spec.sandbox.image : void 0);
552
+ debugLog("image", `${from ?? "none recorded"}${vm?.digest ? ` (${vm.digest})` : ""} · ${kind.helpers[0]}() checked it before the run and starts each attempt from it`);
553
+ } else debugLog("image", kind.capabilities.isolated ? `none: the ${spec.sandbox.kind} sandbox starts from its provider's base` : `none: the ${spec.sandbox.kind} sandbox runs on the host`);
538
554
  }
539
555
  // src/judge/size.ts
540
556
  /** Roughly four characters a token, as `reeval analyze` counts: an estimate, not the model's. */
@@ -609,6 +625,7 @@ function stateOf(d) {
609
625
  ...d.diff !== void 0 ? { diff: d.diff } : {},
610
626
  ...d.contents ? { contents: d.contents } : {},
611
627
  ...d.checks ? { checks: d.checks } : {},
628
+ ...d.observations ? { observations: d.observations } : {},
612
629
  ...transcript !== void 0 ? { transcript } : {}
613
630
  };
614
631
  }
@@ -643,6 +660,7 @@ async function draftState(src) {
643
660
  ...contents ? { inFull: new Set(Object.keys(contents)) } : {}
644
661
  });
645
662
  const checks = checksShown(wanted.checks, src.checks);
663
+ const observations = observationsShown(wanted.observations, src.observations);
646
664
  return {
647
665
  draft: {
648
666
  all: wanted.turns === "all",
@@ -650,7 +668,8 @@ async function draftState(src) {
650
668
  ...wanted.files ? { files: shown.files } : {},
651
669
  ...wanted.diff !== false ? { diff: shown.patch } : {},
652
670
  ...contents ? { contents } : {},
653
- ...checks ? { checks } : {}
671
+ ...checks ? { checks } : {},
672
+ ...observations ? { observations } : {}
654
673
  },
655
674
  dropped: wanted.diff !== false ? shown.dropped : []
656
675
  };
@@ -682,6 +701,10 @@ function checksShown(which, ran) {
682
701
  stderr
683
702
  }));
684
703
  }
704
+ function observationsShown(which, observed = []) {
705
+ if (!which) return void 0;
706
+ return Object.fromEntries(observed.filter((o) => which === true || which.includes(o.name)).map((o) => [o.name, o.value]));
707
+ }
685
708
  /**
686
709
  * Where `order` goes among `calls`, which are in ascending order: calls finish in any order, but
687
710
  * what they record stays in the order they were made.
@@ -706,7 +729,7 @@ function judgeCaller(a, readFile) {
706
729
  if (!subject) throw new Error(on && "turn" in on ? `c.judge() on names turn ${on.turn}, but this attempt has ${a.turns.length}` : "c.judge() has no turn to grade: call c.send() first");
707
730
  const wanted = resolveStateOptions(call);
708
731
  if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives a value to grade as the output, but output is off. Pass output: true.");
709
- const { task: _task, output: _output, files: _files, diff: _diff, contents: _contents, transcript: _transcript, turns: _turns, checks: _checks, artifacts, ...callJudge } = call;
732
+ const { task: _task, output: _output, files: _files, diff: _diff, contents: _contents, transcript: _transcript, turns: _turns, checks: _checks, observations: _observations, artifacts, ...callJudge } = call;
710
733
  const options = mergeJudge(a.spec.judge, callJudge);
711
734
  const { model } = options;
712
735
  if (model === void 0) {
@@ -716,12 +739,15 @@ function judgeCaller(a, readFile) {
716
739
  }
717
740
  const { maxConcurrent, ...judgeOptions } = options;
718
741
  const output = on && "value" in on ? on.value : subject.text;
742
+ const unobserved = Array.isArray(call.observations) ? call.observations.find((n) => !observedHere(a, n)) : void 0;
743
+ if (unobserved !== void 0) throw new Error(`c.judge() observations names a value this attempt hasn't observed: ${JSON.stringify(observedName(unobserved))}`);
719
744
  const { draft, dropped } = await draftState({
720
745
  turns: a.turns,
721
746
  subject,
722
747
  output,
723
748
  wanted,
724
749
  checks: a.record.checks,
750
+ observations: a.record.observations,
725
751
  readFile,
726
752
  listFiles: async () => workspaceFiles(await sandboxFor(a, "c.judge()"), a.signal)
727
753
  });
@@ -844,18 +870,12 @@ function judgeCaller(a, readFile) {
844
870
  }
845
871
  // src/sandbox/index.ts
846
872
  async function createSandbox(descriptor, options) {
873
+ const kind = sandboxKind(descriptor);
847
874
  try {
848
- switch (descriptor.kind) {
849
- case "fake": return await createFakeSandbox(descriptor, options.env);
850
- case "docker": {
851
- const { createDockerSandbox } = await import("./docker-CIzHzwlC.js");
852
- return await createDockerSandbox(descriptor, options);
853
- }
854
- }
875
+ return await kind.create(descriptor, options);
855
876
  } catch (error) {
856
877
  if (error instanceof InfraError || options.signal.aborted) throw error;
857
- const kind = descriptor.kind === "docker" ? descriptor.engine ?? "docker" : descriptor.kind;
858
- throw new InfraError(`Failed to create ${kind} sandbox: ${error.message}`);
878
+ throw new InfraError(`Failed to create ${kind.provider(descriptor)} sandbox: ${error.message}`);
859
879
  }
860
880
  }
861
881
  // src/attempt/record.ts
@@ -1276,6 +1296,9 @@ function createAttempt(spec, signal, progress, start, snapshotDir) {
1276
1296
  ended = true;
1277
1297
  await starting?.catch(() => {});
1278
1298
  },
1299
+ get ended() {
1300
+ return ended;
1301
+ },
1279
1302
  report: (update) => progress({
1280
1303
  at: Date.now(),
1281
1304
  ...update
@@ -1310,7 +1333,7 @@ const startTimeoutMs = (harness) => parseDuration(harness.startTimeout ?? "90s")
1310
1333
  function setupAllowance(planned) {
1311
1334
  const setupMs = planned.environment.layers.flatMap((l) => l.spec.setup ?? []).reduce((ms, step) => ms + parseDuration((typeof step === "string" ? void 0 : step.timeout) ?? "5m"), 0);
1312
1335
  const { sandbox } = planned;
1313
- return setupMs + (sandbox.kind === "docker" && !!(sandbox.from || sandbox.dockerfile || sandbox.install?.length) ? SETUP_BUDGET_MS : 0) + 2 * startTimeoutMs(planned.harness);
1336
+ return setupMs + (sandboxKind(sandbox).buildsAtStart(sandbox) ? SETUP_BUDGET_MS : 0) + 2 * startTimeoutMs(planned.harness);
1314
1337
  }
1315
1338
  /** The eval's timer, paused while the sandbox and its setup steps start. */
1316
1339
  function evalTimer(ms) {
@@ -1392,7 +1415,7 @@ const SETUP_COMMAND_CHARS = 200;
1392
1415
  */
1393
1416
  async function leftRunning(a, box, env) {
1394
1417
  const names = unbrokeredChecks(env);
1395
- if (a.spec.sandbox.kind !== "docker" || !env.setup.length || !names.length) return [];
1418
+ if (!capabilitiesOf(a.spec.sandbox).processTable || !env.setup.length || !names.length) return [];
1396
1419
  const r = await box.exec([
1397
1420
  "for t in tr sed sort; do command -v \"$t\" >/dev/null || { echo \"no $t in the image\" >&2; exit 1; }; done",
1398
1421
  "for p in /proc/[0-9]*; do",
@@ -1628,7 +1651,7 @@ async function startSession(a, runId) {
1628
1651
  })
1629
1652
  });
1630
1653
  a.sandbox = box;
1631
- if (spec.sandbox.kind === "fake") debugLog("container", `fake sandbox at ${box.workdir}`);
1654
+ if (!capabilitiesOf(spec.sandbox).isolated) debugLog("container", `${sandboxKind(spec.sandbox).provider(spec.sandbox)} sandbox at ${box.workdir}`);
1632
1655
  a.phases = brokerPhases(box, env.substitutions);
1633
1656
  if (a.signal.aborted) box.destroy();
1634
1657
  a.signal.throwIfAborted();
@@ -1674,7 +1697,7 @@ async function startSession(a, runId) {
1674
1697
  debugLog("env", "stdio MCP servers start through reeval's MCP timer");
1675
1698
  } else if (timed.untimed) debugLog("env", `stdio MCP servers start untimed: ${timed.untimed}`);
1676
1699
  if (layout.pluginPlacement === "namespaced") await writePluginMcp(box, env, plugins, timed.servers);
1677
- const harness = await createHarness(spec.harness, spec.vertex);
1700
+ const harness = await createHarness(spec.harness, spec.sandbox, spec.vertex);
1678
1701
  const harnessStart = performance.now();
1679
1702
  debugLog("harness", `starting ${harnessId(spec.harness)}: bootstrap and session`);
1680
1703
  const session = await startHarness(a, harness, box, {
@@ -2016,6 +2039,7 @@ function watchMcpStartup(log) {
2016
2039
  function turnSender(a, takeContext, checks) {
2017
2040
  return async (prompt) => {
2018
2041
  const [box, agent] = await a.start("c.send()");
2042
+ const before = await a.timed("diffMs", () => workspaceBlobs(box, a.signal));
2019
2043
  const entry = {
2020
2044
  prompt,
2021
2045
  text: "",
@@ -2056,7 +2080,8 @@ function turnSender(a, takeContext, checks) {
2056
2080
  durationMs: Date.now() - sendStarted
2057
2081
  };
2058
2082
  });
2059
- const { raw, rejectedKey, reasoningMs, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
2083
+ const timedBefore = a.record.timings.agentMs;
2084
+ const { raw, rejectedKey, reasoningMs, agentMs, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
2060
2085
  entry.reasoningMs = null;
2061
2086
  if (watch) entry.mcpStartup = mcpStartupOf((await watch.end()).entries);
2062
2087
  throw error;
@@ -2064,7 +2089,8 @@ function turnSender(a, takeContext, checks) {
2064
2089
  cancel();
2065
2090
  watch?.close();
2066
2091
  });
2067
- const sendWallMs = Date.now() - sendStarted;
2092
+ const sendWallMs = agentMs ?? Date.now() - sendStarted;
2093
+ if (agentMs !== void 0) a.record.timings.agentMs = timedBefore + agentMs;
2068
2094
  const startup = await watch?.end();
2069
2095
  const timedOut = startup?.timedOut && mcpTimeoutMessage(startup.timedOut);
2070
2096
  const seenAt = startup?.at ?? Date.now();
@@ -2089,6 +2115,16 @@ function turnSender(a, takeContext, checks) {
2089
2115
  await takeContext(box, a.signal);
2090
2116
  const count = (...types) => result.transcript.filter((e) => types.includes(e.type)).length;
2091
2117
  const failure = result.transcript.find((e) => e.type === "error");
2118
+ const infra = timedOut || result.status !== "aborted" && rejectedKey;
2119
+ let changes;
2120
+ let diffError;
2121
+ if (!infra && !a.signal.aborted) try {
2122
+ changes = await a.timed("diffMs", () => diffSinceSnapshot(box, a.signal));
2123
+ } catch (error) {
2124
+ diffError = error;
2125
+ }
2126
+ const edited = changes ? changedBetween(before, changes.blobs) : void 0;
2127
+ const filesChanged = edited?.length;
2092
2128
  a.report({
2093
2129
  kind: "turn-end",
2094
2130
  turn: turnNumber,
@@ -2096,18 +2132,19 @@ function turnSender(a, takeContext, checks) {
2096
2132
  ms: result.durationMs,
2097
2133
  tools: count("tool", "web-search", "web-fetch"),
2098
2134
  fileEdits: count("file-edit"),
2135
+ ...filesChanged === void 0 ? {} : { filesChanged },
2099
2136
  ...failure?.type === "error" ? { error: clip(redactSecrets(failure.message.trim()), 300) } : {}
2100
2137
  });
2101
- debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(redactSecrets(String(failure.message)), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
2138
+ debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${filesChanged === void 0 ? "" : `${plural(filesChanged, "file")} changed · `}${plural(count("file-edit"), "edit tool call")}${failure && "message" in failure ? ` · error: ${clip(redactSecrets(String(failure.message)), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
2102
2139
  for (const s of entry.mcpStartup ?? []) debugLog("harness", `turn ${turnNumber} mcp ${s.server} ${s.ms === null ? "didn't answer initialize" : `answered initialize in ${formatDuration(s.ms)}`}`);
2103
- const infra = timedOut || result.status !== "aborted" && rejectedKey;
2104
2140
  if (infra) {
2105
2141
  const error = new InfraError(infra);
2106
2142
  a.startFailure ??= { error };
2107
2143
  throw error;
2108
2144
  }
2109
2145
  a.signal.throwIfAborted();
2110
- const diff = await a.timed("diffMs", () => diffSinceSnapshot(box, a.signal));
2146
+ if (diffError !== void 0 || !changes || !edited) throw diffError;
2147
+ const { diff } = changes;
2111
2148
  a.record.diff = diff;
2112
2149
  if (a.spec.snapshot.when === "turns") await takeSnapshot(a, box, `turn-${turnNumber}`, "harness");
2113
2150
  const data = {
@@ -2115,10 +2152,10 @@ function turnSender(a, takeContext, checks) {
2115
2152
  ...result,
2116
2153
  diff
2117
2154
  };
2118
- const turn = Object.freeze({
2155
+ const turn = Object.freeze(Object.assign({
2119
2156
  ...data,
2120
2157
  ...checks
2121
- });
2158
+ }, { [TURN_EDITED]: edited }));
2122
2159
  a.turns.push(turn);
2123
2160
  return turn;
2124
2161
  };
@@ -2165,6 +2202,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
2165
2202
  exec,
2166
2203
  readFile,
2167
2204
  writeFile,
2205
+ observe: observer(a),
2168
2206
  serve: (command, port) => serveCommand(a, command, port),
2169
2207
  capture: (target, options) => capturePage(a, target, options),
2170
2208
  attempt: {
@@ -2265,6 +2303,37 @@ function bestOfCell(bestOf) {
2265
2303
  settled: (settled) => settle[n - 1]?.(sideOf(settled))
2266
2304
  });
2267
2305
  }
2306
+ /** Characters a case key can't hold: ids and paths split on them, and control characters. */
2307
+ const BAD_KEY = /[#/›\u0000-\u001f\u007f]/;
2308
+ /** The name a case's eval plans under: `adds a health check [nextjs]`. */
2309
+ const caseEvalName = (name, key) => `${name} [${key}]`;
2310
+ /** The key a case gets without `caseKey`: a string, a finite number, or an object's `id` or `name`. */
2311
+ function defaultKey(item) {
2312
+ if (typeof item === "string") return item;
2313
+ if (typeof item === "number" && Number.isFinite(item)) return String(item);
2314
+ if (isObjectLike(item) && !Array.isArray(item)) {
2315
+ if (typeof item.id === "string") return item.id;
2316
+ if (typeof item.name === "string") return item.name;
2317
+ }
2318
+ }
2319
+ /**
2320
+ * Each case's key, in list order: `caseKey(item)` when given, else its default. Trimmed, 1–80
2321
+ * characters, with no `#`, `/`, `›` or control characters, and unique within the eval.
2322
+ */
2323
+ function caseKeys(cases, caseKey, where) {
2324
+ const seen = /* @__PURE__ */ new Map();
2325
+ return cases.map((item, i) => {
2326
+ const raw = caseKey ? caseKey(item) : defaultKey(item);
2327
+ if (raw === void 0) throw new UsageError(`cases[${i}] in ${where} has no key: a case is its own key when it's a string or a number, or an object's id or name. Give it an id, or pass caseKey: (item) => item.slug.`);
2328
+ if (typeof raw !== "string") throw new UsageError(`caseKey in ${where} must return a string (got ${redactSecrets(JSON.stringify(raw) ?? String(raw))} for cases[${i}])`);
2329
+ const key = raw.trim();
2330
+ if (!key || key.length > 80 || BAD_KEY.test(key)) throw new UsageError(`cases[${i}] in ${where} has the key ${redactSecrets(JSON.stringify(raw))}: a case key is 1–80 characters, with no #, /, › or control characters.${caseKey ? "" : " Pass caseKey to give it another."}`);
2331
+ const earlier = seen.get(key);
2332
+ if (earlier !== void 0) throw new UsageError(`cases[${earlier}] and cases[${i}] in ${where} have the same key "${redactSecrets(key)}". Each case needs its own key${caseKey ? "" : ": give them distinct ids or names, or pass caseKey"}.`);
2333
+ seen.set(key, i);
2334
+ return key;
2335
+ });
2336
+ }
2268
2337
  // src/contract/declaration.ts
2269
2338
  /** A node's options: any object, until the cascade checks it. Its JSON Schema is the type's. */
2270
2339
  const evalOptionsDataSchema = z.record(z.string(), z.unknown());
@@ -2276,7 +2345,7 @@ const declaredModeSchema = z.enum([
2276
2345
  const declaredFamilySchema = z.strictObject({
2277
2346
  name: z.string(),
2278
2347
  case: z.number().int().min(0),
2279
- cases: z.number().int().min(2)
2348
+ cases: z.number().int().min(1)
2280
2349
  });
2281
2350
  const declaredTodoSchema = z.strictObject({
2282
2351
  type: z.literal("todo"),
@@ -2285,7 +2354,7 @@ const declaredTodoSchema = z.strictObject({
2285
2354
  const hash = () => z.string().regex(/^[0-9a-f]{64}$/, "must be a sha256 hash: 64 lowercase hex digits");
2286
2355
  const declaredSourceSchema = z.strictObject({
2287
2356
  body: hash(),
2288
- case: hash().optional()
2357
+ caseJson: z.string().optional()
2289
2358
  });
2290
2359
  const declaredEvalSchema = z.strictObject({
2291
2360
  type: z.literal("eval"),
@@ -2321,6 +2390,89 @@ const NODE_SCHEMAS = {
2321
2390
  * tree's `children`. Options are left to the cascade.
2322
2391
  */
2323
2392
  const declarationProblem = (children) => dataProblem(declarationTreeSchema, { children }, "The declaration tree", (node) => Object.keys(NODE_SCHEMAS[String(node.type)]?.shape ?? {}));
2393
+ // src/where.ts
2394
+ /** Keys a trait lookup can hit without naming a trait: JSON, promises and inspection. */
2395
+ const PASSTHROUGH = /* @__PURE__ */ new Set([
2396
+ "toJSON",
2397
+ "then",
2398
+ "constructor"
2399
+ ]);
2400
+ function pickPairs(w, attempts) {
2401
+ const controls = attempts.filter((a) => a.cell.role === "control");
2402
+ const known = [...new Set(controls.flatMap((a) => Object.keys(traitsOf(a.environment.layers))))];
2403
+ const decisions = /* @__PURE__ */ new Map();
2404
+ const decide = (control) => {
2405
+ const id = control.cell.key;
2406
+ const made = decisions.get(id);
2407
+ if (made) return made;
2408
+ const env = control.environment;
2409
+ const traits = traitsOf(env.layers);
2410
+ let readHarness = false;
2411
+ const view = new Proxy(traits, { get(target, key, receiver) {
2412
+ if (typeof key === "string" && !(key in target) && !PASSTHROUGH.has(key)) {
2413
+ if (!known.includes(key)) {
2414
+ const near = closest(key, known);
2415
+ throw new UsageError(`${w.label}'s where reads environment.traits.${key}, which no environment defines (${known.length ? `they define: ${known.join(", ")}` : "none defines any"}).${near ? ` Did you mean "${near}"?` : ""}`);
2416
+ }
2417
+ }
2418
+ return Reflect.get(target, key, receiver);
2419
+ } });
2420
+ const pair = {
2421
+ case: w.item,
2422
+ environment: Object.freeze({
2423
+ name: env.name,
2424
+ traits: view
2425
+ }),
2426
+ get harness() {
2427
+ readHarness = true;
2428
+ return control.cell.harness;
2429
+ }
2430
+ };
2431
+ let result;
2432
+ try {
2433
+ result = w.where(pair);
2434
+ } catch (error) {
2435
+ if (error instanceof UsageError) throw error;
2436
+ const why = error instanceof Error ? error.message : String(error);
2437
+ throw new UsageError(`${w.label}'s where threw for case ${w.key} on ${env.name}: ${why}`);
2438
+ }
2439
+ if (typeof result !== "boolean") throw new UsageError(`${w.label}'s where returned ${JSON.stringify(result) ?? String(result)} for case ${w.key} on ${env.name}; it must return true or false.`);
2440
+ const decided = {
2441
+ keep: result,
2442
+ readHarness
2443
+ };
2444
+ decisions.set(id, decided);
2445
+ return decided;
2446
+ };
2447
+ const controlOf = (a) => a.cell.role === "control" ? a : controls.find((c) => c.cell.harness === a.cell.harness);
2448
+ const cells = [];
2449
+ const keptCells = [];
2450
+ const excluded = [];
2451
+ return {
2452
+ kept: attempts.filter((a) => {
2453
+ const { keep, readHarness } = decide(controlOf(a));
2454
+ const first = !cells.some((c) => c.key === a.cell.key);
2455
+ if (first) cells.push({
2456
+ key: a.cell.key,
2457
+ environment: a.cell.environment
2458
+ });
2459
+ if (keep) {
2460
+ if (first) keptCells.push(a.cell.key);
2461
+ return true;
2462
+ }
2463
+ const pair = {
2464
+ case: w.key,
2465
+ environment: a.cell.environment,
2466
+ ...readHarness ? { harness: a.cell.harness } : {}
2467
+ };
2468
+ if (!excluded.some((e) => e.environment === pair.environment && e.harness === pair.harness)) excluded.push(pair);
2469
+ return false;
2470
+ }),
2471
+ excluded,
2472
+ cells,
2473
+ keptCells
2474
+ };
2475
+ }
2324
2476
  // src/plan-eval.ts
2325
2477
  /** Lets the attempt's own eval timer fire before the engine's backstop, so the message and cleanup are its own. */
2326
2478
  const GRACE_MS = 5e3;
@@ -2329,7 +2481,7 @@ function configLevel(config) {
2329
2481
  return {
2330
2482
  harness: config.harness,
2331
2483
  sandbox: config.sandbox,
2332
- environment: config.environment,
2484
+ environments: config.environments,
2333
2485
  compare: void 0,
2334
2486
  runs,
2335
2487
  bestOf,
@@ -2364,26 +2516,83 @@ function planEval(declaration, ctx) {
2364
2516
  profile: ctx.profile,
2365
2517
  runs: ctx.runs,
2366
2518
  skipped: declaration.mode === "skip" || declaration.mode === "todo",
2367
- pooled: declaration.family !== void 0
2519
+ pooled: (declaration.family?.cases ?? 0) >= 2
2368
2520
  });
2369
2521
  const { harnesses } = resolvedEval;
2370
2522
  if (!matchesTags(resolvedEval.resolved.tags, ctx.tags)) return {
2371
2523
  tests: null,
2372
- harnesses
2524
+ harnesses,
2525
+ traits: []
2526
+ };
2527
+ const { resolved } = resolvedEval;
2528
+ const declared = expand({
2529
+ ...resolved,
2530
+ harness: resolved.harness.map((h) => withDefaultEffort(h, ctx.cursorEfforts))
2531
+ });
2532
+ const traits = controlTraitKeys(declared);
2533
+ const filters = traitFilters(ctx.traits, ctx.profile);
2534
+ const traitSkip = filters.length > 0 && !resolvedEval.skip && !pickTraits(declared, filters).kept.length;
2535
+ const narrowed = traitSkip ? {
2536
+ ...resolvedEval,
2537
+ skip: noCellLeft(declared, filters)
2538
+ } : resolvedEval;
2539
+ const picks = [];
2540
+ if (traitSkip && declaration.where) picks.push(pickPairs(declaration.where, declared));
2541
+ let traitExcluded = [];
2542
+ const pick = (attempts) => {
2543
+ let kept = attempts;
2544
+ if (declaration.where) {
2545
+ const picked = pickPairs(declaration.where, attempts);
2546
+ picks.push(picked);
2547
+ kept = picked.kept;
2548
+ }
2549
+ if (!filters.length) return kept;
2550
+ const byTraits = pickTraits(kept, filters);
2551
+ traitExcluded = byTraits.excluded;
2552
+ return byTraits.kept;
2553
+ };
2554
+ const leftOut = () => `${filtersLabel(filters)} ${filters.length === 1 ? "leaves" : "leave"} out ${traitExcluded.join(", ")}`;
2555
+ let tests = planTests(declaration, ctx, where, narrowed, pick);
2556
+ const allLeftOut = !tests.length && traitExcluded.length > 0;
2557
+ if (allLeftOut) {
2558
+ const skip = `no cell left: ${leftOut()}`;
2559
+ tests = planTests(declaration, ctx, where, {
2560
+ ...narrowed,
2561
+ skip
2562
+ }, pick);
2563
+ }
2564
+ if (traitExcluded.length) {
2565
+ for (const t of tests) t.attempt.plan.traitExcluded = traitExcluded;
2566
+ const notes = new Set(tests.map((t) => t.attempt.plan.notes));
2567
+ if (!allLeftOut) for (const list of notes) list.push(leftOut());
2568
+ }
2569
+ const [picked] = picks;
2570
+ if (!picked) return {
2571
+ tests,
2572
+ harnesses,
2573
+ traits
2373
2574
  };
2575
+ const { excluded, cells, keptCells } = picked;
2374
2576
  return {
2375
- tests: planTests(declaration, ctx, where, resolvedEval),
2376
- harnesses
2577
+ tests,
2578
+ harnesses,
2579
+ traits,
2580
+ picked: {
2581
+ excluded,
2582
+ cells,
2583
+ keptCells
2584
+ }
2377
2585
  };
2378
2586
  }
2379
- function planTests(declaration, ctx, where, resolvedEval) {
2587
+ /** Why an eval none of whose control environments the trait filters match doesn't run. */
2588
+ function noCellLeft(attempts, filters) {
2589
+ const envs = [...new Set(attempts.filter((a) => a.cell.role === "control").map((a) => a.cell.environment))];
2590
+ return `no cell left: ${filtersLabel(filters)} ${filters.length === 1 ? "matches" : "match"} none of ${envs.join(", ")}`;
2591
+ }
2592
+ function planTests(declaration, ctx, where, resolvedEval, pick) {
2380
2593
  const { name, mode, groups: enclosing } = declaration;
2381
2594
  const groups = enclosing.map((g) => g.name);
2382
2595
  const { resolved } = resolvedEval;
2383
- const source = {
2384
- ...declaration.source,
2385
- options: optionsHash([...enclosing.map((g) => g.options), declaration.options])
2386
- };
2387
2596
  const timeoutMs = resolved.timeout === void 0 ? ctx.config.timeoutMs : parseDuration(resolved.timeout);
2388
2597
  const gateways = ctx.profile?.gateway ?? ctx.config.gateway;
2389
2598
  const gatewaySet = ctx.config.gatewaySet || ctx.profile?.gateway !== void 0;
@@ -2416,10 +2625,16 @@ function planTests(declaration, ctx, where, resolvedEval) {
2416
2625
  gateway: route.name
2417
2626
  };
2418
2627
  }).map((harness) => withDefaultEffort(harness, ctx.cursorEfforts));
2419
- const plannedAttempts = resolvedEval.skip ? [] : expand({
2628
+ const plannedAttempts = resolvedEval.skip ? [] : pick(expand({
2420
2629
  ...resolved,
2421
2630
  harness: routed
2422
- }, vertex);
2631
+ }, vertex));
2632
+ const { caseJson, ...declared } = declaration.source;
2633
+ const source = {
2634
+ ...declared,
2635
+ ...caseJson === void 0 ? {} : { case: caseHash(caseJson, process.env, secretValues(resolved.env, resolvedEval.skip ? expand(resolved) : plannedAttempts)) },
2636
+ options: optionsHash([...enclosing.map((g) => g.options), declaration.options])
2637
+ };
2423
2638
  let judge;
2424
2639
  if (plannedAttempts.length) try {
2425
2640
  const gateway = resolveModelGateway(resolved.judge.model, resolved.judge.gateway, { gateways }, "the judge model");
@@ -2445,6 +2660,8 @@ function planTests(declaration, ctx, where, resolvedEval) {
2445
2660
  const why = unenforceable(planned.egress, planned.sandbox, where);
2446
2661
  if (why) throw new UsageError(why);
2447
2662
  const on = `${where} on ${harnessId(planned.harness)}`;
2663
+ const unverified = unverifiedHarness(planned.harness, planned.sandbox);
2664
+ if (unverified) throw new UsageError(`${on}: ${unverified}`);
2448
2665
  const mcp = refusedMcp(planned.harness, planned.environment.layers);
2449
2666
  if (mcp) throw new UsageError(`${on}: ${mcp}`);
2450
2667
  const plugins = planPlugins(planned, ctx.config.root);
@@ -2458,7 +2675,7 @@ function planTests(declaration, ctx, where, resolvedEval) {
2458
2675
  id: evalId(rankFile, [...groups, declaration.family.name]),
2459
2676
  ...declaration.family
2460
2677
  };
2461
- const harnessBy = resolved.sources?.harness ?? ctx.config.file;
2678
+ const harnessBy = resolved.sources?.harnesses ?? ctx.config.file;
2462
2679
  const plan = evalPlan(resolvedEval, plannedAttempts, {
2463
2680
  configFile: ctx.config.file,
2464
2681
  file: path
@@ -2575,6 +2792,13 @@ function planTests(declaration, ctx, where, resolvedEval) {
2575
2792
  });
2576
2793
  }
2577
2794
  /**
2795
+ * The values of the `secret()` variables the attempts read. The record masks them once the run
2796
+ * has planned, and the case hash is taken here, before that, so it masks them itself.
2797
+ */
2798
+ function secretValues(env, attempts) {
2799
+ return [...new Set(attempts.flatMap((a) => secretVarsIn(env, a)))].flatMap((name) => process.env[name] ?? []);
2800
+ }
2801
+ /**
2578
2802
  * What the plan can tell of an attempt's plugins before any is fetched: a build outside Docker, a
2579
2803
  * local plugin's content, and the harness's refusal, as a refusal; the commands that fetch a
2580
2804
  * package at start, as notes under `proxy` and `allow-all`; and a label for each plugin. Git and npm plugins
@@ -2633,7 +2857,8 @@ function planPlugins(planned, root) {
2633
2857
  function attemptLimits(planned, concurrency) {
2634
2858
  const limits = { attempts: concurrency };
2635
2859
  const { sandbox, harness } = planned;
2636
- if (sandbox.maxConcurrent) limits[`sandbox:${sandboxProvider(sandbox)}`] = sandbox.maxConcurrent;
2860
+ const cap = providerLimit(sandbox);
2861
+ if (cap) limits[`sandbox:${sandboxProvider(sandbox)}`] = cap;
2637
2862
  if (harness.maxConcurrent) limits[`harness:${harness.kind}`] = harness.maxConcurrent;
2638
2863
  return limits;
2639
2864
  }
@@ -2668,7 +2893,7 @@ function withPaths(options, where, base = callerDir() ?? process.cwd()) {
2668
2893
  }
2669
2894
  /** `(name, fn)`, `(name, options, fn)` as in Vitest, or `(name, fn, options)`. */
2670
2895
  function splitArgs(what, name, a, b) {
2671
- if (typeof a === "function" && typeof b === "function") throw new UsageError(`${what}("${name}") got two functions. Options must be a plain object; helpers such as secret are only available in defineConfig and defineEnvironment callbacks. Put the secret in an environment (environment: …) or in the config's env.`);
2896
+ if (typeof a === "function" && typeof b === "function") throw new UsageError(`${what}("${name}") got two functions. Options must be a plain object; helpers such as secret are only available in defineConfig and defineEnvironment callbacks. Put the secret in an environment (environments: …) or in the config's env.`);
2672
2897
  if (typeof a === "function") return [b ?? {}, a];
2673
2898
  if (typeof b === "function") return [a ?? {}, b];
2674
2899
  throw new UsageError(what === "group" ? `group("${name}") has no function. Pass the group's body, e.g. group("${name}", () => { … }).` : `${what}("${name}") has no function. Pass the eval's body, e.g. ${what}("${name}", async (c) => { … }).`);
@@ -2680,8 +2905,40 @@ function checkOnly(mode, what) {
2680
2905
  error.name = "OnlyInCiError";
2681
2906
  throw error;
2682
2907
  }
2683
- const defineGroup = (mode) => (name, a, b) => {
2908
+ /** Each name path as its file wrote it, by kind and the path it's shown as. */
2909
+ const givenPaths = /* @__PURE__ */ new Map();
2910
+ /**
2911
+ * `name` as it's declared, recorded and shown: masked as everything reeval writes is, so the
2912
+ * eval's id, its artifact directory and every view hold no secret. Two names that mask to the
2913
+ * same name are refused rather than numbered: the id is the key history, compare and query
2914
+ * group attempts by across runs, and a number would depend on the order of the declarations.
2915
+ */
2916
+ function shownName(kind, name, hint = "") {
2917
+ const shown = redactSecrets(name);
2918
+ const path = [...groupStack.map((g) => g.name), shown];
2919
+ const given = [...groupStack.map((g) => g.given), name].join("\0");
2920
+ const key = [
2921
+ collectingFile() ?? "",
2922
+ kind === "group" ? "group" : "eval",
2923
+ ...path
2924
+ ].join("\0");
2925
+ const earlier = givenPaths.get(key);
2926
+ if (earlier !== void 0 && earlier !== given) throw new UsageError(`${kind === "group" ? "group" : kind === "todo" ? "evaluate.todo" : "evaluate"}("${path.join(" › ")}") names a secret, and its masked name is another ${kind === "group" ? "group" : "eval"}'s in this file: name it without the secret.${hint}`);
2927
+ givenPaths.set(key, given);
2928
+ return shown;
2929
+ }
2930
+ /**
2931
+ * A note naming the variables whose values masked part of an eval's name: unlike a credential
2932
+ * shape, a value is masked only where the variable holds it, so the eval's id differs elsewhere.
2933
+ */
2934
+ function maskedNote(names) {
2935
+ const vars = [...new Set(names.flatMap((n) => maskedVariables(n)))];
2936
+ if (!vars.length) return void 0;
2937
+ return `name masked: it holds the value of ${vars.join(", ")}, so its id differs where ${vars.length === 1 ? "that variable has" : "those variables have"} another value`;
2938
+ }
2939
+ const defineGroup = (mode) => (given, a, b) => {
2684
2940
  checkOnly(mode, "group");
2941
+ const name = shownName("group", given);
2685
2942
  const [written, fn] = splitArgs("group", name, a, b);
2686
2943
  const options = withPaths(written, `group("${name}")`);
2687
2944
  checkOptions(options, `group("${name}")`);
@@ -2695,6 +2952,7 @@ const defineGroup = (mode) => (name, a, b) => {
2695
2952
  declare(node);
2696
2953
  const stack = [...groupStack, {
2697
2954
  name,
2955
+ given,
2698
2956
  options,
2699
2957
  node
2700
2958
  }];
@@ -2717,7 +2975,8 @@ const group = Object.assign(defineGroup("run"), {
2717
2975
  * A placeholder, `evaluate.todo` or `group.todo`: one eval entry that never runs, counted as
2718
2976
  * skipped and shown as todo. The enclosing groups' tags filter it as they do an eval.
2719
2977
  */
2720
- function registerTodo(name) {
2978
+ function registerTodo(given) {
2979
+ const name = shownName("todo", given);
2721
2980
  declare({
2722
2981
  type: "todo",
2723
2982
  name
@@ -2735,7 +2994,7 @@ const families = /* @__PURE__ */ new Map();
2735
2994
  /** Two name paths with one id differ only where a name holds the ` › ` the id joins them with. */
2736
2995
  const separatorHint = (path, earlier) => path.length === earlier.length ? "" : ` " › " joins the group and eval names in an eval's id, so a name containing it can be a grouped eval's id.`;
2737
2996
  /** A run's facts name an eval and a family alike by id, so an eval can't take a family's. */
2738
- const namedLikeFamily = (eval_, family) => new UsageError(`evaluate("${eval_.join(" › ")}") has the same id as the pooled evaluate.each("${family.join(" › ")}") in this file: a family's id is its name template. Give the eval or the template another name.${separatorHint(eval_, family)}`);
2997
+ const namedLikeFamily = (eval_, family) => new UsageError(`evaluate("${eval_.join(" › ")}") has the same id as evaluate("${family.join(" › ")}") with cases in this file. Give one of them another name.${separatorHint(eval_, family)}`);
2739
2998
  function checkUnique(where, groups, name, hint, inFamily) {
2740
2999
  const path = [...groups, name];
2741
3000
  const id = evalId(collectingFile() ?? "", path);
@@ -2766,10 +3025,18 @@ function checkFamily(groups, family) {
2766
3025
  });
2767
3026
  return;
2768
3027
  }
2769
- if (earlier.path.length !== path.length || earlier.cases !== family.cases || earlier.seen.has(family.case)) throw new UsageError(`evaluate.each("${path.join(" › ")}") is declared twice in this file. Each pooled evaluate.each needs its own name, or their cases are pooled into one family. Give each its own name template, or pass evaluate.each(cases, { pool: false }) to keep the cases apart.${separatorHint(path, earlier.path)}`);
3028
+ if (earlier.path.length !== path.length || earlier.cases !== family.cases || earlier.seen.has(family.case)) throw new UsageError(`evaluate("${path.join(" › ")}") with cases is declared twice in this file. Each eval needs its own name, or their cases are pooled into one. Give each its own name.${separatorHint(path, earlier.path)}`);
2770
3029
  earlier.seen.add(family.case);
2771
3030
  }
2772
- function register(mode, name, own, fn, source, hint = "", family) {
3031
+ function register(mode, given, own, fn, source, hint = "", givenFamily) {
3032
+ prepare(mode, given, own, fn, source, hint, givenFamily).commit();
3033
+ }
3034
+ function prepare(mode, given, own, fn, source, hint = "", givenFamily, where) {
3035
+ const name = shownName("eval", given, givenFamily ? " Give each case a key that holds no secret." : "");
3036
+ const family = givenFamily && {
3037
+ ...givenFamily,
3038
+ name: redactSecrets(givenFamily.name)
3039
+ };
2773
3040
  const groups = groupStack.map((g) => g.name);
2774
3041
  if (family) checkFamily(groups, family);
2775
3042
  checkUnique(`evaluate("${[...groups, name].join(" › ")}")`, groups, name, hint, !!family);
@@ -2784,7 +3051,7 @@ function register(mode, name, own, fn, source, hint = "", family) {
2784
3051
  source,
2785
3052
  ...family ? { family } : {}
2786
3053
  });
2787
- const { tests, harnesses } = planEval({
3054
+ const planned = planEval({
2788
3055
  name,
2789
3056
  mode,
2790
3057
  groups: groupStack.map(({ name, options }) => ({
@@ -2793,82 +3060,130 @@ function register(mode, name, own, fn, source, hint = "", family) {
2793
3060
  })),
2794
3061
  options: own,
2795
3062
  source,
2796
- ...family ? { family } : {}
3063
+ ...family ? { family } : {},
3064
+ ...where ? { where } : {}
2797
3065
  }, {
2798
3066
  ...getRunContext(),
2799
3067
  file: collectingFile(),
2800
3068
  order: registered
2801
3069
  });
2802
- declareHarnesses(harnesses);
2803
- if (!tests) return;
2804
- registered++;
2805
- const bestOfCells = /* @__PURE__ */ new Map();
2806
- for (const { name: testName, spec, ...options } of tests) {
2807
- const { cell, n, bestOf } = options.attempt;
2808
- let hooks = {};
2809
- if (bestOf !== void 0) {
2810
- const of = bestOfCells.get(cell.key) ?? bestOfCell(bestOf);
2811
- bestOfCells.set(cell.key, of);
2812
- hooks = of(n);
2813
- }
2814
- registerTest(testName, {
2815
- ...options,
2816
- ...hooks
2817
- }, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
3070
+ if (planned.tests) {
3071
+ const note = maskedNote([...groupStack.map((g) => g.given), given]);
3072
+ if (note) for (const notes of new Set(planned.tests.map((t) => t.attempt.plan.notes))) notes.push(note);
3073
+ registered++;
2818
3074
  }
3075
+ const commit = () => {
3076
+ declarePlanned(planned.harnesses, planned.traits);
3077
+ if (!planned.tests) return;
3078
+ const bestOfCells = /* @__PURE__ */ new Map();
3079
+ for (const { name: testName, spec, ...options } of planned.tests) {
3080
+ const { cell, n, bestOf } = options.attempt;
3081
+ let hooks = {};
3082
+ if (bestOf !== void 0) {
3083
+ const of = bestOfCells.get(cell.key) ?? bestOfCell(bestOf);
3084
+ bestOfCells.set(cell.key, of);
3085
+ hooks = of(n);
3086
+ }
3087
+ registerTest(testName, {
3088
+ ...options,
3089
+ ...hooks
3090
+ }, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
3091
+ }
3092
+ };
3093
+ return {
3094
+ planned,
3095
+ commit
3096
+ };
2819
3097
  }
2820
3098
  /** Evals registered so far in this worker, for the scheduler's tie-break. */
2821
3099
  let registered = 0;
2822
3100
  const defineEval = (mode, declared) => (name, a, b) => {
2823
3101
  checkOnly(mode, "evaluate");
2824
- const [own, fn] = splitArgs("evaluate", name, a, b);
2825
- const where = `evaluate("${name}")`;
2826
- register(mode, name, withPaths(own, where), fn, declared ?? { body: bodyHash(fn, where, collectingFile()) });
3102
+ const shown = redactSecrets(name);
3103
+ const [written, fn] = splitArgs("evaluate", shown, a, b);
3104
+ const where = `evaluate("${shown}")`;
3105
+ if (isObjectLike(written) && "where" in written && !("cases" in written)) throw new UsageError(`${where} sets where without cases. where picks case-and-environment pairs; to drop an environment, leave it out of environments.`);
3106
+ if (isObjectLike(written) && ("cases" in written || "caseKey" in written)) return registerCases(mode, name, written, fn, where);
3107
+ register(mode, name, withPaths(written, where), fn, declared ?? { body: bodyHash(fn, where, collectingFile()) });
2827
3108
  };
2828
- /** `$key` is a property of the case, `%s` the case itself, `%#` its index. */
2829
- function formatName(template, item, index) {
2830
- return template.replace(/%#/g, String(index)).replace(/%s/g, typeof item === "string" ? item : JSON.stringify(item)).replace(/\$([A-Za-z_][A-Za-z0-9_]*)/g, (whole, key) => isObjectLike(item) && key in item ? String(item[key]) : whole);
2831
- }
2832
- function checkEachOptions(options) {
2833
- if (options === void 0) return {};
2834
- if (isObjectLike(options) && Object.keys(options).every((k) => k === "pool")) {
2835
- const { pool } = options;
2836
- if (pool === void 0) return {};
2837
- if (typeof pool === "boolean") return { pool };
2838
- }
2839
- throw new UsageError(`evaluate.each(cases, options) takes { pool?: boolean } (got ${JSON.stringify(options)}). Eval options such as compare go after the name: evaluate.each(cases)(name, { compare }, fn).`);
2840
- }
2841
3109
  /**
2842
- * One eval per case, pooled into one suite:
2843
- * `evaluate.each(cases)('adds $a and $b', async (c, { a, b }) => …)`.
3110
+ * An eval with `cases`: each case plans as an entry of its own named for the eval, which the
3111
+ * record folds back into the eval. Two or more cases pool into one pass rate and, with
3112
+ * `compare`, one lift paired by case.
2844
3113
  */
2845
- const evalEach = (cases, options) => (name, a, b) => {
2846
- const { pool = true } = checkEachOptions(options);
2847
- if (!Array.isArray(cases)) throw new UsageError(`evaluate.each takes an array of cases (got ${typeof cases}), e.g. evaluate.each([{ n: 1 }, { n: 2 }])("case $n", async (c, item) => { … }).`);
2848
- const [written, fn] = splitArgs("evaluate.each", name, a, b);
2849
- const own = withPaths(written, `evaluate.each("${name}")`);
2850
- const hint = ` For evaluate.each, put $key, %s or %# in the name ("${name}") so each case's name differs.`;
2851
- const pooled = pool && cases.length >= 2;
3114
+ function registerCases(mode, name, written, fn, where) {
3115
+ const { cases, caseKey, where: pick, ...rest } = written;
3116
+ if (!Array.isArray(cases)) throw new UsageError(cases === void 0 ? `caseKey in ${where} names each case, but there are no cases. Add cases: [...], or drop caseKey.` : `cases in ${where} must be a list (got ${typeof cases}), e.g. cases: ['users', 'posts'].`);
3117
+ if (cases.length === 0) throw new UsageError(`cases in ${where} is an empty list. Give at least one case.`);
3118
+ if (caseKey !== void 0 && typeof caseKey !== "function") throw new UsageError(`caseKey in ${where} must be a function from a case to its key, e.g. caseKey: (item) => item.slug.`);
3119
+ if (pick !== void 0 && typeof pick !== "function") throw new UsageError(`where in ${where} must be a function from a case and an environment to true or false, e.g. where: ({ case: item, environment }) => item.stacks.includes(environment.traits.stack).`);
3120
+ if (isObjectLike(rest.runs) && "bestOf" in rest.runs) throw new UsageError(`${where} has cases, so it can't take runs: { bestOf }: best-of stops each cell on its own, and the cases pool over the runs. Set a fixed count, e.g. runs: 3.`);
3121
+ const keys = caseKeys(cases, caseKey, where);
3122
+ const shownKeys = keys.map((k) => redactSecrets(k));
3123
+ const own = withPaths(rest, where);
2852
3124
  const file = collectingFile();
2853
- const body = bodyHash(fn, `evaluate.each("${name}")`, file);
2854
- cases.forEach((item, i) => {
2855
- const family = pooled ? {
3125
+ const body = bodyHash(fn, where, file);
3126
+ const prepared = cases.map((item, i) => {
3127
+ const key = shownKeys[i];
3128
+ const family = {
2856
3129
  name,
2857
3130
  case: i,
2858
3131
  cases: cases.length
2859
- } : void 0;
3132
+ };
2860
3133
  const source = {
2861
3134
  body,
2862
- case: caseHash(item, `evaluate.each("${name}") case ${i}`, file)
3135
+ caseJson: caseJson(item, `${where} case "${key}"`, file)
2863
3136
  };
2864
- register("run", formatName(name, item, i), own, (c) => fn(c, item), source, hint, family);
3137
+ const run = (c) => fn(c, item);
3138
+ const picks = pick ? {
3139
+ label: where,
3140
+ key,
3141
+ item,
3142
+ where: pick
3143
+ } : void 0;
3144
+ return prepare(mode, caseEvalName(name, keys[i]), own, run, source, "", family, picks);
2865
3145
  });
2866
- };
3146
+ if (pick) applyWhere(where, shownKeys, prepared);
3147
+ for (const p of prepared) p.commit();
3148
+ }
3149
+ /**
3150
+ * The checks `where` needs across an eval's cases, then the pairs it left out: each case's own on
3151
+ * its plan, and every case's in the eval's notes. A cell that no case runs on is refused, and so
3152
+ * is an eval it leaves nothing to run.
3153
+ */
3154
+ function applyWhere(where, keys, prepared) {
3155
+ const picks = prepared.flatMap((p) => p.planned.picked ? [p.planned.picked] : []);
3156
+ if (!picks.length) return;
3157
+ const excluded = picks.flatMap((p) => p.excluded);
3158
+ const kept = new Set(picks.flatMap((p) => p.keptCells));
3159
+ if (!kept.size) throw new UsageError(`${where}'s where excludes all ${excluded.length} case-and-environment pairs, so it has nothing to run.`);
3160
+ const empty = picks.flatMap((p) => p.cells).find((c) => !kept.has(c.key));
3161
+ if (empty) throw new UsageError(`${where}'s where leaves cell ${empty.key} with no cases. Leave ${empty.environment} out of environments, or widen where.`);
3162
+ if (!excluded.length) return;
3163
+ const byEnv = /* @__PURE__ */ new Map();
3164
+ for (const e of excluded) {
3165
+ const on = e.harness ? `${e.environment} for ${e.harness}` : e.environment;
3166
+ const list = byEnv.get(on) ?? [];
3167
+ if (!list.includes(e.case)) list.push(e.case);
3168
+ byEnv.set(on, list);
3169
+ }
3170
+ const order = (k) => keys.indexOf(k);
3171
+ const notes = [`where excludes ${excluded.length} ${excluded.length === 1 ? "pair" : "pairs"}`, ...[...byEnv].map(([on, list]) => `excluded on ${on}: ${list.sort((a, b) => order(a) - order(b)).join(", ")}`)];
3172
+ const plans = /* @__PURE__ */ new Set();
3173
+ for (const p of prepared) {
3174
+ const own = p.planned.picked?.excluded ?? [];
3175
+ for (const t of p.planned.tests ?? []) {
3176
+ plans.add(t.attempt.plan);
3177
+ if (own.length) t.attempt.plan.excluded = own;
3178
+ if (t.attempt.family) t.attempt.plan.familyExcluded = excluded;
3179
+ }
3180
+ }
3181
+ for (const list of new Set([...plans].map((p) => p.notes))) list.push(...notes);
3182
+ }
2867
3183
  const evaluate = Object.assign(defineEval("run"), {
2868
3184
  skip: defineEval("skip"),
2869
3185
  only: defineEval("only"),
2870
- todo: registerTodo,
2871
- each: evalEach
3186
+ todo: registerTodo
2872
3187
  });
2873
3188
  /**
2874
3189
  * Declares a tree's groups and evals as the calls that made it would, in the file being
@@ -2886,13 +3201,13 @@ function declareNodes(nodes, bodyFor, base) {
2886
3201
  for (const node of nodes) if (node.type === "todo") registerTodo(node.name);
2887
3202
  else if (node.type === "group") {
2888
3203
  const children = node.children;
2889
- const where = `group("${node.name}")`;
3204
+ const where = `group("${redactSecrets(node.name)}")`;
2890
3205
  defineGroup(node.mode)(node.name, optionsFrom(node.options, where, base), () => declareNodes(children, bodyFor, base));
2891
3206
  } else if (node.family) {
2892
- const options = optionsFrom(node.options, `evaluate("${node.name}")`, base);
3207
+ const options = optionsFrom(node.options, `evaluate("${redactSecrets(node.name)}")`, base);
2893
3208
  checkOnly(node.mode, "evaluate");
2894
3209
  register(node.mode, node.name, options, bodyFor(node.body), node.source, "", node.family);
2895
- } else defineEval(node.mode, node.source)(node.name, optionsFrom(node.options, `evaluate("${node.name}")`, base), bodyFor(node.body));
3210
+ } else defineEval(node.mode, node.source)(node.name, optionsFrom(node.options, `evaluate("${redactSecrets(node.name)}")`, base), bodyFor(node.body));
2896
3211
  }
2897
3212
  /**
2898
3213
  * Options from their data forms, with host paths absolute against `base`: environments become
@@ -2900,32 +3215,30 @@ function declareNodes(nodes, bodyFor, base) {
2900
3215
  */
2901
3216
  function optionsFrom(written, where, base) {
2902
3217
  refuseIllegalCompare(written, where);
2903
- const { environment, compare, ...rest } = withPaths(written, where, base);
3218
+ const { environments, compare, ...rest } = withPaths(written, where, base);
2904
3219
  const missing = missingIn(where);
2905
3220
  const asEnv = (env, field) => environmentFrom(environmentPaths(env, base, field, missing));
2906
- const own = environment !== void 0 ? { environment: asEnv(environment, "environment") } : {};
3221
+ const asEnvs = (value, field) => {
3222
+ if (Array.isArray(value)) return value.map((e, i) => asEnv(e, `${field}[${i}]`));
3223
+ if ("only" in value) return value;
3224
+ if ("add" in value) return { add: [value.add].flat().map((e, i) => asEnv(e, `${field}.add[${i}]`)) };
3225
+ return asEnv(value, field);
3226
+ };
3227
+ const own = environments !== void 0 ? { environments: asEnvs(environments, "environments") } : {};
2907
3228
  if (compare === void 0) return {
2908
3229
  ...rest,
2909
3230
  ...own
2910
3231
  };
2911
3232
  const variants = compare.variants.map((v, i) => asEnv(v, `compare.variants[${i}]`));
2912
- if (compare.baseline !== void 0) return {
2913
- ...rest,
2914
- compare: {
2915
- ...compare,
2916
- baseline: asEnv(compare.baseline, "compare.baseline"),
2917
- variants
2918
- }
2919
- };
2920
3233
  return {
2921
3234
  ...rest,
2922
- ...own,
2923
3235
  compare: {
2924
3236
  ...compare,
3237
+ control: asEnv(compare.control, "compare.control"),
2925
3238
  variants
2926
3239
  }
2927
3240
  };
2928
3241
  }
2929
3242
  export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
2930
3243
 
2931
- //# sourceMappingURL=evaluate-CFFdyYZX.js.map
3244
+ //# sourceMappingURL=evaluate-CwKeN_fo.js.map