reeval 0.18.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (380) hide show
  1. package/README.md +1 -1
  2. package/THIRD_PARTY_NOTICES.md +5 -5
  3. package/dist/{adapter-CptK032x.js → adapter-C8xPuXYq.js} +5 -4
  4. package/dist/adapter-C8xPuXYq.js.map +1 -0
  5. package/dist/{agent-B63OzUX9.js → agent-CBoLndM7.js} +30 -10
  6. package/dist/{agent-B63OzUX9.js.map → agent-CBoLndM7.js.map} +1 -1
  7. package/dist/agents-C6zhkxp3.js +1151 -0
  8. package/dist/agents-C6zhkxp3.js.map +1 -0
  9. package/dist/{app-files-sgqwfjtn.js → app-files-DT5mJyV4.js} +3 -3
  10. package/dist/app-files-DT5mJyV4.js.map +1 -0
  11. package/dist/assemble-UPCRrtOz.js +1 -0
  12. package/dist/{assemble-Me7Whp-y.js → assemble-t6h018An.js} +152 -129
  13. package/dist/assemble-t6h018An.js.map +1 -0
  14. package/dist/{attempt-Dp7Wtm8M.js → attempt-CAdlwlNu.js} +2 -2
  15. package/dist/{attempt-Dp7Wtm8M.js.map → attempt-CAdlwlNu.js.map} +1 -1
  16. package/dist/{attempt-host-Cik0SoIn.js → attempt-host-CIm1-11m.js} +6 -6
  17. package/dist/{attempt-host-Cik0SoIn.js.map → attempt-host-CIm1-11m.js.map} +1 -1
  18. package/dist/boat-BjLqFJIZ.js +108 -0
  19. package/dist/boat-BjLqFJIZ.js.map +1 -0
  20. package/dist/boat-CJCtzfB1.js +322 -0
  21. package/dist/boat-CJCtzfB1.js.map +1 -0
  22. package/dist/bridge/index.mjs +300 -78
  23. package/dist/capture-DoxFUN3J.js +1 -0
  24. package/dist/{capture-C3RmjPph.js → capture-iBzFiC0j.js} +27 -25
  25. package/dist/capture-iBzFiC0j.js.map +1 -0
  26. package/dist/cases-Dy0mifal.js +1 -0
  27. package/dist/cases-z9E6DCd5.js +639 -0
  28. package/dist/cases-z9E6DCd5.js.map +1 -0
  29. package/dist/{cli-BBOlXKPG.js → cli-20Fpvs5H.js} +442 -707
  30. package/dist/cli-20Fpvs5H.js.map +1 -0
  31. package/dist/{cli-spec-DZ0bnP0_.js → cli-spec-Dsr79Qy1.js} +38 -11
  32. package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
  33. package/dist/cli.js +2 -2
  34. package/dist/{client-tZ2WBDxf.js → client-KM4C5r5_.js} +93 -3
  35. package/dist/client-KM4C5r5_.js.map +1 -0
  36. package/dist/{command-qGS802uX.js → command-B5OyHpOu.js} +21 -80
  37. package/dist/command-B5OyHpOu.js.map +1 -0
  38. package/dist/{command-DhiwsZl1.js → command-BLNVDiIg.js} +9 -9
  39. package/dist/{command-DhiwsZl1.js.map → command-BLNVDiIg.js.map} +1 -1
  40. package/dist/{command-fO0BFsTq.js → command-CRh-cb6Z.js} +37 -37
  41. package/dist/command-CRh-cb6Z.js.map +1 -0
  42. package/dist/{command-CFQ9c1hk.js → command-aUJvENwy.js} +16 -16
  43. package/dist/command-aUJvENwy.js.map +1 -0
  44. package/dist/{commands-CC60TOT5.js → commands-CWy-WndG.js} +186 -85
  45. package/dist/commands-CWy-WndG.js.map +1 -0
  46. package/dist/{commands-xYTMNTPv.js → commands-CfdBX-mc.js} +1 -1
  47. package/dist/{config-C2UtS9vy.js → config-B9mB1kF5.js} +261 -1708
  48. package/dist/config-B9mB1kF5.js.map +1 -0
  49. package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
  50. package/dist/context-Bkv3ndjq.js.map +1 -0
  51. package/dist/{cost-BuVDamui.js → cost-BdRgySQl.js} +2 -2
  52. package/dist/{cost-BuVDamui.js.map → cost-BdRgySQl.js.map} +1 -1
  53. package/dist/cost-Cr26Alhq.js +1 -0
  54. package/dist/{cursor-catalog-BTK_apL-.js → cursor-catalog-BSn9xjmo.js} +3 -3
  55. package/dist/{cursor-catalog-BTK_apL-.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
  56. package/dist/db-P2fXw4nx.js +2 -0
  57. package/dist/{db-C5SZ0510.js → db-zgC1tvKB.js} +130 -84
  58. package/dist/db-zgC1tvKB.js.map +1 -0
  59. package/dist/debug-CF4p4Lp_.js +1 -0
  60. package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
  61. package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
  62. package/dist/{default-effort-FEidNAIi.js → default-effort-BAy8ixd1.js} +1 -1
  63. package/dist/{default-effort-BVmx7r7L.js → default-effort-DGhQmu95.js} +4 -4
  64. package/dist/{default-effort-BVmx7r7L.js.map → default-effort-DGhQmu95.js.map} +1 -1
  65. package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
  66. package/dist/dist-CAFD1yfv.js.map +1 -0
  67. package/dist/{docker-B2F8-PGC.js → docker-Cqw-CyWz.js} +3 -3
  68. package/dist/{docker-B2F8-PGC.js.map → docker-Cqw-CyWz.js.map} +1 -1
  69. package/dist/{docker-Dto8ROCq.js → docker-DAMnm3L9.js} +103 -73
  70. package/dist/docker-DAMnm3L9.js.map +1 -0
  71. package/dist/{docker-BlSopHpZ.js → docker-Dr4XeDc8.js} +1 -1
  72. package/dist/{doctor-BcvfKpBX.js → doctor-B2PohneB.js} +84 -27
  73. package/dist/doctor-B2PohneB.js.map +1 -0
  74. package/dist/{environment-CC1RV21n.js → environment-DB8o3hGi.js} +79 -39
  75. package/dist/environment-DB8o3hGi.js.map +1 -0
  76. package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
  77. package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
  78. package/dist/{eval-source-CzxdgFBW.js → eval-source-D3sbAzt-.js} +10 -10
  79. package/dist/{eval-source-CzxdgFBW.js.map → eval-source-D3sbAzt-.js.map} +1 -1
  80. package/dist/{evaluate-DToSB2bt.d.ts → evaluate-COdzf7Hl.d.ts} +17 -26
  81. package/dist/{evaluate-NTNJ8T9H.js → evaluate-CwKeN_fo.js} +377 -142
  82. package/dist/evaluate-CwKeN_fo.js.map +1 -0
  83. package/dist/files-B4KQUiM7.js +1 -0
  84. package/dist/files-BEwfjD8J.js +64 -0
  85. package/dist/files-BEwfjD8J.js.map +1 -0
  86. package/dist/{files-DsXULUi5.js → files-BRzV7aPh.js} +5 -5
  87. package/dist/{files-DsXULUi5.js.map → files-BRzV7aPh.js.map} +1 -1
  88. package/dist/{files-aAzoGNCB.js → files-BiFU16BW.js} +5 -5
  89. package/dist/{files-aAzoGNCB.js.map → files-BiFU16BW.js.map} +1 -1
  90. package/dist/files-C1Rk9FYA.js +1 -0
  91. package/dist/files-C3mCC8Mn.js +1 -0
  92. package/dist/{files-CZXrkPja.js → files-Cl7BQ5gE.js} +54 -13
  93. package/dist/{files-CZXrkPja.js.map → files-Cl7BQ5gE.js.map} +1 -1
  94. package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
  95. package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
  96. package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
  97. package/dist/files-DlQshsks.js.map +1 -0
  98. package/dist/files-DnlA1A7N.js +1 -0
  99. package/dist/gate-CgYdaprG.js +49 -0
  100. package/dist/gate-CgYdaprG.js.map +1 -0
  101. package/dist/gateway.d.ts +1 -1
  102. package/dist/{git-C1MB5bjr.js → git-fDiJ8UIV.js} +2 -2
  103. package/dist/{git-C1MB5bjr.js.map → git-fDiJ8UIV.js.map} +1 -1
  104. package/dist/{harness-DGsRZDSo.js → harness-C2berMiM.js} +3 -3
  105. package/dist/{harness-DGsRZDSo.js.map → harness-C2berMiM.js.map} +1 -1
  106. package/dist/{harness-DYSC4zMh.js → harness-DXv2gnaT.js} +4 -2
  107. package/dist/harness-DXv2gnaT.js.map +1 -0
  108. package/dist/harness.d.ts +1 -1
  109. package/dist/harness.js +1 -1
  110. package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
  111. package/dist/ids-C_advdJA.js.map +1 -0
  112. package/dist/{images-CNayXCp2.js → images-BNgcR77K.js} +98 -10
  113. package/dist/images-BNgcR77K.js.map +1 -0
  114. package/dist/index.d.ts +23 -3
  115. package/dist/index.js +5 -5
  116. package/dist/init-4RPeD-K-.js +1 -0
  117. package/dist/{init-D5HtmyMI.js → init-DbuZVRrW.js} +7 -6
  118. package/dist/init-DbuZVRrW.js.map +1 -0
  119. package/dist/{ink-list-Ca-5dunO.js → ink-list-BpfidQ1h.js} +121 -31
  120. package/dist/ink-list-BpfidQ1h.js.map +1 -0
  121. package/dist/{invocations-DrGXF9dT.js → invocations-CxGNOlh0.js} +3 -3
  122. package/dist/{invocations-DrGXF9dT.js.map → invocations-CxGNOlh0.js.map} +1 -1
  123. package/dist/judge-COMXT1Hx.js +230 -0
  124. package/dist/judge-COMXT1Hx.js.map +1 -0
  125. package/dist/judge.d.ts +1 -1
  126. package/dist/judge.js +1 -1
  127. package/dist/{judgments-m-mb0zv8.js → judgments-CzgpsTqv.js} +2 -2
  128. package/dist/{judgments-m-mb0zv8.js.map → judgments-CzgpsTqv.js.map} +1 -1
  129. package/dist/kinds-C1oY3qyC.js +697 -0
  130. package/dist/kinds-C1oY3qyC.js.map +1 -0
  131. package/dist/{summary-Uzc9osfV.js → layout-Cm85tQYk.js} +39 -150
  132. package/dist/layout-Cm85tQYk.js.map +1 -0
  133. package/dist/list-DuThsE28.js +1348 -0
  134. package/dist/list-DuThsE28.js.map +1 -0
  135. package/dist/live-CJCK11nh.js +1 -0
  136. package/dist/{live-B71o2TkX.js → live-CgYbO4Te.js} +6 -4
  137. package/dist/{live-B71o2TkX.js.map → live-CgYbO4Te.js.map} +1 -1
  138. package/dist/{live-uRURFDMZ.js → live-CnGrRM6j.js} +7 -7
  139. package/dist/{live-uRURFDMZ.js.map → live-CnGrRM6j.js.map} +1 -1
  140. package/dist/{matchers-Cs8aDqdU.js → matchers-CXLn-1jm.js} +3 -2
  141. package/dist/{matchers-Cs8aDqdU.js.map → matchers-CXLn-1jm.js.map} +1 -1
  142. package/dist/{model-CXVVB4RK.js → model-047Dcd-a.js} +1 -1
  143. package/dist/model-Cv4GuTsZ.js +1 -0
  144. package/dist/{model-1L19Zukh.js → model-CwwkrCb7.js} +5 -5
  145. package/dist/{model-1L19Zukh.js.map → model-CwwkrCb7.js.map} +1 -1
  146. package/dist/{model-BxBWFOZn.js → model-DJjLw-0r.js} +5 -5
  147. package/dist/{model-BxBWFOZn.js.map → model-DJjLw-0r.js.map} +1 -1
  148. package/dist/{model-Ce9l9_rF.js → model-V3mMXp6x.js} +203 -313
  149. package/dist/model-V3mMXp6x.js.map +1 -0
  150. package/dist/{node-DhIRSrrK.js → node-D7pAU--0.js} +14 -10
  151. package/dist/node-D7pAU--0.js.map +1 -0
  152. package/dist/{opencode-providers-Co7plCea.js → opencode-providers-BGiXt8_B.js} +104 -35
  153. package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
  154. package/dist/{openrouter-models-DNvLRYM7.js → openrouter-models-Cw1z8-sq.js} +2 -2
  155. package/dist/{openrouter-models-DNvLRYM7.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
  156. package/dist/options-CHubslQf.js +1 -0
  157. package/dist/{gate-Cp_pQoEe.js → pairs-BGb134Vu.js} +2 -48
  158. package/dist/pairs-BGb134Vu.js.map +1 -0
  159. package/dist/{patch-ug5HpMOj.js → patch-DH4e_Gkd.js} +2 -2
  160. package/dist/{patch-ug5HpMOj.js.map → patch-DH4e_Gkd.js.map} +1 -1
  161. package/dist/plain-BYUf_fUV.js +267 -0
  162. package/dist/plain-BYUf_fUV.js.map +1 -0
  163. package/dist/{plan-DqnzfDH1.js → plan-Cgq9NCEX.js} +110 -39
  164. package/dist/plan-Cgq9NCEX.js.map +1 -0
  165. package/dist/{plugins-B2KQ0y3P.js → plugins-BkmPxDiY.js} +10 -267
  166. package/dist/plugins-BkmPxDiY.js.map +1 -0
  167. package/dist/preflight-4gWSy0Pe.js +1 -0
  168. package/dist/{preflight-DyzmvK7i.js → preflight-BElE_q0u.js} +16 -9
  169. package/dist/preflight-BElE_q0u.js.map +1 -0
  170. package/dist/preflight-CtlloeOQ.js +68 -0
  171. package/dist/preflight-CtlloeOQ.js.map +1 -0
  172. package/dist/{preflight-DVXEvzFU.js → preflight-R-InZCQb.js} +1 -1
  173. package/dist/{prepare-21WMQZyt.js → prepare-C7f5mPl5.js} +9 -9
  174. package/dist/{prepare-21WMQZyt.js.map → prepare-C7f5mPl5.js.map} +1 -1
  175. package/dist/{prepare-BcSRa7-3.js → prepare-CzkAdbXm.js} +2 -2
  176. package/dist/{prices-DYJA0tJN.js → prices-5m3SzMZ3.js} +2 -2
  177. package/dist/{prices-DYJA0tJN.js.map → prices-5m3SzMZ3.js.map} +1 -1
  178. package/dist/{provenance-iRxjp4Pu.js → provenance-zsc4--dP.js} +4 -3
  179. package/dist/provenance-zsc4--dP.js.map +1 -0
  180. package/dist/provider-BnUoz4KB.js +1 -0
  181. package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
  182. package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
  183. package/dist/{providers-vXMXuwn-.js → providers-B8xYMeah.js} +152 -9
  184. package/dist/providers-B8xYMeah.js.map +1 -0
  185. package/dist/providers-CVu8e0xS.js +1 -0
  186. package/dist/{query-BhaKQ07P.js → query-Bb3SYICk.js} +2 -2
  187. package/dist/{query-BhaKQ07P.js.map → query-Bb3SYICk.js.map} +1 -1
  188. package/dist/query-DIfkUcPh.js +1 -0
  189. package/dist/{questions-BzdLdV3C.js → questions-CWlR9ffu.js} +9 -9
  190. package/dist/{questions-BzdLdV3C.js.map → questions-CWlR9ffu.js.map} +1 -1
  191. package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
  192. package/dist/record-6IUzwP9K.js.map +1 -0
  193. package/dist/{remarks-NMgPd30U.js → remarks-DPCONFxZ.js} +71 -50
  194. package/dist/remarks-DPCONFxZ.js.map +1 -0
  195. package/dist/{remarks-CauU408t.js → remarks-vtkPmQb9.js} +115 -35
  196. package/dist/remarks-vtkPmQb9.js.map +1 -0
  197. package/dist/{remote-Dwy3fyo3.js → remote-BgYWqogI.js} +6 -6
  198. package/dist/{remote-Dwy3fyo3.js.map → remote-BgYWqogI.js.map} +1 -1
  199. package/dist/{resolve-B1zMnPwK.js → resolve-Do0ZaYH6.js} +10 -9
  200. package/dist/resolve-Do0ZaYH6.js.map +1 -0
  201. package/dist/resolve-amw_2Odz.js +1 -0
  202. package/dist/{results-CfC0id_u.js → results-JyuM64gy.js} +6 -6
  203. package/dist/{results-CfC0id_u.js.map → results-JyuM64gy.js.map} +1 -1
  204. package/dist/{resume-Dkhrpjow.js → resume-AW5oXzUc.js} +169 -27
  205. package/dist/resume-AW5oXzUc.js.map +1 -0
  206. package/dist/retry-ChzISxmy.js +105 -0
  207. package/dist/retry-ChzISxmy.js.map +1 -0
  208. package/dist/{runtime-DcvQcr36.js → runtime-BpW1f1Mx.js} +26 -7
  209. package/dist/runtime-BpW1f1Mx.js.map +1 -0
  210. package/dist/sandbox/boat.d.ts +26 -0
  211. package/dist/sandbox/boat.js +37 -0
  212. package/dist/sandbox/boat.js.map +1 -0
  213. package/dist/{sandbox-8zCrtuCf.js → sandbox-kSgr7vo5.js} +31 -3
  214. package/dist/sandbox-kSgr7vo5.js.map +1 -0
  215. package/dist/sandbox.d.ts +20 -2
  216. package/dist/sandbox.js +1 -1
  217. package/dist/server-2_gUnVvf.js +1 -0
  218. package/dist/{server-Dplq8g0r.js → server-Cn8VqO9S.js} +9 -9
  219. package/dist/{server-Dplq8g0r.js.map → server-Cn8VqO9S.js.map} +1 -1
  220. package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
  221. package/dist/shared-Cfi45Bvy.js.map +1 -0
  222. package/dist/slug-IHh3f770.js +136 -0
  223. package/dist/slug-IHh3f770.js.map +1 -0
  224. package/dist/source-CY3etKzw.js +1 -0
  225. package/dist/{source-NWHqn5-o.js → source-CYOhR98B.js} +2 -2
  226. package/dist/{source-NWHqn5-o.js.map → source-CYOhR98B.js.map} +1 -1
  227. package/dist/src-6Zzyg4Qd.js +2 -0
  228. package/dist/{src-B9oE_mNS.js → src-BDhcPtnI.js} +2 -2
  229. package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
  230. package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
  231. package/dist/src-BRMsk9nh.js +2 -0
  232. package/dist/{src-BQXdf-Gw.js → src-CV_7DkPa.js} +47 -45
  233. package/dist/{src-BQXdf-Gw.js.map → src-CV_7DkPa.js.map} +1 -1
  234. package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
  235. package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
  236. package/dist/{src-BQ6BiXoc.js → src-Ck2r_EMT.js} +607 -72
  237. package/dist/src-Ck2r_EMT.js.map +1 -0
  238. package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
  239. package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
  240. package/dist/src-DB7N8TYJ.js +2 -0
  241. package/dist/src-DPDaAYwh.js +2 -0
  242. package/dist/{src-CUpoVVb8.js → src-Dbk19BiP.js} +2 -2
  243. package/dist/src-DtodMT59.js +1 -0
  244. package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
  245. package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
  246. package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
  247. package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
  248. package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
  249. package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
  250. package/dist/src-T-OeuJLr.js +2 -0
  251. package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
  252. package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
  253. package/dist/src-zntEDlnx.js +3 -0
  254. package/dist/{state-DDCGUhva.js → state-BOLt8pJg.js} +17 -17
  255. package/dist/{state-DDCGUhva.js.map → state-BOLt8pJg.js.map} +1 -1
  256. package/dist/state-kimtIhBP.js +1 -0
  257. package/dist/suite-CKBa6MKt.js +2 -0
  258. package/dist/{suite-M8_7xio7.js → suite-COglUhPd.js} +70 -92
  259. package/dist/suite-COglUhPd.js.map +1 -0
  260. package/dist/summary-DU5zaibx.js +148 -0
  261. package/dist/summary-DU5zaibx.js.map +1 -0
  262. package/dist/tart-BTVXp8rz.js +373 -0
  263. package/dist/tart-BTVXp8rz.js.map +1 -0
  264. package/dist/tart-cli-BIC9HqPg.js +37 -0
  265. package/dist/tart-cli-BIC9HqPg.js.map +1 -0
  266. package/dist/tart-cli-BMrkmBjI.js +1 -0
  267. package/dist/tart-images-DZ_Mbh-g.js +233 -0
  268. package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
  269. package/dist/tart-ref-7Qv3cm3B.js +1 -0
  270. package/dist/tart-ref-DYZijjrU.js +55 -0
  271. package/dist/tart-ref-DYZijjrU.js.map +1 -0
  272. package/dist/testing.d.ts +2 -2
  273. package/dist/testing.js +4 -4
  274. package/dist/trace-Dx2NBE1O.js +311 -0
  275. package/dist/trace-Dx2NBE1O.js.map +1 -0
  276. package/dist/traits-DKnYkqeK.js +66 -0
  277. package/dist/traits-DKnYkqeK.js.map +1 -0
  278. package/dist/{transcript-Cg4SeqRX.js → transcript-CCf07UmU.js} +2 -2
  279. package/dist/{transcript-Cg4SeqRX.js.map → transcript-CCf07UmU.js.map} +1 -1
  280. package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
  281. package/dist/types-C2mLpph3.js.map +1 -0
  282. package/dist/{types-DGvexkcp.d.ts → types-DBwFsGem.d.ts} +352 -76
  283. package/dist/{judge-DYOF69Oe.js → vercel-CGlSBv63.js} +3 -229
  284. package/dist/vercel-CGlSBv63.js.map +1 -0
  285. package/dist/{vertex-BLQMayc1.js → vertex-Eb2wqahk.js} +2 -2
  286. package/dist/{vertex-BLQMayc1.js.map → vertex-Eb2wqahk.js.map} +1 -1
  287. package/dist/vertex-O88TCwgY.js +1 -0
  288. package/dist/view/index.d.ts +101 -58
  289. package/dist/view/index.js +9 -8
  290. package/dist/{views-BISrsIjw.js → views-DAIiLiV3.js} +7 -6
  291. package/dist/views-DAIiLiV3.js.map +1 -0
  292. package/dist/views-DtRhj2Tn.js +2 -0
  293. package/dist/{worker-entry-DJD_nHkS.js → worker-entry-BIROTPX1.js} +11 -8
  294. package/dist/{worker-entry-DJD_nHkS.js.map → worker-entry-BIROTPX1.js.map} +1 -1
  295. package/dist/workspace-CcQRT31C.js +1 -0
  296. package/dist/{workspace-BO1VhUR9.js → workspace-uuE51x35.js} +6 -6
  297. package/dist/{workspace-BO1VhUR9.js.map → workspace-uuE51x35.js.map} +1 -1
  298. package/package.json +24 -11
  299. package/skills/eval-rubric/SKILL.md +15 -7
  300. package/dist/adapter-CptK032x.js.map +0 -1
  301. package/dist/app-files-sgqwfjtn.js.map +0 -1
  302. package/dist/assemble-CxBHkavM.js +0 -1
  303. package/dist/assemble-Me7Whp-y.js.map +0 -1
  304. package/dist/capture-C3RmjPph.js.map +0 -1
  305. package/dist/capture-YPwxp_iK.js +0 -1
  306. package/dist/cli-BBOlXKPG.js.map +0 -1
  307. package/dist/cli-spec-DZ0bnP0_.js.map +0 -1
  308. package/dist/client-tZ2WBDxf.js.map +0 -1
  309. package/dist/command-CFQ9c1hk.js.map +0 -1
  310. package/dist/command-fO0BFsTq.js.map +0 -1
  311. package/dist/command-qGS802uX.js.map +0 -1
  312. package/dist/commands-CC60TOT5.js.map +0 -1
  313. package/dist/config-C2UtS9vy.js.map +0 -1
  314. package/dist/context-BajfiH0S.js.map +0 -1
  315. package/dist/contrasts-BjOg7wyr.js +0 -101
  316. package/dist/contrasts-BjOg7wyr.js.map +0 -1
  317. package/dist/cost-BJU0MAji.js +0 -1
  318. package/dist/db-C5SZ0510.js.map +0 -1
  319. package/dist/db-puzLzX4k.js +0 -2
  320. package/dist/debug-BnXV7JbX.js +0 -1
  321. package/dist/dist-ByYk_wAs.js.map +0 -1
  322. package/dist/docker-Dto8ROCq.js.map +0 -1
  323. package/dist/doctor-BcvfKpBX.js.map +0 -1
  324. package/dist/environment-CC1RV21n.js.map +0 -1
  325. package/dist/evaluate-NTNJ8T9H.js.map +0 -1
  326. package/dist/files-COW5JBJM.js +0 -1
  327. package/dist/files-DFtuRQJx.js +0 -1
  328. package/dist/files-oOYNR7ZH.js +0 -1
  329. package/dist/files-p-uTdOjA.js.map +0 -1
  330. package/dist/gate-Cp_pQoEe.js.map +0 -1
  331. package/dist/harness-DYSC4zMh.js.map +0 -1
  332. package/dist/ids-DuK6xl4w.js.map +0 -1
  333. package/dist/images-CNayXCp2.js.map +0 -1
  334. package/dist/init-8o6W3_fn.js +0 -1
  335. package/dist/init-D5HtmyMI.js.map +0 -1
  336. package/dist/ink-list-Ca-5dunO.js.map +0 -1
  337. package/dist/judge-DYOF69Oe.js.map +0 -1
  338. package/dist/list-CM_YSJXq.js +0 -374
  339. package/dist/list-CM_YSJXq.js.map +0 -1
  340. package/dist/live-DQc8jr6o.js +0 -1
  341. package/dist/model-Ce9l9_rF.js.map +0 -1
  342. package/dist/model-Cxzo4cjw.js +0 -1
  343. package/dist/node-DhIRSrrK.js.map +0 -1
  344. package/dist/opencode-providers-Co7plCea.js.map +0 -1
  345. package/dist/plan-DqnzfDH1.js.map +0 -1
  346. package/dist/plugins-B2KQ0y3P.js.map +0 -1
  347. package/dist/preflight-DyzmvK7i.js.map +0 -1
  348. package/dist/provenance-iRxjp4Pu.js.map +0 -1
  349. package/dist/provider-DCdTHQpk.js +0 -1
  350. package/dist/providers-D0ob_tra.js +0 -1
  351. package/dist/providers-vXMXuwn-.js.map +0 -1
  352. package/dist/query-CvgUsvep.js +0 -1
  353. package/dist/record-DhzwkSXc.js.map +0 -1
  354. package/dist/remarks-CauU408t.js.map +0 -1
  355. package/dist/remarks-NMgPd30U.js.map +0 -1
  356. package/dist/resolve-B1zMnPwK.js.map +0 -1
  357. package/dist/resolve-BIIeVT12.js +0 -1
  358. package/dist/resume-Dkhrpjow.js.map +0 -1
  359. package/dist/runtime-DcvQcr36.js.map +0 -1
  360. package/dist/sandbox-8zCrtuCf.js.map +0 -1
  361. package/dist/server-0tuj8VSa.js +0 -1
  362. package/dist/shared-ogqPJ0uo.js.map +0 -1
  363. package/dist/source-llwLewOU.js +0 -1
  364. package/dist/src-BQ6BiXoc.js.map +0 -1
  365. package/dist/src-BXnv0ViU.js +0 -2
  366. package/dist/src-BiS8YNN_.js +0 -3
  367. package/dist/src-BkH4GOxu.js +0 -1
  368. package/dist/src-C-tH5ebd.js +0 -2
  369. package/dist/src-Cg1bFs9g.js +0 -2
  370. package/dist/src-DV_UorZv.js +0 -2
  371. package/dist/src-XxUpr2sa.js +0 -2
  372. package/dist/state-DZbAO95a.js +0 -1
  373. package/dist/suite-M8_7xio7.js.map +0 -1
  374. package/dist/suite-u2KJnGpY.js +0 -2
  375. package/dist/summary-Uzc9osfV.js.map +0 -1
  376. package/dist/types-C4OkYhZg.js.map +0 -1
  377. package/dist/vertex-h_peaZTN.js +0 -1
  378. package/dist/views-BISrsIjw.js.map +0 -1
  379. package/dist/views-Bfab8fM9.js +0 -2
  380. package/dist/workspace-B8EOCuxl.js +0 -1
@@ -1,38 +1,34 @@
1
1
  import { n as UsageError } from "./errors-ttMTnIYX.js";
2
- import { h as parseDuration } from "./shared-ogqPJ0uo.js";
3
- import { D as profileSandbox, P as workspaceLabel, p as environmentSandbox } from "./environment-CC1RV21n.js";
4
- import { C as pluginBuildsLabel } from "./plugins-B2KQ0y3P.js";
5
- import { a as fromCwd, o as newRunId } from "./ids-DuK6xl4w.js";
6
- import { $ as directOnCloud, M as harnessId, O as checkLimitHarness, Q as ciRunner, R as plannedSandboxIds, U as sandboxId, a as configPathFor, b as resolveModelGateway, c as findConfig, d as malformedCredentials, f as missingCredentials, g as selectProfile, h as runJudgeModel, i as configLabel, it as resolveOutput, l as hostCapacity, m as resolveConcurrency, nt as isDirect, p as requireConfig, u as loadConfig, v as gatewayScope, z as plannedSandboxes } from "./config-C2UtS9vy.js";
7
- import { l as judgeModelName, m as unusedGatewayKey } from "./judge-DYOF69Oe.js";
8
- import { y as vertexSettingsIn } from "./vertex-BLQMayc1.js";
9
- import { h as redactSecrets, v as registerSecret } from "./redact-BhjtnjrU.js";
2
+ import { g as parseDuration } from "./shared-Cfi45Bvy.js";
10
3
  import { a as plural, i as formatDuration, n as clip } from "./text-B8-W3Cvn.js";
4
+ import { l as sandboxCapacity, n as capabilitiesOf, r as cleanupRun } from "./kinds-C1oY3qyC.js";
5
+ import { N as workspaceLabel, T as profileSandbox, d as environmentSandbox } from "./environment-DB8o3hGi.js";
6
+ import { a as newRunId, i as fromCwd } from "./ids-C_advdJA.js";
7
+ import { I as checkLimitHarness, V as harnessId, X as plannedSandboxes, Y as plannedSandboxIds, _t as resolveOutput, a as configPathFor, b as resolveModelGateway, c as findConfig, d as malformedCredentials, dt as ciRunner, f as missingCredentials, ft as directOnCloud, g as selectProfile, h as runJudgeModel, ht as isDirect, i as configLabel, l as hostCapacity, m as resolveConcurrency, p as requireConfig, u as loadConfig, v as gatewayScope } from "./config-B9mB1kF5.js";
8
+ import { y as vertexSettingsIn } from "./vertex-Eb2wqahk.js";
9
+ import { h as redactSecrets, v as registerSecret } from "./redact-BhjtnjrU.js";
10
+ import { l as judgeModelName, m as unusedGatewayKey } from "./judge-COMXT1Hx.js";
11
11
  import { n as loadEnvFiles, t as inCI } from "./env-8x1O8I1M.js";
12
12
  import { o as PACKAGE_VERSION } from "./root-Co0jhOT5.js";
13
- import { W as LABEL$1, st as imageInfo } from "./src-BQ6BiXoc.js";
14
- import { r as openRouterFallbacks } from "./openrouter-models-DNvLRYM7.js";
15
- import { t as PrepareError } from "./provider-DZd8t_XW.js";
16
- import { _ as formatUsage, k as runOutcome } from "./model-Ce9l9_rF.js";
17
- import { h as measuresLine, i as effectOf, r as countedOf, s as isNotGatedResult, u as suiteResult, v as observedLine } from "./suite-M8_7xio7.js";
18
- import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-DZ0bnP0_.js";
13
+ import { r as openRouterFallbacks } from "./openrouter-models-Cw1z8-sq.js";
14
+ import { a as traitFilters, o as traitFlags, t as checkTraitKeys } from "./traits-DKnYkqeK.js";
15
+ import { k as runOutcome } from "./model-V3mMXp6x.js";
16
+ import { u as suiteResult } from "./suite-COglUhPd.js";
17
+ import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-Dsr79Qy1.js";
19
18
  import { n as stripAnsi, t as colors } from "./colors-DcXBvAlU.js";
20
- import { i as shownRemarks } from "./remarks-D9Lcd1fF.js";
21
- import { _ as statsLine, c as checkLabel } from "./summary-Uzc9osfV.js";
22
- import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-iRxjp4Pu.js";
19
+ import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-zsc4--dP.js";
23
20
  import { t as budgetMarker } from "./budget-BcxLYY5L.js";
24
- import { a as liveAttempts, d as savedFilters, f as savedInvocation, i as leftOutAttempts, l as resolveResume, m as writeLiveProgress, o as loadPrevious, p as writeCurrent, r as keptAttempts, s as prepareResume, t as completedKeysFor, u as resumeSettings } from "./resume-Dkhrpjow.js";
25
- import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-DyzmvK7i.js";
26
- import { n as imageHarness } from "./prepare-21WMQZyt.js";
27
- import { i as createProviders, n as attemptImage } from "./providers-vXMXuwn-.js";
28
- import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-DrGXF9dT.js";
29
- import { c as attemptAxes, d as failureLines, f as offset, g as tailLines, h as progressLine, l as debugLines, m as phaseOf, p as outcomeOf, r as listLines, s as progressText, t as LiveList, u as errorCheck } from "./list-CM_YSJXq.js";
21
+ import { S as writeLiveProgress, _ as resumeSettings, b as stoppedRetry, c as liveAttempts, d as prepareResume, f as previousAttempt, g as restoreRetry, h as resolveRetry, l as loadPrevious, m as resolveResume, n as erroredAttempts, o as keptAttempts, r as holdRetry, s as leftOutAttempts, t as completedKeysFor, u as noteRetries, v as savedFilters, x as writeCurrent, y as savedInvocation } from "./resume-AW5oXzUc.js";
22
+ import { n as preflight } from "./preflight-CtlloeOQ.js";
23
+ import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-CxGNOlh0.js";
24
+ import { _ as phaseOf, c as pointerLine, d as attemptAxes, f as debugLines, g as outcomeOf, h as offset, i as listLines, m as failureLines, n as SPINNER, p as errorCheck, s as progressText, t as LiveList, u as statusList, v as progressLine, y as tailLines } from "./list-DuThsE28.js";
30
25
  import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
31
26
  import { dirname, join } from "node:path";
32
27
  import { inspect } from "node:util";
33
28
  import { appendFile, readFile, rm } from "node:fs/promises";
34
29
  import { detectAgent } from "std-env";
35
30
  import { Command, Option } from "commander";
31
+ import stringWidth from "string-width";
36
32
  // src/cli-parse.ts
37
33
  const OPTIONS = Object.fromEntries(Object.entries(FLAGS).map(([name, spec]) => {
38
34
  const short = "short" in spec ? `-${spec.short}, ` : "";
@@ -163,62 +159,6 @@ function usageError(error, command) {
163
159
  }
164
160
  return new UsageError(first.charAt(0).toUpperCase() + first.slice(1));
165
161
  }
166
- // src/preflight.ts
167
- /**
168
- * How the terminal names a sandbox to prepare: its cell id, `+ install`, `+ plugin builds`, and a
169
- * harness other than Cursor.
170
- */
171
- function prepareLabel({ sandbox, harness, plugins }) {
172
- if (sandbox.kind !== "docker") return sandboxId(sandbox);
173
- const baked = imageHarness(harness);
174
- const install = sandbox.install?.length ? " + install" : "";
175
- return `${sandboxId(sandbox)}${install}${pluginBuildsLabel(plugins)}${baked === "cursor" ? "" : ` · ${baked}`}`;
176
- }
177
- /** A run that stopped in preflight on an infra failure: exit 3. */
178
- var PreflightInfraError = class extends Error {
179
- name = "PreflightInfraError";
180
- };
181
- /**
182
- * Before any eval starts: checks each sandbox provider, then prepares every distinct sandbox the
183
- * planned attempts need (building Docker images), in parallel, and writes the results for the
184
- * run's workers. A config-level sandbox that fails stops the run (exit 2 for a mistake in the
185
- * config, exit 3 otherwise); any other failure fails only the attempts that need it.
186
- */
187
- async function preflight(uses, options) {
188
- const providers = options.providers ?? createProviders();
189
- const targets = prepareTargets(uses, providers, prepareLabel);
190
- if (!targets.some((t) => t.kind !== "fake")) return { images: [] };
191
- let outcomes;
192
- try {
193
- outcomes = await prepareAll(targets, {
194
- providers,
195
- root: options.root,
196
- ...options.signal ? { signal: options.signal } : {},
197
- ...options.delays ? { delays: options.delays } : {},
198
- ...options.onEvent ? { onEvent: options.onEvent } : {},
199
- log: (line) => process.env.REEVAL_DEBUG && console.error(line)
200
- });
201
- } catch (error) {
202
- if (error instanceof PrepareError) throw error.failure.usage ? new UsageError(error.message) : new PreflightInfraError(error.message);
203
- if (error instanceof RunStopError) throw error.outcome.failure.usage ? new UsageError(error.message) : new PreflightInfraError(error.message);
204
- throw error;
205
- }
206
- writePrepared(options.root, options.runId, outcomes);
207
- const images = [];
208
- for (const o of outcomes) {
209
- if (!o.ok || o.target.sandbox.kind !== "docker") continue;
210
- const { ref, id, agentCli } = attemptImage(o.handle);
211
- if (images.some((i) => i.id === id)) continue;
212
- const info = await imageInfo(ref, o.target.sandbox.engine);
213
- images.push({
214
- ref,
215
- id,
216
- repoDigests: info?.repoDigests ?? [],
217
- agentCli: agentCli ?? info?.labels[LABEL$1.agentVersion] ?? null
218
- });
219
- }
220
- return { images };
221
- }
222
162
  // src/ui/header.ts
223
163
  /** ` ➜ `, ` ⚠ `: the glyph column every line of the block starts with. */
224
164
  const GLYPH = 5;
@@ -254,7 +194,8 @@ function headerLines(c, h, width) {
254
194
  return (values.length > 1 && stripAnsi(joined).length > valueWidth ? values : wrap(joined, valueWidth)).map((v, i) => i === 0 ? ` ${c.teal("➜")} ${c.dim(`${key}:`)}${indent(KEY - key.length - 1)}${v}` : `${indent(16)}${v}`);
255
195
  };
256
196
  const many = (one, n, many = `${one}s`) => n === 1 ? one : many;
257
- const resumed = h.resumed ? c.dim(` (resumed, ${plural(h.resumed, "attempt")} done)`) : "";
197
+ const continued = [h.resumed ? `${plural(h.resumed, "attempt")} done` : "", h.retried ? `${plural(h.retried, "error")} to retry` : ""].filter(Boolean);
198
+ const resumed = continued.length ? c.dim(` (resumed, ${continued.join(", ")})`) : "";
258
199
  const reason = h.profile && h.profile.reason !== "--profile" ? c.dim(` (${h.profile.reason})`) : "";
259
200
  return [
260
201
  h.dryRun ? `${title} ${c.dim("dry run")}` : title,
@@ -289,9 +230,11 @@ function scope(c, h) {
289
230
  }
290
231
  function estimate(c, h) {
291
232
  const serial = `~${formatDuration(h.estimateMs)} ${c.dim("(serial)")}`;
292
- const parallel = Math.min(h.concurrency, h.attempts);
233
+ const most = Math.min(h.concurrency, h.attempts);
234
+ const parallel = h.parallel ?? most;
293
235
  if (parallel <= 1) return serial;
294
- return `${`~${formatDuration(h.estimateMs / parallel)} ${c.dim(`(concurrency: ${h.concurrency})`)}`} · ${serial}`;
236
+ const capped = parallel < most ? `, ${parallel} at once` : "";
237
+ return `${`~${formatDuration(h.estimateMs / parallel)} ${c.dim(`(concurrency: ${h.concurrency}${capped})`)}`} · ${serial}`;
295
238
  }
296
239
  /** `⚠ Warning: …` or `ℹ Hint: …`, wrapped under its first word. */
297
240
  function noteLines(c, kind, text, width) {
@@ -342,15 +285,17 @@ function prepareStepLines(c, e, width, max) {
342
285
  const who = outcome.target.setBy.join(", ");
343
286
  return [` ${c.red("✗")} ${outcome.target.label} failed to prepare ${c.dim(`(set by ${who}; only its attempts fail)`)}`, ...tailLines(outcome.failure.message, max).map((l) => `${indent(GLYPH)}${c.orange(l)}`)];
344
287
  }
288
+ /** The start line's text: how the attempts' logs will print. */
289
+ function startText(output) {
290
+ return output.outputLogs === "none" ? "Running..." : output.outputLogs === "errors-only" ? "Running; logs print for failed attempts only..." : output.logOrder === "stream" ? "Streaming logs..." : "Running; each attempt's logs print as it ends...";
291
+ }
345
292
  /** The last line before the attempts' own: how their logs will print. */
346
293
  function startLine(c, output) {
347
- const text = output.outputLogs === "none" ? "Running..." : output.outputLogs === "errors-only" ? "Running; logs print for failed attempts only..." : output.logOrder === "stream" ? "Streaming logs..." : "Running; each attempt's logs print as it ends...";
348
- return ` ${c.teal("◆")} ${text}`;
294
+ return ` ${c.teal("◆")} ${startText(output)}`;
349
295
  }
350
- /** What the start line becomes once the run ends, on a terminal that redraws it. */
351
- function endLine(c, status) {
352
- const text = status === "interrupted" ? "Canceled." : status === "budget" ? "Stopped at the budget." : "Finished.";
353
- return ` ${c.teal("◆")} ${text}`;
296
+ /** What the start line's text becomes once the run ends, on a terminal that redraws it. */
297
+ function endText(status) {
298
+ return status === "interrupted" ? "Canceled." : status === "budget" ? "Stopped at the budget." : "Finished.";
354
299
  }
355
300
  /** The plan as the header block: its facts, `--dry-run`'s table, then each kind of note. */
356
301
  function planLines(c, e, width) {
@@ -364,509 +309,6 @@ function planLines(c, e, width) {
364
309
  out.push("");
365
310
  return out;
366
311
  }
367
- // src/ui/status.ts
368
- const EFFECT = {
369
- better: "helped",
370
- worse: "made it worse",
371
- same: "same"
372
- };
373
- const ERROR_VERB = {
374
- assertion: "failed",
375
- timeout: "timed out",
376
- aborted: "canceled",
377
- infra: "hit an infrastructure error",
378
- judge: "could not be judged",
379
- turn: "ended a turn with an error"
380
- };
381
- const MESSAGE_LINES = 4;
382
- const LINE_MAX = 160;
383
- const times = (n) => n === 1 ? "once" : `${n} times`;
384
- /** `cursor (composer-2.5)`, from the record's cell. */
385
- function harnessLabel(record, cell) {
386
- const model = record.evals.flatMap((e) => e.cells).find((c) => c.key === cell.key && c.harness === cell.harness)?.model;
387
- return model && model !== cell.harness ? `${cell.harness} (${model})` : cell.harness;
388
- }
389
- /** The cells an eval's outcome reads: the gated ones, or every cell of a study. */
390
- function deciding(cells) {
391
- const gated = cells.filter((c) => c.gated);
392
- return gated.length ? gated : cells;
393
- }
394
- const cellsOf = (e) => e.cells ?? (e.cases ?? []).flatMap((k) => k.cells);
395
- /** Each part's label, adding the environment and sandbox only where the cells differ in them. */
396
- function labeler(record, cells) {
397
- const envs = new Set(cells.map((c) => c.environment)).size > 1;
398
- const sandboxes = new Set(cells.map((c) => c.sandbox)).size > 1;
399
- return (c) => [
400
- harnessLabel(record, c),
401
- envs ? `in ${c.environment}` : "",
402
- sandboxes ? `on ${c.sandbox}` : ""
403
- ].filter(Boolean).join(" ");
404
- }
405
- function tallyBy(cells, label) {
406
- const by = /* @__PURE__ */ new Map();
407
- for (const c of cells) {
408
- const key = label(c);
409
- const t = by.get(key) ?? {
410
- label: key,
411
- passed: 0,
412
- counted: 0
413
- };
414
- t.passed += c.passed;
415
- t.counted += countedOf(c);
416
- by.set(key, t);
417
- }
418
- return [...by.values()];
419
- }
420
- const of = (t) => `${t.passed} of ${t.counted}`;
421
- /** The outcome sentence, per the rules at the top. */
422
- function outcomeSentence(tallies) {
423
- const counted = tallies.filter((t) => t.counted > 0);
424
- if (!counted.length) return "No run finished.";
425
- const [one] = counted;
426
- if (counted.length === 1 && one) {
427
- const once = one.counted === 1;
428
- if (one.passed === one.counted) return `Passed on ${one.label}${once ? "" : " every time"}.`;
429
- if (one.passed === 0) return `Failed on ${one.label}${once ? "" : " every time"}.`;
430
- return `Passed ${of(one)} on ${one.label}.`;
431
- }
432
- const names = counted.map((t) => t.label);
433
- const all = `${names.slice(0, -1).join(", ")} and ${names.at(-1)}`;
434
- if (counted.every((t) => t.passed === t.counted)) return `Passed on ${all} every time.`;
435
- if (counted.every((t) => t.passed === 0)) return `Failed on ${all} every time.`;
436
- const [first, ...rest] = counted;
437
- return `${first?.label} passed ${first ? of(first) : ""}${rest.map((t) => `, ${t.label} ${of(t)}`).join("")}.`;
438
- }
439
- /** `(cursor 4 of 4, codex 1 of 4)` when the harnesses' pass rates differ; empty when they agree. */
440
- function disagreement(record, cells) {
441
- const tallies = tallyBy(cells, (c) => harnessLabel(record, c)).filter((t) => t.counted > 0);
442
- if (tallies.length < 2) return "";
443
- return new Set(tallies.map((t) => t.passed / t.counted)).size > 1 ? ` (${tallies.map((t) => `${t.label} ${of(t)}`).join(", ")})` : "";
444
- }
445
- /** `, not by minLift 60` when the variant's lift fell short of its `minLift`, which fails it. */
446
- function minLiftShort(e, own) {
447
- const keys = new Set(own.map((c) => c.key));
448
- const pooled = (e.pooled ?? []).filter((p) => keys.has(p.key));
449
- const missed = pooled.length ? pooled.find((p) => p.minLift !== null && p.outcome === "fail")?.minLift : own.find((c) => c.minLift !== null && c.interval && c.interval.low < c.minLift)?.minLift;
450
- return missed == null ? "" : `, not by minLift ${Math.round(missed * 100)}`;
451
- }
452
- /**
453
- * Under a variant, its comparison with the baseline (`measuresLine`): `Same pass rate · 5× faster ·
454
- * judge "tidy" +0.12.`, one line, or one per harness (`cursor: …`) when it ran on several. Only
455
- * when some measure besides pass rate holds; `verbose` adds the ones that only lean.
456
- */
457
- function measureLines(e, own, verbose) {
458
- const keys = new Set(own.map((c) => c.key));
459
- const pooled = (e.pooled ?? []).filter((p) => keys.has(p.key));
460
- const sets = pooled.length ? pooled.map((p) => ({
461
- harness: own.find((c) => c.key === p.key)?.harness ?? p.key,
462
- measures: p.measures
463
- })) : own.flatMap((c) => c.measures ? [{
464
- harness: c.harness,
465
- measures: c.measures
466
- }] : []);
467
- const several = new Set(sets.map((x) => x.harness)).size > 1;
468
- return sets.flatMap(({ harness, measures }) => {
469
- if (!measures.filter((m) => m.measure !== "pass" && (m.verdict === "better" || m.verdict === "worse" || verbose && m.verdict === "leans")).length) return [];
470
- const line = measuresLine(measures, verbose);
471
- const text = `${line.slice(0, 1).toUpperCase()}${line.slice(1)}.`;
472
- return [{
473
- depth: 2,
474
- text: several ? `${harness}: ${text}` : text,
475
- tone: "dim"
476
- }];
477
- });
478
- }
479
- function compareLines(record, e, cells, verbose) {
480
- const base = cells.filter((c) => c.baseline);
481
- const name = (c) => c.environment === "-" ? "no environment" : c.environment;
482
- const baseTally = tallyBy(base, () => "baseline")[0] ?? {
483
- passed: 0,
484
- counted: 0
485
- };
486
- const lines = [{
487
- depth: 0,
488
- text: `Baseline ${[...new Set(base.map(name))].join(", ")} passed ${of(baseTally)}${disagreement(record, base)}.`
489
- }];
490
- const variants = [...new Set(cells.filter((c) => c.variant !== null).map((c) => c.variant))];
491
- if (!variants.length) return lines;
492
- const rows = variants.map((v, order) => {
493
- const own = cells.filter((c) => c.variant === v);
494
- const t = tallyBy(own, () => v ?? "")[0] ?? {
495
- passed: 0,
496
- counted: 0
497
- };
498
- const effect = effectOf({
499
- pass: t.passed,
500
- counted: t.counted
501
- }, {
502
- pass: baseTally.passed,
503
- counted: baseTally.counted
504
- });
505
- return {
506
- v: v ?? "",
507
- t,
508
- effect,
509
- own,
510
- order
511
- };
512
- });
513
- const rate = (t) => t.counted ? t.passed / t.counted : -1;
514
- rows.sort((a, b) => rate(b.t) - rate(a.t) || a.order - b.order);
515
- const width = Math.max(...rows.map((r) => r.v.length));
516
- lines.push({
517
- depth: 0,
518
- text: "Variants, by pass rate:",
519
- tone: "heading"
520
- });
521
- for (const r of rows) {
522
- const short = minLiftShort(e, r.own);
523
- const word = r.effect ? ` ${EFFECT[r.effect]}${short}` : short ? ` ${short.slice(2)}` : "";
524
- lines.push({
525
- depth: 1,
526
- text: `${r.v.padEnd(width)} ${of(r.t)}${word}${disagreement(record, r.own)}`
527
- });
528
- lines.push(...measureLines(e, r.own, verbose));
529
- if (!verbose) continue;
530
- for (const cell of r.own) {
531
- const stats = statsLine(cell, cells);
532
- const who = r.own.length > 1 ? `${harnessLabel(record, cell)}: ` : "";
533
- if (stats) lines.push({
534
- depth: 2,
535
- text: `${who}${stats}`,
536
- tone: "dim"
537
- });
538
- }
539
- }
540
- return lines;
541
- }
542
- function failedChecks(cells, groupOf) {
543
- const by = /* @__PURE__ */ new Map();
544
- for (const cell of cells) for (const k of cell.checks) {
545
- if (k.outcome.status !== "ran" || k.outcome.failed + k.outcome.errored === 0) continue;
546
- const n = k.outcome.failed + k.outcome.errored;
547
- const f = by.get(k.label) ?? {
548
- label: k.label,
549
- failed: 0,
550
- byHarness: /* @__PURE__ */ new Map(),
551
- first: k
552
- };
553
- f.failed += n;
554
- const h = groupOf(cell);
555
- f.byHarness.set(h, (f.byHarness.get(h) ?? 0) + n);
556
- if (f.first.outcome.status === "ran" && !f.first.outcome.assertion) f.first = k;
557
- by.set(k.label, f);
558
- }
559
- return [...by.values()].sort((a, b) => b.failed - a.failed || a.first.order - b.first.order);
560
- }
561
- /** `, all on H` or `, M of them on H`, per the rules at the top. */
562
- function where(f, harnesses) {
563
- if (harnesses < 2) return "";
564
- const [top] = [...f.byHarness].sort((a, b) => b[1] - a[1]);
565
- if (!top) return "";
566
- const [h, n] = top;
567
- if (n === f.failed) return f.failed === 1 ? `, on ${h}` : `, all on ${h}`;
568
- return n * 2 > f.failed ? `, ${n} of them on ${h}` : "";
569
- }
570
- /** A message's first lines, without the label it repeats or a command's empty output headings. */
571
- function messageLines(text, label = "", max = LINE_MAX) {
572
- return (label && text.startsWith(`${label}: `) ? text.slice(label.length + 2) : text).split("\n").map((l) => l.trimEnd()).filter((l) => l.trim() && !/^--- (stderr|stdout) ---$/.test(l.trim())).slice(0, MESSAGE_LINES).map((l) => clip(l, max));
573
- }
574
- /** A failure's message is red only on a failed eval: a passing compare's baseline misses are expected. */
575
- function checkLines(cells, groupOf, tone) {
576
- const failed = failedChecks(cells, groupOf);
577
- if (!failed.length) return [];
578
- const harnesses = new Set(cells.map(groupOf)).size;
579
- const lines = [{
580
- depth: 0,
581
- text: failed.length === 1 ? "Every failure was the same check:" : `${failed.length} checks failed:`,
582
- tone: "heading"
583
- }];
584
- for (const f of failed) {
585
- lines.push({
586
- depth: 1,
587
- text: `"${checkLabel(f.label)}" failed ${times(f.failed)}${where(f, harnesses)}`
588
- });
589
- const o = f.first.outcome;
590
- const judge = f.first.judge;
591
- const message = o.status === "ran" && o.assertion ? o.assertion : judge ? `judge: ${judge.question} (scores ${judge.scores.join(", ")}, threshold ${judge.threshold})` : "";
592
- for (const text of messageLines(message, f.label)) lines.push({
593
- depth: 2,
594
- text,
595
- tone
596
- });
597
- if (o.status === "ran" && o.at) lines.push({
598
- depth: 2,
599
- text: `at ${o.at}`,
600
- tone: "dim"
601
- });
602
- }
603
- return lines;
604
- }
605
- const CASES_NAMED = 8;
606
- function failedCaseLines(e) {
607
- const failed = (e.cases ?? []).filter((k) => k.outcome !== "pass" && k.outcome !== "skipped");
608
- if (!failed.length) return [];
609
- const named = failed.slice(0, CASES_NAMED).map((k) => k.label);
610
- const more = failed.length > CASES_NAMED ? ` and ${failed.length - CASES_NAMED} more` : "";
611
- return [{
612
- depth: 0,
613
- text: `Failed cases: ${named.join(", ")}${more}.`
614
- }];
615
- }
616
- /** Errors no failed check explains: timeouts, infrastructure errors, throws. Never clipped: their remedy is often last. */
617
- function errorLines(cells, tone) {
618
- const by = /* @__PURE__ */ new Map();
619
- for (const c of cells) for (const err of c.errors) {
620
- const key = `${err.kind}\n${err.message}`;
621
- const e = by.get(key) ?? {
622
- kind: err.kind,
623
- message: err.message,
624
- n: 0
625
- };
626
- e.n += err.attempts;
627
- by.set(key, e);
628
- }
629
- return [...by.values()].flatMap((e) => [{
630
- depth: 0,
631
- text: `${plural(e.n, "run")} ${ERROR_VERB[e.kind]}:`,
632
- tone: "heading"
633
- }, ...messageLines(e.message, "", Number.POSITIVE_INFINITY).map((text) => ({
634
- depth: 1,
635
- text,
636
- tone
637
- }))]);
638
- }
639
- /** `evals/a.eval.ts:12` from `source:evals/a.eval.ts#L12`. */
640
- const location = (at) => at.replace(/^source:/, "").replace(/#L(\d+)$/, ":$1");
641
- /** `"npm test" passed on try 2 of 3.`: a check that needed its `c.exec` retries is flaky. */
642
- function retriedLines(cells) {
643
- const seen = /* @__PURE__ */ new Set();
644
- return cells.flatMap((c) => c.checks.flatMap((k) => {
645
- if (!k.retried || seen.has(k.label)) return [];
646
- seen.add(k.label);
647
- return [`"${checkLabel(k.label)}" ${k.retried}.`];
648
- }));
649
- }
650
- function pageErrorNotes(cells) {
651
- const [first, ...rest] = cells.flatMap((c) => c.pageErrors ?? []);
652
- if (!first) return [];
653
- const more = rest.length ? ` (+${plural(rest.length, "more capture")})` : "";
654
- return [`${plural(first.errors, "page error")} in capture "${first.capture}" → ${first.file}${more}`];
655
- }
656
- function remarkLines(remarks, notes) {
657
- if (!remarks.length && !notes.length) return [];
658
- return [
659
- {
660
- depth: 0,
661
- text: "Worth knowing:",
662
- tone: "heading"
663
- },
664
- ...remarks.map((r) => ({
665
- depth: 1,
666
- text: r.fix.edit ? `${r.text} Fix at ${location(r.fix.at)}: ${r.fix.edit}.` : `${r.text} At ${location(r.fix.at)}.`
667
- })),
668
- ...notes.map((text) => ({
669
- depth: 1,
670
- text
671
- }))
672
- ];
673
- }
674
- /** A compare's failures are told apart by arm: `the baseline`, or the variant's name. */
675
- const armOf = (c) => c.baseline ? "the baseline" : c.variant ?? c.environment;
676
- const nameOf = (e) => [...e.group, e.template ?? e.name].join(" › ");
677
- function iconOf(e) {
678
- if (e.todo || e.notRun) return "skipped";
679
- if (e.outcome === "skipped") return "skipped";
680
- if (e.outcome === "canceled") return "canceled";
681
- if (e.outcome === "pass") return isNotGatedResult(e) ? "observe" : "pass";
682
- return "fail";
683
- }
684
- function evalRow(record, e, remarks, verbose = false) {
685
- const cells = cellsOf(e);
686
- const decide = deciding(cells);
687
- const passed = decide.reduce((n, c) => n + c.passed, 0);
688
- const ran = decide.reduce((n, c) => n + c.passed + c.failed + c.infraError, 0);
689
- const compare = cells.some((c) => c.baseline);
690
- const counted = compare && decide.length < cells.length ? "variant run" : "run";
691
- const summary = e.todo ? "todo" : e.notRun ? "not run" : ran === 0 ? e.outcome === "canceled" ? "canceled" : "skipped" : `${passed} of ${plural(ran, counted)} passed`;
692
- const tone = iconOf(e) === "fail" ? "red" : "dim";
693
- const notes = ran ? [...retriedLines(decide), ...pageErrorNotes(decide)] : [];
694
- const detail = ran ? [
695
- ...compare ? compareLines(record, e, cells, verbose) : [{
696
- depth: 0,
697
- text: outcomeSentence(tallyBy(decide, labeler(record, decide)))
698
- }],
699
- ...failedCaseLines(e),
700
- ...compare ? checkLines(cells, armOf, tone) : checkLines(decide, (c) => harnessLabel(record, c), tone),
701
- ...errorLines(compare ? cells : decide, tone),
702
- ...iconOf(e) === "fail" ? snapshotLines(cells) : [],
703
- ...questionLines(e.questions ?? []),
704
- ...observedLines(e.observations ?? [], cells, (c) => compare ? armOf(c) : harnessLabel(record, c)),
705
- ...remarkLines(remarks, notes)
706
- ] : remarkLines(remarks, []);
707
- return {
708
- id: e.id,
709
- name: nameOf(e),
710
- icon: iconOf(e),
711
- summary,
712
- detail,
713
- remarks: remarks.length + notes.length,
714
- compare
715
- };
716
- }
717
- /** Free-text answers shown per question; the rest are in the record. */
718
- const ANSWERS_SHOWN = 3;
719
- /** Rows of `[label, text]` at `depth`, the labels padded to one column. */
720
- const aligned = (depth, rows) => {
721
- const w = Math.max(0, ...rows.map(([label]) => label.length));
722
- return rows.map(([label, text]) => ({
723
- depth,
724
- text: `${label.padEnd(w)} ${text}`
725
- }));
726
- };
727
- /** Each question's answer: who won the pairs, the tally of choices, or the first answers. */
728
- function questionLines(questions) {
729
- return questions.flatMap((q) => {
730
- const head = {
731
- depth: 0,
732
- text: `Question: ${clip(q.question, 100)}`,
733
- tone: "heading"
734
- };
735
- if (q.kind === "attempts") {
736
- const failed = q.error === null ? [] : [{
737
- depth: 1,
738
- text: `${q.failed} of ${q.asked} not answered: ${clip(q.error, 200)}`,
739
- tone: "red"
740
- }];
741
- if (q.choices) return [
742
- head,
743
- {
744
- depth: 1,
745
- text: q.choices.map((t) => `${t.choice} ${t.n}`).join(" · ")
746
- },
747
- ...failed
748
- ];
749
- const shown = q.answers.slice(0, ANSWERS_SHOWN);
750
- const more = q.answers.length - shown.length;
751
- return [
752
- head,
753
- ...aligned(1, shown.map((a) => [a.attempt, clip(a.text.replace(/\s+/g, " "), 100)])),
754
- ...more ? [{
755
- depth: 1,
756
- text: `${more} more: reeval query attempt_answers`,
757
- tone: "dim"
758
- }] : [],
759
- ...failed
760
- ];
761
- }
762
- if (q.votes === 0 && q.error === null) return [head, {
763
- depth: 1,
764
- text: "No pairs to judge: tries that failed their checks are left out.",
765
- tone: "dim"
766
- }];
767
- if (q.rates.length === 0 && q.error === null) return [head, {
768
- depth: 1,
769
- text: `${plural(q.votes, "vote")}, every one a tie.`,
770
- tone: "dim"
771
- }];
772
- const cuts = new Set(q.rates.map((r) => r.cut)).size > 1;
773
- return [
774
- head,
775
- ...aligned(1, q.rates.map((r) => [cuts ? `${r.name} (${r.cut})` : r.name, `won ${r.wins} of ${r.n}`])),
776
- ...q.error === null ? [] : [{
777
- depth: 1,
778
- text: `The model call failed: ${clip(q.error, 200)}`,
779
- tone: "red"
780
- }]
781
- ];
782
- });
783
- }
784
- /** Where a failed eval's first failed attempt's final workspace snapshot is, to open it. */
785
- function snapshotLines(cells) {
786
- const [first, ...others] = cells.flatMap((c) => c.snapshots ?? []);
787
- if (!first) return [];
788
- return [{
789
- depth: 0,
790
- text: `Snapshot: ${first}${others.length ? ` (+${others.length} more in summary.md)` : ""}`,
791
- tone: "dim"
792
- }];
793
- }
794
- /** What `c.observe` recorded, each name's tally on one line, or a line per cell under it. */
795
- function observedLines(tallies, cells, label) {
796
- if (!tallies.length) return [];
797
- const byKey = new Map(cells.map((c) => [c.key, c]));
798
- const perCell = new Set(tallies.map((t) => t.cell)).size > 1;
799
- const width = Math.max(...tallies.map((t) => clip(t.name, 40).length));
800
- const lines = [{
801
- depth: 0,
802
- text: "Observed:",
803
- tone: "heading"
804
- }];
805
- if (!perCell) {
806
- for (const t of tallies) lines.push({
807
- depth: 1,
808
- text: `${clip(t.name, 40).padEnd(width)} ${observedLine(t)}`
809
- });
810
- return lines;
811
- }
812
- for (const name of new Set(tallies.map((t) => t.name))) {
813
- lines.push({
814
- depth: 1,
815
- text: clip(name, 40)
816
- });
817
- const rows = tallies.filter((t) => t.name === name).map((t) => {
818
- const cell = byKey.get(t.cell);
819
- return [cell ? label(cell) : t.cell, observedLine(t)];
820
- });
821
- const w = Math.max(...rows.map(([who]) => who.length));
822
- for (const [who, line] of rows) lines.push({
823
- depth: 2,
824
- text: `${who.padEnd(w)} ${line}`
825
- });
826
- }
827
- return lines;
828
- }
829
- /** `3 passed · 1 failed · 1 not gated`, from the suite's tally. */
830
- function tallyText(suite) {
831
- const t = suite.tally;
832
- return [
833
- t.errors ? plural(t.errors, "run error") : "",
834
- t.fail ? `${t.fail} failed` : "",
835
- t.infraError ? plural(t.infraError, "infra error") : "",
836
- t.canceled ? `${t.canceled} canceled` : "",
837
- t.pass ? `${t.pass} passed` : "",
838
- t.notGated ? `${t.notGated} not gated` : "",
839
- t.skipped ? `${t.skipped} skipped` : ""
840
- ].filter(Boolean).join(" · ");
841
- }
842
- function statusList(input) {
843
- const { record, suite } = input;
844
- const { shown, hidden } = shownRemarks(input.remarks?.remarks ?? [], suite.runId, input.verbose);
845
- const files = suite.files.map((f) => ({
846
- path: f.path,
847
- rows: f.evals.map((e) => evalRow(record, e, shown.filter((r) => r.evalId === e.path), input.verbose))
848
- }));
849
- const tokens = formatUsage(record.cost.usage);
850
- const done = [
851
- `Done in ${formatDuration(record.durationMs)}`,
852
- tallyText(suite) || "nothing ran",
853
- tokens
854
- ].filter(Boolean).join(" · ");
855
- const notes = [
856
- ...input.notRun ? [`Incomplete: ${plural(input.notRun, "attempt")} this resume’s filters left out ${input.notRun === 1 ? "is" : "are"} not run; reeval run --resume ${record.runId} runs ${input.notRun === 1 ? "it" : "them"}`] : [],
857
- ...record.status === "budget" && !input.notRun ? ["Budget reached: attempts that hadn’t started were skipped"] : [],
858
- ...record.status === "interrupted" && !input.notRun ? ["Interrupted: attempts that hadn’t finished were canceled"] : []
859
- ];
860
- return {
861
- files,
862
- errors: suite.errors,
863
- done,
864
- notes,
865
- hiddenRemarks: hidden
866
- };
867
- }
868
- /** The pointers under the closing line: the studio and the run's summary.md. */
869
- const pointerLine = (summaryPath) => `Browse it: reeval studio · summary ${summaryPath}`;
870
312
  // src/ui/tags.ts
871
313
  const STOPWORDS = new Set("a an the and or of to in on for with without from by at as is are be it its that this when than then into via not no".split(" "));
872
314
  const MAX_WORDS = 3;
@@ -986,7 +428,8 @@ var AgentRenderer = class {
986
428
  verbose,
987
429
  ...e.notRun ? { notRun: e.notRun } : {}
988
430
  });
989
- for (const line of listLines(list, this.#c, false, pointerLine(e.summaryPath), verbose)) this.#write(line);
431
+ const pointer = pointerLine(e.summaryPath);
432
+ for (const line of listLines(list, this.#c, false, pointer, verbose, true)) this.#write(line);
990
433
  return;
991
434
  }
992
435
  default: return;
@@ -1266,7 +709,7 @@ var PlainRenderer = class {
1266
709
  this.#write(`::error file=${ghEscape(file, true)},line=${line},title=${label}::${ghEscape(x.message)}`);
1267
710
  }
1268
711
  if (!errorCheck(result)?.at) {
1269
- const message = ghEscape(failureLines(colors(false), result, { max: ERROR_LINES }).join("\n"));
712
+ const message = ghEscape(failureLines(colors(false), result, { max: ERROR_LINES }).join("\n").replace(/^✗ failed {2}/, ""));
1270
713
  const at = locate(result);
1271
714
  const where = at ? `,line=${at.line},col=${at.col}` : "";
1272
715
  this.#write(`::error file=${ghEscape(result.file, true)}${where},title=${title}::${message}`);
@@ -1330,6 +773,107 @@ var ListStore = class {
1330
773
  for (const l of this.#listeners) l();
1331
774
  }
1332
775
  };
776
+ const FRAME_MS = 80;
777
+ const ESC = String.fromCharCode(27);
778
+ const BEL = String.fromCharCode(7);
779
+ /** A CSI sequence (colors, cursor moves) or an OSC one (a hyperlink), which take no columns. */
780
+ const ESCAPE = new RegExp(`${ESC}(?:\\[[0-9;?]*[ -/]*[@-~]|\\][^${BEL}${ESC}]*(?:${BEL}|${ESC}\\\\))`, "g");
781
+ const graphemes = new Intl.Segmenter();
782
+ /**
783
+ * Terminal rows one line takes at `columns`, placed as a terminal places it: a wide character that
784
+ * doesn't fit in the row's last column moves to the next row, `\r` returns to the row's start and
785
+ * `\t` goes to the next stop of eight without wrapping.
786
+ */
787
+ function rowsOf(line, columns) {
788
+ let rows = 1;
789
+ let column = 0;
790
+ for (const { segment } of graphemes.segment(line.replace(ESCAPE, ""))) if (segment === "\r") column = 0;
791
+ else if (segment === " ") column = Math.min(columns - 1, (Math.floor(column / 8) + 1) * 8);
792
+ else if (segment === "\b") column = Math.max(0, column - 1);
793
+ else {
794
+ const width = stringWidth(segment);
795
+ if (!width) continue;
796
+ if (column + width > columns) {
797
+ rows += 1;
798
+ column = 0;
799
+ }
800
+ column += width;
801
+ }
802
+ return rows;
803
+ }
804
+ /**
805
+ * Before the list is up, the sandbox prepare step: the glyph of its `◆ Preparing…` line spins in
806
+ * place (the cursor goes up to it and back) until each sandbox is ready or failed, then settles to
807
+ * `◆`. What prints under it is kept and measured in rows as the terminal placed it, so a wide
808
+ * character still finds the step line. Once the step line would be out of reach, or the width
809
+ * changes (some terminals reflow what they show, some don't, so its row is unknown), it stops
810
+ * without drawing.
811
+ */
812
+ var StepSpinner = class {
813
+ #write;
814
+ #c;
815
+ #size;
816
+ #timer;
817
+ #frame = 0;
818
+ /** The step line and everything printed under it since. */
819
+ #text = "";
820
+ /** The width `#text` was printed at. */
821
+ #columns = 0;
822
+ #pending = 0;
823
+ constructor(write, c, size) {
824
+ this.#write = write;
825
+ this.#c = c;
826
+ this.#size = size;
827
+ }
828
+ /** Terminal lines from the step line down to the cursor after `text`. */
829
+ #up(text = this.#text) {
830
+ const columns = Math.max(1, this.#columns);
831
+ const lines = text.split("\n");
832
+ if (lines.at(-1) === "") lines.pop();
833
+ return lines.reduce((n, l) => n + rowsOf(l, columns), 0);
834
+ }
835
+ /** The step line was just written as `text`; it spins until `targets` have reported. */
836
+ start(text, targets) {
837
+ this.settle();
838
+ this.#text = text;
839
+ this.#columns = this.#size().columns;
840
+ this.#pending = targets;
841
+ this.#timer = setInterval(() => this.#draw(), FRAME_MS);
842
+ this.#timer.unref();
843
+ this.#draw();
844
+ }
845
+ /** Before `text` prints under it; settles first if the step line would scroll out of reach. */
846
+ before(text) {
847
+ if (!this.#timer) return;
848
+ const next = this.#text + text;
849
+ if (!this.#reachable(next)) this.settle();
850
+ this.#text = next;
851
+ }
852
+ /** One sandbox reported; the last settles it. */
853
+ reported() {
854
+ this.#pending -= 1;
855
+ if (this.#pending <= 0) this.settle();
856
+ }
857
+ #reachable(text = this.#text) {
858
+ const { rows, columns } = this.#size();
859
+ return columns === this.#columns && this.#up(text) < rows - 1;
860
+ }
861
+ #draw(glyph = SPINNER[this.#frame++ % SPINNER.length] ?? "◆") {
862
+ if (!this.#reachable()) {
863
+ clearInterval(this.#timer);
864
+ this.#timer = void 0;
865
+ return;
866
+ }
867
+ const up = this.#up();
868
+ this.#write(`\x1b[${up}A\r\x1b[2C${this.#c.teal(glyph)}\r\x1b[${up}B`);
869
+ }
870
+ settle() {
871
+ if (!this.#timer) return;
872
+ this.#draw("◆");
873
+ clearInterval(this.#timer);
874
+ this.#timer = void 0;
875
+ }
876
+ };
1333
877
  var InteractiveRenderer = class {
1334
878
  #o;
1335
879
  #c;
@@ -1342,12 +886,20 @@ var InteractiveRenderer = class {
1342
886
  #fallback;
1343
887
  #ended;
1344
888
  #closed = false;
889
+ #spinner;
890
+ /** What the plain renderer wrote since `#written` was reset, while the list isn't up. */
891
+ #written = "";
1345
892
  constructor(options) {
1346
893
  this.#o = options;
1347
894
  this.#c = options.colors;
1348
895
  this.#list = new LiveList(options.harness);
896
+ this.#spinner = new StepSpinner((text) => options.out.write(text), options.colors, () => sizeOf(options.stdout));
1349
897
  const out = { write: (text) => {
1350
- if (!this.#app || this.#fallback) return options.out.write(text);
898
+ if (!this.#app || this.#fallback) {
899
+ this.#spinner.before(redactSecrets(String(text)));
900
+ this.#written += String(text);
901
+ return options.out.write(text);
902
+ }
1351
903
  const lines = String(text).replace(/\n$/, "").split("\n");
1352
904
  this.#store.set({ logs: [...this.#store.get().logs, ...lines] });
1353
905
  return true;
@@ -1367,7 +919,7 @@ var InteractiveRenderer = class {
1367
919
  #mount() {
1368
920
  if (this.#app || this.#closed) return;
1369
921
  const cancel = this.#o.cancel ?? (() => process.kill(process.pid, "SIGINT"));
1370
- const load = this.#o.load ?? (() => import("./ink-list-Ca-5dunO.js"));
922
+ const load = this.#o.load ?? (() => import("./ink-list-BpfidQ1h.js"));
1371
923
  this.#app = load().then((m) => m.mountList({
1372
924
  store: this.#store,
1373
925
  colors: this.#c,
@@ -1384,7 +936,7 @@ var InteractiveRenderer = class {
1384
936
  for (const line of logs) out.write(`${line}\n`);
1385
937
  const reason = error instanceof Error ? error.message : String(error);
1386
938
  out.write(`${this.#c.yellow(`Couldn't draw the interactive list (${reason}).`)}\n`);
1387
- if (phase) out.write(`${phase}\n`);
939
+ if (phase) out.write(` ${this.#c.teal("◆")} ${phase}\n`);
1388
940
  this.#fallback = new PlainRenderer({
1389
941
  ...this.#o,
1390
942
  timers: false,
@@ -1397,12 +949,14 @@ var InteractiveRenderer = class {
1397
949
  if (this.#ended) this.#fallback.onEvent(this.#ended.event);
1398
950
  }
1399
951
  #update() {
1400
- const { done, total } = this.#list.counts();
952
+ const { done, total, failed, failing } = this.#list.counts();
1401
953
  this.#store.set({
1402
954
  files: this.#list.files(),
1403
955
  counts: {
1404
956
  done,
1405
- total
957
+ total,
958
+ failed,
959
+ failing
1406
960
  }
1407
961
  });
1408
962
  }
@@ -1413,9 +967,18 @@ var InteractiveRenderer = class {
1413
967
  }
1414
968
  switch (e.type) {
1415
969
  case "run-start":
1416
- this.#store.set({ phase: startLine(this.#c, this.#o.output) });
970
+ this.#spinner.settle();
971
+ this.#store.set({ phase: startText(this.#o.output) });
972
+ return;
973
+ case "prepare":
974
+ this.#written = "";
975
+ this.#plain.onEvent(e);
976
+ if (this.#app || this.#o.timers === false || process.env.REEVAL_DEBUG) return;
977
+ if (e.state === "start") this.#spinner.start(this.#written, e.targets.length);
978
+ else this.#spinner.reported();
1417
979
  return;
1418
980
  case "collected":
981
+ this.#spinner.settle();
1419
982
  this.#list.collected(e.file, e.evals);
1420
983
  this.#mount();
1421
984
  this.#update();
@@ -1435,25 +998,28 @@ var InteractiveRenderer = class {
1435
998
  return;
1436
999
  case "debug": return;
1437
1000
  case "run-end": {
1438
- const list = statusList({
1001
+ const listOf = (verbose) => statusList({
1439
1002
  record: e.record,
1440
1003
  suite: suiteResult(e.record),
1441
1004
  remarks: e.remarks ?? null,
1442
- verbose: false,
1005
+ verbose,
1443
1006
  ...e.notRun ? { notRun: e.notRun } : {}
1444
1007
  });
1008
+ const list = listOf(false);
1009
+ const verbose = (this.#o.verbosity ?? 0) > 0;
1445
1010
  if (!this.#app) {
1446
1011
  this.#plain.onEvent(e);
1447
1012
  return;
1448
1013
  }
1449
1014
  this.#ended = {
1450
- list,
1015
+ scrollback: verbose ? listOf(true) : list,
1016
+ verbose,
1451
1017
  pointer: pointerLine(e.summaryPath),
1452
1018
  event: e
1453
1019
  };
1454
1020
  this.#store.set({
1455
1021
  files: list.files,
1456
- ...this.#store.get().phase ? { phase: endLine(this.#c, e.record.status) } : {},
1022
+ ...this.#store.get().phase ? { phase: endText(e.record.status) } : {},
1457
1023
  finished: {
1458
1024
  done: list.done,
1459
1025
  notes: list.notes,
@@ -1467,6 +1033,7 @@ var InteractiveRenderer = class {
1467
1033
  }
1468
1034
  }
1469
1035
  close() {
1036
+ this.#spinner.settle();
1470
1037
  this.#plain.close();
1471
1038
  this.#fallback?.close();
1472
1039
  if (!this.#store.get().finished) {
@@ -1480,11 +1047,18 @@ var InteractiveRenderer = class {
1480
1047
  if (!app || !this.#ended) return;
1481
1048
  await app.waitUntilExit();
1482
1049
  const { phase } = this.#store.get();
1483
- const { list, pointer } = this.#ended;
1484
- const text = [...phase ? [phase, ""] : [], ...listLines(list, this.#c, true, pointer)];
1050
+ const { scrollback, verbose, pointer } = this.#ended;
1051
+ const text = [...phase ? [` ${this.#c.teal("◆")} ${phase}`, ""] : [], ...listLines(scrollback, this.#c, true, pointer, verbose)];
1485
1052
  this.#o.out.write(`${text.join("\n")}\n`);
1486
1053
  }
1487
1054
  };
1055
+ const sizeOf = (stdout) => {
1056
+ const { rows, columns } = stdout ?? {};
1057
+ return {
1058
+ rows: rows || 24,
1059
+ columns: columns || 80
1060
+ };
1061
+ };
1488
1062
  // src/ui/index.ts
1489
1063
  const depthOf = (bits = 4) => bits >= 24 ? 24 : bits >= 8 ? 8 : 4;
1490
1064
  function createRenderer(options) {
@@ -1532,7 +1106,7 @@ const EXIT = {
1532
1106
  };
1533
1107
  const version = PACKAGE_VERSION;
1534
1108
  const HELP = helpText(version);
1535
- const { green, red, dim } = colors(!!process.stdout.isTTY && !process.env.NO_COLOR);
1109
+ const { green, red, dim, yellow } = colors(!!process.stdout.isTTY && !process.env.NO_COLOR);
1536
1110
  const startedAt = Date.now();
1537
1111
  const argv = process.argv.slice(2);
1538
1112
  /** What the invocation log needs from the command as it goes. */
@@ -1576,7 +1150,7 @@ async function run(argv) {
1576
1150
  return values.help ? EXIT.ok : EXIT.usage;
1577
1151
  }
1578
1152
  if (command === "init") {
1579
- const { initCommand } = await import("./init-8o6W3_fn.js");
1153
+ const { initCommand } = await import("./init-4RPeD-K-.js");
1580
1154
  return initCommand(values, filters, {
1581
1155
  out: (s) => console.log(s),
1582
1156
  input: process.stdin,
@@ -1585,7 +1159,7 @@ async function run(argv) {
1585
1159
  });
1586
1160
  }
1587
1161
  if (command === "doctor") {
1588
- const { doctorCommand } = await import("./doctor-BcvfKpBX.js");
1162
+ const { doctorCommand } = await import("./doctor-B2PohneB.js");
1589
1163
  return doctorCommand(values, {
1590
1164
  out: (s) => console.log(s),
1591
1165
  color: !!process.stdout.isTTY && !process.env.NO_COLOR,
@@ -1593,7 +1167,7 @@ async function run(argv) {
1593
1167
  });
1594
1168
  }
1595
1169
  if (command === "report") {
1596
- const { reportCommand } = await import("./command-CFQ9c1hk.js");
1170
+ const { reportCommand } = await import("./command-aUJvENwy.js");
1597
1171
  return reportCommand(values, filters, {
1598
1172
  out: (s) => console.log(s),
1599
1173
  err: (s) => console.error(dim(s)),
@@ -1608,7 +1182,7 @@ async function run(argv) {
1608
1182
  }
1609
1183
  if (command === "studio") {
1610
1184
  const config = await loadConfig(configPath, "studio");
1611
- const { studioCommand } = await import("./command-DhiwsZl1.js");
1185
+ const { studioCommand } = await import("./command-BLNVDiIg.js");
1612
1186
  return studioCommand({
1613
1187
  config,
1614
1188
  flags: values,
@@ -1619,7 +1193,7 @@ async function run(argv) {
1619
1193
  if (command === "mcp") {
1620
1194
  if (!values.http) console.log = console.info = console.debug = console.error;
1621
1195
  const config = await loadConfig(configPath, "mcp");
1622
- const { mcpCommand } = await import("./command-qGS802uX.js");
1196
+ const { mcpCommand } = await import("./command-B5OyHpOu.js");
1623
1197
  return mcpCommand({
1624
1198
  config,
1625
1199
  flags: values,
@@ -1628,7 +1202,7 @@ async function run(argv) {
1628
1202
  }
1629
1203
  if (command in HISTORY_COMMANDS) {
1630
1204
  const config = await loadConfig(configPath, command);
1631
- const commands = await import("./commands-xYTMNTPv.js");
1205
+ const commands = await import("./commands-CfdBX-mc.js");
1632
1206
  const ctx = {
1633
1207
  config,
1634
1208
  flags: values,
@@ -1641,7 +1215,7 @@ async function run(argv) {
1641
1215
  case "compare": return commands.compareCommand(ctx, filters);
1642
1216
  case "checkout": return commands.checkoutCommand(ctx, filters);
1643
1217
  case "query": {
1644
- const { queryCommand } = await import("./query-CvgUsvep.js");
1218
+ const { queryCommand } = await import("./query-DIfkUcPh.js");
1645
1219
  return queryCommand({
1646
1220
  ...ctx,
1647
1221
  err: (s) => console.error(dim(s))
@@ -1649,7 +1223,7 @@ async function run(argv) {
1649
1223
  }
1650
1224
  case "analyze": {
1651
1225
  loadEnvFiles(configPath);
1652
- const { analyzeCommand } = await import("./command-fO0BFsTq.js");
1226
+ const { analyzeCommand } = await import("./command-CRh-cb6Z.js");
1653
1227
  return analyzeCommand({
1654
1228
  ...ctx,
1655
1229
  err: (s) => console.error(dim(s))
@@ -1663,12 +1237,12 @@ async function run(argv) {
1663
1237
  }
1664
1238
  /** The dry run's prices, fetched only when a planned attempt or the judge has anything to price. */
1665
1239
  async function dryRunPrices(planned, judgeModel) {
1666
- const { needsPrices } = await import("./cost-BJU0MAji.js");
1240
+ const { needsPrices } = await import("./cost-Cr26Alhq.js");
1667
1241
  if (!needsPrices(planned.filter((p) => !p.attempt.plan.skip).map((p) => p.attempt.harness), judgeModel)) return {
1668
1242
  table: null,
1669
1243
  why: "nothing to price"
1670
1244
  };
1671
- const { loadPrices } = await import("./prices-DYJA0tJN.js");
1245
+ const { loadPrices } = await import("./prices-5m3SzMZ3.js");
1672
1246
  return loadPrices();
1673
1247
  }
1674
1248
  /** Asks a yes/no question on the terminal; anything but y or yes is no. */
@@ -1695,14 +1269,14 @@ async function cleanCommand(flags, configPath) {
1695
1269
  keep = Number(flags.keep);
1696
1270
  if (!(Number.isInteger(keep) && keep >= 0)) throw new UsageError(`--keep must be a whole number (got ${flags.keep})`);
1697
1271
  }
1698
- const { debugRoot, pruneDebug } = await import("./files-COW5JBJM.js");
1699
- const { formatBytes } = await import("./debug-BnXV7JbX.js");
1272
+ const { debugRoot, pruneDebug } = await import("./files-C1Rk9FYA.js");
1273
+ const { formatBytes } = await import("./debug-CF4p4Lp_.js");
1700
1274
  const { runs, bytes } = await pruneDebug(config.root, keep);
1701
1275
  const where = fromCwd(debugRoot(config.root)) || ".";
1702
1276
  console.log(runs.length ? `Removed debug data and snapshots for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where} and .reeval/runs` : `No debug data or snapshots to remove in ${where} or .reeval/runs`);
1703
1277
  if (flags.images) {
1704
- const { dockerAvailable, pruneImages } = await import("./src-BkH4GOxu.js");
1705
- const { requireDocker } = await import("./providers-D0ob_tra.js");
1278
+ const { dockerAvailable, pruneImages } = await import("./src-DtodMT59.js");
1279
+ const { requireDocker } = await import("./providers-CVu8e0xS.js");
1706
1280
  const engines = [];
1707
1281
  for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
1708
1282
  if (!engines.length) await requireDocker();
@@ -1712,24 +1286,25 @@ async function cleanCommand(flags, configPath) {
1712
1286
  }
1713
1287
  return EXIT.ok;
1714
1288
  }
1715
- /** `-v` is 1 and `-vv` (or `--trace`) is 2. */
1289
+ /** `-v` is 1 and `-vv` is 2. */
1716
1290
  function verbosityOf(flags) {
1717
- if (flags.trace) return 2;
1718
1291
  return Math.min(2, flags.verbose?.length ?? 0);
1719
1292
  }
1720
1293
  /**
1721
- * `reeval image build`: collects the evals (as `--dry-run` does) and builds every image their Docker
1722
- * sandboxes need; with `--dry-run`, lists them instead. `--profile` plans as `reeval run --profile`
1723
- * does. Needs a config, as `reeval run` does: there is no default image.
1294
+ * `reeval image build`: collects the evals (as `--dry-run` does), pulls every image their `tart()`
1295
+ * sandboxes clone that Tart doesn't have, and builds every image their Docker sandboxes need; with
1296
+ * `--dry-run`, lists them instead. A plan with no Docker sandbox never runs Docker. `--profile`
1297
+ * plans as `reeval run --profile` does. Needs a config, as `reeval run` does: there is no default
1298
+ * image.
1724
1299
  */
1725
1300
  async function imageBuildCommand(flags, configPath, filters) {
1726
- const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-CNayXCp2.js");
1301
+ const { buildImages, imagePlanLines, imageTargets, plannedImageUses, pullTartImages, tartPlanLines, tartTargets } = await import("./images-BNgcR77K.js");
1727
1302
  const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
1728
1303
  ...flags,
1729
1304
  "dry-run": true
1730
1305
  }, configPath));
1731
1306
  const root = config.root;
1732
- const targets = imageTargets(await plannedImageUses({
1307
+ const uses = await plannedImageUses({
1733
1308
  runId,
1734
1309
  stagingDir,
1735
1310
  config,
@@ -1737,46 +1312,116 @@ async function imageBuildCommand(flags, configPath, filters) {
1737
1312
  tags,
1738
1313
  concurrency,
1739
1314
  profile
1740
- }, filters, (parts) => unmatchedFilter(config, parts)));
1741
- if (targets.length === 0) {
1742
- console.log("No eval runs in a Docker sandbox, so there are no images to build.");
1315
+ }, filters, (parts) => unmatchedFilter(config, parts));
1316
+ const targets = imageTargets(uses);
1317
+ const vms = tartTargets(uses);
1318
+ if (targets.length === 0 && vms.length === 0) {
1319
+ console.log("No eval runs in a Docker or Tart sandbox, so there are no images to build or pull.");
1743
1320
  return EXIT.ok;
1744
1321
  }
1322
+ const { tartTagWarning } = await import("./tart-ref-7Qv3cm3B.js");
1323
+ for (const t of vms) {
1324
+ const warning = tartTagWarning(t.sandbox.image);
1325
+ if (warning) console.log(yellow(`warning: ${warning}`));
1326
+ }
1745
1327
  if (flags["dry-run"]) {
1746
- for (const line of await imagePlanLines(targets, root)) console.log(line);
1328
+ if (vms.length) for (const line of await tartPlanLines(vms)) console.log(line);
1329
+ if (targets.length) for (const line of await imagePlanLines(targets, root)) console.log(line);
1747
1330
  return EXIT.ok;
1748
1331
  }
1749
- const { PrepareError } = await import("./provider-DCdTHQpk.js");
1750
- let built;
1332
+ const { PrepareError } = await import("./provider-BnUoz4KB.js");
1333
+ const outcomes = [];
1334
+ let refused = false;
1335
+ const refusal = (id, of, others) => (error) => {
1336
+ if (!others || !(error instanceof PrepareError)) throw error;
1337
+ if (error.failure.usage) refused = true;
1338
+ const evals = of.reduce((n, t) => n + t.evals, 0);
1339
+ outcomes.push({
1340
+ id,
1341
+ evals,
1342
+ ok: false,
1343
+ failure: error.failure
1344
+ });
1345
+ return [];
1346
+ };
1751
1347
  try {
1752
- built = await buildImages(targets, root, {
1753
- log: (line) => console.log(dim(line)),
1754
- force: !!flags.force
1348
+ const pulled = vms.length ? await pullTartImages(vms, root, { log: (line) => console.log(line.startsWith("warning: ") ? yellow(line) : dim(line)) }).catch(refusal("tart", vms, targets.length)) : [];
1349
+ for (const p of pulled) outcomes.push(p.ok ? {
1350
+ id: p.id,
1351
+ evals: p.evals,
1352
+ ok: true,
1353
+ ref: p.image.ref,
1354
+ id12: p.image.digest ? short12(p.image.digest) : "local VM"
1355
+ } : {
1356
+ id: p.id,
1357
+ evals: p.evals,
1358
+ ok: false,
1359
+ failure: p.failure
1755
1360
  });
1361
+ if (targets.length) {
1362
+ const built = await buildImages(targets, root, {
1363
+ log: (line) => console.log(dim(line)),
1364
+ force: !!flags.force
1365
+ }).catch(refusal("docker", targets, vms.length));
1366
+ for (const b of built) outcomes.push(b.ok ? {
1367
+ id: b.id,
1368
+ evals: b.evals,
1369
+ ok: true,
1370
+ ref: b.image.ref,
1371
+ id12: short12(b.image.id)
1372
+ } : {
1373
+ id: b.id,
1374
+ evals: b.evals,
1375
+ ok: false,
1376
+ failure: b.failure
1377
+ });
1378
+ }
1756
1379
  } catch (error) {
1757
- if (error instanceof PrepareError && error.failure.usage) throw new UsageError(error.message);
1758
- throw error;
1380
+ if (!(error instanceof PrepareError)) throw error;
1381
+ if (error.failure.usage) throw new UsageError(error.message);
1382
+ if (error.failure.cause === "enospc") invocation.machineState = true;
1383
+ const { PreflightInfraError } = await import("./preflight-4gWSy0Pe.js");
1384
+ throw new PreflightInfraError(error.message);
1759
1385
  }
1760
- for (const b of built) {
1761
- if (b.ok) {
1762
- const id = b.image.id.replace(/^sha256:/, "").slice(0, 12);
1763
- console.log(`${green("✓")} ${green(`${b.id}: ${b.image.ref}`)}${dim(` ${id}`)}`);
1386
+ for (const o of outcomes) {
1387
+ if (o.ok) {
1388
+ console.log(`${green("✓")} ${green(`${o.id}: ${o.ref}`)}${dim(` ${o.id12}`)}`);
1764
1389
  continue;
1765
1390
  }
1766
- const uses = b.evals ? ` (${b.evals} eval${b.evals === 1 ? "" : "s"})` : "";
1767
- const [headline = "", ...rest] = b.failure.message.split("\n");
1768
- console.log(`${red("✗")} ${red(`${b.id}${uses}: ${headline}`)}`);
1391
+ const uses = o.evals ? ` (${o.evals} eval${o.evals === 1 ? "" : "s"})` : "";
1392
+ const [headline = "", ...rest] = o.failure.message.split("\n");
1393
+ console.log(`${red("✗")} ${red(`${o.id}${uses}: ${headline}`)}`);
1769
1394
  for (const line of rest) console.log(` ${line}`);
1770
1395
  }
1771
- const failed = built.filter((b) => !b.ok);
1772
- const n = built.length;
1773
- console.log(failed.length ? red(`${failed.length} of ${n} image${n === 1 ? "" : "s"} failed to build`) : dim(`${n} image${n === 1 ? "" : "s"} ready`));
1396
+ const failed = outcomes.filter((o) => !o.ok);
1397
+ const n = outcomes.length;
1398
+ console.log(failed.length ? red(`${failed.length} of ${n} image${n === 1 ? "" : "s"} failed to ${vms.length ? targets.length ? "build or pull" : "pull" : "build"}`) : dim(`${n} image${n === 1 ? "" : "s"} ready`));
1774
1399
  if (!failed.length) return EXIT.ok;
1775
- return failed.every((b) => !b.ok && b.failure.usage) ? EXIT.usage : EXIT.infra;
1400
+ if (refused || failed.every((o) => !o.ok && o.failure.usage)) return EXIT.usage;
1401
+ invocation.machineState = failed.every((o) => !o.ok && o.failure.cause === "enospc");
1402
+ return EXIT.infra;
1776
1403
  }
1404
+ /** The first 12 hex digits of an image id or digest. */
1405
+ const short12 = (id) => id.replace(/^sha256:/, "").slice(0, 12);
1777
1406
  /** `reeval run`: check, plan, run the engine, then write the run and pick the exit code. */
1778
1407
  async function runCommand(flags, configPath, given) {
1779
- const resumed = flags.resume === void 0 ? null : await resumedRun(flags, given, configPath);
1408
+ const retry = flags["retry-errors"];
1409
+ if (retry !== void 0 && flags.resume !== void 0) throw new UsageError("--resume and --retry-errors both continue a run: pass one of them.");
1410
+ if (retry !== void 0 && (given.length || flags.tag?.length || flags.trait?.length)) throw new UsageError("--retry-errors runs every infra-errored attempt of the run again, with its plan: leave out filters, --tag and --trait.");
1411
+ const resumed = flags.resume !== void 0 || retry !== void 0 ? await resumedRun(flags, given, configPath) : null;
1412
+ if (resumed?.nothing) {
1413
+ const { nothing, record } = resumed.nothing;
1414
+ const outcome = runOutcome(record);
1415
+ if (flags.json) {
1416
+ console.error(nothing);
1417
+ const { suiteResult } = await import("./suite-CKBa6MKt.js");
1418
+ process.stdout.write(`${JSON.stringify({
1419
+ ...suiteResult(record),
1420
+ outcome
1421
+ }, null, 2)}\n`);
1422
+ } else console.log(nothing);
1423
+ return outcome === "fail" ? EXIT.failed : outcome === "infra_error" ? EXIT.infra : EXIT.ok;
1424
+ }
1780
1425
  const runFlags = resumed?.settings ? {
1781
1426
  ...flags,
1782
1427
  ...resumed.settings
@@ -1792,7 +1437,8 @@ async function runCommand(flags, configPath, given) {
1792
1437
  ...setup.profile ? { profile: setup.profile.name } : {},
1793
1438
  ...explicit === void 0 ? {} : { concurrency: setup.concurrency },
1794
1439
  filters: savedFilters(setup.config.root, filters),
1795
- tags: runFlags.tag ?? []
1440
+ tags: runFlags.tag ?? [],
1441
+ ...runFlags.trait?.length ? { traits: runFlags.trait } : {}
1796
1442
  };
1797
1443
  const ctx = await startRun(setup);
1798
1444
  invocation.runId = ctx.runId;
@@ -1800,7 +1446,7 @@ async function runCommand(flags, configPath, given) {
1800
1446
  if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
1801
1447
  const stepStart = process.env.GITHUB_STEP_SUMMARY;
1802
1448
  if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
1803
- const { engineVersions } = await import("./node-DhIRSrrK.js");
1449
+ const { engineVersions } = await import("./node-D7pAU--0.js");
1804
1450
  const terminal = setup.json && !setup.dryRun ? process.stderr : process.stdout;
1805
1451
  const renderer = createRenderer({
1806
1452
  out: setup.json && setup.dryRun ? { write: () => true } : terminal,
@@ -1819,28 +1465,37 @@ async function runCommand(flags, configPath, given) {
1819
1465
  executed = await executeRun(ctx, setup, filters, renderer, {
1820
1466
  startedWith,
1821
1467
  notes: resumed?.notes ?? [],
1822
- narrowed: !!resumed && (given.length > 0 || !!flags.tag?.length)
1468
+ narrowed: !!resumed && (given.length > 0 || !!flags.tag?.length || !!flags.trait?.length)
1823
1469
  });
1824
1470
  } catch (error) {
1825
1471
  renderer.close();
1826
- if (setup.resume === void 0) await rm(ctx.stagingDir, {
1472
+ if (ctx.retry) await restoreRetry(setup.config.root, ctx.runId);
1473
+ else if (setup.resume === void 0) await rm(ctx.stagingDir, {
1827
1474
  recursive: true,
1828
1475
  force: true
1829
1476
  });
1830
1477
  throw error;
1831
1478
  }
1832
1479
  const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions, leftOut } = executed;
1480
+ const { retried } = executed;
1833
1481
  invocation.unhandledErrors = engineRun.unhandledErrors;
1834
- const engines = [...new Set(plan?.sandboxes.flatMap((s) => s.kind === "docker" ? [s.engine ?? "docker"] : []))];
1835
- if ((interrupted || engineRun.canceled) && engines.length) {
1836
- const { removeRunContainers } = await import("./docker-BlSopHpZ.js");
1837
- await removeRunContainers(ctx.runId, engines);
1838
- }
1482
+ if ((interrupted || engineRun.canceled) && plan) await cleanupRun(ctx.runId, plan.sandboxes);
1839
1483
  if (engineRun.stopped || !plan) {
1840
1484
  live?.remove();
1841
1485
  renderer.close();
1842
1486
  return EXIT.ok;
1843
1487
  }
1488
+ if (ctx.retry && ctx.previous && retried.size) {
1489
+ const { retryEnvironmentChanges } = await import("./retry-ChzISxmy.js");
1490
+ const changes = retryEnvironmentChanges(ctx.previous, ctx.stagingDir, retried);
1491
+ if (changes.length) {
1492
+ questions?.cancel();
1493
+ live?.remove();
1494
+ renderer.close();
1495
+ await restoreRetry(setup.config.root, ctx.runId);
1496
+ throw new UsageError(`run ${ctx.runId}'s plan changed since it ran: ${changes.join("; ")}. ${retryVerb(ctx.retry, ctx.runId)} only with the plan they ran with, so the run is left as it was: put it back, or start a new run with reeval run.`);
1497
+ }
1498
+ }
1844
1499
  let stopped = cutShort || engineRun.canceled;
1845
1500
  if (stopped) questions?.cancel();
1846
1501
  let answers = stopped ? await questions?.answered() : await questions?.settle();
@@ -1850,8 +1505,8 @@ async function runCommand(flags, configPath, given) {
1850
1505
  }
1851
1506
  finishing = true;
1852
1507
  live?.phase("finishing");
1853
- const { lockfiles } = await import("./state-DZbAO95a.js");
1854
- const { writeScratchRun } = await import("./results-CfC0id_u.js");
1508
+ const { lockfiles } = await import("./state-kimtIhBP.js");
1509
+ const { writeScratchRun } = await import("./results-JyuM64gy.js");
1855
1510
  const run = await writeScratchRun({
1856
1511
  ctx,
1857
1512
  author: ctx.author,
@@ -1866,6 +1521,7 @@ async function runCommand(flags, configPath, given) {
1866
1521
  language: { name: "typescript" },
1867
1522
  enginePackages: engineVersions,
1868
1523
  sandbox: sandboxVersions,
1524
+ ...ctx.retry ? { retry: ctx.retry } : {},
1869
1525
  worktreePatch: ctx.worktreePatch,
1870
1526
  lockfiles: await lockfiles(ctx.config.root),
1871
1527
  git: ctx.git,
@@ -1884,7 +1540,7 @@ async function runCommand(flags, configPath, given) {
1884
1540
  status: "complete"
1885
1541
  }) : outcome;
1886
1542
  const code = stopped ? stopCode : ran === "fail" ? EXIT.failed : ran === "infra_error" ? EXIT.infra : EXIT.ok;
1887
- const { pruneDebug } = await import("./files-COW5JBJM.js");
1543
+ const { pruneDebug } = await import("./files-C1Rk9FYA.js");
1888
1544
  await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
1889
1545
  const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
1890
1546
  renderer.onEvent({
@@ -1897,13 +1553,21 @@ async function runCommand(flags, configPath, given) {
1897
1553
  ...notRun > 0 && !stopped ? { notRun } : {}
1898
1554
  });
1899
1555
  renderer.close();
1556
+ if (retried.size) {
1557
+ const { retryCounts, retryLine } = await import("./retry-ChzISxmy.js");
1558
+ const line = retryLine(retryCounts(record, retried, ctx.previous));
1559
+ if (setup.json) console.error(line);
1560
+ else console.log(line);
1561
+ }
1900
1562
  const stepSummary = process.env.GITHUB_STEP_SUMMARY;
1901
1563
  if (stepSummary && run.artifactDir) await appendFile(stepSummary, await readFile(summary)).catch(() => {});
1902
1564
  const { config } = ctx;
1903
1565
  const mode = setup.record;
1904
- const autoRecord = flags.record || mode === "always" || mode === "ci" && inCI();
1566
+ const { recordFile } = await import("./files-DnlA1A7N.js");
1567
+ const rerecord = (setup.retry !== void 0 || setup.resume !== void 0) && record.status === "complete" && existsSync(recordFile(config.history.dir, record.runId));
1568
+ const autoRecord = flags.record || mode === "always" || mode === "ci" && inCI() || rerecord;
1905
1569
  if (autoRecord && record.status !== "interrupted") {
1906
- const { record: recordCmd } = await import("./commands-xYTMNTPv.js");
1570
+ const { record: recordCmd } = await import("./commands-CfdBX-mc.js");
1907
1571
  const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
1908
1572
  const line = await recordCmd(config, record.runId, {
1909
1573
  ...flags,
@@ -1917,7 +1581,7 @@ async function runCommand(flags, configPath, given) {
1917
1581
  else console.log(line);
1918
1582
  }
1919
1583
  if (setup.json) {
1920
- const { suiteResult } = await import("./suite-u2KJnGpY.js");
1584
+ const { suiteResult } = await import("./suite-CKBa6MKt.js");
1921
1585
  process.stdout.write(`${JSON.stringify({
1922
1586
  ...suiteResult(record),
1923
1587
  outcome: ran
@@ -1934,12 +1598,24 @@ async function resumedRun(flags, filters, configPath) {
1934
1598
  const path = requireConfig(configPath);
1935
1599
  loadEnvFiles(path);
1936
1600
  const root = dirname(path);
1937
- const target = await resolveResume(root, flags.resume || void 0);
1601
+ const target = await continuedRun(root, flags);
1602
+ if (flags["retry-errors"] !== void 0 && target.previous && !erroredAttempts(target.previous).length) return {
1603
+ nothing: {
1604
+ nothing: `run ${target.runId} has no infra errors, so --retry-errors has nothing to run again.`,
1605
+ record: target.previous
1606
+ },
1607
+ saved: null,
1608
+ settings: null,
1609
+ notes: [],
1610
+ profileOf: void 0
1611
+ };
1612
+ if (flags.resume !== void 0 && await stoppedRetry(root, target) && (filters.length || flags.tag?.length || flags.trait?.length)) throw new UsageError(`run ${target.runId} stopped during a --retry-errors, which runs every infra-errored attempt of the run again, with its plan: leave out filters, --tag and --trait.`);
1938
1613
  const saved = await savedInvocation(root, target);
1939
1614
  if (!saved) return {
1615
+ nothing: null,
1940
1616
  saved: null,
1941
1617
  settings: null,
1942
- notes: [`run ${target.runId} saved no --runs, profile or filters, so this resume plans with the ones given now`],
1618
+ notes: [`run ${target.runId} saved no --runs, profile or filters, so this ${flags.resume === void 0 ? "retry" : "resume"} plans with the ones given now`],
1943
1619
  profileOf: void 0
1944
1620
  };
1945
1621
  const given = {
@@ -1947,16 +1623,24 @@ async function resumedRun(flags, filters, configPath) {
1947
1623
  profile: flags.profile,
1948
1624
  concurrency: flags.concurrency,
1949
1625
  tag: flags.tag,
1626
+ trait: flags.trait,
1950
1627
  filters
1951
1628
  };
1952
- const resumed = resumeSettings(target.runId, saved, given, { root });
1629
+ const flag = flags.resume === void 0 ? "--retry-errors" : "--resume";
1630
+ const resumed = resumeSettings(target.runId, saved, given, {
1631
+ root,
1632
+ flag
1633
+ });
1953
1634
  const fromRun = flags.profile === void 0 && resumed.settings.profile !== void 0;
1954
1635
  return {
1636
+ nothing: null,
1955
1637
  saved,
1956
1638
  ...resumed,
1957
1639
  profileOf: fromRun ? target.runId : void 0
1958
1640
  };
1959
1641
  }
1642
+ /** The run `--resume` continues, or the finished one `--retry-errors` runs the errors of again. */
1643
+ const continuedRun = (root, flags) => flags["retry-errors"] !== void 0 ? resolveRetry(root, flags["retry-errors"] || void 0) : resolveResume(root, flags.resume || void 0);
1960
1644
  /**
1961
1645
  * Everything that can fail before the evals are collected: output mode, config, profile, flags.
1962
1646
  * `noProfile` is a resume of a run that had none, which `defaultProfile` mustn't give one;
@@ -1987,12 +1671,9 @@ async function prepareRun(flags, configPath, { noProfile = false, profileOf } =
1987
1671
  runs = Number(flags.runs);
1988
1672
  if (!(Number.isInteger(runs) && runs >= 1)) throw new UsageError(`--runs must be a whole number of at least 1 (got ${flags.runs})`);
1989
1673
  }
1990
- const sandboxes = [environmentSandbox(config.environment, config.sandbox), ...profile ? [profileSandbox(config, profile.defaults)] : []];
1674
+ const sandboxes = [...(config.environments.length ? config.environments : [void 0]).map((e) => environmentSandbox(e, config.sandbox)), ...profile ? [profileSandbox(config, profile.defaults)] : []];
1991
1675
  const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
1992
- const container = sandboxes.find((s) => s.kind === "docker");
1993
- if (!container) return hostCapacity();
1994
- const { dockerCapacity } = await import("./src-BkH4GOxu.js");
1995
- return await dockerCapacity(container.engine) ?? hostCapacity();
1676
+ return await sandboxCapacity(sandboxes) ?? hostCapacity();
1996
1677
  });
1997
1678
  return {
1998
1679
  output,
@@ -2000,6 +1681,7 @@ async function prepareRun(flags, configPath, { noProfile = false, profileOf } =
2000
1681
  profile,
2001
1682
  runs,
2002
1683
  tags: [...flags.tag ?? [], ...profile?.tags ?? []],
1684
+ traits: traitFlags(flags.trait),
2003
1685
  concurrency,
2004
1686
  budget: profile?.budget ?? config.budget,
2005
1687
  record: profile?.record ?? config.record,
@@ -2011,37 +1693,48 @@ async function prepareRun(flags, configPath, { noProfile = false, profileOf } =
2011
1693
  /** Named in the plan line when the project root isn't the current directory. */
2012
1694
  configElsewhere: dirname(path) !== process.cwd(),
2013
1695
  /** `undefined` is a new run; `''` is newest resumable; a string is a run id or prefix. */
2014
- resume: flags.resume
1696
+ resume: flags.resume,
1697
+ /** Like `resume`, for `--retry-errors`. */
1698
+ retry: flags["retry-errors"]
2015
1699
  };
2016
1700
  }
1701
+ /** What a retry, or a resume that finishes one, does to the run's errors, for its refusals. */
1702
+ const retryVerb = (retry, runId) => retry.resume ? `--resume finishes run ${runId}'s --retry-errors, which runs its errors again` : "--retry-errors runs its errors again";
2017
1703
  /** The run's id and the state it starts from: commit, uncommitted changes, author. */
2018
- async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
1704
+ async function startRun({ config, profile, runs, tags, traits, concurrency, verbosity, configFile, resume, retry, dryRun }) {
2019
1705
  const startedAt = /* @__PURE__ */ new Date();
2020
1706
  const git = await gitInfo(config.root, [config.history.dir]);
2021
- const { worktreePatch } = await import("./state-DZbAO95a.js");
1707
+ const { worktreePatch } = await import("./state-kimtIhBP.js");
2022
1708
  const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
2023
1709
  const who = await author(config.root);
2024
- const { stagingDir, sweepStaging } = await import("./results-CfC0id_u.js");
2025
- const { scratchDir } = await import("./files-DFtuRQJx.js");
1710
+ const { stagingDir, sweepStaging } = await import("./results-JyuM64gy.js");
1711
+ const { scratchDir } = await import("./files-DnlA1A7N.js");
2026
1712
  const scratch = scratchDir(config.root);
2027
- if (resume !== void 0) {
2028
- const target = await resolveResume(config.root, resume || void 0);
2029
- if (!dryRun) await prepareResume(target, config.root);
1713
+ if (resume !== void 0 || retry !== void 0) {
1714
+ const target = await continuedRun(config.root, {
1715
+ resume,
1716
+ "retry-errors": retry
1717
+ });
1718
+ if (!dryRun && retry === void 0) await prepareResume(target, config.root);
1719
+ else if (!dryRun) holdRetry(target, config.root);
1720
+ const stopped = retry === void 0 ? await stoppedRetry(config.root, target) : null;
2030
1721
  await sweepStaging(config.root, target.runId);
2031
1722
  const aside = dryRun ? {
2032
1723
  record: target.previous,
2033
1724
  artifacts: target.previousArtifacts
2034
1725
  } : await loadPrevious(stagingDir(config.root, target.runId));
2035
1726
  if (!dryRun) try {
2036
- const { removeRunContainers } = await import("./docker-BlSopHpZ.js");
2037
- await removeRunContainers(target.runId, ["docker", "podman"]);
1727
+ await cleanupRun(target.runId);
2038
1728
  } catch {}
1729
+ const { unfoldCases } = await import("./cases-Dy0mifal.js");
1730
+ const previous = aside.record ?? target.previous;
2039
1731
  return {
2040
1732
  runId: target.runId,
2041
1733
  stagingDir: stagingDir(config.root, target.runId),
2042
1734
  config,
2043
1735
  runs,
2044
1736
  tags,
1737
+ traits,
2045
1738
  concurrency,
2046
1739
  profile,
2047
1740
  verbosity,
@@ -2051,8 +1744,17 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
2051
1744
  worktreePatch: patch,
2052
1745
  author: who,
2053
1746
  scratch,
2054
- previous: aside.record ?? target.previous,
2055
- previousArtifacts: aside.artifacts ?? target.previousArtifacts
1747
+ previous: previous && unfoldCases(previous),
1748
+ previousArtifacts: aside.artifacts ?? target.previousArtifacts,
1749
+ ...retry === void 0 ? {} : { retryTarget: target },
1750
+ ...retry !== void 0 ? { retry: {
1751
+ startedAt,
1752
+ resume: false
1753
+ } } : stopped ? { retry: {
1754
+ startedAt,
1755
+ resume: true,
1756
+ ...stopped.began ? { began: stopped.began } : {}
1757
+ } } : {}
2056
1758
  };
2057
1759
  }
2058
1760
  const stamps = new Set([scratch, config.history.dir].flatMap((dir) => existsSync(dir) ? readdirSync(dir).map((name) => name.slice(0, 16)) : []));
@@ -2065,6 +1767,7 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
2065
1767
  config,
2066
1768
  runs,
2067
1769
  tags,
1770
+ traits,
2068
1771
  concurrency,
2069
1772
  profile,
2070
1773
  verbosity,
@@ -2116,12 +1819,12 @@ function namesDeclaredSecret(evals) {
2116
1819
  */
2117
1820
  async function planEfforts(planned) {
2118
1821
  const live = planned.filter((p) => !p.attempt.plan.skip);
2119
- const { effortNotes, withDefaultEffort } = await import("./default-effort-FEidNAIi.js");
1822
+ const { effortNotes, withDefaultEffort } = await import("./default-effort-BAy8ixd1.js");
2120
1823
  let cursor;
2121
1824
  let unchecked = "CURSOR_API_KEY isn't set";
2122
1825
  const key = process.env.CURSOR_API_KEY;
2123
1826
  if (key && live.some((p) => p.attempt.harness.kind === "cursor")) {
2124
- const { checkCursorPlan } = await import("./cursor-catalog-BTK_apL-.js");
1827
+ const { checkCursorPlan } = await import("./cursor-catalog-BSn9xjmo.js");
2125
1828
  const checked = await checkCursorPlan(key, live);
2126
1829
  if (checked instanceof Error) unchecked = `its request failed: ${checked.message}`;
2127
1830
  else {
@@ -2142,21 +1845,23 @@ async function planEfforts(planned) {
2142
1845
  * or Docker problem (exit 2). Then runs the engine with live output and Ctrl-C handling.
2143
1846
  */
2144
1847
  async function executeRun(ctx, setup, filters, renderer, started) {
2145
- const { runId, stagingDir, config, runs, tags, concurrency, profile, verbosity, configFile } = ctx;
1848
+ const { runId, stagingDir, config, runs, tags, traits, concurrency, profile, verbosity } = ctx;
1849
+ const { configFile } = ctx;
2146
1850
  const run = JSON.parse(JSON.stringify({
2147
1851
  runId,
2148
1852
  stagingDir,
2149
1853
  config,
2150
1854
  runs,
2151
1855
  tags,
1856
+ traits,
2152
1857
  concurrency,
2153
1858
  profile,
2154
1859
  verbosity,
2155
1860
  configFile,
2156
1861
  completed: []
2157
1862
  }));
2158
- const { runEngine } = await import("./node-DhIRSrrK.js");
2159
- const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-DqnzfDH1.js");
1863
+ const { runEngine } = await import("./node-D7pAU--0.js");
1864
+ const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-Cgq9NCEX.js");
2160
1865
  let plan;
2161
1866
  let sandboxVersions = { images: [] };
2162
1867
  let preflightStop;
@@ -2166,13 +1871,21 @@ async function executeRun(ctx, setup, filters, renderer, started) {
2166
1871
  let live;
2167
1872
  let kept = /* @__PURE__ */ new Map();
2168
1873
  let leftOut = [];
1874
+ let retried = /* @__PURE__ */ new Set();
2169
1875
  let questions;
2170
1876
  const onPlan = async (collected) => {
2171
1877
  const { planned, excluded, matched, unmatched } = collected;
2172
1878
  if (unmatched.length) throw unmatchedFilter(config, unmatched);
2173
1879
  if (matched === 0) throw noMatch(config, filters, tags);
2174
1880
  checkLimitHarness(profile, config.file, collected);
1881
+ checkTraitKeys(traitFilters(traits, profile), collected.traits);
1882
+ const continuing = setup.resume !== void 0 || ctx.retryTarget !== void 0;
1883
+ const { errored } = continuing ? noteRetries(planned, stagingDir, ctx.previous, false) : { errored: /* @__PURE__ */ new Set() };
2175
1884
  const completed = completedKeysFor(planned, stagingDir, ctx.previous);
1885
+ if (ctx.retry) for (const { key } of planned) {
1886
+ const was = previousAttempt(ctx.previous, key);
1887
+ if (was && was.outcome !== "infra_error") completed.add(key);
1888
+ }
2176
1889
  const skipped = planned.map((p) => completed.has(p.key) ? {
2177
1890
  ...p,
2178
1891
  attempt: {
@@ -2190,7 +1903,25 @@ async function executeRun(ctx, setup, filters, renderer, started) {
2190
1903
  if (named) throw named;
2191
1904
  const efforts = await planEfforts(skipped);
2192
1905
  if (efforts) run.cursorEfforts = efforts;
2193
- const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-CXVVB4RK.js") : void 0;
1906
+ if (ctx.retry && ctx.previous) {
1907
+ const ranOn = ctx.previous.stack?.reeval;
1908
+ if (ranOn && ranOn !== version) throw new UsageError(`run ${runId} ran on reeval ${ranOn}, and this is ${version}. ${retryVerb(ctx.retry, runId)} only on the reeval they ran on: install reeval ${ranOn}, or start a new run with reeval run.`);
1909
+ const { retryChanges } = await import("./retry-ChzISxmy.js");
1910
+ const { judgePart } = await import("./assemble-UPCRrtOz.js");
1911
+ const { engineVersions } = await import("./node-D7pAU--0.js");
1912
+ const judged = judgePart(config, profile, version, engineVersions);
1913
+ const runs = new Set(planned.map((p) => p.key).filter((key) => !completed.has(key)));
1914
+ const changes = await retryChanges(ctx.previous, planned, runs, config.root, judged);
1915
+ if (changes.length) throw new UsageError(`run ${runId}'s plan changed since it ran: ${changes.join("; ")}. ${retryVerb(ctx.retry, runId)} only with the plan they ran with: put it back, or start a new run with reeval run.`);
1916
+ if (ctx.retryTarget && !setup.dryRun) {
1917
+ await prepareResume(ctx.retryTarget, config.root, { held: true });
1918
+ const aside = await loadPrevious(stagingDir);
1919
+ ctx.previous = aside.record ?? ctx.previous;
1920
+ ctx.previousArtifacts = aside.artifacts;
1921
+ }
1922
+ }
1923
+ retried = continuing && !setup.dryRun ? noteRetries(planned, stagingDir, ctx.previous, true).errored : errored;
1924
+ const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-047Dcd-a.js") : void 0;
2194
1925
  const judge = runJudgeModel(config, profile);
2195
1926
  const judgeModel = judge === void 0 ? null : judgeModelName(judge);
2196
1927
  plan = await buildPlan({
@@ -2207,7 +1938,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
2207
1938
  });
2208
1939
  plan.runId = runId;
2209
1940
  const cloud = ciRunner(process.env);
2210
- const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && p.attempt.sandbox.kind === "docker").filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
1941
+ const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && capabilitiesOf(p.attempt.sandbox).isolated).filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
2211
1942
  if (cloud && direct.length) plan.warnings.push(`${directOnCloud(cloud)}. Under it: ${direct.join(", ")}. Keep 'allow-all' to the evals that need raw TCP or UDP; the default, 'proxy', sends everything through the gateway`);
2212
1943
  plan.warnings.push(...started.notes);
2213
1944
  if (started.narrowed) {
@@ -2215,16 +1946,17 @@ async function executeRun(ctx, setup, filters, renderer, started) {
2215
1946
  const notRun = leftOut.filter((l) => !l.done).length;
2216
1947
  if (notRun) plan.warnings.push(`${notRun === 1 ? "1 attempt" : `${notRun} attempts`} run ${runId} planned ${notRun === 1 ? "isn't" : "aren't"} in this resume's filters: ${notRun === 1 ? "it stays" : "they stay"} not run and the run stays incomplete. reeval run --resume ${runId} runs ${notRun === 1 ? "it" : "them"}`);
2217
1948
  }
2218
- if (completed.size) plan.resume = {
1949
+ if (completed.size || retried.size) plan.resume = {
2219
1950
  runId,
2220
- completed: completed.size
1951
+ completed: completed.size,
1952
+ ...retried.size ? { retried: retried.size } : {}
2221
1953
  };
2222
1954
  run.completed = [...completed];
2223
1955
  const done = leftOut.filter((l) => l.done).map((l) => l.key);
2224
1956
  kept = keptAttempts([...completed, ...done], stagingDir, ctx.previous);
2225
- const { imageStages, imageTargets } = await import("./images-CNayXCp2.js");
1957
+ const { imageStages, imageTargets } = await import("./images-BNgcR77K.js");
2226
1958
  const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
2227
- id: p.id,
1959
+ id: p.attempt.family?.id ?? p.id,
2228
1960
  harness: p.attempt.harness,
2229
1961
  sandbox: p.attempt.sandbox,
2230
1962
  plugins: p.attempt.pluginBuilds ?? []
@@ -2241,13 +1973,13 @@ async function executeRun(ctx, setup, filters, renderer, started) {
2241
1973
  return false;
2242
1974
  }
2243
1975
  const elsewhere = setup.configElsewhere ? configFile : void 0;
2244
- const { scratchDir } = await import("./files-DFtuRQJx.js");
1976
+ const { scratchDir } = await import("./files-DnlA1A7N.js");
2245
1977
  const lines = [];
2246
1978
  if (setup.dryRun) {
2247
1979
  const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
2248
1980
  lines.push(...planTable(plan, todos));
2249
- const { imageStatusLines } = await import("./images-CNayXCp2.js");
2250
- const { dockerAvailable, engineSpec } = await import("./src-BkH4GOxu.js");
1981
+ const { imageStatusLines } = await import("./images-BNgcR77K.js");
1982
+ const { dockerAvailable, engineSpec } = await import("./src-DtodMT59.js");
2251
1983
  const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
2252
1984
  const down = [];
2253
1985
  for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
@@ -2256,7 +1988,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
2256
1988
  const names = down.map((e) => engineSpec(e).name).join(" and ");
2257
1989
  lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
2258
1990
  }
2259
- const { costLine } = await import("./cost-BJU0MAji.js");
1991
+ const { costLine } = await import("./cost-Cr26Alhq.js");
2260
1992
  const cost = plan.cost && costLine(plan.cost);
2261
1993
  if (cost) lines.push(cost);
2262
1994
  }
@@ -2314,6 +2046,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
2314
2046
  }
2315
2047
  const missing = missingCredentials(config, process.env, {
2316
2048
  harnesses: plan.harnessDescriptors,
2049
+ sandboxes: plan.sandboxes,
2317
2050
  profile,
2318
2051
  questionModels: [...questionModels],
2319
2052
  judges: [...judges.values()]
@@ -2330,7 +2063,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
2330
2063
  });
2331
2064
  if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
2332
2065
  if (asked.length) {
2333
- const { questionRunner } = await import("./questions-BzdLdV3C.js");
2066
+ const { questionRunner } = await import("./questions-CWlR9ffu.js");
2334
2067
  const warned = /* @__PURE__ */ new Set();
2335
2068
  questions = questionRunner({
2336
2069
  evals: asked,
@@ -2372,7 +2105,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
2372
2105
  }
2373
2106
  });
2374
2107
  }
2375
- const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-B8EOCuxl.js");
2108
+ const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-CcQRT31C.js");
2376
2109
  const live = skipped.filter((p) => !p.attempt.plan.skip);
2377
2110
  const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
2378
2111
  workspace: a.workspace,
@@ -2550,9 +2283,9 @@ async function executeRun(ctx, setup, filters, renderer, started) {
2550
2283
  throw error;
2551
2284
  } finally {
2552
2285
  clearTimeout(budgetTimer);
2553
- const { removePrepared } = await import("./preflight-DVXEvzFU.js");
2286
+ const { removePrepared } = await import("./preflight-R-InZCQb.js");
2554
2287
  removePrepared(config.root, runId);
2555
- const { removePreparedWorkspaces } = await import("./workspace-B8EOCuxl.js");
2288
+ const { removePreparedWorkspaces } = await import("./workspace-CcQRT31C.js");
2556
2289
  removePreparedWorkspaces(config.root, runId);
2557
2290
  }
2558
2291
  if (preflightStop) throw preflightStop;
@@ -2567,7 +2300,9 @@ async function executeRun(ctx, setup, filters, renderer, started) {
2567
2300
  cutShort,
2568
2301
  live,
2569
2302
  questions,
2570
- leftOut
2303
+ leftOut,
2304
+ /** The attempts that ran again after an infra error, by key; none on a dry run. */
2305
+ retried: setup.dryRun ? /* @__PURE__ */ new Set() : retried
2571
2306
  };
2572
2307
  }
2573
2308
  process.stdout.on("error", (error) => {
@@ -2601,7 +2336,7 @@ function finish(code, error, kind) {
2601
2336
  } : null,
2602
2337
  unhandledErrors: invocation.unhandledErrors.map((e) => redactSecrets(e))
2603
2338
  });
2604
- const broke = kind === "crash" || code === EXIT.infra || invocation.unhandledErrors.length > 0;
2339
+ const broke = kind === "crash" || code === EXIT.infra && !invocation.machineState || invocation.unhandledErrors.length > 0;
2605
2340
  const quiet = !!process.env.REEVAL_NO_REPORT_HINT || command === "report" || !!flags?.json || flags?.ui === "agent" || process.env.REEVAL_UI === "agent" || !flags?.ui && !process.env.REEVAL_UI && !!detectAgent().name;
2606
2341
  if (broke && !quiet) console.error(dim("hint: if this looks like a bug in reeval, reeval report collects a redacted report to file (REEVAL_NO_REPORT_HINT=1 hides this)"));
2607
2342
  }
@@ -2635,4 +2370,4 @@ run(process.argv.slice(2)).finally(() => {
2635
2370
  });
2636
2371
  export {};
2637
2372
 
2638
- //# sourceMappingURL=cli-BBOlXKPG.js.map
2373
+ //# sourceMappingURL=cli-20Fpvs5H.js.map