reeval 0.17.2 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (403) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +5 -5
  3. package/dist/{adapter-DyvDj4ZN.js → adapter-C8xPuXYq.js} +5 -4
  4. package/dist/adapter-C8xPuXYq.js.map +1 -0
  5. package/dist/{agent-v036GEgG.js → agent-CBoLndM7.js} +30 -10
  6. package/dist/{agent-v036GEgG.js.map → agent-CBoLndM7.js.map} +1 -1
  7. package/dist/agents-C6zhkxp3.js +1151 -0
  8. package/dist/agents-C6zhkxp3.js.map +1 -0
  9. package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
  10. package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
  11. package/dist/{app-files-hpRT5lj_.js → app-files-DT5mJyV4.js} +4 -4
  12. package/dist/app-files-DT5mJyV4.js.map +1 -0
  13. package/dist/assemble-UPCRrtOz.js +1 -0
  14. package/dist/{assemble-Dlh6Xv3U.js → assemble-t6h018An.js} +194 -307
  15. package/dist/assemble-t6h018An.js.map +1 -0
  16. package/dist/{attempt-CtKGvaQR.js → attempt-CAdlwlNu.js} +3 -3
  17. package/dist/{attempt-CtKGvaQR.js.map → attempt-CAdlwlNu.js.map} +1 -1
  18. package/dist/{attempt-host-DErH-DYV.js → attempt-host-CIm1-11m.js} +18 -10
  19. package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-CIm1-11m.js.map} +1 -1
  20. package/dist/boat-BjLqFJIZ.js +108 -0
  21. package/dist/boat-BjLqFJIZ.js.map +1 -0
  22. package/dist/boat-CJCtzfB1.js +322 -0
  23. package/dist/boat-CJCtzfB1.js.map +1 -0
  24. package/dist/bridge/index.mjs +300 -78
  25. package/dist/capture-DoxFUN3J.js +1 -0
  26. package/dist/{capture-Bdcx9Gap.js → capture-iBzFiC0j.js} +406 -625
  27. package/dist/capture-iBzFiC0j.js.map +1 -0
  28. package/dist/cases-Dy0mifal.js +1 -0
  29. package/dist/cases-z9E6DCd5.js +639 -0
  30. package/dist/cases-z9E6DCd5.js.map +1 -0
  31. package/dist/{cli-CGd51thd.js → cli-20Fpvs5H.js} +929 -576
  32. package/dist/cli-20Fpvs5H.js.map +1 -0
  33. package/dist/{cli-spec-ChkleEVm.js → cli-spec-Dsr79Qy1.js} +46 -18
  34. package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
  35. package/dist/cli.js +2 -2
  36. package/dist/{client-CF-DOjOU.js → client-KM4C5r5_.js} +111 -14
  37. package/dist/client-KM4C5r5_.js.map +1 -0
  38. package/dist/{command-CsfcwMyc.js → command-B5OyHpOu.js} +24 -81
  39. package/dist/command-B5OyHpOu.js.map +1 -0
  40. package/dist/{command-CtTdCF3B.js → command-BLNVDiIg.js} +10 -10
  41. package/dist/{command-CtTdCF3B.js.map → command-BLNVDiIg.js.map} +1 -1
  42. package/dist/{command-CyIb9BSC.js → command-CRh-cb6Z.js} +120 -55
  43. package/dist/command-CRh-cb6Z.js.map +1 -0
  44. package/dist/{command-TLmY-rYp.js → command-aUJvENwy.js} +18 -18
  45. package/dist/command-aUJvENwy.js.map +1 -0
  46. package/dist/{commands-C54S45d7.js → commands-CWy-WndG.js} +239 -92
  47. package/dist/commands-CWy-WndG.js.map +1 -0
  48. package/dist/{commands-jO9Jj35r.js → commands-CfdBX-mc.js} +1 -1
  49. package/dist/{config-ImY6AlRE.js → config-B9mB1kF5.js} +924 -1729
  50. package/dist/config-B9mB1kF5.js.map +1 -0
  51. package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
  52. package/dist/context-Bkv3ndjq.js.map +1 -0
  53. package/dist/{cost-e4IppZpJ.js → cost-BdRgySQl.js} +4 -4
  54. package/dist/{cost-e4IppZpJ.js.map → cost-BdRgySQl.js.map} +1 -1
  55. package/dist/cost-Cr26Alhq.js +1 -0
  56. package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BSn9xjmo.js} +3 -3
  57. package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
  58. package/dist/db-P2fXw4nx.js +2 -0
  59. package/dist/{db-sHGpaLo9.js → db-zgC1tvKB.js} +178 -74
  60. package/dist/db-zgC1tvKB.js.map +1 -0
  61. package/dist/debug-CF4p4Lp_.js +1 -0
  62. package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
  63. package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
  64. package/dist/{default-effort-fpdbYVKm.js → default-effort-BAy8ixd1.js} +1 -1
  65. package/dist/{default-effort-BZRa-SQP.js → default-effort-DGhQmu95.js} +8 -5
  66. package/dist/default-effort-DGhQmu95.js.map +1 -0
  67. package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
  68. package/dist/dist-CAFD1yfv.js.map +1 -0
  69. package/dist/{docker-CWuSRj7a.js → docker-Cqw-CyWz.js} +4 -4
  70. package/dist/{docker-CWuSRj7a.js.map → docker-Cqw-CyWz.js.map} +1 -1
  71. package/dist/{docker--2ytOHcu.js → docker-DAMnm3L9.js} +103 -73
  72. package/dist/docker-DAMnm3L9.js.map +1 -0
  73. package/dist/{docker-CIzHzwlC.js → docker-Dr4XeDc8.js} +1 -1
  74. package/dist/{doctor-eE65YpEv.js → doctor-B2PohneB.js} +86 -29
  75. package/dist/doctor-B2PohneB.js.map +1 -0
  76. package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
  77. package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
  78. package/dist/{environment-Cy3FVLR5.js → environment-DB8o3hGi.js} +80 -40
  79. package/dist/environment-DB8o3hGi.js.map +1 -0
  80. package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
  81. package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
  82. package/dist/{eval-source-BTsTRiBW.js → eval-source-D3sbAzt-.js} +23 -15
  83. package/dist/eval-source-D3sbAzt-.js.map +1 -0
  84. package/dist/evaluate-COdzf7Hl.d.ts +92 -0
  85. package/dist/{evaluate-CFFdyYZX.js → evaluate-CwKeN_fo.js} +472 -159
  86. package/dist/evaluate-CwKeN_fo.js.map +1 -0
  87. package/dist/files-B4KQUiM7.js +1 -0
  88. package/dist/files-BEwfjD8J.js +64 -0
  89. package/dist/files-BEwfjD8J.js.map +1 -0
  90. package/dist/{files-CQU08uh4.js → files-BRzV7aPh.js} +5 -5
  91. package/dist/{files-CQU08uh4.js.map → files-BRzV7aPh.js.map} +1 -1
  92. package/dist/{files-CIX-pLl4.js → files-BiFU16BW.js} +6 -6
  93. package/dist/{files-CIX-pLl4.js.map → files-BiFU16BW.js.map} +1 -1
  94. package/dist/files-C1Rk9FYA.js +1 -0
  95. package/dist/files-C3mCC8Mn.js +1 -0
  96. package/dist/{files-tnMgu7-X.js → files-Cl7BQ5gE.js} +55 -14
  97. package/dist/{files-tnMgu7-X.js.map → files-Cl7BQ5gE.js.map} +1 -1
  98. package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
  99. package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
  100. package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
  101. package/dist/files-DlQshsks.js.map +1 -0
  102. package/dist/files-DnlA1A7N.js +1 -0
  103. package/dist/gate-CgYdaprG.js +49 -0
  104. package/dist/gate-CgYdaprG.js.map +1 -0
  105. package/dist/gateway.d.ts +1 -1
  106. package/dist/{git-BWVkG4Yl.js → git-fDiJ8UIV.js} +3 -3
  107. package/dist/{git-BWVkG4Yl.js.map → git-fDiJ8UIV.js.map} +1 -1
  108. package/dist/harness-C2berMiM.js +105 -0
  109. package/dist/harness-C2berMiM.js.map +1 -0
  110. package/dist/{harness-Ch2P81sx.js → harness-DXv2gnaT.js} +68 -111
  111. package/dist/harness-DXv2gnaT.js.map +1 -0
  112. package/dist/harness.d.ts +1 -1
  113. package/dist/harness.js +1 -1
  114. package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
  115. package/dist/ids-C_advdJA.js.map +1 -0
  116. package/dist/{images-BVZHDf_z.js → images-BNgcR77K.js} +98 -10
  117. package/dist/images-BNgcR77K.js.map +1 -0
  118. package/dist/index.d.ts +23 -3
  119. package/dist/index.js +5 -5
  120. package/dist/init-4RPeD-K-.js +1 -0
  121. package/dist/{init-BWnMbUnc.js → init-DbuZVRrW.js} +8 -7
  122. package/dist/init-DbuZVRrW.js.map +1 -0
  123. package/dist/ink-list-BpfidQ1h.js +274 -0
  124. package/dist/ink-list-BpfidQ1h.js.map +1 -0
  125. package/dist/{invocations-BGRs1r_C.js → invocations-CxGNOlh0.js} +4 -4
  126. package/dist/{invocations-BGRs1r_C.js.map → invocations-CxGNOlh0.js.map} +1 -1
  127. package/dist/judge-COMXT1Hx.js +230 -0
  128. package/dist/judge-COMXT1Hx.js.map +1 -0
  129. package/dist/judge.d.ts +1 -1
  130. package/dist/judge.js +1 -1
  131. package/dist/judgments-CzgpsTqv.js +246 -0
  132. package/dist/judgments-CzgpsTqv.js.map +1 -0
  133. package/dist/kinds-C1oY3qyC.js +697 -0
  134. package/dist/kinds-C1oY3qyC.js.map +1 -0
  135. package/dist/{summary-zP3cCXDN.js → layout-Cm85tQYk.js} +77 -154
  136. package/dist/layout-Cm85tQYk.js.map +1 -0
  137. package/dist/list-DuThsE28.js +1348 -0
  138. package/dist/list-DuThsE28.js.map +1 -0
  139. package/dist/live-CJCK11nh.js +1 -0
  140. package/dist/{live-d_5a5iUv.js → live-CgYbO4Te.js} +24 -4
  141. package/dist/live-CgYbO4Te.js.map +1 -0
  142. package/dist/{live-ColieOai.js → live-CnGrRM6j.js} +8 -8
  143. package/dist/{live-ColieOai.js.map → live-CnGrRM6j.js.map} +1 -1
  144. package/dist/{matchers-B4ZtlJyP.js → matchers-CXLn-1jm.js} +4 -3
  145. package/dist/{matchers-B4ZtlJyP.js.map → matchers-CXLn-1jm.js.map} +1 -1
  146. package/dist/{model-DOXnWH65.js → model-047Dcd-a.js} +1 -1
  147. package/dist/model-Cv4GuTsZ.js +1 -0
  148. package/dist/{model-CFSjV-HJ.js → model-CwwkrCb7.js} +5 -5
  149. package/dist/{model-CFSjV-HJ.js.map → model-CwwkrCb7.js.map} +1 -1
  150. package/dist/{model-DBpdRXtO.js → model-DJjLw-0r.js} +43 -19
  151. package/dist/{model-DBpdRXtO.js.map → model-DJjLw-0r.js.map} +1 -1
  152. package/dist/{model-DOiCiMCD.js → model-V3mMXp6x.js} +213 -559
  153. package/dist/model-V3mMXp6x.js.map +1 -0
  154. package/dist/{node-CwP1G178.js → node-D7pAU--0.js} +34 -25
  155. package/dist/node-D7pAU--0.js.map +1 -0
  156. package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-BGiXt8_B.js} +107 -49
  157. package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
  158. package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-Cw1z8-sq.js} +2 -2
  159. package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
  160. package/dist/options-CHubslQf.js +1 -0
  161. package/dist/{gate-BoDKeuJu.js → pairs-BGb134Vu.js} +3 -49
  162. package/dist/pairs-BGb134Vu.js.map +1 -0
  163. package/dist/{patch-DvLr8wCZ.js → patch-DH4e_Gkd.js} +3 -3
  164. package/dist/{patch-DvLr8wCZ.js.map → patch-DH4e_Gkd.js.map} +1 -1
  165. package/dist/plain-BYUf_fUV.js +267 -0
  166. package/dist/plain-BYUf_fUV.js.map +1 -0
  167. package/dist/{plan-CIfOFhth.js → plan-Cgq9NCEX.js} +117 -45
  168. package/dist/plan-Cgq9NCEX.js.map +1 -0
  169. package/dist/{plugins-BEs2Am3L.js → plugins-BkmPxDiY.js} +11 -268
  170. package/dist/plugins-BkmPxDiY.js.map +1 -0
  171. package/dist/preflight-4gWSy0Pe.js +1 -0
  172. package/dist/{preflight-50Qw5e5K.js → preflight-BElE_q0u.js} +16 -9
  173. package/dist/preflight-BElE_q0u.js.map +1 -0
  174. package/dist/preflight-CtlloeOQ.js +68 -0
  175. package/dist/preflight-CtlloeOQ.js.map +1 -0
  176. package/dist/{preflight-BAIRsTpl.js → preflight-R-InZCQb.js} +1 -1
  177. package/dist/{prepare-8XhzhLHX.js → prepare-C7f5mPl5.js} +9 -9
  178. package/dist/{prepare-8XhzhLHX.js.map → prepare-C7f5mPl5.js.map} +1 -1
  179. package/dist/{prepare-8nvfDFQx.js → prepare-CzkAdbXm.js} +2 -2
  180. package/dist/{prices-C4BRuSe8.js → prices-5m3SzMZ3.js} +3 -3
  181. package/dist/{prices-C4BRuSe8.js.map → prices-5m3SzMZ3.js.map} +1 -1
  182. package/dist/{provenance-Cc8fFE7Z.js → provenance-zsc4--dP.js} +6 -5
  183. package/dist/provenance-zsc4--dP.js.map +1 -0
  184. package/dist/provider-BnUoz4KB.js +1 -0
  185. package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
  186. package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
  187. package/dist/{providers-DZSTlC9R.js → providers-B8xYMeah.js} +152 -9
  188. package/dist/providers-B8xYMeah.js.map +1 -0
  189. package/dist/providers-CVu8e0xS.js +1 -0
  190. package/dist/{query-DXpxaaTH.js → query-Bb3SYICk.js} +3 -3
  191. package/dist/{query-DXpxaaTH.js.map → query-Bb3SYICk.js.map} +1 -1
  192. package/dist/query-DIfkUcPh.js +1 -0
  193. package/dist/{questions-C_Y-oXW6.js → questions-CWlR9ffu.js} +41 -11
  194. package/dist/questions-CWlR9ffu.js.map +1 -0
  195. package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
  196. package/dist/record-6IUzwP9K.js.map +1 -0
  197. package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
  198. package/dist/redact-BhjtnjrU.js.map +1 -0
  199. package/dist/remarks-CAvTSq16.js +1 -0
  200. package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
  201. package/dist/remarks-D9Lcd1fF.js.map +1 -0
  202. package/dist/remarks-DPCONFxZ.js +138 -0
  203. package/dist/remarks-DPCONFxZ.js.map +1 -0
  204. package/dist/{remarks-E4rpi8Ym.js → remarks-vtkPmQb9.js} +327 -41
  205. package/dist/remarks-vtkPmQb9.js.map +1 -0
  206. package/dist/{remote-DZ00A8V_.js → remote-BgYWqogI.js} +6 -6
  207. package/dist/{remote-DZ00A8V_.js.map → remote-BgYWqogI.js.map} +1 -1
  208. package/dist/{resolve-B_JlQ0La.js → resolve-Do0ZaYH6.js} +11 -10
  209. package/dist/resolve-Do0ZaYH6.js.map +1 -0
  210. package/dist/resolve-amw_2Odz.js +1 -0
  211. package/dist/{results-BMJHaQbi.js → results-JyuM64gy.js} +7 -7
  212. package/dist/{results-BMJHaQbi.js.map → results-JyuM64gy.js.map} +1 -1
  213. package/dist/resume-AW5oXzUc.js +691 -0
  214. package/dist/resume-AW5oXzUc.js.map +1 -0
  215. package/dist/retry-ChzISxmy.js +105 -0
  216. package/dist/retry-ChzISxmy.js.map +1 -0
  217. package/dist/{runtime-CSZ1hjlN.js → runtime-BpW1f1Mx.js} +26 -7
  218. package/dist/runtime-BpW1f1Mx.js.map +1 -0
  219. package/dist/sandbox/boat.d.ts +26 -0
  220. package/dist/sandbox/boat.js +37 -0
  221. package/dist/sandbox/boat.js.map +1 -0
  222. package/dist/{sandbox-B5W8lLL1.js → sandbox-kSgr7vo5.js} +31 -3
  223. package/dist/sandbox-kSgr7vo5.js.map +1 -0
  224. package/dist/sandbox.d.ts +20 -2
  225. package/dist/sandbox.js +1 -1
  226. package/dist/server-2_gUnVvf.js +1 -0
  227. package/dist/{server-BfN6q2lH.js → server-Cn8VqO9S.js} +10 -10
  228. package/dist/{server-BfN6q2lH.js.map → server-Cn8VqO9S.js.map} +1 -1
  229. package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
  230. package/dist/shared-Cfi45Bvy.js.map +1 -0
  231. package/dist/slug-IHh3f770.js +136 -0
  232. package/dist/slug-IHh3f770.js.map +1 -0
  233. package/dist/source-CY3etKzw.js +1 -0
  234. package/dist/{source-BD6Bg0Tr.js → source-CYOhR98B.js} +2 -2
  235. package/dist/{source-BD6Bg0Tr.js.map → source-CYOhR98B.js.map} +1 -1
  236. package/dist/src-6Zzyg4Qd.js +2 -0
  237. package/dist/{src-BC2WXULN.js → src-BDhcPtnI.js} +2 -2
  238. package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
  239. package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
  240. package/dist/src-BRMsk9nh.js +2 -0
  241. package/dist/{src-Dp-fWOfG.js → src-CV_7DkPa.js} +47 -45
  242. package/dist/{src-Dp-fWOfG.js.map → src-CV_7DkPa.js.map} +1 -1
  243. package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
  244. package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
  245. package/dist/{src-Cpy9MNS3.js → src-Ck2r_EMT.js} +617 -74
  246. package/dist/src-Ck2r_EMT.js.map +1 -0
  247. package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
  248. package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
  249. package/dist/src-DB7N8TYJ.js +2 -0
  250. package/dist/src-DPDaAYwh.js +2 -0
  251. package/dist/{src-Cx0HJIJ-.js → src-Dbk19BiP.js} +2 -2
  252. package/dist/src-DtodMT59.js +1 -0
  253. package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
  254. package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
  255. package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
  256. package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
  257. package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
  258. package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
  259. package/dist/src-T-OeuJLr.js +2 -0
  260. package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
  261. package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
  262. package/dist/src-zntEDlnx.js +3 -0
  263. package/dist/{state-CdSjXIfE.js → state-BOLt8pJg.js} +17 -17
  264. package/dist/{state-CdSjXIfE.js.map → state-BOLt8pJg.js.map} +1 -1
  265. package/dist/state-kimtIhBP.js +1 -0
  266. package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
  267. package/dist/stats-C18W4xPX.js.map +1 -0
  268. package/dist/suite-CKBa6MKt.js +2 -0
  269. package/dist/suite-COglUhPd.js +1272 -0
  270. package/dist/suite-COglUhPd.js.map +1 -0
  271. package/dist/summary-DU5zaibx.js +148 -0
  272. package/dist/summary-DU5zaibx.js.map +1 -0
  273. package/dist/tart-BTVXp8rz.js +373 -0
  274. package/dist/tart-BTVXp8rz.js.map +1 -0
  275. package/dist/tart-cli-BIC9HqPg.js +37 -0
  276. package/dist/tart-cli-BIC9HqPg.js.map +1 -0
  277. package/dist/tart-cli-BMrkmBjI.js +1 -0
  278. package/dist/tart-images-DZ_Mbh-g.js +233 -0
  279. package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
  280. package/dist/tart-ref-7Qv3cm3B.js +1 -0
  281. package/dist/tart-ref-DYZijjrU.js +55 -0
  282. package/dist/tart-ref-DYZijjrU.js.map +1 -0
  283. package/dist/testing.d.ts +47 -37
  284. package/dist/testing.js +4 -4
  285. package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
  286. package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
  287. package/dist/trace-Dx2NBE1O.js +311 -0
  288. package/dist/trace-Dx2NBE1O.js.map +1 -0
  289. package/dist/traits-DKnYkqeK.js +66 -0
  290. package/dist/traits-DKnYkqeK.js.map +1 -0
  291. package/dist/{transcript-C_IpYMRk.js → transcript-CCf07UmU.js} +3 -3
  292. package/dist/{transcript-C_IpYMRk.js.map → transcript-CCf07UmU.js.map} +1 -1
  293. package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
  294. package/dist/types-C2mLpph3.js.map +1 -0
  295. package/dist/{types-BHR1lGPh.d.ts → types-DBwFsGem.d.ts} +420 -136
  296. package/dist/{judge-ByEUlzOM.js → vercel-CGlSBv63.js} +3 -229
  297. package/dist/vercel-CGlSBv63.js.map +1 -0
  298. package/dist/{vertex-6uSpZ5L8.js → vertex-Eb2wqahk.js} +5 -2
  299. package/dist/vertex-Eb2wqahk.js.map +1 -0
  300. package/dist/vertex-O88TCwgY.js +1 -0
  301. package/dist/view/index.d.ts +238 -90
  302. package/dist/view/index.js +11 -9
  303. package/dist/{views-CKoN5NAx.js → views-DAIiLiV3.js} +8 -7
  304. package/dist/views-DAIiLiV3.js.map +1 -0
  305. package/dist/views-DtRhj2Tn.js +2 -0
  306. package/dist/{worker-entry-D5dB8cgT.js → worker-entry-BIROTPX1.js} +11 -8
  307. package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-BIROTPX1.js.map} +1 -1
  308. package/dist/workspace-CcQRT31C.js +1 -0
  309. package/dist/{workspace-Cho30qlM.js → workspace-uuE51x35.js} +7 -7
  310. package/dist/{workspace-Cho30qlM.js.map → workspace-uuE51x35.js.map} +1 -1
  311. package/package.json +26 -11
  312. package/skills/eval-rubric/SKILL.md +31 -4
  313. package/dist/adapter-DyvDj4ZN.js.map +0 -1
  314. package/dist/app-files-hpRT5lj_.js.map +0 -1
  315. package/dist/assemble-DTXwCL3m.js +0 -1
  316. package/dist/assemble-Dlh6Xv3U.js.map +0 -1
  317. package/dist/capture-BT2fAmpZ.js +0 -1
  318. package/dist/capture-Bdcx9Gap.js.map +0 -1
  319. package/dist/cli-CGd51thd.js.map +0 -1
  320. package/dist/cli-spec-ChkleEVm.js.map +0 -1
  321. package/dist/client-CF-DOjOU.js.map +0 -1
  322. package/dist/command-CsfcwMyc.js.map +0 -1
  323. package/dist/command-CyIb9BSC.js.map +0 -1
  324. package/dist/command-TLmY-rYp.js.map +0 -1
  325. package/dist/commands-C54S45d7.js.map +0 -1
  326. package/dist/config-ImY6AlRE.js.map +0 -1
  327. package/dist/context-BajfiH0S.js.map +0 -1
  328. package/dist/contrasts-DtSX8E_V.js +0 -101
  329. package/dist/contrasts-DtSX8E_V.js.map +0 -1
  330. package/dist/cost-CMzY9VXv.js +0 -1
  331. package/dist/db-ZIk6S2CD.js +0 -2
  332. package/dist/db-sHGpaLo9.js.map +0 -1
  333. package/dist/debug-BnXV7JbX.js +0 -1
  334. package/dist/default-effort-BZRa-SQP.js.map +0 -1
  335. package/dist/dist-ByYk_wAs.js.map +0 -1
  336. package/dist/docker--2ytOHcu.js.map +0 -1
  337. package/dist/doctor-eE65YpEv.js.map +0 -1
  338. package/dist/environment-Cy3FVLR5.js.map +0 -1
  339. package/dist/eval-source-BTsTRiBW.js.map +0 -1
  340. package/dist/evaluate-CFFdyYZX.js.map +0 -1
  341. package/dist/evaluate-DnGOaGQc.d.ts +0 -43
  342. package/dist/files-B3d-8Fe0.js +0 -1
  343. package/dist/files-COW5JBJM.js +0 -1
  344. package/dist/files-Do4Rp3yZ.js +0 -1
  345. package/dist/files-p-uTdOjA.js.map +0 -1
  346. package/dist/gate-BoDKeuJu.js.map +0 -1
  347. package/dist/harness-Ch2P81sx.js.map +0 -1
  348. package/dist/ids-DuK6xl4w.js.map +0 -1
  349. package/dist/images-BVZHDf_z.js.map +0 -1
  350. package/dist/init-BWnMbUnc.js.map +0 -1
  351. package/dist/init-s1jN-PCI.js +0 -1
  352. package/dist/judge-ByEUlzOM.js.map +0 -1
  353. package/dist/live-DmJix4c8.js +0 -1
  354. package/dist/live-d_5a5iUv.js.map +0 -1
  355. package/dist/model-8uZBYqV5.js +0 -1
  356. package/dist/model-DOiCiMCD.js.map +0 -1
  357. package/dist/node-CwP1G178.js.map +0 -1
  358. package/dist/opencode-providers-yNKyXD49.js.map +0 -1
  359. package/dist/plan-CIfOFhth.js.map +0 -1
  360. package/dist/plugins-BEs2Am3L.js.map +0 -1
  361. package/dist/preflight-50Qw5e5K.js.map +0 -1
  362. package/dist/provenance-Cc8fFE7Z.js.map +0 -1
  363. package/dist/provider-DCdTHQpk.js +0 -1
  364. package/dist/providers-CP_bBtYm.js +0 -1
  365. package/dist/providers-DZSTlC9R.js.map +0 -1
  366. package/dist/query-kFPuqBv1.js +0 -1
  367. package/dist/questions-C_Y-oXW6.js.map +0 -1
  368. package/dist/record-DhzwkSXc.js.map +0 -1
  369. package/dist/redact-BEEGc37e.js.map +0 -1
  370. package/dist/remarks-0EXMh6AD.js.map +0 -1
  371. package/dist/remarks-BL18Id09.js +0 -1
  372. package/dist/remarks-DeZZT8PK.js +0 -116
  373. package/dist/remarks-DeZZT8PK.js.map +0 -1
  374. package/dist/remarks-E4rpi8Ym.js.map +0 -1
  375. package/dist/resolve-BJSay360.js +0 -1
  376. package/dist/resolve-B_JlQ0La.js.map +0 -1
  377. package/dist/resume-CexsWuGK.js +0 -434
  378. package/dist/resume-CexsWuGK.js.map +0 -1
  379. package/dist/runtime-CSZ1hjlN.js.map +0 -1
  380. package/dist/sandbox-B5W8lLL1.js.map +0 -1
  381. package/dist/server-SBzrf35T.js +0 -1
  382. package/dist/shared-ogqPJ0uo.js.map +0 -1
  383. package/dist/source-Dn1uqPon.js +0 -1
  384. package/dist/src-BKDZO3ra.js +0 -2
  385. package/dist/src-BxrHipV0.js +0 -3
  386. package/dist/src-C9elednc.js +0 -2
  387. package/dist/src-CM7xdJFA.js +0 -2
  388. package/dist/src-CWEGknZo.js +0 -2
  389. package/dist/src-Cpy9MNS3.js.map +0 -1
  390. package/dist/src-DVaAKCsH.js +0 -1
  391. package/dist/src-DurP--_D.js +0 -2
  392. package/dist/state-Cicc9Oa5.js +0 -1
  393. package/dist/stats-D-Gqwqt0.js.map +0 -1
  394. package/dist/suite-BYJ2Q0S-.js +0 -2
  395. package/dist/suite-BtHPBFYL.js +0 -544
  396. package/dist/suite-BtHPBFYL.js.map +0 -1
  397. package/dist/summary-zP3cCXDN.js.map +0 -1
  398. package/dist/types-C4OkYhZg.js.map +0 -1
  399. package/dist/vertex-6uSpZ5L8.js.map +0 -1
  400. package/dist/vertex-BaAz44YN.js +0 -1
  401. package/dist/views-CKoN5NAx.js.map +0 -1
  402. package/dist/views-CmyX255j.js +0 -2
  403. package/dist/workspace-Dw9DEZox.js +0 -1
@@ -1,37 +1,34 @@
1
1
  import { n as UsageError } from "./errors-ttMTnIYX.js";
2
- import { h as parseDuration } from "./shared-ogqPJ0uo.js";
3
- import { D as profileSandbox, P as workspaceLabel, p as environmentSandbox } from "./environment-Cy3FVLR5.js";
4
- import { C as pluginBuildsLabel } from "./plugins-BEs2Am3L.js";
5
- import { a as fromCwd, o as newRunId } from "./ids-DuK6xl4w.js";
6
- import { M as harnessId, O as checkLimitHarness, R as plannedSandboxIds, U as sandboxId, a as configPathFor, b as resolveModelGateway, c as findConfig, d as malformedCredentials, et as ciRunner, f as missingCredentials, g as selectProfile, h as runJudgeModel, i as configLabel, it as isDirect, l as hostCapacity, m as resolveConcurrency, ot as resolveOutput, p as requireConfig, tt as directOnCloud, u as loadConfig, v as gatewayScope, z as plannedSandboxes } from "./config-ImY6AlRE.js";
7
- import { l as judgeModelName, m as unusedGatewayKey } from "./judge-ByEUlzOM.js";
8
- import { y as vertexSettingsIn } from "./vertex-6uSpZ5L8.js";
9
- import { _ as registerSecret, m as redactSecrets } from "./redact-BEEGc37e.js";
10
- import { a as plural, i as formatDuration, n as clip, o as sandboxWarningLine } from "./text-D--Q5KsA.js";
11
- import { n as loadEnvFiles, t as inCI } from "./env-T86lAqUD.js";
2
+ import { g as parseDuration } from "./shared-Cfi45Bvy.js";
3
+ import { a as plural, i as formatDuration, n as clip } from "./text-B8-W3Cvn.js";
4
+ import { l as sandboxCapacity, n as capabilitiesOf, r as cleanupRun } from "./kinds-C1oY3qyC.js";
5
+ import { N as workspaceLabel, T as profileSandbox, d as environmentSandbox } from "./environment-DB8o3hGi.js";
6
+ import { a as newRunId, i as fromCwd } from "./ids-C_advdJA.js";
7
+ import { I as checkLimitHarness, V as harnessId, X as plannedSandboxes, Y as plannedSandboxIds, _t as resolveOutput, a as configPathFor, b as resolveModelGateway, c as findConfig, d as malformedCredentials, dt as ciRunner, f as missingCredentials, ft as directOnCloud, g as selectProfile, h as runJudgeModel, ht as isDirect, i as configLabel, l as hostCapacity, m as resolveConcurrency, p as requireConfig, u as loadConfig, v as gatewayScope } from "./config-B9mB1kF5.js";
8
+ import { y as vertexSettingsIn } from "./vertex-Eb2wqahk.js";
9
+ import { h as redactSecrets, v as registerSecret } from "./redact-BhjtnjrU.js";
10
+ import { l as judgeModelName, m as unusedGatewayKey } from "./judge-COMXT1Hx.js";
11
+ import { n as loadEnvFiles, t as inCI } from "./env-8x1O8I1M.js";
12
12
  import { o as PACKAGE_VERSION } from "./root-Co0jhOT5.js";
13
- import { W as LABEL$1, st as imageInfo } from "./src-Cpy9MNS3.js";
14
- import { s as debugTime } from "./debug-B5jDImcW.js";
15
- import { r as openRouterFallbacks } from "./openrouter-models-C8nzPKG2.js";
16
- import { t as PrepareError } from "./provider-DZd8t_XW.js";
17
- import { K as triesNote, O as runOutcome, _ as formatUsage } from "./model-DOiCiMCD.js";
18
- import { u as suiteResult } from "./suite-BtHPBFYL.js";
19
- import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-ChkleEVm.js";
13
+ import { r as openRouterFallbacks } from "./openrouter-models-Cw1z8-sq.js";
14
+ import { a as traitFilters, o as traitFlags, t as checkTraitKeys } from "./traits-DKnYkqeK.js";
15
+ import { k as runOutcome } from "./model-V3mMXp6x.js";
16
+ import { u as suiteResult } from "./suite-COglUhPd.js";
17
+ import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-Dsr79Qy1.js";
20
18
  import { n as stripAnsi, t as colors } from "./colors-DcXBvAlU.js";
21
- import { m as render } from "./summary-zP3cCXDN.js";
22
- import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-Cc8fFE7Z.js";
19
+ import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-zsc4--dP.js";
23
20
  import { t as budgetMarker } from "./budget-BcxLYY5L.js";
24
- import { a as prepareResume, c as writeCurrent, i as loadPrevious, l as writeLiveProgress, r as keptAttempts, s as resolveResume, t as completedKeysFor } from "./resume-CexsWuGK.js";
25
- import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-50Qw5e5K.js";
26
- import { n as imageHarness } from "./prepare-8XhzhLHX.js";
27
- import { i as createProviders, n as attemptImage } from "./providers-DZSTlC9R.js";
28
- import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-BGRs1r_C.js";
21
+ import { S as writeLiveProgress, _ as resumeSettings, b as stoppedRetry, c as liveAttempts, d as prepareResume, f as previousAttempt, g as restoreRetry, h as resolveRetry, l as loadPrevious, m as resolveResume, n as erroredAttempts, o as keptAttempts, r as holdRetry, s as leftOutAttempts, t as completedKeysFor, u as noteRetries, v as savedFilters, x as writeCurrent, y as savedInvocation } from "./resume-AW5oXzUc.js";
22
+ import { n as preflight } from "./preflight-CtlloeOQ.js";
23
+ import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-CxGNOlh0.js";
24
+ import { _ as phaseOf, c as pointerLine, d as attemptAxes, f as debugLines, g as outcomeOf, h as offset, i as listLines, m as failureLines, n as SPINNER, p as errorCheck, s as progressText, t as LiveList, u as statusList, v as progressLine, y as tailLines } from "./list-DuThsE28.js";
29
25
  import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
30
26
  import { dirname, join } from "node:path";
31
27
  import { inspect } from "node:util";
32
28
  import { appendFile, readFile, rm } from "node:fs/promises";
33
29
  import { detectAgent } from "std-env";
34
30
  import { Command, Option } from "commander";
31
+ import stringWidth from "string-width";
35
32
  // src/cli-parse.ts
36
33
  const OPTIONS = Object.fromEntries(Object.entries(FLAGS).map(([name, spec]) => {
37
34
  const short = "short" in spec ? `-${spec.short}, ` : "";
@@ -162,259 +159,155 @@ function usageError(error, command) {
162
159
  }
163
160
  return new UsageError(first.charAt(0).toUpperCase() + first.slice(1));
164
161
  }
165
- // src/preflight.ts
166
- /**
167
- * How the terminal names a sandbox to prepare: its cell id, `+ install`, `+ plugin builds`, and a
168
- * harness other than Cursor.
169
- */
170
- function prepareLabel({ sandbox, harness, plugins }) {
171
- if (sandbox.kind !== "docker") return sandboxId(sandbox);
172
- const baked = imageHarness(harness);
173
- const install = sandbox.install?.length ? " + install" : "";
174
- return `${sandboxId(sandbox)}${install}${pluginBuildsLabel(plugins)}${baked === "cursor" ? "" : ` · ${baked}`}`;
175
- }
176
- /** A run that stopped in preflight on an infra failure: exit 3. */
177
- var PreflightInfraError = class extends Error {
178
- name = "PreflightInfraError";
179
- };
162
+ // src/ui/header.ts
163
+ /** ` ➜ `, ` ⚠ `: the glyph column every line of the block starts with. */
164
+ const GLYPH = 5;
165
+ /** `Est. Time: `, the longest key. */
166
+ const KEY = 11;
167
+ /** `Warning: `, the longer note label. */
168
+ const LABEL = 9;
180
169
  /**
181
- * Before any eval starts: checks each sandbox provider, then prepares every distinct sandbox the
182
- * planned attempts need (building Docker images), in parallel, and writes the results for the
183
- * run's workers. A config-level sandbox that fails stops the run (exit 2 for a mistake in the
184
- * config, exit 3 otherwise); any other failure fails only the attempts that need it.
170
+ * `text` broken at spaces into lines of at most `width` columns, not counting color codes; a
171
+ * longer word keeps a line. A `·` stays with the word before it, so no line starts with one.
185
172
  */
186
- async function preflight(uses, options) {
187
- const providers = options.providers ?? createProviders();
188
- const targets = prepareTargets(uses, providers, prepareLabel);
189
- if (!targets.some((t) => t.kind !== "fake")) return { images: [] };
190
- let outcomes;
191
- try {
192
- outcomes = await prepareAll(targets, {
193
- providers,
194
- root: options.root,
195
- ...options.signal ? { signal: options.signal } : {},
196
- ...options.delays ? { delays: options.delays } : {},
197
- ...options.onEvent ? { onEvent: options.onEvent } : {},
198
- log: (line) => process.env.REEVAL_DEBUG && console.error(line)
199
- });
200
- } catch (error) {
201
- if (error instanceof PrepareError) throw error.failure.usage ? new UsageError(error.message) : new PreflightInfraError(error.message);
202
- if (error instanceof RunStopError) throw error.outcome.failure.usage ? new UsageError(error.message) : new PreflightInfraError(error.message);
203
- throw error;
204
- }
205
- writePrepared(options.root, options.runId, outcomes);
206
- const images = [];
207
- for (const o of outcomes) {
208
- if (!o.ok || o.target.sandbox.kind !== "docker") continue;
209
- const { ref, id, agentCli } = attemptImage(o.handle);
210
- if (images.some((i) => i.id === id)) continue;
211
- const info = await imageInfo(ref, o.target.sandbox.engine);
212
- images.push({
213
- ref,
214
- id,
215
- repoDigests: info?.repoDigests ?? [],
216
- agentCli: agentCli ?? info?.labels[LABEL$1.agentVersion] ?? null
217
- });
218
- }
219
- return { images };
173
+ function wrap(text, width) {
174
+ const words = [];
175
+ for (const word of text.split(/\s+/).filter(Boolean)) if (word === "·" && words.length) words[words.length - 1] += " ·";
176
+ else words.push(word);
177
+ const lines = [];
178
+ let line = "";
179
+ for (const word of words) if (line && stripAnsi(line).length + 1 + stripAnsi(word).length > width) {
180
+ lines.push(line);
181
+ line = word;
182
+ } else line = line ? `${line} ${word}` : word;
183
+ return line ? [...lines, line] : lines;
220
184
  }
221
- // src/ui/format.ts
222
- /** Most detail lines a `-vv` payload prints under its line. */
223
- const DEBUG_DETAIL_LINES = 40;
224
- /**
225
- * A debug line as the terminal shows it, `12:50:01 image cache hit …` (with milliseconds under
226
- * `-vv`), then its detail indented under it.
227
- */
228
- function debugLines(c, line, ms) {
229
- const phase = line.phase.padEnd(9);
230
- const head = `${c.dim(debugTime(line.at, ms))} ${c.dim(phase)} ${line.level === 2 ? c.dim(line.text) : line.text}`;
231
- const detail = (line.detail ?? "").split("\n").filter((l) => l.trim());
232
- const shown = detail.slice(0, DEBUG_DETAIL_LINES).map((l) => c.dim(` ${l}`));
233
- const more = detail.length - shown.length;
185
+ const indent = (n) => " ".repeat(n);
186
+ /** `REEVAL v0.16.1`, then the run's facts, one `➜ Key: value` line each. */
187
+ function headerLines(c, h, width) {
188
+ const name = `REEVAL v${h.version}`;
189
+ const title = c.bold(c.depth === 4 ? name : c.teal(name));
190
+ const valueWidth = width - GLYPH - KEY;
191
+ const row = (key, value) => {
192
+ const values = typeof value === "string" ? [value] : value;
193
+ const joined = values.join(", ");
194
+ return (values.length > 1 && stripAnsi(joined).length > valueWidth ? values : wrap(joined, valueWidth)).map((v, i) => i === 0 ? ` ${c.teal("➜")} ${c.dim(`${key}:`)}${indent(KEY - key.length - 1)}${v}` : `${indent(16)}${v}`);
195
+ };
196
+ const many = (one, n, many = `${one}s`) => n === 1 ? one : many;
197
+ const continued = [h.resumed ? `${plural(h.resumed, "attempt")} done` : "", h.retried ? `${plural(h.retried, "error")} to retry` : ""].filter(Boolean);
198
+ const resumed = continued.length ? c.dim(` (resumed, ${continued.join(", ")})`) : "";
199
+ const reason = h.profile && h.profile.reason !== "--profile" ? c.dim(` (${h.profile.reason})`) : "";
234
200
  return [
235
- head,
236
- ...shown,
237
- ...more > 0 ? [c.dim(` … ${more} more lines in trace.ndjson`)] : []
201
+ h.dryRun ? `${title} ${c.dim("dry run")}` : title,
202
+ "",
203
+ ...row("Run", `${h.runId}${resumed}`),
204
+ ...h.config ? row("Config", h.config) : [],
205
+ ...h.profile ? row("Profile", `${h.profile.name}${reason}`) : [],
206
+ ...row(many("Harness", h.harnesses.length, "Harnesses"), h.harnesses.map((id) => harness(c, h, id))),
207
+ ...row(many("Sandbox", h.sandboxes.length, "Sandboxes"), h.sandboxes),
208
+ ...h.judges.length ? row(many("Judge", h.judges.length), h.judges) : [],
209
+ ...row("Scope", scope(c, h)),
210
+ ...row("Est. Time", estimate(c, h)),
211
+ ...h.record && !h.dryRun ? row("Record", h.record) : []
238
212
  ];
239
213
  }
240
- /** Offset into an attempt, `+1:05`. */
241
- function offset(ms) {
242
- const s = Math.max(0, Math.floor(ms / 1e3));
243
- return `+${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`;
244
- }
245
- function outcomeOf(e) {
246
- if (e.state === "skipped") return "skipped";
247
- if (e.meta?.errorKind === "aborted") return "canceled";
248
- return e.meta?.outcome ?? (e.state === "passed" ? "pass" : "fail");
249
- }
250
- /** What tells an eval's attempts apart: the axes that vary (`cursor:composer-2.5 · with-skill`), else the harness. */
251
- function attemptAxes(a, fallback) {
252
- return a?.label.replace(/\s*#\d+$/, "") || a?.harness.kind || fallback;
253
- }
254
- /** A command on one line: the first line, cut to 80 characters at a word boundary. */
255
- function oneLine(text, max = 80) {
256
- const [first = ""] = text.trim().split("\n");
257
- const cut = clip(first, max);
258
- return cut === first && text.trim().includes("\n") ? `${first}…` : cut;
259
- }
260
- /** One line per worker progress event, without prefix or offset. */
261
- function progressLine(c, p) {
262
- switch (p.kind) {
263
- case "sandbox": return c.dim("sandbox starting");
264
- case "sandbox-ready": return `sandbox ready ${c.dim(formatDuration(p.ms))}`;
265
- case "turn-start": return `turn ${p.turn} started`;
266
- case "turn-end": {
267
- const status = p.status === "completed" ? "done" : c.red(p.status);
268
- const activity = [plural(p.tools, "tool call"), plural(p.fileEdits, "file edit")].join(", ");
269
- const error = p.error ? `: ${c.red(oneLine(p.error, 160))}` : "";
270
- return `turn ${p.turn} ${status}${error} ${c.dim(`${formatDuration(p.ms)} · ${activity}`)}`;
271
- }
272
- case "check-start": return c.dim(`$ ${oneLine(p.command)}`);
273
- case "check-end": {
274
- const exit = p.exitCode === 0 ? c.dim("exit 0") : c.red(`exit ${p.exitCode}`);
275
- const tries = p.try && p.tries ? triesNote(p.exitCode, p.try, p.tries) : null;
276
- return `$ ${oneLine(p.command)} ${exit} ${c.dim([formatDuration(p.ms), tries].filter(Boolean).join(" · "))}`;
277
- }
278
- case "judge-start": return c.dim(`judging: ${oneLine(p.question, 120)}`);
279
- case "judge-end": return judgeLine(c, p);
280
- case "cleanup": return c.dim("cleaning up");
281
- case "warning": return c.yellow(`warning: ${p.text}`);
282
- case "browser-download": {
283
- const total = p.bytes == null ? "?" : formatBytes(p.bytes);
284
- return `downloading ${p.name} ${p.version} ${formatBytes(p.received)} / ${total} → ${p.cache}`;
285
- }
286
- }
287
- }
288
- function formatBytes(n) {
289
- if (n < 1024) return `${n} B`;
290
- if (n < 1048576) return `${(n / 1024).toFixed(1)} KB`;
291
- return `${(n / 1048576).toFixed(1)} MB`;
292
- }
293
- function judgeLine(c, j) {
294
- const verdict = j.verdict === "pass" ? c.green("pass") : c.red(j.verdict);
295
- const score = j.score === null ? "" : ` ${j.score.toFixed(2)}`;
296
- const threshold = j.threshold === null ? "" : ` ${c.dim(`(≥ ${j.threshold})`)}`;
297
- return `${c.dim("judge")} ${verdict}${score}${threshold} ${j.question}`;
298
- }
299
- /** The phase a running attempt is in, for the heartbeat. */
300
- function phaseOf(p) {
301
- switch (p?.kind) {
302
- case void 0: return "starting";
303
- case "sandbox": return "sandbox";
304
- case "sandbox-ready": return "sandbox ready";
305
- case "turn-start": return `turn ${p.turn}`;
306
- case "turn-end": return `turn ${p.turn} done`;
307
- case "check-start":
308
- case "check-end": return "checks";
309
- case "judge-start":
310
- case "judge-end": return "judging";
311
- case "cleanup": return "cleanup";
312
- case "browser-download": return "browser download";
313
- case "warning": return "running";
314
- }
315
- }
316
- function capLines(text, max) {
317
- const lines = text.replace(/\s+$/, "").split("\n");
318
- if (lines.length <= max) return lines;
319
- return [...lines.slice(0, max), `(+${lines.length - max} more lines)`];
214
+ /** A harness id, with the effort it runs at when the id doesn't name it. */
215
+ function harness(c, h, id) {
216
+ const effort = h.efforts?.[id];
217
+ if (effort === void 0 || id.includes("[effort=")) return id;
218
+ return `${id} ${`(effort: ${effort})`.split(" ").map((w) => c.dim(w)).join(" ")}`;
320
219
  }
321
- /** An infra error's headline, then its last lines: the cause is at the end. */
322
- function tailLines(text, max) {
323
- const lines = text.replace(/\s+$/, "").split("\n");
324
- if (lines.length <= max) return lines;
325
- const rest = Math.max(1, max - 1);
326
- return [
327
- lines[0] ?? "",
328
- `(${lines.length - 1 - rest} more lines)`,
329
- ...lines.slice(-rest)
220
+ function scope(c, h) {
221
+ const attempts = h.fewest < h.attempts ? `${h.fewest}–${h.attempts} attempts` : plural(h.attempts, "attempt");
222
+ const parts = [
223
+ plural(h.files, "eval file"),
224
+ plural(h.evals, "eval") + (h.skipped ? ` (${h.skipped} skipped)` : ""),
225
+ plural(h.cells, "cell"),
226
+ attempts
330
227
  ];
228
+ const caps = [h.budget.attempts !== void 0 ? `${h.budget.attempts} ${h.attempts <= h.budget.attempts ? c.green("✓") : c.red("✗")}` : "", h.budget.duration ?? ""].filter(Boolean);
229
+ return `${parts.join(" · ")}${caps.length ? ` (budget: ${caps.join(", ")})` : ""}`;
331
230
  }
332
- /**
333
- * Why an eval failed, as lines: the judge verdict (and reason) for judge failures, Vitest's
334
- * formatted error (diff, code frame) for assertions, the tail of the message for infra errors.
335
- */
336
- function failureLines(c, e, options) {
337
- const outcome = outcomeOf(e);
338
- if (outcome === "canceled") return [c.yellow("canceled")];
339
- const message = e.errors[0] ?? "";
340
- const turns = (e.meta?.turnErrors ?? []).filter((t) => !message.includes(t.replace(/^turn \d+ error: /, ""))).map((t) => c.yellow(t));
341
- return [
342
- ...failureBody(c, e, outcome, message, options),
343
- ...otherChecks(c, e),
344
- ...turns
345
- ];
231
+ function estimate(c, h) {
232
+ const serial = `~${formatDuration(h.estimateMs)} ${c.dim("(serial)")}`;
233
+ const most = Math.min(h.concurrency, h.attempts);
234
+ const parallel = h.parallel ?? most;
235
+ if (parallel <= 1) return serial;
236
+ const capped = parallel < most ? `, ${parallel} at once` : "";
237
+ return `${`~${formatDuration(h.estimateMs / parallel)} ${c.dim(`(concurrency: ${h.concurrency}${capped})`)}`} · ${serial}`;
346
238
  }
347
- /** The failed check the attempt's error is, when it is one. */
348
- function errorCheck(e) {
349
- const message = e.errors[0] ?? "";
350
- return (e.meta?.failedChecks ?? []).find((x) => x.message && message.includes(x.message.replace(/…$/, "")));
239
+ /** `⚠ Warning: …` or `ℹ Hint: …`, wrapped under its first word. */
240
+ function noteLines(c, kind, text, width) {
241
+ const [glyph, label, paint] = kind === "hint" ? [
242
+ "ℹ",
243
+ "Hint:",
244
+ c.blue
245
+ ] : [
246
+ "⚠",
247
+ "Warning:",
248
+ kind === "root" ? c.red : c.orange
249
+ ];
250
+ const basic = {
251
+ warning: c.yellow,
252
+ hint: c.teal,
253
+ root: c.red
254
+ }[kind];
255
+ const lead = c.depth === 4 && kind !== "root" ? `${basic(glyph)} ${c.bold(label)}` : c.bold(paint(`${glyph} ${label}`));
256
+ const at = 14;
257
+ return wrap(text, Math.max(20, width - at)).map((line, i) => i === 0 ? ` ${lead}${indent(LABEL - label.length)}${line}` : `${indent(at)}${line}`);
351
258
  }
352
- /** The failed checks the error isn't, each with its line: an `expect` failure doesn't end the body. */
353
- function otherChecks(c, e) {
354
- const shown = errorCheck(e);
355
- return (e.meta?.failedChecks ?? []).filter((x) => x !== shown).map((x) => {
356
- const why = x.message.startsWith(`${x.label}: `) ? x.message.slice(x.label.length + 2) : x.message;
357
- return [
358
- c.red(`✗ ${x.label}`),
359
- x.at && c.dim(x.at),
360
- why && c.dim(why)
361
- ].filter(Boolean).join(" ");
362
- });
259
+ /** A step line, `◆ Preparing 1 sandbox (docker)...`, its text wrapped under itself. */
260
+ function stepLines(glyph, text, width) {
261
+ return wrap(text, Math.max(20, width - GLYPH)).map((line, i) => i === 0 ? ` ${glyph} ${line}` : `${indent(GLYPH)}${line}`);
363
262
  }
364
- function failureBody(c, e, outcome, message, options) {
365
- if (outcome === "infra_error") {
366
- if (e.meta?.prepare) {
367
- const lines = message.trimEnd().split("\n");
368
- return [
369
- c.yellow("infra error"),
370
- lines[0] ?? "",
371
- lines.at(-1) ?? ""
372
- ].filter((l, i, all) => all.indexOf(l) === i);
373
- }
374
- return [`${c.yellow("infra error")}`, ...tailLines(message, options.max)];
375
- }
376
- const failed = (e.meta?.judgements ?? []).filter((j) => j.verdict !== "pass");
377
- if (e.meta?.errorKind === "judge" && failed.length) return failed.flatMap((j, i) => {
378
- const reason = options.reasons?.[i];
379
- return [judgeLine(c, {
380
- ...j,
381
- threshold: j.threshold
382
- }), ...reason ? [` ${reason}`] : []];
383
- });
384
- const formatted = e.formattedErrors[0];
385
- const body = formatted && e.meta?.errorKind !== "timeout" ? formatted : message;
386
- return capLines((c.enabled ? body : stripAnsi(body)).replace(/^\n+/, ""), options.max).map((l) => c.enabled && body === message ? c.red(l) : l);
263
+ /** A run-level notice (a prepared workspace, the Ctrl-C one) as a step line. */
264
+ function noticeLines(c, e, width) {
265
+ const glyph = {
266
+ info: c.dim("ℹ"),
267
+ warn: c.orange("⚠"),
268
+ error: c.red("✗")
269
+ }[e.level];
270
+ const lines = stepLines(glyph, e.text, width);
271
+ return e.level === "info" ? lines.map((l) => c.dim(l)) : lines;
387
272
  }
388
- /** Preflight's lines: which sandboxes it prepares, the slow ones when ready, and each failure. */
389
- function prepareLines(c, e, max) {
273
+ /** Preflight: which sandboxes it prepares, each one ready, and each failure. */
274
+ function prepareStepLines(c, e, width, max) {
390
275
  if (e.state === "start") {
391
276
  const n = e.targets.length;
392
277
  const names = e.targets.map((t) => t.label).join(", ");
393
- return [c.dim(`preparing ${n} sandbox${n === 1 ? "" : "es"}: ${names}`)];
278
+ return stepLines(c.teal("◆"), `Preparing ${plural(n, "sandbox", "sandboxes")} (${names})...`, width);
394
279
  }
395
280
  const { outcome } = e;
396
- if (outcome.ok) return outcome.ms >= 1e3 ? [c.dim(`${c.green("✓")} ${outcome.target.label} ready ${formatDuration(outcome.ms)}`)] : [];
281
+ if (outcome.ok) {
282
+ const took = outcome.ms >= 1e3 ? ` ${c.dim(formatDuration(outcome.ms))}` : "";
283
+ return [` ${c.green("✓")} ${outcome.target.label} ready${took}`];
284
+ }
397
285
  const who = outcome.target.setBy.join(", ");
398
- return [`${c.red("✗")} ${outcome.target.label} failed to prepare ${c.dim(`(set by ${who}; only its attempts fail)`)}`, ...tailLines(outcome.failure.message, max).map((l) => ` ${c.yellow(l)}`)];
286
+ return [` ${c.red("✗")} ${outcome.target.label} failed to prepare ${c.dim(`(set by ${who}; only its attempts fail)`)}`, ...tailLines(outcome.failure.message, max).map((l) => `${indent(GLYPH)}${c.orange(l)}`)];
287
+ }
288
+ /** The start line's text: how the attempts' logs will print. */
289
+ function startText(output) {
290
+ return output.outputLogs === "none" ? "Running..." : output.outputLogs === "errors-only" ? "Running; logs print for failed attempts only..." : output.logOrder === "stream" ? "Streaming logs..." : "Running; each attempt's logs print as it ends...";
291
+ }
292
+ /** The last line before the attempts' own: how their logs will print. */
293
+ function startLine(c, output) {
294
+ return ` ${c.teal("◆")} ${startText(output)}`;
295
+ }
296
+ /** What the start line's text becomes once the run ends, on a terminal that redraws it. */
297
+ function endText(status) {
298
+ return status === "interrupted" ? "Canceled." : status === "budget" ? "Stopped at the budget." : "Finished.";
399
299
  }
400
- // src/ui/result.ts
401
- function resultLines(c, e, verbose) {
402
- const { record } = e;
403
- const tokens = formatUsage(record.cost.usage);
404
- const notes = [
405
- ...record.status === "budget" ? [c.yellow("Budget reached: attempts that hadn’t started were skipped")] : [],
406
- ...record.status === "interrupted" ? [c.yellow("Interrupted: attempts that hadn’t finished were canceled")] : [],
407
- c.dim([
408
- formatDuration(record.durationMs),
409
- tokens,
410
- `results ${e.summaryPath}`
411
- ].filter(Boolean).join(" · "))
300
+ /** The plan as the header block: its facts, `--dry-run`'s table, then each kind of note. */
301
+ function planLines(c, e, width) {
302
+ const sections = [
303
+ e.details,
304
+ [...e.warnings.flatMap((w) => noteLines(c, "warning", w, width)), ...(e.sandboxWarnings ?? []).flatMap((w) => noteLines(c, w.kind === "root" ? "root" : "warning", w.message, width))],
305
+ e.hints.flatMap((h) => noteLines(c, "hint", h, width))
412
306
  ];
413
- return render(suiteResult(record), e.remarks ?? null, {
414
- color: c.enabled,
415
- verbose,
416
- notes
417
- });
307
+ const out = headerLines(c, e.header, width);
308
+ for (const lines of sections) if (lines.length) out.push("", ...lines);
309
+ out.push("");
310
+ return out;
418
311
  }
419
312
  // src/ui/tags.ts
420
313
  const STOPWORDS = new Set("a an the and or of to in on for with without from by at as is are be it its that this when than then into via not no".split(" "));
@@ -475,6 +368,8 @@ var Tagger = class {
475
368
  };
476
369
  // src/ui/agent.ts
477
370
  const MAX_LINES = 12;
371
+ /** Agents read whole lines, so the header's notes don't wrap. */
372
+ const WIDTH = Number.POSITIVE_INFINITY;
478
373
  var AgentRenderer = class {
479
374
  #o;
480
375
  #c = colors(false);
@@ -490,14 +385,13 @@ var AgentRenderer = class {
490
385
  onEvent(e) {
491
386
  switch (e.type) {
492
387
  case "plan":
493
- this.#write(e.line);
494
- for (const line of e.details) this.#write(line);
495
- for (const w of e.warnings) this.#write(`warning: ${w}`);
496
- for (const h of e.hints) this.#write(`hint: ${h}`);
497
- for (const w of e.sandboxWarnings ?? []) this.#write(sandboxWarningLine(w));
388
+ for (const line of planLines(this.#c, e, WIDTH)) this.#write(line);
498
389
  return;
499
390
  case "run-start":
500
- this.#write(`reeval run: ${e.runId} · ${e.files} eval file${e.files === 1 ? "" : "s"}`);
391
+ this.#write(startLine(this.#c, {
392
+ ...this.#o.output,
393
+ outputLogs: "none"
394
+ }));
501
395
  return;
502
396
  case "collected":
503
397
  this.#tags.addFile(e.evals);
@@ -519,148 +413,30 @@ var AgentRenderer = class {
519
413
  return;
520
414
  }
521
415
  case "prepare":
522
- for (const line of prepareLines(this.#c, e, MAX_LINES)) this.#write(line);
416
+ for (const line of prepareStepLines(this.#c, e, WIDTH, MAX_LINES)) this.#write(line);
523
417
  return;
524
418
  case "notice":
525
- if (e.level !== "info") this.#write(e.text);
419
+ for (const line of noticeLines(this.#c, e, WIDTH)) this.#write(line);
526
420
  return;
527
- case "run-end":
421
+ case "run-end": {
528
422
  this.#write("");
529
- for (const line of resultLines(this.#c, e, (this.#o.verbosity ?? 0) > 0)) this.#write(line);
423
+ const verbose = (this.#o.verbosity ?? 0) > 0;
424
+ const list = statusList({
425
+ record: e.record,
426
+ suite: suiteResult(e.record),
427
+ remarks: e.remarks ?? null,
428
+ verbose,
429
+ ...e.notRun ? { notRun: e.notRun } : {}
430
+ });
431
+ const pointer = pointerLine(e.summaryPath);
432
+ for (const line of listLines(list, this.#c, false, pointer, verbose, true)) this.#write(line);
530
433
  return;
434
+ }
531
435
  default: return;
532
436
  }
533
437
  }
534
438
  close() {}
535
439
  };
536
- // src/ui/header.ts
537
- /** ` ➜ `, ` ⚠ `: the glyph column every line of the block starts with. */
538
- const GLYPH = 5;
539
- /** `Est. Time: `, the longest key. */
540
- const KEY = 11;
541
- /** `Warning: `, the longer note label. */
542
- const LABEL = 9;
543
- /**
544
- * `text` broken at spaces into lines of at most `width` columns, not counting color codes; a
545
- * longer word keeps a line. A `·` stays with the word before it, so no line starts with one.
546
- */
547
- function wrap(text, width) {
548
- const words = [];
549
- for (const word of text.split(/\s+/).filter(Boolean)) if (word === "·" && words.length) words[words.length - 1] += " ·";
550
- else words.push(word);
551
- const lines = [];
552
- let line = "";
553
- for (const word of words) if (line && stripAnsi(line).length + 1 + stripAnsi(word).length > width) {
554
- lines.push(line);
555
- line = word;
556
- } else line = line ? `${line} ${word}` : word;
557
- return line ? [...lines, line] : lines;
558
- }
559
- const indent = (n) => " ".repeat(n);
560
- /** `REEVAL v0.16.1`, then the run's facts, one `➜ Key: value` line each. */
561
- function headerLines(c, h, width) {
562
- const name = `REEVAL v${h.version}`;
563
- const title = c.bold(c.depth === 4 ? name : c.teal(name));
564
- const valueWidth = width - GLYPH - KEY;
565
- const row = (key, value) => {
566
- const values = typeof value === "string" ? [value] : value;
567
- const joined = values.join(", ");
568
- return (values.length > 1 && stripAnsi(joined).length > valueWidth ? values : wrap(joined, valueWidth)).map((v, i) => i === 0 ? ` ${c.teal("➜")} ${c.dim(`${key}:`)}${indent(KEY - key.length - 1)}${v}` : `${indent(16)}${v}`);
569
- };
570
- const many = (one, n, many = `${one}s`) => n === 1 ? one : many;
571
- const resumed = h.resumed ? c.dim(` (resumed, ${plural(h.resumed, "attempt")} done)`) : "";
572
- const reason = h.profile && h.profile.reason !== "--profile" ? c.dim(` (${h.profile.reason})`) : "";
573
- return [
574
- h.dryRun ? `${title} ${c.dim("dry run")}` : title,
575
- "",
576
- ...row("Run", `${h.runId}${resumed}`),
577
- ...h.config ? row("Config", h.config) : [],
578
- ...h.profile ? row("Profile", `${h.profile.name}${reason}`) : [],
579
- ...row(many("Harness", h.harnesses.length, "Harnesses"), h.harnesses.map((id) => harness(c, h, id))),
580
- ...row(many("Sandbox", h.sandboxes.length, "Sandboxes"), h.sandboxes),
581
- ...h.judges.length ? row(many("Judge", h.judges.length), h.judges) : [],
582
- ...row("Scope", scope(c, h)),
583
- ...row("Est. Time", estimate(c, h)),
584
- ...h.record && !h.dryRun ? row("Record", h.record) : []
585
- ];
586
- }
587
- /** A harness id, with the effort it runs at when the id doesn't name it. */
588
- function harness(c, h, id) {
589
- const effort = h.efforts?.[id];
590
- if (effort === void 0 || id.includes("[effort=")) return id;
591
- return `${id} ${`(effort: ${effort})`.split(" ").map((w) => c.dim(w)).join(" ")}`;
592
- }
593
- function scope(c, h) {
594
- const attempts = h.fewest < h.attempts ? `${h.fewest}–${h.attempts} attempts` : plural(h.attempts, "attempt");
595
- const parts = [
596
- plural(h.files, "eval file"),
597
- plural(h.evals, "eval") + (h.skipped ? ` (${h.skipped} skipped)` : ""),
598
- plural(h.cells, "cell"),
599
- attempts
600
- ];
601
- const caps = [h.budget.attempts !== void 0 ? `${h.budget.attempts} ${h.attempts <= h.budget.attempts ? c.green("✓") : c.red("✗")}` : "", h.budget.duration ?? ""].filter(Boolean);
602
- return `${parts.join(" · ")}${caps.length ? ` (budget: ${caps.join(", ")})` : ""}`;
603
- }
604
- function estimate(c, h) {
605
- const serial = `~${formatDuration(h.estimateMs)} ${c.dim("(serial)")}`;
606
- const parallel = Math.min(h.concurrency, h.attempts);
607
- if (parallel <= 1) return serial;
608
- return `${`~${formatDuration(h.estimateMs / parallel)} ${c.dim(`(concurrency: ${h.concurrency})`)}`} · ${serial}`;
609
- }
610
- /** `⚠ Warning: …` or `ℹ Hint: …`, wrapped under its first word. */
611
- function noteLines(c, kind, text, width) {
612
- const [glyph, label, paint] = kind === "hint" ? [
613
- "ℹ",
614
- "Hint:",
615
- c.blue
616
- ] : [
617
- "⚠",
618
- "Warning:",
619
- kind === "root" ? c.red : c.orange
620
- ];
621
- const basic = {
622
- warning: c.yellow,
623
- hint: c.teal,
624
- root: c.red
625
- }[kind];
626
- const lead = c.depth === 4 && kind !== "root" ? `${basic(glyph)} ${c.bold(label)}` : c.bold(paint(`${glyph} ${label}`));
627
- const at = 14;
628
- return wrap(text, Math.max(20, width - at)).map((line, i) => i === 0 ? ` ${lead}${indent(LABEL - label.length)}${line}` : `${indent(at)}${line}`);
629
- }
630
- /** A step line, `◆ Preparing 1 sandbox (docker)...`, its text wrapped under itself. */
631
- function stepLines(glyph, text, width) {
632
- return wrap(text, Math.max(20, width - GLYPH)).map((line, i) => i === 0 ? ` ${glyph} ${line}` : `${indent(GLYPH)}${line}`);
633
- }
634
- /** A run-level notice (a prepared workspace, the Ctrl-C one) as a step line. */
635
- function noticeLines(c, e, width) {
636
- const glyph = {
637
- info: c.dim("ℹ"),
638
- warn: c.orange("⚠"),
639
- error: c.red("✗")
640
- }[e.level];
641
- const lines = stepLines(glyph, e.text, width);
642
- return e.level === "info" ? lines.map((l) => c.dim(l)) : lines;
643
- }
644
- /** Preflight: which sandboxes it prepares, each one ready, and each failure. */
645
- function prepareStepLines(c, e, width, max) {
646
- if (e.state === "start") {
647
- const n = e.targets.length;
648
- const names = e.targets.map((t) => t.label).join(", ");
649
- return stepLines(c.teal("◆"), `Preparing ${plural(n, "sandbox", "sandboxes")} (${names})...`, width);
650
- }
651
- const { outcome } = e;
652
- if (outcome.ok) {
653
- const took = outcome.ms >= 1e3 ? ` ${c.dim(formatDuration(outcome.ms))}` : "";
654
- return [` ${c.green("✓")} ${outcome.target.label} ready${took}`];
655
- }
656
- const who = outcome.target.setBy.join(", ");
657
- return [` ${c.red("✗")} ${outcome.target.label} failed to prepare ${c.dim(`(set by ${who}; only its attempts fail)`)}`, ...tailLines(outcome.failure.message, max).map((l) => `${indent(GLYPH)}${c.orange(l)}`)];
658
- }
659
- /** The last line before the attempts' own: how their logs will print. */
660
- function startLine(c, output) {
661
- const text = output.outputLogs === "none" ? "Running..." : output.outputLogs === "errors-only" ? "Running; logs print for failed attempts only..." : output.logOrder === "stream" ? "Streaming logs..." : "Running; each attempt's logs print as it ends...";
662
- return ` ${c.teal("◆")} ${text}`;
663
- }
664
440
  // src/ui/plain.ts
665
441
  /** In a grouped block, an end line replaces its start line and keeps its start time. */
666
442
  const STARTS = {
@@ -702,6 +478,8 @@ var PlainRenderer = class {
702
478
  #queued = /* @__PURE__ */ new Set();
703
479
  #runStartedAt = 0;
704
480
  #lastWrite = 0;
481
+ /** The last line written was blank, so the list needs no gap above it. */
482
+ #blank = false;
705
483
  #timer;
706
484
  /** On a TTY: the status line is drawn under everything else, and redrawn after each write. */
707
485
  #live;
@@ -710,8 +488,11 @@ var PlainRenderer = class {
710
488
  #evals = /* @__PURE__ */ new Set();
711
489
  #tries = /* @__PURE__ */ new Set();
712
490
  #ended = /* @__PURE__ */ new Set();
491
+ #list;
492
+ #lastProgress = 0;
713
493
  constructor(options) {
714
494
  this.#o = options;
495
+ this.#list = new LiveList(options.harness);
715
496
  this.#c = options.colors;
716
497
  this.#now = options.now ?? Date.now;
717
498
  this.#live = options.output.statusLine;
@@ -725,6 +506,7 @@ var PlainRenderer = class {
725
506
  const line = `${text.trimEnd()}\n`;
726
507
  this.#o.out.write(this.#status ? `${CLEAR}${line}${this.#status}` : line);
727
508
  this.#lastWrite = this.#now();
509
+ this.#blank = line === "\n";
728
510
  }
729
511
  /** `7 evals · 12/28 tries`: what's collected so far and how many tries ended. */
730
512
  #drawStatus() {
@@ -775,31 +557,17 @@ var PlainRenderer = class {
775
557
  onEvent(e) {
776
558
  const c = this.#c;
777
559
  switch (e.type) {
778
- case "run-start": {
560
+ case "run-start":
779
561
  this.#runStartedAt = e.at;
780
- if (this.#o.output.rich) {
781
- this.#write(startLine(c, this.#o.output));
782
- this.#write("");
783
- return;
784
- }
785
- const files = `${e.files} eval file${e.files === 1 ? "" : "s"}`;
786
- const order = `${this.#o.output.logOrder} logs`;
787
- const setup = `harness ${e.harness} · sandbox ${e.sandbox} · concurrency ${e.concurrency}`;
788
- this.#write(c.dim(`reeval ${e.version} · run ${e.runId} · ${files} · ${setup} · ${order}`));
562
+ this.#write(startLine(c, this.#o.output));
563
+ this.#write("");
789
564
  return;
790
- }
791
565
  case "plan":
792
- if (this.#o.output.rich) {
793
- this.#richPlan(e);
794
- return;
795
- }
796
- this.#write(e.line);
797
- for (const line of e.details) this.#write(c.dim(line));
798
- for (const w of e.warnings) this.#write(c.yellow(`warning: ${w}`));
799
- for (const h of e.hints) this.#write(c.dim(`hint: ${h}`));
800
- for (const w of e.sandboxWarnings ?? []) this.#sandboxWarning(w);
566
+ for (const line of planLines(c, e, this.#columns)) this.#write(line);
567
+ if (this.#o.output.githubActions) for (const w of e.sandboxWarnings ?? []) this.#annotate(w);
801
568
  return;
802
569
  case "collected":
570
+ this.#list.collected(e.file, e.evals);
803
571
  this.#tags.addFile(e.evals);
804
572
  for (const x of e.evals) {
805
573
  if (x.skipped) continue;
@@ -811,6 +579,7 @@ var PlainRenderer = class {
811
579
  return;
812
580
  case "eval-start": {
813
581
  const a = this.#attempt(e, e.at);
582
+ this.#list.start(e);
814
583
  a.last = e;
815
584
  this.#queued.delete(e.key);
816
585
  this.#running.set(e.key, a);
@@ -820,6 +589,7 @@ var PlainRenderer = class {
820
589
  case "eval-progress": {
821
590
  const a = this.#running.get(e.key);
822
591
  if (!a) return;
592
+ this.#list.progress(e.key, e.progress);
823
593
  if (e.progress.kind !== "warning") a.last = e;
824
594
  const { progress } = e;
825
595
  const paint = a.ungated ? {
@@ -861,13 +631,13 @@ var PlainRenderer = class {
861
631
  return;
862
632
  }
863
633
  case "eval-end":
634
+ this.#list.end(e.result);
864
635
  this.#end(e.result, e.at);
865
636
  if (this.#tries.has(e.result.key)) this.#ended.add(e.result.key);
866
637
  this.#drawStatus();
867
638
  return;
868
- case "prepare": {
869
- const lines = this.#o.output.rich ? prepareStepLines(c, e, this.#columns, ERROR_LINES) : prepareLines(c, e, ERROR_LINES);
870
- for (const line of lines) this.#write(line);
639
+ case "prepare":
640
+ for (const line of prepareStepLines(c, e, this.#columns, ERROR_LINES)) this.#write(line);
871
641
  if (e.state === "failed" && this.#o.output.githubActions) {
872
642
  const { target, failure } = e.outcome;
873
643
  const what = target.kind === "workspace" ? "The" : "Sandbox";
@@ -875,52 +645,30 @@ var PlainRenderer = class {
875
645
  this.#write(`::error title=${title}::${ghEscape(failure.message)}`);
876
646
  }
877
647
  return;
878
- }
879
- case "notice": {
880
- if (this.#o.output.rich) {
881
- for (const line of noticeLines(c, e, this.#columns)) this.#write(line);
882
- return;
883
- }
884
- const paint = {
885
- info: c.dim,
886
- warn: c.yellow,
887
- error: c.red
888
- }[e.level];
889
- this.#write(paint(e.text));
648
+ case "notice":
649
+ for (const line of noticeLines(c, e, this.#columns)) this.#write(line);
890
650
  return;
891
- }
892
- case "run-end":
651
+ case "run-end": {
893
652
  this.#clearStatus();
894
- this.#write("");
895
- for (const line of resultLines(c, e, (this.#o.verbosity ?? 0) > 0)) this.#write(line);
653
+ if (!this.#blank) this.#write("");
654
+ const verbose = (this.#o.verbosity ?? 0) > 0;
655
+ const list = statusList({
656
+ record: e.record,
657
+ suite: suiteResult(e.record),
658
+ remarks: e.remarks ?? null,
659
+ verbose,
660
+ ...e.notRun ? { notRun: e.notRun } : {}
661
+ });
662
+ for (const line of listLines(list, c, true, pointerLine(e.summaryPath), verbose)) this.#write(line);
896
663
  return;
664
+ }
897
665
  }
898
666
  }
899
667
  get #columns() {
900
- return this.#o.columns || 80;
668
+ return this.#o.columns || Number.POSITIVE_INFINITY;
901
669
  }
902
- /** The plan as the rich header: its facts, `--dry-run`'s table, then each kind of note. */
903
- #richPlan(e) {
904
- const c = this.#c;
905
- const width = this.#columns;
906
- for (const line of headerLines(c, e.header, width)) this.#write(line);
907
- const sections = [
908
- e.details,
909
- [...e.warnings.flatMap((w) => noteLines(c, "warning", w, width)), ...(e.sandboxWarnings ?? []).flatMap((w) => noteLines(c, w.kind === "root" ? "root" : "warning", w.message, width))],
910
- e.hints.flatMap((h) => noteLines(c, "hint", h, width))
911
- ];
912
- for (const lines of sections) {
913
- if (!lines.length) continue;
914
- this.#write("");
915
- for (const line of lines) this.#write(line);
916
- }
917
- this.#write("");
918
- }
919
- #sandboxWarning(w) {
920
- const c = this.#c;
921
- const line = sandboxWarningLine(w);
922
- this.#write(w.kind === "root" ? c.bold(c.red(line)) : c.yellow(line));
923
- if (!this.#o.output.githubActions) return;
670
+ /** The header block names a sandbox warning; GitHub Actions also annotates it. */
671
+ #annotate(w) {
924
672
  const file = w.file ? `file=${ghEscape(w.file, true)},` : "";
925
673
  const title = w.kind === "root" ? "Sandbox runs as root" : "Sandbox has extra capabilities";
926
674
  this.#write(`::warning ${file}title=${ghEscape(title, true)}::${ghEscape(w.message)}`);
@@ -961,20 +709,30 @@ var PlainRenderer = class {
961
709
  this.#write(`::error file=${ghEscape(file, true)},line=${line},title=${label}::${ghEscape(x.message)}`);
962
710
  }
963
711
  if (!errorCheck(result)?.at) {
964
- const message = ghEscape(failureLines(colors(false), result, { max: ERROR_LINES }).join("\n"));
712
+ const message = ghEscape(failureLines(colors(false), result, { max: ERROR_LINES }).join("\n").replace(/^✗ failed {2}/, ""));
965
713
  const at = locate(result);
966
714
  const where = at ? `,line=${at.line},col=${at.col}` : "";
967
715
  this.#write(`::error file=${ghEscape(result.file, true)}${where},title=${title}::${message}`);
968
716
  }
969
717
  }
970
718
  }
971
- /** Prints the heartbeat if nothing was written for `heartbeatMs` and evals are running. */
719
+ /**
720
+ * Every `heartbeatMs` while attempts run. Without attempt logs, how many evals are done; with
721
+ * them, each running attempt's phase, when nothing printed for that long.
722
+ */
972
723
  tick(now = this.#now()) {
973
- const { heartbeatMs } = this.#o.output;
724
+ const { heartbeatMs, outputLogs } = this.#o.output;
974
725
  if (this.#live || heartbeatMs <= 0) return;
975
- if (now - this.#lastWrite < heartbeatMs) return;
976
- const c = this.#c;
977
726
  if (this.#running.size === 0) return;
727
+ const c = this.#c;
728
+ if (outputLogs === "none") {
729
+ if (now - Math.max(this.#lastProgress, this.#runStartedAt) < heartbeatMs) return;
730
+ const { done, total } = this.#list.counts();
731
+ this.#write(c.dim(progressText(done, total, now - this.#runStartedAt)));
732
+ this.#lastProgress = now;
733
+ return;
734
+ }
735
+ if (now - this.#lastWrite < heartbeatMs) return;
978
736
  const running = [...this.#running.values()].map((a) => {
979
737
  const progress = a.last?.type === "eval-progress" ? a.last.progress : void 0;
980
738
  return `${this.#prefix(a)} ${phaseOf(progress)} ${c.dim(formatDuration(now - a.startedAt))}`;
@@ -988,6 +746,319 @@ var PlainRenderer = class {
988
746
  this.#clearStatus();
989
747
  }
990
748
  };
749
+ // src/ui/tty.ts
750
+ /** The list's state, which the Ink app reads with `useSyncExternalStore`. */
751
+ var ListStore = class {
752
+ #state = {
753
+ files: [],
754
+ logs: [],
755
+ counts: {
756
+ done: 0,
757
+ total: 0
758
+ },
759
+ phase: null,
760
+ finished: null
761
+ };
762
+ #listeners = /* @__PURE__ */ new Set();
763
+ get = () => this.#state;
764
+ subscribe = (listener) => {
765
+ this.#listeners.add(listener);
766
+ return () => this.#listeners.delete(listener);
767
+ };
768
+ set(next) {
769
+ this.#state = {
770
+ ...this.#state,
771
+ ...next
772
+ };
773
+ for (const l of this.#listeners) l();
774
+ }
775
+ };
776
+ const FRAME_MS = 80;
777
+ const ESC = String.fromCharCode(27);
778
+ const BEL = String.fromCharCode(7);
779
+ /** A CSI sequence (colors, cursor moves) or an OSC one (a hyperlink), which take no columns. */
780
+ const ESCAPE = new RegExp(`${ESC}(?:\\[[0-9;?]*[ -/]*[@-~]|\\][^${BEL}${ESC}]*(?:${BEL}|${ESC}\\\\))`, "g");
781
+ const graphemes = new Intl.Segmenter();
782
+ /**
783
+ * Terminal rows one line takes at `columns`, placed as a terminal places it: a wide character that
784
+ * doesn't fit in the row's last column moves to the next row, `\r` returns to the row's start and
785
+ * `\t` goes to the next stop of eight without wrapping.
786
+ */
787
+ function rowsOf(line, columns) {
788
+ let rows = 1;
789
+ let column = 0;
790
+ for (const { segment } of graphemes.segment(line.replace(ESCAPE, ""))) if (segment === "\r") column = 0;
791
+ else if (segment === " ") column = Math.min(columns - 1, (Math.floor(column / 8) + 1) * 8);
792
+ else if (segment === "\b") column = Math.max(0, column - 1);
793
+ else {
794
+ const width = stringWidth(segment);
795
+ if (!width) continue;
796
+ if (column + width > columns) {
797
+ rows += 1;
798
+ column = 0;
799
+ }
800
+ column += width;
801
+ }
802
+ return rows;
803
+ }
804
+ /**
805
+ * Before the list is up, the sandbox prepare step: the glyph of its `◆ Preparing…` line spins in
806
+ * place (the cursor goes up to it and back) until each sandbox is ready or failed, then settles to
807
+ * `◆`. What prints under it is kept and measured in rows as the terminal placed it, so a wide
808
+ * character still finds the step line. Once the step line would be out of reach, or the width
809
+ * changes (some terminals reflow what they show, some don't, so its row is unknown), it stops
810
+ * without drawing.
811
+ */
812
+ var StepSpinner = class {
813
+ #write;
814
+ #c;
815
+ #size;
816
+ #timer;
817
+ #frame = 0;
818
+ /** The step line and everything printed under it since. */
819
+ #text = "";
820
+ /** The width `#text` was printed at. */
821
+ #columns = 0;
822
+ #pending = 0;
823
+ constructor(write, c, size) {
824
+ this.#write = write;
825
+ this.#c = c;
826
+ this.#size = size;
827
+ }
828
+ /** Terminal lines from the step line down to the cursor after `text`. */
829
+ #up(text = this.#text) {
830
+ const columns = Math.max(1, this.#columns);
831
+ const lines = text.split("\n");
832
+ if (lines.at(-1) === "") lines.pop();
833
+ return lines.reduce((n, l) => n + rowsOf(l, columns), 0);
834
+ }
835
+ /** The step line was just written as `text`; it spins until `targets` have reported. */
836
+ start(text, targets) {
837
+ this.settle();
838
+ this.#text = text;
839
+ this.#columns = this.#size().columns;
840
+ this.#pending = targets;
841
+ this.#timer = setInterval(() => this.#draw(), FRAME_MS);
842
+ this.#timer.unref();
843
+ this.#draw();
844
+ }
845
+ /** Before `text` prints under it; settles first if the step line would scroll out of reach. */
846
+ before(text) {
847
+ if (!this.#timer) return;
848
+ const next = this.#text + text;
849
+ if (!this.#reachable(next)) this.settle();
850
+ this.#text = next;
851
+ }
852
+ /** One sandbox reported; the last settles it. */
853
+ reported() {
854
+ this.#pending -= 1;
855
+ if (this.#pending <= 0) this.settle();
856
+ }
857
+ #reachable(text = this.#text) {
858
+ const { rows, columns } = this.#size();
859
+ return columns === this.#columns && this.#up(text) < rows - 1;
860
+ }
861
+ #draw(glyph = SPINNER[this.#frame++ % SPINNER.length] ?? "◆") {
862
+ if (!this.#reachable()) {
863
+ clearInterval(this.#timer);
864
+ this.#timer = void 0;
865
+ return;
866
+ }
867
+ const up = this.#up();
868
+ this.#write(`\x1b[${up}A\r\x1b[2C${this.#c.teal(glyph)}\r\x1b[${up}B`);
869
+ }
870
+ settle() {
871
+ if (!this.#timer) return;
872
+ this.#draw("◆");
873
+ clearInterval(this.#timer);
874
+ this.#timer = void 0;
875
+ }
876
+ };
877
+ var InteractiveRenderer = class {
878
+ #o;
879
+ #c;
880
+ #store = new ListStore();
881
+ #list;
882
+ /** The header and run-level lines, as every terminal prints them. */
883
+ #plain;
884
+ #app;
885
+ /** Set once the list couldn't load: from then on, the run prints as a pipe's does. */
886
+ #fallback;
887
+ #ended;
888
+ #closed = false;
889
+ #spinner;
890
+ /** What the plain renderer wrote since `#written` was reset, while the list isn't up. */
891
+ #written = "";
892
+ constructor(options) {
893
+ this.#o = options;
894
+ this.#c = options.colors;
895
+ this.#list = new LiveList(options.harness);
896
+ this.#spinner = new StepSpinner((text) => options.out.write(text), options.colors, () => sizeOf(options.stdout));
897
+ const out = { write: (text) => {
898
+ if (!this.#app || this.#fallback) {
899
+ this.#spinner.before(redactSecrets(String(text)));
900
+ this.#written += String(text);
901
+ return options.out.write(text);
902
+ }
903
+ const lines = String(text).replace(/\n$/, "").split("\n");
904
+ this.#store.set({ logs: [...this.#store.get().logs, ...lines] });
905
+ return true;
906
+ } };
907
+ this.#plain = new PlainRenderer({
908
+ ...options,
909
+ out,
910
+ timers: false,
911
+ output: {
912
+ ...options.output,
913
+ statusLine: false,
914
+ interactive: false,
915
+ outputLogs: "none"
916
+ }
917
+ });
918
+ }
919
+ #mount() {
920
+ if (this.#app || this.#closed) return;
921
+ const cancel = this.#o.cancel ?? (() => process.kill(process.pid, "SIGINT"));
922
+ const load = this.#o.load ?? (() => import("./ink-list-BpfidQ1h.js"));
923
+ this.#app = load().then((m) => m.mountList({
924
+ store: this.#store,
925
+ colors: this.#c,
926
+ cancel,
927
+ stdout: this.#o.stdout,
928
+ stdin: this.#o.stdin
929
+ })).catch((error) => {
930
+ this.#fallBack(error);
931
+ });
932
+ }
933
+ #fallBack(error) {
934
+ const out = this.#o.out;
935
+ const { logs, phase } = this.#store.get();
936
+ for (const line of logs) out.write(`${line}\n`);
937
+ const reason = error instanceof Error ? error.message : String(error);
938
+ out.write(`${this.#c.yellow(`Couldn't draw the interactive list (${reason}).`)}\n`);
939
+ if (phase) out.write(` ${this.#c.teal("◆")} ${phase}\n`);
940
+ this.#fallback = new PlainRenderer({
941
+ ...this.#o,
942
+ timers: false,
943
+ output: {
944
+ ...this.#o.output,
945
+ statusLine: false,
946
+ interactive: false
947
+ }
948
+ });
949
+ if (this.#ended) this.#fallback.onEvent(this.#ended.event);
950
+ }
951
+ #update() {
952
+ const { done, total, failed, failing } = this.#list.counts();
953
+ this.#store.set({
954
+ files: this.#list.files(),
955
+ counts: {
956
+ done,
957
+ total,
958
+ failed,
959
+ failing
960
+ }
961
+ });
962
+ }
963
+ onEvent(e) {
964
+ if (this.#fallback) {
965
+ if (e.type !== "run-start") this.#fallback.onEvent(e);
966
+ return;
967
+ }
968
+ switch (e.type) {
969
+ case "run-start":
970
+ this.#spinner.settle();
971
+ this.#store.set({ phase: startText(this.#o.output) });
972
+ return;
973
+ case "prepare":
974
+ this.#written = "";
975
+ this.#plain.onEvent(e);
976
+ if (this.#app || this.#o.timers === false || process.env.REEVAL_DEBUG) return;
977
+ if (e.state === "start") this.#spinner.start(this.#written, e.targets.length);
978
+ else this.#spinner.reported();
979
+ return;
980
+ case "collected":
981
+ this.#spinner.settle();
982
+ this.#list.collected(e.file, e.evals);
983
+ this.#mount();
984
+ this.#update();
985
+ return;
986
+ case "eval-start":
987
+ this.#list.start(e);
988
+ this.#update();
989
+ return;
990
+ case "eval-progress":
991
+ if (e.progress.kind === "warning" || e.progress.kind === "browser-download") this.#plain.onEvent(e);
992
+ this.#list.progress(e.key, e.progress);
993
+ this.#update();
994
+ return;
995
+ case "eval-end":
996
+ this.#list.end(e.result);
997
+ this.#update();
998
+ return;
999
+ case "debug": return;
1000
+ case "run-end": {
1001
+ const listOf = (verbose) => statusList({
1002
+ record: e.record,
1003
+ suite: suiteResult(e.record),
1004
+ remarks: e.remarks ?? null,
1005
+ verbose,
1006
+ ...e.notRun ? { notRun: e.notRun } : {}
1007
+ });
1008
+ const list = listOf(false);
1009
+ const verbose = (this.#o.verbosity ?? 0) > 0;
1010
+ if (!this.#app) {
1011
+ this.#plain.onEvent(e);
1012
+ return;
1013
+ }
1014
+ this.#ended = {
1015
+ scrollback: verbose ? listOf(true) : list,
1016
+ verbose,
1017
+ pointer: pointerLine(e.summaryPath),
1018
+ event: e
1019
+ };
1020
+ this.#store.set({
1021
+ files: list.files,
1022
+ ...this.#store.get().phase ? { phase: endText(e.record.status) } : {},
1023
+ finished: {
1024
+ done: list.done,
1025
+ notes: list.notes,
1026
+ pointer: pointerLine(e.summaryPath),
1027
+ errors: list.errors
1028
+ }
1029
+ });
1030
+ return;
1031
+ }
1032
+ default: this.#plain.onEvent(e);
1033
+ }
1034
+ }
1035
+ close() {
1036
+ this.#spinner.settle();
1037
+ this.#plain.close();
1038
+ this.#fallback?.close();
1039
+ if (!this.#store.get().finished) {
1040
+ this.#closed = true;
1041
+ this.#app?.then((a) => a?.unmount());
1042
+ }
1043
+ }
1044
+ /** Until the user quits the list, then the whole list into the scrollback; at once when there's none. */
1045
+ async wait() {
1046
+ const app = await this.#app;
1047
+ if (!app || !this.#ended) return;
1048
+ await app.waitUntilExit();
1049
+ const { phase } = this.#store.get();
1050
+ const { scrollback, verbose, pointer } = this.#ended;
1051
+ const text = [...phase ? [` ${this.#c.teal("◆")} ${phase}`, ""] : [], ...listLines(scrollback, this.#c, true, pointer, verbose)];
1052
+ this.#o.out.write(`${text.join("\n")}\n`);
1053
+ }
1054
+ };
1055
+ const sizeOf = (stdout) => {
1056
+ const { rows, columns } = stdout ?? {};
1057
+ return {
1058
+ rows: rows || 24,
1059
+ columns: columns || 80
1060
+ };
1061
+ };
991
1062
  // src/ui/index.ts
992
1063
  const depthOf = (bits = 4) => bits >= 24 ? 24 : bits >= 8 ? 8 : 4;
993
1064
  function createRenderer(options) {
@@ -996,10 +1067,17 @@ function createRenderer(options) {
996
1067
  ...options,
997
1068
  out
998
1069
  });
1070
+ const paint = colors(options.output.color, depthOf(options.colorDepth));
1071
+ if (options.output.interactive && options.tty) return new InteractiveRenderer({
1072
+ ...options,
1073
+ ...options.tty,
1074
+ out,
1075
+ colors: paint
1076
+ });
999
1077
  return new PlainRenderer({
1000
1078
  ...options,
1001
1079
  out,
1002
- colors: colors(options.output.color, depthOf(options.colorDepth))
1080
+ colors: paint
1003
1081
  });
1004
1082
  }
1005
1083
  // src/cli.ts
@@ -1028,7 +1106,7 @@ const EXIT = {
1028
1106
  };
1029
1107
  const version = PACKAGE_VERSION;
1030
1108
  const HELP = helpText(version);
1031
- const { green, red, dim } = colors(!!process.stdout.isTTY && !process.env.NO_COLOR);
1109
+ const { green, red, dim, yellow } = colors(!!process.stdout.isTTY && !process.env.NO_COLOR);
1032
1110
  const startedAt = Date.now();
1033
1111
  const argv = process.argv.slice(2);
1034
1112
  /** What the invocation log needs from the command as it goes. */
@@ -1072,7 +1150,7 @@ async function run(argv) {
1072
1150
  return values.help ? EXIT.ok : EXIT.usage;
1073
1151
  }
1074
1152
  if (command === "init") {
1075
- const { initCommand } = await import("./init-s1jN-PCI.js");
1153
+ const { initCommand } = await import("./init-4RPeD-K-.js");
1076
1154
  return initCommand(values, filters, {
1077
1155
  out: (s) => console.log(s),
1078
1156
  input: process.stdin,
@@ -1081,7 +1159,7 @@ async function run(argv) {
1081
1159
  });
1082
1160
  }
1083
1161
  if (command === "doctor") {
1084
- const { doctorCommand } = await import("./doctor-eE65YpEv.js");
1162
+ const { doctorCommand } = await import("./doctor-B2PohneB.js");
1085
1163
  return doctorCommand(values, {
1086
1164
  out: (s) => console.log(s),
1087
1165
  color: !!process.stdout.isTTY && !process.env.NO_COLOR,
@@ -1089,7 +1167,7 @@ async function run(argv) {
1089
1167
  });
1090
1168
  }
1091
1169
  if (command === "report") {
1092
- const { reportCommand } = await import("./command-TLmY-rYp.js");
1170
+ const { reportCommand } = await import("./command-aUJvENwy.js");
1093
1171
  return reportCommand(values, filters, {
1094
1172
  out: (s) => console.log(s),
1095
1173
  err: (s) => console.error(dim(s)),
@@ -1104,7 +1182,7 @@ async function run(argv) {
1104
1182
  }
1105
1183
  if (command === "studio") {
1106
1184
  const config = await loadConfig(configPath, "studio");
1107
- const { studioCommand } = await import("./command-CtTdCF3B.js");
1185
+ const { studioCommand } = await import("./command-BLNVDiIg.js");
1108
1186
  return studioCommand({
1109
1187
  config,
1110
1188
  flags: values,
@@ -1115,7 +1193,7 @@ async function run(argv) {
1115
1193
  if (command === "mcp") {
1116
1194
  if (!values.http) console.log = console.info = console.debug = console.error;
1117
1195
  const config = await loadConfig(configPath, "mcp");
1118
- const { mcpCommand } = await import("./command-CsfcwMyc.js");
1196
+ const { mcpCommand } = await import("./command-B5OyHpOu.js");
1119
1197
  return mcpCommand({
1120
1198
  config,
1121
1199
  flags: values,
@@ -1124,7 +1202,7 @@ async function run(argv) {
1124
1202
  }
1125
1203
  if (command in HISTORY_COMMANDS) {
1126
1204
  const config = await loadConfig(configPath, command);
1127
- const commands = await import("./commands-jO9Jj35r.js");
1205
+ const commands = await import("./commands-CfdBX-mc.js");
1128
1206
  const ctx = {
1129
1207
  config,
1130
1208
  flags: values,
@@ -1137,7 +1215,7 @@ async function run(argv) {
1137
1215
  case "compare": return commands.compareCommand(ctx, filters);
1138
1216
  case "checkout": return commands.checkoutCommand(ctx, filters);
1139
1217
  case "query": {
1140
- const { queryCommand } = await import("./query-kFPuqBv1.js");
1218
+ const { queryCommand } = await import("./query-DIfkUcPh.js");
1141
1219
  return queryCommand({
1142
1220
  ...ctx,
1143
1221
  err: (s) => console.error(dim(s))
@@ -1145,7 +1223,7 @@ async function run(argv) {
1145
1223
  }
1146
1224
  case "analyze": {
1147
1225
  loadEnvFiles(configPath);
1148
- const { analyzeCommand } = await import("./command-CyIb9BSC.js");
1226
+ const { analyzeCommand } = await import("./command-CRh-cb6Z.js");
1149
1227
  return analyzeCommand({
1150
1228
  ...ctx,
1151
1229
  err: (s) => console.error(dim(s))
@@ -1159,12 +1237,12 @@ async function run(argv) {
1159
1237
  }
1160
1238
  /** The dry run's prices, fetched only when a planned attempt or the judge has anything to price. */
1161
1239
  async function dryRunPrices(planned, judgeModel) {
1162
- const { needsPrices } = await import("./cost-CMzY9VXv.js");
1240
+ const { needsPrices } = await import("./cost-Cr26Alhq.js");
1163
1241
  if (!needsPrices(planned.filter((p) => !p.attempt.plan.skip).map((p) => p.attempt.harness), judgeModel)) return {
1164
1242
  table: null,
1165
1243
  why: "nothing to price"
1166
1244
  };
1167
- const { loadPrices } = await import("./prices-C4BRuSe8.js");
1245
+ const { loadPrices } = await import("./prices-5m3SzMZ3.js");
1168
1246
  return loadPrices();
1169
1247
  }
1170
1248
  /** Asks a yes/no question on the terminal; anything but y or yes is no. */
@@ -1191,14 +1269,14 @@ async function cleanCommand(flags, configPath) {
1191
1269
  keep = Number(flags.keep);
1192
1270
  if (!(Number.isInteger(keep) && keep >= 0)) throw new UsageError(`--keep must be a whole number (got ${flags.keep})`);
1193
1271
  }
1194
- const { debugRoot, pruneDebug } = await import("./files-COW5JBJM.js");
1195
- const { formatBytes } = await import("./debug-BnXV7JbX.js");
1272
+ const { debugRoot, pruneDebug } = await import("./files-C1Rk9FYA.js");
1273
+ const { formatBytes } = await import("./debug-CF4p4Lp_.js");
1196
1274
  const { runs, bytes } = await pruneDebug(config.root, keep);
1197
1275
  const where = fromCwd(debugRoot(config.root)) || ".";
1198
1276
  console.log(runs.length ? `Removed debug data and snapshots for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where} and .reeval/runs` : `No debug data or snapshots to remove in ${where} or .reeval/runs`);
1199
1277
  if (flags.images) {
1200
- const { dockerAvailable, pruneImages } = await import("./src-DVaAKCsH.js");
1201
- const { requireDocker } = await import("./providers-CP_bBtYm.js");
1278
+ const { dockerAvailable, pruneImages } = await import("./src-DtodMT59.js");
1279
+ const { requireDocker } = await import("./providers-CVu8e0xS.js");
1202
1280
  const engines = [];
1203
1281
  for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
1204
1282
  if (!engines.length) await requireDocker();
@@ -1208,24 +1286,25 @@ async function cleanCommand(flags, configPath) {
1208
1286
  }
1209
1287
  return EXIT.ok;
1210
1288
  }
1211
- /** `-v` is 1 and `-vv` (or `--trace`) is 2. */
1289
+ /** `-v` is 1 and `-vv` is 2. */
1212
1290
  function verbosityOf(flags) {
1213
- if (flags.trace) return 2;
1214
1291
  return Math.min(2, flags.verbose?.length ?? 0);
1215
1292
  }
1216
1293
  /**
1217
- * `reeval image build`: collects the evals (as `--dry-run` does) and builds every image their Docker
1218
- * sandboxes need; with `--dry-run`, lists them instead. `--profile` plans as `reeval run --profile`
1219
- * does. Needs a config, as `reeval run` does: there is no default image.
1294
+ * `reeval image build`: collects the evals (as `--dry-run` does), pulls every image their `tart()`
1295
+ * sandboxes clone that Tart doesn't have, and builds every image their Docker sandboxes need; with
1296
+ * `--dry-run`, lists them instead. A plan with no Docker sandbox never runs Docker. `--profile`
1297
+ * plans as `reeval run --profile` does. Needs a config, as `reeval run` does: there is no default
1298
+ * image.
1220
1299
  */
1221
1300
  async function imageBuildCommand(flags, configPath, filters) {
1222
- const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-BVZHDf_z.js");
1301
+ const { buildImages, imagePlanLines, imageTargets, plannedImageUses, pullTartImages, tartPlanLines, tartTargets } = await import("./images-BNgcR77K.js");
1223
1302
  const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
1224
1303
  ...flags,
1225
1304
  "dry-run": true
1226
1305
  }, configPath));
1227
1306
  const root = config.root;
1228
- const targets = imageTargets(await plannedImageUses({
1307
+ const uses = await plannedImageUses({
1229
1308
  runId,
1230
1309
  stagingDir,
1231
1310
  config,
@@ -1233,53 +1312,141 @@ async function imageBuildCommand(flags, configPath, filters) {
1233
1312
  tags,
1234
1313
  concurrency,
1235
1314
  profile
1236
- }, filters, (parts) => unmatchedFilter(config, parts)));
1237
- if (targets.length === 0) {
1238
- console.log("No eval runs in a Docker sandbox, so there are no images to build.");
1315
+ }, filters, (parts) => unmatchedFilter(config, parts));
1316
+ const targets = imageTargets(uses);
1317
+ const vms = tartTargets(uses);
1318
+ if (targets.length === 0 && vms.length === 0) {
1319
+ console.log("No eval runs in a Docker or Tart sandbox, so there are no images to build or pull.");
1239
1320
  return EXIT.ok;
1240
1321
  }
1322
+ const { tartTagWarning } = await import("./tart-ref-7Qv3cm3B.js");
1323
+ for (const t of vms) {
1324
+ const warning = tartTagWarning(t.sandbox.image);
1325
+ if (warning) console.log(yellow(`warning: ${warning}`));
1326
+ }
1241
1327
  if (flags["dry-run"]) {
1242
- for (const line of await imagePlanLines(targets, root)) console.log(line);
1328
+ if (vms.length) for (const line of await tartPlanLines(vms)) console.log(line);
1329
+ if (targets.length) for (const line of await imagePlanLines(targets, root)) console.log(line);
1243
1330
  return EXIT.ok;
1244
1331
  }
1245
- const { PrepareError } = await import("./provider-DCdTHQpk.js");
1246
- let built;
1332
+ const { PrepareError } = await import("./provider-BnUoz4KB.js");
1333
+ const outcomes = [];
1334
+ let refused = false;
1335
+ const refusal = (id, of, others) => (error) => {
1336
+ if (!others || !(error instanceof PrepareError)) throw error;
1337
+ if (error.failure.usage) refused = true;
1338
+ const evals = of.reduce((n, t) => n + t.evals, 0);
1339
+ outcomes.push({
1340
+ id,
1341
+ evals,
1342
+ ok: false,
1343
+ failure: error.failure
1344
+ });
1345
+ return [];
1346
+ };
1247
1347
  try {
1248
- built = await buildImages(targets, root, {
1249
- log: (line) => console.log(dim(line)),
1250
- force: !!flags.force
1348
+ const pulled = vms.length ? await pullTartImages(vms, root, { log: (line) => console.log(line.startsWith("warning: ") ? yellow(line) : dim(line)) }).catch(refusal("tart", vms, targets.length)) : [];
1349
+ for (const p of pulled) outcomes.push(p.ok ? {
1350
+ id: p.id,
1351
+ evals: p.evals,
1352
+ ok: true,
1353
+ ref: p.image.ref,
1354
+ id12: p.image.digest ? short12(p.image.digest) : "local VM"
1355
+ } : {
1356
+ id: p.id,
1357
+ evals: p.evals,
1358
+ ok: false,
1359
+ failure: p.failure
1251
1360
  });
1361
+ if (targets.length) {
1362
+ const built = await buildImages(targets, root, {
1363
+ log: (line) => console.log(dim(line)),
1364
+ force: !!flags.force
1365
+ }).catch(refusal("docker", targets, vms.length));
1366
+ for (const b of built) outcomes.push(b.ok ? {
1367
+ id: b.id,
1368
+ evals: b.evals,
1369
+ ok: true,
1370
+ ref: b.image.ref,
1371
+ id12: short12(b.image.id)
1372
+ } : {
1373
+ id: b.id,
1374
+ evals: b.evals,
1375
+ ok: false,
1376
+ failure: b.failure
1377
+ });
1378
+ }
1252
1379
  } catch (error) {
1253
- if (error instanceof PrepareError && error.failure.usage) throw new UsageError(error.message);
1254
- throw error;
1380
+ if (!(error instanceof PrepareError)) throw error;
1381
+ if (error.failure.usage) throw new UsageError(error.message);
1382
+ if (error.failure.cause === "enospc") invocation.machineState = true;
1383
+ const { PreflightInfraError } = await import("./preflight-4gWSy0Pe.js");
1384
+ throw new PreflightInfraError(error.message);
1255
1385
  }
1256
- for (const b of built) {
1257
- if (b.ok) {
1258
- const id = b.image.id.replace(/^sha256:/, "").slice(0, 12);
1259
- console.log(`${green("✓")} ${green(`${b.id}: ${b.image.ref}`)}${dim(` ${id}`)}`);
1386
+ for (const o of outcomes) {
1387
+ if (o.ok) {
1388
+ console.log(`${green("✓")} ${green(`${o.id}: ${o.ref}`)}${dim(` ${o.id12}`)}`);
1260
1389
  continue;
1261
1390
  }
1262
- const uses = b.evals ? ` (${b.evals} eval${b.evals === 1 ? "" : "s"})` : "";
1263
- const [headline = "", ...rest] = b.failure.message.split("\n");
1264
- console.log(`${red("✗")} ${red(`${b.id}${uses}: ${headline}`)}`);
1391
+ const uses = o.evals ? ` (${o.evals} eval${o.evals === 1 ? "" : "s"})` : "";
1392
+ const [headline = "", ...rest] = o.failure.message.split("\n");
1393
+ console.log(`${red("✗")} ${red(`${o.id}${uses}: ${headline}`)}`);
1265
1394
  for (const line of rest) console.log(` ${line}`);
1266
1395
  }
1267
- const failed = built.filter((b) => !b.ok);
1268
- const n = built.length;
1269
- console.log(failed.length ? red(`${failed.length} of ${n} image${n === 1 ? "" : "s"} failed to build`) : dim(`${n} image${n === 1 ? "" : "s"} ready`));
1396
+ const failed = outcomes.filter((o) => !o.ok);
1397
+ const n = outcomes.length;
1398
+ console.log(failed.length ? red(`${failed.length} of ${n} image${n === 1 ? "" : "s"} failed to ${vms.length ? targets.length ? "build or pull" : "pull" : "build"}`) : dim(`${n} image${n === 1 ? "" : "s"} ready`));
1270
1399
  if (!failed.length) return EXIT.ok;
1271
- return failed.every((b) => !b.ok && b.failure.usage) ? EXIT.usage : EXIT.infra;
1400
+ if (refused || failed.every((o) => !o.ok && o.failure.usage)) return EXIT.usage;
1401
+ invocation.machineState = failed.every((o) => !o.ok && o.failure.cause === "enospc");
1402
+ return EXIT.infra;
1272
1403
  }
1404
+ /** The first 12 hex digits of an image id or digest. */
1405
+ const short12 = (id) => id.replace(/^sha256:/, "").slice(0, 12);
1273
1406
  /** `reeval run`: check, plan, run the engine, then write the run and pick the exit code. */
1274
- async function runCommand(flags, configPath, filters) {
1275
- const setup = await prepareRun(flags, configPath);
1407
+ async function runCommand(flags, configPath, given) {
1408
+ const retry = flags["retry-errors"];
1409
+ if (retry !== void 0 && flags.resume !== void 0) throw new UsageError("--resume and --retry-errors both continue a run: pass one of them.");
1410
+ if (retry !== void 0 && (given.length || flags.tag?.length || flags.trait?.length)) throw new UsageError("--retry-errors runs every infra-errored attempt of the run again, with its plan: leave out filters, --tag and --trait.");
1411
+ const resumed = flags.resume !== void 0 || retry !== void 0 ? await resumedRun(flags, given, configPath) : null;
1412
+ if (resumed?.nothing) {
1413
+ const { nothing, record } = resumed.nothing;
1414
+ const outcome = runOutcome(record);
1415
+ if (flags.json) {
1416
+ console.error(nothing);
1417
+ const { suiteResult } = await import("./suite-CKBa6MKt.js");
1418
+ process.stdout.write(`${JSON.stringify({
1419
+ ...suiteResult(record),
1420
+ outcome
1421
+ }, null, 2)}\n`);
1422
+ } else console.log(nothing);
1423
+ return outcome === "fail" ? EXIT.failed : outcome === "infra_error" ? EXIT.infra : EXIT.ok;
1424
+ }
1425
+ const runFlags = resumed?.settings ? {
1426
+ ...flags,
1427
+ ...resumed.settings
1428
+ } : flags;
1429
+ const filters = resumed?.settings?.filters ?? given;
1430
+ const setup = await prepareRun(runFlags, configPath, {
1431
+ noProfile: !!resumed?.saved && resumed.saved.profile === void 0,
1432
+ ...resumed?.profileOf ? { profileOf: resumed.profileOf } : {}
1433
+ });
1434
+ const explicit = runFlags.concurrency ?? (process.env.REEVAL_CONCURRENCY || void 0);
1435
+ const startedWith = resumed ? resumed.saved ?? void 0 : {
1436
+ ...setup.runs === void 0 ? {} : { runs: setup.runs },
1437
+ ...setup.profile ? { profile: setup.profile.name } : {},
1438
+ ...explicit === void 0 ? {} : { concurrency: setup.concurrency },
1439
+ filters: savedFilters(setup.config.root, filters),
1440
+ tags: runFlags.tag ?? [],
1441
+ ...runFlags.trait?.length ? { traits: runFlags.trait } : {}
1442
+ };
1276
1443
  const ctx = await startRun(setup);
1277
1444
  invocation.runId = ctx.runId;
1278
1445
  const githubOut = process.env.GITHUB_OUTPUT;
1279
1446
  if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
1280
1447
  const stepStart = process.env.GITHUB_STEP_SUMMARY;
1281
1448
  if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
1282
- const { engineVersions } = await import("./node-CwP1G178.js");
1449
+ const { engineVersions } = await import("./node-D7pAU--0.js");
1283
1450
  const terminal = setup.json && !setup.dryRun ? process.stderr : process.stdout;
1284
1451
  const renderer = createRenderer({
1285
1452
  out: setup.json && setup.dryRun ? { write: () => true } : terminal,
@@ -1287,31 +1454,48 @@ async function runCommand(flags, configPath, filters) {
1287
1454
  colorDepth: terminal.isTTY ? terminal.getColorDepth() : void 0,
1288
1455
  output: setup.output,
1289
1456
  verbosity: ctx.verbosity ?? 0,
1290
- harness: setup.config.harness[0]?.kind ?? "agent"
1457
+ harness: setup.config.harness[0]?.kind ?? "agent",
1458
+ tty: {
1459
+ stdout: process.stdout,
1460
+ stdin: process.stdin
1461
+ }
1291
1462
  });
1292
1463
  let executed;
1293
1464
  try {
1294
- executed = await executeRun(ctx, setup, filters, renderer);
1465
+ executed = await executeRun(ctx, setup, filters, renderer, {
1466
+ startedWith,
1467
+ notes: resumed?.notes ?? [],
1468
+ narrowed: !!resumed && (given.length > 0 || !!flags.tag?.length || !!flags.trait?.length)
1469
+ });
1295
1470
  } catch (error) {
1296
1471
  renderer.close();
1297
- if (setup.resume === void 0) await rm(ctx.stagingDir, {
1472
+ if (ctx.retry) await restoreRetry(setup.config.root, ctx.runId);
1473
+ else if (setup.resume === void 0) await rm(ctx.stagingDir, {
1298
1474
  recursive: true,
1299
1475
  force: true
1300
1476
  });
1301
1477
  throw error;
1302
1478
  }
1303
- const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions } = executed;
1479
+ const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions, leftOut } = executed;
1480
+ const { retried } = executed;
1304
1481
  invocation.unhandledErrors = engineRun.unhandledErrors;
1305
- const engines = [...new Set(plan?.sandboxes.flatMap((s) => s.kind === "docker" ? [s.engine ?? "docker"] : []))];
1306
- if ((interrupted || engineRun.canceled) && engines.length) {
1307
- const { removeRunContainers } = await import("./docker-CIzHzwlC.js");
1308
- await removeRunContainers(ctx.runId, engines);
1309
- }
1482
+ if ((interrupted || engineRun.canceled) && plan) await cleanupRun(ctx.runId, plan.sandboxes);
1310
1483
  if (engineRun.stopped || !plan) {
1311
1484
  live?.remove();
1312
1485
  renderer.close();
1313
1486
  return EXIT.ok;
1314
1487
  }
1488
+ if (ctx.retry && ctx.previous && retried.size) {
1489
+ const { retryEnvironmentChanges } = await import("./retry-ChzISxmy.js");
1490
+ const changes = retryEnvironmentChanges(ctx.previous, ctx.stagingDir, retried);
1491
+ if (changes.length) {
1492
+ questions?.cancel();
1493
+ live?.remove();
1494
+ renderer.close();
1495
+ await restoreRetry(setup.config.root, ctx.runId);
1496
+ throw new UsageError(`run ${ctx.runId}'s plan changed since it ran: ${changes.join("; ")}. ${retryVerb(ctx.retry, ctx.runId)} only with the plan they ran with, so the run is left as it was: put it back, or start a new run with reeval run.`);
1497
+ }
1498
+ }
1315
1499
  let stopped = cutShort || engineRun.canceled;
1316
1500
  if (stopped) questions?.cancel();
1317
1501
  let answers = stopped ? await questions?.answered() : await questions?.settle();
@@ -1321,8 +1505,8 @@ async function runCommand(flags, configPath, filters) {
1321
1505
  }
1322
1506
  finishing = true;
1323
1507
  live?.phase("finishing");
1324
- const { lockfiles } = await import("./state-Cicc9Oa5.js");
1325
- const { writeScratchRun } = await import("./results-BMJHaQbi.js");
1508
+ const { lockfiles } = await import("./state-kimtIhBP.js");
1509
+ const { writeScratchRun } = await import("./results-JyuM64gy.js");
1326
1510
  const run = await writeScratchRun({
1327
1511
  ctx,
1328
1512
  author: ctx.author,
@@ -1337,18 +1521,26 @@ async function runCommand(flags, configPath, filters) {
1337
1521
  language: { name: "typescript" },
1338
1522
  enginePackages: engineVersions,
1339
1523
  sandbox: sandboxVersions,
1524
+ ...ctx.retry ? { retry: ctx.retry } : {},
1340
1525
  worktreePatch: ctx.worktreePatch,
1341
1526
  lockfiles: await lockfiles(ctx.config.root),
1342
1527
  git: ctx.git,
1343
1528
  previous: ctx.previous,
1344
1529
  previousArtifacts: ctx.previousArtifacts,
1530
+ leftOut,
1345
1531
  runner: runnerInfo(),
1532
+ ...startedWith ? { invocation: startedWith } : {},
1346
1533
  ...answers && (answers.evals.size || answers.attempts.size) ? { answers } : {}
1347
1534
  }, () => live?.stop());
1348
1535
  const { record } = run;
1349
1536
  const outcome = runOutcome(record);
1350
- const code = stopped ? stopCode : outcome === "fail" ? EXIT.failed : outcome === "infra_error" ? EXIT.infra : EXIT.ok;
1351
- const { pruneDebug } = await import("./files-COW5JBJM.js");
1537
+ const notRun = leftOut.filter((l) => !l.done).length;
1538
+ const ran = notRun > 0 && !stopped && record.status === "interrupted" ? runOutcome({
1539
+ ...record,
1540
+ status: "complete"
1541
+ }) : outcome;
1542
+ const code = stopped ? stopCode : ran === "fail" ? EXIT.failed : ran === "infra_error" ? EXIT.infra : EXIT.ok;
1543
+ const { pruneDebug } = await import("./files-C1Rk9FYA.js");
1352
1544
  await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
1353
1545
  const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
1354
1546
  renderer.onEvent({
@@ -1357,15 +1549,25 @@ async function runCommand(flags, configPath, filters) {
1357
1549
  record,
1358
1550
  summaryPath: fromCwd(summary),
1359
1551
  results: engineRun.evals,
1360
- remarks: run.remarks
1552
+ remarks: run.remarks,
1553
+ ...notRun > 0 && !stopped ? { notRun } : {}
1361
1554
  });
1362
1555
  renderer.close();
1556
+ if (retried.size) {
1557
+ const { retryCounts, retryLine } = await import("./retry-ChzISxmy.js");
1558
+ const line = retryLine(retryCounts(record, retried, ctx.previous));
1559
+ if (setup.json) console.error(line);
1560
+ else console.log(line);
1561
+ }
1363
1562
  const stepSummary = process.env.GITHUB_STEP_SUMMARY;
1364
1563
  if (stepSummary && run.artifactDir) await appendFile(stepSummary, await readFile(summary)).catch(() => {});
1365
1564
  const { config } = ctx;
1366
1565
  const mode = setup.record;
1367
- if ((flags.record || mode === "always" || mode === "ci" && inCI()) && record.status !== "interrupted") {
1368
- const { record: recordCmd } = await import("./commands-jO9Jj35r.js");
1566
+ const { recordFile } = await import("./files-DnlA1A7N.js");
1567
+ const rerecord = (setup.retry !== void 0 || setup.resume !== void 0) && record.status === "complete" && existsSync(recordFile(config.history.dir, record.runId));
1568
+ const autoRecord = flags.record || mode === "always" || mode === "ci" && inCI() || rerecord;
1569
+ if (autoRecord && record.status !== "interrupted") {
1570
+ const { record: recordCmd } = await import("./commands-CfdBX-mc.js");
1369
1571
  const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
1370
1572
  const line = await recordCmd(config, record.runId, {
1371
1573
  ...flags,
@@ -1373,15 +1575,78 @@ async function runCommand(flags, configPath, filters) {
1373
1575
  });
1374
1576
  if (setup.json) console.error(line);
1375
1577
  else console.log(line);
1578
+ } else if (autoRecord && notRun > 0 && !stopped) {
1579
+ const line = `Not recorded: run ${record.runId} is incomplete. reeval run --resume ${record.runId}${flags.record ? " --record" : ""} runs the rest and records it.`;
1580
+ if (setup.json) console.error(line);
1581
+ else console.log(line);
1376
1582
  }
1377
1583
  if (setup.json) {
1378
- const { suiteResult } = await import("./suite-BYJ2Q0S-.js");
1379
- process.stdout.write(`${JSON.stringify(suiteResult(record), null, 2)}\n`);
1584
+ const { suiteResult } = await import("./suite-CKBa6MKt.js");
1585
+ process.stdout.write(`${JSON.stringify({
1586
+ ...suiteResult(record),
1587
+ outcome: ran
1588
+ }, null, 2)}\n`);
1380
1589
  }
1590
+ await renderer.wait?.();
1381
1591
  return code;
1382
1592
  }
1383
- /** Everything that can fail before the evals are collected: output mode, config, profile, flags. */
1384
- async function prepareRun(flags, configPath) {
1593
+ /**
1594
+ * `--resume`: the run's saved settings with the flags given now. An older reeval's run saved none,
1595
+ * so it plans with the flags given now, with a note.
1596
+ */
1597
+ async function resumedRun(flags, filters, configPath) {
1598
+ const path = requireConfig(configPath);
1599
+ loadEnvFiles(path);
1600
+ const root = dirname(path);
1601
+ const target = await continuedRun(root, flags);
1602
+ if (flags["retry-errors"] !== void 0 && target.previous && !erroredAttempts(target.previous).length) return {
1603
+ nothing: {
1604
+ nothing: `run ${target.runId} has no infra errors, so --retry-errors has nothing to run again.`,
1605
+ record: target.previous
1606
+ },
1607
+ saved: null,
1608
+ settings: null,
1609
+ notes: [],
1610
+ profileOf: void 0
1611
+ };
1612
+ if (flags.resume !== void 0 && await stoppedRetry(root, target) && (filters.length || flags.tag?.length || flags.trait?.length)) throw new UsageError(`run ${target.runId} stopped during a --retry-errors, which runs every infra-errored attempt of the run again, with its plan: leave out filters, --tag and --trait.`);
1613
+ const saved = await savedInvocation(root, target);
1614
+ if (!saved) return {
1615
+ nothing: null,
1616
+ saved: null,
1617
+ settings: null,
1618
+ notes: [`run ${target.runId} saved no --runs, profile or filters, so this ${flags.resume === void 0 ? "retry" : "resume"} plans with the ones given now`],
1619
+ profileOf: void 0
1620
+ };
1621
+ const given = {
1622
+ runs: flags.runs,
1623
+ profile: flags.profile,
1624
+ concurrency: flags.concurrency,
1625
+ tag: flags.tag,
1626
+ trait: flags.trait,
1627
+ filters
1628
+ };
1629
+ const flag = flags.resume === void 0 ? "--retry-errors" : "--resume";
1630
+ const resumed = resumeSettings(target.runId, saved, given, {
1631
+ root,
1632
+ flag
1633
+ });
1634
+ const fromRun = flags.profile === void 0 && resumed.settings.profile !== void 0;
1635
+ return {
1636
+ nothing: null,
1637
+ saved,
1638
+ ...resumed,
1639
+ profileOf: fromRun ? target.runId : void 0
1640
+ };
1641
+ }
1642
+ /** The run `--resume` continues, or the finished one `--retry-errors` runs the errors of again. */
1643
+ const continuedRun = (root, flags) => flags["retry-errors"] !== void 0 ? resolveRetry(root, flags["retry-errors"] || void 0) : resolveResume(root, flags.resume || void 0);
1644
+ /**
1645
+ * Everything that can fail before the evals are collected: output mode, config, profile, flags.
1646
+ * `noProfile` is a resume of a run that had none, which `defaultProfile` mustn't give one;
1647
+ * `profileOf` names the resumed run whose profile `flags.profile` is.
1648
+ */
1649
+ async function prepareRun(flags, configPath, { noProfile = false, profileOf } = {}) {
1385
1650
  const output = resolveOutput({
1386
1651
  flags: {
1387
1652
  ui: flags.ui,
@@ -1392,24 +1657,23 @@ async function prepareRun(flags, configPath) {
1392
1657
  },
1393
1658
  env: process.env,
1394
1659
  isTTY: !!(flags.json && !flags["dry-run"] ? process.stderr : process.stdout).isTTY,
1395
- agent: detectAgent().name
1660
+ agent: detectAgent().name,
1661
+ stdinTTY: !!process.stdin.isTTY && !flags.json && !flags["dry-run"]
1396
1662
  });
1397
1663
  const path = requireConfig(configPath);
1398
1664
  loadEnvFiles(path);
1399
- const named = flags.profile ?? (process.env.REEVAL_PROFILE || void 0);
1665
+ const named = noProfile ? void 0 : flags.profile ?? (process.env.REEVAL_PROFILE || void 0);
1400
1666
  const config = await loadConfig(path, "run", named);
1401
- const profile = selectProfile(config, flags.profile);
1667
+ if (profileOf && flags.profile && !config.profiles[flags.profile]) throw new UsageError(`run ${profileOf} was started with profile ${flags.profile}, which ${config.file} no longer defines. Put it back to resume the run, or start a new run without --resume.`);
1668
+ const profile = noProfile ? null : selectProfile(config, flags.profile);
1402
1669
  let runs;
1403
1670
  if (flags.runs !== void 0) {
1404
1671
  runs = Number(flags.runs);
1405
1672
  if (!(Number.isInteger(runs) && runs >= 1)) throw new UsageError(`--runs must be a whole number of at least 1 (got ${flags.runs})`);
1406
1673
  }
1407
- const sandboxes = [environmentSandbox(config.environment, config.sandbox), ...profile ? [profileSandbox(config, profile.defaults)] : []];
1674
+ const sandboxes = [...(config.environments.length ? config.environments : [void 0]).map((e) => environmentSandbox(e, config.sandbox)), ...profile ? [profileSandbox(config, profile.defaults)] : []];
1408
1675
  const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
1409
- const container = sandboxes.find((s) => s.kind === "docker");
1410
- if (!container) return hostCapacity();
1411
- const { dockerCapacity } = await import("./src-DVaAKCsH.js");
1412
- return await dockerCapacity(container.engine) ?? hostCapacity();
1676
+ return await sandboxCapacity(sandboxes) ?? hostCapacity();
1413
1677
  });
1414
1678
  return {
1415
1679
  output,
@@ -1417,6 +1681,7 @@ async function prepareRun(flags, configPath) {
1417
1681
  profile,
1418
1682
  runs,
1419
1683
  tags: [...flags.tag ?? [], ...profile?.tags ?? []],
1684
+ traits: traitFlags(flags.trait),
1420
1685
  concurrency,
1421
1686
  budget: profile?.budget ?? config.budget,
1422
1687
  record: profile?.record ?? config.record,
@@ -1428,37 +1693,48 @@ async function prepareRun(flags, configPath) {
1428
1693
  /** Named in the plan line when the project root isn't the current directory. */
1429
1694
  configElsewhere: dirname(path) !== process.cwd(),
1430
1695
  /** `undefined` is a new run; `''` is newest resumable; a string is a run id or prefix. */
1431
- resume: flags.resume
1696
+ resume: flags.resume,
1697
+ /** Like `resume`, for `--retry-errors`. */
1698
+ retry: flags["retry-errors"]
1432
1699
  };
1433
1700
  }
1701
+ /** What a retry, or a resume that finishes one, does to the run's errors, for its refusals. */
1702
+ const retryVerb = (retry, runId) => retry.resume ? `--resume finishes run ${runId}'s --retry-errors, which runs its errors again` : "--retry-errors runs its errors again";
1434
1703
  /** The run's id and the state it starts from: commit, uncommitted changes, author. */
1435
- async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
1704
+ async function startRun({ config, profile, runs, tags, traits, concurrency, verbosity, configFile, resume, retry, dryRun }) {
1436
1705
  const startedAt = /* @__PURE__ */ new Date();
1437
1706
  const git = await gitInfo(config.root, [config.history.dir]);
1438
- const { worktreePatch } = await import("./state-Cicc9Oa5.js");
1707
+ const { worktreePatch } = await import("./state-kimtIhBP.js");
1439
1708
  const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
1440
1709
  const who = await author(config.root);
1441
- const { stagingDir, sweepStaging } = await import("./results-BMJHaQbi.js");
1442
- const { scratchDir } = await import("./files-Do4Rp3yZ.js");
1710
+ const { stagingDir, sweepStaging } = await import("./results-JyuM64gy.js");
1711
+ const { scratchDir } = await import("./files-DnlA1A7N.js");
1443
1712
  const scratch = scratchDir(config.root);
1444
- if (resume !== void 0) {
1445
- const target = await resolveResume(config.root, resume || void 0);
1446
- if (!dryRun) await prepareResume(target, config.root);
1713
+ if (resume !== void 0 || retry !== void 0) {
1714
+ const target = await continuedRun(config.root, {
1715
+ resume,
1716
+ "retry-errors": retry
1717
+ });
1718
+ if (!dryRun && retry === void 0) await prepareResume(target, config.root);
1719
+ else if (!dryRun) holdRetry(target, config.root);
1720
+ const stopped = retry === void 0 ? await stoppedRetry(config.root, target) : null;
1447
1721
  await sweepStaging(config.root, target.runId);
1448
1722
  const aside = dryRun ? {
1449
1723
  record: target.previous,
1450
1724
  artifacts: target.previousArtifacts
1451
1725
  } : await loadPrevious(stagingDir(config.root, target.runId));
1452
1726
  if (!dryRun) try {
1453
- const { removeRunContainers } = await import("./docker-CIzHzwlC.js");
1454
- await removeRunContainers(target.runId, ["docker", "podman"]);
1727
+ await cleanupRun(target.runId);
1455
1728
  } catch {}
1729
+ const { unfoldCases } = await import("./cases-Dy0mifal.js");
1730
+ const previous = aside.record ?? target.previous;
1456
1731
  return {
1457
1732
  runId: target.runId,
1458
1733
  stagingDir: stagingDir(config.root, target.runId),
1459
1734
  config,
1460
1735
  runs,
1461
1736
  tags,
1737
+ traits,
1462
1738
  concurrency,
1463
1739
  profile,
1464
1740
  verbosity,
@@ -1468,8 +1744,17 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
1468
1744
  worktreePatch: patch,
1469
1745
  author: who,
1470
1746
  scratch,
1471
- previous: aside.record ?? target.previous,
1472
- previousArtifacts: aside.artifacts ?? target.previousArtifacts
1747
+ previous: previous && unfoldCases(previous),
1748
+ previousArtifacts: aside.artifacts ?? target.previousArtifacts,
1749
+ ...retry === void 0 ? {} : { retryTarget: target },
1750
+ ...retry !== void 0 ? { retry: {
1751
+ startedAt,
1752
+ resume: false
1753
+ } } : stopped ? { retry: {
1754
+ startedAt,
1755
+ resume: true,
1756
+ ...stopped.began ? { began: stopped.began } : {}
1757
+ } } : {}
1473
1758
  };
1474
1759
  }
1475
1760
  const stamps = new Set([scratch, config.history.dir].flatMap((dir) => existsSync(dir) ? readdirSync(dir).map((name) => name.slice(0, 16)) : []));
@@ -1482,6 +1767,7 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
1482
1767
  config,
1483
1768
  runs,
1484
1769
  tags,
1770
+ traits,
1485
1771
  concurrency,
1486
1772
  profile,
1487
1773
  verbosity,
@@ -1497,11 +1783,33 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
1497
1783
  }
1498
1784
  function noMatch(config, filters, tags) {
1499
1785
  const narrowed = [filters.length ? `filter "${filters.join(" ")}"` : "", tags.length ? `tags ${tags.join(", ")}` : ""].filter(Boolean);
1500
- return new UsageError(`No evals matched ${config.include.join(", ")}${narrowed.length ? ` with ${narrowed.join(" and ")}` : ""}`);
1786
+ return new UsageError(`No evals matched ${config.include.join(", ")}${narrowed.length ? ` with ${narrowed.join(" and ")}` : ""}${maskedFilterHint(filters)}`);
1787
+ }
1788
+ /**
1789
+ * Why a filter that holds a secret matches nothing: eval names are masked before they're
1790
+ * matched. Printed masked, it's the filter that would match.
1791
+ */
1792
+ function maskedFilterHint(parts) {
1793
+ const masked = parts.filter((p) => redactSecrets(p) !== p);
1794
+ if (!masked.length) return "";
1795
+ return `. Eval names are masked, so a filter that holds a secret can't match one: filter by the masked name (${masked.map((p) => `"${redactSecrets(p)}"`).join(", ")}) or a part of it without the secret`;
1501
1796
  }
1502
1797
  function unmatchedFilter(config, parts) {
1503
1798
  const which = parts.map((p) => `"${p}"`).join(", ");
1504
- return new UsageError(`filter ${which} ${parts.length === 1 ? "matches" : "match"} no eval file, eval name or cell in ${config.include.join(", ")}`);
1799
+ return new UsageError(`filter ${which} ${parts.length === 1 ? "matches" : "match"} no eval file, eval name or cell in ${config.include.join(", ")}${maskedFilterHint(parts)}`);
1800
+ }
1801
+ /**
1802
+ * An eval whose id holds a value declared with `secret()`. `evaluate` masks names as the
1803
+ * worker's environment shows secrets, but a declared one is only known once the plan has the
1804
+ * environments, after the names became ids, so it's refused rather than written.
1805
+ */
1806
+ function namesDeclaredSecret(evals) {
1807
+ for (const { id } of evals) {
1808
+ const [file, ...names] = id.split(" › ");
1809
+ const masked = names.map((n) => redactSecrets(n));
1810
+ if (masked.every((n, i) => n === names[i])) continue;
1811
+ return new UsageError(`${[file, ...masked].join(" › ")} names a secret declared with secret(): name it without the secret.`);
1812
+ }
1505
1813
  }
1506
1814
  /**
1507
1815
  * Reads Cursor's model catalog for the planned Cursor cells, checks their `effort` and `params`
@@ -1511,12 +1819,12 @@ function unmatchedFilter(config, parts) {
1511
1819
  */
1512
1820
  async function planEfforts(planned) {
1513
1821
  const live = planned.filter((p) => !p.attempt.plan.skip);
1514
- const { effortNotes, withDefaultEffort } = await import("./default-effort-fpdbYVKm.js");
1822
+ const { effortNotes, withDefaultEffort } = await import("./default-effort-BAy8ixd1.js");
1515
1823
  let cursor;
1516
1824
  let unchecked = "CURSOR_API_KEY isn't set";
1517
1825
  const key = process.env.CURSOR_API_KEY;
1518
1826
  if (key && live.some((p) => p.attempt.harness.kind === "cursor")) {
1519
- const { checkCursorPlan } = await import("./cursor-catalog-Bw6-pJjR.js");
1827
+ const { checkCursorPlan } = await import("./cursor-catalog-BSn9xjmo.js");
1520
1828
  const checked = await checkCursorPlan(key, live);
1521
1829
  if (checked instanceof Error) unchecked = `its request failed: ${checked.message}`;
1522
1830
  else {
@@ -1536,22 +1844,24 @@ async function planEfforts(planned) {
1536
1844
  * Collects the evals and prints the plan; stops there for `--dry-run`, or on a budget, credential
1537
1845
  * or Docker problem (exit 2). Then runs the engine with live output and Ctrl-C handling.
1538
1846
  */
1539
- async function executeRun(ctx, setup, filters, renderer) {
1540
- const { runId, stagingDir, config, runs, tags, concurrency, profile, verbosity, configFile } = ctx;
1847
+ async function executeRun(ctx, setup, filters, renderer, started) {
1848
+ const { runId, stagingDir, config, runs, tags, traits, concurrency, profile, verbosity } = ctx;
1849
+ const { configFile } = ctx;
1541
1850
  const run = JSON.parse(JSON.stringify({
1542
1851
  runId,
1543
1852
  stagingDir,
1544
1853
  config,
1545
1854
  runs,
1546
1855
  tags,
1856
+ traits,
1547
1857
  concurrency,
1548
1858
  profile,
1549
1859
  verbosity,
1550
1860
  configFile,
1551
1861
  completed: []
1552
1862
  }));
1553
- const { runEngine } = await import("./node-CwP1G178.js");
1554
- const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-CIfOFhth.js");
1863
+ const { runEngine } = await import("./node-D7pAU--0.js");
1864
+ const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-Cgq9NCEX.js");
1555
1865
  let plan;
1556
1866
  let sandboxVersions = { images: [] };
1557
1867
  let preflightStop;
@@ -1560,13 +1870,22 @@ async function executeRun(ctx, setup, filters, renderer) {
1560
1870
  let toRun = [];
1561
1871
  let live;
1562
1872
  let kept = /* @__PURE__ */ new Map();
1873
+ let leftOut = [];
1874
+ let retried = /* @__PURE__ */ new Set();
1563
1875
  let questions;
1564
1876
  const onPlan = async (collected) => {
1565
1877
  const { planned, excluded, matched, unmatched } = collected;
1566
1878
  if (unmatched.length) throw unmatchedFilter(config, unmatched);
1567
1879
  if (matched === 0) throw noMatch(config, filters, tags);
1568
1880
  checkLimitHarness(profile, config.file, collected);
1881
+ checkTraitKeys(traitFilters(traits, profile), collected.traits);
1882
+ const continuing = setup.resume !== void 0 || ctx.retryTarget !== void 0;
1883
+ const { errored } = continuing ? noteRetries(planned, stagingDir, ctx.previous, false) : { errored: /* @__PURE__ */ new Set() };
1569
1884
  const completed = completedKeysFor(planned, stagingDir, ctx.previous);
1885
+ if (ctx.retry) for (const { key } of planned) {
1886
+ const was = previousAttempt(ctx.previous, key);
1887
+ if (was && was.outcome !== "infra_error") completed.add(key);
1888
+ }
1570
1889
  const skipped = planned.map((p) => completed.has(p.key) ? {
1571
1890
  ...p,
1572
1891
  attempt: {
@@ -1580,9 +1899,29 @@ async function executeRun(ctx, setup, filters, renderer) {
1580
1899
  } : p);
1581
1900
  toRun = skipped;
1582
1901
  for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
1902
+ const named = namesDeclaredSecret([...planned, ...excluded]);
1903
+ if (named) throw named;
1583
1904
  const efforts = await planEfforts(skipped);
1584
1905
  if (efforts) run.cursorEfforts = efforts;
1585
- const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-DOXnWH65.js") : void 0;
1906
+ if (ctx.retry && ctx.previous) {
1907
+ const ranOn = ctx.previous.stack?.reeval;
1908
+ if (ranOn && ranOn !== version) throw new UsageError(`run ${runId} ran on reeval ${ranOn}, and this is ${version}. ${retryVerb(ctx.retry, runId)} only on the reeval they ran on: install reeval ${ranOn}, or start a new run with reeval run.`);
1909
+ const { retryChanges } = await import("./retry-ChzISxmy.js");
1910
+ const { judgePart } = await import("./assemble-UPCRrtOz.js");
1911
+ const { engineVersions } = await import("./node-D7pAU--0.js");
1912
+ const judged = judgePart(config, profile, version, engineVersions);
1913
+ const runs = new Set(planned.map((p) => p.key).filter((key) => !completed.has(key)));
1914
+ const changes = await retryChanges(ctx.previous, planned, runs, config.root, judged);
1915
+ if (changes.length) throw new UsageError(`run ${runId}'s plan changed since it ran: ${changes.join("; ")}. ${retryVerb(ctx.retry, runId)} only with the plan they ran with: put it back, or start a new run with reeval run.`);
1916
+ if (ctx.retryTarget && !setup.dryRun) {
1917
+ await prepareResume(ctx.retryTarget, config.root, { held: true });
1918
+ const aside = await loadPrevious(stagingDir);
1919
+ ctx.previous = aside.record ?? ctx.previous;
1920
+ ctx.previousArtifacts = aside.artifacts;
1921
+ }
1922
+ }
1923
+ retried = continuing && !setup.dryRun ? noteRetries(planned, stagingDir, ctx.previous, true).errored : errored;
1924
+ const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-047Dcd-a.js") : void 0;
1586
1925
  const judge = runJudgeModel(config, profile);
1587
1926
  const judgeModel = judge === void 0 ? null : judgeModelName(judge);
1588
1927
  plan = await buildPlan({
@@ -1599,17 +1938,25 @@ async function executeRun(ctx, setup, filters, renderer) {
1599
1938
  });
1600
1939
  plan.runId = runId;
1601
1940
  const cloud = ciRunner(process.env);
1602
- const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && p.attempt.sandbox.kind === "docker").filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
1941
+ const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && capabilitiesOf(p.attempt.sandbox).isolated).filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
1603
1942
  if (cloud && direct.length) plan.warnings.push(`${directOnCloud(cloud)}. Under it: ${direct.join(", ")}. Keep 'allow-all' to the evals that need raw TCP or UDP; the default, 'proxy', sends everything through the gateway`);
1604
- if (completed.size) plan.resume = {
1943
+ plan.warnings.push(...started.notes);
1944
+ if (started.narrowed) {
1945
+ leftOut = leftOutAttempts(planned, stagingDir, ctx.previous, await liveAttempts(stagingDir), new Set(excluded.map((e) => e.id)));
1946
+ const notRun = leftOut.filter((l) => !l.done).length;
1947
+ if (notRun) plan.warnings.push(`${notRun === 1 ? "1 attempt" : `${notRun} attempts`} run ${runId} planned ${notRun === 1 ? "isn't" : "aren't"} in this resume's filters: ${notRun === 1 ? "it stays" : "they stay"} not run and the run stays incomplete. reeval run --resume ${runId} runs ${notRun === 1 ? "it" : "them"}`);
1948
+ }
1949
+ if (completed.size || retried.size) plan.resume = {
1605
1950
  runId,
1606
- completed: completed.size
1951
+ completed: completed.size,
1952
+ ...retried.size ? { retried: retried.size } : {}
1607
1953
  };
1608
1954
  run.completed = [...completed];
1609
- kept = keptAttempts(completed, stagingDir, ctx.previous);
1610
- const { imageStages, imageTargets } = await import("./images-BVZHDf_z.js");
1955
+ const done = leftOut.filter((l) => l.done).map((l) => l.key);
1956
+ kept = keptAttempts([...completed, ...done], stagingDir, ctx.previous);
1957
+ const { imageStages, imageTargets } = await import("./images-BNgcR77K.js");
1611
1958
  const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
1612
- id: p.id,
1959
+ id: p.attempt.family?.id ?? p.id,
1613
1960
  harness: p.attempt.harness,
1614
1961
  sandbox: p.attempt.sandbox,
1615
1962
  plugins: p.attempt.pluginBuilds ?? []
@@ -1626,13 +1973,13 @@ async function executeRun(ctx, setup, filters, renderer) {
1626
1973
  return false;
1627
1974
  }
1628
1975
  const elsewhere = setup.configElsewhere ? configFile : void 0;
1629
- const { scratchDir } = await import("./files-Do4Rp3yZ.js");
1976
+ const { scratchDir } = await import("./files-DnlA1A7N.js");
1630
1977
  const lines = [];
1631
1978
  if (setup.dryRun) {
1632
1979
  const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
1633
1980
  lines.push(...planTable(plan, todos));
1634
- const { imageStatusLines } = await import("./images-BVZHDf_z.js");
1635
- const { dockerAvailable, engineSpec } = await import("./src-DVaAKCsH.js");
1981
+ const { imageStatusLines } = await import("./images-BNgcR77K.js");
1982
+ const { dockerAvailable, engineSpec } = await import("./src-DtodMT59.js");
1636
1983
  const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
1637
1984
  const down = [];
1638
1985
  for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
@@ -1641,7 +1988,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1641
1988
  const names = down.map((e) => engineSpec(e).name).join(" and ");
1642
1989
  lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
1643
1990
  }
1644
- const { costLine } = await import("./cost-CMzY9VXv.js");
1991
+ const { costLine } = await import("./cost-Cr26Alhq.js");
1645
1992
  const cost = plan.cost && costLine(plan.cost);
1646
1993
  if (cost) lines.push(cost);
1647
1994
  }
@@ -1699,6 +2046,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1699
2046
  }
1700
2047
  const missing = missingCredentials(config, process.env, {
1701
2048
  harnesses: plan.harnessDescriptors,
2049
+ sandboxes: plan.sandboxes,
1702
2050
  profile,
1703
2051
  questionModels: [...questionModels],
1704
2052
  judges: [...judges.values()]
@@ -1715,7 +2063,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1715
2063
  });
1716
2064
  if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
1717
2065
  if (asked.length) {
1718
- const { questionRunner } = await import("./questions-C_Y-oXW6.js");
2066
+ const { questionRunner } = await import("./questions-CWlR9ffu.js");
1719
2067
  const warned = /* @__PURE__ */ new Set();
1720
2068
  questions = questionRunner({
1721
2069
  evals: asked,
@@ -1757,7 +2105,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1757
2105
  }
1758
2106
  });
1759
2107
  }
1760
- const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-Dw9DEZox.js");
2108
+ const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-CcQRT31C.js");
1761
2109
  const live = skipped.filter((p) => !p.attempt.plan.skip);
1762
2110
  const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
1763
2111
  workspace: a.workspace,
@@ -1861,7 +2209,9 @@ async function executeRun(ctx, setup, filters, renderer) {
1861
2209
  runId,
1862
2210
  startedAt: ctx.startedAt,
1863
2211
  planned: toRun,
1864
- kept
2212
+ kept,
2213
+ leftOut,
2214
+ ...started.startedWith ? { invocation: started.startedWith } : {}
1865
2215
  });
1866
2216
  renderer.onEvent({
1867
2217
  type: "run-start",
@@ -1933,9 +2283,9 @@ async function executeRun(ctx, setup, filters, renderer) {
1933
2283
  throw error;
1934
2284
  } finally {
1935
2285
  clearTimeout(budgetTimer);
1936
- const { removePrepared } = await import("./preflight-BAIRsTpl.js");
2286
+ const { removePrepared } = await import("./preflight-R-InZCQb.js");
1937
2287
  removePrepared(config.root, runId);
1938
- const { removePreparedWorkspaces } = await import("./workspace-Dw9DEZox.js");
2288
+ const { removePreparedWorkspaces } = await import("./workspace-CcQRT31C.js");
1939
2289
  removePreparedWorkspaces(config.root, runId);
1940
2290
  }
1941
2291
  if (preflightStop) throw preflightStop;
@@ -1949,7 +2299,10 @@ async function executeRun(ctx, setup, filters, renderer) {
1949
2299
  budgetReached: () => reached,
1950
2300
  cutShort,
1951
2301
  live,
1952
- questions
2302
+ questions,
2303
+ leftOut,
2304
+ /** The attempts that ran again after an infra error, by key; none on a dry run. */
2305
+ retried: setup.dryRun ? /* @__PURE__ */ new Set() : retried
1953
2306
  };
1954
2307
  }
1955
2308
  process.stdout.on("error", (error) => {
@@ -1983,7 +2336,7 @@ function finish(code, error, kind) {
1983
2336
  } : null,
1984
2337
  unhandledErrors: invocation.unhandledErrors.map((e) => redactSecrets(e))
1985
2338
  });
1986
- const broke = kind === "crash" || code === EXIT.infra || invocation.unhandledErrors.length > 0;
2339
+ const broke = kind === "crash" || code === EXIT.infra && !invocation.machineState || invocation.unhandledErrors.length > 0;
1987
2340
  const quiet = !!process.env.REEVAL_NO_REPORT_HINT || command === "report" || !!flags?.json || flags?.ui === "agent" || process.env.REEVAL_UI === "agent" || !flags?.ui && !process.env.REEVAL_UI && !!detectAgent().name;
1988
2341
  if (broke && !quiet) console.error(dim("hint: if this looks like a bug in reeval, reeval report collects a redacted report to file (REEVAL_NO_REPORT_HINT=1 hides this)"));
1989
2342
  }
@@ -2017,4 +2370,4 @@ run(process.argv.slice(2)).finally(() => {
2017
2370
  });
2018
2371
  export {};
2019
2372
 
2020
- //# sourceMappingURL=cli-CGd51thd.js.map
2373
+ //# sourceMappingURL=cli-20Fpvs5H.js.map