reeval 0.17.1 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (297) hide show
  1. package/README.md +4 -2
  2. package/THIRD_PARTY_NOTICES.md +5 -5
  3. package/dist/{adapter-BIlJWXff.js → adapter-CptK032x.js} +2 -2
  4. package/dist/{adapter-BIlJWXff.js.map → adapter-CptK032x.js.map} +1 -1
  5. package/dist/{agent-CE-qdvmh.js → agent-B63OzUX9.js} +124 -11
  6. package/dist/agent-B63OzUX9.js.map +1 -0
  7. package/dist/{analysis-CTBb_C0Z.js → analysis-BnG_1x_O.js} +4 -4
  8. package/dist/{analysis-CTBb_C0Z.js.map → analysis-BnG_1x_O.js.map} +1 -1
  9. package/dist/{app-files-YMmDAHA8.js → app-files-sgqwfjtn.js} +4 -4
  10. package/dist/app-files-sgqwfjtn.js.map +1 -0
  11. package/dist/assemble-CxBHkavM.js +1 -0
  12. package/dist/{assemble-7RI7Nuhz.js → assemble-Me7Whp-y.js} +62 -196
  13. package/dist/assemble-Me7Whp-y.js.map +1 -0
  14. package/dist/{attempt-CtKGvaQR.js → attempt-Dp7Wtm8M.js} +2 -2
  15. package/dist/{attempt-CtKGvaQR.js.map → attempt-Dp7Wtm8M.js.map} +1 -1
  16. package/dist/{attempt-host-CJM_wEA4.js → attempt-host-Cik0SoIn.js} +17 -9
  17. package/dist/{attempt-host-CJM_wEA4.js.map → attempt-host-Cik0SoIn.js.map} +1 -1
  18. package/dist/bridge/index.mjs +9 -5
  19. package/dist/{capture-DF8cHsJh.js → capture-C3RmjPph.js} +392 -609
  20. package/dist/capture-C3RmjPph.js.map +1 -0
  21. package/dist/capture-YPwxp_iK.js +1 -0
  22. package/dist/{cli-Bm9ntSxz.js → cli-BBOlXKPG.js} +1078 -459
  23. package/dist/cli-BBOlXKPG.js.map +1 -0
  24. package/dist/{cli-spec-ChkleEVm.js → cli-spec-DZ0bnP0_.js} +9 -8
  25. package/dist/cli-spec-DZ0bnP0_.js.map +1 -0
  26. package/dist/cli.js +2 -2
  27. package/dist/{client-CF-DOjOU.js → client-tZ2WBDxf.js} +20 -13
  28. package/dist/client-tZ2WBDxf.js.map +1 -0
  29. package/dist/{command-Dk6wggV-.js → command-CFQ9c1hk.js} +13 -13
  30. package/dist/{command-Dk6wggV-.js.map → command-CFQ9c1hk.js.map} +1 -1
  31. package/dist/{command-DPF8hnWS.js → command-DhiwsZl1.js} +9 -9
  32. package/dist/{command-DPF8hnWS.js.map → command-DhiwsZl1.js.map} +1 -1
  33. package/dist/{command-DDkz6jsD.js → command-fO0BFsTq.js} +105 -37
  34. package/dist/command-fO0BFsTq.js.map +1 -0
  35. package/dist/{command-B7MslY_w.js → command-qGS802uX.js} +17 -15
  36. package/dist/command-qGS802uX.js.map +1 -0
  37. package/dist/{commands-DHT-nhEJ.js → commands-CC60TOT5.js} +64 -18
  38. package/dist/commands-CC60TOT5.js.map +1 -0
  39. package/dist/{commands-DQBayPCV.js → commands-xYTMNTPv.js} +1 -1
  40. package/dist/{config-BLL5JqeX.js → config-C2UtS9vy.js} +745 -80
  41. package/dist/config-C2UtS9vy.js.map +1 -0
  42. package/dist/{context-I6nq3uS7.js → context-BajfiH0S.js} +3 -3
  43. package/dist/{context-I6nq3uS7.js.map → context-BajfiH0S.js.map} +1 -1
  44. package/dist/{contrasts-CjOU_pfD.js → contrasts-BjOg7wyr.js} +3 -3
  45. package/dist/{contrasts-CjOU_pfD.js.map → contrasts-BjOg7wyr.js.map} +1 -1
  46. package/dist/cost-BJU0MAji.js +1 -0
  47. package/dist/{cost-e4IppZpJ.js → cost-BuVDamui.js} +4 -4
  48. package/dist/{cost-e4IppZpJ.js.map → cost-BuVDamui.js.map} +1 -1
  49. package/dist/{cursor-catalog-BeBJTk6A.js → cursor-catalog-BTK_apL-.js} +2 -2
  50. package/dist/{cursor-catalog-BeBJTk6A.js.map → cursor-catalog-BTK_apL-.js.map} +1 -1
  51. package/dist/{db-DPiWC7Br.js → db-C5SZ0510.js} +79 -18
  52. package/dist/db-C5SZ0510.js.map +1 -0
  53. package/dist/db-puzLzX4k.js +2 -0
  54. package/dist/{default-effort-ciIXMA7h.js → default-effort-BVmx7r7L.js} +8 -5
  55. package/dist/default-effort-BVmx7r7L.js.map +1 -0
  56. package/dist/{default-effort-B7c4V9Qd.js → default-effort-FEidNAIi.js} +1 -1
  57. package/dist/{dist-CGwueB0T.js → dist-ByYk_wAs.js} +58 -290
  58. package/dist/dist-ByYk_wAs.js.map +1 -0
  59. package/dist/{docker-Bcrc39Z8.js → docker-B2F8-PGC.js} +4 -4
  60. package/dist/{docker-Bcrc39Z8.js.map → docker-B2F8-PGC.js.map} +1 -1
  61. package/dist/{docker-D83s_ADn.js → docker-BlSopHpZ.js} +1 -1
  62. package/dist/{docker-BUZOxh62.js → docker-Dto8ROCq.js} +5 -5
  63. package/dist/{docker-BUZOxh62.js.map → docker-Dto8ROCq.js.map} +1 -1
  64. package/dist/{doctor-CIhEAzbw.js → doctor-BcvfKpBX.js} +20 -20
  65. package/dist/{doctor-CIhEAzbw.js.map → doctor-BcvfKpBX.js.map} +1 -1
  66. package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
  67. package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
  68. package/dist/{environment-Ccytlpgf.js → environment-CC1RV21n.js} +3 -3
  69. package/dist/{environment-Ccytlpgf.js.map → environment-CC1RV21n.js.map} +1 -1
  70. package/dist/{eval-source-D4xqc0iO.js → eval-source-CzxdgFBW.js} +18 -10
  71. package/dist/eval-source-CzxdgFBW.js.map +1 -0
  72. package/dist/evaluate-DToSB2bt.d.ts +101 -0
  73. package/dist/{evaluate-DNzyRQFe.js → evaluate-NTNJ8T9H.js} +135 -54
  74. package/dist/evaluate-NTNJ8T9H.js.map +1 -0
  75. package/dist/{files-D_CBK9Zo.js → files-CZXrkPja.js} +6 -6
  76. package/dist/{files-D_CBK9Zo.js.map → files-CZXrkPja.js.map} +1 -1
  77. package/dist/files-DFtuRQJx.js +1 -0
  78. package/dist/{files-3ZqUBbwg.js → files-DsXULUi5.js} +4 -4
  79. package/dist/{files-3ZqUBbwg.js.map → files-DsXULUi5.js.map} +1 -1
  80. package/dist/{files-B2U5RezT.js → files-aAzoGNCB.js} +5 -5
  81. package/dist/{files-B2U5RezT.js.map → files-aAzoGNCB.js.map} +1 -1
  82. package/dist/files-oOYNR7ZH.js +1 -0
  83. package/dist/{gate-BoDKeuJu.js → gate-Cp_pQoEe.js} +2 -2
  84. package/dist/{gate-BoDKeuJu.js.map → gate-Cp_pQoEe.js.map} +1 -1
  85. package/dist/gateway.d.ts +1 -1
  86. package/dist/{git-s-qf-vww.js → git-C1MB5bjr.js} +3 -3
  87. package/dist/{git-s-qf-vww.js.map → git-C1MB5bjr.js.map} +1 -1
  88. package/dist/harness-DGsRZDSo.js +105 -0
  89. package/dist/harness-DGsRZDSo.js.map +1 -0
  90. package/dist/{harness-Ch2P81sx.js → harness-DYSC4zMh.js} +65 -110
  91. package/dist/harness-DYSC4zMh.js.map +1 -0
  92. package/dist/harness.d.ts +1 -1
  93. package/dist/harness.js +1 -1
  94. package/dist/{images-h1VBM7TP.js → images-CNayXCp2.js} +8 -8
  95. package/dist/{images-h1VBM7TP.js.map → images-CNayXCp2.js.map} +1 -1
  96. package/dist/index.d.ts +3 -3
  97. package/dist/index.js +5 -5
  98. package/dist/init-8o6W3_fn.js +1 -0
  99. package/dist/{init-Z1cRnE3E.js → init-D5HtmyMI.js} +5 -5
  100. package/dist/{init-Z1cRnE3E.js.map → init-D5HtmyMI.js.map} +1 -1
  101. package/dist/ink-list-Ca-5dunO.js +184 -0
  102. package/dist/ink-list-Ca-5dunO.js.map +1 -0
  103. package/dist/{invocations-z6hUD15n.js → invocations-DrGXF9dT.js} +4 -4
  104. package/dist/{invocations-z6hUD15n.js.map → invocations-DrGXF9dT.js.map} +1 -1
  105. package/dist/{judge-ByEUlzOM.js → judge-DYOF69Oe.js} +5 -5
  106. package/dist/{judge-ByEUlzOM.js.map → judge-DYOF69Oe.js.map} +1 -1
  107. package/dist/judge.d.ts +1 -1
  108. package/dist/judge.js +1 -1
  109. package/dist/judgments-m-mb0zv8.js +246 -0
  110. package/dist/judgments-m-mb0zv8.js.map +1 -0
  111. package/dist/list-CM_YSJXq.js +374 -0
  112. package/dist/list-CM_YSJXq.js.map +1 -0
  113. package/dist/{live-D22qY6S_.js → live-B71o2TkX.js} +22 -4
  114. package/dist/live-B71o2TkX.js.map +1 -0
  115. package/dist/live-DQc8jr6o.js +1 -0
  116. package/dist/{live-DpfJfc33.js → live-uRURFDMZ.js} +7 -7
  117. package/dist/{live-DpfJfc33.js.map → live-uRURFDMZ.js.map} +1 -1
  118. package/dist/{matchers-B4ZtlJyP.js → matchers-Cs8aDqdU.js} +3 -3
  119. package/dist/{matchers-B4ZtlJyP.js.map → matchers-Cs8aDqdU.js.map} +1 -1
  120. package/dist/{model-CFSjV-HJ.js → model-1L19Zukh.js} +5 -5
  121. package/dist/{model-CFSjV-HJ.js.map → model-1L19Zukh.js.map} +1 -1
  122. package/dist/{model-BL_10Neu.js → model-BxBWFOZn.js} +44 -20
  123. package/dist/{model-BL_10Neu.js.map → model-BxBWFOZn.js.map} +1 -1
  124. package/dist/{model-D1TBKeZj.js → model-CXVVB4RK.js} +1 -1
  125. package/dist/{model-Bn7ndUQZ.js → model-Ce9l9_rF.js} +27 -254
  126. package/dist/model-Ce9l9_rF.js.map +1 -0
  127. package/dist/model-Cxzo4cjw.js +1 -0
  128. package/dist/{node-CwP1G178.js → node-DhIRSrrK.js} +22 -17
  129. package/dist/{node-CwP1G178.js.map → node-DhIRSrrK.js.map} +1 -1
  130. package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-Co7plCea.js} +4 -15
  131. package/dist/{opencode-providers-yNKyXD49.js.map → opencode-providers-Co7plCea.js.map} +1 -1
  132. package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-DNvLRYM7.js} +2 -2
  133. package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-DNvLRYM7.js.map} +1 -1
  134. package/dist/{patch-LFunkZ-L.js → patch-ug5HpMOj.js} +3 -3
  135. package/dist/{patch-LFunkZ-L.js.map → patch-ug5HpMOj.js.map} +1 -1
  136. package/dist/{plan-BrfiMcHh.js → plan-DqnzfDH1.js} +15 -14
  137. package/dist/plan-DqnzfDH1.js.map +1 -0
  138. package/dist/{plugins-CW-sQMg3.js → plugins-B2KQ0y3P.js} +4 -4
  139. package/dist/{plugins-CW-sQMg3.js.map → plugins-B2KQ0y3P.js.map} +1 -1
  140. package/dist/{preflight-CH-7KyhS.js → preflight-DVXEvzFU.js} +1 -1
  141. package/dist/{preflight-BnFpd3Kv.js → preflight-DyzmvK7i.js} +3 -3
  142. package/dist/{preflight-BnFpd3Kv.js.map → preflight-DyzmvK7i.js.map} +1 -1
  143. package/dist/{prepare-0cA77eFD.js → prepare-21WMQZyt.js} +7 -7
  144. package/dist/{prepare-0cA77eFD.js.map → prepare-21WMQZyt.js.map} +1 -1
  145. package/dist/{prepare-CRyEwb5q.js → prepare-BcSRa7-3.js} +2 -2
  146. package/dist/{prices-C4BRuSe8.js → prices-DYJA0tJN.js} +2 -2
  147. package/dist/{prices-C4BRuSe8.js.map → prices-DYJA0tJN.js.map} +1 -1
  148. package/dist/{provenance-CKIG_1LB.js → provenance-iRxjp4Pu.js} +4 -4
  149. package/dist/{provenance-CKIG_1LB.js.map → provenance-iRxjp4Pu.js.map} +1 -1
  150. package/dist/{providers-CCl8ElNW.js → providers-D0ob_tra.js} +1 -1
  151. package/dist/{providers-Dt6BEN_1.js → providers-vXMXuwn-.js} +5 -5
  152. package/dist/{providers-Dt6BEN_1.js.map → providers-vXMXuwn-.js.map} +1 -1
  153. package/dist/{query-BV-zcw2t.js → query-BhaKQ07P.js} +3 -3
  154. package/dist/{query-BV-zcw2t.js.map → query-BhaKQ07P.js.map} +1 -1
  155. package/dist/query-CvgUsvep.js +1 -0
  156. package/dist/{questions-AyLUMHYX.js → questions-BzdLdV3C.js} +39 -9
  157. package/dist/questions-BzdLdV3C.js.map +1 -0
  158. package/dist/{record-J45gJise.js → record-DhzwkSXc.js} +22 -2
  159. package/dist/{record-J45gJise.js.map → record-DhzwkSXc.js.map} +1 -1
  160. package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
  161. package/dist/redact-BhjtnjrU.js.map +1 -0
  162. package/dist/remarks-CAvTSq16.js +1 -0
  163. package/dist/{remarks-UIZTSIhd.js → remarks-CauU408t.js} +231 -25
  164. package/dist/remarks-CauU408t.js.map +1 -0
  165. package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
  166. package/dist/remarks-D9Lcd1fF.js.map +1 -0
  167. package/dist/{remarks-CgO1nIoG.js → remarks-NMgPd30U.js} +4 -3
  168. package/dist/{remarks-CgO1nIoG.js.map → remarks-NMgPd30U.js.map} +1 -1
  169. package/dist/{remote-BA9-hhgB.js → remote-Dwy3fyo3.js} +5 -5
  170. package/dist/{remote-BA9-hhgB.js.map → remote-Dwy3fyo3.js.map} +1 -1
  171. package/dist/{resolve-FHjE-dN3.js → resolve-B1zMnPwK.js} +7 -7
  172. package/dist/{resolve-FHjE-dN3.js.map → resolve-B1zMnPwK.js.map} +1 -1
  173. package/dist/resolve-BIIeVT12.js +1 -0
  174. package/dist/{results-CFE6Nwb0.js → results-CfC0id_u.js} +7 -7
  175. package/dist/{results-CFE6Nwb0.js.map → results-CfC0id_u.js.map} +1 -1
  176. package/dist/{resume-iaTxWRew.js → resume-Dkhrpjow.js} +123 -8
  177. package/dist/resume-Dkhrpjow.js.map +1 -0
  178. package/dist/{runtime-SEkGS1Rx.js → runtime-DcvQcr36.js} +2 -2
  179. package/dist/{runtime-SEkGS1Rx.js.map → runtime-DcvQcr36.js.map} +1 -1
  180. package/dist/{sandbox-CAsXlGro.js → sandbox-8zCrtuCf.js} +2 -2
  181. package/dist/{sandbox-CAsXlGro.js.map → sandbox-8zCrtuCf.js.map} +1 -1
  182. package/dist/sandbox.d.ts +1 -1
  183. package/dist/sandbox.js +1 -1
  184. package/dist/server-0tuj8VSa.js +1 -0
  185. package/dist/{server-C9_kV1IP.js → server-Dplq8g0r.js} +8 -8
  186. package/dist/{server-C9_kV1IP.js.map → server-Dplq8g0r.js.map} +1 -1
  187. package/dist/{source-BszY6yAC.js → source-NWHqn5-o.js} +2 -2
  188. package/dist/{source-BszY6yAC.js.map → source-NWHqn5-o.js.map} +1 -1
  189. package/dist/source-llwLewOU.js +1 -0
  190. package/dist/{src-Dt6BWDcJ.js → src-B9oE_mNS.js} +1 -1
  191. package/dist/{src-DaYQ1O9Y.js → src-BQ6BiXoc.js} +86 -31
  192. package/dist/{src-DaYQ1O9Y.js.map → src-BQ6BiXoc.js.map} +1 -1
  193. package/dist/src-BQXdf-Gw.js +1448 -0
  194. package/dist/src-BQXdf-Gw.js.map +1 -0
  195. package/dist/{src-DVGVWOAB.js → src-BXnv0ViU.js} +1 -1
  196. package/dist/src-BiS8YNN_.js +3 -0
  197. package/dist/{src-CZANB4pb.js → src-BkH4GOxu.js} +1 -1
  198. package/dist/{src-8fy4nSWU.js → src-BvM0jWuZ.js} +28 -2
  199. package/dist/{src-8fy4nSWU.js.map → src-BvM0jWuZ.js.map} +1 -1
  200. package/dist/src-C-tH5ebd.js +2 -0
  201. package/dist/{src-umdk2alV.js → src-CUpoVVb8.js} +2 -2
  202. package/dist/src-Cg1bFs9g.js +2 -0
  203. package/dist/{src-CWrKJO92.js → src-D5t4OiRt.js} +2 -2
  204. package/dist/{src-CWrKJO92.js.map → src-D5t4OiRt.js.map} +1 -1
  205. package/dist/{src-DoxUxkU3.js → src-DV_UorZv.js} +1 -1
  206. package/dist/{src-DMiwtprP.js → src-KlDOGD0B.js} +4 -4
  207. package/dist/{src-DMiwtprP.js.map → src-KlDOGD0B.js.map} +1 -1
  208. package/dist/{src-BYpY_vXK.js → src-XxUpr2sa.js} +1 -1
  209. package/dist/{state-DfcikRK3.js → state-DDCGUhva.js} +16 -16
  210. package/dist/{state-DfcikRK3.js.map → state-DDCGUhva.js.map} +1 -1
  211. package/dist/state-DZbAO95a.js +1 -0
  212. package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
  213. package/dist/stats-C18W4xPX.js.map +1 -0
  214. package/dist/suite-M8_7xio7.js +1294 -0
  215. package/dist/suite-M8_7xio7.js.map +1 -0
  216. package/dist/suite-u2KJnGpY.js +2 -0
  217. package/dist/{summary-BmooCwQv.js → summary-Uzc9osfV.js} +47 -13
  218. package/dist/summary-Uzc9osfV.js.map +1 -0
  219. package/dist/testing.d.ts +51 -37
  220. package/dist/testing.js +4 -4
  221. package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
  222. package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
  223. package/dist/{transcript-C_IpYMRk.js → transcript-Cg4SeqRX.js} +3 -3
  224. package/dist/{transcript-C_IpYMRk.js.map → transcript-Cg4SeqRX.js.map} +1 -1
  225. package/dist/{types-BlM-aN9_.d.ts → types-DGvexkcp.d.ts} +116 -66
  226. package/dist/{vertex-6uSpZ5L8.js → vertex-BLQMayc1.js} +5 -2
  227. package/dist/vertex-BLQMayc1.js.map +1 -0
  228. package/dist/vertex-h_peaZTN.js +1 -0
  229. package/dist/view/index.d.ts +143 -34
  230. package/dist/view/index.js +9 -8
  231. package/dist/{views-DPng-2-z.js → views-BISrsIjw.js} +6 -6
  232. package/dist/{views-DPng-2-z.js.map → views-BISrsIjw.js.map} +1 -1
  233. package/dist/views-Bfab8fM9.js +2 -0
  234. package/dist/{worker-entry-lt5otEb5.js → worker-entry-DJD_nHkS.js} +5 -5
  235. package/dist/{worker-entry-lt5otEb5.js.map → worker-entry-DJD_nHkS.js.map} +1 -1
  236. package/dist/workspace-B8EOCuxl.js +1 -0
  237. package/dist/{workspace-1O8x4oDc.js → workspace-BO1VhUR9.js} +5 -5
  238. package/dist/{workspace-1O8x4oDc.js.map → workspace-BO1VhUR9.js.map} +1 -1
  239. package/docker/Dockerfile +4 -4
  240. package/package.json +14 -12
  241. package/skills/eval-rubric/SKILL.md +22 -3
  242. package/dist/agent-CE-qdvmh.js.map +0 -1
  243. package/dist/app-files-YMmDAHA8.js.map +0 -1
  244. package/dist/assemble-7RI7Nuhz.js.map +0 -1
  245. package/dist/assemble-CANqwtwD.js +0 -1
  246. package/dist/capture-DF8cHsJh.js.map +0 -1
  247. package/dist/capture-DagM1npp.js +0 -1
  248. package/dist/cli-Bm9ntSxz.js.map +0 -1
  249. package/dist/cli-spec-ChkleEVm.js.map +0 -1
  250. package/dist/client-CF-DOjOU.js.map +0 -1
  251. package/dist/command-B7MslY_w.js.map +0 -1
  252. package/dist/command-DDkz6jsD.js.map +0 -1
  253. package/dist/commands-DHT-nhEJ.js.map +0 -1
  254. package/dist/config-BLL5JqeX.js.map +0 -1
  255. package/dist/cost-CMzY9VXv.js +0 -1
  256. package/dist/db-DPiWC7Br.js.map +0 -1
  257. package/dist/db-DysoTexX.js +0 -2
  258. package/dist/default-effort-ciIXMA7h.js.map +0 -1
  259. package/dist/dist-CGwueB0T.js.map +0 -1
  260. package/dist/eval-source-D4xqc0iO.js.map +0 -1
  261. package/dist/evaluate-BEGAQQuo.d.ts +0 -43
  262. package/dist/evaluate-DNzyRQFe.js.map +0 -1
  263. package/dist/files-Bw2NQecs.js +0 -1
  264. package/dist/files-Cp1jp0Nq.js +0 -1
  265. package/dist/harness-Ch2P81sx.js.map +0 -1
  266. package/dist/init-DJ3WFUt8.js +0 -1
  267. package/dist/live-D22qY6S_.js.map +0 -1
  268. package/dist/live-DUpitP_9.js +0 -1
  269. package/dist/model-Bn7ndUQZ.js.map +0 -1
  270. package/dist/model-CR3-qY-z.js +0 -1
  271. package/dist/plan-BrfiMcHh.js.map +0 -1
  272. package/dist/query-DSSMxkmw.js +0 -1
  273. package/dist/questions-AyLUMHYX.js.map +0 -1
  274. package/dist/redact-BEEGc37e.js.map +0 -1
  275. package/dist/remarks-0EXMh6AD.js.map +0 -1
  276. package/dist/remarks-BL18Id09.js +0 -1
  277. package/dist/remarks-UIZTSIhd.js.map +0 -1
  278. package/dist/resolve-C56s8_EF.js +0 -1
  279. package/dist/resume-iaTxWRew.js.map +0 -1
  280. package/dist/server-BYfqpMD0.js +0 -1
  281. package/dist/source-C4Q5psjD.js +0 -1
  282. package/dist/src-BBmCEE-M.js +0 -3
  283. package/dist/src-BWcgyARO.js +0 -640
  284. package/dist/src-BWcgyARO.js.map +0 -1
  285. package/dist/src-CnrLy_r_.js +0 -2
  286. package/dist/src-DqIQQTu1.js +0 -2
  287. package/dist/state-haxTOljN.js +0 -1
  288. package/dist/stats-D-Gqwqt0.js.map +0 -1
  289. package/dist/suite-BVcBo_ua.js +0 -544
  290. package/dist/suite-BVcBo_ua.js.map +0 -1
  291. package/dist/suite-CmeXyn66.js +0 -2
  292. package/dist/summary-BmooCwQv.js.map +0 -1
  293. package/dist/vertex-6uSpZ5L8.js.map +0 -1
  294. package/dist/vertex-BaAz44YN.js +0 -1
  295. package/dist/views-CBAsDkCB.js +0 -2
  296. package/dist/workspace-It7dvpbg.js +0 -1
  297. /package/dist/bridge/{pnpm-lock.yaml → pnpm-lockfile.yaml} +0 -0
@@ -1,31 +1,32 @@
1
1
  import { n as UsageError } from "./errors-ttMTnIYX.js";
2
2
  import { h as parseDuration } from "./shared-ogqPJ0uo.js";
3
- import { D as profileSandbox, P as workspaceLabel, p as environmentSandbox } from "./environment-Ccytlpgf.js";
4
- import { C as pluginBuildsLabel } from "./plugins-CW-sQMg3.js";
3
+ import { D as profileSandbox, P as workspaceLabel, p as environmentSandbox } from "./environment-CC1RV21n.js";
4
+ import { C as pluginBuildsLabel } from "./plugins-B2KQ0y3P.js";
5
5
  import { a as fromCwd, o as newRunId } from "./ids-DuK6xl4w.js";
6
- import { M as harnessId, O as checkLimitHarness, R as plannedSandboxIds, U as sandboxId, a as configPathFor, b as resolveModelGateway, c as findConfig, d as malformedCredentials, et as ciRunner, f as missingCredentials, g as selectProfile, h as runJudgeModel, i as configLabel, it as isDirect, l as hostCapacity, m as resolveConcurrency, ot as resolveOutput, p as requireConfig, tt as directOnCloud, u as loadConfig, v as gatewayScope, z as plannedSandboxes } from "./config-BLL5JqeX.js";
7
- import { l as judgeModelName, m as unusedGatewayKey } from "./judge-ByEUlzOM.js";
8
- import { y as vertexSettingsIn } from "./vertex-6uSpZ5L8.js";
9
- import { _ as registerSecret, m as redactSecrets } from "./redact-BEEGc37e.js";
10
- import { a as plural, i as formatDuration, n as clip, o as sandboxWarningLine } from "./text-D--Q5KsA.js";
11
- import { n as loadEnvFiles, t as inCI } from "./env-T86lAqUD.js";
6
+ import { $ as directOnCloud, M as harnessId, O as checkLimitHarness, Q as ciRunner, R as plannedSandboxIds, U as sandboxId, a as configPathFor, b as resolveModelGateway, c as findConfig, d as malformedCredentials, f as missingCredentials, g as selectProfile, h as runJudgeModel, i as configLabel, it as resolveOutput, l as hostCapacity, m as resolveConcurrency, nt as isDirect, p as requireConfig, u as loadConfig, v as gatewayScope, z as plannedSandboxes } from "./config-C2UtS9vy.js";
7
+ import { l as judgeModelName, m as unusedGatewayKey } from "./judge-DYOF69Oe.js";
8
+ import { y as vertexSettingsIn } from "./vertex-BLQMayc1.js";
9
+ import { h as redactSecrets, v as registerSecret } from "./redact-BhjtnjrU.js";
10
+ import { a as plural, i as formatDuration, n as clip } from "./text-B8-W3Cvn.js";
11
+ import { n as loadEnvFiles, t as inCI } from "./env-8x1O8I1M.js";
12
12
  import { o as PACKAGE_VERSION } from "./root-Co0jhOT5.js";
13
- import { W as LABEL$1, st as imageInfo } from "./src-DaYQ1O9Y.js";
14
- import { s as debugTime } from "./debug-B5jDImcW.js";
15
- import { r as openRouterFallbacks } from "./openrouter-models-C8nzPKG2.js";
13
+ import { W as LABEL$1, st as imageInfo } from "./src-BQ6BiXoc.js";
14
+ import { r as openRouterFallbacks } from "./openrouter-models-DNvLRYM7.js";
16
15
  import { t as PrepareError } from "./provider-DZd8t_XW.js";
17
- import { D as runOutcome, G as triesNote, _ as formatUsage } from "./model-Bn7ndUQZ.js";
18
- import { u as suiteResult } from "./suite-BVcBo_ua.js";
19
- import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-ChkleEVm.js";
16
+ import { _ as formatUsage, k as runOutcome } from "./model-Ce9l9_rF.js";
17
+ import { h as measuresLine, i as effectOf, r as countedOf, s as isNotGatedResult, u as suiteResult, v as observedLine } from "./suite-M8_7xio7.js";
18
+ import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-DZ0bnP0_.js";
20
19
  import { n as stripAnsi, t as colors } from "./colors-DcXBvAlU.js";
21
- import { m as render } from "./summary-BmooCwQv.js";
22
- import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-CKIG_1LB.js";
20
+ import { i as shownRemarks } from "./remarks-D9Lcd1fF.js";
21
+ import { _ as statsLine, c as checkLabel } from "./summary-Uzc9osfV.js";
22
+ import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-iRxjp4Pu.js";
23
23
  import { t as budgetMarker } from "./budget-BcxLYY5L.js";
24
- import { a as prepareResume, c as writeCurrent, i as loadPrevious, l as writeLiveProgress, r as keptAttempts, s as resolveResume, t as completedKeysFor } from "./resume-iaTxWRew.js";
25
- import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-BnFpd3Kv.js";
26
- import { n as imageHarness } from "./prepare-0cA77eFD.js";
27
- import { i as createProviders, n as attemptImage } from "./providers-Dt6BEN_1.js";
28
- import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-z6hUD15n.js";
24
+ import { a as liveAttempts, d as savedFilters, f as savedInvocation, i as leftOutAttempts, l as resolveResume, m as writeLiveProgress, o as loadPrevious, p as writeCurrent, r as keptAttempts, s as prepareResume, t as completedKeysFor, u as resumeSettings } from "./resume-Dkhrpjow.js";
25
+ import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-DyzmvK7i.js";
26
+ import { n as imageHarness } from "./prepare-21WMQZyt.js";
27
+ import { i as createProviders, n as attemptImage } from "./providers-vXMXuwn-.js";
28
+ import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-DrGXF9dT.js";
29
+ import { c as attemptAxes, d as failureLines, f as offset, g as tailLines, h as progressLine, l as debugLines, m as phaseOf, p as outcomeOf, r as listLines, s as progressText, t as LiveList, u as errorCheck } from "./list-CM_YSJXq.js";
29
30
  import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
30
31
  import { dirname, join } from "node:path";
31
32
  import { inspect } from "node:util";
@@ -218,204 +219,654 @@ async function preflight(uses, options) {
218
219
  }
219
220
  return { images };
220
221
  }
221
- // src/ui/format.ts
222
- /** Most detail lines a `-vv` payload prints under its line. */
223
- const DEBUG_DETAIL_LINES = 40;
222
+ // src/ui/header.ts
223
+ /** ` ➜ `, ` ⚠ `: the glyph column every line of the block starts with. */
224
+ const GLYPH = 5;
225
+ /** `Est. Time: `, the longest key. */
226
+ const KEY = 11;
227
+ /** `Warning: `, the longer note label. */
228
+ const LABEL = 9;
224
229
  /**
225
- * A debug line as the terminal shows it, `12:50:01 image cache hit …` (with milliseconds under
226
- * `-vv`), then its detail indented under it.
230
+ * `text` broken at spaces into lines of at most `width` columns, not counting color codes; a
231
+ * longer word keeps a line. A `·` stays with the word before it, so no line starts with one.
227
232
  */
228
- function debugLines(c, line, ms) {
229
- const phase = line.phase.padEnd(9);
230
- const head = `${c.dim(debugTime(line.at, ms))} ${c.dim(phase)} ${line.level === 2 ? c.dim(line.text) : line.text}`;
231
- const detail = (line.detail ?? "").split("\n").filter((l) => l.trim());
232
- const shown = detail.slice(0, DEBUG_DETAIL_LINES).map((l) => c.dim(` ${l}`));
233
- const more = detail.length - shown.length;
233
+ function wrap(text, width) {
234
+ const words = [];
235
+ for (const word of text.split(/\s+/).filter(Boolean)) if (word === "·" && words.length) words[words.length - 1] += " ·";
236
+ else words.push(word);
237
+ const lines = [];
238
+ let line = "";
239
+ for (const word of words) if (line && stripAnsi(line).length + 1 + stripAnsi(word).length > width) {
240
+ lines.push(line);
241
+ line = word;
242
+ } else line = line ? `${line} ${word}` : word;
243
+ return line ? [...lines, line] : lines;
244
+ }
245
+ const indent = (n) => " ".repeat(n);
246
+ /** `REEVAL v0.16.1`, then the run's facts, one `➜ Key: value` line each. */
247
+ function headerLines(c, h, width) {
248
+ const name = `REEVAL v${h.version}`;
249
+ const title = c.bold(c.depth === 4 ? name : c.teal(name));
250
+ const valueWidth = width - GLYPH - KEY;
251
+ const row = (key, value) => {
252
+ const values = typeof value === "string" ? [value] : value;
253
+ const joined = values.join(", ");
254
+ return (values.length > 1 && stripAnsi(joined).length > valueWidth ? values : wrap(joined, valueWidth)).map((v, i) => i === 0 ? ` ${c.teal("➜")} ${c.dim(`${key}:`)}${indent(KEY - key.length - 1)}${v}` : `${indent(16)}${v}`);
255
+ };
256
+ const many = (one, n, many = `${one}s`) => n === 1 ? one : many;
257
+ const resumed = h.resumed ? c.dim(` (resumed, ${plural(h.resumed, "attempt")} done)`) : "";
258
+ const reason = h.profile && h.profile.reason !== "--profile" ? c.dim(` (${h.profile.reason})`) : "";
234
259
  return [
235
- head,
236
- ...shown,
237
- ...more > 0 ? [c.dim(` … ${more} more lines in trace.ndjson`)] : []
260
+ h.dryRun ? `${title} ${c.dim("dry run")}` : title,
261
+ "",
262
+ ...row("Run", `${h.runId}${resumed}`),
263
+ ...h.config ? row("Config", h.config) : [],
264
+ ...h.profile ? row("Profile", `${h.profile.name}${reason}`) : [],
265
+ ...row(many("Harness", h.harnesses.length, "Harnesses"), h.harnesses.map((id) => harness(c, h, id))),
266
+ ...row(many("Sandbox", h.sandboxes.length, "Sandboxes"), h.sandboxes),
267
+ ...h.judges.length ? row(many("Judge", h.judges.length), h.judges) : [],
268
+ ...row("Scope", scope(c, h)),
269
+ ...row("Est. Time", estimate(c, h)),
270
+ ...h.record && !h.dryRun ? row("Record", h.record) : []
238
271
  ];
239
272
  }
240
- /** Offset into an attempt, `+1:05`. */
241
- function offset(ms) {
242
- const s = Math.max(0, Math.floor(ms / 1e3));
243
- return `+${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`;
244
- }
245
- function outcomeOf(e) {
246
- if (e.state === "skipped") return "skipped";
247
- if (e.meta?.errorKind === "aborted") return "canceled";
248
- return e.meta?.outcome ?? (e.state === "passed" ? "pass" : "fail");
249
- }
250
- /** What tells an eval's attempts apart: the axes that vary (`cursor:composer-2.5 · with-skill`), else the harness. */
251
- function attemptAxes(a, fallback) {
252
- return a?.label.replace(/\s*#\d+$/, "") || a?.harness.kind || fallback;
253
- }
254
- /** A command on one line: the first line, cut to 80 characters at a word boundary. */
255
- function oneLine(text, max = 80) {
256
- const [first = ""] = text.trim().split("\n");
257
- const cut = clip(first, max);
258
- return cut === first && text.trim().includes("\n") ? `${first}…` : cut;
259
- }
260
- /** One line per worker progress event, without prefix or offset. */
261
- function progressLine(c, p) {
262
- switch (p.kind) {
263
- case "sandbox": return c.dim("sandbox starting");
264
- case "sandbox-ready": return `sandbox ready ${c.dim(formatDuration(p.ms))}`;
265
- case "turn-start": return `turn ${p.turn} started`;
266
- case "turn-end": {
267
- const status = p.status === "completed" ? "done" : c.red(p.status);
268
- const activity = [plural(p.tools, "tool call"), plural(p.fileEdits, "file edit")].join(", ");
269
- const error = p.error ? `: ${c.red(oneLine(p.error, 160))}` : "";
270
- return `turn ${p.turn} ${status}${error} ${c.dim(`${formatDuration(p.ms)} · ${activity}`)}`;
271
- }
272
- case "check-start": return c.dim(`$ ${oneLine(p.command)}`);
273
- case "check-end": {
274
- const exit = p.exitCode === 0 ? c.dim("exit 0") : c.red(`exit ${p.exitCode}`);
275
- const tries = p.try && p.tries ? triesNote(p.exitCode, p.try, p.tries) : null;
276
- return `$ ${oneLine(p.command)} ${exit} ${c.dim([formatDuration(p.ms), tries].filter(Boolean).join(" · "))}`;
277
- }
278
- case "judge-start": return c.dim(`judging: ${oneLine(p.question, 120)}`);
279
- case "judge-end": return judgeLine(c, p);
280
- case "cleanup": return c.dim("cleaning up");
281
- case "warning": return c.yellow(`warning: ${p.text}`);
282
- case "browser-download": {
283
- const total = p.bytes == null ? "?" : formatBytes(p.bytes);
284
- return `downloading ${p.name} ${p.version} ${formatBytes(p.received)} / ${total} → ${p.cache}`;
285
- }
273
+ /** A harness id, with the effort it runs at when the id doesn't name it. */
274
+ function harness(c, h, id) {
275
+ const effort = h.efforts?.[id];
276
+ if (effort === void 0 || id.includes("[effort=")) return id;
277
+ return `${id} ${`(effort: ${effort})`.split(" ").map((w) => c.dim(w)).join(" ")}`;
278
+ }
279
+ function scope(c, h) {
280
+ const attempts = h.fewest < h.attempts ? `${h.fewest}–${h.attempts} attempts` : plural(h.attempts, "attempt");
281
+ const parts = [
282
+ plural(h.files, "eval file"),
283
+ plural(h.evals, "eval") + (h.skipped ? ` (${h.skipped} skipped)` : ""),
284
+ plural(h.cells, "cell"),
285
+ attempts
286
+ ];
287
+ const caps = [h.budget.attempts !== void 0 ? `${h.budget.attempts} ${h.attempts <= h.budget.attempts ? c.green("✓") : c.red("✗")}` : "", h.budget.duration ?? ""].filter(Boolean);
288
+ return `${parts.join(" · ")}${caps.length ? ` (budget: ${caps.join(", ")})` : ""}`;
289
+ }
290
+ function estimate(c, h) {
291
+ const serial = `~${formatDuration(h.estimateMs)} ${c.dim("(serial)")}`;
292
+ const parallel = Math.min(h.concurrency, h.attempts);
293
+ if (parallel <= 1) return serial;
294
+ return `${`~${formatDuration(h.estimateMs / parallel)} ${c.dim(`(concurrency: ${h.concurrency})`)}`} · ${serial}`;
295
+ }
296
+ /** `⚠ Warning: …` or `ℹ Hint: …`, wrapped under its first word. */
297
+ function noteLines(c, kind, text, width) {
298
+ const [glyph, label, paint] = kind === "hint" ? [
299
+ "ℹ",
300
+ "Hint:",
301
+ c.blue
302
+ ] : [
303
+ "⚠",
304
+ "Warning:",
305
+ kind === "root" ? c.red : c.orange
306
+ ];
307
+ const basic = {
308
+ warning: c.yellow,
309
+ hint: c.teal,
310
+ root: c.red
311
+ }[kind];
312
+ const lead = c.depth === 4 && kind !== "root" ? `${basic(glyph)} ${c.bold(label)}` : c.bold(paint(`${glyph} ${label}`));
313
+ const at = 14;
314
+ return wrap(text, Math.max(20, width - at)).map((line, i) => i === 0 ? ` ${lead}${indent(LABEL - label.length)}${line}` : `${indent(at)}${line}`);
315
+ }
316
+ /** A step line, `◆ Preparing 1 sandbox (docker)...`, its text wrapped under itself. */
317
+ function stepLines(glyph, text, width) {
318
+ return wrap(text, Math.max(20, width - GLYPH)).map((line, i) => i === 0 ? ` ${glyph} ${line}` : `${indent(GLYPH)}${line}`);
319
+ }
320
+ /** A run-level notice (a prepared workspace, the Ctrl-C one) as a step line. */
321
+ function noticeLines(c, e, width) {
322
+ const glyph = {
323
+ info: c.dim("ℹ"),
324
+ warn: c.orange("⚠"),
325
+ error: c.red("✗")
326
+ }[e.level];
327
+ const lines = stepLines(glyph, e.text, width);
328
+ return e.level === "info" ? lines.map((l) => c.dim(l)) : lines;
329
+ }
330
+ /** Preflight: which sandboxes it prepares, each one ready, and each failure. */
331
+ function prepareStepLines(c, e, width, max) {
332
+ if (e.state === "start") {
333
+ const n = e.targets.length;
334
+ const names = e.targets.map((t) => t.label).join(", ");
335
+ return stepLines(c.teal("◆"), `Preparing ${plural(n, "sandbox", "sandboxes")} (${names})...`, width);
286
336
  }
337
+ const { outcome } = e;
338
+ if (outcome.ok) {
339
+ const took = outcome.ms >= 1e3 ? ` ${c.dim(formatDuration(outcome.ms))}` : "";
340
+ return [` ${c.green("✓")} ${outcome.target.label} ready${took}`];
341
+ }
342
+ const who = outcome.target.setBy.join(", ");
343
+ return [` ${c.red("✗")} ${outcome.target.label} failed to prepare ${c.dim(`(set by ${who}; only its attempts fail)`)}`, ...tailLines(outcome.failure.message, max).map((l) => `${indent(GLYPH)}${c.orange(l)}`)];
287
344
  }
288
- function formatBytes(n) {
289
- if (n < 1024) return `${n} B`;
290
- if (n < 1048576) return `${(n / 1024).toFixed(1)} KB`;
291
- return `${(n / 1048576).toFixed(1)} MB`;
292
- }
293
- function judgeLine(c, j) {
294
- const verdict = j.verdict === "pass" ? c.green("pass") : c.red(j.verdict);
295
- const score = j.score === null ? "" : ` ${j.score.toFixed(2)}`;
296
- const threshold = j.threshold === null ? "" : ` ${c.dim(`(≥ ${j.threshold})`)}`;
297
- return `${c.dim("judge")} ${verdict}${score}${threshold} ${j.question}`;
298
- }
299
- /** The phase a running attempt is in, for the heartbeat. */
300
- function phaseOf(p) {
301
- switch (p?.kind) {
302
- case void 0: return "starting";
303
- case "sandbox": return "sandbox";
304
- case "sandbox-ready": return "sandbox ready";
305
- case "turn-start": return `turn ${p.turn}`;
306
- case "turn-end": return `turn ${p.turn} done`;
307
- case "check-start":
308
- case "check-end": return "checks";
309
- case "judge-start":
310
- case "judge-end": return "judging";
311
- case "cleanup": return "cleanup";
312
- case "browser-download": return "browser download";
313
- case "warning": return "running";
314
- }
315
- }
316
- function capLines(text, max) {
317
- const lines = text.replace(/\s+$/, "").split("\n");
318
- if (lines.length <= max) return lines;
319
- return [...lines.slice(0, max), `(+${lines.length - max} more lines)`];
320
- }
321
- /** An infra error's headline, then its last lines: the cause is at the end. */
322
- function tailLines(text, max) {
323
- const lines = text.replace(/\s+$/, "").split("\n");
324
- if (lines.length <= max) return lines;
325
- const rest = Math.max(1, max - 1);
326
- return [
327
- lines[0] ?? "",
328
- `(${lines.length - 1 - rest} more lines)`,
329
- ...lines.slice(-rest)
345
+ /** The last line before the attempts' own: how their logs will print. */
346
+ function startLine(c, output) {
347
+ const text = output.outputLogs === "none" ? "Running..." : output.outputLogs === "errors-only" ? "Running; logs print for failed attempts only..." : output.logOrder === "stream" ? "Streaming logs..." : "Running; each attempt's logs print as it ends...";
348
+ return ` ${c.teal("◆")} ${text}`;
349
+ }
350
+ /** What the start line becomes once the run ends, on a terminal that redraws it. */
351
+ function endLine(c, status) {
352
+ const text = status === "interrupted" ? "Canceled." : status === "budget" ? "Stopped at the budget." : "Finished.";
353
+ return ` ${c.teal("◆")} ${text}`;
354
+ }
355
+ /** The plan as the header block: its facts, `--dry-run`'s table, then each kind of note. */
356
+ function planLines(c, e, width) {
357
+ const sections = [
358
+ e.details,
359
+ [...e.warnings.flatMap((w) => noteLines(c, "warning", w, width)), ...(e.sandboxWarnings ?? []).flatMap((w) => noteLines(c, w.kind === "root" ? "root" : "warning", w.message, width))],
360
+ e.hints.flatMap((h) => noteLines(c, "hint", h, width))
330
361
  ];
362
+ const out = headerLines(c, e.header, width);
363
+ for (const lines of sections) if (lines.length) out.push("", ...lines);
364
+ out.push("");
365
+ return out;
366
+ }
367
+ // src/ui/status.ts
368
+ const EFFECT = {
369
+ better: "helped",
370
+ worse: "made it worse",
371
+ same: "same"
372
+ };
373
+ const ERROR_VERB = {
374
+ assertion: "failed",
375
+ timeout: "timed out",
376
+ aborted: "canceled",
377
+ infra: "hit an infrastructure error",
378
+ judge: "could not be judged",
379
+ turn: "ended a turn with an error"
380
+ };
381
+ const MESSAGE_LINES = 4;
382
+ const LINE_MAX = 160;
383
+ const times = (n) => n === 1 ? "once" : `${n} times`;
384
+ /** `cursor (composer-2.5)`, from the record's cell. */
385
+ function harnessLabel(record, cell) {
386
+ const model = record.evals.flatMap((e) => e.cells).find((c) => c.key === cell.key && c.harness === cell.harness)?.model;
387
+ return model && model !== cell.harness ? `${cell.harness} (${model})` : cell.harness;
388
+ }
389
+ /** The cells an eval's outcome reads: the gated ones, or every cell of a study. */
390
+ function deciding(cells) {
391
+ const gated = cells.filter((c) => c.gated);
392
+ return gated.length ? gated : cells;
393
+ }
394
+ const cellsOf = (e) => e.cells ?? (e.cases ?? []).flatMap((k) => k.cells);
395
+ /** Each part's label, adding the environment and sandbox only where the cells differ in them. */
396
+ function labeler(record, cells) {
397
+ const envs = new Set(cells.map((c) => c.environment)).size > 1;
398
+ const sandboxes = new Set(cells.map((c) => c.sandbox)).size > 1;
399
+ return (c) => [
400
+ harnessLabel(record, c),
401
+ envs ? `in ${c.environment}` : "",
402
+ sandboxes ? `on ${c.sandbox}` : ""
403
+ ].filter(Boolean).join(" ");
404
+ }
405
+ function tallyBy(cells, label) {
406
+ const by = /* @__PURE__ */ new Map();
407
+ for (const c of cells) {
408
+ const key = label(c);
409
+ const t = by.get(key) ?? {
410
+ label: key,
411
+ passed: 0,
412
+ counted: 0
413
+ };
414
+ t.passed += c.passed;
415
+ t.counted += countedOf(c);
416
+ by.set(key, t);
417
+ }
418
+ return [...by.values()];
419
+ }
420
+ const of = (t) => `${t.passed} of ${t.counted}`;
421
+ /** The outcome sentence, per the rules at the top. */
422
+ function outcomeSentence(tallies) {
423
+ const counted = tallies.filter((t) => t.counted > 0);
424
+ if (!counted.length) return "No run finished.";
425
+ const [one] = counted;
426
+ if (counted.length === 1 && one) {
427
+ const once = one.counted === 1;
428
+ if (one.passed === one.counted) return `Passed on ${one.label}${once ? "" : " every time"}.`;
429
+ if (one.passed === 0) return `Failed on ${one.label}${once ? "" : " every time"}.`;
430
+ return `Passed ${of(one)} on ${one.label}.`;
431
+ }
432
+ const names = counted.map((t) => t.label);
433
+ const all = `${names.slice(0, -1).join(", ")} and ${names.at(-1)}`;
434
+ if (counted.every((t) => t.passed === t.counted)) return `Passed on ${all} every time.`;
435
+ if (counted.every((t) => t.passed === 0)) return `Failed on ${all} every time.`;
436
+ const [first, ...rest] = counted;
437
+ return `${first?.label} passed ${first ? of(first) : ""}${rest.map((t) => `, ${t.label} ${of(t)}`).join("")}.`;
438
+ }
439
+ /** `(cursor 4 of 4, codex 1 of 4)` when the harnesses' pass rates differ; empty when they agree. */
440
+ function disagreement(record, cells) {
441
+ const tallies = tallyBy(cells, (c) => harnessLabel(record, c)).filter((t) => t.counted > 0);
442
+ if (tallies.length < 2) return "";
443
+ return new Set(tallies.map((t) => t.passed / t.counted)).size > 1 ? ` (${tallies.map((t) => `${t.label} ${of(t)}`).join(", ")})` : "";
444
+ }
445
+ /** `, not by minLift 60` when the variant's lift fell short of its `minLift`, which fails it. */
446
+ function minLiftShort(e, own) {
447
+ const keys = new Set(own.map((c) => c.key));
448
+ const pooled = (e.pooled ?? []).filter((p) => keys.has(p.key));
449
+ const missed = pooled.length ? pooled.find((p) => p.minLift !== null && p.outcome === "fail")?.minLift : own.find((c) => c.minLift !== null && c.interval && c.interval.low < c.minLift)?.minLift;
450
+ return missed == null ? "" : `, not by minLift ${Math.round(missed * 100)}`;
331
451
  }
332
452
  /**
333
- * Why an eval failed, as lines: the judge verdict (and reason) for judge failures, Vitest's
334
- * formatted error (diff, code frame) for assertions, the tail of the message for infra errors.
453
+ * Under a variant, its comparison with the baseline (`measuresLine`): `Same pass rate · 5× faster ·
454
+ * judge "tidy" +0.12.`, one line, or one per harness (`cursor: …`) when it ran on several. Only
455
+ * when some measure besides pass rate holds; `verbose` adds the ones that only lean.
335
456
  */
336
- function failureLines(c, e, options) {
337
- const outcome = outcomeOf(e);
338
- if (outcome === "canceled") return [c.yellow("canceled")];
339
- const message = e.errors[0] ?? "";
340
- const turns = (e.meta?.turnErrors ?? []).filter((t) => !message.includes(t.replace(/^turn \d+ error: /, ""))).map((t) => c.yellow(t));
457
+ function measureLines(e, own, verbose) {
458
+ const keys = new Set(own.map((c) => c.key));
459
+ const pooled = (e.pooled ?? []).filter((p) => keys.has(p.key));
460
+ const sets = pooled.length ? pooled.map((p) => ({
461
+ harness: own.find((c) => c.key === p.key)?.harness ?? p.key,
462
+ measures: p.measures
463
+ })) : own.flatMap((c) => c.measures ? [{
464
+ harness: c.harness,
465
+ measures: c.measures
466
+ }] : []);
467
+ const several = new Set(sets.map((x) => x.harness)).size > 1;
468
+ return sets.flatMap(({ harness, measures }) => {
469
+ if (!measures.filter((m) => m.measure !== "pass" && (m.verdict === "better" || m.verdict === "worse" || verbose && m.verdict === "leans")).length) return [];
470
+ const line = measuresLine(measures, verbose);
471
+ const text = `${line.slice(0, 1).toUpperCase()}${line.slice(1)}.`;
472
+ return [{
473
+ depth: 2,
474
+ text: several ? `${harness}: ${text}` : text,
475
+ tone: "dim"
476
+ }];
477
+ });
478
+ }
479
+ function compareLines(record, e, cells, verbose) {
480
+ const base = cells.filter((c) => c.baseline);
481
+ const name = (c) => c.environment === "-" ? "no environment" : c.environment;
482
+ const baseTally = tallyBy(base, () => "baseline")[0] ?? {
483
+ passed: 0,
484
+ counted: 0
485
+ };
486
+ const lines = [{
487
+ depth: 0,
488
+ text: `Baseline ${[...new Set(base.map(name))].join(", ")} passed ${of(baseTally)}${disagreement(record, base)}.`
489
+ }];
490
+ const variants = [...new Set(cells.filter((c) => c.variant !== null).map((c) => c.variant))];
491
+ if (!variants.length) return lines;
492
+ const rows = variants.map((v, order) => {
493
+ const own = cells.filter((c) => c.variant === v);
494
+ const t = tallyBy(own, () => v ?? "")[0] ?? {
495
+ passed: 0,
496
+ counted: 0
497
+ };
498
+ const effect = effectOf({
499
+ pass: t.passed,
500
+ counted: t.counted
501
+ }, {
502
+ pass: baseTally.passed,
503
+ counted: baseTally.counted
504
+ });
505
+ return {
506
+ v: v ?? "",
507
+ t,
508
+ effect,
509
+ own,
510
+ order
511
+ };
512
+ });
513
+ const rate = (t) => t.counted ? t.passed / t.counted : -1;
514
+ rows.sort((a, b) => rate(b.t) - rate(a.t) || a.order - b.order);
515
+ const width = Math.max(...rows.map((r) => r.v.length));
516
+ lines.push({
517
+ depth: 0,
518
+ text: "Variants, by pass rate:",
519
+ tone: "heading"
520
+ });
521
+ for (const r of rows) {
522
+ const short = minLiftShort(e, r.own);
523
+ const word = r.effect ? ` ${EFFECT[r.effect]}${short}` : short ? ` ${short.slice(2)}` : "";
524
+ lines.push({
525
+ depth: 1,
526
+ text: `${r.v.padEnd(width)} ${of(r.t)}${word}${disagreement(record, r.own)}`
527
+ });
528
+ lines.push(...measureLines(e, r.own, verbose));
529
+ if (!verbose) continue;
530
+ for (const cell of r.own) {
531
+ const stats = statsLine(cell, cells);
532
+ const who = r.own.length > 1 ? `${harnessLabel(record, cell)}: ` : "";
533
+ if (stats) lines.push({
534
+ depth: 2,
535
+ text: `${who}${stats}`,
536
+ tone: "dim"
537
+ });
538
+ }
539
+ }
540
+ return lines;
541
+ }
542
+ function failedChecks(cells, groupOf) {
543
+ const by = /* @__PURE__ */ new Map();
544
+ for (const cell of cells) for (const k of cell.checks) {
545
+ if (k.outcome.status !== "ran" || k.outcome.failed + k.outcome.errored === 0) continue;
546
+ const n = k.outcome.failed + k.outcome.errored;
547
+ const f = by.get(k.label) ?? {
548
+ label: k.label,
549
+ failed: 0,
550
+ byHarness: /* @__PURE__ */ new Map(),
551
+ first: k
552
+ };
553
+ f.failed += n;
554
+ const h = groupOf(cell);
555
+ f.byHarness.set(h, (f.byHarness.get(h) ?? 0) + n);
556
+ if (f.first.outcome.status === "ran" && !f.first.outcome.assertion) f.first = k;
557
+ by.set(k.label, f);
558
+ }
559
+ return [...by.values()].sort((a, b) => b.failed - a.failed || a.first.order - b.first.order);
560
+ }
561
+ /** `, all on H` or `, M of them on H`, per the rules at the top. */
562
+ function where(f, harnesses) {
563
+ if (harnesses < 2) return "";
564
+ const [top] = [...f.byHarness].sort((a, b) => b[1] - a[1]);
565
+ if (!top) return "";
566
+ const [h, n] = top;
567
+ if (n === f.failed) return f.failed === 1 ? `, on ${h}` : `, all on ${h}`;
568
+ return n * 2 > f.failed ? `, ${n} of them on ${h}` : "";
569
+ }
570
+ /** A message's first lines, without the label it repeats or a command's empty output headings. */
571
+ function messageLines(text, label = "", max = LINE_MAX) {
572
+ return (label && text.startsWith(`${label}: `) ? text.slice(label.length + 2) : text).split("\n").map((l) => l.trimEnd()).filter((l) => l.trim() && !/^--- (stderr|stdout) ---$/.test(l.trim())).slice(0, MESSAGE_LINES).map((l) => clip(l, max));
573
+ }
574
+ /** A failure's message is red only on a failed eval: a passing compare's baseline misses are expected. */
575
+ function checkLines(cells, groupOf, tone) {
576
+ const failed = failedChecks(cells, groupOf);
577
+ if (!failed.length) return [];
578
+ const harnesses = new Set(cells.map(groupOf)).size;
579
+ const lines = [{
580
+ depth: 0,
581
+ text: failed.length === 1 ? "Every failure was the same check:" : `${failed.length} checks failed:`,
582
+ tone: "heading"
583
+ }];
584
+ for (const f of failed) {
585
+ lines.push({
586
+ depth: 1,
587
+ text: `"${checkLabel(f.label)}" failed ${times(f.failed)}${where(f, harnesses)}`
588
+ });
589
+ const o = f.first.outcome;
590
+ const judge = f.first.judge;
591
+ const message = o.status === "ran" && o.assertion ? o.assertion : judge ? `judge: ${judge.question} (scores ${judge.scores.join(", ")}, threshold ${judge.threshold})` : "";
592
+ for (const text of messageLines(message, f.label)) lines.push({
593
+ depth: 2,
594
+ text,
595
+ tone
596
+ });
597
+ if (o.status === "ran" && o.at) lines.push({
598
+ depth: 2,
599
+ text: `at ${o.at}`,
600
+ tone: "dim"
601
+ });
602
+ }
603
+ return lines;
604
+ }
605
+ const CASES_NAMED = 8;
606
+ function failedCaseLines(e) {
607
+ const failed = (e.cases ?? []).filter((k) => k.outcome !== "pass" && k.outcome !== "skipped");
608
+ if (!failed.length) return [];
609
+ const named = failed.slice(0, CASES_NAMED).map((k) => k.label);
610
+ const more = failed.length > CASES_NAMED ? ` and ${failed.length - CASES_NAMED} more` : "";
611
+ return [{
612
+ depth: 0,
613
+ text: `Failed cases: ${named.join(", ")}${more}.`
614
+ }];
615
+ }
616
+ /** Errors no failed check explains: timeouts, infrastructure errors, throws. Never clipped: their remedy is often last. */
617
+ function errorLines(cells, tone) {
618
+ const by = /* @__PURE__ */ new Map();
619
+ for (const c of cells) for (const err of c.errors) {
620
+ const key = `${err.kind}\n${err.message}`;
621
+ const e = by.get(key) ?? {
622
+ kind: err.kind,
623
+ message: err.message,
624
+ n: 0
625
+ };
626
+ e.n += err.attempts;
627
+ by.set(key, e);
628
+ }
629
+ return [...by.values()].flatMap((e) => [{
630
+ depth: 0,
631
+ text: `${plural(e.n, "run")} ${ERROR_VERB[e.kind]}:`,
632
+ tone: "heading"
633
+ }, ...messageLines(e.message, "", Number.POSITIVE_INFINITY).map((text) => ({
634
+ depth: 1,
635
+ text,
636
+ tone
637
+ }))]);
638
+ }
639
+ /** `evals/a.eval.ts:12` from `source:evals/a.eval.ts#L12`. */
640
+ const location = (at) => at.replace(/^source:/, "").replace(/#L(\d+)$/, ":$1");
641
+ /** `"npm test" passed on try 2 of 3.`: a check that needed its `c.exec` retries is flaky. */
642
+ function retriedLines(cells) {
643
+ const seen = /* @__PURE__ */ new Set();
644
+ return cells.flatMap((c) => c.checks.flatMap((k) => {
645
+ if (!k.retried || seen.has(k.label)) return [];
646
+ seen.add(k.label);
647
+ return [`"${checkLabel(k.label)}" ${k.retried}.`];
648
+ }));
649
+ }
650
+ function pageErrorNotes(cells) {
651
+ const [first, ...rest] = cells.flatMap((c) => c.pageErrors ?? []);
652
+ if (!first) return [];
653
+ const more = rest.length ? ` (+${plural(rest.length, "more capture")})` : "";
654
+ return [`${plural(first.errors, "page error")} in capture "${first.capture}" → ${first.file}${more}`];
655
+ }
656
+ function remarkLines(remarks, notes) {
657
+ if (!remarks.length && !notes.length) return [];
341
658
  return [
342
- ...failureBody(c, e, outcome, message, options),
343
- ...otherChecks(c, e),
344
- ...turns
659
+ {
660
+ depth: 0,
661
+ text: "Worth knowing:",
662
+ tone: "heading"
663
+ },
664
+ ...remarks.map((r) => ({
665
+ depth: 1,
666
+ text: r.fix.edit ? `${r.text} Fix at ${location(r.fix.at)}: ${r.fix.edit}.` : `${r.text} At ${location(r.fix.at)}.`
667
+ })),
668
+ ...notes.map((text) => ({
669
+ depth: 1,
670
+ text
671
+ }))
345
672
  ];
346
673
  }
347
- /** The failed check the attempt's error is, when it is one. */
348
- function errorCheck(e) {
349
- const message = e.errors[0] ?? "";
350
- return (e.meta?.failedChecks ?? []).find((x) => x.message && message.includes(x.message.replace(/…$/, "")));
674
+ /** A compare's failures are told apart by arm: `the baseline`, or the variant's name. */
675
+ const armOf = (c) => c.baseline ? "the baseline" : c.variant ?? c.environment;
676
+ const nameOf = (e) => [...e.group, e.template ?? e.name].join(" › ");
677
+ function iconOf(e) {
678
+ if (e.todo || e.notRun) return "skipped";
679
+ if (e.outcome === "skipped") return "skipped";
680
+ if (e.outcome === "canceled") return "canceled";
681
+ if (e.outcome === "pass") return isNotGatedResult(e) ? "observe" : "pass";
682
+ return "fail";
351
683
  }
352
- /** The failed checks the error isn't, each with its line: an `expect` failure doesn't end the body. */
353
- function otherChecks(c, e) {
354
- const shown = errorCheck(e);
355
- return (e.meta?.failedChecks ?? []).filter((x) => x !== shown).map((x) => {
356
- const why = x.message.startsWith(`${x.label}: `) ? x.message.slice(x.label.length + 2) : x.message;
357
- return [
358
- c.red(`✗ ${x.label}`),
359
- x.at && c.dim(x.at),
360
- why && c.dim(why)
361
- ].filter(Boolean).join(" ");
362
- });
684
+ function evalRow(record, e, remarks, verbose = false) {
685
+ const cells = cellsOf(e);
686
+ const decide = deciding(cells);
687
+ const passed = decide.reduce((n, c) => n + c.passed, 0);
688
+ const ran = decide.reduce((n, c) => n + c.passed + c.failed + c.infraError, 0);
689
+ const compare = cells.some((c) => c.baseline);
690
+ const counted = compare && decide.length < cells.length ? "variant run" : "run";
691
+ const summary = e.todo ? "todo" : e.notRun ? "not run" : ran === 0 ? e.outcome === "canceled" ? "canceled" : "skipped" : `${passed} of ${plural(ran, counted)} passed`;
692
+ const tone = iconOf(e) === "fail" ? "red" : "dim";
693
+ const notes = ran ? [...retriedLines(decide), ...pageErrorNotes(decide)] : [];
694
+ const detail = ran ? [
695
+ ...compare ? compareLines(record, e, cells, verbose) : [{
696
+ depth: 0,
697
+ text: outcomeSentence(tallyBy(decide, labeler(record, decide)))
698
+ }],
699
+ ...failedCaseLines(e),
700
+ ...compare ? checkLines(cells, armOf, tone) : checkLines(decide, (c) => harnessLabel(record, c), tone),
701
+ ...errorLines(compare ? cells : decide, tone),
702
+ ...iconOf(e) === "fail" ? snapshotLines(cells) : [],
703
+ ...questionLines(e.questions ?? []),
704
+ ...observedLines(e.observations ?? [], cells, (c) => compare ? armOf(c) : harnessLabel(record, c)),
705
+ ...remarkLines(remarks, notes)
706
+ ] : remarkLines(remarks, []);
707
+ return {
708
+ id: e.id,
709
+ name: nameOf(e),
710
+ icon: iconOf(e),
711
+ summary,
712
+ detail,
713
+ remarks: remarks.length + notes.length,
714
+ compare
715
+ };
363
716
  }
364
- function failureBody(c, e, outcome, message, options) {
365
- if (outcome === "infra_error") {
366
- if (e.meta?.prepare) {
367
- const lines = message.trimEnd().split("\n");
717
+ /** Free-text answers shown per question; the rest are in the record. */
718
+ const ANSWERS_SHOWN = 3;
719
+ /** Rows of `[label, text]` at `depth`, the labels padded to one column. */
720
+ const aligned = (depth, rows) => {
721
+ const w = Math.max(0, ...rows.map(([label]) => label.length));
722
+ return rows.map(([label, text]) => ({
723
+ depth,
724
+ text: `${label.padEnd(w)} ${text}`
725
+ }));
726
+ };
727
+ /** Each question's answer: who won the pairs, the tally of choices, or the first answers. */
728
+ function questionLines(questions) {
729
+ return questions.flatMap((q) => {
730
+ const head = {
731
+ depth: 0,
732
+ text: `Question: ${clip(q.question, 100)}`,
733
+ tone: "heading"
734
+ };
735
+ if (q.kind === "attempts") {
736
+ const failed = q.error === null ? [] : [{
737
+ depth: 1,
738
+ text: `${q.failed} of ${q.asked} not answered: ${clip(q.error, 200)}`,
739
+ tone: "red"
740
+ }];
741
+ if (q.choices) return [
742
+ head,
743
+ {
744
+ depth: 1,
745
+ text: q.choices.map((t) => `${t.choice} ${t.n}`).join(" · ")
746
+ },
747
+ ...failed
748
+ ];
749
+ const shown = q.answers.slice(0, ANSWERS_SHOWN);
750
+ const more = q.answers.length - shown.length;
368
751
  return [
369
- c.yellow("infra error"),
370
- lines[0] ?? "",
371
- lines.at(-1) ?? ""
372
- ].filter((l, i, all) => all.indexOf(l) === i);
752
+ head,
753
+ ...aligned(1, shown.map((a) => [a.attempt, clip(a.text.replace(/\s+/g, " "), 100)])),
754
+ ...more ? [{
755
+ depth: 1,
756
+ text: `${more} more: reeval query attempt_answers`,
757
+ tone: "dim"
758
+ }] : [],
759
+ ...failed
760
+ ];
373
761
  }
374
- return [`${c.yellow("infra error")}`, ...tailLines(message, options.max)];
375
- }
376
- const failed = (e.meta?.judgements ?? []).filter((j) => j.verdict !== "pass");
377
- if (e.meta?.errorKind === "judge" && failed.length) return failed.flatMap((j, i) => {
378
- const reason = options.reasons?.[i];
379
- return [judgeLine(c, {
380
- ...j,
381
- threshold: j.threshold
382
- }), ...reason ? [` ${reason}`] : []];
762
+ if (q.votes === 0 && q.error === null) return [head, {
763
+ depth: 1,
764
+ text: "No pairs to judge: tries that failed their checks are left out.",
765
+ tone: "dim"
766
+ }];
767
+ if (q.rates.length === 0 && q.error === null) return [head, {
768
+ depth: 1,
769
+ text: `${plural(q.votes, "vote")}, every one a tie.`,
770
+ tone: "dim"
771
+ }];
772
+ const cuts = new Set(q.rates.map((r) => r.cut)).size > 1;
773
+ return [
774
+ head,
775
+ ...aligned(1, q.rates.map((r) => [cuts ? `${r.name} (${r.cut})` : r.name, `won ${r.wins} of ${r.n}`])),
776
+ ...q.error === null ? [] : [{
777
+ depth: 1,
778
+ text: `The model call failed: ${clip(q.error, 200)}`,
779
+ tone: "red"
780
+ }]
781
+ ];
383
782
  });
384
- const formatted = e.formattedErrors[0];
385
- const body = formatted && e.meta?.errorKind !== "timeout" ? formatted : message;
386
- return capLines((c.enabled ? body : stripAnsi(body)).replace(/^\n+/, ""), options.max).map((l) => c.enabled && body === message ? c.red(l) : l);
387
783
  }
388
- /** Preflight's lines: which sandboxes it prepares, the slow ones when ready, and each failure. */
389
- function prepareLines(c, e, max) {
390
- if (e.state === "start") {
391
- const n = e.targets.length;
392
- const names = e.targets.map((t) => t.label).join(", ");
393
- return [c.dim(`preparing ${n} sandbox${n === 1 ? "" : "es"}: ${names}`)];
784
+ /** Where a failed eval's first failed attempt's final workspace snapshot is, to open it. */
785
+ function snapshotLines(cells) {
786
+ const [first, ...others] = cells.flatMap((c) => c.snapshots ?? []);
787
+ if (!first) return [];
788
+ return [{
789
+ depth: 0,
790
+ text: `Snapshot: ${first}${others.length ? ` (+${others.length} more in summary.md)` : ""}`,
791
+ tone: "dim"
792
+ }];
793
+ }
794
+ /** What `c.observe` recorded, each name's tally on one line, or a line per cell under it. */
795
+ function observedLines(tallies, cells, label) {
796
+ if (!tallies.length) return [];
797
+ const byKey = new Map(cells.map((c) => [c.key, c]));
798
+ const perCell = new Set(tallies.map((t) => t.cell)).size > 1;
799
+ const width = Math.max(...tallies.map((t) => clip(t.name, 40).length));
800
+ const lines = [{
801
+ depth: 0,
802
+ text: "Observed:",
803
+ tone: "heading"
804
+ }];
805
+ if (!perCell) {
806
+ for (const t of tallies) lines.push({
807
+ depth: 1,
808
+ text: `${clip(t.name, 40).padEnd(width)} ${observedLine(t)}`
809
+ });
810
+ return lines;
394
811
  }
395
- const { outcome } = e;
396
- if (outcome.ok) return outcome.ms >= 1e3 ? [c.dim(`${c.green("✓")} ${outcome.target.label} ready ${formatDuration(outcome.ms)}`)] : [];
397
- const who = outcome.target.setBy.join(", ");
398
- return [`${c.red("✗")} ${outcome.target.label} failed to prepare ${c.dim(`(set by ${who}; only its attempts fail)`)}`, ...tailLines(outcome.failure.message, max).map((l) => ` ${c.yellow(l)}`)];
812
+ for (const name of new Set(tallies.map((t) => t.name))) {
813
+ lines.push({
814
+ depth: 1,
815
+ text: clip(name, 40)
816
+ });
817
+ const rows = tallies.filter((t) => t.name === name).map((t) => {
818
+ const cell = byKey.get(t.cell);
819
+ return [cell ? label(cell) : t.cell, observedLine(t)];
820
+ });
821
+ const w = Math.max(...rows.map(([who]) => who.length));
822
+ for (const [who, line] of rows) lines.push({
823
+ depth: 2,
824
+ text: `${who.padEnd(w)} ${line}`
825
+ });
826
+ }
827
+ return lines;
399
828
  }
400
- // src/ui/result.ts
401
- function resultLines(c, e, verbose) {
402
- const { record } = e;
829
+ /** `3 passed · 1 failed · 1 not gated`, from the suite's tally. */
830
+ function tallyText(suite) {
831
+ const t = suite.tally;
832
+ return [
833
+ t.errors ? plural(t.errors, "run error") : "",
834
+ t.fail ? `${t.fail} failed` : "",
835
+ t.infraError ? plural(t.infraError, "infra error") : "",
836
+ t.canceled ? `${t.canceled} canceled` : "",
837
+ t.pass ? `${t.pass} passed` : "",
838
+ t.notGated ? `${t.notGated} not gated` : "",
839
+ t.skipped ? `${t.skipped} skipped` : ""
840
+ ].filter(Boolean).join(" · ");
841
+ }
842
+ function statusList(input) {
843
+ const { record, suite } = input;
844
+ const { shown, hidden } = shownRemarks(input.remarks?.remarks ?? [], suite.runId, input.verbose);
845
+ const files = suite.files.map((f) => ({
846
+ path: f.path,
847
+ rows: f.evals.map((e) => evalRow(record, e, shown.filter((r) => r.evalId === e.path), input.verbose))
848
+ }));
403
849
  const tokens = formatUsage(record.cost.usage);
850
+ const done = [
851
+ `Done in ${formatDuration(record.durationMs)}`,
852
+ tallyText(suite) || "nothing ran",
853
+ tokens
854
+ ].filter(Boolean).join(" · ");
404
855
  const notes = [
405
- ...record.status === "budget" ? [c.yellow("Budget reached: attempts that hadn’t started were skipped")] : [],
406
- ...record.status === "interrupted" ? [c.yellow("Interrupted: attempts that hadn’t finished were canceled")] : [],
407
- c.dim([
408
- formatDuration(record.durationMs),
409
- tokens,
410
- `results ${e.summaryPath}`
411
- ].filter(Boolean).join(" · "))
856
+ ...input.notRun ? [`Incomplete: ${plural(input.notRun, "attempt")} this resume’s filters left out ${input.notRun === 1 ? "is" : "are"} not run; reeval run --resume ${record.runId} runs ${input.notRun === 1 ? "it" : "them"}`] : [],
857
+ ...record.status === "budget" && !input.notRun ? ["Budget reached: attempts that hadn’t started were skipped"] : [],
858
+ ...record.status === "interrupted" && !input.notRun ? ["Interrupted: attempts that hadn’t finished were canceled"] : []
412
859
  ];
413
- return render(suiteResult(record), e.remarks ?? null, {
414
- color: c.enabled,
415
- verbose,
416
- notes
417
- });
860
+ return {
861
+ files,
862
+ errors: suite.errors,
863
+ done,
864
+ notes,
865
+ hiddenRemarks: hidden
866
+ };
418
867
  }
868
+ /** The pointers under the closing line: the studio and the run's summary.md. */
869
+ const pointerLine = (summaryPath) => `Browse it: reeval studio · summary ${summaryPath}`;
419
870
  // src/ui/tags.ts
420
871
  const STOPWORDS = new Set("a an the and or of to in on for with without from by at as is are be it its that this when than then into via not no".split(" "));
421
872
  const MAX_WORDS = 3;
@@ -475,6 +926,8 @@ var Tagger = class {
475
926
  };
476
927
  // src/ui/agent.ts
477
928
  const MAX_LINES = 12;
929
+ /** Agents read whole lines, so the header's notes don't wrap. */
930
+ const WIDTH = Number.POSITIVE_INFINITY;
478
931
  var AgentRenderer = class {
479
932
  #o;
480
933
  #c = colors(false);
@@ -490,14 +943,13 @@ var AgentRenderer = class {
490
943
  onEvent(e) {
491
944
  switch (e.type) {
492
945
  case "plan":
493
- this.#write(e.line);
494
- for (const line of e.details) this.#write(line);
495
- for (const w of e.warnings) this.#write(`warning: ${w}`);
496
- for (const h of e.hints) this.#write(`hint: ${h}`);
497
- for (const w of e.sandboxWarnings ?? []) this.#write(sandboxWarningLine(w));
946
+ for (const line of planLines(this.#c, e, WIDTH)) this.#write(line);
498
947
  return;
499
948
  case "run-start":
500
- this.#write(`reeval run: ${e.runId} · ${e.files} eval file${e.files === 1 ? "" : "s"}`);
949
+ this.#write(startLine(this.#c, {
950
+ ...this.#o.output,
951
+ outputLogs: "none"
952
+ }));
501
953
  return;
502
954
  case "collected":
503
955
  this.#tags.addFile(e.evals);
@@ -519,148 +971,29 @@ var AgentRenderer = class {
519
971
  return;
520
972
  }
521
973
  case "prepare":
522
- for (const line of prepareLines(this.#c, e, MAX_LINES)) this.#write(line);
974
+ for (const line of prepareStepLines(this.#c, e, WIDTH, MAX_LINES)) this.#write(line);
523
975
  return;
524
976
  case "notice":
525
- if (e.level !== "info") this.#write(e.text);
977
+ for (const line of noticeLines(this.#c, e, WIDTH)) this.#write(line);
526
978
  return;
527
- case "run-end":
979
+ case "run-end": {
528
980
  this.#write("");
529
- for (const line of resultLines(this.#c, e, (this.#o.verbosity ?? 0) > 0)) this.#write(line);
981
+ const verbose = (this.#o.verbosity ?? 0) > 0;
982
+ const list = statusList({
983
+ record: e.record,
984
+ suite: suiteResult(e.record),
985
+ remarks: e.remarks ?? null,
986
+ verbose,
987
+ ...e.notRun ? { notRun: e.notRun } : {}
988
+ });
989
+ for (const line of listLines(list, this.#c, false, pointerLine(e.summaryPath), verbose)) this.#write(line);
530
990
  return;
991
+ }
531
992
  default: return;
532
993
  }
533
994
  }
534
995
  close() {}
535
996
  };
536
- // src/ui/header.ts
537
- /** ` ➜ `, ` ⚠ `: the glyph column every line of the block starts with. */
538
- const GLYPH = 5;
539
- /** `Est. Time: `, the longest key. */
540
- const KEY = 11;
541
- /** `Warning: `, the longer note label. */
542
- const LABEL = 9;
543
- /**
544
- * `text` broken at spaces into lines of at most `width` columns, not counting color codes; a
545
- * longer word keeps a line. A `·` stays with the word before it, so no line starts with one.
546
- */
547
- function wrap(text, width) {
548
- const words = [];
549
- for (const word of text.split(/\s+/).filter(Boolean)) if (word === "·" && words.length) words[words.length - 1] += " ·";
550
- else words.push(word);
551
- const lines = [];
552
- let line = "";
553
- for (const word of words) if (line && stripAnsi(line).length + 1 + stripAnsi(word).length > width) {
554
- lines.push(line);
555
- line = word;
556
- } else line = line ? `${line} ${word}` : word;
557
- return line ? [...lines, line] : lines;
558
- }
559
- const indent = (n) => " ".repeat(n);
560
- /** `REEVAL v0.16.1`, then the run's facts, one `➜ Key: value` line each. */
561
- function headerLines(c, h, width) {
562
- const name = `REEVAL v${h.version}`;
563
- const title = c.bold(c.depth === 4 ? name : c.teal(name));
564
- const valueWidth = width - GLYPH - KEY;
565
- const row = (key, value) => {
566
- const values = typeof value === "string" ? [value] : value;
567
- const joined = values.join(", ");
568
- return (values.length > 1 && stripAnsi(joined).length > valueWidth ? values : wrap(joined, valueWidth)).map((v, i) => i === 0 ? ` ${c.teal("➜")} ${c.dim(`${key}:`)}${indent(KEY - key.length - 1)}${v}` : `${indent(16)}${v}`);
569
- };
570
- const many = (one, n, many = `${one}s`) => n === 1 ? one : many;
571
- const resumed = h.resumed ? c.dim(` (resumed, ${plural(h.resumed, "attempt")} done)`) : "";
572
- const reason = h.profile && h.profile.reason !== "--profile" ? c.dim(` (${h.profile.reason})`) : "";
573
- return [
574
- h.dryRun ? `${title} ${c.dim("dry run")}` : title,
575
- "",
576
- ...row("Run", `${h.runId}${resumed}`),
577
- ...h.config ? row("Config", h.config) : [],
578
- ...h.profile ? row("Profile", `${h.profile.name}${reason}`) : [],
579
- ...row(many("Harness", h.harnesses.length, "Harnesses"), h.harnesses.map((id) => harness(c, h, id))),
580
- ...row(many("Sandbox", h.sandboxes.length, "Sandboxes"), h.sandboxes),
581
- ...h.judges.length ? row(many("Judge", h.judges.length), h.judges) : [],
582
- ...row("Scope", scope(c, h)),
583
- ...row("Est. Time", estimate(c, h)),
584
- ...h.record && !h.dryRun ? row("Record", h.record) : []
585
- ];
586
- }
587
- /** A harness id, with the effort it runs at when the id doesn't name it. */
588
- function harness(c, h, id) {
589
- const effort = h.efforts?.[id];
590
- if (effort === void 0 || id.includes("[effort=")) return id;
591
- return `${id} ${`(effort: ${effort})`.split(" ").map((w) => c.dim(w)).join(" ")}`;
592
- }
593
- function scope(c, h) {
594
- const attempts = h.fewest < h.attempts ? `${h.fewest}–${h.attempts} attempts` : plural(h.attempts, "attempt");
595
- const parts = [
596
- plural(h.files, "eval file"),
597
- plural(h.evals, "eval") + (h.skipped ? ` (${h.skipped} skipped)` : ""),
598
- plural(h.cells, "cell"),
599
- attempts
600
- ];
601
- const caps = [h.budget.attempts !== void 0 ? `${h.budget.attempts} ${h.attempts <= h.budget.attempts ? c.green("✓") : c.red("✗")}` : "", h.budget.duration ?? ""].filter(Boolean);
602
- return `${parts.join(" · ")}${caps.length ? ` (budget: ${caps.join(", ")})` : ""}`;
603
- }
604
- function estimate(c, h) {
605
- const serial = `~${formatDuration(h.estimateMs)} ${c.dim("(serial)")}`;
606
- const parallel = Math.min(h.concurrency, h.attempts);
607
- if (parallel <= 1) return serial;
608
- return `${`~${formatDuration(h.estimateMs / parallel)} ${c.dim(`(concurrency: ${h.concurrency})`)}`} · ${serial}`;
609
- }
610
- /** `⚠ Warning: …` or `ℹ Hint: …`, wrapped under its first word. */
611
- function noteLines(c, kind, text, width) {
612
- const [glyph, label, paint] = kind === "hint" ? [
613
- "ℹ",
614
- "Hint:",
615
- c.blue
616
- ] : [
617
- "⚠",
618
- "Warning:",
619
- kind === "root" ? c.red : c.orange
620
- ];
621
- const basic = {
622
- warning: c.yellow,
623
- hint: c.teal,
624
- root: c.red
625
- }[kind];
626
- const lead = c.depth === 4 && kind !== "root" ? `${basic(glyph)} ${c.bold(label)}` : c.bold(paint(`${glyph} ${label}`));
627
- const at = 14;
628
- return wrap(text, Math.max(20, width - at)).map((line, i) => i === 0 ? ` ${lead}${indent(LABEL - label.length)}${line}` : `${indent(at)}${line}`);
629
- }
630
- /** A step line, `◆ Preparing 1 sandbox (docker)...`, its text wrapped under itself. */
631
- function stepLines(glyph, text, width) {
632
- return wrap(text, Math.max(20, width - GLYPH)).map((line, i) => i === 0 ? ` ${glyph} ${line}` : `${indent(GLYPH)}${line}`);
633
- }
634
- /** A run-level notice (a prepared workspace, the Ctrl-C one) as a step line. */
635
- function noticeLines(c, e, width) {
636
- const glyph = {
637
- info: c.dim("ℹ"),
638
- warn: c.orange("⚠"),
639
- error: c.red("✗")
640
- }[e.level];
641
- const lines = stepLines(glyph, e.text, width);
642
- return e.level === "info" ? lines.map((l) => c.dim(l)) : lines;
643
- }
644
- /** Preflight: which sandboxes it prepares, each one ready, and each failure. */
645
- function prepareStepLines(c, e, width, max) {
646
- if (e.state === "start") {
647
- const n = e.targets.length;
648
- const names = e.targets.map((t) => t.label).join(", ");
649
- return stepLines(c.teal("◆"), `Preparing ${plural(n, "sandbox", "sandboxes")} (${names})...`, width);
650
- }
651
- const { outcome } = e;
652
- if (outcome.ok) {
653
- const took = outcome.ms >= 1e3 ? ` ${c.dim(formatDuration(outcome.ms))}` : "";
654
- return [` ${c.green("✓")} ${outcome.target.label} ready${took}`];
655
- }
656
- const who = outcome.target.setBy.join(", ");
657
- return [` ${c.red("✗")} ${outcome.target.label} failed to prepare ${c.dim(`(set by ${who}; only its attempts fail)`)}`, ...tailLines(outcome.failure.message, max).map((l) => `${indent(GLYPH)}${c.orange(l)}`)];
658
- }
659
- /** The last line before the attempts' own: how their logs will print. */
660
- function startLine(c, output) {
661
- const text = output.outputLogs === "none" ? "Running..." : output.outputLogs === "errors-only" ? "Running; logs print for failed attempts only..." : output.logOrder === "stream" ? "Streaming logs..." : "Running; each attempt's logs print as it ends...";
662
- return ` ${c.teal("◆")} ${text}`;
663
- }
664
997
  // src/ui/plain.ts
665
998
  /** In a grouped block, an end line replaces its start line and keeps its start time. */
666
999
  const STARTS = {
@@ -702,6 +1035,8 @@ var PlainRenderer = class {
702
1035
  #queued = /* @__PURE__ */ new Set();
703
1036
  #runStartedAt = 0;
704
1037
  #lastWrite = 0;
1038
+ /** The last line written was blank, so the list needs no gap above it. */
1039
+ #blank = false;
705
1040
  #timer;
706
1041
  /** On a TTY: the status line is drawn under everything else, and redrawn after each write. */
707
1042
  #live;
@@ -710,8 +1045,11 @@ var PlainRenderer = class {
710
1045
  #evals = /* @__PURE__ */ new Set();
711
1046
  #tries = /* @__PURE__ */ new Set();
712
1047
  #ended = /* @__PURE__ */ new Set();
1048
+ #list;
1049
+ #lastProgress = 0;
713
1050
  constructor(options) {
714
1051
  this.#o = options;
1052
+ this.#list = new LiveList(options.harness);
715
1053
  this.#c = options.colors;
716
1054
  this.#now = options.now ?? Date.now;
717
1055
  this.#live = options.output.statusLine;
@@ -725,6 +1063,7 @@ var PlainRenderer = class {
725
1063
  const line = `${text.trimEnd()}\n`;
726
1064
  this.#o.out.write(this.#status ? `${CLEAR}${line}${this.#status}` : line);
727
1065
  this.#lastWrite = this.#now();
1066
+ this.#blank = line === "\n";
728
1067
  }
729
1068
  /** `7 evals · 12/28 tries`: what's collected so far and how many tries ended. */
730
1069
  #drawStatus() {
@@ -775,31 +1114,17 @@ var PlainRenderer = class {
775
1114
  onEvent(e) {
776
1115
  const c = this.#c;
777
1116
  switch (e.type) {
778
- case "run-start": {
1117
+ case "run-start":
779
1118
  this.#runStartedAt = e.at;
780
- if (this.#o.output.rich) {
781
- this.#write(startLine(c, this.#o.output));
782
- this.#write("");
783
- return;
784
- }
785
- const files = `${e.files} eval file${e.files === 1 ? "" : "s"}`;
786
- const order = `${this.#o.output.logOrder} logs`;
787
- const setup = `harness ${e.harness} · sandbox ${e.sandbox} · concurrency ${e.concurrency}`;
788
- this.#write(c.dim(`reeval ${e.version} · run ${e.runId} · ${files} · ${setup} · ${order}`));
1119
+ this.#write(startLine(c, this.#o.output));
1120
+ this.#write("");
789
1121
  return;
790
- }
791
1122
  case "plan":
792
- if (this.#o.output.rich) {
793
- this.#richPlan(e);
794
- return;
795
- }
796
- this.#write(e.line);
797
- for (const line of e.details) this.#write(c.dim(line));
798
- for (const w of e.warnings) this.#write(c.yellow(`warning: ${w}`));
799
- for (const h of e.hints) this.#write(c.dim(`hint: ${h}`));
800
- for (const w of e.sandboxWarnings ?? []) this.#sandboxWarning(w);
1123
+ for (const line of planLines(c, e, this.#columns)) this.#write(line);
1124
+ if (this.#o.output.githubActions) for (const w of e.sandboxWarnings ?? []) this.#annotate(w);
801
1125
  return;
802
1126
  case "collected":
1127
+ this.#list.collected(e.file, e.evals);
803
1128
  this.#tags.addFile(e.evals);
804
1129
  for (const x of e.evals) {
805
1130
  if (x.skipped) continue;
@@ -811,6 +1136,7 @@ var PlainRenderer = class {
811
1136
  return;
812
1137
  case "eval-start": {
813
1138
  const a = this.#attempt(e, e.at);
1139
+ this.#list.start(e);
814
1140
  a.last = e;
815
1141
  this.#queued.delete(e.key);
816
1142
  this.#running.set(e.key, a);
@@ -820,6 +1146,7 @@ var PlainRenderer = class {
820
1146
  case "eval-progress": {
821
1147
  const a = this.#running.get(e.key);
822
1148
  if (!a) return;
1149
+ this.#list.progress(e.key, e.progress);
823
1150
  if (e.progress.kind !== "warning") a.last = e;
824
1151
  const { progress } = e;
825
1152
  const paint = a.ungated ? {
@@ -861,13 +1188,13 @@ var PlainRenderer = class {
861
1188
  return;
862
1189
  }
863
1190
  case "eval-end":
1191
+ this.#list.end(e.result);
864
1192
  this.#end(e.result, e.at);
865
1193
  if (this.#tries.has(e.result.key)) this.#ended.add(e.result.key);
866
1194
  this.#drawStatus();
867
1195
  return;
868
- case "prepare": {
869
- const lines = this.#o.output.rich ? prepareStepLines(c, e, this.#columns, ERROR_LINES) : prepareLines(c, e, ERROR_LINES);
870
- for (const line of lines) this.#write(line);
1196
+ case "prepare":
1197
+ for (const line of prepareStepLines(c, e, this.#columns, ERROR_LINES)) this.#write(line);
871
1198
  if (e.state === "failed" && this.#o.output.githubActions) {
872
1199
  const { target, failure } = e.outcome;
873
1200
  const what = target.kind === "workspace" ? "The" : "Sandbox";
@@ -875,52 +1202,30 @@ var PlainRenderer = class {
875
1202
  this.#write(`::error title=${title}::${ghEscape(failure.message)}`);
876
1203
  }
877
1204
  return;
878
- }
879
- case "notice": {
880
- if (this.#o.output.rich) {
881
- for (const line of noticeLines(c, e, this.#columns)) this.#write(line);
882
- return;
883
- }
884
- const paint = {
885
- info: c.dim,
886
- warn: c.yellow,
887
- error: c.red
888
- }[e.level];
889
- this.#write(paint(e.text));
1205
+ case "notice":
1206
+ for (const line of noticeLines(c, e, this.#columns)) this.#write(line);
890
1207
  return;
891
- }
892
- case "run-end":
1208
+ case "run-end": {
893
1209
  this.#clearStatus();
894
- this.#write("");
895
- for (const line of resultLines(c, e, (this.#o.verbosity ?? 0) > 0)) this.#write(line);
1210
+ if (!this.#blank) this.#write("");
1211
+ const verbose = (this.#o.verbosity ?? 0) > 0;
1212
+ const list = statusList({
1213
+ record: e.record,
1214
+ suite: suiteResult(e.record),
1215
+ remarks: e.remarks ?? null,
1216
+ verbose,
1217
+ ...e.notRun ? { notRun: e.notRun } : {}
1218
+ });
1219
+ for (const line of listLines(list, c, true, pointerLine(e.summaryPath), verbose)) this.#write(line);
896
1220
  return;
1221
+ }
897
1222
  }
898
1223
  }
899
1224
  get #columns() {
900
- return this.#o.columns || 80;
901
- }
902
- /** The plan as the rich header: its facts, `--dry-run`'s table, then each kind of note. */
903
- #richPlan(e) {
904
- const c = this.#c;
905
- const width = this.#columns;
906
- for (const line of headerLines(c, e.header, width)) this.#write(line);
907
- const sections = [
908
- e.details,
909
- [...e.warnings.flatMap((w) => noteLines(c, "warning", w, width)), ...(e.sandboxWarnings ?? []).flatMap((w) => noteLines(c, w.kind === "root" ? "root" : "warning", w.message, width))],
910
- e.hints.flatMap((h) => noteLines(c, "hint", h, width))
911
- ];
912
- for (const lines of sections) {
913
- if (!lines.length) continue;
914
- this.#write("");
915
- for (const line of lines) this.#write(line);
916
- }
917
- this.#write("");
1225
+ return this.#o.columns || Number.POSITIVE_INFINITY;
918
1226
  }
919
- #sandboxWarning(w) {
920
- const c = this.#c;
921
- const line = sandboxWarningLine(w);
922
- this.#write(w.kind === "root" ? c.bold(c.red(line)) : c.yellow(line));
923
- if (!this.#o.output.githubActions) return;
1227
+ /** The header block names a sandbox warning; GitHub Actions also annotates it. */
1228
+ #annotate(w) {
924
1229
  const file = w.file ? `file=${ghEscape(w.file, true)},` : "";
925
1230
  const title = w.kind === "root" ? "Sandbox runs as root" : "Sandbox has extra capabilities";
926
1231
  this.#write(`::warning ${file}title=${ghEscape(title, true)}::${ghEscape(w.message)}`);
@@ -968,13 +1273,23 @@ var PlainRenderer = class {
968
1273
  }
969
1274
  }
970
1275
  }
971
- /** Prints the heartbeat if nothing was written for `heartbeatMs` and evals are running. */
1276
+ /**
1277
+ * Every `heartbeatMs` while attempts run. Without attempt logs, how many evals are done; with
1278
+ * them, each running attempt's phase, when nothing printed for that long.
1279
+ */
972
1280
  tick(now = this.#now()) {
973
- const { heartbeatMs } = this.#o.output;
1281
+ const { heartbeatMs, outputLogs } = this.#o.output;
974
1282
  if (this.#live || heartbeatMs <= 0) return;
975
- if (now - this.#lastWrite < heartbeatMs) return;
976
- const c = this.#c;
977
1283
  if (this.#running.size === 0) return;
1284
+ const c = this.#c;
1285
+ if (outputLogs === "none") {
1286
+ if (now - Math.max(this.#lastProgress, this.#runStartedAt) < heartbeatMs) return;
1287
+ const { done, total } = this.#list.counts();
1288
+ this.#write(c.dim(progressText(done, total, now - this.#runStartedAt)));
1289
+ this.#lastProgress = now;
1290
+ return;
1291
+ }
1292
+ if (now - this.#lastWrite < heartbeatMs) return;
978
1293
  const running = [...this.#running.values()].map((a) => {
979
1294
  const progress = a.last?.type === "eval-progress" ? a.last.progress : void 0;
980
1295
  return `${this.#prefix(a)} ${phaseOf(progress)} ${c.dim(formatDuration(now - a.startedAt))}`;
@@ -988,6 +1303,188 @@ var PlainRenderer = class {
988
1303
  this.#clearStatus();
989
1304
  }
990
1305
  };
1306
+ // src/ui/tty.ts
1307
+ /** The list's state, which the Ink app reads with `useSyncExternalStore`. */
1308
+ var ListStore = class {
1309
+ #state = {
1310
+ files: [],
1311
+ logs: [],
1312
+ counts: {
1313
+ done: 0,
1314
+ total: 0
1315
+ },
1316
+ phase: null,
1317
+ finished: null
1318
+ };
1319
+ #listeners = /* @__PURE__ */ new Set();
1320
+ get = () => this.#state;
1321
+ subscribe = (listener) => {
1322
+ this.#listeners.add(listener);
1323
+ return () => this.#listeners.delete(listener);
1324
+ };
1325
+ set(next) {
1326
+ this.#state = {
1327
+ ...this.#state,
1328
+ ...next
1329
+ };
1330
+ for (const l of this.#listeners) l();
1331
+ }
1332
+ };
1333
+ var InteractiveRenderer = class {
1334
+ #o;
1335
+ #c;
1336
+ #store = new ListStore();
1337
+ #list;
1338
+ /** The header and run-level lines, as every terminal prints them. */
1339
+ #plain;
1340
+ #app;
1341
+ /** Set once the list couldn't load: from then on, the run prints as a pipe's does. */
1342
+ #fallback;
1343
+ #ended;
1344
+ #closed = false;
1345
+ constructor(options) {
1346
+ this.#o = options;
1347
+ this.#c = options.colors;
1348
+ this.#list = new LiveList(options.harness);
1349
+ const out = { write: (text) => {
1350
+ if (!this.#app || this.#fallback) return options.out.write(text);
1351
+ const lines = String(text).replace(/\n$/, "").split("\n");
1352
+ this.#store.set({ logs: [...this.#store.get().logs, ...lines] });
1353
+ return true;
1354
+ } };
1355
+ this.#plain = new PlainRenderer({
1356
+ ...options,
1357
+ out,
1358
+ timers: false,
1359
+ output: {
1360
+ ...options.output,
1361
+ statusLine: false,
1362
+ interactive: false,
1363
+ outputLogs: "none"
1364
+ }
1365
+ });
1366
+ }
1367
+ #mount() {
1368
+ if (this.#app || this.#closed) return;
1369
+ const cancel = this.#o.cancel ?? (() => process.kill(process.pid, "SIGINT"));
1370
+ const load = this.#o.load ?? (() => import("./ink-list-Ca-5dunO.js"));
1371
+ this.#app = load().then((m) => m.mountList({
1372
+ store: this.#store,
1373
+ colors: this.#c,
1374
+ cancel,
1375
+ stdout: this.#o.stdout,
1376
+ stdin: this.#o.stdin
1377
+ })).catch((error) => {
1378
+ this.#fallBack(error);
1379
+ });
1380
+ }
1381
+ #fallBack(error) {
1382
+ const out = this.#o.out;
1383
+ const { logs, phase } = this.#store.get();
1384
+ for (const line of logs) out.write(`${line}\n`);
1385
+ const reason = error instanceof Error ? error.message : String(error);
1386
+ out.write(`${this.#c.yellow(`Couldn't draw the interactive list (${reason}).`)}\n`);
1387
+ if (phase) out.write(`${phase}\n`);
1388
+ this.#fallback = new PlainRenderer({
1389
+ ...this.#o,
1390
+ timers: false,
1391
+ output: {
1392
+ ...this.#o.output,
1393
+ statusLine: false,
1394
+ interactive: false
1395
+ }
1396
+ });
1397
+ if (this.#ended) this.#fallback.onEvent(this.#ended.event);
1398
+ }
1399
+ #update() {
1400
+ const { done, total } = this.#list.counts();
1401
+ this.#store.set({
1402
+ files: this.#list.files(),
1403
+ counts: {
1404
+ done,
1405
+ total
1406
+ }
1407
+ });
1408
+ }
1409
+ onEvent(e) {
1410
+ if (this.#fallback) {
1411
+ if (e.type !== "run-start") this.#fallback.onEvent(e);
1412
+ return;
1413
+ }
1414
+ switch (e.type) {
1415
+ case "run-start":
1416
+ this.#store.set({ phase: startLine(this.#c, this.#o.output) });
1417
+ return;
1418
+ case "collected":
1419
+ this.#list.collected(e.file, e.evals);
1420
+ this.#mount();
1421
+ this.#update();
1422
+ return;
1423
+ case "eval-start":
1424
+ this.#list.start(e);
1425
+ this.#update();
1426
+ return;
1427
+ case "eval-progress":
1428
+ if (e.progress.kind === "warning" || e.progress.kind === "browser-download") this.#plain.onEvent(e);
1429
+ this.#list.progress(e.key, e.progress);
1430
+ this.#update();
1431
+ return;
1432
+ case "eval-end":
1433
+ this.#list.end(e.result);
1434
+ this.#update();
1435
+ return;
1436
+ case "debug": return;
1437
+ case "run-end": {
1438
+ const list = statusList({
1439
+ record: e.record,
1440
+ suite: suiteResult(e.record),
1441
+ remarks: e.remarks ?? null,
1442
+ verbose: false,
1443
+ ...e.notRun ? { notRun: e.notRun } : {}
1444
+ });
1445
+ if (!this.#app) {
1446
+ this.#plain.onEvent(e);
1447
+ return;
1448
+ }
1449
+ this.#ended = {
1450
+ list,
1451
+ pointer: pointerLine(e.summaryPath),
1452
+ event: e
1453
+ };
1454
+ this.#store.set({
1455
+ files: list.files,
1456
+ ...this.#store.get().phase ? { phase: endLine(this.#c, e.record.status) } : {},
1457
+ finished: {
1458
+ done: list.done,
1459
+ notes: list.notes,
1460
+ pointer: pointerLine(e.summaryPath),
1461
+ errors: list.errors
1462
+ }
1463
+ });
1464
+ return;
1465
+ }
1466
+ default: this.#plain.onEvent(e);
1467
+ }
1468
+ }
1469
+ close() {
1470
+ this.#plain.close();
1471
+ this.#fallback?.close();
1472
+ if (!this.#store.get().finished) {
1473
+ this.#closed = true;
1474
+ this.#app?.then((a) => a?.unmount());
1475
+ }
1476
+ }
1477
+ /** Until the user quits the list, then the whole list into the scrollback; at once when there's none. */
1478
+ async wait() {
1479
+ const app = await this.#app;
1480
+ if (!app || !this.#ended) return;
1481
+ await app.waitUntilExit();
1482
+ const { phase } = this.#store.get();
1483
+ const { list, pointer } = this.#ended;
1484
+ const text = [...phase ? [phase, ""] : [], ...listLines(list, this.#c, true, pointer)];
1485
+ this.#o.out.write(`${text.join("\n")}\n`);
1486
+ }
1487
+ };
991
1488
  // src/ui/index.ts
992
1489
  const depthOf = (bits = 4) => bits >= 24 ? 24 : bits >= 8 ? 8 : 4;
993
1490
  function createRenderer(options) {
@@ -996,10 +1493,17 @@ function createRenderer(options) {
996
1493
  ...options,
997
1494
  out
998
1495
  });
1496
+ const paint = colors(options.output.color, depthOf(options.colorDepth));
1497
+ if (options.output.interactive && options.tty) return new InteractiveRenderer({
1498
+ ...options,
1499
+ ...options.tty,
1500
+ out,
1501
+ colors: paint
1502
+ });
999
1503
  return new PlainRenderer({
1000
1504
  ...options,
1001
1505
  out,
1002
- colors: colors(options.output.color, depthOf(options.colorDepth))
1506
+ colors: paint
1003
1507
  });
1004
1508
  }
1005
1509
  // src/cli.ts
@@ -1072,7 +1576,7 @@ async function run(argv) {
1072
1576
  return values.help ? EXIT.ok : EXIT.usage;
1073
1577
  }
1074
1578
  if (command === "init") {
1075
- const { initCommand } = await import("./init-DJ3WFUt8.js");
1579
+ const { initCommand } = await import("./init-8o6W3_fn.js");
1076
1580
  return initCommand(values, filters, {
1077
1581
  out: (s) => console.log(s),
1078
1582
  input: process.stdin,
@@ -1081,7 +1585,7 @@ async function run(argv) {
1081
1585
  });
1082
1586
  }
1083
1587
  if (command === "doctor") {
1084
- const { doctorCommand } = await import("./doctor-CIhEAzbw.js");
1588
+ const { doctorCommand } = await import("./doctor-BcvfKpBX.js");
1085
1589
  return doctorCommand(values, {
1086
1590
  out: (s) => console.log(s),
1087
1591
  color: !!process.stdout.isTTY && !process.env.NO_COLOR,
@@ -1089,7 +1593,7 @@ async function run(argv) {
1089
1593
  });
1090
1594
  }
1091
1595
  if (command === "report") {
1092
- const { reportCommand } = await import("./command-Dk6wggV-.js");
1596
+ const { reportCommand } = await import("./command-CFQ9c1hk.js");
1093
1597
  return reportCommand(values, filters, {
1094
1598
  out: (s) => console.log(s),
1095
1599
  err: (s) => console.error(dim(s)),
@@ -1104,7 +1608,7 @@ async function run(argv) {
1104
1608
  }
1105
1609
  if (command === "studio") {
1106
1610
  const config = await loadConfig(configPath, "studio");
1107
- const { studioCommand } = await import("./command-DPF8hnWS.js");
1611
+ const { studioCommand } = await import("./command-DhiwsZl1.js");
1108
1612
  return studioCommand({
1109
1613
  config,
1110
1614
  flags: values,
@@ -1115,7 +1619,7 @@ async function run(argv) {
1115
1619
  if (command === "mcp") {
1116
1620
  if (!values.http) console.log = console.info = console.debug = console.error;
1117
1621
  const config = await loadConfig(configPath, "mcp");
1118
- const { mcpCommand } = await import("./command-B7MslY_w.js");
1622
+ const { mcpCommand } = await import("./command-qGS802uX.js");
1119
1623
  return mcpCommand({
1120
1624
  config,
1121
1625
  flags: values,
@@ -1124,7 +1628,7 @@ async function run(argv) {
1124
1628
  }
1125
1629
  if (command in HISTORY_COMMANDS) {
1126
1630
  const config = await loadConfig(configPath, command);
1127
- const commands = await import("./commands-DQBayPCV.js");
1631
+ const commands = await import("./commands-xYTMNTPv.js");
1128
1632
  const ctx = {
1129
1633
  config,
1130
1634
  flags: values,
@@ -1137,7 +1641,7 @@ async function run(argv) {
1137
1641
  case "compare": return commands.compareCommand(ctx, filters);
1138
1642
  case "checkout": return commands.checkoutCommand(ctx, filters);
1139
1643
  case "query": {
1140
- const { queryCommand } = await import("./query-DSSMxkmw.js");
1644
+ const { queryCommand } = await import("./query-CvgUsvep.js");
1141
1645
  return queryCommand({
1142
1646
  ...ctx,
1143
1647
  err: (s) => console.error(dim(s))
@@ -1145,7 +1649,7 @@ async function run(argv) {
1145
1649
  }
1146
1650
  case "analyze": {
1147
1651
  loadEnvFiles(configPath);
1148
- const { analyzeCommand } = await import("./command-DDkz6jsD.js");
1652
+ const { analyzeCommand } = await import("./command-fO0BFsTq.js");
1149
1653
  return analyzeCommand({
1150
1654
  ...ctx,
1151
1655
  err: (s) => console.error(dim(s))
@@ -1159,12 +1663,12 @@ async function run(argv) {
1159
1663
  }
1160
1664
  /** The dry run's prices, fetched only when a planned attempt or the judge has anything to price. */
1161
1665
  async function dryRunPrices(planned, judgeModel) {
1162
- const { needsPrices } = await import("./cost-CMzY9VXv.js");
1666
+ const { needsPrices } = await import("./cost-BJU0MAji.js");
1163
1667
  if (!needsPrices(planned.filter((p) => !p.attempt.plan.skip).map((p) => p.attempt.harness), judgeModel)) return {
1164
1668
  table: null,
1165
1669
  why: "nothing to price"
1166
1670
  };
1167
- const { loadPrices } = await import("./prices-C4BRuSe8.js");
1671
+ const { loadPrices } = await import("./prices-DYJA0tJN.js");
1168
1672
  return loadPrices();
1169
1673
  }
1170
1674
  /** Asks a yes/no question on the terminal; anything but y or yes is no. */
@@ -1197,8 +1701,8 @@ async function cleanCommand(flags, configPath) {
1197
1701
  const where = fromCwd(debugRoot(config.root)) || ".";
1198
1702
  console.log(runs.length ? `Removed debug data and snapshots for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where} and .reeval/runs` : `No debug data or snapshots to remove in ${where} or .reeval/runs`);
1199
1703
  if (flags.images) {
1200
- const { dockerAvailable, pruneImages } = await import("./src-CZANB4pb.js");
1201
- const { requireDocker } = await import("./providers-CCl8ElNW.js");
1704
+ const { dockerAvailable, pruneImages } = await import("./src-BkH4GOxu.js");
1705
+ const { requireDocker } = await import("./providers-D0ob_tra.js");
1202
1706
  const engines = [];
1203
1707
  for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
1204
1708
  if (!engines.length) await requireDocker();
@@ -1219,7 +1723,7 @@ function verbosityOf(flags) {
1219
1723
  * does. Needs a config, as `reeval run` does: there is no default image.
1220
1724
  */
1221
1725
  async function imageBuildCommand(flags, configPath, filters) {
1222
- const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-h1VBM7TP.js");
1726
+ const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-CNayXCp2.js");
1223
1727
  const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
1224
1728
  ...flags,
1225
1729
  "dry-run": true
@@ -1271,15 +1775,32 @@ async function imageBuildCommand(flags, configPath, filters) {
1271
1775
  return failed.every((b) => !b.ok && b.failure.usage) ? EXIT.usage : EXIT.infra;
1272
1776
  }
1273
1777
  /** `reeval run`: check, plan, run the engine, then write the run and pick the exit code. */
1274
- async function runCommand(flags, configPath, filters) {
1275
- const setup = await prepareRun(flags, configPath);
1778
+ async function runCommand(flags, configPath, given) {
1779
+ const resumed = flags.resume === void 0 ? null : await resumedRun(flags, given, configPath);
1780
+ const runFlags = resumed?.settings ? {
1781
+ ...flags,
1782
+ ...resumed.settings
1783
+ } : flags;
1784
+ const filters = resumed?.settings?.filters ?? given;
1785
+ const setup = await prepareRun(runFlags, configPath, {
1786
+ noProfile: !!resumed?.saved && resumed.saved.profile === void 0,
1787
+ ...resumed?.profileOf ? { profileOf: resumed.profileOf } : {}
1788
+ });
1789
+ const explicit = runFlags.concurrency ?? (process.env.REEVAL_CONCURRENCY || void 0);
1790
+ const startedWith = resumed ? resumed.saved ?? void 0 : {
1791
+ ...setup.runs === void 0 ? {} : { runs: setup.runs },
1792
+ ...setup.profile ? { profile: setup.profile.name } : {},
1793
+ ...explicit === void 0 ? {} : { concurrency: setup.concurrency },
1794
+ filters: savedFilters(setup.config.root, filters),
1795
+ tags: runFlags.tag ?? []
1796
+ };
1276
1797
  const ctx = await startRun(setup);
1277
1798
  invocation.runId = ctx.runId;
1278
1799
  const githubOut = process.env.GITHUB_OUTPUT;
1279
1800
  if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
1280
1801
  const stepStart = process.env.GITHUB_STEP_SUMMARY;
1281
1802
  if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
1282
- const { engineVersions } = await import("./node-CwP1G178.js");
1803
+ const { engineVersions } = await import("./node-DhIRSrrK.js");
1283
1804
  const terminal = setup.json && !setup.dryRun ? process.stderr : process.stdout;
1284
1805
  const renderer = createRenderer({
1285
1806
  out: setup.json && setup.dryRun ? { write: () => true } : terminal,
@@ -1287,11 +1808,19 @@ async function runCommand(flags, configPath, filters) {
1287
1808
  colorDepth: terminal.isTTY ? terminal.getColorDepth() : void 0,
1288
1809
  output: setup.output,
1289
1810
  verbosity: ctx.verbosity ?? 0,
1290
- harness: setup.config.harness[0]?.kind ?? "agent"
1811
+ harness: setup.config.harness[0]?.kind ?? "agent",
1812
+ tty: {
1813
+ stdout: process.stdout,
1814
+ stdin: process.stdin
1815
+ }
1291
1816
  });
1292
1817
  let executed;
1293
1818
  try {
1294
- executed = await executeRun(ctx, setup, filters, renderer);
1819
+ executed = await executeRun(ctx, setup, filters, renderer, {
1820
+ startedWith,
1821
+ notes: resumed?.notes ?? [],
1822
+ narrowed: !!resumed && (given.length > 0 || !!flags.tag?.length)
1823
+ });
1295
1824
  } catch (error) {
1296
1825
  renderer.close();
1297
1826
  if (setup.resume === void 0) await rm(ctx.stagingDir, {
@@ -1300,11 +1829,11 @@ async function runCommand(flags, configPath, filters) {
1300
1829
  });
1301
1830
  throw error;
1302
1831
  }
1303
- const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions } = executed;
1832
+ const { engineRun, plan, sandboxVersions, budgetReached, cutShort, live, questions, leftOut } = executed;
1304
1833
  invocation.unhandledErrors = engineRun.unhandledErrors;
1305
1834
  const engines = [...new Set(plan?.sandboxes.flatMap((s) => s.kind === "docker" ? [s.engine ?? "docker"] : []))];
1306
1835
  if ((interrupted || engineRun.canceled) && engines.length) {
1307
- const { removeRunContainers } = await import("./docker-D83s_ADn.js");
1836
+ const { removeRunContainers } = await import("./docker-BlSopHpZ.js");
1308
1837
  await removeRunContainers(ctx.runId, engines);
1309
1838
  }
1310
1839
  if (engineRun.stopped || !plan) {
@@ -1321,8 +1850,8 @@ async function runCommand(flags, configPath, filters) {
1321
1850
  }
1322
1851
  finishing = true;
1323
1852
  live?.phase("finishing");
1324
- const { lockfiles } = await import("./state-haxTOljN.js");
1325
- const { writeScratchRun } = await import("./results-CFE6Nwb0.js");
1853
+ const { lockfiles } = await import("./state-DZbAO95a.js");
1854
+ const { writeScratchRun } = await import("./results-CfC0id_u.js");
1326
1855
  const run = await writeScratchRun({
1327
1856
  ctx,
1328
1857
  author: ctx.author,
@@ -1342,12 +1871,19 @@ async function runCommand(flags, configPath, filters) {
1342
1871
  git: ctx.git,
1343
1872
  previous: ctx.previous,
1344
1873
  previousArtifacts: ctx.previousArtifacts,
1874
+ leftOut,
1345
1875
  runner: runnerInfo(),
1876
+ ...startedWith ? { invocation: startedWith } : {},
1346
1877
  ...answers && (answers.evals.size || answers.attempts.size) ? { answers } : {}
1347
1878
  }, () => live?.stop());
1348
1879
  const { record } = run;
1349
1880
  const outcome = runOutcome(record);
1350
- const code = stopped ? stopCode : outcome === "fail" ? EXIT.failed : outcome === "infra_error" ? EXIT.infra : EXIT.ok;
1881
+ const notRun = leftOut.filter((l) => !l.done).length;
1882
+ const ran = notRun > 0 && !stopped && record.status === "interrupted" ? runOutcome({
1883
+ ...record,
1884
+ status: "complete"
1885
+ }) : outcome;
1886
+ const code = stopped ? stopCode : ran === "fail" ? EXIT.failed : ran === "infra_error" ? EXIT.infra : EXIT.ok;
1351
1887
  const { pruneDebug } = await import("./files-COW5JBJM.js");
1352
1888
  await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
1353
1889
  const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
@@ -1357,15 +1893,17 @@ async function runCommand(flags, configPath, filters) {
1357
1893
  record,
1358
1894
  summaryPath: fromCwd(summary),
1359
1895
  results: engineRun.evals,
1360
- remarks: run.remarks
1896
+ remarks: run.remarks,
1897
+ ...notRun > 0 && !stopped ? { notRun } : {}
1361
1898
  });
1362
1899
  renderer.close();
1363
1900
  const stepSummary = process.env.GITHUB_STEP_SUMMARY;
1364
1901
  if (stepSummary && run.artifactDir) await appendFile(stepSummary, await readFile(summary)).catch(() => {});
1365
1902
  const { config } = ctx;
1366
1903
  const mode = setup.record;
1367
- if ((flags.record || mode === "always" || mode === "ci" && inCI()) && record.status !== "interrupted") {
1368
- const { record: recordCmd } = await import("./commands-DQBayPCV.js");
1904
+ const autoRecord = flags.record || mode === "always" || mode === "ci" && inCI();
1905
+ if (autoRecord && record.status !== "interrupted") {
1906
+ const { record: recordCmd } = await import("./commands-xYTMNTPv.js");
1369
1907
  const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
1370
1908
  const line = await recordCmd(config, record.runId, {
1371
1909
  ...flags,
@@ -1373,15 +1911,58 @@ async function runCommand(flags, configPath, filters) {
1373
1911
  });
1374
1912
  if (setup.json) console.error(line);
1375
1913
  else console.log(line);
1914
+ } else if (autoRecord && notRun > 0 && !stopped) {
1915
+ const line = `Not recorded: run ${record.runId} is incomplete. reeval run --resume ${record.runId}${flags.record ? " --record" : ""} runs the rest and records it.`;
1916
+ if (setup.json) console.error(line);
1917
+ else console.log(line);
1376
1918
  }
1377
1919
  if (setup.json) {
1378
- const { suiteResult } = await import("./suite-CmeXyn66.js");
1379
- process.stdout.write(`${JSON.stringify(suiteResult(record), null, 2)}\n`);
1920
+ const { suiteResult } = await import("./suite-u2KJnGpY.js");
1921
+ process.stdout.write(`${JSON.stringify({
1922
+ ...suiteResult(record),
1923
+ outcome: ran
1924
+ }, null, 2)}\n`);
1380
1925
  }
1926
+ await renderer.wait?.();
1381
1927
  return code;
1382
1928
  }
1383
- /** Everything that can fail before the evals are collected: output mode, config, profile, flags. */
1384
- async function prepareRun(flags, configPath) {
1929
+ /**
1930
+ * `--resume`: the run's saved settings with the flags given now. An older reeval's run saved none,
1931
+ * so it plans with the flags given now, with a note.
1932
+ */
1933
+ async function resumedRun(flags, filters, configPath) {
1934
+ const path = requireConfig(configPath);
1935
+ loadEnvFiles(path);
1936
+ const root = dirname(path);
1937
+ const target = await resolveResume(root, flags.resume || void 0);
1938
+ const saved = await savedInvocation(root, target);
1939
+ if (!saved) return {
1940
+ saved: null,
1941
+ settings: null,
1942
+ notes: [`run ${target.runId} saved no --runs, profile or filters, so this resume plans with the ones given now`],
1943
+ profileOf: void 0
1944
+ };
1945
+ const given = {
1946
+ runs: flags.runs,
1947
+ profile: flags.profile,
1948
+ concurrency: flags.concurrency,
1949
+ tag: flags.tag,
1950
+ filters
1951
+ };
1952
+ const resumed = resumeSettings(target.runId, saved, given, { root });
1953
+ const fromRun = flags.profile === void 0 && resumed.settings.profile !== void 0;
1954
+ return {
1955
+ saved,
1956
+ ...resumed,
1957
+ profileOf: fromRun ? target.runId : void 0
1958
+ };
1959
+ }
1960
+ /**
1961
+ * Everything that can fail before the evals are collected: output mode, config, profile, flags.
1962
+ * `noProfile` is a resume of a run that had none, which `defaultProfile` mustn't give one;
1963
+ * `profileOf` names the resumed run whose profile `flags.profile` is.
1964
+ */
1965
+ async function prepareRun(flags, configPath, { noProfile = false, profileOf } = {}) {
1385
1966
  const output = resolveOutput({
1386
1967
  flags: {
1387
1968
  ui: flags.ui,
@@ -1392,13 +1973,15 @@ async function prepareRun(flags, configPath) {
1392
1973
  },
1393
1974
  env: process.env,
1394
1975
  isTTY: !!(flags.json && !flags["dry-run"] ? process.stderr : process.stdout).isTTY,
1395
- agent: detectAgent().name
1976
+ agent: detectAgent().name,
1977
+ stdinTTY: !!process.stdin.isTTY && !flags.json && !flags["dry-run"]
1396
1978
  });
1397
1979
  const path = requireConfig(configPath);
1398
1980
  loadEnvFiles(path);
1399
- const named = flags.profile ?? (process.env.REEVAL_PROFILE || void 0);
1981
+ const named = noProfile ? void 0 : flags.profile ?? (process.env.REEVAL_PROFILE || void 0);
1400
1982
  const config = await loadConfig(path, "run", named);
1401
- const profile = selectProfile(config, flags.profile);
1983
+ if (profileOf && flags.profile && !config.profiles[flags.profile]) throw new UsageError(`run ${profileOf} was started with profile ${flags.profile}, which ${config.file} no longer defines. Put it back to resume the run, or start a new run without --resume.`);
1984
+ const profile = noProfile ? null : selectProfile(config, flags.profile);
1402
1985
  let runs;
1403
1986
  if (flags.runs !== void 0) {
1404
1987
  runs = Number(flags.runs);
@@ -1408,7 +1991,7 @@ async function prepareRun(flags, configPath) {
1408
1991
  const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
1409
1992
  const container = sandboxes.find((s) => s.kind === "docker");
1410
1993
  if (!container) return hostCapacity();
1411
- const { dockerCapacity } = await import("./src-CZANB4pb.js");
1994
+ const { dockerCapacity } = await import("./src-BkH4GOxu.js");
1412
1995
  return await dockerCapacity(container.engine) ?? hostCapacity();
1413
1996
  });
1414
1997
  return {
@@ -1435,11 +2018,11 @@ async function prepareRun(flags, configPath) {
1435
2018
  async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
1436
2019
  const startedAt = /* @__PURE__ */ new Date();
1437
2020
  const git = await gitInfo(config.root, [config.history.dir]);
1438
- const { worktreePatch } = await import("./state-haxTOljN.js");
2021
+ const { worktreePatch } = await import("./state-DZbAO95a.js");
1439
2022
  const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
1440
2023
  const who = await author(config.root);
1441
- const { stagingDir, sweepStaging } = await import("./results-CFE6Nwb0.js");
1442
- const { scratchDir } = await import("./files-Bw2NQecs.js");
2024
+ const { stagingDir, sweepStaging } = await import("./results-CfC0id_u.js");
2025
+ const { scratchDir } = await import("./files-DFtuRQJx.js");
1443
2026
  const scratch = scratchDir(config.root);
1444
2027
  if (resume !== void 0) {
1445
2028
  const target = await resolveResume(config.root, resume || void 0);
@@ -1450,7 +2033,7 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
1450
2033
  artifacts: target.previousArtifacts
1451
2034
  } : await loadPrevious(stagingDir(config.root, target.runId));
1452
2035
  if (!dryRun) try {
1453
- const { removeRunContainers } = await import("./docker-D83s_ADn.js");
2036
+ const { removeRunContainers } = await import("./docker-BlSopHpZ.js");
1454
2037
  await removeRunContainers(target.runId, ["docker", "podman"]);
1455
2038
  } catch {}
1456
2039
  return {
@@ -1497,11 +2080,33 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
1497
2080
  }
1498
2081
  function noMatch(config, filters, tags) {
1499
2082
  const narrowed = [filters.length ? `filter "${filters.join(" ")}"` : "", tags.length ? `tags ${tags.join(", ")}` : ""].filter(Boolean);
1500
- return new UsageError(`No evals matched ${config.include.join(", ")}${narrowed.length ? ` with ${narrowed.join(" and ")}` : ""}`);
2083
+ return new UsageError(`No evals matched ${config.include.join(", ")}${narrowed.length ? ` with ${narrowed.join(" and ")}` : ""}${maskedFilterHint(filters)}`);
2084
+ }
2085
+ /**
2086
+ * Why a filter that holds a secret matches nothing: eval names are masked before they're
2087
+ * matched. Printed masked, it's the filter that would match.
2088
+ */
2089
+ function maskedFilterHint(parts) {
2090
+ const masked = parts.filter((p) => redactSecrets(p) !== p);
2091
+ if (!masked.length) return "";
2092
+ return `. Eval names are masked, so a filter that holds a secret can't match one: filter by the masked name (${masked.map((p) => `"${redactSecrets(p)}"`).join(", ")}) or a part of it without the secret`;
1501
2093
  }
1502
2094
  function unmatchedFilter(config, parts) {
1503
2095
  const which = parts.map((p) => `"${p}"`).join(", ");
1504
- return new UsageError(`filter ${which} ${parts.length === 1 ? "matches" : "match"} no eval file, eval name or cell in ${config.include.join(", ")}`);
2096
+ return new UsageError(`filter ${which} ${parts.length === 1 ? "matches" : "match"} no eval file, eval name or cell in ${config.include.join(", ")}${maskedFilterHint(parts)}`);
2097
+ }
2098
+ /**
2099
+ * An eval whose id holds a value declared with `secret()`. `evaluate` masks names as the
2100
+ * worker's environment shows secrets, but a declared one is only known once the plan has the
2101
+ * environments, after the names became ids, so it's refused rather than written.
2102
+ */
2103
+ function namesDeclaredSecret(evals) {
2104
+ for (const { id } of evals) {
2105
+ const [file, ...names] = id.split(" › ");
2106
+ const masked = names.map((n) => redactSecrets(n));
2107
+ if (masked.every((n, i) => n === names[i])) continue;
2108
+ return new UsageError(`${[file, ...masked].join(" › ")} names a secret declared with secret(): name it without the secret.`);
2109
+ }
1505
2110
  }
1506
2111
  /**
1507
2112
  * Reads Cursor's model catalog for the planned Cursor cells, checks their `effort` and `params`
@@ -1511,12 +2116,12 @@ function unmatchedFilter(config, parts) {
1511
2116
  */
1512
2117
  async function planEfforts(planned) {
1513
2118
  const live = planned.filter((p) => !p.attempt.plan.skip);
1514
- const { effortNotes, withDefaultEffort } = await import("./default-effort-B7c4V9Qd.js");
2119
+ const { effortNotes, withDefaultEffort } = await import("./default-effort-FEidNAIi.js");
1515
2120
  let cursor;
1516
2121
  let unchecked = "CURSOR_API_KEY isn't set";
1517
2122
  const key = process.env.CURSOR_API_KEY;
1518
2123
  if (key && live.some((p) => p.attempt.harness.kind === "cursor")) {
1519
- const { checkCursorPlan } = await import("./cursor-catalog-BeBJTk6A.js");
2124
+ const { checkCursorPlan } = await import("./cursor-catalog-BTK_apL-.js");
1520
2125
  const checked = await checkCursorPlan(key, live);
1521
2126
  if (checked instanceof Error) unchecked = `its request failed: ${checked.message}`;
1522
2127
  else {
@@ -1536,7 +2141,7 @@ async function planEfforts(planned) {
1536
2141
  * Collects the evals and prints the plan; stops there for `--dry-run`, or on a budget, credential
1537
2142
  * or Docker problem (exit 2). Then runs the engine with live output and Ctrl-C handling.
1538
2143
  */
1539
- async function executeRun(ctx, setup, filters, renderer) {
2144
+ async function executeRun(ctx, setup, filters, renderer, started) {
1540
2145
  const { runId, stagingDir, config, runs, tags, concurrency, profile, verbosity, configFile } = ctx;
1541
2146
  const run = JSON.parse(JSON.stringify({
1542
2147
  runId,
@@ -1550,8 +2155,8 @@ async function executeRun(ctx, setup, filters, renderer) {
1550
2155
  configFile,
1551
2156
  completed: []
1552
2157
  }));
1553
- const { runEngine } = await import("./node-CwP1G178.js");
1554
- const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-BrfiMcHh.js");
2158
+ const { runEngine } = await import("./node-DhIRSrrK.js");
2159
+ const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-DqnzfDH1.js");
1555
2160
  let plan;
1556
2161
  let sandboxVersions = { images: [] };
1557
2162
  let preflightStop;
@@ -1560,6 +2165,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1560
2165
  let toRun = [];
1561
2166
  let live;
1562
2167
  let kept = /* @__PURE__ */ new Map();
2168
+ let leftOut = [];
1563
2169
  let questions;
1564
2170
  const onPlan = async (collected) => {
1565
2171
  const { planned, excluded, matched, unmatched } = collected;
@@ -1580,9 +2186,11 @@ async function executeRun(ctx, setup, filters, renderer) {
1580
2186
  } : p);
1581
2187
  toRun = skipped;
1582
2188
  for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
2189
+ const named = namesDeclaredSecret([...planned, ...excluded]);
2190
+ if (named) throw named;
1583
2191
  const efforts = await planEfforts(skipped);
1584
2192
  if (efforts) run.cursorEfforts = efforts;
1585
- const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-D1TBKeZj.js") : void 0;
2193
+ const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-CXVVB4RK.js") : void 0;
1586
2194
  const judge = runJudgeModel(config, profile);
1587
2195
  const judgeModel = judge === void 0 ? null : judgeModelName(judge);
1588
2196
  plan = await buildPlan({
@@ -1601,13 +2209,20 @@ async function executeRun(ctx, setup, filters, renderer) {
1601
2209
  const cloud = ciRunner(process.env);
1602
2210
  const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && p.attempt.sandbox.kind === "docker").filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
1603
2211
  if (cloud && direct.length) plan.warnings.push(`${directOnCloud(cloud)}. Under it: ${direct.join(", ")}. Keep 'allow-all' to the evals that need raw TCP or UDP; the default, 'proxy', sends everything through the gateway`);
2212
+ plan.warnings.push(...started.notes);
2213
+ if (started.narrowed) {
2214
+ leftOut = leftOutAttempts(planned, stagingDir, ctx.previous, await liveAttempts(stagingDir), new Set(excluded.map((e) => e.id)));
2215
+ const notRun = leftOut.filter((l) => !l.done).length;
2216
+ if (notRun) plan.warnings.push(`${notRun === 1 ? "1 attempt" : `${notRun} attempts`} run ${runId} planned ${notRun === 1 ? "isn't" : "aren't"} in this resume's filters: ${notRun === 1 ? "it stays" : "they stay"} not run and the run stays incomplete. reeval run --resume ${runId} runs ${notRun === 1 ? "it" : "them"}`);
2217
+ }
1604
2218
  if (completed.size) plan.resume = {
1605
2219
  runId,
1606
2220
  completed: completed.size
1607
2221
  };
1608
2222
  run.completed = [...completed];
1609
- kept = keptAttempts(completed, stagingDir, ctx.previous);
1610
- const { imageStages, imageTargets } = await import("./images-h1VBM7TP.js");
2223
+ const done = leftOut.filter((l) => l.done).map((l) => l.key);
2224
+ kept = keptAttempts([...completed, ...done], stagingDir, ctx.previous);
2225
+ const { imageStages, imageTargets } = await import("./images-CNayXCp2.js");
1611
2226
  const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
1612
2227
  id: p.id,
1613
2228
  harness: p.attempt.harness,
@@ -1626,13 +2241,13 @@ async function executeRun(ctx, setup, filters, renderer) {
1626
2241
  return false;
1627
2242
  }
1628
2243
  const elsewhere = setup.configElsewhere ? configFile : void 0;
1629
- const { scratchDir } = await import("./files-Bw2NQecs.js");
2244
+ const { scratchDir } = await import("./files-DFtuRQJx.js");
1630
2245
  const lines = [];
1631
2246
  if (setup.dryRun) {
1632
2247
  const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
1633
2248
  lines.push(...planTable(plan, todos));
1634
- const { imageStatusLines } = await import("./images-h1VBM7TP.js");
1635
- const { dockerAvailable, engineSpec } = await import("./src-CZANB4pb.js");
2249
+ const { imageStatusLines } = await import("./images-CNayXCp2.js");
2250
+ const { dockerAvailable, engineSpec } = await import("./src-BkH4GOxu.js");
1636
2251
  const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
1637
2252
  const down = [];
1638
2253
  for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
@@ -1641,7 +2256,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1641
2256
  const names = down.map((e) => engineSpec(e).name).join(" and ");
1642
2257
  lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
1643
2258
  }
1644
- const { costLine } = await import("./cost-CMzY9VXv.js");
2259
+ const { costLine } = await import("./cost-BJU0MAji.js");
1645
2260
  const cost = plan.cost && costLine(plan.cost);
1646
2261
  if (cost) lines.push(cost);
1647
2262
  }
@@ -1715,7 +2330,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1715
2330
  });
1716
2331
  if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
1717
2332
  if (asked.length) {
1718
- const { questionRunner } = await import("./questions-AyLUMHYX.js");
2333
+ const { questionRunner } = await import("./questions-BzdLdV3C.js");
1719
2334
  const warned = /* @__PURE__ */ new Set();
1720
2335
  questions = questionRunner({
1721
2336
  evals: asked,
@@ -1757,7 +2372,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1757
2372
  }
1758
2373
  });
1759
2374
  }
1760
- const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-It7dvpbg.js");
2375
+ const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-B8EOCuxl.js");
1761
2376
  const live = skipped.filter((p) => !p.attempt.plan.skip);
1762
2377
  const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
1763
2378
  workspace: a.workspace,
@@ -1861,7 +2476,9 @@ async function executeRun(ctx, setup, filters, renderer) {
1861
2476
  runId,
1862
2477
  startedAt: ctx.startedAt,
1863
2478
  planned: toRun,
1864
- kept
2479
+ kept,
2480
+ leftOut,
2481
+ ...started.startedWith ? { invocation: started.startedWith } : {}
1865
2482
  });
1866
2483
  renderer.onEvent({
1867
2484
  type: "run-start",
@@ -1889,8 +2506,9 @@ async function executeRun(ctx, setup, filters, renderer) {
1889
2506
  onCancelable: (cancel) => {
1890
2507
  let first;
1891
2508
  const onSignal = (signal) => {
1892
- if (first && signal !== first.signal && Date.now() - first.at < SAME_PRESS_MS) {
1893
- if (!first.passedOn && !finishing) renderer.onEvent({
2509
+ if (first && !first.passedOn && signal !== first.signal && Date.now() - first.at < SAME_PRESS_MS) {
2510
+ if (signal === "SIGINT") stopCode = EXIT.interrupted;
2511
+ if (!finishing) renderer.onEvent({
1894
2512
  type: "notice",
1895
2513
  at: Date.now(),
1896
2514
  level: "warn",
@@ -1932,9 +2550,9 @@ async function executeRun(ctx, setup, filters, renderer) {
1932
2550
  throw error;
1933
2551
  } finally {
1934
2552
  clearTimeout(budgetTimer);
1935
- const { removePrepared } = await import("./preflight-CH-7KyhS.js");
2553
+ const { removePrepared } = await import("./preflight-DVXEvzFU.js");
1936
2554
  removePrepared(config.root, runId);
1937
- const { removePreparedWorkspaces } = await import("./workspace-It7dvpbg.js");
2555
+ const { removePreparedWorkspaces } = await import("./workspace-B8EOCuxl.js");
1938
2556
  removePreparedWorkspaces(config.root, runId);
1939
2557
  }
1940
2558
  if (preflightStop) throw preflightStop;
@@ -1948,7 +2566,8 @@ async function executeRun(ctx, setup, filters, renderer) {
1948
2566
  budgetReached: () => reached,
1949
2567
  cutShort,
1950
2568
  live,
1951
- questions
2569
+ questions,
2570
+ leftOut
1952
2571
  };
1953
2572
  }
1954
2573
  process.stdout.on("error", (error) => {
@@ -2016,4 +2635,4 @@ run(process.argv.slice(2)).finally(() => {
2016
2635
  });
2017
2636
  export {};
2018
2637
 
2019
- //# sourceMappingURL=cli-Bm9ntSxz.js.map
2638
+ //# sourceMappingURL=cli-BBOlXKPG.js.map