reeval 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (326) hide show
  1. package/README.md +52 -40
  2. package/THIRD_PARTY_NOTICES.md +218 -11
  3. package/assets/package.json +1 -1
  4. package/assets/pnpm-lock.yaml +30 -30
  5. package/dist/adapter-DWlpOtRn.js +279 -0
  6. package/dist/adapter-DWlpOtRn.js.map +1 -0
  7. package/dist/{agent-B72lZ6w5.js → agent-CURAwx_q.js} +6 -6
  8. package/dist/{agent-B72lZ6w5.js.map → agent-CURAwx_q.js.map} +1 -1
  9. package/dist/{agent-Cfj1wD4D.js → agent-CaS_2Sce.js} +92 -26
  10. package/dist/agent-CaS_2Sce.js.map +1 -0
  11. package/dist/{analysis-BTNnKh1w.js → analysis-Bxub9LMT.js} +4 -4
  12. package/dist/{analysis-BTNnKh1w.js.map → analysis-Bxub9LMT.js.map} +1 -1
  13. package/dist/{assemble-BbFVjHeS.js → assemble-B9Alds-z.js} +290 -30
  14. package/dist/assemble-B9Alds-z.js.map +1 -0
  15. package/dist/assemble-J5MukZFY.js +1 -0
  16. package/dist/{client-CwzU9CiW.js → assert-EUIoRdIt.js} +38 -112
  17. package/dist/assert-EUIoRdIt.js.map +1 -0
  18. package/dist/bridge/index.mjs +248 -55
  19. package/dist/{capture-BXhBfmsM.js → capture-CNb9SuU5.js} +204 -39
  20. package/dist/capture-CNb9SuU5.js.map +1 -0
  21. package/dist/capture-jquXZBKK.js +1 -0
  22. package/dist/{cli-BMjTKY-D.js → cli-D303Vj5N.js} +257 -146
  23. package/dist/cli-D303Vj5N.js.map +1 -0
  24. package/dist/{cli-spec-DSvKsOK4.js → cli-spec-BPZbQG2p.js} +38 -19
  25. package/dist/cli-spec-BPZbQG2p.js.map +1 -0
  26. package/dist/cli.js +4 -2
  27. package/dist/cli.js.map +1 -1
  28. package/dist/client-fcwgwOAD.js +106 -0
  29. package/dist/client-fcwgwOAD.js.map +1 -0
  30. package/dist/{command-JPGNzFsA.js → command-BLfT7vnQ.js} +19 -18
  31. package/dist/command-BLfT7vnQ.js.map +1 -0
  32. package/dist/{command-BK4kOgkH.js → command-BT9a37ZR.js} +381 -78
  33. package/dist/command-BT9a37ZR.js.map +1 -0
  34. package/dist/{command-BqJtFiPy.js → command-Bge8F7Ko.js} +384 -101
  35. package/dist/command-Bge8F7Ko.js.map +1 -0
  36. package/dist/command-BkFjijN2.js +1 -0
  37. package/dist/{command-BdstSEHu.js → command-CMJtHKRa.js} +14 -14
  38. package/dist/{command-BdstSEHu.js.map → command-CMJtHKRa.js.map} +1 -1
  39. package/dist/commands-1p-bIWhN.js +1 -0
  40. package/dist/{commands-B618jg3C.js → commands-BFx16WB8.js} +8 -8
  41. package/dist/commands-BFx16WB8.js.map +1 -0
  42. package/dist/config-DEgnUiU-.js +756 -0
  43. package/dist/config-DEgnUiU-.js.map +1 -0
  44. package/dist/{context-CvUv7DUb.js → context-C1mxXh80.js} +11 -7
  45. package/dist/context-C1mxXh80.js.map +1 -0
  46. package/dist/{db-DFbtGIYy.js → db-CdHizc4_.js} +582 -53
  47. package/dist/db-CdHizc4_.js.map +1 -0
  48. package/dist/db-Dm1cRJJP.js +2 -0
  49. package/dist/debug-D7bEv_UV.js +1 -0
  50. package/dist/{dist-CTzThfu_.js → dist-B8hJImYt.js} +17 -4
  51. package/dist/dist-B8hJImYt.js.map +1 -0
  52. package/dist/{docker-2qns7hJ2.js → docker-7b_2SaH7.js} +35 -22
  53. package/dist/docker-7b_2SaH7.js.map +1 -0
  54. package/dist/{docker-IFRW8K5f.js → docker-sZrNrr82.js} +3 -3
  55. package/dist/docker-sZrNrr82.js.map +1 -0
  56. package/dist/docker-yauK-9X8.js +1 -0
  57. package/dist/{doctor-B6oYTPoR.js → doctor-BYNok4UO.js} +263 -84
  58. package/dist/doctor-BYNok4UO.js.map +1 -0
  59. package/dist/{duration-JsJrIWXq.js → duration-CEOQww97.js} +2 -2
  60. package/dist/{duration-JsJrIWXq.js.map → duration-CEOQww97.js.map} +1 -1
  61. package/dist/{egress-IUKdeayc.js → egress-C4PZTSSe.js} +147 -14
  62. package/dist/egress-C4PZTSSe.js.map +1 -0
  63. package/dist/environment-DNjRk0nU.js +1133 -0
  64. package/dist/environment-DNjRk0nU.js.map +1 -0
  65. package/dist/{errors-yDAEwmZo.js → errors-B6BSMs03.js} +2 -2
  66. package/dist/errors-B6BSMs03.js.map +1 -0
  67. package/dist/{errors-C9w_ud7g.js → errors-r025B0Zb.js} +4 -4
  68. package/dist/errors-r025B0Zb.js.map +1 -0
  69. package/dist/evaluate-C1UVL21l.d.ts +92 -0
  70. package/dist/evaluate-D4zeRGBF.js +2351 -0
  71. package/dist/evaluate-D4zeRGBF.js.map +1 -0
  72. package/dist/family-D2b7WD10.js +67 -0
  73. package/dist/family-D2b7WD10.js.map +1 -0
  74. package/dist/files-BAt0i00X.js +1 -0
  75. package/dist/{files-CMSUXAEg.js → files-BXUdY8tr.js} +10 -9
  76. package/dist/{files-CMSUXAEg.js.map → files-BXUdY8tr.js.map} +1 -1
  77. package/dist/files-BhyltlDB.js +1 -0
  78. package/dist/files-Bk1mC3AN.js +43 -0
  79. package/dist/files-Bk1mC3AN.js.map +1 -0
  80. package/dist/{files-B6aVVGrk.js → files-DQmHE7pb.js} +5 -5
  81. package/dist/{files-B6aVVGrk.js.map → files-DQmHE7pb.js.map} +1 -1
  82. package/dist/files-ZZPqfAo6.js +1 -0
  83. package/dist/{git-EmdFJ4DY.js → git-DfG-YJ1u.js} +2 -2
  84. package/dist/{git-EmdFJ4DY.js.map → git-DfG-YJ1u.js.map} +1 -1
  85. package/dist/harness-B8QC9u0O.js +323 -0
  86. package/dist/harness-B8QC9u0O.js.map +1 -0
  87. package/dist/harness.d.ts +19 -31
  88. package/dist/harness.js +1 -2
  89. package/dist/{header-_F0EWob3.js → header-hK-Z_Po9.js} +10 -3
  90. package/dist/{header-_F0EWob3.js.map → header-hK-Z_Po9.js.map} +1 -1
  91. package/dist/{images-CscddrJU.js → images-0On0cztu.js} +114 -45
  92. package/dist/images-0On0cztu.js.map +1 -0
  93. package/dist/index.d.ts +10 -26
  94. package/dist/index.js +6 -2108
  95. package/dist/{init-NQfIvi-0.js → init-B77Rpcqc.js} +26 -8
  96. package/dist/init-B77Rpcqc.js.map +1 -0
  97. package/dist/init-BrOWin3W.js +1 -0
  98. package/dist/{invocations-nPvrlZPj.js → invocations-DbGfBAA_.js} +4 -4
  99. package/dist/{invocations-nPvrlZPj.js.map → invocations-DbGfBAA_.js.map} +1 -1
  100. package/dist/{judge-CeYPN8P_.js → judge-DW_GDOmB.js} +11 -19
  101. package/dist/{judge-CeYPN8P_.js.map → judge-DW_GDOmB.js.map} +1 -1
  102. package/dist/judge.d.ts +2 -2
  103. package/dist/judge.js +1 -1
  104. package/dist/{live-S0qODb6o.js → live-jlfszAbb.js} +9 -20
  105. package/dist/live-jlfszAbb.js.map +1 -0
  106. package/dist/matchers-Ddmc2z7H.js +118 -0
  107. package/dist/matchers-Ddmc2z7H.js.map +1 -0
  108. package/dist/{model-DEQo3FkZ.js → model-DaVCY3rb.js} +2 -2
  109. package/dist/{model-DEQo3FkZ.js.map → model-DaVCY3rb.js.map} +1 -1
  110. package/dist/model-NM3ID0uR.js +1 -0
  111. package/dist/{model-Cl507trk.js → model-h_cDtcRr.js} +19 -18
  112. package/dist/model-h_cDtcRr.js.map +1 -0
  113. package/dist/model-nwWJvaUR.js +1 -0
  114. package/dist/{model-BndnmcLm.js → model-oqcDplJG.js} +271 -34
  115. package/dist/model-oqcDplJG.js.map +1 -0
  116. package/dist/{node-Co7EXge0.js → node-B3ydHo6t.js} +10 -47
  117. package/dist/node-B3ydHo6t.js.map +1 -0
  118. package/dist/object-CLf2-tNq.js +9 -0
  119. package/dist/object-CLf2-tNq.js.map +1 -0
  120. package/dist/{opencode-providers-ByD85Q0l.js → opencode-providers-J9colP0-.js} +96 -21
  121. package/dist/opencode-providers-J9colP0-.js.map +1 -0
  122. package/dist/options-Bh2W63dK.js +1119 -0
  123. package/dist/options-Bh2W63dK.js.map +1 -0
  124. package/dist/{pairs-DDsjgm5F.js → pairs-CnInRu-G.js} +76 -2
  125. package/dist/{pairs-DDsjgm5F.js.map → pairs-CnInRu-G.js.map} +1 -1
  126. package/dist/{patch-CGCVipPu.js → patch-Zac5aIRM.js} +6 -6
  127. package/dist/{patch-CGCVipPu.js.map → patch-Zac5aIRM.js.map} +1 -1
  128. package/dist/{plan-PPc4-6xd.js → plan-C9Cy27Ay.js} +75 -32
  129. package/dist/plan-C9Cy27Ay.js.map +1 -0
  130. package/dist/preflight-CPWpQeBI.js +1 -0
  131. package/dist/{preflight-CddycK92.js → preflight-D7iFPKeK.js} +8 -7
  132. package/dist/preflight-D7iFPKeK.js.map +1 -0
  133. package/dist/prepare-BGjePFQl.js +2 -0
  134. package/dist/{prepare-CsP-057p.js → prepare-doGxaAGR.js} +38 -30
  135. package/dist/prepare-doGxaAGR.js.map +1 -0
  136. package/dist/process-CSVO_jsc.js +119 -0
  137. package/dist/process-CSVO_jsc.js.map +1 -0
  138. package/dist/{provenance-CsX6PHDR.js → provenance-DmE9uPVt.js} +4 -4
  139. package/dist/{provenance-CsX6PHDR.js.map → provenance-DmE9uPVt.js.map} +1 -1
  140. package/dist/provider-DehR4CMN.js +1 -0
  141. package/dist/providers-gDMko8fl.js +1 -0
  142. package/dist/providers-tAOD1NQN.js +165 -0
  143. package/dist/providers-tAOD1NQN.js.map +1 -0
  144. package/dist/{query-BUcwCYsI.js → query-Bes9Kh6Y.js} +3 -3
  145. package/dist/{query-BUcwCYsI.js.map → query-Bes9Kh6Y.js.map} +1 -1
  146. package/dist/query-C_G_svLU.js +1 -0
  147. package/dist/questions-DjAwMxZ3.js +524 -0
  148. package/dist/questions-DjAwMxZ3.js.map +1 -0
  149. package/dist/{record-BRP9vZrj.js → record-BKmMvZ6O.js} +9 -1
  150. package/dist/{record-BRP9vZrj.js.map → record-BKmMvZ6O.js.map} +1 -1
  151. package/dist/remote-HDRt16ew.js +378 -0
  152. package/dist/remote-HDRt16ew.js.map +1 -0
  153. package/dist/{results-BcSHWbgB.js → results-DuauQI3Z.js} +4 -4
  154. package/dist/{results-BcSHWbgB.js.map → results-DuauQI3Z.js.map} +1 -1
  155. package/dist/{resume-DZ9AmfPv.js → resume-BRfa8E_s.js} +10 -10
  156. package/dist/{resume-DZ9AmfPv.js.map → resume-BRfa8E_s.js.map} +1 -1
  157. package/dist/{root-CvdRlRZX.js → root-DICDHCtp.js} +4 -4
  158. package/dist/{root-CvdRlRZX.js.map → root-DICDHCtp.js.map} +1 -1
  159. package/dist/{runtime-Dfb2r-cS.js → runtime-PX_oXiCS.js} +105 -11
  160. package/dist/runtime-PX_oXiCS.js.map +1 -0
  161. package/dist/sandbox-D3A8oUb7.js +70 -0
  162. package/dist/sandbox-D3A8oUb7.js.map +1 -0
  163. package/dist/sandbox.d.ts +22 -8
  164. package/dist/sandbox.js +1 -2
  165. package/dist/server-C7sg1ndo.js +1 -0
  166. package/dist/{server-Da60oe4b.js → server-XDleKaF_.js} +11 -11
  167. package/dist/server-XDleKaF_.js.map +1 -0
  168. package/dist/shared-DYpIb1BI.js +166 -0
  169. package/dist/shared-DYpIb1BI.js.map +1 -0
  170. package/dist/{source-BVOnL3sn.js → source-C8xumIWj.js} +2 -2
  171. package/dist/{source-BVOnL3sn.js.map → source-C8xumIWj.js.map} +1 -1
  172. package/dist/source-fGd0SaKP.js +1 -0
  173. package/dist/{src-TdJuDYqB.js → src-B7treWa8.js} +1597 -292
  174. package/dist/src-B7treWa8.js.map +1 -0
  175. package/dist/{src-C1E9HDWO.js → src-C7_1SqDP.js} +3 -3
  176. package/dist/{src-C1E9HDWO.js.map → src-C7_1SqDP.js.map} +1 -1
  177. package/dist/{src-DDNXsD9H.js → src-CF17Ug08.js} +5 -4
  178. package/dist/src-CF17Ug08.js.map +1 -0
  179. package/dist/{src-D_cXqmnI.js → src-CFp86GtV.js} +60 -60
  180. package/dist/src-CFp86GtV.js.map +1 -0
  181. package/dist/src-C_lPtowT.js +2 -0
  182. package/dist/src-DIL0jST0.js +368 -0
  183. package/dist/src-DIL0jST0.js.map +1 -0
  184. package/dist/src-DNnG4e4R.js +2 -0
  185. package/dist/src-DTbrY1hi.js +2 -0
  186. package/dist/src-DdyJEh2v.js +2 -0
  187. package/dist/src-Dj9PAroc.js +2 -0
  188. package/dist/src-Du8H85Qq.js +2 -0
  189. package/dist/src-DvCg0NBs.js +324 -0
  190. package/dist/src-DvCg0NBs.js.map +1 -0
  191. package/dist/{src-CxTC5kNI.js → src-DvK0EIZx.js} +8 -10
  192. package/dist/{src-CxTC5kNI.js.map → src-DvK0EIZx.js.map} +1 -1
  193. package/dist/{src-DVy58-zZ.js → src-FGXhkEkH.js} +5 -5
  194. package/dist/{src-DVy58-zZ.js.map → src-FGXhkEkH.js.map} +1 -1
  195. package/dist/src-jzagbuqH.js +2 -0
  196. package/dist/src-ofUaYbuY.js +3 -0
  197. package/dist/src-pJZOrYR9.js +1 -0
  198. package/dist/{src-D95O-Smv.js → src-u8_ixn_F.js} +15 -11
  199. package/dist/src-u8_ixn_F.js.map +1 -0
  200. package/dist/{state-B72h_jEw.js → state-C__9swoG.js} +36 -24
  201. package/dist/state-C__9swoG.js.map +1 -0
  202. package/dist/state-CeR6oz0k.js +66 -0
  203. package/dist/state-CeR6oz0k.js.map +1 -0
  204. package/dist/state-CgjQo29K.js +1 -0
  205. package/dist/studio-app/assets/index-CwQN3f_h.css +2 -0
  206. package/dist/studio-app/assets/{index-Bgr-efZm.js → index-DrQc2Yxd.js} +12 -12
  207. package/dist/studio-app/index.html +2 -2
  208. package/dist/{summary-k3mhWWXv.js → summary-BLAVLySU.js} +202 -17
  209. package/dist/summary-BLAVLySU.js.map +1 -0
  210. package/dist/{summary-CJdxaRdM.js → summary-DgyjUh_l.js} +7 -7
  211. package/dist/{summary-CJdxaRdM.js.map → summary-DgyjUh_l.js.map} +1 -1
  212. package/dist/testing.d.ts +3 -2
  213. package/dist/testing.js +3 -2
  214. package/dist/{tool-target-BU1JnSE9.js → tool-target-D2mcYlSQ.js} +14 -7
  215. package/dist/tool-target-D2mcYlSQ.js.map +1 -0
  216. package/dist/{tools-D0gF2QoV.js → tools-GCyNYSc1.js} +7 -7
  217. package/dist/tools-GCyNYSc1.js.map +1 -0
  218. package/dist/transcript-BNNANU8A.js +64 -0
  219. package/dist/transcript-BNNANU8A.js.map +1 -0
  220. package/dist/types-C5vI3g1C.js.map +1 -1
  221. package/dist/{types-_LOFdPUf.d.ts → types-g17bRAzt.d.ts} +578 -49
  222. package/dist/{usage-DDF_6tkV.js → usage-CleNKkly.js} +4 -2
  223. package/dist/{usage-DDF_6tkV.js.map → usage-CleNKkly.js.map} +1 -1
  224. package/dist/view/index.d.ts +83 -17
  225. package/dist/view/index.js +7 -6
  226. package/dist/views-BW6oCDUQ.js +2 -0
  227. package/dist/{views-DZ1YDqey.js → views-BmQF4UCI.js} +13 -8
  228. package/dist/views-BmQF4UCI.js.map +1 -0
  229. package/dist/{worker-entry-DXeyWY-m.js → worker-entry-yOchGYYv.js} +16 -225
  230. package/dist/worker-entry-yOchGYYv.js.map +1 -0
  231. package/dist/{workspace-DpXdrslo.js → workspace-CgyHEM0l.js} +7 -7
  232. package/dist/{workspace-DpXdrslo.js.map → workspace-CgyHEM0l.js.map} +1 -1
  233. package/dist/workspace-D8sSgpiy.js +1 -0
  234. package/docker/Dockerfile +7 -5
  235. package/package.json +12 -12
  236. package/dist/agent-Cfj1wD4D.js.map +0 -1
  237. package/dist/assemble-BbFVjHeS.js.map +0 -1
  238. package/dist/assemble-CXiTVyHD.js +0 -2
  239. package/dist/capture-BXhBfmsM.js.map +0 -1
  240. package/dist/capture-N7r3yeak.js +0 -2
  241. package/dist/cli-BMjTKY-D.js.map +0 -1
  242. package/dist/cli-spec-DSvKsOK4.js.map +0 -1
  243. package/dist/client-CwzU9CiW.js.map +0 -1
  244. package/dist/command-BK4kOgkH.js.map +0 -1
  245. package/dist/command-BqJtFiPy.js.map +0 -1
  246. package/dist/command-CTVGuH8y.js +0 -2
  247. package/dist/command-JPGNzFsA.js.map +0 -1
  248. package/dist/commands-B618jg3C.js.map +0 -1
  249. package/dist/commands-Duo2ur3r.js +0 -2
  250. package/dist/config-B-E0xRkf.js +0 -447
  251. package/dist/config-B-E0xRkf.js.map +0 -1
  252. package/dist/context-CvUv7DUb.js.map +0 -1
  253. package/dist/db-CnoUy-FZ.js +0 -3
  254. package/dist/db-DFbtGIYy.js.map +0 -1
  255. package/dist/debug-SEmW4w8f.js +0 -2
  256. package/dist/dist-CTzThfu_.js.map +0 -1
  257. package/dist/docker-2qns7hJ2.js.map +0 -1
  258. package/dist/docker-D6t23fZl.js +0 -2
  259. package/dist/docker-IFRW8K5f.js.map +0 -1
  260. package/dist/doctor-B6oYTPoR.js.map +0 -1
  261. package/dist/egress-IUKdeayc.js.map +0 -1
  262. package/dist/environment-DiCljXMD.js +0 -657
  263. package/dist/environment-DiCljXMD.js.map +0 -1
  264. package/dist/errors-C9w_ud7g.js.map +0 -1
  265. package/dist/errors-yDAEwmZo.js.map +0 -1
  266. package/dist/fake-instructions-CUI33I_F.js +0 -32
  267. package/dist/fake-instructions-CUI33I_F.js.map +0 -1
  268. package/dist/files-BM78_nnF.js +0 -2
  269. package/dist/files-BpYgF2z0.js +0 -2
  270. package/dist/files-YCCVaTQC.js +0 -2
  271. package/dist/harness-iqWeJQMM.js +0 -154
  272. package/dist/harness-iqWeJQMM.js.map +0 -1
  273. package/dist/images-CscddrJU.js.map +0 -1
  274. package/dist/index.js.map +0 -1
  275. package/dist/init-DBHyi92L.js +0 -2
  276. package/dist/init-NQfIvi-0.js.map +0 -1
  277. package/dist/jiti-BcJHCtrb.js +0 -46
  278. package/dist/jiti-BcJHCtrb.js.map +0 -1
  279. package/dist/live-S0qODb6o.js.map +0 -1
  280. package/dist/model-BbLGRtOs.js +0 -2
  281. package/dist/model-BndnmcLm.js.map +0 -1
  282. package/dist/model-Cl507trk.js.map +0 -1
  283. package/dist/model-DMOraCcm.js +0 -2
  284. package/dist/node-Co7EXge0.js.map +0 -1
  285. package/dist/opencode-providers-ByD85Q0l.js.map +0 -1
  286. package/dist/options-DPUlm4FI.js +0 -735
  287. package/dist/options-DPUlm4FI.js.map +0 -1
  288. package/dist/plan-PPc4-6xd.js.map +0 -1
  289. package/dist/preflight-CddycK92.js.map +0 -1
  290. package/dist/preflight-Zumy0adT.js +0 -2
  291. package/dist/prepare-CKdMoC7e.js +0 -3
  292. package/dist/prepare-CsP-057p.js.map +0 -1
  293. package/dist/provider-BDbNOQhQ.js +0 -2
  294. package/dist/providers-DkwiLKDS.js +0 -107
  295. package/dist/providers-DkwiLKDS.js.map +0 -1
  296. package/dist/providers-yXZZcPfx.js +0 -2
  297. package/dist/query-noGbekGF.js +0 -2
  298. package/dist/questions-BEPj98fZ.js +0 -234
  299. package/dist/questions-BEPj98fZ.js.map +0 -1
  300. package/dist/runtime-Dfb2r-cS.js.map +0 -1
  301. package/dist/sandbox--ZXuscAZ.js +0 -134
  302. package/dist/sandbox--ZXuscAZ.js.map +0 -1
  303. package/dist/server-Da60oe4b.js.map +0 -1
  304. package/dist/server-DiGR4JxK.js +0 -2
  305. package/dist/source-Dqu0jKwx.js +0 -2
  306. package/dist/src-BHNiP-5A.js +0 -4
  307. package/dist/src-BT4dB5CO.js +0 -2
  308. package/dist/src-BT5pwco0.js +0 -3
  309. package/dist/src-BrMbt1B2.js +0 -3
  310. package/dist/src-D13vpMh0.js +0 -3
  311. package/dist/src-D95O-Smv.js.map +0 -1
  312. package/dist/src-DDNXsD9H.js.map +0 -1
  313. package/dist/src-D_cXqmnI.js.map +0 -1
  314. package/dist/src-DdRlaiRc.js +0 -3
  315. package/dist/src-TdJuDYqB.js.map +0 -1
  316. package/dist/src-xczuQbE6.js +0 -3
  317. package/dist/state-B72h_jEw.js.map +0 -1
  318. package/dist/state-Bf_Vq_pV.js +0 -2
  319. package/dist/studio-app/assets/index-CUx6GoL5.css +0 -2
  320. package/dist/summary-k3mhWWXv.js.map +0 -1
  321. package/dist/tool-target-BU1JnSE9.js.map +0 -1
  322. package/dist/tools-D0gF2QoV.js.map +0 -1
  323. package/dist/views-DZ1YDqey.js.map +0 -1
  324. package/dist/views-Ds0GXWtX.js +0 -3
  325. package/dist/worker-entry-DXeyWY-m.js.map +0 -1
  326. package/dist/workspace-DzNL-3iO.js +0 -2
@@ -1,32 +1,34 @@
1
- import { n as UsageError } from "./errors-yDAEwmZo.js";
2
- import { t as parseDuration } from "./duration-JsJrIWXq.js";
1
+ import { n as isOneOf } from "./object-CLf2-tNq.js";
2
+ import { n as UsageError } from "./errors-B6BSMs03.js";
3
+ import { t as parseDuration } from "./duration-CEOQww97.js";
4
+ import { f as sandboxId, s as harnessId } from "./options-Bh2W63dK.js";
5
+ import { A as workspaceLabel, p as environmentSandbox } from "./environment-DNjRk0nU.js";
6
+ import { a as fromCwd, i as evalSlug, n as attemptKey, o as newRunId } from "./ids-Bp6ut7ZD.js";
7
+ import { a as configPathFor, c as findConfig, d as malformedCredentials, f as missingCredentials, h as selectProfile, i as configLabel, l as hostCapacity, m as resolveConcurrency, p as requireConfig, u as loadConfig } from "./config-DEgnUiU-.js";
3
8
  import { n as loadEnvFiles, t as inCI } from "./env-BQqj93gu.js";
4
9
  import { o as redactSecrets, s as registerSecret } from "./redact-DJ9JCShu.js";
5
- import { o as PACKAGE_VERSION } from "./root-CvdRlRZX.js";
6
- import { T as workspaceLabel, u as environmentSandbox } from "./environment-DiCljXMD.js";
7
- import { a as fromCwd, i as evalSlug, n as attemptKey, o as newRunId } from "./ids-Bp6ut7ZD.js";
8
- import { I as LABEL, X as imageInfo } from "./src-TdJuDYqB.js";
9
- import { d as sandboxId, s as harnessId } from "./options-DPUlm4FI.js";
10
- import { c as hostCapacity, d as missingCredentials, f as resolveConcurrency, i as configPathFor, l as loadConfig, p as selectProfile, r as configLabel, s as findConfig, u as malformedCredentials } from "./config-B-E0xRkf.js";
10
+ import { o as PACKAGE_VERSION } from "./root-DICDHCtp.js";
11
+ import { I as LABEL, X as imageInfo } from "./src-B7treWa8.js";
11
12
  import { s as debugTime } from "./debug-D24NOQH1.js";
12
13
  import { i as sandboxWarningLine, n as clip, r as formatDuration } from "./text-gEcKjAyp.js";
13
14
  import { t as PrepareError } from "./provider-CmAYTQQs.js";
14
- import { H as cellContrast, L as judgeRatesLabel, M as cellJudgeRates, N as cellUsage, P as collapsedJudgeLabel, _ as hasUngatedCells, b as isUngatedMiss, h as formatUsage, q as effortLabel, w as runOutcome } from "./model-BndnmcLm.js";
15
- import { a as sampleReasons } from "./pairs-DDsjgm5F.js";
16
- import { a as liftLabel, i as gateLabel, n as answersLabel } from "./summary-k3mhWWXv.js";
17
- import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-DZ9AmfPv.js";
18
- import { n as FLAGS, o as helpText, r as GLOBAL, t as COMMANDS } from "./cli-spec-DSvKsOK4.js";
19
- import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-CsX6PHDR.js";
15
+ import { B as judgeRatesLabel, H as summaryJudgeRates, I as collapsedJudgeLabel, P as cellUsage, T as runOutcome, Y as cellContrast, g as formatUsage, l as evalMarkOutcome, tt as effortLabel, v as hasUngatedCells, x as isUngatedMiss } from "./model-oqcDplJG.js";
16
+ import { a as familyLiftLabel, i as familyHeadLabel, n as familyCases, o as familyRateLabel, r as familyEffortLabel, t as caseLabel } from "./family-D2b7WD10.js";
17
+ import { n as headToHeadLines, s as sampleReasons } from "./pairs-CnInRu-G.js";
18
+ import { a as contrastLines, f as undiscriminating, i as checksLabel, n as answersLabel, o as gateLabel, p as undiscriminatingLabel, s as liftLabel } from "./summary-BLAVLySU.js";
19
+ import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-BRfa8E_s.js";
20
+ import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-BPZbQG2p.js";
21
+ import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-DmE9uPVt.js";
20
22
  import { t as budgetMarker } from "./budget-DBUrH4jX.js";
21
- import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-CddycK92.js";
22
- import { r as imageHarness } from "./prepare-CsP-057p.js";
23
- import { n as createProviders } from "./providers-DkwiLKDS.js";
24
- import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-nPvrlZPj.js";
23
+ import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-D7iFPKeK.js";
24
+ import { n as imageHarness } from "./prepare-doGxaAGR.js";
25
+ import { r as createProviders, t as attemptImage } from "./providers-tAOD1NQN.js";
26
+ import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-DbGfBAA_.js";
25
27
  import { n as stripAnsi, t as colors } from "./colors-QV9dFVhN.js";
26
28
  import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
27
29
  import { dirname, join } from "node:path";
28
- import { appendFile, readFile, rm } from "node:fs/promises";
29
30
  import { inspect } from "node:util";
31
+ import { appendFile, readFile, rm } from "node:fs/promises";
30
32
  import { detectAgent } from "std-env";
31
33
  import { Command, Option } from "commander";
32
34
  //#region src/cli-parse.ts
@@ -64,13 +66,13 @@ function parseCli(argv) {
64
66
  const subcommand = configure(command.command(subName), `reeval ${name} ${subName}`, sub.flags);
65
67
  subcommand.argument("[args...]").action(() => {
66
68
  const [stray] = Object.keys(valuesOf(command, spec.flags));
67
- if (stray) throw new UsageError(`--${stray} isn't an option of reeval ${name} ${subName}`);
69
+ if (stray) throw new UsageError(`--${stray} isn't an option of reeval ${name} ${subName}. Run reeval --help.`);
68
70
  parsed = done(program, subcommand, [name, subName], sub.flags, sub.positionals);
69
71
  });
70
72
  }
71
73
  }
72
74
  program.parse(argv, { from: "user" });
73
- if (!parsed) throw new UsageError("Run reeval --help.");
75
+ if (!parsed) throw new UsageError("reeval couldn't read that command line. Run reeval --help.");
74
76
  return parsed;
75
77
  }
76
78
  function configure(command, name, flags) {
@@ -123,11 +125,16 @@ function usageError(error, command) {
123
125
  const [flag = "", value] = arg.replace(/^--?/, "").split("=");
124
126
  const known = arg.startsWith("--") ? FLAGS[flag] : Object.entries(FLAGS).find(([, s]) => "short" in s && s.short === flag)?.[1];
125
127
  const long = arg.startsWith("--") ? flag : Object.entries(FLAGS).find(([, s]) => "short" in s && s.short === flag)?.[0];
126
- if (known && value !== void 0 && known.type === "boolean") return new UsageError(`Option '--${flag}' does not take an argument`);
128
+ if (known && value !== void 0 && known.type === "boolean") return new UsageError(`--${flag} doesn't take a value: pass --${flag} on its own.`);
127
129
  if (known) return new UsageError(`--${long} isn't an option of ${command}. Run reeval --help.`);
128
130
  const hint = rest.join(" ").replace(/^\(|\)$/g, "");
129
131
  return new UsageError(`Unknown option '${arg}'. ${hint ? `${hint} ` : ""}Run reeval --help.`);
130
132
  }
133
+ if (error.code === "commander.optionMissingArgument") {
134
+ const usage = /'([^']+)'/.exec(first)?.[1] ?? "";
135
+ const flag = usage.split(/[ ,]/).find((part) => part.startsWith("--")) ?? usage;
136
+ return new UsageError(`${flag} needs a value, e.g. ${usage}. Run reeval --help.`);
137
+ }
131
138
  return new UsageError(first.charAt(0).toUpperCase() + first.slice(1));
132
139
  }
133
140
  //#endregion
@@ -171,10 +178,10 @@ async function preflight(uses, options) {
171
178
  writePrepared(options.root, options.runId, outcomes);
172
179
  const images = [];
173
180
  for (const o of outcomes) {
174
- if (!o.ok || o.target.kind !== "docker") continue;
175
- const { ref, id, agentCli } = o.handle;
181
+ if (!o.ok || o.target.sandbox.kind !== "docker") continue;
182
+ const { ref, id, agentCli } = attemptImage(o.handle);
176
183
  if (images.some((i) => i.id === id)) continue;
177
- const info = await imageInfo(ref);
184
+ const info = await imageInfo(ref, o.target.sandbox.engine);
178
185
  images.push({
179
186
  ref,
180
187
  id,
@@ -416,8 +423,10 @@ function resultFor(entry, attempt, results) {
416
423
  function entryOutcome(e) {
417
424
  const outcomes = e.attempts.map(attemptOutcome);
418
425
  if (outcomes.length && outcomes.every((o) => o === "cancelled")) return "cancelled";
419
- return e.outcome;
426
+ return evalMarkOutcome(e);
420
427
  }
428
+ /** Every cell is `gate: 'none'`: the eval reports its attempts but nothing gates it. */
429
+ const allUngated = (e) => e.cells.length > 0 && e.cells.every((c) => c.gate === "none");
421
430
  const pct = (rate) => rate === null ? " – " : `${Math.round(rate * 100)}%`.padStart(4);
422
431
  function evalColumns(c, e) {
423
432
  const [attempt, ...more] = e.attempts;
@@ -426,7 +435,7 @@ function evalColumns(c, e) {
426
435
  if (more.length) {
427
436
  const pass = e.attempts.filter((a) => a.outcome === "pass").length;
428
437
  const counted = e.attempts.filter((a) => a.outcome === "pass" || a.outcome === "fail").length;
429
- const rate = hasUngatedCells(e.cells) ? ` · ${pct(e.passRate).trim()} gated` : ` ${pct(e.passRate)}`;
438
+ const rate = allUngated(e) ? " · not gated" : hasUngatedCells(e.cells) ? ` · pass@1 ${pct(e.passRate).trim()} gated` : ` pass@1 ${pct(e.passRate)}`;
430
439
  return c.dim(`${`${pass}/${counted}`.padStart(7)}${rate} ${e.attempts.length} attempts`);
431
440
  }
432
441
  if (attemptOutcome(attempt) === "cancelled") return c.yellow("cancelled");
@@ -439,7 +448,7 @@ function evalColumns(c, e) {
439
448
  if (tokens) cols.push(tokens);
440
449
  return c.dim(cols.join(" "));
441
450
  }
442
- /** Pass rate per cell for evals with several attempts; with compare, each cell's role and lift. */
451
+ /** pass@1 per cell for evals with several attempts; with compare, each cell's role and lift. */
443
452
  function cellRows(c, e, indent) {
444
453
  if (e.attempts.length < 2) return [];
445
454
  const compare = e.cells.some((cell) => cell.role === "variant");
@@ -458,27 +467,92 @@ function cellRows(c, e, indent) {
458
467
  if (cell.gate !== "all" && cell.gate !== "none") note += ` ${c.dim(`gate ${gateLabel(cell.gate)}`)}`;
459
468
  const tokens = formatUsage(cellUsage(e, cell.key));
460
469
  const usage = tokens ? ` ${c.dim(tokens)}` : "";
461
- const rows = [`${indent}${c.dim(cell.key.padEnd(width))}${role} ${counts} ${pct(cell.passRate)}${note}${infra}${usage}`];
462
- const rates = judgeRatesLabel(cellJudgeRates(e, cell.key));
470
+ const rows = [`${indent}${c.dim(cell.key.padEnd(width))}${role} ${counts} pass@1 ${pct(cell.passRate)}${note}${infra}${usage}`];
471
+ const rates = judgeRatesLabel(summaryJudgeRates(e, cell));
463
472
  if (rates) rows.push(`${indent} ${c.dim(rates)}`);
464
473
  const effort = cellContrast(e, cell.key);
465
474
  if (effort) rows.push(`${indent} ${c.dim(`effort: ${effortLabel(effort)}`)}`);
475
+ const checks = checksLabel(cell);
476
+ if (checks) rows.push(`${indent} ${c.dim(`checks: ${checks}`)}`);
466
477
  if (cell.overrides.length) rows.push(`${indent} ${c.dim(`overrides: ${cell.overrides.join(", ")}`)}`);
467
478
  return rows;
468
479
  });
469
480
  }
481
+ /** With two variants or more, each pair of arms on one harness: lift, checks and effort. */
482
+ function contrastRows(c, e, indent) {
483
+ const lines = contrastLines(e);
484
+ if (!lines.length) return [];
485
+ return [`${indent}${c.dim("contrasts (each pair of arms, same harness)")}`, ...lines.map((line) => `${indent} ${c.dim(line)}`)];
486
+ }
470
487
  /** One row per compare question: its win rates, or why it has none. They never gate. */
471
488
  function questionRows(c, e, indent) {
472
489
  return (e.answers ?? []).flatMap((a) => {
473
490
  const label = answersLabel(a);
474
- return [`${indent}${c.dim("question")} ${a.error !== void 0 ? c.yellow(label) : label}`, ...sampleReasons(a.votes).map((reason) => `${indent} ${c.dim(reason)}`)];
491
+ return [
492
+ `${indent}${c.dim("question")} ${a.error !== void 0 ? c.yellow(label) : label}`,
493
+ ...headToHeadLines(a.votes).map((line) => `${indent} ${line}`),
494
+ ...sampleReasons(a.votes).map((reason) => `${indent} ${c.dim(reason)}`)
495
+ ];
475
496
  });
476
497
  }
477
- /** The run's evals as a file › group tree, one row per eval and per cell of a matrix. */
478
- function tree(c, evals) {
498
+ /** A family's mark: its pooled gate's and its cases' worst. */
499
+ function familyOutcome(family, cases) {
500
+ const outcomes = [family.outcome, ...cases.map(entryOutcome)];
501
+ for (const o of [
502
+ "fail",
503
+ "infra_error",
504
+ "cancelled"
505
+ ]) if (outcomes.includes(o)) return o;
506
+ return outcomes.every((o) => o === "skipped") ? "skipped" : "pass";
507
+ }
508
+ /** The width case marks wrap at. */
509
+ const CASE_WIDTH = 96;
510
+ /** A pooled family's rows under its header: its cases' marks, then each pooled rate or lift. */
511
+ function familyRows(c, family, cases, indent) {
512
+ const marks = cases.map((e) => `${mark(c, entryOutcome(e))} ${caseLabel(family.name, e.name)}`);
513
+ const plain = cases.map((e) => `x ${caseLabel(family.name, e.name)}`);
479
514
  const rows = [];
515
+ let line = [];
516
+ let width = 0;
517
+ marks.forEach((m, i) => {
518
+ const w = plain[i].length + 2;
519
+ if (line.length && width + w > CASE_WIDTH) {
520
+ rows.push(`${indent}${line.join(" ")}`);
521
+ line = [];
522
+ width = 0;
523
+ }
524
+ line.push(m);
525
+ width += w;
526
+ });
527
+ if (line.length) rows.push(`${indent}${line.join(" ")}`);
528
+ if (family.lifts.length) {
529
+ const keyWidth = Math.max(...family.lifts.map((l) => l.variant.length));
530
+ for (const lift of family.lifts) {
531
+ const below = lift.outcome === "fail";
532
+ const gate = lift.minLift !== void 0 ? ` (minLift ${Math.round(lift.minLift * 100)}${below ? " ✗" : ""})` : "";
533
+ const text = `${familyLiftLabel(lift)}${gate}`;
534
+ rows.push(`${indent}${c.dim(lift.variant.padEnd(keyWidth))} ${below ? c.red(text) : text}`);
535
+ const effort = familyEffortLabel(lift);
536
+ if (effort) rows.push(`${indent} ${c.dim(`effort: ${effort}`)}`);
537
+ }
538
+ } else if (family.rates.length === 1) rows.push(`${indent}${familyRateLabel(family.rates[0])}`);
539
+ else {
540
+ const keyWidth = Math.max(...family.rates.map((r) => r.cell.length));
541
+ for (const rate of family.rates) rows.push(`${indent}${c.dim(rate.cell.padEnd(keyWidth))} ${familyRateLabel(rate)}`);
542
+ }
543
+ return rows;
544
+ }
545
+ /**
546
+ * The run's evals as a file › group tree, one row per eval and per cell of a matrix. A pooled
547
+ * family is one row, its template, with its cases' marks and the pooled results under it.
548
+ */
549
+ function tree(c, evals, families = []) {
550
+ const rows = [];
551
+ const shown = /* @__PURE__ */ new Set();
480
552
  let path = [];
481
553
  for (const e of evals) {
554
+ const family = e.family && families.find((f) => f.id === e.family?.id);
555
+ if (family && shown.has(family.id)) continue;
482
556
  const next = [e.file, ...e.group];
483
557
  let same = 0;
484
558
  while (same < path.length && same < next.length && path[same] === next[same]) same++;
@@ -487,19 +561,32 @@ function tree(c, evals) {
487
561
  text: i === 0 ? c.bold(next[i] ?? "") : next[i] ?? ""
488
562
  });
489
563
  path = next;
490
- rows.push({
564
+ if (family) {
565
+ shown.add(family.id);
566
+ rows.push({
567
+ indent: next.length,
568
+ text: family.name,
569
+ family
570
+ });
571
+ } else rows.push({
491
572
  indent: next.length,
492
573
  text: e.name,
493
574
  entry: e
494
575
  });
495
576
  }
496
- const width = Math.max(...rows.filter((r) => r.entry).map((r) => r.indent * 2 + 2 + r.text.length));
497
- return rows.flatMap(({ indent, text, entry }) => {
577
+ const width = Math.max(...rows.filter((r) => r.entry || r.family).map((r) => r.indent * 2 + 2 + r.text.length));
578
+ return rows.flatMap(({ indent, text, entry, family }) => {
498
579
  const pad = " ".repeat(indent);
580
+ const fill = " ".repeat(Math.max(0, width - indent * 2 - 2 - text.length));
581
+ if (family) {
582
+ const cases = familyCases(family, evals);
583
+ return [` ${pad}${mark(c, familyOutcome(family, cases))} ${text}${fill} ${c.dim(familyHeadLabel(family))}`, ...familyRows(c, family, cases, ` ${pad} `)];
584
+ }
499
585
  if (!entry) return [` ${pad}${text}`];
500
586
  return [
501
- ` ${pad}${`${mark(c, entryOutcome(entry))} ${text}`}${" ".repeat(Math.max(0, width - indent * 2 - 2 - text.length))} ${evalColumns(c, entry)}`,
587
+ ` ${pad}${`${entryOutcome(entry) === "pass" && allUngated(entry) && entry.attempts.some((a) => isUngatedMiss(entry, a)) ? MISS_MARK(c) : mark(c, entryOutcome(entry))} ${text}`}${fill} ${evalColumns(c, entry)}`,
502
588
  ...cellRows(c, entry, ` ${pad} `),
589
+ ...contrastRows(c, entry, ` ${pad} `),
503
590
  ...questionRows(c, entry, ` ${pad} `)
504
591
  ];
505
592
  });
@@ -517,6 +604,12 @@ function totals(c, s, cancelled = 0) {
517
604
  const scope = s.attempts === s.evals ? `${s.evals}` : `${s.attempts} attempts, ${s.evals} evals`;
518
605
  return parts.length ? `${parts.join(c.dim(" · "))} ${c.dim(`(${scope})`)}` : "no evals ran";
519
606
  }
607
+ /** `1 passed · 1 failed (2)`: the families' pooled gates. */
608
+ function familyTotals(c, families) {
609
+ const fail = families.filter((f) => f.outcome === "fail").length;
610
+ const pass = families.length - fail;
611
+ return `${[pass && c.green(`${pass} passed`), fail && c.red(`${fail} failed`)].filter(Boolean).join(c.dim(" · "))} ${c.dim(`(${families.length})`)}`;
612
+ }
520
613
  /** Failing attempts, in run order, including expected misses. */
521
614
  function failingAttempts(evals) {
522
615
  return evals.flatMap((entry) => entry.attempts.map((attempt) => ({
@@ -528,7 +621,7 @@ function failingAttempts(evals) {
528
621
  function renderSummary(c, e, attemptDir) {
529
622
  const { record } = e;
530
623
  const lines = [];
531
- if (record.evals.length) lines.push(c.bold("Summary"), ...tree(c, record.evals));
624
+ if (record.evals.length) lines.push(c.bold("Summary"), ...tree(c, record.evals, record.families));
532
625
  const all = failingAttempts(record.evals);
533
626
  const missed = all.filter(({ entry, attempt }) => isUngatedMiss(entry, attempt));
534
627
  const failing = all.filter((f) => !missed.includes(f));
@@ -550,7 +643,18 @@ function renderSummary(c, e, attemptDir) {
550
643
  }
551
644
  if (missed.length) {
552
645
  lines.push("", c.bold(`Expected misses, not gated (${missed.length})`));
646
+ const pooled = /* @__PURE__ */ new Map();
647
+ for (const { entry, attempt } of missed) {
648
+ if (!entry.family) continue;
649
+ const key = `${entry.family.id}\0${attempt.cell}`;
650
+ pooled.set(key, (pooled.get(key) ?? 0) + 1);
651
+ }
652
+ for (const [key, n] of pooled) {
653
+ const [id, cell] = key.split("\0");
654
+ lines.push(` ${MISS_MARK(c)} ${id} ${c.dim(`${n} ${n === 1 ? "attempt" : "attempts"} on ${cell}`)}`);
655
+ }
553
656
  for (const { entry, attempt } of missed) {
657
+ if (entry.family) continue;
554
658
  const which = entry.attempts.length > 1 ? ` ${c.dim(attempt.id)}` : "";
555
659
  const [reason] = failureLines(colors(false), resultFor(entry, attempt, e.results), { max: 1 }).filter((l) => l.trim());
556
660
  lines.push(` ${MISS_MARK(c)} ${entry.id}${which}`);
@@ -562,12 +666,23 @@ function renderSummary(c, e, attemptDir) {
562
666
  const cancelled = failing.filter((f) => f.outcome === "cancelled").length;
563
667
  const label = record.summary.attempts === record.summary.evals ? " Evals" : "Attempts";
564
668
  const lifted = record.evals.flatMap((entry) => entry.cells.flatMap((cell) => cell.lift?.minLift !== void 0 && cell.lift.interval.lo < cell.lift.minLift ? [`${entry.id} ${cell.key} ${liftLabel(cell.lift)}: its lower bound is below minLift ${Math.round(cell.lift.minLift * 100)}`] : []));
669
+ for (const family of record.families ?? []) for (const lift of family.lifts) {
670
+ if (lift.outcome !== "fail" || lift.minLift === void 0) continue;
671
+ lifted.push(`${family.id} ${lift.variant} pooled ${familyLiftLabel(lift)}: its lower bound is below minLift ${Math.round(lift.minLift * 100)}`);
672
+ }
673
+ const idle = undiscriminating(record.evals);
674
+ if (idle.length) {
675
+ lines.push("", c.bold(`Checks that don't discriminate (${idle.length})`));
676
+ for (const x of idle) lines.push(` ${c.dim("·")} ${undiscriminatingLabel(x)}`);
677
+ }
565
678
  if (lifted.length) {
566
679
  lines.push("", c.bold(c.red(`Lift below minLift (${lifted.length})`)));
567
680
  for (const line of lifted) lines.push(` ${c.red("✗")} ${line}`);
568
681
  }
569
682
  if (record.status === "budget") lines.push("", c.yellow("Budget reached: attempts that hadn’t started were skipped"));
570
- lines.push("", `${c.dim(label)} ${totals(c, record.summary, cancelled)}`, `${c.dim("Duration")} ${formatDuration(record.durationMs)}`);
683
+ lines.push("");
684
+ if (record.families?.length) lines.push(`${c.dim("Families")} ${familyTotals(c, record.families)}`);
685
+ lines.push(`${c.dim(label)} ${totals(c, record.summary, cancelled)}`, `${c.dim("Duration")} ${formatDuration(record.durationMs)}`);
571
686
  const tokens = formatUsage(record.cost.usage);
572
687
  if (tokens) lines.push(` ${c.dim("Tokens")} ${tokens}`);
573
688
  lines.push(` ${c.dim("Results")} ${e.summaryPath}`);
@@ -732,8 +847,17 @@ var AgentRenderer = class {
732
847
  for (const error of e.unhandledErrors) this.#write(`ERROR ${error}`);
733
848
  for (const entry of e.record.evals) for (const a of entry.answers ?? []) {
734
849
  this.#write(`question ${entry.id}: ${answersLabel(a)}`);
850
+ for (const line of headToHeadLines(a.votes)) this.#write(` ${line}`);
735
851
  for (const reason of sampleReasons(a.votes)) this.#write(` ${reason}`);
736
852
  }
853
+ for (const x of undiscriminating(e.record.evals)) this.#write(`check doesn't discriminate: ${undiscriminatingLabel(x)}`);
854
+ for (const family of e.record.families ?? []) {
855
+ const pooled = family.lifts.length ? family.lifts.map((l) => {
856
+ const gate = l.minLift === void 0 ? "" : ` (minLift ${Math.round(l.minLift * 100)}${l.outcome === "fail" ? ", FAIL" : ""})`;
857
+ return `${l.variant} ${familyLiftLabel(l)}${gate}`;
858
+ }) : family.rates.map((r) => `${r.cell} ${familyRateLabel(r)}`);
859
+ this.#write(`family ${family.id}: ${familyHeadLabel(family)} · ${pooled.join(" · ")}`);
860
+ }
737
861
  const s = e.record.summary;
738
862
  const cancelled = failing.filter((f) => f.outcome === "cancelled").length;
739
863
  const parts = [
@@ -1009,13 +1133,13 @@ const CI_VARS = [
1009
1133
  const isCI = (env) => CI_VARS.some((name) => set(env, name));
1010
1134
  function pick(name, value, allowed) {
1011
1135
  if (value === void 0 || value === "") return void 0;
1012
- if (value === "auto" || allowed.includes(value)) return value;
1013
- throw new UsageError(`${name} must be one of ${[...allowed, "auto"].join(", ")}`);
1136
+ if (value === "auto" || isOneOf(allowed, value)) return value;
1137
+ throw new UsageError(`${name} is one of ${[...allowed, "auto"].join(", ")} (got ${value})`);
1014
1138
  }
1015
1139
  function parseHeartbeat(value) {
1016
1140
  if (!value) return 6e4;
1017
1141
  const match = /^(\d+(?:\.\d+)?)(ms|s|m)?$/.exec(value);
1018
- if (!match) throw new UsageError(`REEVAL_HEARTBEAT must be a duration like 30s or 0`);
1142
+ if (!match) throw new UsageError(`REEVAL_HEARTBEAT must be a duration like 30s, or 0 for none (got ${value})`);
1019
1143
  const unit = {
1020
1144
  ms: 1,
1021
1145
  s: 1e3,
@@ -1115,12 +1239,16 @@ async function run(argv) {
1115
1239
  return EXIT.ok;
1116
1240
  }
1117
1241
  const [command, ...filters] = positionals;
1242
+ if (values.help && command) {
1243
+ console.log(commandHelpText(version, positionals));
1244
+ return EXIT.ok;
1245
+ }
1118
1246
  if (values.help || !command) {
1119
1247
  console.log(HELP);
1120
1248
  return values.help ? EXIT.ok : EXIT.usage;
1121
1249
  }
1122
1250
  if (command === "init") {
1123
- const { initCommand } = await import("./init-DBHyi92L.js");
1251
+ const { initCommand } = await import("./init-BrOWin3W.js");
1124
1252
  return initCommand(values, filters, {
1125
1253
  out: (s) => console.log(s),
1126
1254
  input: process.stdin,
@@ -1129,7 +1257,7 @@ async function run(argv) {
1129
1257
  });
1130
1258
  }
1131
1259
  if (command === "doctor") {
1132
- const { doctorCommand } = await import("./doctor-B6oYTPoR.js");
1260
+ const { doctorCommand } = await import("./doctor-BYNok4UO.js");
1133
1261
  return doctorCommand(values, {
1134
1262
  out: (s) => console.log(s),
1135
1263
  color: !!process.stdout.isTTY && !process.env.NO_COLOR,
@@ -1137,7 +1265,7 @@ async function run(argv) {
1137
1265
  });
1138
1266
  }
1139
1267
  if (command === "report") {
1140
- const { reportCommand } = await import("./command-JPGNzFsA.js");
1268
+ const { reportCommand } = await import("./command-BLfT7vnQ.js");
1141
1269
  return reportCommand(values, filters, {
1142
1270
  out: (s) => console.log(s),
1143
1271
  err: (s) => console.error(dim(s)),
@@ -1152,7 +1280,7 @@ async function run(argv) {
1152
1280
  }
1153
1281
  if (command === "studio") {
1154
1282
  const config = await loadConfig(configPath, "studio");
1155
- const { studioCommand } = await import("./command-BdstSEHu.js");
1283
+ const { studioCommand } = await import("./command-CMJtHKRa.js");
1156
1284
  return studioCommand({
1157
1285
  config,
1158
1286
  flags: values,
@@ -1163,7 +1291,7 @@ async function run(argv) {
1163
1291
  if (command === "mcp") {
1164
1292
  if (!values.http) console.log = console.info = console.debug = console.error;
1165
1293
  const config = await loadConfig(configPath, "mcp");
1166
- const { mcpCommand } = await import("./command-BK4kOgkH.js");
1294
+ const { mcpCommand } = await import("./command-BT9a37ZR.js");
1167
1295
  return mcpCommand({
1168
1296
  config,
1169
1297
  flags: values,
@@ -1172,7 +1300,7 @@ async function run(argv) {
1172
1300
  }
1173
1301
  if (command in HISTORY_COMMANDS) {
1174
1302
  const config = await loadConfig(configPath, command);
1175
- const commands = await import("./commands-Duo2ur3r.js");
1303
+ const commands = await import("./commands-1p-bIWhN.js");
1176
1304
  const ctx = {
1177
1305
  config,
1178
1306
  flags: values,
@@ -1185,7 +1313,7 @@ async function run(argv) {
1185
1313
  case "compare": return commands.compareCommand(ctx, filters);
1186
1314
  case "checkout": return commands.checkoutCommand(ctx, filters);
1187
1315
  case "query": {
1188
- const { queryCommand } = await import("./query-noGbekGF.js");
1316
+ const { queryCommand } = await import("./query-C_G_svLU.js");
1189
1317
  return queryCommand({
1190
1318
  ...ctx,
1191
1319
  err: (s) => console.error(dim(s))
@@ -1193,7 +1321,7 @@ async function run(argv) {
1193
1321
  }
1194
1322
  case "analyze": {
1195
1323
  loadEnvFiles(configPath);
1196
- const { analyzeCommand } = await import("./command-CTVGuH8y.js");
1324
+ const { analyzeCommand } = await import("./command-BkFjijN2.js");
1197
1325
  return analyzeCommand({
1198
1326
  ...ctx,
1199
1327
  err: (s) => console.error(dim(s))
@@ -1226,16 +1354,19 @@ async function cleanCommand(flags, configPath) {
1226
1354
  keep = Number(flags.keep);
1227
1355
  if (!(Number.isInteger(keep) && keep >= 0)) throw new UsageError(`--keep must be a whole number (got ${flags.keep})`);
1228
1356
  }
1229
- const { debugRoot, pruneDebug } = await import("./files-BM78_nnF.js");
1230
- const { formatBytes } = await import("./debug-SEmW4w8f.js");
1357
+ const { debugRoot, pruneDebug } = await import("./files-BAt0i00X.js");
1358
+ const { formatBytes } = await import("./debug-D7bEv_UV.js");
1231
1359
  const { runs, bytes } = await pruneDebug(config.root, keep);
1232
1360
  const where = fromCwd(debugRoot(config.root)) || ".";
1233
1361
  console.log(runs.length ? `Removed debug data for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where}` : `No debug data to remove in ${where}`);
1234
1362
  if (flags.images) {
1235
- const { pruneImages } = await import("./src-BT4dB5CO.js");
1236
- const { requireDocker } = await import("./providers-yXZZcPfx.js");
1237
- await requireDocker();
1238
- const removed = await pruneImages();
1363
+ const { dockerAvailable, pruneImages } = await import("./src-pJZOrYR9.js");
1364
+ const { requireDocker } = await import("./providers-gDMko8fl.js");
1365
+ const engines = [];
1366
+ for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
1367
+ if (!engines.length) await requireDocker();
1368
+ let removed = 0;
1369
+ for (const engine of engines) removed += await pruneImages(engine);
1239
1370
  console.log(removed ? `Removed ${removed} image${removed === 1 ? "" : "s"} reeval built that nothing uses` : "No leftover images to remove");
1240
1371
  }
1241
1372
  return EXIT.ok;
@@ -1248,55 +1379,24 @@ function verbosityOf(flags) {
1248
1379
  /**
1249
1380
  * `reeval image build`: collects the evals (as `--dry-run` does) and builds every image their Docker
1250
1381
  * sandboxes need; with `--dry-run`, lists them instead. `--profile` plans as `reeval run --profile`
1251
- * does. Without a config, just reeval's image.
1382
+ * does. Needs a config, as `reeval run` does: there is no default image.
1252
1383
  */
1253
1384
  async function imageBuildCommand(flags, configPath, filters) {
1254
- const { buildImages, imagePlanLines, imageTargets } = await import("./images-CscddrJU.js");
1255
- const { docker } = await import("./sandbox.js");
1256
- let root = process.cwd();
1257
- const { fakeHarness } = await import("./harness.js");
1258
- let uses = [{
1259
- id: "",
1260
- harness: fakeHarness(),
1261
- sandbox: docker()
1262
- }];
1263
- if (!configPath && flags.profile) selectProfile(await loadConfig(void 0), flags.profile);
1264
- if (configPath) {
1265
- const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
1266
- ...flags,
1267
- "dry-run": true
1268
- }, configPath));
1269
- root = config.root;
1270
- const { runEngine } = await import("./node-Co7EXge0.js");
1271
- let planned = [];
1272
- await runEngine({
1273
- run: JSON.parse(JSON.stringify({
1274
- runId,
1275
- stagingDir,
1276
- config,
1277
- runs,
1278
- tags,
1279
- concurrency,
1280
- profile
1281
- })),
1282
- filters,
1283
- onPlan: async (collected) => {
1284
- if (collected.unmatched.length) throw unmatchedFilter(config, collected.unmatched);
1285
- planned = collected.planned.filter((p) => !p.attempt.plan.skip);
1286
- return false;
1287
- }
1288
- });
1289
- uses = planned.length ? planned.map((p) => ({
1290
- id: p.id,
1291
- harness: p.attempt.harness,
1292
- sandbox: p.attempt.sandbox
1293
- })) : config.harness.map((harness) => ({
1294
- id: "",
1295
- harness,
1296
- sandbox: config.sandbox
1297
- }));
1298
- }
1299
- const targets = imageTargets(uses);
1385
+ const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-0On0cztu.js");
1386
+ const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
1387
+ ...flags,
1388
+ "dry-run": true
1389
+ }, configPath));
1390
+ const root = config.root;
1391
+ const targets = imageTargets(await plannedImageUses({
1392
+ runId,
1393
+ stagingDir,
1394
+ config,
1395
+ runs,
1396
+ tags,
1397
+ concurrency,
1398
+ profile
1399
+ }, filters, (parts) => unmatchedFilter(config, parts)));
1300
1400
  if (targets.length === 0) {
1301
1401
  console.log("No eval runs in a Docker sandbox, so there are no images to build.");
1302
1402
  return EXIT.ok;
@@ -1305,7 +1405,7 @@ async function imageBuildCommand(flags, configPath, filters) {
1305
1405
  for (const line of await imagePlanLines(targets, root)) console.log(line);
1306
1406
  return EXIT.ok;
1307
1407
  }
1308
- const { PrepareError } = await import("./provider-BDbNOQhQ.js");
1408
+ const { PrepareError } = await import("./provider-DehR4CMN.js");
1309
1409
  let built;
1310
1410
  try {
1311
1411
  built = await buildImages(targets, root, {
@@ -1342,7 +1442,7 @@ async function runCommand(flags, configPath, filters) {
1342
1442
  if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
1343
1443
  const stepStart = process.env.GITHUB_STEP_SUMMARY;
1344
1444
  if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
1345
- const { engineVersions } = await import("./node-Co7EXge0.js");
1445
+ const { engineVersions } = await import("./node-B3ydHo6t.js");
1346
1446
  const renderer = createRenderer({
1347
1447
  out: setup.json ? { write: () => true } : process.stdout,
1348
1448
  output: setup.output,
@@ -1363,15 +1463,16 @@ async function runCommand(flags, configPath, filters) {
1363
1463
  }
1364
1464
  const { engineRun, plan, sandboxVersions, budgetReached, live, notes, questions } = executed;
1365
1465
  invocation.unhandledErrors = engineRun.unhandledErrors;
1366
- if ((interrupted || engineRun.cancelled) && plan?.sandboxes.some((s) => s.kind === "docker")) {
1367
- const { removeRunContainers } = await import("./docker-D6t23fZl.js");
1368
- await removeRunContainers(ctx.runId);
1466
+ const engines = [...new Set(plan?.sandboxes.flatMap((s) => s.kind === "docker" ? [s.engine ?? "docker"] : []))];
1467
+ if ((interrupted || engineRun.cancelled) && engines.length) {
1468
+ const { removeRunContainers } = await import("./docker-yauK-9X8.js");
1469
+ await removeRunContainers(ctx.runId, engines);
1369
1470
  }
1370
1471
  if (notes) {
1371
1472
  const cancelled = interrupted || engineRun.cancelled;
1372
1473
  const written = await notes.finish(cancelled);
1373
1474
  if (!cancelled && written.length) {
1374
- const { notesLine } = await import("./agent-B72lZ6w5.js");
1475
+ const { notesLine } = await import("./agent-CURAwx_q.js");
1375
1476
  const text = notesLine(ctx.config.root, ctx.runId, written);
1376
1477
  renderer.onEvent({
1377
1478
  type: "notice",
@@ -1388,8 +1489,8 @@ async function runCommand(flags, configPath, filters) {
1388
1489
  }
1389
1490
  const stopped = interrupted || engineRun.cancelled;
1390
1491
  const answers = questions && !stopped ? await questions.settle() : void 0;
1391
- const { lockfiles } = await import("./state-Bf_Vq_pV.js");
1392
- const { writeScratchRun } = await import("./results-BcSHWbgB.js");
1492
+ const { lockfiles } = await import("./state-CgjQo29K.js");
1493
+ const { writeScratchRun } = await import("./results-DuauQI3Z.js");
1393
1494
  const run = await writeScratchRun({
1394
1495
  ctx,
1395
1496
  author: ctx.author,
@@ -1399,7 +1500,7 @@ async function runCommand(flags, configPath, filters) {
1399
1500
  interrupted: stopped,
1400
1501
  budgetReached: budgetReached(),
1401
1502
  plan,
1402
- es: version,
1503
+ reeval: version,
1403
1504
  runtime: runtimeInfo(),
1404
1505
  language: { name: "typescript" },
1405
1506
  enginePackages: engineVersions,
@@ -1414,8 +1515,8 @@ async function runCommand(flags, configPath, filters) {
1414
1515
  }, () => live?.stop());
1415
1516
  const { record } = run;
1416
1517
  if (notes && !stopped) {
1417
- const { summarizeRun } = await import("./agent-B72lZ6w5.js");
1418
- const { resolveAnalysisModel } = await import("./model-DMOraCcm.js");
1518
+ const { summarizeRun } = await import("./agent-CURAwx_q.js");
1519
+ const { resolveAnalysisModel } = await import("./model-NM3ID0uR.js");
1419
1520
  const model = resolveAnalysisModel(ctx.config, "agent");
1420
1521
  const { level, text } = await summarizeRun(ctx.config, record.runId, model);
1421
1522
  renderer.onEvent({
@@ -1425,7 +1526,7 @@ async function runCommand(flags, configPath, filters) {
1425
1526
  text
1426
1527
  });
1427
1528
  }
1428
- const { pruneDebug } = await import("./files-BM78_nnF.js");
1529
+ const { pruneDebug } = await import("./files-BAt0i00X.js");
1429
1530
  await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
1430
1531
  const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
1431
1532
  renderer.onEvent({
@@ -1442,7 +1543,7 @@ async function runCommand(flags, configPath, filters) {
1442
1543
  const { config } = ctx;
1443
1544
  const mode = setup.record;
1444
1545
  if ((flags.record || mode === "always" || mode === "ci" && inCI()) && record.status !== "interrupted") {
1445
- const { record: recordCmd } = await import("./commands-Duo2ur3r.js");
1546
+ const { record: recordCmd } = await import("./commands-1p-bIWhN.js");
1446
1547
  const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
1447
1548
  console.log(await recordCmd(config, record.runId, {
1448
1549
  ...flags,
@@ -1468,9 +1569,10 @@ async function prepareRun(flags, configPath) {
1468
1569
  agent: detectAgent().name
1469
1570
  });
1470
1571
  if (flags.json && !flags["dry-run"]) throw new UsageError("reeval run --json only prints the plan: add --dry-run. A run writes its record to .reeval/runs; read it with reeval show --json.");
1471
- loadEnvFiles(configPath);
1572
+ const path = requireConfig(configPath);
1573
+ loadEnvFiles(path);
1472
1574
  const named = flags.profile ?? (process.env.REEVAL_PROFILE || void 0);
1473
- const config = await loadConfig(configPath, "run", named);
1575
+ const config = await loadConfig(path, "run", named);
1474
1576
  const profile = selectProfile(config, flags.profile);
1475
1577
  let runs;
1476
1578
  if (flags.runs !== void 0) {
@@ -1479,9 +1581,10 @@ async function prepareRun(flags, configPath) {
1479
1581
  }
1480
1582
  const sandboxes = [environmentSandbox(config.environment, config.sandbox), ...profile?.defaults.environment ? [environmentSandbox(profile.defaults.environment, config.sandbox)] : []];
1481
1583
  const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
1482
- if (!sandboxes.some((s) => s.kind === "docker")) return hostCapacity();
1483
- const { dockerCapacity } = await import("./src-BT4dB5CO.js");
1484
- return await dockerCapacity() ?? hostCapacity();
1584
+ const container = sandboxes.find((s) => s.kind === "docker");
1585
+ if (!container) return hostCapacity();
1586
+ const { dockerCapacity } = await import("./src-pJZOrYR9.js");
1587
+ return await dockerCapacity(container.engine) ?? hostCapacity();
1485
1588
  });
1486
1589
  return {
1487
1590
  output,
@@ -1496,9 +1599,9 @@ async function prepareRun(flags, configPath) {
1496
1599
  json: !!flags.json,
1497
1600
  force: !!flags.force,
1498
1601
  verbosity: verbosityOf(flags),
1499
- configFile: configPath && configLabel(configPath),
1602
+ configFile: configLabel(path),
1500
1603
  /** Named in the plan line when the project root isn't the current directory. */
1501
- configElsewhere: !!configPath && dirname(configPath) !== process.cwd(),
1604
+ configElsewhere: dirname(path) !== process.cwd(),
1502
1605
  /** `undefined` is a new run; `''` is newest resumable; a string is a run id or prefix. */
1503
1606
  resume: flags.resume
1504
1607
  };
@@ -1507,11 +1610,11 @@ async function prepareRun(flags, configPath) {
1507
1610
  async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
1508
1611
  const startedAt = /* @__PURE__ */ new Date();
1509
1612
  const git = await gitInfo(config.root, [config.history.dir]);
1510
- const { worktreePatch } = await import("./state-Bf_Vq_pV.js");
1613
+ const { worktreePatch } = await import("./state-CgjQo29K.js");
1511
1614
  const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
1512
1615
  const who = await author(config.root);
1513
- const { stagingDir, sweepStaging } = await import("./results-BcSHWbgB.js");
1514
- const { scratchDir } = await import("./files-BpYgF2z0.js");
1616
+ const { stagingDir, sweepStaging } = await import("./results-DuauQI3Z.js");
1617
+ const { scratchDir } = await import("./files-ZZPqfAo6.js");
1515
1618
  const scratch = scratchDir(config.root);
1516
1619
  if (resume !== void 0) {
1517
1620
  const target = await resolveResume(config.root, resume || void 0);
@@ -1522,8 +1625,8 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
1522
1625
  artifacts: target.previousArtifacts
1523
1626
  } : await loadPrevious(stagingDir(config.root, target.runId));
1524
1627
  if (!dryRun) try {
1525
- const { removeRunContainers } = await import("./docker-D6t23fZl.js");
1526
- await removeRunContainers(target.runId);
1628
+ const { removeRunContainers } = await import("./docker-yauK-9X8.js");
1629
+ await removeRunContainers(target.runId, ["docker", "podman"]);
1527
1630
  } catch {}
1528
1631
  return {
1529
1632
  runId: target.runId,
@@ -1593,8 +1696,8 @@ async function executeRun(ctx, setup, filters, renderer) {
1593
1696
  configFile,
1594
1697
  completed: []
1595
1698
  }));
1596
- const { runEngine } = await import("./node-Co7EXge0.js");
1597
- const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable } = await import("./plan-PPc4-6xd.js");
1699
+ const { runEngine } = await import("./node-B3ydHo6t.js");
1700
+ const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable } = await import("./plan-C9Cy27Ay.js");
1598
1701
  let plan;
1599
1702
  let sandboxVersions = { images: [] };
1600
1703
  let preflightStop;
@@ -1621,7 +1724,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1621
1724
  } : p);
1622
1725
  toRun = skipped;
1623
1726
  for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
1624
- const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-DMOraCcm.js") : void 0;
1727
+ const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-NM3ID0uR.js") : void 0;
1625
1728
  plan = await buildPlan({
1626
1729
  planned: skipped,
1627
1730
  profile,
@@ -1637,7 +1740,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1637
1740
  completed: completed.size
1638
1741
  };
1639
1742
  run.completed = [...completed];
1640
- const { imageStages, imageTargets } = await import("./images-CscddrJU.js");
1743
+ const { imageStages, imageTargets } = await import("./images-0On0cztu.js");
1641
1744
  const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
1642
1745
  id: p.id,
1643
1746
  harness: p.attempt.harness,
@@ -1658,10 +1761,16 @@ async function executeRun(ctx, setup, filters, renderer) {
1658
1761
  if (setup.dryRun) {
1659
1762
  const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
1660
1763
  lines.push(...planTable(plan, todos));
1661
- const { imageStatusLines } = await import("./images-CscddrJU.js");
1662
- const { dockerAvailable } = await import("./src-BT4dB5CO.js");
1663
- if (targets.length && await dockerAvailable()) lines.push(...await imageStatusLines(targets, config.root));
1664
- else if (targets.length) lines.push("images not checked: Docker is not running");
1764
+ const { imageStatusLines } = await import("./images-0On0cztu.js");
1765
+ const { dockerAvailable, engineSpec } = await import("./src-pJZOrYR9.js");
1766
+ const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
1767
+ const down = [];
1768
+ for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
1769
+ if (targets.length && !down.length) lines.push(...await imageStatusLines(targets, config.root));
1770
+ else if (targets.length) {
1771
+ const names = down.map((e) => engineSpec(e).name).join(" and ");
1772
+ lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
1773
+ }
1665
1774
  }
1666
1775
  lines.push(...plan.warnings.map((w) => `warning: ${w}`));
1667
1776
  lines.push(...plan.hints.map((h) => `hint: ${h}`));
@@ -1673,7 +1782,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1673
1782
  });
1674
1783
  if (setup.dryRun) return false;
1675
1784
  checkAttemptBudget(plan, setup.force, config.file);
1676
- const analysis = config.analysis.agent ? await import("./model-DMOraCcm.js") : null;
1785
+ const analysis = config.analysis.agent ? await import("./model-NM3ID0uR.js") : null;
1677
1786
  analysis?.analysisModelName(config, "agent");
1678
1787
  const questionModels = /* @__PURE__ */ new Set();
1679
1788
  const asked = [];
@@ -1682,7 +1791,9 @@ async function executeRun(ctx, setup, filters, renderer) {
1682
1791
  const questions = (row.plan.questions ?? []).map((q) => ({
1683
1792
  question: q.question,
1684
1793
  model: models.questionModel(config, q, row.id),
1685
- ...q.across ? { across: q.across } : {}
1794
+ ...q.across ? { across: q.across } : {},
1795
+ ...q.state ? { state: q.state } : {},
1796
+ ...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
1686
1797
  }));
1687
1798
  for (const q of questions) if (typeof q.model === "string") questionModels.add(q.model);
1688
1799
  if (questions.length) asked.push({
@@ -1703,7 +1814,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1703
1814
  });
1704
1815
  if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
1705
1816
  if (analysis) {
1706
- const { startNotes } = await import("./agent-B72lZ6w5.js");
1817
+ const { startNotes } = await import("./agent-CURAwx_q.js");
1707
1818
  notes = startNotes({
1708
1819
  config,
1709
1820
  model: analysis.resolveAnalysisModel(config, "agent"),
@@ -1719,7 +1830,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1719
1830
  });
1720
1831
  }
1721
1832
  if (asked.length) {
1722
- const { questionRunner } = await import("./questions-BEPj98fZ.js");
1833
+ const { questionRunner } = await import("./questions-DjAwMxZ3.js");
1723
1834
  questions = questionRunner({
1724
1835
  evals: asked,
1725
1836
  stagingDir,
@@ -1736,7 +1847,7 @@ async function executeRun(ctx, setup, filters, renderer) {
1736
1847
  }
1737
1848
  });
1738
1849
  }
1739
- const { prepareWorkspaces, workspaceKey } = await import("./workspace-DzNL-3iO.js");
1850
+ const { prepareWorkspaces, workspaceKey } = await import("./workspace-D8sSgpiy.js");
1740
1851
  const live = skipped.filter((p) => !p.attempt.plan.skip);
1741
1852
  const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
1742
1853
  workspace: a.workspace,
@@ -1884,14 +1995,14 @@ async function executeRun(ctx, setup, filters, renderer) {
1884
1995
  throw error;
1885
1996
  } finally {
1886
1997
  clearTimeout(budgetTimer);
1887
- const { removePrepared } = await import("./preflight-Zumy0adT.js");
1998
+ const { removePrepared } = await import("./preflight-CPWpQeBI.js");
1888
1999
  removePrepared(config.root, runId);
1889
- const { removePreparedWorkspaces } = await import("./workspace-DzNL-3iO.js");
2000
+ const { removePreparedWorkspaces } = await import("./workspace-D8sSgpiy.js");
1890
2001
  removePreparedWorkspaces(config.root, runId);
1891
2002
  }
1892
2003
  if (preflightStop) throw preflightStop;
1893
2004
  if (engineRun.noFiles || !engineRun.stopped && engineRun.evals.length === 0 && !interrupted) live?.remove();
1894
- if (engineRun.noFiles && !filters.length && !tags.length) throw new UsageError(`No eval files match ${config.include.join(", ")} (config.include) in ${config.root}. Name eval files to match it, or set include in ${config.file}.`);
2005
+ if (engineRun.noFiles && !filters.length && !tags.length) throw new UsageError(`No eval files in ${config.root} match include: ${config.include.join(", ")}. Name eval files to match it, or set include in ${config.file}.`);
1895
2006
  if (engineRun.noFiles || !engineRun.stopped && engineRun.evals.length === 0 && !interrupted) throw noMatch(config, filters, tags);
1896
2007
  return {
1897
2008
  engineRun,
@@ -1920,7 +2031,7 @@ function finish(code, error, kind) {
1920
2031
  const command = commandOf(argv);
1921
2032
  logInvocation(root, {
1922
2033
  at: new Date(startedAt).toISOString(),
1923
- es: version,
2034
+ reeval: version,
1924
2035
  argv: redactArgv(argv),
1925
2036
  command,
1926
2037
  exitCode: code,
@@ -1969,4 +2080,4 @@ run(process.argv.slice(2)).finally(() => {
1969
2080
  //#endregion
1970
2081
  export {};
1971
2082
 
1972
- //# sourceMappingURL=cli-BMjTKY-D.js.map
2083
+ //# sourceMappingURL=cli-D303Vj5N.js.map