reeval 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +803 -0
  2. package/README.md +15 -15
  3. package/assets/gateway-models.mjs +65 -31
  4. package/dist/{adapter-C60z3wLq.js → adapter-BdzhdRYo.js} +8 -6
  5. package/dist/adapter-BdzhdRYo.js.map +1 -0
  6. package/dist/{agent-BmsMXZXx.js → agent-Dx-bDmlV.js} +179 -31
  7. package/dist/agent-Dx-bDmlV.js.map +1 -0
  8. package/dist/{agents-DXuFxtGC.js → agents-l0-k7vw3.js} +361 -101
  9. package/dist/agents-l0-k7vw3.js.map +1 -0
  10. package/dist/{app-files-DT5mJyV4.js → app-files-B-CQYB0c.js} +3 -3
  11. package/dist/app-files-B-CQYB0c.js.map +1 -0
  12. package/dist/{assemble-CE4pHKIa.js → assemble-CYMpN8LC.js} +1 -1
  13. package/dist/{assemble-DM2PNU4R.js → assemble-CsZIcJo8.js} +171 -18
  14. package/dist/assemble-CsZIcJo8.js.map +1 -0
  15. package/dist/{attempt-CAdlwlNu.js → attempt-CKfKsqqw.js} +2 -2
  16. package/dist/{attempt-CAdlwlNu.js.map → attempt-CKfKsqqw.js.map} +1 -1
  17. package/dist/{attempt-host-7VviAJrl.js → attempt-host-BwSmNYkX.js} +17 -8
  18. package/dist/attempt-host-BwSmNYkX.js.map +1 -0
  19. package/dist/{boat-BjLqFJIZ.js → boat-CZpAreTB.js} +3 -3
  20. package/dist/{boat-BjLqFJIZ.js.map → boat-CZpAreTB.js.map} +1 -1
  21. package/dist/{boat-DSNgZt-t.js → boat-su1Y7LPd.js} +17 -14
  22. package/dist/boat-su1Y7LPd.js.map +1 -0
  23. package/dist/bridge/index.mjs +6 -1
  24. package/dist/{capture-Ccf0Hmyx.js → capture-BqIAeFyC.js} +55 -29
  25. package/dist/{capture-Ccf0Hmyx.js.map → capture-BqIAeFyC.js.map} +1 -1
  26. package/dist/capture-D0Ln5aCP.js +1 -0
  27. package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
  28. package/dist/cases-CM-MQRQp.js.map +1 -0
  29. package/dist/cases-FvhbM8M5.js +1 -0
  30. package/dist/{cli-DTuLSlmW.js → cli-s_3opn_I.js} +219 -109
  31. package/dist/cli-s_3opn_I.js.map +1 -0
  32. package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-DxzIjmq_.js} +109 -26
  33. package/dist/cli-spec-DxzIjmq_.js.map +1 -0
  34. package/dist/cli.js +2 -2
  35. package/dist/client-C5fIzHQa.js +803 -0
  36. package/dist/client-C5fIzHQa.js.map +1 -0
  37. package/dist/{command-DSL7I-SI.js → command-B5OAIjp7.js} +25 -21
  38. package/dist/command-B5OAIjp7.js.map +1 -0
  39. package/dist/{command-C2C7P5qo.js → command-BJ7KqczP.js} +10 -10
  40. package/dist/{command-C2C7P5qo.js.map → command-BJ7KqczP.js.map} +1 -1
  41. package/dist/{command-D8oMB2fa.js → command-BXLZsEwQ.js} +16 -16
  42. package/dist/{command-D8oMB2fa.js.map → command-BXLZsEwQ.js.map} +1 -1
  43. package/dist/{command-Ff0T-1Hi.js → command-DA_SHQUQ.js} +16 -16
  44. package/dist/{command-Ff0T-1Hi.js.map → command-DA_SHQUQ.js.map} +1 -1
  45. package/dist/{commands-xAOi_VEk.js → commands-BKBamOcX.js} +174 -40
  46. package/dist/commands-BKBamOcX.js.map +1 -0
  47. package/dist/{commands-Bk9NG1s_.js → commands-BPvAOEmb.js} +1 -1
  48. package/dist/{config-SvK8pjEK.js → config-C0URar0t.js} +255 -89
  49. package/dist/config-C0URar0t.js.map +1 -0
  50. package/dist/{context-BdgCao9Q.js → context-Sj58aJEu.js} +2 -2
  51. package/dist/{context-BdgCao9Q.js.map → context-Sj58aJEu.js.map} +1 -1
  52. package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
  53. package/dist/cost-C21gGrju.js.map +1 -0
  54. package/dist/cost-IM80LyGD.js +1 -0
  55. package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-FWgSo-T1.js} +4 -4
  56. package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-FWgSo-T1.js.map} +1 -1
  57. package/dist/db-Bc4L-Wcr.js +2 -0
  58. package/dist/{db-BzOJ4Js4.js → db-BjAWxVAG.js} +142 -248
  59. package/dist/db-BjAWxVAG.js.map +1 -0
  60. package/dist/debug-CIXzB4wK.js.map +1 -1
  61. package/dist/{default-effort-BBDPbJ4D.js → default-effort-CWlhZ6U1.js} +1 -1
  62. package/dist/{default-effort-BDKIxJyC.js → default-effort-Ck2GYIxX.js} +4 -4
  63. package/dist/{default-effort-BDKIxJyC.js.map → default-effort-Ck2GYIxX.js.map} +1 -1
  64. package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
  65. package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
  66. package/dist/{docker-DG_uFgn2.js → docker-CDdK6Ity.js} +41 -16
  67. package/dist/docker-CDdK6Ity.js.map +1 -0
  68. package/dist/{docker-DFrjoNs-.js → docker-D74l67k8.js} +4 -4
  69. package/dist/{docker-DFrjoNs-.js.map → docker-D74l67k8.js.map} +1 -1
  70. package/dist/{docker--mukta3L.js → docker-DMug3l-a.js} +1 -1
  71. package/dist/{doctor-DWK4ttex.js → doctor-eFxyn6at.js} +85 -46
  72. package/dist/doctor-eFxyn6at.js.map +1 -0
  73. package/dist/{environment-cPMCOk-h.js → environment-CJIKz-Ud.js} +49 -11
  74. package/dist/environment-CJIKz-Ud.js.map +1 -0
  75. package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
  76. package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
  77. package/dist/{eval-source-C9b1h8L6.js → eval-source-CXIRJ91v.js} +24 -12
  78. package/dist/eval-source-CXIRJ91v.js.map +1 -0
  79. package/dist/{evaluate-B12Tosvq.js → evaluate-DA4QsrHO.js} +792 -281
  80. package/dist/evaluate-DA4QsrHO.js.map +1 -0
  81. package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DHrU4Wld.d.ts} +2 -2
  82. package/dist/fake-bootstrap-CnwWG3kP.js +15 -0
  83. package/dist/fake-bootstrap-CnwWG3kP.js.map +1 -0
  84. package/dist/{files-BOBdC0uI.js → files-BVVWAS0A.js} +6 -6
  85. package/dist/{files-BOBdC0uI.js.map → files-BVVWAS0A.js.map} +1 -1
  86. package/dist/files-BaqXvB88.js +1 -0
  87. package/dist/{files-tiNWe8Ih.js → files-Bey1VAex.js} +3 -3
  88. package/dist/{files-tiNWe8Ih.js.map → files-Bey1VAex.js.map} +1 -1
  89. package/dist/files-BjOJw4sn.js +1 -0
  90. package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
  91. package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
  92. package/dist/files-CNHhSyb3.js +1 -0
  93. package/dist/files-CW1lChNh.js +1 -0
  94. package/dist/{files-F5EG0WPT.js → files-D6j-XpaK.js} +5 -5
  95. package/dist/{files-F5EG0WPT.js.map → files-D6j-XpaK.js.map} +1 -1
  96. package/dist/{files-s28a6czq.js → files-F44NRCN0.js} +143 -20
  97. package/dist/files-F44NRCN0.js.map +1 -0
  98. package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
  99. package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
  100. package/dist/gateway.d.ts +1 -1
  101. package/dist/{git-BpoqEgDV.js → git-Jqkf9SBa.js} +3 -3
  102. package/dist/{git-BpoqEgDV.js.map → git-Jqkf9SBa.js.map} +1 -1
  103. package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
  104. package/dist/harness-FcTyyPco.js.map +1 -0
  105. package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
  106. package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
  107. package/dist/harness.d.ts +11 -8
  108. package/dist/harness.js +1 -1
  109. package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
  110. package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
  111. package/dist/{images-D2jFze2v.js → images-Dsc4Wi-F.js} +65 -30
  112. package/dist/images-Dsc4Wi-F.js.map +1 -0
  113. package/dist/index.d.ts +46 -7
  114. package/dist/index.js +6 -6
  115. package/dist/init-C0oUG3jh.js +1 -0
  116. package/dist/{init-Bi1Q3lRi.js → init-QB5ecPDA.js} +33 -14
  117. package/dist/init-QB5ecPDA.js.map +1 -0
  118. package/dist/{ink-list-BpfidQ1h.js → ink-list-mCJypk3d.js} +2 -2
  119. package/dist/{ink-list-BpfidQ1h.js.map → ink-list-mCJypk3d.js.map} +1 -1
  120. package/dist/{invocations-CxGNOlh0.js → invocations-BSr4lRzI.js} +5 -5
  121. package/dist/{invocations-CxGNOlh0.js.map → invocations-BSr4lRzI.js.map} +1 -1
  122. package/dist/{judge-COMXT1Hx.js → judge-eGH39PaP.js} +12 -10
  123. package/dist/{judge-COMXT1Hx.js.map → judge-eGH39PaP.js.map} +1 -1
  124. package/dist/judge.d.ts +3 -1
  125. package/dist/judge.js +1 -1
  126. package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
  127. package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
  128. package/dist/{layout-Cm85tQYk.js → layout-DfQa4mJs.js} +14 -11
  129. package/dist/layout-DfQa4mJs.js.map +1 -0
  130. package/dist/{list-DuThsE28.js → list-Dh6T0kdr.js} +23 -12
  131. package/dist/{list-DuThsE28.js.map → list-Dh6T0kdr.js.map} +1 -1
  132. package/dist/live-DwjWJTdn.js +1 -0
  133. package/dist/{live-CgYbO4Te.js → live-xTZs0c61.js} +2 -2
  134. package/dist/{live-CgYbO4Te.js.map → live-xTZs0c61.js.map} +1 -1
  135. package/dist/{live-DL0se4Zd.js → live-y1p3oNXH.js} +9 -9
  136. package/dist/{live-DL0se4Zd.js.map → live-y1p3oNXH.js.map} +1 -1
  137. package/dist/{model-CwwkrCb7.js → model-3pZZzloU.js} +7 -5
  138. package/dist/model-3pZZzloU.js.map +1 -0
  139. package/dist/{model-CkNVAxob.js → model-DEjwetkJ.js} +1 -1
  140. package/dist/{model-V3mMXp6x.js → model-DNpKiHqO.js} +6 -6
  141. package/dist/{model-V3mMXp6x.js.map → model-DNpKiHqO.js.map} +1 -1
  142. package/dist/model-calls-B0BoQfJ4.js +95 -0
  143. package/dist/model-calls-B0BoQfJ4.js.map +1 -0
  144. package/dist/{model-cgt5uvu8.js → model-jRKSQBg6.js} +5 -5
  145. package/dist/{model-cgt5uvu8.js.map → model-jRKSQBg6.js.map} +1 -1
  146. package/dist/model-nfewGTzB.js +1 -0
  147. package/dist/{node-D7pAU--0.js → node-DM-mLGdX.js} +7 -7
  148. package/dist/{node-D7pAU--0.js.map → node-DM-mLGdX.js.map} +1 -1
  149. package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
  150. package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
  151. package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-C0aRo633.js} +2 -2
  152. package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-C0aRo633.js.map} +1 -1
  153. package/dist/options-DpsJ_5CU.js +1 -0
  154. package/dist/{patch-DAi1nCuH.js → patch-Bx-qp8Li.js} +2 -2
  155. package/dist/{patch-DAi1nCuH.js.map → patch-Bx-qp8Li.js.map} +1 -1
  156. package/dist/pi-providers-HygmL1y-.js +552 -0
  157. package/dist/pi-providers-HygmL1y-.js.map +1 -0
  158. package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
  159. package/dist/plain-Ctn8oUdp.js.map +1 -0
  160. package/dist/{plan-CsPilzpk.js → plan-ChO0A-xn.js} +63 -12
  161. package/dist/plan-ChO0A-xn.js.map +1 -0
  162. package/dist/{plugins-BfnhazOv.js → plugins-BioMWncI.js} +636 -9
  163. package/dist/plugins-BioMWncI.js.map +1 -0
  164. package/dist/{preflight-D1C-iH0U.js → preflight-BUjk_6wT.js} +1 -1
  165. package/dist/{preflight-GkpgrBxP.js → preflight-CJSGzMzc.js} +8 -9
  166. package/dist/preflight-CJSGzMzc.js.map +1 -0
  167. package/dist/preflight-DD3fI4Hz.js +1 -0
  168. package/dist/{preflight-BixaE4BA.js → preflight-qFCXmxha.js} +4 -4
  169. package/dist/{preflight-BixaE4BA.js.map → preflight-qFCXmxha.js.map} +1 -1
  170. package/dist/{prepare-DQEDXoDp.js → prepare-D7oSSFNm.js} +9 -9
  171. package/dist/{prepare-DQEDXoDp.js.map → prepare-D7oSSFNm.js.map} +1 -1
  172. package/dist/{prepare-C--rM_wO.js → prepare-xHU2BpN-.js} +2 -2
  173. package/dist/prices-B1vm_lPk.js +1 -0
  174. package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
  175. package/dist/prices-BrY_ihLs.js.map +1 -0
  176. package/dist/{provenance-BXyYwUC8.js → provenance-B-flBAzR.js} +5 -5
  177. package/dist/{provenance-BXyYwUC8.js.map → provenance-B-flBAzR.js.map} +1 -1
  178. package/dist/provider-DHT20Gl8.js +1 -0
  179. package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
  180. package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
  181. package/dist/provider-stop-CxzWW7Ft.js +69 -0
  182. package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
  183. package/dist/{providers-1p4HKYIp.js → providers-B6StVyfG.js} +215 -48
  184. package/dist/providers-B6StVyfG.js.map +1 -0
  185. package/dist/providers-Dljf6al6.js +1 -0
  186. package/dist/query-BAc2tQZj.js +1 -0
  187. package/dist/{query-DhhU9ADI.js → query-CkLmJv7R.js} +2 -2
  188. package/dist/{query-DhhU9ADI.js.map → query-CkLmJv7R.js.map} +1 -1
  189. package/dist/{questions-CMAq4pr8.js → questions-CJIh2FSX.js} +7 -7
  190. package/dist/{questions-CMAq4pr8.js.map → questions-CJIh2FSX.js.map} +1 -1
  191. package/dist/{record-6IUzwP9K.js → record-nVVujCW7.js} +2 -2
  192. package/dist/{record-6IUzwP9K.js.map → record-nVVujCW7.js.map} +1 -1
  193. package/dist/{redact-BhjtnjrU.js → redact-DL2BTykw.js} +17 -2
  194. package/dist/redact-DL2BTykw.js.map +1 -0
  195. package/dist/{remarks-BNABzemt.js → remarks-B3wjKscW.js} +46 -11
  196. package/dist/remarks-B3wjKscW.js.map +1 -0
  197. package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
  198. package/dist/remarks-BpdM_2RC.js.map +1 -0
  199. package/dist/remarks-CleoDlYw.js +1 -0
  200. package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
  201. package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
  202. package/dist/{remote-U7fNt0UP.js → remote-C88amFRN.js} +6 -6
  203. package/dist/{remote-U7fNt0UP.js.map → remote-C88amFRN.js.map} +1 -1
  204. package/dist/resolve-Bul8lHaa.js +205 -0
  205. package/dist/resolve-Bul8lHaa.js.map +1 -0
  206. package/dist/resolve-DW92wpIZ.js +1 -0
  207. package/dist/{resolve-Cx3a9UQ2.js → resolve-G576PiFJ.js} +14 -11
  208. package/dist/resolve-G576PiFJ.js.map +1 -0
  209. package/dist/{results-CQKXmHym.js → results-zVHIW6xw.js} +9 -22
  210. package/dist/results-zVHIW6xw.js.map +1 -0
  211. package/dist/{resume-POpZa1ge.js → resume-D_PcUXfG.js} +94 -16
  212. package/dist/resume-D_PcUXfG.js.map +1 -0
  213. package/dist/{retry-CmG809Mp.js → retry-BvTzCxdc.js} +4 -4
  214. package/dist/{retry-CmG809Mp.js.map → retry-BvTzCxdc.js.map} +1 -1
  215. package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
  216. package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
  217. package/dist/{vercel-CGlSBv63.js → routes-B0F2R-K7.js} +91 -56
  218. package/dist/routes-B0F2R-K7.js.map +1 -0
  219. package/dist/{runtime-Xm9oeC7o.js → runtime-B3r_pAq6.js} +4 -4
  220. package/dist/{runtime-Xm9oeC7o.js.map → runtime-B3r_pAq6.js.map} +1 -1
  221. package/dist/sandbox/boat.d.ts +1 -1
  222. package/dist/sandbox/boat.js +1 -1
  223. package/dist/{sandbox-DrlL4yqb.js → sandbox-Duh3z1TV.js} +3 -3
  224. package/dist/{sandbox-DrlL4yqb.js.map → sandbox-Duh3z1TV.js.map} +1 -1
  225. package/dist/sandbox.d.ts +1 -1
  226. package/dist/sandbox.js +1 -1
  227. package/dist/server-CGEsvVRG.js +1 -0
  228. package/dist/{server-C00zd6zl.js → server-DUWMRKik.js} +8 -8
  229. package/dist/{server-C00zd6zl.js.map → server-DUWMRKik.js.map} +1 -1
  230. package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
  231. package/dist/shared-DtZwIAfy.js.map +1 -0
  232. package/dist/{source-DDgr48Gn.js → source-CHuN4Nfg.js} +2 -2
  233. package/dist/{source-DDgr48Gn.js.map → source-CHuN4Nfg.js.map} +1 -1
  234. package/dist/source-Wz1Nrfue.js +1 -0
  235. package/dist/src--yl1OypK.js +2 -0
  236. package/dist/{src-Cb2MWozp.js → src-B-HuDqpH.js} +276 -41
  237. package/dist/src-B-HuDqpH.js.map +1 -0
  238. package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
  239. package/dist/src-B5r1d23Y.js.map +1 -0
  240. package/dist/{src-y0nOeRRE.js → src-BHJ4MWvY.js} +494 -112
  241. package/dist/src-BHJ4MWvY.js.map +1 -0
  242. package/dist/{src-Dw79VLa3.js → src-BI0v8qlZ.js} +202 -56
  243. package/dist/src-BI0v8qlZ.js.map +1 -0
  244. package/dist/src-BL-_0QoE.js +1 -0
  245. package/dist/{src-YvblOrBU.js → src-BqNzHHWn.js} +87 -13
  246. package/dist/src-BqNzHHWn.js.map +1 -0
  247. package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
  248. package/dist/src-C-ChdnG6.js.map +1 -0
  249. package/dist/src-C-mOSczH.js +2 -0
  250. package/dist/src-CZaYu6ER.js +3 -0
  251. package/dist/src-CeBhwD8k.js +2 -0
  252. package/dist/src-CkEiiyON.js +2 -0
  253. package/dist/{src-KxS91P7L.js → src-CmA5Yev6.js} +79 -11
  254. package/dist/src-CmA5Yev6.js.map +1 -0
  255. package/dist/src-CwH_nEaw.js +2 -0
  256. package/dist/src-DA95rM6v.js +4 -0
  257. package/dist/{src-DvGns8cz.js → src-DL-EE4DU.js} +299 -17
  258. package/dist/src-DL-EE4DU.js.map +1 -0
  259. package/dist/src-De0KY5S_.js +2 -0
  260. package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
  261. package/dist/src-boB6D-5T.js.map +1 -0
  262. package/dist/state-B3jex5KU.js +1 -0
  263. package/dist/{state-328JpnHA.js → state-DUBRDyMk.js} +21 -21
  264. package/dist/{state-328JpnHA.js.map → state-DUBRDyMk.js.map} +1 -1
  265. package/dist/store/local.d.ts +8 -0
  266. package/dist/store/local.js +16 -0
  267. package/dist/store/local.js.map +1 -0
  268. package/dist/store/s3.d.ts +10 -0
  269. package/dist/store/s3.js +18 -0
  270. package/dist/store/s3.js.map +1 -0
  271. package/dist/{suite-COglUhPd.js → suite-B9J6Afg6.js} +6 -5
  272. package/dist/{suite-COglUhPd.js.map → suite-B9J6Afg6.js.map} +1 -1
  273. package/dist/suite-Bhs9-Z7O.js +2 -0
  274. package/dist/{summary-DDy9DfyJ.js → summary-CSp4cfnr.js} +25 -7
  275. package/dist/{summary-DDy9DfyJ.js.map → summary-CSp4cfnr.js.map} +1 -1
  276. package/dist/sync-CxAY6Dki.js +559 -0
  277. package/dist/sync-CxAY6Dki.js.map +1 -0
  278. package/dist/{tart-dWoJvWHc.js → tart-CcrrtSrp.js} +179 -61
  279. package/dist/tart-CcrrtSrp.js.map +1 -0
  280. package/dist/tart-CzViW3WZ.js +186 -0
  281. package/dist/tart-CzViW3WZ.js.map +1 -0
  282. package/dist/tart-DmVuVsmX.js +1 -0
  283. package/dist/tart-bake-F-6seRC9.js +192 -0
  284. package/dist/tart-bake-F-6seRC9.js.map +1 -0
  285. package/dist/tart-bake-RK7RaT9E.js +1 -0
  286. package/dist/tart-cli-B86Jqo1t.js +89 -0
  287. package/dist/tart-cli-B86Jqo1t.js.map +1 -0
  288. package/dist/tart-cli-ClvRniLQ.js +1 -0
  289. package/dist/tart-hosts-Brii7zHM.js +1 -0
  290. package/dist/tart-hosts-DPyZsg-1.js +313 -0
  291. package/dist/tart-hosts-DPyZsg-1.js.map +1 -0
  292. package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CWSf03w8.js} +64 -6
  293. package/dist/tart-images-CWSf03w8.js.map +1 -0
  294. package/dist/testing.d.ts +36 -2
  295. package/dist/testing.js +4 -4
  296. package/dist/tool-class-CVDdJy0H.js +433 -0
  297. package/dist/tool-class-CVDdJy0H.js.map +1 -0
  298. package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
  299. package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
  300. package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
  301. package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
  302. package/dist/{transcript-CCf07UmU.js → transcript-BP_2hji-.js} +3 -3
  303. package/dist/{transcript-CCf07UmU.js.map → transcript-BP_2hji-.js.map} +1 -1
  304. package/dist/{types-DBwFsGem.d.ts → types-BJ0wh8wC.d.ts} +408 -44
  305. package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
  306. package/dist/types-CbcnJxCy.js.map +1 -0
  307. package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
  308. package/dist/usage-CwIRPPZ9.js.map +1 -0
  309. package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
  310. package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
  311. package/dist/vertex-DXfcXYKn.js +1 -0
  312. package/dist/view/index.d.ts +27 -3
  313. package/dist/view/index.js +8 -8
  314. package/dist/{views-BROacjBv.js → views-DRxXCyIH.js} +6 -6
  315. package/dist/{views-BROacjBv.js.map → views-DRxXCyIH.js.map} +1 -1
  316. package/dist/views-i67V1ly3.js +2 -0
  317. package/dist/{worker-entry-Smb4bNDc.js → worker-entry-C3Q4U63F.js} +12 -9
  318. package/dist/worker-entry-C3Q4U63F.js.map +1 -0
  319. package/dist/workspace-DhUOW8if.js +1 -0
  320. package/dist/{workspace-DNDTEk8v.js → workspace-P72SXLXr.js} +7 -7
  321. package/dist/{workspace-DNDTEk8v.js.map → workspace-P72SXLXr.js.map} +1 -1
  322. package/package.json +12 -1
  323. package/skills/eval-rubric/SKILL.md +10 -0
  324. package/skills/reeval/SKILL.md +370 -0
  325. package/skills/reeval/references/capture.md +48 -0
  326. package/skills/reeval/references/cli.md +162 -0
  327. package/skills/reeval/references/compare.md +180 -0
  328. package/skills/reeval/references/config.md +147 -0
  329. package/skills/reeval/references/environments.md +218 -0
  330. package/skills/reeval/references/evals.md +241 -0
  331. package/skills/reeval/references/mistakes.md +39 -0
  332. package/skills/reeval/references/vocabulary.md +19 -0
  333. package/skills/setup/SKILL.md +219 -0
  334. package/skills/setup/references/templates.md +258 -0
  335. package/dist/adapter-C60z3wLq.js.map +0 -1
  336. package/dist/agent-BmsMXZXx.js.map +0 -1
  337. package/dist/agents-DXuFxtGC.js.map +0 -1
  338. package/dist/app-files-DT5mJyV4.js.map +0 -1
  339. package/dist/assemble-DM2PNU4R.js.map +0 -1
  340. package/dist/attempt-host-7VviAJrl.js.map +0 -1
  341. package/dist/boat-DSNgZt-t.js.map +0 -1
  342. package/dist/capture-BzotD2hs.js +0 -1
  343. package/dist/cases-Dy0mifal.js +0 -1
  344. package/dist/cases-z9E6DCd5.js.map +0 -1
  345. package/dist/cli-DTuLSlmW.js.map +0 -1
  346. package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
  347. package/dist/client-KM4C5r5_.js +0 -387
  348. package/dist/client-KM4C5r5_.js.map +0 -1
  349. package/dist/command-DSL7I-SI.js.map +0 -1
  350. package/dist/commands-xAOi_VEk.js.map +0 -1
  351. package/dist/config-SvK8pjEK.js.map +0 -1
  352. package/dist/cost-BdRgySQl.js.map +0 -1
  353. package/dist/cost-Cr26Alhq.js +0 -1
  354. package/dist/db-BzOJ4Js4.js.map +0 -1
  355. package/dist/db-CQRZ3URX.js +0 -2
  356. package/dist/docker-DG_uFgn2.js.map +0 -1
  357. package/dist/doctor-DWK4ttex.js.map +0 -1
  358. package/dist/environment-cPMCOk-h.js.map +0 -1
  359. package/dist/eval-source-C9b1h8L6.js.map +0 -1
  360. package/dist/evaluate-B12Tosvq.js.map +0 -1
  361. package/dist/files-C1Rk9FYA.js +0 -1
  362. package/dist/files-DMQ9-Vv5.js +0 -1
  363. package/dist/files-Di0xDbCc.js +0 -1
  364. package/dist/files-G5jf8wBp.js +0 -1
  365. package/dist/files-s28a6czq.js.map +0 -1
  366. package/dist/harness-DXv2gnaT.js.map +0 -1
  367. package/dist/images-D2jFze2v.js.map +0 -1
  368. package/dist/init-Bi1Q3lRi.js.map +0 -1
  369. package/dist/init-CUw65o36.js +0 -1
  370. package/dist/kinds-Bh2I35id.js +0 -698
  371. package/dist/kinds-Bh2I35id.js.map +0 -1
  372. package/dist/layout-Cm85tQYk.js.map +0 -1
  373. package/dist/live-DZLmj0iG.js +0 -1
  374. package/dist/matchers-CXLn-1jm.js +0 -148
  375. package/dist/matchers-CXLn-1jm.js.map +0 -1
  376. package/dist/model-Cv4GuTsZ.js +0 -1
  377. package/dist/model-CwwkrCb7.js.map +0 -1
  378. package/dist/options-DPUmk7v_.js +0 -1
  379. package/dist/plain-BYUf_fUV.js.map +0 -1
  380. package/dist/plan-CsPilzpk.js.map +0 -1
  381. package/dist/plugins-BfnhazOv.js.map +0 -1
  382. package/dist/preflight-Bn3ianye.js +0 -1
  383. package/dist/preflight-GkpgrBxP.js.map +0 -1
  384. package/dist/prices-5m3SzMZ3.js.map +0 -1
  385. package/dist/provider-BnUoz4KB.js +0 -1
  386. package/dist/providers-1p4HKYIp.js.map +0 -1
  387. package/dist/providers-B-ijCktf.js +0 -1
  388. package/dist/query-BYhyf0LR.js +0 -1
  389. package/dist/redact-BhjtnjrU.js.map +0 -1
  390. package/dist/remarks-BNABzemt.js.map +0 -1
  391. package/dist/remarks-CAvTSq16.js +0 -1
  392. package/dist/remarks-D9Lcd1fF.js.map +0 -1
  393. package/dist/resolve-C14Pmtej.js +0 -1
  394. package/dist/resolve-Cx3a9UQ2.js.map +0 -1
  395. package/dist/results-CQKXmHym.js.map +0 -1
  396. package/dist/resume-POpZa1ge.js.map +0 -1
  397. package/dist/server-C1UJcpUh.js +0 -1
  398. package/dist/shared-Cfi45Bvy.js.map +0 -1
  399. package/dist/source-DJHmQv_j.js +0 -1
  400. package/dist/src-1dBuXzs4.js +0 -3
  401. package/dist/src-BKpioayC.js +0 -2
  402. package/dist/src-BNQznj4z.js.map +0 -1
  403. package/dist/src-BY1cCaoJ.js +0 -2
  404. package/dist/src-BbZYaQ8b.js +0 -2
  405. package/dist/src-CDFH2vVN.js +0 -2
  406. package/dist/src-CMW5R9Q0.js +0 -1
  407. package/dist/src-CV_7DkPa.js.map +0 -1
  408. package/dist/src-Cb2MWozp.js.map +0 -1
  409. package/dist/src-CrV2WA2i.js +0 -2
  410. package/dist/src-CsFaawcZ.js +0 -2
  411. package/dist/src-DAzaCTcN.js.map +0 -1
  412. package/dist/src-DvGns8cz.js.map +0 -1
  413. package/dist/src-Dw79VLa3.js.map +0 -1
  414. package/dist/src-KxS91P7L.js.map +0 -1
  415. package/dist/src-YvblOrBU.js.map +0 -1
  416. package/dist/src-wKwznPgt.js +0 -2
  417. package/dist/src-y0nOeRRE.js.map +0 -1
  418. package/dist/state-Ca55Gk_O.js +0 -1
  419. package/dist/suite-CKBa6MKt.js +0 -2
  420. package/dist/tart-cli-C-GsbVsQ.js +0 -1
  421. package/dist/tart-cli-CU0nFS-L.js +0 -55
  422. package/dist/tart-cli-CU0nFS-L.js.map +0 -1
  423. package/dist/tart-dWoJvWHc.js.map +0 -1
  424. package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
  425. package/dist/tool-target-DS5d72eH.js +0 -40
  426. package/dist/tool-target-DS5d72eH.js.map +0 -1
  427. package/dist/types-C2mLpph3.js.map +0 -1
  428. package/dist/usage-DJiDnh2D.js.map +0 -1
  429. package/dist/vercel-CGlSBv63.js.map +0 -1
  430. package/dist/vertex-O88TCwgY.js +0 -1
  431. package/dist/views-B0Yt_Q1S.js +0 -2
  432. package/dist/worker-entry-Smb4bNDc.js.map +0 -1
  433. package/dist/workspace-C0dDP75l.js +0 -1
@@ -1,31 +1,33 @@
1
1
  import { n as UsageError } from "./errors-ttMTnIYX.js";
2
- import { g as parseDuration } from "./shared-Cfi45Bvy.js";
2
+ import { g as parseDuration } from "./shared-DtZwIAfy.js";
3
+ import { o as PACKAGE_VERSION } from "./root-BzeoyfGd.js";
4
+ import { a as newRunId, i as fromCwd } from "./ids-B-JKTCP8.js";
5
+ import { h as redactSecrets, y as registerSecret } from "./redact-DL2BTykw.js";
6
+ import { F as cleanupRun, P as capabilitiesOf, V as sandboxCapacity } from "./plugins-BioMWncI.js";
3
7
  import { a as plural, i as formatDuration, n as clip } from "./text-B8-W3Cvn.js";
4
- import { l as sandboxCapacity, n as capabilitiesOf, r as cleanupRun } from "./kinds-Bh2I35id.js";
5
- import { N as workspaceLabel, T as profileSandbox, d as environmentSandbox } from "./environment-cPMCOk-h.js";
6
- import { a as newRunId, i as fromCwd } from "./ids-C_advdJA.js";
7
- import { I as checkLimitHarness, V as harnessId, X as plannedSandboxes, Y as plannedSandboxIds, _t as resolveOutput, a as configPathFor, b as resolveModelGateway, c as findConfig, d as malformedCredentials, dt as ciRunner, f as missingCredentials, ft as directOnCloud, g as selectProfile, h as runJudgeModel, ht as isDirect, i as configLabel, l as hostCapacity, m as resolveConcurrency, p as requireConfig, u as loadConfig, v as gatewayScope } from "./config-SvK8pjEK.js";
8
- import { y as vertexSettingsIn } from "./vertex-Eb2wqahk.js";
9
- import { h as redactSecrets, v as registerSecret } from "./redact-BhjtnjrU.js";
10
- import { l as judgeModelName, m as unusedGatewayKey } from "./judge-COMXT1Hx.js";
8
+ import { y as vertexSettingsIn } from "./vertex-BEWxHoxH.js";
9
+ import { E as profileSandbox, P as workspaceLabel, d as environmentSandbox } from "./environment-CJIKz-Ud.js";
10
+ import { $ as plannedSandboxIds, Et as setupEgress, H as harnessId, L as checkLimitHarness, Ot as resolveOutput, Tt as rootlessPodmanSwitch, _ as withProfileHistory, _t as directOnCloud, a as configPathFor, bt as isDirect, c as findConfig, d as loadConfig, et as plannedSandboxes, f as malformedCredentials, g as selectProfile, gt as ciRunner, h as runJudgeModel, i as configLabel, l as givenConcurrency, m as requireConfig, o as defaultConcurrency, p as missingCredentials, u as hostCapacity, x as resolveModelGateway, y as gatewayScope } from "./config-C0URar0t.js";
11
+ import { l as judgeModelName, m as unusedGatewayKey } from "./judge-eGH39PaP.js";
11
12
  import { n as loadEnvFiles, t as inCI } from "./env-8x1O8I1M.js";
12
- import { o as PACKAGE_VERSION } from "./root-Co0jhOT5.js";
13
- import { r as openRouterFallbacks } from "./openrouter-models-Cw1z8-sq.js";
14
- import { a as traitFilters, o as traitFlags, t as checkTraitKeys } from "./traits-DKnYkqeK.js";
15
- import { k as runOutcome } from "./model-V3mMXp6x.js";
16
- import { u as suiteResult } from "./suite-COglUhPd.js";
17
- import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-Dsr79Qy1.js";
13
+ import { r as localStudio } from "./resolve-Bul8lHaa.js";
14
+ import { r as openRouterFallbacks } from "./openrouter-models-C0aRo633.js";
15
+ import { a as traitFilters, o as traitFlags, t as checkTraitKeys } from "./traits-CCD8XsmI.js";
16
+ import { k as runOutcome } from "./model-DNpKiHqO.js";
17
+ import { u as suiteResult } from "./suite-B9J6Afg6.js";
18
+ import { c as helpText, i as INSTEAD, n as FLAGS, o as commandHelpText, r as GLOBAL, t as COMMANDS } from "./cli-spec-DxzIjmq_.js";
18
19
  import { n as stripAnsi, t as colors } from "./colors-DcXBvAlU.js";
19
- import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-BXyYwUC8.js";
20
+ import { d as storeCredentials, i as isRemote, u as pullStore } from "./sync-CxAY6Dki.js";
21
+ import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-B-flBAzR.js";
20
22
  import { t as budgetMarker } from "./budget-BcxLYY5L.js";
21
- import { S as writeLiveProgress, _ as resumeSettings, b as stoppedRetry, c as liveAttempts, d as prepareResume, f as previousAttempt, g as restoreRetry, h as resolveRetry, l as loadPrevious, m as resolveResume, n as erroredAttempts, o as keptAttempts, r as holdRetry, s as leftOutAttempts, t as completedKeysFor, u as noteRetries, v as savedFilters, x as writeCurrent, y as savedInvocation } from "./resume-POpZa1ge.js";
22
- import { n as preflight } from "./preflight-GkpgrBxP.js";
23
- import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-CxGNOlh0.js";
24
- import { _ as phaseOf, c as pointerLine, d as attemptAxes, f as debugLines, g as outcomeOf, h as offset, i as listLines, m as failureLines, n as SPINNER, p as errorCheck, s as progressText, t as LiveList, u as statusList, v as progressLine, y as tailLines } from "./list-DuThsE28.js";
23
+ import { C as writeCurrent, S as sweepStaging, _ as restoreRetry, b as savedInvocation, c as liveAttempts, d as prepareResume, f as previousAttempt, g as resolveRetry, h as resolveResume, l as loadPrevious, m as removeStoppedStaging, n as erroredAttempts, o as keptAttempts, r as holdRetry, s as leftOutAttempts, t as completedKeysFor, u as noteRetries, v as resumeSettings, w as writeLiveProgress, x as stoppedRetry, y as savedFilters } from "./resume-D_PcUXfG.js";
24
+ import { n as preflight } from "./preflight-CJSGzMzc.js";
25
+ import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-BSr4lRzI.js";
26
+ import { _ as phaseOf, c as pointerLine, d as attemptAxes, f as debugLines, g as outcomeOf, h as offset, i as listLines, m as failureLines, n as SPINNER, p as errorCheck, s as progressText, t as LiveList, u as statusList, v as progressLine, y as tailLines } from "./list-Dh6T0kdr.js";
25
27
  import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
26
28
  import { dirname, join } from "node:path";
27
- import { inspect } from "node:util";
28
29
  import { appendFile, readFile, rm } from "node:fs/promises";
30
+ import { inspect } from "node:util";
29
31
  import { detectAgent } from "std-env";
30
32
  import { Command, Option } from "commander";
31
33
  import stringWidth from "string-width";
@@ -64,9 +66,17 @@ function parseCli(argv) {
64
66
  for (const [subName, sub] of Object.entries(spec.sub ?? {})) {
65
67
  const subcommand = configure(command.command(subName), `reeval ${name} ${subName}`, sub.flags);
66
68
  subcommand.argument("[args...]").action(() => {
67
- const [stray] = Object.keys(valuesOf(command, spec.flags));
69
+ const own = valuesOf(command, spec.flags);
70
+ const stray = Object.keys(own).find((flag) => !sub.flags.includes(flag));
68
71
  if (stray) throw new UsageError(`--${stray} isn't an option of reeval ${name} ${subName}. Run reeval --help.`);
69
- parsed = done(program, subcommand, [name, subName], sub.flags, sub.positionals);
72
+ const result = done(program, subcommand, [name, subName], sub.flags, sub.positionals);
73
+ parsed = {
74
+ ...result,
75
+ values: {
76
+ ...own,
77
+ ...result.values
78
+ }
79
+ };
70
80
  });
71
81
  }
72
82
  }
@@ -148,7 +158,10 @@ function usageError(error, command) {
148
158
  const known = arg.startsWith("--") ? FLAGS[flag] : Object.entries(FLAGS).find(([, s]) => "short" in s && s.short === flag)?.[1];
149
159
  const long = arg.startsWith("--") ? flag : Object.entries(FLAGS).find(([, s]) => "short" in s && s.short === flag)?.[0];
150
160
  if (known && value !== void 0 && known.type === "boolean") return new UsageError(`--${flag} doesn't take a value: pass --${flag} on its own.`);
151
- if (known) return new UsageError(`--${long} isn't an option of ${command}. Run reeval --help.`);
161
+ if (known) {
162
+ const instead = INSTEAD[`${command} --${long}`];
163
+ return new UsageError(`--${long} isn't an option of ${command}. ${instead ? `${instead}.` : "Run reeval --help."}`);
164
+ }
152
165
  const shown = arg.replace(/\r/g, "\\r").replace(/\n/g, "\\n");
153
166
  return new UsageError(`Unknown option '${shown}'. ${suggestion ? `${suggestion} ` : ""}Run reeval --help.`);
154
167
  }
@@ -484,9 +497,9 @@ var PlainRenderer = class {
484
497
  /** On a TTY: the status line is drawn under everything else, and redrawn after each write. */
485
498
  #live;
486
499
  #status = "";
487
- /** Evals and tries collected (not skipped), and the tries that ended. */
488
- #evals = /* @__PURE__ */ new Set();
500
+ /** Tries collected (not skipped), as many as the plan has, and the tries that ended. */
489
501
  #tries = /* @__PURE__ */ new Set();
502
+ #plannedTries = 0;
490
503
  #ended = /* @__PURE__ */ new Set();
491
504
  #list;
492
505
  #lastProgress = 0;
@@ -508,11 +521,15 @@ var PlainRenderer = class {
508
521
  this.#lastWrite = this.#now();
509
522
  this.#blank = line === "\n";
510
523
  }
511
- /** `7 evals · 12/28 tries`: what's collected so far and how many tries ended. */
524
+ /**
525
+ * `7 evals · 12/28 tries`: what the plan runs (or what's collected, when that's more) and how
526
+ * many tries ended.
527
+ */
512
528
  #drawStatus() {
513
529
  if (!this.#live || !this.#tries.size) return;
514
- const evals = `${this.#evals.size} eval${this.#evals.size === 1 ? "" : "s"}`;
515
- const status = this.#c.dim(`${evals} · ${this.#ended.size}/${this.#tries.size} tries`);
530
+ const { total } = this.#list.counts();
531
+ const tries = Math.max(this.#plannedTries, this.#tries.size);
532
+ const status = this.#c.dim(`${plural(total, "eval")} · ${this.#ended.size}/${tries} tries`);
516
533
  if (status === this.#status) return;
517
534
  this.#o.out.write(`${CLEAR}${status}`);
518
535
  this.#status = status;
@@ -563,6 +580,8 @@ var PlainRenderer = class {
563
580
  this.#write("");
564
581
  return;
565
582
  case "plan":
583
+ this.#list.planned(e.header.evals);
584
+ this.#plannedTries = e.header.attempts;
566
585
  for (const line of planLines(c, e, this.#columns)) this.#write(line);
567
586
  if (this.#o.output.githubActions) for (const w of e.sandboxWarnings ?? []) this.#annotate(w);
568
587
  return;
@@ -572,7 +591,6 @@ var PlainRenderer = class {
572
591
  for (const x of e.evals) {
573
592
  if (x.skipped) continue;
574
593
  this.#queued.add(x.key);
575
- this.#evals.add(x.id);
576
594
  this.#tries.add(x.key);
577
595
  }
578
596
  this.#drawStatus();
@@ -919,7 +937,7 @@ var InteractiveRenderer = class {
919
937
  #mount() {
920
938
  if (this.#app || this.#closed) return;
921
939
  const cancel = this.#o.cancel ?? (() => process.kill(process.pid, "SIGINT"));
922
- const load = this.#o.load ?? (() => import("./ink-list-BpfidQ1h.js"));
940
+ const load = this.#o.load ?? (() => import("./ink-list-mCJypk3d.js"));
923
941
  this.#app = load().then((m) => m.mountList({
924
942
  store: this.#store,
925
943
  colors: this.#c,
@@ -977,6 +995,10 @@ var InteractiveRenderer = class {
977
995
  if (e.state === "start") this.#spinner.start(this.#written, e.targets.length);
978
996
  else this.#spinner.reported();
979
997
  return;
998
+ case "plan":
999
+ this.#list.planned(e.header.evals);
1000
+ this.#plain.onEvent(e);
1001
+ return;
980
1002
  case "collected":
981
1003
  this.#spinner.settle();
982
1004
  this.#list.collected(e.file, e.evals);
@@ -1088,7 +1110,18 @@ const HISTORY_COMMANDS = {
1088
1110
  compare: 1,
1089
1111
  checkout: 1,
1090
1112
  query: 1,
1091
- analyze: 1
1113
+ analyze: 1,
1114
+ sync: 1
1115
+ };
1116
+ /** The commands that pull a remote store's new runs into its copy before reading it. */
1117
+ const PULLS = {
1118
+ history: 1,
1119
+ show: 1,
1120
+ compare: 1,
1121
+ checkout: 1,
1122
+ query: 1,
1123
+ analyze: 1,
1124
+ mcp: 1
1092
1125
  };
1093
1126
  const EXPECTED_ERRORS = [
1094
1127
  "CommandError",
@@ -1096,6 +1129,8 @@ const EXPECTED_ERRORS = [
1096
1129
  "HistoryError",
1097
1130
  "StudioBuildError"
1098
1131
  ];
1132
+ /** Errors that exit 3: reeval couldn't reach what the run needs, not a mistake in the command. */
1133
+ const INFRA_ERRORS = ["PreflightInfraError", "StoreError"];
1099
1134
  const EXIT = {
1100
1135
  ok: 0,
1101
1136
  failed: 1,
@@ -1133,6 +1168,10 @@ function quitWithoutRecord(runId, stagingDir) {
1133
1168
  ` Deleting ${fromCwd(stagingDir)} discards the run.`
1134
1169
  ].join("\n");
1135
1170
  }
1171
+ /** Sweeps stale staging, naming each stale run it keeps because it can still be resumed. */
1172
+ async function sweep(root, keep) {
1173
+ for (const runId of await sweepStaging(root, keep)) process.stderr.write(`reeval: run ${runId} stopped over a day ago and can still be resumed: reeval run --resume ${runId} finishes it, reeval clean --staging deletes it.\n`);
1174
+ }
1136
1175
  async function run(argv) {
1137
1176
  const { values, positionals } = parseCli(argv);
1138
1177
  invocation.flags = values;
@@ -1150,7 +1189,7 @@ async function run(argv) {
1150
1189
  return values.help ? EXIT.ok : EXIT.usage;
1151
1190
  }
1152
1191
  if (command === "init") {
1153
- const { initCommand } = await import("./init-CUw65o36.js");
1192
+ const { initCommand } = await import("./init-C0oUG3jh.js");
1154
1193
  return initCommand(values, filters, {
1155
1194
  out: (s) => console.log(s),
1156
1195
  input: process.stdin,
@@ -1159,7 +1198,7 @@ async function run(argv) {
1159
1198
  });
1160
1199
  }
1161
1200
  if (command === "doctor") {
1162
- const { doctorCommand } = await import("./doctor-DWK4ttex.js");
1201
+ const { doctorCommand } = await import("./doctor-eFxyn6at.js");
1163
1202
  return doctorCommand(values, {
1164
1203
  out: (s) => console.log(s),
1165
1204
  color: !!process.stdout.isTTY && !process.env.NO_COLOR,
@@ -1167,7 +1206,7 @@ async function run(argv) {
1167
1206
  });
1168
1207
  }
1169
1208
  if (command === "report") {
1170
- const { reportCommand } = await import("./command-D8oMB2fa.js");
1209
+ const { reportCommand } = await import("./command-BXLZsEwQ.js");
1171
1210
  return reportCommand(values, filters, {
1172
1211
  out: (s) => console.log(s),
1173
1212
  err: (s) => console.error(dim(s)),
@@ -1181,8 +1220,9 @@ async function run(argv) {
1181
1220
  return imageBuildCommand(values, configPath, filters.slice(1));
1182
1221
  }
1183
1222
  if (command === "studio") {
1184
- const config = await loadConfig(configPath, "studio");
1185
- const { studioCommand } = await import("./command-C2C7P5qo.js");
1223
+ const config = await historyConfig(configPath, "studio", values.profile);
1224
+ localStudio(config);
1225
+ const { studioCommand } = await import("./command-BJ7KqczP.js");
1186
1226
  return studioCommand({
1187
1227
  config,
1188
1228
  flags: values,
@@ -1192,8 +1232,8 @@ async function run(argv) {
1192
1232
  }
1193
1233
  if (command === "mcp") {
1194
1234
  if (!values.http) console.log = console.info = console.debug = console.error;
1195
- const config = await loadConfig(configPath, "mcp");
1196
- const { mcpCommand } = await import("./command-Ff0T-1Hi.js");
1235
+ const config = await historyConfig(configPath, "mcp", values.profile, values);
1236
+ const { mcpCommand } = await import("./command-DA_SHQUQ.js");
1197
1237
  return mcpCommand({
1198
1238
  config,
1199
1239
  flags: values,
@@ -1201,8 +1241,8 @@ async function run(argv) {
1201
1241
  });
1202
1242
  }
1203
1243
  if (command in HISTORY_COMMANDS) {
1204
- const config = await loadConfig(configPath, command);
1205
- const commands = await import("./commands-Bk9NG1s_.js");
1244
+ const config = await historyConfig(configPath, command, values.profile, values);
1245
+ const commands = await import("./commands-BPvAOEmb.js");
1206
1246
  const ctx = {
1207
1247
  config,
1208
1248
  flags: values,
@@ -1214,16 +1254,16 @@ async function run(argv) {
1214
1254
  case "show": return commands.showCommand(ctx, filters);
1215
1255
  case "compare": return commands.compareCommand(ctx, filters);
1216
1256
  case "checkout": return commands.checkoutCommand(ctx, filters);
1257
+ case "sync": return commands.syncCommand(ctx, filters);
1217
1258
  case "query": {
1218
- const { queryCommand } = await import("./query-BYhyf0LR.js");
1259
+ const { queryCommand } = await import("./query-BAc2tQZj.js");
1219
1260
  return queryCommand({
1220
1261
  ...ctx,
1221
1262
  err: (s) => console.error(dim(s))
1222
1263
  }, filters);
1223
1264
  }
1224
1265
  case "analyze": {
1225
- loadEnvFiles(configPath);
1226
- const { analyzeCommand } = await import("./command-DSL7I-SI.js");
1266
+ const { analyzeCommand } = await import("./command-B5OAIjp7.js");
1227
1267
  return analyzeCommand({
1228
1268
  ...ctx,
1229
1269
  err: (s) => console.error(dim(s))
@@ -1235,14 +1275,38 @@ async function run(argv) {
1235
1275
  if (command !== "run") throw new UsageError(`Unknown command "${command}". Run reeval --help.`);
1236
1276
  return runCommand(values, configPath, filters);
1237
1277
  }
1278
+ /**
1279
+ * The config a command that reads the history runs on: the profile `--profile`, `REEVAL_PROFILE`
1280
+ * or `defaultProfile` picks sets which store it reads, as it does for `reeval run`. `.env` and
1281
+ * `.env.local` load first, as for `reeval run`: they can hold the store's keys and the profile.
1282
+ */
1283
+ async function historyConfig(configPath, command, flag, values = {}) {
1284
+ loadEnvFiles(configPath);
1285
+ const named = flag ?? (process.env.REEVAL_PROFILE || void 0);
1286
+ const loaded = await loadConfig(configPath, command, named);
1287
+ const config = withProfileHistory(loaded, selectProfile(loaded, flag));
1288
+ if (!config.history.scope) return config;
1289
+ const history = values["all-branches"] ? {
1290
+ ...config.history,
1291
+ scope: "all"
1292
+ } : config.history;
1293
+ if (isRemote(history) && command in PULLS && !values.offline) {
1294
+ const { cachedAt, unreachable } = await pullStore(history);
1295
+ if (unreachable) console.error(yellow(`${history.store.name} is unreachable (${unreachable}); showing the runs pulled at ${cachedAt}`));
1296
+ }
1297
+ return {
1298
+ ...config,
1299
+ history
1300
+ };
1301
+ }
1238
1302
  /** The dry run's prices, fetched only when a planned attempt or the judge has anything to price. */
1239
1303
  async function dryRunPrices(planned, judgeModel) {
1240
- const { needsPrices } = await import("./cost-Cr26Alhq.js");
1304
+ const { needsPrices } = await import("./cost-IM80LyGD.js");
1241
1305
  if (!needsPrices(planned.filter((p) => !p.attempt.plan.skip).map((p) => p.attempt.harness), judgeModel)) return {
1242
1306
  table: null,
1243
1307
  why: "nothing to price"
1244
1308
  };
1245
- const { loadPrices } = await import("./prices-5m3SzMZ3.js");
1309
+ const { loadPrices } = await import("./prices-B1vm_lPk.js");
1246
1310
  return loadPrices();
1247
1311
  }
1248
1312
  /** Asks a yes/no question on the terminal; anything but y or yes is no. */
@@ -1260,7 +1324,7 @@ async function confirm(question) {
1260
1324
  }
1261
1325
  /**
1262
1326
  * `reeval clean`: removes local debug data and snapshots, all of them or all but the newest
1263
- * `--keep` runs'.
1327
+ * `--keep` runs', and with `--staging` every stopped run's staging.
1264
1328
  */
1265
1329
  async function cleanCommand(flags, configPath) {
1266
1330
  const config = await loadConfig(configPath, "clean");
@@ -1269,20 +1333,31 @@ async function cleanCommand(flags, configPath) {
1269
1333
  keep = Number(flags.keep);
1270
1334
  if (!(Number.isInteger(keep) && keep >= 0)) throw new UsageError(`--keep must be a whole number (got ${flags.keep})`);
1271
1335
  }
1272
- const { debugRoot, pruneDebug } = await import("./files-C1Rk9FYA.js");
1336
+ const { debugRoot, pruneDebug } = await import("./files-BjOJw4sn.js");
1273
1337
  const { formatBytes } = await import("./debug-CF4p4Lp_.js");
1274
1338
  const { runs, bytes } = await pruneDebug(config.root, keep);
1275
1339
  const where = fromCwd(debugRoot(config.root)) || ".";
1276
1340
  console.log(runs.length ? `Removed debug data and snapshots for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where} and .reeval/runs` : `No debug data or snapshots to remove in ${where} or .reeval/runs`);
1341
+ if (flags.staging) {
1342
+ const removed = await removeStoppedStaging(config.root);
1343
+ console.log(removed.length ? `Removed the staging of ${removed.length} stopped run${removed.length === 1 ? "" : "s"}: ${removed.join(", ")}` : "No stopped runs in .reeval/staging to remove");
1344
+ }
1277
1345
  if (flags.images) {
1278
- const { dockerAvailable, pruneImages } = await import("./src-CMW5R9Q0.js");
1279
- const { requireDocker } = await import("./providers-B-ijCktf.js");
1346
+ const { dockerAvailable, pruneImages } = await import("./src-BL-_0QoE.js");
1347
+ const { requireDocker } = await import("./providers-Dljf6al6.js");
1280
1348
  const engines = [];
1281
1349
  for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
1282
- if (!engines.length) await requireDocker();
1350
+ const { tartCli } = await import("./tart-cli-ClvRniLQ.js");
1351
+ const tart = await tartCli(["--version"]).then((r) => r.exitCode === 0, () => false);
1352
+ if (!engines.length && !tart) await requireDocker();
1283
1353
  let removed = 0;
1284
1354
  for (const engine of engines) removed += await pruneImages(engine);
1285
- console.log(removed ? `Removed ${removed} image${removed === 1 ? "" : "s"} reeval built that nothing uses` : "No leftover images to remove");
1355
+ if (engines.length) console.log(removed ? `Removed ${removed} image${removed === 1 ? "" : "s"} reeval built that nothing uses` : "No leftover images to remove");
1356
+ if (tart) {
1357
+ const { deleteBakedImages } = await import("./tart-bake-RK7RaT9E.js");
1358
+ const baked = await deleteBakedImages();
1359
+ console.log(baked.length ? `Deleted ${baked.length} Tart image${baked.length === 1 ? "" : "s"} with a harness baked in (reeval image build bakes them again): ${baked.join(", ")}` : "No Tart images with a harness baked in to delete");
1360
+ }
1286
1361
  }
1287
1362
  return EXIT.ok;
1288
1363
  }
@@ -1298,7 +1373,7 @@ function verbosityOf(flags) {
1298
1373
  * image.
1299
1374
  */
1300
1375
  async function imageBuildCommand(flags, configPath, filters) {
1301
- const { buildImages, imagePlanLines, imageTargets, plannedImageUses, pullTartImages, tartPlanLines, tartTargets } = await import("./images-D2jFze2v.js");
1376
+ const { buildImages, imagePlanLines, imageTargets, plannedImageUses, pullTartImages, tartPlanLines, tartTargets } = await import("./images-Dsc4Wi-F.js");
1302
1377
  const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
1303
1378
  ...flags,
1304
1379
  "dry-run": true
@@ -1320,8 +1395,8 @@ async function imageBuildCommand(flags, configPath, filters) {
1320
1395
  return EXIT.ok;
1321
1396
  }
1322
1397
  const { tartTagWarning } = await import("./tart-ref-7Qv3cm3B.js");
1323
- for (const t of vms) {
1324
- const warning = tartTagWarning(t.sandbox.image);
1398
+ for (const image of new Set(vms.map((t) => t.sandbox.image))) {
1399
+ const warning = tartTagWarning(image);
1325
1400
  if (warning) console.log(yellow(`warning: ${warning}`));
1326
1401
  }
1327
1402
  if (flags["dry-run"]) {
@@ -1329,7 +1404,7 @@ async function imageBuildCommand(flags, configPath, filters) {
1329
1404
  if (targets.length) for (const line of await imagePlanLines(targets, root)) console.log(line);
1330
1405
  return EXIT.ok;
1331
1406
  }
1332
- const { PrepareError } = await import("./provider-BnUoz4KB.js");
1407
+ const { PrepareError } = await import("./provider-DHT20Gl8.js");
1333
1408
  const outcomes = [];
1334
1409
  let refused = false;
1335
1410
  const refusal = (id, of, others) => (error) => {
@@ -1345,7 +1420,10 @@ async function imageBuildCommand(flags, configPath, filters) {
1345
1420
  return [];
1346
1421
  };
1347
1422
  try {
1348
- const pulled = vms.length ? await pullTartImages(vms, root, { log: (line) => console.log(line.startsWith("warning: ") ? yellow(line) : dim(line)) }).catch(refusal("tart", vms, targets.length)) : [];
1423
+ const pulled = vms.length ? await pullTartImages(vms, root, {
1424
+ log: (line) => console.log(line.startsWith("warning: ") ? yellow(line) : dim(line)),
1425
+ force: !!flags.force
1426
+ }).catch(refusal("tart", vms, targets.length)) : [];
1349
1427
  for (const p of pulled) outcomes.push(p.ok ? {
1350
1428
  id: p.id,
1351
1429
  evals: p.evals,
@@ -1380,7 +1458,7 @@ async function imageBuildCommand(flags, configPath, filters) {
1380
1458
  if (!(error instanceof PrepareError)) throw error;
1381
1459
  if (error.failure.usage) throw new UsageError(error.message);
1382
1460
  if (error.failure.cause === "enospc") invocation.machineState = true;
1383
- const { PreflightInfraError } = await import("./preflight-Bn3ianye.js");
1461
+ const { PreflightInfraError } = await import("./preflight-DD3fI4Hz.js");
1384
1462
  throw new PreflightInfraError(error.message);
1385
1463
  }
1386
1464
  for (const o of outcomes) {
@@ -1414,7 +1492,7 @@ async function runCommand(flags, configPath, given) {
1414
1492
  const outcome = runOutcome(record);
1415
1493
  if (flags.json) {
1416
1494
  console.error(nothing);
1417
- const { suiteResult } = await import("./suite-CKBa6MKt.js");
1495
+ const { suiteResult } = await import("./suite-Bhs9-Z7O.js");
1418
1496
  process.stdout.write(`${JSON.stringify({
1419
1497
  ...suiteResult(record),
1420
1498
  outcome
@@ -1432,21 +1510,21 @@ async function runCommand(flags, configPath, given) {
1432
1510
  ...resumed?.profileOf ? { profileOf: resumed.profileOf } : {}
1433
1511
  });
1434
1512
  const explicit = runFlags.concurrency ?? (process.env.REEVAL_CONCURRENCY || void 0);
1513
+ const ctx = await startRun(setup);
1435
1514
  const startedWith = resumed ? resumed.saved ?? void 0 : {
1436
1515
  ...setup.runs === void 0 ? {} : { runs: setup.runs },
1437
1516
  ...setup.profile ? { profile: setup.profile.name } : {},
1438
- ...explicit === void 0 ? {} : { concurrency: setup.concurrency },
1517
+ ...explicit === void 0 ? {} : { concurrency: ctx.concurrency },
1439
1518
  filters: savedFilters(setup.config.root, filters),
1440
1519
  tags: runFlags.tag ?? [],
1441
1520
  ...runFlags.trait?.length ? { traits: runFlags.trait } : {}
1442
1521
  };
1443
- const ctx = await startRun(setup);
1444
1522
  invocation.runId = ctx.runId;
1445
1523
  const githubOut = process.env.GITHUB_OUTPUT;
1446
1524
  if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
1447
1525
  const stepStart = process.env.GITHUB_STEP_SUMMARY;
1448
1526
  if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
1449
- const { engineVersions } = await import("./node-D7pAU--0.js");
1527
+ const { engineVersions } = await import("./node-DM-mLGdX.js");
1450
1528
  const terminal = setup.json && !setup.dryRun ? process.stderr : process.stdout;
1451
1529
  const renderer = createRenderer({
1452
1530
  out: setup.json && setup.dryRun ? { write: () => true } : terminal,
@@ -1486,7 +1564,7 @@ async function runCommand(flags, configPath, given) {
1486
1564
  return EXIT.ok;
1487
1565
  }
1488
1566
  if (ctx.retry && ctx.previous && retried.size) {
1489
- const { retryEnvironmentChanges } = await import("./retry-CmG809Mp.js");
1567
+ const { retryEnvironmentChanges } = await import("./retry-BvTzCxdc.js");
1490
1568
  const changes = retryEnvironmentChanges(ctx.previous, ctx.stagingDir, retried);
1491
1569
  if (changes.length) {
1492
1570
  questions?.cancel();
@@ -1505,8 +1583,8 @@ async function runCommand(flags, configPath, given) {
1505
1583
  }
1506
1584
  finishing = true;
1507
1585
  live?.phase("finishing");
1508
- const { lockfiles } = await import("./state-Ca55Gk_O.js");
1509
- const { writeScratchRun } = await import("./results-CQKXmHym.js");
1586
+ const { lockfiles } = await import("./state-B3jex5KU.js");
1587
+ const { writeScratchRun } = await import("./results-zVHIW6xw.js");
1510
1588
  const run = await writeScratchRun({
1511
1589
  ctx,
1512
1590
  author: ctx.author,
@@ -1539,8 +1617,8 @@ async function runCommand(flags, configPath, given) {
1539
1617
  ...record,
1540
1618
  status: "complete"
1541
1619
  }) : outcome;
1542
- const code = stopped ? stopCode : ran === "fail" ? EXIT.failed : ran === "infra_error" ? EXIT.infra : EXIT.ok;
1543
- const { pruneDebug } = await import("./files-C1Rk9FYA.js");
1620
+ let code = stopped ? stopCode : ran === "fail" ? EXIT.failed : ran === "infra_error" ? EXIT.infra : EXIT.ok;
1621
+ const { pruneDebug } = await import("./files-BjOJw4sn.js");
1544
1622
  await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
1545
1623
  const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
1546
1624
  renderer.onEvent({
@@ -1554,7 +1632,7 @@ async function runCommand(flags, configPath, given) {
1554
1632
  });
1555
1633
  renderer.close();
1556
1634
  if (retried.size) {
1557
- const { retryCounts, retryLine } = await import("./retry-CmG809Mp.js");
1635
+ const { retryCounts, retryLine } = await import("./retry-BvTzCxdc.js");
1558
1636
  const line = retryLine(retryCounts(record, retried, ctx.previous));
1559
1637
  if (setup.json) console.error(line);
1560
1638
  else console.log(line);
@@ -1563,15 +1641,22 @@ async function runCommand(flags, configPath, given) {
1563
1641
  if (stepSummary && run.artifactDir) await appendFile(stepSummary, await readFile(summary)).catch(() => {});
1564
1642
  const { config } = ctx;
1565
1643
  const mode = setup.record;
1566
- const { recordFile } = await import("./files-G5jf8wBp.js");
1644
+ const { readRecord, recordFile } = await import("./files-BaqXvB88.js");
1567
1645
  const rerecord = (setup.retry !== void 0 || setup.resume !== void 0) && record.status === "complete" && existsSync(recordFile(config.history.dir, record.runId));
1568
1646
  const autoRecord = flags.record || mode === "always" || mode === "ci" && inCI() || rerecord;
1569
1647
  if (autoRecord && record.status !== "interrupted") {
1570
- const { record: recordCmd } = await import("./commands-Bk9NG1s_.js");
1648
+ const { record: recordCmd } = await import("./commands-BPvAOEmb.js");
1571
1649
  const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
1650
+ const level = rerecord ? { artifacts: (await readRecord(recordFile(config.history.dir, record.runId))).artifacts } : {};
1572
1651
  const line = await recordCmd(config, record.runId, {
1573
1652
  ...flags,
1574
- ...allow
1653
+ ...allow,
1654
+ ...level
1655
+ }).catch((error) => {
1656
+ if (error.name !== "StoreError") throw error;
1657
+ if (code !== EXIT.interrupted && code !== EXIT.terminated) code = EXIT.infra;
1658
+ invocation.machineState = true;
1659
+ return red(`reeval: ${redactSecrets(error.message)}`);
1575
1660
  });
1576
1661
  if (setup.json) console.error(line);
1577
1662
  else console.log(line);
@@ -1581,7 +1666,7 @@ async function runCommand(flags, configPath, given) {
1581
1666
  else console.log(line);
1582
1667
  }
1583
1668
  if (setup.json) {
1584
- const { suiteResult } = await import("./suite-CKBa6MKt.js");
1669
+ const { suiteResult } = await import("./suite-Bhs9-Z7O.js");
1585
1670
  process.stdout.write(`${JSON.stringify({
1586
1671
  ...suiteResult(record),
1587
1672
  outcome: ran
@@ -1663,18 +1748,18 @@ async function prepareRun(flags, configPath, { noProfile = false, profileOf } =
1663
1748
  const path = requireConfig(configPath);
1664
1749
  loadEnvFiles(path);
1665
1750
  const named = noProfile ? void 0 : flags.profile ?? (process.env.REEVAL_PROFILE || void 0);
1666
- const config = await loadConfig(path, "run", named);
1667
- if (profileOf && flags.profile && !config.profiles[flags.profile]) throw new UsageError(`run ${profileOf} was started with profile ${flags.profile}, which ${config.file} no longer defines. Put it back to resume the run, or start a new run without --resume.`);
1668
- const profile = noProfile ? null : selectProfile(config, flags.profile);
1751
+ const loaded = await loadConfig(path, "run", named);
1752
+ if (profileOf && flags.profile && !loaded.profiles[flags.profile]) throw new UsageError(`run ${profileOf} was started with profile ${flags.profile}, which ${loaded.file} no longer defines. Put it back to resume the run, or start a new run without --resume.`);
1753
+ const profile = noProfile ? null : selectProfile(loaded, flags.profile);
1754
+ const config = withProfileHistory(loaded, profile);
1755
+ if ((flags.record || (profile?.record ?? config.record) === "always" || (profile?.record ?? config.record) === "ci" && inCI()) && !flags["dry-run"]) storeCredentials(config.history);
1669
1756
  let runs;
1670
1757
  if (flags.runs !== void 0) {
1671
1758
  runs = Number(flags.runs);
1672
1759
  if (!(Number.isInteger(runs) && runs >= 1)) throw new UsageError(`--runs must be a whole number of at least 1 (got ${flags.runs})`);
1673
1760
  }
1674
1761
  const sandboxes = [...(config.environments.length ? config.environments : [void 0]).map((e) => environmentSandbox(e, config.sandbox)), ...profile ? [profileSandbox(config, profile.defaults)] : []];
1675
- const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
1676
- return await sandboxCapacity(sandboxes) ?? hostCapacity();
1677
- });
1762
+ const concurrency = givenConcurrency(config, flags.concurrency);
1678
1763
  return {
1679
1764
  output,
1680
1765
  config,
@@ -1682,7 +1767,14 @@ async function prepareRun(flags, configPath, { noProfile = false, profileOf } =
1682
1767
  runs,
1683
1768
  tags: [...flags.tag ?? [], ...profile?.tags ?? []],
1684
1769
  traits: traitFlags(flags.trait),
1685
- concurrency,
1770
+ sandboxes,
1771
+ /**
1772
+ * The run's concurrency. Without one given, it's sized from where the first sandbox that
1773
+ * runs off the host runs (the Docker or Podman VM, the Boat account) when it answers, as a
1774
+ * real run is, so --dry-run shows the same; a resume reads it once the stopped run's
1775
+ * sandboxes are gone, since they count against it.
1776
+ */
1777
+ sizeRun: async () => concurrency ?? defaultConcurrency(await sandboxCapacity(sandboxes) ?? hostCapacity()),
1686
1778
  budget: profile?.budget ?? config.budget,
1687
1779
  record: profile?.record ?? config.record,
1688
1780
  dryRun: !!flags["dry-run"],
@@ -1701,14 +1793,14 @@ async function prepareRun(flags, configPath, { noProfile = false, profileOf } =
1701
1793
  /** What a retry, or a resume that finishes one, does to the run's errors, for its refusals. */
1702
1794
  const retryVerb = (retry, runId) => retry.resume ? `--resume finishes run ${runId}'s --retry-errors, which runs its errors again` : "--retry-errors runs its errors again";
1703
1795
  /** The run's id and the state it starts from: commit, uncommitted changes, author. */
1704
- async function startRun({ config, profile, runs, tags, traits, concurrency, verbosity, configFile, resume, retry, dryRun }) {
1796
+ async function startRun({ config, profile, runs, tags, traits, sandboxes, sizeRun, verbosity, configFile, resume, retry, dryRun }) {
1705
1797
  const startedAt = /* @__PURE__ */ new Date();
1706
1798
  const git = await gitInfo(config.root, [config.history.dir]);
1707
- const { worktreePatch } = await import("./state-Ca55Gk_O.js");
1799
+ const { worktreePatch } = await import("./state-B3jex5KU.js");
1708
1800
  const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
1709
1801
  const who = await author(config.root);
1710
- const { stagingDir, sweepStaging } = await import("./results-CQKXmHym.js");
1711
- const { scratchDir } = await import("./files-G5jf8wBp.js");
1802
+ const { stagingDir } = await import("./results-zVHIW6xw.js");
1803
+ const { scratchDir } = await import("./files-BaqXvB88.js");
1712
1804
  const scratch = scratchDir(config.root);
1713
1805
  if (resume !== void 0 || retry !== void 0) {
1714
1806
  const target = await continuedRun(config.root, {
@@ -1718,15 +1810,18 @@ async function startRun({ config, profile, runs, tags, traits, concurrency, verb
1718
1810
  if (!dryRun && retry === void 0) await prepareResume(target, config.root);
1719
1811
  else if (!dryRun) holdRetry(target, config.root);
1720
1812
  const stopped = retry === void 0 ? await stoppedRetry(config.root, target) : null;
1721
- await sweepStaging(config.root, target.runId);
1813
+ if (!dryRun) await sweep(config.root, target.runId);
1722
1814
  const aside = dryRun ? {
1723
1815
  record: target.previous,
1724
1816
  artifacts: target.previousArtifacts
1725
1817
  } : await loadPrevious(stagingDir(config.root, target.runId));
1726
- if (!dryRun) try {
1727
- await cleanupRun(target.runId);
1728
- } catch {}
1729
- const { unfoldCases } = await import("./cases-Dy0mifal.js");
1818
+ if (!dryRun) {
1819
+ await cleanupRun(target.runId).catch(() => {});
1820
+ const boats = sandboxes.filter((s) => s.kind === "boat");
1821
+ if (boats.length) await cleanupRun(target.runId, boats).catch(() => {});
1822
+ }
1823
+ const concurrency = await sizeRun();
1824
+ const { unfoldCases } = await import("./cases-FvhbM8M5.js");
1730
1825
  const previous = aside.record ?? target.previous;
1731
1826
  return {
1732
1827
  runId: target.runId,
@@ -1760,7 +1855,8 @@ async function startRun({ config, profile, runs, tags, traits, concurrency, verb
1760
1855
  const stamps = new Set([scratch, config.history.dir].flatMap((dir) => existsSync(dir) ? readdirSync(dir).map((name) => name.slice(0, 16)) : []));
1761
1856
  let runId = newRunId(startedAt, git?.sha ?? null, who);
1762
1857
  for (let n = 1; stamps.has(runId.slice(0, 16)); n++) runId = newRunId(new Date(startedAt.getTime() + n * 1e3), git?.sha ?? null, who);
1763
- await sweepStaging(config.root);
1858
+ if (!dryRun) await sweep(config.root);
1859
+ const concurrency = await sizeRun();
1764
1860
  return {
1765
1861
  runId,
1766
1862
  stagingDir: stagingDir(config.root, runId),
@@ -1819,12 +1915,12 @@ function namesDeclaredSecret(evals) {
1819
1915
  */
1820
1916
  async function planEfforts(planned) {
1821
1917
  const live = planned.filter((p) => !p.attempt.plan.skip);
1822
- const { effortNotes, withDefaultEffort } = await import("./default-effort-BBDPbJ4D.js");
1918
+ const { effortNotes, withDefaultEffort } = await import("./default-effort-CWlhZ6U1.js");
1823
1919
  let cursor;
1824
1920
  let unchecked = "CURSOR_API_KEY isn't set";
1825
1921
  const key = process.env.CURSOR_API_KEY;
1826
1922
  if (key && live.some((p) => p.attempt.harness.kind === "cursor")) {
1827
- const { checkCursorPlan } = await import("./cursor-catalog-DEzan4IT.js");
1923
+ const { checkCursorPlan } = await import("./cursor-catalog-FWgSo-T1.js");
1828
1924
  const checked = await checkCursorPlan(key, live);
1829
1925
  if (checked instanceof Error) unchecked = `its request failed: ${checked.message}`;
1830
1926
  else {
@@ -1860,8 +1956,8 @@ async function executeRun(ctx, setup, filters, renderer, started) {
1860
1956
  configFile,
1861
1957
  completed: []
1862
1958
  }));
1863
- const { runEngine } = await import("./node-D7pAU--0.js");
1864
- const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-CsPilzpk.js");
1959
+ const { runEngine } = await import("./node-DM-mLGdX.js");
1960
+ const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable, runHeader } = await import("./plan-ChO0A-xn.js");
1865
1961
  let plan;
1866
1962
  let sandboxVersions = { images: [] };
1867
1963
  let preflightStop;
@@ -1906,9 +2002,9 @@ async function executeRun(ctx, setup, filters, renderer, started) {
1906
2002
  if (ctx.retry && ctx.previous) {
1907
2003
  const ranOn = ctx.previous.stack?.reeval;
1908
2004
  if (ranOn && ranOn !== version) throw new UsageError(`run ${runId} ran on reeval ${ranOn}, and this is ${version}. ${retryVerb(ctx.retry, runId)} only on the reeval they ran on: install reeval ${ranOn}, or start a new run with reeval run.`);
1909
- const { retryChanges } = await import("./retry-CmG809Mp.js");
1910
- const { judgePart } = await import("./assemble-CE4pHKIa.js");
1911
- const { engineVersions } = await import("./node-D7pAU--0.js");
2005
+ const { retryChanges } = await import("./retry-BvTzCxdc.js");
2006
+ const { judgePart } = await import("./assemble-CYMpN8LC.js");
2007
+ const { engineVersions } = await import("./node-DM-mLGdX.js");
1912
2008
  const judged = judgePart(config, profile, version, engineVersions);
1913
2009
  const runs = new Set(planned.map((p) => p.key).filter((key) => !completed.has(key)));
1914
2010
  const changes = await retryChanges(ctx.previous, planned, runs, config.root, judged);
@@ -1921,7 +2017,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
1921
2017
  }
1922
2018
  }
1923
2019
  retried = continuing && !setup.dryRun ? noteRetries(planned, stagingDir, ctx.previous, true).errored : errored;
1924
- const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-CkNVAxob.js") : void 0;
2020
+ const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-DEjwetkJ.js") : void 0;
1925
2021
  const judge = runJudgeModel(config, profile);
1926
2022
  const judgeModel = judge === void 0 ? null : judgeModelName(judge);
1927
2023
  plan = await buildPlan({
@@ -1938,8 +2034,11 @@ async function executeRun(ctx, setup, filters, renderer, started) {
1938
2034
  });
1939
2035
  plan.runId = runId;
1940
2036
  const cloud = ciRunner(process.env);
1941
- const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && capabilitiesOf(p.attempt.sandbox).isolated).filter((p) => isDirect(p.attempt.egress.policy)).map((p) => p.id))] : [];
2037
+ const direct = cloud ? [...new Set(skipped.filter((p) => !p.attempt.plan.skip && capabilitiesOf(p.attempt.sandbox).isolated).filter((p) => isDirect(p.attempt.egress.policy) || isDirect(setupEgress(p.attempt.egress).policy)).map((p) => p.id))] : [];
1942
2038
  if (cloud && direct.length) plan.warnings.push(`${directOnCloud(cloud)}. Under it: ${direct.join(", ")}. Keep 'allow-all' to the evals that need raw TCP or UDP; the default, 'proxy', sends everything through the gateway`);
2039
+ const switching = concurrency > 1 ? [...new Set(skipped.filter(({ attempt }) => !attempt.plan.skip && !!attempt.egress.setup && attempt.sandbox.kind === "docker" && attempt.sandbox.engine === "podman").map((p) => p.id))] : [];
2040
+ const { podmanRootless } = await import("./providers-Dljf6al6.js");
2041
+ if (switching.length && await podmanRootless()) plan.warnings.push(rootlessPodmanSwitch(switching));
1943
2042
  plan.warnings.push(...started.notes);
1944
2043
  if (started.narrowed) {
1945
2044
  leftOut = leftOutAttempts(planned, stagingDir, ctx.previous, await liveAttempts(stagingDir), new Set(excluded.map((e) => e.id)));
@@ -1954,7 +2053,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
1954
2053
  run.completed = [...completed];
1955
2054
  const done = leftOut.filter((l) => l.done).map((l) => l.key);
1956
2055
  kept = keptAttempts([...completed, ...done], stagingDir, ctx.previous);
1957
- const { imageStages, imageTargets } = await import("./images-D2jFze2v.js");
2056
+ const { imageStages, imageTargets } = await import("./images-Dsc4Wi-F.js");
1958
2057
  const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
1959
2058
  id: p.attempt.family?.id ?? p.id,
1960
2059
  harness: p.attempt.harness,
@@ -1973,13 +2072,13 @@ async function executeRun(ctx, setup, filters, renderer, started) {
1973
2072
  return false;
1974
2073
  }
1975
2074
  const elsewhere = setup.configElsewhere ? configFile : void 0;
1976
- const { scratchDir } = await import("./files-G5jf8wBp.js");
2075
+ const { scratchDir } = await import("./files-BaqXvB88.js");
1977
2076
  const lines = [];
1978
2077
  if (setup.dryRun) {
1979
2078
  const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
1980
2079
  lines.push(...planTable(plan, todos));
1981
- const { imageStatusLines } = await import("./images-D2jFze2v.js");
1982
- const { dockerAvailable, engineSpec } = await import("./src-CMW5R9Q0.js");
2080
+ const { imageStatusLines } = await import("./images-Dsc4Wi-F.js");
2081
+ const { dockerAvailable, engineSpec } = await import("./src-BL-_0QoE.js");
1983
2082
  const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
1984
2083
  const down = [];
1985
2084
  for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
@@ -1988,7 +2087,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
1988
2087
  const names = down.map((e) => engineSpec(e).name).join(" and ");
1989
2088
  lines.push(`images not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
1990
2089
  }
1991
- const { costLine } = await import("./cost-Cr26Alhq.js");
2090
+ const { costLine } = await import("./cost-IM80LyGD.js");
1992
2091
  const cost = plan.cost && costLine(plan.cost);
1993
2092
  if (cost) lines.push(cost);
1994
2093
  }
@@ -2011,6 +2110,15 @@ async function executeRun(ctx, setup, filters, renderer, started) {
2011
2110
  sandboxWarnings: plan.sandboxWarnings
2012
2111
  });
2013
2112
  if (setup.dryRun) return false;
2113
+ if (continuing) {
2114
+ const away = /* @__PURE__ */ new Map();
2115
+ for (const { attempt } of planned) {
2116
+ const { sandbox } = attempt;
2117
+ if (sandbox.kind === "tart" && sandbox.hosts) away.set(JSON.stringify(sandbox.hosts), sandbox);
2118
+ if (sandbox.kind === "boat") away.set(JSON.stringify(sandbox), sandbox);
2119
+ }
2120
+ if (away.size) await cleanupRun(runId, [...away.values()]).catch(() => {});
2121
+ }
2014
2122
  checkAttemptBudget(plan, setup.force, config.file);
2015
2123
  const questionModels = /* @__PURE__ */ new Set();
2016
2124
  const asked = [];
@@ -2063,7 +2171,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
2063
2171
  });
2064
2172
  if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
2065
2173
  if (asked.length) {
2066
- const { questionRunner } = await import("./questions-CMAq4pr8.js");
2174
+ const { questionRunner } = await import("./questions-CJIh2FSX.js");
2067
2175
  const warned = /* @__PURE__ */ new Set();
2068
2176
  questions = questionRunner({
2069
2177
  evals: asked,
@@ -2105,7 +2213,7 @@ async function executeRun(ctx, setup, filters, renderer, started) {
2105
2213
  }
2106
2214
  });
2107
2215
  }
2108
- const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-C0dDP75l.js");
2216
+ const { prepareWorkspaces, sameTreeArms, workspaceKey } = await import("./workspace-DhUOW8if.js");
2109
2217
  const live = skipped.filter((p) => !p.attempt.plan.skip);
2110
2218
  const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
2111
2219
  workspace: a.workspace,
@@ -2283,9 +2391,9 @@ async function executeRun(ctx, setup, filters, renderer, started) {
2283
2391
  throw error;
2284
2392
  } finally {
2285
2393
  clearTimeout(budgetTimer);
2286
- const { removePrepared } = await import("./preflight-D1C-iH0U.js");
2394
+ const { removePrepared } = await import("./preflight-BUjk_6wT.js");
2287
2395
  removePrepared(config.root, runId);
2288
- const { removePreparedWorkspaces } = await import("./workspace-C0dDP75l.js");
2396
+ const { removePreparedWorkspaces } = await import("./workspace-DhUOW8if.js");
2289
2397
  removePreparedWorkspaces(config.root, runId);
2290
2398
  }
2291
2399
  if (preflightStop) throw preflightStop;
@@ -2361,13 +2469,15 @@ run(process.argv.slice(2)).finally(() => {
2361
2469
  process.exitCode = code;
2362
2470
  }, (error) => {
2363
2471
  const name = error.name;
2364
- const expected = error instanceof UsageError || EXPECTED_ERRORS.includes(name) || name === "PreflightInfraError";
2472
+ const infra = INFRA_ERRORS.includes(name);
2473
+ if (name === "StoreError") invocation.machineState = true;
2474
+ const expected = error instanceof UsageError || EXPECTED_ERRORS.includes(name) || infra;
2365
2475
  console.error(red(`reeval: ${redactSecrets(error.message)}`));
2366
2476
  if (!expected && process.env.REEVAL_DEBUG) console.error(redactSecrets(inspect(error)));
2367
- const code = name === "PreflightInfraError" ? EXIT.infra : EXIT.usage;
2477
+ const code = infra ? EXIT.infra : EXIT.usage;
2368
2478
  finish(code, error, error instanceof UsageError ? "usage" : expected ? "expected" : "crash");
2369
2479
  process.exitCode = code;
2370
2480
  });
2371
2481
  export {};
2372
2482
 
2373
- //# sourceMappingURL=cli-DTuLSlmW.js.map
2483
+ //# sourceMappingURL=cli-s_3opn_I.js.map