reeval 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +803 -0
  2. package/README.md +15 -15
  3. package/assets/gateway-models.mjs +65 -31
  4. package/dist/{adapter-C60z3wLq.js → adapter-BdzhdRYo.js} +8 -6
  5. package/dist/adapter-BdzhdRYo.js.map +1 -0
  6. package/dist/{agent-BmsMXZXx.js → agent-Dx-bDmlV.js} +179 -31
  7. package/dist/agent-Dx-bDmlV.js.map +1 -0
  8. package/dist/{agents-DXuFxtGC.js → agents-l0-k7vw3.js} +361 -101
  9. package/dist/agents-l0-k7vw3.js.map +1 -0
  10. package/dist/{app-files-DT5mJyV4.js → app-files-B-CQYB0c.js} +3 -3
  11. package/dist/app-files-B-CQYB0c.js.map +1 -0
  12. package/dist/{assemble-CE4pHKIa.js → assemble-CYMpN8LC.js} +1 -1
  13. package/dist/{assemble-DM2PNU4R.js → assemble-CsZIcJo8.js} +171 -18
  14. package/dist/assemble-CsZIcJo8.js.map +1 -0
  15. package/dist/{attempt-CAdlwlNu.js → attempt-CKfKsqqw.js} +2 -2
  16. package/dist/{attempt-CAdlwlNu.js.map → attempt-CKfKsqqw.js.map} +1 -1
  17. package/dist/{attempt-host-7VviAJrl.js → attempt-host-BwSmNYkX.js} +17 -8
  18. package/dist/attempt-host-BwSmNYkX.js.map +1 -0
  19. package/dist/{boat-BjLqFJIZ.js → boat-CZpAreTB.js} +3 -3
  20. package/dist/{boat-BjLqFJIZ.js.map → boat-CZpAreTB.js.map} +1 -1
  21. package/dist/{boat-DSNgZt-t.js → boat-su1Y7LPd.js} +17 -14
  22. package/dist/boat-su1Y7LPd.js.map +1 -0
  23. package/dist/bridge/index.mjs +6 -1
  24. package/dist/{capture-Ccf0Hmyx.js → capture-BqIAeFyC.js} +55 -29
  25. package/dist/{capture-Ccf0Hmyx.js.map → capture-BqIAeFyC.js.map} +1 -1
  26. package/dist/capture-D0Ln5aCP.js +1 -0
  27. package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
  28. package/dist/cases-CM-MQRQp.js.map +1 -0
  29. package/dist/cases-FvhbM8M5.js +1 -0
  30. package/dist/{cli-DTuLSlmW.js → cli-s_3opn_I.js} +219 -109
  31. package/dist/cli-s_3opn_I.js.map +1 -0
  32. package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-DxzIjmq_.js} +109 -26
  33. package/dist/cli-spec-DxzIjmq_.js.map +1 -0
  34. package/dist/cli.js +2 -2
  35. package/dist/client-C5fIzHQa.js +803 -0
  36. package/dist/client-C5fIzHQa.js.map +1 -0
  37. package/dist/{command-DSL7I-SI.js → command-B5OAIjp7.js} +25 -21
  38. package/dist/command-B5OAIjp7.js.map +1 -0
  39. package/dist/{command-C2C7P5qo.js → command-BJ7KqczP.js} +10 -10
  40. package/dist/{command-C2C7P5qo.js.map → command-BJ7KqczP.js.map} +1 -1
  41. package/dist/{command-D8oMB2fa.js → command-BXLZsEwQ.js} +16 -16
  42. package/dist/{command-D8oMB2fa.js.map → command-BXLZsEwQ.js.map} +1 -1
  43. package/dist/{command-Ff0T-1Hi.js → command-DA_SHQUQ.js} +16 -16
  44. package/dist/{command-Ff0T-1Hi.js.map → command-DA_SHQUQ.js.map} +1 -1
  45. package/dist/{commands-xAOi_VEk.js → commands-BKBamOcX.js} +174 -40
  46. package/dist/commands-BKBamOcX.js.map +1 -0
  47. package/dist/{commands-Bk9NG1s_.js → commands-BPvAOEmb.js} +1 -1
  48. package/dist/{config-SvK8pjEK.js → config-C0URar0t.js} +255 -89
  49. package/dist/config-C0URar0t.js.map +1 -0
  50. package/dist/{context-BdgCao9Q.js → context-Sj58aJEu.js} +2 -2
  51. package/dist/{context-BdgCao9Q.js.map → context-Sj58aJEu.js.map} +1 -1
  52. package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
  53. package/dist/cost-C21gGrju.js.map +1 -0
  54. package/dist/cost-IM80LyGD.js +1 -0
  55. package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-FWgSo-T1.js} +4 -4
  56. package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-FWgSo-T1.js.map} +1 -1
  57. package/dist/db-Bc4L-Wcr.js +2 -0
  58. package/dist/{db-BzOJ4Js4.js → db-BjAWxVAG.js} +142 -248
  59. package/dist/db-BjAWxVAG.js.map +1 -0
  60. package/dist/debug-CIXzB4wK.js.map +1 -1
  61. package/dist/{default-effort-BBDPbJ4D.js → default-effort-CWlhZ6U1.js} +1 -1
  62. package/dist/{default-effort-BDKIxJyC.js → default-effort-Ck2GYIxX.js} +4 -4
  63. package/dist/{default-effort-BDKIxJyC.js.map → default-effort-Ck2GYIxX.js.map} +1 -1
  64. package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
  65. package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
  66. package/dist/{docker-DG_uFgn2.js → docker-CDdK6Ity.js} +41 -16
  67. package/dist/docker-CDdK6Ity.js.map +1 -0
  68. package/dist/{docker-DFrjoNs-.js → docker-D74l67k8.js} +4 -4
  69. package/dist/{docker-DFrjoNs-.js.map → docker-D74l67k8.js.map} +1 -1
  70. package/dist/{docker--mukta3L.js → docker-DMug3l-a.js} +1 -1
  71. package/dist/{doctor-DWK4ttex.js → doctor-eFxyn6at.js} +85 -46
  72. package/dist/doctor-eFxyn6at.js.map +1 -0
  73. package/dist/{environment-cPMCOk-h.js → environment-CJIKz-Ud.js} +49 -11
  74. package/dist/environment-CJIKz-Ud.js.map +1 -0
  75. package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
  76. package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
  77. package/dist/{eval-source-C9b1h8L6.js → eval-source-CXIRJ91v.js} +24 -12
  78. package/dist/eval-source-CXIRJ91v.js.map +1 -0
  79. package/dist/{evaluate-B12Tosvq.js → evaluate-DA4QsrHO.js} +792 -281
  80. package/dist/evaluate-DA4QsrHO.js.map +1 -0
  81. package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DHrU4Wld.d.ts} +2 -2
  82. package/dist/fake-bootstrap-CnwWG3kP.js +15 -0
  83. package/dist/fake-bootstrap-CnwWG3kP.js.map +1 -0
  84. package/dist/{files-BOBdC0uI.js → files-BVVWAS0A.js} +6 -6
  85. package/dist/{files-BOBdC0uI.js.map → files-BVVWAS0A.js.map} +1 -1
  86. package/dist/files-BaqXvB88.js +1 -0
  87. package/dist/{files-tiNWe8Ih.js → files-Bey1VAex.js} +3 -3
  88. package/dist/{files-tiNWe8Ih.js.map → files-Bey1VAex.js.map} +1 -1
  89. package/dist/files-BjOJw4sn.js +1 -0
  90. package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
  91. package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
  92. package/dist/files-CNHhSyb3.js +1 -0
  93. package/dist/files-CW1lChNh.js +1 -0
  94. package/dist/{files-F5EG0WPT.js → files-D6j-XpaK.js} +5 -5
  95. package/dist/{files-F5EG0WPT.js.map → files-D6j-XpaK.js.map} +1 -1
  96. package/dist/{files-s28a6czq.js → files-F44NRCN0.js} +143 -20
  97. package/dist/files-F44NRCN0.js.map +1 -0
  98. package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
  99. package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
  100. package/dist/gateway.d.ts +1 -1
  101. package/dist/{git-BpoqEgDV.js → git-Jqkf9SBa.js} +3 -3
  102. package/dist/{git-BpoqEgDV.js.map → git-Jqkf9SBa.js.map} +1 -1
  103. package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
  104. package/dist/harness-FcTyyPco.js.map +1 -0
  105. package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
  106. package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
  107. package/dist/harness.d.ts +11 -8
  108. package/dist/harness.js +1 -1
  109. package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
  110. package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
  111. package/dist/{images-D2jFze2v.js → images-Dsc4Wi-F.js} +65 -30
  112. package/dist/images-Dsc4Wi-F.js.map +1 -0
  113. package/dist/index.d.ts +46 -7
  114. package/dist/index.js +6 -6
  115. package/dist/init-C0oUG3jh.js +1 -0
  116. package/dist/{init-Bi1Q3lRi.js → init-QB5ecPDA.js} +33 -14
  117. package/dist/init-QB5ecPDA.js.map +1 -0
  118. package/dist/{ink-list-BpfidQ1h.js → ink-list-mCJypk3d.js} +2 -2
  119. package/dist/{ink-list-BpfidQ1h.js.map → ink-list-mCJypk3d.js.map} +1 -1
  120. package/dist/{invocations-CxGNOlh0.js → invocations-BSr4lRzI.js} +5 -5
  121. package/dist/{invocations-CxGNOlh0.js.map → invocations-BSr4lRzI.js.map} +1 -1
  122. package/dist/{judge-COMXT1Hx.js → judge-eGH39PaP.js} +12 -10
  123. package/dist/{judge-COMXT1Hx.js.map → judge-eGH39PaP.js.map} +1 -1
  124. package/dist/judge.d.ts +3 -1
  125. package/dist/judge.js +1 -1
  126. package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
  127. package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
  128. package/dist/{layout-Cm85tQYk.js → layout-DfQa4mJs.js} +14 -11
  129. package/dist/layout-DfQa4mJs.js.map +1 -0
  130. package/dist/{list-DuThsE28.js → list-Dh6T0kdr.js} +23 -12
  131. package/dist/{list-DuThsE28.js.map → list-Dh6T0kdr.js.map} +1 -1
  132. package/dist/live-DwjWJTdn.js +1 -0
  133. package/dist/{live-CgYbO4Te.js → live-xTZs0c61.js} +2 -2
  134. package/dist/{live-CgYbO4Te.js.map → live-xTZs0c61.js.map} +1 -1
  135. package/dist/{live-DL0se4Zd.js → live-y1p3oNXH.js} +9 -9
  136. package/dist/{live-DL0se4Zd.js.map → live-y1p3oNXH.js.map} +1 -1
  137. package/dist/{model-CwwkrCb7.js → model-3pZZzloU.js} +7 -5
  138. package/dist/model-3pZZzloU.js.map +1 -0
  139. package/dist/{model-CkNVAxob.js → model-DEjwetkJ.js} +1 -1
  140. package/dist/{model-V3mMXp6x.js → model-DNpKiHqO.js} +6 -6
  141. package/dist/{model-V3mMXp6x.js.map → model-DNpKiHqO.js.map} +1 -1
  142. package/dist/model-calls-B0BoQfJ4.js +95 -0
  143. package/dist/model-calls-B0BoQfJ4.js.map +1 -0
  144. package/dist/{model-cgt5uvu8.js → model-jRKSQBg6.js} +5 -5
  145. package/dist/{model-cgt5uvu8.js.map → model-jRKSQBg6.js.map} +1 -1
  146. package/dist/model-nfewGTzB.js +1 -0
  147. package/dist/{node-D7pAU--0.js → node-DM-mLGdX.js} +7 -7
  148. package/dist/{node-D7pAU--0.js.map → node-DM-mLGdX.js.map} +1 -1
  149. package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
  150. package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
  151. package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-C0aRo633.js} +2 -2
  152. package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-C0aRo633.js.map} +1 -1
  153. package/dist/options-DpsJ_5CU.js +1 -0
  154. package/dist/{patch-DAi1nCuH.js → patch-Bx-qp8Li.js} +2 -2
  155. package/dist/{patch-DAi1nCuH.js.map → patch-Bx-qp8Li.js.map} +1 -1
  156. package/dist/pi-providers-HygmL1y-.js +552 -0
  157. package/dist/pi-providers-HygmL1y-.js.map +1 -0
  158. package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
  159. package/dist/plain-Ctn8oUdp.js.map +1 -0
  160. package/dist/{plan-CsPilzpk.js → plan-ChO0A-xn.js} +63 -12
  161. package/dist/plan-ChO0A-xn.js.map +1 -0
  162. package/dist/{plugins-BfnhazOv.js → plugins-BioMWncI.js} +636 -9
  163. package/dist/plugins-BioMWncI.js.map +1 -0
  164. package/dist/{preflight-D1C-iH0U.js → preflight-BUjk_6wT.js} +1 -1
  165. package/dist/{preflight-GkpgrBxP.js → preflight-CJSGzMzc.js} +8 -9
  166. package/dist/preflight-CJSGzMzc.js.map +1 -0
  167. package/dist/preflight-DD3fI4Hz.js +1 -0
  168. package/dist/{preflight-BixaE4BA.js → preflight-qFCXmxha.js} +4 -4
  169. package/dist/{preflight-BixaE4BA.js.map → preflight-qFCXmxha.js.map} +1 -1
  170. package/dist/{prepare-DQEDXoDp.js → prepare-D7oSSFNm.js} +9 -9
  171. package/dist/{prepare-DQEDXoDp.js.map → prepare-D7oSSFNm.js.map} +1 -1
  172. package/dist/{prepare-C--rM_wO.js → prepare-xHU2BpN-.js} +2 -2
  173. package/dist/prices-B1vm_lPk.js +1 -0
  174. package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
  175. package/dist/prices-BrY_ihLs.js.map +1 -0
  176. package/dist/{provenance-BXyYwUC8.js → provenance-B-flBAzR.js} +5 -5
  177. package/dist/{provenance-BXyYwUC8.js.map → provenance-B-flBAzR.js.map} +1 -1
  178. package/dist/provider-DHT20Gl8.js +1 -0
  179. package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
  180. package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
  181. package/dist/provider-stop-CxzWW7Ft.js +69 -0
  182. package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
  183. package/dist/{providers-1p4HKYIp.js → providers-B6StVyfG.js} +215 -48
  184. package/dist/providers-B6StVyfG.js.map +1 -0
  185. package/dist/providers-Dljf6al6.js +1 -0
  186. package/dist/query-BAc2tQZj.js +1 -0
  187. package/dist/{query-DhhU9ADI.js → query-CkLmJv7R.js} +2 -2
  188. package/dist/{query-DhhU9ADI.js.map → query-CkLmJv7R.js.map} +1 -1
  189. package/dist/{questions-CMAq4pr8.js → questions-CJIh2FSX.js} +7 -7
  190. package/dist/{questions-CMAq4pr8.js.map → questions-CJIh2FSX.js.map} +1 -1
  191. package/dist/{record-6IUzwP9K.js → record-nVVujCW7.js} +2 -2
  192. package/dist/{record-6IUzwP9K.js.map → record-nVVujCW7.js.map} +1 -1
  193. package/dist/{redact-BhjtnjrU.js → redact-DL2BTykw.js} +17 -2
  194. package/dist/redact-DL2BTykw.js.map +1 -0
  195. package/dist/{remarks-BNABzemt.js → remarks-B3wjKscW.js} +46 -11
  196. package/dist/remarks-B3wjKscW.js.map +1 -0
  197. package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
  198. package/dist/remarks-BpdM_2RC.js.map +1 -0
  199. package/dist/remarks-CleoDlYw.js +1 -0
  200. package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
  201. package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
  202. package/dist/{remote-U7fNt0UP.js → remote-C88amFRN.js} +6 -6
  203. package/dist/{remote-U7fNt0UP.js.map → remote-C88amFRN.js.map} +1 -1
  204. package/dist/resolve-Bul8lHaa.js +205 -0
  205. package/dist/resolve-Bul8lHaa.js.map +1 -0
  206. package/dist/resolve-DW92wpIZ.js +1 -0
  207. package/dist/{resolve-Cx3a9UQ2.js → resolve-G576PiFJ.js} +14 -11
  208. package/dist/resolve-G576PiFJ.js.map +1 -0
  209. package/dist/{results-CQKXmHym.js → results-zVHIW6xw.js} +9 -22
  210. package/dist/results-zVHIW6xw.js.map +1 -0
  211. package/dist/{resume-POpZa1ge.js → resume-D_PcUXfG.js} +94 -16
  212. package/dist/resume-D_PcUXfG.js.map +1 -0
  213. package/dist/{retry-CmG809Mp.js → retry-BvTzCxdc.js} +4 -4
  214. package/dist/{retry-CmG809Mp.js.map → retry-BvTzCxdc.js.map} +1 -1
  215. package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
  216. package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
  217. package/dist/{vercel-CGlSBv63.js → routes-B0F2R-K7.js} +91 -56
  218. package/dist/routes-B0F2R-K7.js.map +1 -0
  219. package/dist/{runtime-Xm9oeC7o.js → runtime-B3r_pAq6.js} +4 -4
  220. package/dist/{runtime-Xm9oeC7o.js.map → runtime-B3r_pAq6.js.map} +1 -1
  221. package/dist/sandbox/boat.d.ts +1 -1
  222. package/dist/sandbox/boat.js +1 -1
  223. package/dist/{sandbox-DrlL4yqb.js → sandbox-Duh3z1TV.js} +3 -3
  224. package/dist/{sandbox-DrlL4yqb.js.map → sandbox-Duh3z1TV.js.map} +1 -1
  225. package/dist/sandbox.d.ts +1 -1
  226. package/dist/sandbox.js +1 -1
  227. package/dist/server-CGEsvVRG.js +1 -0
  228. package/dist/{server-C00zd6zl.js → server-DUWMRKik.js} +8 -8
  229. package/dist/{server-C00zd6zl.js.map → server-DUWMRKik.js.map} +1 -1
  230. package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
  231. package/dist/shared-DtZwIAfy.js.map +1 -0
  232. package/dist/{source-DDgr48Gn.js → source-CHuN4Nfg.js} +2 -2
  233. package/dist/{source-DDgr48Gn.js.map → source-CHuN4Nfg.js.map} +1 -1
  234. package/dist/source-Wz1Nrfue.js +1 -0
  235. package/dist/src--yl1OypK.js +2 -0
  236. package/dist/{src-Cb2MWozp.js → src-B-HuDqpH.js} +276 -41
  237. package/dist/src-B-HuDqpH.js.map +1 -0
  238. package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
  239. package/dist/src-B5r1d23Y.js.map +1 -0
  240. package/dist/{src-y0nOeRRE.js → src-BHJ4MWvY.js} +494 -112
  241. package/dist/src-BHJ4MWvY.js.map +1 -0
  242. package/dist/{src-Dw79VLa3.js → src-BI0v8qlZ.js} +202 -56
  243. package/dist/src-BI0v8qlZ.js.map +1 -0
  244. package/dist/src-BL-_0QoE.js +1 -0
  245. package/dist/{src-YvblOrBU.js → src-BqNzHHWn.js} +87 -13
  246. package/dist/src-BqNzHHWn.js.map +1 -0
  247. package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
  248. package/dist/src-C-ChdnG6.js.map +1 -0
  249. package/dist/src-C-mOSczH.js +2 -0
  250. package/dist/src-CZaYu6ER.js +3 -0
  251. package/dist/src-CeBhwD8k.js +2 -0
  252. package/dist/src-CkEiiyON.js +2 -0
  253. package/dist/{src-KxS91P7L.js → src-CmA5Yev6.js} +79 -11
  254. package/dist/src-CmA5Yev6.js.map +1 -0
  255. package/dist/src-CwH_nEaw.js +2 -0
  256. package/dist/src-DA95rM6v.js +4 -0
  257. package/dist/{src-DvGns8cz.js → src-DL-EE4DU.js} +299 -17
  258. package/dist/src-DL-EE4DU.js.map +1 -0
  259. package/dist/src-De0KY5S_.js +2 -0
  260. package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
  261. package/dist/src-boB6D-5T.js.map +1 -0
  262. package/dist/state-B3jex5KU.js +1 -0
  263. package/dist/{state-328JpnHA.js → state-DUBRDyMk.js} +21 -21
  264. package/dist/{state-328JpnHA.js.map → state-DUBRDyMk.js.map} +1 -1
  265. package/dist/store/local.d.ts +8 -0
  266. package/dist/store/local.js +16 -0
  267. package/dist/store/local.js.map +1 -0
  268. package/dist/store/s3.d.ts +10 -0
  269. package/dist/store/s3.js +18 -0
  270. package/dist/store/s3.js.map +1 -0
  271. package/dist/{suite-COglUhPd.js → suite-B9J6Afg6.js} +6 -5
  272. package/dist/{suite-COglUhPd.js.map → suite-B9J6Afg6.js.map} +1 -1
  273. package/dist/suite-Bhs9-Z7O.js +2 -0
  274. package/dist/{summary-DDy9DfyJ.js → summary-CSp4cfnr.js} +25 -7
  275. package/dist/{summary-DDy9DfyJ.js.map → summary-CSp4cfnr.js.map} +1 -1
  276. package/dist/sync-CxAY6Dki.js +559 -0
  277. package/dist/sync-CxAY6Dki.js.map +1 -0
  278. package/dist/{tart-dWoJvWHc.js → tart-CcrrtSrp.js} +179 -61
  279. package/dist/tart-CcrrtSrp.js.map +1 -0
  280. package/dist/tart-CzViW3WZ.js +186 -0
  281. package/dist/tart-CzViW3WZ.js.map +1 -0
  282. package/dist/tart-DmVuVsmX.js +1 -0
  283. package/dist/tart-bake-F-6seRC9.js +192 -0
  284. package/dist/tart-bake-F-6seRC9.js.map +1 -0
  285. package/dist/tart-bake-RK7RaT9E.js +1 -0
  286. package/dist/tart-cli-B86Jqo1t.js +89 -0
  287. package/dist/tart-cli-B86Jqo1t.js.map +1 -0
  288. package/dist/tart-cli-ClvRniLQ.js +1 -0
  289. package/dist/tart-hosts-Brii7zHM.js +1 -0
  290. package/dist/tart-hosts-DPyZsg-1.js +313 -0
  291. package/dist/tart-hosts-DPyZsg-1.js.map +1 -0
  292. package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CWSf03w8.js} +64 -6
  293. package/dist/tart-images-CWSf03w8.js.map +1 -0
  294. package/dist/testing.d.ts +36 -2
  295. package/dist/testing.js +4 -4
  296. package/dist/tool-class-CVDdJy0H.js +433 -0
  297. package/dist/tool-class-CVDdJy0H.js.map +1 -0
  298. package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
  299. package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
  300. package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
  301. package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
  302. package/dist/{transcript-CCf07UmU.js → transcript-BP_2hji-.js} +3 -3
  303. package/dist/{transcript-CCf07UmU.js.map → transcript-BP_2hji-.js.map} +1 -1
  304. package/dist/{types-DBwFsGem.d.ts → types-BJ0wh8wC.d.ts} +408 -44
  305. package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
  306. package/dist/types-CbcnJxCy.js.map +1 -0
  307. package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
  308. package/dist/usage-CwIRPPZ9.js.map +1 -0
  309. package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
  310. package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
  311. package/dist/vertex-DXfcXYKn.js +1 -0
  312. package/dist/view/index.d.ts +27 -3
  313. package/dist/view/index.js +8 -8
  314. package/dist/{views-BROacjBv.js → views-DRxXCyIH.js} +6 -6
  315. package/dist/{views-BROacjBv.js.map → views-DRxXCyIH.js.map} +1 -1
  316. package/dist/views-i67V1ly3.js +2 -0
  317. package/dist/{worker-entry-Smb4bNDc.js → worker-entry-C3Q4U63F.js} +12 -9
  318. package/dist/worker-entry-C3Q4U63F.js.map +1 -0
  319. package/dist/workspace-DhUOW8if.js +1 -0
  320. package/dist/{workspace-DNDTEk8v.js → workspace-P72SXLXr.js} +7 -7
  321. package/dist/{workspace-DNDTEk8v.js.map → workspace-P72SXLXr.js.map} +1 -1
  322. package/package.json +12 -1
  323. package/skills/eval-rubric/SKILL.md +10 -0
  324. package/skills/reeval/SKILL.md +370 -0
  325. package/skills/reeval/references/capture.md +48 -0
  326. package/skills/reeval/references/cli.md +162 -0
  327. package/skills/reeval/references/compare.md +180 -0
  328. package/skills/reeval/references/config.md +147 -0
  329. package/skills/reeval/references/environments.md +218 -0
  330. package/skills/reeval/references/evals.md +241 -0
  331. package/skills/reeval/references/mistakes.md +39 -0
  332. package/skills/reeval/references/vocabulary.md +19 -0
  333. package/skills/setup/SKILL.md +219 -0
  334. package/skills/setup/references/templates.md +258 -0
  335. package/dist/adapter-C60z3wLq.js.map +0 -1
  336. package/dist/agent-BmsMXZXx.js.map +0 -1
  337. package/dist/agents-DXuFxtGC.js.map +0 -1
  338. package/dist/app-files-DT5mJyV4.js.map +0 -1
  339. package/dist/assemble-DM2PNU4R.js.map +0 -1
  340. package/dist/attempt-host-7VviAJrl.js.map +0 -1
  341. package/dist/boat-DSNgZt-t.js.map +0 -1
  342. package/dist/capture-BzotD2hs.js +0 -1
  343. package/dist/cases-Dy0mifal.js +0 -1
  344. package/dist/cases-z9E6DCd5.js.map +0 -1
  345. package/dist/cli-DTuLSlmW.js.map +0 -1
  346. package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
  347. package/dist/client-KM4C5r5_.js +0 -387
  348. package/dist/client-KM4C5r5_.js.map +0 -1
  349. package/dist/command-DSL7I-SI.js.map +0 -1
  350. package/dist/commands-xAOi_VEk.js.map +0 -1
  351. package/dist/config-SvK8pjEK.js.map +0 -1
  352. package/dist/cost-BdRgySQl.js.map +0 -1
  353. package/dist/cost-Cr26Alhq.js +0 -1
  354. package/dist/db-BzOJ4Js4.js.map +0 -1
  355. package/dist/db-CQRZ3URX.js +0 -2
  356. package/dist/docker-DG_uFgn2.js.map +0 -1
  357. package/dist/doctor-DWK4ttex.js.map +0 -1
  358. package/dist/environment-cPMCOk-h.js.map +0 -1
  359. package/dist/eval-source-C9b1h8L6.js.map +0 -1
  360. package/dist/evaluate-B12Tosvq.js.map +0 -1
  361. package/dist/files-C1Rk9FYA.js +0 -1
  362. package/dist/files-DMQ9-Vv5.js +0 -1
  363. package/dist/files-Di0xDbCc.js +0 -1
  364. package/dist/files-G5jf8wBp.js +0 -1
  365. package/dist/files-s28a6czq.js.map +0 -1
  366. package/dist/harness-DXv2gnaT.js.map +0 -1
  367. package/dist/images-D2jFze2v.js.map +0 -1
  368. package/dist/init-Bi1Q3lRi.js.map +0 -1
  369. package/dist/init-CUw65o36.js +0 -1
  370. package/dist/kinds-Bh2I35id.js +0 -698
  371. package/dist/kinds-Bh2I35id.js.map +0 -1
  372. package/dist/layout-Cm85tQYk.js.map +0 -1
  373. package/dist/live-DZLmj0iG.js +0 -1
  374. package/dist/matchers-CXLn-1jm.js +0 -148
  375. package/dist/matchers-CXLn-1jm.js.map +0 -1
  376. package/dist/model-Cv4GuTsZ.js +0 -1
  377. package/dist/model-CwwkrCb7.js.map +0 -1
  378. package/dist/options-DPUmk7v_.js +0 -1
  379. package/dist/plain-BYUf_fUV.js.map +0 -1
  380. package/dist/plan-CsPilzpk.js.map +0 -1
  381. package/dist/plugins-BfnhazOv.js.map +0 -1
  382. package/dist/preflight-Bn3ianye.js +0 -1
  383. package/dist/preflight-GkpgrBxP.js.map +0 -1
  384. package/dist/prices-5m3SzMZ3.js.map +0 -1
  385. package/dist/provider-BnUoz4KB.js +0 -1
  386. package/dist/providers-1p4HKYIp.js.map +0 -1
  387. package/dist/providers-B-ijCktf.js +0 -1
  388. package/dist/query-BYhyf0LR.js +0 -1
  389. package/dist/redact-BhjtnjrU.js.map +0 -1
  390. package/dist/remarks-BNABzemt.js.map +0 -1
  391. package/dist/remarks-CAvTSq16.js +0 -1
  392. package/dist/remarks-D9Lcd1fF.js.map +0 -1
  393. package/dist/resolve-C14Pmtej.js +0 -1
  394. package/dist/resolve-Cx3a9UQ2.js.map +0 -1
  395. package/dist/results-CQKXmHym.js.map +0 -1
  396. package/dist/resume-POpZa1ge.js.map +0 -1
  397. package/dist/server-C1UJcpUh.js +0 -1
  398. package/dist/shared-Cfi45Bvy.js.map +0 -1
  399. package/dist/source-DJHmQv_j.js +0 -1
  400. package/dist/src-1dBuXzs4.js +0 -3
  401. package/dist/src-BKpioayC.js +0 -2
  402. package/dist/src-BNQznj4z.js.map +0 -1
  403. package/dist/src-BY1cCaoJ.js +0 -2
  404. package/dist/src-BbZYaQ8b.js +0 -2
  405. package/dist/src-CDFH2vVN.js +0 -2
  406. package/dist/src-CMW5R9Q0.js +0 -1
  407. package/dist/src-CV_7DkPa.js.map +0 -1
  408. package/dist/src-Cb2MWozp.js.map +0 -1
  409. package/dist/src-CrV2WA2i.js +0 -2
  410. package/dist/src-CsFaawcZ.js +0 -2
  411. package/dist/src-DAzaCTcN.js.map +0 -1
  412. package/dist/src-DvGns8cz.js.map +0 -1
  413. package/dist/src-Dw79VLa3.js.map +0 -1
  414. package/dist/src-KxS91P7L.js.map +0 -1
  415. package/dist/src-YvblOrBU.js.map +0 -1
  416. package/dist/src-wKwznPgt.js +0 -2
  417. package/dist/src-y0nOeRRE.js.map +0 -1
  418. package/dist/state-Ca55Gk_O.js +0 -1
  419. package/dist/suite-CKBa6MKt.js +0 -2
  420. package/dist/tart-cli-C-GsbVsQ.js +0 -1
  421. package/dist/tart-cli-CU0nFS-L.js +0 -55
  422. package/dist/tart-cli-CU0nFS-L.js.map +0 -1
  423. package/dist/tart-dWoJvWHc.js.map +0 -1
  424. package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
  425. package/dist/tool-target-DS5d72eH.js +0 -40
  426. package/dist/tool-target-DS5d72eH.js.map +0 -1
  427. package/dist/types-C2mLpph3.js.map +0 -1
  428. package/dist/usage-DJiDnh2D.js.map +0 -1
  429. package/dist/vercel-CGlSBv63.js.map +0 -1
  430. package/dist/vertex-O88TCwgY.js +0 -1
  431. package/dist/views-B0Yt_Q1S.js +0 -2
  432. package/dist/worker-entry-Smb4bNDc.js.map +0 -1
  433. package/dist/workspace-C0dDP75l.js +0 -1
@@ -1,199 +1,49 @@
1
1
  import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-ttMTnIYX.js";
2
- import { g as parseDuration, m as MAX_TIMER_MS, n as dataProblem, t as closest } from "./shared-Cfi45Bvy.js";
3
- import { t as isObjectLike } from "./object-CeQ8dgUC.js";
4
- import { a as plural, i as formatDuration, n as clip, o as withoutBom, t as capMiddle } from "./text-B8-W3Cvn.js";
5
- import { a as TURN_EDITED, f as limitsLabel, r as withLease } from "./client-KM4C5r5_.js";
6
- import { d as sandboxKind, n as capabilitiesOf, s as providerLimit } from "./kinds-Bh2I35id.js";
7
- import { O as sandboxPaths, P as workspaceOf, f as filesOf, i as callerDir, j as traitsOf, l as environmentFrom, p as globFilter, u as environmentPaths } from "./environment-cPMCOk-h.js";
2
+ import { g as parseDuration, m as MAX_TIMER_MS, n as dataProblem, t as closest } from "./shared-DtZwIAfy.js";
3
+ import { Dt as OPAQUE_OUTPUT, jt as shellQuote$1, n as MCP_TIMER } from "./src-BHJ4MWvY.js";
8
4
  import { t as evalSlug } from "./slug-IHh3f770.js";
9
- import { n as attemptKey, o as recordFileName, r as evalId, s as relativeFile, t as attemptId } from "./ids-C_advdJA.js";
10
- import { $ as runsOf, A as brokeredIn, B as expand, Bt as decideState, Ct as resolveStateOptions, Et as observer, Ft as NO_JUDGE_INPUT, Ht as resolveJudgeArtifacts, It as inputsOn, J as plainHeaders, K as plainHeaderError, L as checkOptions, Lt as judgeProblem, Pt as INPUT_KEYS, Q as refuseIllegalCompare, Rt as artifactRefsOf, St as mergeJudge, Tt as observedName, U as majorityOf, V as harnessId, Vt as filePartsOf, W as matchesTags, _ as fellBackToDirect, at as unbrokered, b as resolveModelGateway, et as sandboxId, gt as unenforceable, ht as isDirect, it as secretVarsIn, j as cascade, nt as sandboxSetBy, ot as unbrokeredError, pt as egressLabel, tt as sandboxProvider, ut as allowedHosts, v as gatewayScope, w as resolveGateway, wt as observedHere, z as evalPlan, zt as badJudgeArtifacts } from "./config-SvK8pjEK.js";
11
- import { l as vercelCredentialProblem } from "./vercel-CGlSBv63.js";
12
- import { v as vertexSettings } from "./vertex-Eb2wqahk.js";
13
- import { _ as redactValue, d as maskedVariables, f as redactJson, g as redactSplitSecrets, h as redactSecrets, o as findSecrets } from "./redact-BhjtnjrU.js";
14
- import { f as runJudge, h as isApiError, l as judgeModelName } from "./judge-COMXT1Hx.js";
5
+ import { c as relativeFile, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-B-JKTCP8.js";
6
+ import { _ as redactValue, d as maskedVariables, f as redactJson, g as redactSplitSecrets, h as redactSecrets, o as findSecrets } from "./redact-DL2BTykw.js";
7
+ import { O as pluginSourceLabel, P as capabilitiesOf, S as pluginBuildId, U as sandboxKind, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall, z as providerLimit } from "./plugins-BioMWncI.js";
8
+ import { a as plural, i as formatDuration, n as clip, o as withoutBom, t as capMiddle } from "./text-B8-W3Cvn.js";
9
+ import { d as vercelCredentialProblem } from "./routes-B0F2R-K7.js";
10
+ import { v as vertexSettings } from "./vertex-BEWxHoxH.js";
11
+ import { n as hasNetworkSession, t as InfraError } from "./types-CbcnJxCy.js";
12
+ import { C as writePluginFiles, S as unverifiedHarness, b as refusedMcp, c as agentMcp, d as brokeringFor, f as claudeMcpServers, h as fakeBrokering, l as agentOf, m as expectedBlockedFor, t as AGENTS, v as keyRejected, x as refusedPlugins, y as layoutOf } from "./agents-l0-k7vw3.js";
13
+ import { t as isObjectLike } from "./object-CeQ8dgUC.js";
14
+ import { d as limitsLabel, f as runMatchLabel, r as withLease, s as judgeFailedMessage, u as TURN_EDITED } from "./client-C5fIzHQa.js";
15
+ import { F as workspaceOf, M as traitsOf, f as filesOf, i as callerDir, k as sandboxPaths, l as environmentFrom, p as globFilter, u as environmentPaths } from "./environment-CJIKz-Ud.js";
16
+ import { $t as resolveJudgeArtifacts, B as evalPlan, Ct as phaseLabel, Dt as unenforceable, Ft as observedHere, G as matchesTags, Gt as INPUT_KEYS, H as harnessId, It as observedName, J as plainEnvError, Jt as judgeProblem, Kt as NO_JUDGE_INPUT, Lt as observer, M as cascade, Nt as mergeJudge, Pt as resolveStateOptions, Q as plainHeaders, Qt as filePartsOf, R as checkOptions, T as resolveGateway, V as expand, W as majorityOf, X as plainHeaderError, Xt as badJudgeArtifacts, Yt as artifactRefsOf, Zt as decideState, at as sandboxProvider, bt as isDirect, ct as secretVarsIn, it as sandboxId, j as brokeredIn, lt as unbrokered, mt as allowedHosts, nt as refuseIllegalCompare, ot as sandboxSetBy, q as plainEnv, qt as inputsOn, rt as runsOf, ut as unbrokeredError, v as fellBackToDirect, vt as egressLabel, x as resolveModelGateway, y as gatewayScope, yt as gatewayPolicy } from "./config-C0URar0t.js";
17
+ import { f as runJudge, g as isTransient, h as isApiError, l as judgeModelName } from "./judge-eGH39PaP.js";
15
18
  import { t as inCI } from "./env-8x1O8I1M.js";
16
- import { bt as shellQuote, gt as OPAQUE_OUTPUT, n as MCP_TIMER } from "./src-y0nOeRRE.js";
17
- import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-BfnhazOv.js";
18
- import { _ as unverifiedHarness, c as brokeringFor, d as expectedBlockedFor, g as refusedPlugins, h as refusedMcp, l as claudeMcpServers, m as layoutOf, o as agentMcp, p as keyRejected, t as AGENTS, v as writePluginFiles } from "./agents-DXuFxtGC.js";
19
- import { n as hasNetworkSession, t as InfraError } from "./types-C2mLpph3.js";
20
- import { a as registerTest, i as registerGroup, n as declarePlanned, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-Xm9oeC7o.js";
19
+ import { a as registerTest, i as registerGroup, n as declarePlanned, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-B3r_pAq6.js";
21
20
  import { c as formatBytes } from "./debug-CIXzB4wK.js";
22
- import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CAdlwlNu.js";
23
- import { t as attemptDebugDir } from "./files-DlQshsks.js";
24
- import { r as openRouterFallbacks } from "./openrouter-models-Cw1z8-sq.js";
25
- import { a as writeCaptureSidecars, c as sandboxFor, d as snapshot, f as workspaceBlobs, l as changedBetween, o as asCheck, p as workspaceFiles, r as serveCommand, s as createChecks, t as capturePage, u as diffSinceSnapshot } from "./capture-Ccf0Hmyx.js";
26
- import { t as contextTaker } from "./context-BdgCao9Q.js";
27
- import { a as skillFiles, i as resolveEnvironment } from "./resolve-Cx3a9UQ2.js";
28
- import { t as PrepareError } from "./provider-BxzZq3Th.js";
29
- import { i as preparedWorkspace } from "./workspace-DNDTEk8v.js";
21
+ import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CKfKsqqw.js";
22
+ import { a as skillFiles, i as resolveEnvironment } from "./resolve-G576PiFJ.js";
23
+ import { n as fakeBootstrapScript } from "./fake-bootstrap-CnwWG3kP.js";
24
+ import { n as providerStop, t as ProviderStopError } from "./provider-stop-CxzWW7Ft.js";
25
+ import { t as PrepareError } from "./provider-ieWOn4YS.js";
26
+ import { i as preparedWorkspace } from "./workspace-P72SXLXr.js";
27
+ import { t as attemptDebugDir } from "./files-g1watgRC.js";
28
+ import { r as openRouterFallbacks } from "./openrouter-models-C0aRo633.js";
29
+ import { a as writeCaptureSidecars, c as sandboxFor, d as snapshot, f as workspaceBlobs, l as changedBetween, o as asCheck, p as workspaceFiles, r as serveCommand, s as createChecks, t as capturePage, u as diffSinceSnapshot } from "./capture-BqIAeFyC.js";
30
+ import { t as contextTaker } from "./context-Sj58aJEu.js";
30
31
  import { t as patchPath } from "./patch-C7yBztZj.js";
31
- import { n as judgeDiff } from "./patch-DAi1nCuH.js";
32
- import { t as transcriptForJudge } from "./transcript-CCf07UmU.js";
32
+ import { n as judgeDiff } from "./patch-Bx-qp8Li.js";
33
+ import { t as transcriptForJudge } from "./transcript-BP_2hji-.js";
34
+ import { n as modelCallEvents } from "./model-calls-B0BoQfJ4.js";
33
35
  import { i as writeTar, n as readTar, t as extractTar } from "./tarball-CESxtK4X.js";
34
- import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-C9b1h8L6.js";
35
- import { i as withDefaultEffort } from "./default-effort-BDKIxJyC.js";
36
- import { a as traitFilters, i as pickTraits, n as controlTraitKeys, r as filtersLabel } from "./traits-DKnYkqeK.js";
36
+ import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-CXIRJ91v.js";
37
+ import { i as withDefaultEffort } from "./default-effort-Ck2GYIxX.js";
38
+ import { a as traitFilters, i as pickTraits, n as controlTraitKeys, r as filtersLabel } from "./traits-CCD8XsmI.js";
37
39
  import { z } from "zod";
38
40
  import { dirname, join } from "node:path";
39
- import { parse, stringify } from "yaml";
40
- import { randomBytes } from "node:crypto";
41
- import { setTimeout as setTimeout$1 } from "node:timers/promises";
41
+ import { randomBytes, randomUUID } from "node:crypto";
42
42
  import { gunzipSync, gzipSync } from "node:zlib";
43
+ import { parse, stringify } from "yaml";
43
44
  import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
44
- // src/attempt/assertions.ts
45
- const MESSAGE_LIMIT = 1024;
46
- const LABEL_LIMIT = 60;
47
- /** Masked first, so a secret the limit would cut through is masked whole. */
48
- const clipped = (text, limit) => {
49
- const s = redactSecrets(text);
50
- return s.length > limit ? `${s.slice(0, limit)}…` : s;
51
- };
52
- const text = (error) => error instanceof Error ? error.message : String(error);
53
- const shown = (value) => {
54
- if (typeof value === "string") return value;
55
- if (value instanceof RegExp) return String(value);
56
- try {
57
- return JSON.stringify(value) ?? String(value);
58
- } catch {
59
- return String(value);
60
- }
61
- };
62
- const shownSubject = (subject) => typeof subject === "string" && subject ? subject.replace(/\s+/g, " ").trim() : void 0;
63
- /**
64
- * The records of an attempt's matcher calls, in call order. Without a label, one is
65
- * `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3`, skipping
66
- * any label already recorded, so each record's label is unique (a given `build #2` after two
67
- * `build`s is `build #2 #2`).
68
- */
69
- function assertionRecords(raw) {
70
- const seen = /* @__PURE__ */ new Map();
71
- const used = /* @__PURE__ */ new Set();
72
- return raw.map((r) => {
73
- const named = r.matcher === "toStayWithin" ? limitsLabel(r.args[0]) : r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
74
- const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
75
- let n = seen.get(base) ?? 0;
76
- let label;
77
- do {
78
- n += 1;
79
- label = n > 1 ? `${base} #${n}` : base;
80
- } while (used.has(label));
81
- seen.set(base, n);
82
- used.add(label);
83
- return {
84
- label,
85
- labelled: r.label !== void 0,
86
- matcher: r.matcher,
87
- status: r.status,
88
- soft: r.soft,
89
- ...r.at ? { at: r.at } : {},
90
- ...r.status === "pass" ? {} : { message: clipped(text(r.error), MESSAGE_LIMIT) },
91
- ...r.durationMs === void 0 ? {} : { durationMs: r.durationMs }
92
- };
93
- });
94
- }
95
- const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
96
- /**
97
- * What fails the attempt at its end for the checks that failed without ending the body: the first
98
- * that didn't pass, an `expect` as what it failed with, an `expect.require` the eval caught with
99
- * `CAUGHT_HINT` after its message.
100
- */
101
- function checkFailure(raw) {
102
- const failed = raw.find((r) => r.status !== "pass");
103
- if (!failed) return void 0;
104
- const error = failed.error instanceof Error ? failed.error : new Error(text(failed.error));
105
- if (failed.soft) return error;
106
- const caught = Object.setPrototypeOf(Object.assign(/* @__PURE__ */ new Error(), error), Object.getPrototypeOf(error));
107
- caught.name = error.name;
108
- caught.message = `${error.message} ${CAUGHT_HINT}`;
109
- caught.stack = error.stack?.replace(error.message, caught.message);
110
- return caught;
111
- }
112
- // src/attempt/debug.ts
113
- /** Which config and profile the attempt runs under, and where each resolved option came from. */
114
- function tracePlan(debug, spec, ctx) {
115
- const config = ctx.configFile ?? "none (no config file found)";
116
- const from = (key, fallback = ctx.configFile ? `${config} or its default` : "default") => `(${spec.sources[key] ?? fallback})`;
117
- const profile = ctx.profile ? `profile ${ctx.profile.name} (${ctx.profile.reason})` : "no profile";
118
- debug.log("plan", `config ${config} · ${profile} · run ${ctx.runId} · attempt ${spec.cell.key}#${spec.n}`);
119
- debug.log("plan", `harness ${harnessId(spec.harness)} ${from("harnesses")}`);
120
- debug.log("plan", `sandbox ${sandboxId(spec.sandbox)} ${from("sandbox")}`);
121
- const env = spec.environment;
122
- const role = spec.cell.role === "variant" ? ` · variant ${spec.cell.variant}` : "";
123
- const envFrom = spec.cell.role === "variant" || spec.sources.compare ? from("compare") : from("environments", "none set");
124
- debug.log("plan", `environment ${env.name}${role} ${envFrom}`);
125
- debug.log("plan", `egress ${egressLabel(spec.egress) || "proxy"} ${from("egress")}`);
126
- const gate = typeof spec.gate === "string" ? spec.gate : `passRate ${spec.gate.passRate}`;
127
- debug.log("plan", `run ${spec.n} · runs ${ctx.runs !== void 0 ? "(--runs)" : from("runs")} · gate ${gate} · timeout ${formatDuration(spec.timeoutMs)} ${from("timeout", "config.timeout or 15m")}`);
128
- const judge = spec.judge;
129
- const model = typeof judge.model === "string" ? judge.model : judge.model && "kind" in judge.model ? "fake judge" : judge.model ? "a model object" : "none";
130
- debug.log("plan", `judge ${model}${judge.threshold !== void 0 ? ` · threshold ${judge.threshold}` : ""} ${from("judge")}`);
131
- for (const note of spec.notes) debug.log("plan", `profile: ${note}`);
132
- debug.trace("plan", "config", {
133
- config: ctx.config,
134
- profile: ctx.profile
135
- }, "resolved config");
136
- debug.trace("plan", "attempt", {
137
- cell: spec.cell,
138
- n: spec.n,
139
- harness: spec.harness,
140
- sandbox: spec.sandbox,
141
- egress: spec.egress,
142
- gate: spec.gate,
143
- timeoutMs: spec.timeoutMs,
144
- judge: spec.judge,
145
- sources: spec.sources,
146
- env: Object.keys(spec.env).sort()
147
- }, "resolved attempt");
148
- }
149
- /** `POST api2.cursor.sh/auth/exchange`, `GET host/v1/*` or `POST host (path ~ /re/)`. */
150
- function endpointLabel({ method, host, path }) {
151
- const where = !path ? host : "exact" in path ? `${host}${path.exact}` : "startsWith" in path ? `${host}${path.startsWith}*` : `${host} (path ~ /${path.regex}/)`;
152
- return `${method?.join("|") ?? "*"} ${where}`;
153
- }
154
- /** The resolved environment and what the gateway brokers, by name. */
155
- function traceEnvironment(debug, spec, env) {
156
- const r = env.record;
157
- debug.log("env", `environment ${r.name} (digest ${r.digest.replace(/^sha256:/, "").slice(0, 12)})${r.parent ? ` · extends ${r.parent}` : ""}${r.overrides.length ? ` · overrides ${r.overrides.join(", ")}` : ""}`);
158
- const unset = r.unset.length ? ` · unset (optional) ${r.unset.join(", ")}` : "";
159
- debug.log("env", `vars ${r.vars.join(", ") || "none"} · secrets ${r.secrets.join(", ") || "none"}${unset}`);
160
- for (const s of env.substitutions) debug.log("broker", `${s.name} is a placeholder in the sandbox · swapped in on HTTPS to ${s.hosts.join(", ")} in the ${s.in.join(", ")} · for ${s.phase === "checks" ? "checks and setup only" : "the agent and checks"}`);
161
- const brokering = brokeringFor(spec.harness, spec.vertex);
162
- if (brokering) {
163
- const endpoints = brokering.endpoints.map(endpointLabel).join(", ");
164
- debug.log("broker", `${brokering.credential} stays on the host · the gateway swaps it in on ${endpoints}`);
165
- }
166
- debug.trace("env", "environment", {
167
- record: r,
168
- vars: Object.keys(env.vars).sort(),
169
- checkVars: Object.keys(env.checkVars).sort(),
170
- files: env.files.map((f) => f.path),
171
- setup: env.setup,
172
- skills: env.skills.map((s) => s.name),
173
- workspaceSkills: env.workspaceSkills.map((s) => s.name),
174
- plugins: env.plugins.map((p) => ({
175
- name: p.name,
176
- parts: p.parts,
177
- files: p.files.length
178
- })),
179
- mcp: Object.fromEntries(Object.entries(env.mcp).map(([name, m]) => [name, m.transport === "stdio" ? {
180
- transport: m.transport,
181
- command: m.command,
182
- args: m.args,
183
- env: Object.keys(m.env)
184
- } : {
185
- transport: m.transport,
186
- url: m.url,
187
- headers: Object.keys(m.headers)
188
- }])),
189
- brokered: env.substitutions.map(({ name, hosts, in: where, phase }) => ({
190
- name,
191
- hosts,
192
- in: where,
193
- phase
194
- }))
195
- }, "resolved environment");
196
- }
45
+ import { setTimeout as setTimeout$1 } from "node:timers/promises";
46
+ import { resolveBridgePort } from "@ai-sdk/harness/utils";
197
47
  // src/harness/fake.ts
198
48
  const lineCount = (text) => text ? text.split("\n").length - (text.endsWith("\n") ? 1 : 0) : 0;
199
49
  /**
@@ -206,12 +56,17 @@ const lineCount = (text) => text ? text.split("\n").length - (text.endsWith("\n"
206
56
  */
207
57
  const createFakeHarness = (descriptor) => {
208
58
  let starts = 0;
209
- return { async start(sandbox, _signal, { skills, workspaceSkills, mcp: all, plugins }) {
59
+ return { async start(sandbox, signal, { skills, workspaceSkills, mcp: all, plugins }) {
210
60
  if (descriptor.failWith) throw new InfraError(descriptor.failWith);
211
61
  const hangs = descriptor.startHangs;
212
62
  if (hangs === true || starts++ < (hangs ?? 0)) return new Promise(() => {});
63
+ if (descriptor.bootstrap) {
64
+ const r = await sandbox.exec(fakeBootstrapScript(descriptor.bootstrap), { signal });
65
+ if (r.exitCode !== 0) throw new InfraError(`The fake agent's bootstrap exited ${r.exitCode}: ${(r.stderr || r.stdout).trim().slice(-500)}`);
66
+ }
213
67
  const layout = layoutOf(descriptor);
214
68
  const brokered = descriptor.brokers ? await brokerKey(descriptor, sandbox) : void 0;
69
+ if (descriptor.brokers && AGENTS[descriptor.layout ?? "cursor"].bridgePort) await reachBridge(sandbox, signal);
215
70
  const mcp = agentMcp(layout, all, plugins);
216
71
  for (const path of await writePluginFiles(sandbox, layout, plugins)) debugLog("env", `plugin settings → ${path} by the fake agent`);
217
72
  const dir = `~/${layout.harnessSkillDir}`;
@@ -231,9 +86,11 @@ const createFakeHarness = (descriptor) => {
231
86
  const files = (skilled ? descriptor.withSkill?.files : void 0) ?? descriptor.files ?? {};
232
87
  const reply = (skilled ? descriptor.withSkill?.text : void 0) ?? descriptor.text ?? "Done.";
233
88
  const flaked = Math.random() < (descriptor.flakeRate ?? 0);
89
+ let running;
234
90
  return {
235
91
  async send(prompt, signal) {
236
92
  const startedAt = Date.now();
93
+ const brokeredBefore = sandbox.brokeredStatuses?.().length ?? 0;
237
94
  const script = descriptor.script ?? [];
238
95
  const steps = script[Math.min(turns++, script.length - 1)] ?? [];
239
96
  const at = () => {
@@ -261,10 +118,19 @@ const createFakeHarness = (descriptor) => {
261
118
  durationMs: descriptor.turnMs ?? Date.now() - startedAt,
262
119
  transcript,
263
120
  usage: descriptor.usage ?? null,
121
+ ...!descriptor.costUsd ? {} : { cost: {
122
+ usd: descriptor.costUsd,
123
+ basis: "reported",
124
+ source: "fake"
125
+ } },
264
126
  model: descriptor.servedModel ?? null,
265
127
  unreported: [],
266
128
  ...descriptor.turnMs === void 0 ? {} : { agentMs: descriptor.turnMs }
267
129
  });
130
+ running = (note) => ({
131
+ ...done("", "aborted"),
132
+ transcript: [...transcript, note]
133
+ });
268
134
  const write = async (path, content) => {
269
135
  const time = at();
270
136
  const before = await sandbox.readFile(path);
@@ -294,8 +160,9 @@ const createFakeHarness = (descriptor) => {
294
160
  tool: tool.tool,
295
161
  input: tool.input,
296
162
  output: tool.output,
163
+ outputView: "model",
297
164
  ok: tool.ok === void 0 ? true : tool.ok,
298
- durationMs: 0
165
+ durationMs: tool.durationMs === void 0 ? 0 : tool.durationMs
299
166
  });
300
167
  if (flaked) {
301
168
  const text = "I ran out of ideas and left the workspace unchanged.";
@@ -323,6 +190,7 @@ const createFakeHarness = (descriptor) => {
323
190
  tool: "shell",
324
191
  input: { command: step.exec },
325
192
  output: r.stdout,
193
+ outputView: "model",
326
194
  ok: r.exitCode === 0,
327
195
  durationMs: Date.now() - startedAt - time.atMs
328
196
  });
@@ -340,6 +208,7 @@ const createFakeHarness = (descriptor) => {
340
208
  tool: `mcp__${server}__${tool}`,
341
209
  input: {},
342
210
  output: r.output,
211
+ outputView: "model",
343
212
  ok: r.ok,
344
213
  durationMs: Date.now() - startedAt - time.atMs
345
214
  });
@@ -350,7 +219,16 @@ const createFakeHarness = (descriptor) => {
350
219
  type: "error",
351
220
  message: step.error
352
221
  });
353
- return done("", "error");
222
+ const statuses = sandbox.brokeredStatuses?.().slice(brokeredBefore) ?? [];
223
+ const stopped = providerStop({
224
+ errors: [step.error],
225
+ status: "error",
226
+ statuses
227
+ });
228
+ return {
229
+ ...done("", "error"),
230
+ ...stopped && { providerStop: stopped }
231
+ };
354
232
  } else if ("refuseKey" in step) {
355
233
  const message = `${keyRejected(step.refuseKey)}. 401 Unauthorized (the fake agent's)`;
356
234
  debugLog("harness", `turn failed: ${message}`);
@@ -379,6 +257,10 @@ const createFakeHarness = (descriptor) => {
379
257
  });
380
258
  return done(text, "completed");
381
259
  },
260
+ stopped: async (note) => {
261
+ if (!running) throw new Error("No turn is running");
262
+ return running(note);
263
+ },
382
264
  async close() {}
383
265
  };
384
266
  } };
@@ -388,7 +270,7 @@ const createFakeHarness = (descriptor) => {
388
270
  * endpoints, as a real adapter does, and returns the variable its `exec` steps get it in.
389
271
  */
390
272
  async function brokerKey(descriptor, sandbox) {
391
- const { credential, endpoints } = AGENTS[descriptor.layout ?? "cursor"];
273
+ const { credential, endpoints } = fakeBrokering(descriptor.layout);
392
274
  const key = process.env[credential];
393
275
  if (!key) throw new InfraError(`${credential} isn't set, so the fake agent has no key to broker. Set it in the environment or .env.local.`);
394
276
  const network = hasNetworkSession(sandbox) ? sandbox.network : void 0;
@@ -408,6 +290,40 @@ async function brokerKey(descriptor, sandbox) {
408
290
  return { [credential]: placeholder };
409
291
  }
410
292
  /**
293
+ * Serves on the port the layout's adapter would put its bridge on, in the sandbox, and reaches it
294
+ * from the host through the session's endpoint, as that adapter's start does.
295
+ */
296
+ async function reachBridge(sandbox, signal) {
297
+ const network = hasNetworkSession(sandbox) ? sandbox.network : void 0;
298
+ if (!network) throw new InfraError("The fake agent's bridge needs a sandbox session");
299
+ let port;
300
+ try {
301
+ port = resolveBridgePort({
302
+ sandboxSession: network,
303
+ portOverride: void 0
304
+ });
305
+ } catch (error) {
306
+ throw new InfraError(`Fake harness failed to start: ${error.message}`);
307
+ }
308
+ const server = await network.spawn({
309
+ command: `node -e ${shellQuote$1(`require('http').createServer((q, s) => s.end('bridge')).listen(${port}, '0.0.0.0', () => console.log('up'))`)}`,
310
+ abortSignal: signal
311
+ });
312
+ try {
313
+ const up = await server.stdout.getReader().read();
314
+ if (!new TextDecoder().decode(up.value).includes("up")) throw new InfraError(`The fake agent's bridge didn't listen on ${port}`);
315
+ const { url } = await network.getPortEndpoint({
316
+ port,
317
+ protocol: "http"
318
+ });
319
+ const reply = await (await fetch(url, { signal })).text();
320
+ if (reply !== "bridge") throw new InfraError(`${url} answered ${JSON.stringify(reply)}`);
321
+ debugLog("harness", `bridge port ${port} reached through ${url}`);
322
+ } finally {
323
+ await server.kill();
324
+ }
325
+ }
326
+ /**
411
327
  * Starts a stdio MCP server in the sandbox, as an agent would, and calls one tool with no
412
328
  * arguments: initialize, then the call.
413
329
  */
@@ -445,8 +361,8 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
445
361
  arguments: {}
446
362
  }
447
363
  }
448
- ].map((m) => shellQuote(JSON.stringify(m)));
449
- const command = [server.command, ...server.args].map(shellQuote).join(" ");
364
+ ].map((m) => shellQuote$1(JSON.stringify(m)));
365
+ const command = [server.command, ...server.args].map(shellQuote$1).join(" ");
450
366
  const script = [
451
367
  "d=$(mktemp -d) && : > \"$d/out\" && : > \"$d/err\" || exit 1",
452
368
  "{",
@@ -486,13 +402,118 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
486
402
  /** The attempt's harness, for the sandbox it runs in; an agent the sandbox can't run is an infra error. */
487
403
  async function createHarness(descriptor, sandbox, vertex) {
488
404
  if (descriptor.kind === "fake") return createFakeHarness(descriptor);
489
- const { createAgentHarness } = await import("./agent-BmsMXZXx.js");
405
+ const { createAgentHarness } = await import("./agent-Dx-bDmlV.js");
490
406
  return createAgentHarness(descriptor, sandbox, vertex);
491
407
  }
492
408
  /** Where each harness's agent looks for repo skills, relative to the workspace. */
493
409
  const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
410
+ // src/attempt/debug.ts
411
+ /** Which config and profile the attempt runs under, and where each resolved option came from. */
412
+ function tracePlan(debug, spec, ctx) {
413
+ const config = ctx.configFile ?? "none (no config file found)";
414
+ const from = (key, fallback = ctx.configFile ? `${config} or its default` : "default") => `(${spec.sources[key] ?? fallback})`;
415
+ const profile = ctx.profile ? `profile ${ctx.profile.name} (${ctx.profile.reason})` : "no profile";
416
+ debug.log("plan", `config ${config} · ${profile} · run ${ctx.runId} · attempt ${spec.cell.key}#${spec.n}`);
417
+ debug.log("plan", `harness ${harnessId(spec.harness)} ${from("harnesses")}`);
418
+ debug.log("plan", `sandbox ${sandboxId(spec.sandbox)} ${from("sandbox")}`);
419
+ const env = spec.environment;
420
+ const role = spec.cell.role === "variant" ? ` · variant ${spec.cell.variant}` : "";
421
+ const envFrom = spec.cell.role === "variant" || spec.sources.compare ? from("compare") : from("environments", "none set");
422
+ debug.log("plan", `environment ${env.name}${role} ${envFrom}`);
423
+ debug.log("plan", `egress ${egressLabel(spec.egress) || "proxy"} ${from("egress")}`);
424
+ const gate = typeof spec.gate === "string" ? spec.gate : `passRate ${spec.gate.passRate}`;
425
+ debug.log("plan", `run ${spec.n} · runs ${ctx.runs !== void 0 ? "(--runs)" : from("runs")} · gate ${gate} · timeout ${formatDuration(spec.timeoutMs)} ${from("timeout", "config.timeout or 15m")}`);
426
+ const judge = spec.judge;
427
+ const model = typeof judge.model === "string" ? judge.model : judge.model && "kind" in judge.model ? "fake judge" : judge.model ? "a model object" : "none";
428
+ debug.log("plan", `judge ${model}${judge.threshold !== void 0 ? ` · threshold ${judge.threshold}` : ""} ${from("judge")}`);
429
+ for (const note of spec.notes) debug.log("plan", `profile: ${note}`);
430
+ debug.trace("plan", "config", {
431
+ config: ctx.config,
432
+ profile: ctx.profile
433
+ }, "resolved config");
434
+ debug.trace("plan", "attempt", {
435
+ cell: spec.cell,
436
+ n: spec.n,
437
+ harness: spec.harness,
438
+ sandbox: spec.sandbox,
439
+ egress: spec.egress,
440
+ gate: spec.gate,
441
+ timeoutMs: spec.timeoutMs,
442
+ judge: spec.judge,
443
+ sources: spec.sources,
444
+ env: Object.keys(spec.env).sort()
445
+ }, "resolved attempt");
446
+ }
447
+ /** `POST api2.cursor.sh/auth/exchange`, `GET host/v1/*` or `POST host (path ~ /re/)`. */
448
+ function endpointLabel({ method, host, path }) {
449
+ const where = !path ? host : "exact" in path ? `${host}${path.exact}` : "startsWith" in path ? `${host}${path.startsWith}*` : `${host} (path ~ /${path.regex}/)`;
450
+ return `${method?.join("|") ?? "*"} ${where}`;
451
+ }
452
+ /** The resolved environment and what the gateway brokers, by name. */
453
+ function traceEnvironment(debug, spec, env) {
454
+ const r = env.record;
455
+ debug.log("env", `environment ${r.name} (digest ${r.digest.replace(/^sha256:/, "").slice(0, 12)})${r.parent ? ` · extends ${r.parent}` : ""}${r.overrides.length ? ` · overrides ${r.overrides.join(", ")}` : ""}`);
456
+ const unset = r.unset.length ? ` · unset (optional) ${r.unset.join(", ")}` : "";
457
+ debug.log("env", `vars ${r.vars.join(", ") || "none"} · secrets ${r.secrets.join(", ") || "none"}${unset}`);
458
+ for (const s of env.substitutions) debug.log("broker", `${s.name} is a placeholder in the sandbox · swapped in on HTTPS to ${s.hosts.join(", ")} in the ${s.in.join(", ")} · for ${s.phase === "checks" ? "checks and setup only" : "the agent and checks"}`);
459
+ const brokering = brokeringFor(spec.harness, spec.vertex);
460
+ if (brokering) {
461
+ const endpoints = brokering.endpoints.map(endpointLabel).join(", ");
462
+ debug.log("broker", `${brokering.credential} stays on the host · the gateway swaps it in on ${endpoints}`);
463
+ }
464
+ debug.trace("env", "environment", {
465
+ record: r,
466
+ vars: Object.keys(env.vars).sort(),
467
+ checkVars: Object.keys(env.checkVars).sort(),
468
+ files: env.files.map((f) => f.path),
469
+ setup: env.setup,
470
+ skills: env.skills.map((s) => s.name),
471
+ workspaceSkills: env.workspaceSkills.map((s) => s.name),
472
+ plugins: env.plugins.map((p) => ({
473
+ name: p.name,
474
+ parts: p.parts,
475
+ files: p.files.length
476
+ })),
477
+ mcp: Object.fromEntries(Object.entries(env.mcp).map(([name, m]) => [name, m.transport === "stdio" ? {
478
+ transport: m.transport,
479
+ command: m.command,
480
+ args: m.args,
481
+ env: Object.keys(m.env)
482
+ } : {
483
+ transport: m.transport,
484
+ url: m.url,
485
+ headers: Object.keys(m.headers)
486
+ }])),
487
+ brokered: env.substitutions.map(({ name, hosts, in: where, phase }) => ({
488
+ name,
489
+ hosts,
490
+ in: where,
491
+ phase
492
+ }))
493
+ }, "resolved environment");
494
+ }
494
495
  // src/attempt/environment.ts
495
496
  /**
497
+ * How many VMs the hosts preflight kept for `spec`'s sandbox run at once, or null when it runs
498
+ * on no hosts or preflight left nothing to read. The plan's cap counts every host it names at its
499
+ * `vms`; a host skipped, or with memory for fewer, runs fewer.
500
+ */
501
+ async function preparedVmCap(spec, root, runId) {
502
+ const { sandbox, harness } = spec;
503
+ if (sandbox.kind !== "tart" || !sandbox.hosts) return null;
504
+ const { preparedFor } = await import("./preflight-BUjk_6wT.js");
505
+ const { createProviders, preparedHosts } = await import("./providers-Dljf6al6.js");
506
+ const provider = createProviders().tart;
507
+ if (!provider) return null;
508
+ try {
509
+ const prepared = preparedFor(root, runId, provider, sandbox, harness);
510
+ const hosts = prepared ? preparedHosts(prepared) : null;
511
+ return hosts ? hosts.reduce((sum, h) => sum + h.vms, 0) : null;
512
+ } catch {
513
+ return null;
514
+ }
515
+ }
516
+ /**
496
517
  * Resolves the attempt's environment (files, skills, secrets and their placeholders)
497
518
  * and finds the image its sandbox starts from, when its kind has one, which preflight prepared.
498
519
  * Runs on the test's signal, before the eval timer starts.
@@ -501,7 +522,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
501
522
  const { spec, record } = a;
502
523
  const env = await resolveEnvironment(spec.environment, spec.env, root, where, layoutOf(spec.harness).rulesFile, spec.egress.policy, (w) => preparedWorkspace(root, runId, w), {
503
524
  harness: spec.harness,
504
- allowed: allowedHosts(spec.egress)
525
+ allowed: allowedHosts(spec.egress),
526
+ ...spec.egress.setup ? { setupEgress: spec.egress.setup.policy } : {}
505
527
  });
506
528
  a.env = env;
507
529
  record.environment = env.record;
@@ -516,8 +538,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
516
538
  record.image = null;
517
539
  const kind = sandboxKind(spec.sandbox);
518
540
  if (kind.capabilities.images) {
519
- const { preparedFor } = await import("./preflight-D1C-iH0U.js");
520
- const { attemptImage, createProviders, preparedImage } = await import("./providers-B-ijCktf.js");
541
+ const { preparedFor } = await import("./preflight-BUjk_6wT.js");
542
+ const { attemptImage, createProviders, preparedImage } = await import("./providers-Dljf6al6.js");
521
543
  signal.throwIfAborted();
522
544
  const provider = createProviders()[kind.provider(spec.sandbox)];
523
545
  const builds = builtPlugins(spec.environment.layers);
@@ -542,16 +564,88 @@ async function prepareEnvironment(a, root, runId, where, signal) {
542
564
  const { ref, id, base, install, agentCli } = record.image;
543
565
  debugLog("image", `ready ${ref} (${id.replace(/^sha256:/, "").slice(0, 12)}) · prepared before the run${ref !== base ? ` · on ${base}` : ""}${agentCli ? ` · agent CLI ${agentCli}` : ""}${install.length ? ` · install: ${install.join(" && ")}` : ""}`);
544
566
  } else if (kind.capabilities.isolated) {
545
- const { preparedFor } = await import("./preflight-D1C-iH0U.js");
546
- const { createProviders, vmImage } = await import("./providers-B-ijCktf.js");
567
+ const { preparedFor } = await import("./preflight-BUjk_6wT.js");
568
+ const { bakedImage, createProviders, preparedHosts, vmImage } = await import("./providers-Dljf6al6.js");
547
569
  const provider = createProviders()[kind.provider(spec.sandbox)];
548
570
  const prepared = provider && preparedFor(root, runId, provider, spec.sandbox, spec.harness);
549
571
  const vm = prepared ? vmImage(prepared) : null;
550
572
  if (vm) record.vmImage = vm;
573
+ const baked = prepared ? bakedImage(prepared) : null;
574
+ if (baked) a.image = baked;
575
+ const hosts = prepared ? preparedHosts(prepared) : null;
576
+ if (hosts) a.hosts = hosts;
551
577
  const from = vm?.ref ?? ("image" in spec.sandbox ? spec.sandbox.image : void 0);
552
- debugLog("image", `${from ?? "none recorded"}${vm?.digest ? ` (${vm.digest})` : ""} · ${kind.helpers[0]}() checked it before the run and starts each attempt from it`);
578
+ debugLog("image", `${from ?? "none recorded"}${vm?.digest ? ` (${vm.digest})` : ""} · ${kind.helpers[0]}() checked it before the run and starts each attempt from ${baked ? `${baked}, which has the harness baked in` : "it"}`);
553
579
  } else debugLog("image", kind.capabilities.isolated ? `none: the ${spec.sandbox.kind} sandbox starts from its provider's base` : `none: the ${spec.sandbox.kind} sandbox runs on the host`);
554
580
  }
581
+ // src/attempt/assertions.ts
582
+ const MESSAGE_LIMIT = 1024;
583
+ const LABEL_LIMIT = 60;
584
+ /** Masked first, so a secret the limit would cut through is masked whole. */
585
+ const clipped = (text, limit) => {
586
+ const s = redactSecrets(text);
587
+ return s.length > limit ? `${s.slice(0, limit)}…` : s;
588
+ };
589
+ const text = (error) => error instanceof Error ? error.message : String(error);
590
+ const shown = (value) => {
591
+ if (typeof value === "string") return value;
592
+ if (value instanceof RegExp) return String(value);
593
+ try {
594
+ return JSON.stringify(value) ?? String(value);
595
+ } catch {
596
+ return String(value);
597
+ }
598
+ };
599
+ const shownSubject = (subject) => typeof subject === "string" && subject ? subject.replace(/\s+/g, " ").trim() : void 0;
600
+ /**
601
+ * The records of an attempt's matcher calls, in call order. Without a label, one is
602
+ * `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3`, skipping
603
+ * any label already recorded, so each record's label is unique (a given `build #2` after two
604
+ * `build`s is `build #2 #2`).
605
+ */
606
+ function assertionRecords(raw) {
607
+ const seen = /* @__PURE__ */ new Map();
608
+ const used = /* @__PURE__ */ new Set();
609
+ return raw.map((r) => {
610
+ const named = r.matcher === "toStayWithin" ? limitsLabel(r.args[0]) : r.matcher === "toHaveRun" && runMatchLabel(r.args[0]) !== void 0 ? runMatchLabel(r.args[0]) : r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
611
+ const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
612
+ let n = seen.get(base) ?? 0;
613
+ let label;
614
+ do {
615
+ n += 1;
616
+ label = n > 1 ? `${base} #${n}` : base;
617
+ } while (used.has(label));
618
+ seen.set(base, n);
619
+ used.add(label);
620
+ return {
621
+ label,
622
+ labelled: r.label !== void 0,
623
+ matcher: r.matcher,
624
+ status: r.status,
625
+ soft: r.soft,
626
+ ...r.at ? { at: r.at } : {},
627
+ ...r.status === "pass" ? {} : { message: clipped(text(r.error), MESSAGE_LIMIT) },
628
+ ...r.durationMs === void 0 ? {} : { durationMs: r.durationMs }
629
+ };
630
+ });
631
+ }
632
+ const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
633
+ /**
634
+ * What fails the attempt at its end for the checks that failed without ending the body: the first
635
+ * that didn't pass, an `expect` as what it failed with, an `expect.require` the eval caught with
636
+ * `CAUGHT_HINT` after its message.
637
+ */
638
+ function checkFailure(raw) {
639
+ const failed = raw.find((r) => r.status !== "pass");
640
+ if (!failed) return void 0;
641
+ const error = failed.error instanceof Error ? failed.error : new Error(text(failed.error));
642
+ if (failed.soft) return error;
643
+ const caught = Object.setPrototypeOf(Object.assign(/* @__PURE__ */ new Error(), error), Object.getPrototypeOf(error));
644
+ caught.name = error.name;
645
+ caught.message = `${error.message} ${CAUGHT_HINT}`;
646
+ caught.stack = error.stack?.replace(error.message, caught.message);
647
+ return caught;
648
+ }
555
649
  // src/judge/size.ts
556
650
  /** Roughly four characters a token, as `reeval analyze` counts: an estimate, not the model's. */
557
651
  const CHARS_PER_TOKEN = 4;
@@ -607,6 +701,19 @@ function stateHint(state, question) {
607
701
  * no model's limit, so this follows every provider error.
608
702
  */
609
703
  const withInputSize = (message, state, question) => `${message}\nThe judge's input was ~${k(inputTokens(state, question))} tokens (estimated): ${describeSize(state, question)}.${stateHint(state, question)}`;
704
+ const count = (digits) => Number(digits.replaceAll(",", ""));
705
+ /**
706
+ * Whether the provider refused the call on a token limit this input alone is over, which no
707
+ * retry fixes: a request too large, OpenAI's `Limit 30000, Requested 45000`, or Anthropic's
708
+ * `of 30,000 input tokens per minute` against the input's estimated size.
709
+ */
710
+ function overTokenLimit(message, state, question) {
711
+ if (/request too large/i.test(message)) return true;
712
+ const asked = /Limit:? ([\d,]+), Requested:? ([\d,]+)/i.exec(message);
713
+ if (asked) return count(asked[2]) > count(asked[1]);
714
+ const perMinute = /([\d,]+) (?:input )?tokens per minute/i.exec(message);
715
+ return !!perMinute && inputTokens(state, question) >= count(perMinute[1]);
716
+ }
610
717
  // src/judge/state.ts
611
718
  function turnState(t) {
612
719
  return {
@@ -778,10 +885,16 @@ function judgeCaller(a, readFile) {
778
885
  debugLog("judge", `asking ${name}: ${JSON.stringify(clip(redactSecrets(question), 160))}`);
779
886
  if (dropped.length) debugLog("judge", `left out of the judge's input: ${dropped.join(" · ")}`);
780
887
  debugTrace("judge", "prompt", input, "judge prompt");
781
- const failed = (message, retries) => {
888
+ const failed = (message, transient, retries) => {
782
889
  a.judgeErrors.push({
783
890
  call: order,
784
- message: `c.judge(${JSON.stringify(clip(redactSecrets(question), 80))}) failed: ${message}`
891
+ message: `c.judge(${JSON.stringify(clip(redactSecrets(question), 80))}) failed: ${message}`,
892
+ transient,
893
+ matcher: judgeFailedMessage({
894
+ question,
895
+ model: name,
896
+ error: message
897
+ })
785
898
  });
786
899
  input.error = message;
787
900
  if (retries?.length) input.retries = retries;
@@ -819,7 +932,7 @@ function judgeCaller(a, readFile) {
819
932
  ...unpinned,
820
933
  gateway: entry || gateway
821
934
  } : unpinned;
822
- const { verdict, retries } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
935
+ const { verdict, retries, usage } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
823
936
  ...routed,
824
937
  model,
825
938
  threshold: atLeast
@@ -832,6 +945,7 @@ function judgeCaller(a, readFile) {
832
945
  judgments.splice(placeFor(judgments.map((j) => j.call ?? 0), order), 0, {
833
946
  ...recorded,
834
947
  ...retries.length ? { retries } : {},
948
+ ...usage ? { usage } : {},
835
949
  call: order
836
950
  });
837
951
  const { score } = verdict;
@@ -853,7 +967,8 @@ function judgeCaller(a, readFile) {
853
967
  if (a.signal.aborted) throw error;
854
968
  const { retries } = error;
855
969
  const message = error instanceof Error ? error.message : String(error);
856
- return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
970
+ const transient = isTransient(error) && !overTokenLimit(message, state, question);
971
+ return failed(isApiError(error) && !transient ? withInputSize(message, state, question) : message, transient, retries);
857
972
  }
858
973
  };
859
974
  return async (question, given) => {
@@ -878,6 +993,191 @@ async function createSandbox(descriptor, options) {
878
993
  throw new InfraError(`Failed to create ${kind.provider(descriptor)} sandbox: ${error.message}`);
879
994
  }
880
995
  }
996
+ // src/view/timeline.ts
997
+ const isToolLike = (e) => e.type === "file-edit" || e.type === "web-search" || e.type === "web-fetch" || e.type === "tool";
998
+ /**
999
+ * The events with each tool call's missing `durationMs` run to the next event's `atMs` (the
1000
+ * turn's end for the last), flagged `durationInferred`. A main model call ends it too: the agent
1001
+ * only calls the model again once the tool's result is back. A side call or a call that carries
1002
+ * the whole turn doesn't.
1003
+ */
1004
+ function inferDurations(events, turnMs) {
1005
+ return events.map((e, i) => {
1006
+ if (!isToolLike(e) || e.durationMs !== null) return e;
1007
+ const next = events.slice(i + 1).find((n) => n.atMs > e.atMs && (n.type !== "model-call" || !n.stream && n.role !== "side"));
1008
+ const end = Math.max(e.atMs, next ? next.atMs : turnMs);
1009
+ return {
1010
+ ...e,
1011
+ durationMs: end - e.atMs,
1012
+ durationInferred: true
1013
+ };
1014
+ });
1015
+ }
1016
+ const SHELL_TOOL = /shell|bash|terminal|command|exec/i;
1017
+ const OUTPUT_TOOL = /^(bash_?output|task_?output|await_?shell)$/i;
1018
+ const CURSOR_TERMINAL = /\.cursor\/projects\/[^"]*\/terminals\/[^"/]+\.txt/;
1019
+ const TERMINAL_COMMAND = /command:\s*("(?:[^"\\]|\\.)*")/;
1020
+ const record = (v) => v && typeof v === "object" ? v : {};
1021
+ /** A shell call's command, or null for another tool. */
1022
+ function commandOf(e) {
1023
+ if (e.type !== "tool" || !SHELL_TOOL.test(e.tool) || OUTPUT_TOOL.test(e.tool)) return null;
1024
+ const input = record(e.input);
1025
+ const command = input.command ?? input.cmd;
1026
+ return typeof command === "string" ? command.trim() : null;
1027
+ }
1028
+ const strings = (v) => typeof v === "string" ? [v] : v && typeof v === "object" ? Object.values(v).flatMap(strings) : [];
1029
+ const textOf = (v) => typeof v === "string" ? v : JSON.stringify(v) ?? "";
1030
+ /** The command a Cursor terminal file's header names, from a read of it. */
1031
+ function terminalCommand(e) {
1032
+ if (e.type !== "tool") return null;
1033
+ const match = strings(e.output).map((text) => TERMINAL_COMMAND.exec(text)).find(Boolean);
1034
+ if (!match?.[1]) return null;
1035
+ try {
1036
+ return String(JSON.parse(match[1])).trim();
1037
+ } catch {
1038
+ return null;
1039
+ }
1040
+ }
1041
+ function backgrounded(e, named) {
1042
+ if (e.type !== "tool") return false;
1043
+ const input = record(e.input);
1044
+ if (input.run_in_background === true || input.is_background === true) return true;
1045
+ const command = commandOf(e);
1046
+ if (command === null) return false;
1047
+ return /(^|[^&])&$/.test(command) || /(^|[\s;&|])(nohup|setsid)\s/.test(command) || named.has(command);
1048
+ }
1049
+ function polls(e) {
1050
+ if (e.type !== "tool") return false;
1051
+ if (OUTPUT_TOOL.test(e.tool)) return true;
1052
+ const command = commandOf(e);
1053
+ if (command !== null) return /^(sleep|wait)(\s|$)/.test(command);
1054
+ return CURSOR_TERMINAL.test(textOf(e.input));
1055
+ }
1056
+ const endOf = (e) => e.atMs + (e.durationMs ?? 0);
1057
+ /** Work that ends a wait no poll has checked yet: an edit, or another command. */
1058
+ const works = (e) => e.type === "file-edit" || commandOf(e) !== null;
1059
+ /**
1060
+ * The background jobs the agent waited on, a heuristic over the transcript: a span opens at a
1061
+ * call run in the background or a poll and takes in each poll after it. Until its first poll,
1062
+ * lookups (reads, searches) leave it open and an edit or another command drops it; after it,
1063
+ * the next call that isn't a poll closes it. A background call no poll followed isn't waited on.
1064
+ */
1065
+ function waitSpans(events) {
1066
+ const named = new Set(events.map(terminalCommand).filter((c) => c !== null));
1067
+ const spans = [];
1068
+ let open = null;
1069
+ const close = () => {
1070
+ if (open?.polls.length) spans.push(open);
1071
+ open = null;
1072
+ };
1073
+ events.forEach((e, i) => {
1074
+ if (!isToolLike(e) || e.subagent) return;
1075
+ if (backgrounded(e, named)) {
1076
+ if (open?.polls.length) close();
1077
+ open ??= {
1078
+ startMs: e.atMs,
1079
+ endMs: endOf(e),
1080
+ background: i,
1081
+ polls: []
1082
+ };
1083
+ } else if (polls(e)) {
1084
+ open ??= {
1085
+ startMs: e.atMs,
1086
+ endMs: e.atMs,
1087
+ background: null,
1088
+ polls: []
1089
+ };
1090
+ open.polls.push(i);
1091
+ open.endMs = Math.max(open.endMs, endOf(e));
1092
+ } else if (open?.polls.length || works(e)) close();
1093
+ });
1094
+ close();
1095
+ return spans;
1096
+ }
1097
+ const PRIORITY = [
1098
+ "waiting",
1099
+ "tool",
1100
+ "model"
1101
+ ];
1102
+ /** The turn from 0 to `turnMs` as segments, each moment the first of waiting, tool, model and other. */
1103
+ function segments(events, turnMs) {
1104
+ const marks = [];
1105
+ const add = (kind, start, end) => {
1106
+ const s = Math.max(0, Math.min(turnMs, start));
1107
+ const e = Math.max(0, Math.min(turnMs, end));
1108
+ if (e <= s) return;
1109
+ marks.push({
1110
+ at: s,
1111
+ kind,
1112
+ delta: 1
1113
+ }, {
1114
+ at: e,
1115
+ kind,
1116
+ delta: -1
1117
+ });
1118
+ };
1119
+ for (const span of waitSpans(events)) add(0, span.startMs, span.endMs);
1120
+ for (const e of events) if (isToolLike(e)) add(1, e.atMs, endOf(e));
1121
+ else if (e.type === "model-call" && !e.stream) add(2, e.atMs, e.atMs + e.durationMs);
1122
+ marks.sort((a, b) => a.at - b.at);
1123
+ const depth = [
1124
+ 0,
1125
+ 0,
1126
+ 0
1127
+ ];
1128
+ const out = [];
1129
+ let at = 0;
1130
+ const push = (kind, end) => {
1131
+ if (end <= at) return;
1132
+ const last = out.at(-1);
1133
+ if (last?.kind === kind) last.endMs = end;
1134
+ else out.push({
1135
+ kind,
1136
+ startMs: at,
1137
+ endMs: end
1138
+ });
1139
+ at = end;
1140
+ };
1141
+ const current = () => PRIORITY[depth.findIndex((d) => d > 0)] ?? "other";
1142
+ for (const m of marks) {
1143
+ push(current(), m.at);
1144
+ depth[m.kind] = (depth[m.kind] ?? 0) + m.delta;
1145
+ }
1146
+ push(current(), turnMs);
1147
+ return out;
1148
+ }
1149
+ /** A turn's `timeSplit`, from its transcript (durations inferred) and length. */
1150
+ function timeSplit(events, turnMs) {
1151
+ const total = {
1152
+ waiting: 0,
1153
+ tool: 0,
1154
+ model: 0,
1155
+ other: 0
1156
+ };
1157
+ for (const s of segments(events, turnMs)) total[s.kind] += s.endMs - s.startMs;
1158
+ const seen = events.some((e) => e.type === "model-call" && !e.stream);
1159
+ const waitingMs = Math.round(total.waiting);
1160
+ const toolMs = Math.round(total.tool);
1161
+ const modelMs = Math.round(total.model);
1162
+ return {
1163
+ modelMs: seen ? modelMs : null,
1164
+ toolMs,
1165
+ waitingMs,
1166
+ otherMs: Math.max(0, Math.round(turnMs) - waitingMs - toolMs - modelMs)
1167
+ };
1168
+ }
1169
+ /** The turns' splits summed; `modelMs` null when any turn's is, undefined with none. */
1170
+ function sumSplits(splits) {
1171
+ const known = splits.filter((s) => s !== void 0);
1172
+ if (known.length === 0) return void 0;
1173
+ const sum = (f) => known.reduce((n, s) => n + f(s), 0);
1174
+ return {
1175
+ modelMs: known.every((s) => s.modelMs !== null) ? sum((s) => s.modelMs ?? 0) : null,
1176
+ toolMs: sum((s) => s.toolMs),
1177
+ waitingMs: sum((s) => s.waitingMs),
1178
+ otherMs: sum((s) => s.otherMs)
1179
+ };
1180
+ }
881
1181
  // src/attempt/record.ts
882
1182
  function classify(error, signal) {
883
1183
  const message = error instanceof Error ? error.message : String(error);
@@ -909,6 +1209,34 @@ function isJudges(a, error, raw) {
909
1209
  if (a.judgeThrown.has(message)) return true;
910
1210
  return raw.some((r) => r.status !== "pass" && judged(r) && (r.error === error || messageOf(r.error) === message));
911
1211
  }
1212
+ /** The error the attempt's failed judge calls fail it with once its body is done, if any failed. */
1213
+ function judgeErrorsMessage(a) {
1214
+ const messages = a.judgeErrors.toSorted((x, y) => x.call - y.call).map((e) => e.message);
1215
+ const [only, ...more] = messages;
1216
+ if (!only) return void 0;
1217
+ return more.length ? `${more.length + 1} judge calls failed:\n${messages.join("\n")}` : only;
1218
+ }
1219
+ /** A matcher's message without the `label: ` its `expect` puts first. */
1220
+ const unlabeled = (r) => {
1221
+ const message = messageOf(r.error);
1222
+ const prefix = r.label === void 0 ? "" : `${r.label}: `;
1223
+ return message.startsWith(prefix) ? message.slice(prefix.length) : message;
1224
+ };
1225
+ /**
1226
+ * Whether judge calls the provider couldn't answer (a rate limit, a 5xx, overload, the network,
1227
+ * after the retries) are all that failed the attempt: every failed call was transient, and every
1228
+ * check that didn't hold is a `toPass()` on one of them, not polled. A verdict that answered, a
1229
+ * non-transient judge error (a bad key or model, an input over the provider's limit) or any other
1230
+ * check makes it the attempt's own failure. By message, since another language's runner hands back
1231
+ * its own copy of the error.
1232
+ */
1233
+ function judgeOutage(a, error, raw) {
1234
+ if (!a.judgeErrors.length || !a.judgeErrors.every((e) => e.transient)) return false;
1235
+ const message = messageOf(error);
1236
+ if (a.judgeThrown.has(message) && message !== judgeErrorsMessage(a)) return false;
1237
+ const unanswered = new Set(a.judgeErrors.map((e) => e.matcher));
1238
+ return raw.every((r) => r.status === "pass" || judged(r) && !r.matcher.startsWith("poll.") && unanswered.has(unlabeled(r)));
1239
+ }
912
1240
  /**
913
1241
  * The error that fails the attempt: a judge's gives way to the first other check that didn't hold
914
1242
  * (an `expect`, or an `expect.require` the eval caught), whichever came first, so the attempt
@@ -921,7 +1249,8 @@ function attemptError(a, error, raw) {
921
1249
  }
922
1250
  /**
923
1251
  * Why the attempt failed, into the record: the eval's own timeout, a cancel, an infra error, or an
924
- * assertion, which is the judge's when `error`, from `attemptError`, is a judge's.
1252
+ * assertion, which is the judge's when `error`, from `attemptError`, is a judge's. A judge's that
1253
+ * only a provider outage caused is an infra error, so `--retry-errors` and `--resume` run it again.
925
1254
  */
926
1255
  function recordFailure(a, error, timedOut, raw) {
927
1256
  const { record } = a;
@@ -941,7 +1270,8 @@ function recordFailure(a, error, timedOut, raw) {
941
1270
  setBy: setBy ?? []
942
1271
  };
943
1272
  }
944
- if (record.error.kind === "assertion" && isJudges(a, error, raw)) record.error.kind = "judge";
1273
+ if (error instanceof ProviderStopError && !timedOut) record.error.provider = { ...error.stop };
1274
+ if (record.error.kind === "assertion" && isJudges(a, error, raw)) record.error.kind = judgeOutage(a, error, raw) ? "infra" : "judge";
945
1275
  record.outcome = record.error.kind === "infra" ? "infra_error" : "fail";
946
1276
  }
947
1277
  /**
@@ -965,6 +1295,8 @@ async function recordOom(a, sandbox) {
965
1295
  a.record.outcome = "infra_error";
966
1296
  return new InfraError(message);
967
1297
  }
1298
+ /** The gateway tags entries with the phase it was switched to, which is always one of the attempt's. */
1299
+ const phased = (entry) => entry;
968
1300
  /**
969
1301
  * The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
970
1302
  * Refused hosts the harness always tries (in a session like this one) are `expectedBlocked`,
@@ -974,17 +1306,23 @@ function recordEgress(a, sandbox) {
974
1306
  const traffic = sandbox?.traffic?.();
975
1307
  const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0, a.spec.vertex);
976
1308
  const refused = traffic?.blocked ?? [];
977
- const isExpected = (h) => !!expected?.hosts.includes(h.host);
1309
+ const isExpected = (h) => h.phase !== "setup" && !!expected?.hosts.includes(h.host);
978
1310
  const expectedBlocked = refused.filter(isExpected);
1311
+ const { setup } = a.spec.egress;
1312
+ const direct = isDirect(a.spec.egress.policy) || !!setup && isDirect(setup.policy);
979
1313
  a.record.egress = {
980
1314
  policy: a.spec.egress.policy,
981
1315
  allowed: allowedHosts(a.spec.egress),
1316
+ ...setup ? { setup: {
1317
+ policy: setup.policy,
1318
+ allowed: allowedHosts(setup)
1319
+ } } : {},
982
1320
  recorded: !!traffic,
983
- ...traffic && isDirect(a.spec.egress.policy) ? { partial: true } : {},
984
- contacted: traffic?.contacted ?? [],
985
- blocked: refused.filter((h) => !isExpected(h)),
986
- expectedBlocked,
987
- unresolved: traffic?.unresolved ?? []
1321
+ ...traffic && direct ? { partial: true } : {},
1322
+ contacted: (traffic?.contacted ?? []).map(phased),
1323
+ blocked: refused.filter((h) => !isExpected(h)).map(phased),
1324
+ expectedBlocked: expectedBlocked.map(phased),
1325
+ unresolved: (traffic?.unresolved ?? []).map(phased)
988
1326
  };
989
1327
  if (expected && expectedBlocked.length) {
990
1328
  const hosts = expectedBlocked.map((h) => `${h.host}:${h.port} ×${h.connections}`).join(", ");
@@ -999,6 +1337,25 @@ function recordEgress(a, sandbox) {
999
1337
  if (cost !== void 0) a.record.reportedCostUsd = cost;
1000
1338
  }
1001
1339
  /**
1340
+ * Each turn's model calls and inferred tool durations in its transcript, and where its time
1341
+ * went. After teardown, so a stream that outlived its turn (Cursor's) has ended.
1342
+ */
1343
+ function recordTimeline(a, sandbox) {
1344
+ const calls = sandbox?.modelCalls?.() ?? [];
1345
+ for (const turn of a.record.turns) {
1346
+ const events = inferDurations([...turn.transcript, ...modelCallEvents(calls, {
1347
+ startedAt: Date.parse(turn.startedAt),
1348
+ durationMs: turn.durationMs,
1349
+ model: turn.model,
1350
+ steps: a.turnSteps.get(turn) ?? []
1351
+ })].sort((x, y) => x.atMs - y.atMs), turn.durationMs);
1352
+ turn.transcript = events;
1353
+ turn.timeSplit = timeSplit(events, turn.durationMs);
1354
+ }
1355
+ const split = sumSplits(a.record.turns.map((t) => t.timeSplit));
1356
+ if (split) a.record.timeSplit = split;
1357
+ }
1358
+ /**
1002
1359
  * The record as its staging file holds it: redacted, with a secret split across a turn's raw
1003
1360
  * stream parts masked first, since masking each part on its own leaves pieces of it to rebuild.
1004
1361
  */
@@ -1169,18 +1526,32 @@ function unpackPlugins(plugins, unpack, hooksFile = null) {
1169
1526
  // src/sandbox/phases.ts
1170
1527
  const strip = ({ phase: _, ...sub }) => sub;
1171
1528
  /**
1172
- * Which brokered secrets the gateway swaps in when. `to: 'checks'` secrets are active only while
1173
- * at least one check or setup step is running, so a request the agent makes between them gets
1174
- * the placeholder.
1529
+ * Which brokered secrets the gateway swaps in when, and which egress policy it enforces when.
1530
+ * `to: 'checks'` secrets are active only while at least one check or setup step is running, so a
1531
+ * request the agent makes between them gets the placeholder. A setup policy of its own holds from
1532
+ * the sandbox's start until `agent()`; every check runs under the agent's.
1175
1533
  */
1176
- function brokerPhases(box, subs) {
1534
+ function brokerPhases(box, subs, egress) {
1177
1535
  const agent = subs.filter((s) => s.phase === "agent").map(strip);
1178
1536
  const all = subs.map(strip);
1179
1537
  const perPhase = box.setSubstitutions && all.length > agent.length;
1180
1538
  let running = 0;
1539
+ const enter = async (phase, planned) => {
1540
+ if (!box.setEgress) throw new InfraError(`Can't switch the sandbox's egress between the setup and agent phases: its egress gateway isn't running`);
1541
+ debugLog("container", `egress → ${phase}: ${phaseLabel(planned) || "proxy"}`);
1542
+ await box.setEgress({
1543
+ policy: gatewayPolicy(planned),
1544
+ direct: isDirect(planned.policy),
1545
+ phase
1546
+ });
1547
+ };
1181
1548
  return {
1182
1549
  async start() {
1183
1550
  if (all.length > 0) await box.setSubstitutions?.(agent);
1551
+ if (egress?.setup) await enter("setup", egress.setup);
1552
+ },
1553
+ async agent() {
1554
+ if (egress?.setup) await enter("agent", egress);
1184
1555
  },
1185
1556
  async checks(fn) {
1186
1557
  if (!perPhase) return fn();
@@ -1321,7 +1692,8 @@ function createAttempt(spec, signal, progress, start, snapshotDir) {
1321
1692
  artifactBlobs: /* @__PURE__ */ new Map(),
1322
1693
  capturePages: /* @__PURE__ */ new Map(),
1323
1694
  stepBlobs: /* @__PURE__ */ new Map(),
1324
- snapshotDir
1695
+ snapshotDir,
1696
+ turnSteps: /* @__PURE__ */ new Map()
1325
1697
  };
1326
1698
  return a;
1327
1699
  }
@@ -1456,7 +1828,7 @@ async function copyWorkspace(box, workspace, signal) {
1456
1828
  if (r.exitCode !== 0) throw new InfraError(`Couldn't unpack the workspace ${workspace.label} into the sandbox (the image needs tar): ${tailOf(r.stderr || r.stdout)}`);
1457
1829
  if (workspace.init) {
1458
1830
  const at = "2000-01-01T00:00:00Z";
1459
- const init = await box.exec(`git init -q -b main && git add -A && git -c commit.gpgsign=false commit -q --no-verify -m ${shellQuote(`${workspace.label} at ${workspace.commit}`)}`, {
1831
+ const init = await box.exec(`git init -q -b main && git add -A && git -c commit.gpgsign=false commit -q --no-verify -m ${shellQuote$1(`${workspace.label} at ${workspace.commit}`)}`, {
1460
1832
  signal,
1461
1833
  env: {
1462
1834
  GIT_AUTHOR_NAME: "reeval",
@@ -1471,13 +1843,87 @@ async function copyWorkspace(box, workspace, signal) {
1471
1843
  }
1472
1844
  debugLog("env", `workspace ${workspace.label} at ${workspace.commit.slice(0, 12)} (${tar.byteLength} bytes of tar${workspace.init ? ", with a .git" : ""}) in ${formatDuration(performance.now() - start)}`);
1473
1845
  }
1846
+ /** Bytes per tar: each is built in memory and piped in whole. */
1847
+ const TAR_BATCH_BYTES = 67108864;
1848
+ /**
1849
+ * Writes `files` (workspace-relative, or `~/` for the home directory) with one tar per
1850
+ * destination: on a container or VM each `writeFile` is a process of its own, which a vendored
1851
+ * directory of thousands of files turns into minutes. A lone file is written as it is, and a
1852
+ * sandbox where the tar doesn't unpack gets a write per file.
1853
+ */
1854
+ async function writeFiles(box, files, signal) {
1855
+ const home = files.flatMap((f) => f.path.startsWith("~/") ? [{
1856
+ ...f,
1857
+ path: f.path.slice(2)
1858
+ }] : []);
1859
+ await unpackFiles(box, "", files.filter((f) => !f.path.startsWith("~/")), signal);
1860
+ await unpackFiles(box, "~/", home, signal);
1861
+ }
1862
+ async function unpackFiles(box, prefix, files, signal) {
1863
+ const one = async () => {
1864
+ for (const f of files) await box.writeFile(`${prefix}${f.path}`, f.content);
1865
+ };
1866
+ if (files.length < 2) return one();
1867
+ const entries = files.map((f) => ({
1868
+ type: "file",
1869
+ path: f.path,
1870
+ mode: 420,
1871
+ mtime: 0,
1872
+ data: typeof f.content === "string" ? Buffer.from(f.content, "utf8") : Buffer.from(f.content.buffer, f.content.byteOffset, f.content.byteLength)
1873
+ }));
1874
+ const batches = [[]];
1875
+ let size = 0;
1876
+ for (const e of entries) {
1877
+ if (size + e.data.length > TAR_BATCH_BYTES && batches[batches.length - 1]?.length) {
1878
+ batches.push([]);
1879
+ size = 0;
1880
+ }
1881
+ batches[batches.length - 1]?.push(e);
1882
+ size += e.data.length;
1883
+ }
1884
+ const dir = shellQuote$1(prefix ? box.home : box.workdir);
1885
+ const name = `.reeval-files-${randomUUID()}.tar`;
1886
+ const unpack = [
1887
+ "unset TAR_OPTIONS",
1888
+ `cd ${dir} || exit 1`,
1889
+ "d=$PWD",
1890
+ `t=$(mktemp -d) || { rm -f ${name}; exit 1; }`,
1891
+ "(",
1892
+ ` tar -xmf "$d/${name}" -C "$t" && cd "$t" && find . -type f > "$t.all" || exit $?`,
1893
+ " : > \"$t.new\"",
1894
+ " while IFS= read -r f; do",
1895
+ " if [ -e \"$d/$f\" ] || [ -L \"$d/$f\" ]; then",
1896
+ " cat \"$f\" > \"$d/$f\" || exit 1",
1897
+ " continue",
1898
+ " fi",
1899
+ ` p=\${f%/*}`,
1900
+ ` while [ "$p" != . ] && [ ! -L "$d/$p" ]; do p=\${p%/*}; done`,
1901
+ " if [ \"$p\" = . ]; then printf '%s\\n' \"$f\" >> \"$t.new\"",
1902
+ ` else mkdir -p "$d/\${f%/*}" && cat "$f" > "$d/$f" || exit 1`,
1903
+ " fi",
1904
+ " done < \"$t.all\"",
1905
+ " [ ! -s \"$t.new\" ] || { tar -cf \"$t.tar\" -T \"$t.new\" && cd \"$d\" && tar -xf \"$t.tar\"; }",
1906
+ ")",
1907
+ "s=$?",
1908
+ `cd "$d"; rm -rf "$t" "$t.all" "$t.new" "$t.tar" ${name}`,
1909
+ "exit $s"
1910
+ ].join("\n");
1911
+ for (const batch of batches) {
1912
+ await box.writeFile(`${prefix}${name}`, writeTar(batch));
1913
+ const r = await box.exec(unpack, { signal });
1914
+ if (r.exitCode !== 0) {
1915
+ debugLog("env", `the files' tar didn't unpack (exit ${r.exitCode}: ${tailOf(r.stderr || r.stdout)}): writing ${files.length} files one at a time`);
1916
+ return one();
1917
+ }
1918
+ }
1919
+ }
1474
1920
  /**
1475
1921
  * A built plugin's directory, copied from where its layer left it in the image, less the parts
1476
1922
  * `only` leaves out, with the manifest as reeval read it.
1477
1923
  */
1478
1924
  async function copyBuiltPlugin(a, box, plugin, dir) {
1479
- const dropped = droppedPaths(plugin.parts).map((p) => shellQuote(`${dir}/${p}`));
1480
- const r = await box.exec(`mkdir -p ${shellQuote(dir)} && cp -a ${shellQuote(plugin.built.dir)}/. ${shellQuote(dir)}/${dropped.length ? ` && rm -rf ${dropped.join(" ")}` : ""}`, { signal: a.signal });
1925
+ const dropped = droppedPaths(plugin.parts).map((p) => shellQuote$1(`${dir}/${p}`));
1926
+ const r = await box.exec(`mkdir -p ${shellQuote$1(dir)} && cp -a ${shellQuote$1(plugin.built.dir)}/. ${shellQuote$1(dir)}/${dropped.length ? ` && rm -rf ${dropped.join(" ")}` : ""}`, { signal: a.signal });
1481
1927
  if (r.exitCode !== 0) throw new InfraError(`Couldn't copy the ${plugin.name} plugin's build from ${plugin.built.tag}: ${tailOf(r.stderr || r.stdout)}`);
1482
1928
  const manifest = plugin.files.find((f) => f.path === PLUGIN_MANIFEST);
1483
1929
  if (manifest) await box.writeFile(`${dir}/${PLUGIN_MANIFEST}`, manifest.content);
@@ -1500,7 +1946,7 @@ async function copyPlugins(a, box, env) {
1500
1946
  else for (const f of plugin.files) await box.writeFile(`${at}/${f.path}`, f.content);
1501
1947
  const executable = built ? [] : plugin.files.flatMap((f) => f.executable ? [`${dir}/${f.path}`] : []);
1502
1948
  if (executable.length) {
1503
- const r = await box.exec(`chmod +x ${executable.map(shellQuote).join(" ")}`, { signal: a.signal });
1949
+ const r = await box.exec(`chmod +x ${executable.map(shellQuote$1).join(" ")}`, { signal: a.signal });
1504
1950
  if (r.exitCode !== 0) throw new InfraError(`Couldn't make the ${plugin.name} plugin's ${executable.length} executable files executable: ${tailOf(r.stderr || r.stdout)}`);
1505
1951
  }
1506
1952
  const root = (v) => v.replaceAll(PLUGIN_ROOT, dir);
@@ -1533,7 +1979,7 @@ async function copyPlugins(a, box, env) {
1533
1979
  * only, so the sandbox has `find`.
1534
1980
  */
1535
1981
  async function unpackedFromCopy(a, box, dir) {
1536
- const r = await box.exec(`cd ${shellQuote(dir)} && find rules agents commands hooks -maxdepth 1 -type f 2>/dev/null; true`, { signal: a.signal });
1982
+ const r = await box.exec(`cd ${shellQuote$1(dir)} && find rules agents commands hooks -maxdepth 1 -type f 2>/dev/null; true`, { signal: a.signal });
1537
1983
  const out = [];
1538
1984
  for (const path of r.stdout.split("\n").filter(Boolean)) {
1539
1985
  const content = await box.readFile(`${dir}/${path}`);
@@ -1562,7 +2008,7 @@ async function writeUnpackedPlugins(a, box, env, plugins, unpack) {
1562
2008
  const files = unpackPlugins(placed, unpack, unpack.hooks ? await box.readFile(unpack.hooks.file) : null);
1563
2009
  const fresh = files.filter((f) => f.path !== unpack.hooks?.file);
1564
2010
  if (fresh.length) {
1565
- const taken = (await box.exec(`for p in ${fresh.map((f) => shellQuote(f.path)).join(" ")}; do { [ -e "$p" ] || [ -L "$p" ]; } && echo "$p"; done; true`, { signal: a.signal })).stdout.trim().split("\n").filter(Boolean);
2011
+ const taken = (await box.exec(`for p in ${fresh.map((f) => shellQuote$1(f.path)).join(" ")}; do { [ -e "$p" ] || [ -L "$p" ]; } && echo "$p"; done; true`, { signal: a.signal })).stdout.trim().split("\n").filter(Boolean);
1566
2012
  if (taken.length) throw new InfraError(`The workspace already has ${taken.join(", ")}, which a plugin would replace: rename the plugin's file, or leave the plugin out of this cell`);
1567
2013
  }
1568
2014
  for (const f of files) {
@@ -1596,40 +2042,71 @@ async function writePluginMcp(box, env, plugins, servers) {
1596
2042
  debugLog("env", `plugin ${placed.name}'s MCP servers → ${at}/.mcp.json, through the timer`);
1597
2043
  }
1598
2044
  }
2045
+ /** How long a start that failed on a transient error waits before it's tried again. */
2046
+ const TRANSIENT_START_DELAY_MS = 2e3;
1599
2047
  /**
1600
- * Starts the harness session within its `startTimeout`. A start that doesn't become ready in time
1601
- * is aborted (and closed, should it finish later) and started once more: a stuck bootstrap is
1602
- * usually transient, and nothing has run yet, so the retry can't change what the attempt
1603
- * measures. A second timeout is an infra error; a start that fails outright isn't retried.
2048
+ * Starts a harness session within `timeoutMs`, at most twice. A start that doesn't become ready in
2049
+ * time is aborted (and closed, should it finish later) and started once more, and so is one that
2050
+ * fails on a transient {@link InfraError} (a 5xx or a network error): nothing has run yet, so the
2051
+ * retry can't change what the attempt measures. Each retry is passed to `onRetry` to record. A
2052
+ * second timeout or transient failure, or any other failure, is the attempt's infra error.
1604
2053
  */
1605
- async function startHarness(a, harness, box, options) {
1606
- const { harness: descriptor } = a.spec;
1607
- const ms = startTimeoutMs(descriptor);
1608
- const late = `${descriptor.kind}: session did not become ready within ${formatDuration(ms)}`;
2054
+ async function startWithRetry(start, { kind, timeoutMs, signal, onRetry, transientDelayMs = TRANSIENT_START_DELAY_MS }) {
2055
+ const late = `${kind}: session did not become ready within ${formatDuration(timeoutMs)}`;
1609
2056
  for (let tries = 1;; tries++) {
1610
2057
  const stop = new AbortController();
1611
2058
  let timer;
1612
2059
  const timedOut = new Promise((resolve) => {
1613
- timer = setTimeout(() => resolve(null), ms);
2060
+ timer = setTimeout(() => resolve(null), timeoutMs);
1614
2061
  });
1615
- const starting = harness.start(box, AbortSignal.any([a.signal, stop.signal]), options);
1616
- const session = await Promise.race([starting, timedOut]).finally(() => clearTimeout(timer));
2062
+ const startedAt = Date.now();
2063
+ const starting = start(AbortSignal.any([signal, stop.signal]));
2064
+ let session;
2065
+ try {
2066
+ session = await Promise.race([starting, timedOut]);
2067
+ } catch (error) {
2068
+ signal.throwIfAborted();
2069
+ if (!(error instanceof InfraError) || !error.transient) throw error;
2070
+ if (tries === 2) throw new InfraError(`${error.message} (tried twice)`);
2071
+ onRetry({
2072
+ error: error.message,
2073
+ waitedMs: Date.now() - startedAt
2074
+ });
2075
+ debugLog("harness", `${error.message}; starting it once more`);
2076
+ await new Promise((resolve) => setTimeout(resolve, transientDelayMs));
2077
+ signal.throwIfAborted();
2078
+ continue;
2079
+ } finally {
2080
+ clearTimeout(timer);
2081
+ }
1617
2082
  if (session) return session;
1618
2083
  stop.abort(new InfraError(late));
1619
2084
  starting.then((s) => s.close().catch(() => {}), () => {});
1620
- a.signal.throwIfAborted();
2085
+ signal.throwIfAborted();
1621
2086
  if (tries === 2) throw new InfraError(`${late} (tried twice)`);
1622
- a.record.startRetries = [{
2087
+ onRetry({
1623
2088
  error: late,
1624
- waitedMs: ms
1625
- }];
2089
+ waitedMs: timeoutMs
2090
+ });
1626
2091
  debugLog("harness", `${late}; starting it once more`);
1627
2092
  }
1628
2093
  }
2094
+ function startHarness(a, harness, box, options) {
2095
+ const { harness: descriptor } = a.spec;
2096
+ return startWithRetry((signal) => harness.start(box, signal, options), {
2097
+ kind: descriptor.kind,
2098
+ timeoutMs: startTimeoutMs(descriptor),
2099
+ signal: a.signal,
2100
+ onRetry: (retry) => {
2101
+ a.record.startRetries = [...a.record.startRetries ?? [], retry];
2102
+ }
2103
+ });
2104
+ }
1629
2105
  /**
1630
2106
  * Starts the sandbox (with the brokered secrets its gateway swaps in), writes the
1631
2107
  * environment's files, workspace skills and plugins, runs its setup steps, snapshots the workspace for
1632
- * the diff, and starts the harness session with the harness-placed skills and MCP servers.
2108
+ * the diff, switches to the agent's egress, and starts the harness session with the
2109
+ * harness-placed skills and MCP servers.
1633
2110
  */
1634
2111
  async function startSession(a, runId) {
1635
2112
  const startedAt = performance.now();
@@ -1641,7 +2118,9 @@ async function startSession(a, runId) {
1641
2118
  signal: a.signal,
1642
2119
  env: env.vars,
1643
2120
  image: a.image,
2121
+ ...a.hosts && { hosts: a.hosts },
1644
2122
  brokering: brokeringFor(spec.harness, spec.vertex),
2123
+ ...agentOf(spec.harness, spec.vertex)?.keychainTrust && { keychainTrust: true },
1645
2124
  expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0, spec.vertex)?.hosts,
1646
2125
  substitutions: env.substitutions,
1647
2126
  egress: spec.egress,
@@ -1652,7 +2131,7 @@ async function startSession(a, runId) {
1652
2131
  });
1653
2132
  a.sandbox = box;
1654
2133
  if (!capabilitiesOf(spec.sandbox).isolated) debugLog("container", `${sandboxKind(spec.sandbox).provider(spec.sandbox)} sandbox at ${box.workdir}`);
1655
- a.phases = brokerPhases(box, env.substitutions);
2134
+ a.phases = brokerPhases(box, env.substitutions, spec.egress);
1656
2135
  if (a.signal.aborted) box.destroy();
1657
2136
  a.signal.throwIfAborted();
1658
2137
  await a.phases.start();
@@ -1661,23 +2140,29 @@ async function startSession(a, runId) {
1661
2140
  debugLog("broker", `gateway swaps in ${agent.join(", ") || "nothing"} for the agent`);
1662
2141
  }
1663
2142
  if (env.workspace) await copyWorkspace(box, env.workspace, a.signal);
2143
+ const skillDir = workspaceSkillDir(spec.harness);
2144
+ const skillFileList = env.workspaceSkills.map((skill) => ({
2145
+ skill,
2146
+ files: skillFiles(skill).map((f) => ({
2147
+ path: `${skillDir}/${skill.name}/${f.path}`,
2148
+ content: f.content
2149
+ }))
2150
+ }));
2151
+ const toWrite = [...env.files, ...skillFileList.flatMap((s) => s.files)];
2152
+ const filesStart = performance.now();
2153
+ await writeFiles(box, toWrite, a.signal);
2154
+ if (toWrite.length) debugLog("env", `${toWrite.length} files written in ${formatDuration(performance.now() - filesStart)}`);
1664
2155
  for (const f of env.files) {
1665
- await box.writeFile(f.path, f.content);
1666
2156
  const size = typeof f.content === "string" ? Buffer.byteLength(f.content) : f.content.byteLength;
1667
2157
  debugLog("env", `file ${f.path} (${size} bytes${f.executable ? ", executable" : ""})`);
1668
2158
  }
1669
2159
  const executable = env.files.flatMap((f) => f.executable ? [f.path.startsWith("~/") ? `${box.home}/${f.path.slice(2)}` : f.path] : []);
1670
2160
  if (executable.length) {
1671
- const paths = executable.map((p) => shellQuote(p.startsWith("-") ? `./${p}` : p));
2161
+ const paths = executable.map((p) => shellQuote$1(p.startsWith("-") ? `./${p}` : p));
1672
2162
  const r = await box.exec(`chmod +x ${paths.join(" ")}`, { signal: a.signal });
1673
2163
  if (r.exitCode !== 0) throw new InfraError(`Couldn't make ${executable.join(", ")} executable: ${tailOf(r.stderr || r.stdout)}`);
1674
2164
  }
1675
- const skillDir = workspaceSkillDir(spec.harness);
1676
- for (const skill of env.workspaceSkills) {
1677
- const files = skillFiles(skill);
1678
- for (const f of files) await box.writeFile(`${skillDir}/${skill.name}/${f.path}`, f.content);
1679
- debugLog("env", `skill ${skill.name} → ${skillDir}/${skill.name}/ (${files.length} files, workspace)`);
1680
- }
2165
+ for (const { skill, files } of skillFileList) debugLog("env", `skill ${skill.name} → ${skillDir}/${skill.name}/ (${files.length} files, workspace)`);
1681
2166
  const plugins = await copyPlugins(a, box, env);
1682
2167
  const layout = layoutOf(spec.harness);
1683
2168
  if (layout.pluginUnpack && plugins.length) await writeUnpackedPlugins(a, box, env, plugins, layout.pluginUnpack);
@@ -1691,6 +2176,7 @@ async function startSession(a, runId) {
1691
2176
  }
1692
2177
  await snapshot(box, a.signal);
1693
2178
  debugLog("env", "workspace snapshot taken: the diff starts here");
2179
+ await a.phases.agent();
1694
2180
  const timed = await timeMcpServers(box, env.mcp, a.signal);
1695
2181
  if (timed.log) {
1696
2182
  a.mcpStartup = timed.log;
@@ -1803,8 +2289,8 @@ function cleanArchive(tar) {
1803
2289
  };
1804
2290
  }
1805
2291
  const tarCommand = (workdir) => {
1806
- const flags = EXCLUDED_DIRS.map((d) => `--exclude=${shellQuote(d)}`).join(" ");
1807
- return `cd ${shellQuote(workdir)} && COPYFILE_DISABLE=1 tar -cf - ${flags} . 2>/dev/null`;
2292
+ const flags = EXCLUDED_DIRS.map((d) => `--exclude=${shellQuote$1(d)}`).join(" ");
2293
+ return `cd ${shellQuote$1(workdir)} && COPYFILE_DISABLE=1 tar -cf - ${flags} . 2>/dev/null`;
1808
2294
  };
1809
2295
  const GZIP = "if command -v gzip >/dev/null 2>&1; then gzip -c; else cat; fi";
1810
2296
  const NEEDS_TAR = "command -v tar >/dev/null 2>&1 || { echo 'the sandbox has no tar' >&2; exit 127; }";
@@ -1833,9 +2319,9 @@ async function archiveWorkspace(box, signal, compress, limits = LIMITS) {
1833
2319
  },
1834
2320
  entries: null
1835
2321
  });
1836
- const names = EXCLUDED_DIRS.map((d) => `-name ${shellQuote(d)}`).join(" -o ");
2322
+ const names = EXCLUDED_DIRS.map((d) => `-name ${shellQuote$1(d)}`).join(" -o ");
1837
2323
  const tar = tarCommand(box.workdir);
1838
- const sized = await box.execOwn(`${NEEDS_TAR}; cd ${shellQuote(box.workdir)} && { find . -type d \\( ${names} \\) -prune -print 2>/dev/null | head -n ${LISTED}; echo ${SIZE_MARK}; ${tar} | wc -c; echo ${SIZE_MARK}; ${tar} | { ${GZIP}; } | wc -c; }`, { signal });
2324
+ const sized = await box.execOwn(`${NEEDS_TAR}; cd ${shellQuote$1(box.workdir)} && { find . -type d \\( ${names} \\) -prune -print 2>/dev/null | head -n ${LISTED}; echo ${SIZE_MARK}; ${tar} | wc -c; echo ${SIZE_MARK}; ${tar} | { ${GZIP}; } | wc -c; }`, { signal });
1839
2325
  const [list = "", raw = "", size = ""] = sized.stdout.split(`${SIZE_MARK}\n`);
1840
2326
  const dirs = list.split("\n").filter(Boolean).map((p) => redactSecrets(`${p.replace(/^\.\//, "")}/`));
1841
2327
  const rawBytes = Number(raw.trim());
@@ -1950,10 +2436,10 @@ async function takeSnapshot(a, box, name, phase, limits = LIMITS, timeoutMs = SN
1950
2436
  /** How long a turn gets to stop once its attempt is aborted, before reeval stops waiting for it. */
1951
2437
  const TURN_STOP_GRACE_MS = 3e3;
1952
2438
  /**
1953
- * Resolves `grace` ms after `signal` aborts, with the turn as cut off; `cancel` drops the timer
1954
- * once the turn settles on its own.
2439
+ * Resolves `grace` ms after `signal` aborts, with the turn as cut off: what the session says it
2440
+ * got to, or an empty turn; `cancel` drops the timer once the turn settles on its own.
1955
2441
  */
1956
- function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
2442
+ function cutOff(signal, startedAt, session, grace = TURN_STOP_GRACE_MS) {
1957
2443
  let timer;
1958
2444
  let onAbort = () => {};
1959
2445
  const cutoff = new Promise((resolve) => {
@@ -1961,22 +2447,25 @@ function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
1961
2447
  timer = setTimeout(() => {
1962
2448
  const message = `The agent didn't stop within ${grace / 1e3}s of being canceled, so reeval stopped waiting for its turn`;
1963
2449
  const at = Date.now() - Date.parse(startedAt);
1964
- resolve({
2450
+ const note = {
2451
+ type: "error",
2452
+ message,
2453
+ ts: (/* @__PURE__ */ new Date()).toISOString(),
2454
+ atMs: at
2455
+ };
2456
+ const empty = {
1965
2457
  text: "",
1966
2458
  status: "aborted",
1967
- transcript: [{
1968
- type: "error",
1969
- message,
1970
- ts: (/* @__PURE__ */ new Date()).toISOString(),
1971
- atMs: at
1972
- }],
2459
+ transcript: [note],
1973
2460
  usage: null,
1974
2461
  model: null,
1975
2462
  unreported: [],
1976
2463
  reasoningMs: null,
1977
2464
  startedAt,
1978
2465
  durationMs: at
1979
- });
2466
+ };
2467
+ if (!session.stopped) return resolve(empty);
2468
+ session.stopped(note).then(resolve, () => resolve(empty));
1980
2469
  }, grace);
1981
2470
  };
1982
2471
  if (signal.aborted) onAbort();
@@ -2064,7 +2553,7 @@ function turnSender(a, takeContext, checks) {
2064
2553
  }, `turn ${turnNumber} prompt`);
2065
2554
  const watch = a.mcpStartup && watchMcpStartup(a.mcpStartup);
2066
2555
  const signal = watch ? AbortSignal.any([a.signal, watch.signal]) : a.signal;
2067
- const { cutoff, cancel } = cutOff(signal, entry.startedAt);
2556
+ const { cutoff, cancel } = cutOff(signal, entry.startedAt, agent);
2068
2557
  const sendStarted = Date.now();
2069
2558
  const sending = agent.send(prompt, signal).catch((error) => {
2070
2559
  if (!watch?.signal.aborted || a.signal.aborted) throw error;
@@ -2081,7 +2570,7 @@ function turnSender(a, takeContext, checks) {
2081
2570
  };
2082
2571
  });
2083
2572
  const timedBefore = a.record.timings.agentMs;
2084
- const { raw, rejectedKey, reasoningMs, agentMs, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
2573
+ const { raw, rejectedKey, providerStop, reasoningMs, agentMs, steps, cost, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
2085
2574
  entry.reasoningMs = null;
2086
2575
  if (watch) entry.mcpStartup = mcpStartupOf((await watch.end()).entries);
2087
2576
  throw error;
@@ -2089,6 +2578,7 @@ function turnSender(a, takeContext, checks) {
2089
2578
  cancel();
2090
2579
  watch?.close();
2091
2580
  });
2581
+ if (steps) a.turnSteps.set(entry, steps);
2092
2582
  const sendWallMs = agentMs ?? Date.now() - sendStarted;
2093
2583
  if (agentMs !== void 0) a.record.timings.agentMs = timedBefore + agentMs;
2094
2584
  const startup = await watch?.end();
@@ -2104,7 +2594,7 @@ function turnSender(a, takeContext, checks) {
2104
2594
  atMs: Math.max(0, seenAt - Date.parse(sent.startedAt))
2105
2595
  }].sort((x, y) => x.atMs - y.atMs)
2106
2596
  } : sent;
2107
- Object.assign(entry, result, raw ? { raw } : {}, reasoningMs !== void 0 ? { reasoningMs } : {}, {
2597
+ Object.assign(entry, result, raw ? { raw } : {}, reasoningMs !== void 0 ? { reasoningMs } : {}, cost ? { cost } : {}, {
2108
2598
  durationMs: Math.max(result.durationMs, sendWallMs),
2109
2599
  ...startup ? { mcpStartup: mcpStartupOf(startup.entries) } : {}
2110
2600
  });
@@ -2115,7 +2605,8 @@ function turnSender(a, takeContext, checks) {
2115
2605
  await takeContext(box, a.signal);
2116
2606
  const count = (...types) => result.transcript.filter((e) => types.includes(e.type)).length;
2117
2607
  const failure = result.transcript.find((e) => e.type === "error");
2118
- const infra = timedOut || result.status !== "aborted" && rejectedKey;
2608
+ const stopped = result.status === "aborted" ? void 0 : providerStop;
2609
+ const infra = timedOut || result.status !== "aborted" && rejectedKey || stopped?.message;
2119
2610
  let changes;
2120
2611
  let diffError;
2121
2612
  if (!infra && !a.signal.aborted) try {
@@ -2138,7 +2629,7 @@ function turnSender(a, takeContext, checks) {
2138
2629
  debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${filesChanged === void 0 ? "" : `${plural(filesChanged, "file")} changed · `}${plural(count("file-edit"), "edit tool call")}${failure && "message" in failure ? ` · error: ${clip(redactSecrets(String(failure.message)), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
2139
2630
  for (const s of entry.mcpStartup ?? []) debugLog("harness", `turn ${turnNumber} mcp ${s.server} ${s.ms === null ? "didn't answer initialize" : `answered initialize in ${formatDuration(s.ms)}`}`);
2140
2631
  if (infra) {
2141
- const error = new InfraError(infra);
2632
+ const error = !timedOut && stopped ? new ProviderStopError(stopped) : new InfraError(infra);
2142
2633
  a.startFailure ??= { error };
2143
2634
  throw error;
2144
2635
  }
@@ -2190,7 +2681,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
2190
2681
  clock?.resume();
2191
2682
  }
2192
2683
  }, snapshotSidecar(stagingDir, attemptKey(id, spec.cell.key, spec.n)));
2193
- const destroy = () => void a.sandbox?.destroy();
2684
+ const destroy = () => void Promise.resolve(a.session?.salvage?.()).catch(() => {}).finally(() => a.sandbox?.destroy());
2194
2685
  testSignal.addEventListener("abort", destroy, { once: true });
2195
2686
  const takeContext = contextTaker(a);
2196
2687
  const { exec, readFile, writeFile } = createChecks(a);
@@ -2234,12 +2725,10 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
2234
2725
  const missed = await unawaited() ?? checkFailure(assertions()) ?? expectError();
2235
2726
  if (a.startFailure) throw a.startFailure.error;
2236
2727
  if (missed) throw missed;
2237
- const judgeErrors = a.judgeErrors.toSorted((x, y) => x.call - y.call).map((e) => e.message);
2238
- const [only, ...more] = judgeErrors;
2239
- if (only) {
2240
- const message = more.length ? `${more.length + 1} judge calls failed:\n${judgeErrors.join("\n")}` : only;
2241
- a.judgeThrown.add(message);
2242
- throw new Error(message);
2728
+ const judgeErrors = judgeErrorsMessage(a);
2729
+ if (judgeErrors) {
2730
+ a.judgeThrown.add(judgeErrors);
2731
+ throw new Error(judgeErrors);
2243
2732
  }
2244
2733
  } catch (thrown) {
2245
2734
  const error = attemptError(a, thrown, assertions());
@@ -2261,6 +2750,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
2261
2750
  oom = a.signal.aborted ? void 0 : await recordOom(a, sandbox);
2262
2751
  if (sandbox && spec.snapshot.when && !a.signal.aborted) await takeSnapshot(a, sandbox, "final", "teardown");
2263
2752
  await sandbox?.destroy();
2753
+ recordTimeline(a, sandbox);
2264
2754
  if (sandbox) debug.log("teardown", `sandbox destroyed (${formatDuration(performance.now() - teardown)} for teardown)`);
2265
2755
  const { record } = a;
2266
2756
  const checked = assertionRecords(assertions());
@@ -2303,8 +2793,11 @@ function bestOfCell(bestOf) {
2303
2793
  settled: (settled) => settle[n - 1]?.(sideOf(settled))
2304
2794
  });
2305
2795
  }
2306
- /** Characters a case key can't hold: ids and paths split on them, and control characters. */
2307
- const BAD_KEY = /[#/›\u0000-\u001f\u007f]/;
2796
+ /**
2797
+ * Characters a case key can't hold: attempt ids split on `#` and eval ids on `›`, and control
2798
+ * characters. A `/` is allowed: a key is its own path segment wherever it's in one.
2799
+ */
2800
+ const BAD_KEY = /[#›\u0000-\u001f\u007f]/;
2308
2801
  /** The name a case's eval plans under: `adds a health check [nextjs]`. */
2309
2802
  const caseEvalName = (name, key) => `${name} [${key}]`;
2310
2803
  /** The key a case gets without `caseKey`: a string, a finite number, or an object's `id` or `name`. */
@@ -2318,7 +2811,7 @@ function defaultKey(item) {
2318
2811
  }
2319
2812
  /**
2320
2813
  * Each case's key, in list order: `caseKey(item)` when given, else its default. Trimmed, 1–80
2321
- * characters, with no `#`, `/`, `›` or control characters, and unique within the eval.
2814
+ * characters, with no `#`, `›` or control characters, and unique within the eval.
2322
2815
  */
2323
2816
  function caseKeys(cases, caseKey, where) {
2324
2817
  const seen = /* @__PURE__ */ new Map();
@@ -2327,7 +2820,7 @@ function caseKeys(cases, caseKey, where) {
2327
2820
  if (raw === void 0) throw new UsageError(`cases[${i}] in ${where} has no key: a case is its own key when it's a string or a number, or an object's id or name. Give it an id, or pass caseKey: (item) => item.slug.`);
2328
2821
  if (typeof raw !== "string") throw new UsageError(`caseKey in ${where} must return a string (got ${redactSecrets(JSON.stringify(raw) ?? String(raw))} for cases[${i}])`);
2329
2822
  const key = raw.trim();
2330
- if (!key || key.length > 80 || BAD_KEY.test(key)) throw new UsageError(`cases[${i}] in ${where} has the key ${redactSecrets(JSON.stringify(raw))}: a case key is 1–80 characters, with no #, /, › or control characters.${caseKey ? "" : " Pass caseKey to give it another."}`);
2823
+ if (!key || key.length > 80 || BAD_KEY.test(key)) throw new UsageError(`cases[${i}] in ${where} has the key ${redactSecrets(JSON.stringify(raw))}: a case key is 1–80 characters, with no #, › or control characters.${caseKey ? "" : " Pass caseKey to give it another."}`);
2331
2824
  const earlier = seen.get(key);
2332
2825
  if (earlier !== void 0) throw new UsageError(`cases[${earlier}] and cases[${i}] in ${where} have the same key "${redactSecrets(key)}". Each case needs its own key${caseKey ? "" : ": give them distinct ids or names, or pass caseKey"}.`);
2333
2826
  seen.set(key, i);
@@ -2653,6 +3146,8 @@ function planTests(declaration, ctx, where, resolvedEval, pick) {
2653
3146
  if (first) throw new UsageError(unbrokeredError(first, where, ctx.config.file));
2654
3147
  const [plain] = plainHeaders(plannedAttempts);
2655
3148
  if (plain) throw new UsageError(plainHeaderError(plain, where, ctx.config.file));
3149
+ const [plainVar] = plainEnv(resolved.env, plannedAttempts);
3150
+ if (plainVar) throw new UsageError(plainEnvError(plainVar, where, ctx.config.file));
2656
3151
  }
2657
3152
  const pluginNotes = [];
2658
3153
  const pluginLabels = /* @__PURE__ */ new Map();
@@ -3084,10 +3579,26 @@ function prepare(mode, given, own, fn, source, hint = "", givenFamily, where) {
3084
3579
  bestOfCells.set(cell.key, of);
3085
3580
  hooks = of(n);
3086
3581
  }
3087
- registerTest(testName, {
3582
+ const test = {
3088
3583
  ...options,
3089
3584
  ...hooks
3090
- }, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
3585
+ };
3586
+ if (spec && test.gate && spec.sandbox.kind === "tart" && spec.sandbox.hosts) {
3587
+ const gate = {
3588
+ ...test.gate,
3589
+ limits: { ...test.gate.limits }
3590
+ };
3591
+ const before = test.ready;
3592
+ test.gate = gate;
3593
+ test.ready = async () => {
3594
+ const { config, runId } = getRunContext();
3595
+ const cap = await preparedVmCap(spec, config.root, runId);
3596
+ const key = "sandbox:tart";
3597
+ if (cap && gate.limits[key]) gate.limits[key] = Math.min(gate.limits[key], cap);
3598
+ return before ? before() : true;
3599
+ };
3600
+ }
3601
+ registerTest(testName, test, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
3091
3602
  }
3092
3603
  };
3093
3604
  return {
@@ -3241,4 +3752,4 @@ function optionsFrom(written, where, base) {
3241
3752
  }
3242
3753
  export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
3243
3754
 
3244
- //# sourceMappingURL=evaluate-B12Tosvq.js.map
3755
+ //# sourceMappingURL=evaluate-DA4QsrHO.js.map