reeval 0.20.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (481) hide show
  1. package/CHANGELOG.md +841 -0
  2. package/README.md +15 -15
  3. package/assets/gateway-models.mjs +65 -31
  4. package/dist/{adapter-C60z3wLq.js → adapter-Di9H2etg.js} +8 -49
  5. package/dist/adapter-Di9H2etg.js.map +1 -0
  6. package/dist/{agent-BmsMXZXx.js → agent-B-6GFDqA.js} +185 -38
  7. package/dist/agent-B-6GFDqA.js.map +1 -0
  8. package/dist/{agents-DXuFxtGC.js → agents-KBTYhSSh.js} +385 -116
  9. package/dist/agents-KBTYhSSh.js.map +1 -0
  10. package/dist/{app-files-DT5mJyV4.js → app-files-Cmh9FGr5.js} +3 -3
  11. package/dist/app-files-Cmh9FGr5.js.map +1 -0
  12. package/dist/{assemble-CE4pHKIa.js → assemble-CBMRIhso.js} +1 -1
  13. package/dist/{assemble-DM2PNU4R.js → assemble-DNVBTBp_.js} +172 -18
  14. package/dist/assemble-DNVBTBp_.js.map +1 -0
  15. package/dist/attach-EbDGKGTO.js +152 -0
  16. package/dist/attach-EbDGKGTO.js.map +1 -0
  17. package/dist/{attempt-CAdlwlNu.js → attempt-_dUsblVx.js} +2 -2
  18. package/dist/{attempt-CAdlwlNu.js.map → attempt-_dUsblVx.js.map} +1 -1
  19. package/dist/{attempt-host-7VviAJrl.js → attempt-host-DQvynDHk.js} +17 -8
  20. package/dist/attempt-host-DQvynDHk.js.map +1 -0
  21. package/dist/boat-Bzz8L-KQ.js +701 -0
  22. package/dist/boat-Bzz8L-KQ.js.map +1 -0
  23. package/dist/{boat-BjLqFJIZ.js → boat-Dek8ftIt.js} +20 -8
  24. package/dist/boat-Dek8ftIt.js.map +1 -0
  25. package/dist/bridge/index.mjs +6 -1
  26. package/dist/bundle-DZstHDeW.js +718 -0
  27. package/dist/bundle-DZstHDeW.js.map +1 -0
  28. package/dist/{capture-Ccf0Hmyx.js → capture-BkbPsfej.js} +61 -32
  29. package/dist/capture-BkbPsfej.js.map +1 -0
  30. package/dist/capture-CUAvk9KM.js +1 -0
  31. package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
  32. package/dist/cases-CM-MQRQp.js.map +1 -0
  33. package/dist/cases-FvhbM8M5.js +1 -0
  34. package/dist/{cli-DTuLSlmW.js → cli-BgBBuxxJ.js} +412 -1044
  35. package/dist/cli-BgBBuxxJ.js.map +1 -0
  36. package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-B5L9n-DV.js} +199 -35
  37. package/dist/cli-spec-B5L9n-DV.js.map +1 -0
  38. package/dist/cli.js +2 -2
  39. package/dist/client-DMEYR3MA.js +807 -0
  40. package/dist/client-DMEYR3MA.js.map +1 -0
  41. package/dist/{command-C2C7P5qo.js → command-BKb8eN_X.js} +10 -10
  42. package/dist/{command-C2C7P5qo.js.map → command-BKb8eN_X.js.map} +1 -1
  43. package/dist/{command-DSL7I-SI.js → command-D_w_kDpI.js} +25 -21
  44. package/dist/command-D_w_kDpI.js.map +1 -0
  45. package/dist/{command-Ff0T-1Hi.js → command-Dwlh0Vhu.js} +17 -16
  46. package/dist/command-Dwlh0Vhu.js.map +1 -0
  47. package/dist/{command-D8oMB2fa.js → command-loXH2jHr.js} +16 -16
  48. package/dist/{command-D8oMB2fa.js.map → command-loXH2jHr.js.map} +1 -1
  49. package/dist/{commands-xAOi_VEk.js → commands-Cuf57OYb.js} +174 -40
  50. package/dist/commands-Cuf57OYb.js.map +1 -0
  51. package/dist/{commands-Bk9NG1s_.js → commands-Dqli45sR.js} +1 -1
  52. package/dist/{config-SvK8pjEK.js → config-C0RKt00M.js} +288 -106
  53. package/dist/config-C0RKt00M.js.map +1 -0
  54. package/dist/{context-BdgCao9Q.js → context-DrEAtUyg.js} +2 -2
  55. package/dist/{context-BdgCao9Q.js.map → context-DrEAtUyg.js.map} +1 -1
  56. package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
  57. package/dist/cost-C21gGrju.js.map +1 -0
  58. package/dist/cost-IM80LyGD.js +1 -0
  59. package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-CRsFxorF.js} +4 -4
  60. package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-CRsFxorF.js.map} +1 -1
  61. package/dist/db-3o8s2EyD.js +2 -0
  62. package/dist/{db-BzOJ4Js4.js → db-ErnJ-gMD.js} +168 -248
  63. package/dist/db-ErnJ-gMD.js.map +1 -0
  64. package/dist/debug-CIXzB4wK.js.map +1 -1
  65. package/dist/{default-effort-BDKIxJyC.js → default-effort-D70cTG00.js} +4 -4
  66. package/dist/{default-effort-BDKIxJyC.js.map → default-effort-D70cTG00.js.map} +1 -1
  67. package/dist/{default-effort-BBDPbJ4D.js → default-effort-DHzj84dD.js} +1 -1
  68. package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
  69. package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
  70. package/dist/{docker-DFrjoNs-.js → docker-CFWzZOEk.js} +4 -4
  71. package/dist/{docker-DFrjoNs-.js.map → docker-CFWzZOEk.js.map} +1 -1
  72. package/dist/docker-DrMnpL84.js +1 -0
  73. package/dist/{doctor-DWK4ttex.js → doctor-B7utCSdh.js} +121 -100
  74. package/dist/doctor-B7utCSdh.js.map +1 -0
  75. package/dist/{environment-cPMCOk-h.js → environment-C0Gg8kyN.js} +49 -11
  76. package/dist/environment-C0Gg8kyN.js.map +1 -0
  77. package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
  78. package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
  79. package/dist/{eval-source-C9b1h8L6.js → eval-source-CsW5VIXS.js} +24 -12
  80. package/dist/eval-source-CsW5VIXS.js.map +1 -0
  81. package/dist/{evaluate-B12Tosvq.js → evaluate-BHQyAszQ.js} +679 -142
  82. package/dist/evaluate-BHQyAszQ.js.map +1 -0
  83. package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DzHq4EsU.d.ts} +2 -2
  84. package/dist/fake-bootstrap-VpisF1WE.js +15 -0
  85. package/dist/fake-bootstrap-VpisF1WE.js.map +1 -0
  86. package/dist/{files-tiNWe8Ih.js → files-BCraBodb.js} +3 -3
  87. package/dist/{files-tiNWe8Ih.js.map → files-BCraBodb.js.map} +1 -1
  88. package/dist/files-BjOJw4sn.js +1 -0
  89. package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
  90. package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
  91. package/dist/{files-BOBdC0uI.js → files-BuKWZCzY.js} +6 -6
  92. package/dist/{files-BOBdC0uI.js.map → files-BuKWZCzY.js.map} +1 -1
  93. package/dist/files-CAzVG3Rr.js +1 -0
  94. package/dist/{files-F5EG0WPT.js → files-Cv856jLN.js} +5 -5
  95. package/dist/{files-F5EG0WPT.js.map → files-Cv856jLN.js.map} +1 -1
  96. package/dist/files-D-Lir4aE.js +1 -0
  97. package/dist/files-DRlJ0Xr2.js +1 -0
  98. package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
  99. package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
  100. package/dist/{files-s28a6czq.js → files-qjeqXnfE.js} +143 -20
  101. package/dist/files-qjeqXnfE.js.map +1 -0
  102. package/dist/gateway.d.ts +1 -1
  103. package/dist/{git-BpoqEgDV.js → git-DOrOtAiD.js} +3 -3
  104. package/dist/{git-BpoqEgDV.js.map → git-DOrOtAiD.js.map} +1 -1
  105. package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
  106. package/dist/harness-FcTyyPco.js.map +1 -0
  107. package/dist/harness-install-Dxho1yaU.js +53 -0
  108. package/dist/harness-install-Dxho1yaU.js.map +1 -0
  109. package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
  110. package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
  111. package/dist/harness.d.ts +11 -8
  112. package/dist/harness.js +1 -1
  113. package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
  114. package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
  115. package/dist/{images-D2jFze2v.js → images-Ch1y1t6_.js} +122 -73
  116. package/dist/images-Ch1y1t6_.js.map +1 -0
  117. package/dist/index.d.ts +46 -7
  118. package/dist/index.js +6 -6
  119. package/dist/{init-Bi1Q3lRi.js → init-C2HqCwnt.js} +33 -14
  120. package/dist/init-C2HqCwnt.js.map +1 -0
  121. package/dist/init-CM9xwyre.js +1 -0
  122. package/dist/{ink-list-BpfidQ1h.js → ink-list-BbKXElcQ.js} +54 -15
  123. package/dist/ink-list-BbKXElcQ.js.map +1 -0
  124. package/dist/inputs-5UIDpjz-.js +190 -0
  125. package/dist/inputs-5UIDpjz-.js.map +1 -0
  126. package/dist/inputs-load-C82HOKnb.js +1 -0
  127. package/dist/inputs-load-CGPyigMC.js +95 -0
  128. package/dist/inputs-load-CGPyigMC.js.map +1 -0
  129. package/dist/{invocations-CxGNOlh0.js → invocations-BrZAMuTb.js} +5 -5
  130. package/dist/{invocations-CxGNOlh0.js.map → invocations-BrZAMuTb.js.map} +1 -1
  131. package/dist/jiti-radm1Soi.js +52 -0
  132. package/dist/jiti-radm1Soi.js.map +1 -0
  133. package/dist/{judge-COMXT1Hx.js → judge-DwzqucLS.js} +17 -12
  134. package/dist/{judge-COMXT1Hx.js.map → judge-DwzqucLS.js.map} +1 -1
  135. package/dist/judge.d.ts +3 -1
  136. package/dist/judge.js +1 -1
  137. package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
  138. package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
  139. package/dist/{layout-Cm85tQYk.js → layout-Czh5wOpz.js} +14 -11
  140. package/dist/layout-Czh5wOpz.js.map +1 -0
  141. package/dist/{list-DuThsE28.js → list-B_C0k5ex.js} +33 -19
  142. package/dist/{list-DuThsE28.js.map → list-B_C0k5ex.js.map} +1 -1
  143. package/dist/live-C9Et_BVT.js +1 -0
  144. package/dist/{live-CgYbO4Te.js → live-DGgIrVw0.js} +7 -5
  145. package/dist/live-DGgIrVw0.js.map +1 -0
  146. package/dist/{live-DL0se4Zd.js → live-DgFm0H90.js} +9 -9
  147. package/dist/{live-DL0se4Zd.js.map → live-DgFm0H90.js.map} +1 -1
  148. package/dist/machine-service-2lIv5UWF.js +1 -0
  149. package/dist/machine-service-Ct_1XMMi.js +520 -0
  150. package/dist/machine-service-Ct_1XMMi.js.map +1 -0
  151. package/dist/machines-BnquHcBZ.js +120 -0
  152. package/dist/machines-BnquHcBZ.js.map +1 -0
  153. package/dist/{model-CkNVAxob.js → model-B1tdv9-d.js} +1 -1
  154. package/dist/model-Bcy4Qjmm.js +1 -0
  155. package/dist/{model-V3mMXp6x.js → model-DF77SgZv.js} +6 -6
  156. package/dist/{model-V3mMXp6x.js.map → model-DF77SgZv.js.map} +1 -1
  157. package/dist/{model-CwwkrCb7.js → model-DRa7yv6K.js} +9 -6
  158. package/dist/model-DRa7yv6K.js.map +1 -0
  159. package/dist/{model-cgt5uvu8.js → model-E5X8YBJM.js} +7 -6
  160. package/dist/model-E5X8YBJM.js.map +1 -0
  161. package/dist/model-calls-B0BoQfJ4.js +95 -0
  162. package/dist/model-calls-B0BoQfJ4.js.map +1 -0
  163. package/dist/{node-D7pAU--0.js → node-BjuhqQwa.js} +21 -10
  164. package/dist/node-BjuhqQwa.js.map +1 -0
  165. package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
  166. package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
  167. package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-B1sIOTD2.js} +2 -2
  168. package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-B1sIOTD2.js.map} +1 -1
  169. package/dist/options-C989kuVE.js +1 -0
  170. package/dist/own-reads-CYmBlrHK.js +15 -0
  171. package/dist/own-reads-CYmBlrHK.js.map +1 -0
  172. package/dist/{patch-DAi1nCuH.js → patch-BR8rdl7r.js} +2 -2
  173. package/dist/{patch-DAi1nCuH.js.map → patch-BR8rdl7r.js.map} +1 -1
  174. package/dist/pi-providers-HygmL1y-.js +552 -0
  175. package/dist/pi-providers-HygmL1y-.js.map +1 -0
  176. package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
  177. package/dist/plain-Ctn8oUdp.js.map +1 -0
  178. package/dist/{plan-CsPilzpk.js → plan-BcT4G5jj.js} +104 -15
  179. package/dist/plan-BcT4G5jj.js.map +1 -0
  180. package/dist/{plugins-BfnhazOv.js → plugins-cC0sqoDA.js} +608 -12
  181. package/dist/plugins-cC0sqoDA.js.map +1 -0
  182. package/dist/preflight-BMVDBcux.js +1 -0
  183. package/dist/{preflight-BixaE4BA.js → preflight-C9OtTb7k.js} +4 -4
  184. package/dist/{preflight-BixaE4BA.js.map → preflight-C9OtTb7k.js.map} +1 -1
  185. package/dist/{preflight-GkpgrBxP.js → preflight-DySHXeFJ.js} +9 -10
  186. package/dist/preflight-DySHXeFJ.js.map +1 -0
  187. package/dist/{preflight-D1C-iH0U.js → preflight-zsAn3heN.js} +1 -1
  188. package/dist/{prepare-C--rM_wO.js → prepare-MaRWRhNT.js} +2 -2
  189. package/dist/prepare-nbX1CXXg.js +409 -0
  190. package/dist/prepare-nbX1CXXg.js.map +1 -0
  191. package/dist/prices-B1vm_lPk.js +1 -0
  192. package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
  193. package/dist/prices-BrY_ihLs.js.map +1 -0
  194. package/dist/{provenance-BXyYwUC8.js → provenance-CkykifD9.js} +5 -5
  195. package/dist/{provenance-BXyYwUC8.js.map → provenance-CkykifD9.js.map} +1 -1
  196. package/dist/provider-DHT20Gl8.js +1 -0
  197. package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
  198. package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
  199. package/dist/provider-stop-CxzWW7Ft.js +69 -0
  200. package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
  201. package/dist/providers-BkLGzaSY.js +1 -0
  202. package/dist/{providers-1p4HKYIp.js → providers-bNgfKa6m.js} +224 -49
  203. package/dist/providers-bNgfKa6m.js.map +1 -0
  204. package/dist/query-B2SQddoV.js +1 -0
  205. package/dist/{query-DhhU9ADI.js → query-DhV3a8lM.js} +2 -2
  206. package/dist/{query-DhhU9ADI.js.map → query-DhV3a8lM.js.map} +1 -1
  207. package/dist/{questions-CMAq4pr8.js → questions-Csv64WCb.js} +7 -7
  208. package/dist/{questions-CMAq4pr8.js.map → questions-Csv64WCb.js.map} +1 -1
  209. package/dist/{record-6IUzwP9K.js → record--vHreTLk.js} +11 -5
  210. package/dist/record--vHreTLk.js.map +1 -0
  211. package/dist/{redact-BhjtnjrU.js → redact-t-L3VGy8.js} +106 -2
  212. package/dist/redact-t-L3VGy8.js.map +1 -0
  213. package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
  214. package/dist/remarks-BpdM_2RC.js.map +1 -0
  215. package/dist/{remarks-BNABzemt.js → remarks-CYFOsnz5.js} +46 -11
  216. package/dist/remarks-CYFOsnz5.js.map +1 -0
  217. package/dist/remarks-CleoDlYw.js +1 -0
  218. package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
  219. package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
  220. package/dist/{remote-U7fNt0UP.js → remote-0CHog7Oq.js} +6 -6
  221. package/dist/{remote-U7fNt0UP.js.map → remote-0CHog7Oq.js.map} +1 -1
  222. package/dist/resolve-Bm9ZyFR-.js +1 -0
  223. package/dist/resolve-Bul8lHaa.js +205 -0
  224. package/dist/resolve-Bul8lHaa.js.map +1 -0
  225. package/dist/{resolve-Cx3a9UQ2.js → resolve-DorSHDBs.js} +14 -11
  226. package/dist/resolve-DorSHDBs.js.map +1 -0
  227. package/dist/resources-DqdwihhT.js +62 -0
  228. package/dist/resources-DqdwihhT.js.map +1 -0
  229. package/dist/{results-CQKXmHym.js → results-DhMgRkR0.js} +9 -22
  230. package/dist/results-DhMgRkR0.js.map +1 -0
  231. package/dist/{resume-POpZa1ge.js → resume-C8Ihfy-C.js} +166 -17
  232. package/dist/resume-C8Ihfy-C.js.map +1 -0
  233. package/dist/{retry-CmG809Mp.js → retry-Bfk7pDti.js} +4 -4
  234. package/dist/{retry-CmG809Mp.js.map → retry-Bfk7pDti.js.map} +1 -1
  235. package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
  236. package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
  237. package/dist/routes-Cp4gAjBZ.js +427 -0
  238. package/dist/routes-Cp4gAjBZ.js.map +1 -0
  239. package/dist/{runtime-Xm9oeC7o.js → runtime-CPUH0Vk5.js} +46 -5
  240. package/dist/runtime-CPUH0Vk5.js.map +1 -0
  241. package/dist/sandbox/boat.d.ts +1 -1
  242. package/dist/sandbox/boat.js +1 -1
  243. package/dist/{sandbox-DrlL4yqb.js → sandbox-z8q_4Ywn.js} +3 -3
  244. package/dist/{sandbox-DrlL4yqb.js.map → sandbox-z8q_4Ywn.js.map} +1 -1
  245. package/dist/sandbox.d.ts +1 -1
  246. package/dist/sandbox.js +1 -1
  247. package/dist/{server-C00zd6zl.js → server-BsQZtuF9.js} +9 -9
  248. package/dist/{server-C00zd6zl.js.map → server-BsQZtuF9.js.map} +1 -1
  249. package/dist/server-ZaMzUNmV.js +1 -0
  250. package/dist/services-BG8xuU82.js +39 -0
  251. package/dist/services-BG8xuU82.js.map +1 -0
  252. package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
  253. package/dist/shared-DtZwIAfy.js.map +1 -0
  254. package/dist/{source-DDgr48Gn.js → source-9HWaTEgV.js} +2 -2
  255. package/dist/{source-DDgr48Gn.js.map → source-9HWaTEgV.js.map} +1 -1
  256. package/dist/source-BcBDj-R0.js +1 -0
  257. package/dist/src-B2eHTCnp.js +3 -0
  258. package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
  259. package/dist/src-B5r1d23Y.js.map +1 -0
  260. package/dist/{src-Dw79VLa3.js → src-Bb3k9Kn_.js} +208 -59
  261. package/dist/src-Bb3k9Kn_.js.map +1 -0
  262. package/dist/src-BfOlH_Bu.js +2 -0
  263. package/dist/src-BuGc40IF.js +2 -0
  264. package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
  265. package/dist/src-C-ChdnG6.js.map +1 -0
  266. package/dist/src-C2K5_afU.js +2 -0
  267. package/dist/{src-YvblOrBU.js → src-C3OhA5pw.js} +87 -13
  268. package/dist/src-C3OhA5pw.js.map +1 -0
  269. package/dist/src-CSVJjghq.js +4 -0
  270. package/dist/{src-y0nOeRRE.js → src-CWfy8ct8.js} +1572 -487
  271. package/dist/src-CWfy8ct8.js.map +1 -0
  272. package/dist/src-CcrQQB0_.js +2 -0
  273. package/dist/src-CotUxxQV.js +2 -0
  274. package/dist/src-D-PwBaBB.js +2 -0
  275. package/dist/{src-Cb2MWozp.js → src-DNWiU__5.js} +276 -41
  276. package/dist/src-DNWiU__5.js.map +1 -0
  277. package/dist/{src-KxS91P7L.js → src-DwcTn4JY.js} +79 -11
  278. package/dist/src-DwcTn4JY.js.map +1 -0
  279. package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
  280. package/dist/src-boB6D-5T.js.map +1 -0
  281. package/dist/{src-DvGns8cz.js → src-c3ZQu7Uu.js} +299 -17
  282. package/dist/src-c3ZQu7Uu.js.map +1 -0
  283. package/dist/src-jYqNkT-F.js +1 -0
  284. package/dist/start-Db5VRcRZ.js +660 -0
  285. package/dist/start-Db5VRcRZ.js.map +1 -0
  286. package/dist/state-CKIjZZRK.js +1 -0
  287. package/dist/{state-328JpnHA.js → state-DUYdzNA7.js} +21 -21
  288. package/dist/{state-328JpnHA.js.map → state-DUYdzNA7.js.map} +1 -1
  289. package/dist/store/local.d.ts +8 -0
  290. package/dist/store/local.js +16 -0
  291. package/dist/store/local.js.map +1 -0
  292. package/dist/store/s3.d.ts +10 -0
  293. package/dist/store/s3.js +18 -0
  294. package/dist/store/s3.js.map +1 -0
  295. package/dist/suite-CH1aaAbs.js +2 -0
  296. package/dist/{suite-COglUhPd.js → suite-DjGxmDSY.js} +6 -5
  297. package/dist/{suite-COglUhPd.js.map → suite-DjGxmDSY.js.map} +1 -1
  298. package/dist/{summary-DDy9DfyJ.js → summary-BK5Sa_N0.js} +28 -7
  299. package/dist/summary-BK5Sa_N0.js.map +1 -0
  300. package/dist/sync-DUi_P2UJ.js +559 -0
  301. package/dist/sync-DUi_P2UJ.js.map +1 -0
  302. package/dist/{tarball-CESxtK4X.js → tarball-C8BSTAMk.js} +2 -2
  303. package/dist/{tarball-CESxtK4X.js.map → tarball-C8BSTAMk.js.map} +1 -1
  304. package/dist/tart-BQ_Ap0l4.js +508 -0
  305. package/dist/tart-BQ_Ap0l4.js.map +1 -0
  306. package/dist/tart-CzViW3WZ.js +186 -0
  307. package/dist/tart-CzViW3WZ.js.map +1 -0
  308. package/dist/tart-DmVuVsmX.js +1 -0
  309. package/dist/tart-bake-D1RttYfM.js +148 -0
  310. package/dist/tart-bake-D1RttYfM.js.map +1 -0
  311. package/dist/tart-bake-DjFcsG3N.js +1 -0
  312. package/dist/tart-cli-BOcluha1.js +1 -0
  313. package/dist/tart-cli-gA9xEaQL.js +89 -0
  314. package/dist/tart-cli-gA9xEaQL.js.map +1 -0
  315. package/dist/tart-hosts-CHNrfV6u.js +313 -0
  316. package/dist/tart-hosts-CHNrfV6u.js.map +1 -0
  317. package/dist/tart-hosts-CWse-ASo.js +1 -0
  318. package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CLBjfF9u.js} +64 -6
  319. package/dist/tart-images-CLBjfF9u.js.map +1 -0
  320. package/dist/testing.d.ts +36 -2
  321. package/dist/testing.js +4 -4
  322. package/dist/tool-class-krsmEo44.js +433 -0
  323. package/dist/tool-class-krsmEo44.js.map +1 -0
  324. package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
  325. package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
  326. package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
  327. package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
  328. package/dist/{transcript-CCf07UmU.js → transcript-BXWZiMuC.js} +3 -3
  329. package/dist/{transcript-CCf07UmU.js.map → transcript-BXWZiMuC.js.map} +1 -1
  330. package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
  331. package/dist/types-CbcnJxCy.js.map +1 -0
  332. package/dist/{types-DBwFsGem.d.ts → types-DKK8jj--.d.ts} +517 -45
  333. package/dist/ui-DKYjTVpn.js +1455 -0
  334. package/dist/ui-DKYjTVpn.js.map +1 -0
  335. package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
  336. package/dist/usage-CwIRPPZ9.js.map +1 -0
  337. package/dist/vercel-models-Ci33llXz.js +21 -0
  338. package/dist/vercel-models-Ci33llXz.js.map +1 -0
  339. package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
  340. package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
  341. package/dist/vertex-DXfcXYKn.js +1 -0
  342. package/dist/view/index.d.ts +32 -3
  343. package/dist/view/index.js +8 -8
  344. package/dist/{views-BROacjBv.js → views-B1zVLJsF.js} +6 -6
  345. package/dist/{views-BROacjBv.js.map → views-B1zVLJsF.js.map} +1 -1
  346. package/dist/views-DCjtw1jF.js +2 -0
  347. package/dist/{worker-entry-Smb4bNDc.js → worker-entry-DrAYYt0M.js} +37 -14
  348. package/dist/worker-entry-DrAYYt0M.js.map +1 -0
  349. package/dist/workspace-CnRbV8cM.js +1 -0
  350. package/dist/{workspace-DNDTEk8v.js → workspace-DwTe1zo6.js} +7 -7
  351. package/dist/{workspace-DNDTEk8v.js.map → workspace-DwTe1zo6.js.map} +1 -1
  352. package/docker/Dockerfile +1 -1
  353. package/package.json +15 -1
  354. package/skills/eval-rubric/SKILL.md +10 -0
  355. package/skills/reeval/SKILL.md +370 -0
  356. package/skills/reeval/references/capture.md +48 -0
  357. package/skills/reeval/references/cli.md +162 -0
  358. package/skills/reeval/references/compare.md +180 -0
  359. package/skills/reeval/references/config.md +147 -0
  360. package/skills/reeval/references/environments.md +218 -0
  361. package/skills/reeval/references/evals.md +241 -0
  362. package/skills/reeval/references/mistakes.md +39 -0
  363. package/skills/reeval/references/vocabulary.md +19 -0
  364. package/skills/setup/SKILL.md +219 -0
  365. package/skills/setup/references/templates.md +258 -0
  366. package/dist/adapter-C60z3wLq.js.map +0 -1
  367. package/dist/agent-BmsMXZXx.js.map +0 -1
  368. package/dist/agents-DXuFxtGC.js.map +0 -1
  369. package/dist/app-files-DT5mJyV4.js.map +0 -1
  370. package/dist/assemble-DM2PNU4R.js.map +0 -1
  371. package/dist/attempt-host-7VviAJrl.js.map +0 -1
  372. package/dist/boat-BjLqFJIZ.js.map +0 -1
  373. package/dist/boat-DSNgZt-t.js +0 -322
  374. package/dist/boat-DSNgZt-t.js.map +0 -1
  375. package/dist/capture-BzotD2hs.js +0 -1
  376. package/dist/capture-Ccf0Hmyx.js.map +0 -1
  377. package/dist/cases-Dy0mifal.js +0 -1
  378. package/dist/cases-z9E6DCd5.js.map +0 -1
  379. package/dist/cli-DTuLSlmW.js.map +0 -1
  380. package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
  381. package/dist/client-KM4C5r5_.js +0 -387
  382. package/dist/client-KM4C5r5_.js.map +0 -1
  383. package/dist/command-DSL7I-SI.js.map +0 -1
  384. package/dist/command-Ff0T-1Hi.js.map +0 -1
  385. package/dist/commands-xAOi_VEk.js.map +0 -1
  386. package/dist/config-SvK8pjEK.js.map +0 -1
  387. package/dist/cost-BdRgySQl.js.map +0 -1
  388. package/dist/cost-Cr26Alhq.js +0 -1
  389. package/dist/db-BzOJ4Js4.js.map +0 -1
  390. package/dist/db-CQRZ3URX.js +0 -2
  391. package/dist/docker--mukta3L.js +0 -1
  392. package/dist/docker-DG_uFgn2.js +0 -246
  393. package/dist/docker-DG_uFgn2.js.map +0 -1
  394. package/dist/doctor-DWK4ttex.js.map +0 -1
  395. package/dist/environment-cPMCOk-h.js.map +0 -1
  396. package/dist/eval-source-C9b1h8L6.js.map +0 -1
  397. package/dist/evaluate-B12Tosvq.js.map +0 -1
  398. package/dist/files-C1Rk9FYA.js +0 -1
  399. package/dist/files-DMQ9-Vv5.js +0 -1
  400. package/dist/files-Di0xDbCc.js +0 -1
  401. package/dist/files-G5jf8wBp.js +0 -1
  402. package/dist/files-s28a6czq.js.map +0 -1
  403. package/dist/harness-DXv2gnaT.js.map +0 -1
  404. package/dist/images-D2jFze2v.js.map +0 -1
  405. package/dist/init-Bi1Q3lRi.js.map +0 -1
  406. package/dist/init-CUw65o36.js +0 -1
  407. package/dist/ink-list-BpfidQ1h.js.map +0 -1
  408. package/dist/kinds-Bh2I35id.js +0 -698
  409. package/dist/kinds-Bh2I35id.js.map +0 -1
  410. package/dist/layout-Cm85tQYk.js.map +0 -1
  411. package/dist/live-CgYbO4Te.js.map +0 -1
  412. package/dist/live-DZLmj0iG.js +0 -1
  413. package/dist/matchers-CXLn-1jm.js +0 -148
  414. package/dist/matchers-CXLn-1jm.js.map +0 -1
  415. package/dist/model-Cv4GuTsZ.js +0 -1
  416. package/dist/model-CwwkrCb7.js.map +0 -1
  417. package/dist/model-cgt5uvu8.js.map +0 -1
  418. package/dist/node-D7pAU--0.js.map +0 -1
  419. package/dist/options-DPUmk7v_.js +0 -1
  420. package/dist/plain-BYUf_fUV.js.map +0 -1
  421. package/dist/plan-CsPilzpk.js.map +0 -1
  422. package/dist/plugins-BfnhazOv.js.map +0 -1
  423. package/dist/preflight-Bn3ianye.js +0 -1
  424. package/dist/preflight-GkpgrBxP.js.map +0 -1
  425. package/dist/prepare-DQEDXoDp.js +0 -268
  426. package/dist/prepare-DQEDXoDp.js.map +0 -1
  427. package/dist/prices-5m3SzMZ3.js.map +0 -1
  428. package/dist/provider-BnUoz4KB.js +0 -1
  429. package/dist/providers-1p4HKYIp.js.map +0 -1
  430. package/dist/providers-B-ijCktf.js +0 -1
  431. package/dist/query-BYhyf0LR.js +0 -1
  432. package/dist/record-6IUzwP9K.js.map +0 -1
  433. package/dist/redact-BhjtnjrU.js.map +0 -1
  434. package/dist/remarks-BNABzemt.js.map +0 -1
  435. package/dist/remarks-CAvTSq16.js +0 -1
  436. package/dist/remarks-D9Lcd1fF.js.map +0 -1
  437. package/dist/resolve-C14Pmtej.js +0 -1
  438. package/dist/resolve-Cx3a9UQ2.js.map +0 -1
  439. package/dist/results-CQKXmHym.js.map +0 -1
  440. package/dist/resume-POpZa1ge.js.map +0 -1
  441. package/dist/runtime-Xm9oeC7o.js.map +0 -1
  442. package/dist/server-C1UJcpUh.js +0 -1
  443. package/dist/shared-Cfi45Bvy.js.map +0 -1
  444. package/dist/source-DJHmQv_j.js +0 -1
  445. package/dist/src-1dBuXzs4.js +0 -3
  446. package/dist/src-BKpioayC.js +0 -2
  447. package/dist/src-BNQznj4z.js.map +0 -1
  448. package/dist/src-BY1cCaoJ.js +0 -2
  449. package/dist/src-BbZYaQ8b.js +0 -2
  450. package/dist/src-CDFH2vVN.js +0 -2
  451. package/dist/src-CMW5R9Q0.js +0 -1
  452. package/dist/src-CV_7DkPa.js.map +0 -1
  453. package/dist/src-Cb2MWozp.js.map +0 -1
  454. package/dist/src-CrV2WA2i.js +0 -2
  455. package/dist/src-CsFaawcZ.js +0 -2
  456. package/dist/src-DAzaCTcN.js.map +0 -1
  457. package/dist/src-DvGns8cz.js.map +0 -1
  458. package/dist/src-Dw79VLa3.js.map +0 -1
  459. package/dist/src-KxS91P7L.js.map +0 -1
  460. package/dist/src-YvblOrBU.js.map +0 -1
  461. package/dist/src-wKwznPgt.js +0 -2
  462. package/dist/src-y0nOeRRE.js.map +0 -1
  463. package/dist/state-Ca55Gk_O.js +0 -1
  464. package/dist/suite-CKBa6MKt.js +0 -2
  465. package/dist/summary-DDy9DfyJ.js.map +0 -1
  466. package/dist/tart-cli-C-GsbVsQ.js +0 -1
  467. package/dist/tart-cli-CU0nFS-L.js +0 -55
  468. package/dist/tart-cli-CU0nFS-L.js.map +0 -1
  469. package/dist/tart-dWoJvWHc.js +0 -586
  470. package/dist/tart-dWoJvWHc.js.map +0 -1
  471. package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
  472. package/dist/tool-target-DS5d72eH.js +0 -40
  473. package/dist/tool-target-DS5d72eH.js.map +0 -1
  474. package/dist/types-C2mLpph3.js.map +0 -1
  475. package/dist/usage-DJiDnh2D.js.map +0 -1
  476. package/dist/vercel-CGlSBv63.js +0 -344
  477. package/dist/vercel-CGlSBv63.js.map +0 -1
  478. package/dist/vertex-O88TCwgY.js +0 -1
  479. package/dist/views-B0Yt_Q1S.js +0 -2
  480. package/dist/worker-entry-Smb4bNDc.js.map +0 -1
  481. package/dist/workspace-C0dDP75l.js +0 -1
@@ -1,46 +1,51 @@
1
1
  import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-ttMTnIYX.js";
2
- import { g as parseDuration, m as MAX_TIMER_MS, n as dataProblem, t as closest } from "./shared-Cfi45Bvy.js";
3
- import { t as isObjectLike } from "./object-CeQ8dgUC.js";
4
- import { a as plural, i as formatDuration, n as clip, o as withoutBom, t as capMiddle } from "./text-B8-W3Cvn.js";
5
- import { a as TURN_EDITED, f as limitsLabel, r as withLease } from "./client-KM4C5r5_.js";
6
- import { d as sandboxKind, n as capabilitiesOf, s as providerLimit } from "./kinds-Bh2I35id.js";
7
- import { O as sandboxPaths, P as workspaceOf, f as filesOf, i as callerDir, j as traitsOf, l as environmentFrom, p as globFilter, u as environmentPaths } from "./environment-cPMCOk-h.js";
2
+ import { g as parseDuration, m as MAX_TIMER_MS, n as dataProblem, t as closest } from "./shared-DtZwIAfy.js";
3
+ import { Gt as OPAQUE_OUTPUT, Yt as shellQuote$1, u as MCP_TIMER } from "./src-CWfy8ct8.js";
8
4
  import { t as evalSlug } from "./slug-IHh3f770.js";
9
- import { n as attemptKey, o as recordFileName, r as evalId, s as relativeFile, t as attemptId } from "./ids-C_advdJA.js";
10
- import { $ as runsOf, A as brokeredIn, B as expand, Bt as decideState, Ct as resolveStateOptions, Et as observer, Ft as NO_JUDGE_INPUT, Ht as resolveJudgeArtifacts, It as inputsOn, J as plainHeaders, K as plainHeaderError, L as checkOptions, Lt as judgeProblem, Pt as INPUT_KEYS, Q as refuseIllegalCompare, Rt as artifactRefsOf, St as mergeJudge, Tt as observedName, U as majorityOf, V as harnessId, Vt as filePartsOf, W as matchesTags, _ as fellBackToDirect, at as unbrokered, b as resolveModelGateway, et as sandboxId, gt as unenforceable, ht as isDirect, it as secretVarsIn, j as cascade, nt as sandboxSetBy, ot as unbrokeredError, pt as egressLabel, tt as sandboxProvider, ut as allowedHosts, v as gatewayScope, w as resolveGateway, wt as observedHere, z as evalPlan, zt as badJudgeArtifacts } from "./config-SvK8pjEK.js";
11
- import { l as vercelCredentialProblem } from "./vercel-CGlSBv63.js";
12
- import { v as vertexSettings } from "./vertex-Eb2wqahk.js";
13
- import { _ as redactValue, d as maskedVariables, f as redactJson, g as redactSplitSecrets, h as redactSecrets, o as findSecrets } from "./redact-BhjtnjrU.js";
14
- import { f as runJudge, h as isApiError, l as judgeModelName } from "./judge-COMXT1Hx.js";
5
+ import { c as relativeFile, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-B-JKTCP8.js";
6
+ import { _ as redactSplitSecrets, d as maskedVariables, f as redactFields, g as redactSecrets, o as findSecrets, p as redactJson, y as redactValue } from "./redact-t-L3VGy8.js";
7
+ import { B as providerLimit, I as capabilitiesOf, S as pluginBuildId, W as sandboxKind, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, k as pluginSourceLabel, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-cC0sqoDA.js";
8
+ import { a as plural, i as formatDuration, n as clip, o as withoutBom, t as capMiddle } from "./text-B8-W3Cvn.js";
9
+ import { h as vercelCredentialProblem, m as vercelBaseUrlProblem, p as vercelBaseUrl } from "./routes-Cp4gAjBZ.js";
10
+ import { v as vertexSettings } from "./vertex-BEWxHoxH.js";
11
+ import { n as hasNetworkSession, t as InfraError } from "./types-CbcnJxCy.js";
12
+ import { C as writePluginFiles, S as refusedSharedMachine, b as refusedMcp, c as agentMcp, d as brokeringFor, f as claudeMcpServers, h as fakeBrokering, l as agentOf, m as expectedBlockedFor, t as AGENTS, v as keyRejected, x as refusedPlugins, y as layoutOf } from "./agents-KBTYhSSh.js";
13
+ import { t as isObjectLike } from "./object-CeQ8dgUC.js";
14
+ import { d as limitsLabel, f as runMatchLabel, r as withLease, s as judgeFailedMessage, u as TURN_EDITED } from "./client-DMEYR3MA.js";
15
+ import { A as sandboxPaths, I as workspaceOf, N as traitsOf, f as filesOf, i as callerDir, l as environmentFrom, p as globFilter, u as environmentPaths } from "./environment-C0Gg8kyN.js";
16
+ import { $ as plainHeaders, $t as decideState, E as resolveGateway, Ft as mergeJudge, G as majorityOf, H as expand, It as resolveStateOptions, J as plainEnv, Jt as NO_JUDGE_INPUT, K as matchesTags, Lt as observedHere, M as brokeredIn, N as cascade, Qt as badJudgeArtifacts, Rt as observedName, S as resolveModelGateway, Tt as phaseLabel, U as harnessId, V as evalPlan, Xt as judgeProblem, Y as plainEnvError, Yt as inputsOn, Z as plainHeaderError, Zt as artifactRefsOf, at as sandboxId, b as gatewayScope, bt as gatewayPolicy, dt as unbrokeredError, en as filePartsOf, ht as allowedHosts, it as runsOf, kt as unenforceable, lt as secretVarsIn, ot as sandboxProvider, qt as INPUT_KEYS, rt as refuseIllegalCompare, st as sandboxSetBy, tn as resolveJudgeArtifacts, ut as unbrokered, xt as isDirect, y as fellBackToDirect, yt as egressLabel, z as checkOptions, zt as observer } from "./config-C0RKt00M.js";
17
+ import { f as runJudge, g as isTransient, h as isApiError, l as judgeModelName } from "./judge-DwzqucLS.js";
15
18
  import { t as inCI } from "./env-8x1O8I1M.js";
16
- import { bt as shellQuote, gt as OPAQUE_OUTPUT, n as MCP_TIMER } from "./src-y0nOeRRE.js";
17
- import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-BfnhazOv.js";
18
- import { _ as unverifiedHarness, c as brokeringFor, d as expectedBlockedFor, g as refusedPlugins, h as refusedMcp, l as claudeMcpServers, m as layoutOf, o as agentMcp, p as keyRejected, t as AGENTS, v as writePluginFiles } from "./agents-DXuFxtGC.js";
19
- import { n as hasNetworkSession, t as InfraError } from "./types-C2mLpph3.js";
20
- import { a as registerTest, i as registerGroup, n as declarePlanned, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-Xm9oeC7o.js";
19
+ import { a as registerTest, i as registerGroup, n as declarePlanned, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-CPUH0Vk5.js";
21
20
  import { c as formatBytes } from "./debug-CIXzB4wK.js";
22
- import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CAdlwlNu.js";
23
- import { t as attemptDebugDir } from "./files-DlQshsks.js";
24
- import { r as openRouterFallbacks } from "./openrouter-models-Cw1z8-sq.js";
25
- import { a as writeCaptureSidecars, c as sandboxFor, d as snapshot, f as workspaceBlobs, l as changedBetween, o as asCheck, p as workspaceFiles, r as serveCommand, s as createChecks, t as capturePage, u as diffSinceSnapshot } from "./capture-Ccf0Hmyx.js";
26
- import { t as contextTaker } from "./context-BdgCao9Q.js";
27
- import { a as skillFiles, i as resolveEnvironment } from "./resolve-Cx3a9UQ2.js";
28
- import { t as PrepareError } from "./provider-BxzZq3Th.js";
29
- import { i as preparedWorkspace } from "./workspace-DNDTEk8v.js";
21
+ import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-_dUsblVx.js";
22
+ import { t as attemptDebugDir } from "./files-g1watgRC.js";
23
+ import { r as openRouterFallbacks } from "./openrouter-models-B1sIOTD2.js";
24
+ import { a as writeCaptureSidecars, c as sandboxFor, d as snapshot, f as workspaceBlobs, l as changedBetween, o as asCheck, p as workspaceFiles, r as serveCommand, s as createChecks, t as capturePage, u as diffSinceSnapshot } from "./capture-BkbPsfej.js";
25
+ import { t as contextTaker } from "./context-DrEAtUyg.js";
26
+ import { a as skillFiles, i as resolveEnvironment } from "./resolve-DorSHDBs.js";
27
+ import { n as fakeBootstrapScript } from "./fake-bootstrap-VpisF1WE.js";
28
+ import { n as providerStop, t as ProviderStopError } from "./provider-stop-CxzWW7Ft.js";
29
+ import { t as PrepareError } from "./provider-ieWOn4YS.js";
30
+ import { i as preparedWorkspace } from "./workspace-DwTe1zo6.js";
30
31
  import { t as patchPath } from "./patch-C7yBztZj.js";
31
- import { n as judgeDiff } from "./patch-DAi1nCuH.js";
32
- import { t as transcriptForJudge } from "./transcript-CCf07UmU.js";
33
- import { i as writeTar, n as readTar, t as extractTar } from "./tarball-CESxtK4X.js";
34
- import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-C9b1h8L6.js";
35
- import { i as withDefaultEffort } from "./default-effort-BDKIxJyC.js";
36
- import { a as traitFilters, i as pickTraits, n as controlTraitKeys, r as filtersLabel } from "./traits-DKnYkqeK.js";
32
+ import { n as judgeDiff } from "./patch-BR8rdl7r.js";
33
+ import { t as transcriptForJudge } from "./transcript-BXWZiMuC.js";
34
+ import { n as modelCallEvents } from "./model-calls-B0BoQfJ4.js";
35
+ import { t as resourcesLine } from "./resources-DqdwihhT.js";
36
+ import { a as writeTar, n as readTar, t as extractTar } from "./tarball-C8BSTAMk.js";
37
+ import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-CsW5VIXS.js";
38
+ import { t as environmentInputs } from "./inputs-5UIDpjz-.js";
39
+ import { i as withDefaultEffort } from "./default-effort-D70cTG00.js";
40
+ import { a as traitFilters, i as pickTraits, n as controlTraitKeys, r as filtersLabel } from "./traits-CCD8XsmI.js";
37
41
  import { z } from "zod";
38
42
  import { dirname, join } from "node:path";
39
- import { parse, stringify } from "yaml";
40
- import { randomBytes } from "node:crypto";
41
- import { setTimeout as setTimeout$1 } from "node:timers/promises";
43
+ import { randomBytes, randomUUID } from "node:crypto";
42
44
  import { gunzipSync, gzipSync } from "node:zlib";
45
+ import { parse, stringify } from "yaml";
43
46
  import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
47
+ import { setTimeout as setTimeout$1 } from "node:timers/promises";
48
+ import { resolveBridgePort } from "@ai-sdk/harness/utils";
44
49
  // src/attempt/assertions.ts
45
50
  const MESSAGE_LIMIT = 1024;
46
51
  const LABEL_LIMIT = 60;
@@ -70,7 +75,7 @@ function assertionRecords(raw) {
70
75
  const seen = /* @__PURE__ */ new Map();
71
76
  const used = /* @__PURE__ */ new Set();
72
77
  return raw.map((r) => {
73
- const named = r.matcher === "toStayWithin" ? limitsLabel(r.args[0]) : r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
78
+ const named = r.matcher === "toStayWithin" ? limitsLabel(r.args[0]) : r.matcher === "toHaveRun" && runMatchLabel(r.args[0]) !== void 0 ? runMatchLabel(r.args[0]) : r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
74
79
  const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
75
80
  let n = seen.get(base) ?? 0;
76
81
  let label;
@@ -103,12 +108,25 @@ function checkFailure(raw) {
103
108
  if (!failed) return void 0;
104
109
  const error = failed.error instanceof Error ? failed.error : new Error(text(failed.error));
105
110
  if (failed.soft) return error;
106
- const caught = Object.setPrototypeOf(Object.assign(/* @__PURE__ */ new Error(), error), Object.getPrototypeOf(error));
107
- caught.name = error.name;
108
- caught.message = `${error.message} ${CAUGHT_HINT}`;
109
- caught.stack = error.stack?.replace(error.message, caught.message);
110
- return caught;
111
- }
111
+ const message = `${error.message} ${CAUGHT_HINT}`;
112
+ const stack = error.stack?.replace(error.message, () => message);
113
+ if (error instanceof DOMException) return Object.defineProperties(new Error(message, { cause: error }), {
114
+ name: own(error.name),
115
+ stack: own(stack)
116
+ });
117
+ return Object.setPrototypeOf(Object.defineProperties(/* @__PURE__ */ new Error(), {
118
+ ...Object.getOwnPropertyDescriptors(error),
119
+ name: own(error.name),
120
+ message: own(message),
121
+ stack: own(stack)
122
+ }), Object.getPrototypeOf(error));
123
+ }
124
+ const own = (value) => ({
125
+ value,
126
+ writable: true,
127
+ configurable: true,
128
+ enumerable: false
129
+ });
112
130
  // src/attempt/debug.ts
113
131
  /** Which config and profile the attempt runs under, and where each resolved option came from. */
114
132
  function tracePlan(debug, spec, ctx) {
@@ -206,12 +224,17 @@ const lineCount = (text) => text ? text.split("\n").length - (text.endsWith("\n"
206
224
  */
207
225
  const createFakeHarness = (descriptor) => {
208
226
  let starts = 0;
209
- return { async start(sandbox, _signal, { skills, workspaceSkills, mcp: all, plugins }) {
227
+ return { async start(sandbox, signal, { skills, workspaceSkills, mcp: all, plugins }) {
210
228
  if (descriptor.failWith) throw new InfraError(descriptor.failWith);
211
229
  const hangs = descriptor.startHangs;
212
230
  if (hangs === true || starts++ < (hangs ?? 0)) return new Promise(() => {});
231
+ if (descriptor.bootstrap) {
232
+ const r = await sandbox.exec(fakeBootstrapScript(descriptor.bootstrap), { signal });
233
+ if (r.exitCode !== 0) throw new InfraError(`The fake agent's bootstrap exited ${r.exitCode}: ${redactSecrets(r.stderr || r.stdout).trim().slice(-500)}`);
234
+ }
213
235
  const layout = layoutOf(descriptor);
214
236
  const brokered = descriptor.brokers ? await brokerKey(descriptor, sandbox) : void 0;
237
+ if (descriptor.brokers && AGENTS[descriptor.layout ?? "cursor"].bridgePort) await reachBridge(sandbox, signal);
215
238
  const mcp = agentMcp(layout, all, plugins);
216
239
  for (const path of await writePluginFiles(sandbox, layout, plugins)) debugLog("env", `plugin settings → ${path} by the fake agent`);
217
240
  const dir = `~/${layout.harnessSkillDir}`;
@@ -231,9 +254,11 @@ const createFakeHarness = (descriptor) => {
231
254
  const files = (skilled ? descriptor.withSkill?.files : void 0) ?? descriptor.files ?? {};
232
255
  const reply = (skilled ? descriptor.withSkill?.text : void 0) ?? descriptor.text ?? "Done.";
233
256
  const flaked = Math.random() < (descriptor.flakeRate ?? 0);
257
+ let running;
234
258
  return {
235
259
  async send(prompt, signal) {
236
260
  const startedAt = Date.now();
261
+ const brokeredBefore = sandbox.brokeredStatuses?.().length ?? 0;
237
262
  const script = descriptor.script ?? [];
238
263
  const steps = script[Math.min(turns++, script.length - 1)] ?? [];
239
264
  const at = () => {
@@ -261,10 +286,19 @@ const createFakeHarness = (descriptor) => {
261
286
  durationMs: descriptor.turnMs ?? Date.now() - startedAt,
262
287
  transcript,
263
288
  usage: descriptor.usage ?? null,
289
+ ...!descriptor.costUsd ? {} : { cost: {
290
+ usd: descriptor.costUsd,
291
+ basis: "reported",
292
+ source: "fake"
293
+ } },
264
294
  model: descriptor.servedModel ?? null,
265
295
  unreported: [],
266
296
  ...descriptor.turnMs === void 0 ? {} : { agentMs: descriptor.turnMs }
267
297
  });
298
+ running = (note) => ({
299
+ ...done("", "aborted"),
300
+ transcript: [...transcript, note]
301
+ });
268
302
  const write = async (path, content) => {
269
303
  const time = at();
270
304
  const before = await sandbox.readFile(path);
@@ -294,8 +328,9 @@ const createFakeHarness = (descriptor) => {
294
328
  tool: tool.tool,
295
329
  input: tool.input,
296
330
  output: tool.output,
331
+ outputView: "model",
297
332
  ok: tool.ok === void 0 ? true : tool.ok,
298
- durationMs: 0
333
+ durationMs: tool.durationMs === void 0 ? 0 : tool.durationMs
299
334
  });
300
335
  if (flaked) {
301
336
  const text = "I ran out of ideas and left the workspace unchanged.";
@@ -323,6 +358,7 @@ const createFakeHarness = (descriptor) => {
323
358
  tool: "shell",
324
359
  input: { command: step.exec },
325
360
  output: r.stdout,
361
+ outputView: "model",
326
362
  ok: r.exitCode === 0,
327
363
  durationMs: Date.now() - startedAt - time.atMs
328
364
  });
@@ -340,6 +376,7 @@ const createFakeHarness = (descriptor) => {
340
376
  tool: `mcp__${server}__${tool}`,
341
377
  input: {},
342
378
  output: r.output,
379
+ outputView: "model",
343
380
  ok: r.ok,
344
381
  durationMs: Date.now() - startedAt - time.atMs
345
382
  });
@@ -350,7 +387,16 @@ const createFakeHarness = (descriptor) => {
350
387
  type: "error",
351
388
  message: step.error
352
389
  });
353
- return done("", "error");
390
+ const statuses = sandbox.brokeredStatuses?.().slice(brokeredBefore) ?? [];
391
+ const stopped = providerStop({
392
+ errors: [step.error],
393
+ status: "error",
394
+ statuses
395
+ });
396
+ return {
397
+ ...done("", "error"),
398
+ ...stopped && { providerStop: stopped }
399
+ };
354
400
  } else if ("refuseKey" in step) {
355
401
  const message = `${keyRejected(step.refuseKey)}. 401 Unauthorized (the fake agent's)`;
356
402
  debugLog("harness", `turn failed: ${message}`);
@@ -379,6 +425,10 @@ const createFakeHarness = (descriptor) => {
379
425
  });
380
426
  return done(text, "completed");
381
427
  },
428
+ stopped: async (note) => {
429
+ if (!running) throw new Error("No turn is running");
430
+ return running(note);
431
+ },
382
432
  async close() {}
383
433
  };
384
434
  } };
@@ -388,7 +438,7 @@ const createFakeHarness = (descriptor) => {
388
438
  * endpoints, as a real adapter does, and returns the variable its `exec` steps get it in.
389
439
  */
390
440
  async function brokerKey(descriptor, sandbox) {
391
- const { credential, endpoints } = AGENTS[descriptor.layout ?? "cursor"];
441
+ const { credential, endpoints } = fakeBrokering(descriptor.layout);
392
442
  const key = process.env[credential];
393
443
  if (!key) throw new InfraError(`${credential} isn't set, so the fake agent has no key to broker. Set it in the environment or .env.local.`);
394
444
  const network = hasNetworkSession(sandbox) ? sandbox.network : void 0;
@@ -408,6 +458,40 @@ async function brokerKey(descriptor, sandbox) {
408
458
  return { [credential]: placeholder };
409
459
  }
410
460
  /**
461
+ * Serves on the port the layout's adapter would put its bridge on, in the sandbox, and reaches it
462
+ * from the host through the session's endpoint, as that adapter's start does.
463
+ */
464
+ async function reachBridge(sandbox, signal) {
465
+ const network = hasNetworkSession(sandbox) ? sandbox.network : void 0;
466
+ if (!network) throw new InfraError("The fake agent's bridge needs a sandbox session");
467
+ let port;
468
+ try {
469
+ port = resolveBridgePort({
470
+ sandboxSession: network,
471
+ portOverride: void 0
472
+ });
473
+ } catch (error) {
474
+ throw new InfraError(`Fake harness failed to start: ${error.message}`);
475
+ }
476
+ const server = await network.spawn({
477
+ command: `node -e ${shellQuote$1(`require('http').createServer((q, s) => s.end('bridge')).listen(${port}, '0.0.0.0', () => console.log('up'))`)}`,
478
+ abortSignal: signal
479
+ });
480
+ try {
481
+ const up = await server.stdout.getReader().read();
482
+ if (!new TextDecoder().decode(up.value).includes("up")) throw new InfraError(`The fake agent's bridge didn't listen on ${port}`);
483
+ const { url } = await network.getPortEndpoint({
484
+ port,
485
+ protocol: "http"
486
+ });
487
+ const reply = await (await fetch(url, { signal })).text();
488
+ if (reply !== "bridge") throw new InfraError(`${url} answered ${JSON.stringify(reply)}`);
489
+ debugLog("harness", `bridge port ${port} reached through ${url}`);
490
+ } finally {
491
+ await server.kill();
492
+ }
493
+ }
494
+ /**
411
495
  * Starts a stdio MCP server in the sandbox, as an agent would, and calls one tool with no
412
496
  * arguments: initialize, then the call.
413
497
  */
@@ -445,8 +529,8 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
445
529
  arguments: {}
446
530
  }
447
531
  }
448
- ].map((m) => shellQuote(JSON.stringify(m)));
449
- const command = [server.command, ...server.args].map(shellQuote).join(" ");
532
+ ].map((m) => shellQuote$1(JSON.stringify(m)));
533
+ const command = [server.command, ...server.args].map(shellQuote$1).join(" ");
450
534
  const script = [
451
535
  "d=$(mktemp -d) && : > \"$d/out\" && : > \"$d/err\" || exit 1",
452
536
  "{",
@@ -486,7 +570,7 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
486
570
  /** The attempt's harness, for the sandbox it runs in; an agent the sandbox can't run is an infra error. */
487
571
  async function createHarness(descriptor, sandbox, vertex) {
488
572
  if (descriptor.kind === "fake") return createFakeHarness(descriptor);
489
- const { createAgentHarness } = await import("./agent-BmsMXZXx.js");
573
+ const { createAgentHarness } = await import("./agent-B-6GFDqA.js");
490
574
  return createAgentHarness(descriptor, sandbox, vertex);
491
575
  }
492
576
  /** Where each harness's agent looks for repo skills, relative to the workspace. */
@@ -501,7 +585,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
501
585
  const { spec, record } = a;
502
586
  const env = await resolveEnvironment(spec.environment, spec.env, root, where, layoutOf(spec.harness).rulesFile, spec.egress.policy, (w) => preparedWorkspace(root, runId, w), {
503
587
  harness: spec.harness,
504
- allowed: allowedHosts(spec.egress)
588
+ allowed: allowedHosts(spec.egress),
589
+ ...spec.egress.setup ? { setupEgress: spec.egress.setup.policy } : {}
505
590
  });
506
591
  a.env = env;
507
592
  record.environment = env.record;
@@ -516,8 +601,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
516
601
  record.image = null;
517
602
  const kind = sandboxKind(spec.sandbox);
518
603
  if (kind.capabilities.images) {
519
- const { preparedFor } = await import("./preflight-D1C-iH0U.js");
520
- const { attemptImage, createProviders, preparedImage } = await import("./providers-B-ijCktf.js");
604
+ const { preparedFor } = await import("./preflight-zsAn3heN.js");
605
+ const { attemptImage, createProviders, preparedImage } = await import("./providers-BkLGzaSY.js");
521
606
  signal.throwIfAborted();
522
607
  const provider = createProviders()[kind.provider(spec.sandbox)];
523
608
  const builds = builtPlugins(spec.environment.layers);
@@ -542,14 +627,18 @@ async function prepareEnvironment(a, root, runId, where, signal) {
542
627
  const { ref, id, base, install, agentCli } = record.image;
543
628
  debugLog("image", `ready ${ref} (${id.replace(/^sha256:/, "").slice(0, 12)}) · prepared before the run${ref !== base ? ` · on ${base}` : ""}${agentCli ? ` · agent CLI ${agentCli}` : ""}${install.length ? ` · install: ${install.join(" && ")}` : ""}`);
544
629
  } else if (kind.capabilities.isolated) {
545
- const { preparedFor } = await import("./preflight-D1C-iH0U.js");
546
- const { createProviders, vmImage } = await import("./providers-B-ijCktf.js");
630
+ const { preparedFor } = await import("./preflight-zsAn3heN.js");
631
+ const { bakedImage, createProviders, preparedHosts, vmImage } = await import("./providers-BkLGzaSY.js");
547
632
  const provider = createProviders()[kind.provider(spec.sandbox)];
548
633
  const prepared = provider && preparedFor(root, runId, provider, spec.sandbox, spec.harness);
549
634
  const vm = prepared ? vmImage(prepared) : null;
550
635
  if (vm) record.vmImage = vm;
636
+ const baked = prepared ? bakedImage(prepared) : null;
637
+ if (baked) a.image = baked;
638
+ const hosts = prepared ? preparedHosts(prepared) : null;
639
+ if (hosts) a.hosts = hosts;
551
640
  const from = vm?.ref ?? ("image" in spec.sandbox ? spec.sandbox.image : void 0);
552
- debugLog("image", `${from ?? "none recorded"}${vm?.digest ? ` (${vm.digest})` : ""} · ${kind.helpers[0]}() checked it before the run and starts each attempt from it`);
641
+ debugLog("image", `${from ?? "none recorded"}${vm?.digest ? ` (${vm.digest})` : ""} · ${kind.helpers[0]}() checked it before the run and starts each attempt from ${baked ? `${baked}, which has the harness baked in` : "it"}`);
553
642
  } else debugLog("image", kind.capabilities.isolated ? `none: the ${spec.sandbox.kind} sandbox starts from its provider's base` : `none: the ${spec.sandbox.kind} sandbox runs on the host`);
554
643
  }
555
644
  // src/judge/size.ts
@@ -607,6 +696,19 @@ function stateHint(state, question) {
607
696
  * no model's limit, so this follows every provider error.
608
697
  */
609
698
  const withInputSize = (message, state, question) => `${message}\nThe judge's input was ~${k(inputTokens(state, question))} tokens (estimated): ${describeSize(state, question)}.${stateHint(state, question)}`;
699
+ const count = (digits) => Number(digits.replaceAll(",", ""));
700
+ /**
701
+ * Whether the provider refused the call on a token limit this input alone is over, which no
702
+ * retry fixes: a request too large, OpenAI's `Limit 30000, Requested 45000`, or Anthropic's
703
+ * `of 30,000 input tokens per minute` against the input's estimated size.
704
+ */
705
+ function overTokenLimit(message, state, question) {
706
+ if (/request too large/i.test(message)) return true;
707
+ const asked = /Limit:? ([\d,]+), Requested:? ([\d,]+)/i.exec(message);
708
+ if (asked) return count(asked[2]) > count(asked[1]);
709
+ const perMinute = /([\d,]+) (?:input )?tokens per minute/i.exec(message);
710
+ return !!perMinute && inputTokens(state, question) >= count(perMinute[1]);
711
+ }
610
712
  // src/judge/state.ts
611
713
  function turnState(t) {
612
714
  return {
@@ -778,10 +880,16 @@ function judgeCaller(a, readFile) {
778
880
  debugLog("judge", `asking ${name}: ${JSON.stringify(clip(redactSecrets(question), 160))}`);
779
881
  if (dropped.length) debugLog("judge", `left out of the judge's input: ${dropped.join(" · ")}`);
780
882
  debugTrace("judge", "prompt", input, "judge prompt");
781
- const failed = (message, retries) => {
883
+ const failed = (message, transient, retries) => {
782
884
  a.judgeErrors.push({
783
885
  call: order,
784
- message: `c.judge(${JSON.stringify(clip(redactSecrets(question), 80))}) failed: ${message}`
886
+ message: `c.judge(${JSON.stringify(clip(redactSecrets(question), 80))}) failed: ${message}`,
887
+ transient,
888
+ matcher: judgeFailedMessage({
889
+ question,
890
+ model: name,
891
+ error: message
892
+ })
785
893
  });
786
894
  input.error = message;
787
895
  if (retries?.length) input.retries = retries;
@@ -819,7 +927,7 @@ function judgeCaller(a, readFile) {
819
927
  ...unpinned,
820
928
  gateway: entry || gateway
821
929
  } : unpinned;
822
- const { verdict, retries } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
930
+ const { verdict, retries, usage } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
823
931
  ...routed,
824
932
  model,
825
933
  threshold: atLeast
@@ -832,6 +940,7 @@ function judgeCaller(a, readFile) {
832
940
  judgments.splice(placeFor(judgments.map((j) => j.call ?? 0), order), 0, {
833
941
  ...recorded,
834
942
  ...retries.length ? { retries } : {},
943
+ ...usage ? { usage } : {},
835
944
  call: order
836
945
  });
837
946
  const { score } = verdict;
@@ -853,7 +962,8 @@ function judgeCaller(a, readFile) {
853
962
  if (a.signal.aborted) throw error;
854
963
  const { retries } = error;
855
964
  const message = error instanceof Error ? error.message : String(error);
856
- return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
965
+ const transient = isTransient(error) && !overTokenLimit(message, state, question);
966
+ return failed(isApiError(error) && !transient ? withInputSize(message, state, question) : message, transient, retries);
857
967
  }
858
968
  };
859
969
  return async (question, given) => {
@@ -878,6 +988,191 @@ async function createSandbox(descriptor, options) {
878
988
  throw new InfraError(`Failed to create ${kind.provider(descriptor)} sandbox: ${error.message}`);
879
989
  }
880
990
  }
991
+ // src/view/timeline.ts
992
+ const isToolLike = (e) => e.type === "file-edit" || e.type === "web-search" || e.type === "web-fetch" || e.type === "tool";
993
+ /**
994
+ * The events with each tool call's missing `durationMs` run to the next event's `atMs` (the
995
+ * turn's end for the last), flagged `durationInferred`. A main model call ends it too: the agent
996
+ * only calls the model again once the tool's result is back. A side call or a call that carries
997
+ * the whole turn doesn't.
998
+ */
999
+ function inferDurations(events, turnMs) {
1000
+ return events.map((e, i) => {
1001
+ if (!isToolLike(e) || e.durationMs !== null) return e;
1002
+ const next = events.slice(i + 1).find((n) => n.atMs > e.atMs && (n.type !== "model-call" || !n.stream && n.role !== "side"));
1003
+ const end = Math.max(e.atMs, next ? next.atMs : turnMs);
1004
+ return {
1005
+ ...e,
1006
+ durationMs: end - e.atMs,
1007
+ durationInferred: true
1008
+ };
1009
+ });
1010
+ }
1011
+ const SHELL_TOOL = /shell|bash|terminal|command|exec/i;
1012
+ const OUTPUT_TOOL = /^(bash_?output|task_?output|await_?shell)$/i;
1013
+ const CURSOR_TERMINAL = /\.cursor\/projects\/[^"]*\/terminals\/[^"/]+\.txt/;
1014
+ const TERMINAL_COMMAND = /command:\s*("(?:[^"\\]|\\.)*")/;
1015
+ const record = (v) => v && typeof v === "object" ? v : {};
1016
+ /** A shell call's command, or null for another tool. */
1017
+ function commandOf(e) {
1018
+ if (e.type !== "tool" || !SHELL_TOOL.test(e.tool) || OUTPUT_TOOL.test(e.tool)) return null;
1019
+ const input = record(e.input);
1020
+ const command = input.command ?? input.cmd;
1021
+ return typeof command === "string" ? command.trim() : null;
1022
+ }
1023
+ const strings = (v) => typeof v === "string" ? [v] : v && typeof v === "object" ? Object.values(v).flatMap(strings) : [];
1024
+ const textOf = (v) => typeof v === "string" ? v : JSON.stringify(v) ?? "";
1025
+ /** The command a Cursor terminal file's header names, from a read of it. */
1026
+ function terminalCommand(e) {
1027
+ if (e.type !== "tool") return null;
1028
+ const match = strings(e.output).map((text) => TERMINAL_COMMAND.exec(text)).find(Boolean);
1029
+ if (!match?.[1]) return null;
1030
+ try {
1031
+ return String(JSON.parse(match[1])).trim();
1032
+ } catch {
1033
+ return null;
1034
+ }
1035
+ }
1036
+ function backgrounded(e, named) {
1037
+ if (e.type !== "tool") return false;
1038
+ const input = record(e.input);
1039
+ if (input.run_in_background === true || input.is_background === true) return true;
1040
+ const command = commandOf(e);
1041
+ if (command === null) return false;
1042
+ return /(^|[^&])&$/.test(command) || /(^|[\s;&|])(nohup|setsid)\s/.test(command) || named.has(command);
1043
+ }
1044
+ function polls(e) {
1045
+ if (e.type !== "tool") return false;
1046
+ if (OUTPUT_TOOL.test(e.tool)) return true;
1047
+ const command = commandOf(e);
1048
+ if (command !== null) return /^(sleep|wait)(\s|$)/.test(command);
1049
+ return CURSOR_TERMINAL.test(textOf(e.input));
1050
+ }
1051
+ const endOf = (e) => e.atMs + (e.durationMs ?? 0);
1052
+ /** Work that ends a wait no poll has checked yet: an edit, or another command. */
1053
+ const works = (e) => e.type === "file-edit" || commandOf(e) !== null;
1054
+ /**
1055
+ * The background jobs the agent waited on, a heuristic over the transcript: a span opens at a
1056
+ * call run in the background or a poll and takes in each poll after it. Until its first poll,
1057
+ * lookups (reads, searches) leave it open and an edit or another command drops it; after it,
1058
+ * the next call that isn't a poll closes it. A background call no poll followed isn't waited on.
1059
+ */
1060
+ function waitSpans(events) {
1061
+ const named = new Set(events.map(terminalCommand).filter((c) => c !== null));
1062
+ const spans = [];
1063
+ let open = null;
1064
+ const close = () => {
1065
+ if (open?.polls.length) spans.push(open);
1066
+ open = null;
1067
+ };
1068
+ events.forEach((e, i) => {
1069
+ if (!isToolLike(e) || e.subagent) return;
1070
+ if (backgrounded(e, named)) {
1071
+ if (open?.polls.length) close();
1072
+ open ??= {
1073
+ startMs: e.atMs,
1074
+ endMs: endOf(e),
1075
+ background: i,
1076
+ polls: []
1077
+ };
1078
+ } else if (polls(e)) {
1079
+ open ??= {
1080
+ startMs: e.atMs,
1081
+ endMs: e.atMs,
1082
+ background: null,
1083
+ polls: []
1084
+ };
1085
+ open.polls.push(i);
1086
+ open.endMs = Math.max(open.endMs, endOf(e));
1087
+ } else if (open?.polls.length || works(e)) close();
1088
+ });
1089
+ close();
1090
+ return spans;
1091
+ }
1092
+ const PRIORITY = [
1093
+ "waiting",
1094
+ "tool",
1095
+ "model"
1096
+ ];
1097
+ /** The turn from 0 to `turnMs` as segments, each moment the first of waiting, tool, model and other. */
1098
+ function segments(events, turnMs) {
1099
+ const marks = [];
1100
+ const add = (kind, start, end) => {
1101
+ const s = Math.max(0, Math.min(turnMs, start));
1102
+ const e = Math.max(0, Math.min(turnMs, end));
1103
+ if (e <= s) return;
1104
+ marks.push({
1105
+ at: s,
1106
+ kind,
1107
+ delta: 1
1108
+ }, {
1109
+ at: e,
1110
+ kind,
1111
+ delta: -1
1112
+ });
1113
+ };
1114
+ for (const span of waitSpans(events)) add(0, span.startMs, span.endMs);
1115
+ for (const e of events) if (isToolLike(e)) add(1, e.atMs, endOf(e));
1116
+ else if (e.type === "model-call" && !e.stream) add(2, e.atMs, e.atMs + e.durationMs);
1117
+ marks.sort((a, b) => a.at - b.at);
1118
+ const depth = [
1119
+ 0,
1120
+ 0,
1121
+ 0
1122
+ ];
1123
+ const out = [];
1124
+ let at = 0;
1125
+ const push = (kind, end) => {
1126
+ if (end <= at) return;
1127
+ const last = out.at(-1);
1128
+ if (last?.kind === kind) last.endMs = end;
1129
+ else out.push({
1130
+ kind,
1131
+ startMs: at,
1132
+ endMs: end
1133
+ });
1134
+ at = end;
1135
+ };
1136
+ const current = () => PRIORITY[depth.findIndex((d) => d > 0)] ?? "other";
1137
+ for (const m of marks) {
1138
+ push(current(), m.at);
1139
+ depth[m.kind] = (depth[m.kind] ?? 0) + m.delta;
1140
+ }
1141
+ push(current(), turnMs);
1142
+ return out;
1143
+ }
1144
+ /** A turn's `timeSplit`, from its transcript (durations inferred) and length. */
1145
+ function timeSplit(events, turnMs) {
1146
+ const total = {
1147
+ waiting: 0,
1148
+ tool: 0,
1149
+ model: 0,
1150
+ other: 0
1151
+ };
1152
+ for (const s of segments(events, turnMs)) total[s.kind] += s.endMs - s.startMs;
1153
+ const seen = events.some((e) => e.type === "model-call" && !e.stream);
1154
+ const waitingMs = Math.round(total.waiting);
1155
+ const toolMs = Math.round(total.tool);
1156
+ const modelMs = Math.round(total.model);
1157
+ return {
1158
+ modelMs: seen ? modelMs : null,
1159
+ toolMs,
1160
+ waitingMs,
1161
+ otherMs: Math.max(0, Math.round(turnMs) - waitingMs - toolMs - modelMs)
1162
+ };
1163
+ }
1164
+ /** The turns' splits summed; `modelMs` null when any turn's is, undefined with none. */
1165
+ function sumSplits(splits) {
1166
+ const known = splits.filter((s) => s !== void 0);
1167
+ if (known.length === 0) return void 0;
1168
+ const sum = (f) => known.reduce((n, s) => n + f(s), 0);
1169
+ return {
1170
+ modelMs: known.every((s) => s.modelMs !== null) ? sum((s) => s.modelMs ?? 0) : null,
1171
+ toolMs: sum((s) => s.toolMs),
1172
+ waitingMs: sum((s) => s.waitingMs),
1173
+ otherMs: sum((s) => s.otherMs)
1174
+ };
1175
+ }
881
1176
  // src/attempt/record.ts
882
1177
  function classify(error, signal) {
883
1178
  const message = error instanceof Error ? error.message : String(error);
@@ -909,6 +1204,34 @@ function isJudges(a, error, raw) {
909
1204
  if (a.judgeThrown.has(message)) return true;
910
1205
  return raw.some((r) => r.status !== "pass" && judged(r) && (r.error === error || messageOf(r.error) === message));
911
1206
  }
1207
+ /** The error the attempt's failed judge calls fail it with once its body is done, if any failed. */
1208
+ function judgeErrorsMessage(a) {
1209
+ const messages = a.judgeErrors.toSorted((x, y) => x.call - y.call).map((e) => e.message);
1210
+ const [only, ...more] = messages;
1211
+ if (!only) return void 0;
1212
+ return more.length ? `${more.length + 1} judge calls failed:\n${messages.join("\n")}` : only;
1213
+ }
1214
+ /** A matcher's message without the `label: ` its `expect` puts first. */
1215
+ const unlabeled = (r) => {
1216
+ const message = messageOf(r.error);
1217
+ const prefix = r.label === void 0 ? "" : `${r.label}: `;
1218
+ return message.startsWith(prefix) ? message.slice(prefix.length) : message;
1219
+ };
1220
+ /**
1221
+ * Whether judge calls the provider couldn't answer (a rate limit, a 5xx, overload, the network,
1222
+ * after the retries) are all that failed the attempt: every failed call was transient, and every
1223
+ * check that didn't hold is a `toPass()` on one of them, not polled. A verdict that answered, a
1224
+ * non-transient judge error (a bad key or model, an input over the provider's limit) or any other
1225
+ * check makes it the attempt's own failure. By message, since another language's runner hands back
1226
+ * its own copy of the error.
1227
+ */
1228
+ function judgeOutage(a, error, raw) {
1229
+ if (!a.judgeErrors.length || !a.judgeErrors.every((e) => e.transient)) return false;
1230
+ const message = messageOf(error);
1231
+ if (a.judgeThrown.has(message) && message !== judgeErrorsMessage(a)) return false;
1232
+ const unanswered = new Set(a.judgeErrors.map((e) => e.matcher));
1233
+ return raw.every((r) => r.status === "pass" || judged(r) && !r.matcher.startsWith("poll.") && unanswered.has(unlabeled(r)));
1234
+ }
912
1235
  /**
913
1236
  * The error that fails the attempt: a judge's gives way to the first other check that didn't hold
914
1237
  * (an `expect`, or an `expect.require` the eval caught), whichever came first, so the attempt
@@ -921,7 +1244,8 @@ function attemptError(a, error, raw) {
921
1244
  }
922
1245
  /**
923
1246
  * Why the attempt failed, into the record: the eval's own timeout, a cancel, an infra error, or an
924
- * assertion, which is the judge's when `error`, from `attemptError`, is a judge's.
1247
+ * assertion, which is the judge's when `error`, from `attemptError`, is a judge's. A judge's that
1248
+ * only a provider outage caused is an infra error, so `--retry-errors` and `--resume` run it again.
925
1249
  */
926
1250
  function recordFailure(a, error, timedOut, raw) {
927
1251
  const { record } = a;
@@ -941,9 +1265,17 @@ function recordFailure(a, error, timedOut, raw) {
941
1265
  setBy: setBy ?? []
942
1266
  };
943
1267
  }
944
- if (record.error.kind === "assertion" && isJudges(a, error, raw)) record.error.kind = "judge";
1268
+ if (error instanceof ProviderStopError && !timedOut) record.error.provider = { ...error.stop };
1269
+ if (record.error.kind === "assertion" && isJudges(a, error, raw)) record.error.kind = judgeOutage(a, error, raw) ? "infra" : "judge";
945
1270
  record.outcome = record.error.kind === "infra" ? "infra_error" : "fail";
946
1271
  }
1272
+ /** What the sandbox gave the attempt and what it used, into the record, before the sandbox goes. */
1273
+ async function recordResources(a) {
1274
+ const resources = await a.meter?.stop({ gone: a.signal.aborted }).catch(() => void 0);
1275
+ if (!resources) return;
1276
+ a.record.resources = resources;
1277
+ debugLog("teardown", `resources ${resourcesLine(resources)}`);
1278
+ }
947
1279
  /**
948
1280
  * A process the kernel killed at the sandbox's memory limit makes the attempt an infra error,
949
1281
  * whatever its checks said: the limit, not the agent, decided it. Returns the error to fail it with.
@@ -965,6 +1297,8 @@ async function recordOom(a, sandbox) {
965
1297
  a.record.outcome = "infra_error";
966
1298
  return new InfraError(message);
967
1299
  }
1300
+ /** The gateway tags entries with the phase it was switched to, which is always one of the attempt's. */
1301
+ const phased = (entry) => entry;
968
1302
  /**
969
1303
  * The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
970
1304
  * Refused hosts the harness always tries (in a session like this one) are `expectedBlocked`,
@@ -974,17 +1308,23 @@ function recordEgress(a, sandbox) {
974
1308
  const traffic = sandbox?.traffic?.();
975
1309
  const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0, a.spec.vertex);
976
1310
  const refused = traffic?.blocked ?? [];
977
- const isExpected = (h) => !!expected?.hosts.includes(h.host);
1311
+ const isExpected = (h) => h.phase !== "setup" && !!expected?.hosts.includes(h.host);
978
1312
  const expectedBlocked = refused.filter(isExpected);
1313
+ const { setup } = a.spec.egress;
1314
+ const direct = isDirect(a.spec.egress.policy) || !!setup && isDirect(setup.policy);
979
1315
  a.record.egress = {
980
1316
  policy: a.spec.egress.policy,
981
1317
  allowed: allowedHosts(a.spec.egress),
1318
+ ...setup ? { setup: {
1319
+ policy: setup.policy,
1320
+ allowed: allowedHosts(setup)
1321
+ } } : {},
982
1322
  recorded: !!traffic,
983
- ...traffic && isDirect(a.spec.egress.policy) ? { partial: true } : {},
984
- contacted: traffic?.contacted ?? [],
985
- blocked: refused.filter((h) => !isExpected(h)),
986
- expectedBlocked,
987
- unresolved: traffic?.unresolved ?? []
1323
+ ...traffic && direct ? { partial: true } : {},
1324
+ contacted: (traffic?.contacted ?? []).map(phased),
1325
+ blocked: refused.filter((h) => !isExpected(h)).map(phased),
1326
+ expectedBlocked: expectedBlocked.map(phased),
1327
+ unresolved: (traffic?.unresolved ?? []).map(phased)
988
1328
  };
989
1329
  if (expected && expectedBlocked.length) {
990
1330
  const hosts = expectedBlocked.map((h) => `${h.host}:${h.port} ×${h.connections}`).join(", ");
@@ -999,6 +1339,25 @@ function recordEgress(a, sandbox) {
999
1339
  if (cost !== void 0) a.record.reportedCostUsd = cost;
1000
1340
  }
1001
1341
  /**
1342
+ * Each turn's model calls and inferred tool durations in its transcript, and where its time
1343
+ * went. After teardown, so a stream that outlived its turn (Cursor's) has ended.
1344
+ */
1345
+ function recordTimeline(a, sandbox) {
1346
+ const calls = sandbox?.modelCalls?.() ?? [];
1347
+ for (const turn of a.record.turns) {
1348
+ const events = inferDurations([...turn.transcript, ...modelCallEvents(calls, {
1349
+ startedAt: Date.parse(turn.startedAt),
1350
+ durationMs: turn.durationMs,
1351
+ model: turn.model,
1352
+ steps: a.turnSteps.get(turn) ?? []
1353
+ })].sort((x, y) => x.atMs - y.atMs), turn.durationMs);
1354
+ turn.transcript = events;
1355
+ turn.timeSplit = timeSplit(events, turn.durationMs);
1356
+ }
1357
+ const split = sumSplits(a.record.turns.map((t) => t.timeSplit));
1358
+ if (split) a.record.timeSplit = split;
1359
+ }
1360
+ /**
1002
1361
  * The record as its staging file holds it: redacted, with a secret split across a turn's raw
1003
1362
  * stream parts masked first, since masking each part on its own leaves pieces of it to rebuild.
1004
1363
  */
@@ -1028,7 +1387,7 @@ async function writeRecord(a, stagingDir, id) {
1028
1387
  verdict: j.verdict,
1029
1388
  score: j.score,
1030
1389
  threshold: j.threshold,
1031
- question: clip(j.question, 200)
1390
+ question: clip(redactSecrets(j.question), 200)
1032
1391
  })),
1033
1392
  turnErrors: turnErrors(record.turns.map((t) => t.transcript)),
1034
1393
  ...failedChecks(record),
@@ -1169,18 +1528,32 @@ function unpackPlugins(plugins, unpack, hooksFile = null) {
1169
1528
  // src/sandbox/phases.ts
1170
1529
  const strip = ({ phase: _, ...sub }) => sub;
1171
1530
  /**
1172
- * Which brokered secrets the gateway swaps in when. `to: 'checks'` secrets are active only while
1173
- * at least one check or setup step is running, so a request the agent makes between them gets
1174
- * the placeholder.
1531
+ * Which brokered secrets the gateway swaps in when, and which egress policy it enforces when.
1532
+ * `to: 'checks'` secrets are active only while at least one check or setup step is running, so a
1533
+ * request the agent makes between them gets the placeholder. A setup policy of its own holds from
1534
+ * the sandbox's start until `agent()`; every check runs under the agent's.
1175
1535
  */
1176
- function brokerPhases(box, subs) {
1536
+ function brokerPhases(box, subs, egress) {
1177
1537
  const agent = subs.filter((s) => s.phase === "agent").map(strip);
1178
1538
  const all = subs.map(strip);
1179
1539
  const perPhase = box.setSubstitutions && all.length > agent.length;
1180
1540
  let running = 0;
1541
+ const enter = async (phase, planned) => {
1542
+ if (!box.setEgress) throw new InfraError(`Can't switch the sandbox's egress between the setup and agent phases: its egress gateway isn't running`);
1543
+ debugLog("container", `egress → ${phase}: ${phaseLabel(planned) || "proxy"}`);
1544
+ await box.setEgress({
1545
+ policy: gatewayPolicy(planned),
1546
+ direct: isDirect(planned.policy),
1547
+ phase
1548
+ });
1549
+ };
1181
1550
  return {
1182
1551
  async start() {
1183
1552
  if (all.length > 0) await box.setSubstitutions?.(agent);
1553
+ if (egress?.setup) await enter("setup", egress.setup);
1554
+ },
1555
+ async agent() {
1556
+ if (egress?.setup) await enter("agent", egress);
1184
1557
  },
1185
1558
  async checks(fn) {
1186
1559
  if (!perPhase) return fn();
@@ -1301,7 +1674,7 @@ function createAttempt(spec, signal, progress, start, snapshotDir) {
1301
1674
  },
1302
1675
  report: (update) => progress({
1303
1676
  at: Date.now(),
1304
- ...update
1677
+ ...redactFields(update, ["kind"])
1305
1678
  }),
1306
1679
  async timed(phase, fn) {
1307
1680
  const start = performance.now();
@@ -1321,7 +1694,8 @@ function createAttempt(spec, signal, progress, start, snapshotDir) {
1321
1694
  artifactBlobs: /* @__PURE__ */ new Map(),
1322
1695
  capturePages: /* @__PURE__ */ new Map(),
1323
1696
  stepBlobs: /* @__PURE__ */ new Map(),
1324
- snapshotDir
1697
+ snapshotDir,
1698
+ turnSteps: /* @__PURE__ */ new Map()
1325
1699
  };
1326
1700
  return a;
1327
1701
  }
@@ -1456,7 +1830,7 @@ async function copyWorkspace(box, workspace, signal) {
1456
1830
  if (r.exitCode !== 0) throw new InfraError(`Couldn't unpack the workspace ${workspace.label} into the sandbox (the image needs tar): ${tailOf(r.stderr || r.stdout)}`);
1457
1831
  if (workspace.init) {
1458
1832
  const at = "2000-01-01T00:00:00Z";
1459
- const init = await box.exec(`git init -q -b main && git add -A && git -c commit.gpgsign=false commit -q --no-verify -m ${shellQuote(`${workspace.label} at ${workspace.commit}`)}`, {
1833
+ const init = await box.exec(`git init -q -b main && git add -A && git -c commit.gpgsign=false commit -q --no-verify -m ${shellQuote$1(`${workspace.label} at ${workspace.commit}`)}`, {
1460
1834
  signal,
1461
1835
  env: {
1462
1836
  GIT_AUTHOR_NAME: "reeval",
@@ -1471,13 +1845,87 @@ async function copyWorkspace(box, workspace, signal) {
1471
1845
  }
1472
1846
  debugLog("env", `workspace ${workspace.label} at ${workspace.commit.slice(0, 12)} (${tar.byteLength} bytes of tar${workspace.init ? ", with a .git" : ""}) in ${formatDuration(performance.now() - start)}`);
1473
1847
  }
1848
+ /** Bytes per tar: each is built in memory and piped in whole. */
1849
+ const TAR_BATCH_BYTES = 67108864;
1850
+ /**
1851
+ * Writes `files` (workspace-relative, or `~/` for the home directory) with one tar per
1852
+ * destination: on a container or VM each `writeFile` is a process of its own, which a vendored
1853
+ * directory of thousands of files turns into minutes. A lone file is written as it is, and a
1854
+ * sandbox where the tar doesn't unpack gets a write per file.
1855
+ */
1856
+ async function writeFiles(box, files, signal) {
1857
+ const home = files.flatMap((f) => f.path.startsWith("~/") ? [{
1858
+ ...f,
1859
+ path: f.path.slice(2)
1860
+ }] : []);
1861
+ await unpackFiles(box, "", files.filter((f) => !f.path.startsWith("~/")), signal);
1862
+ await unpackFiles(box, "~/", home, signal);
1863
+ }
1864
+ async function unpackFiles(box, prefix, files, signal) {
1865
+ const one = async () => {
1866
+ for (const f of files) await box.writeFile(`${prefix}${f.path}`, f.content);
1867
+ };
1868
+ if (files.length < 2) return one();
1869
+ const entries = files.map((f) => ({
1870
+ type: "file",
1871
+ path: f.path,
1872
+ mode: 420,
1873
+ mtime: 0,
1874
+ data: typeof f.content === "string" ? Buffer.from(f.content, "utf8") : Buffer.from(f.content.buffer, f.content.byteOffset, f.content.byteLength)
1875
+ }));
1876
+ const batches = [[]];
1877
+ let size = 0;
1878
+ for (const e of entries) {
1879
+ if (size + e.data.length > TAR_BATCH_BYTES && batches[batches.length - 1]?.length) {
1880
+ batches.push([]);
1881
+ size = 0;
1882
+ }
1883
+ batches[batches.length - 1]?.push(e);
1884
+ size += e.data.length;
1885
+ }
1886
+ const dir = shellQuote$1(prefix ? box.home : box.workdir);
1887
+ const name = `.reeval-files-${randomUUID()}.tar`;
1888
+ const unpack = [
1889
+ "unset TAR_OPTIONS",
1890
+ `cd ${dir} || exit 1`,
1891
+ "d=$PWD",
1892
+ `t=$(mktemp -d) || { rm -f ${name}; exit 1; }`,
1893
+ "(",
1894
+ ` tar -xmf "$d/${name}" -C "$t" && cd "$t" && find . -type f > "$t.all" || exit $?`,
1895
+ " : > \"$t.new\"",
1896
+ " while IFS= read -r f; do",
1897
+ " if [ -e \"$d/$f\" ] || [ -L \"$d/$f\" ]; then",
1898
+ " cat \"$f\" > \"$d/$f\" || exit 1",
1899
+ " continue",
1900
+ " fi",
1901
+ ` p=\${f%/*}`,
1902
+ ` while [ "$p" != . ] && [ ! -L "$d/$p" ]; do p=\${p%/*}; done`,
1903
+ " if [ \"$p\" = . ]; then printf '%s\\n' \"$f\" >> \"$t.new\"",
1904
+ ` else mkdir -p "$d/\${f%/*}" && cat "$f" > "$d/$f" || exit 1`,
1905
+ " fi",
1906
+ " done < \"$t.all\"",
1907
+ " [ ! -s \"$t.new\" ] || { tar -cf \"$t.tar\" -T \"$t.new\" && cd \"$d\" && tar -xf \"$t.tar\"; }",
1908
+ ")",
1909
+ "s=$?",
1910
+ `cd "$d"; rm -rf "$t" "$t.all" "$t.new" "$t.tar" ${name}`,
1911
+ "exit $s"
1912
+ ].join("\n");
1913
+ for (const batch of batches) {
1914
+ await box.writeFile(`${prefix}${name}`, writeTar(batch));
1915
+ const r = await box.exec(unpack, { signal });
1916
+ if (r.exitCode !== 0) {
1917
+ debugLog("env", `the files' tar didn't unpack (exit ${r.exitCode}: ${tailOf(r.stderr || r.stdout)}): writing ${files.length} files one at a time`);
1918
+ return one();
1919
+ }
1920
+ }
1921
+ }
1474
1922
  /**
1475
1923
  * A built plugin's directory, copied from where its layer left it in the image, less the parts
1476
1924
  * `only` leaves out, with the manifest as reeval read it.
1477
1925
  */
1478
1926
  async function copyBuiltPlugin(a, box, plugin, dir) {
1479
- const dropped = droppedPaths(plugin.parts).map((p) => shellQuote(`${dir}/${p}`));
1480
- const r = await box.exec(`mkdir -p ${shellQuote(dir)} && cp -a ${shellQuote(plugin.built.dir)}/. ${shellQuote(dir)}/${dropped.length ? ` && rm -rf ${dropped.join(" ")}` : ""}`, { signal: a.signal });
1927
+ const dropped = droppedPaths(plugin.parts).map((p) => shellQuote$1(`${dir}/${p}`));
1928
+ const r = await box.exec(`mkdir -p ${shellQuote$1(dir)} && cp -a ${shellQuote$1(plugin.built.dir)}/. ${shellQuote$1(dir)}/${dropped.length ? ` && rm -rf ${dropped.join(" ")}` : ""}`, { signal: a.signal });
1481
1929
  if (r.exitCode !== 0) throw new InfraError(`Couldn't copy the ${plugin.name} plugin's build from ${plugin.built.tag}: ${tailOf(r.stderr || r.stdout)}`);
1482
1930
  const manifest = plugin.files.find((f) => f.path === PLUGIN_MANIFEST);
1483
1931
  if (manifest) await box.writeFile(`${dir}/${PLUGIN_MANIFEST}`, manifest.content);
@@ -1500,7 +1948,7 @@ async function copyPlugins(a, box, env) {
1500
1948
  else for (const f of plugin.files) await box.writeFile(`${at}/${f.path}`, f.content);
1501
1949
  const executable = built ? [] : plugin.files.flatMap((f) => f.executable ? [`${dir}/${f.path}`] : []);
1502
1950
  if (executable.length) {
1503
- const r = await box.exec(`chmod +x ${executable.map(shellQuote).join(" ")}`, { signal: a.signal });
1951
+ const r = await box.exec(`chmod +x ${executable.map(shellQuote$1).join(" ")}`, { signal: a.signal });
1504
1952
  if (r.exitCode !== 0) throw new InfraError(`Couldn't make the ${plugin.name} plugin's ${executable.length} executable files executable: ${tailOf(r.stderr || r.stdout)}`);
1505
1953
  }
1506
1954
  const root = (v) => v.replaceAll(PLUGIN_ROOT, dir);
@@ -1533,7 +1981,7 @@ async function copyPlugins(a, box, env) {
1533
1981
  * only, so the sandbox has `find`.
1534
1982
  */
1535
1983
  async function unpackedFromCopy(a, box, dir) {
1536
- const r = await box.exec(`cd ${shellQuote(dir)} && find rules agents commands hooks -maxdepth 1 -type f 2>/dev/null; true`, { signal: a.signal });
1984
+ const r = await box.exec(`cd ${shellQuote$1(dir)} && find rules agents commands hooks -maxdepth 1 -type f 2>/dev/null; true`, { signal: a.signal });
1537
1985
  const out = [];
1538
1986
  for (const path of r.stdout.split("\n").filter(Boolean)) {
1539
1987
  const content = await box.readFile(`${dir}/${path}`);
@@ -1562,7 +2010,7 @@ async function writeUnpackedPlugins(a, box, env, plugins, unpack) {
1562
2010
  const files = unpackPlugins(placed, unpack, unpack.hooks ? await box.readFile(unpack.hooks.file) : null);
1563
2011
  const fresh = files.filter((f) => f.path !== unpack.hooks?.file);
1564
2012
  if (fresh.length) {
1565
- const taken = (await box.exec(`for p in ${fresh.map((f) => shellQuote(f.path)).join(" ")}; do { [ -e "$p" ] || [ -L "$p" ]; } && echo "$p"; done; true`, { signal: a.signal })).stdout.trim().split("\n").filter(Boolean);
2013
+ const taken = (await box.exec(`for p in ${fresh.map((f) => shellQuote$1(f.path)).join(" ")}; do { [ -e "$p" ] || [ -L "$p" ]; } && echo "$p"; done; true`, { signal: a.signal })).stdout.trim().split("\n").filter(Boolean);
1566
2014
  if (taken.length) throw new InfraError(`The workspace already has ${taken.join(", ")}, which a plugin would replace: rename the plugin's file, or leave the plugin out of this cell`);
1567
2015
  }
1568
2016
  for (const f of files) {
@@ -1596,40 +2044,71 @@ async function writePluginMcp(box, env, plugins, servers) {
1596
2044
  debugLog("env", `plugin ${placed.name}'s MCP servers → ${at}/.mcp.json, through the timer`);
1597
2045
  }
1598
2046
  }
2047
+ /** How long a start that failed on a transient error waits before it's tried again. */
2048
+ const TRANSIENT_START_DELAY_MS = 2e3;
1599
2049
  /**
1600
- * Starts the harness session within its `startTimeout`. A start that doesn't become ready in time
1601
- * is aborted (and closed, should it finish later) and started once more: a stuck bootstrap is
1602
- * usually transient, and nothing has run yet, so the retry can't change what the attempt
1603
- * measures. A second timeout is an infra error; a start that fails outright isn't retried.
2050
+ * Starts a harness session within `timeoutMs`, at most twice. A start that doesn't become ready in
2051
+ * time is aborted (and closed, should it finish later) and started once more, and so is one that
2052
+ * fails on a transient {@link InfraError} (a 5xx or a network error): nothing has run yet, so the
2053
+ * retry can't change what the attempt measures. Each retry is passed to `onRetry` to record. A
2054
+ * second timeout or transient failure, or any other failure, is the attempt's infra error.
1604
2055
  */
1605
- async function startHarness(a, harness, box, options) {
1606
- const { harness: descriptor } = a.spec;
1607
- const ms = startTimeoutMs(descriptor);
1608
- const late = `${descriptor.kind}: session did not become ready within ${formatDuration(ms)}`;
2056
+ async function startWithRetry(start, { kind, timeoutMs, signal, onRetry, transientDelayMs = TRANSIENT_START_DELAY_MS }) {
2057
+ const late = `${kind}: session did not become ready within ${formatDuration(timeoutMs)}`;
1609
2058
  for (let tries = 1;; tries++) {
1610
2059
  const stop = new AbortController();
1611
2060
  let timer;
1612
2061
  const timedOut = new Promise((resolve) => {
1613
- timer = setTimeout(() => resolve(null), ms);
2062
+ timer = setTimeout(() => resolve(null), timeoutMs);
1614
2063
  });
1615
- const starting = harness.start(box, AbortSignal.any([a.signal, stop.signal]), options);
1616
- const session = await Promise.race([starting, timedOut]).finally(() => clearTimeout(timer));
2064
+ const startedAt = Date.now();
2065
+ const starting = start(AbortSignal.any([signal, stop.signal]));
2066
+ let session;
2067
+ try {
2068
+ session = await Promise.race([starting, timedOut]);
2069
+ } catch (error) {
2070
+ signal.throwIfAborted();
2071
+ if (!(error instanceof InfraError) || !error.transient) throw error;
2072
+ if (tries === 2) throw new InfraError(`${error.message} (tried twice)`);
2073
+ onRetry({
2074
+ error: error.message,
2075
+ waitedMs: Date.now() - startedAt
2076
+ });
2077
+ debugLog("harness", `${error.message}; starting it once more`);
2078
+ await new Promise((resolve) => setTimeout(resolve, transientDelayMs));
2079
+ signal.throwIfAborted();
2080
+ continue;
2081
+ } finally {
2082
+ clearTimeout(timer);
2083
+ }
1617
2084
  if (session) return session;
1618
2085
  stop.abort(new InfraError(late));
1619
2086
  starting.then((s) => s.close().catch(() => {}), () => {});
1620
- a.signal.throwIfAborted();
2087
+ signal.throwIfAborted();
1621
2088
  if (tries === 2) throw new InfraError(`${late} (tried twice)`);
1622
- a.record.startRetries = [{
2089
+ onRetry({
1623
2090
  error: late,
1624
- waitedMs: ms
1625
- }];
2091
+ waitedMs: timeoutMs
2092
+ });
1626
2093
  debugLog("harness", `${late}; starting it once more`);
1627
2094
  }
1628
2095
  }
2096
+ function startHarness(a, harness, box, options) {
2097
+ const { harness: descriptor } = a.spec;
2098
+ return startWithRetry((signal) => harness.start(box, signal, options), {
2099
+ kind: descriptor.kind,
2100
+ timeoutMs: startTimeoutMs(descriptor),
2101
+ signal: a.signal,
2102
+ onRetry: (retry) => {
2103
+ a.record.startRetries = [...a.record.startRetries ?? [], retry];
2104
+ }
2105
+ });
2106
+ }
1629
2107
  /**
1630
2108
  * Starts the sandbox (with the brokered secrets its gateway swaps in), writes the
1631
2109
  * environment's files, workspace skills and plugins, runs its setup steps, snapshots the workspace for
1632
- * the diff, and starts the harness session with the harness-placed skills and MCP servers.
2110
+ * the diff, switches to the agent's egress, and starts the harness session with the
2111
+ * harness-placed skills and MCP servers.
1633
2112
  */
1634
2113
  async function startSession(a, runId) {
1635
2114
  const startedAt = performance.now();
@@ -1641,7 +2120,10 @@ async function startSession(a, runId) {
1641
2120
  signal: a.signal,
1642
2121
  env: env.vars,
1643
2122
  image: a.image,
2123
+ ...a.hosts && { hosts: a.hosts },
2124
+ harness: spec.harness,
1644
2125
  brokering: brokeringFor(spec.harness, spec.vertex),
2126
+ ...agentOf(spec.harness, spec.vertex)?.keychainTrust && { keychainTrust: true },
1645
2127
  expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0, spec.vertex)?.hosts,
1646
2128
  substitutions: env.substitutions,
1647
2129
  egress: spec.egress,
@@ -1651,8 +2133,9 @@ async function startSession(a, runId) {
1651
2133
  })
1652
2134
  });
1653
2135
  a.sandbox = box;
2136
+ a.meter = box.meter?.();
1654
2137
  if (!capabilitiesOf(spec.sandbox).isolated) debugLog("container", `${sandboxKind(spec.sandbox).provider(spec.sandbox)} sandbox at ${box.workdir}`);
1655
- a.phases = brokerPhases(box, env.substitutions);
2138
+ a.phases = brokerPhases(box, env.substitutions, spec.egress);
1656
2139
  if (a.signal.aborted) box.destroy();
1657
2140
  a.signal.throwIfAborted();
1658
2141
  await a.phases.start();
@@ -1661,23 +2144,29 @@ async function startSession(a, runId) {
1661
2144
  debugLog("broker", `gateway swaps in ${agent.join(", ") || "nothing"} for the agent`);
1662
2145
  }
1663
2146
  if (env.workspace) await copyWorkspace(box, env.workspace, a.signal);
2147
+ const skillDir = workspaceSkillDir(spec.harness);
2148
+ const skillFileList = env.workspaceSkills.map((skill) => ({
2149
+ skill,
2150
+ files: skillFiles(skill).map((f) => ({
2151
+ path: `${skillDir}/${skill.name}/${f.path}`,
2152
+ content: f.content
2153
+ }))
2154
+ }));
2155
+ const toWrite = [...env.files, ...skillFileList.flatMap((s) => s.files)];
2156
+ const filesStart = performance.now();
2157
+ await writeFiles(box, toWrite, a.signal);
2158
+ if (toWrite.length) debugLog("env", `${toWrite.length} files written in ${formatDuration(performance.now() - filesStart)}`);
1664
2159
  for (const f of env.files) {
1665
- await box.writeFile(f.path, f.content);
1666
2160
  const size = typeof f.content === "string" ? Buffer.byteLength(f.content) : f.content.byteLength;
1667
2161
  debugLog("env", `file ${f.path} (${size} bytes${f.executable ? ", executable" : ""})`);
1668
2162
  }
1669
2163
  const executable = env.files.flatMap((f) => f.executable ? [f.path.startsWith("~/") ? `${box.home}/${f.path.slice(2)}` : f.path] : []);
1670
2164
  if (executable.length) {
1671
- const paths = executable.map((p) => shellQuote(p.startsWith("-") ? `./${p}` : p));
2165
+ const paths = executable.map((p) => shellQuote$1(p.startsWith("-") ? `./${p}` : p));
1672
2166
  const r = await box.exec(`chmod +x ${paths.join(" ")}`, { signal: a.signal });
1673
2167
  if (r.exitCode !== 0) throw new InfraError(`Couldn't make ${executable.join(", ")} executable: ${tailOf(r.stderr || r.stdout)}`);
1674
2168
  }
1675
- const skillDir = workspaceSkillDir(spec.harness);
1676
- for (const skill of env.workspaceSkills) {
1677
- const files = skillFiles(skill);
1678
- for (const f of files) await box.writeFile(`${skillDir}/${skill.name}/${f.path}`, f.content);
1679
- debugLog("env", `skill ${skill.name} → ${skillDir}/${skill.name}/ (${files.length} files, workspace)`);
1680
- }
2169
+ for (const { skill, files } of skillFileList) debugLog("env", `skill ${skill.name} → ${skillDir}/${skill.name}/ (${files.length} files, workspace)`);
1681
2170
  const plugins = await copyPlugins(a, box, env);
1682
2171
  const layout = layoutOf(spec.harness);
1683
2172
  if (layout.pluginUnpack && plugins.length) await writeUnpackedPlugins(a, box, env, plugins, layout.pluginUnpack);
@@ -1691,6 +2180,7 @@ async function startSession(a, runId) {
1691
2180
  }
1692
2181
  await snapshot(box, a.signal);
1693
2182
  debugLog("env", "workspace snapshot taken: the diff starts here");
2183
+ await a.phases.agent();
1694
2184
  const timed = await timeMcpServers(box, env.mcp, a.signal);
1695
2185
  if (timed.log) {
1696
2186
  a.mcpStartup = timed.log;
@@ -1803,8 +2293,8 @@ function cleanArchive(tar) {
1803
2293
  };
1804
2294
  }
1805
2295
  const tarCommand = (workdir) => {
1806
- const flags = EXCLUDED_DIRS.map((d) => `--exclude=${shellQuote(d)}`).join(" ");
1807
- return `cd ${shellQuote(workdir)} && COPYFILE_DISABLE=1 tar -cf - ${flags} . 2>/dev/null`;
2296
+ const flags = EXCLUDED_DIRS.map((d) => `--exclude=${shellQuote$1(d)}`).join(" ");
2297
+ return `cd ${shellQuote$1(workdir)} && COPYFILE_DISABLE=1 tar -cf - ${flags} . 2>/dev/null`;
1808
2298
  };
1809
2299
  const GZIP = "if command -v gzip >/dev/null 2>&1; then gzip -c; else cat; fi";
1810
2300
  const NEEDS_TAR = "command -v tar >/dev/null 2>&1 || { echo 'the sandbox has no tar' >&2; exit 127; }";
@@ -1833,9 +2323,9 @@ async function archiveWorkspace(box, signal, compress, limits = LIMITS) {
1833
2323
  },
1834
2324
  entries: null
1835
2325
  });
1836
- const names = EXCLUDED_DIRS.map((d) => `-name ${shellQuote(d)}`).join(" -o ");
2326
+ const names = EXCLUDED_DIRS.map((d) => `-name ${shellQuote$1(d)}`).join(" -o ");
1837
2327
  const tar = tarCommand(box.workdir);
1838
- const sized = await box.execOwn(`${NEEDS_TAR}; cd ${shellQuote(box.workdir)} && { find . -type d \\( ${names} \\) -prune -print 2>/dev/null | head -n ${LISTED}; echo ${SIZE_MARK}; ${tar} | wc -c; echo ${SIZE_MARK}; ${tar} | { ${GZIP}; } | wc -c; }`, { signal });
2328
+ const sized = await box.execOwn(`${NEEDS_TAR}; cd ${shellQuote$1(box.workdir)} && { find . -type d \\( ${names} \\) -prune -print 2>/dev/null | head -n ${LISTED}; echo ${SIZE_MARK}; ${tar} | wc -c; echo ${SIZE_MARK}; ${tar} | { ${GZIP}; } | wc -c; }`, { signal });
1839
2329
  const [list = "", raw = "", size = ""] = sized.stdout.split(`${SIZE_MARK}\n`);
1840
2330
  const dirs = list.split("\n").filter(Boolean).map((p) => redactSecrets(`${p.replace(/^\.\//, "")}/`));
1841
2331
  const rawBytes = Number(raw.trim());
@@ -1950,10 +2440,10 @@ async function takeSnapshot(a, box, name, phase, limits = LIMITS, timeoutMs = SN
1950
2440
  /** How long a turn gets to stop once its attempt is aborted, before reeval stops waiting for it. */
1951
2441
  const TURN_STOP_GRACE_MS = 3e3;
1952
2442
  /**
1953
- * Resolves `grace` ms after `signal` aborts, with the turn as cut off; `cancel` drops the timer
1954
- * once the turn settles on its own.
2443
+ * Resolves `grace` ms after `signal` aborts, with the turn as cut off: what the session says it
2444
+ * got to, or an empty turn; `cancel` drops the timer once the turn settles on its own.
1955
2445
  */
1956
- function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
2446
+ function cutOff(signal, startedAt, session, grace = TURN_STOP_GRACE_MS) {
1957
2447
  let timer;
1958
2448
  let onAbort = () => {};
1959
2449
  const cutoff = new Promise((resolve) => {
@@ -1961,22 +2451,25 @@ function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
1961
2451
  timer = setTimeout(() => {
1962
2452
  const message = `The agent didn't stop within ${grace / 1e3}s of being canceled, so reeval stopped waiting for its turn`;
1963
2453
  const at = Date.now() - Date.parse(startedAt);
1964
- resolve({
2454
+ const note = {
2455
+ type: "error",
2456
+ message,
2457
+ ts: (/* @__PURE__ */ new Date()).toISOString(),
2458
+ atMs: at
2459
+ };
2460
+ const empty = {
1965
2461
  text: "",
1966
2462
  status: "aborted",
1967
- transcript: [{
1968
- type: "error",
1969
- message,
1970
- ts: (/* @__PURE__ */ new Date()).toISOString(),
1971
- atMs: at
1972
- }],
2463
+ transcript: [note],
1973
2464
  usage: null,
1974
2465
  model: null,
1975
2466
  unreported: [],
1976
2467
  reasoningMs: null,
1977
2468
  startedAt,
1978
2469
  durationMs: at
1979
- });
2470
+ };
2471
+ if (!session.stopped) return resolve(empty);
2472
+ session.stopped(note).then(resolve, () => resolve(empty));
1980
2473
  }, grace);
1981
2474
  };
1982
2475
  if (signal.aborted) onAbort();
@@ -2064,7 +2557,7 @@ function turnSender(a, takeContext, checks) {
2064
2557
  }, `turn ${turnNumber} prompt`);
2065
2558
  const watch = a.mcpStartup && watchMcpStartup(a.mcpStartup);
2066
2559
  const signal = watch ? AbortSignal.any([a.signal, watch.signal]) : a.signal;
2067
- const { cutoff, cancel } = cutOff(signal, entry.startedAt);
2560
+ const { cutoff, cancel } = cutOff(signal, entry.startedAt, agent);
2068
2561
  const sendStarted = Date.now();
2069
2562
  const sending = agent.send(prompt, signal).catch((error) => {
2070
2563
  if (!watch?.signal.aborted || a.signal.aborted) throw error;
@@ -2081,7 +2574,7 @@ function turnSender(a, takeContext, checks) {
2081
2574
  };
2082
2575
  });
2083
2576
  const timedBefore = a.record.timings.agentMs;
2084
- const { raw, rejectedKey, reasoningMs, agentMs, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
2577
+ const { raw, rejectedKey, providerStop, reasoningMs, agentMs, steps, cost, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
2085
2578
  entry.reasoningMs = null;
2086
2579
  if (watch) entry.mcpStartup = mcpStartupOf((await watch.end()).entries);
2087
2580
  throw error;
@@ -2089,6 +2582,7 @@ function turnSender(a, takeContext, checks) {
2089
2582
  cancel();
2090
2583
  watch?.close();
2091
2584
  });
2585
+ if (steps) a.turnSteps.set(entry, steps);
2092
2586
  const sendWallMs = agentMs ?? Date.now() - sendStarted;
2093
2587
  if (agentMs !== void 0) a.record.timings.agentMs = timedBefore + agentMs;
2094
2588
  const startup = await watch?.end();
@@ -2104,7 +2598,7 @@ function turnSender(a, takeContext, checks) {
2104
2598
  atMs: Math.max(0, seenAt - Date.parse(sent.startedAt))
2105
2599
  }].sort((x, y) => x.atMs - y.atMs)
2106
2600
  } : sent;
2107
- Object.assign(entry, result, raw ? { raw } : {}, reasoningMs !== void 0 ? { reasoningMs } : {}, {
2601
+ Object.assign(entry, result, raw ? { raw } : {}, reasoningMs !== void 0 ? { reasoningMs } : {}, cost ? { cost } : {}, {
2108
2602
  durationMs: Math.max(result.durationMs, sendWallMs),
2109
2603
  ...startup ? { mcpStartup: mcpStartupOf(startup.entries) } : {}
2110
2604
  });
@@ -2115,7 +2609,8 @@ function turnSender(a, takeContext, checks) {
2115
2609
  await takeContext(box, a.signal);
2116
2610
  const count = (...types) => result.transcript.filter((e) => types.includes(e.type)).length;
2117
2611
  const failure = result.transcript.find((e) => e.type === "error");
2118
- const infra = timedOut || result.status !== "aborted" && rejectedKey;
2612
+ const stopped = result.status === "aborted" ? void 0 : providerStop;
2613
+ const infra = timedOut || result.status !== "aborted" && rejectedKey || stopped?.message;
2119
2614
  let changes;
2120
2615
  let diffError;
2121
2616
  if (!infra && !a.signal.aborted) try {
@@ -2138,7 +2633,7 @@ function turnSender(a, takeContext, checks) {
2138
2633
  debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${filesChanged === void 0 ? "" : `${plural(filesChanged, "file")} changed · `}${plural(count("file-edit"), "edit tool call")}${failure && "message" in failure ? ` · error: ${clip(redactSecrets(String(failure.message)), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
2139
2634
  for (const s of entry.mcpStartup ?? []) debugLog("harness", `turn ${turnNumber} mcp ${s.server} ${s.ms === null ? "didn't answer initialize" : `answered initialize in ${formatDuration(s.ms)}`}`);
2140
2635
  if (infra) {
2141
- const error = new InfraError(infra);
2636
+ const error = !timedOut && stopped ? new ProviderStopError(stopped) : new InfraError(infra);
2142
2637
  a.startFailure ??= { error };
2143
2638
  throw error;
2144
2639
  }
@@ -2190,7 +2685,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
2190
2685
  clock?.resume();
2191
2686
  }
2192
2687
  }, snapshotSidecar(stagingDir, attemptKey(id, spec.cell.key, spec.n)));
2193
- const destroy = () => void a.sandbox?.destroy();
2688
+ const destroy = () => void Promise.resolve(a.session?.salvage?.()).catch(() => {}).finally(() => a.sandbox?.destroy());
2194
2689
  testSignal.addEventListener("abort", destroy, { once: true });
2195
2690
  const takeContext = contextTaker(a);
2196
2691
  const { exec, readFile, writeFile } = createChecks(a);
@@ -2234,12 +2729,10 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
2234
2729
  const missed = await unawaited() ?? checkFailure(assertions()) ?? expectError();
2235
2730
  if (a.startFailure) throw a.startFailure.error;
2236
2731
  if (missed) throw missed;
2237
- const judgeErrors = a.judgeErrors.toSorted((x, y) => x.call - y.call).map((e) => e.message);
2238
- const [only, ...more] = judgeErrors;
2239
- if (only) {
2240
- const message = more.length ? `${more.length + 1} judge calls failed:\n${judgeErrors.join("\n")}` : only;
2241
- a.judgeThrown.add(message);
2242
- throw new Error(message);
2732
+ const judgeErrors = judgeErrorsMessage(a);
2733
+ if (judgeErrors) {
2734
+ a.judgeThrown.add(judgeErrors);
2735
+ throw new Error(judgeErrors);
2243
2736
  }
2244
2737
  } catch (thrown) {
2245
2738
  const error = attemptError(a, thrown, assertions());
@@ -2250,6 +2743,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
2250
2743
  clock?.clear();
2251
2744
  await a.endStarts();
2252
2745
  for (const bg of a.backgrounds) await bg.kill().catch(() => {});
2746
+ await recordResources(a);
2253
2747
  const { sandbox } = a;
2254
2748
  if (sandbox && !a.signal.aborted) await takeContext(sandbox, AbortSignal.timeout(3e4));
2255
2749
  if (sandbox) a.report({ kind: "cleanup" });
@@ -2261,6 +2755,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
2261
2755
  oom = a.signal.aborted ? void 0 : await recordOom(a, sandbox);
2262
2756
  if (sandbox && spec.snapshot.when && !a.signal.aborted) await takeSnapshot(a, sandbox, "final", "teardown");
2263
2757
  await sandbox?.destroy();
2758
+ recordTimeline(a, sandbox);
2264
2759
  if (sandbox) debug.log("teardown", `sandbox destroyed (${formatDuration(performance.now() - teardown)} for teardown)`);
2265
2760
  const { record } = a;
2266
2761
  const checked = assertionRecords(assertions());
@@ -2303,8 +2798,11 @@ function bestOfCell(bestOf) {
2303
2798
  settled: (settled) => settle[n - 1]?.(sideOf(settled))
2304
2799
  });
2305
2800
  }
2306
- /** Characters a case key can't hold: ids and paths split on them, and control characters. */
2307
- const BAD_KEY = /[#/›\u0000-\u001f\u007f]/;
2801
+ /**
2802
+ * Characters a case key can't hold: attempt ids split on `#` and eval ids on `›`, and control
2803
+ * characters. A `/` is allowed: a key is its own path segment wherever it's in one.
2804
+ */
2805
+ const BAD_KEY = /[#›\u0000-\u001f\u007f]/;
2308
2806
  /** The name a case's eval plans under: `adds a health check [nextjs]`. */
2309
2807
  const caseEvalName = (name, key) => `${name} [${key}]`;
2310
2808
  /** The key a case gets without `caseKey`: a string, a finite number, or an object's `id` or `name`. */
@@ -2318,7 +2816,7 @@ function defaultKey(item) {
2318
2816
  }
2319
2817
  /**
2320
2818
  * Each case's key, in list order: `caseKey(item)` when given, else its default. Trimmed, 1–80
2321
- * characters, with no `#`, `/`, `›` or control characters, and unique within the eval.
2819
+ * characters, with no `#`, `›` or control characters, and unique within the eval.
2322
2820
  */
2323
2821
  function caseKeys(cases, caseKey, where) {
2324
2822
  const seen = /* @__PURE__ */ new Map();
@@ -2327,7 +2825,7 @@ function caseKeys(cases, caseKey, where) {
2327
2825
  if (raw === void 0) throw new UsageError(`cases[${i}] in ${where} has no key: a case is its own key when it's a string or a number, or an object's id or name. Give it an id, or pass caseKey: (item) => item.slug.`);
2328
2826
  if (typeof raw !== "string") throw new UsageError(`caseKey in ${where} must return a string (got ${redactSecrets(JSON.stringify(raw) ?? String(raw))} for cases[${i}])`);
2329
2827
  const key = raw.trim();
2330
- if (!key || key.length > 80 || BAD_KEY.test(key)) throw new UsageError(`cases[${i}] in ${where} has the key ${redactSecrets(JSON.stringify(raw))}: a case key is 1–80 characters, with no #, /, › or control characters.${caseKey ? "" : " Pass caseKey to give it another."}`);
2828
+ if (!key || key.length > 80 || BAD_KEY.test(key)) throw new UsageError(`cases[${i}] in ${where} has the key ${redactSecrets(JSON.stringify(raw))}: a case key is 1–80 characters, with no #, › or control characters.${caseKey ? "" : " Pass caseKey to give it another."}`);
2331
2829
  const earlier = seen.get(key);
2332
2830
  if (earlier !== void 0) throw new UsageError(`cases[${earlier}] and cases[${i}] in ${where} have the same key "${redactSecrets(key)}". Each case needs its own key${caseKey ? "" : ": give them distinct ids or names, or pass caseKey"}.`);
2333
2831
  seen.set(key, i);
@@ -2604,12 +3102,13 @@ function planTests(declaration, ctx, where, resolvedEval, pick) {
2604
3102
  const hid = harnessId(harness);
2605
3103
  if (gatewaySet || harness.gateway !== void 0) {
2606
3104
  routes.set(hid, route);
2607
- gatewayNotes.push(`gateway: ${hid} → ${route.name}${route.name === "direct" && route.skipped.length ? " (fallback)" : ""}`);
3105
+ const at = route.name === "vercel" && !vercelBaseUrlProblem() ? vercelBaseUrl() : void 0;
3106
+ gatewayNotes.push(`gateway: ${hid} → ${route.name}${at ? ` at ${at}` : ""}${route.name === "direct" && route.skipped.length ? " (fallback)" : ""}`);
2608
3107
  }
2609
3108
  if (route.name === "direct") return harness;
2610
3109
  if (harness.kind !== "fake") try {
2611
3110
  if (route.name === "vercel") {
2612
- const problem = vercelCredentialProblem();
3111
+ const problem = vercelCredentialProblem() ?? vercelBaseUrlProblem();
2613
3112
  if (problem) throw new UsageError(problem);
2614
3113
  }
2615
3114
  if (route.name === "vertex") {
@@ -2653,6 +3152,8 @@ function planTests(declaration, ctx, where, resolvedEval, pick) {
2653
3152
  if (first) throw new UsageError(unbrokeredError(first, where, ctx.config.file));
2654
3153
  const [plain] = plainHeaders(plannedAttempts);
2655
3154
  if (plain) throw new UsageError(plainHeaderError(plain, where, ctx.config.file));
3155
+ const [plainVar] = plainEnv(resolved.env, plannedAttempts);
3156
+ if (plainVar) throw new UsageError(plainEnvError(plainVar, where, ctx.config.file));
2656
3157
  }
2657
3158
  const pluginNotes = [];
2658
3159
  const pluginLabels = /* @__PURE__ */ new Map();
@@ -2660,10 +3161,11 @@ function planTests(declaration, ctx, where, resolvedEval, pick) {
2660
3161
  const why = unenforceable(planned.egress, planned.sandbox, where);
2661
3162
  if (why) throw new UsageError(why);
2662
3163
  const on = `${where} on ${harnessId(planned.harness)}`;
2663
- const unverified = unverifiedHarness(planned.harness, planned.sandbox);
2664
- if (unverified) throw new UsageError(`${on}: ${unverified}`);
2665
3164
  const mcp = refusedMcp(planned.harness, planned.environment.layers);
2666
3165
  if (mcp) throw new UsageError(`${on}: ${mcp}`);
3166
+ const slots = sandboxKind(planned.sandbox).slotsPerMachine?.(planned.sandbox) ?? 1;
3167
+ const shared = refusedSharedMachine(planned.harness, slots);
3168
+ if (shared) throw new UsageError(`${on}: ${shared}`);
2667
3169
  const plugins = planPlugins(planned, ctx.config.root);
2668
3170
  if (plugins.refusal) throw new UsageError(`${on}: ${plugins.refusal}`);
2669
3171
  for (const note of plugins.notes) if (!pluginNotes.includes(note)) pluginNotes.push(note);
@@ -3084,10 +3586,45 @@ function prepare(mode, given, own, fn, source, hint = "", givenFamily, where) {
3084
3586
  bestOfCells.set(cell.key, of);
3085
3587
  hooks = of(n);
3086
3588
  }
3087
- registerTest(testName, {
3589
+ const test = {
3088
3590
  ...options,
3089
3591
  ...hooks
3090
- }, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
3592
+ };
3593
+ if (spec && getRunContext().inputs) {
3594
+ const { root } = getRunContext().config;
3595
+ const found = environmentInputs(spec.environment.layers, spec.sandbox, root);
3596
+ const { attempt } = options;
3597
+ const warnings = [.../* @__PURE__ */ new Set([...attempt.plan.warnings, ...found.warnings])];
3598
+ test.attempt = {
3599
+ ...attempt,
3600
+ plan: {
3601
+ ...attempt.plan,
3602
+ warnings
3603
+ },
3604
+ inputs: found.inputs
3605
+ };
3606
+ }
3607
+ const readCap = spec && sandboxKind(spec.sandbox).attemptCap?.(spec.sandbox);
3608
+ if (spec && test.gate && readCap) {
3609
+ const gate = {
3610
+ ...test.gate,
3611
+ limits: { ...test.gate.limits }
3612
+ };
3613
+ const before = test.ready;
3614
+ test.gate = gate;
3615
+ test.ready = async () => {
3616
+ const { config, runId } = getRunContext();
3617
+ const cap = await readCap({
3618
+ harness: spec.harness,
3619
+ root: config.root,
3620
+ runId
3621
+ });
3622
+ const key = `sandbox:${sandboxProvider(spec.sandbox)}`;
3623
+ if (cap && gate.limits[key]) gate.limits[key] = Math.min(gate.limits[key], cap);
3624
+ return before ? before() : true;
3625
+ };
3626
+ }
3627
+ registerTest(testName, test, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
3091
3628
  }
3092
3629
  };
3093
3630
  return {
@@ -3241,4 +3778,4 @@ function optionsFrom(written, where, base) {
3241
3778
  }
3242
3779
  export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
3243
3780
 
3244
- //# sourceMappingURL=evaluate-B12Tosvq.js.map
3781
+ //# sourceMappingURL=evaluate-BHQyAszQ.js.map