reeval 0.20.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (481) hide show
  1. package/CHANGELOG.md +841 -0
  2. package/README.md +15 -15
  3. package/assets/gateway-models.mjs +65 -31
  4. package/dist/{adapter-C60z3wLq.js → adapter-Di9H2etg.js} +8 -49
  5. package/dist/adapter-Di9H2etg.js.map +1 -0
  6. package/dist/{agent-BmsMXZXx.js → agent-B-6GFDqA.js} +185 -38
  7. package/dist/agent-B-6GFDqA.js.map +1 -0
  8. package/dist/{agents-DXuFxtGC.js → agents-KBTYhSSh.js} +385 -116
  9. package/dist/agents-KBTYhSSh.js.map +1 -0
  10. package/dist/{app-files-DT5mJyV4.js → app-files-Cmh9FGr5.js} +3 -3
  11. package/dist/app-files-Cmh9FGr5.js.map +1 -0
  12. package/dist/{assemble-CE4pHKIa.js → assemble-CBMRIhso.js} +1 -1
  13. package/dist/{assemble-DM2PNU4R.js → assemble-DNVBTBp_.js} +172 -18
  14. package/dist/assemble-DNVBTBp_.js.map +1 -0
  15. package/dist/attach-EbDGKGTO.js +152 -0
  16. package/dist/attach-EbDGKGTO.js.map +1 -0
  17. package/dist/{attempt-CAdlwlNu.js → attempt-_dUsblVx.js} +2 -2
  18. package/dist/{attempt-CAdlwlNu.js.map → attempt-_dUsblVx.js.map} +1 -1
  19. package/dist/{attempt-host-7VviAJrl.js → attempt-host-DQvynDHk.js} +17 -8
  20. package/dist/attempt-host-DQvynDHk.js.map +1 -0
  21. package/dist/boat-Bzz8L-KQ.js +701 -0
  22. package/dist/boat-Bzz8L-KQ.js.map +1 -0
  23. package/dist/{boat-BjLqFJIZ.js → boat-Dek8ftIt.js} +20 -8
  24. package/dist/boat-Dek8ftIt.js.map +1 -0
  25. package/dist/bridge/index.mjs +6 -1
  26. package/dist/bundle-DZstHDeW.js +718 -0
  27. package/dist/bundle-DZstHDeW.js.map +1 -0
  28. package/dist/{capture-Ccf0Hmyx.js → capture-BkbPsfej.js} +61 -32
  29. package/dist/capture-BkbPsfej.js.map +1 -0
  30. package/dist/capture-CUAvk9KM.js +1 -0
  31. package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
  32. package/dist/cases-CM-MQRQp.js.map +1 -0
  33. package/dist/cases-FvhbM8M5.js +1 -0
  34. package/dist/{cli-DTuLSlmW.js → cli-BgBBuxxJ.js} +412 -1044
  35. package/dist/cli-BgBBuxxJ.js.map +1 -0
  36. package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-B5L9n-DV.js} +199 -35
  37. package/dist/cli-spec-B5L9n-DV.js.map +1 -0
  38. package/dist/cli.js +2 -2
  39. package/dist/client-DMEYR3MA.js +807 -0
  40. package/dist/client-DMEYR3MA.js.map +1 -0
  41. package/dist/{command-C2C7P5qo.js → command-BKb8eN_X.js} +10 -10
  42. package/dist/{command-C2C7P5qo.js.map → command-BKb8eN_X.js.map} +1 -1
  43. package/dist/{command-DSL7I-SI.js → command-D_w_kDpI.js} +25 -21
  44. package/dist/command-D_w_kDpI.js.map +1 -0
  45. package/dist/{command-Ff0T-1Hi.js → command-Dwlh0Vhu.js} +17 -16
  46. package/dist/command-Dwlh0Vhu.js.map +1 -0
  47. package/dist/{command-D8oMB2fa.js → command-loXH2jHr.js} +16 -16
  48. package/dist/{command-D8oMB2fa.js.map → command-loXH2jHr.js.map} +1 -1
  49. package/dist/{commands-xAOi_VEk.js → commands-Cuf57OYb.js} +174 -40
  50. package/dist/commands-Cuf57OYb.js.map +1 -0
  51. package/dist/{commands-Bk9NG1s_.js → commands-Dqli45sR.js} +1 -1
  52. package/dist/{config-SvK8pjEK.js → config-C0RKt00M.js} +288 -106
  53. package/dist/config-C0RKt00M.js.map +1 -0
  54. package/dist/{context-BdgCao9Q.js → context-DrEAtUyg.js} +2 -2
  55. package/dist/{context-BdgCao9Q.js.map → context-DrEAtUyg.js.map} +1 -1
  56. package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
  57. package/dist/cost-C21gGrju.js.map +1 -0
  58. package/dist/cost-IM80LyGD.js +1 -0
  59. package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-CRsFxorF.js} +4 -4
  60. package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-CRsFxorF.js.map} +1 -1
  61. package/dist/db-3o8s2EyD.js +2 -0
  62. package/dist/{db-BzOJ4Js4.js → db-ErnJ-gMD.js} +168 -248
  63. package/dist/db-ErnJ-gMD.js.map +1 -0
  64. package/dist/debug-CIXzB4wK.js.map +1 -1
  65. package/dist/{default-effort-BDKIxJyC.js → default-effort-D70cTG00.js} +4 -4
  66. package/dist/{default-effort-BDKIxJyC.js.map → default-effort-D70cTG00.js.map} +1 -1
  67. package/dist/{default-effort-BBDPbJ4D.js → default-effort-DHzj84dD.js} +1 -1
  68. package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
  69. package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
  70. package/dist/{docker-DFrjoNs-.js → docker-CFWzZOEk.js} +4 -4
  71. package/dist/{docker-DFrjoNs-.js.map → docker-CFWzZOEk.js.map} +1 -1
  72. package/dist/docker-DrMnpL84.js +1 -0
  73. package/dist/{doctor-DWK4ttex.js → doctor-B7utCSdh.js} +121 -100
  74. package/dist/doctor-B7utCSdh.js.map +1 -0
  75. package/dist/{environment-cPMCOk-h.js → environment-C0Gg8kyN.js} +49 -11
  76. package/dist/environment-C0Gg8kyN.js.map +1 -0
  77. package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
  78. package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
  79. package/dist/{eval-source-C9b1h8L6.js → eval-source-CsW5VIXS.js} +24 -12
  80. package/dist/eval-source-CsW5VIXS.js.map +1 -0
  81. package/dist/{evaluate-B12Tosvq.js → evaluate-BHQyAszQ.js} +679 -142
  82. package/dist/evaluate-BHQyAszQ.js.map +1 -0
  83. package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DzHq4EsU.d.ts} +2 -2
  84. package/dist/fake-bootstrap-VpisF1WE.js +15 -0
  85. package/dist/fake-bootstrap-VpisF1WE.js.map +1 -0
  86. package/dist/{files-tiNWe8Ih.js → files-BCraBodb.js} +3 -3
  87. package/dist/{files-tiNWe8Ih.js.map → files-BCraBodb.js.map} +1 -1
  88. package/dist/files-BjOJw4sn.js +1 -0
  89. package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
  90. package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
  91. package/dist/{files-BOBdC0uI.js → files-BuKWZCzY.js} +6 -6
  92. package/dist/{files-BOBdC0uI.js.map → files-BuKWZCzY.js.map} +1 -1
  93. package/dist/files-CAzVG3Rr.js +1 -0
  94. package/dist/{files-F5EG0WPT.js → files-Cv856jLN.js} +5 -5
  95. package/dist/{files-F5EG0WPT.js.map → files-Cv856jLN.js.map} +1 -1
  96. package/dist/files-D-Lir4aE.js +1 -0
  97. package/dist/files-DRlJ0Xr2.js +1 -0
  98. package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
  99. package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
  100. package/dist/{files-s28a6czq.js → files-qjeqXnfE.js} +143 -20
  101. package/dist/files-qjeqXnfE.js.map +1 -0
  102. package/dist/gateway.d.ts +1 -1
  103. package/dist/{git-BpoqEgDV.js → git-DOrOtAiD.js} +3 -3
  104. package/dist/{git-BpoqEgDV.js.map → git-DOrOtAiD.js.map} +1 -1
  105. package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
  106. package/dist/harness-FcTyyPco.js.map +1 -0
  107. package/dist/harness-install-Dxho1yaU.js +53 -0
  108. package/dist/harness-install-Dxho1yaU.js.map +1 -0
  109. package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
  110. package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
  111. package/dist/harness.d.ts +11 -8
  112. package/dist/harness.js +1 -1
  113. package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
  114. package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
  115. package/dist/{images-D2jFze2v.js → images-Ch1y1t6_.js} +122 -73
  116. package/dist/images-Ch1y1t6_.js.map +1 -0
  117. package/dist/index.d.ts +46 -7
  118. package/dist/index.js +6 -6
  119. package/dist/{init-Bi1Q3lRi.js → init-C2HqCwnt.js} +33 -14
  120. package/dist/init-C2HqCwnt.js.map +1 -0
  121. package/dist/init-CM9xwyre.js +1 -0
  122. package/dist/{ink-list-BpfidQ1h.js → ink-list-BbKXElcQ.js} +54 -15
  123. package/dist/ink-list-BbKXElcQ.js.map +1 -0
  124. package/dist/inputs-5UIDpjz-.js +190 -0
  125. package/dist/inputs-5UIDpjz-.js.map +1 -0
  126. package/dist/inputs-load-C82HOKnb.js +1 -0
  127. package/dist/inputs-load-CGPyigMC.js +95 -0
  128. package/dist/inputs-load-CGPyigMC.js.map +1 -0
  129. package/dist/{invocations-CxGNOlh0.js → invocations-BrZAMuTb.js} +5 -5
  130. package/dist/{invocations-CxGNOlh0.js.map → invocations-BrZAMuTb.js.map} +1 -1
  131. package/dist/jiti-radm1Soi.js +52 -0
  132. package/dist/jiti-radm1Soi.js.map +1 -0
  133. package/dist/{judge-COMXT1Hx.js → judge-DwzqucLS.js} +17 -12
  134. package/dist/{judge-COMXT1Hx.js.map → judge-DwzqucLS.js.map} +1 -1
  135. package/dist/judge.d.ts +3 -1
  136. package/dist/judge.js +1 -1
  137. package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
  138. package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
  139. package/dist/{layout-Cm85tQYk.js → layout-Czh5wOpz.js} +14 -11
  140. package/dist/layout-Czh5wOpz.js.map +1 -0
  141. package/dist/{list-DuThsE28.js → list-B_C0k5ex.js} +33 -19
  142. package/dist/{list-DuThsE28.js.map → list-B_C0k5ex.js.map} +1 -1
  143. package/dist/live-C9Et_BVT.js +1 -0
  144. package/dist/{live-CgYbO4Te.js → live-DGgIrVw0.js} +7 -5
  145. package/dist/live-DGgIrVw0.js.map +1 -0
  146. package/dist/{live-DL0se4Zd.js → live-DgFm0H90.js} +9 -9
  147. package/dist/{live-DL0se4Zd.js.map → live-DgFm0H90.js.map} +1 -1
  148. package/dist/machine-service-2lIv5UWF.js +1 -0
  149. package/dist/machine-service-Ct_1XMMi.js +520 -0
  150. package/dist/machine-service-Ct_1XMMi.js.map +1 -0
  151. package/dist/machines-BnquHcBZ.js +120 -0
  152. package/dist/machines-BnquHcBZ.js.map +1 -0
  153. package/dist/{model-CkNVAxob.js → model-B1tdv9-d.js} +1 -1
  154. package/dist/model-Bcy4Qjmm.js +1 -0
  155. package/dist/{model-V3mMXp6x.js → model-DF77SgZv.js} +6 -6
  156. package/dist/{model-V3mMXp6x.js.map → model-DF77SgZv.js.map} +1 -1
  157. package/dist/{model-CwwkrCb7.js → model-DRa7yv6K.js} +9 -6
  158. package/dist/model-DRa7yv6K.js.map +1 -0
  159. package/dist/{model-cgt5uvu8.js → model-E5X8YBJM.js} +7 -6
  160. package/dist/model-E5X8YBJM.js.map +1 -0
  161. package/dist/model-calls-B0BoQfJ4.js +95 -0
  162. package/dist/model-calls-B0BoQfJ4.js.map +1 -0
  163. package/dist/{node-D7pAU--0.js → node-BjuhqQwa.js} +21 -10
  164. package/dist/node-BjuhqQwa.js.map +1 -0
  165. package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
  166. package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
  167. package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-B1sIOTD2.js} +2 -2
  168. package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-B1sIOTD2.js.map} +1 -1
  169. package/dist/options-C989kuVE.js +1 -0
  170. package/dist/own-reads-CYmBlrHK.js +15 -0
  171. package/dist/own-reads-CYmBlrHK.js.map +1 -0
  172. package/dist/{patch-DAi1nCuH.js → patch-BR8rdl7r.js} +2 -2
  173. package/dist/{patch-DAi1nCuH.js.map → patch-BR8rdl7r.js.map} +1 -1
  174. package/dist/pi-providers-HygmL1y-.js +552 -0
  175. package/dist/pi-providers-HygmL1y-.js.map +1 -0
  176. package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
  177. package/dist/plain-Ctn8oUdp.js.map +1 -0
  178. package/dist/{plan-CsPilzpk.js → plan-BcT4G5jj.js} +104 -15
  179. package/dist/plan-BcT4G5jj.js.map +1 -0
  180. package/dist/{plugins-BfnhazOv.js → plugins-cC0sqoDA.js} +608 -12
  181. package/dist/plugins-cC0sqoDA.js.map +1 -0
  182. package/dist/preflight-BMVDBcux.js +1 -0
  183. package/dist/{preflight-BixaE4BA.js → preflight-C9OtTb7k.js} +4 -4
  184. package/dist/{preflight-BixaE4BA.js.map → preflight-C9OtTb7k.js.map} +1 -1
  185. package/dist/{preflight-GkpgrBxP.js → preflight-DySHXeFJ.js} +9 -10
  186. package/dist/preflight-DySHXeFJ.js.map +1 -0
  187. package/dist/{preflight-D1C-iH0U.js → preflight-zsAn3heN.js} +1 -1
  188. package/dist/{prepare-C--rM_wO.js → prepare-MaRWRhNT.js} +2 -2
  189. package/dist/prepare-nbX1CXXg.js +409 -0
  190. package/dist/prepare-nbX1CXXg.js.map +1 -0
  191. package/dist/prices-B1vm_lPk.js +1 -0
  192. package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
  193. package/dist/prices-BrY_ihLs.js.map +1 -0
  194. package/dist/{provenance-BXyYwUC8.js → provenance-CkykifD9.js} +5 -5
  195. package/dist/{provenance-BXyYwUC8.js.map → provenance-CkykifD9.js.map} +1 -1
  196. package/dist/provider-DHT20Gl8.js +1 -0
  197. package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
  198. package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
  199. package/dist/provider-stop-CxzWW7Ft.js +69 -0
  200. package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
  201. package/dist/providers-BkLGzaSY.js +1 -0
  202. package/dist/{providers-1p4HKYIp.js → providers-bNgfKa6m.js} +224 -49
  203. package/dist/providers-bNgfKa6m.js.map +1 -0
  204. package/dist/query-B2SQddoV.js +1 -0
  205. package/dist/{query-DhhU9ADI.js → query-DhV3a8lM.js} +2 -2
  206. package/dist/{query-DhhU9ADI.js.map → query-DhV3a8lM.js.map} +1 -1
  207. package/dist/{questions-CMAq4pr8.js → questions-Csv64WCb.js} +7 -7
  208. package/dist/{questions-CMAq4pr8.js.map → questions-Csv64WCb.js.map} +1 -1
  209. package/dist/{record-6IUzwP9K.js → record--vHreTLk.js} +11 -5
  210. package/dist/record--vHreTLk.js.map +1 -0
  211. package/dist/{redact-BhjtnjrU.js → redact-t-L3VGy8.js} +106 -2
  212. package/dist/redact-t-L3VGy8.js.map +1 -0
  213. package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
  214. package/dist/remarks-BpdM_2RC.js.map +1 -0
  215. package/dist/{remarks-BNABzemt.js → remarks-CYFOsnz5.js} +46 -11
  216. package/dist/remarks-CYFOsnz5.js.map +1 -0
  217. package/dist/remarks-CleoDlYw.js +1 -0
  218. package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
  219. package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
  220. package/dist/{remote-U7fNt0UP.js → remote-0CHog7Oq.js} +6 -6
  221. package/dist/{remote-U7fNt0UP.js.map → remote-0CHog7Oq.js.map} +1 -1
  222. package/dist/resolve-Bm9ZyFR-.js +1 -0
  223. package/dist/resolve-Bul8lHaa.js +205 -0
  224. package/dist/resolve-Bul8lHaa.js.map +1 -0
  225. package/dist/{resolve-Cx3a9UQ2.js → resolve-DorSHDBs.js} +14 -11
  226. package/dist/resolve-DorSHDBs.js.map +1 -0
  227. package/dist/resources-DqdwihhT.js +62 -0
  228. package/dist/resources-DqdwihhT.js.map +1 -0
  229. package/dist/{results-CQKXmHym.js → results-DhMgRkR0.js} +9 -22
  230. package/dist/results-DhMgRkR0.js.map +1 -0
  231. package/dist/{resume-POpZa1ge.js → resume-C8Ihfy-C.js} +166 -17
  232. package/dist/resume-C8Ihfy-C.js.map +1 -0
  233. package/dist/{retry-CmG809Mp.js → retry-Bfk7pDti.js} +4 -4
  234. package/dist/{retry-CmG809Mp.js.map → retry-Bfk7pDti.js.map} +1 -1
  235. package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
  236. package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
  237. package/dist/routes-Cp4gAjBZ.js +427 -0
  238. package/dist/routes-Cp4gAjBZ.js.map +1 -0
  239. package/dist/{runtime-Xm9oeC7o.js → runtime-CPUH0Vk5.js} +46 -5
  240. package/dist/runtime-CPUH0Vk5.js.map +1 -0
  241. package/dist/sandbox/boat.d.ts +1 -1
  242. package/dist/sandbox/boat.js +1 -1
  243. package/dist/{sandbox-DrlL4yqb.js → sandbox-z8q_4Ywn.js} +3 -3
  244. package/dist/{sandbox-DrlL4yqb.js.map → sandbox-z8q_4Ywn.js.map} +1 -1
  245. package/dist/sandbox.d.ts +1 -1
  246. package/dist/sandbox.js +1 -1
  247. package/dist/{server-C00zd6zl.js → server-BsQZtuF9.js} +9 -9
  248. package/dist/{server-C00zd6zl.js.map → server-BsQZtuF9.js.map} +1 -1
  249. package/dist/server-ZaMzUNmV.js +1 -0
  250. package/dist/services-BG8xuU82.js +39 -0
  251. package/dist/services-BG8xuU82.js.map +1 -0
  252. package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
  253. package/dist/shared-DtZwIAfy.js.map +1 -0
  254. package/dist/{source-DDgr48Gn.js → source-9HWaTEgV.js} +2 -2
  255. package/dist/{source-DDgr48Gn.js.map → source-9HWaTEgV.js.map} +1 -1
  256. package/dist/source-BcBDj-R0.js +1 -0
  257. package/dist/src-B2eHTCnp.js +3 -0
  258. package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
  259. package/dist/src-B5r1d23Y.js.map +1 -0
  260. package/dist/{src-Dw79VLa3.js → src-Bb3k9Kn_.js} +208 -59
  261. package/dist/src-Bb3k9Kn_.js.map +1 -0
  262. package/dist/src-BfOlH_Bu.js +2 -0
  263. package/dist/src-BuGc40IF.js +2 -0
  264. package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
  265. package/dist/src-C-ChdnG6.js.map +1 -0
  266. package/dist/src-C2K5_afU.js +2 -0
  267. package/dist/{src-YvblOrBU.js → src-C3OhA5pw.js} +87 -13
  268. package/dist/src-C3OhA5pw.js.map +1 -0
  269. package/dist/src-CSVJjghq.js +4 -0
  270. package/dist/{src-y0nOeRRE.js → src-CWfy8ct8.js} +1572 -487
  271. package/dist/src-CWfy8ct8.js.map +1 -0
  272. package/dist/src-CcrQQB0_.js +2 -0
  273. package/dist/src-CotUxxQV.js +2 -0
  274. package/dist/src-D-PwBaBB.js +2 -0
  275. package/dist/{src-Cb2MWozp.js → src-DNWiU__5.js} +276 -41
  276. package/dist/src-DNWiU__5.js.map +1 -0
  277. package/dist/{src-KxS91P7L.js → src-DwcTn4JY.js} +79 -11
  278. package/dist/src-DwcTn4JY.js.map +1 -0
  279. package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
  280. package/dist/src-boB6D-5T.js.map +1 -0
  281. package/dist/{src-DvGns8cz.js → src-c3ZQu7Uu.js} +299 -17
  282. package/dist/src-c3ZQu7Uu.js.map +1 -0
  283. package/dist/src-jYqNkT-F.js +1 -0
  284. package/dist/start-Db5VRcRZ.js +660 -0
  285. package/dist/start-Db5VRcRZ.js.map +1 -0
  286. package/dist/state-CKIjZZRK.js +1 -0
  287. package/dist/{state-328JpnHA.js → state-DUYdzNA7.js} +21 -21
  288. package/dist/{state-328JpnHA.js.map → state-DUYdzNA7.js.map} +1 -1
  289. package/dist/store/local.d.ts +8 -0
  290. package/dist/store/local.js +16 -0
  291. package/dist/store/local.js.map +1 -0
  292. package/dist/store/s3.d.ts +10 -0
  293. package/dist/store/s3.js +18 -0
  294. package/dist/store/s3.js.map +1 -0
  295. package/dist/suite-CH1aaAbs.js +2 -0
  296. package/dist/{suite-COglUhPd.js → suite-DjGxmDSY.js} +6 -5
  297. package/dist/{suite-COglUhPd.js.map → suite-DjGxmDSY.js.map} +1 -1
  298. package/dist/{summary-DDy9DfyJ.js → summary-BK5Sa_N0.js} +28 -7
  299. package/dist/summary-BK5Sa_N0.js.map +1 -0
  300. package/dist/sync-DUi_P2UJ.js +559 -0
  301. package/dist/sync-DUi_P2UJ.js.map +1 -0
  302. package/dist/{tarball-CESxtK4X.js → tarball-C8BSTAMk.js} +2 -2
  303. package/dist/{tarball-CESxtK4X.js.map → tarball-C8BSTAMk.js.map} +1 -1
  304. package/dist/tart-BQ_Ap0l4.js +508 -0
  305. package/dist/tart-BQ_Ap0l4.js.map +1 -0
  306. package/dist/tart-CzViW3WZ.js +186 -0
  307. package/dist/tart-CzViW3WZ.js.map +1 -0
  308. package/dist/tart-DmVuVsmX.js +1 -0
  309. package/dist/tart-bake-D1RttYfM.js +148 -0
  310. package/dist/tart-bake-D1RttYfM.js.map +1 -0
  311. package/dist/tart-bake-DjFcsG3N.js +1 -0
  312. package/dist/tart-cli-BOcluha1.js +1 -0
  313. package/dist/tart-cli-gA9xEaQL.js +89 -0
  314. package/dist/tart-cli-gA9xEaQL.js.map +1 -0
  315. package/dist/tart-hosts-CHNrfV6u.js +313 -0
  316. package/dist/tart-hosts-CHNrfV6u.js.map +1 -0
  317. package/dist/tart-hosts-CWse-ASo.js +1 -0
  318. package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CLBjfF9u.js} +64 -6
  319. package/dist/tart-images-CLBjfF9u.js.map +1 -0
  320. package/dist/testing.d.ts +36 -2
  321. package/dist/testing.js +4 -4
  322. package/dist/tool-class-krsmEo44.js +433 -0
  323. package/dist/tool-class-krsmEo44.js.map +1 -0
  324. package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
  325. package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
  326. package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
  327. package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
  328. package/dist/{transcript-CCf07UmU.js → transcript-BXWZiMuC.js} +3 -3
  329. package/dist/{transcript-CCf07UmU.js.map → transcript-BXWZiMuC.js.map} +1 -1
  330. package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
  331. package/dist/types-CbcnJxCy.js.map +1 -0
  332. package/dist/{types-DBwFsGem.d.ts → types-DKK8jj--.d.ts} +517 -45
  333. package/dist/ui-DKYjTVpn.js +1455 -0
  334. package/dist/ui-DKYjTVpn.js.map +1 -0
  335. package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
  336. package/dist/usage-CwIRPPZ9.js.map +1 -0
  337. package/dist/vercel-models-Ci33llXz.js +21 -0
  338. package/dist/vercel-models-Ci33llXz.js.map +1 -0
  339. package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
  340. package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
  341. package/dist/vertex-DXfcXYKn.js +1 -0
  342. package/dist/view/index.d.ts +32 -3
  343. package/dist/view/index.js +8 -8
  344. package/dist/{views-BROacjBv.js → views-B1zVLJsF.js} +6 -6
  345. package/dist/{views-BROacjBv.js.map → views-B1zVLJsF.js.map} +1 -1
  346. package/dist/views-DCjtw1jF.js +2 -0
  347. package/dist/{worker-entry-Smb4bNDc.js → worker-entry-DrAYYt0M.js} +37 -14
  348. package/dist/worker-entry-DrAYYt0M.js.map +1 -0
  349. package/dist/workspace-CnRbV8cM.js +1 -0
  350. package/dist/{workspace-DNDTEk8v.js → workspace-DwTe1zo6.js} +7 -7
  351. package/dist/{workspace-DNDTEk8v.js.map → workspace-DwTe1zo6.js.map} +1 -1
  352. package/docker/Dockerfile +1 -1
  353. package/package.json +15 -1
  354. package/skills/eval-rubric/SKILL.md +10 -0
  355. package/skills/reeval/SKILL.md +370 -0
  356. package/skills/reeval/references/capture.md +48 -0
  357. package/skills/reeval/references/cli.md +162 -0
  358. package/skills/reeval/references/compare.md +180 -0
  359. package/skills/reeval/references/config.md +147 -0
  360. package/skills/reeval/references/environments.md +218 -0
  361. package/skills/reeval/references/evals.md +241 -0
  362. package/skills/reeval/references/mistakes.md +39 -0
  363. package/skills/reeval/references/vocabulary.md +19 -0
  364. package/skills/setup/SKILL.md +219 -0
  365. package/skills/setup/references/templates.md +258 -0
  366. package/dist/adapter-C60z3wLq.js.map +0 -1
  367. package/dist/agent-BmsMXZXx.js.map +0 -1
  368. package/dist/agents-DXuFxtGC.js.map +0 -1
  369. package/dist/app-files-DT5mJyV4.js.map +0 -1
  370. package/dist/assemble-DM2PNU4R.js.map +0 -1
  371. package/dist/attempt-host-7VviAJrl.js.map +0 -1
  372. package/dist/boat-BjLqFJIZ.js.map +0 -1
  373. package/dist/boat-DSNgZt-t.js +0 -322
  374. package/dist/boat-DSNgZt-t.js.map +0 -1
  375. package/dist/capture-BzotD2hs.js +0 -1
  376. package/dist/capture-Ccf0Hmyx.js.map +0 -1
  377. package/dist/cases-Dy0mifal.js +0 -1
  378. package/dist/cases-z9E6DCd5.js.map +0 -1
  379. package/dist/cli-DTuLSlmW.js.map +0 -1
  380. package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
  381. package/dist/client-KM4C5r5_.js +0 -387
  382. package/dist/client-KM4C5r5_.js.map +0 -1
  383. package/dist/command-DSL7I-SI.js.map +0 -1
  384. package/dist/command-Ff0T-1Hi.js.map +0 -1
  385. package/dist/commands-xAOi_VEk.js.map +0 -1
  386. package/dist/config-SvK8pjEK.js.map +0 -1
  387. package/dist/cost-BdRgySQl.js.map +0 -1
  388. package/dist/cost-Cr26Alhq.js +0 -1
  389. package/dist/db-BzOJ4Js4.js.map +0 -1
  390. package/dist/db-CQRZ3URX.js +0 -2
  391. package/dist/docker--mukta3L.js +0 -1
  392. package/dist/docker-DG_uFgn2.js +0 -246
  393. package/dist/docker-DG_uFgn2.js.map +0 -1
  394. package/dist/doctor-DWK4ttex.js.map +0 -1
  395. package/dist/environment-cPMCOk-h.js.map +0 -1
  396. package/dist/eval-source-C9b1h8L6.js.map +0 -1
  397. package/dist/evaluate-B12Tosvq.js.map +0 -1
  398. package/dist/files-C1Rk9FYA.js +0 -1
  399. package/dist/files-DMQ9-Vv5.js +0 -1
  400. package/dist/files-Di0xDbCc.js +0 -1
  401. package/dist/files-G5jf8wBp.js +0 -1
  402. package/dist/files-s28a6czq.js.map +0 -1
  403. package/dist/harness-DXv2gnaT.js.map +0 -1
  404. package/dist/images-D2jFze2v.js.map +0 -1
  405. package/dist/init-Bi1Q3lRi.js.map +0 -1
  406. package/dist/init-CUw65o36.js +0 -1
  407. package/dist/ink-list-BpfidQ1h.js.map +0 -1
  408. package/dist/kinds-Bh2I35id.js +0 -698
  409. package/dist/kinds-Bh2I35id.js.map +0 -1
  410. package/dist/layout-Cm85tQYk.js.map +0 -1
  411. package/dist/live-CgYbO4Te.js.map +0 -1
  412. package/dist/live-DZLmj0iG.js +0 -1
  413. package/dist/matchers-CXLn-1jm.js +0 -148
  414. package/dist/matchers-CXLn-1jm.js.map +0 -1
  415. package/dist/model-Cv4GuTsZ.js +0 -1
  416. package/dist/model-CwwkrCb7.js.map +0 -1
  417. package/dist/model-cgt5uvu8.js.map +0 -1
  418. package/dist/node-D7pAU--0.js.map +0 -1
  419. package/dist/options-DPUmk7v_.js +0 -1
  420. package/dist/plain-BYUf_fUV.js.map +0 -1
  421. package/dist/plan-CsPilzpk.js.map +0 -1
  422. package/dist/plugins-BfnhazOv.js.map +0 -1
  423. package/dist/preflight-Bn3ianye.js +0 -1
  424. package/dist/preflight-GkpgrBxP.js.map +0 -1
  425. package/dist/prepare-DQEDXoDp.js +0 -268
  426. package/dist/prepare-DQEDXoDp.js.map +0 -1
  427. package/dist/prices-5m3SzMZ3.js.map +0 -1
  428. package/dist/provider-BnUoz4KB.js +0 -1
  429. package/dist/providers-1p4HKYIp.js.map +0 -1
  430. package/dist/providers-B-ijCktf.js +0 -1
  431. package/dist/query-BYhyf0LR.js +0 -1
  432. package/dist/record-6IUzwP9K.js.map +0 -1
  433. package/dist/redact-BhjtnjrU.js.map +0 -1
  434. package/dist/remarks-BNABzemt.js.map +0 -1
  435. package/dist/remarks-CAvTSq16.js +0 -1
  436. package/dist/remarks-D9Lcd1fF.js.map +0 -1
  437. package/dist/resolve-C14Pmtej.js +0 -1
  438. package/dist/resolve-Cx3a9UQ2.js.map +0 -1
  439. package/dist/results-CQKXmHym.js.map +0 -1
  440. package/dist/resume-POpZa1ge.js.map +0 -1
  441. package/dist/runtime-Xm9oeC7o.js.map +0 -1
  442. package/dist/server-C1UJcpUh.js +0 -1
  443. package/dist/shared-Cfi45Bvy.js.map +0 -1
  444. package/dist/source-DJHmQv_j.js +0 -1
  445. package/dist/src-1dBuXzs4.js +0 -3
  446. package/dist/src-BKpioayC.js +0 -2
  447. package/dist/src-BNQznj4z.js.map +0 -1
  448. package/dist/src-BY1cCaoJ.js +0 -2
  449. package/dist/src-BbZYaQ8b.js +0 -2
  450. package/dist/src-CDFH2vVN.js +0 -2
  451. package/dist/src-CMW5R9Q0.js +0 -1
  452. package/dist/src-CV_7DkPa.js.map +0 -1
  453. package/dist/src-Cb2MWozp.js.map +0 -1
  454. package/dist/src-CrV2WA2i.js +0 -2
  455. package/dist/src-CsFaawcZ.js +0 -2
  456. package/dist/src-DAzaCTcN.js.map +0 -1
  457. package/dist/src-DvGns8cz.js.map +0 -1
  458. package/dist/src-Dw79VLa3.js.map +0 -1
  459. package/dist/src-KxS91P7L.js.map +0 -1
  460. package/dist/src-YvblOrBU.js.map +0 -1
  461. package/dist/src-wKwznPgt.js +0 -2
  462. package/dist/src-y0nOeRRE.js.map +0 -1
  463. package/dist/state-Ca55Gk_O.js +0 -1
  464. package/dist/suite-CKBa6MKt.js +0 -2
  465. package/dist/summary-DDy9DfyJ.js.map +0 -1
  466. package/dist/tart-cli-C-GsbVsQ.js +0 -1
  467. package/dist/tart-cli-CU0nFS-L.js +0 -55
  468. package/dist/tart-cli-CU0nFS-L.js.map +0 -1
  469. package/dist/tart-dWoJvWHc.js +0 -586
  470. package/dist/tart-dWoJvWHc.js.map +0 -1
  471. package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
  472. package/dist/tool-target-DS5d72eH.js +0 -40
  473. package/dist/tool-target-DS5d72eH.js.map +0 -1
  474. package/dist/types-C2mLpph3.js.map +0 -1
  475. package/dist/usage-DJiDnh2D.js.map +0 -1
  476. package/dist/vercel-CGlSBv63.js +0 -344
  477. package/dist/vercel-CGlSBv63.js.map +0 -1
  478. package/dist/vertex-O88TCwgY.js +0 -1
  479. package/dist/views-B0Yt_Q1S.js +0 -2
  480. package/dist/worker-entry-Smb4bNDc.js.map +0 -1
  481. package/dist/workspace-C0dDP75l.js +0 -1
@@ -11,6 +11,10 @@ type EffortSample = {
11
11
  filesChanged: number;
12
12
  /** Total tokens when this attempt reported usage; omitted when the harness dropped it. */
13
13
  tokens?: number;
14
+ /** Where its agent time went; omitted for an attempt recorded without it. */
15
+ split?: TimeSplit;
16
+ /** The agent's cost in US dollars (`cost.usd`); omitted when the attempt has none. */
17
+ usd?: number;
14
18
  };
15
19
  type Spread = {
16
20
  median: number;
@@ -28,6 +32,13 @@ type CellEffort = {
28
32
  filesChanged: number;
29
33
  /** Over the counted attempts that reported usage; omitted when none did. */
30
34
  tokens?: Spread;
35
+ /**
36
+ * Median model, tool, waiting and other time over the counted attempts that have a split;
37
+ * omitted when none does. `modelMs` is null when any of them saw no model calls.
38
+ */
39
+ split?: TimeSplit;
40
+ /** The agent's cost in US dollars over the counted attempts that have one; omitted when none does. */
41
+ usd?: Spread;
31
42
  };
32
43
  /**
33
44
  * `fewer` / `more`: the exact rank test's p is below `EFFORT_ALPHA`. `noise`: it isn't.
@@ -67,11 +78,13 @@ type EffortContrast = {
67
78
  /** Set when some counted attempts on either side reported no usage. Not a verdict. */
68
79
  tokensNote?: typeof TOKENS_UNREPORTED;
69
80
  };
70
- declare const effortSample: (a: Pick<AttemptRecord, 'activity' | 'timings' | 'usage' | 'changedFiles'>) => EffortSample;
81
+ declare const effortSample: (a: Pick<AttemptRecord, 'activity' | 'timings' | 'usage' | 'changedFiles' | 'timeSplit' | 'cost'>) => EffortSample;
71
82
  /** The files the workspace diff changed, a collapsed directory counting each of its files. */
72
83
  declare const filesChanged: (files: AttemptRecord['changedFiles']) => number;
73
84
  /** Token totals from the attempts that reported usage; empty when none did. */
74
85
  declare const tokenValues: (samples: EffortSample[]) => number[];
86
+ /** The samples' median split, `modelMs` null when any sample's is; undefined when none has one. */
87
+ declare function splitOf(samples: EffortSample[]): TimeSplit | undefined;
75
88
  declare function effortOf(samples: EffortSample[]): CellEffort | null;
76
89
  /**
77
90
  * The verdict of an exact rank test of `xs` (variant) against `ys` (control). Its direction is
@@ -109,6 +122,11 @@ declare const formatRatio: (r: number | null) => string;
109
122
  declare const bothUnderATenth: (a: number, b: number) => boolean;
110
123
  /** Seconds for agent time: `<0.1s`, `0.4s`, `21s`, `1m 05s`. */
111
124
  declare function formatSeconds(ms: number): string;
125
+ /**
126
+ * `model 1m 05s · tools 12s · waiting 1m 21s · other 3m 22s`: model left out when unknown,
127
+ * waiting when there was none.
128
+ */
129
+ declare const splitLabel: (s: TimeSplit) => string;
112
130
  declare const VERDICT_TEXT: Record<EffortVerdict, (what: string) => string>;
113
131
  /**
114
132
  * `tools 5 vs 43 (×0.11), agent 21s vs 46s (×0.47): too few attempts to test` for a variant
@@ -118,10 +136,16 @@ declare const VERDICT_TEXT: Record<EffortVerdict, (what: string) => string>;
118
136
  * times both under a tenth of a second have no ratio: `agent <0.1s vs <0.1s`.
119
137
  */
120
138
  declare function effortLabel(c: EffortContrast): string;
121
- /** One cell's effort on its own: `tools 43 · agent 46s`, plus tokens when any attempt reported them. */
122
- declare const effortSummary: (e: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens'>) => string;
123
- /** The same cell between two runs: `tools 44 → 10 (×0.23) · agent 45s → 21s (×0.47)`. */
124
- declare function effortChangeLabel(base: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens'>, head: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens'>): string;
139
+ /**
140
+ * One cell's effort on its own: `tools 43 · agent 46s`, plus tokens and cost when any attempt
141
+ * reported them.
142
+ */
143
+ declare const effortSummary: (e: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens' | 'usd'>) => string;
144
+ /**
145
+ * The same cell between two runs: `tools 44 → 10 (×0.23) · agent 45s → 21s (×0.47)`, then
146
+ * `waiting 0s → 1m 21s` when either side waited on a background job.
147
+ */
148
+ declare function effortChangeLabel(base: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens' | 'split' | 'usd'>, head: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens' | 'split' | 'usd'>): string;
125
149
  /** Stays 1 while reeval is pre-release: the record changes in place. */
126
150
  declare const SCHEMA_VERSION = 1;
127
151
  type ArtifactLevel = 'all' | 'failures' | 'none';
@@ -132,6 +156,16 @@ type FileRef = {
132
156
  sha256: string;
133
157
  bytes: number;
134
158
  };
159
+ type RunDebug = {
160
+ /** `requests`: each attempt's `requests.ndjson`; `all`: its `debug.log` and `trace.ndjson` too. */
161
+ level: 'requests' | 'all';
162
+ files: FileRef[];
163
+ /** Files left out because the secret scan found something in them, with what it found. */
164
+ dropped?: Array<{
165
+ path: string;
166
+ note: string;
167
+ }>;
168
+ };
135
169
  /** A generic attempt file from `c.capture()`. */
136
170
  type ArtifactRef = FileRef & {
137
171
  name: string;
@@ -222,6 +256,10 @@ type JudgmentRecord = {
222
256
  upstream?: string;
223
257
  /** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
224
258
  costUsd?: number;
259
+ /** The call's tokens; unset when the provider reported none, and for the fake judge. */
260
+ usage?: TokenUsage;
261
+ /** `costUsd` as reported, else the call's tokens at the judge model's list price; unset when neither is known. */
262
+ cost?: CostUsd;
225
263
  /**
226
264
  * `<index>:<sha256(question) prefix>`: the same question at the same place in the evaluation, so a
227
265
  * verdict can be followed across runs (and an edited question starts a new series). `<index>`
@@ -292,6 +330,13 @@ type AttemptRecord = {
292
330
  * other sandboxes, and in attempts that didn't reach their sandbox.
293
331
  */
294
332
  vmImage?: VmImage;
333
+ /**
334
+ * The container or VM the attempt ran on and the CPU, memory and disk it used there, measured
335
+ * out of the agent's reach: a container's from a sibling container, a VM's as root
336
+ * (`AttemptResources`). Absent on the fake sandbox, which runs on the host, and in attempts that
337
+ * didn't reach their sandbox.
338
+ */
339
+ resources?: AttemptResources;
295
340
  /** Names of the variables and secrets set in the sandbox, and the brokered ones; never values. */
296
341
  env: {
297
342
  vars: string[];
@@ -348,8 +393,7 @@ type AttemptRecord = {
348
393
  * `text` is the agent's final message, cut at 8 KB; the whole of it is in the `transcript.jsonl`
349
394
  * file.
350
395
  * `mcpStartup` times each stdio MCP server's start in the turn (`McpStartup`).
351
- * `reasoningMs` is how long the turn's streamed reasoning took, null when the harness can't
352
- * time it.
396
+ * `reasoningMs` is how long the turn's reasoning took, null when the harness can't time it.
353
397
  */
354
398
  turns: Array<{
355
399
  prompt: string;
@@ -361,6 +405,11 @@ type AttemptRecord = {
361
405
  usage: TokenUsage | null;
362
406
  /** The model the harness says served the turn; null when it reports none. */
363
407
  model: string | null;
408
+ /**
409
+ * What the harness says the turn cost (Claude Code, pi), else its tokens at the list price of
410
+ * its model (or the cell's); unset when neither is known.
411
+ */
412
+ cost?: CostUsd;
364
413
  /** Event types the harness can't report; see `Turn.unreported`. */
365
414
  unreported: WebEventType[];
366
415
  /** The turn's first error, such as a provider's billing or auth error, cut at 1 KB; else null. */
@@ -377,9 +426,25 @@ type AttemptRecord = {
377
426
  * script's.
378
427
  */
379
428
  reasoningMs?: number | null;
429
+ /**
430
+ * Where the turn's time went: its model calls, tool calls, the background jobs it waited on,
431
+ * and the rest, summing to `durationMs` (`TimeSplit`). Absent in records from before reeval
432
+ * split it.
433
+ */
434
+ timeSplit?: TimeSplit;
380
435
  }>;
436
+ /**
437
+ * The turns' `timeSplit` added up, its `modelMs` null when any turn's is. Absent when no turn
438
+ * has one.
439
+ */
440
+ timeSplit?: TimeSplit;
381
441
  /** Tokens across the attempt's turns; null when the harness reported none. */
382
442
  usage: TokenUsage | null;
443
+ /**
444
+ * The gateway's billed `reportedCostUsd`, else the turns' costs added up; unset when a turn with
445
+ * tokens has none.
446
+ */
447
+ cost?: CostUsd;
383
448
  /**
384
449
  * Tokens the capture vision loop used. Infrastructure only: never included
385
450
  * in `usage`, effort, compare, or lift.
@@ -418,7 +483,10 @@ type AttemptRecord = {
418
483
  maxToolMs: number | null;
419
484
  };
420
485
  judgements: JudgmentRecord[];
421
- /** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
486
+ /**
487
+ * Harness session starts that didn't become ready within `startTimeout`, or failed on a transient
488
+ * error (a 5xx or a network error), and were started again (at most one).
489
+ */
422
490
  startRetries?: StartRetry[];
423
491
  /**
424
492
  * The infra errors this attempt ended with before `reeval run --retry-errors` or `--resume` ran
@@ -810,6 +878,11 @@ type CellRecord = Cell & {
810
878
  * gates on that majority. `runs` still counts every planned attempt, the skipped ones included.
811
879
  */
812
880
  bestOf?: number;
881
+ /** The cell's attempts' agent and judge costs added up; unset where none has one. */
882
+ cost?: {
883
+ agent?: CostTotal;
884
+ judge?: CostTotal;
885
+ };
813
886
  runs: number;
814
887
  pass: number;
815
888
  fail: number;
@@ -867,7 +940,7 @@ type RunSummary = {
867
940
  infraError: number;
868
941
  skipped: number;
869
942
  };
870
- /** The tokens the run used, as the harnesses count them; reeval prices nothing. */
943
+ /** The tokens the run used, as the harnesses count them, and what they cost. */
871
944
  type Cost = {
872
945
  /** Prompt tokens, cache reads and writes included. */
873
946
  inputTokens: number | null;
@@ -875,6 +948,10 @@ type Cost = {
875
948
  /** The same totals split as each turn reports them. */
876
949
  usage: TokenUsage | null;
877
950
  source: 'reported' | 'unreported';
951
+ /** The attempts' costs added up; unset when none has one. */
952
+ agentUsd?: CostTotal;
953
+ /** The judge calls' costs added up; unset when none has one. */
954
+ judgeUsd?: CostTotal;
878
955
  };
879
956
  /**
880
957
  * The `reeval run` settings that shaped a run's plan, as it was first started. A resume keeps
@@ -893,6 +970,11 @@ type RunInvocation = {
893
970
  tags: string[];
894
971
  /** `--trait`, as given (`key=value`); unset without any. */
895
972
  traits?: string[];
973
+ /**
974
+ * The file `--events` wrote the run's event stream to, from the project root in posix form
975
+ * (`../…` outside it), so a resume appends to it; unset without `--events`.
976
+ */
977
+ events?: string;
896
978
  };
897
979
  type RunRecord = {
898
980
  kind: 'reeval-run';
@@ -988,6 +1070,11 @@ type RunRecord = {
988
1070
  cost: Cost;
989
1071
  /** Which artifacts sit next to this record. Refs to files that weren't kept are removed. */
990
1072
  artifacts: ArtifactLevel;
1073
+ /**
1074
+ * The attempts' debug data uploaded with the run to its store (`history.debug`), each file by
1075
+ * its path in `.reeval/debug/<run>/` (`<eval>/<cell>/<n>/requests.ndjson`); absent when none was.
1076
+ */
1077
+ debug?: RunDebug;
991
1078
  redaction: {
992
1079
  version: 1;
993
1080
  };
@@ -1205,6 +1292,11 @@ type TranscriptEvent = EventTime & ({
1205
1292
  durationMs: number | null;
1206
1293
  /** True when a shell-like tool or edit ran at least two minutes. */
1207
1294
  stalled?: boolean;
1295
+ /**
1296
+ * Set when the harness never reported the call's end, so `durationMs` runs to the next
1297
+ * event's `atMs` (the turn's end for the last).
1298
+ */
1299
+ durationInferred?: true;
1208
1300
  /** The `toolCallId` of the subagent call whose agent made this call; absent for the agent's own. */
1209
1301
  subagent?: string;
1210
1302
  }) | {
@@ -1221,6 +1313,11 @@ type TranscriptEvent = EventTime & ({
1221
1313
  durationMs: number | null;
1222
1314
  /** True when a shell-like tool or edit ran at least two minutes. */
1223
1315
  stalled?: boolean;
1316
+ /**
1317
+ * Set when the harness never reported the call's end, so `durationMs` runs to the next
1318
+ * event's `atMs` (the turn's end for the last).
1319
+ */
1320
+ durationInferred?: true;
1224
1321
  /** The `toolCallId` of the subagent call whose agent made this call; absent for the agent's own. */
1225
1322
  subagent?: string;
1226
1323
  } | {
@@ -1232,6 +1329,11 @@ type TranscriptEvent = EventTime & ({
1232
1329
  durationMs: number | null;
1233
1330
  /** True when a shell-like tool or edit ran at least two minutes. */
1234
1331
  stalled?: boolean;
1332
+ /**
1333
+ * Set when the harness never reported the call's end, so `durationMs` runs to the next
1334
+ * event's `atMs` (the turn's end for the last).
1335
+ */
1336
+ durationInferred?: true;
1235
1337
  /** The `toolCallId` of the subagent call whose agent made this call; absent for the agent's own. */
1236
1338
  subagent?: string;
1237
1339
  } | {
@@ -1240,11 +1342,31 @@ type TranscriptEvent = EventTime & ({
1240
1342
  tool: string;
1241
1343
  input: unknown;
1242
1344
  output: unknown;
1345
+ /**
1346
+ * What `output` holds: `model`, the result as the harness gave it to the model; `full`,
1347
+ * the tool's whole output, which can be more than the model saw (the harness cut it for
1348
+ * the model, or saved it to a file and gave the model a pointer, which reeval read).
1349
+ */
1350
+ outputView?: 'model' | 'full';
1351
+ /**
1352
+ * Set when `output` is only part of the output, with a line where the rest was cut:
1353
+ * `transcript.jsonl` keeps the first 96K and last 16K characters of one past 128K (of its
1354
+ * JSON, for structured output), and the harness adapter keeps the first and last 512 KiB
1355
+ * of a saved output past 1 MiB. `bytes` is the whole output's size in UTF-8.
1356
+ */
1357
+ outputTruncated?: {
1358
+ bytes: number;
1359
+ };
1243
1360
  /** Whether the call succeeded; null when the harness never reported its result. */
1244
1361
  ok: boolean | null;
1245
1362
  durationMs: number | null;
1246
1363
  /** True when a shell-like tool or edit ran at least two minutes. */
1247
1364
  stalled?: boolean;
1365
+ /**
1366
+ * Set when the harness never reported the call's end, so `durationMs` runs to the next
1367
+ * event's `atMs` (the turn's end for the last).
1368
+ */
1369
+ durationInferred?: true;
1248
1370
  /** The `toolCallId` of the subagent call whose agent made this call; absent for the agent's own. */
1249
1371
  subagent?: string;
1250
1372
  } | {
@@ -1255,7 +1377,40 @@ type TranscriptEvent = EventTime & ({
1255
1377
  {
1256
1378
  type: 'retry';
1257
1379
  message: string;
1258
- });
1380
+ } | ModelCallEvent);
1381
+ /**
1382
+ * One of the agent's calls to its model API, as the egress gateway saw it (in a sandbox behind
1383
+ * it, such as Docker's): when it started, when its response's first byte came (on a streamed call,
1384
+ * about its first token) and when it ended, with the tokens and the model its response named.
1385
+ * Cursor sends a whole turn over one long stream to `api2.cursor.sh` (`AgentService/RunSSE`),
1386
+ * so a Cursor turn has one call, with `stream: true`, its tool calls inside it and its tokens
1387
+ * only in the turn's `usage`.
1388
+ */
1389
+ type ModelCallEvent = {
1390
+ type: 'model-call';
1391
+ /** `POST api.anthropic.com/v1/messages`: the method, host and path. */
1392
+ request: string;
1393
+ /** The response's HTTP status; null when none came. */
1394
+ status: number | null;
1395
+ durationMs: number;
1396
+ /** Milliseconds from the call's start to its response body's first byte; null when none came. */
1397
+ firstByteMs: number | null;
1398
+ /** The tokens its response body named; null when it named none (Cursor's are protobuf). */
1399
+ usage: TokenUsage | null;
1400
+ /** The model that served it, from its request or response; null when neither named one. */
1401
+ model: string | null;
1402
+ /** Set on a call that carries the whole turn (Cursor's), so it isn't counted as model time. */
1403
+ stream?: true;
1404
+ /**
1405
+ * `main` for one of the agent's own steps, `side` for a call its CLI made around them: a
1406
+ * session title, a web search's own model call, Antigravity's Flash Lite calls. A call is
1407
+ * `main` when its tokens are one of the turn's steps' as the harness reported them. Otherwise
1408
+ * it's `side` when its request declared at most one tool while another call in the attempt
1409
+ * declared more, or when another model served it than served the turn's other calls, and `main`
1410
+ * when neither holds. Absent in records from before roles.
1411
+ */
1412
+ role?: 'main' | 'side';
1413
+ };
1259
1414
  /** The transcript events a harness may be unable to report. */
1260
1415
  type WebEventType = 'web-search' | 'web-fetch';
1261
1416
  type ExecResult = {
@@ -1270,6 +1425,13 @@ type ExecRetry = {
1270
1425
  /** Milliseconds to wait before each extra try. Default 1000. */
1271
1426
  interval?: number;
1272
1427
  };
1428
+ /**
1429
+ * Variables `c.exec` sets for one command, over the environment's own. A value whose name looks
1430
+ * like a secret's (`KEY`, `TOKEN`, `SECRET`, `PASSWORD`, `CREDENTIAL`) is masked in everything
1431
+ * reeval writes, as a host variable's is. The host value of a secret the environment brokers
1432
+ * goes in as its placeholder, which the gateway swaps back on that secret's hosts.
1433
+ */
1434
+ type ExecEnv = Record<string, string>;
1273
1435
  /** One run of a `c.exec` command that had `retries`. */
1274
1436
  type ExecTry = {
1275
1437
  exitCode: number;
@@ -1487,6 +1649,45 @@ type TokenUsage = {
1487
1649
  cacheWrite: number;
1488
1650
  reasoning: number | null;
1489
1651
  };
1652
+ /**
1653
+ * Where a turn's time went, in ms, summing to its `durationMs`. Each moment counts once, as the
1654
+ * first of: `waitingMs`, a background job the agent waited on (from the call that started it
1655
+ * through the polls that checked on it), `toolMs`, a tool call, `modelMs`, a model call, and
1656
+ * `otherMs`, the rest (the harness's own work, and time between calls). `modelMs` is null when
1657
+ * the turn's model calls weren't seen one by one: on a sandbox without the egress gateway, and
1658
+ * on Cursor, whose turn is one stream; that time is in `otherMs`. Which calls wait is a heuristic
1659
+ * over the transcript: a shell call run in the background (`run_in_background`, a trailing `&`,
1660
+ * `nohup`, or the command a Cursor terminal file names), then `sleep` and `wait` commands, output
1661
+ * tools (`BashOutput`, `TaskOutput`) and reads of a Cursor terminal file.
1662
+ */
1663
+ type TimeSplit = {
1664
+ modelMs: number | null;
1665
+ toolMs: number;
1666
+ waitingMs: number;
1667
+ otherMs: number;
1668
+ };
1669
+ /** A dollar figure and where it came from. */
1670
+ type CostUsd = {
1671
+ /** US dollars. */
1672
+ usd: number;
1673
+ /**
1674
+ * `reported`: what the harness (Claude Code, pi) or the gateway (OpenRouter) said it cost;
1675
+ * `estimated`: the tokens at list prices, which leave out discounts and long-context rates.
1676
+ */
1677
+ basis: 'reported' | 'estimated';
1678
+ /**
1679
+ * Who reported it (`claude-code`, `pi`, `openrouter`, `fake`), or the prices it was estimated
1680
+ * at (`models.dev 2026-11-05`, a `REEVAL_PRICES` file); several, joined with `, `, in a sum.
1681
+ */
1682
+ source: string;
1683
+ };
1684
+ /**
1685
+ * Costs added up: `basis` is `reported` only when every part was. `unpriced` counts the parts
1686
+ * (attempts, or judge calls) that used tokens but have no cost, which `usd` leaves out.
1687
+ */
1688
+ type CostTotal = CostUsd & {
1689
+ unpriced?: number;
1690
+ };
1490
1691
  /** One agent turn. Immutable once `c.send()` resolves. */
1491
1692
  /** A turn as data, as a record or another language's runner carries it: `Turn` without its methods. */
1492
1693
  type TurnData = {
@@ -1517,7 +1718,9 @@ type TurnData = {
1517
1718
  };
1518
1719
  type Turn = TurnData & {
1519
1720
  /** Runs in the eval's sandbox. Same as `c.exec`. */
1520
- exec(command: string, options?: ExecRetry): Promise<ExecResult>;
1721
+ exec(command: string, options?: ExecRetry & {
1722
+ env?: ExecEnv;
1723
+ }): Promise<ExecResult>;
1521
1724
  readFile(path: string): Promise<string | null>;
1522
1725
  };
1523
1726
  /** Any JSON value. */
@@ -1695,15 +1898,19 @@ type AttemptContext = {
1695
1898
  judge(question: string, options: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
1696
1899
  /**
1697
1900
  * Runs `command` in the sandbox. With `retries`, a non-zero exit runs it again, up to that
1698
- * many extra times, `interval` ms apart; it resolves with the last try.
1901
+ * many extra times, `interval` ms apart; it resolves with the last try. `env` adds variables
1902
+ * for this command only.
1699
1903
  */
1700
1904
  exec(command: string, options?: ExecRetry & {
1905
+ env?: ExecEnv;
1701
1906
  background?: false;
1702
1907
  }): Promise<ExecResult>;
1703
1908
  exec(command: string, options: {
1704
1909
  background: true;
1910
+ env?: ExecEnv;
1705
1911
  }): Promise<BackgroundHandle>;
1706
1912
  exec(command: string, options?: ExecRetry & {
1913
+ env?: ExecEnv;
1707
1914
  background?: boolean;
1708
1915
  }): Promise<ExecResult | BackgroundHandle>;
1709
1916
  /**
@@ -1815,6 +2022,12 @@ type FakeHarnessDescriptor = {
1815
2022
  failWith?: string;
1816
2023
  /** The first this many session starts of an attempt never become ready, even when aborted (`true`: every one). */
1817
2024
  startHangs?: number | true;
2025
+ /**
2026
+ * A shell command the fake agent runs as its session starts, once per sandbox, as a real
2027
+ * adapter's bootstrap installs its CLI: `tart()` bakes it into the image it clones, as it does a
2028
+ * real harness's, so an attempt there finds it done.
2029
+ */
2030
+ bootstrap?: string;
1818
2031
  /** A reasoning block at the start of every turn. */
1819
2032
  reasoning?: string;
1820
2033
  /**
@@ -1829,13 +2042,15 @@ type FakeHarnessDescriptor = {
1829
2042
  effort?: string;
1830
2043
  /**
1831
2044
  * Tool calls the fake agent reports on every turn, before writing files. `ok` defaults to true;
1832
- * null reports a call whose result never came.
2045
+ * null reports a call whose result never came. `durationMs` defaults to 0; null reports a call
2046
+ * whose end the harness never said.
1833
2047
  */
1834
2048
  tools?: Array<{
1835
2049
  tool: string;
1836
2050
  input: unknown;
1837
2051
  output: unknown;
1838
2052
  ok?: boolean | null;
2053
+ durationMs?: number | null;
1839
2054
  }>;
1840
2055
  /** Files and message used instead when the environment gives the agent a skill. */
1841
2056
  withSkill?: {
@@ -1844,6 +2059,8 @@ type FakeHarnessDescriptor = {
1844
2059
  };
1845
2060
  /** Chance (0–1) that an attempt writes no files and says so, for flaky pass rates. */
1846
2061
  flakeRate?: number;
2062
+ /** US dollars the fake agent says every turn cost, as a harness that reports its cost does; 0 reports none. */
2063
+ costUsd?: number;
1847
2064
  /** Tokens the fake agent reports for every turn, as `turn.usage`; without it, null. */
1848
2065
  usage?: {
1849
2066
  input: number;
@@ -1875,9 +2092,13 @@ type FakeHarnessDescriptor = {
1875
2092
  layout?: 'cursor' | 'claude-code' | 'antigravity' | 'opencode' | 'goose' | 'pi' | 'codex' | 'fx' | 'grok-build';
1876
2093
  /**
1877
2094
  * In Docker, broker the key of the agent `layout` names (`OPENAI_API_KEY` for `codex`, Cursor's
1878
- * when unset) on that agent's endpoints, as its adapter does: the key stays on the host, and
2095
+ * when unset) on that agent's endpoints, as its adapter does; OpenCode, Goose and pi take their
2096
+ * model's provider's key, so with their layouts it's `FAKE_AGENT_API_KEY`, on no endpoint. The key stays on the host, and
1879
2097
  * `exec` steps get a placeholder in its variable, which the egress gateway swaps in on those
1880
2098
  * endpoints when sent as `Authorization: Bearer <placeholder>`, and refuses on any other request.
2099
+ * For an agent whose bridge is a server in the sandbox (Claude Code, OpenCode, Goose, fx, Grok
2100
+ * Build), it also serves on the session's first exposed port and reaches it from the host, as
2101
+ * that adapter's start does.
1881
2102
  */
1882
2103
  brokers?: true;
1883
2104
  };
@@ -1886,7 +2107,8 @@ type FakeHarnessDescriptor = {
1886
2107
  * the command's stdout, `write` as file edits, and `reply` is the turn's final message. `mcp`
1887
2108
  * (`'<server>/<tool>'`) starts the environment's stdio MCP server in the sandbox, as an agent
1888
2109
  * would, and calls the tool with no arguments; it's reported as `mcp__<server>__<tool>`.
1889
- * `error` ends the turn there with status `error` and that message, as a provider error would.
2110
+ * `error` ends the turn there with status `error` and that message, as a provider error would; one
2111
+ * worded as a provider's credit, quota or rate limit makes the attempt an infra error.
1890
2112
  * `refuseKey` ends it as the model API rejecting the key in that variable would, which makes the
1891
2113
  * attempt an infra error.
1892
2114
  */
@@ -2021,8 +2243,9 @@ type OpenCodeHarnessDescriptor = {
2021
2243
  startTimeout?: Duration;
2022
2244
  };
2023
2245
  /**
2024
- * Goose through `@reeval/harness-goose` (`goose acp`), made by `goose()`. It runs Google models
2025
- * on a Gemini API key, and reports no usage (its ACP server counts one call).
2246
+ * Goose through `@reeval/harness-goose` (`goose acp`), made by `goose()`. It runs a model on its
2247
+ * provider's key, and reads each turn's usage from Goose's `usage_ledger` (its ACP server counts
2248
+ * one call).
2026
2249
  */
2027
2250
  type GooseHarnessDescriptor = {
2028
2251
  kind: 'goose';
@@ -2031,7 +2254,10 @@ type GooseHarnessDescriptor = {
2031
2254
  * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
2032
2255
  */
2033
2256
  gateway?: GatewayName;
2034
- /** A Gemini model id, run on Goose's `google` provider, e.g. `gemini-3.8-flash`. */
2257
+ /**
2258
+ * `<provider>/<model>`: Goose's `google`, `anthropic`, `openai` or `xai` provider on its own key,
2259
+ * or the gateway's id on `vercel()` and `openrouter()`, e.g. `google/gemini-3.8-flash`.
2260
+ */
2035
2261
  model: string;
2036
2262
  /**
2037
2263
  * Reasoning effort as `GOOSE_THINKING_EFFORT`, e.g. `off`, `low`, `medium`, `high` or `max`.
@@ -2048,8 +2274,8 @@ type GooseHarnessDescriptor = {
2048
2274
  startTimeout?: Duration;
2049
2275
  };
2050
2276
  /**
2051
- * pi through `@reeval/harness-pi` (`pi --mode json` in the sandbox), made by `pi()`. It runs
2052
- * Google models on a Gemini API key and reports each model call's tokens. It takes stdio and
2277
+ * pi through `@reeval/harness-pi` (`pi --mode json` in the sandbox), made by `pi()`. It runs a
2278
+ * model on its provider's key and reports each model call's tokens. It takes stdio and
2053
2279
  * HTTP MCP servers, not SSE ones.
2054
2280
  */
2055
2281
  type PiHarnessDescriptor = {
@@ -2059,7 +2285,10 @@ type PiHarnessDescriptor = {
2059
2285
  * fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
2060
2286
  */
2061
2287
  gateway?: GatewayName;
2062
- /** A Gemini model id, run on pi's `google` provider, e.g. `gemini-3.8-flash`. */
2288
+ /**
2289
+ * `<provider>/<model>`: one of pi's providers whose key it sends in a header on a fixed host, or
2290
+ * the gateway's id on `vercel()` and `openrouter()`, e.g. `google/gemini-3.8-flash`.
2291
+ */
2063
2292
  model: string;
2064
2293
  /**
2065
2294
  * Reasoning effort, passed as pi's `--thinking` (the CLI's name). Values are whatever that
@@ -2309,6 +2538,27 @@ type TartSandboxDescriptor = {
2309
2538
  user?: string;
2310
2539
  key: string;
2311
2540
  };
2541
+ /**
2542
+ * Macs that run the VMs instead of this machine, each one plain `ssh` reaches without a prompt
2543
+ * (an `~/.ssh/config` alias, the agent or a key file; no passwords): `'mac-studio'`,
2544
+ * `'elliot@mac-mini.local'`, `'ssh://ci@10.0.0.5:2222'` or `{ host, user?, port?, identityFile?, vms? }`.
2545
+ * reeval runs `tart` on each over SSH and keeps the egress gateway, and every key, here. Each
2546
+ * host needs the image; preflight bakes the harness onto it there. This machine when unset.
2547
+ */
2548
+ hosts?: TartHost[];
2549
+ };
2550
+ /** A Mac that runs `tart()`'s VMs over SSH: an SSH destination, or its parts. */
2551
+ type TartHost = string | {
2552
+ /** The host name, address or `~/.ssh/config` alias. */
2553
+ host: string;
2554
+ /** The login user; `ssh`'s own (the config's, or this machine's user) when unset. */
2555
+ user?: string;
2556
+ /** The SSH port; `ssh`'s own (the config's, or 22) when unset. */
2557
+ port?: number;
2558
+ /** The private key `ssh -i` takes: absolute, or `~/…`; the agent's or the config's when unset. */
2559
+ identityFile?: string;
2560
+ /** The most of reeval's macOS VMs it runs at once: 1 or 2, and 2 when unset. */
2561
+ vms?: number;
2312
2562
  };
2313
2563
  /** A boat.dev machine size: `small` is 2 vCPUs and 4 GB. */
2314
2564
  type BoatMachineType = 'small' | 'default' | 'large' | 'xlarge';
@@ -2331,8 +2581,21 @@ type BoatSandboxDescriptor = {
2331
2581
  prefix: string;
2332
2582
  /** The machine size. Default `small`. */
2333
2583
  type?: BoatMachineType;
2334
- /** The most Boat sandboxes at once, across the run. */
2584
+ /** The most attempts on Boat at once, across the run: machines times `attemptsPerMachine`. */
2335
2585
  maxConcurrent?: number;
2586
+ /**
2587
+ * A machine takes another of the run's attempts once one ends, instead of being deleted: its
2588
+ * user's processes, cron jobs and files are removed and its home put back as it was after the
2589
+ * harness installed, which isn't installed again. Default false, one attempt per slot.
2590
+ */
2591
+ reuse?: boolean;
2592
+ /**
2593
+ * The attempts one machine runs at once, each as a user of its own with its own home (which
2594
+ * each sees at `/home/reeval`), workspace and egress gateway. The harness installs once per
2595
+ * machine, and every other slot's home gets its own copy of the install. Default 1. A harness whose bridge takes a fixed port in the
2596
+ * machine (Claude Code, OpenCode, Goose, fx, Grok Build) runs one per machine.
2597
+ */
2598
+ attemptsPerMachine?: number;
2336
2599
  };
2337
2600
  type SandboxDescriptor = FakeSandboxDescriptor | DockerSandboxDescriptor | TartSandboxDescriptor | BoatSandboxDescriptor;
2338
2601
  /**
@@ -2492,6 +2755,17 @@ type SetupStep = string | {
2492
2755
  type EgressPolicy = 'proxy' | 'allow-all' | 'deny-all' | {
2493
2756
  allow: readonly string[];
2494
2757
  };
2758
+ /**
2759
+ * An egress policy per phase of the attempt: `setup` covers its files and setup steps, `agent`
2760
+ * the harness's start, every turn and every check after them. A phase it leaves unset keeps the
2761
+ * policy it inherits.
2762
+ */
2763
+ type PhasedEgress = {
2764
+ setup?: EgressPolicy;
2765
+ agent?: EgressPolicy;
2766
+ };
2767
+ /** One policy for the whole attempt, or one per phase. */
2768
+ type Egress = EgressPolicy | PhasedEgress;
2495
2769
  /**
2496
2770
  * A git repository (or one directory of it) the workspace starts as, from the `git` helper. `url`
2497
2771
  * is what reeval fetches: the clone URL, or an absolute path for a local repository.
@@ -2528,8 +2802,11 @@ type EnvironmentSpec = {
2528
2802
  workspace?: GitWorkspace;
2529
2803
  /** Variables and secrets for the sandbox, merged by name over the parent's. */
2530
2804
  env?: EnvVars;
2531
- /** Replaces the inherited policy; an `extend` child's replaces its parent's. */
2532
- egress?: EgressPolicy;
2805
+ /**
2806
+ * Replaces the inherited policy, per phase: a plain policy sets both, `{ setup, agent }` the
2807
+ * phases it names. An `extend` child's replaces its parent's the same way.
2808
+ */
2809
+ egress?: Egress;
2533
2810
  /** Workspace-relative, or `~/` for home. */
2534
2811
  files?: Record<string, string | HostFile>;
2535
2812
  /** Project rules, written where the harness's agent reads them: `AGENTS.md`, or `CLAUDE.md` for Claude Code. */
@@ -2998,10 +3275,10 @@ type EvalOptionFields<N extends number = number> = {
2998
3275
  /** Variables and secrets for the sandbox, merged by name down the cascade. */
2999
3276
  env?: EnvVars;
3000
3277
  /**
3001
- * Where the sandbox may connect. Replaces the inherited policy: config, profile defaults, the
3002
- * environment, then groups and the eval.
3278
+ * Where the sandbox may connect. Replaces the inherited policy, per phase: config, profile
3279
+ * defaults, the environment, then groups and the eval.
3003
3280
  */
3004
- egress?: EgressPolicy;
3281
+ egress?: Egress;
3005
3282
  /** Defaults for `c.judge()`, merged field by field down the cascade. */
3006
3283
  judge?: JudgeOptions;
3007
3284
  /** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
@@ -3089,11 +3366,78 @@ type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
3089
3366
  /** `'provider/model'`, or a fake judge. */
3090
3367
  model?: string | FakeJudgeModel;
3091
3368
  };
3092
- type HistoryConfig = {
3093
- /** Committed history folder (default: `.history` in the directory the evals live in). */
3369
+ /** A history folder committed with the evals: `local()` from `reeval/store/local`. */
3370
+ type LocalStoreDescriptor = {
3371
+ kind: 'local';
3372
+ /**
3373
+ * The folder, relative to the config's directory (default: `.history` in the directory the
3374
+ * evals live in).
3375
+ */
3094
3376
  dir?: string;
3095
- /** Artifacts recorded next to each run: every attempt's, failed attempts' only, or none. */
3377
+ };
3378
+ /**
3379
+ * The host variables an `s3()` store reads. Each names a variable, never a value: credentials
3380
+ * come from the environment only.
3381
+ */
3382
+ type S3StoreEnv = {
3383
+ /** The access key id (default `AWS_ACCESS_KEY_ID`). */
3384
+ accessKeyId?: string;
3385
+ /** The secret access key (default `AWS_SECRET_ACCESS_KEY`). */
3386
+ secretAccessKey?: string;
3387
+ /** A session token for temporary credentials (default `AWS_SESSION_TOKEN`), when it's set. */
3388
+ sessionToken?: string;
3389
+ /** The endpoint, when `endpoint` isn't set (default `AWS_ENDPOINT_URL_S3`, then `AWS_ENDPOINT_URL`). */
3390
+ endpoint?: string;
3391
+ /** The bucket, when `bucket` isn't set. */
3392
+ bucket?: string;
3393
+ /** The region, when `region` isn't set (default `AWS_REGION`, then `AWS_DEFAULT_REGION`). */
3394
+ region?: string;
3395
+ };
3396
+ /**
3397
+ * An S3-compatible bucket, such as AWS S3, Cloudflare R2 or MinIO: `s3()` from `reeval/store/s3`.
3398
+ * reeval keeps a copy of it in `.reeval/store/` that every history command reads.
3399
+ */
3400
+ type S3StoreDescriptor = {
3401
+ kind: 's3';
3402
+ /** The bucket; `env.bucket` names a variable with it instead. */
3403
+ bucket?: string;
3404
+ /**
3405
+ * The endpoint's origin, such as `https://<account>.r2.cloudflarestorage.com` or
3406
+ * `http://127.0.0.1:9000`; without it, `env.endpoint`'s variable, else AWS S3 in the region.
3407
+ */
3408
+ endpoint?: string;
3409
+ /** The signing region (default: `env.region`'s variable, else `auto`). */
3410
+ region?: string;
3411
+ /** A key prefix every object goes under, so several projects can share a bucket (default none). */
3412
+ prefix?: string;
3413
+ /** Which host variables hold the credentials, and the bucket, endpoint or region when they aren't set here. */
3414
+ env?: S3StoreEnv;
3415
+ };
3416
+ /** A history store as data, which `reeval.yaml` can name too. */
3417
+ type HistoryStoreDescriptor = LocalStoreDescriptor | S3StoreDescriptor;
3418
+ type HistoryConfig = {
3419
+ /**
3420
+ * Where recorded runs are kept: `local()` (the default, a folder committed with the evals) or
3421
+ * `s3()`.
3422
+ */
3423
+ store?: HistoryStoreDescriptor;
3424
+ /**
3425
+ * Artifacts recorded next to each run: every attempt's, failed attempts' only (the default), or
3426
+ * none. The setup log and each `c.exec` log are an attempt's artifacts too.
3427
+ */
3096
3428
  artifacts?: ArtifactLevel;
3429
+ /**
3430
+ * Each attempt's debug data uploaded with the run to a remote store: none (the default), its
3431
+ * gateway requests (`requests.ndjson`, which holds prompts and responses), or all of it
3432
+ * (`debug.log` and `trace.ndjson` too). Rescanned for secrets first; `local()` takes `'none'` only.
3433
+ */
3434
+ debug?: DebugUpload;
3435
+ };
3436
+ type DebugUpload = 'none' | 'requests' | 'all';
3437
+ /** `HistoryConfig` as data: the store is `local` or `s3`. */
3438
+ type HistoryConfigData = Omit<HistoryConfig, 'store'> & {
3439
+ /** Where recorded runs are kept: `{ kind: local }` (the default) or `{ kind: s3, … }`. */
3440
+ store?: HistoryStoreDescriptor;
3097
3441
  };
3098
3442
  /** Run-level caps. `attempts` is checked against the plan; `duration` while running. */
3099
3443
  type Budget = {
@@ -3116,8 +3460,8 @@ type ProfileDefaults = {
3116
3460
  gate?: GatePolicy;
3117
3461
  /** Variables and secrets for the sandbox, merged by name over the config's. */
3118
3462
  env?: EnvVars;
3119
- /** Where the sandbox may connect; replaces the config's policy. */
3120
- egress?: EgressPolicy;
3463
+ /** Where the sandbox may connect; replaces the config's policy, per phase. */
3464
+ egress?: Egress;
3121
3465
  /** Defaults for `c.judge()`, merged field by field over the config's. */
3122
3466
  judge?: ConfigJudgeOptions;
3123
3467
  /** Per attempt. */
@@ -3158,6 +3502,11 @@ type Profile = {
3158
3502
  budget?: Budget;
3159
3503
  /** When this profile's runs are copied into the history folder. */
3160
3504
  record?: RecordMode;
3505
+ /**
3506
+ * The history this profile records to and every history command reads with `--profile`, merged
3507
+ * over the config's `history` key by key.
3508
+ */
3509
+ history?: HistoryConfig;
3161
3510
  /** Tag filters for this profile, like `--tag` (`'!slow'` excludes). */
3162
3511
  tags?: string[];
3163
3512
  /** Replaces the config's `gateway` list for this profile's runs. */
@@ -3292,8 +3641,8 @@ type ReevalConfig = {
3292
3641
  gate?: GatePolicy;
3293
3642
  /** Variables and secrets for the sandbox, merged by name down the cascade. */
3294
3643
  env?: EnvVars;
3295
- /** Where the sandbox may connect; default `'proxy'`. */
3296
- egress?: EgressPolicy;
3644
+ /** Where the sandbox may connect; default `'proxy'` in both phases. */
3645
+ egress?: Egress;
3297
3646
  /** Defaults for `c.judge()`, merged field by field down the cascade. */
3298
3647
  judge?: ConfigJudgeOptions;
3299
3648
  /** Per attempt; default `'15m'`. */
@@ -3314,7 +3663,7 @@ type ReevalConfig = {
3314
3663
  concurrency?: number;
3315
3664
  /** When runs are copied into the history folder: `reeval record` only, in CI, or on every run. */
3316
3665
  record?: RecordMode;
3317
- /** The history folder, and which attempts keep their artifacts there. */
3666
+ /** Where recorded runs are kept, and which attempts keep their artifacts there. */
3318
3667
  history?: HistoryConfig;
3319
3668
  /** Caps on a run; a profile's `budget` is merged over it. */
3320
3669
  budget?: Budget;
@@ -3325,7 +3674,7 @@ type ReevalConfig = {
3325
3674
  ci?: string;
3326
3675
  local?: string;
3327
3676
  };
3328
- /** `unbrokered: 'error'` refuses Docker secrets without `hosts`, and credential-like MCP headers written as plain strings, instead of warning (default `'warn'`). */
3677
+ /** `unbrokered: 'error'` refuses Docker secrets without `hosts`, and credential-like MCP headers and `env` values written as plain strings, instead of warning (default `'warn'`). */
3329
3678
  secrets?: {
3330
3679
  unbrokered?: 'warn' | 'error';
3331
3680
  };
@@ -3382,15 +3731,26 @@ type ResourceFailureRecord = {
3382
3731
  cause: 'oom';
3383
3732
  kills: number;
3384
3733
  };
3734
+ /**
3735
+ * The model provider ended a turn, not the agent: out of credit, over its quota, rate limited or
3736
+ * overloaded, with the HTTP status of the last model call the egress gateway brokered when that
3737
+ * call failed (`null` when it didn't, as in a stream that failed inside an HTTP 200, or none was).
3738
+ * The attempt is then an infra error, so `--retry-errors` runs it again.
3739
+ */
3740
+ type ProviderStopRecord = {
3741
+ cause: 'credit' | 'quota' | 'rate_limit' | 'overloaded';
3742
+ status: number | null;
3743
+ };
3385
3744
  /**
3386
3745
  * Why an attempt failed; `prepare` when its sandbox couldn't be prepared, `resource` when it ran
3387
- * out of one.
3746
+ * out of one, `provider` when the model provider stopped a turn.
3388
3747
  */
3389
3748
  type AttemptError = {
3390
3749
  kind: ErrorKind;
3391
3750
  message: string;
3392
3751
  prepare?: PrepareFailureRecord;
3393
3752
  resource?: ResourceFailureRecord;
3753
+ provider?: ProviderStopRecord;
3394
3754
  };
3395
3755
  /**
3396
3756
  * One start of a stdio MCP server during a turn (`turns[].mcpStartup`, in start order), timed by
@@ -3527,8 +3887,16 @@ type EnvironmentRecord = {
3527
3887
  brokered: string[];
3528
3888
  /** Optional secrets whose host variable was unset, so they weren't injected. */
3529
3889
  unset: string[];
3530
- /** The policy the attempt ran under, from the options and the environment; in the digest unless `proxy`. */
3890
+ /**
3891
+ * The policy the attempt's agent phase ran under (the harness's start, its turns and the checks),
3892
+ * from the options and the environment; in the digest unless `proxy`.
3893
+ */
3531
3894
  egress: EgressPolicy;
3895
+ /**
3896
+ * The policy the setup phase (files and setup steps) ran under, when it wasn't the agent's; in
3897
+ * the digest when set.
3898
+ */
3899
+ setupEgress?: EgressPolicy;
3532
3900
  /**
3533
3901
  * This records the Git repository the workspace started as: the repo as written (a local one
3534
3902
  * relative to the project root), the ref and path asked for (null when unset), the commit the
@@ -3553,11 +3921,21 @@ type EnvironmentRecord = {
3553
3921
  * contains only unexpected refusals. A request that carries the harness's brokered key off the
3554
3922
  * endpoints reeval brokers it for is refused on a host the policy let through: it's in `blocked`
3555
3923
  * once per method and path, its `reason` names them, and `connections` counts its requests. Past
3556
- * the first 20 pairs, the rest share one entry per host.
3924
+ * the first 20 pairs, the rest share one entry per host. With a setup policy of its own, every
3925
+ * entry names the `phase` it happened in.
3557
3926
  */
3558
3927
  type EgressRecord = {
3928
+ /** The agent phase's policy: the harness's start, its turns and the checks. */
3559
3929
  policy: EgressPolicy;
3560
3930
  allowed: string[] | null;
3931
+ /**
3932
+ * The setup phase's policy (files and setup steps) and the hosts it allowed, when it wasn't the
3933
+ * agent's.
3934
+ */
3935
+ setup?: {
3936
+ policy: EgressPolicy;
3937
+ allowed: string[] | null;
3938
+ };
3561
3939
  recorded: boolean;
3562
3940
  /**
3563
3941
  * True when traffic could leave around the gateway (`allow-all` in a container that kept its
@@ -3571,23 +3949,29 @@ type EgressRecord = {
3571
3949
  connections: number;
3572
3950
  sent: number;
3573
3951
  received: number;
3952
+ /** With a setup policy of its own: the phase the connections were made in. */
3953
+ phase?: 'setup' | 'agent';
3574
3954
  }>;
3575
3955
  /**
3576
3956
  * Hosts the policy refused, and requests that carry the harness's brokered key off the endpoints
3577
3957
  * reeval brokers it for (one entry per method and path), each with why. A name that didn't
3578
- * resolve is in `unresolved` instead.
3958
+ * resolve is in `unresolved` instead. A tunnel setup opened to a host the agent's policy refuses
3959
+ * is closed as the agent phase starts, and is here with `closed: true`.
3579
3960
  */
3580
3961
  blocked: Array<{
3581
3962
  host: string;
3582
3963
  port: number;
3583
3964
  connections: number;
3584
3965
  reason: string;
3966
+ phase?: 'setup' | 'agent';
3967
+ closed?: true;
3585
3968
  }>;
3586
3969
  expectedBlocked: Array<{
3587
3970
  host: string;
3588
3971
  port: number;
3589
3972
  connections: number;
3590
3973
  reason: string;
3974
+ phase?: 'setup' | 'agent';
3591
3975
  }>;
3592
3976
  /**
3593
3977
  * Hosts the policy let through whose names didn't resolve (`metadata.google.internal` outside
@@ -3600,6 +3984,7 @@ type EgressRecord = {
3600
3984
  port: number;
3601
3985
  connections: number;
3602
3986
  reason: string;
3987
+ phase?: 'setup' | 'agent';
3603
3988
  }>;
3604
3989
  };
3605
3990
  /** The image an attempt's container started from, and how reeval made it. */
@@ -3633,11 +4018,98 @@ type VmImage = {
3633
4018
  /** The OCI manifest digest (`sha256:…`) Tart had for it; null for a local VM, which has none. */
3634
4019
  digest: string | null;
3635
4020
  };
3636
- /** A harness session start that timed out and was started again. */
4021
+ /**
4022
+ * What an attempt's sandbox gave it and what it used, measured from when the sandbox started
4023
+ * until teardown. A value the sandbox can't measure is left out, never estimated.
4024
+ */
4025
+ type AttemptResources = {
4026
+ /**
4027
+ * `container`: the container's own cgroup, so only the attempt's processes count (Docker,
4028
+ * Podman). `machine`: the whole VM, its system processes included (`boat()`, `tart()`).
4029
+ */
4030
+ scope: 'container' | 'machine';
4031
+ /**
4032
+ * The VM ran other attempts at the same time (`boat({ attemptsPerMachine })` above 1), so its
4033
+ * whole-VM usage isn't this attempt's: `usage` is left out.
4034
+ */
4035
+ shared?: true;
4036
+ /** What the attempt ran on. */
4037
+ machine: {
4038
+ /** The sandbox's provider: `docker`, `podman`, `boat` or `tart`. */
4039
+ sandbox: string;
4040
+ /** The provider's machine size, where it has sizes: `boat({ type })`. */
4041
+ size?: string;
4042
+ /**
4043
+ * CPUs the attempt could use: the container's CPU quota, else the CPUs the VM has (a macOS
4044
+ * VM's `sysctl hw.ncpu`).
4045
+ */
4046
+ vcpus?: number;
4047
+ /**
4048
+ * Memory the attempt could use, in bytes: the container's limit, else the VM's total (a
4049
+ * macOS VM's `sysctl hw.memsize`).
4050
+ */
4051
+ memoryBytes?: number;
4052
+ };
4053
+ /** What it used; absent when the sandbox gave no readings (a macOS VM has no `/proc`). */
4054
+ usage?: {
4055
+ /** Milliseconds between the first reading and the last. */
4056
+ durationMs: number;
4057
+ /** Milliseconds between readings while the attempt ran. */
4058
+ sampleMs: number;
4059
+ /** CPU time over `durationMs`, in seconds: user and system, every CPU added up. */
4060
+ cpuSeconds: number;
4061
+ /** `cpuSeconds` over `durationMs`: how many CPUs it kept busy on average. */
4062
+ avgCores: number;
4063
+ /** The most CPUs it kept busy between two readings; absent with fewer than two. */
4064
+ peakCores?: number;
4065
+ /**
4066
+ * The 95th percentile of the CPUs it kept busy between two readings, a second apart: what it
4067
+ * needed but for its busiest twentieth of the time. Absent with fewer than two readings.
4068
+ */
4069
+ p95Cores?: number;
4070
+ /**
4071
+ * The highest memory reading, in bytes, page cache the kernel can reclaim left out: the
4072
+ * cgroup's `memory.current` less its `inactive_file`, or the VM's `MemTotal` less
4073
+ * `MemAvailable`. Sampled, so a spike between readings is missed.
4074
+ */
4075
+ peakMemoryBytes?: number;
4076
+ /**
4077
+ * cgroup v2's `memory.peak`, in bytes: the highest `memory.current` the kernel saw since the
4078
+ * container started, page cache included, so no spike is missed but it reads higher than
4079
+ * `peakMemoryBytes`. `container` scope on Linux 5.19 or later only.
4080
+ */
4081
+ cgroupPeakMemoryBytes?: number;
4082
+ /**
4083
+ * How often the container's CPU quota held it back over `durationMs`, from cgroup v2's
4084
+ * `cpu.stat`: `periods` is `nr_periods` (quota periods it ran in), `throttledPeriods`
4085
+ * `nr_throttled` (those it used up its quota in and waited out) and `throttledSeconds`
4086
+ * `throttled_usec`. `container` scope only.
4087
+ */
4088
+ throttling?: {
4089
+ periods: number;
4090
+ throttledPeriods: number;
4091
+ throttledSeconds: number;
4092
+ };
4093
+ /** Bytes the VM's disks read over `durationMs`; `machine` scope only. */
4094
+ diskReadBytes?: number;
4095
+ /** Bytes the VM's disks wrote over `durationMs`; `machine` scope only. */
4096
+ diskWriteBytes?: number;
4097
+ /**
4098
+ * The sandbox went before the last reading (a timeout or cancel): the values stop at the last
4099
+ * sample.
4100
+ */
4101
+ partial?: true;
4102
+ };
4103
+ };
4104
+ /** A harness session start that timed out, or failed on a transient error, and was started again. */
3637
4105
  type StartRetry = {
3638
- /** `<harness kind>: session did not become ready within <startTimeout>`. */
4106
+ /**
4107
+ * `<harness kind>: session did not become ready within <startTimeout>`, or the transient error
4108
+ * the start failed on, such as `Cursor harness failed to start: Cursor's model catalog request
4109
+ * failed: HTTP 502 (tried 3 times)`.
4110
+ */
3639
4111
  error: string;
3640
- /** How long it waited before giving up on that start. */
4112
+ /** How long it waited before giving up on that start, or until that start failed. */
3641
4113
  waitedMs: number;
3642
4114
  };
3643
4115
  /**
@@ -3800,5 +4272,5 @@ type ResolvedGateway = {
3800
4272
  why: string;
3801
4273
  }>;
3802
4274
  };
3803
- export { EvalFn as $, directional as $n, Secret as $t, CellRole as A, HistoryEntry as An, JudgeState as At, Diff as B, EFFORT_MIN_RUNS as Bn, PinnableKey as Bt, CaptureUseAction as C, FamilyLift as Cn, JSONValue as Ct, CaseFn as D, FamilyVerdict as Dn, JudgeInputs as Dt, CaptureWindow as E, FamilyRecord as En, JudgeFailure as Et, CompareCut as F, StackSummary as Fn, OpenRouterGatewayDescriptor as Ft, EgressPolicy as G, Spread as Gn, ProfileLimit as Gt, DirectoryOptions as H, EffortMetric as Hn, PluginSource as Ht, CompareOptions as I, UpstreamCount as In, Outcome as It, Environment as J, bothUnderATenth as Jn, QuestionOptions as Jt, EgressRecord as K, TOKENS_UNREPORTED as Kn, Question as Kt, ConfigHelpers as L, CellEffort as Ln, PairFactor as Lt, ClaudeCodeHarnessDescriptor as M, Lift as Mn, McpServer as Mt, CodexHarnessDescriptor as N, RunInvocation as Nn, Observation as Nt, CaseOptions as O, FileRef as On, JudgeModel as Ot, CompareAnswer as P, RunRecord as Pn, OpenCodeHarnessDescriptor as Pt, ErrorKind as Q, contrastEffort as Qn, SandboxWarning as Qt, ConfigJudgeOptions as R, DIRECTIONAL_MIN as Rn, PairVote as Rt, CaptureUse as S, EvalEntry as Sn, HostFile as St, CaptureWait as T, FamilyRate as Tn, JudgeCallOptions as Tt, DockerSandboxDescriptor as U, EffortSample as Un, Profile as Ut, DirectGatewayDescriptor as V, EffortContrast as Vn, PluginPart as Vt, Duration as W, EffortVerdict as Wn, ProfileDefaults as Wt, EnvironmentInput as X, cellEffort as Xn, ResolvedGateway as Xt, EnvironmentHelpers as Y, cellContrast as Yn, ReevalConfig as Yt, EnvironmentSpec as Z, cellSamples as Zn, SandboxDescriptor as Zt, CaptureInstruction as _, WebEventType as _n, HarnessData as _t, AttemptContext as a, TartSandboxDescriptor as an, effortVerdict as ar, FakeAnswer as at, CapturePage as b, CellRecord as bn, HarnessOption as bt, BlindPair as c, Turn as cn, formatRatio as cr, FakeSandboxDescriptor as ct, BoatSandboxDescriptor as d, VercelGatewayDescriptor as dn, FxHarnessDescriptor as dt, SecretTemplate as en, effortChangeLabel as er, EvalOptions as et, Budget as f, Verdict as fn, GatewayDescriptor as ft, CaptureFailedRequest as g, Wait as gn, GroupOptions as gt, CaptureConsoleMessage as h, Viewport as hn, GrokBuildHarnessDescriptor as ht, AttemptAnswer as i, StatusRange as in, effortSummary as ir, FakeAnalysisModel as it, ClaudeCodeEffort as j, JudgmentRecord as jn, JudgeSubject as jt, Cell as k, HistoryCell as kn, JudgeOptions as kt, BoatApiKey as l, TurnData as ln, formatSeconds as lr, FakeScriptStep as lt, CaptureCall as m, VertexPublisher as mn, GooseHarnessDescriptor as mt, AnalysisConfig as n, SetupStep as nn, effortOf as nr, ExecResult as nt, BackgroundHandle as o, TokenUsage as on, filesChanged as or, FakeHarnessDescriptor as ot, Capture as p, VertexGatewayDescriptor as pn, GatewayName as pt, EnvVars as q, VERDICT_TEXT as qn, QuestionInputs as qt, AntigravityHarnessDescriptor as r, SkillSource as rn, effortSample as rr, ExecRetry as rt, BlindAttempt as s, TranscriptEvent as sn, formatCount as sr, FakeJudgeModel as st, AgentHarnessDescriptor as t, ServeHandle as tn, effortLabel as tr, EvalOptionsData as tt, BoatMachineType as u, TurnStatus as un, tokenValues as ur, FakeVote as ut, CaptureKey as v, ArtifactRef as vn, HarnessDescriptor as vt, CaptureUseScript as w, FamilyLiftSummary as wn, JudgeArtifactRef as wt, CapturePageFile as x, CheckTally as xn, HistoryConfig as xt, CaptureOptions as y, AttemptRecord as yn, HarnessEdit as yt, CursorHarnessDescriptor as z, EFFORT_ALPHA as zn, PiHarnessDescriptor as zt };
3804
- //# sourceMappingURL=types-DBwFsGem.d.ts.map
4275
+ export { EnvironmentInput as $, EffortSample as $n, Question as $t, CellRole as A, CheckTally as An, JudgeArtifactRef as At, CostUsd as B, JudgmentRecord as Bn, Observation as Bt, CaptureUseAction as C, VertexPublisher as Cn, HarnessEdit as Ct, CaseFn as D, ArtifactRef as Dn, HistoryStoreDescriptor as Dt, CaptureWindow as E, WebEventType as En, HistoryConfigData as Et, CompareCut as F, FamilyRecord as Fn, JudgeOptions as Ft, DirectoryOptions as G, StackSummary as Gn, PairVote as Gt, DebugUpload as H, RunDebug as Hn, OpenRouterGatewayDescriptor as Ht, CompareOptions as I, FamilyVerdict as In, JudgeState as It, EgressPolicy as J, DIRECTIONAL_MIN as Jn, PluginPart as Jt, DockerSandboxDescriptor as K, UpstreamCount as Kn, PiHarnessDescriptor as Kt, ConfigHelpers as L, FileRef as Ln, JudgeSubject as Lt, ClaudeCodeHarnessDescriptor as M, FamilyLift as Mn, JudgeFailure as Mt, CodexHarnessDescriptor as N, FamilyLiftSummary as Nn, JudgeInputs as Nt, CaseOptions as O, AttemptRecord as On, HostFile as Ot, CompareAnswer as P, FamilyRate as Pn, JudgeModel as Pt, EnvironmentHelpers as Q, EffortMetric as Qn, ProfileLimit as Qt, ConfigJudgeOptions as R, HistoryCell as Rn, LocalStoreDescriptor as Rt, CaptureUse as S, VertexGatewayDescriptor as Sn, tokenValues as Sr, HarnessDescriptor as St, CaptureWait as T, Wait as Tn, HistoryConfig as Tt, Diff as U, RunInvocation as Un, Outcome as Ut, CursorHarnessDescriptor as V, Lift as Vn, OpenCodeHarnessDescriptor as Vt, DirectGatewayDescriptor as W, RunRecord as Wn, PairFactor as Wt, EnvVars as X, EFFORT_MIN_RUNS as Xn, Profile as Xt, EgressRecord as Y, EFFORT_ALPHA as Yn, PluginSource as Yt, Environment as Z, EffortContrast as Zn, ProfileDefaults as Zt, CaptureInstruction as _, Turn as _n, formatCount as _r, GatewayName as _t, AttemptContext as a, S3StoreEnv as an, cellContrast as ar, ExecEnv as at, CapturePage as b, VercelGatewayDescriptor as bn, splitLabel as br, GroupOptions as bt, BlindPair as c, Secret as cn, contrastEffort as cr, FakeAnalysisModel as ct, BoatSandboxDescriptor as d, SetupStep as dn, effortLabel as dr, FakeJudgeModel as dt, QuestionInputs as en, EffortVerdict as er, EnvironmentSpec as et, Budget as f, SkillSource as fn, effortOf as fr, FakeSandboxDescriptor as ft, CaptureFailedRequest as g, TranscriptEvent as gn, filesChanged as gr, GatewayDescriptor as gt, CaptureConsoleMessage as h, TokenUsage as hn, effortVerdict as hr, FxHarnessDescriptor as ht, AttemptAnswer as i, S3StoreDescriptor as in, bothUnderATenth as ir, EvalOptionsData as it, ClaudeCodeEffort as j, EvalEntry as jn, JudgeCallOptions as jt, Cell as k, CellRecord as kn, JSONValue as kt, BoatApiKey as l, SecretTemplate as ln, directional as lr, FakeAnswer as lt, CaptureCall as m, TartSandboxDescriptor as mn, effortSummary as mr, FakeVote as mt, AnalysisConfig as n, ReevalConfig as nn, TOKENS_UNREPORTED as nr, EvalFn as nt, BackgroundHandle as o, SandboxDescriptor as on, cellEffort as or, ExecResult as ot, Capture as p, StatusRange as pn, effortSample as pr, FakeScriptStep as pt, Duration as q, CellEffort as qn, PinnableKey as qt, AntigravityHarnessDescriptor as r, ResolvedGateway as rn, VERDICT_TEXT as rr, EvalOptions as rt, BlindAttempt as s, SandboxWarning as sn, cellSamples as sr, ExecRetry as st, AgentHarnessDescriptor as t, QuestionOptions as tn, Spread as tr, ErrorKind as tt, BoatMachineType as u, ServeHandle as un, effortChangeLabel as ur, FakeHarnessDescriptor as ut, CaptureKey as v, TurnData as vn, formatRatio as vr, GooseHarnessDescriptor as vt, CaptureUseScript as w, Viewport as wn, HarnessOption as wt, CapturePageFile as x, Verdict as xn, splitOf as xr, HarnessData as xt, CaptureOptions as y, TurnStatus as yn, formatSeconds as yr, GrokBuildHarnessDescriptor as yt, CostTotal as z, HistoryEntry as zn, McpServer as zt };
4276
+ //# sourceMappingURL=types-DKK8jj--.d.ts.map