reeval 0.20.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (481) hide show
  1. package/CHANGELOG.md +841 -0
  2. package/README.md +15 -15
  3. package/assets/gateway-models.mjs +65 -31
  4. package/dist/{adapter-C60z3wLq.js → adapter-Di9H2etg.js} +8 -49
  5. package/dist/adapter-Di9H2etg.js.map +1 -0
  6. package/dist/{agent-BmsMXZXx.js → agent-B-6GFDqA.js} +185 -38
  7. package/dist/agent-B-6GFDqA.js.map +1 -0
  8. package/dist/{agents-DXuFxtGC.js → agents-KBTYhSSh.js} +385 -116
  9. package/dist/agents-KBTYhSSh.js.map +1 -0
  10. package/dist/{app-files-DT5mJyV4.js → app-files-Cmh9FGr5.js} +3 -3
  11. package/dist/app-files-Cmh9FGr5.js.map +1 -0
  12. package/dist/{assemble-CE4pHKIa.js → assemble-CBMRIhso.js} +1 -1
  13. package/dist/{assemble-DM2PNU4R.js → assemble-DNVBTBp_.js} +172 -18
  14. package/dist/assemble-DNVBTBp_.js.map +1 -0
  15. package/dist/attach-EbDGKGTO.js +152 -0
  16. package/dist/attach-EbDGKGTO.js.map +1 -0
  17. package/dist/{attempt-CAdlwlNu.js → attempt-_dUsblVx.js} +2 -2
  18. package/dist/{attempt-CAdlwlNu.js.map → attempt-_dUsblVx.js.map} +1 -1
  19. package/dist/{attempt-host-7VviAJrl.js → attempt-host-DQvynDHk.js} +17 -8
  20. package/dist/attempt-host-DQvynDHk.js.map +1 -0
  21. package/dist/boat-Bzz8L-KQ.js +701 -0
  22. package/dist/boat-Bzz8L-KQ.js.map +1 -0
  23. package/dist/{boat-BjLqFJIZ.js → boat-Dek8ftIt.js} +20 -8
  24. package/dist/boat-Dek8ftIt.js.map +1 -0
  25. package/dist/bridge/index.mjs +6 -1
  26. package/dist/bundle-DZstHDeW.js +718 -0
  27. package/dist/bundle-DZstHDeW.js.map +1 -0
  28. package/dist/{capture-Ccf0Hmyx.js → capture-BkbPsfej.js} +61 -32
  29. package/dist/capture-BkbPsfej.js.map +1 -0
  30. package/dist/capture-CUAvk9KM.js +1 -0
  31. package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
  32. package/dist/cases-CM-MQRQp.js.map +1 -0
  33. package/dist/cases-FvhbM8M5.js +1 -0
  34. package/dist/{cli-DTuLSlmW.js → cli-BgBBuxxJ.js} +412 -1044
  35. package/dist/cli-BgBBuxxJ.js.map +1 -0
  36. package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-B5L9n-DV.js} +199 -35
  37. package/dist/cli-spec-B5L9n-DV.js.map +1 -0
  38. package/dist/cli.js +2 -2
  39. package/dist/client-DMEYR3MA.js +807 -0
  40. package/dist/client-DMEYR3MA.js.map +1 -0
  41. package/dist/{command-C2C7P5qo.js → command-BKb8eN_X.js} +10 -10
  42. package/dist/{command-C2C7P5qo.js.map → command-BKb8eN_X.js.map} +1 -1
  43. package/dist/{command-DSL7I-SI.js → command-D_w_kDpI.js} +25 -21
  44. package/dist/command-D_w_kDpI.js.map +1 -0
  45. package/dist/{command-Ff0T-1Hi.js → command-Dwlh0Vhu.js} +17 -16
  46. package/dist/command-Dwlh0Vhu.js.map +1 -0
  47. package/dist/{command-D8oMB2fa.js → command-loXH2jHr.js} +16 -16
  48. package/dist/{command-D8oMB2fa.js.map → command-loXH2jHr.js.map} +1 -1
  49. package/dist/{commands-xAOi_VEk.js → commands-Cuf57OYb.js} +174 -40
  50. package/dist/commands-Cuf57OYb.js.map +1 -0
  51. package/dist/{commands-Bk9NG1s_.js → commands-Dqli45sR.js} +1 -1
  52. package/dist/{config-SvK8pjEK.js → config-C0RKt00M.js} +288 -106
  53. package/dist/config-C0RKt00M.js.map +1 -0
  54. package/dist/{context-BdgCao9Q.js → context-DrEAtUyg.js} +2 -2
  55. package/dist/{context-BdgCao9Q.js.map → context-DrEAtUyg.js.map} +1 -1
  56. package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
  57. package/dist/cost-C21gGrju.js.map +1 -0
  58. package/dist/cost-IM80LyGD.js +1 -0
  59. package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-CRsFxorF.js} +4 -4
  60. package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-CRsFxorF.js.map} +1 -1
  61. package/dist/db-3o8s2EyD.js +2 -0
  62. package/dist/{db-BzOJ4Js4.js → db-ErnJ-gMD.js} +168 -248
  63. package/dist/db-ErnJ-gMD.js.map +1 -0
  64. package/dist/debug-CIXzB4wK.js.map +1 -1
  65. package/dist/{default-effort-BDKIxJyC.js → default-effort-D70cTG00.js} +4 -4
  66. package/dist/{default-effort-BDKIxJyC.js.map → default-effort-D70cTG00.js.map} +1 -1
  67. package/dist/{default-effort-BBDPbJ4D.js → default-effort-DHzj84dD.js} +1 -1
  68. package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
  69. package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
  70. package/dist/{docker-DFrjoNs-.js → docker-CFWzZOEk.js} +4 -4
  71. package/dist/{docker-DFrjoNs-.js.map → docker-CFWzZOEk.js.map} +1 -1
  72. package/dist/docker-DrMnpL84.js +1 -0
  73. package/dist/{doctor-DWK4ttex.js → doctor-B7utCSdh.js} +121 -100
  74. package/dist/doctor-B7utCSdh.js.map +1 -0
  75. package/dist/{environment-cPMCOk-h.js → environment-C0Gg8kyN.js} +49 -11
  76. package/dist/environment-C0Gg8kyN.js.map +1 -0
  77. package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
  78. package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
  79. package/dist/{eval-source-C9b1h8L6.js → eval-source-CsW5VIXS.js} +24 -12
  80. package/dist/eval-source-CsW5VIXS.js.map +1 -0
  81. package/dist/{evaluate-B12Tosvq.js → evaluate-BHQyAszQ.js} +679 -142
  82. package/dist/evaluate-BHQyAszQ.js.map +1 -0
  83. package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DzHq4EsU.d.ts} +2 -2
  84. package/dist/fake-bootstrap-VpisF1WE.js +15 -0
  85. package/dist/fake-bootstrap-VpisF1WE.js.map +1 -0
  86. package/dist/{files-tiNWe8Ih.js → files-BCraBodb.js} +3 -3
  87. package/dist/{files-tiNWe8Ih.js.map → files-BCraBodb.js.map} +1 -1
  88. package/dist/files-BjOJw4sn.js +1 -0
  89. package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
  90. package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
  91. package/dist/{files-BOBdC0uI.js → files-BuKWZCzY.js} +6 -6
  92. package/dist/{files-BOBdC0uI.js.map → files-BuKWZCzY.js.map} +1 -1
  93. package/dist/files-CAzVG3Rr.js +1 -0
  94. package/dist/{files-F5EG0WPT.js → files-Cv856jLN.js} +5 -5
  95. package/dist/{files-F5EG0WPT.js.map → files-Cv856jLN.js.map} +1 -1
  96. package/dist/files-D-Lir4aE.js +1 -0
  97. package/dist/files-DRlJ0Xr2.js +1 -0
  98. package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
  99. package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
  100. package/dist/{files-s28a6czq.js → files-qjeqXnfE.js} +143 -20
  101. package/dist/files-qjeqXnfE.js.map +1 -0
  102. package/dist/gateway.d.ts +1 -1
  103. package/dist/{git-BpoqEgDV.js → git-DOrOtAiD.js} +3 -3
  104. package/dist/{git-BpoqEgDV.js.map → git-DOrOtAiD.js.map} +1 -1
  105. package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
  106. package/dist/harness-FcTyyPco.js.map +1 -0
  107. package/dist/harness-install-Dxho1yaU.js +53 -0
  108. package/dist/harness-install-Dxho1yaU.js.map +1 -0
  109. package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
  110. package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
  111. package/dist/harness.d.ts +11 -8
  112. package/dist/harness.js +1 -1
  113. package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
  114. package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
  115. package/dist/{images-D2jFze2v.js → images-Ch1y1t6_.js} +122 -73
  116. package/dist/images-Ch1y1t6_.js.map +1 -0
  117. package/dist/index.d.ts +46 -7
  118. package/dist/index.js +6 -6
  119. package/dist/{init-Bi1Q3lRi.js → init-C2HqCwnt.js} +33 -14
  120. package/dist/init-C2HqCwnt.js.map +1 -0
  121. package/dist/init-CM9xwyre.js +1 -0
  122. package/dist/{ink-list-BpfidQ1h.js → ink-list-BbKXElcQ.js} +54 -15
  123. package/dist/ink-list-BbKXElcQ.js.map +1 -0
  124. package/dist/inputs-5UIDpjz-.js +190 -0
  125. package/dist/inputs-5UIDpjz-.js.map +1 -0
  126. package/dist/inputs-load-C82HOKnb.js +1 -0
  127. package/dist/inputs-load-CGPyigMC.js +95 -0
  128. package/dist/inputs-load-CGPyigMC.js.map +1 -0
  129. package/dist/{invocations-CxGNOlh0.js → invocations-BrZAMuTb.js} +5 -5
  130. package/dist/{invocations-CxGNOlh0.js.map → invocations-BrZAMuTb.js.map} +1 -1
  131. package/dist/jiti-radm1Soi.js +52 -0
  132. package/dist/jiti-radm1Soi.js.map +1 -0
  133. package/dist/{judge-COMXT1Hx.js → judge-DwzqucLS.js} +17 -12
  134. package/dist/{judge-COMXT1Hx.js.map → judge-DwzqucLS.js.map} +1 -1
  135. package/dist/judge.d.ts +3 -1
  136. package/dist/judge.js +1 -1
  137. package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
  138. package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
  139. package/dist/{layout-Cm85tQYk.js → layout-Czh5wOpz.js} +14 -11
  140. package/dist/layout-Czh5wOpz.js.map +1 -0
  141. package/dist/{list-DuThsE28.js → list-B_C0k5ex.js} +33 -19
  142. package/dist/{list-DuThsE28.js.map → list-B_C0k5ex.js.map} +1 -1
  143. package/dist/live-C9Et_BVT.js +1 -0
  144. package/dist/{live-CgYbO4Te.js → live-DGgIrVw0.js} +7 -5
  145. package/dist/live-DGgIrVw0.js.map +1 -0
  146. package/dist/{live-DL0se4Zd.js → live-DgFm0H90.js} +9 -9
  147. package/dist/{live-DL0se4Zd.js.map → live-DgFm0H90.js.map} +1 -1
  148. package/dist/machine-service-2lIv5UWF.js +1 -0
  149. package/dist/machine-service-Ct_1XMMi.js +520 -0
  150. package/dist/machine-service-Ct_1XMMi.js.map +1 -0
  151. package/dist/machines-BnquHcBZ.js +120 -0
  152. package/dist/machines-BnquHcBZ.js.map +1 -0
  153. package/dist/{model-CkNVAxob.js → model-B1tdv9-d.js} +1 -1
  154. package/dist/model-Bcy4Qjmm.js +1 -0
  155. package/dist/{model-V3mMXp6x.js → model-DF77SgZv.js} +6 -6
  156. package/dist/{model-V3mMXp6x.js.map → model-DF77SgZv.js.map} +1 -1
  157. package/dist/{model-CwwkrCb7.js → model-DRa7yv6K.js} +9 -6
  158. package/dist/model-DRa7yv6K.js.map +1 -0
  159. package/dist/{model-cgt5uvu8.js → model-E5X8YBJM.js} +7 -6
  160. package/dist/model-E5X8YBJM.js.map +1 -0
  161. package/dist/model-calls-B0BoQfJ4.js +95 -0
  162. package/dist/model-calls-B0BoQfJ4.js.map +1 -0
  163. package/dist/{node-D7pAU--0.js → node-BjuhqQwa.js} +21 -10
  164. package/dist/node-BjuhqQwa.js.map +1 -0
  165. package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
  166. package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
  167. package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-B1sIOTD2.js} +2 -2
  168. package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-B1sIOTD2.js.map} +1 -1
  169. package/dist/options-C989kuVE.js +1 -0
  170. package/dist/own-reads-CYmBlrHK.js +15 -0
  171. package/dist/own-reads-CYmBlrHK.js.map +1 -0
  172. package/dist/{patch-DAi1nCuH.js → patch-BR8rdl7r.js} +2 -2
  173. package/dist/{patch-DAi1nCuH.js.map → patch-BR8rdl7r.js.map} +1 -1
  174. package/dist/pi-providers-HygmL1y-.js +552 -0
  175. package/dist/pi-providers-HygmL1y-.js.map +1 -0
  176. package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
  177. package/dist/plain-Ctn8oUdp.js.map +1 -0
  178. package/dist/{plan-CsPilzpk.js → plan-BcT4G5jj.js} +104 -15
  179. package/dist/plan-BcT4G5jj.js.map +1 -0
  180. package/dist/{plugins-BfnhazOv.js → plugins-cC0sqoDA.js} +608 -12
  181. package/dist/plugins-cC0sqoDA.js.map +1 -0
  182. package/dist/preflight-BMVDBcux.js +1 -0
  183. package/dist/{preflight-BixaE4BA.js → preflight-C9OtTb7k.js} +4 -4
  184. package/dist/{preflight-BixaE4BA.js.map → preflight-C9OtTb7k.js.map} +1 -1
  185. package/dist/{preflight-GkpgrBxP.js → preflight-DySHXeFJ.js} +9 -10
  186. package/dist/preflight-DySHXeFJ.js.map +1 -0
  187. package/dist/{preflight-D1C-iH0U.js → preflight-zsAn3heN.js} +1 -1
  188. package/dist/{prepare-C--rM_wO.js → prepare-MaRWRhNT.js} +2 -2
  189. package/dist/prepare-nbX1CXXg.js +409 -0
  190. package/dist/prepare-nbX1CXXg.js.map +1 -0
  191. package/dist/prices-B1vm_lPk.js +1 -0
  192. package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
  193. package/dist/prices-BrY_ihLs.js.map +1 -0
  194. package/dist/{provenance-BXyYwUC8.js → provenance-CkykifD9.js} +5 -5
  195. package/dist/{provenance-BXyYwUC8.js.map → provenance-CkykifD9.js.map} +1 -1
  196. package/dist/provider-DHT20Gl8.js +1 -0
  197. package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
  198. package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
  199. package/dist/provider-stop-CxzWW7Ft.js +69 -0
  200. package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
  201. package/dist/providers-BkLGzaSY.js +1 -0
  202. package/dist/{providers-1p4HKYIp.js → providers-bNgfKa6m.js} +224 -49
  203. package/dist/providers-bNgfKa6m.js.map +1 -0
  204. package/dist/query-B2SQddoV.js +1 -0
  205. package/dist/{query-DhhU9ADI.js → query-DhV3a8lM.js} +2 -2
  206. package/dist/{query-DhhU9ADI.js.map → query-DhV3a8lM.js.map} +1 -1
  207. package/dist/{questions-CMAq4pr8.js → questions-Csv64WCb.js} +7 -7
  208. package/dist/{questions-CMAq4pr8.js.map → questions-Csv64WCb.js.map} +1 -1
  209. package/dist/{record-6IUzwP9K.js → record--vHreTLk.js} +11 -5
  210. package/dist/record--vHreTLk.js.map +1 -0
  211. package/dist/{redact-BhjtnjrU.js → redact-t-L3VGy8.js} +106 -2
  212. package/dist/redact-t-L3VGy8.js.map +1 -0
  213. package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
  214. package/dist/remarks-BpdM_2RC.js.map +1 -0
  215. package/dist/{remarks-BNABzemt.js → remarks-CYFOsnz5.js} +46 -11
  216. package/dist/remarks-CYFOsnz5.js.map +1 -0
  217. package/dist/remarks-CleoDlYw.js +1 -0
  218. package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
  219. package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
  220. package/dist/{remote-U7fNt0UP.js → remote-0CHog7Oq.js} +6 -6
  221. package/dist/{remote-U7fNt0UP.js.map → remote-0CHog7Oq.js.map} +1 -1
  222. package/dist/resolve-Bm9ZyFR-.js +1 -0
  223. package/dist/resolve-Bul8lHaa.js +205 -0
  224. package/dist/resolve-Bul8lHaa.js.map +1 -0
  225. package/dist/{resolve-Cx3a9UQ2.js → resolve-DorSHDBs.js} +14 -11
  226. package/dist/resolve-DorSHDBs.js.map +1 -0
  227. package/dist/resources-DqdwihhT.js +62 -0
  228. package/dist/resources-DqdwihhT.js.map +1 -0
  229. package/dist/{results-CQKXmHym.js → results-DhMgRkR0.js} +9 -22
  230. package/dist/results-DhMgRkR0.js.map +1 -0
  231. package/dist/{resume-POpZa1ge.js → resume-C8Ihfy-C.js} +166 -17
  232. package/dist/resume-C8Ihfy-C.js.map +1 -0
  233. package/dist/{retry-CmG809Mp.js → retry-Bfk7pDti.js} +4 -4
  234. package/dist/{retry-CmG809Mp.js.map → retry-Bfk7pDti.js.map} +1 -1
  235. package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
  236. package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
  237. package/dist/routes-Cp4gAjBZ.js +427 -0
  238. package/dist/routes-Cp4gAjBZ.js.map +1 -0
  239. package/dist/{runtime-Xm9oeC7o.js → runtime-CPUH0Vk5.js} +46 -5
  240. package/dist/runtime-CPUH0Vk5.js.map +1 -0
  241. package/dist/sandbox/boat.d.ts +1 -1
  242. package/dist/sandbox/boat.js +1 -1
  243. package/dist/{sandbox-DrlL4yqb.js → sandbox-z8q_4Ywn.js} +3 -3
  244. package/dist/{sandbox-DrlL4yqb.js.map → sandbox-z8q_4Ywn.js.map} +1 -1
  245. package/dist/sandbox.d.ts +1 -1
  246. package/dist/sandbox.js +1 -1
  247. package/dist/{server-C00zd6zl.js → server-BsQZtuF9.js} +9 -9
  248. package/dist/{server-C00zd6zl.js.map → server-BsQZtuF9.js.map} +1 -1
  249. package/dist/server-ZaMzUNmV.js +1 -0
  250. package/dist/services-BG8xuU82.js +39 -0
  251. package/dist/services-BG8xuU82.js.map +1 -0
  252. package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
  253. package/dist/shared-DtZwIAfy.js.map +1 -0
  254. package/dist/{source-DDgr48Gn.js → source-9HWaTEgV.js} +2 -2
  255. package/dist/{source-DDgr48Gn.js.map → source-9HWaTEgV.js.map} +1 -1
  256. package/dist/source-BcBDj-R0.js +1 -0
  257. package/dist/src-B2eHTCnp.js +3 -0
  258. package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
  259. package/dist/src-B5r1d23Y.js.map +1 -0
  260. package/dist/{src-Dw79VLa3.js → src-Bb3k9Kn_.js} +208 -59
  261. package/dist/src-Bb3k9Kn_.js.map +1 -0
  262. package/dist/src-BfOlH_Bu.js +2 -0
  263. package/dist/src-BuGc40IF.js +2 -0
  264. package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
  265. package/dist/src-C-ChdnG6.js.map +1 -0
  266. package/dist/src-C2K5_afU.js +2 -0
  267. package/dist/{src-YvblOrBU.js → src-C3OhA5pw.js} +87 -13
  268. package/dist/src-C3OhA5pw.js.map +1 -0
  269. package/dist/src-CSVJjghq.js +4 -0
  270. package/dist/{src-y0nOeRRE.js → src-CWfy8ct8.js} +1572 -487
  271. package/dist/src-CWfy8ct8.js.map +1 -0
  272. package/dist/src-CcrQQB0_.js +2 -0
  273. package/dist/src-CotUxxQV.js +2 -0
  274. package/dist/src-D-PwBaBB.js +2 -0
  275. package/dist/{src-Cb2MWozp.js → src-DNWiU__5.js} +276 -41
  276. package/dist/src-DNWiU__5.js.map +1 -0
  277. package/dist/{src-KxS91P7L.js → src-DwcTn4JY.js} +79 -11
  278. package/dist/src-DwcTn4JY.js.map +1 -0
  279. package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
  280. package/dist/src-boB6D-5T.js.map +1 -0
  281. package/dist/{src-DvGns8cz.js → src-c3ZQu7Uu.js} +299 -17
  282. package/dist/src-c3ZQu7Uu.js.map +1 -0
  283. package/dist/src-jYqNkT-F.js +1 -0
  284. package/dist/start-Db5VRcRZ.js +660 -0
  285. package/dist/start-Db5VRcRZ.js.map +1 -0
  286. package/dist/state-CKIjZZRK.js +1 -0
  287. package/dist/{state-328JpnHA.js → state-DUYdzNA7.js} +21 -21
  288. package/dist/{state-328JpnHA.js.map → state-DUYdzNA7.js.map} +1 -1
  289. package/dist/store/local.d.ts +8 -0
  290. package/dist/store/local.js +16 -0
  291. package/dist/store/local.js.map +1 -0
  292. package/dist/store/s3.d.ts +10 -0
  293. package/dist/store/s3.js +18 -0
  294. package/dist/store/s3.js.map +1 -0
  295. package/dist/suite-CH1aaAbs.js +2 -0
  296. package/dist/{suite-COglUhPd.js → suite-DjGxmDSY.js} +6 -5
  297. package/dist/{suite-COglUhPd.js.map → suite-DjGxmDSY.js.map} +1 -1
  298. package/dist/{summary-DDy9DfyJ.js → summary-BK5Sa_N0.js} +28 -7
  299. package/dist/summary-BK5Sa_N0.js.map +1 -0
  300. package/dist/sync-DUi_P2UJ.js +559 -0
  301. package/dist/sync-DUi_P2UJ.js.map +1 -0
  302. package/dist/{tarball-CESxtK4X.js → tarball-C8BSTAMk.js} +2 -2
  303. package/dist/{tarball-CESxtK4X.js.map → tarball-C8BSTAMk.js.map} +1 -1
  304. package/dist/tart-BQ_Ap0l4.js +508 -0
  305. package/dist/tart-BQ_Ap0l4.js.map +1 -0
  306. package/dist/tart-CzViW3WZ.js +186 -0
  307. package/dist/tart-CzViW3WZ.js.map +1 -0
  308. package/dist/tart-DmVuVsmX.js +1 -0
  309. package/dist/tart-bake-D1RttYfM.js +148 -0
  310. package/dist/tart-bake-D1RttYfM.js.map +1 -0
  311. package/dist/tart-bake-DjFcsG3N.js +1 -0
  312. package/dist/tart-cli-BOcluha1.js +1 -0
  313. package/dist/tart-cli-gA9xEaQL.js +89 -0
  314. package/dist/tart-cli-gA9xEaQL.js.map +1 -0
  315. package/dist/tart-hosts-CHNrfV6u.js +313 -0
  316. package/dist/tart-hosts-CHNrfV6u.js.map +1 -0
  317. package/dist/tart-hosts-CWse-ASo.js +1 -0
  318. package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CLBjfF9u.js} +64 -6
  319. package/dist/tart-images-CLBjfF9u.js.map +1 -0
  320. package/dist/testing.d.ts +36 -2
  321. package/dist/testing.js +4 -4
  322. package/dist/tool-class-krsmEo44.js +433 -0
  323. package/dist/tool-class-krsmEo44.js.map +1 -0
  324. package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
  325. package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
  326. package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
  327. package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
  328. package/dist/{transcript-CCf07UmU.js → transcript-BXWZiMuC.js} +3 -3
  329. package/dist/{transcript-CCf07UmU.js.map → transcript-BXWZiMuC.js.map} +1 -1
  330. package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
  331. package/dist/types-CbcnJxCy.js.map +1 -0
  332. package/dist/{types-DBwFsGem.d.ts → types-DKK8jj--.d.ts} +517 -45
  333. package/dist/ui-DKYjTVpn.js +1455 -0
  334. package/dist/ui-DKYjTVpn.js.map +1 -0
  335. package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
  336. package/dist/usage-CwIRPPZ9.js.map +1 -0
  337. package/dist/vercel-models-Ci33llXz.js +21 -0
  338. package/dist/vercel-models-Ci33llXz.js.map +1 -0
  339. package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
  340. package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
  341. package/dist/vertex-DXfcXYKn.js +1 -0
  342. package/dist/view/index.d.ts +32 -3
  343. package/dist/view/index.js +8 -8
  344. package/dist/{views-BROacjBv.js → views-B1zVLJsF.js} +6 -6
  345. package/dist/{views-BROacjBv.js.map → views-B1zVLJsF.js.map} +1 -1
  346. package/dist/views-DCjtw1jF.js +2 -0
  347. package/dist/{worker-entry-Smb4bNDc.js → worker-entry-DrAYYt0M.js} +37 -14
  348. package/dist/worker-entry-DrAYYt0M.js.map +1 -0
  349. package/dist/workspace-CnRbV8cM.js +1 -0
  350. package/dist/{workspace-DNDTEk8v.js → workspace-DwTe1zo6.js} +7 -7
  351. package/dist/{workspace-DNDTEk8v.js.map → workspace-DwTe1zo6.js.map} +1 -1
  352. package/docker/Dockerfile +1 -1
  353. package/package.json +15 -1
  354. package/skills/eval-rubric/SKILL.md +10 -0
  355. package/skills/reeval/SKILL.md +370 -0
  356. package/skills/reeval/references/capture.md +48 -0
  357. package/skills/reeval/references/cli.md +162 -0
  358. package/skills/reeval/references/compare.md +180 -0
  359. package/skills/reeval/references/config.md +147 -0
  360. package/skills/reeval/references/environments.md +218 -0
  361. package/skills/reeval/references/evals.md +241 -0
  362. package/skills/reeval/references/mistakes.md +39 -0
  363. package/skills/reeval/references/vocabulary.md +19 -0
  364. package/skills/setup/SKILL.md +219 -0
  365. package/skills/setup/references/templates.md +258 -0
  366. package/dist/adapter-C60z3wLq.js.map +0 -1
  367. package/dist/agent-BmsMXZXx.js.map +0 -1
  368. package/dist/agents-DXuFxtGC.js.map +0 -1
  369. package/dist/app-files-DT5mJyV4.js.map +0 -1
  370. package/dist/assemble-DM2PNU4R.js.map +0 -1
  371. package/dist/attempt-host-7VviAJrl.js.map +0 -1
  372. package/dist/boat-BjLqFJIZ.js.map +0 -1
  373. package/dist/boat-DSNgZt-t.js +0 -322
  374. package/dist/boat-DSNgZt-t.js.map +0 -1
  375. package/dist/capture-BzotD2hs.js +0 -1
  376. package/dist/capture-Ccf0Hmyx.js.map +0 -1
  377. package/dist/cases-Dy0mifal.js +0 -1
  378. package/dist/cases-z9E6DCd5.js.map +0 -1
  379. package/dist/cli-DTuLSlmW.js.map +0 -1
  380. package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
  381. package/dist/client-KM4C5r5_.js +0 -387
  382. package/dist/client-KM4C5r5_.js.map +0 -1
  383. package/dist/command-DSL7I-SI.js.map +0 -1
  384. package/dist/command-Ff0T-1Hi.js.map +0 -1
  385. package/dist/commands-xAOi_VEk.js.map +0 -1
  386. package/dist/config-SvK8pjEK.js.map +0 -1
  387. package/dist/cost-BdRgySQl.js.map +0 -1
  388. package/dist/cost-Cr26Alhq.js +0 -1
  389. package/dist/db-BzOJ4Js4.js.map +0 -1
  390. package/dist/db-CQRZ3URX.js +0 -2
  391. package/dist/docker--mukta3L.js +0 -1
  392. package/dist/docker-DG_uFgn2.js +0 -246
  393. package/dist/docker-DG_uFgn2.js.map +0 -1
  394. package/dist/doctor-DWK4ttex.js.map +0 -1
  395. package/dist/environment-cPMCOk-h.js.map +0 -1
  396. package/dist/eval-source-C9b1h8L6.js.map +0 -1
  397. package/dist/evaluate-B12Tosvq.js.map +0 -1
  398. package/dist/files-C1Rk9FYA.js +0 -1
  399. package/dist/files-DMQ9-Vv5.js +0 -1
  400. package/dist/files-Di0xDbCc.js +0 -1
  401. package/dist/files-G5jf8wBp.js +0 -1
  402. package/dist/files-s28a6czq.js.map +0 -1
  403. package/dist/harness-DXv2gnaT.js.map +0 -1
  404. package/dist/images-D2jFze2v.js.map +0 -1
  405. package/dist/init-Bi1Q3lRi.js.map +0 -1
  406. package/dist/init-CUw65o36.js +0 -1
  407. package/dist/ink-list-BpfidQ1h.js.map +0 -1
  408. package/dist/kinds-Bh2I35id.js +0 -698
  409. package/dist/kinds-Bh2I35id.js.map +0 -1
  410. package/dist/layout-Cm85tQYk.js.map +0 -1
  411. package/dist/live-CgYbO4Te.js.map +0 -1
  412. package/dist/live-DZLmj0iG.js +0 -1
  413. package/dist/matchers-CXLn-1jm.js +0 -148
  414. package/dist/matchers-CXLn-1jm.js.map +0 -1
  415. package/dist/model-Cv4GuTsZ.js +0 -1
  416. package/dist/model-CwwkrCb7.js.map +0 -1
  417. package/dist/model-cgt5uvu8.js.map +0 -1
  418. package/dist/node-D7pAU--0.js.map +0 -1
  419. package/dist/options-DPUmk7v_.js +0 -1
  420. package/dist/plain-BYUf_fUV.js.map +0 -1
  421. package/dist/plan-CsPilzpk.js.map +0 -1
  422. package/dist/plugins-BfnhazOv.js.map +0 -1
  423. package/dist/preflight-Bn3ianye.js +0 -1
  424. package/dist/preflight-GkpgrBxP.js.map +0 -1
  425. package/dist/prepare-DQEDXoDp.js +0 -268
  426. package/dist/prepare-DQEDXoDp.js.map +0 -1
  427. package/dist/prices-5m3SzMZ3.js.map +0 -1
  428. package/dist/provider-BnUoz4KB.js +0 -1
  429. package/dist/providers-1p4HKYIp.js.map +0 -1
  430. package/dist/providers-B-ijCktf.js +0 -1
  431. package/dist/query-BYhyf0LR.js +0 -1
  432. package/dist/record-6IUzwP9K.js.map +0 -1
  433. package/dist/redact-BhjtnjrU.js.map +0 -1
  434. package/dist/remarks-BNABzemt.js.map +0 -1
  435. package/dist/remarks-CAvTSq16.js +0 -1
  436. package/dist/remarks-D9Lcd1fF.js.map +0 -1
  437. package/dist/resolve-C14Pmtej.js +0 -1
  438. package/dist/resolve-Cx3a9UQ2.js.map +0 -1
  439. package/dist/results-CQKXmHym.js.map +0 -1
  440. package/dist/resume-POpZa1ge.js.map +0 -1
  441. package/dist/runtime-Xm9oeC7o.js.map +0 -1
  442. package/dist/server-C1UJcpUh.js +0 -1
  443. package/dist/shared-Cfi45Bvy.js.map +0 -1
  444. package/dist/source-DJHmQv_j.js +0 -1
  445. package/dist/src-1dBuXzs4.js +0 -3
  446. package/dist/src-BKpioayC.js +0 -2
  447. package/dist/src-BNQznj4z.js.map +0 -1
  448. package/dist/src-BY1cCaoJ.js +0 -2
  449. package/dist/src-BbZYaQ8b.js +0 -2
  450. package/dist/src-CDFH2vVN.js +0 -2
  451. package/dist/src-CMW5R9Q0.js +0 -1
  452. package/dist/src-CV_7DkPa.js.map +0 -1
  453. package/dist/src-Cb2MWozp.js.map +0 -1
  454. package/dist/src-CrV2WA2i.js +0 -2
  455. package/dist/src-CsFaawcZ.js +0 -2
  456. package/dist/src-DAzaCTcN.js.map +0 -1
  457. package/dist/src-DvGns8cz.js.map +0 -1
  458. package/dist/src-Dw79VLa3.js.map +0 -1
  459. package/dist/src-KxS91P7L.js.map +0 -1
  460. package/dist/src-YvblOrBU.js.map +0 -1
  461. package/dist/src-wKwznPgt.js +0 -2
  462. package/dist/src-y0nOeRRE.js.map +0 -1
  463. package/dist/state-Ca55Gk_O.js +0 -1
  464. package/dist/suite-CKBa6MKt.js +0 -2
  465. package/dist/summary-DDy9DfyJ.js.map +0 -1
  466. package/dist/tart-cli-C-GsbVsQ.js +0 -1
  467. package/dist/tart-cli-CU0nFS-L.js +0 -55
  468. package/dist/tart-cli-CU0nFS-L.js.map +0 -1
  469. package/dist/tart-dWoJvWHc.js +0 -586
  470. package/dist/tart-dWoJvWHc.js.map +0 -1
  471. package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
  472. package/dist/tool-target-DS5d72eH.js +0 -40
  473. package/dist/tool-target-DS5d72eH.js.map +0 -1
  474. package/dist/types-C2mLpph3.js.map +0 -1
  475. package/dist/usage-DJiDnh2D.js.map +0 -1
  476. package/dist/vercel-CGlSBv63.js +0 -344
  477. package/dist/vercel-CGlSBv63.js.map +0 -1
  478. package/dist/vertex-O88TCwgY.js +0 -1
  479. package/dist/views-B0Yt_Q1S.js +0 -2
  480. package/dist/worker-entry-Smb4bNDc.js.map +0 -1
  481. package/dist/workspace-C0dDP75l.js +0 -1
@@ -1,33 +1,35 @@
1
1
  import { n as UsageError } from "./errors-ttMTnIYX.js";
2
- import { a as fieldPath, c as jsonValueSchema, d as refusal, f as tokenUsageSchema, g as parseDuration, h as TIMER_LIMIT, i as durationSchema, l as nonEmpty$1, m as MAX_TIMER_MS, n as dataProblem, o as genericProblem, p as valueAt, s as got, t as closest, u as oneOf } from "./shared-Cfi45Bvy.js";
2
+ import { a as fieldPath, c as jsonValueSchema, d as refusal, f as tokenUsageSchema, g as parseDuration, h as TIMER_LIMIT, i as durationSchema, l as nonEmpty$1, m as MAX_TIMER_MS, n as dataProblem, o as genericProblem, p as valueAt, s as got, t as closest, u as oneOf } from "./shared-DtZwIAfy.js";
3
+ import { n as openCodeEffortBody, r as openCodeEffortRefusal, u as resolveOpenCodeProvider } from "./opencode-providers-ChJTQRmz.js";
4
+ import { u as ownFileURL } from "./root-BzeoyfGd.js";
5
+ import { mt as PACKAGE_HOSTS, pt as HARNESS_HOSTS } from "./src-CWfy8ct8.js";
6
+ import { _ as snapshotProblem, c as gatePolicySchema, g as snapshotOptionSchema, l as gateProblem, o as egressSchema, s as envVarsSchema, t as VAR } from "./plain-Ctn8oUdp.js";
7
+ import { i as fromCwd, l as sha256 } from "./ids-B-JKTCP8.js";
8
+ import { g as redactSecrets, r as badCharacterAt, t as REDACTED, y as redactValue } from "./redact-t-L3VGy8.js";
9
+ import { F as SANDBOX_KINDS, G as suggestedSandbox, I as capabilitiesOf, R as isSandboxKind, U as sandboxHelpers, W as sandboxKind, y as localPluginServers, z as maxConcurrentProblem } from "./plugins-cC0sqoDA.js";
10
+ import { i as idPart } from "./machines-BnquHcBZ.js";
11
+ import { m as vercelBaseUrlProblem, n as isRoutingGateway, t as ROUTING_GATEWAYS } from "./routes-Cp4gAjBZ.js";
12
+ import { c as GATEWAY_NAMES, d as gatewayDescriptorSchema, f as gatewayEntryDataSchema, i as claudeCodeTakesEffort, o as harnessDescriptorSchema, p as gatewayNameSchema, r as claudeCodeModelName, s as harnessProblem, t as FACTORY, u as directOffPin } from "./harness-FcTyyPco.js";
13
+ import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV } from "./vertex-BEWxHoxH.js";
14
+ import { _ as isAgentKind, g as harnessCredentials, i as HARNESS_EXAMPLE, p as directProviderRefusal, r as DEFAULT_HARNESS, s as agentFor, t as AGENTS } from "./agents-KBTYhSSh.js";
3
15
  import { n as isOneOf, t as isObjectLike } from "./object-CeQ8dgUC.js";
4
- import { _ as snapshotProblem, a as egressPolicySchema, c as gatePolicySchema, g as snapshotOptionSchema, l as gateProblem, s as envVarsSchema } from "./plain-BYUf_fUV.js";
5
- import { a as isSandboxKind, d as sandboxKind, f as suggestedSandbox, i as idPart, n as capabilitiesOf, o as maxConcurrentProblem, t as SANDBOX_KINDS, u as sandboxHelpers } from "./kinds-Bh2I35id.js";
6
- import { C as plainCredentialHeaders, D as sandboxOf, E as sameWorkspaceOnly, F as environmentDataSchema, I as sandboxDescriptorSchema, L as traitsSchema, M as unbrokeredSecrets, N as workspaceLabel, O as sandboxPaths, P as workspaceOf, _ as isMovingRef, a as checkEgress, c as environmentDataProblem, h as isEnvironment, j as traitsOf, l as environmentFrom, m as helpersFor, o as checkEnv, r as applied, u as environmentPaths, v as isSecret, w as plannedSecrets, z as sandboxProblem } from "./environment-cPMCOk-h.js";
7
- import { c as GATEWAY_NAMES, d as gatewayEntryDataSchema, f as gatewayNameSchema, i as claudeCodeTakesEffort, l as directOffPin, o as harnessDescriptorSchema, r as claudeCodeModelName, s as harnessProblem, t as FACTORY, u as gatewayDescriptorSchema } from "./harness-DXv2gnaT.js";
8
- import { c as sha256, i as fromCwd } from "./ids-C_advdJA.js";
9
- import { n as openCodeEffortBody, r as openCodeEffortRefusal, u as resolveOpenCodeProvider } from "./opencode-providers-BGiXt8_B.js";
10
- import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV } from "./vertex-Eb2wqahk.js";
11
- import { _ as redactValue, h as redactSecrets, r as badCharacterAt, t as REDACTED } from "./redact-BhjtnjrU.js";
12
- import { a as RECOMMENDED_JUDGE_MODEL, c as judgeCredentials, d as modelKeyOptions, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL } from "./judge-COMXT1Hx.js";
16
+ import { A as sandboxPaths, E as plannedSecrets, F as workspaceLabel, I as workspaceOf, L as environmentDataSchema, N as traitsOf, O as sameWorkspaceOnly, P as unbrokeredSecrets, R as sandboxDescriptorSchema, T as plainCredentialHeaders, V as sandboxProblem, a as checkEgress, c as environmentDataProblem, h as isEnvironment, k as sandboxOf, l as environmentFrom, m as helpersFor, o as checkEnv, r as applied, u as environmentPaths, v as isMovingRef, w as plainCredentialEnv, y as isSecret, z as traitsSchema } from "./environment-C0Gg8kyN.js";
17
+ import { a as RECOMMENDED_JUDGE_MODEL, c as judgeCredentials, d as modelKeyOptions, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL } from "./judge-DwzqucLS.js";
13
18
  import { t as inCI } from "./env-8x1O8I1M.js";
14
- import { u as ownFileURL } from "./root-Co0jhOT5.js";
15
- import { B as HARNESS_HOSTS, V as PACKAGE_HOSTS } from "./src-y0nOeRRE.js";
16
- import { y as localPluginServers } from "./plugins-BfnhazOv.js";
17
- import { a as agentFor, b as isRoutingGateway, f as isAgentKind, n as DEFAULT_HARNESS, r as HARNESS_EXAMPLE, t as AGENTS, u as credentialOf, y as ROUTING_GATEWAYS } from "./agents-DXuFxtGC.js";
18
19
  import { i as pairFactors } from "./pairs-BGb134Vu.js";
19
- import { u as opencodeModelError } from "./harness-C2berMiM.js";
20
- import { r as tartTagWarning } from "./tart-ref-DYZijjrU.js";
20
+ import { u as opencodeModelError } from "./harness-nwnyghJZ.js";
21
21
  import "./stats-C18W4xPX.js";
22
22
  import { n as gateLabel } from "./gate-CgYdaprG.js";
23
- import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-C60z3wLq.js";
24
- import { t as docker } from "./sandbox-DrlL4yqb.js";
23
+ import { t as reevalJiti } from "./jiti-radm1Soi.js";
24
+ import { n as defaultIncludeGlobs } from "./adapter-Di9H2etg.js";
25
+ import { t as docker } from "./sandbox-z8q_4Ywn.js";
26
+ import { i as resolveHistory } from "./resolve-Bul8lHaa.js";
25
27
  import { z } from "zod";
26
28
  import { existsSync, readFileSync } from "node:fs";
27
- import { availableParallelism, totalmem } from "node:os";
28
29
  import { dirname, join, resolve } from "node:path";
29
- import { parseDocument } from "yaml";
30
+ import { availableParallelism, totalmem } from "node:os";
30
31
  import { isIP } from "node:net";
32
+ import { parseDocument } from "yaml";
31
33
  // src/judge/artifacts.ts
32
34
  /** A capture handle: a name plus the file the attempt wrote. */
33
35
  function isCaptureHandle(value) {
@@ -408,12 +410,38 @@ const budgetSchema = z.strictObject({
408
410
  attempts: count().optional(),
409
411
  duration: durationSchema.optional()
410
412
  });
411
- const historyConfigSchema = z.strictObject({
412
- dir: z.string().optional(),
413
+ const localStoreDescriptorSchema = z.strictObject({
414
+ kind: z.literal("local"),
415
+ dir: z.string().optional()
416
+ });
417
+ const s3StoreEnvSchema = z.strictObject({
418
+ accessKeyId: z.string().optional(),
419
+ secretAccessKey: z.string().optional(),
420
+ sessionToken: z.string().optional(),
421
+ endpoint: z.string().optional(),
422
+ bucket: z.string().optional(),
423
+ region: z.string().optional()
424
+ });
425
+ const s3StoreDescriptorSchema = z.strictObject({
426
+ kind: z.literal("s3"),
427
+ bucket: z.string().optional(),
428
+ endpoint: z.string().optional(),
429
+ region: z.string().optional(),
430
+ prefix: z.string().optional(),
431
+ env: s3StoreEnvSchema.optional()
432
+ });
433
+ const historyStoreDescriptorSchema = z.discriminatedUnion("kind", [localStoreDescriptorSchema, s3StoreDescriptorSchema]);
434
+ const historyConfigDataSchema = z.strictObject({
435
+ store: historyStoreDescriptorSchema.optional(),
413
436
  artifacts: z.enum([
414
437
  "all",
415
438
  "failures",
416
439
  "none"
440
+ ]).optional(),
441
+ debug: z.enum([
442
+ "none",
443
+ "requests",
444
+ "all"
417
445
  ]).optional()
418
446
  });
419
447
  const recordMode = z.enum([
@@ -428,7 +456,7 @@ const profileDefaultsDataSchema = z.strictObject({
428
456
  runs: runs().optional(),
429
457
  gate: gatePolicySchema.optional(),
430
458
  env: envVarsSchema.optional(),
431
- egress: egressPolicySchema.optional(),
459
+ egress: egressSchema.optional(),
432
460
  judge: configJudgeDataSchema.optional(),
433
461
  timeout: durationSchema.optional(),
434
462
  capture: captureOptionsSchema.optional(),
@@ -448,7 +476,8 @@ const profileDataSchema = z.strictObject({
448
476
  budget: budgetSchema.optional(),
449
477
  record: recordMode.optional(),
450
478
  tags: z.array(z.string()).optional(),
451
- gateway: z.array(gatewayEntryDataSchema).min(1).optional()
479
+ gateway: z.array(gatewayEntryDataSchema).min(1).optional(),
480
+ history: historyConfigDataSchema.optional()
452
481
  });
453
482
  const fakeVote = z.enum([
454
483
  "A",
@@ -490,7 +519,7 @@ const reevalConfigDataSchema = z.strictObject({
490
519
  runs: runs().optional(),
491
520
  gate: gatePolicySchema.optional(),
492
521
  env: envVarsSchema.optional(),
493
- egress: egressPolicySchema.optional(),
522
+ egress: egressSchema.optional(),
494
523
  judge: configJudgeDataSchema.optional(),
495
524
  timeout: durationSchema.optional(),
496
525
  capture: captureOptionsSchema.optional(),
@@ -498,7 +527,7 @@ const reevalConfigDataSchema = z.strictObject({
498
527
  snapshot: snapshotOptionSchema.optional(),
499
528
  concurrency: count().optional(),
500
529
  record: recordMode.optional(),
501
- history: historyConfigSchema.optional(),
530
+ history: historyConfigDataSchema.optional(),
502
531
  budget: budgetSchema.optional(),
503
532
  profiles: z.record(z.string(), profileDataSchema).optional(),
504
533
  defaultProfile: z.strictObject({
@@ -676,7 +705,8 @@ const transcriptEventSchema = z.discriminatedUnion("type", [
676
705
  removed: nullableNumber,
677
706
  durationMs: nullableNumber,
678
707
  stalled: z.boolean().optional(),
679
- subagent: z.string().optional()
708
+ subagent: z.string().optional(),
709
+ durationInferred: z.literal(true).optional()
680
710
  }),
681
711
  z.strictObject({
682
712
  ...at$1,
@@ -687,7 +717,8 @@ const transcriptEventSchema = z.discriminatedUnion("type", [
687
717
  ok: z.boolean().nullable().optional(),
688
718
  durationMs: nullableNumber,
689
719
  stalled: z.boolean().optional(),
690
- subagent: z.string().optional()
720
+ subagent: z.string().optional(),
721
+ durationInferred: z.literal(true).optional()
691
722
  }),
692
723
  z.strictObject({
693
724
  ...at$1,
@@ -697,7 +728,8 @@ const transcriptEventSchema = z.discriminatedUnion("type", [
697
728
  ok: z.boolean().nullable(),
698
729
  durationMs: nullableNumber,
699
730
  stalled: z.boolean().optional(),
700
- subagent: z.string().optional()
731
+ subagent: z.string().optional(),
732
+ durationInferred: z.literal(true).optional()
701
733
  }),
702
734
  z.strictObject({
703
735
  ...at$1,
@@ -706,10 +738,13 @@ const transcriptEventSchema = z.discriminatedUnion("type", [
706
738
  tool: z.string(),
707
739
  input: z.unknown(),
708
740
  output: z.unknown(),
741
+ outputView: z.enum(["model", "full"]).optional(),
742
+ outputTruncated: z.strictObject({ bytes: z.number() }).optional(),
709
743
  ok: z.boolean().nullable(),
710
744
  durationMs: nullableNumber,
711
745
  stalled: z.boolean().optional(),
712
- subagent: z.string().optional()
746
+ subagent: z.string().optional(),
747
+ durationInferred: z.literal(true).optional()
713
748
  }),
714
749
  z.strictObject({
715
750
  ...at$1,
@@ -720,6 +755,18 @@ const transcriptEventSchema = z.discriminatedUnion("type", [
720
755
  ...at$1,
721
756
  type: z.literal("retry"),
722
757
  message: z.string()
758
+ }),
759
+ z.strictObject({
760
+ ...at$1,
761
+ type: z.literal("model-call"),
762
+ request: z.string(),
763
+ status: nullableNumber,
764
+ durationMs: z.number(),
765
+ firstByteMs: nullableNumber,
766
+ usage: tokenUsageSchema.nullable(),
767
+ model: z.string().nullable(),
768
+ stream: z.literal(true).optional(),
769
+ role: z.enum(["main", "side"]).optional()
723
770
  })
724
771
  ]);
725
772
  const turnDataSchema = z.strictObject({
@@ -743,6 +790,7 @@ const execRetrySchema = z.strictObject({
743
790
  retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional(),
744
791
  interval: z.number().min(0, "must be 0 or more milliseconds").max(MAX_TIMER_MS, `must be ${TIMER_LIMIT}`).optional()
745
792
  });
793
+ const execEnvSchema = z.record(z.string().regex(VAR, "must be a variable name: letters, digits and _, not starting with a digit"), z.string().refine((v) => !v.includes("\0"), "must not hold a NUL character"));
746
794
  /** A background command has no exit for `retries` to wait on. */
747
795
  const noBackgroundRetry = (options, ctx) => {
748
796
  if (!options.background) return;
@@ -757,6 +805,7 @@ const noBackgroundRetry = (options, ctx) => {
757
805
  };
758
806
  const execOptionsSchema = z.strictObject({
759
807
  background: z.boolean().optional(),
808
+ env: execEnvSchema.optional(),
760
809
  ...execRetrySchema.shape
761
810
  }).superRefine(noBackgroundRetry);
762
811
  /** What's wrong with `c.exec()`'s options, or undefined. */
@@ -873,6 +922,7 @@ const matcherSubjectSchema = z.union([
873
922
  z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
874
923
  z.strictObject({
875
924
  exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
925
+ env: execEnvSchema.optional(),
876
926
  result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
877
927
  }),
878
928
  z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
@@ -920,8 +970,9 @@ const attemptCallSchema = z.discriminatedUnion("op", [
920
970
  op: z.literal("exec"),
921
971
  command: nonEmpty.describe("A shell command, run in the workspace."),
922
972
  background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
973
+ env: execEnvSchema.optional(),
923
974
  ...execRetrySchema.shape
924
- }).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
975
+ }).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, env?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
925
976
  z.strictObject({
926
977
  op: z.literal("serve"),
927
978
  command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
@@ -1457,6 +1508,47 @@ const DEFAULT_EGRESS_ALLOW = PACKAGE_HOSTS;
1457
1508
  Object.fromEntries(Object.entries(SANDBOX_KINDS).map(([kind, k]) => [kind, k.capabilities.egress]));
1458
1509
  /** The policy an attempt runs under when nothing sets one. */
1459
1510
  const DEFAULT_EGRESS = "proxy";
1511
+ /** Whether `egress` sets a policy per phase, `{ setup, agent }`, rather than one for both. */
1512
+ const isPhased = (egress) => typeof egress === "object" && !("allow" in egress);
1513
+ /** The policy `egress` sets for `phase`: a plain policy sets both; undefined when it sets none. */
1514
+ const phaseOf = (egress, phase) => egress === void 0 ? void 0 : isPhased(egress) ? egress[phase] : egress;
1515
+ /**
1516
+ * `over` set at the same level as `under`, later: a plain policy replaces both phases,
1517
+ * `{ setup, agent }` only the phases it names.
1518
+ */
1519
+ function layerEgress(under, over) {
1520
+ if (!isPhased(over)) return over;
1521
+ const setup = over.setup ?? phaseOf(under, "setup");
1522
+ const agent = over.agent ?? phaseOf(under, "agent");
1523
+ return {
1524
+ ...setup !== void 0 ? { setup } : {},
1525
+ ...agent !== void 0 ? { agent } : {}
1526
+ };
1527
+ }
1528
+ /** Each phase's policy `egress` sets, the default where it sets none. */
1529
+ const phasePolicies = (egress) => ({
1530
+ setup: phaseOf(egress, "setup") ?? "proxy",
1531
+ agent: phaseOf(egress, "agent") ?? "proxy"
1532
+ });
1533
+ /** `egress` planned with no automatic hosts: what a config-level check can tell of it. */
1534
+ function bareEgress(egress) {
1535
+ const { setup, agent } = phasePolicies(egress);
1536
+ return samePolicy(setup, agent) ? {
1537
+ policy: agent,
1538
+ auto: []
1539
+ } : {
1540
+ policy: agent,
1541
+ auto: [],
1542
+ setup: {
1543
+ policy: setup,
1544
+ auto: []
1545
+ }
1546
+ };
1547
+ }
1548
+ const policyKey = (p) => typeof p === "string" ? p : JSON.stringify([...new Set(p.allow)].sort());
1549
+ const samePolicy = (a, b) => policyKey(a) === policyKey(b);
1550
+ /** The setup phase's planned policy: its own when it differs from the agent's. */
1551
+ const setupEgress = (planned) => planned.setup ?? planned;
1460
1552
  /** Whether the policy allows only some hosts: `deny-all` and allowlists. */
1461
1553
  const isRestricted = (policy) => policy !== "allow-all" && policy !== "proxy";
1462
1554
  /**
@@ -1465,7 +1557,8 @@ const isRestricted = (policy) => policy !== "allow-all" && policy !== "proxy";
1465
1557
  */
1466
1558
  const isDirect = (policy) => policy === "allow-all";
1467
1559
  /**
1468
- * The hosts the harness needs: a real agent's own API (its model's provider's, for OpenCode), or
1560
+ * The hosts the harness needs: a real agent's own API (its model's provider's, for OpenCode, Goose
1561
+ * and pi), or
1469
1562
  * a fake agent's with `layout` (it behaves like that agent). A plain fake agent talks to nothing.
1470
1563
  */
1471
1564
  function harnessHosts(harness, vertex) {
@@ -1481,26 +1574,34 @@ function mcpServerHosts(layers) {
1481
1574
  return [...urls.values()].flatMap((url) => URL.parse(url)?.hostname ?? []);
1482
1575
  }
1483
1576
  /**
1484
- * The policy an attempt runs under, and the hosts a restricted one allows on its own: the
1485
- * harness's, the brokered secrets' `hosts` and the MCP servers'. The cascade is config and
1486
- * profile defaults (`inherited`), then the environment's layers, then groups and the eval (`own`).
1577
+ * The policy each phase of an attempt runs under, and the hosts a restricted one allows on its
1578
+ * own: the harness's, the brokered secrets' `hosts` and the MCP servers'. Each phase cascades on
1579
+ * its own: config and profile defaults (`inherited`), then the environment's layers, then groups
1580
+ * and the eval (`own`). `setup` is set only when the setup phase's policy differs from the agent's.
1487
1581
  */
1488
1582
  function plannedEgress(egress, harness, env, layers, vertex) {
1489
- const fromLayers = layers.findLast((l) => l.spec.egress !== void 0)?.spec.egress;
1490
- const policy = egress.own ?? fromLayers ?? egress.inherited ?? "proxy";
1491
- if (!isRestricted(policy)) return {
1492
- policy,
1493
- auto: []
1583
+ const pick = (phase) => phaseOf(egress.own, phase) ?? layers.map((l) => phaseOf(l.spec.egress, phase)).findLast((p) => p !== void 0) ?? phaseOf(egress.inherited, phase) ?? "proxy";
1584
+ const phase = (policy) => {
1585
+ if (!isRestricted(policy)) return {
1586
+ policy,
1587
+ auto: []
1588
+ };
1589
+ const own = policy === "deny-all" ? [] : policy.allow;
1590
+ const auto = [
1591
+ ...harnessHosts(harness, vertex),
1592
+ ...plannedSecrets(env, layers).flatMap((s) => s.hosts ?? []),
1593
+ ...mcpServerHosts(layers)
1594
+ ];
1595
+ return {
1596
+ policy,
1597
+ auto: [...new Set(auto)].filter((h) => !own.includes(h))
1598
+ };
1494
1599
  };
1495
- const own = policy === "deny-all" ? [] : policy.allow;
1496
- const auto = [
1497
- ...harnessHosts(harness, vertex),
1498
- ...plannedSecrets(env, layers).flatMap((s) => s.hosts ?? []),
1499
- ...mcpServerHosts(layers)
1500
- ];
1501
- return {
1502
- policy,
1503
- auto: [...new Set(auto)].filter((h) => !own.includes(h))
1600
+ const agent = phase(pick("agent"));
1601
+ const setup = pick("setup");
1602
+ return samePolicy(setup, agent.policy) ? agent : {
1603
+ ...agent,
1604
+ setup: phase(setup)
1504
1605
  };
1505
1606
  }
1506
1607
  /** Every host a restricted policy lets through, or null under `proxy` and `allow-all`. */
@@ -1535,19 +1636,24 @@ const LEVELS = [
1535
1636
  "all-or-nothing",
1536
1637
  "allowlist"
1537
1638
  ];
1538
- /** Why the sandbox can't enforce the attempt's policy, or undefined when it can. */
1639
+ /** Why the sandbox can't enforce the attempt's policies, or undefined when it can. */
1539
1640
  function unenforceable(planned, sandbox, where) {
1641
+ if (!planned.setup) return unenforceablePhase(planned, sandbox, where, "egress");
1642
+ if (!capabilitiesOf(sandbox).egress.switches) return `${where} runs under egress ${describeEgress(planned.setup.policy)} for setup and ${describeEgress(planned.policy)} for the agent, but the ${sandbox.kind} sandbox can't change its egress policy during an attempt. Give both phases one policy, or run it in ${suggestedSandbox((c) => c.isolated && c.egress.switches)}.`;
1643
+ return unenforceablePhase(planned.setup, sandbox, where, "egress.setup") ?? unenforceablePhase(planned, sandbox, where, "egress.agent");
1644
+ }
1645
+ function unenforceablePhase(planned, sandbox, where, key) {
1540
1646
  const capabilities = capabilitiesOf(sandbox);
1541
1647
  const can = capabilities.egress.enforce;
1542
- if (planned.policy === "proxy" && capabilities.isolated && can === "none") return `${where} runs under egress 'proxy', where the egress gateway is the only way out, but the ${sandbox.kind} sandbox enforces no egress policy: its traffic that ignores the proxy goes out directly. Set egress: 'allow-all' to run it there, or run it in ${suggestedSandbox((c) => c.isolated && c.egress.enforce !== "none")}.`;
1648
+ if (planned.policy === "proxy" && capabilities.isolated && can === "none") return `${where} runs under ${key} 'proxy', where the egress gateway is the only way out, but the ${sandbox.kind} sandbox enforces no egress policy: its traffic that ignores the proxy goes out directly. Set ${key}: 'allow-all' to run it there, or run it in ${suggestedSandbox((c) => c.isolated && c.egress.enforce !== "none")}.`;
1543
1649
  const needs = egressNeeds(planned);
1544
1650
  if (LEVELS.indexOf(can) >= LEVELS.indexOf(needs)) return void 0;
1545
1651
  const hosts = allowedHosts(planned)?.join(", ");
1546
1652
  const what = needs !== "allowlist" ? "turning egress off" : planned.policy === "deny-all" ? `a host allowlist for its automatic hosts (${hosts})` : `a host allowlist (${hosts})`;
1547
1653
  const enforces = can === "none" ? "enforces no egress policy" : `only enforces ${can === "all-or-nothing" ? "'allow-all' or 'deny-all'" : can}`;
1548
1654
  const enforcing = suggestedSandbox((c) => LEVELS.indexOf(c.egress.enforce) >= LEVELS.indexOf(needs));
1549
- const hint = capabilities.isolated ? "" : ` (its commands run on the host). ${enforcing ? `Run it in ${enforcing}, or set` : "Set"} egress: 'allow-all'`;
1550
- return `${where} runs under egress ${describeEgress(planned.policy)}, which needs ${what}, but the ${sandbox.kind} sandbox ${enforces}${hint}.`;
1655
+ const hint = capabilities.isolated ? "" : ` (its commands run on the host). ${enforcing ? `Run it in ${enforcing}, or set` : "Set"} ${key}: 'allow-all'`;
1656
+ return `${where} runs under ${key} ${describeEgress(planned.policy)}, which needs ${what}, but the ${sandbox.kind} sandbox ${enforces}${hint}.`;
1551
1657
  }
1552
1658
  const describeEgress = (policy) => typeof policy === "string" ? `'${policy}'` : `{ allow: [${policy.allow.map((h) => `'${h}'`).join(", ")}] }`;
1553
1659
  /** A CI runner as the environment names it, which usually runs on a cloud VM; null elsewhere. */
@@ -1563,8 +1669,35 @@ function ciRunner(env) {
1563
1669
  * so anything in it reaches cloud metadata and the host's network around the gateway.
1564
1670
  */
1565
1671
  const directOnCloud = (where) => `egress 'allow-all' on ${where}: anything in the sandbox can reach the cloud metadata service (169.254.169.254, which hands out the machine's credentials), the host's network and its services around the egress gateway, unfiltered and unrecorded`;
1566
- /** `deny-all + auto: api2.cursor.sh, …` or `allow-all` for the plan; '' under `proxy`, the default. */
1567
- function egressLabel({ policy, auto }) {
1672
+ /**
1673
+ * What running attempts with a setup policy of their own (`ids`) side by side on rootless Podman
1674
+ * risks: a container leaving Podman's bridge, as one does at the switch, can cut the others on it
1675
+ * off from the host for seconds.
1676
+ */
1677
+ const rootlessPodmanSwitch = (ids) => `phased egress on rootless Podman: at the switch to the agent's policy an attempt's container leaves Podman's bridge network, which can cut the other containers on it off from the host for seconds, failing their direct connections (allow-all, and setup steps under it). Under it: ${ids.join(", ")}. Run with --concurrency 1, or on docker(), when those connections matter`;
1678
+ /**
1679
+ * `deny-all + auto: api2.cursor.sh, …` or `allow-all` for the plan; '' under `proxy`, the default.
1680
+ * With a setup policy of its own, `setup allow-all · agent deny-all + auto: …`.
1681
+ */
1682
+ function egressLabel(planned) {
1683
+ if (!planned.setup) return phaseLabel(planned);
1684
+ return `setup ${phaseLabel(planned.setup) || "proxy"} · agent ${phaseLabel(planned) || "proxy"}`;
1685
+ }
1686
+ /** {@link egressLabel} without the automatic hosts, which {@link autoHosts} lists. */
1687
+ function ownEgressLabel(planned) {
1688
+ const own = (p) => ({
1689
+ ...p,
1690
+ auto: []
1691
+ });
1692
+ return egressLabel({
1693
+ ...own(planned),
1694
+ ...planned.setup ? { setup: own(planned.setup) } : {}
1695
+ });
1696
+ }
1697
+ /** The hosts reeval allows on top of the policy, in either phase. */
1698
+ const autoHosts = (planned) => [.../* @__PURE__ */ new Set([...planned.setup?.auto ?? [], ...planned.auto])];
1699
+ /** One phase's policy and automatic hosts, or `''` for the default, `proxy`. */
1700
+ function phaseLabel({ policy, auto }) {
1568
1701
  if (policy === "proxy") return "";
1569
1702
  if (policy === "allow-all") return "allow-all";
1570
1703
  const own = policy === "deny-all" ? "deny-all" : policy.allow.length ? `allow ${policy.allow.join(", ")}` : "allow none";
@@ -1921,7 +2054,7 @@ function normalizeCompare(value) {
1921
2054
  }
1922
2055
  /**
1923
2056
  * Runs per cell when no level sets `runs`: a compare with variants runs `COMPARE_RUNS` a side;
1924
- * anything else, a pooled family's cases included, runs once.
2057
+ * anything else runs once, two or more cases with a compare included, whose lift pairs cases.
1925
2058
  */
1926
2059
  const COMPARE_DEFAULT_RUNS = 4;
1927
2060
  function applyLevel(out, options, where, egress = "own") {
@@ -1929,7 +2062,7 @@ function applyLevel(out, options, where, egress = "own") {
1929
2062
  ...out.sources,
1930
2063
  [key]: where
1931
2064
  };
1932
- if (options.egress !== void 0) out.egress[egress] = options.egress;
2065
+ if (options.egress !== void 0) out.egress[egress] = layerEgress(out.egress[egress], options.egress);
1933
2066
  out.harness = applyHarness(out.harness, options.harnesses, where);
1934
2067
  if (options.sandbox !== void 0) out.sandbox = options.sandbox;
1935
2068
  out.environments = applyEnvironments(out.environments, options.environments, where);
@@ -2118,8 +2251,13 @@ function cascade(base, levels, where, options = {}) {
2118
2251
  /** Gives `out` its default runs when no level set them and `--runs` doesn't replace them. */
2119
2252
  function defaultRuns(out, notes, options) {
2120
2253
  if (out.runs !== void 0 || options.runs !== void 0) return;
2121
- if (!out.compare?.variants.length || options.pooled) {
2254
+ if (!out.compare?.variants.length) {
2255
+ out.runs = 1;
2256
+ return;
2257
+ }
2258
+ if (options.pooled) {
2122
2259
  out.runs = 1;
2260
+ notes.push("runs 1 a case (compare with cases: the lift pairs cases)");
2123
2261
  return;
2124
2262
  }
2125
2263
  out.runs = COMPARE_DEFAULT_RUNS;
@@ -2254,8 +2392,7 @@ function environmentWarnings(attempts) {
2254
2392
  }
2255
2393
  if (s.source === "registry" && !s.version) out.add(`skill.registry(${s.id}) has no version: results aren't comparable over time. Pin one (the resolved commit is recorded either way).`);
2256
2394
  }
2257
- const tag = a.sandbox.kind === "tart" ? tartTagWarning(a.sandbox.image) : null;
2258
- if (tag) out.add(tag);
2395
+ for (const warning of sandboxKind(a.sandbox).planWarnings?.(a.sandbox) ?? []) out.add(warning);
2259
2396
  }
2260
2397
  return [...out];
2261
2398
  }
@@ -2302,6 +2439,18 @@ function plainHeaders(attempts) {
2302
2439
  const templateHint = (h) => `read it from a host variable with secret.template, e.g. secret.template('Bearer \${VAR}')${h.https ? "" : " and give the MCP server an https:// URL"}`;
2303
2440
  const plainHeaderWarning = (h) => `${h.label} is a plain string, so it's in the sandbox as its real value. To give the agent only a placeholder, ${templateHint(h)}.`;
2304
2441
  const plainHeaderError = (h, where, configFile) => `${h.label} in ${where} is a plain string, and secrets.unbrokered is 'error' in ${configFile}. To broker it, ${templateHint(h)}.`;
2442
+ /** The attempts' `env` strings that look like credentials, once each. */
2443
+ function plainEnv(env, attempts) {
2444
+ const out = /* @__PURE__ */ new Map();
2445
+ for (const a of attempts) for (const e of plainCredentialEnv(env, a.environment.layers)) out.set(e.label, e);
2446
+ return [...out.values()];
2447
+ }
2448
+ /** A placeholder only works for a value sent to an HTTPS API: the gateway swaps it in nowhere else. */
2449
+ const hostsExample = (e) => `secret.fromEnv('${e.name}', { hosts: ['api.example.com'] })`;
2450
+ const maskHint = (e) => e.plugin ? `write it as "\${${e.name}}" in the plugin's .mcp.json, so reeval reads it from the host variable ${e.name}` : `read it from a host variable with secret.fromEnv('${e.name}'); to give the agent only a placeholder too, add the hosts it's sent to if that's an HTTPS API, e.g. ${hostsExample(e)}`;
2451
+ const plainEnvWarning = (e) => `${e.label} is a plain string that looks like a credential, so it's in the sandbox as its real value. To mask it in everything reeval writes, ${maskHint(e)}.`;
2452
+ const brokerEnvHint = (e) => e.plugin ? `declare the server with mcp.stdio() instead of the plugin's .mcp.json, whose host variables can't set hosts, and read it with ${hostsExample(e)}, naming the HTTPS hosts it's sent to` : `read it from a host variable with ${hostsExample(e)}, naming the HTTPS hosts it's sent to`;
2453
+ const plainEnvError = (e, where, configFile) => `${e.label} in ${where} is a plain string that looks like a credential, and secrets.unbrokered is 'error' in ${configFile}. To broker it, ${brokerEnvHint(e)}.`;
2305
2454
  const levelOf = (where) => where.startsWith("evaluate(") ? "eval" : where.startsWith("group(") ? "group" : where.startsWith("profiles.") ? "profile" : "config";
2306
2455
  /**
2307
2456
  * A warning per privilege an attempt's Docker sandbox adds, naming the level that set the
@@ -2375,11 +2524,20 @@ function evalPlan(r, attempts, origin) {
2375
2524
  ...environmentWarnings(attempts),
2376
2525
  ...skip ? [] : questionWarnings(resolved),
2377
2526
  ...unbrokered(r.resolved.env, attempts).filter(reachesSandbox).map(unbrokeredWarning),
2378
- ...plainHeaders(attempts).map(plainHeaderWarning)
2527
+ ...plainHeaders(attempts).map(plainHeaderWarning),
2528
+ ...plainEnv(r.resolved.env, attempts).map(plainEnvWarning)
2379
2529
  ],
2380
2530
  sandboxWarnings: skip ? [] : sandboxWarnings(attempts, resolved, origin),
2381
2531
  brokered: [...new Set(attempts.flatMap((a) => brokeredIn(r.resolved.env, a)))],
2382
- egress: [...new Set(attempts.map((a) => egressLabel(a.egress)).filter(Boolean))],
2532
+ egress: [...new Map(attempts.map((a) => {
2533
+ const e = {
2534
+ environment: a.environment.name,
2535
+ harness: harnessId(a.harness),
2536
+ policy: ownEgressLabel(a.egress) || "proxy",
2537
+ auto: autoHosts(a.egress)
2538
+ };
2539
+ return [JSON.stringify(e), e];
2540
+ })).values()],
2383
2541
  workspaces: [...new Set(attempts.flatMap((a) => {
2384
2542
  const w = workspaceOf(a.environment.layers);
2385
2543
  return w ? [workspaceLabel(w)] : [];
@@ -2414,6 +2572,7 @@ const GATEWAYS = {
2414
2572
  const fxEffort = fxEffortRefusal(h, "direct()");
2415
2573
  if (fxEffort) return fxEffort;
2416
2574
  if (h.kind === "fx" && h.model.includes("/")) return `fx runs a gateway id such as ${JSON.stringify(h.model)} only through Vercel AI Gateway or OpenRouter`;
2575
+ if (h.kind === "pi" || h.kind === "goose") return directProviderRefusal(h.kind, h.model);
2417
2576
  if (h.kind === "opencode") {
2418
2577
  const resolved = resolveOpenCodeProvider(h.model, process.env);
2419
2578
  if ("error" in resolved) return resolved.error;
@@ -2438,7 +2597,7 @@ const GATEWAYS = {
2438
2597
  /** OpenCode's id for `model` on `gateway`, as that gateway's route writes it. */
2439
2598
  function openCodeModelOn(model, gateway) {
2440
2599
  if (gateway === "vertex") return openCodeVertexId(model);
2441
- return gateway === "vercel" || gateway === "openrouter" ? `${gateway}/${model}` : model;
2600
+ return isRoutingGateway(gateway) ? `${gateway}/${model}` : model;
2442
2601
  }
2443
2602
  /** Why the API behind `gateway` answers OpenCode's `effort` on `model` with a 400, or undefined. */
2444
2603
  function openCodeEffortOn(model, gateway, effort) {
@@ -2471,6 +2630,8 @@ function routingEntry(name) {
2471
2630
  const fxEffort = fxEffortRefusal(h, gateway.name);
2472
2631
  if (fxEffort) return fxEffort;
2473
2632
  if (h.kind !== "fake" && !gateway.routes[h.kind]) return gateway.refusals[h.kind] ?? `reeval doesn't run ${agentName(h)} on ${gateway.name} yet`;
2633
+ const here = h.kind === "fake" ? void 0 : gateway.refusedAt?.[h.kind]?.(process.env);
2634
+ if (here) return here;
2474
2635
  const effort = harnessEffort(h);
2475
2636
  if (effort !== void 0 && h.kind !== "fake" && gateway.unprovenEffort[h.kind]?.(h.model)) return `reeval doesn't send ${agentName(h)}'s effort (${JSON.stringify(effort)}) to ${h.model} through ${gateway.name} yet: nothing checks the model gets it as its own setting, so drop effort or run ${agentName(h)} on another gateway`;
2476
2637
  return h.kind === "opencode" ? openCodeEffortOn(h.model, name, effort) : void 0;
@@ -2563,6 +2724,12 @@ const gatewayScope = (config, profile) => ({ gateways: profile?.gateway ?? confi
2563
2724
  * naming each gateway tried.
2564
2725
  */
2565
2726
  function resolveModelGateway(model, pin, scope, what) {
2727
+ const gateway = pickGateway(model, pin, scope, what);
2728
+ const problem = gateway === "vercel" ? vercelBaseUrlProblem() : void 0;
2729
+ if (problem) throw new UsageError(`${what} "${model}": ${problem}`);
2730
+ return gateway;
2731
+ }
2732
+ function pickGateway(model, pin, scope, what) {
2566
2733
  if (typeof model !== "string" || !model.includes("/")) return void 0;
2567
2734
  const on = `${what} "${model}"`;
2568
2735
  const listed = scope.gateways.map((g) => g.kind);
@@ -2627,7 +2794,8 @@ const PROFILE_KEYS = [
2627
2794
  "budget",
2628
2795
  "record",
2629
2796
  "tags",
2630
- "gateway"
2797
+ "gateway",
2798
+ "history"
2631
2799
  ];
2632
2800
  const DEFAULTS_KEYS = [
2633
2801
  "harnesses",
@@ -2649,11 +2817,9 @@ const LIMIT_KEYS = [
2649
2817
  "harnesses",
2650
2818
  "traits"
2651
2819
  ];
2652
- /**
2653
- * The config: a plain object, or a callback that destructures the helpers it needs, e.g.
2654
- * `defineConfig(({ secret }) => ({ env: { KEY: secret.fromEnv('KEY') } }))`.
2655
- */
2656
- const defineConfig = (config) => config;
2820
+ function defineConfig(config) {
2821
+ return config;
2822
+ }
2657
2823
  /** The script configs `reeval` looks for in each directory, first match first, as Vitest orders its own. */
2658
2824
  const SCRIPT_FILES = [
2659
2825
  "ts",
@@ -2845,11 +3011,6 @@ function evalRoot(include) {
2845
3011
  for (let i = 0; dirs.every((d) => d[i] !== void 0 && d[i] === dirs[0]?.[i]); i++) common.push(dirs[0]?.[i]);
2846
3012
  return common.join("/");
2847
3013
  }
2848
- const ARTIFACTS = [
2849
- "all",
2850
- "failures",
2851
- "none"
2852
- ];
2853
3014
  const RECORD = [
2854
3015
  "manual",
2855
3016
  "ci",
@@ -2869,7 +3030,7 @@ const unknownKey = (value, allowed, where, file) => {
2869
3030
  function checkBudget(budget, where, file) {
2870
3031
  if (budget === void 0) return {};
2871
3032
  if (typeof budget !== "object" || budget === null) throw new UsageError(`${where} in ${file} must be an object like { attempts: 20, duration: '1h' } (got ${JSON.stringify(budget)})`);
2872
- if ("usd" in budget || "tokens" in budget) throw new UsageError(`${where}.${"usd" in budget ? "usd" : "tokens"} in ${file} isn't supported yet: reeval records the tokens harnesses report, but prices none and enforces no token budget. Cap the run with attempts or duration.`);
3033
+ if ("usd" in budget || "tokens" in budget) throw new UsageError(`${where}.${"usd" in budget ? "usd" : "tokens"} in ${file} isn't supported yet: reeval records the tokens and cost of each attempt, but enforces no cost or token budget. Cap the run with attempts or duration.`);
2873
3034
  unknownKey(budget, ["attempts", "duration"], where, file);
2874
3035
  const { attempts, duration } = budget;
2875
3036
  if (attempts !== void 0 && !(Number.isInteger(attempts) && attempts >= 1)) throw new UsageError(`${where}.attempts in ${file} must be a whole number of at least 1 (got ${JSON.stringify(attempts)})`);
@@ -3002,9 +3163,17 @@ function selectProfile(config, flag, env = process.env) {
3002
3163
  ...profile.budget
3003
3164
  },
3004
3165
  ...profile.record ? { record: profile.record } : {},
3005
- ...profile.gateway ? { gateway: profile.gateway } : {}
3166
+ ...profile.gateway ? { gateway: profile.gateway } : {},
3167
+ ...config.profileHistory[name] ? { history: config.profileHistory[name] } : {}
3006
3168
  };
3007
3169
  }
3170
+ /** The config with the profile's `history` in place of its own, when the profile sets one. */
3171
+ function withProfileHistory(config, profile) {
3172
+ return profile?.history ? {
3173
+ ...config,
3174
+ history: profile.history
3175
+ } : config;
3176
+ }
3008
3177
  /**
3009
3178
  * The config with its defaults filled in and checked; `root` is its directory, which relative
3010
3179
  * host paths written as data resolve against.
@@ -3018,16 +3187,23 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3018
3187
  if (!Array.isArray(include) || include.some((p) => typeof p !== "string")) throw new UsageError(`include in ${file} must be a list of glob patterns, e.g. include: ['evals/**/*.eval.ts'] (got ${JSON.stringify(include)})`);
3019
3188
  const record = user.record ?? "ci";
3020
3189
  if (!RECORD.includes(record)) throw new UsageError(`record in ${file} is ${oneOf(RECORD)} (got ${JSON.stringify(record)})`);
3021
- const artifacts = user.history?.artifacts ?? "all";
3022
- if (!ARTIFACTS.includes(artifacts)) throw new UsageError(`history.artifacts in ${file} is ${oneOf(ARTIFACTS)} (got ${JSON.stringify(artifacts)})`);
3023
- const historyDir = resolve(root, user.history?.dir ?? join(evalRoot(include), ".history"));
3190
+ const historyDefault = join(evalRoot(include), ".history");
3191
+ const history = resolveHistory(user.history, "history", file, root, historyDefault);
3024
3192
  const harness = user.harnesses === void 0 ? [DEFAULT_HARNESS] : [user.harnesses].flat();
3025
3193
  if (harness.length === 0) throw new UsageError(`harnesses is an empty list in ${file}. Give at least one, e.g. harnesses: ${HARNESS_EXAMPLE}`);
3026
3194
  checkHarnesses(harness, file);
3027
3195
  const profiles = {};
3196
+ const profileHistory = {};
3028
3197
  if (user.profiles !== void 0) {
3029
3198
  if (typeof user.profiles !== "object" || user.profiles === null) throw new UsageError(`profiles in ${file} must be an object of named profiles, e.g. profiles: { pr: { limit: { runs: 1 } } } (got ${JSON.stringify(user.profiles)})`);
3030
- for (const [name, profile] of Object.entries(user.profiles)) profiles[name] = checkProfile(name, profile, file);
3199
+ for (const [name, profile] of Object.entries(user.profiles)) {
3200
+ const { history: own, ...checked } = checkProfile(name, profile, file);
3201
+ profiles[name] = checked;
3202
+ if (own !== void 0) {
3203
+ const where = `profiles.${name}.history`;
3204
+ profileHistory[name] = resolveHistory(own, where, file, root, historyDefault, history);
3205
+ }
3206
+ }
3031
3207
  }
3032
3208
  const defaultProfile = user.defaultProfile ?? {};
3033
3209
  unknownKey(defaultProfile, ["ci", "local"], "defaultProfile", file);
@@ -3063,10 +3239,8 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3063
3239
  }, user.snapshot),
3064
3240
  concurrency: user.concurrency === void 0 ? void 0 : checkConcurrency(user.concurrency, `concurrency in ${file}`),
3065
3241
  record,
3066
- history: {
3067
- dir: historyDir,
3068
- artifacts
3069
- },
3242
+ history,
3243
+ profileHistory,
3070
3244
  secrets: checkSecrets(user.secrets, file),
3071
3245
  debug: checkDebug(user.debug, file),
3072
3246
  analysis: checkAnalysis(user.analysis, file),
@@ -3136,9 +3310,11 @@ const hostCapacity = () => ({
3136
3310
  /**
3137
3311
  * Attempts mostly wait on a model, so the default is 8, but no more than one per CPU or per GB
3138
3312
  * of memory where the sandboxes run (the Docker VM, under Docker Desktop): an idle agent
3139
- * container takes about 300 MB, and the rest is headroom for what the agent runs.
3313
+ * container takes about 300 MB, and the rest is headroom for what the agent runs. A pooled
3314
+ * provider's machines run its attempts, so its `attempts` is the default instead.
3140
3315
  */
3141
- function defaultConcurrency({ cpus, memoryBytes }) {
3316
+ function defaultConcurrency({ cpus, memoryBytes, attempts }) {
3317
+ if (attempts) return Math.max(1, attempts);
3142
3318
  return Math.max(1, Math.min(8, cpus, Math.floor(memoryBytes / 1e9)));
3143
3319
  }
3144
3320
  function checkConcurrency(value, where) {
@@ -3146,11 +3322,11 @@ function checkConcurrency(value, where) {
3146
3322
  if (typeof n !== "number" || !Number.isInteger(n) || n < 1) throw new UsageError(`${where} must be a whole number of at least 1 (got ${String(value)})`);
3147
3323
  return n;
3148
3324
  }
3149
- /** `--concurrency`, then `REEVAL_CONCURRENCY`, then `concurrency` in the config, then the default. */
3150
- async function resolveConcurrency(config, flag, capacity, env = process.env) {
3325
+ /** `--concurrency`, then `REEVAL_CONCURRENCY`, then `concurrency` in the config, or undefined. */
3326
+ function givenConcurrency(config, flag, env = process.env) {
3151
3327
  if (flag !== void 0) return checkConcurrency(flag, "--concurrency");
3152
3328
  if (env.REEVAL_CONCURRENCY) return checkConcurrency(env.REEVAL_CONCURRENCY, "REEVAL_CONCURRENCY");
3153
- return config.concurrency ?? defaultConcurrency(await capacity());
3329
+ return config.concurrency;
3154
3330
  }
3155
3331
  function resolveJudge(judge, file, path = "judge") {
3156
3332
  judge ??= {};
@@ -3186,14 +3362,22 @@ function runModelGateway(config, profile, model, judge = false) {
3186
3362
  * that answer questions, and required config and profile secrets.
3187
3363
  */
3188
3364
  function missingCredentials(config, env = process.env, options = {}) {
3189
- const missing = [];
3190
- for (const s of options.sandboxes ?? []) {
3191
- if (s.kind !== "boat" || env[s.apiKey.from]) continue;
3192
- const line = `${s.apiKey.from} (for boat())`;
3193
- if (!missing.includes(line)) missing.push(line);
3194
- }
3195
- const keys = new Set((options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []));
3196
- for (const key of keys) if (!env[key]) missing.push(key);
3365
+ return neededCredentials(config, options).filter(({ names }) => !names.some((k) => env[k])).map(({ line }) => line);
3366
+ }
3367
+ /**
3368
+ * Each credential the run needs, as `missingCredentials` names it, whether it's set or not:
3369
+ * `names` are the variables any one of which provides it.
3370
+ */
3371
+ function neededCredentials(config, options = {}) {
3372
+ const needed = [];
3373
+ const add = (names, line) => {
3374
+ if (!needed.some((n) => n.line === line)) needed.push({
3375
+ names,
3376
+ line
3377
+ });
3378
+ };
3379
+ for (const s of options.sandboxes ?? []) for (const { names, line } of sandboxKind(s).credentials?.(s) ?? []) add(names, line);
3380
+ for (const names of (options.harnesses ?? config.harness).map(harnessCredentials)) if (names.length) add(names, names.join(" or "));
3197
3381
  const judge = runJudgeModel(config, options.profile);
3198
3382
  const models = [
3199
3383
  ...typeof judge === "string" ? [{
@@ -3214,23 +3398,21 @@ function missingCredentials(config, env = process.env, options = {}) {
3214
3398
  const named = /* @__PURE__ */ new Set();
3215
3399
  for (const { model, gateway, what } of models) {
3216
3400
  if (gateway === "vertex") {
3217
- const gateways = gatewayScope(config, options.profile).gateways;
3218
- const line = `${VERTEX_PROJECT_ENV} (for vertex()) for ${what}, "${model}"`;
3219
- if (!gateways.some((g) => g.kind === "vertex" && g.project) && !env["GOOGLE_CLOUD_PROJECT"] && !missing.includes(line)) missing.push(line);
3401
+ if (!gatewayScope(config, options.profile).gateways.some((g) => g.kind === "vertex" && g.project)) add([VERTEX_PROJECT_ENV], `${VERTEX_PROJECT_ENV} (for vertex()) for ${what}, "${model}"`);
3220
3402
  continue;
3221
3403
  }
3222
3404
  const keys = judgeCredentials(model, gateway);
3223
3405
  const once = `${keys.join(" ")}\0${model}`;
3224
3406
  if (named.has(once)) continue;
3225
3407
  named.add(once);
3226
- if (!keys.some((k) => env[k])) missing.push(`${modelKeyOptions(model, gateway)} for ${what}, "${model}"`);
3408
+ add(keys, `${modelKeyOptions(model, gateway)} for ${what}, "${model}"`);
3227
3409
  }
3228
3410
  const vars = {
3229
3411
  ...config.env,
3230
3412
  ...options.profile?.defaults.env
3231
3413
  };
3232
- for (const [name, from] of secretSources(vars)) if (!env[from]) missing.push(from === name ? name : `${from} (for ${name})`);
3233
- return missing;
3414
+ for (const [name, from] of secretSources(vars)) add([from], from === name ? name : `${from} (for ${name})`);
3415
+ return needed;
3234
3416
  }
3235
3417
  /**
3236
3418
  * Keys the run sends in request headers (the agents' credentials, the run's and attempts' judges'
@@ -3239,7 +3421,7 @@ function missingCredentials(config, env = process.env, options = {}) {
3239
3421
  * then hangs on a request it can't send rather than failing to authenticate.
3240
3422
  */
3241
3423
  function malformedCredentials(config, env = process.env, options = {}) {
3242
- const names = (options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []);
3424
+ const names = (options.harnesses ?? config.harness).flatMap((h) => harnessCredentials(h).find((name) => env[name]) ?? []);
3243
3425
  const judge = runJudgeModel(config, options.profile);
3244
3426
  if (judge !== void 0) names.push(...judgeCredentials(judge, runModelGateway(config, options.profile, judge, true)));
3245
3427
  for (const { model, gateway } of options.judges ?? []) names.push(...judgeCredentials(model, gateway));
@@ -3249,6 +3431,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
3249
3431
  return at ? [`${name} (character ${at})`] : [];
3250
3432
  });
3251
3433
  }
3252
- export { runsOf as $, brokeredIn as A, execOptionsProblem as At, expand as B, decideState as Bt, openCodeModelOn as C, resolveStateOptions as Ct, EVAL_OPTION_KEYS as D, CAPTURE_NAME as Dt, COMPARE_DEFAULT_RUNS as E, observer as Et, checkHarnesses as F, NO_JUDGE_INPUT as Ft, mergeSnapshot as G, levelOf as H, resolveJudgeArtifacts as Ht, checkLimitHarness as I, inputsOn as It, plainHeaders as J, plainHeaderError as K, checkOptions as L, judgeProblem as Lt, cellKey as M, resolvedEffortOf as Mt, checkCapture as N, servedEffortOf as Nt, MAJORITY_GATE as O, CAPTURE_NAME_RULE as Ot, checkGate as P, INPUT_KEYS as Pt, refuseIllegalCompare as Q, environmentWarnings as R, artifactRefsOf as Rt, fallbackHint as S, mergeJudge as St, routedHarness as T, observedName as Tt, majorityOf as U, harnessId as V, filePartsOf as Vt, matchesTags as W, plannedSandboxes as X, plannedSandboxIds as Y, profileJudge as Z, fellBackToDirect as _, resolveOutput as _t, configPathFor as a, unbrokered as at, resolveModelGateway as b, questionShows as bt, findConfig as c, DEFAULT_EGRESS as ct, malformedCredentials as d, ciRunner as dt, sandboxId as et, missingCredentials as f, directOnCloud as ft, selectProfile as g, unenforceable as gt, runJudgeModel as h, isDirect as ht, configLabel as i, secretVarsIn as it, cascade as j, harnessEffort as jt, attemptSandbox as k, attemptRequestProblem as kt, hostCapacity as l, DEFAULT_EGRESS_ALLOW as lt, resolveConcurrency as m, gatewayPolicy as mt, analysisModelOf as n, sandboxSetBy as nt, defaultConcurrency as o, unbrokeredError as ot, requireConfig as p, egressLabel as pt, plainHeaderWarning as q, configFilesPhrase as r, sandboxWarnings as rt, defineConfig as s, unbrokeredWarning as st, CONFIG_FILES as t, sandboxProvider as tt, loadConfig as u, allowedHosts as ut, gatewayScope as v, STOPPED_FIRST as vt, resolveGateway as w, observedHere as wt, effortRefusedOn as x, resolveQuestionState as xt, modelGatewayOrNone as y, UNRECORDED_QUESTION_STATE as yt, evalPlan as z, badJudgeArtifacts as zt };
3434
+ export { plainHeaders as $, decideState as $t, MAJORITY_GATE as A, resolveOutput as At, environmentWarnings as B, CAPTURE_NAME as Bt, effortRefusedOn as C, isRestricted as Ct, routedHarness as D, rootlessPodmanSwitch as Dt, resolveGateway as E, phasePolicies as Et, checkCapture as F, mergeJudge as Ft, majorityOf as G, resolvedEffortOf as Gt, expand as H, attemptRequestProblem as Ht, checkGate as I, resolveStateOptions as It, plainEnv as J, NO_JUDGE_INPUT as Jt, matchesTags as K, servedEffortOf as Kt, checkHarnesses as L, observedHere as Lt, brokeredIn as M, UNRECORDED_QUESTION_STATE as Mt, cascade as N, questionShows as Nt, COMPARE_DEFAULT_RUNS as O, setupEgress as Ot, cellKey as P, resolveQuestionState as Pt, plainHeaderWarning as Q, badJudgeArtifacts as Qt, checkLimitHarness as R, observedName as Rt, resolveModelGateway as S, isPhased as St, openCodeModelOn as T, phaseLabel as Tt, harnessId as U, execOptionsProblem as Ut, evalPlan as V, CAPTURE_NAME_RULE as Vt, levelOf as W, harnessEffort as Wt, plainEnvWarning as X, judgeProblem as Xt, plainEnvError as Y, inputsOn as Yt, plainHeaderError as Z, artifactRefsOf as Zt, selectProfile as _, ciRunner as _t, configPathFor as a, sandboxId as at, gatewayScope as b, gatewayPolicy as bt, findConfig as c, sandboxWarnings as ct, loadConfig as d, unbrokeredError as dt, filePartsOf as en, plannedSandboxIds as et, malformedCredentials as f, unbrokeredWarning as ft, runJudgeModel as g, bareEgress as gt, requireConfig as h, allowedHosts as ht, configLabel as i, runsOf as it, attemptSandbox as j, STOPPED_FIRST as jt, EVAL_OPTION_KEYS as k, unenforceable as kt, givenConcurrency as l, secretVarsIn as lt, neededCredentials as m, DEFAULT_EGRESS_ALLOW as mt, analysisModelOf as n, profileJudge as nt, defaultConcurrency as o, sandboxProvider as ot, missingCredentials as p, DEFAULT_EGRESS as pt, mergeSnapshot as q, INPUT_KEYS as qt, configFilesPhrase as r, refuseIllegalCompare as rt, defineConfig as s, sandboxSetBy as st, CONFIG_FILES as t, resolveJudgeArtifacts as tn, plannedSandboxes as tt, hostCapacity as u, unbrokered as ut, withProfileHistory as v, directOnCloud as vt, fallbackHint as w, layerEgress as wt, modelGatewayOrNone as x, isDirect as xt, fellBackToDirect as y, egressLabel as yt, checkOptions as z, observer as zt };
3253
3435
 
3254
- //# sourceMappingURL=config-SvK8pjEK.js.map
3436
+ //# sourceMappingURL=config-C0RKt00M.js.map