reeval 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +803 -0
  2. package/README.md +15 -15
  3. package/assets/gateway-models.mjs +65 -31
  4. package/dist/{adapter-C60z3wLq.js → adapter-BdzhdRYo.js} +8 -6
  5. package/dist/adapter-BdzhdRYo.js.map +1 -0
  6. package/dist/{agent-BmsMXZXx.js → agent-Dx-bDmlV.js} +179 -31
  7. package/dist/agent-Dx-bDmlV.js.map +1 -0
  8. package/dist/{agents-DXuFxtGC.js → agents-l0-k7vw3.js} +361 -101
  9. package/dist/agents-l0-k7vw3.js.map +1 -0
  10. package/dist/{app-files-DT5mJyV4.js → app-files-B-CQYB0c.js} +3 -3
  11. package/dist/app-files-B-CQYB0c.js.map +1 -0
  12. package/dist/{assemble-CE4pHKIa.js → assemble-CYMpN8LC.js} +1 -1
  13. package/dist/{assemble-DM2PNU4R.js → assemble-CsZIcJo8.js} +171 -18
  14. package/dist/assemble-CsZIcJo8.js.map +1 -0
  15. package/dist/{attempt-CAdlwlNu.js → attempt-CKfKsqqw.js} +2 -2
  16. package/dist/{attempt-CAdlwlNu.js.map → attempt-CKfKsqqw.js.map} +1 -1
  17. package/dist/{attempt-host-7VviAJrl.js → attempt-host-BwSmNYkX.js} +17 -8
  18. package/dist/attempt-host-BwSmNYkX.js.map +1 -0
  19. package/dist/{boat-BjLqFJIZ.js → boat-CZpAreTB.js} +3 -3
  20. package/dist/{boat-BjLqFJIZ.js.map → boat-CZpAreTB.js.map} +1 -1
  21. package/dist/{boat-DSNgZt-t.js → boat-su1Y7LPd.js} +17 -14
  22. package/dist/boat-su1Y7LPd.js.map +1 -0
  23. package/dist/bridge/index.mjs +6 -1
  24. package/dist/{capture-Ccf0Hmyx.js → capture-BqIAeFyC.js} +55 -29
  25. package/dist/{capture-Ccf0Hmyx.js.map → capture-BqIAeFyC.js.map} +1 -1
  26. package/dist/capture-D0Ln5aCP.js +1 -0
  27. package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
  28. package/dist/cases-CM-MQRQp.js.map +1 -0
  29. package/dist/cases-FvhbM8M5.js +1 -0
  30. package/dist/{cli-DTuLSlmW.js → cli-s_3opn_I.js} +219 -109
  31. package/dist/cli-s_3opn_I.js.map +1 -0
  32. package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-DxzIjmq_.js} +109 -26
  33. package/dist/cli-spec-DxzIjmq_.js.map +1 -0
  34. package/dist/cli.js +2 -2
  35. package/dist/client-C5fIzHQa.js +803 -0
  36. package/dist/client-C5fIzHQa.js.map +1 -0
  37. package/dist/{command-DSL7I-SI.js → command-B5OAIjp7.js} +25 -21
  38. package/dist/command-B5OAIjp7.js.map +1 -0
  39. package/dist/{command-C2C7P5qo.js → command-BJ7KqczP.js} +10 -10
  40. package/dist/{command-C2C7P5qo.js.map → command-BJ7KqczP.js.map} +1 -1
  41. package/dist/{command-D8oMB2fa.js → command-BXLZsEwQ.js} +16 -16
  42. package/dist/{command-D8oMB2fa.js.map → command-BXLZsEwQ.js.map} +1 -1
  43. package/dist/{command-Ff0T-1Hi.js → command-DA_SHQUQ.js} +16 -16
  44. package/dist/{command-Ff0T-1Hi.js.map → command-DA_SHQUQ.js.map} +1 -1
  45. package/dist/{commands-xAOi_VEk.js → commands-BKBamOcX.js} +174 -40
  46. package/dist/commands-BKBamOcX.js.map +1 -0
  47. package/dist/{commands-Bk9NG1s_.js → commands-BPvAOEmb.js} +1 -1
  48. package/dist/{config-SvK8pjEK.js → config-C0URar0t.js} +255 -89
  49. package/dist/config-C0URar0t.js.map +1 -0
  50. package/dist/{context-BdgCao9Q.js → context-Sj58aJEu.js} +2 -2
  51. package/dist/{context-BdgCao9Q.js.map → context-Sj58aJEu.js.map} +1 -1
  52. package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
  53. package/dist/cost-C21gGrju.js.map +1 -0
  54. package/dist/cost-IM80LyGD.js +1 -0
  55. package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-FWgSo-T1.js} +4 -4
  56. package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-FWgSo-T1.js.map} +1 -1
  57. package/dist/db-Bc4L-Wcr.js +2 -0
  58. package/dist/{db-BzOJ4Js4.js → db-BjAWxVAG.js} +142 -248
  59. package/dist/db-BjAWxVAG.js.map +1 -0
  60. package/dist/debug-CIXzB4wK.js.map +1 -1
  61. package/dist/{default-effort-BBDPbJ4D.js → default-effort-CWlhZ6U1.js} +1 -1
  62. package/dist/{default-effort-BDKIxJyC.js → default-effort-Ck2GYIxX.js} +4 -4
  63. package/dist/{default-effort-BDKIxJyC.js.map → default-effort-Ck2GYIxX.js.map} +1 -1
  64. package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
  65. package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
  66. package/dist/{docker-DG_uFgn2.js → docker-CDdK6Ity.js} +41 -16
  67. package/dist/docker-CDdK6Ity.js.map +1 -0
  68. package/dist/{docker-DFrjoNs-.js → docker-D74l67k8.js} +4 -4
  69. package/dist/{docker-DFrjoNs-.js.map → docker-D74l67k8.js.map} +1 -1
  70. package/dist/{docker--mukta3L.js → docker-DMug3l-a.js} +1 -1
  71. package/dist/{doctor-DWK4ttex.js → doctor-eFxyn6at.js} +85 -46
  72. package/dist/doctor-eFxyn6at.js.map +1 -0
  73. package/dist/{environment-cPMCOk-h.js → environment-CJIKz-Ud.js} +49 -11
  74. package/dist/environment-CJIKz-Ud.js.map +1 -0
  75. package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
  76. package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
  77. package/dist/{eval-source-C9b1h8L6.js → eval-source-CXIRJ91v.js} +24 -12
  78. package/dist/eval-source-CXIRJ91v.js.map +1 -0
  79. package/dist/{evaluate-B12Tosvq.js → evaluate-DA4QsrHO.js} +792 -281
  80. package/dist/evaluate-DA4QsrHO.js.map +1 -0
  81. package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DHrU4Wld.d.ts} +2 -2
  82. package/dist/fake-bootstrap-CnwWG3kP.js +15 -0
  83. package/dist/fake-bootstrap-CnwWG3kP.js.map +1 -0
  84. package/dist/{files-BOBdC0uI.js → files-BVVWAS0A.js} +6 -6
  85. package/dist/{files-BOBdC0uI.js.map → files-BVVWAS0A.js.map} +1 -1
  86. package/dist/files-BaqXvB88.js +1 -0
  87. package/dist/{files-tiNWe8Ih.js → files-Bey1VAex.js} +3 -3
  88. package/dist/{files-tiNWe8Ih.js.map → files-Bey1VAex.js.map} +1 -1
  89. package/dist/files-BjOJw4sn.js +1 -0
  90. package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
  91. package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
  92. package/dist/files-CNHhSyb3.js +1 -0
  93. package/dist/files-CW1lChNh.js +1 -0
  94. package/dist/{files-F5EG0WPT.js → files-D6j-XpaK.js} +5 -5
  95. package/dist/{files-F5EG0WPT.js.map → files-D6j-XpaK.js.map} +1 -1
  96. package/dist/{files-s28a6czq.js → files-F44NRCN0.js} +143 -20
  97. package/dist/files-F44NRCN0.js.map +1 -0
  98. package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
  99. package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
  100. package/dist/gateway.d.ts +1 -1
  101. package/dist/{git-BpoqEgDV.js → git-Jqkf9SBa.js} +3 -3
  102. package/dist/{git-BpoqEgDV.js.map → git-Jqkf9SBa.js.map} +1 -1
  103. package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
  104. package/dist/harness-FcTyyPco.js.map +1 -0
  105. package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
  106. package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
  107. package/dist/harness.d.ts +11 -8
  108. package/dist/harness.js +1 -1
  109. package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
  110. package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
  111. package/dist/{images-D2jFze2v.js → images-Dsc4Wi-F.js} +65 -30
  112. package/dist/images-Dsc4Wi-F.js.map +1 -0
  113. package/dist/index.d.ts +46 -7
  114. package/dist/index.js +6 -6
  115. package/dist/init-C0oUG3jh.js +1 -0
  116. package/dist/{init-Bi1Q3lRi.js → init-QB5ecPDA.js} +33 -14
  117. package/dist/init-QB5ecPDA.js.map +1 -0
  118. package/dist/{ink-list-BpfidQ1h.js → ink-list-mCJypk3d.js} +2 -2
  119. package/dist/{ink-list-BpfidQ1h.js.map → ink-list-mCJypk3d.js.map} +1 -1
  120. package/dist/{invocations-CxGNOlh0.js → invocations-BSr4lRzI.js} +5 -5
  121. package/dist/{invocations-CxGNOlh0.js.map → invocations-BSr4lRzI.js.map} +1 -1
  122. package/dist/{judge-COMXT1Hx.js → judge-eGH39PaP.js} +12 -10
  123. package/dist/{judge-COMXT1Hx.js.map → judge-eGH39PaP.js.map} +1 -1
  124. package/dist/judge.d.ts +3 -1
  125. package/dist/judge.js +1 -1
  126. package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
  127. package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
  128. package/dist/{layout-Cm85tQYk.js → layout-DfQa4mJs.js} +14 -11
  129. package/dist/layout-DfQa4mJs.js.map +1 -0
  130. package/dist/{list-DuThsE28.js → list-Dh6T0kdr.js} +23 -12
  131. package/dist/{list-DuThsE28.js.map → list-Dh6T0kdr.js.map} +1 -1
  132. package/dist/live-DwjWJTdn.js +1 -0
  133. package/dist/{live-CgYbO4Te.js → live-xTZs0c61.js} +2 -2
  134. package/dist/{live-CgYbO4Te.js.map → live-xTZs0c61.js.map} +1 -1
  135. package/dist/{live-DL0se4Zd.js → live-y1p3oNXH.js} +9 -9
  136. package/dist/{live-DL0se4Zd.js.map → live-y1p3oNXH.js.map} +1 -1
  137. package/dist/{model-CwwkrCb7.js → model-3pZZzloU.js} +7 -5
  138. package/dist/model-3pZZzloU.js.map +1 -0
  139. package/dist/{model-CkNVAxob.js → model-DEjwetkJ.js} +1 -1
  140. package/dist/{model-V3mMXp6x.js → model-DNpKiHqO.js} +6 -6
  141. package/dist/{model-V3mMXp6x.js.map → model-DNpKiHqO.js.map} +1 -1
  142. package/dist/model-calls-B0BoQfJ4.js +95 -0
  143. package/dist/model-calls-B0BoQfJ4.js.map +1 -0
  144. package/dist/{model-cgt5uvu8.js → model-jRKSQBg6.js} +5 -5
  145. package/dist/{model-cgt5uvu8.js.map → model-jRKSQBg6.js.map} +1 -1
  146. package/dist/model-nfewGTzB.js +1 -0
  147. package/dist/{node-D7pAU--0.js → node-DM-mLGdX.js} +7 -7
  148. package/dist/{node-D7pAU--0.js.map → node-DM-mLGdX.js.map} +1 -1
  149. package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
  150. package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
  151. package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-C0aRo633.js} +2 -2
  152. package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-C0aRo633.js.map} +1 -1
  153. package/dist/options-DpsJ_5CU.js +1 -0
  154. package/dist/{patch-DAi1nCuH.js → patch-Bx-qp8Li.js} +2 -2
  155. package/dist/{patch-DAi1nCuH.js.map → patch-Bx-qp8Li.js.map} +1 -1
  156. package/dist/pi-providers-HygmL1y-.js +552 -0
  157. package/dist/pi-providers-HygmL1y-.js.map +1 -0
  158. package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
  159. package/dist/plain-Ctn8oUdp.js.map +1 -0
  160. package/dist/{plan-CsPilzpk.js → plan-ChO0A-xn.js} +63 -12
  161. package/dist/plan-ChO0A-xn.js.map +1 -0
  162. package/dist/{plugins-BfnhazOv.js → plugins-BioMWncI.js} +636 -9
  163. package/dist/plugins-BioMWncI.js.map +1 -0
  164. package/dist/{preflight-D1C-iH0U.js → preflight-BUjk_6wT.js} +1 -1
  165. package/dist/{preflight-GkpgrBxP.js → preflight-CJSGzMzc.js} +8 -9
  166. package/dist/preflight-CJSGzMzc.js.map +1 -0
  167. package/dist/preflight-DD3fI4Hz.js +1 -0
  168. package/dist/{preflight-BixaE4BA.js → preflight-qFCXmxha.js} +4 -4
  169. package/dist/{preflight-BixaE4BA.js.map → preflight-qFCXmxha.js.map} +1 -1
  170. package/dist/{prepare-DQEDXoDp.js → prepare-D7oSSFNm.js} +9 -9
  171. package/dist/{prepare-DQEDXoDp.js.map → prepare-D7oSSFNm.js.map} +1 -1
  172. package/dist/{prepare-C--rM_wO.js → prepare-xHU2BpN-.js} +2 -2
  173. package/dist/prices-B1vm_lPk.js +1 -0
  174. package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
  175. package/dist/prices-BrY_ihLs.js.map +1 -0
  176. package/dist/{provenance-BXyYwUC8.js → provenance-B-flBAzR.js} +5 -5
  177. package/dist/{provenance-BXyYwUC8.js.map → provenance-B-flBAzR.js.map} +1 -1
  178. package/dist/provider-DHT20Gl8.js +1 -0
  179. package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
  180. package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
  181. package/dist/provider-stop-CxzWW7Ft.js +69 -0
  182. package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
  183. package/dist/{providers-1p4HKYIp.js → providers-B6StVyfG.js} +215 -48
  184. package/dist/providers-B6StVyfG.js.map +1 -0
  185. package/dist/providers-Dljf6al6.js +1 -0
  186. package/dist/query-BAc2tQZj.js +1 -0
  187. package/dist/{query-DhhU9ADI.js → query-CkLmJv7R.js} +2 -2
  188. package/dist/{query-DhhU9ADI.js.map → query-CkLmJv7R.js.map} +1 -1
  189. package/dist/{questions-CMAq4pr8.js → questions-CJIh2FSX.js} +7 -7
  190. package/dist/{questions-CMAq4pr8.js.map → questions-CJIh2FSX.js.map} +1 -1
  191. package/dist/{record-6IUzwP9K.js → record-nVVujCW7.js} +2 -2
  192. package/dist/{record-6IUzwP9K.js.map → record-nVVujCW7.js.map} +1 -1
  193. package/dist/{redact-BhjtnjrU.js → redact-DL2BTykw.js} +17 -2
  194. package/dist/redact-DL2BTykw.js.map +1 -0
  195. package/dist/{remarks-BNABzemt.js → remarks-B3wjKscW.js} +46 -11
  196. package/dist/remarks-B3wjKscW.js.map +1 -0
  197. package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
  198. package/dist/remarks-BpdM_2RC.js.map +1 -0
  199. package/dist/remarks-CleoDlYw.js +1 -0
  200. package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
  201. package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
  202. package/dist/{remote-U7fNt0UP.js → remote-C88amFRN.js} +6 -6
  203. package/dist/{remote-U7fNt0UP.js.map → remote-C88amFRN.js.map} +1 -1
  204. package/dist/resolve-Bul8lHaa.js +205 -0
  205. package/dist/resolve-Bul8lHaa.js.map +1 -0
  206. package/dist/resolve-DW92wpIZ.js +1 -0
  207. package/dist/{resolve-Cx3a9UQ2.js → resolve-G576PiFJ.js} +14 -11
  208. package/dist/resolve-G576PiFJ.js.map +1 -0
  209. package/dist/{results-CQKXmHym.js → results-zVHIW6xw.js} +9 -22
  210. package/dist/results-zVHIW6xw.js.map +1 -0
  211. package/dist/{resume-POpZa1ge.js → resume-D_PcUXfG.js} +94 -16
  212. package/dist/resume-D_PcUXfG.js.map +1 -0
  213. package/dist/{retry-CmG809Mp.js → retry-BvTzCxdc.js} +4 -4
  214. package/dist/{retry-CmG809Mp.js.map → retry-BvTzCxdc.js.map} +1 -1
  215. package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
  216. package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
  217. package/dist/{vercel-CGlSBv63.js → routes-B0F2R-K7.js} +91 -56
  218. package/dist/routes-B0F2R-K7.js.map +1 -0
  219. package/dist/{runtime-Xm9oeC7o.js → runtime-B3r_pAq6.js} +4 -4
  220. package/dist/{runtime-Xm9oeC7o.js.map → runtime-B3r_pAq6.js.map} +1 -1
  221. package/dist/sandbox/boat.d.ts +1 -1
  222. package/dist/sandbox/boat.js +1 -1
  223. package/dist/{sandbox-DrlL4yqb.js → sandbox-Duh3z1TV.js} +3 -3
  224. package/dist/{sandbox-DrlL4yqb.js.map → sandbox-Duh3z1TV.js.map} +1 -1
  225. package/dist/sandbox.d.ts +1 -1
  226. package/dist/sandbox.js +1 -1
  227. package/dist/server-CGEsvVRG.js +1 -0
  228. package/dist/{server-C00zd6zl.js → server-DUWMRKik.js} +8 -8
  229. package/dist/{server-C00zd6zl.js.map → server-DUWMRKik.js.map} +1 -1
  230. package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
  231. package/dist/shared-DtZwIAfy.js.map +1 -0
  232. package/dist/{source-DDgr48Gn.js → source-CHuN4Nfg.js} +2 -2
  233. package/dist/{source-DDgr48Gn.js.map → source-CHuN4Nfg.js.map} +1 -1
  234. package/dist/source-Wz1Nrfue.js +1 -0
  235. package/dist/src--yl1OypK.js +2 -0
  236. package/dist/{src-Cb2MWozp.js → src-B-HuDqpH.js} +276 -41
  237. package/dist/src-B-HuDqpH.js.map +1 -0
  238. package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
  239. package/dist/src-B5r1d23Y.js.map +1 -0
  240. package/dist/{src-y0nOeRRE.js → src-BHJ4MWvY.js} +494 -112
  241. package/dist/src-BHJ4MWvY.js.map +1 -0
  242. package/dist/{src-Dw79VLa3.js → src-BI0v8qlZ.js} +202 -56
  243. package/dist/src-BI0v8qlZ.js.map +1 -0
  244. package/dist/src-BL-_0QoE.js +1 -0
  245. package/dist/{src-YvblOrBU.js → src-BqNzHHWn.js} +87 -13
  246. package/dist/src-BqNzHHWn.js.map +1 -0
  247. package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
  248. package/dist/src-C-ChdnG6.js.map +1 -0
  249. package/dist/src-C-mOSczH.js +2 -0
  250. package/dist/src-CZaYu6ER.js +3 -0
  251. package/dist/src-CeBhwD8k.js +2 -0
  252. package/dist/src-CkEiiyON.js +2 -0
  253. package/dist/{src-KxS91P7L.js → src-CmA5Yev6.js} +79 -11
  254. package/dist/src-CmA5Yev6.js.map +1 -0
  255. package/dist/src-CwH_nEaw.js +2 -0
  256. package/dist/src-DA95rM6v.js +4 -0
  257. package/dist/{src-DvGns8cz.js → src-DL-EE4DU.js} +299 -17
  258. package/dist/src-DL-EE4DU.js.map +1 -0
  259. package/dist/src-De0KY5S_.js +2 -0
  260. package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
  261. package/dist/src-boB6D-5T.js.map +1 -0
  262. package/dist/state-B3jex5KU.js +1 -0
  263. package/dist/{state-328JpnHA.js → state-DUBRDyMk.js} +21 -21
  264. package/dist/{state-328JpnHA.js.map → state-DUBRDyMk.js.map} +1 -1
  265. package/dist/store/local.d.ts +8 -0
  266. package/dist/store/local.js +16 -0
  267. package/dist/store/local.js.map +1 -0
  268. package/dist/store/s3.d.ts +10 -0
  269. package/dist/store/s3.js +18 -0
  270. package/dist/store/s3.js.map +1 -0
  271. package/dist/{suite-COglUhPd.js → suite-B9J6Afg6.js} +6 -5
  272. package/dist/{suite-COglUhPd.js.map → suite-B9J6Afg6.js.map} +1 -1
  273. package/dist/suite-Bhs9-Z7O.js +2 -0
  274. package/dist/{summary-DDy9DfyJ.js → summary-CSp4cfnr.js} +25 -7
  275. package/dist/{summary-DDy9DfyJ.js.map → summary-CSp4cfnr.js.map} +1 -1
  276. package/dist/sync-CxAY6Dki.js +559 -0
  277. package/dist/sync-CxAY6Dki.js.map +1 -0
  278. package/dist/{tart-dWoJvWHc.js → tart-CcrrtSrp.js} +179 -61
  279. package/dist/tart-CcrrtSrp.js.map +1 -0
  280. package/dist/tart-CzViW3WZ.js +186 -0
  281. package/dist/tart-CzViW3WZ.js.map +1 -0
  282. package/dist/tart-DmVuVsmX.js +1 -0
  283. package/dist/tart-bake-F-6seRC9.js +192 -0
  284. package/dist/tart-bake-F-6seRC9.js.map +1 -0
  285. package/dist/tart-bake-RK7RaT9E.js +1 -0
  286. package/dist/tart-cli-B86Jqo1t.js +89 -0
  287. package/dist/tart-cli-B86Jqo1t.js.map +1 -0
  288. package/dist/tart-cli-ClvRniLQ.js +1 -0
  289. package/dist/tart-hosts-Brii7zHM.js +1 -0
  290. package/dist/tart-hosts-DPyZsg-1.js +313 -0
  291. package/dist/tart-hosts-DPyZsg-1.js.map +1 -0
  292. package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CWSf03w8.js} +64 -6
  293. package/dist/tart-images-CWSf03w8.js.map +1 -0
  294. package/dist/testing.d.ts +36 -2
  295. package/dist/testing.js +4 -4
  296. package/dist/tool-class-CVDdJy0H.js +433 -0
  297. package/dist/tool-class-CVDdJy0H.js.map +1 -0
  298. package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
  299. package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
  300. package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
  301. package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
  302. package/dist/{transcript-CCf07UmU.js → transcript-BP_2hji-.js} +3 -3
  303. package/dist/{transcript-CCf07UmU.js.map → transcript-BP_2hji-.js.map} +1 -1
  304. package/dist/{types-DBwFsGem.d.ts → types-BJ0wh8wC.d.ts} +408 -44
  305. package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
  306. package/dist/types-CbcnJxCy.js.map +1 -0
  307. package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
  308. package/dist/usage-CwIRPPZ9.js.map +1 -0
  309. package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
  310. package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
  311. package/dist/vertex-DXfcXYKn.js +1 -0
  312. package/dist/view/index.d.ts +27 -3
  313. package/dist/view/index.js +8 -8
  314. package/dist/{views-BROacjBv.js → views-DRxXCyIH.js} +6 -6
  315. package/dist/{views-BROacjBv.js.map → views-DRxXCyIH.js.map} +1 -1
  316. package/dist/views-i67V1ly3.js +2 -0
  317. package/dist/{worker-entry-Smb4bNDc.js → worker-entry-C3Q4U63F.js} +12 -9
  318. package/dist/worker-entry-C3Q4U63F.js.map +1 -0
  319. package/dist/workspace-DhUOW8if.js +1 -0
  320. package/dist/{workspace-DNDTEk8v.js → workspace-P72SXLXr.js} +7 -7
  321. package/dist/{workspace-DNDTEk8v.js.map → workspace-P72SXLXr.js.map} +1 -1
  322. package/package.json +12 -1
  323. package/skills/eval-rubric/SKILL.md +10 -0
  324. package/skills/reeval/SKILL.md +370 -0
  325. package/skills/reeval/references/capture.md +48 -0
  326. package/skills/reeval/references/cli.md +162 -0
  327. package/skills/reeval/references/compare.md +180 -0
  328. package/skills/reeval/references/config.md +147 -0
  329. package/skills/reeval/references/environments.md +218 -0
  330. package/skills/reeval/references/evals.md +241 -0
  331. package/skills/reeval/references/mistakes.md +39 -0
  332. package/skills/reeval/references/vocabulary.md +19 -0
  333. package/skills/setup/SKILL.md +219 -0
  334. package/skills/setup/references/templates.md +258 -0
  335. package/dist/adapter-C60z3wLq.js.map +0 -1
  336. package/dist/agent-BmsMXZXx.js.map +0 -1
  337. package/dist/agents-DXuFxtGC.js.map +0 -1
  338. package/dist/app-files-DT5mJyV4.js.map +0 -1
  339. package/dist/assemble-DM2PNU4R.js.map +0 -1
  340. package/dist/attempt-host-7VviAJrl.js.map +0 -1
  341. package/dist/boat-DSNgZt-t.js.map +0 -1
  342. package/dist/capture-BzotD2hs.js +0 -1
  343. package/dist/cases-Dy0mifal.js +0 -1
  344. package/dist/cases-z9E6DCd5.js.map +0 -1
  345. package/dist/cli-DTuLSlmW.js.map +0 -1
  346. package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
  347. package/dist/client-KM4C5r5_.js +0 -387
  348. package/dist/client-KM4C5r5_.js.map +0 -1
  349. package/dist/command-DSL7I-SI.js.map +0 -1
  350. package/dist/commands-xAOi_VEk.js.map +0 -1
  351. package/dist/config-SvK8pjEK.js.map +0 -1
  352. package/dist/cost-BdRgySQl.js.map +0 -1
  353. package/dist/cost-Cr26Alhq.js +0 -1
  354. package/dist/db-BzOJ4Js4.js.map +0 -1
  355. package/dist/db-CQRZ3URX.js +0 -2
  356. package/dist/docker-DG_uFgn2.js.map +0 -1
  357. package/dist/doctor-DWK4ttex.js.map +0 -1
  358. package/dist/environment-cPMCOk-h.js.map +0 -1
  359. package/dist/eval-source-C9b1h8L6.js.map +0 -1
  360. package/dist/evaluate-B12Tosvq.js.map +0 -1
  361. package/dist/files-C1Rk9FYA.js +0 -1
  362. package/dist/files-DMQ9-Vv5.js +0 -1
  363. package/dist/files-Di0xDbCc.js +0 -1
  364. package/dist/files-G5jf8wBp.js +0 -1
  365. package/dist/files-s28a6czq.js.map +0 -1
  366. package/dist/harness-DXv2gnaT.js.map +0 -1
  367. package/dist/images-D2jFze2v.js.map +0 -1
  368. package/dist/init-Bi1Q3lRi.js.map +0 -1
  369. package/dist/init-CUw65o36.js +0 -1
  370. package/dist/kinds-Bh2I35id.js +0 -698
  371. package/dist/kinds-Bh2I35id.js.map +0 -1
  372. package/dist/layout-Cm85tQYk.js.map +0 -1
  373. package/dist/live-DZLmj0iG.js +0 -1
  374. package/dist/matchers-CXLn-1jm.js +0 -148
  375. package/dist/matchers-CXLn-1jm.js.map +0 -1
  376. package/dist/model-Cv4GuTsZ.js +0 -1
  377. package/dist/model-CwwkrCb7.js.map +0 -1
  378. package/dist/options-DPUmk7v_.js +0 -1
  379. package/dist/plain-BYUf_fUV.js.map +0 -1
  380. package/dist/plan-CsPilzpk.js.map +0 -1
  381. package/dist/plugins-BfnhazOv.js.map +0 -1
  382. package/dist/preflight-Bn3ianye.js +0 -1
  383. package/dist/preflight-GkpgrBxP.js.map +0 -1
  384. package/dist/prices-5m3SzMZ3.js.map +0 -1
  385. package/dist/provider-BnUoz4KB.js +0 -1
  386. package/dist/providers-1p4HKYIp.js.map +0 -1
  387. package/dist/providers-B-ijCktf.js +0 -1
  388. package/dist/query-BYhyf0LR.js +0 -1
  389. package/dist/redact-BhjtnjrU.js.map +0 -1
  390. package/dist/remarks-BNABzemt.js.map +0 -1
  391. package/dist/remarks-CAvTSq16.js +0 -1
  392. package/dist/remarks-D9Lcd1fF.js.map +0 -1
  393. package/dist/resolve-C14Pmtej.js +0 -1
  394. package/dist/resolve-Cx3a9UQ2.js.map +0 -1
  395. package/dist/results-CQKXmHym.js.map +0 -1
  396. package/dist/resume-POpZa1ge.js.map +0 -1
  397. package/dist/server-C1UJcpUh.js +0 -1
  398. package/dist/shared-Cfi45Bvy.js.map +0 -1
  399. package/dist/source-DJHmQv_j.js +0 -1
  400. package/dist/src-1dBuXzs4.js +0 -3
  401. package/dist/src-BKpioayC.js +0 -2
  402. package/dist/src-BNQznj4z.js.map +0 -1
  403. package/dist/src-BY1cCaoJ.js +0 -2
  404. package/dist/src-BbZYaQ8b.js +0 -2
  405. package/dist/src-CDFH2vVN.js +0 -2
  406. package/dist/src-CMW5R9Q0.js +0 -1
  407. package/dist/src-CV_7DkPa.js.map +0 -1
  408. package/dist/src-Cb2MWozp.js.map +0 -1
  409. package/dist/src-CrV2WA2i.js +0 -2
  410. package/dist/src-CsFaawcZ.js +0 -2
  411. package/dist/src-DAzaCTcN.js.map +0 -1
  412. package/dist/src-DvGns8cz.js.map +0 -1
  413. package/dist/src-Dw79VLa3.js.map +0 -1
  414. package/dist/src-KxS91P7L.js.map +0 -1
  415. package/dist/src-YvblOrBU.js.map +0 -1
  416. package/dist/src-wKwznPgt.js +0 -2
  417. package/dist/src-y0nOeRRE.js.map +0 -1
  418. package/dist/state-Ca55Gk_O.js +0 -1
  419. package/dist/suite-CKBa6MKt.js +0 -2
  420. package/dist/tart-cli-C-GsbVsQ.js +0 -1
  421. package/dist/tart-cli-CU0nFS-L.js +0 -55
  422. package/dist/tart-cli-CU0nFS-L.js.map +0 -1
  423. package/dist/tart-dWoJvWHc.js.map +0 -1
  424. package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
  425. package/dist/tool-target-DS5d72eH.js +0 -40
  426. package/dist/tool-target-DS5d72eH.js.map +0 -1
  427. package/dist/types-C2mLpph3.js.map +0 -1
  428. package/dist/usage-DJiDnh2D.js.map +0 -1
  429. package/dist/vercel-CGlSBv63.js.map +0 -1
  430. package/dist/vertex-O88TCwgY.js +0 -1
  431. package/dist/views-B0Yt_Q1S.js +0 -2
  432. package/dist/worker-entry-Smb4bNDc.js.map +0 -1
  433. package/dist/workspace-C0dDP75l.js +0 -1
@@ -1,33 +1,34 @@
1
1
  import { n as UsageError } from "./errors-ttMTnIYX.js";
2
- import { a as fieldPath, c as jsonValueSchema, d as refusal, f as tokenUsageSchema, g as parseDuration, h as TIMER_LIMIT, i as durationSchema, l as nonEmpty$1, m as MAX_TIMER_MS, n as dataProblem, o as genericProblem, p as valueAt, s as got, t as closest, u as oneOf } from "./shared-Cfi45Bvy.js";
2
+ import { a as fieldPath, c as jsonValueSchema, d as refusal, f as tokenUsageSchema, g as parseDuration, h as TIMER_LIMIT, i as durationSchema, l as nonEmpty$1, m as MAX_TIMER_MS, n as dataProblem, o as genericProblem, p as valueAt, s as got, t as closest, u as oneOf } from "./shared-DtZwIAfy.js";
3
+ import { n as openCodeEffortBody, r as openCodeEffortRefusal, u as resolveOpenCodeProvider } from "./opencode-providers-ChJTQRmz.js";
4
+ import { u as ownFileURL } from "./root-BzeoyfGd.js";
5
+ import { X as HARNESS_HOSTS, Z as PACKAGE_HOSTS } from "./src-BHJ4MWvY.js";
6
+ import { _ as snapshotProblem, c as gatePolicySchema, g as snapshotOptionSchema, l as gateProblem, o as egressSchema, s as envVarsSchema, t as VAR } from "./plain-Ctn8oUdp.js";
7
+ import { i as fromCwd, l as sha256 } from "./ids-B-JKTCP8.js";
8
+ import { _ as redactValue, h as redactSecrets, r as badCharacterAt, t as REDACTED } from "./redact-DL2BTykw.js";
9
+ import { H as sandboxHelpers, I as idPart, L as isSandboxKind, N as SANDBOX_KINDS, P as capabilitiesOf, R as maxConcurrentProblem, U as sandboxKind, W as suggestedSandbox, y as localPluginServers } from "./plugins-BioMWncI.js";
10
+ import { n as isRoutingGateway, t as ROUTING_GATEWAYS } from "./routes-B0F2R-K7.js";
11
+ import { c as GATEWAY_NAMES, d as gatewayDescriptorSchema, f as gatewayEntryDataSchema, i as claudeCodeTakesEffort, o as harnessDescriptorSchema, p as gatewayNameSchema, r as claudeCodeModelName, s as harnessProblem, t as FACTORY, u as directOffPin } from "./harness-FcTyyPco.js";
12
+ import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV } from "./vertex-BEWxHoxH.js";
13
+ import { _ as isAgentKind, g as harnessCredentials, i as HARNESS_EXAMPLE, p as directProviderRefusal, r as DEFAULT_HARNESS, s as agentFor, t as AGENTS } from "./agents-l0-k7vw3.js";
3
14
  import { n as isOneOf, t as isObjectLike } from "./object-CeQ8dgUC.js";
4
- import { _ as snapshotProblem, a as egressPolicySchema, c as gatePolicySchema, g as snapshotOptionSchema, l as gateProblem, s as envVarsSchema } from "./plain-BYUf_fUV.js";
5
- import { a as isSandboxKind, d as sandboxKind, f as suggestedSandbox, i as idPart, n as capabilitiesOf, o as maxConcurrentProblem, t as SANDBOX_KINDS, u as sandboxHelpers } from "./kinds-Bh2I35id.js";
6
- import { C as plainCredentialHeaders, D as sandboxOf, E as sameWorkspaceOnly, F as environmentDataSchema, I as sandboxDescriptorSchema, L as traitsSchema, M as unbrokeredSecrets, N as workspaceLabel, O as sandboxPaths, P as workspaceOf, _ as isMovingRef, a as checkEgress, c as environmentDataProblem, h as isEnvironment, j as traitsOf, l as environmentFrom, m as helpersFor, o as checkEnv, r as applied, u as environmentPaths, v as isSecret, w as plannedSecrets, z as sandboxProblem } from "./environment-cPMCOk-h.js";
7
- import { c as GATEWAY_NAMES, d as gatewayEntryDataSchema, f as gatewayNameSchema, i as claudeCodeTakesEffort, l as directOffPin, o as harnessDescriptorSchema, r as claudeCodeModelName, s as harnessProblem, t as FACTORY, u as gatewayDescriptorSchema } from "./harness-DXv2gnaT.js";
8
- import { c as sha256, i as fromCwd } from "./ids-C_advdJA.js";
9
- import { n as openCodeEffortBody, r as openCodeEffortRefusal, u as resolveOpenCodeProvider } from "./opencode-providers-BGiXt8_B.js";
10
- import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV } from "./vertex-Eb2wqahk.js";
11
- import { _ as redactValue, h as redactSecrets, r as badCharacterAt, t as REDACTED } from "./redact-BhjtnjrU.js";
12
- import { a as RECOMMENDED_JUDGE_MODEL, c as judgeCredentials, d as modelKeyOptions, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL } from "./judge-COMXT1Hx.js";
15
+ import { B as sandboxProblem, C as plainCredentialEnv, D as sameWorkspaceOnly, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, M as traitsOf, N as unbrokeredSecrets, O as sandboxOf, P as workspaceLabel, R as traitsSchema, T as plannedSecrets, _ as isMovingRef, a as checkEgress, c as environmentDataProblem, h as isEnvironment, k as sandboxPaths, l as environmentFrom, m as helpersFor, o as checkEnv, r as applied, u as environmentPaths, v as isSecret, w as plainCredentialHeaders } from "./environment-CJIKz-Ud.js";
16
+ import { a as RECOMMENDED_JUDGE_MODEL, c as judgeCredentials, d as modelKeyOptions, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL } from "./judge-eGH39PaP.js";
13
17
  import { t as inCI } from "./env-8x1O8I1M.js";
14
- import { u as ownFileURL } from "./root-Co0jhOT5.js";
15
- import { B as HARNESS_HOSTS, V as PACKAGE_HOSTS } from "./src-y0nOeRRE.js";
16
- import { y as localPluginServers } from "./plugins-BfnhazOv.js";
17
- import { a as agentFor, b as isRoutingGateway, f as isAgentKind, n as DEFAULT_HARNESS, r as HARNESS_EXAMPLE, t as AGENTS, u as credentialOf, y as ROUTING_GATEWAYS } from "./agents-DXuFxtGC.js";
18
18
  import { i as pairFactors } from "./pairs-BGb134Vu.js";
19
- import { u as opencodeModelError } from "./harness-C2berMiM.js";
19
+ import { u as opencodeModelError } from "./harness-nwnyghJZ.js";
20
20
  import { r as tartTagWarning } from "./tart-ref-DYZijjrU.js";
21
21
  import "./stats-C18W4xPX.js";
22
22
  import { n as gateLabel } from "./gate-CgYdaprG.js";
23
- import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-C60z3wLq.js";
24
- import { t as docker } from "./sandbox-DrlL4yqb.js";
23
+ import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-BdzhdRYo.js";
24
+ import { t as docker } from "./sandbox-Duh3z1TV.js";
25
+ import { i as resolveHistory } from "./resolve-Bul8lHaa.js";
25
26
  import { z } from "zod";
26
27
  import { existsSync, readFileSync } from "node:fs";
27
- import { availableParallelism, totalmem } from "node:os";
28
28
  import { dirname, join, resolve } from "node:path";
29
- import { parseDocument } from "yaml";
29
+ import { availableParallelism, totalmem } from "node:os";
30
30
  import { isIP } from "node:net";
31
+ import { parseDocument } from "yaml";
31
32
  // src/judge/artifacts.ts
32
33
  /** A capture handle: a name plus the file the attempt wrote. */
33
34
  function isCaptureHandle(value) {
@@ -408,12 +409,38 @@ const budgetSchema = z.strictObject({
408
409
  attempts: count().optional(),
409
410
  duration: durationSchema.optional()
410
411
  });
411
- const historyConfigSchema = z.strictObject({
412
- dir: z.string().optional(),
412
+ const localStoreDescriptorSchema = z.strictObject({
413
+ kind: z.literal("local"),
414
+ dir: z.string().optional()
415
+ });
416
+ const s3StoreEnvSchema = z.strictObject({
417
+ accessKeyId: z.string().optional(),
418
+ secretAccessKey: z.string().optional(),
419
+ sessionToken: z.string().optional(),
420
+ endpoint: z.string().optional(),
421
+ bucket: z.string().optional(),
422
+ region: z.string().optional()
423
+ });
424
+ const s3StoreDescriptorSchema = z.strictObject({
425
+ kind: z.literal("s3"),
426
+ bucket: z.string().optional(),
427
+ endpoint: z.string().optional(),
428
+ region: z.string().optional(),
429
+ prefix: z.string().optional(),
430
+ env: s3StoreEnvSchema.optional()
431
+ });
432
+ const historyStoreDescriptorSchema = z.discriminatedUnion("kind", [localStoreDescriptorSchema, s3StoreDescriptorSchema]);
433
+ const historyConfigDataSchema = z.strictObject({
434
+ store: historyStoreDescriptorSchema.optional(),
413
435
  artifacts: z.enum([
414
436
  "all",
415
437
  "failures",
416
438
  "none"
439
+ ]).optional(),
440
+ debug: z.enum([
441
+ "none",
442
+ "requests",
443
+ "all"
417
444
  ]).optional()
418
445
  });
419
446
  const recordMode = z.enum([
@@ -428,7 +455,7 @@ const profileDefaultsDataSchema = z.strictObject({
428
455
  runs: runs().optional(),
429
456
  gate: gatePolicySchema.optional(),
430
457
  env: envVarsSchema.optional(),
431
- egress: egressPolicySchema.optional(),
458
+ egress: egressSchema.optional(),
432
459
  judge: configJudgeDataSchema.optional(),
433
460
  timeout: durationSchema.optional(),
434
461
  capture: captureOptionsSchema.optional(),
@@ -448,7 +475,8 @@ const profileDataSchema = z.strictObject({
448
475
  budget: budgetSchema.optional(),
449
476
  record: recordMode.optional(),
450
477
  tags: z.array(z.string()).optional(),
451
- gateway: z.array(gatewayEntryDataSchema).min(1).optional()
478
+ gateway: z.array(gatewayEntryDataSchema).min(1).optional(),
479
+ history: historyConfigDataSchema.optional()
452
480
  });
453
481
  const fakeVote = z.enum([
454
482
  "A",
@@ -490,7 +518,7 @@ const reevalConfigDataSchema = z.strictObject({
490
518
  runs: runs().optional(),
491
519
  gate: gatePolicySchema.optional(),
492
520
  env: envVarsSchema.optional(),
493
- egress: egressPolicySchema.optional(),
521
+ egress: egressSchema.optional(),
494
522
  judge: configJudgeDataSchema.optional(),
495
523
  timeout: durationSchema.optional(),
496
524
  capture: captureOptionsSchema.optional(),
@@ -498,7 +526,7 @@ const reevalConfigDataSchema = z.strictObject({
498
526
  snapshot: snapshotOptionSchema.optional(),
499
527
  concurrency: count().optional(),
500
528
  record: recordMode.optional(),
501
- history: historyConfigSchema.optional(),
529
+ history: historyConfigDataSchema.optional(),
502
530
  budget: budgetSchema.optional(),
503
531
  profiles: z.record(z.string(), profileDataSchema).optional(),
504
532
  defaultProfile: z.strictObject({
@@ -676,7 +704,8 @@ const transcriptEventSchema = z.discriminatedUnion("type", [
676
704
  removed: nullableNumber,
677
705
  durationMs: nullableNumber,
678
706
  stalled: z.boolean().optional(),
679
- subagent: z.string().optional()
707
+ subagent: z.string().optional(),
708
+ durationInferred: z.literal(true).optional()
680
709
  }),
681
710
  z.strictObject({
682
711
  ...at$1,
@@ -687,7 +716,8 @@ const transcriptEventSchema = z.discriminatedUnion("type", [
687
716
  ok: z.boolean().nullable().optional(),
688
717
  durationMs: nullableNumber,
689
718
  stalled: z.boolean().optional(),
690
- subagent: z.string().optional()
719
+ subagent: z.string().optional(),
720
+ durationInferred: z.literal(true).optional()
691
721
  }),
692
722
  z.strictObject({
693
723
  ...at$1,
@@ -697,7 +727,8 @@ const transcriptEventSchema = z.discriminatedUnion("type", [
697
727
  ok: z.boolean().nullable(),
698
728
  durationMs: nullableNumber,
699
729
  stalled: z.boolean().optional(),
700
- subagent: z.string().optional()
730
+ subagent: z.string().optional(),
731
+ durationInferred: z.literal(true).optional()
701
732
  }),
702
733
  z.strictObject({
703
734
  ...at$1,
@@ -706,10 +737,13 @@ const transcriptEventSchema = z.discriminatedUnion("type", [
706
737
  tool: z.string(),
707
738
  input: z.unknown(),
708
739
  output: z.unknown(),
740
+ outputView: z.enum(["model", "full"]).optional(),
741
+ outputTruncated: z.strictObject({ bytes: z.number() }).optional(),
709
742
  ok: z.boolean().nullable(),
710
743
  durationMs: nullableNumber,
711
744
  stalled: z.boolean().optional(),
712
- subagent: z.string().optional()
745
+ subagent: z.string().optional(),
746
+ durationInferred: z.literal(true).optional()
713
747
  }),
714
748
  z.strictObject({
715
749
  ...at$1,
@@ -720,6 +754,18 @@ const transcriptEventSchema = z.discriminatedUnion("type", [
720
754
  ...at$1,
721
755
  type: z.literal("retry"),
722
756
  message: z.string()
757
+ }),
758
+ z.strictObject({
759
+ ...at$1,
760
+ type: z.literal("model-call"),
761
+ request: z.string(),
762
+ status: nullableNumber,
763
+ durationMs: z.number(),
764
+ firstByteMs: nullableNumber,
765
+ usage: tokenUsageSchema.nullable(),
766
+ model: z.string().nullable(),
767
+ stream: z.literal(true).optional(),
768
+ role: z.enum(["main", "side"]).optional()
723
769
  })
724
770
  ]);
725
771
  const turnDataSchema = z.strictObject({
@@ -743,6 +789,7 @@ const execRetrySchema = z.strictObject({
743
789
  retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional(),
744
790
  interval: z.number().min(0, "must be 0 or more milliseconds").max(MAX_TIMER_MS, `must be ${TIMER_LIMIT}`).optional()
745
791
  });
792
+ const execEnvSchema = z.record(z.string().regex(VAR, "must be a variable name: letters, digits and _, not starting with a digit"), z.string().refine((v) => !v.includes("\0"), "must not hold a NUL character"));
746
793
  /** A background command has no exit for `retries` to wait on. */
747
794
  const noBackgroundRetry = (options, ctx) => {
748
795
  if (!options.background) return;
@@ -757,6 +804,7 @@ const noBackgroundRetry = (options, ctx) => {
757
804
  };
758
805
  const execOptionsSchema = z.strictObject({
759
806
  background: z.boolean().optional(),
807
+ env: execEnvSchema.optional(),
760
808
  ...execRetrySchema.shape
761
809
  }).superRefine(noBackgroundRetry);
762
810
  /** What's wrong with `c.exec()`'s options, or undefined. */
@@ -873,6 +921,7 @@ const matcherSubjectSchema = z.union([
873
921
  z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
874
922
  z.strictObject({
875
923
  exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
924
+ env: execEnvSchema.optional(),
876
925
  result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
877
926
  }),
878
927
  z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
@@ -920,8 +969,9 @@ const attemptCallSchema = z.discriminatedUnion("op", [
920
969
  op: z.literal("exec"),
921
970
  command: nonEmpty.describe("A shell command, run in the workspace."),
922
971
  background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
972
+ env: execEnvSchema.optional(),
923
973
  ...execRetrySchema.shape
924
- }).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
974
+ }).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, env?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
925
975
  z.strictObject({
926
976
  op: z.literal("serve"),
927
977
  command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
@@ -1457,6 +1507,47 @@ const DEFAULT_EGRESS_ALLOW = PACKAGE_HOSTS;
1457
1507
  Object.fromEntries(Object.entries(SANDBOX_KINDS).map(([kind, k]) => [kind, k.capabilities.egress]));
1458
1508
  /** The policy an attempt runs under when nothing sets one. */
1459
1509
  const DEFAULT_EGRESS = "proxy";
1510
+ /** Whether `egress` sets a policy per phase, `{ setup, agent }`, rather than one for both. */
1511
+ const isPhased = (egress) => typeof egress === "object" && !("allow" in egress);
1512
+ /** The policy `egress` sets for `phase`: a plain policy sets both; undefined when it sets none. */
1513
+ const phaseOf = (egress, phase) => egress === void 0 ? void 0 : isPhased(egress) ? egress[phase] : egress;
1514
+ /**
1515
+ * `over` set at the same level as `under`, later: a plain policy replaces both phases,
1516
+ * `{ setup, agent }` only the phases it names.
1517
+ */
1518
+ function layerEgress(under, over) {
1519
+ if (!isPhased(over)) return over;
1520
+ const setup = over.setup ?? phaseOf(under, "setup");
1521
+ const agent = over.agent ?? phaseOf(under, "agent");
1522
+ return {
1523
+ ...setup !== void 0 ? { setup } : {},
1524
+ ...agent !== void 0 ? { agent } : {}
1525
+ };
1526
+ }
1527
+ /** Each phase's policy `egress` sets, the default where it sets none. */
1528
+ const phasePolicies = (egress) => ({
1529
+ setup: phaseOf(egress, "setup") ?? "proxy",
1530
+ agent: phaseOf(egress, "agent") ?? "proxy"
1531
+ });
1532
+ /** `egress` planned with no automatic hosts: what a config-level check can tell of it. */
1533
+ function bareEgress(egress) {
1534
+ const { setup, agent } = phasePolicies(egress);
1535
+ return samePolicy(setup, agent) ? {
1536
+ policy: agent,
1537
+ auto: []
1538
+ } : {
1539
+ policy: agent,
1540
+ auto: [],
1541
+ setup: {
1542
+ policy: setup,
1543
+ auto: []
1544
+ }
1545
+ };
1546
+ }
1547
+ const policyKey = (p) => typeof p === "string" ? p : JSON.stringify([...new Set(p.allow)].sort());
1548
+ const samePolicy = (a, b) => policyKey(a) === policyKey(b);
1549
+ /** The setup phase's planned policy: its own when it differs from the agent's. */
1550
+ const setupEgress = (planned) => planned.setup ?? planned;
1460
1551
  /** Whether the policy allows only some hosts: `deny-all` and allowlists. */
1461
1552
  const isRestricted = (policy) => policy !== "allow-all" && policy !== "proxy";
1462
1553
  /**
@@ -1465,7 +1556,8 @@ const isRestricted = (policy) => policy !== "allow-all" && policy !== "proxy";
1465
1556
  */
1466
1557
  const isDirect = (policy) => policy === "allow-all";
1467
1558
  /**
1468
- * The hosts the harness needs: a real agent's own API (its model's provider's, for OpenCode), or
1559
+ * The hosts the harness needs: a real agent's own API (its model's provider's, for OpenCode, Goose
1560
+ * and pi), or
1469
1561
  * a fake agent's with `layout` (it behaves like that agent). A plain fake agent talks to nothing.
1470
1562
  */
1471
1563
  function harnessHosts(harness, vertex) {
@@ -1481,26 +1573,34 @@ function mcpServerHosts(layers) {
1481
1573
  return [...urls.values()].flatMap((url) => URL.parse(url)?.hostname ?? []);
1482
1574
  }
1483
1575
  /**
1484
- * The policy an attempt runs under, and the hosts a restricted one allows on its own: the
1485
- * harness's, the brokered secrets' `hosts` and the MCP servers'. The cascade is config and
1486
- * profile defaults (`inherited`), then the environment's layers, then groups and the eval (`own`).
1576
+ * The policy each phase of an attempt runs under, and the hosts a restricted one allows on its
1577
+ * own: the harness's, the brokered secrets' `hosts` and the MCP servers'. Each phase cascades on
1578
+ * its own: config and profile defaults (`inherited`), then the environment's layers, then groups
1579
+ * and the eval (`own`). `setup` is set only when the setup phase's policy differs from the agent's.
1487
1580
  */
1488
1581
  function plannedEgress(egress, harness, env, layers, vertex) {
1489
- const fromLayers = layers.findLast((l) => l.spec.egress !== void 0)?.spec.egress;
1490
- const policy = egress.own ?? fromLayers ?? egress.inherited ?? "proxy";
1491
- if (!isRestricted(policy)) return {
1492
- policy,
1493
- auto: []
1582
+ const pick = (phase) => phaseOf(egress.own, phase) ?? layers.map((l) => phaseOf(l.spec.egress, phase)).findLast((p) => p !== void 0) ?? phaseOf(egress.inherited, phase) ?? "proxy";
1583
+ const phase = (policy) => {
1584
+ if (!isRestricted(policy)) return {
1585
+ policy,
1586
+ auto: []
1587
+ };
1588
+ const own = policy === "deny-all" ? [] : policy.allow;
1589
+ const auto = [
1590
+ ...harnessHosts(harness, vertex),
1591
+ ...plannedSecrets(env, layers).flatMap((s) => s.hosts ?? []),
1592
+ ...mcpServerHosts(layers)
1593
+ ];
1594
+ return {
1595
+ policy,
1596
+ auto: [...new Set(auto)].filter((h) => !own.includes(h))
1597
+ };
1494
1598
  };
1495
- const own = policy === "deny-all" ? [] : policy.allow;
1496
- const auto = [
1497
- ...harnessHosts(harness, vertex),
1498
- ...plannedSecrets(env, layers).flatMap((s) => s.hosts ?? []),
1499
- ...mcpServerHosts(layers)
1500
- ];
1501
- return {
1502
- policy,
1503
- auto: [...new Set(auto)].filter((h) => !own.includes(h))
1599
+ const agent = phase(pick("agent"));
1600
+ const setup = pick("setup");
1601
+ return samePolicy(setup, agent.policy) ? agent : {
1602
+ ...agent,
1603
+ setup: phase(setup)
1504
1604
  };
1505
1605
  }
1506
1606
  /** Every host a restricted policy lets through, or null under `proxy` and `allow-all`. */
@@ -1535,19 +1635,24 @@ const LEVELS = [
1535
1635
  "all-or-nothing",
1536
1636
  "allowlist"
1537
1637
  ];
1538
- /** Why the sandbox can't enforce the attempt's policy, or undefined when it can. */
1638
+ /** Why the sandbox can't enforce the attempt's policies, or undefined when it can. */
1539
1639
  function unenforceable(planned, sandbox, where) {
1640
+ if (!planned.setup) return unenforceablePhase(planned, sandbox, where, "egress");
1641
+ if (!capabilitiesOf(sandbox).egress.switches) return `${where} runs under egress ${describeEgress(planned.setup.policy)} for setup and ${describeEgress(planned.policy)} for the agent, but the ${sandbox.kind} sandbox can't change its egress policy during an attempt. Give both phases one policy, or run it in ${suggestedSandbox((c) => c.isolated && c.egress.switches)}.`;
1642
+ return unenforceablePhase(planned.setup, sandbox, where, "egress.setup") ?? unenforceablePhase(planned, sandbox, where, "egress.agent");
1643
+ }
1644
+ function unenforceablePhase(planned, sandbox, where, key) {
1540
1645
  const capabilities = capabilitiesOf(sandbox);
1541
1646
  const can = capabilities.egress.enforce;
1542
- if (planned.policy === "proxy" && capabilities.isolated && can === "none") return `${where} runs under egress 'proxy', where the egress gateway is the only way out, but the ${sandbox.kind} sandbox enforces no egress policy: its traffic that ignores the proxy goes out directly. Set egress: 'allow-all' to run it there, or run it in ${suggestedSandbox((c) => c.isolated && c.egress.enforce !== "none")}.`;
1647
+ if (planned.policy === "proxy" && capabilities.isolated && can === "none") return `${where} runs under ${key} 'proxy', where the egress gateway is the only way out, but the ${sandbox.kind} sandbox enforces no egress policy: its traffic that ignores the proxy goes out directly. Set ${key}: 'allow-all' to run it there, or run it in ${suggestedSandbox((c) => c.isolated && c.egress.enforce !== "none")}.`;
1543
1648
  const needs = egressNeeds(planned);
1544
1649
  if (LEVELS.indexOf(can) >= LEVELS.indexOf(needs)) return void 0;
1545
1650
  const hosts = allowedHosts(planned)?.join(", ");
1546
1651
  const what = needs !== "allowlist" ? "turning egress off" : planned.policy === "deny-all" ? `a host allowlist for its automatic hosts (${hosts})` : `a host allowlist (${hosts})`;
1547
1652
  const enforces = can === "none" ? "enforces no egress policy" : `only enforces ${can === "all-or-nothing" ? "'allow-all' or 'deny-all'" : can}`;
1548
1653
  const enforcing = suggestedSandbox((c) => LEVELS.indexOf(c.egress.enforce) >= LEVELS.indexOf(needs));
1549
- const hint = capabilities.isolated ? "" : ` (its commands run on the host). ${enforcing ? `Run it in ${enforcing}, or set` : "Set"} egress: 'allow-all'`;
1550
- return `${where} runs under egress ${describeEgress(planned.policy)}, which needs ${what}, but the ${sandbox.kind} sandbox ${enforces}${hint}.`;
1654
+ const hint = capabilities.isolated ? "" : ` (its commands run on the host). ${enforcing ? `Run it in ${enforcing}, or set` : "Set"} ${key}: 'allow-all'`;
1655
+ return `${where} runs under ${key} ${describeEgress(planned.policy)}, which needs ${what}, but the ${sandbox.kind} sandbox ${enforces}${hint}.`;
1551
1656
  }
1552
1657
  const describeEgress = (policy) => typeof policy === "string" ? `'${policy}'` : `{ allow: [${policy.allow.map((h) => `'${h}'`).join(", ")}] }`;
1553
1658
  /** A CI runner as the environment names it, which usually runs on a cloud VM; null elsewhere. */
@@ -1563,8 +1668,35 @@ function ciRunner(env) {
1563
1668
  * so anything in it reaches cloud metadata and the host's network around the gateway.
1564
1669
  */
1565
1670
  const directOnCloud = (where) => `egress 'allow-all' on ${where}: anything in the sandbox can reach the cloud metadata service (169.254.169.254, which hands out the machine's credentials), the host's network and its services around the egress gateway, unfiltered and unrecorded`;
1566
- /** `deny-all + auto: api2.cursor.sh, …` or `allow-all` for the plan; '' under `proxy`, the default. */
1567
- function egressLabel({ policy, auto }) {
1671
+ /**
1672
+ * What running attempts with a setup policy of their own (`ids`) side by side on rootless Podman
1673
+ * risks: a container leaving Podman's bridge, as one does at the switch, can cut the others on it
1674
+ * off from the host for seconds.
1675
+ */
1676
+ const rootlessPodmanSwitch = (ids) => `phased egress on rootless Podman: at the switch to the agent's policy an attempt's container leaves Podman's bridge network, which can cut the other containers on it off from the host for seconds, failing their direct connections (allow-all, and setup steps under it). Under it: ${ids.join(", ")}. Run with --concurrency 1, or on docker(), when those connections matter`;
1677
+ /**
1678
+ * `deny-all + auto: api2.cursor.sh, …` or `allow-all` for the plan; '' under `proxy`, the default.
1679
+ * With a setup policy of its own, `setup allow-all · agent deny-all + auto: …`.
1680
+ */
1681
+ function egressLabel(planned) {
1682
+ if (!planned.setup) return phaseLabel(planned);
1683
+ return `setup ${phaseLabel(planned.setup) || "proxy"} · agent ${phaseLabel(planned) || "proxy"}`;
1684
+ }
1685
+ /** {@link egressLabel} without the automatic hosts, which {@link autoHosts} lists. */
1686
+ function ownEgressLabel(planned) {
1687
+ const own = (p) => ({
1688
+ ...p,
1689
+ auto: []
1690
+ });
1691
+ return egressLabel({
1692
+ ...own(planned),
1693
+ ...planned.setup ? { setup: own(planned.setup) } : {}
1694
+ });
1695
+ }
1696
+ /** The hosts reeval allows on top of the policy, in either phase. */
1697
+ const autoHosts = (planned) => [.../* @__PURE__ */ new Set([...planned.setup?.auto ?? [], ...planned.auto])];
1698
+ /** One phase's policy and automatic hosts, or `''` for the default, `proxy`. */
1699
+ function phaseLabel({ policy, auto }) {
1568
1700
  if (policy === "proxy") return "";
1569
1701
  if (policy === "allow-all") return "allow-all";
1570
1702
  const own = policy === "deny-all" ? "deny-all" : policy.allow.length ? `allow ${policy.allow.join(", ")}` : "allow none";
@@ -1921,7 +2053,7 @@ function normalizeCompare(value) {
1921
2053
  }
1922
2054
  /**
1923
2055
  * Runs per cell when no level sets `runs`: a compare with variants runs `COMPARE_RUNS` a side;
1924
- * anything else, a pooled family's cases included, runs once.
2056
+ * anything else runs once, two or more cases with a compare included, whose lift pairs cases.
1925
2057
  */
1926
2058
  const COMPARE_DEFAULT_RUNS = 4;
1927
2059
  function applyLevel(out, options, where, egress = "own") {
@@ -1929,7 +2061,7 @@ function applyLevel(out, options, where, egress = "own") {
1929
2061
  ...out.sources,
1930
2062
  [key]: where
1931
2063
  };
1932
- if (options.egress !== void 0) out.egress[egress] = options.egress;
2064
+ if (options.egress !== void 0) out.egress[egress] = layerEgress(out.egress[egress], options.egress);
1933
2065
  out.harness = applyHarness(out.harness, options.harnesses, where);
1934
2066
  if (options.sandbox !== void 0) out.sandbox = options.sandbox;
1935
2067
  out.environments = applyEnvironments(out.environments, options.environments, where);
@@ -2118,10 +2250,15 @@ function cascade(base, levels, where, options = {}) {
2118
2250
  /** Gives `out` its default runs when no level set them and `--runs` doesn't replace them. */
2119
2251
  function defaultRuns(out, notes, options) {
2120
2252
  if (out.runs !== void 0 || options.runs !== void 0) return;
2121
- if (!out.compare?.variants.length || options.pooled) {
2253
+ if (!out.compare?.variants.length) {
2122
2254
  out.runs = 1;
2123
2255
  return;
2124
2256
  }
2257
+ if (options.pooled) {
2258
+ out.runs = 1;
2259
+ notes.push("runs 1 a case (compare with cases: the lift pairs cases)");
2260
+ return;
2261
+ }
2125
2262
  out.runs = COMPARE_DEFAULT_RUNS;
2126
2263
  out.sources = {
2127
2264
  ...out.sources,
@@ -2302,6 +2439,18 @@ function plainHeaders(attempts) {
2302
2439
  const templateHint = (h) => `read it from a host variable with secret.template, e.g. secret.template('Bearer \${VAR}')${h.https ? "" : " and give the MCP server an https:// URL"}`;
2303
2440
  const plainHeaderWarning = (h) => `${h.label} is a plain string, so it's in the sandbox as its real value. To give the agent only a placeholder, ${templateHint(h)}.`;
2304
2441
  const plainHeaderError = (h, where, configFile) => `${h.label} in ${where} is a plain string, and secrets.unbrokered is 'error' in ${configFile}. To broker it, ${templateHint(h)}.`;
2442
+ /** The attempts' `env` strings that look like credentials, once each. */
2443
+ function plainEnv(env, attempts) {
2444
+ const out = /* @__PURE__ */ new Map();
2445
+ for (const a of attempts) for (const e of plainCredentialEnv(env, a.environment.layers)) out.set(e.label, e);
2446
+ return [...out.values()];
2447
+ }
2448
+ /** A placeholder only works for a value sent to an HTTPS API: the gateway swaps it in nowhere else. */
2449
+ const hostsExample = (e) => `secret.fromEnv('${e.name}', { hosts: ['api.example.com'] })`;
2450
+ const maskHint = (e) => e.plugin ? `write it as "\${${e.name}}" in the plugin's .mcp.json, so reeval reads it from the host variable ${e.name}` : `read it from a host variable with secret.fromEnv('${e.name}'); to give the agent only a placeholder too, add the hosts it's sent to if that's an HTTPS API, e.g. ${hostsExample(e)}`;
2451
+ const plainEnvWarning = (e) => `${e.label} is a plain string that looks like a credential, so it's in the sandbox as its real value. To mask it in everything reeval writes, ${maskHint(e)}.`;
2452
+ const brokerEnvHint = (e) => e.plugin ? `declare the server with mcp.stdio() instead of the plugin's .mcp.json, whose host variables can't set hosts, and read it with ${hostsExample(e)}, naming the HTTPS hosts it's sent to` : `read it from a host variable with ${hostsExample(e)}, naming the HTTPS hosts it's sent to`;
2453
+ const plainEnvError = (e, where, configFile) => `${e.label} in ${where} is a plain string that looks like a credential, and secrets.unbrokered is 'error' in ${configFile}. To broker it, ${brokerEnvHint(e)}.`;
2305
2454
  const levelOf = (where) => where.startsWith("evaluate(") ? "eval" : where.startsWith("group(") ? "group" : where.startsWith("profiles.") ? "profile" : "config";
2306
2455
  /**
2307
2456
  * A warning per privilege an attempt's Docker sandbox adds, naming the level that set the
@@ -2375,11 +2524,20 @@ function evalPlan(r, attempts, origin) {
2375
2524
  ...environmentWarnings(attempts),
2376
2525
  ...skip ? [] : questionWarnings(resolved),
2377
2526
  ...unbrokered(r.resolved.env, attempts).filter(reachesSandbox).map(unbrokeredWarning),
2378
- ...plainHeaders(attempts).map(plainHeaderWarning)
2527
+ ...plainHeaders(attempts).map(plainHeaderWarning),
2528
+ ...plainEnv(r.resolved.env, attempts).map(plainEnvWarning)
2379
2529
  ],
2380
2530
  sandboxWarnings: skip ? [] : sandboxWarnings(attempts, resolved, origin),
2381
2531
  brokered: [...new Set(attempts.flatMap((a) => brokeredIn(r.resolved.env, a)))],
2382
- egress: [...new Set(attempts.map((a) => egressLabel(a.egress)).filter(Boolean))],
2532
+ egress: [...new Map(attempts.map((a) => {
2533
+ const e = {
2534
+ environment: a.environment.name,
2535
+ harness: harnessId(a.harness),
2536
+ policy: ownEgressLabel(a.egress) || "proxy",
2537
+ auto: autoHosts(a.egress)
2538
+ };
2539
+ return [JSON.stringify(e), e];
2540
+ })).values()],
2383
2541
  workspaces: [...new Set(attempts.flatMap((a) => {
2384
2542
  const w = workspaceOf(a.environment.layers);
2385
2543
  return w ? [workspaceLabel(w)] : [];
@@ -2414,6 +2572,7 @@ const GATEWAYS = {
2414
2572
  const fxEffort = fxEffortRefusal(h, "direct()");
2415
2573
  if (fxEffort) return fxEffort;
2416
2574
  if (h.kind === "fx" && h.model.includes("/")) return `fx runs a gateway id such as ${JSON.stringify(h.model)} only through Vercel AI Gateway or OpenRouter`;
2575
+ if (h.kind === "pi" || h.kind === "goose") return directProviderRefusal(h.kind, h.model);
2417
2576
  if (h.kind === "opencode") {
2418
2577
  const resolved = resolveOpenCodeProvider(h.model, process.env);
2419
2578
  if ("error" in resolved) return resolved.error;
@@ -2438,7 +2597,7 @@ const GATEWAYS = {
2438
2597
  /** OpenCode's id for `model` on `gateway`, as that gateway's route writes it. */
2439
2598
  function openCodeModelOn(model, gateway) {
2440
2599
  if (gateway === "vertex") return openCodeVertexId(model);
2441
- return gateway === "vercel" || gateway === "openrouter" ? `${gateway}/${model}` : model;
2600
+ return isRoutingGateway(gateway) ? `${gateway}/${model}` : model;
2442
2601
  }
2443
2602
  /** Why the API behind `gateway` answers OpenCode's `effort` on `model` with a 400, or undefined. */
2444
2603
  function openCodeEffortOn(model, gateway, effort) {
@@ -2627,7 +2786,8 @@ const PROFILE_KEYS = [
2627
2786
  "budget",
2628
2787
  "record",
2629
2788
  "tags",
2630
- "gateway"
2789
+ "gateway",
2790
+ "history"
2631
2791
  ];
2632
2792
  const DEFAULTS_KEYS = [
2633
2793
  "harnesses",
@@ -2649,11 +2809,9 @@ const LIMIT_KEYS = [
2649
2809
  "harnesses",
2650
2810
  "traits"
2651
2811
  ];
2652
- /**
2653
- * The config: a plain object, or a callback that destructures the helpers it needs, e.g.
2654
- * `defineConfig(({ secret }) => ({ env: { KEY: secret.fromEnv('KEY') } }))`.
2655
- */
2656
- const defineConfig = (config) => config;
2812
+ function defineConfig(config) {
2813
+ return config;
2814
+ }
2657
2815
  /** The script configs `reeval` looks for in each directory, first match first, as Vitest orders its own. */
2658
2816
  const SCRIPT_FILES = [
2659
2817
  "ts",
@@ -2845,11 +3003,6 @@ function evalRoot(include) {
2845
3003
  for (let i = 0; dirs.every((d) => d[i] !== void 0 && d[i] === dirs[0]?.[i]); i++) common.push(dirs[0]?.[i]);
2846
3004
  return common.join("/");
2847
3005
  }
2848
- const ARTIFACTS = [
2849
- "all",
2850
- "failures",
2851
- "none"
2852
- ];
2853
3006
  const RECORD = [
2854
3007
  "manual",
2855
3008
  "ci",
@@ -2869,7 +3022,7 @@ const unknownKey = (value, allowed, where, file) => {
2869
3022
  function checkBudget(budget, where, file) {
2870
3023
  if (budget === void 0) return {};
2871
3024
  if (typeof budget !== "object" || budget === null) throw new UsageError(`${where} in ${file} must be an object like { attempts: 20, duration: '1h' } (got ${JSON.stringify(budget)})`);
2872
- if ("usd" in budget || "tokens" in budget) throw new UsageError(`${where}.${"usd" in budget ? "usd" : "tokens"} in ${file} isn't supported yet: reeval records the tokens harnesses report, but prices none and enforces no token budget. Cap the run with attempts or duration.`);
3025
+ if ("usd" in budget || "tokens" in budget) throw new UsageError(`${where}.${"usd" in budget ? "usd" : "tokens"} in ${file} isn't supported yet: reeval records the tokens and cost of each attempt, but enforces no cost or token budget. Cap the run with attempts or duration.`);
2873
3026
  unknownKey(budget, ["attempts", "duration"], where, file);
2874
3027
  const { attempts, duration } = budget;
2875
3028
  if (attempts !== void 0 && !(Number.isInteger(attempts) && attempts >= 1)) throw new UsageError(`${where}.attempts in ${file} must be a whole number of at least 1 (got ${JSON.stringify(attempts)})`);
@@ -3002,9 +3155,17 @@ function selectProfile(config, flag, env = process.env) {
3002
3155
  ...profile.budget
3003
3156
  },
3004
3157
  ...profile.record ? { record: profile.record } : {},
3005
- ...profile.gateway ? { gateway: profile.gateway } : {}
3158
+ ...profile.gateway ? { gateway: profile.gateway } : {},
3159
+ ...config.profileHistory[name] ? { history: config.profileHistory[name] } : {}
3006
3160
  };
3007
3161
  }
3162
+ /** The config with the profile's `history` in place of its own, when the profile sets one. */
3163
+ function withProfileHistory(config, profile) {
3164
+ return profile?.history ? {
3165
+ ...config,
3166
+ history: profile.history
3167
+ } : config;
3168
+ }
3008
3169
  /**
3009
3170
  * The config with its defaults filled in and checked; `root` is its directory, which relative
3010
3171
  * host paths written as data resolve against.
@@ -3018,16 +3179,23 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3018
3179
  if (!Array.isArray(include) || include.some((p) => typeof p !== "string")) throw new UsageError(`include in ${file} must be a list of glob patterns, e.g. include: ['evals/**/*.eval.ts'] (got ${JSON.stringify(include)})`);
3019
3180
  const record = user.record ?? "ci";
3020
3181
  if (!RECORD.includes(record)) throw new UsageError(`record in ${file} is ${oneOf(RECORD)} (got ${JSON.stringify(record)})`);
3021
- const artifacts = user.history?.artifacts ?? "all";
3022
- if (!ARTIFACTS.includes(artifacts)) throw new UsageError(`history.artifacts in ${file} is ${oneOf(ARTIFACTS)} (got ${JSON.stringify(artifacts)})`);
3023
- const historyDir = resolve(root, user.history?.dir ?? join(evalRoot(include), ".history"));
3182
+ const historyDefault = join(evalRoot(include), ".history");
3183
+ const history = resolveHistory(user.history, "history", file, root, historyDefault);
3024
3184
  const harness = user.harnesses === void 0 ? [DEFAULT_HARNESS] : [user.harnesses].flat();
3025
3185
  if (harness.length === 0) throw new UsageError(`harnesses is an empty list in ${file}. Give at least one, e.g. harnesses: ${HARNESS_EXAMPLE}`);
3026
3186
  checkHarnesses(harness, file);
3027
3187
  const profiles = {};
3188
+ const profileHistory = {};
3028
3189
  if (user.profiles !== void 0) {
3029
3190
  if (typeof user.profiles !== "object" || user.profiles === null) throw new UsageError(`profiles in ${file} must be an object of named profiles, e.g. profiles: { pr: { limit: { runs: 1 } } } (got ${JSON.stringify(user.profiles)})`);
3030
- for (const [name, profile] of Object.entries(user.profiles)) profiles[name] = checkProfile(name, profile, file);
3191
+ for (const [name, profile] of Object.entries(user.profiles)) {
3192
+ const { history: own, ...checked } = checkProfile(name, profile, file);
3193
+ profiles[name] = checked;
3194
+ if (own !== void 0) {
3195
+ const where = `profiles.${name}.history`;
3196
+ profileHistory[name] = resolveHistory(own, where, file, root, historyDefault, history);
3197
+ }
3198
+ }
3031
3199
  }
3032
3200
  const defaultProfile = user.defaultProfile ?? {};
3033
3201
  unknownKey(defaultProfile, ["ci", "local"], "defaultProfile", file);
@@ -3063,10 +3231,8 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
3063
3231
  }, user.snapshot),
3064
3232
  concurrency: user.concurrency === void 0 ? void 0 : checkConcurrency(user.concurrency, `concurrency in ${file}`),
3065
3233
  record,
3066
- history: {
3067
- dir: historyDir,
3068
- artifacts
3069
- },
3234
+ history,
3235
+ profileHistory,
3070
3236
  secrets: checkSecrets(user.secrets, file),
3071
3237
  debug: checkDebug(user.debug, file),
3072
3238
  analysis: checkAnalysis(user.analysis, file),
@@ -3146,11 +3312,11 @@ function checkConcurrency(value, where) {
3146
3312
  if (typeof n !== "number" || !Number.isInteger(n) || n < 1) throw new UsageError(`${where} must be a whole number of at least 1 (got ${String(value)})`);
3147
3313
  return n;
3148
3314
  }
3149
- /** `--concurrency`, then `REEVAL_CONCURRENCY`, then `concurrency` in the config, then the default. */
3150
- async function resolveConcurrency(config, flag, capacity, env = process.env) {
3315
+ /** `--concurrency`, then `REEVAL_CONCURRENCY`, then `concurrency` in the config, or undefined. */
3316
+ function givenConcurrency(config, flag, env = process.env) {
3151
3317
  if (flag !== void 0) return checkConcurrency(flag, "--concurrency");
3152
3318
  if (env.REEVAL_CONCURRENCY) return checkConcurrency(env.REEVAL_CONCURRENCY, "REEVAL_CONCURRENCY");
3153
- return config.concurrency ?? defaultConcurrency(await capacity());
3319
+ return config.concurrency;
3154
3320
  }
3155
3321
  function resolveJudge(judge, file, path = "judge") {
3156
3322
  judge ??= {};
@@ -3192,8 +3358,8 @@ function missingCredentials(config, env = process.env, options = {}) {
3192
3358
  const line = `${s.apiKey.from} (for boat())`;
3193
3359
  if (!missing.includes(line)) missing.push(line);
3194
3360
  }
3195
- const keys = new Set((options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []));
3196
- for (const key of keys) if (!env[key]) missing.push(key);
3361
+ const keys = new Map((options.harnesses ?? config.harness).map(harnessCredentials).filter((names) => names.length > 0).map((names) => [names.join(" or "), names]));
3362
+ for (const [line, names] of keys) if (!names.some((k) => env[k])) missing.push(line);
3197
3363
  const judge = runJudgeModel(config, options.profile);
3198
3364
  const models = [
3199
3365
  ...typeof judge === "string" ? [{
@@ -3239,7 +3405,7 @@ function missingCredentials(config, env = process.env, options = {}) {
3239
3405
  * then hangs on a request it can't send rather than failing to authenticate.
3240
3406
  */
3241
3407
  function malformedCredentials(config, env = process.env, options = {}) {
3242
- const names = (options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []);
3408
+ const names = (options.harnesses ?? config.harness).flatMap((h) => harnessCredentials(h).find((name) => env[name]) ?? []);
3243
3409
  const judge = runJudgeModel(config, options.profile);
3244
3410
  if (judge !== void 0) names.push(...judgeCredentials(judge, runModelGateway(config, options.profile, judge, true)));
3245
3411
  for (const { model, gateway } of options.judges ?? []) names.push(...judgeCredentials(model, gateway));
@@ -3249,6 +3415,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
3249
3415
  return at ? [`${name} (character ${at})`] : [];
3250
3416
  });
3251
3417
  }
3252
- export { runsOf as $, brokeredIn as A, execOptionsProblem as At, expand as B, decideState as Bt, openCodeModelOn as C, resolveStateOptions as Ct, EVAL_OPTION_KEYS as D, CAPTURE_NAME as Dt, COMPARE_DEFAULT_RUNS as E, observer as Et, checkHarnesses as F, NO_JUDGE_INPUT as Ft, mergeSnapshot as G, levelOf as H, resolveJudgeArtifacts as Ht, checkLimitHarness as I, inputsOn as It, plainHeaders as J, plainHeaderError as K, checkOptions as L, judgeProblem as Lt, cellKey as M, resolvedEffortOf as Mt, checkCapture as N, servedEffortOf as Nt, MAJORITY_GATE as O, CAPTURE_NAME_RULE as Ot, checkGate as P, INPUT_KEYS as Pt, refuseIllegalCompare as Q, environmentWarnings as R, artifactRefsOf as Rt, fallbackHint as S, mergeJudge as St, routedHarness as T, observedName as Tt, majorityOf as U, harnessId as V, filePartsOf as Vt, matchesTags as W, plannedSandboxes as X, plannedSandboxIds as Y, profileJudge as Z, fellBackToDirect as _, resolveOutput as _t, configPathFor as a, unbrokered as at, resolveModelGateway as b, questionShows as bt, findConfig as c, DEFAULT_EGRESS as ct, malformedCredentials as d, ciRunner as dt, sandboxId as et, missingCredentials as f, directOnCloud as ft, selectProfile as g, unenforceable as gt, runJudgeModel as h, isDirect as ht, configLabel as i, secretVarsIn as it, cascade as j, harnessEffort as jt, attemptSandbox as k, attemptRequestProblem as kt, hostCapacity as l, DEFAULT_EGRESS_ALLOW as lt, resolveConcurrency as m, gatewayPolicy as mt, analysisModelOf as n, sandboxSetBy as nt, defaultConcurrency as o, unbrokeredError as ot, requireConfig as p, egressLabel as pt, plainHeaderWarning as q, configFilesPhrase as r, sandboxWarnings as rt, defineConfig as s, unbrokeredWarning as st, CONFIG_FILES as t, sandboxProvider as tt, loadConfig as u, allowedHosts as ut, gatewayScope as v, STOPPED_FIRST as vt, resolveGateway as w, observedHere as wt, effortRefusedOn as x, resolveQuestionState as xt, modelGatewayOrNone as y, UNRECORDED_QUESTION_STATE as yt, evalPlan as z, badJudgeArtifacts as zt };
3418
+ export { plannedSandboxIds as $, resolveJudgeArtifacts as $t, attemptSandbox as A, UNRECORDED_QUESTION_STATE as At, evalPlan as B, attemptRequestProblem as Bt, fallbackHint as C, phaseLabel as Ct, COMPARE_DEFAULT_RUNS as D, unenforceable as Dt, routedHarness as E, setupEgress as Et, checkGate as F, observedHere as Ft, matchesTags as G, INPUT_KEYS as Gt, harnessId as H, harnessEffort as Ht, checkHarnesses as I, observedName as It, plainEnvError as J, judgeProblem as Jt, mergeSnapshot as K, NO_JUDGE_INPUT as Kt, checkLimitHarness as L, observer as Lt, cascade as M, resolveQuestionState as Mt, cellKey as N, mergeJudge as Nt, EVAL_OPTION_KEYS as O, resolveOutput as Ot, checkCapture as P, resolveStateOptions as Pt, plainHeaders as Q, filePartsOf as Qt, checkOptions as R, CAPTURE_NAME as Rt, effortRefusedOn as S, layerEgress as St, resolveGateway as T, rootlessPodmanSwitch as Tt, levelOf as U, resolvedEffortOf as Ut, expand as V, execOptionsProblem as Vt, majorityOf as W, servedEffortOf as Wt, plainHeaderError as X, badJudgeArtifacts as Xt, plainEnvWarning as Y, artifactRefsOf as Yt, plainHeaderWarning as Z, decideState as Zt, withProfileHistory as _, directOnCloud as _t, configPathFor as a, sandboxProvider as at, modelGatewayOrNone as b, isDirect as bt, findConfig as c, secretVarsIn as ct, loadConfig as d, unbrokeredWarning as dt, plannedSandboxes as et, malformedCredentials as f, DEFAULT_EGRESS as ft, selectProfile as g, ciRunner as gt, runJudgeModel as h, bareEgress as ht, configLabel as i, sandboxId as it, brokeredIn as j, questionShows as jt, MAJORITY_GATE as k, STOPPED_FIRST as kt, givenConcurrency as l, unbrokered as lt, requireConfig as m, allowedHosts as mt, analysisModelOf as n, refuseIllegalCompare as nt, defaultConcurrency as o, sandboxSetBy as ot, missingCredentials as p, DEFAULT_EGRESS_ALLOW as pt, plainEnv as q, inputsOn as qt, configFilesPhrase as r, runsOf as rt, defineConfig as s, sandboxWarnings as st, CONFIG_FILES as t, profileJudge as tt, hostCapacity as u, unbrokeredError as ut, fellBackToDirect as v, egressLabel as vt, openCodeModelOn as w, phasePolicies as wt, resolveModelGateway as x, isPhased as xt, gatewayScope as y, gatewayPolicy as yt, environmentWarnings as z, CAPTURE_NAME_RULE as zt };
3253
3419
 
3254
- //# sourceMappingURL=config-SvK8pjEK.js.map
3420
+ //# sourceMappingURL=config-C0URar0t.js.map