reeval 0.19.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (436) hide show
  1. package/CHANGELOG.md +803 -0
  2. package/README.md +15 -15
  3. package/THIRD_PARTY_NOTICES.md +5 -5
  4. package/assets/gateway-models.mjs +65 -31
  5. package/dist/{adapter-C8xPuXYq.js → adapter-BdzhdRYo.js} +8 -6
  6. package/dist/adapter-BdzhdRYo.js.map +1 -0
  7. package/dist/{agent-CBoLndM7.js → agent-Dx-bDmlV.js} +179 -31
  8. package/dist/agent-Dx-bDmlV.js.map +1 -0
  9. package/dist/{agents-C6zhkxp3.js → agents-l0-k7vw3.js} +362 -102
  10. package/dist/agents-l0-k7vw3.js.map +1 -0
  11. package/dist/{app-files-DT5mJyV4.js → app-files-B-CQYB0c.js} +3 -3
  12. package/dist/app-files-B-CQYB0c.js.map +1 -0
  13. package/dist/{assemble-UPCRrtOz.js → assemble-CYMpN8LC.js} +1 -1
  14. package/dist/{assemble-t6h018An.js → assemble-CsZIcJo8.js} +171 -18
  15. package/dist/assemble-CsZIcJo8.js.map +1 -0
  16. package/dist/{attempt-CAdlwlNu.js → attempt-CKfKsqqw.js} +2 -2
  17. package/dist/{attempt-CAdlwlNu.js.map → attempt-CKfKsqqw.js.map} +1 -1
  18. package/dist/{attempt-host-CIm1-11m.js → attempt-host-BwSmNYkX.js} +17 -8
  19. package/dist/attempt-host-BwSmNYkX.js.map +1 -0
  20. package/dist/{boat-BjLqFJIZ.js → boat-CZpAreTB.js} +3 -3
  21. package/dist/{boat-BjLqFJIZ.js.map → boat-CZpAreTB.js.map} +1 -1
  22. package/dist/{boat-CJCtzfB1.js → boat-su1Y7LPd.js} +17 -14
  23. package/dist/boat-su1Y7LPd.js.map +1 -0
  24. package/dist/bridge/index.mjs +113 -8
  25. package/dist/{capture-iBzFiC0j.js → capture-BqIAeFyC.js} +55 -29
  26. package/dist/{capture-iBzFiC0j.js.map → capture-BqIAeFyC.js.map} +1 -1
  27. package/dist/capture-D0Ln5aCP.js +1 -0
  28. package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
  29. package/dist/cases-CM-MQRQp.js.map +1 -0
  30. package/dist/cases-FvhbM8M5.js +1 -0
  31. package/dist/{cli-20Fpvs5H.js → cli-s_3opn_I.js} +219 -109
  32. package/dist/cli-s_3opn_I.js.map +1 -0
  33. package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-DxzIjmq_.js} +109 -26
  34. package/dist/cli-spec-DxzIjmq_.js.map +1 -0
  35. package/dist/cli.js +2 -2
  36. package/dist/client-C5fIzHQa.js +803 -0
  37. package/dist/client-C5fIzHQa.js.map +1 -0
  38. package/dist/{command-CRh-cb6Z.js → command-B5OAIjp7.js} +25 -21
  39. package/dist/command-B5OAIjp7.js.map +1 -0
  40. package/dist/{command-BLNVDiIg.js → command-BJ7KqczP.js} +10 -10
  41. package/dist/{command-BLNVDiIg.js.map → command-BJ7KqczP.js.map} +1 -1
  42. package/dist/{command-aUJvENwy.js → command-BXLZsEwQ.js} +16 -16
  43. package/dist/{command-aUJvENwy.js.map → command-BXLZsEwQ.js.map} +1 -1
  44. package/dist/{command-B5OyHpOu.js → command-DA_SHQUQ.js} +16 -16
  45. package/dist/{command-B5OyHpOu.js.map → command-DA_SHQUQ.js.map} +1 -1
  46. package/dist/{commands-CWy-WndG.js → commands-BKBamOcX.js} +174 -40
  47. package/dist/commands-BKBamOcX.js.map +1 -0
  48. package/dist/{commands-CfdBX-mc.js → commands-BPvAOEmb.js} +1 -1
  49. package/dist/{config-B9mB1kF5.js → config-C0URar0t.js} +255 -89
  50. package/dist/config-C0URar0t.js.map +1 -0
  51. package/dist/{context-Bkv3ndjq.js → context-Sj58aJEu.js} +2 -2
  52. package/dist/{context-Bkv3ndjq.js.map → context-Sj58aJEu.js.map} +1 -1
  53. package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
  54. package/dist/cost-C21gGrju.js.map +1 -0
  55. package/dist/cost-IM80LyGD.js +1 -0
  56. package/dist/{cursor-catalog-BSn9xjmo.js → cursor-catalog-FWgSo-T1.js} +4 -4
  57. package/dist/{cursor-catalog-BSn9xjmo.js.map → cursor-catalog-FWgSo-T1.js.map} +1 -1
  58. package/dist/db-Bc4L-Wcr.js +2 -0
  59. package/dist/{db-zgC1tvKB.js → db-BjAWxVAG.js} +142 -248
  60. package/dist/db-BjAWxVAG.js.map +1 -0
  61. package/dist/debug-CIXzB4wK.js.map +1 -1
  62. package/dist/{default-effort-BAy8ixd1.js → default-effort-CWlhZ6U1.js} +1 -1
  63. package/dist/{default-effort-DGhQmu95.js → default-effort-Ck2GYIxX.js} +4 -4
  64. package/dist/{default-effort-DGhQmu95.js.map → default-effort-Ck2GYIxX.js.map} +1 -1
  65. package/dist/{dist-CAFD1yfv.js → dist-DeqeCUZe.js} +15 -8
  66. package/dist/dist-DeqeCUZe.js.map +1 -0
  67. package/dist/{docker-DAMnm3L9.js → docker-CDdK6Ity.js} +49 -22
  68. package/dist/docker-CDdK6Ity.js.map +1 -0
  69. package/dist/{docker-Cqw-CyWz.js → docker-D74l67k8.js} +4 -4
  70. package/dist/{docker-Cqw-CyWz.js.map → docker-D74l67k8.js.map} +1 -1
  71. package/dist/{docker-Dr4XeDc8.js → docker-DMug3l-a.js} +1 -1
  72. package/dist/{doctor-B2PohneB.js → doctor-eFxyn6at.js} +85 -46
  73. package/dist/doctor-eFxyn6at.js.map +1 -0
  74. package/dist/{environment-DB8o3hGi.js → environment-CJIKz-Ud.js} +49 -11
  75. package/dist/environment-CJIKz-Ud.js.map +1 -0
  76. package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
  77. package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
  78. package/dist/{eval-source-D3sbAzt-.js → eval-source-CXIRJ91v.js} +24 -12
  79. package/dist/eval-source-CXIRJ91v.js.map +1 -0
  80. package/dist/{evaluate-CwKeN_fo.js → evaluate-DA4QsrHO.js} +792 -281
  81. package/dist/evaluate-DA4QsrHO.js.map +1 -0
  82. package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DHrU4Wld.d.ts} +2 -2
  83. package/dist/fake-bootstrap-CnwWG3kP.js +15 -0
  84. package/dist/fake-bootstrap-CnwWG3kP.js.map +1 -0
  85. package/dist/{files-BiFU16BW.js → files-BVVWAS0A.js} +6 -6
  86. package/dist/{files-BiFU16BW.js.map → files-BVVWAS0A.js.map} +1 -1
  87. package/dist/files-BaqXvB88.js +1 -0
  88. package/dist/{files-BEwfjD8J.js → files-Bey1VAex.js} +3 -3
  89. package/dist/{files-BEwfjD8J.js.map → files-Bey1VAex.js.map} +1 -1
  90. package/dist/files-BjOJw4sn.js +1 -0
  91. package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
  92. package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
  93. package/dist/files-CNHhSyb3.js +1 -0
  94. package/dist/files-CW1lChNh.js +1 -0
  95. package/dist/{files-BRzV7aPh.js → files-D6j-XpaK.js} +5 -5
  96. package/dist/{files-BRzV7aPh.js.map → files-D6j-XpaK.js.map} +1 -1
  97. package/dist/{files-Cl7BQ5gE.js → files-F44NRCN0.js} +143 -20
  98. package/dist/files-F44NRCN0.js.map +1 -0
  99. package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
  100. package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
  101. package/dist/gateway.d.ts +1 -1
  102. package/dist/{git-fDiJ8UIV.js → git-Jqkf9SBa.js} +3 -3
  103. package/dist/{git-fDiJ8UIV.js.map → git-Jqkf9SBa.js.map} +1 -1
  104. package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
  105. package/dist/harness-FcTyyPco.js.map +1 -0
  106. package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
  107. package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
  108. package/dist/harness.d.ts +11 -8
  109. package/dist/harness.js +1 -1
  110. package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
  111. package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
  112. package/dist/{images-BNgcR77K.js → images-Dsc4Wi-F.js} +65 -30
  113. package/dist/images-Dsc4Wi-F.js.map +1 -0
  114. package/dist/index.d.ts +46 -7
  115. package/dist/index.js +6 -6
  116. package/dist/init-C0oUG3jh.js +1 -0
  117. package/dist/{init-DbuZVRrW.js → init-QB5ecPDA.js} +33 -14
  118. package/dist/init-QB5ecPDA.js.map +1 -0
  119. package/dist/{ink-list-BpfidQ1h.js → ink-list-mCJypk3d.js} +2 -2
  120. package/dist/{ink-list-BpfidQ1h.js.map → ink-list-mCJypk3d.js.map} +1 -1
  121. package/dist/{invocations-CxGNOlh0.js → invocations-BSr4lRzI.js} +5 -5
  122. package/dist/{invocations-CxGNOlh0.js.map → invocations-BSr4lRzI.js.map} +1 -1
  123. package/dist/{judge-COMXT1Hx.js → judge-eGH39PaP.js} +12 -10
  124. package/dist/{judge-COMXT1Hx.js.map → judge-eGH39PaP.js.map} +1 -1
  125. package/dist/judge.d.ts +3 -1
  126. package/dist/judge.js +1 -1
  127. package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
  128. package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
  129. package/dist/{layout-Cm85tQYk.js → layout-DfQa4mJs.js} +14 -11
  130. package/dist/layout-DfQa4mJs.js.map +1 -0
  131. package/dist/{list-DuThsE28.js → list-Dh6T0kdr.js} +23 -12
  132. package/dist/{list-DuThsE28.js.map → list-Dh6T0kdr.js.map} +1 -1
  133. package/dist/live-DwjWJTdn.js +1 -0
  134. package/dist/{live-CgYbO4Te.js → live-xTZs0c61.js} +2 -2
  135. package/dist/{live-CgYbO4Te.js.map → live-xTZs0c61.js.map} +1 -1
  136. package/dist/{live-CnGrRM6j.js → live-y1p3oNXH.js} +9 -9
  137. package/dist/{live-CnGrRM6j.js.map → live-y1p3oNXH.js.map} +1 -1
  138. package/dist/{model-CwwkrCb7.js → model-3pZZzloU.js} +7 -5
  139. package/dist/model-3pZZzloU.js.map +1 -0
  140. package/dist/{model-047Dcd-a.js → model-DEjwetkJ.js} +1 -1
  141. package/dist/{model-V3mMXp6x.js → model-DNpKiHqO.js} +6 -6
  142. package/dist/{model-V3mMXp6x.js.map → model-DNpKiHqO.js.map} +1 -1
  143. package/dist/model-calls-B0BoQfJ4.js +95 -0
  144. package/dist/model-calls-B0BoQfJ4.js.map +1 -0
  145. package/dist/{model-DJjLw-0r.js → model-jRKSQBg6.js} +5 -5
  146. package/dist/{model-DJjLw-0r.js.map → model-jRKSQBg6.js.map} +1 -1
  147. package/dist/model-nfewGTzB.js +1 -0
  148. package/dist/{node-D7pAU--0.js → node-DM-mLGdX.js} +7 -7
  149. package/dist/{node-D7pAU--0.js.map → node-DM-mLGdX.js.map} +1 -1
  150. package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
  151. package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
  152. package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-C0aRo633.js} +2 -2
  153. package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-C0aRo633.js.map} +1 -1
  154. package/dist/options-DpsJ_5CU.js +1 -0
  155. package/dist/{patch-DH4e_Gkd.js → patch-Bx-qp8Li.js} +2 -2
  156. package/dist/{patch-DH4e_Gkd.js.map → patch-Bx-qp8Li.js.map} +1 -1
  157. package/dist/pi-providers-HygmL1y-.js +552 -0
  158. package/dist/pi-providers-HygmL1y-.js.map +1 -0
  159. package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
  160. package/dist/plain-Ctn8oUdp.js.map +1 -0
  161. package/dist/{plan-Cgq9NCEX.js → plan-ChO0A-xn.js} +63 -12
  162. package/dist/plan-ChO0A-xn.js.map +1 -0
  163. package/dist/{plugins-BkmPxDiY.js → plugins-BioMWncI.js} +636 -9
  164. package/dist/plugins-BioMWncI.js.map +1 -0
  165. package/dist/{preflight-R-InZCQb.js → preflight-BUjk_6wT.js} +1 -1
  166. package/dist/{preflight-CtlloeOQ.js → preflight-CJSGzMzc.js} +8 -9
  167. package/dist/preflight-CJSGzMzc.js.map +1 -0
  168. package/dist/preflight-DD3fI4Hz.js +1 -0
  169. package/dist/{preflight-BElE_q0u.js → preflight-qFCXmxha.js} +4 -4
  170. package/dist/{preflight-BElE_q0u.js.map → preflight-qFCXmxha.js.map} +1 -1
  171. package/dist/{prepare-C7f5mPl5.js → prepare-D7oSSFNm.js} +9 -9
  172. package/dist/{prepare-C7f5mPl5.js.map → prepare-D7oSSFNm.js.map} +1 -1
  173. package/dist/{prepare-CzkAdbXm.js → prepare-xHU2BpN-.js} +2 -2
  174. package/dist/prices-B1vm_lPk.js +1 -0
  175. package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
  176. package/dist/prices-BrY_ihLs.js.map +1 -0
  177. package/dist/{provenance-zsc4--dP.js → provenance-B-flBAzR.js} +5 -5
  178. package/dist/{provenance-zsc4--dP.js.map → provenance-B-flBAzR.js.map} +1 -1
  179. package/dist/provider-DHT20Gl8.js +1 -0
  180. package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
  181. package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
  182. package/dist/provider-stop-CxzWW7Ft.js +69 -0
  183. package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
  184. package/dist/{providers-B8xYMeah.js → providers-B6StVyfG.js} +215 -48
  185. package/dist/providers-B6StVyfG.js.map +1 -0
  186. package/dist/providers-Dljf6al6.js +1 -0
  187. package/dist/query-BAc2tQZj.js +1 -0
  188. package/dist/{query-Bb3SYICk.js → query-CkLmJv7R.js} +2 -2
  189. package/dist/{query-Bb3SYICk.js.map → query-CkLmJv7R.js.map} +1 -1
  190. package/dist/{questions-CWlR9ffu.js → questions-CJIh2FSX.js} +7 -7
  191. package/dist/{questions-CWlR9ffu.js.map → questions-CJIh2FSX.js.map} +1 -1
  192. package/dist/{record-6IUzwP9K.js → record-nVVujCW7.js} +2 -2
  193. package/dist/{record-6IUzwP9K.js.map → record-nVVujCW7.js.map} +1 -1
  194. package/dist/{redact-BhjtnjrU.js → redact-DL2BTykw.js} +17 -2
  195. package/dist/redact-DL2BTykw.js.map +1 -0
  196. package/dist/{remarks-vtkPmQb9.js → remarks-B3wjKscW.js} +46 -11
  197. package/dist/remarks-B3wjKscW.js.map +1 -0
  198. package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
  199. package/dist/remarks-BpdM_2RC.js.map +1 -0
  200. package/dist/remarks-CleoDlYw.js +1 -0
  201. package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
  202. package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
  203. package/dist/{remote-BgYWqogI.js → remote-C88amFRN.js} +6 -6
  204. package/dist/{remote-BgYWqogI.js.map → remote-C88amFRN.js.map} +1 -1
  205. package/dist/resolve-Bul8lHaa.js +205 -0
  206. package/dist/resolve-Bul8lHaa.js.map +1 -0
  207. package/dist/resolve-DW92wpIZ.js +1 -0
  208. package/dist/{resolve-Do0ZaYH6.js → resolve-G576PiFJ.js} +14 -11
  209. package/dist/resolve-G576PiFJ.js.map +1 -0
  210. package/dist/{results-JyuM64gy.js → results-zVHIW6xw.js} +9 -22
  211. package/dist/results-zVHIW6xw.js.map +1 -0
  212. package/dist/{resume-AW5oXzUc.js → resume-D_PcUXfG.js} +94 -16
  213. package/dist/resume-D_PcUXfG.js.map +1 -0
  214. package/dist/{retry-ChzISxmy.js → retry-BvTzCxdc.js} +4 -4
  215. package/dist/{retry-ChzISxmy.js.map → retry-BvTzCxdc.js.map} +1 -1
  216. package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
  217. package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
  218. package/dist/{vercel-CGlSBv63.js → routes-B0F2R-K7.js} +91 -56
  219. package/dist/routes-B0F2R-K7.js.map +1 -0
  220. package/dist/{runtime-BpW1f1Mx.js → runtime-B3r_pAq6.js} +4 -4
  221. package/dist/{runtime-BpW1f1Mx.js.map → runtime-B3r_pAq6.js.map} +1 -1
  222. package/dist/sandbox/boat.d.ts +1 -1
  223. package/dist/sandbox/boat.js +1 -1
  224. package/dist/{sandbox-kSgr7vo5.js → sandbox-Duh3z1TV.js} +3 -3
  225. package/dist/{sandbox-kSgr7vo5.js.map → sandbox-Duh3z1TV.js.map} +1 -1
  226. package/dist/sandbox.d.ts +1 -1
  227. package/dist/sandbox.js +1 -1
  228. package/dist/server-CGEsvVRG.js +1 -0
  229. package/dist/{server-Cn8VqO9S.js → server-DUWMRKik.js} +8 -8
  230. package/dist/{server-Cn8VqO9S.js.map → server-DUWMRKik.js.map} +1 -1
  231. package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
  232. package/dist/shared-DtZwIAfy.js.map +1 -0
  233. package/dist/{source-CYOhR98B.js → source-CHuN4Nfg.js} +2 -2
  234. package/dist/{source-CYOhR98B.js.map → source-CHuN4Nfg.js.map} +1 -1
  235. package/dist/source-Wz1Nrfue.js +1 -0
  236. package/dist/src--yl1OypK.js +2 -0
  237. package/dist/{src-Cb2MWozp.js → src-B-HuDqpH.js} +276 -41
  238. package/dist/src-B-HuDqpH.js.map +1 -0
  239. package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
  240. package/dist/src-B5r1d23Y.js.map +1 -0
  241. package/dist/{src-Ck2r_EMT.js → src-BHJ4MWvY.js} +504 -119
  242. package/dist/src-BHJ4MWvY.js.map +1 -0
  243. package/dist/{src-Dw79VLa3.js → src-BI0v8qlZ.js} +202 -56
  244. package/dist/src-BI0v8qlZ.js.map +1 -0
  245. package/dist/src-BL-_0QoE.js +1 -0
  246. package/dist/{src-YvblOrBU.js → src-BqNzHHWn.js} +87 -13
  247. package/dist/src-BqNzHHWn.js.map +1 -0
  248. package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
  249. package/dist/src-C-ChdnG6.js.map +1 -0
  250. package/dist/src-C-mOSczH.js +2 -0
  251. package/dist/src-CZaYu6ER.js +3 -0
  252. package/dist/src-CeBhwD8k.js +2 -0
  253. package/dist/src-CkEiiyON.js +2 -0
  254. package/dist/{src-KxS91P7L.js → src-CmA5Yev6.js} +79 -11
  255. package/dist/src-CmA5Yev6.js.map +1 -0
  256. package/dist/src-CwH_nEaw.js +2 -0
  257. package/dist/src-DA95rM6v.js +4 -0
  258. package/dist/{src-DvGns8cz.js → src-DL-EE4DU.js} +299 -17
  259. package/dist/src-DL-EE4DU.js.map +1 -0
  260. package/dist/src-De0KY5S_.js +2 -0
  261. package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
  262. package/dist/src-boB6D-5T.js.map +1 -0
  263. package/dist/state-B3jex5KU.js +1 -0
  264. package/dist/{state-BOLt8pJg.js → state-DUBRDyMk.js} +22 -22
  265. package/dist/{state-BOLt8pJg.js.map → state-DUBRDyMk.js.map} +1 -1
  266. package/dist/store/local.d.ts +8 -0
  267. package/dist/store/local.js +16 -0
  268. package/dist/store/local.js.map +1 -0
  269. package/dist/store/s3.d.ts +10 -0
  270. package/dist/store/s3.js +18 -0
  271. package/dist/store/s3.js.map +1 -0
  272. package/dist/{suite-COglUhPd.js → suite-B9J6Afg6.js} +6 -5
  273. package/dist/{suite-COglUhPd.js.map → suite-B9J6Afg6.js.map} +1 -1
  274. package/dist/suite-Bhs9-Z7O.js +2 -0
  275. package/dist/{summary-DU5zaibx.js → summary-CSp4cfnr.js} +25 -7
  276. package/dist/{summary-DU5zaibx.js.map → summary-CSp4cfnr.js.map} +1 -1
  277. package/dist/sync-CxAY6Dki.js +559 -0
  278. package/dist/sync-CxAY6Dki.js.map +1 -0
  279. package/dist/tart-CcrrtSrp.js +704 -0
  280. package/dist/tart-CcrrtSrp.js.map +1 -0
  281. package/dist/tart-CzViW3WZ.js +186 -0
  282. package/dist/tart-CzViW3WZ.js.map +1 -0
  283. package/dist/tart-DmVuVsmX.js +1 -0
  284. package/dist/tart-bake-F-6seRC9.js +192 -0
  285. package/dist/tart-bake-F-6seRC9.js.map +1 -0
  286. package/dist/tart-bake-RK7RaT9E.js +1 -0
  287. package/dist/tart-cli-B86Jqo1t.js +89 -0
  288. package/dist/tart-cli-B86Jqo1t.js.map +1 -0
  289. package/dist/tart-cli-ClvRniLQ.js +1 -0
  290. package/dist/tart-hosts-Brii7zHM.js +1 -0
  291. package/dist/tart-hosts-DPyZsg-1.js +313 -0
  292. package/dist/tart-hosts-DPyZsg-1.js.map +1 -0
  293. package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CWSf03w8.js} +64 -6
  294. package/dist/tart-images-CWSf03w8.js.map +1 -0
  295. package/dist/testing.d.ts +36 -2
  296. package/dist/testing.js +4 -4
  297. package/dist/tool-class-CVDdJy0H.js +433 -0
  298. package/dist/tool-class-CVDdJy0H.js.map +1 -0
  299. package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
  300. package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
  301. package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
  302. package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
  303. package/dist/{transcript-CCf07UmU.js → transcript-BP_2hji-.js} +3 -3
  304. package/dist/{transcript-CCf07UmU.js.map → transcript-BP_2hji-.js.map} +1 -1
  305. package/dist/{types-DBwFsGem.d.ts → types-BJ0wh8wC.d.ts} +408 -44
  306. package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
  307. package/dist/types-CbcnJxCy.js.map +1 -0
  308. package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
  309. package/dist/usage-CwIRPPZ9.js.map +1 -0
  310. package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
  311. package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
  312. package/dist/vertex-DXfcXYKn.js +1 -0
  313. package/dist/view/index.d.ts +27 -3
  314. package/dist/view/index.js +8 -8
  315. package/dist/{views-DAIiLiV3.js → views-DRxXCyIH.js} +6 -6
  316. package/dist/{views-DAIiLiV3.js.map → views-DRxXCyIH.js.map} +1 -1
  317. package/dist/views-i67V1ly3.js +2 -0
  318. package/dist/{worker-entry-BIROTPX1.js → worker-entry-C3Q4U63F.js} +12 -9
  319. package/dist/worker-entry-C3Q4U63F.js.map +1 -0
  320. package/dist/workspace-DhUOW8if.js +1 -0
  321. package/dist/{workspace-uuE51x35.js → workspace-P72SXLXr.js} +7 -7
  322. package/dist/{workspace-uuE51x35.js.map → workspace-P72SXLXr.js.map} +1 -1
  323. package/package.json +21 -10
  324. package/skills/eval-rubric/SKILL.md +10 -0
  325. package/skills/reeval/SKILL.md +370 -0
  326. package/skills/reeval/references/capture.md +48 -0
  327. package/skills/reeval/references/cli.md +162 -0
  328. package/skills/reeval/references/compare.md +180 -0
  329. package/skills/reeval/references/config.md +147 -0
  330. package/skills/reeval/references/environments.md +218 -0
  331. package/skills/reeval/references/evals.md +241 -0
  332. package/skills/reeval/references/mistakes.md +39 -0
  333. package/skills/reeval/references/vocabulary.md +19 -0
  334. package/skills/setup/SKILL.md +219 -0
  335. package/skills/setup/references/templates.md +258 -0
  336. package/dist/adapter-C8xPuXYq.js.map +0 -1
  337. package/dist/agent-CBoLndM7.js.map +0 -1
  338. package/dist/agents-C6zhkxp3.js.map +0 -1
  339. package/dist/app-files-DT5mJyV4.js.map +0 -1
  340. package/dist/assemble-t6h018An.js.map +0 -1
  341. package/dist/attempt-host-CIm1-11m.js.map +0 -1
  342. package/dist/boat-CJCtzfB1.js.map +0 -1
  343. package/dist/capture-DoxFUN3J.js +0 -1
  344. package/dist/cases-Dy0mifal.js +0 -1
  345. package/dist/cases-z9E6DCd5.js.map +0 -1
  346. package/dist/cli-20Fpvs5H.js.map +0 -1
  347. package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
  348. package/dist/client-KM4C5r5_.js +0 -387
  349. package/dist/client-KM4C5r5_.js.map +0 -1
  350. package/dist/command-CRh-cb6Z.js.map +0 -1
  351. package/dist/commands-CWy-WndG.js.map +0 -1
  352. package/dist/config-B9mB1kF5.js.map +0 -1
  353. package/dist/cost-BdRgySQl.js.map +0 -1
  354. package/dist/cost-Cr26Alhq.js +0 -1
  355. package/dist/db-P2fXw4nx.js +0 -2
  356. package/dist/db-zgC1tvKB.js.map +0 -1
  357. package/dist/dist-CAFD1yfv.js.map +0 -1
  358. package/dist/docker-DAMnm3L9.js.map +0 -1
  359. package/dist/doctor-B2PohneB.js.map +0 -1
  360. package/dist/environment-DB8o3hGi.js.map +0 -1
  361. package/dist/eval-source-D3sbAzt-.js.map +0 -1
  362. package/dist/evaluate-CwKeN_fo.js.map +0 -1
  363. package/dist/files-B4KQUiM7.js +0 -1
  364. package/dist/files-C1Rk9FYA.js +0 -1
  365. package/dist/files-C3mCC8Mn.js +0 -1
  366. package/dist/files-Cl7BQ5gE.js.map +0 -1
  367. package/dist/files-DnlA1A7N.js +0 -1
  368. package/dist/harness-DXv2gnaT.js.map +0 -1
  369. package/dist/images-BNgcR77K.js.map +0 -1
  370. package/dist/init-4RPeD-K-.js +0 -1
  371. package/dist/init-DbuZVRrW.js.map +0 -1
  372. package/dist/kinds-C1oY3qyC.js +0 -697
  373. package/dist/kinds-C1oY3qyC.js.map +0 -1
  374. package/dist/layout-Cm85tQYk.js.map +0 -1
  375. package/dist/live-CJCK11nh.js +0 -1
  376. package/dist/matchers-CXLn-1jm.js +0 -148
  377. package/dist/matchers-CXLn-1jm.js.map +0 -1
  378. package/dist/model-Cv4GuTsZ.js +0 -1
  379. package/dist/model-CwwkrCb7.js.map +0 -1
  380. package/dist/options-CHubslQf.js +0 -1
  381. package/dist/plain-BYUf_fUV.js.map +0 -1
  382. package/dist/plan-Cgq9NCEX.js.map +0 -1
  383. package/dist/plugins-BkmPxDiY.js.map +0 -1
  384. package/dist/preflight-4gWSy0Pe.js +0 -1
  385. package/dist/preflight-CtlloeOQ.js.map +0 -1
  386. package/dist/prices-5m3SzMZ3.js.map +0 -1
  387. package/dist/provider-BnUoz4KB.js +0 -1
  388. package/dist/providers-B8xYMeah.js.map +0 -1
  389. package/dist/providers-CVu8e0xS.js +0 -1
  390. package/dist/query-DIfkUcPh.js +0 -1
  391. package/dist/redact-BhjtnjrU.js.map +0 -1
  392. package/dist/remarks-CAvTSq16.js +0 -1
  393. package/dist/remarks-D9Lcd1fF.js.map +0 -1
  394. package/dist/remarks-vtkPmQb9.js.map +0 -1
  395. package/dist/resolve-Do0ZaYH6.js.map +0 -1
  396. package/dist/resolve-amw_2Odz.js +0 -1
  397. package/dist/results-JyuM64gy.js.map +0 -1
  398. package/dist/resume-AW5oXzUc.js.map +0 -1
  399. package/dist/server-2_gUnVvf.js +0 -1
  400. package/dist/shared-Cfi45Bvy.js.map +0 -1
  401. package/dist/source-CY3etKzw.js +0 -1
  402. package/dist/src-6Zzyg4Qd.js +0 -2
  403. package/dist/src-BDhcPtnI.js +0 -2
  404. package/dist/src-BNQznj4z.js.map +0 -1
  405. package/dist/src-BRMsk9nh.js +0 -2
  406. package/dist/src-CV_7DkPa.js.map +0 -1
  407. package/dist/src-Cb2MWozp.js.map +0 -1
  408. package/dist/src-Ck2r_EMT.js.map +0 -1
  409. package/dist/src-DAzaCTcN.js.map +0 -1
  410. package/dist/src-DB7N8TYJ.js +0 -2
  411. package/dist/src-DPDaAYwh.js +0 -2
  412. package/dist/src-Dbk19BiP.js +0 -2
  413. package/dist/src-DtodMT59.js +0 -1
  414. package/dist/src-DvGns8cz.js.map +0 -1
  415. package/dist/src-Dw79VLa3.js.map +0 -1
  416. package/dist/src-KxS91P7L.js.map +0 -1
  417. package/dist/src-T-OeuJLr.js +0 -2
  418. package/dist/src-YvblOrBU.js.map +0 -1
  419. package/dist/src-zntEDlnx.js +0 -3
  420. package/dist/state-kimtIhBP.js +0 -1
  421. package/dist/suite-CKBa6MKt.js +0 -2
  422. package/dist/tart-BTVXp8rz.js +0 -373
  423. package/dist/tart-BTVXp8rz.js.map +0 -1
  424. package/dist/tart-cli-BIC9HqPg.js +0 -37
  425. package/dist/tart-cli-BIC9HqPg.js.map +0 -1
  426. package/dist/tart-cli-BMrkmBjI.js +0 -1
  427. package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
  428. package/dist/tool-target-DS5d72eH.js +0 -40
  429. package/dist/tool-target-DS5d72eH.js.map +0 -1
  430. package/dist/types-C2mLpph3.js.map +0 -1
  431. package/dist/usage-DJiDnh2D.js.map +0 -1
  432. package/dist/vercel-CGlSBv63.js.map +0 -1
  433. package/dist/vertex-O88TCwgY.js +0 -1
  434. package/dist/views-DtRhj2Tn.js +0 -2
  435. package/dist/worker-entry-BIROTPX1.js.map +0 -1
  436. package/dist/workspace-CcQRT31C.js +0 -1
package/README.md CHANGED
@@ -27,11 +27,11 @@ Evals are TypeScript files in `evals/`, next to the tool they test. Each run wri
27
27
 
28
28
  Cursor ⋅ Claude Code ⋅ Codex ⋅ Antigravity ⋅ OpenCode ⋅ Goose ⋅ pi ⋅ Grok Build
29
29
 
30
- - **Real agents**: each harness runs the agent itself, with its own tools, skills, and rules, as it works on a developer's machine ([Choose an agent](docs/configuration/agents/index.mdx)).
31
- - **Keys stay on the host**: each attempt starts in a fresh sandbox that holds only a placeholder key. reeval's egress gateway swaps in the real key on requests to that provider and nowhere else ([Pass secrets](docs/configuration/environments/secrets.mdx)).
32
- - **Compare a change**: one eval runs a baseline and its variants with the same prompt and checks, then prints whether each variant helped, made it worse, or did the same ([Measure lift against a baseline](docs/results/compare/lift.mdx)).
33
- - **Pinned environments**: an environment starts from a workspace, such as a git repository at a fixed ref, then adds files, setup steps, rules, skills, and MCP servers, so every attempt starts from the same state ([Define an environment](docs/configuration/environments/index.mdx)).
34
- - **History in Git**: recorded runs sit beside the code. `reeval query` reads them with SQL, and `reeval studio` shows each attempt's transcript, diff, and checks ([Record runs in Git](docs/evaluations/runs/recording.mdx)).
30
+ - **Real agents**: each harness runs the agent itself, with its own tools, skills, and rules, as it works on a developer's machine ([Choose an agent](https://github.com/invertase/reeval/blob/v0.21.0/docs/configuration/agents/index.mdx)).
31
+ - **Keys stay on the host**: each attempt starts in a fresh sandbox that holds only a placeholder key. reeval's egress gateway swaps in the real key on requests to that provider and nowhere else ([Pass secrets](https://github.com/invertase/reeval/blob/v0.21.0/docs/configuration/environments/secrets.mdx)).
32
+ - **Compare a change**: one eval runs a baseline and its variants with the same prompt and checks, then prints whether each variant helped, made it worse, or did the same ([Measure lift against a baseline](https://github.com/invertase/reeval/blob/v0.21.0/docs/results/compare/lift.mdx)).
33
+ - **Pinned environments**: an environment starts from a workspace, such as a git repository at a fixed ref, then adds files, setup steps, rules, skills, and MCP servers, so every attempt starts from the same state ([Define an environment](https://github.com/invertase/reeval/blob/v0.21.0/docs/configuration/environments/index.mdx)).
34
+ - **History in Git**: recorded runs sit beside the code. `reeval query` reads them with SQL, and `reeval studio` shows each attempt's transcript, diff, and checks ([Record runs in Git](https://github.com/invertase/reeval/blob/v0.21.0/docs/evaluations/runs/recording.mdx)).
35
35
 
36
36
  ## Install
37
37
 
@@ -140,23 +140,23 @@ Environments load extensions from where teams already keep them:
140
140
 
141
141
  ## Examples
142
142
 
143
- The recipes in [`docs/evaluations/recipes`](docs/evaluations/recipes) are complete evals for common jobs:
143
+ The recipes in [`docs/evaluations/recipes`](https://github.com/invertase/reeval/blob/v0.21.0/docs/evaluations/recipes) are complete evals for common jobs:
144
144
 
145
- - [Measure a skill](docs/evaluations/recipes/measure-a-skill.mdx): with and without a skill, gated on a minimum lift.
146
- - [Compare harnesses](docs/evaluations/recipes/compare-harnesses.mdx): one suite on several agents at once, with pass rate and effort per agent.
147
- - [Measure MCP servers](docs/evaluations/recipes/measure-mcp-servers.mdx): check the agent calls the server's tools, and whether answers improve.
148
- - [Check a recommendation](docs/evaluations/recipes/test-recommendations.mdx): ask which library to use, and let a judge decide whether the agent picks the right one.
149
- - [Fail a pull request](docs/evaluations/recipes/gate-pull-requests.mdx): run on every pull request and fail the check on a regression.
145
+ - [Measure a skill](https://github.com/invertase/reeval/blob/v0.21.0/docs/evaluations/recipes/measure-a-skill.mdx): with and without a skill, gated on a minimum lift.
146
+ - [Compare harnesses](https://github.com/invertase/reeval/blob/v0.21.0/docs/evaluations/recipes/compare-harnesses.mdx): one suite on several agents at once, with pass rate and effort per agent.
147
+ - [Measure MCP servers](https://github.com/invertase/reeval/blob/v0.21.0/docs/evaluations/recipes/measure-mcp-servers.mdx): check the agent calls the server's tools, and whether answers improve.
148
+ - [Check a recommendation](https://github.com/invertase/reeval/blob/v0.21.0/docs/evaluations/recipes/test-recommendations.mdx): ask which library to use, and let a judge decide whether the agent picks the right one.
149
+ - [Fail a pull request](https://github.com/invertase/reeval/blob/v0.21.0/docs/evaluations/recipes/gate-pull-requests.mdx): run on every pull request and fail the check on a regression.
150
150
 
151
- The [`examples/`](examples) directory has 80 runnable projects, one idea each, from [`with-checks`](examples/with-checks) (deterministic checks) and [`with-skill-lift`](examples/with-skill-lift) (does a skill help?) to sandboxes, egress, judges, captures and real-world tasks. Most run on a fake agent with no keys or Docker; [`examples/README.md`](examples/README.md) lists them all.
151
+ The [`examples/`](https://github.com/invertase/reeval/blob/v0.21.0/examples) directory has 80 runnable projects, one idea each, from [`with-checks`](https://github.com/invertase/reeval/blob/v0.21.0/examples/with-checks) (deterministic checks) and [`with-skill-lift`](https://github.com/invertase/reeval/blob/v0.21.0/examples/with-skill-lift) (does a skill help?) to sandboxes, egress, judges, captures and real-world tasks. Most run on a fake agent with no keys or Docker; [`examples/README.md`](https://github.com/invertase/reeval/blob/v0.21.0/examples/README.md) lists them all.
152
152
 
153
153
  ## Documentation
154
154
 
155
- The manual in [`docs/`](docs/overview.mdx) covers harnesses, sandboxes, environments, secrets and egress, checks and judges, profiles, history, and every CLI command and flag.
155
+ The manual in [`docs/`](https://github.com/invertase/reeval/blob/v0.21.0/docs/overview.mdx) covers harnesses, sandboxes, environments, secrets and egress, checks and judges, profiles, history, and every CLI command and flag.
156
156
 
157
157
  ## For coding agents
158
158
 
159
- To write or update evals, load [`skills/reeval/SKILL.md`](skills/reeval/SKILL.md). For every command, flag, and exit code, run `npx reeval --help`.
159
+ To write or update evals, load [`skills/reeval/SKILL.md`](https://github.com/invertase/reeval/blob/v0.21.0/skills/reeval/SKILL.md). For every command, flag, and exit code, run `npx reeval --help`.
160
160
 
161
161
  To preview a run without starting sandboxes or spending tokens, run `npx reeval run --dry-run --json`. It prints the planned evals, cells, and attempts as one JSON document and exits 0.
162
162
 
@@ -166,4 +166,4 @@ To read past runs without calling a model, run `npx reeval query failing_attempt
166
166
 
167
167
  ## License
168
168
 
169
- [Apache-2.0](LICENSE). Copyright 2026 Invertase Limited. The npm package ships third-party licenses and attributions in `THIRD_PARTY_NOTICES.md`. Questions about licensing: [oss@invertase.io](mailto:oss@invertase.io).
169
+ [Apache-2.0](https://github.com/invertase/reeval/blob/v0.21.0/LICENSE). Copyright 2026 Invertase Limited. The npm package ships third-party licenses and attributions in `THIRD_PARTY_NOTICES.md`. Questions about licensing: [oss@invertase.io](mailto:oss@invertase.io).
@@ -7,8 +7,8 @@ with its own license.
7
7
 
8
8
  | Package | Version | License | Inlined into |
9
9
  |---|---|---|---|
10
- | @ai-sdk/harness | 1.0.170 | Apache-2.0 | dist/bridge |
11
- | @ai-sdk/harness-claude-code | 1.0.174 | Apache-2.0 | dist, dist/bridge |
10
+ | @ai-sdk/harness | 1.0.172 | Apache-2.0 | dist/bridge |
11
+ | @ai-sdk/harness-claude-code | 1.0.176 | Apache-2.0 | dist, dist/bridge |
12
12
  | @floating-ui/core | 1.8.0 | MIT | the studio app |
13
13
  | @floating-ui/dom | 1.8.0 | MIT | the studio app |
14
14
  | @floating-ui/react-dom | 2.1.9 | MIT | the studio app |
@@ -72,7 +72,7 @@ with its own license.
72
72
  | shadcn/ui | | MIT | 11 source files |
73
73
  | moby/profiles (Docker default seccomp profile) | | Apache-2.0 | 1 source files |
74
74
 
75
- ## @ai-sdk/harness@1.0.170
75
+ ## @ai-sdk/harness@1.0.172
76
76
 
77
77
  - License: Apache-2.0
78
78
  - Source: https://github.com/vercel/ai
@@ -94,12 +94,12 @@ See the License for the specific language governing permissions and
94
94
  limitations under the License.
95
95
  ```
96
96
 
97
- ## @ai-sdk/harness-claude-code@1.0.174
97
+ ## @ai-sdk/harness-claude-code@1.0.176
98
98
 
99
99
  - License: Apache-2.0
100
100
  - Source: https://github.com/vercel/ai
101
101
  - Inlined into: dist, dist/bridge
102
- - Modified: dist/bridge/index.mjs, with patches/@ai-sdk__harness-claude-code@1.0.174.patch
102
+ - Modified: dist/bridge/index.mjs, with patches/@ai-sdk__harness-claude-code@1.0.176.patch
103
103
 
104
104
  ```text
105
105
  Copyright 2023 Vercel, Inc.
@@ -1,14 +1,17 @@
1
- // Writes pi's models.json for a Gemini model on an AI gateway's Chat Completions API, run in the
2
- // sandbox before each turn: pi's own catalog entry for the model (its thinking levels, limits and
3
- // prices, as on Gemini's API), with each thinking level sent as the gateway's native Gemini
4
- // `thinkingConfig`, so the model gets the level pi sends it there.
1
+ // Writes pi's models.json for a model on an AI gateway, run in the sandbox before each turn, with
2
+ // the model in a provider of reeval's, on the API its provider's route names (`gateway.routes`),
3
+ // else Chat Completions. Its thinking levels, limits and prices are pi's catalog entry for it, on
4
+ // the gateway or its provider, else the host's description of it. A route with `thinkingConfig`
5
+ // sends each thinking level as the model's native `thinkingConfig`, so the model gets the level pi
6
+ // sends it there.
5
7
  //
6
- // node gateway-models.mjs <bootstrap dir> <models.json> <provider json> <model>
7
- import { mkdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'
8
+ // node gateway-models.mjs <bootstrap dir> <models.json> <gateway json> <model> <model info json>
9
+ import { existsSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'
8
10
  import { dirname, join } from 'node:path'
9
11
 
10
- const [bootstrapDir, out, providerJson, model] = process.argv.slice(2)
11
- const provider = JSON.parse(providerJson)
12
+ const [bootstrapDir, out, gatewayJson, model, infoJson] = process.argv.slice(2)
13
+ const gateway = JSON.parse(gatewayJson)
14
+ const info = JSON.parse(infoJson ?? 'null')
12
15
  const fail = (why) => {
13
16
  process.stderr.write(`reeval: ${why}\n`)
14
17
  process.exit(1)
@@ -16,52 +19,83 @@ const fail = (why) => {
16
19
 
17
20
  // pnpm puts pi's own dependencies beside it, so pi-ai is pi-coding-agent's sibling.
18
21
  const agent = realpathSync(join(bootstrapDir, 'node_modules/@earendil-works/pi-coding-agent'))
19
- const catalog = JSON.parse(
20
- readFileSync(join(dirname(agent), 'pi-ai/dist/providers/data/google.json'), 'utf8'),
21
- )
22
- const id = model.replace(/^google\//, '')
23
- const entry = Object.values(catalog['google-generative-ai'] ?? {}).find((m) => m.id === id)
24
- if (!entry)
25
- fail(`pi's catalog has no Gemini model ${id}, so pi can't run ${model} on ${provider.name}`)
26
- if (entry.reasoning && !entry.thinkingLevelMap) {
27
- fail(
28
- `${id} takes a thinking budget, not a thinking level, so pi can't run it on ${provider.name}`,
22
+ const data = join(dirname(agent), 'pi-ai/dist/providers/data')
23
+ /** The chat models of one of pi's catalogs, whatever API each is on. */
24
+ const catalog = (name) => {
25
+ const file = join(data, `${name}.json`)
26
+ if (!/^[\w-]+$/.test(name) || !existsSync(file)) return []
27
+ return Object.values(JSON.parse(readFileSync(file, 'utf8'))).flatMap((models) =>
28
+ Object.entries(models)
29
+ .filter(([key]) => key.startsWith('chat:'))
30
+ .map(([, m]) => m),
29
31
  )
30
32
  }
31
33
 
34
+ const slash = model.indexOf('/')
35
+ const provider = slash < 0 ? '' : model.slice(0, slash)
36
+ const id = model.slice(slash + 1)
37
+ const route = gateway.routes[provider] ?? {}
38
+ const native = route.thinkingConfig
39
+
40
+ let entry
41
+ if (native) {
42
+ entry = catalog(native.catalog).find((m) => m.id === id)
43
+ if (!entry)
44
+ fail(
45
+ `pi's ${native.catalog} catalog has no model ${id}, so pi can't run ${model} on ${gateway.name}`,
46
+ )
47
+ if (entry.reasoning && !entry.thinkingLevelMap) {
48
+ fail(
49
+ `${id} takes a thinking budget, not a thinking level, so pi can't run it on ${gateway.name}`,
50
+ )
51
+ }
52
+ } else {
53
+ // A provider's own catalog names a version with a dash where a gateway dots it (claude-haiku-4-5).
54
+ const dashed = id.replace(/(\d)\.(\d)/g, '$1-$2')
55
+ entry =
56
+ catalog(gateway.catalog).find((m) => m.id === model) ??
57
+ catalog(provider).find((m) => m.id === id || m.id === dashed) ??
58
+ info ??
59
+ fail(
60
+ `neither pi's catalog nor models.dev describes ${model}, so pi can't run it on ${gateway.name}`,
61
+ )
62
+ }
63
+
32
64
  const thinking = (level) => ({ thinkingConfig: { thinkingLevel: level, includeThoughts: true } })
33
65
  const byLevel = {}
34
- for (const level of entry.reasoning ? ['minimal', 'low', 'medium', 'high'] : []) {
66
+ for (const level of native && entry.reasoning ? ['minimal', 'low', 'medium', 'high'] : []) {
35
67
  const mapped = entry.thinkingLevelMap[level]
36
68
  if (mapped === null) continue
37
- const native = (typeof mapped === 'string' ? mapped : level).toLowerCase()
69
+ const sent = (typeof mapped === 'string' ? mapped : level).toLowerCase()
38
70
  byLevel[level] = {
39
- providerOptions: Object.fromEntries(provider.upstreams.map((u) => [u, thinking(native)])),
71
+ providerOptions: Object.fromEntries(native.upstreams.map((u) => [u, thinking(sent)])),
40
72
  }
41
73
  }
42
74
 
75
+ const api = route.api ?? 'openai-completions'
43
76
  const models = {
44
77
  providers: {
45
- [provider.id]: {
46
- name: provider.name,
47
- baseUrl: provider.baseUrl,
48
- api: 'openai-completions',
49
- apiKey: `$${provider.credentialEnv}`,
50
- // Each level goes as the native thinkingConfig above; the gateway's shared effort field
51
- // would map Gemini's levels onto its own.
78
+ [gateway.id]: {
79
+ name: gateway.name,
80
+ baseUrl: api === 'anthropic-messages' ? gateway.anthropic : gateway.openai,
81
+ api,
82
+ apiKey: `$${gateway.credentialEnv}`,
83
+ authHeader: true,
52
84
  compat: {
53
- supportsReasoningEffort: false,
85
+ // The level goes as the thinkingConfig above; the gateway's shared effort field would map
86
+ // the model's levels onto its own.
87
+ ...(native ? { supportsReasoningEffort: false } : {}),
54
88
  supportsStore: false,
55
89
  supportsDeveloperRole: false,
56
90
  },
57
91
  models: [
58
92
  {
59
93
  id: model,
60
- name: entry.name,
94
+ ...(entry.name ? { name: entry.name } : {}),
61
95
  reasoning: entry.reasoning,
62
96
  ...(entry.thinkingLevelMap ? { thinkingLevelMap: entry.thinkingLevelMap } : {}),
63
97
  input: entry.input,
64
- cost: entry.cost,
98
+ ...(entry.cost ? { cost: entry.cost } : {}),
65
99
  contextWindow: entry.contextWindow,
66
100
  maxTokens: entry.maxTokens,
67
101
  ...(Object.keys(byLevel).length ? { samplingParamsByThinkingLevel: byLevel } : {}),
@@ -1,11 +1,11 @@
1
- import { s as relativeFile } from "./ids-C_advdJA.js";
2
- import { a as PACKAGE_NAMES, c as ownDir, d as ownRequire, l as ownFile, n as COMPILED } from "./root-Co0jhOT5.js";
3
- import { t as codeFrame } from "./errors-BZgLSZtV.js";
1
+ import { a as PACKAGE_NAMES, c as ownDir, d as ownRequire, l as ownFile, n as COMPILED } from "./root-BzeoyfGd.js";
2
+ import { c as relativeFile } from "./ids-B-JKTCP8.js";
3
+ import { t as codeFrame } from "./errors-C1GBcWE2.js";
4
4
  import { createRequire } from "node:module";
5
5
  import { existsSync, readFileSync } from "node:fs";
6
6
  import { delimiter, dirname, join, resolve } from "node:path";
7
- import { stripVTControlCharacters } from "node:util";
8
7
  import { spawnSync } from "node:child_process";
8
+ import { stripVTControlCharacters } from "node:util";
9
9
  import { createJiti } from "jiti";
10
10
  import { DecodedMap, getOriginalPosition } from "@vitest/utils/source-map";
11
11
  import { getTsconfig } from "get-tsconfig";
@@ -41,6 +41,8 @@ async function ownModules() {
41
41
  "/harness": import("./harness.js"),
42
42
  "/sandbox": import("./sandbox.js"),
43
43
  "/sandbox/boat": import("./sandbox/boat.js"),
44
+ "/store/local": import("./store/local.js"),
45
+ "/store/s3": import("./store/s3.js"),
44
46
  "/judge": import("./judge.js"),
45
47
  "/gateway": import("./gateway.js"),
46
48
  "/view": import("./view/index.js")
@@ -288,7 +290,7 @@ const LANGUAGE_ADAPTERS = [TYPESCRIPT_ADAPTER, {
288
290
  extensions: [".eval.py"],
289
291
  includeGlobs: ["evals/**/*.eval.py"],
290
292
  remote: true,
291
- load: (file, root) => import("./remote-BgYWqogI.js").then((m) => m.collectLanguageFile(file, root)),
293
+ load: (file, root) => import("./remote-C88amFRN.js").then((m) => m.collectLanguageFile(file, root)),
292
294
  runnerCommand: (root) => resolvePythonRunnerCommand(root),
293
295
  setupHint: PYTHON_RUNNER_HINT
294
296
  }];
@@ -314,4 +316,4 @@ function runnerEnv(root, adapter) {
314
316
  }
315
317
  export { originalText as a, originalStack as i, defaultIncludeGlobs as n, reevalJiti as o, runnerEnv as r, adapterForFile as t };
316
318
 
317
- //# sourceMappingURL=adapter-C8xPuXYq.js.map
319
+ //# sourceMappingURL=adapter-BdzhdRYo.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"adapter-BdzhdRYo.js","names":[],"sources":["../src/jiti.ts","../src/engine/loader.ts","../src/language-runner/python-runner.ts","../src/language-runner/adapter.ts"],"mappings":";;;;;;;;;;;;AASA,IAAI;;;;;;AAOJ,MAAa,iBAA4B,YAAY;CACnD,IAAI,CAAC,OAAO;EACV,MAAM,OAAO,WACT,KAAK,OAAO,aAAa,GAAG,WAAW,IACvC,KAAK,QAAQ,WAAW,QAAQ,mBAAmB,CAAC,GAAG,gBAAgB;EAC3E,QAAQ,cAAc,IAAI,CAAC,CAAC,IAAI;CAClC;CACA,OAAO,MAAM,OAAO;AACtB;;;;;AAMA,eAAsB,WAAW,IAAY,UAAuB,CAAC,GAAkB;CACrF,MAAM,iBAAiB,WAAW,MAAM,WAAW,IAAI,KAAA;CACvD,OAAO,WAAW,IAAI;EACpB,WAAW;EACX,GAAG;EACH,GAAI,kBAAkB,EAAE,eAAe;CACzC,CAAC;AACH;AAEA,eAAe,aAA+C;CAC5D,MAAM,UAA4C;EAChD,IAAI,OAAO;EACX,YAAY,OAAO;EACnB,YAAY,OAAO;EACnB,iBAAiB,OAAO;EACxB,gBAAgB,OAAO;EACvB,aAAa,OAAO;EACpB,UAAU,OAAO;EACjB,YAAY,OAAO;EACnB,SAAS,OAAO;CAClB;CACA,MAAM,UAAmC,CAAC;CAC1C,KAAK,MAAM,CAAC,SAAS,WAAW,OAAO,QAAQ,OAAO,GACpD,KAAK,MAAM,QAAQ,eAAe,QAAQ,OAAO,WAAW,MAAM;CACpE,OAAO;AACT;;AC3CA,IAAa,gBAAb,cAAmC,MAAM;CACvC,OAAgB;AAClB;AAOA,MAAM,8BAAc,IAAI,IAAyB;AAEjD,MAAM,aACJ;;AAGF,eAAsB,aAAa,MAAc,MAAgC;CAC/E,cAAc,QAAQ,IAAI,GAAG,IAAI;CACjC,MAAM,UAAU;EAAE,aAAa;EAAM,SAAS;EAAO,YAAY;CAAK;CACtE,MAAM,OAAO,MAAM,WAAW,MAAM,OAAO;CAC3C,MAAM,OAAO,MAAM,WAAW,MAAM;EAClC,GAAG;EACH,YAAY,SAAS;GACnB,MAAM,OAAO,KAAK,UAAU,IAAI;GAChC,MAAM,SAAS,WAAW,KAAK,IAAI;GACnC,IAAI,KAAK,YAAY,SAAS,IAAI;IAChC,MAAM,MAAM,KAAK,MAAM,OAAO,KAAK,OAAO,IAAI,QAAQ,CAAC,CAAC,SAAS,MAAM,CAAC;IAExE,MAAM,QACJ,IAAI,KAAK,QAAQ,WAAW,GAAG,2FAC5B;IACL,YAAY,IAAI,KAAK,UAAU;KAC7B,KAAK,IAAI,WAAW,KAAK,KAAK,QAAQ;KACtC;KACA,QAAQ,KAAK;KACb;IACF,CAAC;GACH;GACA,OAAO,EAAE,KAAK;EAChB;CACF,CAAC;CACD,IAAI;EACF,OAAO,MAAM,KAAK,OAAO,IAAI;CAC/B,SAAS,OAAO;EACd,MAAM,YAAY,OAAO,IAAI,KAAK,cAAc,KAAK;CACvD;AACF;;AAGA,SAAS,cAAc,OAAyB;CAC9C,MAAM,IAAI;CACV,IAAI,GAAG,SAAS,oBAAoB,OAAO;CAC3C,MAAM,QAAQ,wDAAwD,KAAK,EAAE,OAAO;CACpF,IAAI,OAAO,EAAE,UAAU,GAAG,MAAM,GAAG,iBAAiB,MAAM;CAC1D,OAAO;AACT;;;;;AAMA,SAAgB,YAAY,OAAgB,MAAuC;CACjF,MAAM,SAAS,8CAA8C,KAAM,OAAiB,WAAW,EAAE;CACjG,IAAI,CAAC,QAAQ,OAAO,KAAA;CACpB,MAAM,GAAG,MAAM,MAAM,MAAM,UAAU;CAErC,MAAM,SAAS,KAAK,QAAQ,8BAA8B,EAAE;CAC5D,MAAM,KAAK;EAAE,MAAM,OAAO,IAAI;EAAG,QAAQ,OAAO,MAAM,IAAI;CAAE;CAC5D,IAAI,QAAQ;CACZ,IAAI;EACF,QAAQ,yBAAyB,UAAU,aAAa,MAAM,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC;CACpF,QAAQ,CAAC;CACT,uBAAO,IAAI,YACT,GAAG,OAAO,IAAI,aAAa,MAAM,IAAI,EAAE,GAAG,GAAG,KAAK,GAAG,GAAG,OAAO,GAAG,QAAQ,KAAK,UAAU,IAC3F;AACF;;AAGA,SAAgB,cAAc,OAAuB;CACnD,OAAO,MAAM,QACX,gDACC,OAAO,QAAgB,MAAc,MAAc,QAAgB,UAAkB;EACpF,MAAM,SAAS,YAAY,IAAI,IAAI;EACnC,MAAM,MAAM,OAAO,MAAM,IAAI,KAAK,SAAS,MAAO,QAAQ,SAAS,IAAK;EAExE,IAAI,CAAC,UAAU,MAAM,GAAG,OAAO;EAC/B,MAAM,KAAK,oBAAoB,OAAO,KAAK;GAAE,MAAM,OAAO,IAAI;GAAG,QAAQ;EAAI,CAAC;EAC9E,IAAI,CAAC,IAAI,OAAO;EAChB,MAAM,WAAW,GAAG;EACpB,OAAO,GAAG,SAAS,KAAK,GAAG,SAAS,GAAG,OAAO,MAAM,UAAU,GAAG,MAAgB,IAAI,IAAI;CAC3F,CACF;AACF;;;;;;;AAQA,SAAgB,aAAa,MAAc,MAAmC;CAC5E,MAAM,YAAY,SAAS,KAAA,IAAY,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;CAClE,KAAK,MAAM,UAAU,CAAC,GAAG,WAAW,GAAG,YAAY,OAAO,CAAC,GAAG;EAC5D,MAAM,KAAK,QAAQ,KAAK,QAAQ,IAAI,KAAK;EACzC,IAAI,CAAC,UAAU,KAAK,GAAG;EAGvB,MAAM,OAAO,KAAK,KAAK,SAAS;EAChC,IAAI,aAAa;EACjB,MAAM,MAAM,eAAe,QAAQ,IAAI;EACvC,OAAO,aAAa,MAAM,eAAe,QAAQ,aAAa,CAAC,MAAM,KAAK;EAC1E,OAAO,OAAO,OAAO,MAAM,eAAe,QAAQ,EAAE,GAAG,MAAM,OAAO,aAAa,CAAC;CACpF;AAEF;;AAGA,SAAS,eAAe,QAAqB,OAAuB;CAClE,MAAM,QAAQ,OAAO,KAAK,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,IAAI;CACpD,MAAM,SAAU,MAAM,GAAG,EAAE,CAAC,CAAY;CACxC,MAAM,KAAK,oBAAoB,OAAO,KAAK;EAAE,MAAM,MAAM;EAAQ;CAAO,CAAC;CAGzE,OADoB,OAAO,OAAO,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI,QAAQ,KAAK,CACxD,CAAC,CAAC,QAAQ,GAAG,SAAS,IAAI,KAAK,SAAS,GAAG,CAAC,KAAK,IAAI,UAAU;AAClF;;;;;AAMA,SAAS,OAAO,MAAc,MAAc,QAAwB;CAElE,MAAM,OAAO,aAAa,MAAM,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,OAAO,MAAM;CACpE,IAAI,KAAK,YAAY,KAAK,OAAO;CACjC,IAAI,QAAQ;CACZ,OAAO,QAAQ,KAAK,QAAQ,KAAK,KAAK,QAAQ,EAAY,GAAG;CAC7D,OAAO;AACT;;AAGA,SAAgB,cAAc,KAAa,MAAoB;CAC7D,IAAI;CACJ,IAAI;EACF,QAAQ,YAAY,GAAG;CACzB,SAAS,OAAO;EACd,MAAM,UAAW,MAAgB;EACjC,MAAM,UAAU,4BAA4B,KAAK,OAAO;EACxD,MAAM,IAAI,cACR,UACI,4BAA4B,QAAQ,GAAG,yBACvC,4BAA4B,SAClC;CACF;CACA,IAAI,CAAC,OAAO;CACZ,IAAI;EAEF,KAAK,MAAM,kBAAkB,aAAa,MAAM,MAAM,MAAM,CAAC,CAAC,QAAQ,WAAW,EAAE,CAAC,CAAC;CACvF,SAAS,OAAO;EACd,MAAM,IAAI,cACR,4BAA4B,aAAa,MAAM,MAAM,IAAI,EAAE,KAAM,MAAgB,SACnF;CACF;AACF;;AAGA,SAAgB,kBAAkB,MAAsB;CACtD,OAAO,KACL,KAAK,OAAO,SAAS;EACnB,IAAI,KAAK,WAAW,IAAI,GAAG;GACzB,MAAM,MAAM,KAAK,QAAQ,IAAI;GAC7B,OAAO,CAAC,MAAM,IAAI,KAAK,SAAS,KAAK,EAAE;EACzC;EACA,IAAI,KAAK,WAAW,IAAI,GAAG;GACzB,MAAM,MAAM,KAAK,QAAQ,MAAM,CAAC;GAChC,OAAO,CAAC,MAAM,IAAI,KAAK,SAAS,MAAM,GAAG,GAAG;EAC9C;CAEF,CAAC,IACA,SAAU,aAAa,KAAK,IAAI,IAAI,CAAC,GAAG,GAAG,IAAI,KAAA,CAClD;AACF;;AAGA,SAAS,KAAK,MAAc,IAA4D;CACtF,IAAI,MAAM;CACV,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,SAAU;EACjC,IAAI,KAAK,OAAO,MAAK;GACnB,IAAI,MAAM,IAAI;GACd,OAAO,MAAM,KAAK,UAAU,KAAK,SAAS,MAAK,OAAO,KAAK,SAAS,OAAO,IAAI;GAC/E,OAAO,KAAK,MAAM,GAAG,MAAM,CAAC;GAC5B,IAAI,MAAM;GACV;EACF;EACA,MAAM,MAAM,GAAG,KAAK,MAAM,CAAC,CAAC;EAC5B,IAAI,KAAK;GACP,OAAO,IAAI;GACX,KAAK,IAAI;EACX,OAAO,OAAO,KAAK;CACrB;CACA,OAAO;AACT;;AC7MA,MAAa,uBAAuB;AAEpC,MAAM,cAAc,CAAC,MAAM,oBAAoB;AAE/C,IAAI;AAEJ,SAAgB,cAAuB;CACrC,IAAI,aAAa,KAAA,GAAW,OAAO;CAMnC,WALe,UAAU,MAAM,CAAC,WAAW,GAAG;EAC5C,UAAU;EACV,OAAO;EACP,aAAa;CACf,CACgB,CAAC,CAAC,WAAW;CAC7B,OAAO;AACT;AAEA,SAAgB,cAAc,MAAuB;CACnD,OAAO,WAAW,KAAK,MAAM,SAAS,CAAC,KAAK,WAAW,KAAK,MAAM,gBAAgB,CAAC;AACrF;AAEA,SAAS,WAAW,MAAkC;CACpD,MAAM,OAAO,KAAK,MAAM,SAAS,OAAO,QAAQ;CAChD,IAAI,WAAW,IAAI,GAAG,OAAO;CAC7B,MAAM,MAAM,KAAK,MAAM,SAAS,WAAW,YAAY;CACvD,IAAI,WAAW,GAAG,GAAG,OAAO;AAE9B;;AAGA,SAAgB,2BAA2B,MAA6B;CACtE,MAAM,WAAW,QAAQ,IAAI,eAAe,KAAK;CACjD,IAAI,UAAU,OAAO,CAAC,UAAU,GAAG,WAAW;CAE9C,IAAI,cAAc,IAAI,KAAK,YAAY,GACrC,OAAO;EAAC;EAAM;EAAO;EAAa;EAAM;EAAU,GAAG;CAAW;CAGlE,MAAM,OAAO,WAAW,IAAI;CAC5B,IAAI,MAAM,OAAO,CAAC,MAAM,GAAG,WAAW;CAGtC,OAAO,CAAC,QAAQ,aAAa,UAAU,WAAW,WAAW,GAAG,WAAW;AAC7E;;AAGA,MAAa,qBAAqB;CAChC,kCAAkC,qBAAqB;CACvD;CACA;CACA;AACF,CAAC,CAAC,KAAK,GAAG;;AC7BV,MAAa,qBAAuC;CAClD,IAAI;CACJ,YAAY;EAAC;EAAY;EAAa;EAAa;EAAY;EAAa;CAAW;CACvF,cAAc,CAAC,oBAAoB;CACnC,QAAQ;CACR,OAAO,MAAM,SAAS,aAAa,MAAM,IAAI;AAC/C;AAEA,MAAM,sBAA0C;CAC9C,MAAM,UAAU,KAAK,QAAQ,IAAI,GAAG,eAAe;CACnD,IAAI,WAAW,KAAK,SAAS,gBAAgB,CAAC,GAAG,OAAO;AAE1D;;AAaA,MAAa,oBAAgD,CAAC,oBAAoB;CAVhF,IAAI;CACJ,YAAY,CAAC,UAAU;CACvB,cAAc,CAAC,oBAAoB;CACnC,QAAQ;CACR,OAAO,MAAM,SAAS,OAAO,uBAAc,CAAC,MAAM,MAAM,EAAE,oBAAoB,MAAM,IAAI,CAAC;CACzF,gBAAgB,SAAS,2BAA2B,IAAI;CACxD,WAAW;AAIqE,CAAc;AAEhG,SAAgB,eAAe,MAA+B;CAE5D,OADc,kBAAkB,MAAM,MAAM,EAAE,WAAW,MAAM,QAAQ,KAAK,SAAS,GAAG,CAAC,CAC9E,KAAK;AAClB;AAEA,SAAgB,sBAAgC;CAC9C,OAAO,kBAAkB,SAAS,MAAM,CAAC,GAAG,EAAE,YAAY,CAAC;AAC7D;AAEA,SAAgB,UAAU,MAAc,SAA6C;CACnF,MAAM,MAAM,QAAQ,IAAI,qBAAqB,cAAc;CAC3D,IAAI,QAAQ,OAAO,YAAY,CAAC,KAAK,OAAO,QAAQ;CACpD,MAAM,MAAM,KAAK,KAAK,KAAK;CAC3B,OAAO;EACL,GAAG,QAAQ;EACX,YAAY;GAAC;GAAK;GAAK,QAAQ,IAAI;EAAU,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK,SAAS;EAC7E,qBAAqB,QAAQ,IAAI;CACnC;AACF"}
@@ -1,11 +1,13 @@
1
+ import { ct as SANDBOX_ROOT, dt as WORKSPACE_DIR } from "./src-BHJ4MWvY.js";
2
+ import { B as runsAgent, P as capabilitiesOf, U as sandboxKind, W as suggestedSandbox } from "./plugins-BioMWncI.js";
3
+ import { a as plural } from "./text-B8-W3Cvn.js";
4
+ import { S as brokerToken } from "./vertex-BEWxHoxH.js";
5
+ import { n as hasNetworkSession, t as InfraError } from "./types-CbcnJxCy.js";
6
+ import { C as writePluginFiles, S as unverifiedHarness, c as agentMcp, o as agentAuth, s as agentFor, u as authHint } from "./agents-l0-k7vw3.js";
1
7
  import { t as isObjectLike } from "./object-CeQ8dgUC.js";
2
- import { t as addUsage } from "./usage-DJiDnh2D.js";
3
- import { c as runsAgent, d as sandboxKind, f as suggestedSandbox, n as capabilitiesOf } from "./kinds-C1oY3qyC.js";
4
- import { S as brokerToken } from "./vertex-Eb2wqahk.js";
5
- import { $ as WORKSPACE_DIR, X as SANDBOX_ROOT } from "./src-Ck2r_EMT.js";
6
- import { _ as unverifiedHarness, a as agentFor, i as agentAuth, o as agentMcp, s as authHint, v as writePluginFiles } from "./agents-C6zhkxp3.js";
7
- import { n as hasNetworkSession, t as InfraError } from "./types-C2mLpph3.js";
8
- import { i as debugTrace, r as debugLog } from "./attempt-CAdlwlNu.js";
8
+ import { a as tokenUsage, t as addUsage } from "./usage-CwIRPPZ9.js";
9
+ import { i as debugTrace, r as debugLog } from "./attempt-CKfKsqqw.js";
10
+ import { n as providerStop } from "./provider-stop-CxzWW7Ft.js";
9
11
  import { HarnessAgent } from "@ai-sdk/harness/agent";
10
12
  const SHELL_TOOLS = /* @__PURE__ */ new Set([
11
13
  "bash",
@@ -29,6 +31,21 @@ function stallFlags(tool, durationMs) {
29
31
  return {};
30
32
  }
31
33
  const record = (value) => isObjectLike(value) ? value : {};
34
+ /**
35
+ * A reasoning block recovered after the turn (Antigravity's, from its stored conversation) says
36
+ * in its provider metadata how long before its own part it started, and how long the agent says
37
+ * it took: it's placed there, not where it arrived.
38
+ */
39
+ function recoveredTiming(part) {
40
+ for (const meta of Object.values(record(part.providerMetadata))) {
41
+ const { startedBeforeMs, durationMs } = record(meta);
42
+ if (typeof startedBeforeMs !== "number") continue;
43
+ return {
44
+ startedBeforeMs,
45
+ durationMs: typeof durationMs === "number" ? durationMs : null
46
+ };
47
+ }
48
+ }
32
49
  /** The harness's synthetic "fileChange" twin of every edit, which always says "modify". */
33
50
  const isFileChangeEcho = (part) => part.toolName === "fileChange" || str(part.toolCallId).startsWith("harness-file-change-");
34
51
  function lines(text) {
@@ -284,18 +301,49 @@ function subagentParts(raw) {
284
301
  return [];
285
302
  });
286
303
  }
304
+ const toolOutputOf = (value) => {
305
+ const said = record(value);
306
+ if (said.view !== "full") return {};
307
+ return {
308
+ view: "full",
309
+ ...typeof said.bytes === "number" ? { bytes: said.bytes } : {}
310
+ };
311
+ };
312
+ /**
313
+ * What an adapter says about a tool result's output, as `toolOutput` in the result's
314
+ * `providerMetadata` under its own key: `view: 'full'` for output it read from where the CLI
315
+ * saved it (the model got a pointer or nothing), and `bytes`, the saved output's size in UTF-8,
316
+ * when it kept only part of it. An adapter that finds the saved output only after the result
317
+ * went out sends a `raw` part whose `toolOutput` also has the `toolCallId` and the `output` that
318
+ * replaces the result's.
319
+ */
320
+ function adapterToolOutput(metadata) {
321
+ for (const entry of Object.values(record(metadata))) {
322
+ const said = toolOutputOf(record(entry).toolOutput);
323
+ if (said.view) return said;
324
+ }
325
+ return {};
326
+ }
287
327
  /**
288
328
  * Turns an AI SDK harness stream into reeval transcript events: text and reasoning blocks joined,
289
329
  * each tool call merged with its result, edits normalized to `file-edit`, web tools given their
290
330
  * query or URL, and the harness's duplicate `fileChange` events dropped. Cursor's tool durations
291
331
  * come from its runner's own `tool_call` events, because tool results are only flushed at the
292
- * end of a step.
332
+ * end of a step. Each tool event's `outputView` is `full` for the tools in `fullToolOutput` and
333
+ * for output an adapter read from where its CLI saved it, else `model`.
293
334
  */
294
- function normalizeStream(parts, startedAt) {
335
+ function normalizeStream(parts, startedAt, options = {}) {
336
+ const fullByHarness = (tool) => options.fullToolOutput === "all" || options.fullToolOutput?.includes(tool) === true;
295
337
  const timing = /* @__PURE__ */ new Map();
338
+ const replaced = /* @__PURE__ */ new Map();
296
339
  for (const { ms, part } of parts) {
297
340
  if (part.type !== "raw") continue;
298
341
  const raw = record(part.rawValue);
342
+ const later = record(raw.toolOutput);
343
+ if (typeof later.toolCallId === "string" && "output" in later && toolOutputOf(later).view) replaced.set(later.toolCallId, {
344
+ ...toolOutputOf(later),
345
+ output: later.output
346
+ });
299
347
  const event = raw.type === "event" ? record(raw.event) : {};
300
348
  const id = event.type === "tool_call" ? str(event.call_id) : "";
301
349
  if (!id) continue;
@@ -327,7 +375,8 @@ function normalizeStream(parts, startedAt) {
327
375
  };
328
376
  const input = record(call.part.input);
329
377
  const name = toolName(call.part);
330
- const raw = failed ? errorText(end.part.error) : end.part.output;
378
+ const replacement = failed ? void 0 : replaced.get(id);
379
+ const raw = failed ? errorText(end.part.error) : replacement ? replacement.output : end.part.output;
331
380
  const subagent = str(call.part.subagent);
332
381
  const structured = record(end.part.structured);
333
382
  const output = subagent && (name === "edit" || name === "write") && !failed ? typeof structured.filePath === "string" ? {
@@ -380,6 +429,7 @@ function normalizeStream(parts, startedAt) {
380
429
  });
381
430
  return;
382
431
  }
432
+ const said = replacement ?? adapterToolOutput(end.part.providerMetadata);
383
433
  out.push({
384
434
  at: call.ms,
385
435
  event: {
@@ -389,6 +439,8 @@ function normalizeStream(parts, startedAt) {
389
439
  tool: name,
390
440
  input: toolInput(call.part),
391
441
  output,
442
+ outputView: said.view ?? (fullByHarness(name) ? "full" : "model"),
443
+ ...said.bytes !== void 0 ? { outputTruncated: { bytes: said.bytes } } : {},
392
444
  ok: !failed
393
445
  }
394
446
  });
@@ -416,13 +468,16 @@ function normalizeStream(parts, startedAt) {
416
468
  }
417
469
  break;
418
470
  case "text-start":
419
- case "reasoning-start":
471
+ case "reasoning-start": {
472
+ const recovered = part.type === "reasoning-start" ? recoveredTiming(part) : void 0;
420
473
  blocks.set(str(part.id), {
421
474
  type: part.type === "text-start" ? "text" : "reasoning",
422
- ms,
423
- text: ""
475
+ ms: recovered ? Math.max(0, ms - recovered.startedBeforeMs) : ms,
476
+ text: "",
477
+ ...recovered ? { durationMs: recovered.durationMs } : {}
424
478
  });
425
479
  break;
480
+ }
426
481
  case "text-delta":
427
482
  case "reasoning-delta": {
428
483
  const block = blocks.get(str(part.id));
@@ -439,7 +494,7 @@ function normalizeStream(parts, startedAt) {
439
494
  ...stamp(block.ms),
440
495
  type: block.type,
441
496
  text: block.text,
442
- durationMs: ms - block.ms
497
+ durationMs: block.durationMs !== void 0 ? block.durationMs : ms - block.ms
443
498
  }
444
499
  });
445
500
  break;
@@ -548,14 +603,16 @@ function normalizeStream(parts, startedAt) {
548
603
  });
549
604
  return out.sort((a, b) => a.at - b.at).map((e) => e.event);
550
605
  }
606
+ /** Whether a harness's reasoning blocks carry their time: streamed, or recovered with it. */
607
+ const timesReasoning = (delivery) => delivery === "streamed" || delivery === "recovered";
551
608
  /**
552
609
  * A turn's reasoning time, from the reasoning blocks the harness streamed; null when it can't be
553
610
  * known. A harness whose reasoning arrives whole (`whole`) or not at all (`none`) can't time it.
554
- * A streamed turn with no reasoning block has spent none only when its usage says it used no
555
- * reasoning tokens; otherwise the model may have reasoned without the harness showing it.
611
+ * A streamed or recovered turn with no reasoning block has spent none only when its usage says it
612
+ * used no reasoning tokens; otherwise the model may have reasoned without the harness showing it.
556
613
  */
557
614
  function reasoningTime(delivery, events, usage) {
558
- if (delivery !== "streamed") return null;
615
+ if (!timesReasoning(delivery)) return null;
559
616
  const blocks = events.flatMap((e) => e.type === "reasoning" ? [e.durationMs] : []);
560
617
  if (blocks.length === 0) return usage?.reasoning === 0 ? 0 : null;
561
618
  if (blocks.some((ms) => ms === null)) return null;
@@ -583,17 +640,49 @@ function traceStreamPart(part, ms, tools) {
583
640
  debugLog("harness", `tool ${call?.name ?? p.toolName ?? "tool"} ${how}${took} (${p.toolCallId})`);
584
641
  } else if (p.type === "error") debugLog("harness", `stream error: ${errorText(p.error)}`);
585
642
  }
586
- /** The AI SDK's usage as reeval's, or null when the harness reported none (every field undefined). */
587
- function tokenUsage(usage) {
588
- if (usage.inputTokens === void 0 && usage.outputTokens === void 0) return null;
589
- const cacheRead = usage.inputTokenDetails?.cacheReadTokens ?? 0;
590
- const cacheWrite = usage.inputTokenDetails?.cacheWriteTokens ?? 0;
643
+ /**
644
+ * Each turn's cost from what the harness reports on its finish: a session's running total, as
645
+ * Claude Code's is, less what its earlier turns reported. A total that fell, or a session other
646
+ * than the attempt's first, can't be split into turns, so that turn has no reported cost; nor
647
+ * has one the harness put at $0, which it says for a model its own prices don't list.
648
+ */
649
+ function turnCosts(reported, source) {
650
+ let session;
651
+ let before = 0;
652
+ return (harnessMetadata) => {
653
+ const said = reported?.(harnessMetadata);
654
+ if (!said) return void 0;
655
+ let usd = said.usd;
656
+ if (said.session !== void 0) {
657
+ const same = session === void 0 || session === said.session;
658
+ const fell = said.usd < before;
659
+ session ??= said.session;
660
+ if (!same) return void 0;
661
+ usd = Math.round((said.usd - before) * 1e9) / 1e9;
662
+ before = said.usd;
663
+ if (fell) return void 0;
664
+ }
665
+ return usd > 0 ? {
666
+ usd,
667
+ basis: "reported",
668
+ source
669
+ } : void 0;
670
+ };
671
+ }
672
+ /** How long reading a CLI's stored usage gets, past which the turn keeps none. */
673
+ const STORED_USAGE_MS = 5e3;
674
+ /** An adapter's usage, in the AI SDK's provider shape, as reeval's; null when it counts nothing. */
675
+ function harnessUsage(usage) {
676
+ const { inputTokens: input, outputTokens: output } = usage;
677
+ if (input.total === void 0 && output.total === void 0) return null;
678
+ const cacheRead = input.cacheRead ?? 0;
679
+ const cacheWrite = input.cacheWrite ?? 0;
591
680
  return {
592
- input: usage.inputTokenDetails?.noCacheTokens ?? Math.max((usage.inputTokens ?? 0) - cacheRead - cacheWrite, 0),
593
- output: usage.outputTokens ?? 0,
681
+ input: input.noCache ?? Math.max((input.total ?? 0) - cacheRead - cacheWrite, 0),
682
+ output: output.total ?? 0,
594
683
  cacheRead,
595
684
  cacheWrite,
596
- reasoning: usage.outputTokenDetails?.reasoningTokens ?? null
685
+ reasoning: output.reasoning ?? null
597
686
  };
598
687
  }
599
688
  /**
@@ -686,6 +775,7 @@ function createAgentHarness(descriptor, sandbox, vertex) {
686
775
  const auth = agentAuth(catalog);
687
776
  let servedModel = null;
688
777
  let finishMetadata;
778
+ const turnCost = turnCosts(catalog.reportedCost, descriptor.kind);
689
779
  const servers = agentMcp(catalog, mcp, plugins);
690
780
  const model = catalog.modelId?.(descriptor.model) ?? descriptor.model;
691
781
  const vertex = catalog.adc ? await writeAdc(sandbox, catalog.adc) : void 0;
@@ -737,21 +827,54 @@ function createAgentHarness(descriptor, sandbox, vertex) {
737
827
  if (signal.aborted) throw error;
738
828
  const message = error.message;
739
829
  const hint = authHint(catalog, message, refusedSince(0));
740
- throw new InfraError(`${hint ? `${hint}. ` : ""}${catalog.name} harness failed to start: ${message}`);
830
+ throw new InfraError(`${hint ? `${hint}. ` : ""}${catalog.name} harness failed to start: ${message}`, { transient: !hint && error.transient === true });
741
831
  }
832
+ const network = sandbox.network;
833
+ let usageMark;
834
+ const storedUsage = async (since) => {
835
+ const store = await catalog.storedUsage?.();
836
+ if (!store) return null;
837
+ const command = store.command({
838
+ since,
839
+ ...usageMark ? { mark: usageMark } : {}
840
+ });
841
+ const read = await Promise.resolve(network.run({
842
+ command,
843
+ abortSignal: AbortSignal.timeout(STORED_USAGE_MS)
844
+ })).then(({ exitCode, stdout, stderr }) => {
845
+ if (exitCode !== 0) throw new Error(`exit ${exitCode}: ${stderr.trim().slice(-300)}`);
846
+ return store.parse(stdout);
847
+ }).catch((error) => {
848
+ debugLog("harness", `reading ${catalog.name}'s stored usage failed: ${errorText(error)}`);
849
+ });
850
+ if (!read) {
851
+ debugLog("harness", `no usage in ${catalog.name}'s store since the turn began`);
852
+ return null;
853
+ }
854
+ if (read.mark) usageMark = read.mark;
855
+ const usage = harnessUsage(read.usage);
856
+ debugLog("harness", `usage read back from ${catalog.name}'s store: ${plural(read.calls, "model call")}${usage ? ` · ${usage.input} in · ${usage.output} out` : ""}`);
857
+ return usage;
858
+ };
859
+ let running;
860
+ let turn;
742
861
  return {
743
862
  async send(prompt, turnSignal) {
744
863
  const startedAt = Date.now();
864
+ const current = { startedAt };
865
+ turn = current;
745
866
  const parts = [];
746
867
  const servedBefore = sandbox.servedRequests?.().length ?? 0;
747
868
  const brokeredBefore = sandbox.brokeredStatuses?.().length ?? 0;
748
869
  servedModel = null;
749
870
  finishMetadata = void 0;
750
871
  let stepUsage = null;
751
- const finish = (text, status, extra = [], usage = stepUsage) => {
872
+ const steps = [];
873
+ const finish = async (text, status, extra = [], reported = stepUsage) => {
874
+ const usage = reported ?? await (current.salvaged ?? storedUsage(startedAt));
752
875
  const statuses = sandbox.brokeredStatuses?.().slice(brokeredBefore) ?? [];
753
876
  const refused = statuses.includes(401);
754
- const events = [...normalizeStream(parts, startedAt), ...extra].map((e) => e.type === "reasoning" && catalog.reasoning !== "streamed" ? {
877
+ const events = [...normalizeStream(parts, startedAt, { ...catalog.fullToolOutput ? { fullToolOutput: catalog.fullToolOutput } : {} }), ...extra].map((e) => e.type === "reasoning" && !timesReasoning(catalog.reasoning) ? {
755
878
  ...e,
756
879
  durationMs: null
757
880
  } : e);
@@ -783,20 +906,30 @@ function createAgentHarness(descriptor, sandbox, vertex) {
783
906
  status,
784
907
  statuses
785
908
  });
909
+ const stopped = rejected ? void 0 : providerStop({
910
+ errors: transcript.flatMap((e) => e.type === "error" ? [e.message] : []),
911
+ status,
912
+ statuses
913
+ });
914
+ const cost = turnCost(finishMetadata);
786
915
  return {
787
916
  text,
788
917
  status,
789
918
  usage,
919
+ ...cost ? { cost } : {},
790
920
  model: servedModel ?? sandbox.servedRequests?.().slice(servedBefore).at(-1) ?? null,
791
921
  unreported: catalog.unreported?.(finishMetadata) ?? [],
922
+ steps,
792
923
  reasoningMs: reasoningTime(catalog.reasoning, transcript, usage),
793
924
  startedAt: new Date(startedAt).toISOString(),
794
925
  durationMs: Date.now() - startedAt,
795
926
  transcript,
796
927
  ...rejected ? { rejectedKey: rejected } : {},
928
+ ...stopped ? { providerStop: stopped } : {},
797
929
  ...process.env.REEVAL_RAW_TRANSCRIPT ? { raw: parts } : {}
798
930
  };
799
931
  };
932
+ running = (note) => finish("", "aborted", [note]);
800
933
  try {
801
934
  const result = await agent.stream({
802
935
  session,
@@ -811,9 +944,16 @@ function createAgentHarness(descriptor, sandbox, vertex) {
811
944
  part
812
945
  });
813
946
  traceStreamPart(part, ms, tools);
814
- if (part.type === "finish-step") stepUsage = addUsage(stepUsage, tokenUsage(part.usage));
947
+ if (part.type === "finish-step") {
948
+ const step = tokenUsage(part.usage);
949
+ if (step) steps.push(step);
950
+ stepUsage = addUsage(stepUsage, step);
951
+ }
815
952
  }
816
- const total = tokenUsage(await result.totalUsage);
953
+ const reported = await result.totalUsage;
954
+ const total = tokenUsage(reported);
955
+ const mark = (await catalog.storedUsage?.())?.mark?.(reported.raw);
956
+ if (mark) usageMark = mark;
817
957
  const status = turnSignal.aborted ? "aborted" : "completed";
818
958
  return finish(await result.text, status, [], total ?? stepUsage);
819
959
  } catch (error) {
@@ -826,8 +966,16 @@ function createAgentHarness(descriptor, sandbox, vertex) {
826
966
  ts: new Date(startedAt + at).toISOString(),
827
967
  atMs: at
828
968
  }]);
969
+ } finally {
970
+ if (turn === current) turn = void 0;
829
971
  }
830
972
  },
973
+ stopped: (note) => running ? running(note) : Promise.reject(/* @__PURE__ */ new Error("No turn is running")),
974
+ salvage: async () => {
975
+ if (!turn) return;
976
+ turn.salvaged ??= storedUsage(turn.startedAt);
977
+ await turn.salvaged;
978
+ },
831
979
  close: () => {
832
980
  stopToken();
833
981
  return session.destroy().catch(() => {});
@@ -837,4 +985,4 @@ function createAgentHarness(descriptor, sandbox, vertex) {
837
985
  }
838
986
  export { createAgentHarness };
839
987
 
840
- //# sourceMappingURL=agent-CBoLndM7.js.map
988
+ //# sourceMappingURL=agent-Dx-bDmlV.js.map