reeval 0.20.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (481) hide show
  1. package/CHANGELOG.md +841 -0
  2. package/README.md +15 -15
  3. package/assets/gateway-models.mjs +65 -31
  4. package/dist/{adapter-C60z3wLq.js → adapter-Di9H2etg.js} +8 -49
  5. package/dist/adapter-Di9H2etg.js.map +1 -0
  6. package/dist/{agent-BmsMXZXx.js → agent-B-6GFDqA.js} +185 -38
  7. package/dist/agent-B-6GFDqA.js.map +1 -0
  8. package/dist/{agents-DXuFxtGC.js → agents-KBTYhSSh.js} +385 -116
  9. package/dist/agents-KBTYhSSh.js.map +1 -0
  10. package/dist/{app-files-DT5mJyV4.js → app-files-Cmh9FGr5.js} +3 -3
  11. package/dist/app-files-Cmh9FGr5.js.map +1 -0
  12. package/dist/{assemble-CE4pHKIa.js → assemble-CBMRIhso.js} +1 -1
  13. package/dist/{assemble-DM2PNU4R.js → assemble-DNVBTBp_.js} +172 -18
  14. package/dist/assemble-DNVBTBp_.js.map +1 -0
  15. package/dist/attach-EbDGKGTO.js +152 -0
  16. package/dist/attach-EbDGKGTO.js.map +1 -0
  17. package/dist/{attempt-CAdlwlNu.js → attempt-_dUsblVx.js} +2 -2
  18. package/dist/{attempt-CAdlwlNu.js.map → attempt-_dUsblVx.js.map} +1 -1
  19. package/dist/{attempt-host-7VviAJrl.js → attempt-host-DQvynDHk.js} +17 -8
  20. package/dist/attempt-host-DQvynDHk.js.map +1 -0
  21. package/dist/boat-Bzz8L-KQ.js +701 -0
  22. package/dist/boat-Bzz8L-KQ.js.map +1 -0
  23. package/dist/{boat-BjLqFJIZ.js → boat-Dek8ftIt.js} +20 -8
  24. package/dist/boat-Dek8ftIt.js.map +1 -0
  25. package/dist/bridge/index.mjs +6 -1
  26. package/dist/bundle-DZstHDeW.js +718 -0
  27. package/dist/bundle-DZstHDeW.js.map +1 -0
  28. package/dist/{capture-Ccf0Hmyx.js → capture-BkbPsfej.js} +61 -32
  29. package/dist/capture-BkbPsfej.js.map +1 -0
  30. package/dist/capture-CUAvk9KM.js +1 -0
  31. package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
  32. package/dist/cases-CM-MQRQp.js.map +1 -0
  33. package/dist/cases-FvhbM8M5.js +1 -0
  34. package/dist/{cli-DTuLSlmW.js → cli-BgBBuxxJ.js} +412 -1044
  35. package/dist/cli-BgBBuxxJ.js.map +1 -0
  36. package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-B5L9n-DV.js} +199 -35
  37. package/dist/cli-spec-B5L9n-DV.js.map +1 -0
  38. package/dist/cli.js +2 -2
  39. package/dist/client-DMEYR3MA.js +807 -0
  40. package/dist/client-DMEYR3MA.js.map +1 -0
  41. package/dist/{command-C2C7P5qo.js → command-BKb8eN_X.js} +10 -10
  42. package/dist/{command-C2C7P5qo.js.map → command-BKb8eN_X.js.map} +1 -1
  43. package/dist/{command-DSL7I-SI.js → command-D_w_kDpI.js} +25 -21
  44. package/dist/command-D_w_kDpI.js.map +1 -0
  45. package/dist/{command-Ff0T-1Hi.js → command-Dwlh0Vhu.js} +17 -16
  46. package/dist/command-Dwlh0Vhu.js.map +1 -0
  47. package/dist/{command-D8oMB2fa.js → command-loXH2jHr.js} +16 -16
  48. package/dist/{command-D8oMB2fa.js.map → command-loXH2jHr.js.map} +1 -1
  49. package/dist/{commands-xAOi_VEk.js → commands-Cuf57OYb.js} +174 -40
  50. package/dist/commands-Cuf57OYb.js.map +1 -0
  51. package/dist/{commands-Bk9NG1s_.js → commands-Dqli45sR.js} +1 -1
  52. package/dist/{config-SvK8pjEK.js → config-C0RKt00M.js} +288 -106
  53. package/dist/config-C0RKt00M.js.map +1 -0
  54. package/dist/{context-BdgCao9Q.js → context-DrEAtUyg.js} +2 -2
  55. package/dist/{context-BdgCao9Q.js.map → context-DrEAtUyg.js.map} +1 -1
  56. package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
  57. package/dist/cost-C21gGrju.js.map +1 -0
  58. package/dist/cost-IM80LyGD.js +1 -0
  59. package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-CRsFxorF.js} +4 -4
  60. package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-CRsFxorF.js.map} +1 -1
  61. package/dist/db-3o8s2EyD.js +2 -0
  62. package/dist/{db-BzOJ4Js4.js → db-ErnJ-gMD.js} +168 -248
  63. package/dist/db-ErnJ-gMD.js.map +1 -0
  64. package/dist/debug-CIXzB4wK.js.map +1 -1
  65. package/dist/{default-effort-BDKIxJyC.js → default-effort-D70cTG00.js} +4 -4
  66. package/dist/{default-effort-BDKIxJyC.js.map → default-effort-D70cTG00.js.map} +1 -1
  67. package/dist/{default-effort-BBDPbJ4D.js → default-effort-DHzj84dD.js} +1 -1
  68. package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
  69. package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
  70. package/dist/{docker-DFrjoNs-.js → docker-CFWzZOEk.js} +4 -4
  71. package/dist/{docker-DFrjoNs-.js.map → docker-CFWzZOEk.js.map} +1 -1
  72. package/dist/docker-DrMnpL84.js +1 -0
  73. package/dist/{doctor-DWK4ttex.js → doctor-B7utCSdh.js} +121 -100
  74. package/dist/doctor-B7utCSdh.js.map +1 -0
  75. package/dist/{environment-cPMCOk-h.js → environment-C0Gg8kyN.js} +49 -11
  76. package/dist/environment-C0Gg8kyN.js.map +1 -0
  77. package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
  78. package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
  79. package/dist/{eval-source-C9b1h8L6.js → eval-source-CsW5VIXS.js} +24 -12
  80. package/dist/eval-source-CsW5VIXS.js.map +1 -0
  81. package/dist/{evaluate-B12Tosvq.js → evaluate-BHQyAszQ.js} +679 -142
  82. package/dist/evaluate-BHQyAszQ.js.map +1 -0
  83. package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DzHq4EsU.d.ts} +2 -2
  84. package/dist/fake-bootstrap-VpisF1WE.js +15 -0
  85. package/dist/fake-bootstrap-VpisF1WE.js.map +1 -0
  86. package/dist/{files-tiNWe8Ih.js → files-BCraBodb.js} +3 -3
  87. package/dist/{files-tiNWe8Ih.js.map → files-BCraBodb.js.map} +1 -1
  88. package/dist/files-BjOJw4sn.js +1 -0
  89. package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
  90. package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
  91. package/dist/{files-BOBdC0uI.js → files-BuKWZCzY.js} +6 -6
  92. package/dist/{files-BOBdC0uI.js.map → files-BuKWZCzY.js.map} +1 -1
  93. package/dist/files-CAzVG3Rr.js +1 -0
  94. package/dist/{files-F5EG0WPT.js → files-Cv856jLN.js} +5 -5
  95. package/dist/{files-F5EG0WPT.js.map → files-Cv856jLN.js.map} +1 -1
  96. package/dist/files-D-Lir4aE.js +1 -0
  97. package/dist/files-DRlJ0Xr2.js +1 -0
  98. package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
  99. package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
  100. package/dist/{files-s28a6czq.js → files-qjeqXnfE.js} +143 -20
  101. package/dist/files-qjeqXnfE.js.map +1 -0
  102. package/dist/gateway.d.ts +1 -1
  103. package/dist/{git-BpoqEgDV.js → git-DOrOtAiD.js} +3 -3
  104. package/dist/{git-BpoqEgDV.js.map → git-DOrOtAiD.js.map} +1 -1
  105. package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
  106. package/dist/harness-FcTyyPco.js.map +1 -0
  107. package/dist/harness-install-Dxho1yaU.js +53 -0
  108. package/dist/harness-install-Dxho1yaU.js.map +1 -0
  109. package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
  110. package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
  111. package/dist/harness.d.ts +11 -8
  112. package/dist/harness.js +1 -1
  113. package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
  114. package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
  115. package/dist/{images-D2jFze2v.js → images-Ch1y1t6_.js} +122 -73
  116. package/dist/images-Ch1y1t6_.js.map +1 -0
  117. package/dist/index.d.ts +46 -7
  118. package/dist/index.js +6 -6
  119. package/dist/{init-Bi1Q3lRi.js → init-C2HqCwnt.js} +33 -14
  120. package/dist/init-C2HqCwnt.js.map +1 -0
  121. package/dist/init-CM9xwyre.js +1 -0
  122. package/dist/{ink-list-BpfidQ1h.js → ink-list-BbKXElcQ.js} +54 -15
  123. package/dist/ink-list-BbKXElcQ.js.map +1 -0
  124. package/dist/inputs-5UIDpjz-.js +190 -0
  125. package/dist/inputs-5UIDpjz-.js.map +1 -0
  126. package/dist/inputs-load-C82HOKnb.js +1 -0
  127. package/dist/inputs-load-CGPyigMC.js +95 -0
  128. package/dist/inputs-load-CGPyigMC.js.map +1 -0
  129. package/dist/{invocations-CxGNOlh0.js → invocations-BrZAMuTb.js} +5 -5
  130. package/dist/{invocations-CxGNOlh0.js.map → invocations-BrZAMuTb.js.map} +1 -1
  131. package/dist/jiti-radm1Soi.js +52 -0
  132. package/dist/jiti-radm1Soi.js.map +1 -0
  133. package/dist/{judge-COMXT1Hx.js → judge-DwzqucLS.js} +17 -12
  134. package/dist/{judge-COMXT1Hx.js.map → judge-DwzqucLS.js.map} +1 -1
  135. package/dist/judge.d.ts +3 -1
  136. package/dist/judge.js +1 -1
  137. package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
  138. package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
  139. package/dist/{layout-Cm85tQYk.js → layout-Czh5wOpz.js} +14 -11
  140. package/dist/layout-Czh5wOpz.js.map +1 -0
  141. package/dist/{list-DuThsE28.js → list-B_C0k5ex.js} +33 -19
  142. package/dist/{list-DuThsE28.js.map → list-B_C0k5ex.js.map} +1 -1
  143. package/dist/live-C9Et_BVT.js +1 -0
  144. package/dist/{live-CgYbO4Te.js → live-DGgIrVw0.js} +7 -5
  145. package/dist/live-DGgIrVw0.js.map +1 -0
  146. package/dist/{live-DL0se4Zd.js → live-DgFm0H90.js} +9 -9
  147. package/dist/{live-DL0se4Zd.js.map → live-DgFm0H90.js.map} +1 -1
  148. package/dist/machine-service-2lIv5UWF.js +1 -0
  149. package/dist/machine-service-Ct_1XMMi.js +520 -0
  150. package/dist/machine-service-Ct_1XMMi.js.map +1 -0
  151. package/dist/machines-BnquHcBZ.js +120 -0
  152. package/dist/machines-BnquHcBZ.js.map +1 -0
  153. package/dist/{model-CkNVAxob.js → model-B1tdv9-d.js} +1 -1
  154. package/dist/model-Bcy4Qjmm.js +1 -0
  155. package/dist/{model-V3mMXp6x.js → model-DF77SgZv.js} +6 -6
  156. package/dist/{model-V3mMXp6x.js.map → model-DF77SgZv.js.map} +1 -1
  157. package/dist/{model-CwwkrCb7.js → model-DRa7yv6K.js} +9 -6
  158. package/dist/model-DRa7yv6K.js.map +1 -0
  159. package/dist/{model-cgt5uvu8.js → model-E5X8YBJM.js} +7 -6
  160. package/dist/model-E5X8YBJM.js.map +1 -0
  161. package/dist/model-calls-B0BoQfJ4.js +95 -0
  162. package/dist/model-calls-B0BoQfJ4.js.map +1 -0
  163. package/dist/{node-D7pAU--0.js → node-BjuhqQwa.js} +21 -10
  164. package/dist/node-BjuhqQwa.js.map +1 -0
  165. package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
  166. package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
  167. package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-B1sIOTD2.js} +2 -2
  168. package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-B1sIOTD2.js.map} +1 -1
  169. package/dist/options-C989kuVE.js +1 -0
  170. package/dist/own-reads-CYmBlrHK.js +15 -0
  171. package/dist/own-reads-CYmBlrHK.js.map +1 -0
  172. package/dist/{patch-DAi1nCuH.js → patch-BR8rdl7r.js} +2 -2
  173. package/dist/{patch-DAi1nCuH.js.map → patch-BR8rdl7r.js.map} +1 -1
  174. package/dist/pi-providers-HygmL1y-.js +552 -0
  175. package/dist/pi-providers-HygmL1y-.js.map +1 -0
  176. package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
  177. package/dist/plain-Ctn8oUdp.js.map +1 -0
  178. package/dist/{plan-CsPilzpk.js → plan-BcT4G5jj.js} +104 -15
  179. package/dist/plan-BcT4G5jj.js.map +1 -0
  180. package/dist/{plugins-BfnhazOv.js → plugins-cC0sqoDA.js} +608 -12
  181. package/dist/plugins-cC0sqoDA.js.map +1 -0
  182. package/dist/preflight-BMVDBcux.js +1 -0
  183. package/dist/{preflight-BixaE4BA.js → preflight-C9OtTb7k.js} +4 -4
  184. package/dist/{preflight-BixaE4BA.js.map → preflight-C9OtTb7k.js.map} +1 -1
  185. package/dist/{preflight-GkpgrBxP.js → preflight-DySHXeFJ.js} +9 -10
  186. package/dist/preflight-DySHXeFJ.js.map +1 -0
  187. package/dist/{preflight-D1C-iH0U.js → preflight-zsAn3heN.js} +1 -1
  188. package/dist/{prepare-C--rM_wO.js → prepare-MaRWRhNT.js} +2 -2
  189. package/dist/prepare-nbX1CXXg.js +409 -0
  190. package/dist/prepare-nbX1CXXg.js.map +1 -0
  191. package/dist/prices-B1vm_lPk.js +1 -0
  192. package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
  193. package/dist/prices-BrY_ihLs.js.map +1 -0
  194. package/dist/{provenance-BXyYwUC8.js → provenance-CkykifD9.js} +5 -5
  195. package/dist/{provenance-BXyYwUC8.js.map → provenance-CkykifD9.js.map} +1 -1
  196. package/dist/provider-DHT20Gl8.js +1 -0
  197. package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
  198. package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
  199. package/dist/provider-stop-CxzWW7Ft.js +69 -0
  200. package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
  201. package/dist/providers-BkLGzaSY.js +1 -0
  202. package/dist/{providers-1p4HKYIp.js → providers-bNgfKa6m.js} +224 -49
  203. package/dist/providers-bNgfKa6m.js.map +1 -0
  204. package/dist/query-B2SQddoV.js +1 -0
  205. package/dist/{query-DhhU9ADI.js → query-DhV3a8lM.js} +2 -2
  206. package/dist/{query-DhhU9ADI.js.map → query-DhV3a8lM.js.map} +1 -1
  207. package/dist/{questions-CMAq4pr8.js → questions-Csv64WCb.js} +7 -7
  208. package/dist/{questions-CMAq4pr8.js.map → questions-Csv64WCb.js.map} +1 -1
  209. package/dist/{record-6IUzwP9K.js → record--vHreTLk.js} +11 -5
  210. package/dist/record--vHreTLk.js.map +1 -0
  211. package/dist/{redact-BhjtnjrU.js → redact-t-L3VGy8.js} +106 -2
  212. package/dist/redact-t-L3VGy8.js.map +1 -0
  213. package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
  214. package/dist/remarks-BpdM_2RC.js.map +1 -0
  215. package/dist/{remarks-BNABzemt.js → remarks-CYFOsnz5.js} +46 -11
  216. package/dist/remarks-CYFOsnz5.js.map +1 -0
  217. package/dist/remarks-CleoDlYw.js +1 -0
  218. package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
  219. package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
  220. package/dist/{remote-U7fNt0UP.js → remote-0CHog7Oq.js} +6 -6
  221. package/dist/{remote-U7fNt0UP.js.map → remote-0CHog7Oq.js.map} +1 -1
  222. package/dist/resolve-Bm9ZyFR-.js +1 -0
  223. package/dist/resolve-Bul8lHaa.js +205 -0
  224. package/dist/resolve-Bul8lHaa.js.map +1 -0
  225. package/dist/{resolve-Cx3a9UQ2.js → resolve-DorSHDBs.js} +14 -11
  226. package/dist/resolve-DorSHDBs.js.map +1 -0
  227. package/dist/resources-DqdwihhT.js +62 -0
  228. package/dist/resources-DqdwihhT.js.map +1 -0
  229. package/dist/{results-CQKXmHym.js → results-DhMgRkR0.js} +9 -22
  230. package/dist/results-DhMgRkR0.js.map +1 -0
  231. package/dist/{resume-POpZa1ge.js → resume-C8Ihfy-C.js} +166 -17
  232. package/dist/resume-C8Ihfy-C.js.map +1 -0
  233. package/dist/{retry-CmG809Mp.js → retry-Bfk7pDti.js} +4 -4
  234. package/dist/{retry-CmG809Mp.js.map → retry-Bfk7pDti.js.map} +1 -1
  235. package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
  236. package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
  237. package/dist/routes-Cp4gAjBZ.js +427 -0
  238. package/dist/routes-Cp4gAjBZ.js.map +1 -0
  239. package/dist/{runtime-Xm9oeC7o.js → runtime-CPUH0Vk5.js} +46 -5
  240. package/dist/runtime-CPUH0Vk5.js.map +1 -0
  241. package/dist/sandbox/boat.d.ts +1 -1
  242. package/dist/sandbox/boat.js +1 -1
  243. package/dist/{sandbox-DrlL4yqb.js → sandbox-z8q_4Ywn.js} +3 -3
  244. package/dist/{sandbox-DrlL4yqb.js.map → sandbox-z8q_4Ywn.js.map} +1 -1
  245. package/dist/sandbox.d.ts +1 -1
  246. package/dist/sandbox.js +1 -1
  247. package/dist/{server-C00zd6zl.js → server-BsQZtuF9.js} +9 -9
  248. package/dist/{server-C00zd6zl.js.map → server-BsQZtuF9.js.map} +1 -1
  249. package/dist/server-ZaMzUNmV.js +1 -0
  250. package/dist/services-BG8xuU82.js +39 -0
  251. package/dist/services-BG8xuU82.js.map +1 -0
  252. package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
  253. package/dist/shared-DtZwIAfy.js.map +1 -0
  254. package/dist/{source-DDgr48Gn.js → source-9HWaTEgV.js} +2 -2
  255. package/dist/{source-DDgr48Gn.js.map → source-9HWaTEgV.js.map} +1 -1
  256. package/dist/source-BcBDj-R0.js +1 -0
  257. package/dist/src-B2eHTCnp.js +3 -0
  258. package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
  259. package/dist/src-B5r1d23Y.js.map +1 -0
  260. package/dist/{src-Dw79VLa3.js → src-Bb3k9Kn_.js} +208 -59
  261. package/dist/src-Bb3k9Kn_.js.map +1 -0
  262. package/dist/src-BfOlH_Bu.js +2 -0
  263. package/dist/src-BuGc40IF.js +2 -0
  264. package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
  265. package/dist/src-C-ChdnG6.js.map +1 -0
  266. package/dist/src-C2K5_afU.js +2 -0
  267. package/dist/{src-YvblOrBU.js → src-C3OhA5pw.js} +87 -13
  268. package/dist/src-C3OhA5pw.js.map +1 -0
  269. package/dist/src-CSVJjghq.js +4 -0
  270. package/dist/{src-y0nOeRRE.js → src-CWfy8ct8.js} +1572 -487
  271. package/dist/src-CWfy8ct8.js.map +1 -0
  272. package/dist/src-CcrQQB0_.js +2 -0
  273. package/dist/src-CotUxxQV.js +2 -0
  274. package/dist/src-D-PwBaBB.js +2 -0
  275. package/dist/{src-Cb2MWozp.js → src-DNWiU__5.js} +276 -41
  276. package/dist/src-DNWiU__5.js.map +1 -0
  277. package/dist/{src-KxS91P7L.js → src-DwcTn4JY.js} +79 -11
  278. package/dist/src-DwcTn4JY.js.map +1 -0
  279. package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
  280. package/dist/src-boB6D-5T.js.map +1 -0
  281. package/dist/{src-DvGns8cz.js → src-c3ZQu7Uu.js} +299 -17
  282. package/dist/src-c3ZQu7Uu.js.map +1 -0
  283. package/dist/src-jYqNkT-F.js +1 -0
  284. package/dist/start-Db5VRcRZ.js +660 -0
  285. package/dist/start-Db5VRcRZ.js.map +1 -0
  286. package/dist/state-CKIjZZRK.js +1 -0
  287. package/dist/{state-328JpnHA.js → state-DUYdzNA7.js} +21 -21
  288. package/dist/{state-328JpnHA.js.map → state-DUYdzNA7.js.map} +1 -1
  289. package/dist/store/local.d.ts +8 -0
  290. package/dist/store/local.js +16 -0
  291. package/dist/store/local.js.map +1 -0
  292. package/dist/store/s3.d.ts +10 -0
  293. package/dist/store/s3.js +18 -0
  294. package/dist/store/s3.js.map +1 -0
  295. package/dist/suite-CH1aaAbs.js +2 -0
  296. package/dist/{suite-COglUhPd.js → suite-DjGxmDSY.js} +6 -5
  297. package/dist/{suite-COglUhPd.js.map → suite-DjGxmDSY.js.map} +1 -1
  298. package/dist/{summary-DDy9DfyJ.js → summary-BK5Sa_N0.js} +28 -7
  299. package/dist/summary-BK5Sa_N0.js.map +1 -0
  300. package/dist/sync-DUi_P2UJ.js +559 -0
  301. package/dist/sync-DUi_P2UJ.js.map +1 -0
  302. package/dist/{tarball-CESxtK4X.js → tarball-C8BSTAMk.js} +2 -2
  303. package/dist/{tarball-CESxtK4X.js.map → tarball-C8BSTAMk.js.map} +1 -1
  304. package/dist/tart-BQ_Ap0l4.js +508 -0
  305. package/dist/tart-BQ_Ap0l4.js.map +1 -0
  306. package/dist/tart-CzViW3WZ.js +186 -0
  307. package/dist/tart-CzViW3WZ.js.map +1 -0
  308. package/dist/tart-DmVuVsmX.js +1 -0
  309. package/dist/tart-bake-D1RttYfM.js +148 -0
  310. package/dist/tart-bake-D1RttYfM.js.map +1 -0
  311. package/dist/tart-bake-DjFcsG3N.js +1 -0
  312. package/dist/tart-cli-BOcluha1.js +1 -0
  313. package/dist/tart-cli-gA9xEaQL.js +89 -0
  314. package/dist/tart-cli-gA9xEaQL.js.map +1 -0
  315. package/dist/tart-hosts-CHNrfV6u.js +313 -0
  316. package/dist/tart-hosts-CHNrfV6u.js.map +1 -0
  317. package/dist/tart-hosts-CWse-ASo.js +1 -0
  318. package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CLBjfF9u.js} +64 -6
  319. package/dist/tart-images-CLBjfF9u.js.map +1 -0
  320. package/dist/testing.d.ts +36 -2
  321. package/dist/testing.js +4 -4
  322. package/dist/tool-class-krsmEo44.js +433 -0
  323. package/dist/tool-class-krsmEo44.js.map +1 -0
  324. package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
  325. package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
  326. package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
  327. package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
  328. package/dist/{transcript-CCf07UmU.js → transcript-BXWZiMuC.js} +3 -3
  329. package/dist/{transcript-CCf07UmU.js.map → transcript-BXWZiMuC.js.map} +1 -1
  330. package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
  331. package/dist/types-CbcnJxCy.js.map +1 -0
  332. package/dist/{types-DBwFsGem.d.ts → types-DKK8jj--.d.ts} +517 -45
  333. package/dist/ui-DKYjTVpn.js +1455 -0
  334. package/dist/ui-DKYjTVpn.js.map +1 -0
  335. package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
  336. package/dist/usage-CwIRPPZ9.js.map +1 -0
  337. package/dist/vercel-models-Ci33llXz.js +21 -0
  338. package/dist/vercel-models-Ci33llXz.js.map +1 -0
  339. package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
  340. package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
  341. package/dist/vertex-DXfcXYKn.js +1 -0
  342. package/dist/view/index.d.ts +32 -3
  343. package/dist/view/index.js +8 -8
  344. package/dist/{views-BROacjBv.js → views-B1zVLJsF.js} +6 -6
  345. package/dist/{views-BROacjBv.js.map → views-B1zVLJsF.js.map} +1 -1
  346. package/dist/views-DCjtw1jF.js +2 -0
  347. package/dist/{worker-entry-Smb4bNDc.js → worker-entry-DrAYYt0M.js} +37 -14
  348. package/dist/worker-entry-DrAYYt0M.js.map +1 -0
  349. package/dist/workspace-CnRbV8cM.js +1 -0
  350. package/dist/{workspace-DNDTEk8v.js → workspace-DwTe1zo6.js} +7 -7
  351. package/dist/{workspace-DNDTEk8v.js.map → workspace-DwTe1zo6.js.map} +1 -1
  352. package/docker/Dockerfile +1 -1
  353. package/package.json +15 -1
  354. package/skills/eval-rubric/SKILL.md +10 -0
  355. package/skills/reeval/SKILL.md +370 -0
  356. package/skills/reeval/references/capture.md +48 -0
  357. package/skills/reeval/references/cli.md +162 -0
  358. package/skills/reeval/references/compare.md +180 -0
  359. package/skills/reeval/references/config.md +147 -0
  360. package/skills/reeval/references/environments.md +218 -0
  361. package/skills/reeval/references/evals.md +241 -0
  362. package/skills/reeval/references/mistakes.md +39 -0
  363. package/skills/reeval/references/vocabulary.md +19 -0
  364. package/skills/setup/SKILL.md +219 -0
  365. package/skills/setup/references/templates.md +258 -0
  366. package/dist/adapter-C60z3wLq.js.map +0 -1
  367. package/dist/agent-BmsMXZXx.js.map +0 -1
  368. package/dist/agents-DXuFxtGC.js.map +0 -1
  369. package/dist/app-files-DT5mJyV4.js.map +0 -1
  370. package/dist/assemble-DM2PNU4R.js.map +0 -1
  371. package/dist/attempt-host-7VviAJrl.js.map +0 -1
  372. package/dist/boat-BjLqFJIZ.js.map +0 -1
  373. package/dist/boat-DSNgZt-t.js +0 -322
  374. package/dist/boat-DSNgZt-t.js.map +0 -1
  375. package/dist/capture-BzotD2hs.js +0 -1
  376. package/dist/capture-Ccf0Hmyx.js.map +0 -1
  377. package/dist/cases-Dy0mifal.js +0 -1
  378. package/dist/cases-z9E6DCd5.js.map +0 -1
  379. package/dist/cli-DTuLSlmW.js.map +0 -1
  380. package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
  381. package/dist/client-KM4C5r5_.js +0 -387
  382. package/dist/client-KM4C5r5_.js.map +0 -1
  383. package/dist/command-DSL7I-SI.js.map +0 -1
  384. package/dist/command-Ff0T-1Hi.js.map +0 -1
  385. package/dist/commands-xAOi_VEk.js.map +0 -1
  386. package/dist/config-SvK8pjEK.js.map +0 -1
  387. package/dist/cost-BdRgySQl.js.map +0 -1
  388. package/dist/cost-Cr26Alhq.js +0 -1
  389. package/dist/db-BzOJ4Js4.js.map +0 -1
  390. package/dist/db-CQRZ3URX.js +0 -2
  391. package/dist/docker--mukta3L.js +0 -1
  392. package/dist/docker-DG_uFgn2.js +0 -246
  393. package/dist/docker-DG_uFgn2.js.map +0 -1
  394. package/dist/doctor-DWK4ttex.js.map +0 -1
  395. package/dist/environment-cPMCOk-h.js.map +0 -1
  396. package/dist/eval-source-C9b1h8L6.js.map +0 -1
  397. package/dist/evaluate-B12Tosvq.js.map +0 -1
  398. package/dist/files-C1Rk9FYA.js +0 -1
  399. package/dist/files-DMQ9-Vv5.js +0 -1
  400. package/dist/files-Di0xDbCc.js +0 -1
  401. package/dist/files-G5jf8wBp.js +0 -1
  402. package/dist/files-s28a6czq.js.map +0 -1
  403. package/dist/harness-DXv2gnaT.js.map +0 -1
  404. package/dist/images-D2jFze2v.js.map +0 -1
  405. package/dist/init-Bi1Q3lRi.js.map +0 -1
  406. package/dist/init-CUw65o36.js +0 -1
  407. package/dist/ink-list-BpfidQ1h.js.map +0 -1
  408. package/dist/kinds-Bh2I35id.js +0 -698
  409. package/dist/kinds-Bh2I35id.js.map +0 -1
  410. package/dist/layout-Cm85tQYk.js.map +0 -1
  411. package/dist/live-CgYbO4Te.js.map +0 -1
  412. package/dist/live-DZLmj0iG.js +0 -1
  413. package/dist/matchers-CXLn-1jm.js +0 -148
  414. package/dist/matchers-CXLn-1jm.js.map +0 -1
  415. package/dist/model-Cv4GuTsZ.js +0 -1
  416. package/dist/model-CwwkrCb7.js.map +0 -1
  417. package/dist/model-cgt5uvu8.js.map +0 -1
  418. package/dist/node-D7pAU--0.js.map +0 -1
  419. package/dist/options-DPUmk7v_.js +0 -1
  420. package/dist/plain-BYUf_fUV.js.map +0 -1
  421. package/dist/plan-CsPilzpk.js.map +0 -1
  422. package/dist/plugins-BfnhazOv.js.map +0 -1
  423. package/dist/preflight-Bn3ianye.js +0 -1
  424. package/dist/preflight-GkpgrBxP.js.map +0 -1
  425. package/dist/prepare-DQEDXoDp.js +0 -268
  426. package/dist/prepare-DQEDXoDp.js.map +0 -1
  427. package/dist/prices-5m3SzMZ3.js.map +0 -1
  428. package/dist/provider-BnUoz4KB.js +0 -1
  429. package/dist/providers-1p4HKYIp.js.map +0 -1
  430. package/dist/providers-B-ijCktf.js +0 -1
  431. package/dist/query-BYhyf0LR.js +0 -1
  432. package/dist/record-6IUzwP9K.js.map +0 -1
  433. package/dist/redact-BhjtnjrU.js.map +0 -1
  434. package/dist/remarks-BNABzemt.js.map +0 -1
  435. package/dist/remarks-CAvTSq16.js +0 -1
  436. package/dist/remarks-D9Lcd1fF.js.map +0 -1
  437. package/dist/resolve-C14Pmtej.js +0 -1
  438. package/dist/resolve-Cx3a9UQ2.js.map +0 -1
  439. package/dist/results-CQKXmHym.js.map +0 -1
  440. package/dist/resume-POpZa1ge.js.map +0 -1
  441. package/dist/runtime-Xm9oeC7o.js.map +0 -1
  442. package/dist/server-C1UJcpUh.js +0 -1
  443. package/dist/shared-Cfi45Bvy.js.map +0 -1
  444. package/dist/source-DJHmQv_j.js +0 -1
  445. package/dist/src-1dBuXzs4.js +0 -3
  446. package/dist/src-BKpioayC.js +0 -2
  447. package/dist/src-BNQznj4z.js.map +0 -1
  448. package/dist/src-BY1cCaoJ.js +0 -2
  449. package/dist/src-BbZYaQ8b.js +0 -2
  450. package/dist/src-CDFH2vVN.js +0 -2
  451. package/dist/src-CMW5R9Q0.js +0 -1
  452. package/dist/src-CV_7DkPa.js.map +0 -1
  453. package/dist/src-Cb2MWozp.js.map +0 -1
  454. package/dist/src-CrV2WA2i.js +0 -2
  455. package/dist/src-CsFaawcZ.js +0 -2
  456. package/dist/src-DAzaCTcN.js.map +0 -1
  457. package/dist/src-DvGns8cz.js.map +0 -1
  458. package/dist/src-Dw79VLa3.js.map +0 -1
  459. package/dist/src-KxS91P7L.js.map +0 -1
  460. package/dist/src-YvblOrBU.js.map +0 -1
  461. package/dist/src-wKwznPgt.js +0 -2
  462. package/dist/src-y0nOeRRE.js.map +0 -1
  463. package/dist/state-Ca55Gk_O.js +0 -1
  464. package/dist/suite-CKBa6MKt.js +0 -2
  465. package/dist/summary-DDy9DfyJ.js.map +0 -1
  466. package/dist/tart-cli-C-GsbVsQ.js +0 -1
  467. package/dist/tart-cli-CU0nFS-L.js +0 -55
  468. package/dist/tart-cli-CU0nFS-L.js.map +0 -1
  469. package/dist/tart-dWoJvWHc.js +0 -586
  470. package/dist/tart-dWoJvWHc.js.map +0 -1
  471. package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
  472. package/dist/tool-target-DS5d72eH.js +0 -40
  473. package/dist/tool-target-DS5d72eH.js.map +0 -1
  474. package/dist/types-C2mLpph3.js.map +0 -1
  475. package/dist/usage-DJiDnh2D.js.map +0 -1
  476. package/dist/vercel-CGlSBv63.js +0 -344
  477. package/dist/vercel-CGlSBv63.js.map +0 -1
  478. package/dist/vertex-O88TCwgY.js +0 -1
  479. package/dist/views-B0Yt_Q1S.js +0 -2
  480. package/dist/worker-entry-Smb4bNDc.js.map +0 -1
  481. package/dist/workspace-C0dDP75l.js +0 -1
package/README.md CHANGED
@@ -27,11 +27,11 @@ Evals are TypeScript files in `evals/`, next to the tool they test. Each run wri
27
27
 
28
28
  Cursor ⋅ Claude Code ⋅ Codex ⋅ Antigravity ⋅ OpenCode ⋅ Goose ⋅ pi ⋅ Grok Build
29
29
 
30
- - **Real agents**: each harness runs the agent itself, with its own tools, skills, and rules, as it works on a developer's machine ([Choose an agent](docs/configuration/agents/index.mdx)).
31
- - **Keys stay on the host**: each attempt starts in a fresh sandbox that holds only a placeholder key. reeval's egress gateway swaps in the real key on requests to that provider and nowhere else ([Pass secrets](docs/configuration/environments/secrets.mdx)).
32
- - **Compare a change**: one eval runs a baseline and its variants with the same prompt and checks, then prints whether each variant helped, made it worse, or did the same ([Measure lift against a baseline](docs/results/compare/lift.mdx)).
33
- - **Pinned environments**: an environment starts from a workspace, such as a git repository at a fixed ref, then adds files, setup steps, rules, skills, and MCP servers, so every attempt starts from the same state ([Define an environment](docs/configuration/environments/index.mdx)).
34
- - **History in Git**: recorded runs sit beside the code. `reeval query` reads them with SQL, and `reeval studio` shows each attempt's transcript, diff, and checks ([Record runs in Git](docs/evaluations/runs/recording.mdx)).
30
+ - **Real agents**: each harness runs the agent itself, with its own tools, skills, and rules, as it works on a developer's machine ([Choose an agent](https://github.com/invertase/reeval/blob/v0.22.0/docs/configuration/agents/index.mdx)).
31
+ - **Keys stay on the host**: each attempt starts in a fresh sandbox that holds only a placeholder key. reeval's egress gateway swaps in the real key on requests to that provider and nowhere else ([Pass secrets](https://github.com/invertase/reeval/blob/v0.22.0/docs/configuration/environments/secrets.mdx)).
32
+ - **Compare a change**: one eval runs a baseline and its variants with the same prompt and checks, then prints whether each variant helped, made it worse, or did the same ([Measure lift against a baseline](https://github.com/invertase/reeval/blob/v0.22.0/docs/results/compare/lift.mdx)).
33
+ - **Pinned environments**: an environment starts from a workspace, such as a git repository at a fixed ref, then adds files, setup steps, rules, skills, and MCP servers, so every attempt starts from the same state ([Define an environment](https://github.com/invertase/reeval/blob/v0.22.0/docs/configuration/environments/index.mdx)).
34
+ - **History in Git**: recorded runs sit beside the code. `reeval query` reads them with SQL, and `reeval studio` shows each attempt's transcript, diff, and checks ([Record runs in Git](https://github.com/invertase/reeval/blob/v0.22.0/docs/evaluations/runs/recording.mdx)).
35
35
 
36
36
  ## Install
37
37
 
@@ -140,23 +140,23 @@ Environments load extensions from where teams already keep them:
140
140
 
141
141
  ## Examples
142
142
 
143
- The recipes in [`docs/evaluations/recipes`](docs/evaluations/recipes) are complete evals for common jobs:
143
+ The recipes in [`docs/evaluations/recipes`](https://github.com/invertase/reeval/blob/v0.22.0/docs/evaluations/recipes) are complete evals for common jobs:
144
144
 
145
- - [Measure a skill](docs/evaluations/recipes/measure-a-skill.mdx): with and without a skill, gated on a minimum lift.
146
- - [Compare harnesses](docs/evaluations/recipes/compare-harnesses.mdx): one suite on several agents at once, with pass rate and effort per agent.
147
- - [Measure MCP servers](docs/evaluations/recipes/measure-mcp-servers.mdx): check the agent calls the server's tools, and whether answers improve.
148
- - [Check a recommendation](docs/evaluations/recipes/test-recommendations.mdx): ask which library to use, and let a judge decide whether the agent picks the right one.
149
- - [Fail a pull request](docs/evaluations/recipes/gate-pull-requests.mdx): run on every pull request and fail the check on a regression.
145
+ - [Measure a skill](https://github.com/invertase/reeval/blob/v0.22.0/docs/evaluations/recipes/measure-a-skill.mdx): with and without a skill, gated on a minimum lift.
146
+ - [Compare harnesses](https://github.com/invertase/reeval/blob/v0.22.0/docs/evaluations/recipes/compare-harnesses.mdx): one suite on several agents at once, with pass rate and effort per agent.
147
+ - [Measure MCP servers](https://github.com/invertase/reeval/blob/v0.22.0/docs/evaluations/recipes/measure-mcp-servers.mdx): check the agent calls the server's tools, and whether answers improve.
148
+ - [Check a recommendation](https://github.com/invertase/reeval/blob/v0.22.0/docs/evaluations/recipes/test-recommendations.mdx): ask which library to use, and let a judge decide whether the agent picks the right one.
149
+ - [Fail a pull request](https://github.com/invertase/reeval/blob/v0.22.0/docs/evaluations/recipes/gate-pull-requests.mdx): run on every pull request and fail the check on a regression.
150
150
 
151
- The [`examples/`](examples) directory has 80 runnable projects, one idea each, from [`with-checks`](examples/with-checks) (deterministic checks) and [`with-skill-lift`](examples/with-skill-lift) (does a skill help?) to sandboxes, egress, judges, captures and real-world tasks. Most run on a fake agent with no keys or Docker; [`examples/README.md`](examples/README.md) lists them all.
151
+ The [`examples/`](https://github.com/invertase/reeval/blob/v0.22.0/examples) directory has 80 runnable projects, one idea each, from [`with-checks`](https://github.com/invertase/reeval/blob/v0.22.0/examples/with-checks) (deterministic checks) and [`with-skill-lift`](https://github.com/invertase/reeval/blob/v0.22.0/examples/with-skill-lift) (does a skill help?) to sandboxes, egress, judges, captures and real-world tasks. Most run on a fake agent with no keys or Docker; [`examples/README.md`](https://github.com/invertase/reeval/blob/v0.22.0/examples/README.md) lists them all.
152
152
 
153
153
  ## Documentation
154
154
 
155
- The manual in [`docs/`](docs/overview.mdx) covers harnesses, sandboxes, environments, secrets and egress, checks and judges, profiles, history, and every CLI command and flag.
155
+ The manual in [`docs/`](https://github.com/invertase/reeval/blob/v0.22.0/docs/overview.mdx) covers harnesses, sandboxes, environments, secrets and egress, checks and judges, profiles, history, and every CLI command and flag.
156
156
 
157
157
  ## For coding agents
158
158
 
159
- To write or update evals, load [`skills/reeval/SKILL.md`](skills/reeval/SKILL.md). For every command, flag, and exit code, run `npx reeval --help`.
159
+ To write or update evals, load [`skills/reeval/SKILL.md`](https://github.com/invertase/reeval/blob/v0.22.0/skills/reeval/SKILL.md). For every command, flag, and exit code, run `npx reeval --help`.
160
160
 
161
161
  To preview a run without starting sandboxes or spending tokens, run `npx reeval run --dry-run --json`. It prints the planned evals, cells, and attempts as one JSON document and exits 0.
162
162
 
@@ -166,4 +166,4 @@ To read past runs without calling a model, run `npx reeval query failing_attempt
166
166
 
167
167
  ## License
168
168
 
169
- [Apache-2.0](LICENSE). Copyright 2026 Invertase Limited. The npm package ships third-party licenses and attributions in `THIRD_PARTY_NOTICES.md`. Questions about licensing: [oss@invertase.io](mailto:oss@invertase.io).
169
+ [Apache-2.0](https://github.com/invertase/reeval/blob/v0.22.0/LICENSE). Copyright 2026 Invertase Limited. The npm package ships third-party licenses and attributions in `THIRD_PARTY_NOTICES.md`. Questions about licensing: [oss@invertase.io](mailto:oss@invertase.io).
@@ -1,14 +1,17 @@
1
- // Writes pi's models.json for a Gemini model on an AI gateway's Chat Completions API, run in the
2
- // sandbox before each turn: pi's own catalog entry for the model (its thinking levels, limits and
3
- // prices, as on Gemini's API), with each thinking level sent as the gateway's native Gemini
4
- // `thinkingConfig`, so the model gets the level pi sends it there.
1
+ // Writes pi's models.json for a model on an AI gateway, run in the sandbox before each turn, with
2
+ // the model in a provider of reeval's, on the API its provider's route names (`gateway.routes`),
3
+ // else Chat Completions. Its thinking levels, limits and prices are pi's catalog entry for it, on
4
+ // the gateway or its provider, else the host's description of it. A route with `thinkingConfig`
5
+ // sends each thinking level as the model's native `thinkingConfig`, so the model gets the level pi
6
+ // sends it there.
5
7
  //
6
- // node gateway-models.mjs <bootstrap dir> <models.json> <provider json> <model>
7
- import { mkdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'
8
+ // node gateway-models.mjs <bootstrap dir> <models.json> <gateway json> <model> <model info json>
9
+ import { existsSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'
8
10
  import { dirname, join } from 'node:path'
9
11
 
10
- const [bootstrapDir, out, providerJson, model] = process.argv.slice(2)
11
- const provider = JSON.parse(providerJson)
12
+ const [bootstrapDir, out, gatewayJson, model, infoJson] = process.argv.slice(2)
13
+ const gateway = JSON.parse(gatewayJson)
14
+ const info = JSON.parse(infoJson ?? 'null')
12
15
  const fail = (why) => {
13
16
  process.stderr.write(`reeval: ${why}\n`)
14
17
  process.exit(1)
@@ -16,52 +19,83 @@ const fail = (why) => {
16
19
 
17
20
  // pnpm puts pi's own dependencies beside it, so pi-ai is pi-coding-agent's sibling.
18
21
  const agent = realpathSync(join(bootstrapDir, 'node_modules/@earendil-works/pi-coding-agent'))
19
- const catalog = JSON.parse(
20
- readFileSync(join(dirname(agent), 'pi-ai/dist/providers/data/google.json'), 'utf8'),
21
- )
22
- const id = model.replace(/^google\//, '')
23
- const entry = Object.values(catalog['google-generative-ai'] ?? {}).find((m) => m.id === id)
24
- if (!entry)
25
- fail(`pi's catalog has no Gemini model ${id}, so pi can't run ${model} on ${provider.name}`)
26
- if (entry.reasoning && !entry.thinkingLevelMap) {
27
- fail(
28
- `${id} takes a thinking budget, not a thinking level, so pi can't run it on ${provider.name}`,
22
+ const data = join(dirname(agent), 'pi-ai/dist/providers/data')
23
+ /** The chat models of one of pi's catalogs, whatever API each is on. */
24
+ const catalog = (name) => {
25
+ const file = join(data, `${name}.json`)
26
+ if (!/^[\w-]+$/.test(name) || !existsSync(file)) return []
27
+ return Object.values(JSON.parse(readFileSync(file, 'utf8'))).flatMap((models) =>
28
+ Object.entries(models)
29
+ .filter(([key]) => key.startsWith('chat:'))
30
+ .map(([, m]) => m),
29
31
  )
30
32
  }
31
33
 
34
+ const slash = model.indexOf('/')
35
+ const provider = slash < 0 ? '' : model.slice(0, slash)
36
+ const id = model.slice(slash + 1)
37
+ const route = gateway.routes[provider] ?? {}
38
+ const native = route.thinkingConfig
39
+
40
+ let entry
41
+ if (native) {
42
+ entry = catalog(native.catalog).find((m) => m.id === id)
43
+ if (!entry)
44
+ fail(
45
+ `pi's ${native.catalog} catalog has no model ${id}, so pi can't run ${model} on ${gateway.name}`,
46
+ )
47
+ if (entry.reasoning && !entry.thinkingLevelMap) {
48
+ fail(
49
+ `${id} takes a thinking budget, not a thinking level, so pi can't run it on ${gateway.name}`,
50
+ )
51
+ }
52
+ } else {
53
+ // A provider's own catalog names a version with a dash where a gateway dots it (claude-haiku-4-5).
54
+ const dashed = id.replace(/(\d)\.(\d)/g, '$1-$2')
55
+ entry =
56
+ catalog(gateway.catalog).find((m) => m.id === model) ??
57
+ catalog(provider).find((m) => m.id === id || m.id === dashed) ??
58
+ info ??
59
+ fail(
60
+ `neither pi's catalog nor models.dev describes ${model}, so pi can't run it on ${gateway.name}`,
61
+ )
62
+ }
63
+
32
64
  const thinking = (level) => ({ thinkingConfig: { thinkingLevel: level, includeThoughts: true } })
33
65
  const byLevel = {}
34
- for (const level of entry.reasoning ? ['minimal', 'low', 'medium', 'high'] : []) {
66
+ for (const level of native && entry.reasoning ? ['minimal', 'low', 'medium', 'high'] : []) {
35
67
  const mapped = entry.thinkingLevelMap[level]
36
68
  if (mapped === null) continue
37
- const native = (typeof mapped === 'string' ? mapped : level).toLowerCase()
69
+ const sent = (typeof mapped === 'string' ? mapped : level).toLowerCase()
38
70
  byLevel[level] = {
39
- providerOptions: Object.fromEntries(provider.upstreams.map((u) => [u, thinking(native)])),
71
+ providerOptions: Object.fromEntries(native.upstreams.map((u) => [u, thinking(sent)])),
40
72
  }
41
73
  }
42
74
 
75
+ const api = route.api ?? 'openai-completions'
43
76
  const models = {
44
77
  providers: {
45
- [provider.id]: {
46
- name: provider.name,
47
- baseUrl: provider.baseUrl,
48
- api: 'openai-completions',
49
- apiKey: `$${provider.credentialEnv}`,
50
- // Each level goes as the native thinkingConfig above; the gateway's shared effort field
51
- // would map Gemini's levels onto its own.
78
+ [gateway.id]: {
79
+ name: gateway.name,
80
+ baseUrl: api === 'anthropic-messages' ? gateway.anthropic : gateway.openai,
81
+ api,
82
+ apiKey: `$${gateway.credentialEnv}`,
83
+ authHeader: true,
52
84
  compat: {
53
- supportsReasoningEffort: false,
85
+ // The level goes as the thinkingConfig above; the gateway's shared effort field would map
86
+ // the model's levels onto its own.
87
+ ...(native ? { supportsReasoningEffort: false } : {}),
54
88
  supportsStore: false,
55
89
  supportsDeveloperRole: false,
56
90
  },
57
91
  models: [
58
92
  {
59
93
  id: model,
60
- name: entry.name,
94
+ ...(entry.name ? { name: entry.name } : {}),
61
95
  reasoning: entry.reasoning,
62
96
  ...(entry.thinkingLevelMap ? { thinkingLevelMap: entry.thinkingLevelMap } : {}),
63
97
  input: entry.input,
64
- cost: entry.cost,
98
+ ...(entry.cost ? { cost: entry.cost } : {}),
65
99
  contextWindow: entry.contextWindow,
66
100
  maxTokens: entry.maxTokens,
67
101
  ...(Object.keys(byLevel).length ? { samplingParamsByThinkingLevel: byLevel } : {}),
@@ -1,54 +1,13 @@
1
- import { s as relativeFile } from "./ids-C_advdJA.js";
2
- import { a as PACKAGE_NAMES, c as ownDir, d as ownRequire, l as ownFile, n as COMPILED } from "./root-Co0jhOT5.js";
3
- import { t as codeFrame } from "./errors-BZgLSZtV.js";
4
- import { createRequire } from "node:module";
1
+ import { l as ownFile } from "./root-BzeoyfGd.js";
2
+ import { c as relativeFile } from "./ids-B-JKTCP8.js";
3
+ import { t as reevalJiti } from "./jiti-radm1Soi.js";
4
+ import { t as codeFrame } from "./errors-C1GBcWE2.js";
5
5
  import { existsSync, readFileSync } from "node:fs";
6
6
  import { delimiter, dirname, join, resolve } from "node:path";
7
- import { stripVTControlCharacters } from "node:util";
8
7
  import { spawnSync } from "node:child_process";
9
- import { createJiti } from "jiti";
8
+ import { stripVTControlCharacters } from "node:util";
10
9
  import { DecodedMap, getOriginalPosition } from "@vitest/utils/source-map";
11
10
  import { getTsconfig } from "get-tsconfig";
12
- // src/jiti.ts
13
- let babel;
14
- /**
15
- * jiti's own Babel transform. jiti requires it lazily from beside its code, which a compiled reeval
16
- * doesn't have, so reeval requires the same file: from the jiti package on disk, or, compiled, from
17
- * the copy embedded in it (`vendor/jiti/babel.cjs`), copied out to disk.
18
- */
19
- const jitiTransform = (options) => {
20
- if (!babel) {
21
- const file = COMPILED ? join(ownDir("vendor/jiti"), "babel.cjs") : join(dirname(ownRequire.resolve("jiti/package.json")), "dist/babel.cjs");
22
- babel = createRequire(file)(file);
23
- }
24
- return babel(options);
25
- };
26
- /**
27
- * A jiti that imports from `id`. Compiled, a file's imports of reeval itself get this reeval's own
28
- * modules: a copy on disk would register evals, and hold state, the executable can't see.
29
- */
30
- async function reevalJiti(id, options = {}) {
31
- const virtualModules = COMPILED ? await ownModules() : void 0;
32
- return createJiti(id, {
33
- transform: jitiTransform,
34
- ...options,
35
- ...virtualModules && { virtualModules }
36
- });
37
- }
38
- async function ownModules() {
39
- const exports = {
40
- "": import("./index.js"),
41
- "/harness": import("./harness.js"),
42
- "/sandbox": import("./sandbox.js"),
43
- "/sandbox/boat": import("./sandbox/boat.js"),
44
- "/judge": import("./judge.js"),
45
- "/gateway": import("./gateway.js"),
46
- "/view": import("./view/index.js")
47
- };
48
- const modules = {};
49
- for (const [subpath, module] of Object.entries(exports)) for (const name of PACKAGE_NAMES) modules[name + subpath] = await module;
50
- return modules;
51
- }
52
11
  // src/engine/loader.ts
53
12
  var TsconfigError = class extends Error {
54
13
  name = "TsconfigError";
@@ -288,7 +247,7 @@ const LANGUAGE_ADAPTERS = [TYPESCRIPT_ADAPTER, {
288
247
  extensions: [".eval.py"],
289
248
  includeGlobs: ["evals/**/*.eval.py"],
290
249
  remote: true,
291
- load: (file, root) => import("./remote-U7fNt0UP.js").then((m) => m.collectLanguageFile(file, root)),
250
+ load: (file, root) => import("./remote-0CHog7Oq.js").then((m) => m.collectLanguageFile(file, root)),
292
251
  runnerCommand: (root) => resolvePythonRunnerCommand(root),
293
252
  setupHint: PYTHON_RUNNER_HINT
294
253
  }];
@@ -312,6 +271,6 @@ function runnerEnv(root, adapter) {
312
271
  REEVAL_PROJECT_ROOT: resolve(root)
313
272
  };
314
273
  }
315
- export { originalText as a, originalStack as i, defaultIncludeGlobs as n, reevalJiti as o, runnerEnv as r, adapterForFile as t };
274
+ export { originalText as a, originalStack as i, defaultIncludeGlobs as n, runnerEnv as r, adapterForFile as t };
316
275
 
317
- //# sourceMappingURL=adapter-C60z3wLq.js.map
276
+ //# sourceMappingURL=adapter-Di9H2etg.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"adapter-Di9H2etg.js","names":[],"sources":["../src/engine/loader.ts","../src/language-runner/python-runner.ts","../src/language-runner/adapter.ts"],"mappings":";;;;;;;;;;;AAYA,IAAa,gBAAb,cAAmC,MAAM;CACvC,OAAgB;AAClB;AAOA,MAAM,8BAAc,IAAI,IAAyB;AAEjD,MAAM,aACJ;;AAGF,eAAsB,aAAa,MAAc,MAAgC;CAC/E,cAAc,QAAQ,IAAI,GAAG,IAAI;CACjC,MAAM,UAAU;EAAE,aAAa;EAAM,SAAS;EAAO,YAAY;CAAK;CACtE,MAAM,OAAO,MAAM,WAAW,MAAM,OAAO;CAC3C,MAAM,OAAO,MAAM,WAAW,MAAM;EAClC,GAAG;EACH,YAAY,SAAS;GACnB,MAAM,OAAO,KAAK,UAAU,IAAI;GAChC,MAAM,SAAS,WAAW,KAAK,IAAI;GACnC,IAAI,KAAK,YAAY,SAAS,IAAI;IAChC,MAAM,MAAM,KAAK,MAAM,OAAO,KAAK,OAAO,IAAI,QAAQ,CAAC,CAAC,SAAS,MAAM,CAAC;IAExE,MAAM,QACJ,IAAI,KAAK,QAAQ,WAAW,GAAG,2FAC5B;IACL,YAAY,IAAI,KAAK,UAAU;KAC7B,KAAK,IAAI,WAAW,KAAK,KAAK,QAAQ;KACtC;KACA,QAAQ,KAAK;KACb;IACF,CAAC;GACH;GACA,OAAO,EAAE,KAAK;EAChB;CACF,CAAC;CACD,IAAI;EACF,OAAO,MAAM,KAAK,OAAO,IAAI;CAC/B,SAAS,OAAO;EACd,MAAM,YAAY,OAAO,IAAI,KAAK,cAAc,KAAK;CACvD;AACF;;AAGA,SAAS,cAAc,OAAyB;CAC9C,MAAM,IAAI;CACV,IAAI,GAAG,SAAS,oBAAoB,OAAO;CAC3C,MAAM,QAAQ,wDAAwD,KAAK,EAAE,OAAO;CACpF,IAAI,OAAO,EAAE,UAAU,GAAG,MAAM,GAAG,iBAAiB,MAAM;CAC1D,OAAO;AACT;;;;;AAMA,SAAgB,YAAY,OAAgB,MAAuC;CACjF,MAAM,SAAS,8CAA8C,KAAM,OAAiB,WAAW,EAAE;CACjG,IAAI,CAAC,QAAQ,OAAO,KAAA;CACpB,MAAM,GAAG,MAAM,MAAM,MAAM,UAAU;CAErC,MAAM,SAAS,KAAK,QAAQ,8BAA8B,EAAE;CAC5D,MAAM,KAAK;EAAE,MAAM,OAAO,IAAI;EAAG,QAAQ,OAAO,MAAM,IAAI;CAAE;CAC5D,IAAI,QAAQ;CACZ,IAAI;EACF,QAAQ,yBAAyB,UAAU,aAAa,MAAM,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC;CACpF,QAAQ,CAAC;CACT,uBAAO,IAAI,YACT,GAAG,OAAO,IAAI,aAAa,MAAM,IAAI,EAAE,GAAG,GAAG,KAAK,GAAG,GAAG,OAAO,GAAG,QAAQ,KAAK,UAAU,IAC3F;AACF;;AAGA,SAAgB,cAAc,OAAuB;CACnD,OAAO,MAAM,QACX,gDACC,OAAO,QAAgB,MAAc,MAAc,QAAgB,UAAkB;EACpF,MAAM,SAAS,YAAY,IAAI,IAAI;EACnC,MAAM,MAAM,OAAO,MAAM,IAAI,KAAK,SAAS,MAAO,QAAQ,SAAS,IAAK;EAExE,IAAI,CAAC,UAAU,MAAM,GAAG,OAAO;EAC/B,MAAM,KAAK,oBAAoB,OAAO,KAAK;GAAE,MAAM,OAAO,IAAI;GAAG,QAAQ;EAAI,CAAC;EAC9E,IAAI,CAAC,IAAI,OAAO;EAChB,MAAM,WAAW,GAAG;EACpB,OAAO,GAAG,SAAS,KAAK,GAAG,SAAS,GAAG,OAAO,MAAM,UAAU,GAAG,MAAgB,IAAI,IAAI;CAC3F,CACF;AACF;;;;;;;AAQA,SAAgB,aAAa,MAAc,MAAmC;CAC5E,MAAM,YAAY,SAAS,KAAA,IAAY,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;CAClE,KAAK,MAAM,UAAU,CAAC,GAAG,WAAW,GAAG,YAAY,OAAO,CAAC,GAAG;EAC5D,MAAM,KAAK,QAAQ,KAAK,QAAQ,IAAI,KAAK;EACzC,IAAI,CAAC,UAAU,KAAK,GAAG;EAGvB,MAAM,OAAO,KAAK,KAAK,SAAS;EAChC,IAAI,aAAa;EACjB,MAAM,MAAM,eAAe,QAAQ,IAAI;EACvC,OAAO,aAAa,MAAM,eAAe,QAAQ,aAAa,CAAC,MAAM,KAAK;EAC1E,OAAO,OAAO,OAAO,MAAM,eAAe,QAAQ,EAAE,GAAG,MAAM,OAAO,aAAa,CAAC;CACpF;AAEF;;AAGA,SAAS,eAAe,QAAqB,OAAuB;CAClE,MAAM,QAAQ,OAAO,KAAK,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,IAAI;CACpD,MAAM,SAAU,MAAM,GAAG,EAAE,CAAC,CAAY;CACxC,MAAM,KAAK,oBAAoB,OAAO,KAAK;EAAE,MAAM,MAAM;EAAQ;CAAO,CAAC;CAGzE,OADoB,OAAO,OAAO,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI,QAAQ,KAAK,CACxD,CAAC,CAAC,QAAQ,GAAG,SAAS,IAAI,KAAK,SAAS,GAAG,CAAC,KAAK,IAAI,UAAU;AAClF;;;;;AAMA,SAAS,OAAO,MAAc,MAAc,QAAwB;CAElE,MAAM,OAAO,aAAa,MAAM,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,OAAO,MAAM;CACpE,IAAI,KAAK,YAAY,KAAK,OAAO;CACjC,IAAI,QAAQ;CACZ,OAAO,QAAQ,KAAK,QAAQ,KAAK,KAAK,QAAQ,EAAY,GAAG;CAC7D,OAAO;AACT;;AAGA,SAAgB,cAAc,KAAa,MAAoB;CAC7D,IAAI;CACJ,IAAI;EACF,QAAQ,YAAY,GAAG;CACzB,SAAS,OAAO;EACd,MAAM,UAAW,MAAgB;EACjC,MAAM,UAAU,4BAA4B,KAAK,OAAO;EACxD,MAAM,IAAI,cACR,UACI,4BAA4B,QAAQ,GAAG,yBACvC,4BAA4B,SAClC;CACF;CACA,IAAI,CAAC,OAAO;CACZ,IAAI;EAEF,KAAK,MAAM,kBAAkB,aAAa,MAAM,MAAM,MAAM,CAAC,CAAC,QAAQ,WAAW,EAAE,CAAC,CAAC;CACvF,SAAS,OAAO;EACd,MAAM,IAAI,cACR,4BAA4B,aAAa,MAAM,MAAM,IAAI,EAAE,KAAM,MAAgB,SACnF;CACF;AACF;;AAGA,SAAgB,kBAAkB,MAAsB;CACtD,OAAO,KACL,KAAK,OAAO,SAAS;EACnB,IAAI,KAAK,WAAW,IAAI,GAAG;GACzB,MAAM,MAAM,KAAK,QAAQ,IAAI;GAC7B,OAAO,CAAC,MAAM,IAAI,KAAK,SAAS,KAAK,EAAE;EACzC;EACA,IAAI,KAAK,WAAW,IAAI,GAAG;GACzB,MAAM,MAAM,KAAK,QAAQ,MAAM,CAAC;GAChC,OAAO,CAAC,MAAM,IAAI,KAAK,SAAS,MAAM,GAAG,GAAG;EAC9C;CAEF,CAAC,IACA,SAAU,aAAa,KAAK,IAAI,IAAI,CAAC,GAAG,GAAG,IAAI,KAAA,CAClD;AACF;;AAGA,SAAS,KAAK,MAAc,IAA4D;CACtF,IAAI,MAAM;CACV,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,SAAU;EACjC,IAAI,KAAK,OAAO,MAAK;GACnB,IAAI,MAAM,IAAI;GACd,OAAO,MAAM,KAAK,UAAU,KAAK,SAAS,MAAK,OAAO,KAAK,SAAS,OAAO,IAAI;GAC/E,OAAO,KAAK,MAAM,GAAG,MAAM,CAAC;GAC5B,IAAI,MAAM;GACV;EACF;EACA,MAAM,MAAM,GAAG,KAAK,MAAM,CAAC,CAAC;EAC5B,IAAI,KAAK;GACP,OAAO,IAAI;GACX,KAAK,IAAI;EACX,OAAO,OAAO,KAAK;CACrB;CACA,OAAO;AACT;;AC7MA,MAAa,uBAAuB;AAEpC,MAAM,cAAc,CAAC,MAAM,oBAAoB;AAE/C,IAAI;AAEJ,SAAgB,cAAuB;CACrC,IAAI,aAAa,KAAA,GAAW,OAAO;CAMnC,WALe,UAAU,MAAM,CAAC,WAAW,GAAG;EAC5C,UAAU;EACV,OAAO;EACP,aAAa;CACf,CACgB,CAAC,CAAC,WAAW;CAC7B,OAAO;AACT;AAEA,SAAgB,cAAc,MAAuB;CACnD,OAAO,WAAW,KAAK,MAAM,SAAS,CAAC,KAAK,WAAW,KAAK,MAAM,gBAAgB,CAAC;AACrF;AAEA,SAAS,WAAW,MAAkC;CACpD,MAAM,OAAO,KAAK,MAAM,SAAS,OAAO,QAAQ;CAChD,IAAI,WAAW,IAAI,GAAG,OAAO;CAC7B,MAAM,MAAM,KAAK,MAAM,SAAS,WAAW,YAAY;CACvD,IAAI,WAAW,GAAG,GAAG,OAAO;AAE9B;;AAGA,SAAgB,2BAA2B,MAA6B;CACtE,MAAM,WAAW,QAAQ,IAAI,eAAe,KAAK;CACjD,IAAI,UAAU,OAAO,CAAC,UAAU,GAAG,WAAW;CAE9C,IAAI,cAAc,IAAI,KAAK,YAAY,GACrC,OAAO;EAAC;EAAM;EAAO;EAAa;EAAM;EAAU,GAAG;CAAW;CAGlE,MAAM,OAAO,WAAW,IAAI;CAC5B,IAAI,MAAM,OAAO,CAAC,MAAM,GAAG,WAAW;CAGtC,OAAO,CAAC,QAAQ,aAAa,UAAU,WAAW,WAAW,GAAG,WAAW;AAC7E;;AAGA,MAAa,qBAAqB;CAChC,kCAAkC,qBAAqB;CACvD;CACA;CACA;AACF,CAAC,CAAC,KAAK,GAAG;;AC7BV,MAAa,qBAAuC;CAClD,IAAI;CACJ,YAAY;EAAC;EAAY;EAAa;EAAa;EAAY;EAAa;CAAW;CACvF,cAAc,CAAC,oBAAoB;CACnC,QAAQ;CACR,OAAO,MAAM,SAAS,aAAa,MAAM,IAAI;AAC/C;AAEA,MAAM,sBAA0C;CAC9C,MAAM,UAAU,KAAK,QAAQ,IAAI,GAAG,eAAe;CACnD,IAAI,WAAW,KAAK,SAAS,gBAAgB,CAAC,GAAG,OAAO;AAE1D;;AAaA,MAAa,oBAAgD,CAAC,oBAAoB;CAVhF,IAAI;CACJ,YAAY,CAAC,UAAU;CACvB,cAAc,CAAC,oBAAoB;CACnC,QAAQ;CACR,OAAO,MAAM,SAAS,OAAO,uBAAc,CAAC,MAAM,MAAM,EAAE,oBAAoB,MAAM,IAAI,CAAC;CACzF,gBAAgB,SAAS,2BAA2B,IAAI;CACxD,WAAW;AAIqE,CAAc;AAEhG,SAAgB,eAAe,MAA+B;CAE5D,OADc,kBAAkB,MAAM,MAAM,EAAE,WAAW,MAAM,QAAQ,KAAK,SAAS,GAAG,CAAC,CAC9E,KAAK;AAClB;AAEA,SAAgB,sBAAgC;CAC9C,OAAO,kBAAkB,SAAS,MAAM,CAAC,GAAG,EAAE,YAAY,CAAC;AAC7D;AAEA,SAAgB,UAAU,MAAc,SAA6C;CACnF,MAAM,MAAM,QAAQ,IAAI,qBAAqB,cAAc;CAC3D,IAAI,QAAQ,OAAO,YAAY,CAAC,KAAK,OAAO,QAAQ;CACpD,MAAM,MAAM,KAAK,KAAK,KAAK;CAC3B,OAAO;EACL,GAAG,QAAQ;EACX,YAAY;GAAC;GAAK;GAAK,QAAQ,IAAI;EAAU,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK,SAAS;EAC7E,qBAAqB,QAAQ,IAAI;CACnC;AACF"}