reeval 0.14.0 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (364) hide show
  1. package/README.md +15 -15
  2. package/THIRD_PARTY_NOTICES.md +69 -69
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-BloGVZ_a.js → adapter-CRKYPkRt.js} +37 -8
  5. package/dist/adapter-CRKYPkRt.js.map +1 -0
  6. package/dist/{agent-m-_t-IuW.js → agent-BgH4-kFc.js} +58 -16
  7. package/dist/agent-BgH4-kFc.js.map +1 -0
  8. package/dist/{app-files-C-jwhRDt.js → app-files-BMVBr079.js} +4 -4
  9. package/dist/app-files-BMVBr079.js.map +1 -0
  10. package/dist/{assemble-BDEpelpi.js → assemble-BVbI6NW-.js} +214 -128
  11. package/dist/assemble-BVbI6NW-.js.map +1 -0
  12. package/dist/assemble-SiHa4__F.js +1 -0
  13. package/dist/{attempt-jI0OWxlz.js → attempt-CpbZ9arV.js} +10 -10
  14. package/dist/{attempt-jI0OWxlz.js.map → attempt-CpbZ9arV.js.map} +1 -1
  15. package/dist/{match-Bz0nSdzN.js → attempt-host-CIxjs2Be.js} +104 -6
  16. package/dist/attempt-host-CIxjs2Be.js.map +1 -0
  17. package/dist/{capture-CVRwRTnb.js → capture-iQ6kYJV0.js} +2360 -1523
  18. package/dist/capture-iQ6kYJV0.js.map +1 -0
  19. package/dist/capture-px7Ey95_.js +1 -0
  20. package/dist/{cli-C_4gaJ50.js → cli-1YQU-66F.js} +153 -180
  21. package/dist/cli-1YQU-66F.js.map +1 -0
  22. package/dist/{cli-spec-CpnbATtI.js → cli-spec-CPWT1mAG.js} +10 -21
  23. package/dist/cli-spec-CPWT1mAG.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-D5hNjpEx.js → client-DMmOkbRA.js} +14 -7
  26. package/dist/client-DMmOkbRA.js.map +1 -0
  27. package/dist/{command-BXLUfw4h.js → command-8_Cg5v6b.js} +35 -22
  28. package/dist/command-8_Cg5v6b.js.map +1 -0
  29. package/dist/{command-I1euLLio.js → command-BlYCAqu-.js} +7 -7
  30. package/dist/{command-I1euLLio.js.map → command-BlYCAqu-.js.map} +1 -1
  31. package/dist/{validate-BhwqrVsm.js → command-C-HeGlap.js} +388 -60
  32. package/dist/command-C-HeGlap.js.map +1 -0
  33. package/dist/{command-Bxdy62dW.js → command-Cr1zjRvl.js} +32 -31
  34. package/dist/command-Cr1zjRvl.js.map +1 -0
  35. package/dist/{commands-p1S4nw_H.js → commands-Bj9NbmsO.js} +1 -1
  36. package/dist/{commands-jzigbNFb.js → commands-BvTNZ2-b.js} +181 -73
  37. package/dist/commands-BvTNZ2-b.js.map +1 -0
  38. package/dist/{config-BrdHA9bJ.js → config-CneBZB84.js} +883 -433
  39. package/dist/config-CneBZB84.js.map +1 -0
  40. package/dist/{contrasts-DRxMQQTC.js → contrasts-lhFsekco.js} +3 -3
  41. package/dist/{contrasts-DRxMQQTC.js.map → contrasts-lhFsekco.js.map} +1 -1
  42. package/dist/cost-C5RBTTJr.js +1 -0
  43. package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
  44. package/dist/cost-ChPa8-Fp.js.map +1 -0
  45. package/dist/db-BnXyamuj.js +2 -0
  46. package/dist/db-Cnh3jIv_.js +1530 -0
  47. package/dist/db-Cnh3jIv_.js.map +1 -0
  48. package/dist/debug-CAx3qcaP.js +1 -0
  49. package/dist/{debug-D24NOQH1.js → debug-CgG3T071.js} +5 -2
  50. package/dist/debug-CgG3T071.js.map +1 -0
  51. package/dist/{docker-BnRlTpZA.js → docker-BihzJwbV.js} +11 -6
  52. package/dist/docker-BihzJwbV.js.map +1 -0
  53. package/dist/{docker-D01slDF3.js → docker-Cqq03H72.js} +26 -8
  54. package/dist/docker-Cqq03H72.js.map +1 -0
  55. package/dist/{docker-DivfZl2u.js → docker-DCAtFMo9.js} +1 -1
  56. package/dist/{doctor-DQ2p0euG.js → doctor-Dl4vMeqt.js} +275 -37
  57. package/dist/doctor-Dl4vMeqt.js.map +1 -0
  58. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  59. package/dist/env-CARj5Axq.js.map +1 -0
  60. package/dist/{environment-CO0P0soF.js → environment-Cf5HMjib.js} +116 -297
  61. package/dist/environment-Cf5HMjib.js.map +1 -0
  62. package/dist/{errors-r025B0Zb.js → errors-CHebohjc.js} +10 -4
  63. package/dist/{errors-r025B0Zb.js.map → errors-CHebohjc.js.map} +1 -1
  64. package/dist/eval-source-CAE4y2Sr.js +146 -0
  65. package/dist/eval-source-CAE4y2Sr.js.map +1 -0
  66. package/dist/{evaluate-BUmsN7yG.js → evaluate-CUtgyr43.js} +682 -150
  67. package/dist/evaluate-CUtgyr43.js.map +1 -0
  68. package/dist/evaluate-D35T51AC.d.ts +45 -0
  69. package/dist/files-BEnUJW1e.js +1 -0
  70. package/dist/{files-B-CJeeEh.js → files-BRViE7m-.js} +200 -87
  71. package/dist/files-BRViE7m-.js.map +1 -0
  72. package/dist/{files-kUCrswI6.js → files-BSqA2sZE.js} +20 -9
  73. package/dist/files-BSqA2sZE.js.map +1 -0
  74. package/dist/files-C3tQtD26.js +1 -0
  75. package/dist/files-CNiJ_fZf.js +37 -0
  76. package/dist/files-CNiJ_fZf.js.map +1 -0
  77. package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
  78. package/dist/files-D1y8Pg4Z.js.map +1 -0
  79. package/dist/files-Dts7EY3H.js +1 -0
  80. package/dist/{pairs-CHfZNRT6.js → gate-BC6SdQZs.js} +58 -4
  81. package/dist/gate-BC6SdQZs.js.map +1 -0
  82. package/dist/gateway.d.ts +22 -9
  83. package/dist/gateway.js +30 -8
  84. package/dist/gateway.js.map +1 -1
  85. package/dist/{git-Bk8DSwHy.js → git-UNxT_3z0.js} +3 -3
  86. package/dist/{git-Bk8DSwHy.js.map → git-UNxT_3z0.js.map} +1 -1
  87. package/dist/harness-3NVyTSGK.js +392 -0
  88. package/dist/harness-3NVyTSGK.js.map +1 -0
  89. package/dist/harness.d.ts +12 -8
  90. package/dist/harness.js +1 -1
  91. package/dist/{images-DzLeFjLx.js → images-8A_LzVfw.js} +8 -8
  92. package/dist/{images-DzLeFjLx.js.map → images-8A_LzVfw.js.map} +1 -1
  93. package/dist/index.d.ts +99 -11
  94. package/dist/index.js +5 -5
  95. package/dist/init-okoALIah.js +1 -0
  96. package/dist/{init-SRalM5xF.js → init-sFBgODhu.js} +10 -11
  97. package/dist/init-sFBgODhu.js.map +1 -0
  98. package/dist/{invocations-DzyKSJBa.js → invocations-C9ejlOYl.js} +10 -6
  99. package/dist/invocations-C9ejlOYl.js.map +1 -0
  100. package/dist/{judge-yLvjC5A3.js → judge-Bg4v4e_K.js} +236 -46
  101. package/dist/judge-Bg4v4e_K.js.map +1 -0
  102. package/dist/judge.d.ts +5 -7
  103. package/dist/judge.js +4 -8
  104. package/dist/judge.js.map +1 -1
  105. package/dist/{live-DYd2QEyR.js → live-DeMvnDD5.js} +3 -3
  106. package/dist/{live-DYd2QEyR.js.map → live-DeMvnDD5.js.map} +1 -1
  107. package/dist/{matchers-D4eUqx4j.js → matchers-BKPuaXlj.js} +14 -13
  108. package/dist/matchers-BKPuaXlj.js.map +1 -0
  109. package/dist/{model-1ZLr9Dcv.js → model-2rBZrdZA.js} +29 -36
  110. package/dist/model-2rBZrdZA.js.map +1 -0
  111. package/dist/{model-B45lDUV1.js → model-3MJtpho8.js} +22 -13
  112. package/dist/model-3MJtpho8.js.map +1 -0
  113. package/dist/{model-CgnOpss4.js → model-Bz7-j71_.js} +245 -68
  114. package/dist/model-Bz7-j71_.js.map +1 -0
  115. package/dist/model-DkPwypN4.js +1 -0
  116. package/dist/model-bREXVXwT.js +1 -0
  117. package/dist/{node-BSq5qHnQ.js → node-fP-H2uxn.js} +72 -50
  118. package/dist/node-fP-H2uxn.js.map +1 -0
  119. package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
  120. package/dist/object-BkXGPKWY.js.map +1 -0
  121. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  122. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  123. package/dist/openrouter-models-Bo8PZ4Ew.js +34 -0
  124. package/dist/openrouter-models-Bo8PZ4Ew.js.map +1 -0
  125. package/dist/{patch-DWf-I_Q_.js → patch-6T9XAipm.js} +7 -9
  126. package/dist/patch-6T9XAipm.js.map +1 -0
  127. package/dist/patch-CquGAGX5.js +76 -0
  128. package/dist/patch-CquGAGX5.js.map +1 -0
  129. package/dist/{plan-BIwUzlB8.js → plan-Dw2BjafJ.js} +79 -35
  130. package/dist/plan-Dw2BjafJ.js.map +1 -0
  131. package/dist/{plugins-Dkp_H3pB.js → plugins-ngQEJBjJ.js} +274 -17
  132. package/dist/plugins-ngQEJBjJ.js.map +1 -0
  133. package/dist/{preflight-D9y-HvZQ.js → preflight-BYXkweNN.js} +1 -1
  134. package/dist/{preflight-DJCD_BDn.js → preflight-CDRpog9o.js} +3 -3
  135. package/dist/{preflight-DJCD_BDn.js.map → preflight-CDRpog9o.js.map} +1 -1
  136. package/dist/{prepare-BJG_pD54.js → prepare-B3btUlT6.js} +2 -2
  137. package/dist/{prepare-CIxCGG0M.js → prepare-PkARUGoF.js} +8 -8
  138. package/dist/{prepare-CIxCGG0M.js.map → prepare-PkARUGoF.js.map} +1 -1
  139. package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
  140. package/dist/prices-QaS-kjSQ.js.map +1 -0
  141. package/dist/{provenance-CfZpFpAh.js → provenance-7Hzg88f4.js} +4 -4
  142. package/dist/{provenance-CfZpFpAh.js.map → provenance-7Hzg88f4.js.map} +1 -1
  143. package/dist/{providers-D1Zxp2he.js → providers-DDNMbv1T.js} +1 -1
  144. package/dist/{providers-DFk_6W4K.js → providers-PKoJtGK0.js} +5 -5
  145. package/dist/{providers-DFk_6W4K.js.map → providers-PKoJtGK0.js.map} +1 -1
  146. package/dist/query-Bt6vLJ3m.js +1 -0
  147. package/dist/{query-CemclveB.js → query-nBXrgEwl.js} +3 -3
  148. package/dist/{query-CemclveB.js.map → query-nBXrgEwl.js.map} +1 -1
  149. package/dist/{questions-bAudIc04.js → questions-CkAwePPR.js} +110 -36
  150. package/dist/questions-CkAwePPR.js.map +1 -0
  151. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  152. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  153. package/dist/redact-DVIaahLv.js +658 -0
  154. package/dist/redact-DVIaahLv.js.map +1 -0
  155. package/dist/remarks-Bp6AwPgb.js +1 -0
  156. package/dist/{rules-VhftPOOT.js → remarks-Bt0zwiQB.js} +330 -66
  157. package/dist/remarks-Bt0zwiQB.js.map +1 -0
  158. package/dist/{header-BLPP6aCU.js → remarks-IFFe_oty.js} +35 -9
  159. package/dist/remarks-IFFe_oty.js.map +1 -0
  160. package/dist/{remarks-mujJv1Ey.js → remarks-eBcrIZRu.js} +10 -5
  161. package/dist/remarks-eBcrIZRu.js.map +1 -0
  162. package/dist/{remote-CKcEKja4.js → remote-DfJYiP-R.js} +39 -99
  163. package/dist/remote-DfJYiP-R.js.map +1 -0
  164. package/dist/resolve-CT4XZIfV.js +1 -0
  165. package/dist/{resolve-CBoKzQMP.js → resolve-D_cLa4tW.js} +46 -30
  166. package/dist/resolve-D_cLa4tW.js.map +1 -0
  167. package/dist/results-DrKJuIQF.js +63 -0
  168. package/dist/results-DrKJuIQF.js.map +1 -0
  169. package/dist/{resume-B9SOSnK9.js → resume-BmIM52MJ.js} +47 -17
  170. package/dist/resume-BmIM52MJ.js.map +1 -0
  171. package/dist/{runtime-BgQRyCLl.js → runtime-CZVpkr04.js} +139 -48
  172. package/dist/runtime-CZVpkr04.js.map +1 -0
  173. package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmi6IQML.js} +2 -2
  174. package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmi6IQML.js.map} +1 -1
  175. package/dist/sandbox.d.ts +1 -1
  176. package/dist/sandbox.js +1 -1
  177. package/dist/server-CKJOnIB0.js +1 -0
  178. package/dist/{server-TGW4WyAz.js → server-DS8BGhGU.js} +75 -33
  179. package/dist/server-DS8BGhGU.js.map +1 -0
  180. package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
  181. package/dist/shared-B_lvGqWz.js.map +1 -0
  182. package/dist/source-CgUqpumI.js +1 -0
  183. package/dist/source-DcXv4de-.js +59 -0
  184. package/dist/source-DcXv4de-.js.map +1 -0
  185. package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
  186. package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
  187. package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
  188. package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
  189. package/dist/{src-DjRI-IeE.js → src-A8HLtO0a.js} +135 -35
  190. package/dist/{src-DjRI-IeE.js.map → src-A8HLtO0a.js.map} +1 -1
  191. package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
  192. package/dist/src-BO8AYqJ5.js.map +1 -0
  193. package/dist/src-BTqJbvj6.js +2 -0
  194. package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
  195. package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BkiComGh.js +2 -0
  197. package/dist/src-BvgeZ4tS.js +2 -0
  198. package/dist/{src-8zoX_qrb.js → src-C0ySiej5.js} +572 -1067
  199. package/dist/src-C0ySiej5.js.map +1 -0
  200. package/dist/src-CgJrz5p0.js +3 -0
  201. package/dist/src-Cp-2Hdib.js +2 -0
  202. package/dist/{src-CyaDNG9r.js → src-DMo2eCOJ.js} +1 -1
  203. package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
  204. package/dist/src-DSZyg0Uq.js.map +1 -0
  205. package/dist/src-Dk2Na0b-.js +2 -0
  206. package/dist/src-OESRtvOi.js +2 -0
  207. package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
  208. package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
  209. package/dist/src-h5uDlk3m.js +1 -0
  210. package/dist/state-C97udoh3.js +1 -0
  211. package/dist/{state-B1t143wG.js → state-C_kdhp5v.js} +15 -15
  212. package/dist/{state-B1t143wG.js.map → state-C_kdhp5v.js.map} +1 -1
  213. package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
  214. package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
  215. package/dist/suite-ByT1lsdD.js +2 -0
  216. package/dist/{suite-WgT7m6Vs.js → suite-DieLXvCT.js} +61 -65
  217. package/dist/suite-DieLXvCT.js.map +1 -0
  218. package/dist/{summary-DIGEVr08.js → summary-CSzyBVsF.js} +113 -277
  219. package/dist/summary-CSzyBVsF.js.map +1 -0
  220. package/dist/tarball-CHcahyNK.js +258 -0
  221. package/dist/tarball-CHcahyNK.js.map +1 -0
  222. package/dist/testing.d.ts +426 -143
  223. package/dist/testing.js +12 -10
  224. package/dist/testing.js.map +1 -1
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
  228. package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
  229. package/dist/{transcript-BTxJpW4R.js → transcript-BJrKfgRW.js} +13 -5
  230. package/dist/transcript-BJrKfgRW.js.map +1 -0
  231. package/dist/types-C5vI3g1C.js.map +1 -1
  232. package/dist/{types-DXBbP9Mk.d.ts → types-DVvW66ci.d.ts} +569 -895
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +548 -121
  238. package/dist/view/index.js +9 -9
  239. package/dist/{views-sXpGE6a2.js → views-B9STsLB4.js} +9 -10
  240. package/dist/{views-sXpGE6a2.js.map → views-B9STsLB4.js.map} +1 -1
  241. package/dist/views-Ca-rTixp.js +2 -0
  242. package/dist/{worker-entry-Duo9oAeT.js → worker-entry-C5bLCv6s.js} +48 -20
  243. package/dist/worker-entry-C5bLCv6s.js.map +1 -0
  244. package/dist/workspace-8uHtBSve.js +1 -0
  245. package/dist/{workspace-CDI68wtX.js → workspace-CBuI9Uds.js} +5 -5
  246. package/dist/{workspace-CDI68wtX.js.map → workspace-CBuI9Uds.js.map} +1 -1
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +5 -1
  249. package/skills/eval-rubric/SKILL.md +46 -104
  250. package/dist/adapter-BloGVZ_a.js.map +0 -1
  251. package/dist/agent-m-_t-IuW.js.map +0 -1
  252. package/dist/app-files-C-jwhRDt.js.map +0 -1
  253. package/dist/assemble-BDEpelpi.js.map +0 -1
  254. package/dist/assemble-C5bfncHU.js +0 -1
  255. package/dist/capture-CVRwRTnb.js.map +0 -1
  256. package/dist/capture-CherbjzB.js +0 -1
  257. package/dist/cli-C_4gaJ50.js.map +0 -1
  258. package/dist/cli-spec-CpnbATtI.js.map +0 -1
  259. package/dist/client-D5hNjpEx.js.map +0 -1
  260. package/dist/command-BWAuwQNk.js +0 -91
  261. package/dist/command-BWAuwQNk.js.map +0 -1
  262. package/dist/command-BXLUfw4h.js.map +0 -1
  263. package/dist/command-Be6BSE2L.js +0 -335
  264. package/dist/command-Be6BSE2L.js.map +0 -1
  265. package/dist/command-Bxdy62dW.js.map +0 -1
  266. package/dist/commands-jzigbNFb.js.map +0 -1
  267. package/dist/config-BrdHA9bJ.js.map +0 -1
  268. package/dist/cost-D4elx933.js.map +0 -1
  269. package/dist/cost-DmnAod9x.js +0 -1
  270. package/dist/db-CAfksinX.js +0 -2
  271. package/dist/db-DO0p7xgP.js +0 -619
  272. package/dist/db-DO0p7xgP.js.map +0 -1
  273. package/dist/debug-D24NOQH1.js.map +0 -1
  274. package/dist/debug-D7bEv_UV.js +0 -1
  275. package/dist/docker-BnRlTpZA.js.map +0 -1
  276. package/dist/docker-D01slDF3.js.map +0 -1
  277. package/dist/doctor-DQ2p0euG.js.map +0 -1
  278. package/dist/env-BQqj93gu.js.map +0 -1
  279. package/dist/environment-CO0P0soF.js.map +0 -1
  280. package/dist/evaluate-BUmsN7yG.js.map +0 -1
  281. package/dist/evaluate-C_sgRA-V.d.ts +0 -134
  282. package/dist/facts-DzOUTreB.js +0 -906
  283. package/dist/facts-DzOUTreB.js.map +0 -1
  284. package/dist/files-B-CJeeEh.js.map +0 -1
  285. package/dist/files-BAt0i00X.js +0 -1
  286. package/dist/files-Bp5j5BqI.js +0 -1
  287. package/dist/files-C2sjmXhk.js +0 -1
  288. package/dist/files-C_gQNVB-.js +0 -65
  289. package/dist/files-C_gQNVB-.js.map +0 -1
  290. package/dist/files-Ca0d6Uq3.js.map +0 -1
  291. package/dist/files-kUCrswI6.js.map +0 -1
  292. package/dist/generate-DZ7vp69F.js +0 -1647
  293. package/dist/generate-DZ7vp69F.js.map +0 -1
  294. package/dist/harness-COVXSwJ6.js +0 -354
  295. package/dist/harness-COVXSwJ6.js.map +0 -1
  296. package/dist/header-BLPP6aCU.js.map +0 -1
  297. package/dist/init-RgCQXZhb.js +0 -1
  298. package/dist/init-SRalM5xF.js.map +0 -1
  299. package/dist/insights-TyAvz9aV.js +0 -68
  300. package/dist/insights-TyAvz9aV.js.map +0 -1
  301. package/dist/invocations-DzyKSJBa.js.map +0 -1
  302. package/dist/judge-yLvjC5A3.js.map +0 -1
  303. package/dist/match-Bz0nSdzN.js.map +0 -1
  304. package/dist/matchers-D4eUqx4j.js.map +0 -1
  305. package/dist/model-1ZLr9Dcv.js.map +0 -1
  306. package/dist/model-B45lDUV1.js.map +0 -1
  307. package/dist/model-BTZq8HIZ.js +0 -1
  308. package/dist/model-CgnOpss4.js.map +0 -1
  309. package/dist/model-Ch8kX87u.js +0 -1
  310. package/dist/node-BSq5qHnQ.js.map +0 -1
  311. package/dist/object-CLf2-tNq.js.map +0 -1
  312. package/dist/opencode-providers-klkqgzvb.js +0 -3276
  313. package/dist/opencode-providers-klkqgzvb.js.map +0 -1
  314. package/dist/pairs-CHfZNRT6.js.map +0 -1
  315. package/dist/patch-DWf-I_Q_.js.map +0 -1
  316. package/dist/plan-BIwUzlB8.js.map +0 -1
  317. package/dist/plugins-Dkp_H3pB.js.map +0 -1
  318. package/dist/prices-BcLEXvdY.js.map +0 -1
  319. package/dist/query-Cvbwl804.js +0 -1
  320. package/dist/questions-bAudIc04.js.map +0 -1
  321. package/dist/redact-CRGdfV2s.js +0 -240
  322. package/dist/redact-CRGdfV2s.js.map +0 -1
  323. package/dist/remarks-CXwSnSjR.js +0 -278
  324. package/dist/remarks-CXwSnSjR.js.map +0 -1
  325. package/dist/remarks-CyAu2DiB.js +0 -1
  326. package/dist/remarks-mujJv1Ey.js.map +0 -1
  327. package/dist/remote-CKcEKja4.js.map +0 -1
  328. package/dist/resolve-B5-uqmJ6.js +0 -1
  329. package/dist/resolve-CBoKzQMP.js.map +0 -1
  330. package/dist/results-CMgZEANy.js +0 -78
  331. package/dist/results-CMgZEANy.js.map +0 -1
  332. package/dist/resume-B9SOSnK9.js.map +0 -1
  333. package/dist/rules-VhftPOOT.js.map +0 -1
  334. package/dist/run-CwIEK16q.js +0 -88
  335. package/dist/run-CwIEK16q.js.map +0 -1
  336. package/dist/runtime-BgQRyCLl.js.map +0 -1
  337. package/dist/server-CHRSgKRp.js +0 -1
  338. package/dist/server-TGW4WyAz.js.map +0 -1
  339. package/dist/shared-BDUzStad.js.map +0 -1
  340. package/dist/source-B1WvBIQt.js +0 -40
  341. package/dist/source-B1WvBIQt.js.map +0 -1
  342. package/dist/source-DAzztZwU.js +0 -1
  343. package/dist/src-1jHbFaGW.js +0 -1
  344. package/dist/src-8zoX_qrb.js.map +0 -1
  345. package/dist/src-C_9wgowC.js +0 -2
  346. package/dist/src-ChhtI6BP.js.map +0 -1
  347. package/dist/src-D005Cbqz.js +0 -2
  348. package/dist/src-DIfRbp5a.js +0 -3
  349. package/dist/src-Dg8121Xy.js +0 -2
  350. package/dist/src-ElBqEN-S.js +0 -2
  351. package/dist/src-NSdQ9zI6.js.map +0 -1
  352. package/dist/src-YHGaUrGE.js +0 -2
  353. package/dist/src-mJVxw1Jm.js +0 -2
  354. package/dist/state-vznMh2wl.js +0 -1
  355. package/dist/suite-BsWGHzlG.js +0 -2
  356. package/dist/suite-WgT7m6Vs.js.map +0 -1
  357. package/dist/summary-DIGEVr08.js.map +0 -1
  358. package/dist/text-CrVy5ioL.js.map +0 -1
  359. package/dist/transcript-BTxJpW4R.js.map +0 -1
  360. package/dist/usage-DF3_p5pE.js +0 -1
  361. package/dist/validate-BhwqrVsm.js.map +0 -1
  362. package/dist/views-BW_C5eVH.js +0 -2
  363. package/dist/worker-entry-Duo9oAeT.js.map +0 -1
  364. package/dist/workspace-QZCacP8A.js +0 -1
@@ -1,6 +1,8 @@
1
1
  import { n as UsageError } from "./errors-B6BSMs03.js";
2
- import { n as analysisModelOf, v as gatewayScope, y as resolveModelGateway } from "./config-BrdHA9bJ.js";
3
- import { a as checkModelString, l as judgeRoute, o as isFakeJudge, p as unusedGatewayKey, s as judgeCredentials, u as modelKeyOptions } from "./judge-yLvjC5A3.js";
2
+ import { b as resolveModelGateway, n as analysisModelOf, v as gatewayScope } from "./config-CneBZB84.js";
3
+ import { s as isFakeJudge, u as judgeRoute } from "./judge-Bg4v4e_K.js";
4
+ import { p as vertexLanguageModel, v as vertexSettings, y as vertexSettingsIn } from "./vertex-DypYjx71.js";
5
+ import { i as openRouterLanguageModel, r as openRouterFallbacks } from "./openrouter-models-Bo8PZ4Ew.js";
4
6
  import { z } from "zod";
5
7
  import { Output, generateText } from "ai";
6
8
  //#region src/analysis/schema.ts
@@ -63,7 +65,7 @@ const analysisSchema = z.object({
63
65
  why: z.string()
64
66
  })
65
67
  });
66
- const SYSTEM_PROMPT = `You analyse the results of a reeval run for the engineer who wrote the evals.
68
+ const SYSTEM_PROMPT = `You analyze the results of a reeval run for the engineer who wrote the evals.
67
69
  reeval runs coding agents in sandboxes. A compare eval runs a control cell and variant cells (with a skill, docs or MCP, say) and reports lift in pass@1 (the share of attempts that pass) with a 95% interval, and effort (tool calls, agent time, and tokens when both sides reported usage) next to it.
68
70
 
69
71
  You get a DIGEST: facts computed deterministically by reeval (ids F1...), evidence excerpts (ids E1...; transcript lines start with their line number L<n>), an attempts table (ids like r2/fix-config/control#1), the RULES reeval used for every verdict, and the CLI. PROJECT INSTRUCTIONS, when present, come first: they set the question this analysis answers. Follow them within these rules.
@@ -129,34 +131,6 @@ function questionModel(config, question, where, judge = config.judge.model) {
129
131
  if (typeof model === "string" && isEvaluationModel(model)) throw new UsageError(`questions "${question.question}" (${where}) would be answered by analysis.model in ${config.file}, "${model}", an evaluation model: it scores yes/no questions and can't ${task}. Use a language model, e.g. 'google/gemini-3.8-flash', or ${fix}.`);
130
132
  return model;
131
133
  }
132
- /**
133
- * Why `analysis.model`'s calls would fail for want of a usable key: none of the keys that would
134
- * work for its provider is set, or the one reeval would send has whitespace or a line break in it.
135
- * Null when a usable key is set, or for any other model. `brief` names the first two keys only,
136
- * for the line a run ends with; the plan's warning has them all.
137
- */
138
- function analysisKeyProblem(config, env = process.env, brief = false) {
139
- const model = analysisModelOf(config);
140
- if (typeof model !== "string" || isEvaluationModel(model) || checkModelString(model)) return null;
141
- let gateway;
142
- try {
143
- gateway = resolveModelGateway(model, void 0, gatewayScope(config), "analysis.model");
144
- } catch (error) {
145
- return brief ? "add vercel() to gateway, or analysis: { insights: false }" : `${error.message} Or analysis: { insights: false }.`;
146
- }
147
- const keys = judgeCredentials(model, gateway);
148
- if (!keys.some((k) => env[k])) {
149
- if (brief) return `set ${keys.slice(0, 2).join(" or ")}, or analysis: { insights: false }`;
150
- const unused = unusedGatewayKey(env, gateway);
151
- return `set ${modelKeyOptions(model, gateway)} for analysis.model, "${model}", in .env.local, or analysis: { insights: false }${unused ? `.${unused}` : ""}`;
152
- }
153
- try {
154
- judgeRoute(model, env, "analysis model", gateway);
155
- return null;
156
- } catch (error) {
157
- return error.message;
158
- }
159
- }
160
134
  /** Checks the model and its credentials before any work; throws a UsageError (exit 2). */
161
135
  function resolveAnalysisModel(config) {
162
136
  const model = analysisModelName(config);
@@ -166,11 +140,15 @@ function resolveAnalysisModel(config) {
166
140
  fake: model
167
141
  };
168
142
  if (isEvaluationModel(model)) throw new UsageError(`analysis.model in ${config.file} is "${model}", an evaluation model: it scores yes/no questions and can't write an analysis. Use a language model, e.g. 'google/gemini-3.8-flash'.`);
169
- const gateway = resolveModelGateway(model, void 0, gatewayScope(config), "analysis.model");
143
+ const scope = gatewayScope(config);
144
+ const gateway = resolveModelGateway(model, void 0, scope, "analysis.model");
170
145
  try {
146
+ const route = judgeRoute(model, process.env, "analysis model", gateway).kind;
171
147
  return {
172
148
  name: model,
173
- route: judgeRoute(model, process.env, "analysis model", gateway).kind
149
+ route,
150
+ ...route === "openrouter" ? { fallbacks: openRouterFallbacks(scope.gateways) } : {},
151
+ ...route === "vertex" ? { vertex: vertexSettingsIn(scope.gateways) } : {}
174
152
  };
175
153
  } catch (error) {
176
154
  throw new UsageError(`${error.message} Or run reeval analyze --context, which needs no key.`);
@@ -241,9 +219,14 @@ async function languageModel(model, digest) {
241
219
  if (model.fake) return fakeModel(model.fake, digest);
242
220
  return providerModel(model);
243
221
  }
244
- /** The provider's own model for a direct route, or the AI Gateway's by name on vercel. */
222
+ /**
223
+ * The provider's own model for a direct route, the AI Gateway's by name on vercel, OpenRouter's
224
+ * on openrouter, or Vertex AI's on vertex.
225
+ */
245
226
  async function providerModel(model) {
246
227
  if (model.route === "gateway") return model.name;
228
+ if (model.route === "openrouter") return openRouterLanguageModel(model.name, !!model.fallbacks);
229
+ if (model.route === "vertex") return vertexLanguageModel(model.name, model.vertex ?? vertexSettings());
247
230
  const provider = model.name.slice(0, model.name.indexOf("/"));
248
231
  const id = model.name.slice(provider.length + 1);
249
232
  switch (provider) {
@@ -253,6 +236,16 @@ async function providerModel(model) {
253
236
  default: return model.name;
254
237
  }
255
238
  }
239
+ /** The language model `name` is on `gateway`, checked for its key: what the judge's route says. */
240
+ function hostModel(name, gateway, what, { fallbacks = false, vertex } = {}) {
241
+ const route = judgeRoute(name, process.env, what, gateway).kind;
242
+ return providerModel({
243
+ name,
244
+ route,
245
+ ...route === "openrouter" ? { fallbacks } : {},
246
+ ...route === "vertex" && vertex ? { vertex } : {}
247
+ });
248
+ }
256
249
  async function callAnalysisModel(model, digest, prompt, maxOutputTokens) {
257
250
  const started = Date.now();
258
251
  const result = await generateText({
@@ -277,6 +270,6 @@ async function callAnalysisModel(model, digest, prompt, maxOutputTokens) {
277
270
  };
278
271
  }
279
272
  //#endregion
280
- export { isFake as a, questionModelLabel as c, cannedModel as i, resolveAnalysisModel as l, analysisModelName as n, providerModel as o, callAnalysisModel as r, questionModel as s, analysisKeyProblem as t, SYSTEM_PROMPT as u };
273
+ export { isFake as a, questionModelLabel as c, hostModel as i, resolveAnalysisModel as l, callAnalysisModel as n, providerModel as o, cannedModel as r, questionModel as s, analysisModelName as t, SYSTEM_PROMPT as u };
281
274
 
282
- //# sourceMappingURL=model-1ZLr9Dcv.js.map
275
+ //# sourceMappingURL=model-2rBZrdZA.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"model-2rBZrdZA.js","names":[],"sources":["../src/analysis/schema.ts","../src/analysis/model.ts"],"mappings":";;;;;;;;AAGA,MAAM,aAAa,EAAE,KAAK;CAAC;CAAQ;CAAU;AAAK,CAAC;AACnD,MAAM,QAAQ,EACX,MAAM,EAAE,OAAO,CAAC,CAAC,CACjB,SAAS,uFAAuF;AAEnG,MAAM,QAAQ,EAAE,OAAO;CACrB,MAAM,EAAE,OAAO;CACf;CACA;CACA,QAAQ,EACL,MAAM,EAAE,OAAO,CAAC,CAAC,CACjB,SAAS,CAAC,CACV,SAAS,qEAAqE;AACnF,CAAC;AAED,MAAa,iBAAiB,EAAE,OAAO;CACrC,MAAM,EACH,OAAO,CAAC,CACR,SACC,2HACF;CACF,cAAc,EAAE,MAAM,KAAK;CAC3B,SAAS,EACN,MAAM,KAAK,CAAC,CACZ,SACC,uLACF;CACF,KAAK,EACF,MAAM,KAAK,CAAC,CACZ,SAAS,yEAAyE;CACrF,WAAW,EAAE,MACX,EAAE,OAAO;EACP,MAAM,EAAE,OAAO;EACf,MAAM,EAAE,KAAK;GAAC;GAAc;GAAe;GAAS;GAAS;EAAO,CAAC;EACrE;CACF,CAAC,CACH;CACA,SAAS,EACN,MACC,EAAE,OAAO;EACP,OAAO,EAAE,OAAO;EAChB,QAAQ,EAAE,OAAO,CAAC,CAAC,SAAS,mDAAmD;EAC/E,MAAM,EAAE,KAAK;GAAC;GAAS;GAAQ;GAAQ;GAAS;GAAU;EAAS,CAAC;EACpE,QAAQ,EAAE,OAAO;EACjB,gBAAgB,EAAE,OAAO,CAAC,CAAC,SAAS,+BAA+B;EACnE,QAAQ,EACL,OAAO,CAAC,CACR,SACC,sIACF;EACF,QAAQ,EAAE,KAAK;GAAC;GAAQ;GAAU;EAAK,CAAC;EACxC,QAAQ,EAAE,KAAK;GAAC;GAAO;GAAU;EAAM,CAAC;EACxC;CACF,CAAC,CACH,CAAC,CACA,IAAI,CAAC;CACR,YAAY,EAAE,OAAO;EAAE,SAAS;EAAY,KAAK,EAAE,OAAO;CAAE,CAAC;AAC/D,CAAC;AAID,MAAa,gBAAgB;;;;;;;;;;;;;;;;;;;;;ACnC7B,MAAa,UAAU,MACrB,OAAO,MAAM,YAAY,MAAM,QAAS,EAAwB,SAAS;AAE3E,MAAM,qBAAqB,UAAkB,MAAM,WAAW,cAAc;;;;;;AAO5E,SAAS,cACP,QACA,QAAgC,OAAO,MAAM,OACL;CACxC,MAAM,QAAQ,gBAAgB,MAAM;CACpC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,OAAO,OAAO,UAAU,YAAY,CAAC,kBAAkB,KAAK,IAAI,QAAQ,KAAA;AAC1E;;AAGA,SAAS,WAAW,OAA+B,MAAsB;CACvE,IAAI,OAAO,UAAU,YAAY,CAAC,YAAY,KAAK,GAAG,OAAO;CAC7D,OAAO,gBAAgB,YAAY,KAAK,IAAI,iBAAiB,MAAM,iEAAiE,KAAK;AAC3I;;AAGA,SAAgB,kBAAkB,QAAoD;CACpF,MAAM,QAAQ,cAAc,MAAM;CAClC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,MAAM,IAAI,WACR,gGAAgG,OAAO,OAAO,WAAW,OAAO,MAAM,OAAO,mBAAmB,EAAE,+EACpK;AACF;;;;;AAMA,SAAgB,mBACd,QACA,QAAgC,OAAO,MAAM,OAC9B;CACf,MAAM,QAAQ,cAAc,QAAQ,KAAK;CACzC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,OAAO,OAAO,KAAK,IAAI,SAAS;AAClC;;;;;AAMA,SAAgB,cACd,QACA,UACA,OACA,QAAgC,OAAO,MAAM,OACjB;CAC5B,IAAI,SAAS,UAAU,KAAA,GAAW,OAAO,SAAS;CAClD,MAAM,OAAO,SAAS,WAAW,KAAA,IAAY,8BAA8B;CAC3E,MAAM,QAAQ,cAAc,QAAQ,KAAK;CACzC,MAAM,MAAM;CACZ,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,WACR,cAAc,SAAS,SAAS,KAAK,MAAM,+FAA+F,OAAO,OAAO,WAAW,OAAO,IAAI,EAAE,OAAO,IAAI,EAC7L;CAEF,IAAI,OAAO,UAAU,YAAY,kBAAkB,KAAK,GACtD,MAAM,IAAI,WACR,cAAc,SAAS,SAAS,KAAK,MAAM,2CAA2C,OAAO,KAAK,KAAK,MAAM,+DAA+D,KAAK,6DAA6D,IAAI,EACpP;CAEF,OAAO;AACT;;AAGA,SAAgB,qBAAqB,QAA+C;CAClF,MAAM,QAAQ,kBAAkB,MAAM;CACtC,IAAI,OAAO,KAAK,GAAG,OAAO;EAAE,MAAM;EAAQ,OAAO;EAAQ,MAAM;CAAM;CACrE,IAAI,kBAAkB,KAAK,GACzB,MAAM,IAAI,WACR,qBAAqB,OAAO,KAAK,OAAO,MAAM,sIAChD;CAEF,MAAM,QAAQ,aAAa,MAAM;CACjC,MAAM,UAAU,oBAAoB,OAAO,KAAA,GAAW,OAAO,gBAAgB;CAC7E,IAAI;EACF,MAAM,QAAQ,WAAW,OAAO,QAAQ,KAAK,kBAAkB,OAAO,CAAC,CAAC;EACxE,OAAO;GACL,MAAM;GACN;GACA,GAAI,UAAU,eAAe,EAAE,WAAW,oBAAoB,MAAM,QAAQ,EAAE,IAAI,CAAC;GACnF,GAAI,UAAU,WAAW,EAAE,QAAQ,iBAAiB,MAAM,QAAQ,EAAE,IAAI,CAAC;EAC3E;CACF,SAAS,OAAO;EACd,MAAM,IAAI,WACR,GAAI,MAAgB,QAAQ,sDAC9B;CACF;AACF;;;;;;AAOA,SAAgB,YACd,QACA,cAAc,OACC;CACf,OAAO;EACL,sBAAsB;EACtB,UAAU;EACV,SAAS;EACT,eAAe,CAAC;EAChB,YAAY,OAAO,YAAY;GAC7B,IAAI,WAAW,QAAQ,MAAM,IAAI,MAAM,OAAO,KAAK;GACnD,MAAM,QAAQ,cAAc,KAAK,KAAK,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC,SAAS,CAAC,IAAI;GACnF,IAAI,SAAS,cAAc,KAAK,KAAK,OAAO,KAAK,SAAS,CAAC,IAAI;GAC/D,IAAI,OAAO,OAAO;GAElB,MAAM,MACJ,eAAe,QAAQ,oBAAoB,KAAA,KAAa,SAAS,QAAQ;GAC3E,IAAI,KAAK;IACP,SAAS,QAAQ;IACjB,OAAO,KAAK,MAAM,GAAG,SAAS,CAAC;GACjC;GACA,OAAO;IACL,SAAS,CAAC;KAAE,MAAM;KAAQ;IAAK,CAAC;IAChC,cAAc,MACV;KAAE,SAAS;KAAU,KAAK;IAAa,IACvC;KAAE,SAAS;KAAQ,KAAK;IAAO;IACnC,OAAO;KACL,aAAa;MAAE,OAAO;MAAO,SAAS;MAAO,WAAW;MAAG,YAAY;KAAE;KACzE,cAAc;MAAE,OAAO;MAAQ,MAAM;MAAQ,WAAW;KAAE;IAC5D;IACA,UAAU,CAAC;IACX,UAAU,EAAE,SAAS,gBAAgB;GACvC;EACF;EACA,UAAU,YAAY;GACpB,MAAM,IAAI,MAAM,yCAAyC;EAC3D;CACF;AACF;;AAGA,SAAS,UAAU,MAAyB,QAA+B;CACzE,IAAI,KAAK,WAAW,KAAA,GAClB,MAAM,IAAI,MAAM,+DAA+D;CAEjF,MAAM,SAAS,OAAO,KAAK,WAAW,aAAa,KAAK,OAAO,MAAM,IAAI,KAAK;CAC9E,OAAO,YAAY,EAAE,MAAM,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM,EAAE,CAAC;AAC3F;AAEA,eAAe,cAAc,OAA8B,QAAwC;CACjG,IAAI,MAAM,MAAM,OAAO,UAAU,MAAM,MAAM,MAAM;CACnD,OAAO,cAAc,KAAK;AAC5B;;;;;AAMA,eAAsB,cAAc,OAAsD;CACxF,IAAI,MAAM,UAAU,WAAW,OAAO,MAAM;CAC5C,IAAI,MAAM,UAAU,cAAc,OAAO,wBAAwB,MAAM,MAAM,CAAC,CAAC,MAAM,SAAS;CAC9F,IAAI,MAAM,UAAU,UAClB,OAAO,oBAAoB,MAAM,MAAM,MAAM,UAAU,eAAe,CAAC;CAEzE,MAAM,WAAW,MAAM,KAAK,MAAM,GAAG,MAAM,KAAK,QAAQ,GAAG,CAAC;CAC5D,MAAM,KAAK,MAAM,KAAK,MAAM,SAAS,SAAS,CAAC;CAC/C,QAAQ,UAAR;EACE,KAAK,aACH,QAAQ,MAAM,OAAO,qBAAA,CAAsB,UAAU,EAAE;EACzD,KAAK,UACH,QAAQ,MAAM,OAAO,kBAAA,CAAmB,OAAO,EAAE;EACnD,KAAK,UACH,QAAQ,MAAM,OAAO,kBAAA,CAAmB,OAAO,EAAE;EACnD,SACE,OAAO,MAAM;CACjB;AACF;;AAMA,SAAgB,UACd,MACA,SACA,MACA,EAAE,YAAY,OAAO,WAA8B,CAAC,GAC5B;CACxB,MAAM,QAAQ,WAAW,MAAM,QAAQ,KAAK,MAAM,OAAO,CAAC,CAAC;CAC3D,OAAO,cAAc;EACnB;EACA;EACA,GAAI,UAAU,eAAe,EAAE,UAAU,IAAI,CAAC;EAC9C,GAAI,UAAU,YAAY,SAAS,EAAE,OAAO,IAAI,CAAC;CACnD,CAAC;AACH;AASA,eAAsB,kBACpB,OACA,QACA,QACA,iBACsB;CACtB,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,SAAS,MAAM,aAAa;EAChC,OAAO,MAAM,cAAc,OAAO,MAAM;EACxC,QAAQ;EACR;EACA,QAAQ,OAAO,OAAO;GAAE,QAAQ;GAAgB,MAAM;EAAW,CAAC;EAClE;EACA,YAAY;CACd,CAAC;CACD,OAAO;EACL,QAAQ,OAAO;EACf,eAAe,OAAO,UAAU,WAAW;EAC3C,OAAO;GACL,aAAa,OAAO,OAAO,eAAe;GAC1C,cAAc,OAAO,OAAO,gBAAgB;EAC9C;EACA,WAAW,KAAK,IAAI,IAAI;CAC1B;AACF"}
@@ -1,12 +1,19 @@
1
- import { i as RECOMMENDED_JUDGE_MODEL, l as judgeRoute, n as DIRECT_PROVIDERS } from "./judge-yLvjC5A3.js";
2
- import { i as debugTrace } from "./attempt-jI0OWxlz.js";
1
+ import { a as RECOMMENDED_JUDGE_MODEL, n as DIRECT_PROVIDERS, r as OPENROUTER_JUDGE_MODEL, u as judgeRoute } from "./judge-Bg4v4e_K.js";
2
+ import { u as vertexDecisionModel, v as vertexSettings } from "./vertex-DypYjx71.js";
3
+ import { i as debugTrace } from "./attempt-CpbZ9arV.js";
4
+ import { n as openRouterDecisionModel, t as openRouterBilling } from "./openrouter-models-Bo8PZ4Ew.js";
3
5
  import { experimental_decide } from "ai";
4
6
  //#region src/judge/model.ts
5
- /** See `judgeRoute`: the provider's own package on direct, the AI Gateway on vercel. */
6
- async function resolveModel(model, gateway) {
7
+ /**
8
+ * See `judgeRoute`: the provider's own package on direct, the AI Gateway on vercel, OpenRouter's
9
+ * Decisions API on openrouter, Vertex AI on vertex.
10
+ */
11
+ async function resolveModel(model, gateway, fallbacks, vertex) {
7
12
  if (typeof model !== "string") return model;
8
13
  const route = judgeRoute(model, process.env, "judge model", gateway);
9
14
  if (route.kind === "gateway") return model;
15
+ if (route.kind === "openrouter") return openRouterDecisionModel(model, fallbacks);
16
+ if (route.kind === "vertex") return vertexDecisionModel(model, vertex ?? vertexSettings());
10
17
  const id = model.slice(route.provider.length + 1);
11
18
  switch (route.provider) {
12
19
  case "anthropic": return (await import("@ai-sdk/anthropic")).anthropic.decisionModel(id);
@@ -19,22 +26,23 @@ async function resolveModel(model, gateway) {
19
26
  const OPENAI_DECISION_MODEL = "gpt-6-luna";
20
27
  /**
21
28
  * reeval's sentence when the judge's route answers only decision models and `model` isn't one:
22
- * Vercel AI Gateway refuses a language model as a judge, and OpenAI's Decisions API answers only
23
- * its decision model. Undefined for any other error.
29
+ * Vercel AI Gateway and OpenRouter refuse a language model as a judge, and OpenAI's Decisions API
30
+ * answers only its decision model. Undefined for any other error.
24
31
  */
25
32
  function judgeModelRefusal(model, gateway, error) {
26
33
  const e = error;
27
34
  const provider = model.slice(0, model.indexOf("/"));
28
- if (gateway && /is a language model, not a decision model/.test(e?.message ?? "")) {
35
+ if (gateway === "openrouter" && (/not a decision model/i.test(e?.message ?? "") || e?.statusCode === 404)) return `the judge model "${model}" isn't a decision model OpenRouter answers on its Decisions API (OpenRouter said: ${e.message}). Use a decision model on openrouter(), such as '${OPENROUTER_JUDGE_MODEL}'.`;
36
+ if (gateway === "vercel" && /is a language model, not a decision model/.test(e?.message ?? "")) {
29
37
  const direct = DIRECT_PROVIDERS[provider];
30
38
  const instead = provider === "openai" ? `, or 'openai/${OPENAI_DECISION_MODEL}' pinned to direct() with judge: { gateway: 'direct' } on ${direct?.key}` : direct ? `, or pin this one to direct() with judge: { gateway: 'direct' } to call it on ${direct.key}` : "";
31
39
  return `the judge model "${model}" is a language model, and Vercel AI Gateway judges only with decision models. Use a decision model on vercel(), such as '${RECOMMENDED_JUDGE_MODEL}' or 'openai/${OPENAI_DECISION_MODEL}-decisions'${instead}.`;
32
40
  }
33
41
  const id = model.slice(provider.length + 1);
34
- if (!gateway && provider === "openai" && id !== OPENAI_DECISION_MODEL && e?.data?.error?.code === "model_not_found") return `the judge model "${model}" isn't a decision model: reeval's openai/* judges call OpenAI's Decisions API, which answers with '${OPENAI_DECISION_MODEL}' (OpenAI said: ${e.message}). Use 'openai/${OPENAI_DECISION_MODEL}'.`;
42
+ if ((gateway ?? "direct") === "direct" && provider === "openai" && id !== OPENAI_DECISION_MODEL && e?.data?.error?.code === "model_not_found") return `the judge model "${model}" isn't a decision model: reeval's openai/* judges call OpenAI's Decisions API, which answers with '${OPENAI_DECISION_MODEL}' (OpenAI said: ${e.message}). Use 'openai/${OPENAI_DECISION_MODEL}'.`;
35
43
  }
36
- async function scoreWithModel(model, question, state, providerOptions, signal, gateway) {
37
- const resolved = await resolveModel(model, gateway);
44
+ async function scoreWithModel(model, question, state, providerOptions, signal, gateway, fallbacks = false, vertex) {
45
+ const resolved = await resolveModel(model, gateway, fallbacks, vertex);
38
46
  const result = await experimental_decide({
39
47
  model: resolved,
40
48
  state,
@@ -46,7 +54,7 @@ async function scoreWithModel(model, question, state, providerOptions, signal, g
46
54
  abortSignal: signal,
47
55
  maxRetries: 0
48
56
  }).catch((error) => {
49
- const refusal = typeof model === "string" && judgeModelRefusal(model, typeof resolved === "string", error);
57
+ const refusal = typeof model === "string" && judgeModelRefusal(model, gateway, error);
50
58
  throw refusal ? new Error(refusal, { cause: error }) : error;
51
59
  });
52
60
  const { answers, usage, warnings, response, providerMetadata } = result;
@@ -59,10 +67,11 @@ async function scoreWithModel(model, question, state, providerOptions, signal, g
59
67
  }, "judge provider response");
60
68
  return {
61
69
  score: result.answers.answer.probability,
62
- resolvedModel: result.response?.modelId
70
+ resolvedModel: result.response?.modelId,
71
+ ...gateway === "openrouter" ? openRouterBilling(providerMetadata) : {}
63
72
  };
64
73
  }
65
74
  //#endregion
66
75
  export { scoreWithModel };
67
76
 
68
- //# sourceMappingURL=model-B45lDUV1.js.map
77
+ //# sourceMappingURL=model-3MJtpho8.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"model-3MJtpho8.js","names":["decide"],"sources":["../src/judge/model.ts"],"mappings":";;;;;;;;;;AAmBA,eAAe,aACb,OACA,SACA,WACA,QACwB;CACxB,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,MAAM,QAAQ,WAAW,OAAO,QAAQ,KAAK,eAAe,OAAO;CACnE,IAAI,MAAM,SAAS,WAAW,OAAO;CACrC,IAAI,MAAM,SAAS,cAAc,OAAO,wBAAwB,OAAO,SAAS;CAChF,IAAI,MAAM,SAAS,UAAU,OAAO,oBAAoB,OAAO,UAAU,eAAe,CAAC;CACzF,MAAM,KAAK,MAAM,MAAM,MAAM,SAAS,SAAS,CAAC;CAChD,QAAQ,MAAM,UAAd;EACE,KAAK,aACH,QAAQ,MAAM,OAAO,qBAAA,CAAsB,UAAU,cAAc,EAAE;EACvE,KAAK,UACH,QAAQ,MAAM,OAAO,kBAAA,CAAmB,OAAO,cAAc,EAAE;EACjE,KAAK,UACH,QAAQ,MAAM,OAAO,kBAAA,CAAmB,OAAO,cAAc,EAAE;EACjE,SACE,QAAQ,MAAM,OAAO,uBAAA,CAAwB,WAAW,cAAc,EAAE;CAC5E;AACF;;AAGA,MAAM,wBAAwB;;;;;;AAS9B,SAAS,kBACP,OACA,SACA,OACoB;CACpB,MAAM,IAAI;CACV,MAAM,WAAW,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC;CAClD,IACE,YAAY,iBACX,wBAAwB,KAAK,GAAG,WAAW,EAAE,KAAK,GAAG,eAAe,MAErE,OAAO,oBAAoB,MAAM,qFAAqF,EAAE,QAAQ,oDAAoD,uBAAuB;CAE7M,IAAI,YAAY,YAAY,4CAA4C,KAAK,GAAG,WAAW,EAAE,GAAG;EAC9F,MAAM,SAAS,iBAAiB;EAChC,MAAM,UACJ,aAAa,WACT,gBAAgB,sBAAsB,4DAA4D,QAAQ,QAC1G,SACE,iFAAiF,OAAO,QACxF;EACR,OAAO,oBAAoB,MAAM,4HAA4H,wBAAwB,eAAe,sBAAsB,aAAa,QAAQ;CACjP;CACA,MAAM,KAAK,MAAM,MAAM,SAAS,SAAS,CAAC;CAC1C,KACG,WAAW,cAAc,YAC1B,aAAa,YACb,OAAO,yBACP,GAAG,MAAM,OAAO,SAAS,mBAEzB,OAAO,oBAAoB,MAAM,sGAAsG,sBAAsB,kBAAkB,EAAE,QAAQ,iBAAiB,sBAAsB;AAGpO;AAEA,eAAsB,eACpB,OACA,UACA,OACA,iBACA,QACA,SACA,YAAY,OACZ,QAMC;CACD,MAAM,WAAW,MAAM,aAAa,OAAO,SAAS,WAAW,MAAM;CACrE,MAAM,SAAS,MAAMA,oBAAO;EAC1B,OAAO;EACP;EACA,WAAW,EAAE,QAAQ;GAAE,MAAM;GAAW,cAAc;EAAS,EAAE;EACjE;EACA,aAAa;EAEb,YAAY;CACd,CAAC,CAAC,CAAC,OAAO,UAAmB;EAC3B,MAAM,UAAU,OAAO,UAAU,YAAY,kBAAkB,OAAO,SAAS,KAAK;EACpF,MAAM,UAAU,IAAI,MAAM,SAAS,EAAE,OAAO,MAAM,CAAC,IAAI;CACzD,CAAC;CACD,MAAM,EAAE,SAAS,OAAO,UAAU,UAAU,qBAAqB;CACjE,WACE,SACA,qBACA;EAAE;EAAS;EAAO;EAAU;EAAU;CAAiB,GACvD,yBACF;CACA,OAAO;EACL,OAAO,OAAO,QAAQ,OAAO;EAC7B,eAAe,OAAO,UAAU;EAChC,GAAI,YAAY,eAAe,kBAAkB,gBAAgB,IAAI,CAAC;CACxE;AACF"}