reeval 0.14.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (347) hide show
  1. package/README.md +3 -3
  2. package/THIRD_PARTY_NOTICES.md +69 -69
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-BloGVZ_a.js → adapter-CbqbMR7y.js} +37 -8
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-m-_t-IuW.js → agent-Zgn-kiqq.js} +57 -15
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{app-files-C-jwhRDt.js → app-files-ChbvRx3P.js} +4 -4
  9. package/dist/app-files-ChbvRx3P.js.map +1 -0
  10. package/dist/{assemble-BDEpelpi.js → assemble-SKQGW_QF.js} +136 -116
  11. package/dist/assemble-SKQGW_QF.js.map +1 -0
  12. package/dist/assemble-XBWuK3Ed.js +1 -0
  13. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  14. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  15. package/dist/{match-Bz0nSdzN.js → attempt-host-BOp8JklD.js} +96 -5
  16. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  17. package/dist/{capture-CVRwRTnb.js → capture-BM_3100g.js} +1218 -1207
  18. package/dist/capture-BM_3100g.js.map +1 -0
  19. package/dist/capture-DSK8O5bA.js +1 -0
  20. package/dist/{cli-C_4gaJ50.js → cli-PdYrcoKM.js} +106 -170
  21. package/dist/cli-PdYrcoKM.js.map +1 -0
  22. package/dist/{cli-spec-CpnbATtI.js → cli-spec-D0qyrPZi.js} +8 -20
  23. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  24. package/dist/cli.js +2 -2
  25. package/dist/{client-D5hNjpEx.js → client-Ct6s6OnF.js} +2 -2
  26. package/dist/{client-D5hNjpEx.js.map → client-Ct6s6OnF.js.map} +1 -1
  27. package/dist/{command-BXLUfw4h.js → command-4pJYVCT4.js} +33 -20
  28. package/dist/command-4pJYVCT4.js.map +1 -0
  29. package/dist/{validate-BhwqrVsm.js → command-9xm5ImqI.js} +387 -59
  30. package/dist/command-9xm5ImqI.js.map +1 -0
  31. package/dist/{command-Bxdy62dW.js → command-BqBxLgTc.js} +31 -30
  32. package/dist/command-BqBxLgTc.js.map +1 -0
  33. package/dist/{command-I1euLLio.js → command-DbSMcxZL.js} +7 -7
  34. package/dist/{command-I1euLLio.js.map → command-DbSMcxZL.js.map} +1 -1
  35. package/dist/{commands-jzigbNFb.js → commands-BU6YJCz9.js} +175 -72
  36. package/dist/commands-BU6YJCz9.js.map +1 -0
  37. package/dist/{commands-p1S4nw_H.js → commands-Bi2SEQtr.js} +1 -1
  38. package/dist/{config-BrdHA9bJ.js → config-k6m7XbAz.js} +499 -271
  39. package/dist/config-k6m7XbAz.js.map +1 -0
  40. package/dist/{contrasts-DRxMQQTC.js → contrasts-DLZF4dGF.js} +3 -3
  41. package/dist/{contrasts-DRxMQQTC.js.map → contrasts-DLZF4dGF.js.map} +1 -1
  42. package/dist/cost-C5RBTTJr.js +1 -0
  43. package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
  44. package/dist/cost-ChPa8-Fp.js.map +1 -0
  45. package/dist/db-CpsifVln.js +1530 -0
  46. package/dist/db-CpsifVln.js.map +1 -0
  47. package/dist/db-DkD-F-X-.js +2 -0
  48. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  49. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  50. package/dist/debug-DeM5pswK.js +1 -0
  51. package/dist/{docker-BnRlTpZA.js → docker-51WvhN_X.js} +5 -4
  52. package/dist/docker-51WvhN_X.js.map +1 -0
  53. package/dist/{docker-D01slDF3.js → docker-B4-rx4dA.js} +26 -8
  54. package/dist/docker-B4-rx4dA.js.map +1 -0
  55. package/dist/{docker-DivfZl2u.js → docker-YFj4H1Cr.js} +1 -1
  56. package/dist/{doctor-DQ2p0euG.js → doctor-BuQnfz4e.js} +275 -37
  57. package/dist/doctor-BuQnfz4e.js.map +1 -0
  58. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  59. package/dist/env-CARj5Axq.js.map +1 -0
  60. package/dist/{environment-CO0P0soF.js → environment-BOFgzFDg.js} +115 -296
  61. package/dist/environment-BOFgzFDg.js.map +1 -0
  62. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  63. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  64. package/dist/eval-source-Dje3rERU.js +145 -0
  65. package/dist/eval-source-Dje3rERU.js.map +1 -0
  66. package/dist/{evaluate-BUmsN7yG.js → evaluate-BPP9s3s6.js} +367 -130
  67. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  68. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  69. package/dist/{files-B-CJeeEh.js → files-Bi461Be2.js} +197 -91
  70. package/dist/files-Bi461Be2.js.map +1 -0
  71. package/dist/files-C2h2Cw4F.js +1 -0
  72. package/dist/files-Clipx4OW.js +37 -0
  73. package/dist/files-Clipx4OW.js.map +1 -0
  74. package/dist/files-DmUrRmmL.js +1 -0
  75. package/dist/{files-kUCrswI6.js → files-H24pz0hL.js} +20 -9
  76. package/dist/files-H24pz0hL.js.map +1 -0
  77. package/dist/gateway.d.ts +22 -9
  78. package/dist/gateway.js +30 -8
  79. package/dist/gateway.js.map +1 -1
  80. package/dist/{git-Bk8DSwHy.js → git-CsVZkNNL.js} +3 -3
  81. package/dist/{git-Bk8DSwHy.js.map → git-CsVZkNNL.js.map} +1 -1
  82. package/dist/harness-3NVyTSGK.js +392 -0
  83. package/dist/harness-3NVyTSGK.js.map +1 -0
  84. package/dist/harness.d.ts +12 -8
  85. package/dist/harness.js +1 -1
  86. package/dist/{images-DzLeFjLx.js → images-4mwhNkvh.js} +8 -8
  87. package/dist/{images-DzLeFjLx.js.map → images-4mwhNkvh.js.map} +1 -1
  88. package/dist/index.d.ts +51 -10
  89. package/dist/index.js +5 -5
  90. package/dist/init-BDEOopK7.js +1 -0
  91. package/dist/{init-SRalM5xF.js → init-cOk3D44b.js} +10 -11
  92. package/dist/init-cOk3D44b.js.map +1 -0
  93. package/dist/{invocations-DzyKSJBa.js → invocations-DW-lTk2f.js} +10 -6
  94. package/dist/invocations-DW-lTk2f.js.map +1 -0
  95. package/dist/{judge-yLvjC5A3.js → judge-CKdgEXN7.js} +236 -46
  96. package/dist/judge-CKdgEXN7.js.map +1 -0
  97. package/dist/judge.d.ts +5 -7
  98. package/dist/judge.js +4 -8
  99. package/dist/judge.js.map +1 -1
  100. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  101. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  102. package/dist/{matchers-D4eUqx4j.js → matchers-DT2TaGix.js} +2 -2
  103. package/dist/{matchers-D4eUqx4j.js.map → matchers-DT2TaGix.js.map} +1 -1
  104. package/dist/{model-1ZLr9Dcv.js → model-Bq3G0OG1.js} +29 -36
  105. package/dist/model-Bq3G0OG1.js.map +1 -0
  106. package/dist/{model-CgnOpss4.js → model-CTnyvh7H.js} +292 -68
  107. package/dist/model-CTnyvh7H.js.map +1 -0
  108. package/dist/model-D-SWrLZG.js +1 -0
  109. package/dist/model-g3XnzUv0.js +1 -0
  110. package/dist/{model-B45lDUV1.js → model-vRiUycWh.js} +22 -13
  111. package/dist/model-vRiUycWh.js.map +1 -0
  112. package/dist/{node-BSq5qHnQ.js → node-BVbbnFtv.js} +72 -50
  113. package/dist/node-BVbbnFtv.js.map +1 -0
  114. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  115. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  116. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  117. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  118. package/dist/{pairs-CHfZNRT6.js → pairs-puYi-zj2.js} +11 -4
  119. package/dist/pairs-puYi-zj2.js.map +1 -0
  120. package/dist/patch-CquGAGX5.js +76 -0
  121. package/dist/patch-CquGAGX5.js.map +1 -0
  122. package/dist/{patch-DWf-I_Q_.js → patch-DKJiC5Sq.js} +5 -7
  123. package/dist/patch-DKJiC5Sq.js.map +1 -0
  124. package/dist/{plan-BIwUzlB8.js → plan-DdWkM6SW.js} +64 -29
  125. package/dist/plan-DdWkM6SW.js.map +1 -0
  126. package/dist/{plugins-Dkp_H3pB.js → plugins-D-CP6mMd.js} +253 -17
  127. package/dist/plugins-D-CP6mMd.js.map +1 -0
  128. package/dist/{preflight-DJCD_BDn.js → preflight-9ixuXzsx.js} +3 -3
  129. package/dist/{preflight-DJCD_BDn.js.map → preflight-9ixuXzsx.js.map} +1 -1
  130. package/dist/{preflight-D9y-HvZQ.js → preflight-DTiVv0gY.js} +1 -1
  131. package/dist/{prepare-CIxCGG0M.js → prepare-BfyikQit.js} +8 -8
  132. package/dist/{prepare-CIxCGG0M.js.map → prepare-BfyikQit.js.map} +1 -1
  133. package/dist/{prepare-BJG_pD54.js → prepare-tmBBf4W8.js} +2 -2
  134. package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
  135. package/dist/prices-QaS-kjSQ.js.map +1 -0
  136. package/dist/{provenance-CfZpFpAh.js → provenance-BizwUn3h.js} +4 -4
  137. package/dist/{provenance-CfZpFpAh.js.map → provenance-BizwUn3h.js.map} +1 -1
  138. package/dist/{providers-DFk_6W4K.js → providers-B6b9Tnyi.js} +5 -5
  139. package/dist/{providers-DFk_6W4K.js.map → providers-B6b9Tnyi.js.map} +1 -1
  140. package/dist/{providers-D1Zxp2he.js → providers-DkOupEBl.js} +1 -1
  141. package/dist/query-Clm2mNSm.js +1 -0
  142. package/dist/{query-CemclveB.js → query-DyM-VExS.js} +3 -3
  143. package/dist/{query-CemclveB.js.map → query-DyM-VExS.js.map} +1 -1
  144. package/dist/{questions-bAudIc04.js → questions-BRhU8MzB.js} +100 -25
  145. package/dist/questions-BRhU8MzB.js.map +1 -0
  146. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  147. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  148. package/dist/redact-DVIaahLv.js +658 -0
  149. package/dist/redact-DVIaahLv.js.map +1 -0
  150. package/dist/remarks-BwJ8pcTQ.js +1 -0
  151. package/dist/{rules-VhftPOOT.js → remarks-C7i2LMRb.js} +296 -66
  152. package/dist/remarks-C7i2LMRb.js.map +1 -0
  153. package/dist/{header-BLPP6aCU.js → remarks-D_3qhBWU.js} +34 -9
  154. package/dist/remarks-D_3qhBWU.js.map +1 -0
  155. package/dist/{remarks-mujJv1Ey.js → remarks-dA8LyoPc.js} +9 -5
  156. package/dist/remarks-dA8LyoPc.js.map +1 -0
  157. package/dist/{remote-CKcEKja4.js → remote-DFIzcx_6.js} +39 -99
  158. package/dist/remote-DFIzcx_6.js.map +1 -0
  159. package/dist/resolve--rQZ8u6Z.js +1 -0
  160. package/dist/{resolve-CBoKzQMP.js → resolve-DWSJpKHX.js} +46 -30
  161. package/dist/resolve-DWSJpKHX.js.map +1 -0
  162. package/dist/{results-CMgZEANy.js → results-BC3pcG_Z.js} +15 -30
  163. package/dist/results-BC3pcG_Z.js.map +1 -0
  164. package/dist/{resume-B9SOSnK9.js → resume-j-rcIryi.js} +47 -17
  165. package/dist/resume-j-rcIryi.js.map +1 -0
  166. package/dist/{runtime-BgQRyCLl.js → runtime-d56Gg3J5.js} +30 -13
  167. package/dist/runtime-d56Gg3J5.js.map +1 -0
  168. package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmln11gU.js} +2 -2
  169. package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmln11gU.js.map} +1 -1
  170. package/dist/sandbox.d.ts +1 -1
  171. package/dist/sandbox.js +1 -1
  172. package/dist/{server-TGW4WyAz.js → server-BOoH10J8.js} +51 -33
  173. package/dist/server-BOoH10J8.js.map +1 -0
  174. package/dist/server-DpuVAbrO.js +1 -0
  175. package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
  176. package/dist/shared-B_lvGqWz.js.map +1 -0
  177. package/dist/source-BFd6f0-R.js +1 -0
  178. package/dist/{source-B1WvBIQt.js → source-wD04-rRG.js} +15 -10
  179. package/dist/source-wD04-rRG.js.map +1 -0
  180. package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
  181. package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
  182. package/dist/{src-CyaDNG9r.js → src-6_zz7oJs.js} +1 -1
  183. package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
  184. package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
  185. package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
  186. package/dist/src-BO8AYqJ5.js.map +1 -0
  187. package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
  188. package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
  189. package/dist/src-BYPMM1x7.js +2 -0
  190. package/dist/src-Bp4aN3rA.js +2 -0
  191. package/dist/src-C7upnZSd.js +2 -0
  192. package/dist/{src-8zoX_qrb.js → src-D91Hd6hT.js} +546 -1060
  193. package/dist/src-D91Hd6hT.js.map +1 -0
  194. package/dist/src-DH-9IO7f.js +1 -0
  195. package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
  196. package/dist/src-DSZyg0Uq.js.map +1 -0
  197. package/dist/{src-DjRI-IeE.js → src-DgHuStEi.js} +134 -34
  198. package/dist/{src-DjRI-IeE.js.map → src-DgHuStEi.js.map} +1 -1
  199. package/dist/src-HSxBazcg.js +3 -0
  200. package/dist/src-LD1BAJFa.js +2 -0
  201. package/dist/src-dlX-Lmxg.js +2 -0
  202. package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
  203. package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
  204. package/dist/src-kYeaJJrl.js +2 -0
  205. package/dist/state-B5PLm5qY.js +1 -0
  206. package/dist/{state-B1t143wG.js → state-DrJ687ut.js} +15 -15
  207. package/dist/{state-B1t143wG.js.map → state-DrJ687ut.js.map} +1 -1
  208. package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
  209. package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
  210. package/dist/suite-22Kt9Grx.js +2 -0
  211. package/dist/{suite-WgT7m6Vs.js → suite-Co8cwelb.js} +48 -63
  212. package/dist/suite-Co8cwelb.js.map +1 -0
  213. package/dist/{summary-DIGEVr08.js → summary-DLFHVoux.js} +80 -270
  214. package/dist/summary-DLFHVoux.js.map +1 -0
  215. package/dist/testing.d.ts +417 -142
  216. package/dist/testing.js +12 -10
  217. package/dist/testing.js.map +1 -1
  218. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  219. package/dist/text-DTigcA64.js.map +1 -0
  220. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  221. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  222. package/dist/types-C5vI3g1C.js.map +1 -1
  223. package/dist/{types-DXBbP9Mk.d.ts → types-coMWHaU2.d.ts} +253 -733
  224. package/dist/vertex-Bv7vhMgs.js +1 -0
  225. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  226. package/dist/vertex-DypYjx71.js +456 -0
  227. package/dist/vertex-DypYjx71.js.map +1 -0
  228. package/dist/view/index.d.ts +536 -121
  229. package/dist/view/index.js +9 -9
  230. package/dist/{views-sXpGE6a2.js → views-BFO1BLKu.js} +9 -10
  231. package/dist/{views-sXpGE6a2.js.map → views-BFO1BLKu.js.map} +1 -1
  232. package/dist/views-BOf8qEL8.js +2 -0
  233. package/dist/{worker-entry-Duo9oAeT.js → worker-entry-CJtdsKBP.js} +19 -12
  234. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  235. package/dist/{workspace-CDI68wtX.js → workspace-BRxjfa8N.js} +5 -5
  236. package/dist/{workspace-CDI68wtX.js.map → workspace-BRxjfa8N.js.map} +1 -1
  237. package/dist/workspace-DDHzkMjm.js +1 -0
  238. package/docker/seccomp-default.json +1 -0
  239. package/package.json +5 -1
  240. package/skills/eval-rubric/SKILL.md +35 -104
  241. package/dist/adapter-BloGVZ_a.js.map +0 -1
  242. package/dist/agent-m-_t-IuW.js.map +0 -1
  243. package/dist/app-files-C-jwhRDt.js.map +0 -1
  244. package/dist/assemble-BDEpelpi.js.map +0 -1
  245. package/dist/assemble-C5bfncHU.js +0 -1
  246. package/dist/capture-CVRwRTnb.js.map +0 -1
  247. package/dist/capture-CherbjzB.js +0 -1
  248. package/dist/cli-C_4gaJ50.js.map +0 -1
  249. package/dist/cli-spec-CpnbATtI.js.map +0 -1
  250. package/dist/command-BWAuwQNk.js +0 -91
  251. package/dist/command-BWAuwQNk.js.map +0 -1
  252. package/dist/command-BXLUfw4h.js.map +0 -1
  253. package/dist/command-Be6BSE2L.js +0 -335
  254. package/dist/command-Be6BSE2L.js.map +0 -1
  255. package/dist/command-Bxdy62dW.js.map +0 -1
  256. package/dist/commands-jzigbNFb.js.map +0 -1
  257. package/dist/config-BrdHA9bJ.js.map +0 -1
  258. package/dist/cost-D4elx933.js.map +0 -1
  259. package/dist/cost-DmnAod9x.js +0 -1
  260. package/dist/db-CAfksinX.js +0 -2
  261. package/dist/db-DO0p7xgP.js +0 -619
  262. package/dist/db-DO0p7xgP.js.map +0 -1
  263. package/dist/debug-D7bEv_UV.js +0 -1
  264. package/dist/docker-BnRlTpZA.js.map +0 -1
  265. package/dist/docker-D01slDF3.js.map +0 -1
  266. package/dist/doctor-DQ2p0euG.js.map +0 -1
  267. package/dist/env-BQqj93gu.js.map +0 -1
  268. package/dist/environment-CO0P0soF.js.map +0 -1
  269. package/dist/evaluate-BUmsN7yG.js.map +0 -1
  270. package/dist/evaluate-C_sgRA-V.d.ts +0 -134
  271. package/dist/facts-DzOUTreB.js +0 -906
  272. package/dist/facts-DzOUTreB.js.map +0 -1
  273. package/dist/files-B-CJeeEh.js.map +0 -1
  274. package/dist/files-Bp5j5BqI.js +0 -1
  275. package/dist/files-C2sjmXhk.js +0 -1
  276. package/dist/files-C_gQNVB-.js +0 -65
  277. package/dist/files-C_gQNVB-.js.map +0 -1
  278. package/dist/files-kUCrswI6.js.map +0 -1
  279. package/dist/generate-DZ7vp69F.js +0 -1647
  280. package/dist/generate-DZ7vp69F.js.map +0 -1
  281. package/dist/harness-COVXSwJ6.js +0 -354
  282. package/dist/harness-COVXSwJ6.js.map +0 -1
  283. package/dist/header-BLPP6aCU.js.map +0 -1
  284. package/dist/init-RgCQXZhb.js +0 -1
  285. package/dist/init-SRalM5xF.js.map +0 -1
  286. package/dist/insights-TyAvz9aV.js +0 -68
  287. package/dist/insights-TyAvz9aV.js.map +0 -1
  288. package/dist/invocations-DzyKSJBa.js.map +0 -1
  289. package/dist/judge-yLvjC5A3.js.map +0 -1
  290. package/dist/match-Bz0nSdzN.js.map +0 -1
  291. package/dist/model-1ZLr9Dcv.js.map +0 -1
  292. package/dist/model-B45lDUV1.js.map +0 -1
  293. package/dist/model-BTZq8HIZ.js +0 -1
  294. package/dist/model-CgnOpss4.js.map +0 -1
  295. package/dist/model-Ch8kX87u.js +0 -1
  296. package/dist/node-BSq5qHnQ.js.map +0 -1
  297. package/dist/opencode-providers-klkqgzvb.js +0 -3276
  298. package/dist/opencode-providers-klkqgzvb.js.map +0 -1
  299. package/dist/pairs-CHfZNRT6.js.map +0 -1
  300. package/dist/patch-DWf-I_Q_.js.map +0 -1
  301. package/dist/plan-BIwUzlB8.js.map +0 -1
  302. package/dist/plugins-Dkp_H3pB.js.map +0 -1
  303. package/dist/prices-BcLEXvdY.js.map +0 -1
  304. package/dist/query-Cvbwl804.js +0 -1
  305. package/dist/questions-bAudIc04.js.map +0 -1
  306. package/dist/redact-CRGdfV2s.js +0 -240
  307. package/dist/redact-CRGdfV2s.js.map +0 -1
  308. package/dist/remarks-CXwSnSjR.js +0 -278
  309. package/dist/remarks-CXwSnSjR.js.map +0 -1
  310. package/dist/remarks-CyAu2DiB.js +0 -1
  311. package/dist/remarks-mujJv1Ey.js.map +0 -1
  312. package/dist/remote-CKcEKja4.js.map +0 -1
  313. package/dist/resolve-B5-uqmJ6.js +0 -1
  314. package/dist/resolve-CBoKzQMP.js.map +0 -1
  315. package/dist/results-CMgZEANy.js.map +0 -1
  316. package/dist/resume-B9SOSnK9.js.map +0 -1
  317. package/dist/rules-VhftPOOT.js.map +0 -1
  318. package/dist/run-CwIEK16q.js +0 -88
  319. package/dist/run-CwIEK16q.js.map +0 -1
  320. package/dist/runtime-BgQRyCLl.js.map +0 -1
  321. package/dist/server-CHRSgKRp.js +0 -1
  322. package/dist/server-TGW4WyAz.js.map +0 -1
  323. package/dist/shared-BDUzStad.js.map +0 -1
  324. package/dist/source-B1WvBIQt.js.map +0 -1
  325. package/dist/source-DAzztZwU.js +0 -1
  326. package/dist/src-1jHbFaGW.js +0 -1
  327. package/dist/src-8zoX_qrb.js.map +0 -1
  328. package/dist/src-C_9wgowC.js +0 -2
  329. package/dist/src-ChhtI6BP.js.map +0 -1
  330. package/dist/src-D005Cbqz.js +0 -2
  331. package/dist/src-DIfRbp5a.js +0 -3
  332. package/dist/src-Dg8121Xy.js +0 -2
  333. package/dist/src-ElBqEN-S.js +0 -2
  334. package/dist/src-NSdQ9zI6.js.map +0 -1
  335. package/dist/src-YHGaUrGE.js +0 -2
  336. package/dist/src-mJVxw1Jm.js +0 -2
  337. package/dist/state-vznMh2wl.js +0 -1
  338. package/dist/suite-BsWGHzlG.js +0 -2
  339. package/dist/suite-WgT7m6Vs.js.map +0 -1
  340. package/dist/summary-DIGEVr08.js.map +0 -1
  341. package/dist/text-CrVy5ioL.js.map +0 -1
  342. package/dist/transcript-BTxJpW4R.js.map +0 -1
  343. package/dist/usage-DF3_p5pE.js +0 -1
  344. package/dist/validate-BhwqrVsm.js.map +0 -1
  345. package/dist/views-BW_C5eVH.js +0 -2
  346. package/dist/worker-entry-Duo9oAeT.js.map +0 -1
  347. package/dist/workspace-QZCacP8A.js +0 -1
@@ -1,6 +1,154 @@
1
- import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-klkqgzvb.js";
2
- import { r as badCharacterAt } from "./redact-CRGdfV2s.js";
1
+ import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-Bz0l7OZI.js";
2
+ import { v as vertexSettings } from "./vertex-DypYjx71.js";
3
+ import { m as redactSecrets, r as badCharacterAt } from "./redact-DVIaahLv.js";
3
4
  import { setTimeout } from "node:timers/promises";
5
+ //#region src/gateway/effort.ts
6
+ /** Every model: no run has checked the harness's effort reaches any of them there. */
7
+ const everyModel = () => true;
8
+ /** Any but a Gemini model (`gemini-3.8-flash`, `google/gemini-3.8-flash`). */
9
+ const notGemini = (model) => !/^(?:google\/)?gemini-/.test(model);
10
+ //#endregion
11
+ //#region src/gateway/openrouter.ts
12
+ const OPENROUTER_HOST = "openrouter.ai";
13
+ const OPENROUTER_KEY = "OPENROUTER_API_KEY";
14
+ const onOpenRouter = (path, method = "POST") => ({
15
+ host: OPENROUTER_HOST,
16
+ path: { exact: path },
17
+ method: [method]
18
+ });
19
+ /** The response names the model that served it on OpenRouter's host, in OpenRouter's form. */
20
+ const servedOnOpenRouter = ({ host, responseModel, responseEffort }) => host === "openrouter.ai" && responseModel ? `${responseModel}${responseEffort ? `[effort=${responseEffort}]` : ""}` : void 0;
21
+ /** The upstream provider OpenRouter's response says served the call, and what it charged. */
22
+ const upstreamOnOpenRouter = ({ host, responseProvider, responseCost }) => host === "openrouter.ai" && responseProvider ? {
23
+ provider: responseProvider,
24
+ ...responseCost !== void 0 ? { costUsd: responseCost } : {}
25
+ } : void 0;
26
+ /** What every OpenRouter route shares: the key under its own name, and what the responses say. */
27
+ const route = (endpoints, calls) => ({
28
+ credential: OPENROUTER_KEY,
29
+ authAs: OPENROUTER_KEY,
30
+ endpoints,
31
+ modelResponses: calls,
32
+ servedModel: servedOnOpenRouter,
33
+ upstream: upstreamOnOpenRouter,
34
+ hosts: [OPENROUTER_HOST]
35
+ });
36
+ /** An id as OpenRouter names it: as written when it names a provider, else under `provider`. */
37
+ const under = (provider) => (model) => model.includes("/") ? model : `${provider}/${model}`;
38
+ /**
39
+ * An Anthropic id as OpenRouter names it, with the version dotted (`anthropic/claude-haiku-4.5`
40
+ * for `claude-haiku-4-5`), unless the id names a provider.
41
+ */
42
+ const anthropicOnOpenRouter = (model) => model.includes("/") ? model : `anthropic/${model.replace(/-(\d{1,2})-(\d{1,2})$/, "-$1.$2")}`;
43
+ /**
44
+ * Claude Code on OpenRouter's Anthropic Messages API: `ANTHROPIC_BASE_URL` at OpenRouter and the
45
+ * key as `ANTHROPIC_AUTH_TOKEN`, which the CLI sends as a Bearer token. On another base URL it
46
+ * makes none of its `/api/claude_code/*` calls.
47
+ */
48
+ function claudeCode$1() {
49
+ const calls = [onOpenRouter("/api/v1/messages")];
50
+ return {
51
+ ...route([...calls, onOpenRouter("/api/v1/messages/count_tokens")], calls),
52
+ authAs: "ANTHROPIC_AUTH_TOKEN",
53
+ authEnv: { ANTHROPIC_BASE_URL: `https://${OPENROUTER_HOST}/api` },
54
+ modelId: anthropicOnOpenRouter
55
+ };
56
+ }
57
+ /** Codex on a Responses API provider at OpenRouter, whose ids name the provider (`openai/…`). */
58
+ function codex$1() {
59
+ const calls = [onOpenRouter("/api/v1/responses")];
60
+ return {
61
+ ...route(calls, calls),
62
+ modelId: under("openai")
63
+ };
64
+ }
65
+ /**
66
+ * Goose on its `openrouter` provider: Chat Completions, and the model list it fetches each turn.
67
+ */
68
+ function goose$1() {
69
+ const calls = [onOpenRouter("/api/v1/chat/completions")];
70
+ return {
71
+ ...route([...calls, onOpenRouter("/api/v1/models", "GET")], calls),
72
+ modelId: under("google")
73
+ };
74
+ }
75
+ /**
76
+ * OpenCode on its own `openrouter` provider, with the model as `openrouter/<id>`: the requests the
77
+ * pinned CLI sends that provider's key on (harness-opencode's provider table), plus Exa's host for
78
+ * web search.
79
+ */
80
+ function opencode$1() {
81
+ const resolved = resolveOpenCodeProvider("openrouter/model");
82
+ if ("error" in resolved) throw new Error(resolved.error);
83
+ const { provider } = resolved;
84
+ return {
85
+ credential: OPENROUTER_KEY,
86
+ authAs: OPENROUTER_KEY,
87
+ endpoints: providerEndpoints(provider),
88
+ modelResponses: providerModelResponses(provider),
89
+ servedModel: (request) => providerServedModel(provider, request),
90
+ upstream: upstreamOnOpenRouter,
91
+ hosts: [...providerHosts(provider), "mcp.exa.ai"],
92
+ modelId: (model) => `openrouter/${model}`
93
+ };
94
+ }
95
+ /**
96
+ * pi on its own `openrouter` provider: Chat Completions, a Bearer key, a model from pi's catalog
97
+ * for OpenRouter, each thinking level sent as OpenRouter's `reasoning.effort`. `--offline` skips
98
+ * the model list pi would fetch to fit that catalog to the key.
99
+ */
100
+ function pi$1() {
101
+ const calls = [onOpenRouter("/api/v1/chat/completions")];
102
+ return {
103
+ ...route(calls, calls),
104
+ modelId: under("google")
105
+ };
106
+ }
107
+ /**
108
+ * Grok Build with the model as a `[model.<id>]` table on OpenRouter's Chat Completions API, keyed
109
+ * by `env_key`; it then calls nothing on xAI's API.
110
+ */
111
+ function grokBuild$1() {
112
+ const calls = [onOpenRouter("/api/v1/chat/completions")];
113
+ return {
114
+ ...route(calls, calls),
115
+ modelId: under("x-ai")
116
+ };
117
+ }
118
+ /**
119
+ * fx with OpenRouter's Chat Completions API as its configured provider `openrouter`, through the
120
+ * relay, and an OpenAI id as `openai/<id>`. fx sends no `reasoning_effort` to a configured provider.
121
+ */
122
+ function fx$1() {
123
+ const calls = [onOpenRouter("/api/v1/chat/completions")];
124
+ return {
125
+ ...route(calls, calls),
126
+ modelId: under("openai")
127
+ };
128
+ }
129
+ /** OpenRouter: the harnesses reeval runs there, and why it doesn't run the rest. */
130
+ const OPENROUTER_GATEWAY = {
131
+ name: "OpenRouter",
132
+ routes: {
133
+ "claude-code": claudeCode$1,
134
+ codex: codex$1,
135
+ goose: goose$1,
136
+ opencode: opencode$1,
137
+ fx: fx$1,
138
+ "grok-build": grokBuild$1,
139
+ pi: pi$1
140
+ },
141
+ refusals: {
142
+ cursor: "Cursor's agent only calls Cursor's own API",
143
+ antigravity: "Antigravity's CLI only calls Google's API, with a Gemini key, and OpenRouter has no Gemini API"
144
+ },
145
+ unprovenEffort: {
146
+ "claude-code": everyModel,
147
+ goose: notGemini,
148
+ opencode: notGemini
149
+ }
150
+ };
151
+ //#endregion
4
152
  //#region src/gateway/vercel.ts
5
153
  const VERCEL_HOST = "ai-gateway.vercel.sh";
6
154
  /** What the adapters take Vercel's key as, whichever host variable it came from. */
@@ -120,6 +268,23 @@ function goose(env) {
120
268
  };
121
269
  }
122
270
  /**
271
+ * pi on a provider of its own on Vercel's Chat Completions API, a Bearer key: a Gemini model from
272
+ * pi's catalog, each thinking level sent as Gemini's own `thinkingConfig` under Vercel's
273
+ * `providerOptions` (Vercel's `reasoning_effort` maps every Gemini 3 level but `low` to `high`).
274
+ */
275
+ function pi(env) {
276
+ const endpoints = [onVercel("/v1/chat/completions", "POST")];
277
+ return {
278
+ credential: vercelCredential(env),
279
+ authAs: VERCEL_KEY,
280
+ endpoints,
281
+ modelResponses: endpoints,
282
+ servedModel: servedOnVercel,
283
+ hosts: [VERCEL_HOST],
284
+ modelId: googleId
285
+ };
286
+ }
287
+ /**
123
288
  * OpenCode on its own `vercel` provider, with the model as `vercel/<id>`: the requests the pinned
124
289
  * CLI sends that provider's key on (harness-opencode's provider table), plus Exa's host for web
125
290
  * search.
@@ -158,31 +323,27 @@ function grokBuild(env) {
158
323
  modelId: (model) => model.includes("/") ? model : `xai/${model}`
159
324
  };
160
325
  }
161
- /** The harnesses reeval runs on Vercel, and what each needs there. */
162
- const VERCEL_ROUTES = {
163
- "claude-code": claudeCode,
164
- codex,
165
- goose,
166
- opencode,
167
- fx,
168
- "grok-build": grokBuild
169
- };
170
- /** Why Vercel can't serve an agent reeval doesn't route there, for the plan's messages. */
171
- const VERCEL_REFUSALS = {
172
- cursor: "Cursor's agent only calls Cursor's own API",
173
- antigravity: "Antigravity's CLI only calls Google's API, with a Gemini key",
174
- pi: "pi reaches a Gemini model on Vercel AI Gateway only through Anthropic's Messages API, with Anthropic's thinking budget for Gemini's own thinking level, and doesn't behave as it does on Gemini's API; reeval runs pi on its Gemini key"
326
+ /** Vercel AI Gateway: the harnesses reeval runs there, and why it doesn't run the rest. */
327
+ const VERCEL_GATEWAY = {
328
+ name: "Vercel AI Gateway",
329
+ routes: {
330
+ "claude-code": claudeCode,
331
+ codex,
332
+ goose,
333
+ opencode,
334
+ fx,
335
+ "grok-build": grokBuild,
336
+ pi
337
+ },
338
+ refusals: {
339
+ cursor: "Cursor's agent only calls Cursor's own API",
340
+ antigravity: "Antigravity's CLI only calls Google's API, with a Gemini key"
341
+ },
342
+ unprovenEffort: {
343
+ goose: everyModel,
344
+ opencode: notGemini
345
+ }
175
346
  };
176
- /**
177
- * The harnesses on Vercel whose `effort` reaches the model by a path no real run has checked: the
178
- * plan refuses `effort` with them on `vercel()`. Claude Code's goes as on Anthropic's API, and
179
- * Codex's response names the effort it ran at.
180
- */
181
- const VERCEL_UNPROVEN_EFFORT = [
182
- "goose",
183
- "opencode",
184
- "grok-build"
185
- ];
186
347
  //#endregion
187
348
  //#region src/judge/retry.ts
188
349
  /** Up to six tries over about a minute: long enough to ride out a provider's "high demand". */
@@ -250,10 +411,16 @@ async function withRetries(call, policy, signal) {
250
411
  //#region src/judge/index.ts
251
412
  /** The judge the docs recommend, pinned so a model update never silently moves scores. */
252
413
  const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.14.0";
253
- /** The language model `reeval init` sets as `analysis.model` and the docs recommend for insights. */
414
+ /** The same judge as OpenRouter names it. */
415
+ const OPENROUTER_JUDGE_MODEL = "typesafe/jev-1.14";
416
+ /** The language model `reeval init` sets as `analysis.model` and `reeval analyze` uses. */
254
417
  const RECOMMENDED_ANALYSIS_MODEL = "google/gemini-3.8-flash";
255
418
  const DEFAULT_THRESHOLD = .8;
256
- const GATEWAY_KEYS = [VERCEL_KEY, VERCEL_OIDC];
419
+ /** Each routing gateway's keys, in the order reeval prefers them. */
420
+ const ROUTED_KEYS = {
421
+ vercel: [VERCEL_KEY, VERCEL_OIDC],
422
+ openrouter: [OPENROUTER_KEY]
423
+ };
257
424
  /**
258
425
  * Providers `direct()` calls, each through its own AI SDK package and key. Any other prefix needs
259
426
  * `vercel()` in the config's gateway list.
@@ -299,37 +466,52 @@ const providerOf = (model) => model.slice(0, model.indexOf("/"));
299
466
  /**
300
467
  * Environment variables a model needs on `gateway`, in the order reeval prefers them; any one is
301
468
  * enough. On direct, its provider's own key (none for a prefix reeval can't call directly); on
302
- * vercel, the Gateway's.
469
+ * vertex none, since its token comes from Application Default Credentials; on another gateway,
470
+ * that gateway's.
303
471
  */
304
472
  function judgeCredentials(model, gateway = "direct") {
305
- if (isFakeJudge(model)) return [];
306
- if (gateway === "vercel") return GATEWAY_KEYS;
473
+ if (isFakeJudge(model) || gateway === "vertex") return [];
474
+ if (gateway !== "direct") return [...ROUTED_KEYS[gateway]];
307
475
  const direct = DIRECT_PROVIDERS[providerOf(model)];
308
476
  return direct ? [direct.key] : [];
309
477
  }
310
478
  /** The keys that would work for a model on `on`: `ANTHROPIC_API_KEY (for direct())`. */
311
479
  function modelKeyOptions(model, on = "direct") {
312
- if (on === "vercel") return `${GATEWAY_KEYS.join(" or ")} (for vercel())`;
480
+ if (on === "vertex") return "Application Default Credentials (for vertex())";
481
+ if (on !== "direct") return `${ROUTED_KEYS[on].join(" or ")} (for ${on}())`;
313
482
  const direct = DIRECT_PROVIDERS[providerOf(model)];
314
483
  return direct ? `${direct.key} (for direct())` : `vercel() in gateway (direct() calls only ${Object.keys(DIRECT_PROVIDERS).map((p) => `${p}/*`).join(", ")} models)`;
315
484
  }
316
- /** The Vercel credential set in `env`, if any. */
317
- const setGatewayKey = (env) => GATEWAY_KEYS.find((k) => env[k]);
485
+ /** A gateway's credential set in `env`, if any, with the gateway it's for. */
486
+ function setGatewayKey(env) {
487
+ for (const [gateway, keys] of Object.entries(ROUTED_KEYS)) {
488
+ const key = keys.find((k) => env[k]);
489
+ if (key) return {
490
+ key,
491
+ gateway
492
+ };
493
+ }
494
+ }
318
495
  /**
319
- * ` AI_GATEWAY_API_KEY is set, but …` when a Vercel credential is set and the model goes direct, so
320
- * a key that worked before reeval stopped falling back to the Gateway says how to use it.
496
+ * ` AI_GATEWAY_API_KEY is set, but …` when a gateway's credential is set and the model goes
497
+ * direct, so the key says how to use it.
321
498
  */
322
499
  function unusedGatewayKey(env, gateway = "direct") {
323
500
  if (gateway !== "direct") return "";
324
501
  const set = setGatewayKey(env);
325
- return set ? ` ${set} is set, but no gateway in the config uses it: add vercel() to gateway to call the model through Vercel AI Gateway.` : "";
502
+ if (!set) return "";
503
+ const name = set.gateway === "vercel" ? "Vercel AI Gateway" : "OpenRouter";
504
+ return ` ${set.key} is set, but no gateway in the config uses it: add ${set.gateway}() to gateway to call the model through ${name}.`;
326
505
  }
327
506
  /**
328
- * The provider's own package on direct, or the Vercel AI Gateway on vercel. Throws, naming the
329
- * model, its prefix and the keys that would work, when none is set, when the key it would send
330
- * has whitespace or a line break in it, or when Vercel's project token has expired.
507
+ * The provider's own package on direct, the Vercel AI Gateway on vercel, or OpenRouter's package
508
+ * on openrouter, or Vertex AI on vertex, whose token is minted when the model is called.
509
+ * Throws, naming the model, its prefix and the keys that would work, when none is
510
+ * set, when the key it would send has whitespace or a line break in it, or when Vercel's project
511
+ * token has expired.
331
512
  */
332
513
  function judgeRoute(model, env = process.env, what = "judge model", gateway = "direct") {
514
+ if (gateway === "vertex") return { kind: "vertex" };
333
515
  const provider = providerOf(model);
334
516
  const direct = DIRECT_PROVIDERS[provider];
335
517
  const keys = judgeCredentials(model, gateway);
@@ -342,6 +524,7 @@ function judgeRoute(model, env = process.env, what = "judge model", gateway = "d
342
524
  }
343
525
  const at = badCharacterAt(env[key]);
344
526
  if (at) throw new Error(`${key} for ${what} "${model}" has whitespace or a line break in it at character ${at}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
527
+ if (gateway === "openrouter") return { kind: "openrouter" };
345
528
  return direct && key === direct.key ? {
346
529
  kind: "direct",
347
530
  provider,
@@ -352,8 +535,11 @@ function judgeRoute(model, env = process.env, what = "judge model", gateway = "d
352
535
  async function runJudge(question, state, options, signal, retry = JUDGE_RETRY) {
353
536
  const { model, threshold } = options;
354
537
  const gateway = typeof options.gateway === "object" ? options.gateway.kind : options.gateway;
538
+ const fallbacks = typeof options.gateway === "object" && options.gateway.kind === "openrouter" && options.gateway.fallbacks === true;
539
+ const vertex = gateway === "vertex" ? vertexSettings(typeof options.gateway === "object" && options.gateway.kind === "vertex" ? options.gateway : {}) : void 0;
355
540
  let score;
356
541
  let resolvedModel;
542
+ let billing = {};
357
543
  let retries = [];
358
544
  if (isFakeJudge(model)) score = typeof model.score === "function" ? model.score(state) : model.score;
359
545
  else {
@@ -361,9 +547,12 @@ async function runJudge(question, state, options, signal, retry = JUDGE_RETRY) {
361
547
  const problem = checkModelString(model);
362
548
  if (problem) throw new Error(problem);
363
549
  }
364
- const { scoreWithModel } = await import("./model-B45lDUV1.js");
365
- const scored = await withRetries(() => scoreWithModel(model, question, state, options.providerOptions, signal, gateway), retry, signal);
366
- ({score, resolvedModel} = scored.value);
550
+ const { scoreWithModel } = await import("./model-vRiUycWh.js");
551
+ const scored = await withRetries(() => scoreWithModel(model, redactSecrets(question), state, options.providerOptions, signal, gateway, fallbacks, vertex), retry, signal);
552
+ const { score: scoredAt, resolvedModel: served, ...billed } = scored.value;
553
+ score = scoredAt;
554
+ resolvedModel = served;
555
+ billing = billed;
367
556
  retries = scored.retries;
368
557
  }
369
558
  return {
@@ -374,12 +563,13 @@ async function runJudge(question, state, options, signal, retry = JUDGE_RETRY) {
374
563
  question,
375
564
  model: judgeModelName(model),
376
565
  ...resolvedModel ? { resolvedModel } : {},
377
- ...gateway && typeof model === "string" ? { gateway } : {}
566
+ ...gateway && typeof model === "string" ? { gateway } : {},
567
+ ...billing
378
568
  },
379
569
  retries
380
570
  };
381
571
  }
382
572
  //#endregion
383
- export { servedOnVercel as C, fxAskedModel as S, VERCEL_HOST as _, checkModelString as a, VERCEL_ROUTES as b, judgeModelName as c, runJudge as d, setGatewayKey as f, FX_VERCEL_ENDPOINTS as g, FX_VERCEL_CALL as h, RECOMMENDED_JUDGE_MODEL as i, judgeRoute as l, isApiError as m, DIRECT_PROVIDERS as n, isFakeJudge as o, unusedGatewayKey as p, RECOMMENDED_ANALYSIS_MODEL as r, judgeCredentials as s, DEFAULT_THRESHOLD as t, modelKeyOptions as u, VERCEL_KEY as v, vercelCredentialProblem as w, VERCEL_UNPROVEN_EFFORT as x, VERCEL_REFUSALS as y };
573
+ export { vercelCredentialProblem as C, servedOnVercel as S, OPENROUTER_KEY as T, FX_VERCEL_ENDPOINTS as _, RECOMMENDED_JUDGE_MODEL as a, VERCEL_KEY as b, judgeCredentials as c, modelKeyOptions as d, runJudge as f, FX_VERCEL_CALL as g, isApiError as h, RECOMMENDED_ANALYSIS_MODEL as i, judgeModelName as l, unusedGatewayKey as m, DIRECT_PROVIDERS as n, checkModelString as o, setGatewayKey as p, OPENROUTER_JUDGE_MODEL as r, isFakeJudge as s, DEFAULT_THRESHOLD as t, judgeRoute as u, VERCEL_GATEWAY as v, OPENROUTER_GATEWAY as w, fxAskedModel as x, VERCEL_HOST as y };
384
574
 
385
- //# sourceMappingURL=judge-yLvjC5A3.js.map
575
+ //# sourceMappingURL=judge-CKdgEXN7.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"judge-CKdgEXN7.js","names":["claudeCode","codex","goose","opencode","pi","grokBuild","fx","sleep"],"sources":["../src/gateway/effort.ts","../src/gateway/openrouter.ts","../src/gateway/vercel.ts","../src/judge/retry.ts","../src/judge/index.ts"],"mappings":";;;;;;AAGA,MAAa,mBAAmB;;AAGhC,MAAa,aAAa,UAAkB,CAAC,wBAAwB,KAAK,KAAK;;;ACS/E,MAAa,kBAAkB;AAC/B,MAAa,iBAAiB;AAE9B,MAAM,gBAAgB,MAAc,SAAyB,YAC1D;CACC,MAAM;CACN,MAAM,EAAE,OAAO,KAAK;CACpB,QAAQ,CAAC,MAAM;AACjB;;AAGF,MAAa,sBAAmC,EAAE,MAAM,eAAe,qBACrE,SAAA,mBAA4B,gBACxB,GAAG,gBAAgB,iBAAiB,WAAW,eAAe,KAAK,OACnE,KAAA;;AAGN,MAAa,wBAAkC,EAAE,MAAM,kBAAkB,mBACvE,SAAA,mBAA4B,mBACxB;CACE,UAAU;CACV,GAAI,iBAAiB,KAAA,IAAY,EAAE,SAAS,aAAa,IAAI,CAAC;AAChE,IACA,KAAA;;AAGN,MAAM,SAAS,WAAqC,WAAqC;CACvF,YAAY;CACZ,QAAQ;CACR;CACA,gBAAgB;CAChB,aAAa;CACb,UAAU;CACV,OAAO,CAAC,eAAe;AACzB;;AAGA,MAAM,SAAS,cAAsB,UACnC,MAAM,SAAS,GAAG,IAAI,QAAQ,GAAG,SAAS,GAAG;;;;;AAM/C,MAAa,yBAAyB,UACpC,MAAM,SAAS,GAAG,IAAI,QAAQ,aAAa,MAAM,QAAQ,yBAAyB,QAAQ;;;;;;AAO5F,SAASA,eAA2B;CAClC,MAAM,QAAQ,CAAC,aAAa,kBAAkB,CAAC;CAC/C,OAAO;EACL,GAAG,MAAM,CAAC,GAAG,OAAO,aAAa,+BAA+B,CAAC,GAAG,KAAK;EACzE,QAAQ;EACR,SAAS,EAAE,oBAAoB,WAAW,gBAAgB,MAAM;EAChE,SAAS;CACX;AACF;;AAGA,SAASC,UAAsB;CAC7B,MAAM,QAAQ,CAAC,aAAa,mBAAmB,CAAC;CAChD,OAAO;EAAE,GAAG,MAAM,OAAO,KAAK;EAAG,SAAS,MAAM,QAAQ;CAAE;AAC5D;;;;AAKA,SAASC,UAAsB;CAC7B,MAAM,QAAQ,CAAC,aAAa,0BAA0B,CAAC;CACvD,OAAO;EACL,GAAG,MAAM,CAAC,GAAG,OAAO,aAAa,kBAAkB,KAAK,CAAC,GAAG,KAAK;EACjE,SAAS,MAAM,QAAQ;CACzB;AACF;;;;;;AAOA,SAASC,aAAyB;CAChC,MAAM,WAAW,wBAAwB,kBAAkB;CAC3D,IAAI,WAAW,UAAU,MAAM,IAAI,MAAM,SAAS,KAAK;CACvD,MAAM,EAAE,aAAa;CACrB,OAAO;EACL,YAAY;EACZ,QAAQ;EACR,WAAW,kBAAkB,QAAQ;EACrC,gBAAgB,uBAAuB,QAAQ;EAC/C,cAAc,YAAY,oBAAoB,UAAU,OAAO;EAC/D,UAAU;EACV,OAAO,CAAC,GAAG,cAAc,QAAQ,GAAG,YAAY;EAChD,UAAU,UAAU,cAAc;CACpC;AACF;;;;;;AAOA,SAASC,OAAmB;CAC1B,MAAM,QAAQ,CAAC,aAAa,0BAA0B,CAAC;CACvD,OAAO;EAAE,GAAG,MAAM,OAAO,KAAK;EAAG,SAAS,MAAM,QAAQ;CAAE;AAC5D;;;;;AAMA,SAASC,cAA0B;CACjC,MAAM,QAAQ,CAAC,aAAa,0BAA0B,CAAC;CACvD,OAAO;EAAE,GAAG,MAAM,OAAO,KAAK;EAAG,SAAS,MAAM,MAAM;CAAE;AAC1D;;;;;AAMA,SAASC,OAAmB;CAC1B,MAAM,QAAQ,CAAC,aAAa,0BAA0B,CAAC;CACvD,OAAO;EAAE,GAAG,MAAM,OAAO,KAAK;EAAG,SAAS,MAAM,QAAQ;CAAE;AAC5D;;AAGA,MAAa,qBAAqC;CAChD,MAAM;CACN,QAAQ;EACN,eAAeN;EACf,OAAA;EACA,OAAA;EACA,UAAA;EACA,IAAA;EACA,cAAcK;EACd,IAAA;CACF;CACA,UAAU;EACR,QAAQ;EACR,aACE;CACJ;CAKA,gBAAgB;EAAE,eAAe;EAAY,OAAO;EAAW,UAAU;CAAU;AACrF;;;ACvJA,MAAa,cAAc;;AAG3B,MAAa,aAAa;;AAE1B,MAAa,cAAc;;AAG3B,MAAa,oBAAoB,MAAyB,QAAQ,QAChE,CAAC,IAAA,yBAAmB,IAAA,uBAAmB,cAAc;;AAGvD,SAAgB,UAAU,OAAiC;CACzD,MAAM,GAAG,WAAW,MAAM,MAAM,GAAG;CACnC,IAAI,CAAC,SAAS,OAAO,KAAA;CACrB,IAAI;EACF,MAAM,EAAE,QAAQ,KAAK,MAAM,OAAO,KAAK,SAAS,WAAW,CAAC,CAAC,SAAS,MAAM,CAAC;EAG7E,OAAO,OAAO,QAAQ,2BAAW,IAAI,KAAK,MAAM,GAAI,IAAI,KAAA;CAC1D,QAAQ;EACN;CACF;AACF;;;;;AAMA,SAAgB,wBACd,MAAyB,QAAQ,KACjC,sBAAM,IAAI,KAAK,GACK;CACpB,IAAI,iBAAiB,GAAG,MAAA,qBAAmB,OAAO,KAAA;CAClD,MAAM,SAAS,UAAU,IAAA,wBAAoB,EAAE;CAC/C,IAAI,CAAC,UAAU,SAAS,KAAK,OAAO,KAAA;CACpC,OAAO,GAAG,YAAY,cAAc,OAAO,YAAY,EAAE,8EAA8E,WAAW;AACpJ;AAEA,MAAM,YAAY,MAAc,YAC7B;CAAE,MAAM;CAAa,MAAM,EAAE,OAAO,KAAK;CAAG,QAAQ,CAAC,MAAM;AAAE;;;;;AAMhE,MAAa,kBAAgD,EAC3D,MACA,eACA,qBAEA,SAAA,0BAAwB,gBACpB,GAAG,gBAAgB,iBAAiB,WAAW,eAAe,KAAK,OACnE,KAAA;;;;;;AAON,SAAS,WAAW,KAAsC;CACxD,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR,WAAW,CAAC,SAAS,gBAAgB,MAAM,GAAG,SAAS,6BAA6B,MAAM,CAAC;EAC3F,OAAO,CAAC,WAAW;CACrB;AACF;;AAGA,SAAS,MAAM,KAAsC;CACnD,MAAM,YAAY,CAAC,SAAS,iBAAiB,MAAM,CAAC;CACpD,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR;EACA,gBAAgB;EAChB,aAAa;EACb,OAAO,CAAC,WAAW;EACnB,UAAU,UAAW,MAAM,SAAS,GAAG,IAAI,QAAQ,UAAU;CAC/D;AACF;;AAGA,MAAa,iBAAiB;CAC5B,MAAM;CACN,MAAM,EAAE,OAAO,6BAA6B;CAC5C,QAAQ,CAAC,MAAM;AACjB;;;;;AAMA,MAAa,gBAA8C,YAAY;CACrE,IAAI,QAAQ,SAAA,0BAAwB,CAAC,IAAI,OAAO,eAAe,KAAK,KAAK,CAAC,CAAC,KAAK,QAAQ,IAAI,GAC1F,OAAO,KAAA;CACT,IAAI,QAAQ,WAAW,KAAA,KAAa,QAAQ,SAAS,OAAO,QAAQ,SAAS,KAAK,OAAO,KAAA;CACzF,MAAM,QAAQ,QAAQ,UAAU;CAChC,QAAQ,OAAO,UAAU,WAAW,QAAQ,QAAQ,OAAO,KAAA;AAC7D;;AAGA,MAAa,sBAAsB,CACjC;CAAE,MAAM;CAAa,MAAM,EAAE,OAAO,qBAAqB;CAAG,QAAQ,CAAC,KAAK;AAAE,GAC5E,cACF;;AAGA,SAAS,GAAG,KAAsC;CAChD,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR,WAAW;EACX,gBAAgB,CAAC,cAAc;EAC/B,aAAa;EACb,YAAY;EACZ,OAAO,CAAC,WAAW;EACnB,UAAU,UAAW,MAAM,SAAS,GAAG,IAAI,QAAQ,UAAU;CAC/D;AACF;;AAGA,MAAM,YAAY,UAAmB,MAAM,SAAS,GAAG,IAAI,QAAQ,UAAU;;;;;AAM7E,SAAS,MAAM,KAAsC;CACnD,MAAM,QAAQ,CAAC,SAAS,wBAAwB,MAAM,GAAG,SAAS,iBAAiB,MAAM,CAAC;CAC1F,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR,WAAW,CAAC,GAAG,OAAO,SAAS,cAAc,KAAK,CAAC;EACnD,gBAAgB;EAChB,aAAa;EACb,OAAO,CAAC,WAAW;EACnB,SAAS;CACX;AACF;;;;;;AAOA,SAAS,GAAG,KAAsC;CAChD,MAAM,YAAY,CAAC,SAAS,wBAAwB,MAAM,CAAC;CAC3D,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR;EACA,gBAAgB;EAChB,aAAa;EACb,OAAO,CAAC,WAAW;EACnB,SAAS;CACX;AACF;;;;;;AAOA,SAAS,SAAS,KAAsC;CACtD,MAAM,WAAW,wBAAwB,cAAc;CACvD,IAAI,WAAW,UAAU,MAAM,IAAI,MAAM,SAAS,KAAK;CACvD,MAAM,EAAE,aAAa;CACrB,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR,WAAW,kBAAkB,QAAQ;EACrC,gBAAgB,uBAAuB,QAAQ;EAC/C,cAAc,YAAY,oBAAoB,UAAU,OAAO;EAC/D,OAAO,CAAC,GAAG,cAAc,QAAQ,GAAG,YAAY;EAChD,UAAU,UAAU,UAAU;CAChC;AACF;;;;;AAMA,SAAS,UAAU,KAAsC;CACvD,MAAM,YAAY,CAAC,SAAS,wBAAwB,MAAM,CAAC;CAC3D,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR;EACA,gBAAgB;EAChB,aAAa;EACb,OAAO,CAAC,WAAW;EAEnB,iBAAiB;GACf,OAAO,CAAC,YAAY;GACpB,KAAK;EACP;EACA,UAAU,UAAW,MAAM,SAAS,GAAG,IAAI,QAAQ,OAAO;CAC5D;AACF;;AAGA,MAAa,iBAAiC;CAC5C,MAAM;CACN,QAAQ;EACN,eAAe;EACf;EACA;EACA;EACA;EACA,cAAc;EACd;CACF;CACA,UAAU;EACR,QAAQ;EACR,aAAa;CACf;CAIA,gBAAgB;EAAE,OAAO;EAAY,UAAU;CAAU;AAC3D;;;;ACrOA,MAAa,cAA2B;CAAE,UAAU;CAAG,QAAQ;CAAO,OAAO;AAAO;AAEpF,MAAM,mCAAmB,IAAI,IAAI;CAAC;CAAK;CAAK;CAAK;AAAG,CAAC;AACrD,MAAM,oBACJ;;AAUF,MAAM,aAAa,UAA4B;CAC7C,MAAM,IAAI;CACV,OAAO,GAAG,SAAS,mBAAmB,EAAE,YAAY,UAAU,EAAE,SAAS,IAAI;AAC/E;;AAGA,MAAa,cAAc,UACzB,OAAQ,UAAU,KAAK,CAAC,EAAoB,eAAe;;AAG7D,SAAgB,YAAY,OAAyB;CACnD,MAAM,IAAI,UAAU,KAAK;CACzB,IAAI,CAAC,KAAK,EAAE,SAAS,gBAAgB,EAAE,SAAS,gBAAgB,OAAO;CACvE,IAAI,OAAO,EAAE,eAAe,UAC1B,OAAO,EAAE,cAAc,OAAO,iBAAiB,IAAI,EAAE,UAAU,KAAK,EAAE,gBAAgB;CAExF,OAAO,EAAE,gBAAgB,QAAQ,kBAAkB,KAAK,EAAE,WAAW,OAAO,CAAC,CAAC;AAChF;AAEA,SAAS,aAAa,OAAwB;CAC5C,MAAM,SAAU,UAAU,KAAK,CAAC,EAAoB,kBAAkB;CACtE,MAAM,UAAU,OAAO,MAAM;CAC7B,OAAO,OAAO,SAAS,OAAO,KAAK,UAAU,IAAI,UAAU,MAAO;AACpE;;AAGA,SAAgB,UAAU,QAAqB,OAAe,OAAwB;CACpF,MAAM,MAAM,KAAK,IAAI,OAAO,OAAO,OAAO,SAAS,KAAK,KAAK;CAC7D,MAAM,WAAW,MAAM,IAAI,KAAK,OAAO,KAAK,MAAM;CAClD,OAAO,KAAK,MAAM,KAAK,IAAI,OAAO,OAAO,KAAK,IAAI,UAAU,aAAa,KAAK,CAAC,CAAC,CAAC;AACnF;;AAKA,eAAsB,YACpB,MACA,QACA,QAC8C;CAC9C,MAAM,UAAwB,CAAC;CAC/B,SACE,IAAI;EACF,OAAO;GAAE,OAAO,MAAM,KAAK;GAAG;EAAQ;CACxC,SAAS,OAAO;EACd,IAAI,OAAO,WAAW,QAAQ,SAAS,KAAK,OAAO,YAAY,CAAC,YAAY,KAAK,GAAG;GAClF,MAAM,SAAU,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;GACxE,OAAO,UAAU;GACjB,MAAM;EACR;EACA,MAAM,UAAU,UAAU,QAAQ,QAAQ,QAAQ,KAAK;EACvD,MAAM,IAAI,UAAU,KAAK;EACzB,QAAQ,KAAK;GAAE,OAAO,GAAG,WAAW,OAAO,CAAC;GAAG;EAAQ,CAAC;EACxD,MAAME,WAAM,SAAS,KAAA,GAAW,EAAE,OAAO,CAAC;CAC5C;AAEJ;;;;AC5DA,MAAa,0BAA0B;;AAEvC,MAAa,yBAAyB;;AAEtC,MAAa,6BAA6B;AAC1C,MAAa,oBAAoB;;AAKjC,MAAM,cAAuE;CAC3E,QAAQ,CAJY,YAAY,WAIb;CACnB,YAAY,CAAC,cAAc;AAC7B;;;;;AAMA,MAAa,mBAA+E;CAC1F,WAAW;EAAE,KAAK;EAAqB,KAAK;EAAqB,MAAM;CAAY;CACnF,QAAQ;EAAE,KAAK;EAAgC,KAAK;EAAkB,MAAM;CAAS;CACrF,QAAQ;EAAE,KAAK;EAAkB,KAAK;EAAkB,MAAM;CAAS;CACvE,eAAe;EAAE,KAAK;EAAuB,KAAK;EAAuB,MAAM;CAAc;AAC/F;AAcA,MAAa,eAAe,MAC1B,OAAO,MAAM,YAAY,MAAM,QAAS,EAAqB,SAAS;AAExE,SAAgB,eAAe,OAA2B;CACxD,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,IAAI,YAAY,KAAK,GAAG,OAAO;CAC/B,MAAM,IAAI;CACV,OAAO,EAAE,YAAY,EAAE,UAAU,GAAG,EAAE,SAAS,GAAG,EAAE,YAAY;AAClE;;;;;AAMA,SAAgB,iBACd,OACA,OAAO,mBACP,WAAW,CAAC,yBAAyB,UAAU,OAAO,GAClC;CACpB,IAAI,MAAM,SAAS,GAAG,GAAG,OAAO,KAAA;CAChC,OAAO,GAAG,KAAK,iCAAiC,SAAS,KAAK,MAAM,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,EAAE,SAAS,MAAM;AAC5G;AAEA,MAAM,cAAc,UAAkB,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC;;;;;;;AAQvE,SAAgB,iBACd,OACA,UAAuB,UACb;CACV,IAAI,YAAY,KAAK,KAAK,YAAY,UAAU,OAAO,CAAC;CACxD,IAAI,YAAY,UAAU,OAAO,CAAC,GAAG,YAAY,QAAQ;CACzD,MAAM,SAAS,iBAAiB,WAAW,KAAK;CAChD,OAAO,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC;AAClC;;AAGA,SAAgB,gBAAgB,OAAe,KAAkB,UAAkB;CACjF,IAAI,OAAO,UAAU,OAAO;CAC5B,IAAI,OAAO,UAAU,OAAO,GAAG,YAAY,GAAG,CAAC,KAAK,MAAM,EAAE,QAAQ,GAAG;CACvE,MAAM,SAAS,iBAAiB,WAAW,KAAK;CAChD,OAAO,SACH,GAAG,OAAO,IAAI,mBACd,4CAA4C,OAAO,KAAK,gBAAgB,CAAC,CACtE,KAAK,MAAM,GAAG,EAAE,GAAG,CAAC,CACpB,KAAK,IAAI,EAAE;AACpB;;AAGA,SAAgB,cACd,KAC0D;CAC1D,KAAK,MAAM,CAAC,SAAS,SAAS,OAAO,QAAQ,WAAW,GAAG;EACzD,MAAM,MAAM,KAAK,MAAM,MAAM,IAAI,EAAE;EACnC,IAAI,KAAK,OAAO;GAAE;GAAc;EAA8B;CAChE;AAEF;;;;;AAMA,SAAgB,iBAAiB,KAAwB,UAAuB,UAAkB;CAChG,IAAI,YAAY,UAAU,OAAO;CACjC,MAAM,MAAM,cAAc,GAAG;CAC7B,IAAI,CAAC,KAAK,OAAO;CACjB,MAAM,OAAO,IAAI,YAAY,WAAW,sBAAsB;CAC9D,OAAO,IAAI,IAAI,IAAI,qDAAqD,IAAI,QAAQ,0CAA0C,KAAK;AACrI;;;;;;;;AASA,SAAgB,WACd,OACA,MAAyB,QAAQ,KACjC,OAAO,eACP,UAAuB,UACX;CACZ,IAAI,YAAY,UAAU,OAAO,EAAE,MAAM,SAAS;CAClD,MAAM,WAAW,WAAW,KAAK;CACjC,MAAM,SAAS,iBAAiB;CAChC,MAAM,OAAO,iBAAiB,OAAO,OAAO;CAC5C,IAAI,CAAC,KAAK,QACR,MAAM,IAAI,MACR,GAAG,KAAK,IAAI,MAAM,qEAAqE,OAAO,KAC5F,gBACF,CAAC,CACE,KAAK,MAAM,GAAG,EAAE,GAAG,CAAC,CACpB,KAAK,IAAI,EAAE,SAChB;CAEF,MAAM,MAAM,KAAK,MAAM,MAAM,IAAI,EAAE;CACnC,IAAI,QAAQ,KAAA,GACV,MAAM,IAAI,MACR,sBAAsB,KAAK,IAAI,MAAM,eAAe,SAAS,UAAU,gBAAgB,OAAO,OAAO,EAAE,qCAAqC,iBAAiB,KAAK,OAAO,GAC3K;CAEF,IAAI,QAAA,qBAAqB;EACvB,MAAM,UAAU,wBAAwB,GAAG;EAC3C,IAAI,SAAS,MAAM,IAAI,MAAM,GAAG,KAAK,IAAI,MAAM,KAAK,SAAS;CAC/D;CACA,MAAM,KAAK,eAAe,IAAI,IAAc;CAC5C,IAAI,IACF,MAAM,IAAI,MACR,GAAG,IAAI,OAAO,KAAK,IAAI,MAAM,sDAAsD,GAAG,sFACxF;CAEF,IAAI,YAAY,cAAc,OAAO,EAAE,MAAM,aAAa;CAC1D,OAAO,UAAU,QAAQ,OAAO,MAC5B;EAAE,MAAM;EAAU;EAAU,KAAK,OAAO;CAAI,IAC5C,EAAE,MAAM,UAAU;AACxB;;AAMA,eAAsB,SACpB,UACA,OACA,SACA,QACA,QAAqB,aACC;CACtB,MAAM,EAAE,OAAO,cAAc;CAC7B,MAAM,UAAU,OAAO,QAAQ,YAAY,WAAW,QAAQ,QAAQ,OAAO,QAAQ;CACrF,MAAM,YACJ,OAAO,QAAQ,YAAY,YAC3B,QAAQ,QAAQ,SAAS,gBACzB,QAAQ,QAAQ,cAAc;CAChC,MAAM,SACJ,YAAY,WACR,eACE,OAAO,QAAQ,YAAY,YAAY,QAAQ,QAAQ,SAAS,WAC5D,QAAQ,UACR,CAAC,CACP,IACA,KAAA;CACN,IAAI;CACJ,IAAI;CACJ,IAAI,UAAmD,CAAC;CACxD,IAAI,UAAwB,CAAC;CAC7B,IAAI,YAAY,KAAK,GACnB,QAAQ,OAAO,MAAM,UAAU,aAAa,MAAM,MAAM,KAAK,IAAI,MAAM;MAClE;EACL,IAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,UAAU,iBAAiB,KAAK;GACtC,IAAI,SAAS,MAAM,IAAI,MAAM,OAAO;EACtC;EACA,MAAM,EAAE,mBAAmB,MAAM,OAAO;EACxC,MAAM,SAAS,MAAM,kBAEjB,eACE,OACA,cAAc,QAAQ,GACtB,OACA,QAAQ,iBACR,QACA,SACA,WACA,MACF,GACF,OACA,MACF;EACA,MAAM,EAAE,OAAO,UAAU,eAAe,QAAQ,GAAG,WAAW,OAAO;EACrE,QAAQ;EACR,gBAAgB;EAChB,UAAU;EACV,UAAU,OAAO;CACnB;CAWA,OAAO;EAAE,SAAA;GATP,SAAS,SAAS,YAAY,SAAS;GACvC;GACA;GACA;GACA,OAAO,eAAe,KAAK;GAC3B,GAAI,gBAAgB,EAAE,cAAc,IAAI,CAAC;GACzC,GAAI,WAAW,OAAO,UAAU,WAAW,EAAE,QAAQ,IAAI,CAAC;GAC1D,GAAG;EAEU;EAAG;CAAQ;AAC5B"}
package/dist/judge.d.ts CHANGED
@@ -1,4 +1,5 @@
1
- import { $ as FakeJudgeModel, $t as Verdict, J as FakeAnalysisModel, Q as FakeInsightsInput, Y as FakeAnswer, Z as FakeInsights, _t as JudgeCallOptions, bt as JudgeOptions, c as BlindAttempt, gt as JudgeArtifactRef, k as ConfigJudgeOptions, l as BlindPair, n as AnalysisConfig, nt as FakeVote, vt as JudgeFailure, xt as JudgeState, yt as JudgeModel } from "./types-DXBbP9Mk.js";
1
+ import { $ as FakeVote, J as FakeAnswer, O as ConfigJudgeOptions, X as FakeJudgeModel, Zt as Verdict, _t as JudgeOptions, c as BlindPair, gt as JudgeModel, ht as JudgeFailure, mt as JudgeCallOptions, n as AnalysisConfig, pt as JudgeArtifactRef, q as FakeAnalysisModel, s as BlindAttempt, vt as JudgeState } from "./types-coMWHaU2.js";
2
+ import "./vertex-C7PI-WPc.js";
2
3
  //#region src/judge/index.d.ts
3
4
  /** The judge the docs recommend, pinned so a model update never silently moves scores. */
4
5
  export declare const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.14.0";
@@ -17,12 +18,9 @@ export declare const fakeJudge: ({ score }?: {
17
18
  * model's structured answer, each pairwise question's pair gets `vote` (a fixed `'A'`, `'B'` or
18
19
  * `'tie'`, `{ winner, reason? }`, a function of the blinded pair, or `{ error }`), and each
19
20
  * question asked of one attempt gets `answer` (its text, `{ choice?, text? }`, a function of the
20
- * blinded attempt, or `{ error }`), through the same parsing and validation as a real one. Insights (in `reeval run` and `reeval analyze --insights`)
21
- * get `insights` (or `insights(input)`, called per eval for its classification and its sentences,
22
- * then once for the headline), through the same validator: an uncited sentence is dropped, and one
23
- * with a banned word is asked for again, then dropped.
21
+ * blinded attempt, or `{ error }`), through the same parsing and validation as a real one.
24
22
  */
25
- export declare const fakeAnalysis: ({ output, vote, answer, insights }: Pick<FakeAnalysisModel, 'output' | 'vote' | 'answer' | 'insights'>) => FakeAnalysisModel;
23
+ export declare const fakeAnalysis: ({ output, vote, answer }: Pick<FakeAnalysisModel, 'output' | 'vote' | 'answer'>) => FakeAnalysisModel;
26
24
  //#endregion
27
- export type { AnalysisConfig, BlindAttempt, BlindPair, ConfigJudgeOptions, FakeAnalysisModel, FakeAnswer, FakeInsights, FakeInsightsInput, FakeJudgeModel, FakeVote, JudgeArtifactRef, JudgeCallOptions, JudgeFailure, JudgeModel, JudgeOptions, JudgeState, Verdict };
25
+ export type { AnalysisConfig, BlindAttempt, BlindPair, ConfigJudgeOptions, FakeAnalysisModel, FakeAnswer, FakeJudgeModel, FakeVote, JudgeArtifactRef, JudgeCallOptions, JudgeFailure, JudgeModel, JudgeOptions, JudgeState, Verdict };
28
26
  //# sourceMappingURL=judge.d.ts.map
package/dist/judge.js CHANGED
@@ -1,4 +1,4 @@
1
- import { i as RECOMMENDED_JUDGE_MODEL, t as DEFAULT_THRESHOLD } from "./judge-yLvjC5A3.js";
1
+ import { a as RECOMMENDED_JUDGE_MODEL, t as DEFAULT_THRESHOLD } from "./judge-CKdgEXN7.js";
2
2
  //#region src/judge.ts
3
3
  /**
4
4
  * A judge that returns `score` without calling a model, for tests and dry runs. `score` can be a
@@ -13,17 +13,13 @@ const fakeJudge = ({ score = 1 } = {}) => ({
13
13
  * model's structured answer, each pairwise question's pair gets `vote` (a fixed `'A'`, `'B'` or
14
14
  * `'tie'`, `{ winner, reason? }`, a function of the blinded pair, or `{ error }`), and each
15
15
  * question asked of one attempt gets `answer` (its text, `{ choice?, text? }`, a function of the
16
- * blinded attempt, or `{ error }`), through the same parsing and validation as a real one. Insights (in `reeval run` and `reeval analyze --insights`)
17
- * get `insights` (or `insights(input)`, called per eval for its classification and its sentences,
18
- * then once for the headline), through the same validator: an uncited sentence is dropped, and one
19
- * with a banned word is asked for again, then dropped.
16
+ * blinded attempt, or `{ error }`), through the same parsing and validation as a real one.
20
17
  */
21
- const fakeAnalysis = ({ output, vote, answer, insights }) => ({
18
+ const fakeAnalysis = ({ output, vote, answer }) => ({
22
19
  kind: "fake-analysis",
23
20
  ...output !== void 0 ? { output } : {},
24
21
  ...vote !== void 0 ? { vote } : {},
25
- ...answer !== void 0 ? { answer } : {},
26
- ...insights !== void 0 ? { insights } : {}
22
+ ...answer !== void 0 ? { answer } : {}
27
23
  });
28
24
  //#endregion
29
25
  export { DEFAULT_THRESHOLD, RECOMMENDED_JUDGE_MODEL, fakeAnalysis, fakeJudge };
package/dist/judge.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"judge.js","names":[],"sources":["../src/judge.ts"],"mappings":";;;;;;AAQA,MAAa,aAAa,EACxB,QAAQ,MAGN,CAAC,OAAuB;CAC1B,MAAM;CACN;AACF;;;;;;;;;;;AAYA,MAAa,gBAAgB,EAC3B,QACA,MACA,QACA,gBAC4F;CAC5F,MAAM;CACN,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;CACzC,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;CACrC,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;CACzC,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;AAC/C"}
1
+ {"version":3,"file":"judge.js","names":[],"sources":["../src/judge.ts"],"mappings":";;;;;;AAQA,MAAa,aAAa,EACxB,QAAQ,MAGN,CAAC,OAAuB;CAC1B,MAAM;CACN;AACF;;;;;;;;AASA,MAAa,gBAAgB,EAC3B,QACA,MACA,cAC+E;CAC/E,MAAM;CACN,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;CACzC,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;CACrC,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;AAC3C"}
@@ -1,5 +1,5 @@
1
1
  import { n as isOneOf } from "./object-CLf2-tNq.js";
2
- import { n as fillDefaults } from "./record-kk9M1hj-.js";
2
+ import { n as fillDefaults } from "./record-DZBCjuwm.js";
3
3
  //#region src/view/live.ts
4
4
  /** The progress file's name inside the run's staging directory. */
5
5
  const LIVE_FILE = "live.json";
@@ -84,4 +84,4 @@ function readableLiveProgress(value) {
84
84
  //#endregion
85
85
  export { liveApiPath as a, readableLiveProgress as c, isFinished as i, LIVE_FILE as n, liveCounts as o, LIVE_SCHEMA_VERSION as r, liveTranscriptPath as s, FINISHED as t };
86
86
 
87
- //# sourceMappingURL=live-DYd2QEyR.js.map
87
+ //# sourceMappingURL=live-XJQ6nqK4.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"live-DYd2QEyR.js","names":[],"sources":["../src/view/live.ts"],"mappings":";;;;AAQA,MAAa,YAAY;AACzB,MAAa,sBAAsB;AAsDnC,MAAa,cAAc;AAC3B,MAAa,sBAAsB,OAAe,eAChD,YAAY,MAAM,sBAAsB,mBAAmB,UAAU;AAEvE,MAAa,WAAkC;CAAC;CAAQ;CAAQ;CAAe;AAAS;AACxF,MAAa,cAAc,WAAuB,SAAS,SAAS,MAAM;AAE1E,SAAgB,WAAW,UAA8D;CACvF,MAAM,SAAS,GAAG,aAChB,SAAS,QAAQ,MAAM,SAAS,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;CACtD,OAAO;EACL,MAAM,MAAM,GAAG,QAAQ;EACvB,QAAQ,MAAM,MAAM;EACpB,QAAQ,MAAM,MAAM;EACpB,SAAS,MAAM,aAAa;EAC5B,SAAS,MAAM,SAAS;EACxB,SAAS,MAAM,SAAS;CAC1B;AACF;AAEA,MAAM,WAAkC;CAAC;CAAU;CAAW,GAAG;AAAQ;AACzE,MAAM,SAA+B;CAAC;CAAW;CAAc;AAAW;AAC1E,MAAM,SAAS,MACb,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC;AACzD,MAAM,OAAO,MAA4B,OAAO,MAAM;AACtD,MAAM,OAAO,MAA4B,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC;AACnF,MAAM,UAAa,GAAY,OAC7B,MAAM,QAAQ,MAAM,KAAA,KAAa,GAAG,CAAC;AAEvC,SAAS,gBAAgB,GAAgC;CACvD,IAAI,CAAC,MAAM,CAAC,GAAG,OAAO;CACtB,MAAM,EAAE,KAAK,QAAQ,MAAM,MAAM,MAAM,GAAG,QAAQ,WAAW,eAAe;CAC5E,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO;CAC3F,IAAI,CAAC,QAAQ,UAAU,MAAM,GAAG,OAAO;CACvC,IAAI,CAAC,OAAO,WAAW,GAAG,KAAK,CAAC,OAAO,YAAY,GAAG,GAAG,OAAO;CAChE,OAAO;EACL;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW,aAAa;EACxB,YAAY,cAAc;CAC5B;AACF;;;;;AAMA,SAAgB,qBAAqB,OAAqC;CACxE,IAAI,CAAC,MAAM,KAAK,KAAK,MAAM,kBAAA,GAAuC,OAAO;CACzE,aAAa,iBAAiB,KAAK;CACnC,MAAM,EAAE,OAAO,WAAW,KAAK,MAAM,aAAa,OAAO,aAAa;CACtE,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,SAAS,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,WAAW,KAAK,CAAC,IAAI,GAAG,GAAG,OAAO;CAC3F,IAAI,CAAC,QAAQ,QAAQ,KAAK,KAAK,CAAC,MAAM,QAAQ,QAAQ,GAAG,OAAO;CAChE,MAAM,OAAO,SAAS,IAAI,eAAe;CACzC,IAAI,CAAC,KAAK,OAAO,MAAM,MAAM,IAAI,GAAG,OAAO;CAC3C,OAAO;EACL,eAAA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,SAAS,KAAK;EACd,QAAQ,WAAW,IAAI;EACvB,UAAU;CACZ;AACF"}
1
+ {"version":3,"file":"live-XJQ6nqK4.js","names":[],"sources":["../src/view/live.ts"],"mappings":";;;;AAQA,MAAa,YAAY;AACzB,MAAa,sBAAsB;AAsDnC,MAAa,cAAc;AAC3B,MAAa,sBAAsB,OAAe,eAChD,YAAY,MAAM,sBAAsB,mBAAmB,UAAU;AAEvE,MAAa,WAAkC;CAAC;CAAQ;CAAQ;CAAe;AAAS;AACxF,MAAa,cAAc,WAAuB,SAAS,SAAS,MAAM;AAE1E,SAAgB,WAAW,UAA8D;CACvF,MAAM,SAAS,GAAG,aAChB,SAAS,QAAQ,MAAM,SAAS,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;CACtD,OAAO;EACL,MAAM,MAAM,GAAG,QAAQ;EACvB,QAAQ,MAAM,MAAM;EACpB,QAAQ,MAAM,MAAM;EACpB,SAAS,MAAM,aAAa;EAC5B,SAAS,MAAM,SAAS;EACxB,SAAS,MAAM,SAAS;CAC1B;AACF;AAEA,MAAM,WAAkC;CAAC;CAAU;CAAW,GAAG;AAAQ;AACzE,MAAM,SAA+B;CAAC;CAAW;CAAc;AAAW;AAC1E,MAAM,SAAS,MACb,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC;AACzD,MAAM,OAAO,MAA4B,OAAO,MAAM;AACtD,MAAM,OAAO,MAA4B,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC;AACnF,MAAM,UAAa,GAAY,OAC7B,MAAM,QAAQ,MAAM,KAAA,KAAa,GAAG,CAAC;AAEvC,SAAS,gBAAgB,GAAgC;CACvD,IAAI,CAAC,MAAM,CAAC,GAAG,OAAO;CACtB,MAAM,EAAE,KAAK,QAAQ,MAAM,MAAM,MAAM,GAAG,QAAQ,WAAW,eAAe;CAC5E,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO;CAC3F,IAAI,CAAC,QAAQ,UAAU,MAAM,GAAG,OAAO;CACvC,IAAI,CAAC,OAAO,WAAW,GAAG,KAAK,CAAC,OAAO,YAAY,GAAG,GAAG,OAAO;CAChE,OAAO;EACL;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW,aAAa;EACxB,YAAY,cAAc;CAC5B;AACF;;;;;AAMA,SAAgB,qBAAqB,OAAqC;CACxE,IAAI,CAAC,MAAM,KAAK,KAAK,MAAM,kBAAA,GAAuC,OAAO;CACzE,aAAa,iBAAiB,KAAK;CACnC,MAAM,EAAE,OAAO,WAAW,KAAK,MAAM,aAAa,OAAO,aAAa;CACtE,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,SAAS,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,WAAW,KAAK,CAAC,IAAI,GAAG,GAAG,OAAO;CAC3F,IAAI,CAAC,QAAQ,QAAQ,KAAK,KAAK,CAAC,MAAM,QAAQ,QAAQ,GAAG,OAAO;CAChE,MAAM,OAAO,SAAS,IAAI,eAAe;CACzC,IAAI,CAAC,KAAK,OAAO,MAAM,MAAM,IAAI,GAAG,OAAO;CAC3C,OAAO;EACL,eAAA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,SAAS,KAAK;EACd,QAAQ,WAAW,IAAI;EACvB,UAAU;CACZ;AACF"}
@@ -1,5 +1,5 @@
1
1
  import { t as isObjectLike } from "./object-CLf2-tNq.js";
2
- import { a as calledMcp, c as changed, d as ran, f as searched, l as edited, o as calledSkill, s as calledTool, u as fetched } from "./client-D5hNjpEx.js";
2
+ import { a as calledMcp, c as changed, d as ran, f as searched, l as edited, o as calledSkill, s as calledTool, u as fetched } from "./client-Ct6s6OnF.js";
3
3
  import { isDeepStrictEqual } from "node:util";
4
4
  //#region src/matchers.ts
5
5
  const matcher = (run) => run;
@@ -115,4 +115,4 @@ const evalMatchers = Object.fromEntries(Object.keys(matchers).filter((name) => !
115
115
  //#endregion
116
116
  export { runMatcher as i, evalMatchers as n, isMatcherName as r, MATCHERS as t };
117
117
 
118
- //# sourceMappingURL=matchers-D4eUqx4j.js.map
118
+ //# sourceMappingURL=matchers-DT2TaGix.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"matchers-D4eUqx4j.js","names":[],"sources":["../src/matchers.ts"],"mappings":";;;;AA+BA,MAAM,WAAoE,QAAmB;;AAG7F,MAAM,QAAQ,GAAW,QAAQ,IAAI,QAAQ,QAAS;CACpD,MAAM,OAAO,EAAE,QAAQ,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI;CAC5D,OAAO,KAAK,UAAU,QAClB,OACA,MAAM,KAAK,SAAS,MAAM,sBAAsB,KAAK,MAAM,CAAC,KAAK;AACvE;AAIA,MAAM,YAAY,UAChB,aAAa,KAAK,KAAK,OAAO,MAAM,aAAa;AAEnD,MAAM,QAAQ,UAA2B;CACvC,IAAI,OAAO,UAAU,UAAU,OAAO,IAAI,MAAM;CAChD,IAAI;EACF,OAAO,KAAK,UAAU,KAAK,KAAK,OAAO,KAAK;CAC9C,QAAQ;EACN,OAAO,OAAO,KAAK;CACrB;AACF;AAEA,MAAa,WAAW;CACtB,MAAM,SAAS,QAAQ,CAAC,cAAyB;EAC/C,MAAM,OAAO,OAAO,GAAG,QAAQ,QAAQ;EACvC,OAAO;GACL;GACA,eACE,OACI,YAAY,KAAK,MAAM,EAAE,aAAa,KAAK,QAAQ,EAAE,0BACrD,YAAY,KAAK,MAAM,EAAE,SAAS,KAAK,QAAQ,EAAE;EACzD;CACF,CAAC;CAED,SAAS,SAAS,QAAQ,CAAC,cAAyB;EAClD,MAAM,OAAO,kBAAkB,QAAQ,QAAQ;EAC/C,OAAO;GACL;GACA,eACE,OACI,YAAY,KAAK,MAAM,EAAE,uBAAuB,KAAK,QAAQ,MAC7D,YAAY,KAAK,MAAM,EAAE,mBAAmB,KAAK,QAAQ;EACjE;CACF,CAAC;CAED,UAAU,SAAS,QAAQ,OAAW;EACpC,MAAM,WAAW;EACjB,eACE,WAAW,OACP,YAAY,KAAK,MAAM,EAAE,mBACzB,YAAY,KAAK,MAAM,EAAE;CACjC,EAAE;CAEF,YAAY,SAAS,QAAQ,OAAW;EACtC,MAAM,CAAC,CAAC;EACR,eACE,SAAS,YAAY,KAAK,MAAM,EAAE,gBAAgB,YAAY,KAAK,MAAM,EAAE;CAC/E,EAAE;CAEF,WAAW,SAAS,QAAQ,CAAC,YAAuB;EAClD,MAAM,OACJ,OAAO,WAAW,YAAY,OAAO,WAAW,WAC5C,OAAO,SAAS,MAAM,IACtB,MAAM,QAAQ,MAAM,KAAK,OAAO,SAAS,MAAM;EACrD,OAAO;GACL;GACA,eACE,OACI,YAAY,KAAK,MAAM,EAAE,kBAAkB,KAAK,MAAM,MACtD,YAAY,KAAK,MAAM,EAAE,cAAc,KAAK,MAAM;EAC1D;CACF,CAAC;CAED,YAAY,QACV,OAAO,QAAQ,CAAC,MAAM,aAAwD;EAC5E,IAAI,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI,UAAU,iDAAiD;EAC5F,MAAM,UAAU,MAAM,OAAO,SAAS,IAAI;EAC1C,IAAI,YAAY,MAAM,OAAO;GAAE,MAAM;GAAO,eAAe,iBAAiB,KAAK;EAAW;EAC5F,IAAI,YAAY,KAAA,GAAW,OAAO;GAAE,MAAM;GAAM,eAAe,uBAAuB;EAAO;EAC7F,MAAM,OAAO,OAAO,YAAY,WAAW,QAAQ,SAAS,OAAO,IAAI,QAAQ,KAAK,OAAO;EAC3F,OAAO;GACL;GACA,eACE,YAAY,KAAK,GAAG,OAAO,SAAS,GAAG,WAAW,QAAQ,QAAQ,KAAK,QAAQ,KAAK,OAAO;EAC/F;CACF,CACF;CAEA,WAAW,QAAQ,OAAO,QAAQ,MAAU;EAC1C,MAAM,IAAK,MAAM;EACjB,IAAI,OAAO,GAAG,aAAa,UAAU,MAAM,IAAI,UAAU,mCAAmC;EAC5F,OAAO;GACL,MAAM,EAAE,aAAa;GACrB,eACE,EAAE,aAAa,IACX,6BACA,0CAA0C,EAAE,SAAS,oBAClC,KAAK,EAAE,MAAM,EAAE,oBAAoB,KAAK,EAAE,MAAM;EAC3E;CACF,CAAC;CAED,gBAAgB,SAAS,QAAQ,CAAC,WAAsC,SAAS,QAAQ,KAAK,CAAC;CAE/F,eAAe,SAAS,QAAQ,CAAC,SAAkC,QAAQ,QAAQ,GAAG,CAAC;CAEvF,mBAAmB,SAAS,QAAQ,CAAC,UAA0B,YAAY,QAAQ,IAAI,CAAC;CAExF,iBAAiB,SACd,QAAQ,CAAC,QAAQ,MAAM,WACtB,UAAU,QAAQ,QAAQ,MAAM,KAAK,CACzC;CAEA,kBAAkB,SACf,QAAQ,CAAC,MAAM,WACd,WAAW,QAAQ,MAAM,KAAK,CAClC;CAEA,WAAW,SAAS,QAAQ,CAAC,aAA0C,IAAI,QAAQ,OAAO,CAAC;CAE3F,cAAc,SAAS,QAAQ,CAAC,UAAoC,OAAO,QAAQ,IAAI,CAAC;CAExF,eAAe,SAAS,QAAQ,CAAC,WAAgC,QAAQ,QAAQ,KAAK,CAAC;CAEvF,QAAQ,SAAS,QAAQ,GAAO,EAAE,UAAkB;EAClD,MAAM,UAAU;EAChB,IAAI,SAAS,YAAY,WAAW,OAAO,QAAQ,UAAU,UAE3D,OAAO;GACL,MAAM;GACN,eACE,qBAAqB,QAAQ,SAAS,KAAK,QAAQ,MAAM,KAAK,QAAQ;EAC1E;EAEF,MAAM,IAAI;EACV,IAAI,OAAO,GAAG,UAAU,UAAU,MAAM,IAAI,UAAU,sCAAsC;EAC5F,MAAM,QAAQ,EAAE,MAAM,QAAQ,CAAC;EAC/B,OAAO;GACL,MAAM,EAAE,YAAY;GACpB,eACE,EAAE,YAAY,SACV,iCAAiC,MAAM,KAAK,EAAE,UAAU,QAAQ,EAAE,SAAS,KAC3E,eAAe,MAAM,YAAY,EAAE,UAAU,QAAQ,EAAE,SAAS,KAAK,EAAE,MAAM;EACrF;CACF,CAAC;AACH;;AAKA,MAAM,mCAAmB,IAAI,IAAiB;CAC5C;CACA;CACA;CACA;CACA;AACF,CAAC;AAKD,MAAa,iBAAiB,SAAsC,OAAO,OAAO,UAAU,IAAI;;AAGhG,MAAa,cACX,MACA,QACA,MACA,UAEC,SAAS,KAAK,CAA8C,QAAQ,MAAM,KAAK;;AASlF,MAAa,WAAW,OAAO,YAC5B,OAAO,KAAK,QAAQ,CAAC,CAAmB,KAAK,SAAS,CACrD,MACA,SAAiD,QAAiB,GAAG,MAAiB;CACpF,OAAO,WAAW,MAAM,QAAQ,MAAM,EAAE,KAAK,CAAC,CAAC,MAAM,MAAM,CAAC;AAC9D,CACF,CAAC,CACH;;AAGA,MAAa,eAAe,OAAO,YAChC,OAAO,KAAK,QAAQ,CAAC,CACnB,QAAQ,SAAS,CAAC,iBAAiB,IAAI,IAAI,CAAC,CAAC,CAC7C,KAAK,SAAS,CAAC,MAAM,SAAS,KAAK,CAAC,CACzC"}
1
+ {"version":3,"file":"matchers-DT2TaGix.js","names":[],"sources":["../src/matchers.ts"],"mappings":";;;;AA+BA,MAAM,WAAoE,QAAmB;;AAG7F,MAAM,QAAQ,GAAW,QAAQ,IAAI,QAAQ,QAAS;CACpD,MAAM,OAAO,EAAE,QAAQ,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI;CAC5D,OAAO,KAAK,UAAU,QAClB,OACA,MAAM,KAAK,SAAS,MAAM,sBAAsB,KAAK,MAAM,CAAC,KAAK;AACvE;AAIA,MAAM,YAAY,UAChB,aAAa,KAAK,KAAK,OAAO,MAAM,aAAa;AAEnD,MAAM,QAAQ,UAA2B;CACvC,IAAI,OAAO,UAAU,UAAU,OAAO,IAAI,MAAM;CAChD,IAAI;EACF,OAAO,KAAK,UAAU,KAAK,KAAK,OAAO,KAAK;CAC9C,QAAQ;EACN,OAAO,OAAO,KAAK;CACrB;AACF;AAEA,MAAa,WAAW;CACtB,MAAM,SAAS,QAAQ,CAAC,cAAyB;EAC/C,MAAM,OAAO,OAAO,GAAG,QAAQ,QAAQ;EACvC,OAAO;GACL;GACA,eACE,OACI,YAAY,KAAK,MAAM,EAAE,aAAa,KAAK,QAAQ,EAAE,0BACrD,YAAY,KAAK,MAAM,EAAE,SAAS,KAAK,QAAQ,EAAE;EACzD;CACF,CAAC;CAED,SAAS,SAAS,QAAQ,CAAC,cAAyB;EAClD,MAAM,OAAO,kBAAkB,QAAQ,QAAQ;EAC/C,OAAO;GACL;GACA,eACE,OACI,YAAY,KAAK,MAAM,EAAE,uBAAuB,KAAK,QAAQ,MAC7D,YAAY,KAAK,MAAM,EAAE,mBAAmB,KAAK,QAAQ;EACjE;CACF,CAAC;CAED,UAAU,SAAS,QAAQ,OAAW;EACpC,MAAM,WAAW;EACjB,eACE,WAAW,OACP,YAAY,KAAK,MAAM,EAAE,mBACzB,YAAY,KAAK,MAAM,EAAE;CACjC,EAAE;CAEF,YAAY,SAAS,QAAQ,OAAW;EACtC,MAAM,CAAC,CAAC;EACR,eACE,SAAS,YAAY,KAAK,MAAM,EAAE,gBAAgB,YAAY,KAAK,MAAM,EAAE;CAC/E,EAAE;CAEF,WAAW,SAAS,QAAQ,CAAC,YAAuB;EAClD,MAAM,OACJ,OAAO,WAAW,YAAY,OAAO,WAAW,WAC5C,OAAO,SAAS,MAAM,IACtB,MAAM,QAAQ,MAAM,KAAK,OAAO,SAAS,MAAM;EACrD,OAAO;GACL;GACA,eACE,OACI,YAAY,KAAK,MAAM,EAAE,kBAAkB,KAAK,MAAM,MACtD,YAAY,KAAK,MAAM,EAAE,cAAc,KAAK,MAAM;EAC1D;CACF,CAAC;CAED,YAAY,QACV,OAAO,QAAQ,CAAC,MAAM,aAAwD;EAC5E,IAAI,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI,UAAU,iDAAiD;EAC5F,MAAM,UAAU,MAAM,OAAO,SAAS,IAAI;EAC1C,IAAI,YAAY,MAAM,OAAO;GAAE,MAAM;GAAO,eAAe,iBAAiB,KAAK;EAAW;EAC5F,IAAI,YAAY,KAAA,GAAW,OAAO;GAAE,MAAM;GAAM,eAAe,uBAAuB;EAAO;EAC7F,MAAM,OAAO,OAAO,YAAY,WAAW,QAAQ,SAAS,OAAO,IAAI,QAAQ,KAAK,OAAO;EAC3F,OAAO;GACL;GACA,eACE,YAAY,KAAK,GAAG,OAAO,SAAS,GAAG,WAAW,QAAQ,QAAQ,KAAK,QAAQ,KAAK,OAAO;EAC/F;CACF,CACF;CAEA,WAAW,QAAQ,OAAO,QAAQ,MAAU;EAC1C,MAAM,IAAK,MAAM;EACjB,IAAI,OAAO,GAAG,aAAa,UAAU,MAAM,IAAI,UAAU,mCAAmC;EAC5F,OAAO;GACL,MAAM,EAAE,aAAa;GACrB,eACE,EAAE,aAAa,IACX,6BACA,0CAA0C,EAAE,SAAS,oBAClC,KAAK,EAAE,MAAM,EAAE,oBAAoB,KAAK,EAAE,MAAM;EAC3E;CACF,CAAC;CAED,gBAAgB,SAAS,QAAQ,CAAC,WAAsC,SAAS,QAAQ,KAAK,CAAC;CAE/F,eAAe,SAAS,QAAQ,CAAC,SAAkC,QAAQ,QAAQ,GAAG,CAAC;CAEvF,mBAAmB,SAAS,QAAQ,CAAC,UAA0B,YAAY,QAAQ,IAAI,CAAC;CAExF,iBAAiB,SACd,QAAQ,CAAC,QAAQ,MAAM,WACtB,UAAU,QAAQ,QAAQ,MAAM,KAAK,CACzC;CAEA,kBAAkB,SACf,QAAQ,CAAC,MAAM,WACd,WAAW,QAAQ,MAAM,KAAK,CAClC;CAEA,WAAW,SAAS,QAAQ,CAAC,aAA0C,IAAI,QAAQ,OAAO,CAAC;CAE3F,cAAc,SAAS,QAAQ,CAAC,UAAoC,OAAO,QAAQ,IAAI,CAAC;CAExF,eAAe,SAAS,QAAQ,CAAC,WAAgC,QAAQ,QAAQ,KAAK,CAAC;CAEvF,QAAQ,SAAS,QAAQ,GAAO,EAAE,UAAkB;EAClD,MAAM,UAAU;EAChB,IAAI,SAAS,YAAY,WAAW,OAAO,QAAQ,UAAU,UAE3D,OAAO;GACL,MAAM;GACN,eACE,qBAAqB,QAAQ,SAAS,KAAK,QAAQ,MAAM,KAAK,QAAQ;EAC1E;EAEF,MAAM,IAAI;EACV,IAAI,OAAO,GAAG,UAAU,UAAU,MAAM,IAAI,UAAU,sCAAsC;EAC5F,MAAM,QAAQ,EAAE,MAAM,QAAQ,CAAC;EAC/B,OAAO;GACL,MAAM,EAAE,YAAY;GACpB,eACE,EAAE,YAAY,SACV,iCAAiC,MAAM,KAAK,EAAE,UAAU,QAAQ,EAAE,SAAS,KAC3E,eAAe,MAAM,YAAY,EAAE,UAAU,QAAQ,EAAE,SAAS,KAAK,EAAE,MAAM;EACrF;CACF,CAAC;AACH;;AAKA,MAAM,mCAAmB,IAAI,IAAiB;CAC5C;CACA;CACA;CACA;CACA;AACF,CAAC;AAKD,MAAa,iBAAiB,SAAsC,OAAO,OAAO,UAAU,IAAI;;AAGhG,MAAa,cACX,MACA,QACA,MACA,UAEC,SAAS,KAAK,CAA8C,QAAQ,MAAM,KAAK;;AASlF,MAAa,WAAW,OAAO,YAC5B,OAAO,KAAK,QAAQ,CAAC,CAAmB,KAAK,SAAS,CACrD,MACA,SAAiD,QAAiB,GAAG,MAAiB;CACpF,OAAO,WAAW,MAAM,QAAQ,MAAM,EAAE,KAAK,CAAC,CAAC,MAAM,MAAM,CAAC;AAC9D,CACF,CAAC,CACH;;AAGA,MAAa,eAAe,OAAO,YAChC,OAAO,KAAK,QAAQ,CAAC,CACnB,QAAQ,SAAS,CAAC,iBAAiB,IAAI,IAAI,CAAC,CAAC,CAC7C,KAAK,SAAS,CAAC,MAAM,SAAS,KAAK,CAAC,CACzC"}
@@ -1,6 +1,8 @@
1
1
  import { n as UsageError } from "./errors-B6BSMs03.js";
2
- import { n as analysisModelOf, v as gatewayScope, y as resolveModelGateway } from "./config-BrdHA9bJ.js";
3
- import { a as checkModelString, l as judgeRoute, o as isFakeJudge, p as unusedGatewayKey, s as judgeCredentials, u as modelKeyOptions } from "./judge-yLvjC5A3.js";
2
+ import { b as resolveModelGateway, n as analysisModelOf, v as gatewayScope } from "./config-k6m7XbAz.js";
3
+ import { s as isFakeJudge, u as judgeRoute } from "./judge-CKdgEXN7.js";
4
+ import { p as vertexLanguageModel, v as vertexSettings, y as vertexSettingsIn } from "./vertex-DypYjx71.js";
5
+ import { i as openRouterLanguageModel, r as openRouterFallbacks } from "./openrouter-models-CWRwLX-G.js";
4
6
  import { z } from "zod";
5
7
  import { Output, generateText } from "ai";
6
8
  //#region src/analysis/schema.ts
@@ -63,7 +65,7 @@ const analysisSchema = z.object({
63
65
  why: z.string()
64
66
  })
65
67
  });
66
- const SYSTEM_PROMPT = `You analyse the results of a reeval run for the engineer who wrote the evals.
68
+ const SYSTEM_PROMPT = `You analyze the results of a reeval run for the engineer who wrote the evals.
67
69
  reeval runs coding agents in sandboxes. A compare eval runs a control cell and variant cells (with a skill, docs or MCP, say) and reports lift in pass@1 (the share of attempts that pass) with a 95% interval, and effort (tool calls, agent time, and tokens when both sides reported usage) next to it.
68
70
 
69
71
  You get a DIGEST: facts computed deterministically by reeval (ids F1...), evidence excerpts (ids E1...; transcript lines start with their line number L<n>), an attempts table (ids like r2/fix-config/control#1), the RULES reeval used for every verdict, and the CLI. PROJECT INSTRUCTIONS, when present, come first: they set the question this analysis answers. Follow them within these rules.
@@ -129,34 +131,6 @@ function questionModel(config, question, where, judge = config.judge.model) {
129
131
  if (typeof model === "string" && isEvaluationModel(model)) throw new UsageError(`questions "${question.question}" (${where}) would be answered by analysis.model in ${config.file}, "${model}", an evaluation model: it scores yes/no questions and can't ${task}. Use a language model, e.g. 'google/gemini-3.8-flash', or ${fix}.`);
130
132
  return model;
131
133
  }
132
- /**
133
- * Why `analysis.model`'s calls would fail for want of a usable key: none of the keys that would
134
- * work for its provider is set, or the one reeval would send has whitespace or a line break in it.
135
- * Null when a usable key is set, or for any other model. `brief` names the first two keys only,
136
- * for the line a run ends with; the plan's warning has them all.
137
- */
138
- function analysisKeyProblem(config, env = process.env, brief = false) {
139
- const model = analysisModelOf(config);
140
- if (typeof model !== "string" || isEvaluationModel(model) || checkModelString(model)) return null;
141
- let gateway;
142
- try {
143
- gateway = resolveModelGateway(model, void 0, gatewayScope(config), "analysis.model");
144
- } catch (error) {
145
- return brief ? "add vercel() to gateway, or analysis: { insights: false }" : `${error.message} Or analysis: { insights: false }.`;
146
- }
147
- const keys = judgeCredentials(model, gateway);
148
- if (!keys.some((k) => env[k])) {
149
- if (brief) return `set ${keys.slice(0, 2).join(" or ")}, or analysis: { insights: false }`;
150
- const unused = unusedGatewayKey(env, gateway);
151
- return `set ${modelKeyOptions(model, gateway)} for analysis.model, "${model}", in .env.local, or analysis: { insights: false }${unused ? `.${unused}` : ""}`;
152
- }
153
- try {
154
- judgeRoute(model, env, "analysis model", gateway);
155
- return null;
156
- } catch (error) {
157
- return error.message;
158
- }
159
- }
160
134
  /** Checks the model and its credentials before any work; throws a UsageError (exit 2). */
161
135
  function resolveAnalysisModel(config) {
162
136
  const model = analysisModelName(config);
@@ -166,11 +140,15 @@ function resolveAnalysisModel(config) {
166
140
  fake: model
167
141
  };
168
142
  if (isEvaluationModel(model)) throw new UsageError(`analysis.model in ${config.file} is "${model}", an evaluation model: it scores yes/no questions and can't write an analysis. Use a language model, e.g. 'google/gemini-3.8-flash'.`);
169
- const gateway = resolveModelGateway(model, void 0, gatewayScope(config), "analysis.model");
143
+ const scope = gatewayScope(config);
144
+ const gateway = resolveModelGateway(model, void 0, scope, "analysis.model");
170
145
  try {
146
+ const route = judgeRoute(model, process.env, "analysis model", gateway).kind;
171
147
  return {
172
148
  name: model,
173
- route: judgeRoute(model, process.env, "analysis model", gateway).kind
149
+ route,
150
+ ...route === "openrouter" ? { fallbacks: openRouterFallbacks(scope.gateways) } : {},
151
+ ...route === "vertex" ? { vertex: vertexSettingsIn(scope.gateways) } : {}
174
152
  };
175
153
  } catch (error) {
176
154
  throw new UsageError(`${error.message} Or run reeval analyze --context, which needs no key.`);
@@ -241,9 +219,14 @@ async function languageModel(model, digest) {
241
219
  if (model.fake) return fakeModel(model.fake, digest);
242
220
  return providerModel(model);
243
221
  }
244
- /** The provider's own model for a direct route, or the AI Gateway's by name on vercel. */
222
+ /**
223
+ * The provider's own model for a direct route, the AI Gateway's by name on vercel, OpenRouter's
224
+ * on openrouter, or Vertex AI's on vertex.
225
+ */
245
226
  async function providerModel(model) {
246
227
  if (model.route === "gateway") return model.name;
228
+ if (model.route === "openrouter") return openRouterLanguageModel(model.name, !!model.fallbacks);
229
+ if (model.route === "vertex") return vertexLanguageModel(model.name, model.vertex ?? vertexSettings());
247
230
  const provider = model.name.slice(0, model.name.indexOf("/"));
248
231
  const id = model.name.slice(provider.length + 1);
249
232
  switch (provider) {
@@ -253,6 +236,16 @@ async function providerModel(model) {
253
236
  default: return model.name;
254
237
  }
255
238
  }
239
+ /** The language model `name` is on `gateway`, checked for its key: what the judge's route says. */
240
+ function hostModel(name, gateway, what, { fallbacks = false, vertex } = {}) {
241
+ const route = judgeRoute(name, process.env, what, gateway).kind;
242
+ return providerModel({
243
+ name,
244
+ route,
245
+ ...route === "openrouter" ? { fallbacks } : {},
246
+ ...route === "vertex" && vertex ? { vertex } : {}
247
+ });
248
+ }
256
249
  async function callAnalysisModel(model, digest, prompt, maxOutputTokens) {
257
250
  const started = Date.now();
258
251
  const result = await generateText({
@@ -277,6 +270,6 @@ async function callAnalysisModel(model, digest, prompt, maxOutputTokens) {
277
270
  };
278
271
  }
279
272
  //#endregion
280
- export { isFake as a, questionModelLabel as c, cannedModel as i, resolveAnalysisModel as l, analysisModelName as n, providerModel as o, callAnalysisModel as r, questionModel as s, analysisKeyProblem as t, SYSTEM_PROMPT as u };
273
+ export { isFake as a, questionModelLabel as c, hostModel as i, resolveAnalysisModel as l, callAnalysisModel as n, providerModel as o, cannedModel as r, questionModel as s, analysisModelName as t, SYSTEM_PROMPT as u };
281
274
 
282
- //# sourceMappingURL=model-1ZLr9Dcv.js.map
275
+ //# sourceMappingURL=model-Bq3G0OG1.js.map