reeval 0.13.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. package/README.md +5 -3
  2. package/THIRD_PARTY_NOTICES.md +190 -190
  3. package/assets/gateway-models.mjs +74 -0
  4. package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
  5. package/dist/adapter-CbqbMR7y.js.map +1 -0
  6. package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
  7. package/dist/agent-Zgn-kiqq.js.map +1 -0
  8. package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
  9. package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
  10. package/dist/app-files-ChbvRx3P.js +11 -0
  11. package/dist/app-files-ChbvRx3P.js.map +1 -0
  12. package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
  13. package/dist/assemble-SKQGW_QF.js.map +1 -0
  14. package/dist/assemble-XBWuK3Ed.js +1 -0
  15. package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
  16. package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
  17. package/dist/attempt-host-BOp8JklD.js +295 -0
  18. package/dist/attempt-host-BOp8JklD.js.map +1 -0
  19. package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
  20. package/dist/capture-BM_3100g.js.map +1 -0
  21. package/dist/capture-DSK8O5bA.js +1 -0
  22. package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
  23. package/dist/cli-PdYrcoKM.js.map +1 -0
  24. package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
  25. package/dist/cli-spec-D0qyrPZi.js.map +1 -0
  26. package/dist/cli.js +2 -2
  27. package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
  28. package/dist/client-Ct6s6OnF.js.map +1 -0
  29. package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
  30. package/dist/command-4pJYVCT4.js.map +1 -0
  31. package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
  32. package/dist/command-9xm5ImqI.js.map +1 -0
  33. package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
  34. package/dist/command-BqBxLgTc.js.map +1 -0
  35. package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
  36. package/dist/command-DbSMcxZL.js.map +1 -0
  37. package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
  38. package/dist/commands-BU6YJCz9.js.map +1 -0
  39. package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
  40. package/dist/config-k6m7XbAz.js +3777 -0
  41. package/dist/config-k6m7XbAz.js.map +1 -0
  42. package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
  43. package/dist/contrasts-DLZF4dGF.js.map +1 -0
  44. package/dist/cost-C5RBTTJr.js +1 -0
  45. package/dist/cost-ChPa8-Fp.js +359 -0
  46. package/dist/cost-ChPa8-Fp.js.map +1 -0
  47. package/dist/db-CpsifVln.js +1530 -0
  48. package/dist/db-CpsifVln.js.map +1 -0
  49. package/dist/db-DkD-F-X-.js +2 -0
  50. package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
  51. package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
  52. package/dist/debug-DeM5pswK.js +1 -0
  53. package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
  54. package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
  55. package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
  56. package/dist/docker-51WvhN_X.js.map +1 -0
  57. package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
  58. package/dist/docker-B4-rx4dA.js.map +1 -0
  59. package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
  60. package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
  61. package/dist/doctor-BuQnfz4e.js.map +1 -0
  62. package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
  63. package/dist/env-CARj5Axq.js.map +1 -0
  64. package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
  65. package/dist/environment-BOFgzFDg.js.map +1 -0
  66. package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
  67. package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
  68. package/dist/eval-source-Dje3rERU.js +145 -0
  69. package/dist/eval-source-Dje3rERU.js.map +1 -0
  70. package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
  71. package/dist/evaluate-BPP9s3s6.js.map +1 -0
  72. package/dist/evaluate-dqXccn4k.d.ts +44 -0
  73. package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
  74. package/dist/files-Bi461Be2.js.map +1 -0
  75. package/dist/files-C2h2Cw4F.js +1 -0
  76. package/dist/files-Clipx4OW.js +37 -0
  77. package/dist/files-Clipx4OW.js.map +1 -0
  78. package/dist/files-DmUrRmmL.js +1 -0
  79. package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
  80. package/dist/files-H24pz0hL.js.map +1 -0
  81. package/dist/gateway.d.ts +30 -0
  82. package/dist/gateway.js +39 -0
  83. package/dist/gateway.js.map +1 -0
  84. package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
  85. package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
  86. package/dist/harness-3NVyTSGK.js +392 -0
  87. package/dist/harness-3NVyTSGK.js.map +1 -0
  88. package/dist/harness.d.ts +33 -21
  89. package/dist/harness.js +1 -1
  90. package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
  91. package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
  92. package/dist/index.d.ts +34 -35
  93. package/dist/index.js +5 -6
  94. package/dist/init-BDEOopK7.js +1 -0
  95. package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
  96. package/dist/init-cOk3D44b.js.map +1 -0
  97. package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
  98. package/dist/invocations-DW-lTk2f.js.map +1 -0
  99. package/dist/judge-CKdgEXN7.js +575 -0
  100. package/dist/judge-CKdgEXN7.js.map +1 -0
  101. package/dist/judge.d.ts +8 -9
  102. package/dist/judge.js +7 -9
  103. package/dist/judge.js.map +1 -1
  104. package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
  105. package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
  106. package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
  107. package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
  108. package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
  109. package/dist/model-Bq3G0OG1.js.map +1 -0
  110. package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
  111. package/dist/model-CTnyvh7H.js.map +1 -0
  112. package/dist/model-D-SWrLZG.js +1 -0
  113. package/dist/model-g3XnzUv0.js +1 -0
  114. package/dist/model-vRiUycWh.js +77 -0
  115. package/dist/model-vRiUycWh.js.map +1 -0
  116. package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
  117. package/dist/node-BVbbnFtv.js.map +1 -0
  118. package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
  119. package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
  120. package/dist/openrouter-models-CWRwLX-G.js +34 -0
  121. package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
  122. package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
  123. package/dist/pairs-puYi-zj2.js.map +1 -0
  124. package/dist/patch-CquGAGX5.js +76 -0
  125. package/dist/patch-CquGAGX5.js.map +1 -0
  126. package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
  127. package/dist/patch-DKJiC5Sq.js.map +1 -0
  128. package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
  129. package/dist/plan-DdWkM6SW.js.map +1 -0
  130. package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
  131. package/dist/plugins-D-CP6mMd.js.map +1 -0
  132. package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
  133. package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
  134. package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
  135. package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
  136. package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
  137. package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
  138. package/dist/prices-QaS-kjSQ.js +253 -0
  139. package/dist/prices-QaS-kjSQ.js.map +1 -0
  140. package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
  141. package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
  142. package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
  143. package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
  144. package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
  145. package/dist/query-Clm2mNSm.js +1 -0
  146. package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
  147. package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
  148. package/dist/questions-BRhU8MzB.js +861 -0
  149. package/dist/questions-BRhU8MzB.js.map +1 -0
  150. package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
  151. package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
  152. package/dist/redact-DVIaahLv.js +658 -0
  153. package/dist/redact-DVIaahLv.js.map +1 -0
  154. package/dist/remarks-BwJ8pcTQ.js +1 -0
  155. package/dist/remarks-C7i2LMRb.js +596 -0
  156. package/dist/remarks-C7i2LMRb.js.map +1 -0
  157. package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
  158. package/dist/remarks-D_3qhBWU.js.map +1 -0
  159. package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
  160. package/dist/remarks-dA8LyoPc.js.map +1 -0
  161. package/dist/remote-DFIzcx_6.js +273 -0
  162. package/dist/remote-DFIzcx_6.js.map +1 -0
  163. package/dist/resolve--rQZ8u6Z.js +1 -0
  164. package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
  165. package/dist/resolve-DWSJpKHX.js.map +1 -0
  166. package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
  167. package/dist/results-BC3pcG_Z.js.map +1 -0
  168. package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
  169. package/dist/resume-j-rcIryi.js.map +1 -0
  170. package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
  171. package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
  172. package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
  173. package/dist/runtime-d56Gg3J5.js.map +1 -0
  174. package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
  175. package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
  176. package/dist/sandbox.d.ts +2 -1
  177. package/dist/sandbox.js +1 -1
  178. package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
  179. package/dist/server-BOoH10J8.js.map +1 -0
  180. package/dist/server-DpuVAbrO.js +1 -0
  181. package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
  182. package/dist/shared-B_lvGqWz.js.map +1 -0
  183. package/dist/shared-CHvO7eLi.d.ts +1 -0
  184. package/dist/source-BFd6f0-R.js +1 -0
  185. package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
  186. package/dist/source-wD04-rRG.js.map +1 -0
  187. package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
  188. package/dist/src-1lpTocbd.js.map +1 -0
  189. package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
  190. package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
  191. package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
  192. package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
  193. package/dist/src-BO8AYqJ5.js.map +1 -0
  194. package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
  195. package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
  196. package/dist/src-BYPMM1x7.js +2 -0
  197. package/dist/src-Bp4aN3rA.js +2 -0
  198. package/dist/src-C7upnZSd.js +2 -0
  199. package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
  200. package/dist/src-D91Hd6hT.js.map +1 -0
  201. package/dist/src-DH-9IO7f.js +1 -0
  202. package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
  203. package/dist/src-DSZyg0Uq.js.map +1 -0
  204. package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
  205. package/dist/src-DgHuStEi.js.map +1 -0
  206. package/dist/src-HSxBazcg.js +3 -0
  207. package/dist/src-LD1BAJFa.js +2 -0
  208. package/dist/src-dlX-Lmxg.js +2 -0
  209. package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
  210. package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
  211. package/dist/src-kYeaJJrl.js +2 -0
  212. package/dist/state-B5PLm5qY.js +1 -0
  213. package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
  214. package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
  215. package/dist/stats-hfpuYsAM.js +315 -0
  216. package/dist/stats-hfpuYsAM.js.map +1 -0
  217. package/dist/suite-22Kt9Grx.js +2 -0
  218. package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
  219. package/dist/suite-Co8cwelb.js.map +1 -0
  220. package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
  221. package/dist/summary-DLFHVoux.js.map +1 -0
  222. package/dist/testing.d.ts +449 -2
  223. package/dist/testing.js +16 -3
  224. package/dist/testing.js.map +1 -0
  225. package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
  226. package/dist/text-DTigcA64.js.map +1 -0
  227. package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
  228. package/dist/transcript-DxPaWHTZ.js.map +1 -0
  229. package/dist/types-C5vI3g1C.js.map +1 -1
  230. package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
  231. package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
  232. package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
  233. package/dist/vertex-Bv7vhMgs.js +1 -0
  234. package/dist/vertex-C7PI-WPc.d.ts +3 -0
  235. package/dist/vertex-DypYjx71.js +456 -0
  236. package/dist/vertex-DypYjx71.js.map +1 -0
  237. package/dist/view/index.d.ts +505 -43
  238. package/dist/view/index.js +10 -9
  239. package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
  240. package/dist/views-BFO1BLKu.js.map +1 -0
  241. package/dist/views-BOf8qEL8.js +2 -0
  242. package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
  243. package/dist/worker-entry-CJtdsKBP.js.map +1 -0
  244. package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
  245. package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
  246. package/dist/workspace-DDHzkMjm.js +1 -0
  247. package/docker/seccomp-default.json +1 -0
  248. package/package.json +10 -2
  249. package/skills/eval-rubric/SKILL.md +43 -103
  250. package/dist/adapter-IKKo4sZj.js.map +0 -1
  251. package/dist/agent-ClE-g9VL.js.map +0 -1
  252. package/dist/assemble-BO93VAk1.js.map +0 -1
  253. package/dist/assemble-VQxyoDxK.js +0 -1
  254. package/dist/assert-EUIoRdIt.js.map +0 -1
  255. package/dist/capture-dn2hjSLj.js.map +0 -1
  256. package/dist/capture-nIJ-7kUE.js +0 -1
  257. package/dist/cli-BCE1xlm4.js.map +0 -1
  258. package/dist/cli-spec-BXIBUlaK.js.map +0 -1
  259. package/dist/client-fcwgwOAD.js +0 -106
  260. package/dist/client-fcwgwOAD.js.map +0 -1
  261. package/dist/command-Bh3yhN_b.js.map +0 -1
  262. package/dist/command-C6vwmjS7.js +0 -91
  263. package/dist/command-C6vwmjS7.js.map +0 -1
  264. package/dist/command-CANun0m0.js.map +0 -1
  265. package/dist/command-CBaHukyl.js +0 -335
  266. package/dist/command-CBaHukyl.js.map +0 -1
  267. package/dist/command-D_kEnYcX.js.map +0 -1
  268. package/dist/commands-7tH3EsVu.js.map +0 -1
  269. package/dist/config-D7TFvUpL.js +0 -759
  270. package/dist/config-D7TFvUpL.js.map +0 -1
  271. package/dist/contrasts-B4nOhn0Z.js.map +0 -1
  272. package/dist/db-DJUuFoWQ.js +0 -602
  273. package/dist/db-DJUuFoWQ.js.map +0 -1
  274. package/dist/db-W9EFlXnn.js +0 -2
  275. package/dist/debug-D7bEv_UV.js +0 -1
  276. package/dist/docker-CQxucRPe.js.map +0 -1
  277. package/dist/docker-DuHcQidD.js.map +0 -1
  278. package/dist/doctor-De_QlOTM.js.map +0 -1
  279. package/dist/egress-DEAvq8Fl.js +0 -1258
  280. package/dist/egress-DEAvq8Fl.js.map +0 -1
  281. package/dist/env-BQqj93gu.js.map +0 -1
  282. package/dist/environment-B4IuURK1.js.map +0 -1
  283. package/dist/evaluate-C9-ritA5.d.ts +0 -92
  284. package/dist/evaluate-Djk1YPFL.js.map +0 -1
  285. package/dist/facts-Cfx1H70y.js +0 -898
  286. package/dist/facts-Cfx1H70y.js.map +0 -1
  287. package/dist/files-Bs6H3HiG.js +0 -65
  288. package/dist/files-Bs6H3HiG.js.map +0 -1
  289. package/dist/files-CN1QfenC.js.map +0 -1
  290. package/dist/files-CtjEXWXm.js +0 -1
  291. package/dist/files-D27nUk0K.js.map +0 -1
  292. package/dist/files-D3gJa91B.js +0 -1
  293. package/dist/generate-BnrvBAzF.js +0 -1646
  294. package/dist/generate-BnrvBAzF.js.map +0 -1
  295. package/dist/harness-1OkumOAm.js +0 -323
  296. package/dist/harness-1OkumOAm.js.map +0 -1
  297. package/dist/header-BVzbRnNv.js.map +0 -1
  298. package/dist/init-hiTkiM4L.js.map +0 -1
  299. package/dist/init-kMkTgTZc.js +0 -1
  300. package/dist/insights-B2Xmhz9o.js +0 -64
  301. package/dist/insights-B2Xmhz9o.js.map +0 -1
  302. package/dist/invocations-D_ePcwyL.js.map +0 -1
  303. package/dist/judge-CEDIfRiU.js +0 -186
  304. package/dist/judge-CEDIfRiU.js.map +0 -1
  305. package/dist/model-B8H3gL69.js +0 -1
  306. package/dist/model-CTiuzkWw.js +0 -46
  307. package/dist/model-CTiuzkWw.js.map +0 -1
  308. package/dist/model-DcIbmggK.js +0 -1
  309. package/dist/model-DpkclE4r.js.map +0 -1
  310. package/dist/model-UgHDMhI1.js.map +0 -1
  311. package/dist/node-CGZYh4Fj.js.map +0 -1
  312. package/dist/opencode-providers-snfeWd6Z.js +0 -3305
  313. package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
  314. package/dist/options-CyAyLjgJ.js +0 -1129
  315. package/dist/options-CyAyLjgJ.js.map +0 -1
  316. package/dist/pairs-DP6XOJby.js.map +0 -1
  317. package/dist/patch-uMq9rCl_.js.map +0 -1
  318. package/dist/plan-CZYgXnZh.js.map +0 -1
  319. package/dist/plugins-P-uh54VF.js.map +0 -1
  320. package/dist/query-D_1ARx5q.js +0 -1
  321. package/dist/questions-KENpWnfV.js +0 -523
  322. package/dist/questions-KENpWnfV.js.map +0 -1
  323. package/dist/redact-CRGdfV2s.js +0 -240
  324. package/dist/redact-CRGdfV2s.js.map +0 -1
  325. package/dist/remarks-1521zwfW.js.map +0 -1
  326. package/dist/remarks-BhAV1H4V.js +0 -1
  327. package/dist/remarks-Ml6bcSdV.js +0 -278
  328. package/dist/remarks-Ml6bcSdV.js.map +0 -1
  329. package/dist/remote-Dkeghk8k.js +0 -378
  330. package/dist/remote-Dkeghk8k.js.map +0 -1
  331. package/dist/resolve-CMmhACr6.js +0 -1
  332. package/dist/resolve-DzdWcIL6.js.map +0 -1
  333. package/dist/results-D-naai_Q.js.map +0 -1
  334. package/dist/resume-8-nnW8iO.js.map +0 -1
  335. package/dist/rules-CcV3zLDR.js +0 -318
  336. package/dist/rules-CcV3zLDR.js.map +0 -1
  337. package/dist/run-Bel7Vde4.js +0 -88
  338. package/dist/run-Bel7Vde4.js.map +0 -1
  339. package/dist/runtime-PX_oXiCS.js.map +0 -1
  340. package/dist/server-C-ysqkLv.js +0 -1
  341. package/dist/server-De8oH3bt.js.map +0 -1
  342. package/dist/shared-CA61BuPJ.js.map +0 -1
  343. package/dist/source-C686eZpY.js.map +0 -1
  344. package/dist/source-DjgypUHW.js +0 -1
  345. package/dist/src-3mP821Pt.js +0 -2
  346. package/dist/src-B3KCLZVk.js +0 -3
  347. package/dist/src-BAyg6CtQ.js +0 -2
  348. package/dist/src-BezgGX4X.js +0 -2
  349. package/dist/src-CIpKL45U.js.map +0 -1
  350. package/dist/src-ChBhk06Q.js +0 -2
  351. package/dist/src-CoqZYspK.js +0 -2
  352. package/dist/src-CzQrAcF8.js.map +0 -1
  353. package/dist/src-D9fSTxlc.js +0 -1
  354. package/dist/src-DIL0jST0.js.map +0 -1
  355. package/dist/src-Xj_BZKbq.js.map +0 -1
  356. package/dist/src-dhSTRN50.js +0 -2
  357. package/dist/src-djNNYIdP.js.map +0 -1
  358. package/dist/state-BGWGAwdo.js +0 -1
  359. package/dist/state-CeR6oz0k.js +0 -66
  360. package/dist/state-CeR6oz0k.js.map +0 -1
  361. package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
  362. package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
  363. package/dist/studio-app/favicon.svg +0 -1
  364. package/dist/studio-app/index.html +0 -15
  365. package/dist/suite-DmuF_ApO.js.map +0 -1
  366. package/dist/suite-mn9g4veR.js +0 -2
  367. package/dist/summary-BFMHMTz2.js.map +0 -1
  368. package/dist/text-CrVy5ioL.js.map +0 -1
  369. package/dist/transcript-BTxJpW4R.js.map +0 -1
  370. package/dist/usage-BDo2q_yu.js +0 -1
  371. package/dist/validate-DWjkshl1.js.map +0 -1
  372. package/dist/views-DE2A1jEX.js +0 -2
  373. package/dist/views-Dk8MCQKr.js.map +0 -1
  374. package/dist/worker-entry-CqrzGPF4.js.map +0 -1
  375. package/dist/workspace-B1eGO46M.js +0 -1
@@ -0,0 +1,575 @@
1
+ import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-Bz0l7OZI.js";
2
+ import { v as vertexSettings } from "./vertex-DypYjx71.js";
3
+ import { m as redactSecrets, r as badCharacterAt } from "./redact-DVIaahLv.js";
4
+ import { setTimeout } from "node:timers/promises";
5
+ //#region src/gateway/effort.ts
6
+ /** Every model: no run has checked the harness's effort reaches any of them there. */
7
+ const everyModel = () => true;
8
+ /** Any but a Gemini model (`gemini-3.8-flash`, `google/gemini-3.8-flash`). */
9
+ const notGemini = (model) => !/^(?:google\/)?gemini-/.test(model);
10
+ //#endregion
11
+ //#region src/gateway/openrouter.ts
12
+ const OPENROUTER_HOST = "openrouter.ai";
13
+ const OPENROUTER_KEY = "OPENROUTER_API_KEY";
14
+ const onOpenRouter = (path, method = "POST") => ({
15
+ host: OPENROUTER_HOST,
16
+ path: { exact: path },
17
+ method: [method]
18
+ });
19
+ /** The response names the model that served it on OpenRouter's host, in OpenRouter's form. */
20
+ const servedOnOpenRouter = ({ host, responseModel, responseEffort }) => host === "openrouter.ai" && responseModel ? `${responseModel}${responseEffort ? `[effort=${responseEffort}]` : ""}` : void 0;
21
+ /** The upstream provider OpenRouter's response says served the call, and what it charged. */
22
+ const upstreamOnOpenRouter = ({ host, responseProvider, responseCost }) => host === "openrouter.ai" && responseProvider ? {
23
+ provider: responseProvider,
24
+ ...responseCost !== void 0 ? { costUsd: responseCost } : {}
25
+ } : void 0;
26
+ /** What every OpenRouter route shares: the key under its own name, and what the responses say. */
27
+ const route = (endpoints, calls) => ({
28
+ credential: OPENROUTER_KEY,
29
+ authAs: OPENROUTER_KEY,
30
+ endpoints,
31
+ modelResponses: calls,
32
+ servedModel: servedOnOpenRouter,
33
+ upstream: upstreamOnOpenRouter,
34
+ hosts: [OPENROUTER_HOST]
35
+ });
36
+ /** An id as OpenRouter names it: as written when it names a provider, else under `provider`. */
37
+ const under = (provider) => (model) => model.includes("/") ? model : `${provider}/${model}`;
38
+ /**
39
+ * An Anthropic id as OpenRouter names it, with the version dotted (`anthropic/claude-haiku-4.5`
40
+ * for `claude-haiku-4-5`), unless the id names a provider.
41
+ */
42
+ const anthropicOnOpenRouter = (model) => model.includes("/") ? model : `anthropic/${model.replace(/-(\d{1,2})-(\d{1,2})$/, "-$1.$2")}`;
43
+ /**
44
+ * Claude Code on OpenRouter's Anthropic Messages API: `ANTHROPIC_BASE_URL` at OpenRouter and the
45
+ * key as `ANTHROPIC_AUTH_TOKEN`, which the CLI sends as a Bearer token. On another base URL it
46
+ * makes none of its `/api/claude_code/*` calls.
47
+ */
48
+ function claudeCode$1() {
49
+ const calls = [onOpenRouter("/api/v1/messages")];
50
+ return {
51
+ ...route([...calls, onOpenRouter("/api/v1/messages/count_tokens")], calls),
52
+ authAs: "ANTHROPIC_AUTH_TOKEN",
53
+ authEnv: { ANTHROPIC_BASE_URL: `https://${OPENROUTER_HOST}/api` },
54
+ modelId: anthropicOnOpenRouter
55
+ };
56
+ }
57
+ /** Codex on a Responses API provider at OpenRouter, whose ids name the provider (`openai/…`). */
58
+ function codex$1() {
59
+ const calls = [onOpenRouter("/api/v1/responses")];
60
+ return {
61
+ ...route(calls, calls),
62
+ modelId: under("openai")
63
+ };
64
+ }
65
+ /**
66
+ * Goose on its `openrouter` provider: Chat Completions, and the model list it fetches each turn.
67
+ */
68
+ function goose$1() {
69
+ const calls = [onOpenRouter("/api/v1/chat/completions")];
70
+ return {
71
+ ...route([...calls, onOpenRouter("/api/v1/models", "GET")], calls),
72
+ modelId: under("google")
73
+ };
74
+ }
75
+ /**
76
+ * OpenCode on its own `openrouter` provider, with the model as `openrouter/<id>`: the requests the
77
+ * pinned CLI sends that provider's key on (harness-opencode's provider table), plus Exa's host for
78
+ * web search.
79
+ */
80
+ function opencode$1() {
81
+ const resolved = resolveOpenCodeProvider("openrouter/model");
82
+ if ("error" in resolved) throw new Error(resolved.error);
83
+ const { provider } = resolved;
84
+ return {
85
+ credential: OPENROUTER_KEY,
86
+ authAs: OPENROUTER_KEY,
87
+ endpoints: providerEndpoints(provider),
88
+ modelResponses: providerModelResponses(provider),
89
+ servedModel: (request) => providerServedModel(provider, request),
90
+ upstream: upstreamOnOpenRouter,
91
+ hosts: [...providerHosts(provider), "mcp.exa.ai"],
92
+ modelId: (model) => `openrouter/${model}`
93
+ };
94
+ }
95
+ /**
96
+ * pi on its own `openrouter` provider: Chat Completions, a Bearer key, a model from pi's catalog
97
+ * for OpenRouter, each thinking level sent as OpenRouter's `reasoning.effort`. `--offline` skips
98
+ * the model list pi would fetch to fit that catalog to the key.
99
+ */
100
+ function pi$1() {
101
+ const calls = [onOpenRouter("/api/v1/chat/completions")];
102
+ return {
103
+ ...route(calls, calls),
104
+ modelId: under("google")
105
+ };
106
+ }
107
+ /**
108
+ * Grok Build with the model as a `[model.<id>]` table on OpenRouter's Chat Completions API, keyed
109
+ * by `env_key`; it then calls nothing on xAI's API.
110
+ */
111
+ function grokBuild$1() {
112
+ const calls = [onOpenRouter("/api/v1/chat/completions")];
113
+ return {
114
+ ...route(calls, calls),
115
+ modelId: under("x-ai")
116
+ };
117
+ }
118
+ /**
119
+ * fx with OpenRouter's Chat Completions API as its configured provider `openrouter`, through the
120
+ * relay, and an OpenAI id as `openai/<id>`. fx sends no `reasoning_effort` to a configured provider.
121
+ */
122
+ function fx$1() {
123
+ const calls = [onOpenRouter("/api/v1/chat/completions")];
124
+ return {
125
+ ...route(calls, calls),
126
+ modelId: under("openai")
127
+ };
128
+ }
129
+ /** OpenRouter: the harnesses reeval runs there, and why it doesn't run the rest. */
130
+ const OPENROUTER_GATEWAY = {
131
+ name: "OpenRouter",
132
+ routes: {
133
+ "claude-code": claudeCode$1,
134
+ codex: codex$1,
135
+ goose: goose$1,
136
+ opencode: opencode$1,
137
+ fx: fx$1,
138
+ "grok-build": grokBuild$1,
139
+ pi: pi$1
140
+ },
141
+ refusals: {
142
+ cursor: "Cursor's agent only calls Cursor's own API",
143
+ antigravity: "Antigravity's CLI only calls Google's API, with a Gemini key, and OpenRouter has no Gemini API"
144
+ },
145
+ unprovenEffort: {
146
+ "claude-code": everyModel,
147
+ goose: notGemini,
148
+ opencode: notGemini
149
+ }
150
+ };
151
+ //#endregion
152
+ //#region src/gateway/vercel.ts
153
+ const VERCEL_HOST = "ai-gateway.vercel.sh";
154
+ /** What the adapters take Vercel's key as, whichever host variable it came from. */
155
+ const VERCEL_KEY = "AI_GATEWAY_API_KEY";
156
+ /** A Vercel project's short-lived token (`vercel env pull`), used when there's no key. */
157
+ const VERCEL_OIDC = "VERCEL_OIDC_TOKEN";
158
+ /** The host variable the key comes from: `AI_GATEWAY_API_KEY`, else `VERCEL_OIDC_TOKEN`. */
159
+ const vercelCredential = (env = process.env) => !env["AI_GATEWAY_API_KEY"] && env["VERCEL_OIDC_TOKEN"] ? VERCEL_OIDC : VERCEL_KEY;
160
+ /** When a JWT's `exp` says it expires, or undefined when it isn't one or doesn't say. */
161
+ function jwtExpiry(token) {
162
+ const [, payload] = token.split(".");
163
+ if (!payload) return void 0;
164
+ try {
165
+ const { exp } = JSON.parse(Buffer.from(payload, "base64url").toString("utf8"));
166
+ return typeof exp === "number" ? /* @__PURE__ */ new Date(exp * 1e3) : void 0;
167
+ } catch {
168
+ return;
169
+ }
170
+ }
171
+ /**
172
+ * Why the run can't use Vercel's key, or undefined when it can: `VERCEL_OIDC_TOKEN` lasts about
173
+ * 12 hours, and a run that starts on an expired one fails every attempt.
174
+ */
175
+ function vercelCredentialProblem(env = process.env, now = /* @__PURE__ */ new Date()) {
176
+ if (vercelCredential(env) !== "VERCEL_OIDC_TOKEN") return void 0;
177
+ const expiry = jwtExpiry(env["VERCEL_OIDC_TOKEN"] ?? "");
178
+ if (!expiry || expiry > now) return void 0;
179
+ return `${VERCEL_OIDC} expired at ${expiry.toISOString()}, so vercel() can't authenticate. Run vercel env pull for a new one, or set ${VERCEL_KEY}.`;
180
+ }
181
+ const onVercel = (path, method) => ({
182
+ host: VERCEL_HOST,
183
+ path: { exact: path },
184
+ method: [method]
185
+ });
186
+ /**
187
+ * The response names the model that served it on Vercel's host, in Vercel's form
188
+ * (`openai/gpt-5.6-luna`), with the reasoning effort when it says one.
189
+ */
190
+ const servedOnVercel = ({ host, responseModel, responseEffort }) => host === "ai-gateway.vercel.sh" && responseModel ? `${responseModel}${responseEffort ? `[effort=${responseEffort}]` : ""}` : void 0;
191
+ /**
192
+ * Claude Code with `ANTHROPIC_BASE_URL` at Vercel's host: its adapter switches to it when given
193
+ * the key as `AI_GATEWAY_API_KEY`, and the CLI sends it in `x-api-key` on Anthropic's message
194
+ * paths. On another base URL it makes none of its `/api/claude_code/*` calls.
195
+ */
196
+ function claudeCode(env) {
197
+ return {
198
+ credential: vercelCredential(env),
199
+ authAs: VERCEL_KEY,
200
+ endpoints: [onVercel("/v1/messages", "POST"), onVercel("/v1/messages/count_tokens", "POST")],
201
+ hosts: [VERCEL_HOST]
202
+ };
203
+ }
204
+ /** Codex on a Responses API provider at Vercel, whose ids name the provider (`openai/…`). */
205
+ function codex(env) {
206
+ const endpoints = [onVercel("/v1/responses", "POST")];
207
+ return {
208
+ credential: vercelCredential(env),
209
+ authAs: VERCEL_KEY,
210
+ endpoints,
211
+ modelResponses: endpoints,
212
+ servedModel: servedOnVercel,
213
+ hosts: [VERCEL_HOST],
214
+ modelId: (model) => model.includes("/") ? model : `openai/${model}`
215
+ };
216
+ }
217
+ /** fx's language-model calls on Vercel, whose responses name the model that served them. */
218
+ const FX_VERCEL_CALL = {
219
+ host: VERCEL_HOST,
220
+ path: { regex: "^/v\\d+/ai/language-model$" },
221
+ method: ["POST"]
222
+ };
223
+ /**
224
+ * The model an fx call asked the gateway for, in `ai-language-model-id`: what reeval records when
225
+ * a provider's stream (Gemini's) names no served model. Only a call that succeeded asked anything.
226
+ */
227
+ const fxAskedModel = (request) => {
228
+ if (request.host !== "ai-gateway.vercel.sh" || !new RegExp(FX_VERCEL_CALL.path.regex).test(request.path)) return void 0;
229
+ if (request.status === void 0 || request.status < 200 || request.status > 299) return void 0;
230
+ const asked = request.headers?.["ai-language-model-id"];
231
+ return (typeof asked === "string" ? asked : asked?.[0]) || void 0;
232
+ };
233
+ /** The model list fx fetches before a turn, and its language-model calls. */
234
+ const FX_VERCEL_ENDPOINTS = [{
235
+ host: VERCEL_HOST,
236
+ path: { regex: "^/coding-agent/v1/" },
237
+ method: ["GET"]
238
+ }, FX_VERCEL_CALL];
239
+ /** fx on its own AI Gateway provider, which takes a gateway id; a bare one is OpenAI's. */
240
+ function fx(env) {
241
+ return {
242
+ credential: vercelCredential(env),
243
+ authAs: VERCEL_KEY,
244
+ endpoints: FX_VERCEL_ENDPOINTS,
245
+ modelResponses: [FX_VERCEL_CALL],
246
+ servedModel: servedOnVercel,
247
+ askedModel: fxAskedModel,
248
+ hosts: [VERCEL_HOST],
249
+ modelId: (model) => model.includes("/") ? model : `openai/${model}`
250
+ };
251
+ }
252
+ /** A Gemini id as Vercel names it (`google/gemini-3-flash`), unless the id names a provider. */
253
+ const googleId = (model) => model.includes("/") ? model : `google/${model}`;
254
+ /**
255
+ * Goose on its `vercel_ai_gateway` provider, an OpenAI-compatible one: Chat Completions, the
256
+ * Responses API for OpenAI's reasoning models, and the model list it fetches each turn.
257
+ */
258
+ function goose(env) {
259
+ const calls = [onVercel("/v1/chat/completions", "POST"), onVercel("/v1/responses", "POST")];
260
+ return {
261
+ credential: vercelCredential(env),
262
+ authAs: VERCEL_KEY,
263
+ endpoints: [...calls, onVercel("/v1/models", "GET")],
264
+ modelResponses: calls,
265
+ servedModel: servedOnVercel,
266
+ hosts: [VERCEL_HOST],
267
+ modelId: googleId
268
+ };
269
+ }
270
+ /**
271
+ * pi on a provider of its own on Vercel's Chat Completions API, a Bearer key: a Gemini model from
272
+ * pi's catalog, each thinking level sent as Gemini's own `thinkingConfig` under Vercel's
273
+ * `providerOptions` (Vercel's `reasoning_effort` maps every Gemini 3 level but `low` to `high`).
274
+ */
275
+ function pi(env) {
276
+ const endpoints = [onVercel("/v1/chat/completions", "POST")];
277
+ return {
278
+ credential: vercelCredential(env),
279
+ authAs: VERCEL_KEY,
280
+ endpoints,
281
+ modelResponses: endpoints,
282
+ servedModel: servedOnVercel,
283
+ hosts: [VERCEL_HOST],
284
+ modelId: googleId
285
+ };
286
+ }
287
+ /**
288
+ * OpenCode on its own `vercel` provider, with the model as `vercel/<id>`: the requests the pinned
289
+ * CLI sends that provider's key on (harness-opencode's provider table), plus Exa's host for web
290
+ * search.
291
+ */
292
+ function opencode(env) {
293
+ const resolved = resolveOpenCodeProvider("vercel/model");
294
+ if ("error" in resolved) throw new Error(resolved.error);
295
+ const { provider } = resolved;
296
+ return {
297
+ credential: vercelCredential(env),
298
+ authAs: VERCEL_KEY,
299
+ endpoints: providerEndpoints(provider),
300
+ modelResponses: providerModelResponses(provider),
301
+ servedModel: (request) => providerServedModel(provider, request),
302
+ hosts: [...providerHosts(provider), "mcp.exa.ai"],
303
+ modelId: (model) => `vercel/${model}`
304
+ };
305
+ }
306
+ /**
307
+ * Grok Build with the model as a `[model.<id>]` table on Vercel's Chat Completions API, keyed by
308
+ * `env_key`; it then calls nothing on xAI's API.
309
+ */
310
+ function grokBuild(env) {
311
+ const endpoints = [onVercel("/v1/chat/completions", "POST")];
312
+ return {
313
+ credential: vercelCredential(env),
314
+ authAs: VERCEL_KEY,
315
+ endpoints,
316
+ modelResponses: endpoints,
317
+ servedModel: servedOnVercel,
318
+ hosts: [VERCEL_HOST],
319
+ expectedBlocked: {
320
+ hosts: ["vercel.com"],
321
+ why: "Grok Build's probe of the gateway's root, which Vercel redirects to its site"
322
+ },
323
+ modelId: (model) => model.includes("/") ? model : `xai/${model}`
324
+ };
325
+ }
326
+ /** Vercel AI Gateway: the harnesses reeval runs there, and why it doesn't run the rest. */
327
+ const VERCEL_GATEWAY = {
328
+ name: "Vercel AI Gateway",
329
+ routes: {
330
+ "claude-code": claudeCode,
331
+ codex,
332
+ goose,
333
+ opencode,
334
+ fx,
335
+ "grok-build": grokBuild,
336
+ pi
337
+ },
338
+ refusals: {
339
+ cursor: "Cursor's agent only calls Cursor's own API",
340
+ antigravity: "Antigravity's CLI only calls Google's API, with a Gemini key"
341
+ },
342
+ unprovenEffort: {
343
+ goose: everyModel,
344
+ opencode: notGemini
345
+ }
346
+ };
347
+ //#endregion
348
+ //#region src/judge/retry.ts
349
+ /** Up to six tries over about a minute: long enough to ride out a provider's "high demand". */
350
+ const JUDGE_RETRY = {
351
+ attempts: 6,
352
+ baseMs: 2e3,
353
+ maxMs: 3e4
354
+ };
355
+ const TRANSIENT_STATUS = /* @__PURE__ */ new Set([
356
+ 408,
357
+ 409,
358
+ 425,
359
+ 429
360
+ ]);
361
+ const TRANSIENT_MESSAGE = /high demand|overloaded|unavailable|rate.?limit|resource.?exhausted|try again|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|socket hang up|fetch failed/i;
362
+ /** The AI SDK wraps the provider's error once it has retried itself; look through that. */
363
+ const innermost = (error) => {
364
+ const e = error;
365
+ return e?.name === "AI_RetryError" && e.lastError ? innermost(e.lastError) : error;
366
+ };
367
+ /** An error from the provider's API (an AI SDK `APICallError`), not one reeval raised itself. */
368
+ const isApiError = (error) => typeof innermost(error)?.statusCode === "number";
369
+ /** Rate limits, 5xx, overload and network blips. Never a bad request, bad key, abort or timeout. */
370
+ function isTransient(error) {
371
+ const e = innermost(error);
372
+ if (!e || e.name === "AbortError" || e.name === "TimeoutError") return false;
373
+ if (typeof e.statusCode === "number") return e.statusCode >= 500 || TRANSIENT_STATUS.has(e.statusCode) || e.isRetryable === true;
374
+ return e.isRetryable === true || TRANSIENT_MESSAGE.test(e.message ?? String(e));
375
+ }
376
+ function retryAfterMs(error) {
377
+ const header = innermost(error)?.responseHeaders?.["retry-after"];
378
+ const seconds = Number(header);
379
+ return Number.isFinite(seconds) && seconds > 0 ? seconds * 1e3 : 0;
380
+ }
381
+ /** Exponential backoff with equal jitter, at least any `retry-after`, never above `maxMs`. */
382
+ function backoffMs(policy, retry, error) {
383
+ const cap = Math.min(policy.maxMs, policy.baseMs * 2 ** retry);
384
+ const jittered = cap / 2 + Math.random() * (cap / 2);
385
+ return Math.round(Math.min(policy.maxMs, Math.max(jittered, retryAfterMs(error))));
386
+ }
387
+ /** Runs `call` until it succeeds, fails for good, or runs out of tries. */
388
+ async function withRetries(call, policy, signal) {
389
+ const retries = [];
390
+ for (;;) try {
391
+ return {
392
+ value: await call(),
393
+ retries
394
+ };
395
+ } catch (error) {
396
+ if (signal.aborted || retries.length + 1 >= policy.attempts || !isTransient(error)) {
397
+ const failed = error instanceof Error ? error : new Error(String(error));
398
+ failed.retries = retries;
399
+ throw failed;
400
+ }
401
+ const delayMs = backoffMs(policy, retries.length, error);
402
+ const e = innermost(error);
403
+ retries.push({
404
+ error: e?.message ?? String(e),
405
+ delayMs
406
+ });
407
+ await setTimeout(delayMs, void 0, { signal });
408
+ }
409
+ }
410
+ //#endregion
411
+ //#region src/judge/index.ts
412
+ /** The judge the docs recommend, pinned so a model update never silently moves scores. */
413
+ const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.14.0";
414
+ /** The same judge as OpenRouter names it. */
415
+ const OPENROUTER_JUDGE_MODEL = "typesafe/jev-1.14";
416
+ /** The language model `reeval init` sets as `analysis.model` and `reeval analyze` uses. */
417
+ const RECOMMENDED_ANALYSIS_MODEL = "google/gemini-3.8-flash";
418
+ const DEFAULT_THRESHOLD = .8;
419
+ /** Each routing gateway's keys, in the order reeval prefers them. */
420
+ const ROUTED_KEYS = {
421
+ vercel: [VERCEL_KEY, VERCEL_OIDC],
422
+ openrouter: [OPENROUTER_KEY]
423
+ };
424
+ /**
425
+ * Providers `direct()` calls, each through its own AI SDK package and key. Any other prefix needs
426
+ * `vercel()` in the config's gateway list.
427
+ */
428
+ const DIRECT_PROVIDERS = {
429
+ anthropic: {
430
+ key: "ANTHROPIC_API_KEY",
431
+ pkg: "@ai-sdk/anthropic",
432
+ name: "Anthropic"
433
+ },
434
+ google: {
435
+ key: "GOOGLE_GENERATIVE_AI_API_KEY",
436
+ pkg: "@ai-sdk/google",
437
+ name: "Google"
438
+ },
439
+ openai: {
440
+ key: "OPENAI_API_KEY",
441
+ pkg: "@ai-sdk/openai",
442
+ name: "OpenAI"
443
+ },
444
+ "typesafe-ai": {
445
+ key: "TYPESAFE_AI_API_KEY",
446
+ pkg: "@ai-sdk/typesafe-ai",
447
+ name: "TypeSafe AI"
448
+ }
449
+ };
450
+ const isFakeJudge = (m) => typeof m === "object" && m !== null && m.kind === "fake-judge";
451
+ function judgeModelName(model) {
452
+ if (typeof model === "string") return model;
453
+ if (isFakeJudge(model)) return "fake";
454
+ const m = model;
455
+ return m.provider && m.modelId ? `${m.provider}/${m.modelId}` : "custom";
456
+ }
457
+ /**
458
+ * Checks a model string's shape; returns an error message naming it as `what` (with where it was
459
+ * set), or undefined when it's valid.
460
+ */
461
+ function checkModelString(model, what = "The judge model", examples = [RECOMMENDED_JUDGE_MODEL, `google/${model}`]) {
462
+ if (model.includes("/")) return void 0;
463
+ return `${what} needs a provider prefix, e.g. ${examples.map((e) => `"${e}"`).join(" or ")} (got "${model}")`;
464
+ }
465
+ const providerOf = (model) => model.slice(0, model.indexOf("/"));
466
+ /**
467
+ * Environment variables a model needs on `gateway`, in the order reeval prefers them; any one is
468
+ * enough. On direct, its provider's own key (none for a prefix reeval can't call directly); on
469
+ * vertex none, since its token comes from Application Default Credentials; on another gateway,
470
+ * that gateway's.
471
+ */
472
+ function judgeCredentials(model, gateway = "direct") {
473
+ if (isFakeJudge(model) || gateway === "vertex") return [];
474
+ if (gateway !== "direct") return [...ROUTED_KEYS[gateway]];
475
+ const direct = DIRECT_PROVIDERS[providerOf(model)];
476
+ return direct ? [direct.key] : [];
477
+ }
478
+ /** The keys that would work for a model on `on`: `ANTHROPIC_API_KEY (for direct())`. */
479
+ function modelKeyOptions(model, on = "direct") {
480
+ if (on === "vertex") return "Application Default Credentials (for vertex())";
481
+ if (on !== "direct") return `${ROUTED_KEYS[on].join(" or ")} (for ${on}())`;
482
+ const direct = DIRECT_PROVIDERS[providerOf(model)];
483
+ return direct ? `${direct.key} (for direct())` : `vercel() in gateway (direct() calls only ${Object.keys(DIRECT_PROVIDERS).map((p) => `${p}/*`).join(", ")} models)`;
484
+ }
485
+ /** A gateway's credential set in `env`, if any, with the gateway it's for. */
486
+ function setGatewayKey(env) {
487
+ for (const [gateway, keys] of Object.entries(ROUTED_KEYS)) {
488
+ const key = keys.find((k) => env[k]);
489
+ if (key) return {
490
+ key,
491
+ gateway
492
+ };
493
+ }
494
+ }
495
+ /**
496
+ * ` AI_GATEWAY_API_KEY is set, but …` when a gateway's credential is set and the model goes
497
+ * direct, so the key says how to use it.
498
+ */
499
+ function unusedGatewayKey(env, gateway = "direct") {
500
+ if (gateway !== "direct") return "";
501
+ const set = setGatewayKey(env);
502
+ if (!set) return "";
503
+ const name = set.gateway === "vercel" ? "Vercel AI Gateway" : "OpenRouter";
504
+ return ` ${set.key} is set, but no gateway in the config uses it: add ${set.gateway}() to gateway to call the model through ${name}.`;
505
+ }
506
+ /**
507
+ * The provider's own package on direct, the Vercel AI Gateway on vercel, or OpenRouter's package
508
+ * on openrouter, or Vertex AI on vertex, whose token is minted when the model is called.
509
+ * Throws, naming the model, its prefix and the keys that would work, when none is
510
+ * set, when the key it would send has whitespace or a line break in it, or when Vercel's project
511
+ * token has expired.
512
+ */
513
+ function judgeRoute(model, env = process.env, what = "judge model", gateway = "direct") {
514
+ if (gateway === "vertex") return { kind: "vertex" };
515
+ const provider = providerOf(model);
516
+ const direct = DIRECT_PROVIDERS[provider];
517
+ const keys = judgeCredentials(model, gateway);
518
+ if (!keys.length) throw new Error(`${what} "${model}" needs vercel() in the config's gateway list: direct() calls only ${Object.keys(DIRECT_PROVIDERS).map((p) => `${p}/*`).join(", ")} models.`);
519
+ const key = keys.find((k) => env[k]);
520
+ if (key === void 0) throw new Error(`No credentials for ${what} "${model}" (provider "${provider}"). Set ${modelKeyOptions(model, gateway)}, in the environment or .env.local.${unusedGatewayKey(env, gateway)}`);
521
+ if (key === "VERCEL_OIDC_TOKEN") {
522
+ const expired = vercelCredentialProblem(env);
523
+ if (expired) throw new Error(`${what} "${model}": ${expired}`);
524
+ }
525
+ const at = badCharacterAt(env[key]);
526
+ if (at) throw new Error(`${key} for ${what} "${model}" has whitespace or a line break in it at character ${at}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
527
+ if (gateway === "openrouter") return { kind: "openrouter" };
528
+ return direct && key === direct.key ? {
529
+ kind: "direct",
530
+ provider,
531
+ pkg: direct.pkg
532
+ } : { kind: "gateway" };
533
+ }
534
+ /** Transient provider errors (429, 5xx, "high demand") are retried with backoff and recorded. */
535
+ async function runJudge(question, state, options, signal, retry = JUDGE_RETRY) {
536
+ const { model, threshold } = options;
537
+ const gateway = typeof options.gateway === "object" ? options.gateway.kind : options.gateway;
538
+ const fallbacks = typeof options.gateway === "object" && options.gateway.kind === "openrouter" && options.gateway.fallbacks === true;
539
+ const vertex = gateway === "vertex" ? vertexSettings(typeof options.gateway === "object" && options.gateway.kind === "vertex" ? options.gateway : {}) : void 0;
540
+ let score;
541
+ let resolvedModel;
542
+ let billing = {};
543
+ let retries = [];
544
+ if (isFakeJudge(model)) score = typeof model.score === "function" ? model.score(state) : model.score;
545
+ else {
546
+ if (typeof model === "string") {
547
+ const problem = checkModelString(model);
548
+ if (problem) throw new Error(problem);
549
+ }
550
+ const { scoreWithModel } = await import("./model-vRiUycWh.js");
551
+ const scored = await withRetries(() => scoreWithModel(model, redactSecrets(question), state, options.providerOptions, signal, gateway, fallbacks, vertex), retry, signal);
552
+ const { score: scoredAt, resolvedModel: served, ...billed } = scored.value;
553
+ score = scoredAt;
554
+ resolvedModel = served;
555
+ billing = billed;
556
+ retries = scored.retries;
557
+ }
558
+ return {
559
+ verdict: {
560
+ verdict: score >= threshold ? "pass" : "fail",
561
+ score,
562
+ threshold,
563
+ question,
564
+ model: judgeModelName(model),
565
+ ...resolvedModel ? { resolvedModel } : {},
566
+ ...gateway && typeof model === "string" ? { gateway } : {},
567
+ ...billing
568
+ },
569
+ retries
570
+ };
571
+ }
572
+ //#endregion
573
+ export { vercelCredentialProblem as C, servedOnVercel as S, OPENROUTER_KEY as T, FX_VERCEL_ENDPOINTS as _, RECOMMENDED_JUDGE_MODEL as a, VERCEL_KEY as b, judgeCredentials as c, modelKeyOptions as d, runJudge as f, FX_VERCEL_CALL as g, isApiError as h, RECOMMENDED_ANALYSIS_MODEL as i, judgeModelName as l, unusedGatewayKey as m, DIRECT_PROVIDERS as n, checkModelString as o, setGatewayKey as p, OPENROUTER_JUDGE_MODEL as r, isFakeJudge as s, DEFAULT_THRESHOLD as t, judgeRoute as u, VERCEL_GATEWAY as v, OPENROUTER_GATEWAY as w, fxAskedModel as x, VERCEL_HOST as y };
574
+
575
+ //# sourceMappingURL=judge-CKdgEXN7.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"judge-CKdgEXN7.js","names":["claudeCode","codex","goose","opencode","pi","grokBuild","fx","sleep"],"sources":["../src/gateway/effort.ts","../src/gateway/openrouter.ts","../src/gateway/vercel.ts","../src/judge/retry.ts","../src/judge/index.ts"],"mappings":";;;;;;AAGA,MAAa,mBAAmB;;AAGhC,MAAa,aAAa,UAAkB,CAAC,wBAAwB,KAAK,KAAK;;;ACS/E,MAAa,kBAAkB;AAC/B,MAAa,iBAAiB;AAE9B,MAAM,gBAAgB,MAAc,SAAyB,YAC1D;CACC,MAAM;CACN,MAAM,EAAE,OAAO,KAAK;CACpB,QAAQ,CAAC,MAAM;AACjB;;AAGF,MAAa,sBAAmC,EAAE,MAAM,eAAe,qBACrE,SAAA,mBAA4B,gBACxB,GAAG,gBAAgB,iBAAiB,WAAW,eAAe,KAAK,OACnE,KAAA;;AAGN,MAAa,wBAAkC,EAAE,MAAM,kBAAkB,mBACvE,SAAA,mBAA4B,mBACxB;CACE,UAAU;CACV,GAAI,iBAAiB,KAAA,IAAY,EAAE,SAAS,aAAa,IAAI,CAAC;AAChE,IACA,KAAA;;AAGN,MAAM,SAAS,WAAqC,WAAqC;CACvF,YAAY;CACZ,QAAQ;CACR;CACA,gBAAgB;CAChB,aAAa;CACb,UAAU;CACV,OAAO,CAAC,eAAe;AACzB;;AAGA,MAAM,SAAS,cAAsB,UACnC,MAAM,SAAS,GAAG,IAAI,QAAQ,GAAG,SAAS,GAAG;;;;;AAM/C,MAAa,yBAAyB,UACpC,MAAM,SAAS,GAAG,IAAI,QAAQ,aAAa,MAAM,QAAQ,yBAAyB,QAAQ;;;;;;AAO5F,SAASA,eAA2B;CAClC,MAAM,QAAQ,CAAC,aAAa,kBAAkB,CAAC;CAC/C,OAAO;EACL,GAAG,MAAM,CAAC,GAAG,OAAO,aAAa,+BAA+B,CAAC,GAAG,KAAK;EACzE,QAAQ;EACR,SAAS,EAAE,oBAAoB,WAAW,gBAAgB,MAAM;EAChE,SAAS;CACX;AACF;;AAGA,SAASC,UAAsB;CAC7B,MAAM,QAAQ,CAAC,aAAa,mBAAmB,CAAC;CAChD,OAAO;EAAE,GAAG,MAAM,OAAO,KAAK;EAAG,SAAS,MAAM,QAAQ;CAAE;AAC5D;;;;AAKA,SAASC,UAAsB;CAC7B,MAAM,QAAQ,CAAC,aAAa,0BAA0B,CAAC;CACvD,OAAO;EACL,GAAG,MAAM,CAAC,GAAG,OAAO,aAAa,kBAAkB,KAAK,CAAC,GAAG,KAAK;EACjE,SAAS,MAAM,QAAQ;CACzB;AACF;;;;;;AAOA,SAASC,aAAyB;CAChC,MAAM,WAAW,wBAAwB,kBAAkB;CAC3D,IAAI,WAAW,UAAU,MAAM,IAAI,MAAM,SAAS,KAAK;CACvD,MAAM,EAAE,aAAa;CACrB,OAAO;EACL,YAAY;EACZ,QAAQ;EACR,WAAW,kBAAkB,QAAQ;EACrC,gBAAgB,uBAAuB,QAAQ;EAC/C,cAAc,YAAY,oBAAoB,UAAU,OAAO;EAC/D,UAAU;EACV,OAAO,CAAC,GAAG,cAAc,QAAQ,GAAG,YAAY;EAChD,UAAU,UAAU,cAAc;CACpC;AACF;;;;;;AAOA,SAASC,OAAmB;CAC1B,MAAM,QAAQ,CAAC,aAAa,0BAA0B,CAAC;CACvD,OAAO;EAAE,GAAG,MAAM,OAAO,KAAK;EAAG,SAAS,MAAM,QAAQ;CAAE;AAC5D;;;;;AAMA,SAASC,cAA0B;CACjC,MAAM,QAAQ,CAAC,aAAa,0BAA0B,CAAC;CACvD,OAAO;EAAE,GAAG,MAAM,OAAO,KAAK;EAAG,SAAS,MAAM,MAAM;CAAE;AAC1D;;;;;AAMA,SAASC,OAAmB;CAC1B,MAAM,QAAQ,CAAC,aAAa,0BAA0B,CAAC;CACvD,OAAO;EAAE,GAAG,MAAM,OAAO,KAAK;EAAG,SAAS,MAAM,QAAQ;CAAE;AAC5D;;AAGA,MAAa,qBAAqC;CAChD,MAAM;CACN,QAAQ;EACN,eAAeN;EACf,OAAA;EACA,OAAA;EACA,UAAA;EACA,IAAA;EACA,cAAcK;EACd,IAAA;CACF;CACA,UAAU;EACR,QAAQ;EACR,aACE;CACJ;CAKA,gBAAgB;EAAE,eAAe;EAAY,OAAO;EAAW,UAAU;CAAU;AACrF;;;ACvJA,MAAa,cAAc;;AAG3B,MAAa,aAAa;;AAE1B,MAAa,cAAc;;AAG3B,MAAa,oBAAoB,MAAyB,QAAQ,QAChE,CAAC,IAAA,yBAAmB,IAAA,uBAAmB,cAAc;;AAGvD,SAAgB,UAAU,OAAiC;CACzD,MAAM,GAAG,WAAW,MAAM,MAAM,GAAG;CACnC,IAAI,CAAC,SAAS,OAAO,KAAA;CACrB,IAAI;EACF,MAAM,EAAE,QAAQ,KAAK,MAAM,OAAO,KAAK,SAAS,WAAW,CAAC,CAAC,SAAS,MAAM,CAAC;EAG7E,OAAO,OAAO,QAAQ,2BAAW,IAAI,KAAK,MAAM,GAAI,IAAI,KAAA;CAC1D,QAAQ;EACN;CACF;AACF;;;;;AAMA,SAAgB,wBACd,MAAyB,QAAQ,KACjC,sBAAM,IAAI,KAAK,GACK;CACpB,IAAI,iBAAiB,GAAG,MAAA,qBAAmB,OAAO,KAAA;CAClD,MAAM,SAAS,UAAU,IAAA,wBAAoB,EAAE;CAC/C,IAAI,CAAC,UAAU,SAAS,KAAK,OAAO,KAAA;CACpC,OAAO,GAAG,YAAY,cAAc,OAAO,YAAY,EAAE,8EAA8E,WAAW;AACpJ;AAEA,MAAM,YAAY,MAAc,YAC7B;CAAE,MAAM;CAAa,MAAM,EAAE,OAAO,KAAK;CAAG,QAAQ,CAAC,MAAM;AAAE;;;;;AAMhE,MAAa,kBAAgD,EAC3D,MACA,eACA,qBAEA,SAAA,0BAAwB,gBACpB,GAAG,gBAAgB,iBAAiB,WAAW,eAAe,KAAK,OACnE,KAAA;;;;;;AAON,SAAS,WAAW,KAAsC;CACxD,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR,WAAW,CAAC,SAAS,gBAAgB,MAAM,GAAG,SAAS,6BAA6B,MAAM,CAAC;EAC3F,OAAO,CAAC,WAAW;CACrB;AACF;;AAGA,SAAS,MAAM,KAAsC;CACnD,MAAM,YAAY,CAAC,SAAS,iBAAiB,MAAM,CAAC;CACpD,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR;EACA,gBAAgB;EAChB,aAAa;EACb,OAAO,CAAC,WAAW;EACnB,UAAU,UAAW,MAAM,SAAS,GAAG,IAAI,QAAQ,UAAU;CAC/D;AACF;;AAGA,MAAa,iBAAiB;CAC5B,MAAM;CACN,MAAM,EAAE,OAAO,6BAA6B;CAC5C,QAAQ,CAAC,MAAM;AACjB;;;;;AAMA,MAAa,gBAA8C,YAAY;CACrE,IAAI,QAAQ,SAAA,0BAAwB,CAAC,IAAI,OAAO,eAAe,KAAK,KAAK,CAAC,CAAC,KAAK,QAAQ,IAAI,GAC1F,OAAO,KAAA;CACT,IAAI,QAAQ,WAAW,KAAA,KAAa,QAAQ,SAAS,OAAO,QAAQ,SAAS,KAAK,OAAO,KAAA;CACzF,MAAM,QAAQ,QAAQ,UAAU;CAChC,QAAQ,OAAO,UAAU,WAAW,QAAQ,QAAQ,OAAO,KAAA;AAC7D;;AAGA,MAAa,sBAAsB,CACjC;CAAE,MAAM;CAAa,MAAM,EAAE,OAAO,qBAAqB;CAAG,QAAQ,CAAC,KAAK;AAAE,GAC5E,cACF;;AAGA,SAAS,GAAG,KAAsC;CAChD,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR,WAAW;EACX,gBAAgB,CAAC,cAAc;EAC/B,aAAa;EACb,YAAY;EACZ,OAAO,CAAC,WAAW;EACnB,UAAU,UAAW,MAAM,SAAS,GAAG,IAAI,QAAQ,UAAU;CAC/D;AACF;;AAGA,MAAM,YAAY,UAAmB,MAAM,SAAS,GAAG,IAAI,QAAQ,UAAU;;;;;AAM7E,SAAS,MAAM,KAAsC;CACnD,MAAM,QAAQ,CAAC,SAAS,wBAAwB,MAAM,GAAG,SAAS,iBAAiB,MAAM,CAAC;CAC1F,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR,WAAW,CAAC,GAAG,OAAO,SAAS,cAAc,KAAK,CAAC;EACnD,gBAAgB;EAChB,aAAa;EACb,OAAO,CAAC,WAAW;EACnB,SAAS;CACX;AACF;;;;;;AAOA,SAAS,GAAG,KAAsC;CAChD,MAAM,YAAY,CAAC,SAAS,wBAAwB,MAAM,CAAC;CAC3D,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR;EACA,gBAAgB;EAChB,aAAa;EACb,OAAO,CAAC,WAAW;EACnB,SAAS;CACX;AACF;;;;;;AAOA,SAAS,SAAS,KAAsC;CACtD,MAAM,WAAW,wBAAwB,cAAc;CACvD,IAAI,WAAW,UAAU,MAAM,IAAI,MAAM,SAAS,KAAK;CACvD,MAAM,EAAE,aAAa;CACrB,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR,WAAW,kBAAkB,QAAQ;EACrC,gBAAgB,uBAAuB,QAAQ;EAC/C,cAAc,YAAY,oBAAoB,UAAU,OAAO;EAC/D,OAAO,CAAC,GAAG,cAAc,QAAQ,GAAG,YAAY;EAChD,UAAU,UAAU,UAAU;CAChC;AACF;;;;;AAMA,SAAS,UAAU,KAAsC;CACvD,MAAM,YAAY,CAAC,SAAS,wBAAwB,MAAM,CAAC;CAC3D,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR;EACA,gBAAgB;EAChB,aAAa;EACb,OAAO,CAAC,WAAW;EAEnB,iBAAiB;GACf,OAAO,CAAC,YAAY;GACpB,KAAK;EACP;EACA,UAAU,UAAW,MAAM,SAAS,GAAG,IAAI,QAAQ,OAAO;CAC5D;AACF;;AAGA,MAAa,iBAAiC;CAC5C,MAAM;CACN,QAAQ;EACN,eAAe;EACf;EACA;EACA;EACA;EACA,cAAc;EACd;CACF;CACA,UAAU;EACR,QAAQ;EACR,aAAa;CACf;CAIA,gBAAgB;EAAE,OAAO;EAAY,UAAU;CAAU;AAC3D;;;;ACrOA,MAAa,cAA2B;CAAE,UAAU;CAAG,QAAQ;CAAO,OAAO;AAAO;AAEpF,MAAM,mCAAmB,IAAI,IAAI;CAAC;CAAK;CAAK;CAAK;AAAG,CAAC;AACrD,MAAM,oBACJ;;AAUF,MAAM,aAAa,UAA4B;CAC7C,MAAM,IAAI;CACV,OAAO,GAAG,SAAS,mBAAmB,EAAE,YAAY,UAAU,EAAE,SAAS,IAAI;AAC/E;;AAGA,MAAa,cAAc,UACzB,OAAQ,UAAU,KAAK,CAAC,EAAoB,eAAe;;AAG7D,SAAgB,YAAY,OAAyB;CACnD,MAAM,IAAI,UAAU,KAAK;CACzB,IAAI,CAAC,KAAK,EAAE,SAAS,gBAAgB,EAAE,SAAS,gBAAgB,OAAO;CACvE,IAAI,OAAO,EAAE,eAAe,UAC1B,OAAO,EAAE,cAAc,OAAO,iBAAiB,IAAI,EAAE,UAAU,KAAK,EAAE,gBAAgB;CAExF,OAAO,EAAE,gBAAgB,QAAQ,kBAAkB,KAAK,EAAE,WAAW,OAAO,CAAC,CAAC;AAChF;AAEA,SAAS,aAAa,OAAwB;CAC5C,MAAM,SAAU,UAAU,KAAK,CAAC,EAAoB,kBAAkB;CACtE,MAAM,UAAU,OAAO,MAAM;CAC7B,OAAO,OAAO,SAAS,OAAO,KAAK,UAAU,IAAI,UAAU,MAAO;AACpE;;AAGA,SAAgB,UAAU,QAAqB,OAAe,OAAwB;CACpF,MAAM,MAAM,KAAK,IAAI,OAAO,OAAO,OAAO,SAAS,KAAK,KAAK;CAC7D,MAAM,WAAW,MAAM,IAAI,KAAK,OAAO,KAAK,MAAM;CAClD,OAAO,KAAK,MAAM,KAAK,IAAI,OAAO,OAAO,KAAK,IAAI,UAAU,aAAa,KAAK,CAAC,CAAC,CAAC;AACnF;;AAKA,eAAsB,YACpB,MACA,QACA,QAC8C;CAC9C,MAAM,UAAwB,CAAC;CAC/B,SACE,IAAI;EACF,OAAO;GAAE,OAAO,MAAM,KAAK;GAAG;EAAQ;CACxC,SAAS,OAAO;EACd,IAAI,OAAO,WAAW,QAAQ,SAAS,KAAK,OAAO,YAAY,CAAC,YAAY,KAAK,GAAG;GAClF,MAAM,SAAU,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;GACxE,OAAO,UAAU;GACjB,MAAM;EACR;EACA,MAAM,UAAU,UAAU,QAAQ,QAAQ,QAAQ,KAAK;EACvD,MAAM,IAAI,UAAU,KAAK;EACzB,QAAQ,KAAK;GAAE,OAAO,GAAG,WAAW,OAAO,CAAC;GAAG;EAAQ,CAAC;EACxD,MAAME,WAAM,SAAS,KAAA,GAAW,EAAE,OAAO,CAAC;CAC5C;AAEJ;;;;AC5DA,MAAa,0BAA0B;;AAEvC,MAAa,yBAAyB;;AAEtC,MAAa,6BAA6B;AAC1C,MAAa,oBAAoB;;AAKjC,MAAM,cAAuE;CAC3E,QAAQ,CAJY,YAAY,WAIb;CACnB,YAAY,CAAC,cAAc;AAC7B;;;;;AAMA,MAAa,mBAA+E;CAC1F,WAAW;EAAE,KAAK;EAAqB,KAAK;EAAqB,MAAM;CAAY;CACnF,QAAQ;EAAE,KAAK;EAAgC,KAAK;EAAkB,MAAM;CAAS;CACrF,QAAQ;EAAE,KAAK;EAAkB,KAAK;EAAkB,MAAM;CAAS;CACvE,eAAe;EAAE,KAAK;EAAuB,KAAK;EAAuB,MAAM;CAAc;AAC/F;AAcA,MAAa,eAAe,MAC1B,OAAO,MAAM,YAAY,MAAM,QAAS,EAAqB,SAAS;AAExE,SAAgB,eAAe,OAA2B;CACxD,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,IAAI,YAAY,KAAK,GAAG,OAAO;CAC/B,MAAM,IAAI;CACV,OAAO,EAAE,YAAY,EAAE,UAAU,GAAG,EAAE,SAAS,GAAG,EAAE,YAAY;AAClE;;;;;AAMA,SAAgB,iBACd,OACA,OAAO,mBACP,WAAW,CAAC,yBAAyB,UAAU,OAAO,GAClC;CACpB,IAAI,MAAM,SAAS,GAAG,GAAG,OAAO,KAAA;CAChC,OAAO,GAAG,KAAK,iCAAiC,SAAS,KAAK,MAAM,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,EAAE,SAAS,MAAM;AAC5G;AAEA,MAAM,cAAc,UAAkB,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC;;;;;;;AAQvE,SAAgB,iBACd,OACA,UAAuB,UACb;CACV,IAAI,YAAY,KAAK,KAAK,YAAY,UAAU,OAAO,CAAC;CACxD,IAAI,YAAY,UAAU,OAAO,CAAC,GAAG,YAAY,QAAQ;CACzD,MAAM,SAAS,iBAAiB,WAAW,KAAK;CAChD,OAAO,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC;AAClC;;AAGA,SAAgB,gBAAgB,OAAe,KAAkB,UAAkB;CACjF,IAAI,OAAO,UAAU,OAAO;CAC5B,IAAI,OAAO,UAAU,OAAO,GAAG,YAAY,GAAG,CAAC,KAAK,MAAM,EAAE,QAAQ,GAAG;CACvE,MAAM,SAAS,iBAAiB,WAAW,KAAK;CAChD,OAAO,SACH,GAAG,OAAO,IAAI,mBACd,4CAA4C,OAAO,KAAK,gBAAgB,CAAC,CACtE,KAAK,MAAM,GAAG,EAAE,GAAG,CAAC,CACpB,KAAK,IAAI,EAAE;AACpB;;AAGA,SAAgB,cACd,KAC0D;CAC1D,KAAK,MAAM,CAAC,SAAS,SAAS,OAAO,QAAQ,WAAW,GAAG;EACzD,MAAM,MAAM,KAAK,MAAM,MAAM,IAAI,EAAE;EACnC,IAAI,KAAK,OAAO;GAAE;GAAc;EAA8B;CAChE;AAEF;;;;;AAMA,SAAgB,iBAAiB,KAAwB,UAAuB,UAAkB;CAChG,IAAI,YAAY,UAAU,OAAO;CACjC,MAAM,MAAM,cAAc,GAAG;CAC7B,IAAI,CAAC,KAAK,OAAO;CACjB,MAAM,OAAO,IAAI,YAAY,WAAW,sBAAsB;CAC9D,OAAO,IAAI,IAAI,IAAI,qDAAqD,IAAI,QAAQ,0CAA0C,KAAK;AACrI;;;;;;;;AASA,SAAgB,WACd,OACA,MAAyB,QAAQ,KACjC,OAAO,eACP,UAAuB,UACX;CACZ,IAAI,YAAY,UAAU,OAAO,EAAE,MAAM,SAAS;CAClD,MAAM,WAAW,WAAW,KAAK;CACjC,MAAM,SAAS,iBAAiB;CAChC,MAAM,OAAO,iBAAiB,OAAO,OAAO;CAC5C,IAAI,CAAC,KAAK,QACR,MAAM,IAAI,MACR,GAAG,KAAK,IAAI,MAAM,qEAAqE,OAAO,KAC5F,gBACF,CAAC,CACE,KAAK,MAAM,GAAG,EAAE,GAAG,CAAC,CACpB,KAAK,IAAI,EAAE,SAChB;CAEF,MAAM,MAAM,KAAK,MAAM,MAAM,IAAI,EAAE;CACnC,IAAI,QAAQ,KAAA,GACV,MAAM,IAAI,MACR,sBAAsB,KAAK,IAAI,MAAM,eAAe,SAAS,UAAU,gBAAgB,OAAO,OAAO,EAAE,qCAAqC,iBAAiB,KAAK,OAAO,GAC3K;CAEF,IAAI,QAAA,qBAAqB;EACvB,MAAM,UAAU,wBAAwB,GAAG;EAC3C,IAAI,SAAS,MAAM,IAAI,MAAM,GAAG,KAAK,IAAI,MAAM,KAAK,SAAS;CAC/D;CACA,MAAM,KAAK,eAAe,IAAI,IAAc;CAC5C,IAAI,IACF,MAAM,IAAI,MACR,GAAG,IAAI,OAAO,KAAK,IAAI,MAAM,sDAAsD,GAAG,sFACxF;CAEF,IAAI,YAAY,cAAc,OAAO,EAAE,MAAM,aAAa;CAC1D,OAAO,UAAU,QAAQ,OAAO,MAC5B;EAAE,MAAM;EAAU;EAAU,KAAK,OAAO;CAAI,IAC5C,EAAE,MAAM,UAAU;AACxB;;AAMA,eAAsB,SACpB,UACA,OACA,SACA,QACA,QAAqB,aACC;CACtB,MAAM,EAAE,OAAO,cAAc;CAC7B,MAAM,UAAU,OAAO,QAAQ,YAAY,WAAW,QAAQ,QAAQ,OAAO,QAAQ;CACrF,MAAM,YACJ,OAAO,QAAQ,YAAY,YAC3B,QAAQ,QAAQ,SAAS,gBACzB,QAAQ,QAAQ,cAAc;CAChC,MAAM,SACJ,YAAY,WACR,eACE,OAAO,QAAQ,YAAY,YAAY,QAAQ,QAAQ,SAAS,WAC5D,QAAQ,UACR,CAAC,CACP,IACA,KAAA;CACN,IAAI;CACJ,IAAI;CACJ,IAAI,UAAmD,CAAC;CACxD,IAAI,UAAwB,CAAC;CAC7B,IAAI,YAAY,KAAK,GACnB,QAAQ,OAAO,MAAM,UAAU,aAAa,MAAM,MAAM,KAAK,IAAI,MAAM;MAClE;EACL,IAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,UAAU,iBAAiB,KAAK;GACtC,IAAI,SAAS,MAAM,IAAI,MAAM,OAAO;EACtC;EACA,MAAM,EAAE,mBAAmB,MAAM,OAAO;EACxC,MAAM,SAAS,MAAM,kBAEjB,eACE,OACA,cAAc,QAAQ,GACtB,OACA,QAAQ,iBACR,QACA,SACA,WACA,MACF,GACF,OACA,MACF;EACA,MAAM,EAAE,OAAO,UAAU,eAAe,QAAQ,GAAG,WAAW,OAAO;EACrE,QAAQ;EACR,gBAAgB;EAChB,UAAU;EACV,UAAU,OAAO;CACnB;CAWA,OAAO;EAAE,SAAA;GATP,SAAS,SAAS,YAAY,SAAS;GACvC;GACA;GACA;GACA,OAAO,eAAe,KAAK;GAC3B,GAAI,gBAAgB,EAAE,cAAc,IAAI,CAAC;GACzC,GAAI,WAAW,OAAO,UAAU,WAAW,EAAE,QAAQ,IAAI,CAAC;GAC1D,GAAG;EAEU;EAAG;CAAQ;AAC5B"}
package/dist/judge.d.ts CHANGED
@@ -1,4 +1,5 @@
1
- import { Bt as Verdict, D as ConfigJudgeOptions, J as FakeJudgeModel, K as FakeInsights, W as FakeAnalysisModel, Z as FakeVote, ct as JudgeArtifactRef, dt as JudgeModel, ft as JudgeOptions, lt as JudgeCallOptions, n as AnalysisConfig, o as BlindPair, pt as JudgeState, q as FakeInsightsInput, ut as JudgeFailure } from "./types-B-9b1MDV.js";
1
+ import { $ as FakeVote, J as FakeAnswer, O as ConfigJudgeOptions, X as FakeJudgeModel, Zt as Verdict, _t as JudgeOptions, c as BlindPair, gt as JudgeModel, ht as JudgeFailure, mt as JudgeCallOptions, n as AnalysisConfig, pt as JudgeArtifactRef, q as FakeAnalysisModel, s as BlindAttempt, vt as JudgeState } from "./types-coMWHaU2.js";
2
+ import "./vertex-C7PI-WPc.js";
2
3
  //#region src/judge/index.d.ts
3
4
  /** The judge the docs recommend, pinned so a model update never silently moves scores. */
4
5
  export declare const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.14.0";
@@ -14,14 +15,12 @@ export declare const fakeJudge: ({ score }?: {
14
15
  }) => FakeJudgeModel;
15
16
  /**
16
17
  * An analysis model for tests: `reeval analyze` gets `output` (or `output(digest)`) back as the
17
- * model's structured answer, and each compare question's pair gets `vote` (a fixed `'A'`, `'B'` or
18
- * `'tie'`, `{ winner, reason? }`, a function of the blinded pair, or `{ error }`), through the same
19
- * parsing and validation as a real one. Insights (in `reeval run` and `reeval analyze --insights`)
20
- * get `insights` (or `insights(input)`, called per eval for its classification and its sentences,
21
- * then once for the headline), through the same validator: an uncited sentence is dropped, and one
22
- * with a banned word is asked for again, then dropped.
18
+ * model's structured answer, each pairwise question's pair gets `vote` (a fixed `'A'`, `'B'` or
19
+ * `'tie'`, `{ winner, reason? }`, a function of the blinded pair, or `{ error }`), and each
20
+ * question asked of one attempt gets `answer` (its text, `{ choice?, text? }`, a function of the
21
+ * blinded attempt, or `{ error }`), through the same parsing and validation as a real one.
23
22
  */
24
- export declare const fakeAnalysis: ({ output, vote, insights }: Pick<FakeAnalysisModel, 'output' | 'vote' | 'insights'>) => FakeAnalysisModel;
23
+ export declare const fakeAnalysis: ({ output, vote, answer }: Pick<FakeAnalysisModel, 'output' | 'vote' | 'answer'>) => FakeAnalysisModel;
25
24
  //#endregion
26
- export type { AnalysisConfig, BlindPair, ConfigJudgeOptions, FakeAnalysisModel, FakeInsights, FakeInsightsInput, FakeJudgeModel, FakeVote, JudgeArtifactRef, JudgeCallOptions, JudgeFailure, JudgeModel, JudgeOptions, JudgeState, Verdict };
25
+ export type { AnalysisConfig, BlindAttempt, BlindPair, ConfigJudgeOptions, FakeAnalysisModel, FakeAnswer, FakeJudgeModel, FakeVote, JudgeArtifactRef, JudgeCallOptions, JudgeFailure, JudgeModel, JudgeOptions, JudgeState, Verdict };
27
26
  //# sourceMappingURL=judge.d.ts.map
package/dist/judge.js CHANGED
@@ -1,4 +1,4 @@
1
- import { i as RECOMMENDED_JUDGE_MODEL, t as DEFAULT_THRESHOLD } from "./judge-CEDIfRiU.js";
1
+ import { a as RECOMMENDED_JUDGE_MODEL, t as DEFAULT_THRESHOLD } from "./judge-CKdgEXN7.js";
2
2
  //#region src/judge.ts
3
3
  /**
4
4
  * A judge that returns `score` without calling a model, for tests and dry runs. `score` can be a
@@ -10,18 +10,16 @@ const fakeJudge = ({ score = 1 } = {}) => ({
10
10
  });
11
11
  /**
12
12
  * An analysis model for tests: `reeval analyze` gets `output` (or `output(digest)`) back as the
13
- * model's structured answer, and each compare question's pair gets `vote` (a fixed `'A'`, `'B'` or
14
- * `'tie'`, `{ winner, reason? }`, a function of the blinded pair, or `{ error }`), through the same
15
- * parsing and validation as a real one. Insights (in `reeval run` and `reeval analyze --insights`)
16
- * get `insights` (or `insights(input)`, called per eval for its classification and its sentences,
17
- * then once for the headline), through the same validator: an uncited sentence is dropped, and one
18
- * with a banned word is asked for again, then dropped.
13
+ * model's structured answer, each pairwise question's pair gets `vote` (a fixed `'A'`, `'B'` or
14
+ * `'tie'`, `{ winner, reason? }`, a function of the blinded pair, or `{ error }`), and each
15
+ * question asked of one attempt gets `answer` (its text, `{ choice?, text? }`, a function of the
16
+ * blinded attempt, or `{ error }`), through the same parsing and validation as a real one.
19
17
  */
20
- const fakeAnalysis = ({ output, vote, insights }) => ({
18
+ const fakeAnalysis = ({ output, vote, answer }) => ({
21
19
  kind: "fake-analysis",
22
20
  ...output !== void 0 ? { output } : {},
23
21
  ...vote !== void 0 ? { vote } : {},
24
- ...insights !== void 0 ? { insights } : {}
22
+ ...answer !== void 0 ? { answer } : {}
25
23
  });
26
24
  //#endregion
27
25
  export { DEFAULT_THRESHOLD, RECOMMENDED_JUDGE_MODEL, fakeAnalysis, fakeJudge };
package/dist/judge.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"judge.js","names":[],"sources":["../src/judge.ts"],"mappings":";;;;;;AAQA,MAAa,aAAa,EACxB,QAAQ,MAGN,CAAC,OAAuB;CAC1B,MAAM;CACN;AACF;;;;;;;;;;AAWA,MAAa,gBAAgB,EAC3B,QACA,MACA,gBACiF;CACjF,MAAM;CACN,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;CACzC,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;CACrC,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;AAC/C"}
1
+ {"version":3,"file":"judge.js","names":[],"sources":["../src/judge.ts"],"mappings":";;;;;;AAQA,MAAa,aAAa,EACxB,QAAQ,MAGN,CAAC,OAAuB;CAC1B,MAAM;CACN;AACF;;;;;;;;AASA,MAAa,gBAAgB,EAC3B,QACA,MACA,cAC+E;CAC/E,MAAM;CACN,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;CACzC,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;CACrC,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;AAC3C"}