reeval 0.14.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/THIRD_PARTY_NOTICES.md +69 -69
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-BloGVZ_a.js → adapter-CbqbMR7y.js} +37 -8
- package/dist/adapter-CbqbMR7y.js.map +1 -0
- package/dist/{agent-m-_t-IuW.js → agent-Zgn-kiqq.js} +57 -15
- package/dist/agent-Zgn-kiqq.js.map +1 -0
- package/dist/{app-files-C-jwhRDt.js → app-files-ChbvRx3P.js} +4 -4
- package/dist/app-files-ChbvRx3P.js.map +1 -0
- package/dist/{assemble-BDEpelpi.js → assemble-SKQGW_QF.js} +136 -116
- package/dist/assemble-SKQGW_QF.js.map +1 -0
- package/dist/assemble-XBWuK3Ed.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
- package/dist/{match-Bz0nSdzN.js → attempt-host-BOp8JklD.js} +96 -5
- package/dist/attempt-host-BOp8JklD.js.map +1 -0
- package/dist/{capture-CVRwRTnb.js → capture-BM_3100g.js} +1218 -1207
- package/dist/capture-BM_3100g.js.map +1 -0
- package/dist/capture-DSK8O5bA.js +1 -0
- package/dist/{cli-C_4gaJ50.js → cli-PdYrcoKM.js} +106 -170
- package/dist/cli-PdYrcoKM.js.map +1 -0
- package/dist/{cli-spec-CpnbATtI.js → cli-spec-D0qyrPZi.js} +8 -20
- package/dist/cli-spec-D0qyrPZi.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-D5hNjpEx.js → client-Ct6s6OnF.js} +2 -2
- package/dist/{client-D5hNjpEx.js.map → client-Ct6s6OnF.js.map} +1 -1
- package/dist/{command-BXLUfw4h.js → command-4pJYVCT4.js} +33 -20
- package/dist/command-4pJYVCT4.js.map +1 -0
- package/dist/{validate-BhwqrVsm.js → command-9xm5ImqI.js} +387 -59
- package/dist/command-9xm5ImqI.js.map +1 -0
- package/dist/{command-Bxdy62dW.js → command-BqBxLgTc.js} +31 -30
- package/dist/command-BqBxLgTc.js.map +1 -0
- package/dist/{command-I1euLLio.js → command-DbSMcxZL.js} +7 -7
- package/dist/{command-I1euLLio.js.map → command-DbSMcxZL.js.map} +1 -1
- package/dist/{commands-jzigbNFb.js → commands-BU6YJCz9.js} +175 -72
- package/dist/commands-BU6YJCz9.js.map +1 -0
- package/dist/{commands-p1S4nw_H.js → commands-Bi2SEQtr.js} +1 -1
- package/dist/{config-BrdHA9bJ.js → config-k6m7XbAz.js} +499 -271
- package/dist/config-k6m7XbAz.js.map +1 -0
- package/dist/{contrasts-DRxMQQTC.js → contrasts-DLZF4dGF.js} +3 -3
- package/dist/{contrasts-DRxMQQTC.js.map → contrasts-DLZF4dGF.js.map} +1 -1
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-CpsifVln.js +1530 -0
- package/dist/db-CpsifVln.js.map +1 -0
- package/dist/db-DkD-F-X-.js +2 -0
- package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
- package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
- package/dist/debug-DeM5pswK.js +1 -0
- package/dist/{docker-BnRlTpZA.js → docker-51WvhN_X.js} +5 -4
- package/dist/docker-51WvhN_X.js.map +1 -0
- package/dist/{docker-D01slDF3.js → docker-B4-rx4dA.js} +26 -8
- package/dist/docker-B4-rx4dA.js.map +1 -0
- package/dist/{docker-DivfZl2u.js → docker-YFj4H1Cr.js} +1 -1
- package/dist/{doctor-DQ2p0euG.js → doctor-BuQnfz4e.js} +275 -37
- package/dist/doctor-BuQnfz4e.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-CO0P0soF.js → environment-BOFgzFDg.js} +115 -296
- package/dist/environment-BOFgzFDg.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
- package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
- package/dist/eval-source-Dje3rERU.js +145 -0
- package/dist/eval-source-Dje3rERU.js.map +1 -0
- package/dist/{evaluate-BUmsN7yG.js → evaluate-BPP9s3s6.js} +367 -130
- package/dist/evaluate-BPP9s3s6.js.map +1 -0
- package/dist/evaluate-dqXccn4k.d.ts +44 -0
- package/dist/{files-B-CJeeEh.js → files-Bi461Be2.js} +197 -91
- package/dist/files-Bi461Be2.js.map +1 -0
- package/dist/files-C2h2Cw4F.js +1 -0
- package/dist/files-Clipx4OW.js +37 -0
- package/dist/files-Clipx4OW.js.map +1 -0
- package/dist/files-DmUrRmmL.js +1 -0
- package/dist/{files-kUCrswI6.js → files-H24pz0hL.js} +20 -9
- package/dist/files-H24pz0hL.js.map +1 -0
- package/dist/gateway.d.ts +22 -9
- package/dist/gateway.js +30 -8
- package/dist/gateway.js.map +1 -1
- package/dist/{git-Bk8DSwHy.js → git-CsVZkNNL.js} +3 -3
- package/dist/{git-Bk8DSwHy.js.map → git-CsVZkNNL.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +12 -8
- package/dist/harness.js +1 -1
- package/dist/{images-DzLeFjLx.js → images-4mwhNkvh.js} +8 -8
- package/dist/{images-DzLeFjLx.js.map → images-4mwhNkvh.js.map} +1 -1
- package/dist/index.d.ts +51 -10
- package/dist/index.js +5 -5
- package/dist/init-BDEOopK7.js +1 -0
- package/dist/{init-SRalM5xF.js → init-cOk3D44b.js} +10 -11
- package/dist/init-cOk3D44b.js.map +1 -0
- package/dist/{invocations-DzyKSJBa.js → invocations-DW-lTk2f.js} +10 -6
- package/dist/invocations-DW-lTk2f.js.map +1 -0
- package/dist/{judge-yLvjC5A3.js → judge-CKdgEXN7.js} +236 -46
- package/dist/judge-CKdgEXN7.js.map +1 -0
- package/dist/judge.d.ts +5 -7
- package/dist/judge.js +4 -8
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
- package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
- package/dist/{matchers-D4eUqx4j.js → matchers-DT2TaGix.js} +2 -2
- package/dist/{matchers-D4eUqx4j.js.map → matchers-DT2TaGix.js.map} +1 -1
- package/dist/{model-1ZLr9Dcv.js → model-Bq3G0OG1.js} +29 -36
- package/dist/model-Bq3G0OG1.js.map +1 -0
- package/dist/{model-CgnOpss4.js → model-CTnyvh7H.js} +292 -68
- package/dist/model-CTnyvh7H.js.map +1 -0
- package/dist/model-D-SWrLZG.js +1 -0
- package/dist/model-g3XnzUv0.js +1 -0
- package/dist/{model-B45lDUV1.js → model-vRiUycWh.js} +22 -13
- package/dist/model-vRiUycWh.js.map +1 -0
- package/dist/{node-BSq5qHnQ.js → node-BVbbnFtv.js} +72 -50
- package/dist/node-BVbbnFtv.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-CWRwLX-G.js +34 -0
- package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
- package/dist/{pairs-CHfZNRT6.js → pairs-puYi-zj2.js} +11 -4
- package/dist/pairs-puYi-zj2.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{patch-DWf-I_Q_.js → patch-DKJiC5Sq.js} +5 -7
- package/dist/patch-DKJiC5Sq.js.map +1 -0
- package/dist/{plan-BIwUzlB8.js → plan-DdWkM6SW.js} +64 -29
- package/dist/plan-DdWkM6SW.js.map +1 -0
- package/dist/{plugins-Dkp_H3pB.js → plugins-D-CP6mMd.js} +253 -17
- package/dist/plugins-D-CP6mMd.js.map +1 -0
- package/dist/{preflight-DJCD_BDn.js → preflight-9ixuXzsx.js} +3 -3
- package/dist/{preflight-DJCD_BDn.js.map → preflight-9ixuXzsx.js.map} +1 -1
- package/dist/{preflight-D9y-HvZQ.js → preflight-DTiVv0gY.js} +1 -1
- package/dist/{prepare-CIxCGG0M.js → prepare-BfyikQit.js} +8 -8
- package/dist/{prepare-CIxCGG0M.js.map → prepare-BfyikQit.js.map} +1 -1
- package/dist/{prepare-BJG_pD54.js → prepare-tmBBf4W8.js} +2 -2
- package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-CfZpFpAh.js → provenance-BizwUn3h.js} +4 -4
- package/dist/{provenance-CfZpFpAh.js.map → provenance-BizwUn3h.js.map} +1 -1
- package/dist/{providers-DFk_6W4K.js → providers-B6b9Tnyi.js} +5 -5
- package/dist/{providers-DFk_6W4K.js.map → providers-B6b9Tnyi.js.map} +1 -1
- package/dist/{providers-D1Zxp2he.js → providers-DkOupEBl.js} +1 -1
- package/dist/query-Clm2mNSm.js +1 -0
- package/dist/{query-CemclveB.js → query-DyM-VExS.js} +3 -3
- package/dist/{query-CemclveB.js.map → query-DyM-VExS.js.map} +1 -1
- package/dist/{questions-bAudIc04.js → questions-BRhU8MzB.js} +100 -25
- package/dist/questions-BRhU8MzB.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-BwJ8pcTQ.js +1 -0
- package/dist/{rules-VhftPOOT.js → remarks-C7i2LMRb.js} +296 -66
- package/dist/remarks-C7i2LMRb.js.map +1 -0
- package/dist/{header-BLPP6aCU.js → remarks-D_3qhBWU.js} +34 -9
- package/dist/remarks-D_3qhBWU.js.map +1 -0
- package/dist/{remarks-mujJv1Ey.js → remarks-dA8LyoPc.js} +9 -5
- package/dist/remarks-dA8LyoPc.js.map +1 -0
- package/dist/{remote-CKcEKja4.js → remote-DFIzcx_6.js} +39 -99
- package/dist/remote-DFIzcx_6.js.map +1 -0
- package/dist/resolve--rQZ8u6Z.js +1 -0
- package/dist/{resolve-CBoKzQMP.js → resolve-DWSJpKHX.js} +46 -30
- package/dist/resolve-DWSJpKHX.js.map +1 -0
- package/dist/{results-CMgZEANy.js → results-BC3pcG_Z.js} +15 -30
- package/dist/results-BC3pcG_Z.js.map +1 -0
- package/dist/{resume-B9SOSnK9.js → resume-j-rcIryi.js} +47 -17
- package/dist/resume-j-rcIryi.js.map +1 -0
- package/dist/{runtime-BgQRyCLl.js → runtime-d56Gg3J5.js} +30 -13
- package/dist/runtime-d56Gg3J5.js.map +1 -0
- package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmln11gU.js} +2 -2
- package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmln11gU.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-TGW4WyAz.js → server-BOoH10J8.js} +51 -33
- package/dist/server-BOoH10J8.js.map +1 -0
- package/dist/server-DpuVAbrO.js +1 -0
- package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/source-BFd6f0-R.js +1 -0
- package/dist/{source-B1WvBIQt.js → source-wD04-rRG.js} +15 -10
- package/dist/source-wD04-rRG.js.map +1 -0
- package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
- package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
- package/dist/{src-CyaDNG9r.js → src-6_zz7oJs.js} +1 -1
- package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
- package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
- package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BYPMM1x7.js +2 -0
- package/dist/src-Bp4aN3rA.js +2 -0
- package/dist/src-C7upnZSd.js +2 -0
- package/dist/{src-8zoX_qrb.js → src-D91Hd6hT.js} +546 -1060
- package/dist/src-D91Hd6hT.js.map +1 -0
- package/dist/src-DH-9IO7f.js +1 -0
- package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/{src-DjRI-IeE.js → src-DgHuStEi.js} +134 -34
- package/dist/{src-DjRI-IeE.js.map → src-DgHuStEi.js.map} +1 -1
- package/dist/src-HSxBazcg.js +3 -0
- package/dist/src-LD1BAJFa.js +2 -0
- package/dist/src-dlX-Lmxg.js +2 -0
- package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
- package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-kYeaJJrl.js +2 -0
- package/dist/state-B5PLm5qY.js +1 -0
- package/dist/{state-B1t143wG.js → state-DrJ687ut.js} +15 -15
- package/dist/{state-B1t143wG.js.map → state-DrJ687ut.js.map} +1 -1
- package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
- package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
- package/dist/suite-22Kt9Grx.js +2 -0
- package/dist/{suite-WgT7m6Vs.js → suite-Co8cwelb.js} +48 -63
- package/dist/suite-Co8cwelb.js.map +1 -0
- package/dist/{summary-DIGEVr08.js → summary-DLFHVoux.js} +80 -270
- package/dist/summary-DLFHVoux.js.map +1 -0
- package/dist/testing.d.ts +417 -142
- package/dist/testing.js +12 -10
- package/dist/testing.js.map +1 -1
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
- package/dist/transcript-DxPaWHTZ.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-DXBbP9Mk.d.ts → types-coMWHaU2.d.ts} +253 -733
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +536 -121
- package/dist/view/index.js +9 -9
- package/dist/{views-sXpGE6a2.js → views-BFO1BLKu.js} +9 -10
- package/dist/{views-sXpGE6a2.js.map → views-BFO1BLKu.js.map} +1 -1
- package/dist/views-BOf8qEL8.js +2 -0
- package/dist/{worker-entry-Duo9oAeT.js → worker-entry-CJtdsKBP.js} +19 -12
- package/dist/worker-entry-CJtdsKBP.js.map +1 -0
- package/dist/{workspace-CDI68wtX.js → workspace-BRxjfa8N.js} +5 -5
- package/dist/{workspace-CDI68wtX.js.map → workspace-BRxjfa8N.js.map} +1 -1
- package/dist/workspace-DDHzkMjm.js +1 -0
- package/docker/seccomp-default.json +1 -0
- package/package.json +5 -1
- package/skills/eval-rubric/SKILL.md +35 -104
- package/dist/adapter-BloGVZ_a.js.map +0 -1
- package/dist/agent-m-_t-IuW.js.map +0 -1
- package/dist/app-files-C-jwhRDt.js.map +0 -1
- package/dist/assemble-BDEpelpi.js.map +0 -1
- package/dist/assemble-C5bfncHU.js +0 -1
- package/dist/capture-CVRwRTnb.js.map +0 -1
- package/dist/capture-CherbjzB.js +0 -1
- package/dist/cli-C_4gaJ50.js.map +0 -1
- package/dist/cli-spec-CpnbATtI.js.map +0 -1
- package/dist/command-BWAuwQNk.js +0 -91
- package/dist/command-BWAuwQNk.js.map +0 -1
- package/dist/command-BXLUfw4h.js.map +0 -1
- package/dist/command-Be6BSE2L.js +0 -335
- package/dist/command-Be6BSE2L.js.map +0 -1
- package/dist/command-Bxdy62dW.js.map +0 -1
- package/dist/commands-jzigbNFb.js.map +0 -1
- package/dist/config-BrdHA9bJ.js.map +0 -1
- package/dist/cost-D4elx933.js.map +0 -1
- package/dist/cost-DmnAod9x.js +0 -1
- package/dist/db-CAfksinX.js +0 -2
- package/dist/db-DO0p7xgP.js +0 -619
- package/dist/db-DO0p7xgP.js.map +0 -1
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-BnRlTpZA.js.map +0 -1
- package/dist/docker-D01slDF3.js.map +0 -1
- package/dist/doctor-DQ2p0euG.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-CO0P0soF.js.map +0 -1
- package/dist/evaluate-BUmsN7yG.js.map +0 -1
- package/dist/evaluate-C_sgRA-V.d.ts +0 -134
- package/dist/facts-DzOUTreB.js +0 -906
- package/dist/facts-DzOUTreB.js.map +0 -1
- package/dist/files-B-CJeeEh.js.map +0 -1
- package/dist/files-Bp5j5BqI.js +0 -1
- package/dist/files-C2sjmXhk.js +0 -1
- package/dist/files-C_gQNVB-.js +0 -65
- package/dist/files-C_gQNVB-.js.map +0 -1
- package/dist/files-kUCrswI6.js.map +0 -1
- package/dist/generate-DZ7vp69F.js +0 -1647
- package/dist/generate-DZ7vp69F.js.map +0 -1
- package/dist/harness-COVXSwJ6.js +0 -354
- package/dist/harness-COVXSwJ6.js.map +0 -1
- package/dist/header-BLPP6aCU.js.map +0 -1
- package/dist/init-RgCQXZhb.js +0 -1
- package/dist/init-SRalM5xF.js.map +0 -1
- package/dist/insights-TyAvz9aV.js +0 -68
- package/dist/insights-TyAvz9aV.js.map +0 -1
- package/dist/invocations-DzyKSJBa.js.map +0 -1
- package/dist/judge-yLvjC5A3.js.map +0 -1
- package/dist/match-Bz0nSdzN.js.map +0 -1
- package/dist/model-1ZLr9Dcv.js.map +0 -1
- package/dist/model-B45lDUV1.js.map +0 -1
- package/dist/model-BTZq8HIZ.js +0 -1
- package/dist/model-CgnOpss4.js.map +0 -1
- package/dist/model-Ch8kX87u.js +0 -1
- package/dist/node-BSq5qHnQ.js.map +0 -1
- package/dist/opencode-providers-klkqgzvb.js +0 -3276
- package/dist/opencode-providers-klkqgzvb.js.map +0 -1
- package/dist/pairs-CHfZNRT6.js.map +0 -1
- package/dist/patch-DWf-I_Q_.js.map +0 -1
- package/dist/plan-BIwUzlB8.js.map +0 -1
- package/dist/plugins-Dkp_H3pB.js.map +0 -1
- package/dist/prices-BcLEXvdY.js.map +0 -1
- package/dist/query-Cvbwl804.js +0 -1
- package/dist/questions-bAudIc04.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-CXwSnSjR.js +0 -278
- package/dist/remarks-CXwSnSjR.js.map +0 -1
- package/dist/remarks-CyAu2DiB.js +0 -1
- package/dist/remarks-mujJv1Ey.js.map +0 -1
- package/dist/remote-CKcEKja4.js.map +0 -1
- package/dist/resolve-B5-uqmJ6.js +0 -1
- package/dist/resolve-CBoKzQMP.js.map +0 -1
- package/dist/results-CMgZEANy.js.map +0 -1
- package/dist/resume-B9SOSnK9.js.map +0 -1
- package/dist/rules-VhftPOOT.js.map +0 -1
- package/dist/run-CwIEK16q.js +0 -88
- package/dist/run-CwIEK16q.js.map +0 -1
- package/dist/runtime-BgQRyCLl.js.map +0 -1
- package/dist/server-CHRSgKRp.js +0 -1
- package/dist/server-TGW4WyAz.js.map +0 -1
- package/dist/shared-BDUzStad.js.map +0 -1
- package/dist/source-B1WvBIQt.js.map +0 -1
- package/dist/source-DAzztZwU.js +0 -1
- package/dist/src-1jHbFaGW.js +0 -1
- package/dist/src-8zoX_qrb.js.map +0 -1
- package/dist/src-C_9wgowC.js +0 -2
- package/dist/src-ChhtI6BP.js.map +0 -1
- package/dist/src-D005Cbqz.js +0 -2
- package/dist/src-DIfRbp5a.js +0 -3
- package/dist/src-Dg8121Xy.js +0 -2
- package/dist/src-ElBqEN-S.js +0 -2
- package/dist/src-NSdQ9zI6.js.map +0 -1
- package/dist/src-YHGaUrGE.js +0 -2
- package/dist/src-mJVxw1Jm.js +0 -2
- package/dist/state-vznMh2wl.js +0 -1
- package/dist/suite-BsWGHzlG.js +0 -2
- package/dist/suite-WgT7m6Vs.js.map +0 -1
- package/dist/summary-DIGEVr08.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-DF3_p5pE.js +0 -1
- package/dist/validate-BhwqrVsm.js.map +0 -1
- package/dist/views-BW_C5eVH.js +0 -2
- package/dist/worker-entry-Duo9oAeT.js.map +0 -1
- package/dist/workspace-QZCacP8A.js +0 -1
|
@@ -1,6 +1,154 @@
|
|
|
1
|
-
import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-
|
|
2
|
-
import {
|
|
1
|
+
import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-Bz0l7OZI.js";
|
|
2
|
+
import { v as vertexSettings } from "./vertex-DypYjx71.js";
|
|
3
|
+
import { m as redactSecrets, r as badCharacterAt } from "./redact-DVIaahLv.js";
|
|
3
4
|
import { setTimeout } from "node:timers/promises";
|
|
5
|
+
//#region src/gateway/effort.ts
|
|
6
|
+
/** Every model: no run has checked the harness's effort reaches any of them there. */
|
|
7
|
+
const everyModel = () => true;
|
|
8
|
+
/** Any but a Gemini model (`gemini-3.8-flash`, `google/gemini-3.8-flash`). */
|
|
9
|
+
const notGemini = (model) => !/^(?:google\/)?gemini-/.test(model);
|
|
10
|
+
//#endregion
|
|
11
|
+
//#region src/gateway/openrouter.ts
|
|
12
|
+
const OPENROUTER_HOST = "openrouter.ai";
|
|
13
|
+
const OPENROUTER_KEY = "OPENROUTER_API_KEY";
|
|
14
|
+
const onOpenRouter = (path, method = "POST") => ({
|
|
15
|
+
host: OPENROUTER_HOST,
|
|
16
|
+
path: { exact: path },
|
|
17
|
+
method: [method]
|
|
18
|
+
});
|
|
19
|
+
/** The response names the model that served it on OpenRouter's host, in OpenRouter's form. */
|
|
20
|
+
const servedOnOpenRouter = ({ host, responseModel, responseEffort }) => host === "openrouter.ai" && responseModel ? `${responseModel}${responseEffort ? `[effort=${responseEffort}]` : ""}` : void 0;
|
|
21
|
+
/** The upstream provider OpenRouter's response says served the call, and what it charged. */
|
|
22
|
+
const upstreamOnOpenRouter = ({ host, responseProvider, responseCost }) => host === "openrouter.ai" && responseProvider ? {
|
|
23
|
+
provider: responseProvider,
|
|
24
|
+
...responseCost !== void 0 ? { costUsd: responseCost } : {}
|
|
25
|
+
} : void 0;
|
|
26
|
+
/** What every OpenRouter route shares: the key under its own name, and what the responses say. */
|
|
27
|
+
const route = (endpoints, calls) => ({
|
|
28
|
+
credential: OPENROUTER_KEY,
|
|
29
|
+
authAs: OPENROUTER_KEY,
|
|
30
|
+
endpoints,
|
|
31
|
+
modelResponses: calls,
|
|
32
|
+
servedModel: servedOnOpenRouter,
|
|
33
|
+
upstream: upstreamOnOpenRouter,
|
|
34
|
+
hosts: [OPENROUTER_HOST]
|
|
35
|
+
});
|
|
36
|
+
/** An id as OpenRouter names it: as written when it names a provider, else under `provider`. */
|
|
37
|
+
const under = (provider) => (model) => model.includes("/") ? model : `${provider}/${model}`;
|
|
38
|
+
/**
|
|
39
|
+
* An Anthropic id as OpenRouter names it, with the version dotted (`anthropic/claude-haiku-4.5`
|
|
40
|
+
* for `claude-haiku-4-5`), unless the id names a provider.
|
|
41
|
+
*/
|
|
42
|
+
const anthropicOnOpenRouter = (model) => model.includes("/") ? model : `anthropic/${model.replace(/-(\d{1,2})-(\d{1,2})$/, "-$1.$2")}`;
|
|
43
|
+
/**
|
|
44
|
+
* Claude Code on OpenRouter's Anthropic Messages API: `ANTHROPIC_BASE_URL` at OpenRouter and the
|
|
45
|
+
* key as `ANTHROPIC_AUTH_TOKEN`, which the CLI sends as a Bearer token. On another base URL it
|
|
46
|
+
* makes none of its `/api/claude_code/*` calls.
|
|
47
|
+
*/
|
|
48
|
+
function claudeCode$1() {
|
|
49
|
+
const calls = [onOpenRouter("/api/v1/messages")];
|
|
50
|
+
return {
|
|
51
|
+
...route([...calls, onOpenRouter("/api/v1/messages/count_tokens")], calls),
|
|
52
|
+
authAs: "ANTHROPIC_AUTH_TOKEN",
|
|
53
|
+
authEnv: { ANTHROPIC_BASE_URL: `https://${OPENROUTER_HOST}/api` },
|
|
54
|
+
modelId: anthropicOnOpenRouter
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
/** Codex on a Responses API provider at OpenRouter, whose ids name the provider (`openai/…`). */
|
|
58
|
+
function codex$1() {
|
|
59
|
+
const calls = [onOpenRouter("/api/v1/responses")];
|
|
60
|
+
return {
|
|
61
|
+
...route(calls, calls),
|
|
62
|
+
modelId: under("openai")
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Goose on its `openrouter` provider: Chat Completions, and the model list it fetches each turn.
|
|
67
|
+
*/
|
|
68
|
+
function goose$1() {
|
|
69
|
+
const calls = [onOpenRouter("/api/v1/chat/completions")];
|
|
70
|
+
return {
|
|
71
|
+
...route([...calls, onOpenRouter("/api/v1/models", "GET")], calls),
|
|
72
|
+
modelId: under("google")
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* OpenCode on its own `openrouter` provider, with the model as `openrouter/<id>`: the requests the
|
|
77
|
+
* pinned CLI sends that provider's key on (harness-opencode's provider table), plus Exa's host for
|
|
78
|
+
* web search.
|
|
79
|
+
*/
|
|
80
|
+
function opencode$1() {
|
|
81
|
+
const resolved = resolveOpenCodeProvider("openrouter/model");
|
|
82
|
+
if ("error" in resolved) throw new Error(resolved.error);
|
|
83
|
+
const { provider } = resolved;
|
|
84
|
+
return {
|
|
85
|
+
credential: OPENROUTER_KEY,
|
|
86
|
+
authAs: OPENROUTER_KEY,
|
|
87
|
+
endpoints: providerEndpoints(provider),
|
|
88
|
+
modelResponses: providerModelResponses(provider),
|
|
89
|
+
servedModel: (request) => providerServedModel(provider, request),
|
|
90
|
+
upstream: upstreamOnOpenRouter,
|
|
91
|
+
hosts: [...providerHosts(provider), "mcp.exa.ai"],
|
|
92
|
+
modelId: (model) => `openrouter/${model}`
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* pi on its own `openrouter` provider: Chat Completions, a Bearer key, a model from pi's catalog
|
|
97
|
+
* for OpenRouter, each thinking level sent as OpenRouter's `reasoning.effort`. `--offline` skips
|
|
98
|
+
* the model list pi would fetch to fit that catalog to the key.
|
|
99
|
+
*/
|
|
100
|
+
function pi$1() {
|
|
101
|
+
const calls = [onOpenRouter("/api/v1/chat/completions")];
|
|
102
|
+
return {
|
|
103
|
+
...route(calls, calls),
|
|
104
|
+
modelId: under("google")
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Grok Build with the model as a `[model.<id>]` table on OpenRouter's Chat Completions API, keyed
|
|
109
|
+
* by `env_key`; it then calls nothing on xAI's API.
|
|
110
|
+
*/
|
|
111
|
+
function grokBuild$1() {
|
|
112
|
+
const calls = [onOpenRouter("/api/v1/chat/completions")];
|
|
113
|
+
return {
|
|
114
|
+
...route(calls, calls),
|
|
115
|
+
modelId: under("x-ai")
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* fx with OpenRouter's Chat Completions API as its configured provider `openrouter`, through the
|
|
120
|
+
* relay, and an OpenAI id as `openai/<id>`. fx sends no `reasoning_effort` to a configured provider.
|
|
121
|
+
*/
|
|
122
|
+
function fx$1() {
|
|
123
|
+
const calls = [onOpenRouter("/api/v1/chat/completions")];
|
|
124
|
+
return {
|
|
125
|
+
...route(calls, calls),
|
|
126
|
+
modelId: under("openai")
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
/** OpenRouter: the harnesses reeval runs there, and why it doesn't run the rest. */
|
|
130
|
+
const OPENROUTER_GATEWAY = {
|
|
131
|
+
name: "OpenRouter",
|
|
132
|
+
routes: {
|
|
133
|
+
"claude-code": claudeCode$1,
|
|
134
|
+
codex: codex$1,
|
|
135
|
+
goose: goose$1,
|
|
136
|
+
opencode: opencode$1,
|
|
137
|
+
fx: fx$1,
|
|
138
|
+
"grok-build": grokBuild$1,
|
|
139
|
+
pi: pi$1
|
|
140
|
+
},
|
|
141
|
+
refusals: {
|
|
142
|
+
cursor: "Cursor's agent only calls Cursor's own API",
|
|
143
|
+
antigravity: "Antigravity's CLI only calls Google's API, with a Gemini key, and OpenRouter has no Gemini API"
|
|
144
|
+
},
|
|
145
|
+
unprovenEffort: {
|
|
146
|
+
"claude-code": everyModel,
|
|
147
|
+
goose: notGemini,
|
|
148
|
+
opencode: notGemini
|
|
149
|
+
}
|
|
150
|
+
};
|
|
151
|
+
//#endregion
|
|
4
152
|
//#region src/gateway/vercel.ts
|
|
5
153
|
const VERCEL_HOST = "ai-gateway.vercel.sh";
|
|
6
154
|
/** What the adapters take Vercel's key as, whichever host variable it came from. */
|
|
@@ -120,6 +268,23 @@ function goose(env) {
|
|
|
120
268
|
};
|
|
121
269
|
}
|
|
122
270
|
/**
|
|
271
|
+
* pi on a provider of its own on Vercel's Chat Completions API, a Bearer key: a Gemini model from
|
|
272
|
+
* pi's catalog, each thinking level sent as Gemini's own `thinkingConfig` under Vercel's
|
|
273
|
+
* `providerOptions` (Vercel's `reasoning_effort` maps every Gemini 3 level but `low` to `high`).
|
|
274
|
+
*/
|
|
275
|
+
function pi(env) {
|
|
276
|
+
const endpoints = [onVercel("/v1/chat/completions", "POST")];
|
|
277
|
+
return {
|
|
278
|
+
credential: vercelCredential(env),
|
|
279
|
+
authAs: VERCEL_KEY,
|
|
280
|
+
endpoints,
|
|
281
|
+
modelResponses: endpoints,
|
|
282
|
+
servedModel: servedOnVercel,
|
|
283
|
+
hosts: [VERCEL_HOST],
|
|
284
|
+
modelId: googleId
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
/**
|
|
123
288
|
* OpenCode on its own `vercel` provider, with the model as `vercel/<id>`: the requests the pinned
|
|
124
289
|
* CLI sends that provider's key on (harness-opencode's provider table), plus Exa's host for web
|
|
125
290
|
* search.
|
|
@@ -158,31 +323,27 @@ function grokBuild(env) {
|
|
|
158
323
|
modelId: (model) => model.includes("/") ? model : `xai/${model}`
|
|
159
324
|
};
|
|
160
325
|
}
|
|
161
|
-
/**
|
|
162
|
-
const
|
|
163
|
-
"
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
326
|
+
/** Vercel AI Gateway: the harnesses reeval runs there, and why it doesn't run the rest. */
|
|
327
|
+
const VERCEL_GATEWAY = {
|
|
328
|
+
name: "Vercel AI Gateway",
|
|
329
|
+
routes: {
|
|
330
|
+
"claude-code": claudeCode,
|
|
331
|
+
codex,
|
|
332
|
+
goose,
|
|
333
|
+
opencode,
|
|
334
|
+
fx,
|
|
335
|
+
"grok-build": grokBuild,
|
|
336
|
+
pi
|
|
337
|
+
},
|
|
338
|
+
refusals: {
|
|
339
|
+
cursor: "Cursor's agent only calls Cursor's own API",
|
|
340
|
+
antigravity: "Antigravity's CLI only calls Google's API, with a Gemini key"
|
|
341
|
+
},
|
|
342
|
+
unprovenEffort: {
|
|
343
|
+
goose: everyModel,
|
|
344
|
+
opencode: notGemini
|
|
345
|
+
}
|
|
175
346
|
};
|
|
176
|
-
/**
|
|
177
|
-
* The harnesses on Vercel whose `effort` reaches the model by a path no real run has checked: the
|
|
178
|
-
* plan refuses `effort` with them on `vercel()`. Claude Code's goes as on Anthropic's API, and
|
|
179
|
-
* Codex's response names the effort it ran at.
|
|
180
|
-
*/
|
|
181
|
-
const VERCEL_UNPROVEN_EFFORT = [
|
|
182
|
-
"goose",
|
|
183
|
-
"opencode",
|
|
184
|
-
"grok-build"
|
|
185
|
-
];
|
|
186
347
|
//#endregion
|
|
187
348
|
//#region src/judge/retry.ts
|
|
188
349
|
/** Up to six tries over about a minute: long enough to ride out a provider's "high demand". */
|
|
@@ -250,10 +411,16 @@ async function withRetries(call, policy, signal) {
|
|
|
250
411
|
//#region src/judge/index.ts
|
|
251
412
|
/** The judge the docs recommend, pinned so a model update never silently moves scores. */
|
|
252
413
|
const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.14.0";
|
|
253
|
-
/** The
|
|
414
|
+
/** The same judge as OpenRouter names it. */
|
|
415
|
+
const OPENROUTER_JUDGE_MODEL = "typesafe/jev-1.14";
|
|
416
|
+
/** The language model `reeval init` sets as `analysis.model` and `reeval analyze` uses. */
|
|
254
417
|
const RECOMMENDED_ANALYSIS_MODEL = "google/gemini-3.8-flash";
|
|
255
418
|
const DEFAULT_THRESHOLD = .8;
|
|
256
|
-
|
|
419
|
+
/** Each routing gateway's keys, in the order reeval prefers them. */
|
|
420
|
+
const ROUTED_KEYS = {
|
|
421
|
+
vercel: [VERCEL_KEY, VERCEL_OIDC],
|
|
422
|
+
openrouter: [OPENROUTER_KEY]
|
|
423
|
+
};
|
|
257
424
|
/**
|
|
258
425
|
* Providers `direct()` calls, each through its own AI SDK package and key. Any other prefix needs
|
|
259
426
|
* `vercel()` in the config's gateway list.
|
|
@@ -299,37 +466,52 @@ const providerOf = (model) => model.slice(0, model.indexOf("/"));
|
|
|
299
466
|
/**
|
|
300
467
|
* Environment variables a model needs on `gateway`, in the order reeval prefers them; any one is
|
|
301
468
|
* enough. On direct, its provider's own key (none for a prefix reeval can't call directly); on
|
|
302
|
-
*
|
|
469
|
+
* vertex none, since its token comes from Application Default Credentials; on another gateway,
|
|
470
|
+
* that gateway's.
|
|
303
471
|
*/
|
|
304
472
|
function judgeCredentials(model, gateway = "direct") {
|
|
305
|
-
if (isFakeJudge(model)) return [];
|
|
306
|
-
if (gateway
|
|
473
|
+
if (isFakeJudge(model) || gateway === "vertex") return [];
|
|
474
|
+
if (gateway !== "direct") return [...ROUTED_KEYS[gateway]];
|
|
307
475
|
const direct = DIRECT_PROVIDERS[providerOf(model)];
|
|
308
476
|
return direct ? [direct.key] : [];
|
|
309
477
|
}
|
|
310
478
|
/** The keys that would work for a model on `on`: `ANTHROPIC_API_KEY (for direct())`. */
|
|
311
479
|
function modelKeyOptions(model, on = "direct") {
|
|
312
|
-
if (on === "
|
|
480
|
+
if (on === "vertex") return "Application Default Credentials (for vertex())";
|
|
481
|
+
if (on !== "direct") return `${ROUTED_KEYS[on].join(" or ")} (for ${on}())`;
|
|
313
482
|
const direct = DIRECT_PROVIDERS[providerOf(model)];
|
|
314
483
|
return direct ? `${direct.key} (for direct())` : `vercel() in gateway (direct() calls only ${Object.keys(DIRECT_PROVIDERS).map((p) => `${p}/*`).join(", ")} models)`;
|
|
315
484
|
}
|
|
316
|
-
/**
|
|
317
|
-
|
|
485
|
+
/** A gateway's credential set in `env`, if any, with the gateway it's for. */
|
|
486
|
+
function setGatewayKey(env) {
|
|
487
|
+
for (const [gateway, keys] of Object.entries(ROUTED_KEYS)) {
|
|
488
|
+
const key = keys.find((k) => env[k]);
|
|
489
|
+
if (key) return {
|
|
490
|
+
key,
|
|
491
|
+
gateway
|
|
492
|
+
};
|
|
493
|
+
}
|
|
494
|
+
}
|
|
318
495
|
/**
|
|
319
|
-
* ` AI_GATEWAY_API_KEY is set, but …` when a
|
|
320
|
-
*
|
|
496
|
+
* ` AI_GATEWAY_API_KEY is set, but …` when a gateway's credential is set and the model goes
|
|
497
|
+
* direct, so the key says how to use it.
|
|
321
498
|
*/
|
|
322
499
|
function unusedGatewayKey(env, gateway = "direct") {
|
|
323
500
|
if (gateway !== "direct") return "";
|
|
324
501
|
const set = setGatewayKey(env);
|
|
325
|
-
|
|
502
|
+
if (!set) return "";
|
|
503
|
+
const name = set.gateway === "vercel" ? "Vercel AI Gateway" : "OpenRouter";
|
|
504
|
+
return ` ${set.key} is set, but no gateway in the config uses it: add ${set.gateway}() to gateway to call the model through ${name}.`;
|
|
326
505
|
}
|
|
327
506
|
/**
|
|
328
|
-
* The provider's own package on direct,
|
|
329
|
-
*
|
|
330
|
-
*
|
|
507
|
+
* The provider's own package on direct, the Vercel AI Gateway on vercel, or OpenRouter's package
|
|
508
|
+
* on openrouter, or Vertex AI on vertex, whose token is minted when the model is called.
|
|
509
|
+
* Throws, naming the model, its prefix and the keys that would work, when none is
|
|
510
|
+
* set, when the key it would send has whitespace or a line break in it, or when Vercel's project
|
|
511
|
+
* token has expired.
|
|
331
512
|
*/
|
|
332
513
|
function judgeRoute(model, env = process.env, what = "judge model", gateway = "direct") {
|
|
514
|
+
if (gateway === "vertex") return { kind: "vertex" };
|
|
333
515
|
const provider = providerOf(model);
|
|
334
516
|
const direct = DIRECT_PROVIDERS[provider];
|
|
335
517
|
const keys = judgeCredentials(model, gateway);
|
|
@@ -342,6 +524,7 @@ function judgeRoute(model, env = process.env, what = "judge model", gateway = "d
|
|
|
342
524
|
}
|
|
343
525
|
const at = badCharacterAt(env[key]);
|
|
344
526
|
if (at) throw new Error(`${key} for ${what} "${model}" has whitespace or a line break in it at character ${at}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
|
|
527
|
+
if (gateway === "openrouter") return { kind: "openrouter" };
|
|
345
528
|
return direct && key === direct.key ? {
|
|
346
529
|
kind: "direct",
|
|
347
530
|
provider,
|
|
@@ -352,8 +535,11 @@ function judgeRoute(model, env = process.env, what = "judge model", gateway = "d
|
|
|
352
535
|
async function runJudge(question, state, options, signal, retry = JUDGE_RETRY) {
|
|
353
536
|
const { model, threshold } = options;
|
|
354
537
|
const gateway = typeof options.gateway === "object" ? options.gateway.kind : options.gateway;
|
|
538
|
+
const fallbacks = typeof options.gateway === "object" && options.gateway.kind === "openrouter" && options.gateway.fallbacks === true;
|
|
539
|
+
const vertex = gateway === "vertex" ? vertexSettings(typeof options.gateway === "object" && options.gateway.kind === "vertex" ? options.gateway : {}) : void 0;
|
|
355
540
|
let score;
|
|
356
541
|
let resolvedModel;
|
|
542
|
+
let billing = {};
|
|
357
543
|
let retries = [];
|
|
358
544
|
if (isFakeJudge(model)) score = typeof model.score === "function" ? model.score(state) : model.score;
|
|
359
545
|
else {
|
|
@@ -361,9 +547,12 @@ async function runJudge(question, state, options, signal, retry = JUDGE_RETRY) {
|
|
|
361
547
|
const problem = checkModelString(model);
|
|
362
548
|
if (problem) throw new Error(problem);
|
|
363
549
|
}
|
|
364
|
-
const { scoreWithModel } = await import("./model-
|
|
365
|
-
const scored = await withRetries(() => scoreWithModel(model, question, state, options.providerOptions, signal, gateway), retry, signal);
|
|
366
|
-
|
|
550
|
+
const { scoreWithModel } = await import("./model-vRiUycWh.js");
|
|
551
|
+
const scored = await withRetries(() => scoreWithModel(model, redactSecrets(question), state, options.providerOptions, signal, gateway, fallbacks, vertex), retry, signal);
|
|
552
|
+
const { score: scoredAt, resolvedModel: served, ...billed } = scored.value;
|
|
553
|
+
score = scoredAt;
|
|
554
|
+
resolvedModel = served;
|
|
555
|
+
billing = billed;
|
|
367
556
|
retries = scored.retries;
|
|
368
557
|
}
|
|
369
558
|
return {
|
|
@@ -374,12 +563,13 @@ async function runJudge(question, state, options, signal, retry = JUDGE_RETRY) {
|
|
|
374
563
|
question,
|
|
375
564
|
model: judgeModelName(model),
|
|
376
565
|
...resolvedModel ? { resolvedModel } : {},
|
|
377
|
-
...gateway && typeof model === "string" ? { gateway } : {}
|
|
566
|
+
...gateway && typeof model === "string" ? { gateway } : {},
|
|
567
|
+
...billing
|
|
378
568
|
},
|
|
379
569
|
retries
|
|
380
570
|
};
|
|
381
571
|
}
|
|
382
572
|
//#endregion
|
|
383
|
-
export {
|
|
573
|
+
export { vercelCredentialProblem as C, servedOnVercel as S, OPENROUTER_KEY as T, FX_VERCEL_ENDPOINTS as _, RECOMMENDED_JUDGE_MODEL as a, VERCEL_KEY as b, judgeCredentials as c, modelKeyOptions as d, runJudge as f, FX_VERCEL_CALL as g, isApiError as h, RECOMMENDED_ANALYSIS_MODEL as i, judgeModelName as l, unusedGatewayKey as m, DIRECT_PROVIDERS as n, checkModelString as o, setGatewayKey as p, OPENROUTER_JUDGE_MODEL as r, isFakeJudge as s, DEFAULT_THRESHOLD as t, judgeRoute as u, VERCEL_GATEWAY as v, OPENROUTER_GATEWAY as w, fxAskedModel as x, VERCEL_HOST as y };
|
|
384
574
|
|
|
385
|
-
//# sourceMappingURL=judge-
|
|
575
|
+
//# sourceMappingURL=judge-CKdgEXN7.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge-CKdgEXN7.js","names":["claudeCode","codex","goose","opencode","pi","grokBuild","fx","sleep"],"sources":["../src/gateway/effort.ts","../src/gateway/openrouter.ts","../src/gateway/vercel.ts","../src/judge/retry.ts","../src/judge/index.ts"],"mappings":";;;;;;AAGA,MAAa,mBAAmB;;AAGhC,MAAa,aAAa,UAAkB,CAAC,wBAAwB,KAAK,KAAK;;;ACS/E,MAAa,kBAAkB;AAC/B,MAAa,iBAAiB;AAE9B,MAAM,gBAAgB,MAAc,SAAyB,YAC1D;CACC,MAAM;CACN,MAAM,EAAE,OAAO,KAAK;CACpB,QAAQ,CAAC,MAAM;AACjB;;AAGF,MAAa,sBAAmC,EAAE,MAAM,eAAe,qBACrE,SAAA,mBAA4B,gBACxB,GAAG,gBAAgB,iBAAiB,WAAW,eAAe,KAAK,OACnE,KAAA;;AAGN,MAAa,wBAAkC,EAAE,MAAM,kBAAkB,mBACvE,SAAA,mBAA4B,mBACxB;CACE,UAAU;CACV,GAAI,iBAAiB,KAAA,IAAY,EAAE,SAAS,aAAa,IAAI,CAAC;AAChE,IACA,KAAA;;AAGN,MAAM,SAAS,WAAqC,WAAqC;CACvF,YAAY;CACZ,QAAQ;CACR;CACA,gBAAgB;CAChB,aAAa;CACb,UAAU;CACV,OAAO,CAAC,eAAe;AACzB;;AAGA,MAAM,SAAS,cAAsB,UACnC,MAAM,SAAS,GAAG,IAAI,QAAQ,GAAG,SAAS,GAAG;;;;;AAM/C,MAAa,yBAAyB,UACpC,MAAM,SAAS,GAAG,IAAI,QAAQ,aAAa,MAAM,QAAQ,yBAAyB,QAAQ;;;;;;AAO5F,SAASA,eAA2B;CAClC,MAAM,QAAQ,CAAC,aAAa,kBAAkB,CAAC;CAC/C,OAAO;EACL,GAAG,MAAM,CAAC,GAAG,OAAO,aAAa,+BAA+B,CAAC,GAAG,KAAK;EACzE,QAAQ;EACR,SAAS,EAAE,oBAAoB,WAAW,gBAAgB,MAAM;EAChE,SAAS;CACX;AACF;;AAGA,SAASC,UAAsB;CAC7B,MAAM,QAAQ,CAAC,aAAa,mBAAmB,CAAC;CAChD,OAAO;EAAE,GAAG,MAAM,OAAO,KAAK;EAAG,SAAS,MAAM,QAAQ;CAAE;AAC5D;;;;AAKA,SAASC,UAAsB;CAC7B,MAAM,QAAQ,CAAC,aAAa,0BAA0B,CAAC;CACvD,OAAO;EACL,GAAG,MAAM,CAAC,GAAG,OAAO,aAAa,kBAAkB,KAAK,CAAC,GAAG,KAAK;EACjE,SAAS,MAAM,QAAQ;CACzB;AACF;;;;;;AAOA,SAASC,aAAyB;CAChC,MAAM,WAAW,wBAAwB,kBAAkB;CAC3D,IAAI,WAAW,UAAU,MAAM,IAAI,MAAM,SAAS,KAAK;CACvD,MAAM,EAAE,aAAa;CACrB,OAAO;EACL,YAAY;EACZ,QAAQ;EACR,WAAW,kBAAkB,QAAQ;EACrC,gBAAgB,uBAAuB,QAAQ;EAC/C,cAAc,YAAY,oBAAoB,UAAU,OAAO;EAC/D,UAAU;EACV,OAAO,CAAC,GAAG,cAAc,QAAQ,GAAG,YAAY;EAChD,UAAU,UAAU,cAAc;CACpC;AACF;;;;;;AAOA,SAASC,OAAmB;CAC1B,MAAM,QAAQ,CAAC,aAAa,0BAA0B,CAAC;CACvD,OAAO;EAAE,GAAG,MAAM,OAAO,KAAK;EAAG,SAAS,MAAM,QAAQ;CAAE;AAC5D;;;;;AAMA,SAASC,cAA0B;CACjC,MAAM,QAAQ,CAAC,aAAa,0BAA0B,CAAC;CACvD,OAAO;EAAE,GAAG,MAAM,OAAO,KAAK;EAAG,SAAS,MAAM,MAAM;CAAE;AAC1D;;;;;AAMA,SAASC,OAAmB;CAC1B,MAAM,QAAQ,CAAC,aAAa,0BAA0B,CAAC;CACvD,OAAO;EAAE,GAAG,MAAM,OAAO,KAAK;EAAG,SAAS,MAAM,QAAQ;CAAE;AAC5D;;AAGA,MAAa,qBAAqC;CAChD,MAAM;CACN,QAAQ;EACN,eAAeN;EACf,OAAA;EACA,OAAA;EACA,UAAA;EACA,IAAA;EACA,cAAcK;EACd,IAAA;CACF;CACA,UAAU;EACR,QAAQ;EACR,aACE;CACJ;CAKA,gBAAgB;EAAE,eAAe;EAAY,OAAO;EAAW,UAAU;CAAU;AACrF;;;ACvJA,MAAa,cAAc;;AAG3B,MAAa,aAAa;;AAE1B,MAAa,cAAc;;AAG3B,MAAa,oBAAoB,MAAyB,QAAQ,QAChE,CAAC,IAAA,yBAAmB,IAAA,uBAAmB,cAAc;;AAGvD,SAAgB,UAAU,OAAiC;CACzD,MAAM,GAAG,WAAW,MAAM,MAAM,GAAG;CACnC,IAAI,CAAC,SAAS,OAAO,KAAA;CACrB,IAAI;EACF,MAAM,EAAE,QAAQ,KAAK,MAAM,OAAO,KAAK,SAAS,WAAW,CAAC,CAAC,SAAS,MAAM,CAAC;EAG7E,OAAO,OAAO,QAAQ,2BAAW,IAAI,KAAK,MAAM,GAAI,IAAI,KAAA;CAC1D,QAAQ;EACN;CACF;AACF;;;;;AAMA,SAAgB,wBACd,MAAyB,QAAQ,KACjC,sBAAM,IAAI,KAAK,GACK;CACpB,IAAI,iBAAiB,GAAG,MAAA,qBAAmB,OAAO,KAAA;CAClD,MAAM,SAAS,UAAU,IAAA,wBAAoB,EAAE;CAC/C,IAAI,CAAC,UAAU,SAAS,KAAK,OAAO,KAAA;CACpC,OAAO,GAAG,YAAY,cAAc,OAAO,YAAY,EAAE,8EAA8E,WAAW;AACpJ;AAEA,MAAM,YAAY,MAAc,YAC7B;CAAE,MAAM;CAAa,MAAM,EAAE,OAAO,KAAK;CAAG,QAAQ,CAAC,MAAM;AAAE;;;;;AAMhE,MAAa,kBAAgD,EAC3D,MACA,eACA,qBAEA,SAAA,0BAAwB,gBACpB,GAAG,gBAAgB,iBAAiB,WAAW,eAAe,KAAK,OACnE,KAAA;;;;;;AAON,SAAS,WAAW,KAAsC;CACxD,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR,WAAW,CAAC,SAAS,gBAAgB,MAAM,GAAG,SAAS,6BAA6B,MAAM,CAAC;EAC3F,OAAO,CAAC,WAAW;CACrB;AACF;;AAGA,SAAS,MAAM,KAAsC;CACnD,MAAM,YAAY,CAAC,SAAS,iBAAiB,MAAM,CAAC;CACpD,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR;EACA,gBAAgB;EAChB,aAAa;EACb,OAAO,CAAC,WAAW;EACnB,UAAU,UAAW,MAAM,SAAS,GAAG,IAAI,QAAQ,UAAU;CAC/D;AACF;;AAGA,MAAa,iBAAiB;CAC5B,MAAM;CACN,MAAM,EAAE,OAAO,6BAA6B;CAC5C,QAAQ,CAAC,MAAM;AACjB;;;;;AAMA,MAAa,gBAA8C,YAAY;CACrE,IAAI,QAAQ,SAAA,0BAAwB,CAAC,IAAI,OAAO,eAAe,KAAK,KAAK,CAAC,CAAC,KAAK,QAAQ,IAAI,GAC1F,OAAO,KAAA;CACT,IAAI,QAAQ,WAAW,KAAA,KAAa,QAAQ,SAAS,OAAO,QAAQ,SAAS,KAAK,OAAO,KAAA;CACzF,MAAM,QAAQ,QAAQ,UAAU;CAChC,QAAQ,OAAO,UAAU,WAAW,QAAQ,QAAQ,OAAO,KAAA;AAC7D;;AAGA,MAAa,sBAAsB,CACjC;CAAE,MAAM;CAAa,MAAM,EAAE,OAAO,qBAAqB;CAAG,QAAQ,CAAC,KAAK;AAAE,GAC5E,cACF;;AAGA,SAAS,GAAG,KAAsC;CAChD,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR,WAAW;EACX,gBAAgB,CAAC,cAAc;EAC/B,aAAa;EACb,YAAY;EACZ,OAAO,CAAC,WAAW;EACnB,UAAU,UAAW,MAAM,SAAS,GAAG,IAAI,QAAQ,UAAU;CAC/D;AACF;;AAGA,MAAM,YAAY,UAAmB,MAAM,SAAS,GAAG,IAAI,QAAQ,UAAU;;;;;AAM7E,SAAS,MAAM,KAAsC;CACnD,MAAM,QAAQ,CAAC,SAAS,wBAAwB,MAAM,GAAG,SAAS,iBAAiB,MAAM,CAAC;CAC1F,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR,WAAW,CAAC,GAAG,OAAO,SAAS,cAAc,KAAK,CAAC;EACnD,gBAAgB;EAChB,aAAa;EACb,OAAO,CAAC,WAAW;EACnB,SAAS;CACX;AACF;;;;;;AAOA,SAAS,GAAG,KAAsC;CAChD,MAAM,YAAY,CAAC,SAAS,wBAAwB,MAAM,CAAC;CAC3D,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR;EACA,gBAAgB;EAChB,aAAa;EACb,OAAO,CAAC,WAAW;EACnB,SAAS;CACX;AACF;;;;;;AAOA,SAAS,SAAS,KAAsC;CACtD,MAAM,WAAW,wBAAwB,cAAc;CACvD,IAAI,WAAW,UAAU,MAAM,IAAI,MAAM,SAAS,KAAK;CACvD,MAAM,EAAE,aAAa;CACrB,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR,WAAW,kBAAkB,QAAQ;EACrC,gBAAgB,uBAAuB,QAAQ;EAC/C,cAAc,YAAY,oBAAoB,UAAU,OAAO;EAC/D,OAAO,CAAC,GAAG,cAAc,QAAQ,GAAG,YAAY;EAChD,UAAU,UAAU,UAAU;CAChC;AACF;;;;;AAMA,SAAS,UAAU,KAAsC;CACvD,MAAM,YAAY,CAAC,SAAS,wBAAwB,MAAM,CAAC;CAC3D,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR;EACA,gBAAgB;EAChB,aAAa;EACb,OAAO,CAAC,WAAW;EAEnB,iBAAiB;GACf,OAAO,CAAC,YAAY;GACpB,KAAK;EACP;EACA,UAAU,UAAW,MAAM,SAAS,GAAG,IAAI,QAAQ,OAAO;CAC5D;AACF;;AAGA,MAAa,iBAAiC;CAC5C,MAAM;CACN,QAAQ;EACN,eAAe;EACf;EACA;EACA;EACA;EACA,cAAc;EACd;CACF;CACA,UAAU;EACR,QAAQ;EACR,aAAa;CACf;CAIA,gBAAgB;EAAE,OAAO;EAAY,UAAU;CAAU;AAC3D;;;;ACrOA,MAAa,cAA2B;CAAE,UAAU;CAAG,QAAQ;CAAO,OAAO;AAAO;AAEpF,MAAM,mCAAmB,IAAI,IAAI;CAAC;CAAK;CAAK;CAAK;AAAG,CAAC;AACrD,MAAM,oBACJ;;AAUF,MAAM,aAAa,UAA4B;CAC7C,MAAM,IAAI;CACV,OAAO,GAAG,SAAS,mBAAmB,EAAE,YAAY,UAAU,EAAE,SAAS,IAAI;AAC/E;;AAGA,MAAa,cAAc,UACzB,OAAQ,UAAU,KAAK,CAAC,EAAoB,eAAe;;AAG7D,SAAgB,YAAY,OAAyB;CACnD,MAAM,IAAI,UAAU,KAAK;CACzB,IAAI,CAAC,KAAK,EAAE,SAAS,gBAAgB,EAAE,SAAS,gBAAgB,OAAO;CACvE,IAAI,OAAO,EAAE,eAAe,UAC1B,OAAO,EAAE,cAAc,OAAO,iBAAiB,IAAI,EAAE,UAAU,KAAK,EAAE,gBAAgB;CAExF,OAAO,EAAE,gBAAgB,QAAQ,kBAAkB,KAAK,EAAE,WAAW,OAAO,CAAC,CAAC;AAChF;AAEA,SAAS,aAAa,OAAwB;CAC5C,MAAM,SAAU,UAAU,KAAK,CAAC,EAAoB,kBAAkB;CACtE,MAAM,UAAU,OAAO,MAAM;CAC7B,OAAO,OAAO,SAAS,OAAO,KAAK,UAAU,IAAI,UAAU,MAAO;AACpE;;AAGA,SAAgB,UAAU,QAAqB,OAAe,OAAwB;CACpF,MAAM,MAAM,KAAK,IAAI,OAAO,OAAO,OAAO,SAAS,KAAK,KAAK;CAC7D,MAAM,WAAW,MAAM,IAAI,KAAK,OAAO,KAAK,MAAM;CAClD,OAAO,KAAK,MAAM,KAAK,IAAI,OAAO,OAAO,KAAK,IAAI,UAAU,aAAa,KAAK,CAAC,CAAC,CAAC;AACnF;;AAKA,eAAsB,YACpB,MACA,QACA,QAC8C;CAC9C,MAAM,UAAwB,CAAC;CAC/B,SACE,IAAI;EACF,OAAO;GAAE,OAAO,MAAM,KAAK;GAAG;EAAQ;CACxC,SAAS,OAAO;EACd,IAAI,OAAO,WAAW,QAAQ,SAAS,KAAK,OAAO,YAAY,CAAC,YAAY,KAAK,GAAG;GAClF,MAAM,SAAU,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;GACxE,OAAO,UAAU;GACjB,MAAM;EACR;EACA,MAAM,UAAU,UAAU,QAAQ,QAAQ,QAAQ,KAAK;EACvD,MAAM,IAAI,UAAU,KAAK;EACzB,QAAQ,KAAK;GAAE,OAAO,GAAG,WAAW,OAAO,CAAC;GAAG;EAAQ,CAAC;EACxD,MAAME,WAAM,SAAS,KAAA,GAAW,EAAE,OAAO,CAAC;CAC5C;AAEJ;;;;AC5DA,MAAa,0BAA0B;;AAEvC,MAAa,yBAAyB;;AAEtC,MAAa,6BAA6B;AAC1C,MAAa,oBAAoB;;AAKjC,MAAM,cAAuE;CAC3E,QAAQ,CAJY,YAAY,WAIb;CACnB,YAAY,CAAC,cAAc;AAC7B;;;;;AAMA,MAAa,mBAA+E;CAC1F,WAAW;EAAE,KAAK;EAAqB,KAAK;EAAqB,MAAM;CAAY;CACnF,QAAQ;EAAE,KAAK;EAAgC,KAAK;EAAkB,MAAM;CAAS;CACrF,QAAQ;EAAE,KAAK;EAAkB,KAAK;EAAkB,MAAM;CAAS;CACvE,eAAe;EAAE,KAAK;EAAuB,KAAK;EAAuB,MAAM;CAAc;AAC/F;AAcA,MAAa,eAAe,MAC1B,OAAO,MAAM,YAAY,MAAM,QAAS,EAAqB,SAAS;AAExE,SAAgB,eAAe,OAA2B;CACxD,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,IAAI,YAAY,KAAK,GAAG,OAAO;CAC/B,MAAM,IAAI;CACV,OAAO,EAAE,YAAY,EAAE,UAAU,GAAG,EAAE,SAAS,GAAG,EAAE,YAAY;AAClE;;;;;AAMA,SAAgB,iBACd,OACA,OAAO,mBACP,WAAW,CAAC,yBAAyB,UAAU,OAAO,GAClC;CACpB,IAAI,MAAM,SAAS,GAAG,GAAG,OAAO,KAAA;CAChC,OAAO,GAAG,KAAK,iCAAiC,SAAS,KAAK,MAAM,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,EAAE,SAAS,MAAM;AAC5G;AAEA,MAAM,cAAc,UAAkB,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC;;;;;;;AAQvE,SAAgB,iBACd,OACA,UAAuB,UACb;CACV,IAAI,YAAY,KAAK,KAAK,YAAY,UAAU,OAAO,CAAC;CACxD,IAAI,YAAY,UAAU,OAAO,CAAC,GAAG,YAAY,QAAQ;CACzD,MAAM,SAAS,iBAAiB,WAAW,KAAK;CAChD,OAAO,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC;AAClC;;AAGA,SAAgB,gBAAgB,OAAe,KAAkB,UAAkB;CACjF,IAAI,OAAO,UAAU,OAAO;CAC5B,IAAI,OAAO,UAAU,OAAO,GAAG,YAAY,GAAG,CAAC,KAAK,MAAM,EAAE,QAAQ,GAAG;CACvE,MAAM,SAAS,iBAAiB,WAAW,KAAK;CAChD,OAAO,SACH,GAAG,OAAO,IAAI,mBACd,4CAA4C,OAAO,KAAK,gBAAgB,CAAC,CACtE,KAAK,MAAM,GAAG,EAAE,GAAG,CAAC,CACpB,KAAK,IAAI,EAAE;AACpB;;AAGA,SAAgB,cACd,KAC0D;CAC1D,KAAK,MAAM,CAAC,SAAS,SAAS,OAAO,QAAQ,WAAW,GAAG;EACzD,MAAM,MAAM,KAAK,MAAM,MAAM,IAAI,EAAE;EACnC,IAAI,KAAK,OAAO;GAAE;GAAc;EAA8B;CAChE;AAEF;;;;;AAMA,SAAgB,iBAAiB,KAAwB,UAAuB,UAAkB;CAChG,IAAI,YAAY,UAAU,OAAO;CACjC,MAAM,MAAM,cAAc,GAAG;CAC7B,IAAI,CAAC,KAAK,OAAO;CACjB,MAAM,OAAO,IAAI,YAAY,WAAW,sBAAsB;CAC9D,OAAO,IAAI,IAAI,IAAI,qDAAqD,IAAI,QAAQ,0CAA0C,KAAK;AACrI;;;;;;;;AASA,SAAgB,WACd,OACA,MAAyB,QAAQ,KACjC,OAAO,eACP,UAAuB,UACX;CACZ,IAAI,YAAY,UAAU,OAAO,EAAE,MAAM,SAAS;CAClD,MAAM,WAAW,WAAW,KAAK;CACjC,MAAM,SAAS,iBAAiB;CAChC,MAAM,OAAO,iBAAiB,OAAO,OAAO;CAC5C,IAAI,CAAC,KAAK,QACR,MAAM,IAAI,MACR,GAAG,KAAK,IAAI,MAAM,qEAAqE,OAAO,KAC5F,gBACF,CAAC,CACE,KAAK,MAAM,GAAG,EAAE,GAAG,CAAC,CACpB,KAAK,IAAI,EAAE,SAChB;CAEF,MAAM,MAAM,KAAK,MAAM,MAAM,IAAI,EAAE;CACnC,IAAI,QAAQ,KAAA,GACV,MAAM,IAAI,MACR,sBAAsB,KAAK,IAAI,MAAM,eAAe,SAAS,UAAU,gBAAgB,OAAO,OAAO,EAAE,qCAAqC,iBAAiB,KAAK,OAAO,GAC3K;CAEF,IAAI,QAAA,qBAAqB;EACvB,MAAM,UAAU,wBAAwB,GAAG;EAC3C,IAAI,SAAS,MAAM,IAAI,MAAM,GAAG,KAAK,IAAI,MAAM,KAAK,SAAS;CAC/D;CACA,MAAM,KAAK,eAAe,IAAI,IAAc;CAC5C,IAAI,IACF,MAAM,IAAI,MACR,GAAG,IAAI,OAAO,KAAK,IAAI,MAAM,sDAAsD,GAAG,sFACxF;CAEF,IAAI,YAAY,cAAc,OAAO,EAAE,MAAM,aAAa;CAC1D,OAAO,UAAU,QAAQ,OAAO,MAC5B;EAAE,MAAM;EAAU;EAAU,KAAK,OAAO;CAAI,IAC5C,EAAE,MAAM,UAAU;AACxB;;AAMA,eAAsB,SACpB,UACA,OACA,SACA,QACA,QAAqB,aACC;CACtB,MAAM,EAAE,OAAO,cAAc;CAC7B,MAAM,UAAU,OAAO,QAAQ,YAAY,WAAW,QAAQ,QAAQ,OAAO,QAAQ;CACrF,MAAM,YACJ,OAAO,QAAQ,YAAY,YAC3B,QAAQ,QAAQ,SAAS,gBACzB,QAAQ,QAAQ,cAAc;CAChC,MAAM,SACJ,YAAY,WACR,eACE,OAAO,QAAQ,YAAY,YAAY,QAAQ,QAAQ,SAAS,WAC5D,QAAQ,UACR,CAAC,CACP,IACA,KAAA;CACN,IAAI;CACJ,IAAI;CACJ,IAAI,UAAmD,CAAC;CACxD,IAAI,UAAwB,CAAC;CAC7B,IAAI,YAAY,KAAK,GACnB,QAAQ,OAAO,MAAM,UAAU,aAAa,MAAM,MAAM,KAAK,IAAI,MAAM;MAClE;EACL,IAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,UAAU,iBAAiB,KAAK;GACtC,IAAI,SAAS,MAAM,IAAI,MAAM,OAAO;EACtC;EACA,MAAM,EAAE,mBAAmB,MAAM,OAAO;EACxC,MAAM,SAAS,MAAM,kBAEjB,eACE,OACA,cAAc,QAAQ,GACtB,OACA,QAAQ,iBACR,QACA,SACA,WACA,MACF,GACF,OACA,MACF;EACA,MAAM,EAAE,OAAO,UAAU,eAAe,QAAQ,GAAG,WAAW,OAAO;EACrE,QAAQ;EACR,gBAAgB;EAChB,UAAU;EACV,UAAU,OAAO;CACnB;CAWA,OAAO;EAAE,SAAA;GATP,SAAS,SAAS,YAAY,SAAS;GACvC;GACA;GACA;GACA,OAAO,eAAe,KAAK;GAC3B,GAAI,gBAAgB,EAAE,cAAc,IAAI,CAAC;GACzC,GAAI,WAAW,OAAO,UAAU,WAAW,EAAE,QAAQ,IAAI,CAAC;GAC1D,GAAG;EAEU;EAAG;CAAQ;AAC5B"}
|
package/dist/judge.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as FakeVote, J as FakeAnswer, O as ConfigJudgeOptions, X as FakeJudgeModel, Zt as Verdict, _t as JudgeOptions, c as BlindPair, gt as JudgeModel, ht as JudgeFailure, mt as JudgeCallOptions, n as AnalysisConfig, pt as JudgeArtifactRef, q as FakeAnalysisModel, s as BlindAttempt, vt as JudgeState } from "./types-coMWHaU2.js";
|
|
2
|
+
import "./vertex-C7PI-WPc.js";
|
|
2
3
|
//#region src/judge/index.d.ts
|
|
3
4
|
/** The judge the docs recommend, pinned so a model update never silently moves scores. */
|
|
4
5
|
export declare const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.14.0";
|
|
@@ -17,12 +18,9 @@ export declare const fakeJudge: ({ score }?: {
|
|
|
17
18
|
* model's structured answer, each pairwise question's pair gets `vote` (a fixed `'A'`, `'B'` or
|
|
18
19
|
* `'tie'`, `{ winner, reason? }`, a function of the blinded pair, or `{ error }`), and each
|
|
19
20
|
* question asked of one attempt gets `answer` (its text, `{ choice?, text? }`, a function of the
|
|
20
|
-
* blinded attempt, or `{ error }`), through the same parsing and validation as a real one.
|
|
21
|
-
* get `insights` (or `insights(input)`, called per eval for its classification and its sentences,
|
|
22
|
-
* then once for the headline), through the same validator: an uncited sentence is dropped, and one
|
|
23
|
-
* with a banned word is asked for again, then dropped.
|
|
21
|
+
* blinded attempt, or `{ error }`), through the same parsing and validation as a real one.
|
|
24
22
|
*/
|
|
25
|
-
export declare const fakeAnalysis: ({ output, vote, answer
|
|
23
|
+
export declare const fakeAnalysis: ({ output, vote, answer }: Pick<FakeAnalysisModel, 'output' | 'vote' | 'answer'>) => FakeAnalysisModel;
|
|
26
24
|
//#endregion
|
|
27
|
-
export type { AnalysisConfig, BlindAttempt, BlindPair, ConfigJudgeOptions, FakeAnalysisModel, FakeAnswer,
|
|
25
|
+
export type { AnalysisConfig, BlindAttempt, BlindPair, ConfigJudgeOptions, FakeAnalysisModel, FakeAnswer, FakeJudgeModel, FakeVote, JudgeArtifactRef, JudgeCallOptions, JudgeFailure, JudgeModel, JudgeOptions, JudgeState, Verdict };
|
|
28
26
|
//# sourceMappingURL=judge.d.ts.map
|
package/dist/judge.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { a as RECOMMENDED_JUDGE_MODEL, t as DEFAULT_THRESHOLD } from "./judge-CKdgEXN7.js";
|
|
2
2
|
//#region src/judge.ts
|
|
3
3
|
/**
|
|
4
4
|
* A judge that returns `score` without calling a model, for tests and dry runs. `score` can be a
|
|
@@ -13,17 +13,13 @@ const fakeJudge = ({ score = 1 } = {}) => ({
|
|
|
13
13
|
* model's structured answer, each pairwise question's pair gets `vote` (a fixed `'A'`, `'B'` or
|
|
14
14
|
* `'tie'`, `{ winner, reason? }`, a function of the blinded pair, or `{ error }`), and each
|
|
15
15
|
* question asked of one attempt gets `answer` (its text, `{ choice?, text? }`, a function of the
|
|
16
|
-
* blinded attempt, or `{ error }`), through the same parsing and validation as a real one.
|
|
17
|
-
* get `insights` (or `insights(input)`, called per eval for its classification and its sentences,
|
|
18
|
-
* then once for the headline), through the same validator: an uncited sentence is dropped, and one
|
|
19
|
-
* with a banned word is asked for again, then dropped.
|
|
16
|
+
* blinded attempt, or `{ error }`), through the same parsing and validation as a real one.
|
|
20
17
|
*/
|
|
21
|
-
const fakeAnalysis = ({ output, vote, answer
|
|
18
|
+
const fakeAnalysis = ({ output, vote, answer }) => ({
|
|
22
19
|
kind: "fake-analysis",
|
|
23
20
|
...output !== void 0 ? { output } : {},
|
|
24
21
|
...vote !== void 0 ? { vote } : {},
|
|
25
|
-
...answer !== void 0 ? { answer } : {}
|
|
26
|
-
...insights !== void 0 ? { insights } : {}
|
|
22
|
+
...answer !== void 0 ? { answer } : {}
|
|
27
23
|
});
|
|
28
24
|
//#endregion
|
|
29
25
|
export { DEFAULT_THRESHOLD, RECOMMENDED_JUDGE_MODEL, fakeAnalysis, fakeJudge };
|
package/dist/judge.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge.js","names":[],"sources":["../src/judge.ts"],"mappings":";;;;;;AAQA,MAAa,aAAa,EACxB,QAAQ,MAGN,CAAC,OAAuB;CAC1B,MAAM;CACN;AACF
|
|
1
|
+
{"version":3,"file":"judge.js","names":[],"sources":["../src/judge.ts"],"mappings":";;;;;;AAQA,MAAa,aAAa,EACxB,QAAQ,MAGN,CAAC,OAAuB;CAC1B,MAAM;CACN;AACF;;;;;;;;AASA,MAAa,gBAAgB,EAC3B,QACA,MACA,cAC+E;CAC/E,MAAM;CACN,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;CACzC,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;CACrC,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;AAC3C"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { n as isOneOf } from "./object-CLf2-tNq.js";
|
|
2
|
-
import { n as fillDefaults } from "./record-
|
|
2
|
+
import { n as fillDefaults } from "./record-DZBCjuwm.js";
|
|
3
3
|
//#region src/view/live.ts
|
|
4
4
|
/** The progress file's name inside the run's staging directory. */
|
|
5
5
|
const LIVE_FILE = "live.json";
|
|
@@ -84,4 +84,4 @@ function readableLiveProgress(value) {
|
|
|
84
84
|
//#endregion
|
|
85
85
|
export { liveApiPath as a, readableLiveProgress as c, isFinished as i, LIVE_FILE as n, liveCounts as o, LIVE_SCHEMA_VERSION as r, liveTranscriptPath as s, FINISHED as t };
|
|
86
86
|
|
|
87
|
-
//# sourceMappingURL=live-
|
|
87
|
+
//# sourceMappingURL=live-XJQ6nqK4.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"live-
|
|
1
|
+
{"version":3,"file":"live-XJQ6nqK4.js","names":[],"sources":["../src/view/live.ts"],"mappings":";;;;AAQA,MAAa,YAAY;AACzB,MAAa,sBAAsB;AAsDnC,MAAa,cAAc;AAC3B,MAAa,sBAAsB,OAAe,eAChD,YAAY,MAAM,sBAAsB,mBAAmB,UAAU;AAEvE,MAAa,WAAkC;CAAC;CAAQ;CAAQ;CAAe;AAAS;AACxF,MAAa,cAAc,WAAuB,SAAS,SAAS,MAAM;AAE1E,SAAgB,WAAW,UAA8D;CACvF,MAAM,SAAS,GAAG,aAChB,SAAS,QAAQ,MAAM,SAAS,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;CACtD,OAAO;EACL,MAAM,MAAM,GAAG,QAAQ;EACvB,QAAQ,MAAM,MAAM;EACpB,QAAQ,MAAM,MAAM;EACpB,SAAS,MAAM,aAAa;EAC5B,SAAS,MAAM,SAAS;EACxB,SAAS,MAAM,SAAS;CAC1B;AACF;AAEA,MAAM,WAAkC;CAAC;CAAU;CAAW,GAAG;AAAQ;AACzE,MAAM,SAA+B;CAAC;CAAW;CAAc;AAAW;AAC1E,MAAM,SAAS,MACb,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC;AACzD,MAAM,OAAO,MAA4B,OAAO,MAAM;AACtD,MAAM,OAAO,MAA4B,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC;AACnF,MAAM,UAAa,GAAY,OAC7B,MAAM,QAAQ,MAAM,KAAA,KAAa,GAAG,CAAC;AAEvC,SAAS,gBAAgB,GAAgC;CACvD,IAAI,CAAC,MAAM,CAAC,GAAG,OAAO;CACtB,MAAM,EAAE,KAAK,QAAQ,MAAM,MAAM,MAAM,GAAG,QAAQ,WAAW,eAAe;CAC5E,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO;CAC3F,IAAI,CAAC,QAAQ,UAAU,MAAM,GAAG,OAAO;CACvC,IAAI,CAAC,OAAO,WAAW,GAAG,KAAK,CAAC,OAAO,YAAY,GAAG,GAAG,OAAO;CAChE,OAAO;EACL;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW,aAAa;EACxB,YAAY,cAAc;CAC5B;AACF;;;;;AAMA,SAAgB,qBAAqB,OAAqC;CACxE,IAAI,CAAC,MAAM,KAAK,KAAK,MAAM,kBAAA,GAAuC,OAAO;CACzE,aAAa,iBAAiB,KAAK;CACnC,MAAM,EAAE,OAAO,WAAW,KAAK,MAAM,aAAa,OAAO,aAAa;CACtE,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,SAAS,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,WAAW,KAAK,CAAC,IAAI,GAAG,GAAG,OAAO;CAC3F,IAAI,CAAC,QAAQ,QAAQ,KAAK,KAAK,CAAC,MAAM,QAAQ,QAAQ,GAAG,OAAO;CAChE,MAAM,OAAO,SAAS,IAAI,eAAe;CACzC,IAAI,CAAC,KAAK,OAAO,MAAM,MAAM,IAAI,GAAG,OAAO;CAC3C,OAAO;EACL,eAAA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,SAAS,KAAK;EACd,QAAQ,WAAW,IAAI;EACvB,UAAU;CACZ;AACF"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { t as isObjectLike } from "./object-CLf2-tNq.js";
|
|
2
|
-
import { a as calledMcp, c as changed, d as ran, f as searched, l as edited, o as calledSkill, s as calledTool, u as fetched } from "./client-
|
|
2
|
+
import { a as calledMcp, c as changed, d as ran, f as searched, l as edited, o as calledSkill, s as calledTool, u as fetched } from "./client-Ct6s6OnF.js";
|
|
3
3
|
import { isDeepStrictEqual } from "node:util";
|
|
4
4
|
//#region src/matchers.ts
|
|
5
5
|
const matcher = (run) => run;
|
|
@@ -115,4 +115,4 @@ const evalMatchers = Object.fromEntries(Object.keys(matchers).filter((name) => !
|
|
|
115
115
|
//#endregion
|
|
116
116
|
export { runMatcher as i, evalMatchers as n, isMatcherName as r, MATCHERS as t };
|
|
117
117
|
|
|
118
|
-
//# sourceMappingURL=matchers-
|
|
118
|
+
//# sourceMappingURL=matchers-DT2TaGix.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers-
|
|
1
|
+
{"version":3,"file":"matchers-DT2TaGix.js","names":[],"sources":["../src/matchers.ts"],"mappings":";;;;AA+BA,MAAM,WAAoE,QAAmB;;AAG7F,MAAM,QAAQ,GAAW,QAAQ,IAAI,QAAQ,QAAS;CACpD,MAAM,OAAO,EAAE,QAAQ,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI;CAC5D,OAAO,KAAK,UAAU,QAClB,OACA,MAAM,KAAK,SAAS,MAAM,sBAAsB,KAAK,MAAM,CAAC,KAAK;AACvE;AAIA,MAAM,YAAY,UAChB,aAAa,KAAK,KAAK,OAAO,MAAM,aAAa;AAEnD,MAAM,QAAQ,UAA2B;CACvC,IAAI,OAAO,UAAU,UAAU,OAAO,IAAI,MAAM;CAChD,IAAI;EACF,OAAO,KAAK,UAAU,KAAK,KAAK,OAAO,KAAK;CAC9C,QAAQ;EACN,OAAO,OAAO,KAAK;CACrB;AACF;AAEA,MAAa,WAAW;CACtB,MAAM,SAAS,QAAQ,CAAC,cAAyB;EAC/C,MAAM,OAAO,OAAO,GAAG,QAAQ,QAAQ;EACvC,OAAO;GACL;GACA,eACE,OACI,YAAY,KAAK,MAAM,EAAE,aAAa,KAAK,QAAQ,EAAE,0BACrD,YAAY,KAAK,MAAM,EAAE,SAAS,KAAK,QAAQ,EAAE;EACzD;CACF,CAAC;CAED,SAAS,SAAS,QAAQ,CAAC,cAAyB;EAClD,MAAM,OAAO,kBAAkB,QAAQ,QAAQ;EAC/C,OAAO;GACL;GACA,eACE,OACI,YAAY,KAAK,MAAM,EAAE,uBAAuB,KAAK,QAAQ,MAC7D,YAAY,KAAK,MAAM,EAAE,mBAAmB,KAAK,QAAQ;EACjE;CACF,CAAC;CAED,UAAU,SAAS,QAAQ,OAAW;EACpC,MAAM,WAAW;EACjB,eACE,WAAW,OACP,YAAY,KAAK,MAAM,EAAE,mBACzB,YAAY,KAAK,MAAM,EAAE;CACjC,EAAE;CAEF,YAAY,SAAS,QAAQ,OAAW;EACtC,MAAM,CAAC,CAAC;EACR,eACE,SAAS,YAAY,KAAK,MAAM,EAAE,gBAAgB,YAAY,KAAK,MAAM,EAAE;CAC/E,EAAE;CAEF,WAAW,SAAS,QAAQ,CAAC,YAAuB;EAClD,MAAM,OACJ,OAAO,WAAW,YAAY,OAAO,WAAW,WAC5C,OAAO,SAAS,MAAM,IACtB,MAAM,QAAQ,MAAM,KAAK,OAAO,SAAS,MAAM;EACrD,OAAO;GACL;GACA,eACE,OACI,YAAY,KAAK,MAAM,EAAE,kBAAkB,KAAK,MAAM,MACtD,YAAY,KAAK,MAAM,EAAE,cAAc,KAAK,MAAM;EAC1D;CACF,CAAC;CAED,YAAY,QACV,OAAO,QAAQ,CAAC,MAAM,aAAwD;EAC5E,IAAI,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI,UAAU,iDAAiD;EAC5F,MAAM,UAAU,MAAM,OAAO,SAAS,IAAI;EAC1C,IAAI,YAAY,MAAM,OAAO;GAAE,MAAM;GAAO,eAAe,iBAAiB,KAAK;EAAW;EAC5F,IAAI,YAAY,KAAA,GAAW,OAAO;GAAE,MAAM;GAAM,eAAe,uBAAuB;EAAO;EAC7F,MAAM,OAAO,OAAO,YAAY,WAAW,QAAQ,SAAS,OAAO,IAAI,QAAQ,KAAK,OAAO;EAC3F,OAAO;GACL;GACA,eACE,YAAY,KAAK,GAAG,OAAO,SAAS,GAAG,WAAW,QAAQ,QAAQ,KAAK,QAAQ,KAAK,OAAO;EAC/F;CACF,CACF;CAEA,WAAW,QAAQ,OAAO,QAAQ,MAAU;EAC1C,MAAM,IAAK,MAAM;EACjB,IAAI,OAAO,GAAG,aAAa,UAAU,MAAM,IAAI,UAAU,mCAAmC;EAC5F,OAAO;GACL,MAAM,EAAE,aAAa;GACrB,eACE,EAAE,aAAa,IACX,6BACA,0CAA0C,EAAE,SAAS,oBAClC,KAAK,EAAE,MAAM,EAAE,oBAAoB,KAAK,EAAE,MAAM;EAC3E;CACF,CAAC;CAED,gBAAgB,SAAS,QAAQ,CAAC,WAAsC,SAAS,QAAQ,KAAK,CAAC;CAE/F,eAAe,SAAS,QAAQ,CAAC,SAAkC,QAAQ,QAAQ,GAAG,CAAC;CAEvF,mBAAmB,SAAS,QAAQ,CAAC,UAA0B,YAAY,QAAQ,IAAI,CAAC;CAExF,iBAAiB,SACd,QAAQ,CAAC,QAAQ,MAAM,WACtB,UAAU,QAAQ,QAAQ,MAAM,KAAK,CACzC;CAEA,kBAAkB,SACf,QAAQ,CAAC,MAAM,WACd,WAAW,QAAQ,MAAM,KAAK,CAClC;CAEA,WAAW,SAAS,QAAQ,CAAC,aAA0C,IAAI,QAAQ,OAAO,CAAC;CAE3F,cAAc,SAAS,QAAQ,CAAC,UAAoC,OAAO,QAAQ,IAAI,CAAC;CAExF,eAAe,SAAS,QAAQ,CAAC,WAAgC,QAAQ,QAAQ,KAAK,CAAC;CAEvF,QAAQ,SAAS,QAAQ,GAAO,EAAE,UAAkB;EAClD,MAAM,UAAU;EAChB,IAAI,SAAS,YAAY,WAAW,OAAO,QAAQ,UAAU,UAE3D,OAAO;GACL,MAAM;GACN,eACE,qBAAqB,QAAQ,SAAS,KAAK,QAAQ,MAAM,KAAK,QAAQ;EAC1E;EAEF,MAAM,IAAI;EACV,IAAI,OAAO,GAAG,UAAU,UAAU,MAAM,IAAI,UAAU,sCAAsC;EAC5F,MAAM,QAAQ,EAAE,MAAM,QAAQ,CAAC;EAC/B,OAAO;GACL,MAAM,EAAE,YAAY;GACpB,eACE,EAAE,YAAY,SACV,iCAAiC,MAAM,KAAK,EAAE,UAAU,QAAQ,EAAE,SAAS,KAC3E,eAAe,MAAM,YAAY,EAAE,UAAU,QAAQ,EAAE,SAAS,KAAK,EAAE,MAAM;EACrF;CACF,CAAC;AACH;;AAKA,MAAM,mCAAmB,IAAI,IAAiB;CAC5C;CACA;CACA;CACA;CACA;AACF,CAAC;AAKD,MAAa,iBAAiB,SAAsC,OAAO,OAAO,UAAU,IAAI;;AAGhG,MAAa,cACX,MACA,QACA,MACA,UAEC,SAAS,KAAK,CAA8C,QAAQ,MAAM,KAAK;;AASlF,MAAa,WAAW,OAAO,YAC5B,OAAO,KAAK,QAAQ,CAAC,CAAmB,KAAK,SAAS,CACrD,MACA,SAAiD,QAAiB,GAAG,MAAiB;CACpF,OAAO,WAAW,MAAM,QAAQ,MAAM,EAAE,KAAK,CAAC,CAAC,MAAM,MAAM,CAAC;AAC9D,CACF,CAAC,CACH;;AAGA,MAAa,eAAe,OAAO,YAChC,OAAO,KAAK,QAAQ,CAAC,CACnB,QAAQ,SAAS,CAAC,iBAAiB,IAAI,IAAI,CAAC,CAAC,CAC7C,KAAK,SAAS,CAAC,MAAM,SAAS,KAAK,CAAC,CACzC"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { b as resolveModelGateway, n as analysisModelOf, v as gatewayScope } from "./config-k6m7XbAz.js";
|
|
3
|
+
import { s as isFakeJudge, u as judgeRoute } from "./judge-CKdgEXN7.js";
|
|
4
|
+
import { p as vertexLanguageModel, v as vertexSettings, y as vertexSettingsIn } from "./vertex-DypYjx71.js";
|
|
5
|
+
import { i as openRouterLanguageModel, r as openRouterFallbacks } from "./openrouter-models-CWRwLX-G.js";
|
|
4
6
|
import { z } from "zod";
|
|
5
7
|
import { Output, generateText } from "ai";
|
|
6
8
|
//#region src/analysis/schema.ts
|
|
@@ -63,7 +65,7 @@ const analysisSchema = z.object({
|
|
|
63
65
|
why: z.string()
|
|
64
66
|
})
|
|
65
67
|
});
|
|
66
|
-
const SYSTEM_PROMPT = `You
|
|
68
|
+
const SYSTEM_PROMPT = `You analyze the results of a reeval run for the engineer who wrote the evals.
|
|
67
69
|
reeval runs coding agents in sandboxes. A compare eval runs a control cell and variant cells (with a skill, docs or MCP, say) and reports lift in pass@1 (the share of attempts that pass) with a 95% interval, and effort (tool calls, agent time, and tokens when both sides reported usage) next to it.
|
|
68
70
|
|
|
69
71
|
You get a DIGEST: facts computed deterministically by reeval (ids F1...), evidence excerpts (ids E1...; transcript lines start with their line number L<n>), an attempts table (ids like r2/fix-config/control#1), the RULES reeval used for every verdict, and the CLI. PROJECT INSTRUCTIONS, when present, come first: they set the question this analysis answers. Follow them within these rules.
|
|
@@ -129,34 +131,6 @@ function questionModel(config, question, where, judge = config.judge.model) {
|
|
|
129
131
|
if (typeof model === "string" && isEvaluationModel(model)) throw new UsageError(`questions "${question.question}" (${where}) would be answered by analysis.model in ${config.file}, "${model}", an evaluation model: it scores yes/no questions and can't ${task}. Use a language model, e.g. 'google/gemini-3.8-flash', or ${fix}.`);
|
|
130
132
|
return model;
|
|
131
133
|
}
|
|
132
|
-
/**
|
|
133
|
-
* Why `analysis.model`'s calls would fail for want of a usable key: none of the keys that would
|
|
134
|
-
* work for its provider is set, or the one reeval would send has whitespace or a line break in it.
|
|
135
|
-
* Null when a usable key is set, or for any other model. `brief` names the first two keys only,
|
|
136
|
-
* for the line a run ends with; the plan's warning has them all.
|
|
137
|
-
*/
|
|
138
|
-
function analysisKeyProblem(config, env = process.env, brief = false) {
|
|
139
|
-
const model = analysisModelOf(config);
|
|
140
|
-
if (typeof model !== "string" || isEvaluationModel(model) || checkModelString(model)) return null;
|
|
141
|
-
let gateway;
|
|
142
|
-
try {
|
|
143
|
-
gateway = resolveModelGateway(model, void 0, gatewayScope(config), "analysis.model");
|
|
144
|
-
} catch (error) {
|
|
145
|
-
return brief ? "add vercel() to gateway, or analysis: { insights: false }" : `${error.message} Or analysis: { insights: false }.`;
|
|
146
|
-
}
|
|
147
|
-
const keys = judgeCredentials(model, gateway);
|
|
148
|
-
if (!keys.some((k) => env[k])) {
|
|
149
|
-
if (brief) return `set ${keys.slice(0, 2).join(" or ")}, or analysis: { insights: false }`;
|
|
150
|
-
const unused = unusedGatewayKey(env, gateway);
|
|
151
|
-
return `set ${modelKeyOptions(model, gateway)} for analysis.model, "${model}", in .env.local, or analysis: { insights: false }${unused ? `.${unused}` : ""}`;
|
|
152
|
-
}
|
|
153
|
-
try {
|
|
154
|
-
judgeRoute(model, env, "analysis model", gateway);
|
|
155
|
-
return null;
|
|
156
|
-
} catch (error) {
|
|
157
|
-
return error.message;
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
134
|
/** Checks the model and its credentials before any work; throws a UsageError (exit 2). */
|
|
161
135
|
function resolveAnalysisModel(config) {
|
|
162
136
|
const model = analysisModelName(config);
|
|
@@ -166,11 +140,15 @@ function resolveAnalysisModel(config) {
|
|
|
166
140
|
fake: model
|
|
167
141
|
};
|
|
168
142
|
if (isEvaluationModel(model)) throw new UsageError(`analysis.model in ${config.file} is "${model}", an evaluation model: it scores yes/no questions and can't write an analysis. Use a language model, e.g. 'google/gemini-3.8-flash'.`);
|
|
169
|
-
const
|
|
143
|
+
const scope = gatewayScope(config);
|
|
144
|
+
const gateway = resolveModelGateway(model, void 0, scope, "analysis.model");
|
|
170
145
|
try {
|
|
146
|
+
const route = judgeRoute(model, process.env, "analysis model", gateway).kind;
|
|
171
147
|
return {
|
|
172
148
|
name: model,
|
|
173
|
-
route
|
|
149
|
+
route,
|
|
150
|
+
...route === "openrouter" ? { fallbacks: openRouterFallbacks(scope.gateways) } : {},
|
|
151
|
+
...route === "vertex" ? { vertex: vertexSettingsIn(scope.gateways) } : {}
|
|
174
152
|
};
|
|
175
153
|
} catch (error) {
|
|
176
154
|
throw new UsageError(`${error.message} Or run reeval analyze --context, which needs no key.`);
|
|
@@ -241,9 +219,14 @@ async function languageModel(model, digest) {
|
|
|
241
219
|
if (model.fake) return fakeModel(model.fake, digest);
|
|
242
220
|
return providerModel(model);
|
|
243
221
|
}
|
|
244
|
-
/**
|
|
222
|
+
/**
|
|
223
|
+
* The provider's own model for a direct route, the AI Gateway's by name on vercel, OpenRouter's
|
|
224
|
+
* on openrouter, or Vertex AI's on vertex.
|
|
225
|
+
*/
|
|
245
226
|
async function providerModel(model) {
|
|
246
227
|
if (model.route === "gateway") return model.name;
|
|
228
|
+
if (model.route === "openrouter") return openRouterLanguageModel(model.name, !!model.fallbacks);
|
|
229
|
+
if (model.route === "vertex") return vertexLanguageModel(model.name, model.vertex ?? vertexSettings());
|
|
247
230
|
const provider = model.name.slice(0, model.name.indexOf("/"));
|
|
248
231
|
const id = model.name.slice(provider.length + 1);
|
|
249
232
|
switch (provider) {
|
|
@@ -253,6 +236,16 @@ async function providerModel(model) {
|
|
|
253
236
|
default: return model.name;
|
|
254
237
|
}
|
|
255
238
|
}
|
|
239
|
+
/** The language model `name` is on `gateway`, checked for its key: what the judge's route says. */
|
|
240
|
+
function hostModel(name, gateway, what, { fallbacks = false, vertex } = {}) {
|
|
241
|
+
const route = judgeRoute(name, process.env, what, gateway).kind;
|
|
242
|
+
return providerModel({
|
|
243
|
+
name,
|
|
244
|
+
route,
|
|
245
|
+
...route === "openrouter" ? { fallbacks } : {},
|
|
246
|
+
...route === "vertex" && vertex ? { vertex } : {}
|
|
247
|
+
});
|
|
248
|
+
}
|
|
256
249
|
async function callAnalysisModel(model, digest, prompt, maxOutputTokens) {
|
|
257
250
|
const started = Date.now();
|
|
258
251
|
const result = await generateText({
|
|
@@ -277,6 +270,6 @@ async function callAnalysisModel(model, digest, prompt, maxOutputTokens) {
|
|
|
277
270
|
};
|
|
278
271
|
}
|
|
279
272
|
//#endregion
|
|
280
|
-
export { isFake as a, questionModelLabel as c,
|
|
273
|
+
export { isFake as a, questionModelLabel as c, hostModel as i, resolveAnalysisModel as l, callAnalysisModel as n, providerModel as o, cannedModel as r, questionModel as s, analysisModelName as t, SYSTEM_PROMPT as u };
|
|
281
274
|
|
|
282
|
-
//# sourceMappingURL=model-
|
|
275
|
+
//# sourceMappingURL=model-Bq3G0OG1.js.map
|