reeval 0.14.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/THIRD_PARTY_NOTICES.md +69 -69
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-BloGVZ_a.js → adapter-CRKYPkRt.js} +37 -8
- package/dist/adapter-CRKYPkRt.js.map +1 -0
- package/dist/{agent-m-_t-IuW.js → agent-BgH4-kFc.js} +58 -16
- package/dist/agent-BgH4-kFc.js.map +1 -0
- package/dist/{app-files-C-jwhRDt.js → app-files-BMVBr079.js} +4 -4
- package/dist/app-files-BMVBr079.js.map +1 -0
- package/dist/{assemble-BDEpelpi.js → assemble-BVbI6NW-.js} +214 -128
- package/dist/assemble-BVbI6NW-.js.map +1 -0
- package/dist/assemble-SiHa4__F.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-CpbZ9arV.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-CpbZ9arV.js.map} +1 -1
- package/dist/{match-Bz0nSdzN.js → attempt-host-CIxjs2Be.js} +104 -6
- package/dist/attempt-host-CIxjs2Be.js.map +1 -0
- package/dist/{capture-CVRwRTnb.js → capture-iQ6kYJV0.js} +2360 -1523
- package/dist/capture-iQ6kYJV0.js.map +1 -0
- package/dist/capture-px7Ey95_.js +1 -0
- package/dist/{cli-C_4gaJ50.js → cli-1YQU-66F.js} +153 -180
- package/dist/cli-1YQU-66F.js.map +1 -0
- package/dist/{cli-spec-CpnbATtI.js → cli-spec-CPWT1mAG.js} +10 -21
- package/dist/cli-spec-CPWT1mAG.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-D5hNjpEx.js → client-DMmOkbRA.js} +14 -7
- package/dist/client-DMmOkbRA.js.map +1 -0
- package/dist/{command-BXLUfw4h.js → command-8_Cg5v6b.js} +35 -22
- package/dist/command-8_Cg5v6b.js.map +1 -0
- package/dist/{command-I1euLLio.js → command-BlYCAqu-.js} +7 -7
- package/dist/{command-I1euLLio.js.map → command-BlYCAqu-.js.map} +1 -1
- package/dist/{validate-BhwqrVsm.js → command-C-HeGlap.js} +388 -60
- package/dist/command-C-HeGlap.js.map +1 -0
- package/dist/{command-Bxdy62dW.js → command-Cr1zjRvl.js} +32 -31
- package/dist/command-Cr1zjRvl.js.map +1 -0
- package/dist/{commands-p1S4nw_H.js → commands-Bj9NbmsO.js} +1 -1
- package/dist/{commands-jzigbNFb.js → commands-BvTNZ2-b.js} +181 -73
- package/dist/commands-BvTNZ2-b.js.map +1 -0
- package/dist/{config-BrdHA9bJ.js → config-CneBZB84.js} +883 -433
- package/dist/config-CneBZB84.js.map +1 -0
- package/dist/{contrasts-DRxMQQTC.js → contrasts-lhFsekco.js} +3 -3
- package/dist/{contrasts-DRxMQQTC.js.map → contrasts-lhFsekco.js.map} +1 -1
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-BnXyamuj.js +2 -0
- package/dist/db-Cnh3jIv_.js +1530 -0
- package/dist/db-Cnh3jIv_.js.map +1 -0
- package/dist/debug-CAx3qcaP.js +1 -0
- package/dist/{debug-D24NOQH1.js → debug-CgG3T071.js} +5 -2
- package/dist/debug-CgG3T071.js.map +1 -0
- package/dist/{docker-BnRlTpZA.js → docker-BihzJwbV.js} +11 -6
- package/dist/docker-BihzJwbV.js.map +1 -0
- package/dist/{docker-D01slDF3.js → docker-Cqq03H72.js} +26 -8
- package/dist/docker-Cqq03H72.js.map +1 -0
- package/dist/{docker-DivfZl2u.js → docker-DCAtFMo9.js} +1 -1
- package/dist/{doctor-DQ2p0euG.js → doctor-Dl4vMeqt.js} +275 -37
- package/dist/doctor-Dl4vMeqt.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-CO0P0soF.js → environment-Cf5HMjib.js} +116 -297
- package/dist/environment-Cf5HMjib.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-CHebohjc.js} +10 -4
- package/dist/{errors-r025B0Zb.js.map → errors-CHebohjc.js.map} +1 -1
- package/dist/eval-source-CAE4y2Sr.js +146 -0
- package/dist/eval-source-CAE4y2Sr.js.map +1 -0
- package/dist/{evaluate-BUmsN7yG.js → evaluate-CUtgyr43.js} +682 -150
- package/dist/evaluate-CUtgyr43.js.map +1 -0
- package/dist/evaluate-D35T51AC.d.ts +45 -0
- package/dist/files-BEnUJW1e.js +1 -0
- package/dist/{files-B-CJeeEh.js → files-BRViE7m-.js} +200 -87
- package/dist/files-BRViE7m-.js.map +1 -0
- package/dist/{files-kUCrswI6.js → files-BSqA2sZE.js} +20 -9
- package/dist/files-BSqA2sZE.js.map +1 -0
- package/dist/files-C3tQtD26.js +1 -0
- package/dist/files-CNiJ_fZf.js +37 -0
- package/dist/files-CNiJ_fZf.js.map +1 -0
- package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
- package/dist/files-D1y8Pg4Z.js.map +1 -0
- package/dist/files-Dts7EY3H.js +1 -0
- package/dist/{pairs-CHfZNRT6.js → gate-BC6SdQZs.js} +58 -4
- package/dist/gate-BC6SdQZs.js.map +1 -0
- package/dist/gateway.d.ts +22 -9
- package/dist/gateway.js +30 -8
- package/dist/gateway.js.map +1 -1
- package/dist/{git-Bk8DSwHy.js → git-UNxT_3z0.js} +3 -3
- package/dist/{git-Bk8DSwHy.js.map → git-UNxT_3z0.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +12 -8
- package/dist/harness.js +1 -1
- package/dist/{images-DzLeFjLx.js → images-8A_LzVfw.js} +8 -8
- package/dist/{images-DzLeFjLx.js.map → images-8A_LzVfw.js.map} +1 -1
- package/dist/index.d.ts +99 -11
- package/dist/index.js +5 -5
- package/dist/init-okoALIah.js +1 -0
- package/dist/{init-SRalM5xF.js → init-sFBgODhu.js} +10 -11
- package/dist/init-sFBgODhu.js.map +1 -0
- package/dist/{invocations-DzyKSJBa.js → invocations-C9ejlOYl.js} +10 -6
- package/dist/invocations-C9ejlOYl.js.map +1 -0
- package/dist/{judge-yLvjC5A3.js → judge-Bg4v4e_K.js} +236 -46
- package/dist/judge-Bg4v4e_K.js.map +1 -0
- package/dist/judge.d.ts +5 -7
- package/dist/judge.js +4 -8
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-DeMvnDD5.js} +3 -3
- package/dist/{live-DYd2QEyR.js.map → live-DeMvnDD5.js.map} +1 -1
- package/dist/{matchers-D4eUqx4j.js → matchers-BKPuaXlj.js} +14 -13
- package/dist/matchers-BKPuaXlj.js.map +1 -0
- package/dist/{model-1ZLr9Dcv.js → model-2rBZrdZA.js} +29 -36
- package/dist/model-2rBZrdZA.js.map +1 -0
- package/dist/{model-B45lDUV1.js → model-3MJtpho8.js} +22 -13
- package/dist/model-3MJtpho8.js.map +1 -0
- package/dist/{model-CgnOpss4.js → model-Bz7-j71_.js} +245 -68
- package/dist/model-Bz7-j71_.js.map +1 -0
- package/dist/model-DkPwypN4.js +1 -0
- package/dist/model-bREXVXwT.js +1 -0
- package/dist/{node-BSq5qHnQ.js → node-fP-H2uxn.js} +72 -50
- package/dist/node-fP-H2uxn.js.map +1 -0
- package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
- package/dist/object-BkXGPKWY.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js +34 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js.map +1 -0
- package/dist/{patch-DWf-I_Q_.js → patch-6T9XAipm.js} +7 -9
- package/dist/patch-6T9XAipm.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{plan-BIwUzlB8.js → plan-Dw2BjafJ.js} +79 -35
- package/dist/plan-Dw2BjafJ.js.map +1 -0
- package/dist/{plugins-Dkp_H3pB.js → plugins-ngQEJBjJ.js} +274 -17
- package/dist/plugins-ngQEJBjJ.js.map +1 -0
- package/dist/{preflight-D9y-HvZQ.js → preflight-BYXkweNN.js} +1 -1
- package/dist/{preflight-DJCD_BDn.js → preflight-CDRpog9o.js} +3 -3
- package/dist/{preflight-DJCD_BDn.js.map → preflight-CDRpog9o.js.map} +1 -1
- package/dist/{prepare-BJG_pD54.js → prepare-B3btUlT6.js} +2 -2
- package/dist/{prepare-CIxCGG0M.js → prepare-PkARUGoF.js} +8 -8
- package/dist/{prepare-CIxCGG0M.js.map → prepare-PkARUGoF.js.map} +1 -1
- package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-CfZpFpAh.js → provenance-7Hzg88f4.js} +4 -4
- package/dist/{provenance-CfZpFpAh.js.map → provenance-7Hzg88f4.js.map} +1 -1
- package/dist/{providers-D1Zxp2he.js → providers-DDNMbv1T.js} +1 -1
- package/dist/{providers-DFk_6W4K.js → providers-PKoJtGK0.js} +5 -5
- package/dist/{providers-DFk_6W4K.js.map → providers-PKoJtGK0.js.map} +1 -1
- package/dist/query-Bt6vLJ3m.js +1 -0
- package/dist/{query-CemclveB.js → query-nBXrgEwl.js} +3 -3
- package/dist/{query-CemclveB.js.map → query-nBXrgEwl.js.map} +1 -1
- package/dist/{questions-bAudIc04.js → questions-CkAwePPR.js} +110 -36
- package/dist/questions-CkAwePPR.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-Bp6AwPgb.js +1 -0
- package/dist/{rules-VhftPOOT.js → remarks-Bt0zwiQB.js} +330 -66
- package/dist/remarks-Bt0zwiQB.js.map +1 -0
- package/dist/{header-BLPP6aCU.js → remarks-IFFe_oty.js} +35 -9
- package/dist/remarks-IFFe_oty.js.map +1 -0
- package/dist/{remarks-mujJv1Ey.js → remarks-eBcrIZRu.js} +10 -5
- package/dist/remarks-eBcrIZRu.js.map +1 -0
- package/dist/{remote-CKcEKja4.js → remote-DfJYiP-R.js} +39 -99
- package/dist/remote-DfJYiP-R.js.map +1 -0
- package/dist/resolve-CT4XZIfV.js +1 -0
- package/dist/{resolve-CBoKzQMP.js → resolve-D_cLa4tW.js} +46 -30
- package/dist/resolve-D_cLa4tW.js.map +1 -0
- package/dist/results-DrKJuIQF.js +63 -0
- package/dist/results-DrKJuIQF.js.map +1 -0
- package/dist/{resume-B9SOSnK9.js → resume-BmIM52MJ.js} +47 -17
- package/dist/resume-BmIM52MJ.js.map +1 -0
- package/dist/{runtime-BgQRyCLl.js → runtime-CZVpkr04.js} +139 -48
- package/dist/runtime-CZVpkr04.js.map +1 -0
- package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmi6IQML.js} +2 -2
- package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmi6IQML.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CKJOnIB0.js +1 -0
- package/dist/{server-TGW4WyAz.js → server-DS8BGhGU.js} +75 -33
- package/dist/server-DS8BGhGU.js.map +1 -0
- package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/source-CgUqpumI.js +1 -0
- package/dist/source-DcXv4de-.js +59 -0
- package/dist/source-DcXv4de-.js.map +1 -0
- package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
- package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
- package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
- package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-DjRI-IeE.js → src-A8HLtO0a.js} +135 -35
- package/dist/{src-DjRI-IeE.js.map → src-A8HLtO0a.js.map} +1 -1
- package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/src-BTqJbvj6.js +2 -0
- package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
- package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BkiComGh.js +2 -0
- package/dist/src-BvgeZ4tS.js +2 -0
- package/dist/{src-8zoX_qrb.js → src-C0ySiej5.js} +572 -1067
- package/dist/src-C0ySiej5.js.map +1 -0
- package/dist/src-CgJrz5p0.js +3 -0
- package/dist/src-Cp-2Hdib.js +2 -0
- package/dist/{src-CyaDNG9r.js → src-DMo2eCOJ.js} +1 -1
- package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/src-Dk2Na0b-.js +2 -0
- package/dist/src-OESRtvOi.js +2 -0
- package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
- package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-h5uDlk3m.js +1 -0
- package/dist/state-C97udoh3.js +1 -0
- package/dist/{state-B1t143wG.js → state-C_kdhp5v.js} +15 -15
- package/dist/{state-B1t143wG.js.map → state-C_kdhp5v.js.map} +1 -1
- package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
- package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
- package/dist/suite-ByT1lsdD.js +2 -0
- package/dist/{suite-WgT7m6Vs.js → suite-DieLXvCT.js} +61 -65
- package/dist/suite-DieLXvCT.js.map +1 -0
- package/dist/{summary-DIGEVr08.js → summary-CSzyBVsF.js} +113 -277
- package/dist/summary-CSzyBVsF.js.map +1 -0
- package/dist/tarball-CHcahyNK.js +258 -0
- package/dist/tarball-CHcahyNK.js.map +1 -0
- package/dist/testing.d.ts +426 -143
- package/dist/testing.js +12 -10
- package/dist/testing.js.map +1 -1
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
- package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
- package/dist/{transcript-BTxJpW4R.js → transcript-BJrKfgRW.js} +13 -5
- package/dist/transcript-BJrKfgRW.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-DXBbP9Mk.d.ts → types-DVvW66ci.d.ts} +569 -895
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +548 -121
- package/dist/view/index.js +9 -9
- package/dist/{views-sXpGE6a2.js → views-B9STsLB4.js} +9 -10
- package/dist/{views-sXpGE6a2.js.map → views-B9STsLB4.js.map} +1 -1
- package/dist/views-Ca-rTixp.js +2 -0
- package/dist/{worker-entry-Duo9oAeT.js → worker-entry-C5bLCv6s.js} +48 -20
- package/dist/worker-entry-C5bLCv6s.js.map +1 -0
- package/dist/workspace-8uHtBSve.js +1 -0
- package/dist/{workspace-CDI68wtX.js → workspace-CBuI9Uds.js} +5 -5
- package/dist/{workspace-CDI68wtX.js.map → workspace-CBuI9Uds.js.map} +1 -1
- package/docker/seccomp-default.json +1 -0
- package/package.json +5 -1
- package/skills/eval-rubric/SKILL.md +46 -104
- package/dist/adapter-BloGVZ_a.js.map +0 -1
- package/dist/agent-m-_t-IuW.js.map +0 -1
- package/dist/app-files-C-jwhRDt.js.map +0 -1
- package/dist/assemble-BDEpelpi.js.map +0 -1
- package/dist/assemble-C5bfncHU.js +0 -1
- package/dist/capture-CVRwRTnb.js.map +0 -1
- package/dist/capture-CherbjzB.js +0 -1
- package/dist/cli-C_4gaJ50.js.map +0 -1
- package/dist/cli-spec-CpnbATtI.js.map +0 -1
- package/dist/client-D5hNjpEx.js.map +0 -1
- package/dist/command-BWAuwQNk.js +0 -91
- package/dist/command-BWAuwQNk.js.map +0 -1
- package/dist/command-BXLUfw4h.js.map +0 -1
- package/dist/command-Be6BSE2L.js +0 -335
- package/dist/command-Be6BSE2L.js.map +0 -1
- package/dist/command-Bxdy62dW.js.map +0 -1
- package/dist/commands-jzigbNFb.js.map +0 -1
- package/dist/config-BrdHA9bJ.js.map +0 -1
- package/dist/cost-D4elx933.js.map +0 -1
- package/dist/cost-DmnAod9x.js +0 -1
- package/dist/db-CAfksinX.js +0 -2
- package/dist/db-DO0p7xgP.js +0 -619
- package/dist/db-DO0p7xgP.js.map +0 -1
- package/dist/debug-D24NOQH1.js.map +0 -1
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-BnRlTpZA.js.map +0 -1
- package/dist/docker-D01slDF3.js.map +0 -1
- package/dist/doctor-DQ2p0euG.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-CO0P0soF.js.map +0 -1
- package/dist/evaluate-BUmsN7yG.js.map +0 -1
- package/dist/evaluate-C_sgRA-V.d.ts +0 -134
- package/dist/facts-DzOUTreB.js +0 -906
- package/dist/facts-DzOUTreB.js.map +0 -1
- package/dist/files-B-CJeeEh.js.map +0 -1
- package/dist/files-BAt0i00X.js +0 -1
- package/dist/files-Bp5j5BqI.js +0 -1
- package/dist/files-C2sjmXhk.js +0 -1
- package/dist/files-C_gQNVB-.js +0 -65
- package/dist/files-C_gQNVB-.js.map +0 -1
- package/dist/files-Ca0d6Uq3.js.map +0 -1
- package/dist/files-kUCrswI6.js.map +0 -1
- package/dist/generate-DZ7vp69F.js +0 -1647
- package/dist/generate-DZ7vp69F.js.map +0 -1
- package/dist/harness-COVXSwJ6.js +0 -354
- package/dist/harness-COVXSwJ6.js.map +0 -1
- package/dist/header-BLPP6aCU.js.map +0 -1
- package/dist/init-RgCQXZhb.js +0 -1
- package/dist/init-SRalM5xF.js.map +0 -1
- package/dist/insights-TyAvz9aV.js +0 -68
- package/dist/insights-TyAvz9aV.js.map +0 -1
- package/dist/invocations-DzyKSJBa.js.map +0 -1
- package/dist/judge-yLvjC5A3.js.map +0 -1
- package/dist/match-Bz0nSdzN.js.map +0 -1
- package/dist/matchers-D4eUqx4j.js.map +0 -1
- package/dist/model-1ZLr9Dcv.js.map +0 -1
- package/dist/model-B45lDUV1.js.map +0 -1
- package/dist/model-BTZq8HIZ.js +0 -1
- package/dist/model-CgnOpss4.js.map +0 -1
- package/dist/model-Ch8kX87u.js +0 -1
- package/dist/node-BSq5qHnQ.js.map +0 -1
- package/dist/object-CLf2-tNq.js.map +0 -1
- package/dist/opencode-providers-klkqgzvb.js +0 -3276
- package/dist/opencode-providers-klkqgzvb.js.map +0 -1
- package/dist/pairs-CHfZNRT6.js.map +0 -1
- package/dist/patch-DWf-I_Q_.js.map +0 -1
- package/dist/plan-BIwUzlB8.js.map +0 -1
- package/dist/plugins-Dkp_H3pB.js.map +0 -1
- package/dist/prices-BcLEXvdY.js.map +0 -1
- package/dist/query-Cvbwl804.js +0 -1
- package/dist/questions-bAudIc04.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-CXwSnSjR.js +0 -278
- package/dist/remarks-CXwSnSjR.js.map +0 -1
- package/dist/remarks-CyAu2DiB.js +0 -1
- package/dist/remarks-mujJv1Ey.js.map +0 -1
- package/dist/remote-CKcEKja4.js.map +0 -1
- package/dist/resolve-B5-uqmJ6.js +0 -1
- package/dist/resolve-CBoKzQMP.js.map +0 -1
- package/dist/results-CMgZEANy.js +0 -78
- package/dist/results-CMgZEANy.js.map +0 -1
- package/dist/resume-B9SOSnK9.js.map +0 -1
- package/dist/rules-VhftPOOT.js.map +0 -1
- package/dist/run-CwIEK16q.js +0 -88
- package/dist/run-CwIEK16q.js.map +0 -1
- package/dist/runtime-BgQRyCLl.js.map +0 -1
- package/dist/server-CHRSgKRp.js +0 -1
- package/dist/server-TGW4WyAz.js.map +0 -1
- package/dist/shared-BDUzStad.js.map +0 -1
- package/dist/source-B1WvBIQt.js +0 -40
- package/dist/source-B1WvBIQt.js.map +0 -1
- package/dist/source-DAzztZwU.js +0 -1
- package/dist/src-1jHbFaGW.js +0 -1
- package/dist/src-8zoX_qrb.js.map +0 -1
- package/dist/src-C_9wgowC.js +0 -2
- package/dist/src-ChhtI6BP.js.map +0 -1
- package/dist/src-D005Cbqz.js +0 -2
- package/dist/src-DIfRbp5a.js +0 -3
- package/dist/src-Dg8121Xy.js +0 -2
- package/dist/src-ElBqEN-S.js +0 -2
- package/dist/src-NSdQ9zI6.js.map +0 -1
- package/dist/src-YHGaUrGE.js +0 -2
- package/dist/src-mJVxw1Jm.js +0 -2
- package/dist/state-vznMh2wl.js +0 -1
- package/dist/suite-BsWGHzlG.js +0 -2
- package/dist/suite-WgT7m6Vs.js.map +0 -1
- package/dist/summary-DIGEVr08.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-DF3_p5pE.js +0 -1
- package/dist/validate-BhwqrVsm.js.map +0 -1
- package/dist/views-BW_C5eVH.js +0 -2
- package/dist/worker-entry-Duo9oAeT.js.map +0 -1
- package/dist/workspace-QZCacP8A.js +0 -1
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { b as resolveModelGateway, n as analysisModelOf, v as gatewayScope } from "./config-CneBZB84.js";
|
|
3
|
+
import { s as isFakeJudge, u as judgeRoute } from "./judge-Bg4v4e_K.js";
|
|
4
|
+
import { p as vertexLanguageModel, v as vertexSettings, y as vertexSettingsIn } from "./vertex-DypYjx71.js";
|
|
5
|
+
import { i as openRouterLanguageModel, r as openRouterFallbacks } from "./openrouter-models-Bo8PZ4Ew.js";
|
|
4
6
|
import { z } from "zod";
|
|
5
7
|
import { Output, generateText } from "ai";
|
|
6
8
|
//#region src/analysis/schema.ts
|
|
@@ -63,7 +65,7 @@ const analysisSchema = z.object({
|
|
|
63
65
|
why: z.string()
|
|
64
66
|
})
|
|
65
67
|
});
|
|
66
|
-
const SYSTEM_PROMPT = `You
|
|
68
|
+
const SYSTEM_PROMPT = `You analyze the results of a reeval run for the engineer who wrote the evals.
|
|
67
69
|
reeval runs coding agents in sandboxes. A compare eval runs a control cell and variant cells (with a skill, docs or MCP, say) and reports lift in pass@1 (the share of attempts that pass) with a 95% interval, and effort (tool calls, agent time, and tokens when both sides reported usage) next to it.
|
|
68
70
|
|
|
69
71
|
You get a DIGEST: facts computed deterministically by reeval (ids F1...), evidence excerpts (ids E1...; transcript lines start with their line number L<n>), an attempts table (ids like r2/fix-config/control#1), the RULES reeval used for every verdict, and the CLI. PROJECT INSTRUCTIONS, when present, come first: they set the question this analysis answers. Follow them within these rules.
|
|
@@ -129,34 +131,6 @@ function questionModel(config, question, where, judge = config.judge.model) {
|
|
|
129
131
|
if (typeof model === "string" && isEvaluationModel(model)) throw new UsageError(`questions "${question.question}" (${where}) would be answered by analysis.model in ${config.file}, "${model}", an evaluation model: it scores yes/no questions and can't ${task}. Use a language model, e.g. 'google/gemini-3.8-flash', or ${fix}.`);
|
|
130
132
|
return model;
|
|
131
133
|
}
|
|
132
|
-
/**
|
|
133
|
-
* Why `analysis.model`'s calls would fail for want of a usable key: none of the keys that would
|
|
134
|
-
* work for its provider is set, or the one reeval would send has whitespace or a line break in it.
|
|
135
|
-
* Null when a usable key is set, or for any other model. `brief` names the first two keys only,
|
|
136
|
-
* for the line a run ends with; the plan's warning has them all.
|
|
137
|
-
*/
|
|
138
|
-
function analysisKeyProblem(config, env = process.env, brief = false) {
|
|
139
|
-
const model = analysisModelOf(config);
|
|
140
|
-
if (typeof model !== "string" || isEvaluationModel(model) || checkModelString(model)) return null;
|
|
141
|
-
let gateway;
|
|
142
|
-
try {
|
|
143
|
-
gateway = resolveModelGateway(model, void 0, gatewayScope(config), "analysis.model");
|
|
144
|
-
} catch (error) {
|
|
145
|
-
return brief ? "add vercel() to gateway, or analysis: { insights: false }" : `${error.message} Or analysis: { insights: false }.`;
|
|
146
|
-
}
|
|
147
|
-
const keys = judgeCredentials(model, gateway);
|
|
148
|
-
if (!keys.some((k) => env[k])) {
|
|
149
|
-
if (brief) return `set ${keys.slice(0, 2).join(" or ")}, or analysis: { insights: false }`;
|
|
150
|
-
const unused = unusedGatewayKey(env, gateway);
|
|
151
|
-
return `set ${modelKeyOptions(model, gateway)} for analysis.model, "${model}", in .env.local, or analysis: { insights: false }${unused ? `.${unused}` : ""}`;
|
|
152
|
-
}
|
|
153
|
-
try {
|
|
154
|
-
judgeRoute(model, env, "analysis model", gateway);
|
|
155
|
-
return null;
|
|
156
|
-
} catch (error) {
|
|
157
|
-
return error.message;
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
134
|
/** Checks the model and its credentials before any work; throws a UsageError (exit 2). */
|
|
161
135
|
function resolveAnalysisModel(config) {
|
|
162
136
|
const model = analysisModelName(config);
|
|
@@ -166,11 +140,15 @@ function resolveAnalysisModel(config) {
|
|
|
166
140
|
fake: model
|
|
167
141
|
};
|
|
168
142
|
if (isEvaluationModel(model)) throw new UsageError(`analysis.model in ${config.file} is "${model}", an evaluation model: it scores yes/no questions and can't write an analysis. Use a language model, e.g. 'google/gemini-3.8-flash'.`);
|
|
169
|
-
const
|
|
143
|
+
const scope = gatewayScope(config);
|
|
144
|
+
const gateway = resolveModelGateway(model, void 0, scope, "analysis.model");
|
|
170
145
|
try {
|
|
146
|
+
const route = judgeRoute(model, process.env, "analysis model", gateway).kind;
|
|
171
147
|
return {
|
|
172
148
|
name: model,
|
|
173
|
-
route
|
|
149
|
+
route,
|
|
150
|
+
...route === "openrouter" ? { fallbacks: openRouterFallbacks(scope.gateways) } : {},
|
|
151
|
+
...route === "vertex" ? { vertex: vertexSettingsIn(scope.gateways) } : {}
|
|
174
152
|
};
|
|
175
153
|
} catch (error) {
|
|
176
154
|
throw new UsageError(`${error.message} Or run reeval analyze --context, which needs no key.`);
|
|
@@ -241,9 +219,14 @@ async function languageModel(model, digest) {
|
|
|
241
219
|
if (model.fake) return fakeModel(model.fake, digest);
|
|
242
220
|
return providerModel(model);
|
|
243
221
|
}
|
|
244
|
-
/**
|
|
222
|
+
/**
|
|
223
|
+
* The provider's own model for a direct route, the AI Gateway's by name on vercel, OpenRouter's
|
|
224
|
+
* on openrouter, or Vertex AI's on vertex.
|
|
225
|
+
*/
|
|
245
226
|
async function providerModel(model) {
|
|
246
227
|
if (model.route === "gateway") return model.name;
|
|
228
|
+
if (model.route === "openrouter") return openRouterLanguageModel(model.name, !!model.fallbacks);
|
|
229
|
+
if (model.route === "vertex") return vertexLanguageModel(model.name, model.vertex ?? vertexSettings());
|
|
247
230
|
const provider = model.name.slice(0, model.name.indexOf("/"));
|
|
248
231
|
const id = model.name.slice(provider.length + 1);
|
|
249
232
|
switch (provider) {
|
|
@@ -253,6 +236,16 @@ async function providerModel(model) {
|
|
|
253
236
|
default: return model.name;
|
|
254
237
|
}
|
|
255
238
|
}
|
|
239
|
+
/** The language model `name` is on `gateway`, checked for its key: what the judge's route says. */
|
|
240
|
+
function hostModel(name, gateway, what, { fallbacks = false, vertex } = {}) {
|
|
241
|
+
const route = judgeRoute(name, process.env, what, gateway).kind;
|
|
242
|
+
return providerModel({
|
|
243
|
+
name,
|
|
244
|
+
route,
|
|
245
|
+
...route === "openrouter" ? { fallbacks } : {},
|
|
246
|
+
...route === "vertex" && vertex ? { vertex } : {}
|
|
247
|
+
});
|
|
248
|
+
}
|
|
256
249
|
async function callAnalysisModel(model, digest, prompt, maxOutputTokens) {
|
|
257
250
|
const started = Date.now();
|
|
258
251
|
const result = await generateText({
|
|
@@ -277,6 +270,6 @@ async function callAnalysisModel(model, digest, prompt, maxOutputTokens) {
|
|
|
277
270
|
};
|
|
278
271
|
}
|
|
279
272
|
//#endregion
|
|
280
|
-
export { isFake as a, questionModelLabel as c,
|
|
273
|
+
export { isFake as a, questionModelLabel as c, hostModel as i, resolveAnalysisModel as l, callAnalysisModel as n, providerModel as o, cannedModel as r, questionModel as s, analysisModelName as t, SYSTEM_PROMPT as u };
|
|
281
274
|
|
|
282
|
-
//# sourceMappingURL=model-
|
|
275
|
+
//# sourceMappingURL=model-2rBZrdZA.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-2rBZrdZA.js","names":[],"sources":["../src/analysis/schema.ts","../src/analysis/model.ts"],"mappings":";;;;;;;;AAGA,MAAM,aAAa,EAAE,KAAK;CAAC;CAAQ;CAAU;AAAK,CAAC;AACnD,MAAM,QAAQ,EACX,MAAM,EAAE,OAAO,CAAC,CAAC,CACjB,SAAS,uFAAuF;AAEnG,MAAM,QAAQ,EAAE,OAAO;CACrB,MAAM,EAAE,OAAO;CACf;CACA;CACA,QAAQ,EACL,MAAM,EAAE,OAAO,CAAC,CAAC,CACjB,SAAS,CAAC,CACV,SAAS,qEAAqE;AACnF,CAAC;AAED,MAAa,iBAAiB,EAAE,OAAO;CACrC,MAAM,EACH,OAAO,CAAC,CACR,SACC,2HACF;CACF,cAAc,EAAE,MAAM,KAAK;CAC3B,SAAS,EACN,MAAM,KAAK,CAAC,CACZ,SACC,uLACF;CACF,KAAK,EACF,MAAM,KAAK,CAAC,CACZ,SAAS,yEAAyE;CACrF,WAAW,EAAE,MACX,EAAE,OAAO;EACP,MAAM,EAAE,OAAO;EACf,MAAM,EAAE,KAAK;GAAC;GAAc;GAAe;GAAS;GAAS;EAAO,CAAC;EACrE;CACF,CAAC,CACH;CACA,SAAS,EACN,MACC,EAAE,OAAO;EACP,OAAO,EAAE,OAAO;EAChB,QAAQ,EAAE,OAAO,CAAC,CAAC,SAAS,mDAAmD;EAC/E,MAAM,EAAE,KAAK;GAAC;GAAS;GAAQ;GAAQ;GAAS;GAAU;EAAS,CAAC;EACpE,QAAQ,EAAE,OAAO;EACjB,gBAAgB,EAAE,OAAO,CAAC,CAAC,SAAS,+BAA+B;EACnE,QAAQ,EACL,OAAO,CAAC,CACR,SACC,sIACF;EACF,QAAQ,EAAE,KAAK;GAAC;GAAQ;GAAU;EAAK,CAAC;EACxC,QAAQ,EAAE,KAAK;GAAC;GAAO;GAAU;EAAM,CAAC;EACxC;CACF,CAAC,CACH,CAAC,CACA,IAAI,CAAC;CACR,YAAY,EAAE,OAAO;EAAE,SAAS;EAAY,KAAK,EAAE,OAAO;CAAE,CAAC;AAC/D,CAAC;AAID,MAAa,gBAAgB;;;;;;;;;;;;;;;;;;;;;ACnC7B,MAAa,UAAU,MACrB,OAAO,MAAM,YAAY,MAAM,QAAS,EAAwB,SAAS;AAE3E,MAAM,qBAAqB,UAAkB,MAAM,WAAW,cAAc;;;;;;AAO5E,SAAS,cACP,QACA,QAAgC,OAAO,MAAM,OACL;CACxC,MAAM,QAAQ,gBAAgB,MAAM;CACpC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,OAAO,OAAO,UAAU,YAAY,CAAC,kBAAkB,KAAK,IAAI,QAAQ,KAAA;AAC1E;;AAGA,SAAS,WAAW,OAA+B,MAAsB;CACvE,IAAI,OAAO,UAAU,YAAY,CAAC,YAAY,KAAK,GAAG,OAAO;CAC7D,OAAO,gBAAgB,YAAY,KAAK,IAAI,iBAAiB,MAAM,iEAAiE,KAAK;AAC3I;;AAGA,SAAgB,kBAAkB,QAAoD;CACpF,MAAM,QAAQ,cAAc,MAAM;CAClC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,MAAM,IAAI,WACR,gGAAgG,OAAO,OAAO,WAAW,OAAO,MAAM,OAAO,mBAAmB,EAAE,+EACpK;AACF;;;;;AAMA,SAAgB,mBACd,QACA,QAAgC,OAAO,MAAM,OAC9B;CACf,MAAM,QAAQ,cAAc,QAAQ,KAAK;CACzC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,OAAO,OAAO,KAAK,IAAI,SAAS;AAClC;;;;;AAMA,SAAgB,cACd,QACA,UACA,OACA,QAAgC,OAAO,MAAM,OACjB;CAC5B,IAAI,SAAS,UAAU,KAAA,GAAW,OAAO,SAAS;CAClD,MAAM,OAAO,SAAS,WAAW,KAAA,IAAY,8BAA8B;CAC3E,MAAM,QAAQ,cAAc,QAAQ,KAAK;CACzC,MAAM,MAAM;CACZ,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,WACR,cAAc,SAAS,SAAS,KAAK,MAAM,+FAA+F,OAAO,OAAO,WAAW,OAAO,IAAI,EAAE,OAAO,IAAI,EAC7L;CAEF,IAAI,OAAO,UAAU,YAAY,kBAAkB,KAAK,GACtD,MAAM,IAAI,WACR,cAAc,SAAS,SAAS,KAAK,MAAM,2CAA2C,OAAO,KAAK,KAAK,MAAM,+DAA+D,KAAK,6DAA6D,IAAI,EACpP;CAEF,OAAO;AACT;;AAGA,SAAgB,qBAAqB,QAA+C;CAClF,MAAM,QAAQ,kBAAkB,MAAM;CACtC,IAAI,OAAO,KAAK,GAAG,OAAO;EAAE,MAAM;EAAQ,OAAO;EAAQ,MAAM;CAAM;CACrE,IAAI,kBAAkB,KAAK,GACzB,MAAM,IAAI,WACR,qBAAqB,OAAO,KAAK,OAAO,MAAM,sIAChD;CAEF,MAAM,QAAQ,aAAa,MAAM;CACjC,MAAM,UAAU,oBAAoB,OAAO,KAAA,GAAW,OAAO,gBAAgB;CAC7E,IAAI;EACF,MAAM,QAAQ,WAAW,OAAO,QAAQ,KAAK,kBAAkB,OAAO,CAAC,CAAC;EACxE,OAAO;GACL,MAAM;GACN;GACA,GAAI,UAAU,eAAe,EAAE,WAAW,oBAAoB,MAAM,QAAQ,EAAE,IAAI,CAAC;GACnF,GAAI,UAAU,WAAW,EAAE,QAAQ,iBAAiB,MAAM,QAAQ,EAAE,IAAI,CAAC;EAC3E;CACF,SAAS,OAAO;EACd,MAAM,IAAI,WACR,GAAI,MAAgB,QAAQ,sDAC9B;CACF;AACF;;;;;;AAOA,SAAgB,YACd,QACA,cAAc,OACC;CACf,OAAO;EACL,sBAAsB;EACtB,UAAU;EACV,SAAS;EACT,eAAe,CAAC;EAChB,YAAY,OAAO,YAAY;GAC7B,IAAI,WAAW,QAAQ,MAAM,IAAI,MAAM,OAAO,KAAK;GACnD,MAAM,QAAQ,cAAc,KAAK,KAAK,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC,SAAS,CAAC,IAAI;GACnF,IAAI,SAAS,cAAc,KAAK,KAAK,OAAO,KAAK,SAAS,CAAC,IAAI;GAC/D,IAAI,OAAO,OAAO;GAElB,MAAM,MACJ,eAAe,QAAQ,oBAAoB,KAAA,KAAa,SAAS,QAAQ;GAC3E,IAAI,KAAK;IACP,SAAS,QAAQ;IACjB,OAAO,KAAK,MAAM,GAAG,SAAS,CAAC;GACjC;GACA,OAAO;IACL,SAAS,CAAC;KAAE,MAAM;KAAQ;IAAK,CAAC;IAChC,cAAc,MACV;KAAE,SAAS;KAAU,KAAK;IAAa,IACvC;KAAE,SAAS;KAAQ,KAAK;IAAO;IACnC,OAAO;KACL,aAAa;MAAE,OAAO;MAAO,SAAS;MAAO,WAAW;MAAG,YAAY;KAAE;KACzE,cAAc;MAAE,OAAO;MAAQ,MAAM;MAAQ,WAAW;KAAE;IAC5D;IACA,UAAU,CAAC;IACX,UAAU,EAAE,SAAS,gBAAgB;GACvC;EACF;EACA,UAAU,YAAY;GACpB,MAAM,IAAI,MAAM,yCAAyC;EAC3D;CACF;AACF;;AAGA,SAAS,UAAU,MAAyB,QAA+B;CACzE,IAAI,KAAK,WAAW,KAAA,GAClB,MAAM,IAAI,MAAM,+DAA+D;CAEjF,MAAM,SAAS,OAAO,KAAK,WAAW,aAAa,KAAK,OAAO,MAAM,IAAI,KAAK;CAC9E,OAAO,YAAY,EAAE,MAAM,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM,EAAE,CAAC;AAC3F;AAEA,eAAe,cAAc,OAA8B,QAAwC;CACjG,IAAI,MAAM,MAAM,OAAO,UAAU,MAAM,MAAM,MAAM;CACnD,OAAO,cAAc,KAAK;AAC5B;;;;;AAMA,eAAsB,cAAc,OAAsD;CACxF,IAAI,MAAM,UAAU,WAAW,OAAO,MAAM;CAC5C,IAAI,MAAM,UAAU,cAAc,OAAO,wBAAwB,MAAM,MAAM,CAAC,CAAC,MAAM,SAAS;CAC9F,IAAI,MAAM,UAAU,UAClB,OAAO,oBAAoB,MAAM,MAAM,MAAM,UAAU,eAAe,CAAC;CAEzE,MAAM,WAAW,MAAM,KAAK,MAAM,GAAG,MAAM,KAAK,QAAQ,GAAG,CAAC;CAC5D,MAAM,KAAK,MAAM,KAAK,MAAM,SAAS,SAAS,CAAC;CAC/C,QAAQ,UAAR;EACE,KAAK,aACH,QAAQ,MAAM,OAAO,qBAAA,CAAsB,UAAU,EAAE;EACzD,KAAK,UACH,QAAQ,MAAM,OAAO,kBAAA,CAAmB,OAAO,EAAE;EACnD,KAAK,UACH,QAAQ,MAAM,OAAO,kBAAA,CAAmB,OAAO,EAAE;EACnD,SACE,OAAO,MAAM;CACjB;AACF;;AAMA,SAAgB,UACd,MACA,SACA,MACA,EAAE,YAAY,OAAO,WAA8B,CAAC,GAC5B;CACxB,MAAM,QAAQ,WAAW,MAAM,QAAQ,KAAK,MAAM,OAAO,CAAC,CAAC;CAC3D,OAAO,cAAc;EACnB;EACA;EACA,GAAI,UAAU,eAAe,EAAE,UAAU,IAAI,CAAC;EAC9C,GAAI,UAAU,YAAY,SAAS,EAAE,OAAO,IAAI,CAAC;CACnD,CAAC;AACH;AASA,eAAsB,kBACpB,OACA,QACA,QACA,iBACsB;CACtB,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,SAAS,MAAM,aAAa;EAChC,OAAO,MAAM,cAAc,OAAO,MAAM;EACxC,QAAQ;EACR;EACA,QAAQ,OAAO,OAAO;GAAE,QAAQ;GAAgB,MAAM;EAAW,CAAC;EAClE;EACA,YAAY;CACd,CAAC;CACD,OAAO;EACL,QAAQ,OAAO;EACf,eAAe,OAAO,UAAU,WAAW;EAC3C,OAAO;GACL,aAAa,OAAO,OAAO,eAAe;GAC1C,cAAc,OAAO,OAAO,gBAAgB;EAC9C;EACA,WAAW,KAAK,IAAI,IAAI;CAC1B;AACF"}
|
|
@@ -1,12 +1,19 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { a as RECOMMENDED_JUDGE_MODEL, n as DIRECT_PROVIDERS, r as OPENROUTER_JUDGE_MODEL, u as judgeRoute } from "./judge-Bg4v4e_K.js";
|
|
2
|
+
import { u as vertexDecisionModel, v as vertexSettings } from "./vertex-DypYjx71.js";
|
|
3
|
+
import { i as debugTrace } from "./attempt-CpbZ9arV.js";
|
|
4
|
+
import { n as openRouterDecisionModel, t as openRouterBilling } from "./openrouter-models-Bo8PZ4Ew.js";
|
|
3
5
|
import { experimental_decide } from "ai";
|
|
4
6
|
//#region src/judge/model.ts
|
|
5
|
-
/**
|
|
6
|
-
|
|
7
|
+
/**
|
|
8
|
+
* See `judgeRoute`: the provider's own package on direct, the AI Gateway on vercel, OpenRouter's
|
|
9
|
+
* Decisions API on openrouter, Vertex AI on vertex.
|
|
10
|
+
*/
|
|
11
|
+
async function resolveModel(model, gateway, fallbacks, vertex) {
|
|
7
12
|
if (typeof model !== "string") return model;
|
|
8
13
|
const route = judgeRoute(model, process.env, "judge model", gateway);
|
|
9
14
|
if (route.kind === "gateway") return model;
|
|
15
|
+
if (route.kind === "openrouter") return openRouterDecisionModel(model, fallbacks);
|
|
16
|
+
if (route.kind === "vertex") return vertexDecisionModel(model, vertex ?? vertexSettings());
|
|
10
17
|
const id = model.slice(route.provider.length + 1);
|
|
11
18
|
switch (route.provider) {
|
|
12
19
|
case "anthropic": return (await import("@ai-sdk/anthropic")).anthropic.decisionModel(id);
|
|
@@ -19,22 +26,23 @@ async function resolveModel(model, gateway) {
|
|
|
19
26
|
const OPENAI_DECISION_MODEL = "gpt-6-luna";
|
|
20
27
|
/**
|
|
21
28
|
* reeval's sentence when the judge's route answers only decision models and `model` isn't one:
|
|
22
|
-
* Vercel AI Gateway
|
|
23
|
-
* its decision model. Undefined for any other error.
|
|
29
|
+
* Vercel AI Gateway and OpenRouter refuse a language model as a judge, and OpenAI's Decisions API
|
|
30
|
+
* answers only its decision model. Undefined for any other error.
|
|
24
31
|
*/
|
|
25
32
|
function judgeModelRefusal(model, gateway, error) {
|
|
26
33
|
const e = error;
|
|
27
34
|
const provider = model.slice(0, model.indexOf("/"));
|
|
28
|
-
if (gateway && /
|
|
35
|
+
if (gateway === "openrouter" && (/not a decision model/i.test(e?.message ?? "") || e?.statusCode === 404)) return `the judge model "${model}" isn't a decision model OpenRouter answers on its Decisions API (OpenRouter said: ${e.message}). Use a decision model on openrouter(), such as '${OPENROUTER_JUDGE_MODEL}'.`;
|
|
36
|
+
if (gateway === "vercel" && /is a language model, not a decision model/.test(e?.message ?? "")) {
|
|
29
37
|
const direct = DIRECT_PROVIDERS[provider];
|
|
30
38
|
const instead = provider === "openai" ? `, or 'openai/${OPENAI_DECISION_MODEL}' pinned to direct() with judge: { gateway: 'direct' } on ${direct?.key}` : direct ? `, or pin this one to direct() with judge: { gateway: 'direct' } to call it on ${direct.key}` : "";
|
|
31
39
|
return `the judge model "${model}" is a language model, and Vercel AI Gateway judges only with decision models. Use a decision model on vercel(), such as '${RECOMMENDED_JUDGE_MODEL}' or 'openai/${OPENAI_DECISION_MODEL}-decisions'${instead}.`;
|
|
32
40
|
}
|
|
33
41
|
const id = model.slice(provider.length + 1);
|
|
34
|
-
if (
|
|
42
|
+
if ((gateway ?? "direct") === "direct" && provider === "openai" && id !== OPENAI_DECISION_MODEL && e?.data?.error?.code === "model_not_found") return `the judge model "${model}" isn't a decision model: reeval's openai/* judges call OpenAI's Decisions API, which answers with '${OPENAI_DECISION_MODEL}' (OpenAI said: ${e.message}). Use 'openai/${OPENAI_DECISION_MODEL}'.`;
|
|
35
43
|
}
|
|
36
|
-
async function scoreWithModel(model, question, state, providerOptions, signal, gateway) {
|
|
37
|
-
const resolved = await resolveModel(model, gateway);
|
|
44
|
+
async function scoreWithModel(model, question, state, providerOptions, signal, gateway, fallbacks = false, vertex) {
|
|
45
|
+
const resolved = await resolveModel(model, gateway, fallbacks, vertex);
|
|
38
46
|
const result = await experimental_decide({
|
|
39
47
|
model: resolved,
|
|
40
48
|
state,
|
|
@@ -46,7 +54,7 @@ async function scoreWithModel(model, question, state, providerOptions, signal, g
|
|
|
46
54
|
abortSignal: signal,
|
|
47
55
|
maxRetries: 0
|
|
48
56
|
}).catch((error) => {
|
|
49
|
-
const refusal = typeof model === "string" && judgeModelRefusal(model,
|
|
57
|
+
const refusal = typeof model === "string" && judgeModelRefusal(model, gateway, error);
|
|
50
58
|
throw refusal ? new Error(refusal, { cause: error }) : error;
|
|
51
59
|
});
|
|
52
60
|
const { answers, usage, warnings, response, providerMetadata } = result;
|
|
@@ -59,10 +67,11 @@ async function scoreWithModel(model, question, state, providerOptions, signal, g
|
|
|
59
67
|
}, "judge provider response");
|
|
60
68
|
return {
|
|
61
69
|
score: result.answers.answer.probability,
|
|
62
|
-
resolvedModel: result.response?.modelId
|
|
70
|
+
resolvedModel: result.response?.modelId,
|
|
71
|
+
...gateway === "openrouter" ? openRouterBilling(providerMetadata) : {}
|
|
63
72
|
};
|
|
64
73
|
}
|
|
65
74
|
//#endregion
|
|
66
75
|
export { scoreWithModel };
|
|
67
76
|
|
|
68
|
-
//# sourceMappingURL=model-
|
|
77
|
+
//# sourceMappingURL=model-3MJtpho8.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-3MJtpho8.js","names":["decide"],"sources":["../src/judge/model.ts"],"mappings":";;;;;;;;;;AAmBA,eAAe,aACb,OACA,SACA,WACA,QACwB;CACxB,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,MAAM,QAAQ,WAAW,OAAO,QAAQ,KAAK,eAAe,OAAO;CACnE,IAAI,MAAM,SAAS,WAAW,OAAO;CACrC,IAAI,MAAM,SAAS,cAAc,OAAO,wBAAwB,OAAO,SAAS;CAChF,IAAI,MAAM,SAAS,UAAU,OAAO,oBAAoB,OAAO,UAAU,eAAe,CAAC;CACzF,MAAM,KAAK,MAAM,MAAM,MAAM,SAAS,SAAS,CAAC;CAChD,QAAQ,MAAM,UAAd;EACE,KAAK,aACH,QAAQ,MAAM,OAAO,qBAAA,CAAsB,UAAU,cAAc,EAAE;EACvE,KAAK,UACH,QAAQ,MAAM,OAAO,kBAAA,CAAmB,OAAO,cAAc,EAAE;EACjE,KAAK,UACH,QAAQ,MAAM,OAAO,kBAAA,CAAmB,OAAO,cAAc,EAAE;EACjE,SACE,QAAQ,MAAM,OAAO,uBAAA,CAAwB,WAAW,cAAc,EAAE;CAC5E;AACF;;AAGA,MAAM,wBAAwB;;;;;;AAS9B,SAAS,kBACP,OACA,SACA,OACoB;CACpB,MAAM,IAAI;CACV,MAAM,WAAW,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC;CAClD,IACE,YAAY,iBACX,wBAAwB,KAAK,GAAG,WAAW,EAAE,KAAK,GAAG,eAAe,MAErE,OAAO,oBAAoB,MAAM,qFAAqF,EAAE,QAAQ,oDAAoD,uBAAuB;CAE7M,IAAI,YAAY,YAAY,4CAA4C,KAAK,GAAG,WAAW,EAAE,GAAG;EAC9F,MAAM,SAAS,iBAAiB;EAChC,MAAM,UACJ,aAAa,WACT,gBAAgB,sBAAsB,4DAA4D,QAAQ,QAC1G,SACE,iFAAiF,OAAO,QACxF;EACR,OAAO,oBAAoB,MAAM,4HAA4H,wBAAwB,eAAe,sBAAsB,aAAa,QAAQ;CACjP;CACA,MAAM,KAAK,MAAM,MAAM,SAAS,SAAS,CAAC;CAC1C,KACG,WAAW,cAAc,YAC1B,aAAa,YACb,OAAO,yBACP,GAAG,MAAM,OAAO,SAAS,mBAEzB,OAAO,oBAAoB,MAAM,sGAAsG,sBAAsB,kBAAkB,EAAE,QAAQ,iBAAiB,sBAAsB;AAGpO;AAEA,eAAsB,eACpB,OACA,UACA,OACA,iBACA,QACA,SACA,YAAY,OACZ,QAMC;CACD,MAAM,WAAW,MAAM,aAAa,OAAO,SAAS,WAAW,MAAM;CACrE,MAAM,SAAS,MAAMA,oBAAO;EAC1B,OAAO;EACP;EACA,WAAW,EAAE,QAAQ;GAAE,MAAM;GAAW,cAAc;EAAS,EAAE;EACjE;EACA,aAAa;EAEb,YAAY;CACd,CAAC,CAAC,CAAC,OAAO,UAAmB;EAC3B,MAAM,UAAU,OAAO,UAAU,YAAY,kBAAkB,OAAO,SAAS,KAAK;EACpF,MAAM,UAAU,IAAI,MAAM,SAAS,EAAE,OAAO,MAAM,CAAC,IAAI;CACzD,CAAC;CACD,MAAM,EAAE,SAAS,OAAO,UAAU,UAAU,qBAAqB;CACjE,WACE,SACA,qBACA;EAAE;EAAS;EAAO;EAAU;EAAU;CAAiB,GACvD,yBACF;CACA,OAAO;EACL,OAAO,OAAO,QAAQ,OAAO;EAC7B,eAAe,OAAO,UAAU;EAChC,GAAI,YAAY,eAAe,kBAAkB,gBAAgB,IAAI,CAAC;CACxE;AACF"}
|