reeval 0.14.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/THIRD_PARTY_NOTICES.md +69 -69
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-BloGVZ_a.js → adapter-CbqbMR7y.js} +37 -8
- package/dist/adapter-CbqbMR7y.js.map +1 -0
- package/dist/{agent-m-_t-IuW.js → agent-Zgn-kiqq.js} +57 -15
- package/dist/agent-Zgn-kiqq.js.map +1 -0
- package/dist/{app-files-C-jwhRDt.js → app-files-ChbvRx3P.js} +4 -4
- package/dist/app-files-ChbvRx3P.js.map +1 -0
- package/dist/{assemble-BDEpelpi.js → assemble-SKQGW_QF.js} +136 -116
- package/dist/assemble-SKQGW_QF.js.map +1 -0
- package/dist/assemble-XBWuK3Ed.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
- package/dist/{match-Bz0nSdzN.js → attempt-host-BOp8JklD.js} +96 -5
- package/dist/attempt-host-BOp8JklD.js.map +1 -0
- package/dist/{capture-CVRwRTnb.js → capture-BM_3100g.js} +1218 -1207
- package/dist/capture-BM_3100g.js.map +1 -0
- package/dist/capture-DSK8O5bA.js +1 -0
- package/dist/{cli-C_4gaJ50.js → cli-PdYrcoKM.js} +106 -170
- package/dist/cli-PdYrcoKM.js.map +1 -0
- package/dist/{cli-spec-CpnbATtI.js → cli-spec-D0qyrPZi.js} +8 -20
- package/dist/cli-spec-D0qyrPZi.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-D5hNjpEx.js → client-Ct6s6OnF.js} +2 -2
- package/dist/{client-D5hNjpEx.js.map → client-Ct6s6OnF.js.map} +1 -1
- package/dist/{command-BXLUfw4h.js → command-4pJYVCT4.js} +33 -20
- package/dist/command-4pJYVCT4.js.map +1 -0
- package/dist/{validate-BhwqrVsm.js → command-9xm5ImqI.js} +387 -59
- package/dist/command-9xm5ImqI.js.map +1 -0
- package/dist/{command-Bxdy62dW.js → command-BqBxLgTc.js} +31 -30
- package/dist/command-BqBxLgTc.js.map +1 -0
- package/dist/{command-I1euLLio.js → command-DbSMcxZL.js} +7 -7
- package/dist/{command-I1euLLio.js.map → command-DbSMcxZL.js.map} +1 -1
- package/dist/{commands-jzigbNFb.js → commands-BU6YJCz9.js} +175 -72
- package/dist/commands-BU6YJCz9.js.map +1 -0
- package/dist/{commands-p1S4nw_H.js → commands-Bi2SEQtr.js} +1 -1
- package/dist/{config-BrdHA9bJ.js → config-k6m7XbAz.js} +499 -271
- package/dist/config-k6m7XbAz.js.map +1 -0
- package/dist/{contrasts-DRxMQQTC.js → contrasts-DLZF4dGF.js} +3 -3
- package/dist/{contrasts-DRxMQQTC.js.map → contrasts-DLZF4dGF.js.map} +1 -1
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-CpsifVln.js +1530 -0
- package/dist/db-CpsifVln.js.map +1 -0
- package/dist/db-DkD-F-X-.js +2 -0
- package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
- package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
- package/dist/debug-DeM5pswK.js +1 -0
- package/dist/{docker-BnRlTpZA.js → docker-51WvhN_X.js} +5 -4
- package/dist/docker-51WvhN_X.js.map +1 -0
- package/dist/{docker-D01slDF3.js → docker-B4-rx4dA.js} +26 -8
- package/dist/docker-B4-rx4dA.js.map +1 -0
- package/dist/{docker-DivfZl2u.js → docker-YFj4H1Cr.js} +1 -1
- package/dist/{doctor-DQ2p0euG.js → doctor-BuQnfz4e.js} +275 -37
- package/dist/doctor-BuQnfz4e.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-CO0P0soF.js → environment-BOFgzFDg.js} +115 -296
- package/dist/environment-BOFgzFDg.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
- package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
- package/dist/eval-source-Dje3rERU.js +145 -0
- package/dist/eval-source-Dje3rERU.js.map +1 -0
- package/dist/{evaluate-BUmsN7yG.js → evaluate-BPP9s3s6.js} +367 -130
- package/dist/evaluate-BPP9s3s6.js.map +1 -0
- package/dist/evaluate-dqXccn4k.d.ts +44 -0
- package/dist/{files-B-CJeeEh.js → files-Bi461Be2.js} +197 -91
- package/dist/files-Bi461Be2.js.map +1 -0
- package/dist/files-C2h2Cw4F.js +1 -0
- package/dist/files-Clipx4OW.js +37 -0
- package/dist/files-Clipx4OW.js.map +1 -0
- package/dist/files-DmUrRmmL.js +1 -0
- package/dist/{files-kUCrswI6.js → files-H24pz0hL.js} +20 -9
- package/dist/files-H24pz0hL.js.map +1 -0
- package/dist/gateway.d.ts +22 -9
- package/dist/gateway.js +30 -8
- package/dist/gateway.js.map +1 -1
- package/dist/{git-Bk8DSwHy.js → git-CsVZkNNL.js} +3 -3
- package/dist/{git-Bk8DSwHy.js.map → git-CsVZkNNL.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +12 -8
- package/dist/harness.js +1 -1
- package/dist/{images-DzLeFjLx.js → images-4mwhNkvh.js} +8 -8
- package/dist/{images-DzLeFjLx.js.map → images-4mwhNkvh.js.map} +1 -1
- package/dist/index.d.ts +51 -10
- package/dist/index.js +5 -5
- package/dist/init-BDEOopK7.js +1 -0
- package/dist/{init-SRalM5xF.js → init-cOk3D44b.js} +10 -11
- package/dist/init-cOk3D44b.js.map +1 -0
- package/dist/{invocations-DzyKSJBa.js → invocations-DW-lTk2f.js} +10 -6
- package/dist/invocations-DW-lTk2f.js.map +1 -0
- package/dist/{judge-yLvjC5A3.js → judge-CKdgEXN7.js} +236 -46
- package/dist/judge-CKdgEXN7.js.map +1 -0
- package/dist/judge.d.ts +5 -7
- package/dist/judge.js +4 -8
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
- package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
- package/dist/{matchers-D4eUqx4j.js → matchers-DT2TaGix.js} +2 -2
- package/dist/{matchers-D4eUqx4j.js.map → matchers-DT2TaGix.js.map} +1 -1
- package/dist/{model-1ZLr9Dcv.js → model-Bq3G0OG1.js} +29 -36
- package/dist/model-Bq3G0OG1.js.map +1 -0
- package/dist/{model-CgnOpss4.js → model-CTnyvh7H.js} +292 -68
- package/dist/model-CTnyvh7H.js.map +1 -0
- package/dist/model-D-SWrLZG.js +1 -0
- package/dist/model-g3XnzUv0.js +1 -0
- package/dist/{model-B45lDUV1.js → model-vRiUycWh.js} +22 -13
- package/dist/model-vRiUycWh.js.map +1 -0
- package/dist/{node-BSq5qHnQ.js → node-BVbbnFtv.js} +72 -50
- package/dist/node-BVbbnFtv.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-CWRwLX-G.js +34 -0
- package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
- package/dist/{pairs-CHfZNRT6.js → pairs-puYi-zj2.js} +11 -4
- package/dist/pairs-puYi-zj2.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{patch-DWf-I_Q_.js → patch-DKJiC5Sq.js} +5 -7
- package/dist/patch-DKJiC5Sq.js.map +1 -0
- package/dist/{plan-BIwUzlB8.js → plan-DdWkM6SW.js} +64 -29
- package/dist/plan-DdWkM6SW.js.map +1 -0
- package/dist/{plugins-Dkp_H3pB.js → plugins-D-CP6mMd.js} +253 -17
- package/dist/plugins-D-CP6mMd.js.map +1 -0
- package/dist/{preflight-DJCD_BDn.js → preflight-9ixuXzsx.js} +3 -3
- package/dist/{preflight-DJCD_BDn.js.map → preflight-9ixuXzsx.js.map} +1 -1
- package/dist/{preflight-D9y-HvZQ.js → preflight-DTiVv0gY.js} +1 -1
- package/dist/{prepare-CIxCGG0M.js → prepare-BfyikQit.js} +8 -8
- package/dist/{prepare-CIxCGG0M.js.map → prepare-BfyikQit.js.map} +1 -1
- package/dist/{prepare-BJG_pD54.js → prepare-tmBBf4W8.js} +2 -2
- package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-CfZpFpAh.js → provenance-BizwUn3h.js} +4 -4
- package/dist/{provenance-CfZpFpAh.js.map → provenance-BizwUn3h.js.map} +1 -1
- package/dist/{providers-DFk_6W4K.js → providers-B6b9Tnyi.js} +5 -5
- package/dist/{providers-DFk_6W4K.js.map → providers-B6b9Tnyi.js.map} +1 -1
- package/dist/{providers-D1Zxp2he.js → providers-DkOupEBl.js} +1 -1
- package/dist/query-Clm2mNSm.js +1 -0
- package/dist/{query-CemclveB.js → query-DyM-VExS.js} +3 -3
- package/dist/{query-CemclveB.js.map → query-DyM-VExS.js.map} +1 -1
- package/dist/{questions-bAudIc04.js → questions-BRhU8MzB.js} +100 -25
- package/dist/questions-BRhU8MzB.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-BwJ8pcTQ.js +1 -0
- package/dist/{rules-VhftPOOT.js → remarks-C7i2LMRb.js} +296 -66
- package/dist/remarks-C7i2LMRb.js.map +1 -0
- package/dist/{header-BLPP6aCU.js → remarks-D_3qhBWU.js} +34 -9
- package/dist/remarks-D_3qhBWU.js.map +1 -0
- package/dist/{remarks-mujJv1Ey.js → remarks-dA8LyoPc.js} +9 -5
- package/dist/remarks-dA8LyoPc.js.map +1 -0
- package/dist/{remote-CKcEKja4.js → remote-DFIzcx_6.js} +39 -99
- package/dist/remote-DFIzcx_6.js.map +1 -0
- package/dist/resolve--rQZ8u6Z.js +1 -0
- package/dist/{resolve-CBoKzQMP.js → resolve-DWSJpKHX.js} +46 -30
- package/dist/resolve-DWSJpKHX.js.map +1 -0
- package/dist/{results-CMgZEANy.js → results-BC3pcG_Z.js} +15 -30
- package/dist/results-BC3pcG_Z.js.map +1 -0
- package/dist/{resume-B9SOSnK9.js → resume-j-rcIryi.js} +47 -17
- package/dist/resume-j-rcIryi.js.map +1 -0
- package/dist/{runtime-BgQRyCLl.js → runtime-d56Gg3J5.js} +30 -13
- package/dist/runtime-d56Gg3J5.js.map +1 -0
- package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmln11gU.js} +2 -2
- package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmln11gU.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-TGW4WyAz.js → server-BOoH10J8.js} +51 -33
- package/dist/server-BOoH10J8.js.map +1 -0
- package/dist/server-DpuVAbrO.js +1 -0
- package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/source-BFd6f0-R.js +1 -0
- package/dist/{source-B1WvBIQt.js → source-wD04-rRG.js} +15 -10
- package/dist/source-wD04-rRG.js.map +1 -0
- package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
- package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
- package/dist/{src-CyaDNG9r.js → src-6_zz7oJs.js} +1 -1
- package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
- package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
- package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BYPMM1x7.js +2 -0
- package/dist/src-Bp4aN3rA.js +2 -0
- package/dist/src-C7upnZSd.js +2 -0
- package/dist/{src-8zoX_qrb.js → src-D91Hd6hT.js} +546 -1060
- package/dist/src-D91Hd6hT.js.map +1 -0
- package/dist/src-DH-9IO7f.js +1 -0
- package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/{src-DjRI-IeE.js → src-DgHuStEi.js} +134 -34
- package/dist/{src-DjRI-IeE.js.map → src-DgHuStEi.js.map} +1 -1
- package/dist/src-HSxBazcg.js +3 -0
- package/dist/src-LD1BAJFa.js +2 -0
- package/dist/src-dlX-Lmxg.js +2 -0
- package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
- package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-kYeaJJrl.js +2 -0
- package/dist/state-B5PLm5qY.js +1 -0
- package/dist/{state-B1t143wG.js → state-DrJ687ut.js} +15 -15
- package/dist/{state-B1t143wG.js.map → state-DrJ687ut.js.map} +1 -1
- package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
- package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
- package/dist/suite-22Kt9Grx.js +2 -0
- package/dist/{suite-WgT7m6Vs.js → suite-Co8cwelb.js} +48 -63
- package/dist/suite-Co8cwelb.js.map +1 -0
- package/dist/{summary-DIGEVr08.js → summary-DLFHVoux.js} +80 -270
- package/dist/summary-DLFHVoux.js.map +1 -0
- package/dist/testing.d.ts +417 -142
- package/dist/testing.js +12 -10
- package/dist/testing.js.map +1 -1
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
- package/dist/transcript-DxPaWHTZ.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-DXBbP9Mk.d.ts → types-coMWHaU2.d.ts} +253 -733
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +536 -121
- package/dist/view/index.js +9 -9
- package/dist/{views-sXpGE6a2.js → views-BFO1BLKu.js} +9 -10
- package/dist/{views-sXpGE6a2.js.map → views-BFO1BLKu.js.map} +1 -1
- package/dist/views-BOf8qEL8.js +2 -0
- package/dist/{worker-entry-Duo9oAeT.js → worker-entry-CJtdsKBP.js} +19 -12
- package/dist/worker-entry-CJtdsKBP.js.map +1 -0
- package/dist/{workspace-CDI68wtX.js → workspace-BRxjfa8N.js} +5 -5
- package/dist/{workspace-CDI68wtX.js.map → workspace-BRxjfa8N.js.map} +1 -1
- package/dist/workspace-DDHzkMjm.js +1 -0
- package/docker/seccomp-default.json +1 -0
- package/package.json +5 -1
- package/skills/eval-rubric/SKILL.md +35 -104
- package/dist/adapter-BloGVZ_a.js.map +0 -1
- package/dist/agent-m-_t-IuW.js.map +0 -1
- package/dist/app-files-C-jwhRDt.js.map +0 -1
- package/dist/assemble-BDEpelpi.js.map +0 -1
- package/dist/assemble-C5bfncHU.js +0 -1
- package/dist/capture-CVRwRTnb.js.map +0 -1
- package/dist/capture-CherbjzB.js +0 -1
- package/dist/cli-C_4gaJ50.js.map +0 -1
- package/dist/cli-spec-CpnbATtI.js.map +0 -1
- package/dist/command-BWAuwQNk.js +0 -91
- package/dist/command-BWAuwQNk.js.map +0 -1
- package/dist/command-BXLUfw4h.js.map +0 -1
- package/dist/command-Be6BSE2L.js +0 -335
- package/dist/command-Be6BSE2L.js.map +0 -1
- package/dist/command-Bxdy62dW.js.map +0 -1
- package/dist/commands-jzigbNFb.js.map +0 -1
- package/dist/config-BrdHA9bJ.js.map +0 -1
- package/dist/cost-D4elx933.js.map +0 -1
- package/dist/cost-DmnAod9x.js +0 -1
- package/dist/db-CAfksinX.js +0 -2
- package/dist/db-DO0p7xgP.js +0 -619
- package/dist/db-DO0p7xgP.js.map +0 -1
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-BnRlTpZA.js.map +0 -1
- package/dist/docker-D01slDF3.js.map +0 -1
- package/dist/doctor-DQ2p0euG.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-CO0P0soF.js.map +0 -1
- package/dist/evaluate-BUmsN7yG.js.map +0 -1
- package/dist/evaluate-C_sgRA-V.d.ts +0 -134
- package/dist/facts-DzOUTreB.js +0 -906
- package/dist/facts-DzOUTreB.js.map +0 -1
- package/dist/files-B-CJeeEh.js.map +0 -1
- package/dist/files-Bp5j5BqI.js +0 -1
- package/dist/files-C2sjmXhk.js +0 -1
- package/dist/files-C_gQNVB-.js +0 -65
- package/dist/files-C_gQNVB-.js.map +0 -1
- package/dist/files-kUCrswI6.js.map +0 -1
- package/dist/generate-DZ7vp69F.js +0 -1647
- package/dist/generate-DZ7vp69F.js.map +0 -1
- package/dist/harness-COVXSwJ6.js +0 -354
- package/dist/harness-COVXSwJ6.js.map +0 -1
- package/dist/header-BLPP6aCU.js.map +0 -1
- package/dist/init-RgCQXZhb.js +0 -1
- package/dist/init-SRalM5xF.js.map +0 -1
- package/dist/insights-TyAvz9aV.js +0 -68
- package/dist/insights-TyAvz9aV.js.map +0 -1
- package/dist/invocations-DzyKSJBa.js.map +0 -1
- package/dist/judge-yLvjC5A3.js.map +0 -1
- package/dist/match-Bz0nSdzN.js.map +0 -1
- package/dist/model-1ZLr9Dcv.js.map +0 -1
- package/dist/model-B45lDUV1.js.map +0 -1
- package/dist/model-BTZq8HIZ.js +0 -1
- package/dist/model-CgnOpss4.js.map +0 -1
- package/dist/model-Ch8kX87u.js +0 -1
- package/dist/node-BSq5qHnQ.js.map +0 -1
- package/dist/opencode-providers-klkqgzvb.js +0 -3276
- package/dist/opencode-providers-klkqgzvb.js.map +0 -1
- package/dist/pairs-CHfZNRT6.js.map +0 -1
- package/dist/patch-DWf-I_Q_.js.map +0 -1
- package/dist/plan-BIwUzlB8.js.map +0 -1
- package/dist/plugins-Dkp_H3pB.js.map +0 -1
- package/dist/prices-BcLEXvdY.js.map +0 -1
- package/dist/query-Cvbwl804.js +0 -1
- package/dist/questions-bAudIc04.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-CXwSnSjR.js +0 -278
- package/dist/remarks-CXwSnSjR.js.map +0 -1
- package/dist/remarks-CyAu2DiB.js +0 -1
- package/dist/remarks-mujJv1Ey.js.map +0 -1
- package/dist/remote-CKcEKja4.js.map +0 -1
- package/dist/resolve-B5-uqmJ6.js +0 -1
- package/dist/resolve-CBoKzQMP.js.map +0 -1
- package/dist/results-CMgZEANy.js.map +0 -1
- package/dist/resume-B9SOSnK9.js.map +0 -1
- package/dist/rules-VhftPOOT.js.map +0 -1
- package/dist/run-CwIEK16q.js +0 -88
- package/dist/run-CwIEK16q.js.map +0 -1
- package/dist/runtime-BgQRyCLl.js.map +0 -1
- package/dist/server-CHRSgKRp.js +0 -1
- package/dist/server-TGW4WyAz.js.map +0 -1
- package/dist/shared-BDUzStad.js.map +0 -1
- package/dist/source-B1WvBIQt.js.map +0 -1
- package/dist/source-DAzztZwU.js +0 -1
- package/dist/src-1jHbFaGW.js +0 -1
- package/dist/src-8zoX_qrb.js.map +0 -1
- package/dist/src-C_9wgowC.js +0 -2
- package/dist/src-ChhtI6BP.js.map +0 -1
- package/dist/src-D005Cbqz.js +0 -2
- package/dist/src-DIfRbp5a.js +0 -3
- package/dist/src-Dg8121Xy.js +0 -2
- package/dist/src-ElBqEN-S.js +0 -2
- package/dist/src-NSdQ9zI6.js.map +0 -1
- package/dist/src-YHGaUrGE.js +0 -2
- package/dist/src-mJVxw1Jm.js +0 -2
- package/dist/state-vznMh2wl.js +0 -1
- package/dist/suite-BsWGHzlG.js +0 -2
- package/dist/suite-WgT7m6Vs.js.map +0 -1
- package/dist/summary-DIGEVr08.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-DF3_p5pE.js +0 -1
- package/dist/validate-BhwqrVsm.js.map +0 -1
- package/dist/views-BW_C5eVH.js +0 -2
- package/dist/worker-entry-Duo9oAeT.js.map +0 -1
- package/dist/workspace-QZCacP8A.js +0 -1
|
@@ -1,20 +1,23 @@
|
|
|
1
1
|
import { n as isOneOf, t as isObjectLike } from "./object-CLf2-tNq.js";
|
|
2
2
|
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
3
|
-
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, i as durationSchema, l as nonEmpty, o as genericProblem,
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
3
|
+
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, h as parseDuration, i as durationSchema, l as nonEmpty, o as genericProblem, s as got, t as closest, u as oneOf } from "./shared-B_lvGqWz.js";
|
|
4
|
+
import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-BOFgzFDg.js";
|
|
5
|
+
import { B as gatePolicySchema, D as pluginServerName, L as egressPolicySchema, V as gateProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-D-CP6mMd.js";
|
|
6
|
+
import { _ as gatewayDescriptorSchema, f as FACTORY, g as directOffPin, h as GATEWAY_NAMES, m as harnessProblem, p as harnessDescriptorSchema, u as opencodeModelError, v as gatewayEntryDataSchema, y as gatewayNameSchema } from "./harness-3NVyTSGK.js";
|
|
6
7
|
import { a as fromCwd, l as sha256 } from "./ids-Bp6ut7ZD.js";
|
|
7
|
-
import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-
|
|
8
|
-
import {
|
|
9
|
-
import { r as
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
8
|
+
import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-Bz0l7OZI.js";
|
|
9
|
+
import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-CKdgEXN7.js";
|
|
10
|
+
import { a as VERTEX_ROUTES, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV, t as SANDBOX_ACCESS_TOKEN } from "./vertex-DypYjx71.js";
|
|
11
|
+
import { r as badCharacterAt } from "./redact-DVIaahLv.js";
|
|
12
|
+
import { s as withoutBom } from "./text-DTigcA64.js";
|
|
13
|
+
import { t as inCI } from "./env-CARj5Axq.js";
|
|
12
14
|
import { u as ownFileURL } from "./root-Dlff-zp8.js";
|
|
13
|
-
import {
|
|
15
|
+
import { F as PACKAGE_HOSTS, N as DOCKER_EGRESS, P as HARNESS_HOSTS, dt as run, ft as shellQuote } from "./src-D91Hd6hT.js";
|
|
14
16
|
import { t as InfraError } from "./types-C5vI3g1C.js";
|
|
15
|
-
import {
|
|
16
|
-
import
|
|
17
|
-
import {
|
|
17
|
+
import { i as pairFactors } from "./pairs-puYi-zj2.js";
|
|
18
|
+
import "./stats-hfpuYsAM.js";
|
|
19
|
+
import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-CbqbMR7y.js";
|
|
20
|
+
import { t as docker } from "./sandbox-Cmln11gU.js";
|
|
18
21
|
import { existsSync, readFileSync } from "node:fs";
|
|
19
22
|
import { availableParallelism, tmpdir, totalmem } from "node:os";
|
|
20
23
|
import { dirname, join, resolve, win32 } from "node:path";
|
|
@@ -82,31 +85,31 @@ function decideState(json, files) {
|
|
|
82
85
|
//#endregion
|
|
83
86
|
//#region src/contract/judge.ts
|
|
84
87
|
const judgeStateSchema = z.strictObject({
|
|
85
|
-
task: z.boolean().optional()
|
|
86
|
-
output: z.boolean().optional()
|
|
87
|
-
files: z.boolean().optional()
|
|
88
|
-
diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional()
|
|
89
|
-
contents: z.array(nonEmpty()).optional()
|
|
90
|
-
transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional()
|
|
91
|
-
turns: z.enum(["last", "all"]).optional()
|
|
92
|
-
checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
|
|
93
|
-
})
|
|
88
|
+
task: z.boolean().optional(),
|
|
89
|
+
output: z.boolean().optional(),
|
|
90
|
+
files: z.boolean().optional(),
|
|
91
|
+
diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
|
|
92
|
+
contents: z.array(nonEmpty()).optional(),
|
|
93
|
+
transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
|
|
94
|
+
turns: z.enum(["last", "all"]).optional(),
|
|
95
|
+
checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
|
|
96
|
+
});
|
|
94
97
|
const judgeFields = {
|
|
95
|
-
threshold: z.number().min(0, "must be between 0 and 1").max(1, "must be between 0 and 1").optional()
|
|
96
|
-
providerOptions: z.record(z.string(), z.record(z.string(), jsonValueSchema)).optional()
|
|
97
|
-
maxConcurrent: z.number().int().min(1).optional()
|
|
98
|
-
state: judgeStateSchema.optional()
|
|
99
|
-
gateway: z.union([gatewayNameSchema, gatewayDescriptorSchema]).optional()
|
|
98
|
+
threshold: z.number().min(0, "must be between 0 and 1").max(1, "must be between 0 and 1").optional(),
|
|
99
|
+
providerOptions: z.record(z.string(), z.record(z.string(), jsonValueSchema)).optional(),
|
|
100
|
+
maxConcurrent: z.number().int().min(1).optional(),
|
|
101
|
+
state: judgeStateSchema.optional(),
|
|
102
|
+
gateway: z.union([gatewayNameSchema, gatewayDescriptorSchema]).optional()
|
|
100
103
|
};
|
|
101
104
|
const fakeJudgeDataSchema = z.strictObject({
|
|
102
105
|
kind: z.literal("fake-judge"),
|
|
103
|
-
score: z.number().min(0).max(1)
|
|
104
|
-
})
|
|
106
|
+
score: z.number().min(0).max(1)
|
|
107
|
+
});
|
|
105
108
|
const judgeModelNameSchema = z.string().regex(/\//, "must be a 'provider/model' string, e.g. 'google/gemini-3.8-flash'");
|
|
106
109
|
const configJudgeDataSchema = z.strictObject({
|
|
107
|
-
model: z.union([z.string(), fakeJudgeDataSchema]).optional()
|
|
110
|
+
model: z.union([z.string(), fakeJudgeDataSchema]).optional(),
|
|
108
111
|
...judgeFields
|
|
109
|
-
})
|
|
112
|
+
});
|
|
110
113
|
const judgeOptionsSchema = z.strictObject({
|
|
111
114
|
model: z.custom().optional(),
|
|
112
115
|
...judgeFields
|
|
@@ -222,25 +225,25 @@ function judgeProblem(judge, site) {
|
|
|
222
225
|
}
|
|
223
226
|
//#endregion
|
|
224
227
|
//#region src/contract/questions.ts
|
|
225
|
-
const pairFactorSchema = z.enum(["environment", "harness"])
|
|
228
|
+
const pairFactorSchema = z.enum(["environment", "harness"]);
|
|
226
229
|
/** Evaluation models score yes/no questions only, so they can't answer a question. */
|
|
227
230
|
const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
|
|
228
231
|
const questionStateSchema = z.strictObject({
|
|
229
|
-
task: z.boolean().optional()
|
|
230
|
-
output: z.boolean().optional()
|
|
231
|
-
files: z.boolean().optional()
|
|
232
|
-
diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional()
|
|
233
|
-
transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional()
|
|
234
|
-
turns: z.enum(["last", "all"]).optional()
|
|
235
|
-
checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
|
|
236
|
-
})
|
|
232
|
+
task: z.boolean().optional(),
|
|
233
|
+
output: z.boolean().optional(),
|
|
234
|
+
files: z.boolean().optional(),
|
|
235
|
+
diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
|
|
236
|
+
transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
|
|
237
|
+
turns: z.enum(["last", "all"]).optional(),
|
|
238
|
+
checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
|
|
239
|
+
});
|
|
237
240
|
const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty()).min(1)]);
|
|
238
241
|
const questionModelSchema = z.string().refine((m) => m.includes("/"), { params: { provider: true } }).refine((m) => !isEvaluationModel(m), { params: { evaluation: true } });
|
|
239
242
|
const questionOptionsSchema = z.strictObject({
|
|
240
|
-
model: questionModelSchema.optional()
|
|
241
|
-
state: questionStateSchema.optional()
|
|
242
|
-
artifacts: questionArtifactsSchema.optional()
|
|
243
|
-
})
|
|
243
|
+
model: questionModelSchema.optional(),
|
|
244
|
+
state: questionStateSchema.optional(),
|
|
245
|
+
artifacts: questionArtifactsSchema.optional()
|
|
246
|
+
});
|
|
244
247
|
/** The first item a list holds twice. */
|
|
245
248
|
const twice = (items) => items.find((k, i) => items.indexOf(k) !== i);
|
|
246
249
|
const unique = (list, ctx) => {
|
|
@@ -253,11 +256,11 @@ const acrossSchema = z.union([pairFactorSchema, z.array(pairFactorSchema).min(1)
|
|
|
253
256
|
const choicesSchema = z.array(nonEmpty()).min(2).superRefine(unique);
|
|
254
257
|
const questionObjectSchema = z.strictObject({
|
|
255
258
|
question: nonEmpty(),
|
|
256
|
-
model: questionModelSchema.optional()
|
|
257
|
-
across: acrossSchema.optional()
|
|
258
|
-
choices: choicesSchema.optional()
|
|
259
|
-
state: questionStateSchema.optional()
|
|
260
|
-
artifacts: questionArtifactsSchema.optional()
|
|
259
|
+
model: questionModelSchema.optional(),
|
|
260
|
+
across: acrossSchema.optional(),
|
|
261
|
+
choices: choicesSchema.optional(),
|
|
262
|
+
state: questionStateSchema.optional(),
|
|
263
|
+
artifacts: questionArtifactsSchema.optional()
|
|
261
264
|
}).superRefine((q, ctx) => {
|
|
262
265
|
if (q.across !== void 0 && q.choices !== void 0) ctx.addIssue({
|
|
263
266
|
code: "custom",
|
|
@@ -265,7 +268,7 @@ const questionObjectSchema = z.strictObject({
|
|
|
265
268
|
message: "with across"
|
|
266
269
|
});
|
|
267
270
|
});
|
|
268
|
-
const questionSchema = z.union([nonEmpty(), questionObjectSchema])
|
|
271
|
+
const questionSchema = z.union([nonEmpty(), questionObjectSchema]);
|
|
269
272
|
const questionText = (q) => typeof q === "string" ? q : q.question;
|
|
270
273
|
const questionsSchema = z.array(questionSchema).superRefine((list, ctx) => {
|
|
271
274
|
if (twice(list.map(questionText)) !== void 0) ctx.addIssue({
|
|
@@ -387,27 +390,27 @@ function questionsProblem(value, where) {
|
|
|
387
390
|
//#region src/contract/config.ts
|
|
388
391
|
const count = () => z.number().int().min(1);
|
|
389
392
|
const captureWindowSchema = z.strictObject({
|
|
390
|
-
width: count()
|
|
391
|
-
height: count()
|
|
392
|
-
})
|
|
393
|
+
width: count(),
|
|
394
|
+
height: count()
|
|
395
|
+
});
|
|
393
396
|
const captureOptionsSchema = z.strictObject({
|
|
394
|
-
viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional()
|
|
395
|
-
timeout: durationSchema.optional()
|
|
396
|
-
browser: z.boolean().optional()
|
|
397
|
-
model: z.string().includes("/").optional()
|
|
398
|
-
})
|
|
397
|
+
viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
|
|
398
|
+
timeout: durationSchema.optional(),
|
|
399
|
+
browser: z.boolean().optional(),
|
|
400
|
+
model: z.string().includes("/").optional()
|
|
401
|
+
});
|
|
399
402
|
const harnessList = z.union([harnessDescriptorSchema, z.array(harnessDescriptorSchema).min(1)]);
|
|
400
403
|
const budgetSchema = z.strictObject({
|
|
401
|
-
attempts: count().optional()
|
|
402
|
-
duration: durationSchema.optional()
|
|
403
|
-
})
|
|
404
|
+
attempts: count().optional(),
|
|
405
|
+
duration: durationSchema.optional()
|
|
406
|
+
});
|
|
404
407
|
const historyConfigSchema = z.strictObject({
|
|
405
|
-
dir: z.string().optional()
|
|
408
|
+
dir: z.string().optional(),
|
|
406
409
|
artifacts: z.enum([
|
|
407
410
|
"all",
|
|
408
411
|
"failures",
|
|
409
412
|
"none"
|
|
410
|
-
]).optional()
|
|
413
|
+
]).optional()
|
|
411
414
|
});
|
|
412
415
|
const recordMode = z.enum([
|
|
413
416
|
"manual",
|
|
@@ -415,32 +418,32 @@ const recordMode = z.enum([
|
|
|
415
418
|
"always"
|
|
416
419
|
]);
|
|
417
420
|
const profileDefaultsDataSchema = z.strictObject({
|
|
418
|
-
harness: harnessList.optional()
|
|
419
|
-
sandbox: sandboxDescriptorSchema.optional()
|
|
420
|
-
environment: environmentDataSchema.optional()
|
|
421
|
-
runs: count().optional()
|
|
422
|
-
gate: gatePolicySchema.optional()
|
|
423
|
-
env: envVarsSchema.optional()
|
|
424
|
-
egress: egressPolicySchema.optional()
|
|
425
|
-
judge: configJudgeDataSchema.optional()
|
|
426
|
-
timeout: durationSchema.optional()
|
|
427
|
-
capture: captureOptionsSchema.optional()
|
|
428
|
-
question: questionOptionsSchema.optional()
|
|
429
|
-
})
|
|
421
|
+
harness: harnessList.optional(),
|
|
422
|
+
sandbox: sandboxDescriptorSchema.optional(),
|
|
423
|
+
environment: environmentDataSchema.optional(),
|
|
424
|
+
runs: count().optional(),
|
|
425
|
+
gate: gatePolicySchema.optional(),
|
|
426
|
+
env: envVarsSchema.optional(),
|
|
427
|
+
egress: egressPolicySchema.optional(),
|
|
428
|
+
judge: configJudgeDataSchema.optional(),
|
|
429
|
+
timeout: durationSchema.optional(),
|
|
430
|
+
capture: captureOptionsSchema.optional(),
|
|
431
|
+
question: questionOptionsSchema.optional()
|
|
432
|
+
});
|
|
430
433
|
const profileLimitSchema = z.strictObject({
|
|
431
|
-
runs: count().optional()
|
|
432
|
-
compare: z.union([z.literal(false), z.strictObject({ maxVariants: z.number().int().min(0) })]).optional()
|
|
433
|
-
harness: z.array(z.string()).optional()
|
|
434
|
-
})
|
|
434
|
+
runs: count().optional(),
|
|
435
|
+
compare: z.union([z.literal(false), z.strictObject({ maxVariants: z.number().int().min(0) })]).optional(),
|
|
436
|
+
harness: z.array(z.string()).optional()
|
|
437
|
+
});
|
|
435
438
|
const profileDataSchema = z.strictObject({
|
|
436
|
-
defaults: profileDefaultsDataSchema.optional()
|
|
437
|
-
limit: profileLimitSchema.optional()
|
|
438
|
-
judge: configJudgeDataSchema.optional()
|
|
439
|
-
budget: budgetSchema.optional()
|
|
440
|
-
record: recordMode.optional()
|
|
441
|
-
tags: z.array(z.string()).optional()
|
|
442
|
-
gateway: z.array(gatewayEntryDataSchema).min(1).optional()
|
|
443
|
-
})
|
|
439
|
+
defaults: profileDefaultsDataSchema.optional(),
|
|
440
|
+
limit: profileLimitSchema.optional(),
|
|
441
|
+
judge: configJudgeDataSchema.optional(),
|
|
442
|
+
budget: budgetSchema.optional(),
|
|
443
|
+
record: recordMode.optional(),
|
|
444
|
+
tags: z.array(z.string()).optional(),
|
|
445
|
+
gateway: z.array(gatewayEntryDataSchema).min(1).optional()
|
|
446
|
+
});
|
|
444
447
|
const fakeVote = z.enum([
|
|
445
448
|
"A",
|
|
446
449
|
"B",
|
|
@@ -449,7 +452,7 @@ const fakeVote = z.enum([
|
|
|
449
452
|
const failure = z.strictObject({ error: z.string() });
|
|
450
453
|
const fakeAnalysisDataSchema = z.strictObject({
|
|
451
454
|
kind: z.literal("fake-analysis"),
|
|
452
|
-
output: z.unknown().optional()
|
|
455
|
+
output: z.unknown().optional(),
|
|
453
456
|
vote: z.union([
|
|
454
457
|
fakeVote,
|
|
455
458
|
z.strictObject({
|
|
@@ -457,7 +460,7 @@ const fakeAnalysisDataSchema = z.strictObject({
|
|
|
457
460
|
reason: z.string().optional()
|
|
458
461
|
}),
|
|
459
462
|
failure
|
|
460
|
-
]).optional()
|
|
463
|
+
]).optional(),
|
|
461
464
|
answer: z.union([
|
|
462
465
|
z.string(),
|
|
463
466
|
z.strictObject({
|
|
@@ -465,46 +468,44 @@ const fakeAnalysisDataSchema = z.strictObject({
|
|
|
465
468
|
text: z.string().optional()
|
|
466
469
|
}),
|
|
467
470
|
failure
|
|
468
|
-
]).optional()
|
|
469
|
-
})
|
|
471
|
+
]).optional()
|
|
472
|
+
});
|
|
470
473
|
const analysisConfigDataSchema = z.strictObject({
|
|
471
|
-
model: z.union([z.string(), fakeAnalysisDataSchema]).optional()
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
maxOutputTokens: z.number().int().min(500).optional().describe("The most tokens the model may answer with (default 8000). reeval has no prices, so tokens are the cap.")
|
|
477
|
-
}).describe("`AnalysisConfig` as data: the model by name, or a fake analysis.");
|
|
474
|
+
model: z.union([z.string(), fakeAnalysisDataSchema]).optional(),
|
|
475
|
+
instructions: z.string().optional(),
|
|
476
|
+
maxInputTokens: z.number().int().min(1e3).optional(),
|
|
477
|
+
maxOutputTokens: z.number().int().min(500).optional()
|
|
478
|
+
});
|
|
478
479
|
const reevalConfigDataSchema = z.strictObject({
|
|
479
|
-
include: z.array(z.string()).optional()
|
|
480
|
-
harness: harnessList.optional()
|
|
481
|
-
sandbox: sandboxDescriptorSchema.optional()
|
|
482
|
-
environment: environmentDataSchema.optional()
|
|
483
|
-
runs: count().optional()
|
|
484
|
-
gate: gatePolicySchema.optional()
|
|
485
|
-
env: envVarsSchema.optional()
|
|
486
|
-
egress: egressPolicySchema.optional()
|
|
487
|
-
judge: configJudgeDataSchema.optional()
|
|
488
|
-
timeout: durationSchema.optional()
|
|
489
|
-
capture: captureOptionsSchema.optional()
|
|
490
|
-
question: questionOptionsSchema.optional()
|
|
491
|
-
concurrency: count().optional()
|
|
492
|
-
record: recordMode.optional()
|
|
493
|
-
history: historyConfigSchema.optional()
|
|
494
|
-
budget: budgetSchema.optional()
|
|
495
|
-
profiles: z.record(z.string(), profileDataSchema).optional()
|
|
480
|
+
include: z.array(z.string()).optional(),
|
|
481
|
+
harness: harnessList.optional(),
|
|
482
|
+
sandbox: sandboxDescriptorSchema.optional(),
|
|
483
|
+
environment: environmentDataSchema.optional(),
|
|
484
|
+
runs: count().optional(),
|
|
485
|
+
gate: gatePolicySchema.optional(),
|
|
486
|
+
env: envVarsSchema.optional(),
|
|
487
|
+
egress: egressPolicySchema.optional(),
|
|
488
|
+
judge: configJudgeDataSchema.optional(),
|
|
489
|
+
timeout: durationSchema.optional(),
|
|
490
|
+
capture: captureOptionsSchema.optional(),
|
|
491
|
+
question: questionOptionsSchema.optional(),
|
|
492
|
+
concurrency: count().optional(),
|
|
493
|
+
record: recordMode.optional(),
|
|
494
|
+
history: historyConfigSchema.optional(),
|
|
495
|
+
budget: budgetSchema.optional(),
|
|
496
|
+
profiles: z.record(z.string(), profileDataSchema).optional(),
|
|
496
497
|
defaultProfile: z.strictObject({
|
|
497
498
|
ci: z.string().optional(),
|
|
498
499
|
local: z.string().optional()
|
|
499
|
-
}).optional()
|
|
500
|
-
secrets: z.strictObject({ unbrokered: z.enum(["warn", "error"]).optional() }).optional()
|
|
500
|
+
}).optional(),
|
|
501
|
+
secrets: z.strictObject({ unbrokered: z.enum(["warn", "error"]).optional() }).optional(),
|
|
501
502
|
debug: z.strictObject({
|
|
502
503
|
keep: count().optional(),
|
|
503
504
|
traceBufferMb: z.number().positive().optional()
|
|
504
|
-
}).optional()
|
|
505
|
-
analysis: analysisConfigDataSchema.optional()
|
|
506
|
-
gateway: z.array(gatewayEntryDataSchema).min(1).optional()
|
|
507
|
-
})
|
|
505
|
+
}).optional(),
|
|
506
|
+
analysis: analysisConfigDataSchema.optional(),
|
|
507
|
+
gateway: z.array(gatewayEntryDataSchema).min(1).optional()
|
|
508
|
+
});
|
|
508
509
|
/** The object schema an issue's path leads to through `schema`, for "Did you mean". */
|
|
509
510
|
function keysAt(schema, path, value) {
|
|
510
511
|
const def = schema.def;
|
|
@@ -573,6 +574,15 @@ function configDataProblem(value, file) {
|
|
|
573
574
|
}, v)}`;
|
|
574
575
|
}
|
|
575
576
|
//#endregion
|
|
577
|
+
//#region src/gateway/routes.ts
|
|
578
|
+
/** Every gateway that brokers a key, by name. */
|
|
579
|
+
const ROUTING_GATEWAYS = {
|
|
580
|
+
vercel: VERCEL_GATEWAY,
|
|
581
|
+
openrouter: OPENROUTER_GATEWAY
|
|
582
|
+
};
|
|
583
|
+
/** Whether a harness's `gateway` names one of {@link ROUTING_GATEWAYS}. */
|
|
584
|
+
const isRoutingGateway = (name) => name !== void 0 && Object.hasOwn(ROUTING_GATEWAYS, name);
|
|
585
|
+
//#endregion
|
|
576
586
|
//#region src/harness/agents.ts
|
|
577
587
|
const WEB_EVENTS = {
|
|
578
588
|
webSearch: "web-search",
|
|
@@ -710,12 +720,40 @@ function antigravityMcpServers(servers) {
|
|
|
710
720
|
}]));
|
|
711
721
|
}
|
|
712
722
|
/**
|
|
723
|
+
* A `{env:…}` or `{file:…}` OpenCode would substitute. It runs `\{env:[^}]+\}`, then
|
|
724
|
+
* `\{file:[^}]+\}`, over its whole config text with no escape, so a reference left open at the end
|
|
725
|
+
* of a value runs on into the JSON after it, and only `{env:}` and `{file:}` are left alone.
|
|
726
|
+
*/
|
|
727
|
+
const OPENCODE_REFERENCE = /\{(?:env|file):(?!\})/;
|
|
728
|
+
const fieldKey = (key) => /^[A-Za-z_$][\w$]*$/.test(key) ? `.${key}` : `[${JSON.stringify(key)}]`;
|
|
729
|
+
/**
|
|
730
|
+
* The first part of a server OpenCode would substitute, as what the server wouldn't get: its name,
|
|
731
|
+
* a value as `mcp.<name>.<field>`, or an env or header name. Names are in the same config text as
|
|
732
|
+
* values, so they're substituted too; a secret's value is never named.
|
|
733
|
+
*/
|
|
734
|
+
function openCodeReference(name, s) {
|
|
735
|
+
if (OPENCODE_REFERENCE.test(name)) return "its name";
|
|
736
|
+
const server = `mcp${fieldKey(name)}`;
|
|
737
|
+
const [map, kind, values] = s.transport === "stdio" ? [
|
|
738
|
+
"env",
|
|
739
|
+
"env",
|
|
740
|
+
s.env ?? {}
|
|
741
|
+
] : [
|
|
742
|
+
"headers",
|
|
743
|
+
"header",
|
|
744
|
+
s.headers ?? {}
|
|
745
|
+
];
|
|
746
|
+
return [...s.transport === "stdio" ? [[`${server}.command`, s.command], ...s.args.map((a, i) => [`${server}.args[${i}]`, a])] : [[`${server}.url`, s.url]], ...Object.entries(values).flatMap(([k, v]) => [[`the ${kind} name ${k} in ${server}.${map}`, k], [`${server}.${map}${fieldKey(k)}`, v]])].find(([, text]) => OPENCODE_REFERENCE.test(text))?.[0];
|
|
747
|
+
}
|
|
748
|
+
/**
|
|
713
749
|
* OpenCode's config shape (its `mcp.servers`), which the adapter sets `codemode: false` on.
|
|
714
|
-
* OpenCode has no SSE transport.
|
|
750
|
+
* OpenCode has no SSE transport, and refuses a value it would substitute.
|
|
715
751
|
*/
|
|
716
752
|
function opencodeMcpServers(servers) {
|
|
717
753
|
return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
|
|
718
754
|
if (s.transport === "sse") throw new InfraError(`OpenCode has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
|
|
755
|
+
const field = openCodeReference(name, s);
|
|
756
|
+
if (field) throw new InfraError(`OpenCode substitutes {env:…} and {file:…} anywhere in its config, with no escape, so the ${name} server wouldn't get ${field} as written: take the reference out, or leave OpenCode out of this cell`);
|
|
719
757
|
return s.transport === "stdio" ? [name, {
|
|
720
758
|
type: "local",
|
|
721
759
|
command: [s.command, ...s.args],
|
|
@@ -975,7 +1013,7 @@ async function writePluginFiles(sandbox, layout, plugins) {
|
|
|
975
1013
|
if (existing?.trim()) {
|
|
976
1014
|
let base;
|
|
977
1015
|
try {
|
|
978
|
-
base = JSON.parse(existing);
|
|
1016
|
+
base = JSON.parse(withoutBom(existing));
|
|
979
1017
|
} catch {}
|
|
980
1018
|
if (!isObject(base)) throw new InfraError(`${f.path} isn't a JSON object, so reeval can't enable the session's plugins in it: write it as one in files, or leave it out`);
|
|
981
1019
|
const ours = JSON.parse(f.content);
|
|
@@ -1087,7 +1125,7 @@ const AGENTS = {
|
|
|
1087
1125
|
},
|
|
1088
1126
|
unreported: cursorUnreported,
|
|
1089
1127
|
mcpServers: cursorMcpServers,
|
|
1090
|
-
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-
|
|
1128
|
+
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-6_zz7oJs.js")).createCursorSdk({
|
|
1091
1129
|
auth,
|
|
1092
1130
|
...mcpServers ? { mcpServers } : {},
|
|
1093
1131
|
...descriptor.kind === "cursor" && descriptor.params ? { modelParams: descriptor.params } : {},
|
|
@@ -1124,13 +1162,14 @@ const AGENTS = {
|
|
|
1124
1162
|
}
|
|
1125
1163
|
],
|
|
1126
1164
|
mcpServers: claudeMcpServers,
|
|
1127
|
-
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./dist-CYrBwyHA.js")).createClaudeCode({
|
|
1165
|
+
adapter: async ({ auth, env, mcpServers, descriptor }) => (await import("./dist-CYrBwyHA.js")).createClaudeCode({
|
|
1128
1166
|
auth,
|
|
1129
1167
|
...mcpServers ? { mcpServers } : {},
|
|
1130
1168
|
...descriptor.kind === "claude-code" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1131
1169
|
env: {
|
|
1132
1170
|
IS_SANDBOX: "1",
|
|
1133
|
-
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1"
|
|
1171
|
+
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",
|
|
1172
|
+
...env
|
|
1134
1173
|
}
|
|
1135
1174
|
})
|
|
1136
1175
|
},
|
|
@@ -1153,8 +1192,9 @@ const AGENTS = {
|
|
|
1153
1192
|
},
|
|
1154
1193
|
servedModel: geminiServedModel,
|
|
1155
1194
|
mcpServers: antigravityMcpServers,
|
|
1156
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-
|
|
1195
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-C7upnZSd.js")).createAntigravity({
|
|
1157
1196
|
auth,
|
|
1197
|
+
...vertex ? { vertex } : {},
|
|
1158
1198
|
...mcpServers ? { mcpServers } : {},
|
|
1159
1199
|
...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
|
|
1160
1200
|
name,
|
|
@@ -1179,10 +1219,10 @@ const AGENTS = {
|
|
|
1179
1219
|
servedModel: geminiServedModel,
|
|
1180
1220
|
forModel: opencodeForModel,
|
|
1181
1221
|
mcpServers: opencodeMcpServers,
|
|
1182
|
-
adapter: async ({ auth, mcpServers, descriptor, model }) => (await import("./src-
|
|
1222
|
+
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-HSxBazcg.js")).createOpenCode({
|
|
1183
1223
|
auth,
|
|
1184
1224
|
model,
|
|
1185
|
-
vars: opencodeVars(model),
|
|
1225
|
+
...vertex ? { vertex } : { vars: opencodeVars(model) },
|
|
1186
1226
|
...descriptor.kind === "opencode" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1187
1227
|
...mcpServers ? { mcpServers } : {},
|
|
1188
1228
|
requireCredentialBrokering: true
|
|
@@ -1197,14 +1237,18 @@ const AGENTS = {
|
|
|
1197
1237
|
endpoints: GOOSE_ENDPOINTS,
|
|
1198
1238
|
expectedBlocked: {
|
|
1199
1239
|
hosts: ["models.dev"],
|
|
1200
|
-
why: "Goose's model
|
|
1240
|
+
why: "Goose's model catalog refresh, which no setting turns off"
|
|
1201
1241
|
},
|
|
1202
1242
|
servedModel: geminiServedModel,
|
|
1203
1243
|
mcpServers: gooseMcpServers,
|
|
1204
|
-
adapter: async ({ auth, mcpServers, descriptor, model }) => (await import("./src-
|
|
1244
|
+
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-kYeaJJrl.js")).createGoose({
|
|
1245
|
+
...vertex ? { vertex } : {},
|
|
1205
1246
|
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1206
1247
|
auth,
|
|
1207
|
-
provider: (await import("./src-
|
|
1248
|
+
provider: (await import("./src-kYeaJJrl.js")).AI_GATEWAY_PROVIDER
|
|
1249
|
+
} : auth["OPENROUTER_API_KEY"] ? {
|
|
1250
|
+
auth,
|
|
1251
|
+
provider: (await import("./src-kYeaJJrl.js")).OPENROUTER_PROVIDER
|
|
1208
1252
|
} : { auth: { GOOGLE_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" } },
|
|
1209
1253
|
model,
|
|
1210
1254
|
...descriptor.kind === "goose" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
@@ -1225,8 +1269,17 @@ const AGENTS = {
|
|
|
1225
1269
|
endpoints: GEMINI_ENDPOINTS,
|
|
1226
1270
|
servedModel: geminiServedModel,
|
|
1227
1271
|
mcpServers: piMcpServers,
|
|
1228
|
-
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-
|
|
1229
|
-
auth
|
|
1272
|
+
adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-Bp4aN3rA.js")).createPi({
|
|
1273
|
+
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1274
|
+
auth,
|
|
1275
|
+
gateway: (await import("./src-Bp4aN3rA.js")).PI_AI_GATEWAY
|
|
1276
|
+
} : auth["OPENROUTER_API_KEY"] ? {
|
|
1277
|
+
auth,
|
|
1278
|
+
gateway: (await import("./src-Bp4aN3rA.js")).PI_OPENROUTER
|
|
1279
|
+
} : {
|
|
1280
|
+
auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
|
|
1281
|
+
...vertex ? { vertex } : {}
|
|
1282
|
+
},
|
|
1230
1283
|
...descriptor.kind === "pi" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1231
1284
|
...mcpServers ? { mcpServers } : {},
|
|
1232
1285
|
requireCredentialBrokering: true
|
|
@@ -1255,9 +1308,9 @@ const AGENTS = {
|
|
|
1255
1308
|
},
|
|
1256
1309
|
servedModel: openaiServedModel,
|
|
1257
1310
|
mcpServers: codexMcpServers,
|
|
1258
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-
|
|
1311
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-dlX-Lmxg.js")).createCodex({
|
|
1259
1312
|
auth,
|
|
1260
|
-
...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-
|
|
1313
|
+
...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-dlX-Lmxg.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-dlX-Lmxg.js")).OPENROUTER_PROVIDER } : {},
|
|
1261
1314
|
...descriptor.kind === "codex" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1262
1315
|
...mcpServers ? { mcpServers } : {},
|
|
1263
1316
|
...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
|
|
@@ -1275,9 +1328,15 @@ const AGENTS = {
|
|
|
1275
1328
|
...fxForModel("provider/model"),
|
|
1276
1329
|
forModel: fxForModel,
|
|
1277
1330
|
mcpServers: acpMcpServers,
|
|
1278
|
-
adapter: async ({ auth, mcpServers, model }) => (await import("./src-
|
|
1331
|
+
adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-LD1BAJFa.js")).createFx({
|
|
1279
1332
|
auth,
|
|
1280
1333
|
model,
|
|
1334
|
+
...auth["OPENROUTER_API_KEY"] ? { provider: "openrouter" } : {},
|
|
1335
|
+
...env?.FX_VERTEX_PROJECT ? { vertex: {
|
|
1336
|
+
project: env.FX_VERTEX_PROJECT,
|
|
1337
|
+
location: env.FX_VERTEX_LOCATION ?? ""
|
|
1338
|
+
} } : {},
|
|
1339
|
+
...descriptor.kind === "fx" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1281
1340
|
...mcpServers ? { mcpServers } : {},
|
|
1282
1341
|
requireCredentialBrokering: true
|
|
1283
1342
|
})
|
|
@@ -1293,18 +1352,26 @@ const AGENTS = {
|
|
|
1293
1352
|
modelResponses: XAI_MODEL_RESPONSES,
|
|
1294
1353
|
servedModel: xaiServedModel,
|
|
1295
1354
|
mcpServers: acpMcpServers,
|
|
1296
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins, model }) =>
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1355
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins, model, vertexChat }) => {
|
|
1356
|
+
const grok = await import("./src-BYPMM1x7.js");
|
|
1357
|
+
return grok.createGrokBuild({
|
|
1358
|
+
...vertexChat ? {
|
|
1359
|
+
auth: { [grok.VERTEX_TOKEN_ENV]: SANDBOX_ACCESS_TOKEN },
|
|
1360
|
+
provider: grok.vertexProvider(vertexChat.project, vertexChat.region)
|
|
1361
|
+
} : {
|
|
1362
|
+
auth,
|
|
1363
|
+
...auth["AI_GATEWAY_API_KEY"] ? { provider: grok.AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: grok.OPENROUTER_PROVIDER } : {}
|
|
1364
|
+
},
|
|
1365
|
+
model,
|
|
1366
|
+
...plugins.length ? {
|
|
1367
|
+
pluginDirs: plugins.map((p) => p.dir),
|
|
1368
|
+
pluginMcpServers: plugins.flatMap((p) => p.servers.map((s) => s.slice(p.name.length + 1)))
|
|
1369
|
+
} : {},
|
|
1370
|
+
...descriptor.kind === "grok-build" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1371
|
+
...mcpServers ? { mcpServers } : {},
|
|
1372
|
+
requireCredentialBrokering: true
|
|
1373
|
+
});
|
|
1374
|
+
}
|
|
1308
1375
|
}
|
|
1309
1376
|
};
|
|
1310
1377
|
/** The harness of a config that names none: Cursor, on Cursor's own model. */
|
|
@@ -1316,20 +1383,31 @@ const DEFAULT_HARNESS = {
|
|
|
1316
1383
|
const HARNESS_EXAMPLE = "cursor({ model: 'composer-2.5' })";
|
|
1317
1384
|
const isAgentKind = (kind) => typeof kind === "string" && Object.hasOwn(AGENTS, kind);
|
|
1318
1385
|
/**
|
|
1386
|
+
* A server as the plan knows it: its names, literal values and each `secret.template`'s text. A
|
|
1387
|
+
* host secret's value is read when the attempt starts, so the plan has it as `''`.
|
|
1388
|
+
*/
|
|
1389
|
+
function plannedMcpServer(s) {
|
|
1390
|
+
const literal = (values) => Object.fromEntries(Object.entries(values).map(([k, v]) => [k, typeof v === "string" ? v : v.kind === "secret-template" ? v.template : ""]));
|
|
1391
|
+
return s.transport === "stdio" ? {
|
|
1392
|
+
transport: "stdio",
|
|
1393
|
+
command: s.command,
|
|
1394
|
+
args: s.args,
|
|
1395
|
+
env: literal(s.env)
|
|
1396
|
+
} : {
|
|
1397
|
+
transport: s.transport,
|
|
1398
|
+
url: s.url,
|
|
1399
|
+
headers: literal(s.headers)
|
|
1400
|
+
};
|
|
1401
|
+
}
|
|
1402
|
+
/**
|
|
1319
1403
|
* Why a harness's agent can't take the environment's MCP servers (Goose, OpenCode, pi and Codex
|
|
1320
|
-
* take no `mcp.sse
|
|
1321
|
-
* starts; undefined when it can.
|
|
1404
|
+
* take no `mcp.sse`; OpenCode would substitute a `{env:…}` or `{file:…}` in a name or value), in the
|
|
1405
|
+
* agent's own words, so the plan refuses them before any attempt starts; undefined when it can.
|
|
1322
1406
|
*/
|
|
1323
1407
|
function refusedMcp(harness, layers) {
|
|
1324
1408
|
if (!isAgentKind(harness.kind)) return void 0;
|
|
1325
1409
|
const servers = /* @__PURE__ */ new Map();
|
|
1326
|
-
for (const { spec } of layers) for (const [name, server] of Object.entries(spec.mcp ?? {})) servers.set(name, server
|
|
1327
|
-
...server,
|
|
1328
|
-
env: {}
|
|
1329
|
-
} : {
|
|
1330
|
-
...server,
|
|
1331
|
-
headers: {}
|
|
1332
|
-
});
|
|
1410
|
+
for (const { spec } of layers) for (const [name, server] of Object.entries(spec.mcp ?? {})) servers.set(name, plannedMcpServer(server));
|
|
1333
1411
|
if (servers.size === 0) return void 0;
|
|
1334
1412
|
try {
|
|
1335
1413
|
AGENTS[harness.kind].mcpServers(Object.fromEntries(servers));
|
|
@@ -1392,13 +1470,7 @@ function refusedPlugins(harness, plugins, own, unread = []) {
|
|
|
1392
1470
|
const fix = rest.length ? `use only: [${quoted(rest)}], or leave` : "leave";
|
|
1393
1471
|
return event ? `${name} can't run the ${p.name} plugin's ${event} hooks (hooks/hooks.json): it runs ${runs?.join(", ")} hooks; ${fix} the plugin out of this cell` : `${name} can't run the ${p.name} plugin's ${type} hooks (hooks/hooks.json): it runs only command hooks; ${fix} the plugin out of this cell`;
|
|
1394
1472
|
}
|
|
1395
|
-
const servers = Object.fromEntries(plugins.flatMap((p) => Object.entries(p.mcp).map(([server, s]) => [pluginServerName(p.name, server), s
|
|
1396
|
-
...s,
|
|
1397
|
-
env: {}
|
|
1398
|
-
} : {
|
|
1399
|
-
...s,
|
|
1400
|
-
headers: {}
|
|
1401
|
-
}])));
|
|
1473
|
+
const servers = Object.fromEntries(plugins.flatMap((p) => Object.entries(p.mcp).map(([server, s]) => [pluginServerName(p.name, server), plannedMcpServer(s)])));
|
|
1402
1474
|
if (agent && Object.keys(servers).length) {
|
|
1403
1475
|
const alongside = agent.pluginPlacement === "flat" ? Object.fromEntries(own.mcp.map((n) => [n, {
|
|
1404
1476
|
transport: "stdio",
|
|
@@ -1472,12 +1544,25 @@ function refusedPlugins(harness, plugins, own, unread = []) {
|
|
|
1472
1544
|
ruleOwner.set(file, what);
|
|
1473
1545
|
}
|
|
1474
1546
|
}
|
|
1475
|
-
/**
|
|
1476
|
-
|
|
1547
|
+
/**
|
|
1548
|
+
* A real agent's catalog entry, with what its model's provider decides, or its gateway's route:
|
|
1549
|
+
* on Vertex, the attempt's `vertex()` settings decide the project and region.
|
|
1550
|
+
*/
|
|
1551
|
+
function agentFor(h, vertex) {
|
|
1477
1552
|
const agent = AGENTS[h.kind];
|
|
1478
|
-
if (h.gateway === "
|
|
1479
|
-
const route =
|
|
1480
|
-
if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on
|
|
1553
|
+
if (h.gateway === "vertex") {
|
|
1554
|
+
const route = VERTEX_ROUTES[h.kind];
|
|
1555
|
+
if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on Vertex AI`);
|
|
1556
|
+
if (!vertex) throw new InfraError(`${agent.name} on Vertex AI has no vertex() settings`);
|
|
1557
|
+
return {
|
|
1558
|
+
...agent,
|
|
1559
|
+
...route(vertex)
|
|
1560
|
+
};
|
|
1561
|
+
}
|
|
1562
|
+
if (isRoutingGateway(h.gateway)) {
|
|
1563
|
+
const gateway = ROUTING_GATEWAYS[h.gateway];
|
|
1564
|
+
const route = gateway.routes[h.kind];
|
|
1565
|
+
if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on ${gateway.name}`);
|
|
1481
1566
|
return {
|
|
1482
1567
|
...agent,
|
|
1483
1568
|
...route(process.env)
|
|
@@ -1489,9 +1574,12 @@ function agentFor(h) {
|
|
|
1489
1574
|
} : agent;
|
|
1490
1575
|
}
|
|
1491
1576
|
/** The catalog entry for a harness's agent; undefined for the fake harness. */
|
|
1492
|
-
const agentOf = (h) => h.kind === "fake" ? void 0 : agentFor(h);
|
|
1493
|
-
/**
|
|
1494
|
-
|
|
1577
|
+
const agentOf = (h, vertex) => h.kind === "fake" ? void 0 : agentFor(h, vertex);
|
|
1578
|
+
/**
|
|
1579
|
+
* The host variable a harness authenticates with; undefined for the fake harness, and on Vertex,
|
|
1580
|
+
* whose token comes from Application Default Credentials.
|
|
1581
|
+
*/
|
|
1582
|
+
const credentialOf = (h) => h.kind === "fake" || h.gateway === "vertex" ? void 0 : agentFor(h).credential;
|
|
1495
1583
|
/** The fake agent's own layout. */
|
|
1496
1584
|
const FAKE_LAYOUT = {
|
|
1497
1585
|
rulesFile: "AGENTS.md",
|
|
@@ -1506,10 +1594,10 @@ function layoutOf(descriptor) {
|
|
|
1506
1594
|
}
|
|
1507
1595
|
/**
|
|
1508
1596
|
* The hosts a harness's agent is known to try in a session with or without plugins; a fake agent
|
|
1509
|
-
* with `layout` expects that agent's.
|
|
1597
|
+
* with `layout` expects that agent's. On Vertex, the attempt's `vertex()` settings pick the route.
|
|
1510
1598
|
*/
|
|
1511
|
-
function expectedBlockedFor(descriptor, plugins) {
|
|
1512
|
-
const expected = descriptor.kind !== "fake" ? agentFor(descriptor).expectedBlocked : descriptor.layout ? AGENTS[descriptor.layout].expectedBlocked : void 0;
|
|
1599
|
+
function expectedBlockedFor(descriptor, plugins, vertex) {
|
|
1600
|
+
const expected = descriptor.kind !== "fake" ? agentFor(descriptor, vertex).expectedBlocked : descriptor.layout ? AGENTS[descriptor.layout].expectedBlocked : void 0;
|
|
1513
1601
|
return expected?.withPlugins && !plugins ? void 0 : expected;
|
|
1514
1602
|
}
|
|
1515
1603
|
/**
|
|
@@ -1527,18 +1615,27 @@ const REFUSED_KEY = /\bHTTP 401\b|\b401\b.{0,40}\bunauthori[sz]ed\b|\bunauthori[
|
|
|
1527
1615
|
*/
|
|
1528
1616
|
function authHint(agent, message, refused = false) {
|
|
1529
1617
|
if (!refused && !REFUSED_KEY.test(message)) return void 0;
|
|
1618
|
+
if (agent.token) return `The API rejected the access token reeval minted from ${agent.credential}: check gcloud auth application-default login (or GOOGLE_APPLICATION_CREDENTIALS) and the project's Vertex AI access`;
|
|
1530
1619
|
return `The API rejected ${agent.credential}: set a valid key in the environment or .env.local`;
|
|
1531
1620
|
}
|
|
1532
|
-
/**
|
|
1621
|
+
/**
|
|
1622
|
+
* The agent's credential from the host, as the adapter's explicit `auth`. Fails closed. Empty for
|
|
1623
|
+
* a brokered token, which the adapter never sees.
|
|
1624
|
+
*/
|
|
1533
1625
|
function agentAuth(agent, env = process.env) {
|
|
1626
|
+
if (agent.token) return {};
|
|
1534
1627
|
const value = env[agent.credential];
|
|
1535
1628
|
if (!value) throw new InfraError(`${agent.credential} isn't set, so the ${agent.name} harness can't authenticate. Set it in the environment or .env.local.`);
|
|
1536
|
-
return {
|
|
1629
|
+
return {
|
|
1630
|
+
...agent.authEnv,
|
|
1631
|
+
[agent.authAs ?? agent.credential]: value
|
|
1632
|
+
};
|
|
1537
1633
|
}
|
|
1538
|
-
function brokeringFor(h) {
|
|
1539
|
-
const agent = agentOf(h);
|
|
1634
|
+
function brokeringFor(h, vertex) {
|
|
1635
|
+
const agent = agentOf(h, vertex);
|
|
1540
1636
|
if (!agent) return void 0;
|
|
1541
|
-
const { credential, endpoints, tokens, servedModel, askedModel, modelResponses } = agent;
|
|
1637
|
+
const { credential, endpoints, tokens, servedModel, askedModel, modelResponses, upstream } = agent;
|
|
1638
|
+
const { localResponses } = agent;
|
|
1542
1639
|
return {
|
|
1543
1640
|
credential,
|
|
1544
1641
|
endpoints,
|
|
@@ -1548,7 +1645,9 @@ function brokeringFor(h) {
|
|
|
1548
1645
|
} } : {},
|
|
1549
1646
|
...servedModel ? { servedModel } : {},
|
|
1550
1647
|
...askedModel ? { askedModel } : {},
|
|
1551
|
-
...modelResponses ? { modelResponses } : {}
|
|
1648
|
+
...modelResponses ? { modelResponses } : {},
|
|
1649
|
+
...upstream ? { upstream } : {},
|
|
1650
|
+
...localResponses?.length ? { localResponses } : {}
|
|
1552
1651
|
};
|
|
1553
1652
|
}
|
|
1554
1653
|
//#endregion
|
|
@@ -1631,7 +1730,7 @@ const DEFAULT_ATTEMPT_QUESTION_STATE = {
|
|
|
1631
1730
|
const KEYS = Object.keys(DEFAULT_QUESTION_STATE);
|
|
1632
1731
|
/** A pairwise question's defaults when `pairwise`, a per-attempt question's otherwise. */
|
|
1633
1732
|
const defaultQuestionState = (pairwise) => pairwise ? DEFAULT_QUESTION_STATE : DEFAULT_ATTEMPT_QUESTION_STATE;
|
|
1634
|
-
/** Every key set, in `DEFAULT_QUESTION_STATE`'s order, so equal selections
|
|
1733
|
+
/** Every key set, in `DEFAULT_QUESTION_STATE`'s order, so equal selections serialize the same. */
|
|
1635
1734
|
function resolveQuestionState(state, pairwise) {
|
|
1636
1735
|
const defaults = defaultQuestionState(pairwise);
|
|
1637
1736
|
const out = {};
|
|
@@ -1951,13 +2050,10 @@ function resolveOutput(input) {
|
|
|
1951
2050
|
const { flags, env } = input;
|
|
1952
2051
|
const ci = isCI(env);
|
|
1953
2052
|
const githubActions = set(env, "GITHUB_ACTIONS");
|
|
1954
|
-
const ui = pick("--ui", flags.ui ?? env.REEVAL_UI, [
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
]);
|
|
1959
|
-
const requested = ui && ui !== "auto" ? ui : input.agent ? "agent" : ci || !input.isTTY ? "plain" : "live";
|
|
1960
|
-
const mode = requested === "live" ? "plain" : requested;
|
|
2053
|
+
const ui = pick("--ui", flags.ui ?? env.REEVAL_UI, ["plain", "agent"]);
|
|
2054
|
+
const named = ui !== void 0 && ui !== "auto";
|
|
2055
|
+
const mode = named ? ui : input.agent ? "agent" : "plain";
|
|
2056
|
+
const statusLine = !named && mode === "plain" && !ci && input.isTTY;
|
|
1961
2057
|
const order = pick("--log-order", flags.logOrder ?? env.REEVAL_LOG_ORDER, ["stream", "grouped"]);
|
|
1962
2058
|
const logOrder = order && order !== "auto" ? order : ci ? "grouped" : "stream";
|
|
1963
2059
|
const prefix = pick("--log-prefix", flags.logPrefix ?? env.REEVAL_LOG_PREFIX, ["task", "none"]);
|
|
@@ -1968,8 +2064,8 @@ function resolveOutput(input) {
|
|
|
1968
2064
|
"none"
|
|
1969
2065
|
]);
|
|
1970
2066
|
return {
|
|
1971
|
-
requested,
|
|
1972
2067
|
mode,
|
|
2068
|
+
statusLine,
|
|
1973
2069
|
logOrder,
|
|
1974
2070
|
logPrefix,
|
|
1975
2071
|
outputLogs: logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only",
|
|
@@ -2003,8 +2099,8 @@ const isDirect = (policy) => policy === "allow-all";
|
|
|
2003
2099
|
* The hosts the harness needs: a real agent's own API (its model's provider's, for OpenCode), or
|
|
2004
2100
|
* a fake agent's with `layout` (it behaves like that agent). A plain fake agent talks to nothing.
|
|
2005
2101
|
*/
|
|
2006
|
-
function harnessHosts(harness) {
|
|
2007
|
-
if (harness.kind !== "fake" && isAgentKind(harness.kind)) return agentFor(harness).hosts ?? HARNESS_HOSTS[harness.kind];
|
|
2102
|
+
function harnessHosts(harness, vertex) {
|
|
2103
|
+
if (harness.kind !== "fake" && isAgentKind(harness.kind)) return agentFor(harness, vertex).hosts ?? HARNESS_HOSTS[harness.kind];
|
|
2008
2104
|
return harness.kind === "fake" && harness.layout ? HARNESS_HOSTS[harness.layout] : [];
|
|
2009
2105
|
}
|
|
2010
2106
|
/** MCP `http`/`sse` servers' hosts, local plugins' included: the agent connects to them from the sandbox. */
|
|
@@ -2020,7 +2116,7 @@ function mcpServerHosts(layers) {
|
|
|
2020
2116
|
* harness's, the brokered secrets' `hosts` and the MCP servers'. The cascade is config and
|
|
2021
2117
|
* profile defaults (`inherited`), then the environment's layers, then groups and the eval (`own`).
|
|
2022
2118
|
*/
|
|
2023
|
-
function plannedEgress(egress, harness, env, layers) {
|
|
2119
|
+
function plannedEgress(egress, harness, env, layers, vertex) {
|
|
2024
2120
|
const fromLayers = layers.findLast((l) => l.spec.egress !== void 0)?.spec.egress;
|
|
2025
2121
|
const policy = egress.own ?? fromLayers ?? egress.inherited ?? "proxy";
|
|
2026
2122
|
if (!isRestricted(policy)) return {
|
|
@@ -2029,7 +2125,7 @@ function plannedEgress(egress, harness, env, layers) {
|
|
|
2029
2125
|
};
|
|
2030
2126
|
const own = policy === "deny-all" ? [] : policy.allow;
|
|
2031
2127
|
const auto = [
|
|
2032
|
-
...harnessHosts(harness),
|
|
2128
|
+
...harnessHosts(harness, vertex),
|
|
2033
2129
|
...plannedSecrets(env, layers).flatMap((s) => s.hosts ?? []),
|
|
2034
2130
|
...mcpServerHosts(layers)
|
|
2035
2131
|
];
|
|
@@ -2114,7 +2210,7 @@ const DEFAULT_STATE = {
|
|
|
2114
2210
|
turns: "last",
|
|
2115
2211
|
checks: false
|
|
2116
2212
|
};
|
|
2117
|
-
/** Every key set, in `STATE_KEYS` order, so equal selections
|
|
2213
|
+
/** Every key set, in `STATE_KEYS` order, so equal selections serialize the same. */
|
|
2118
2214
|
function resolveStateOptions(state) {
|
|
2119
2215
|
const out = {};
|
|
2120
2216
|
for (const key of STATE_KEYS$1) out[key] = state?.[key] ?? DEFAULT_STATE[key];
|
|
@@ -2148,6 +2244,7 @@ const KNOWN = [...[
|
|
|
2148
2244
|
"questions",
|
|
2149
2245
|
"timeout",
|
|
2150
2246
|
"tags",
|
|
2247
|
+
"revision",
|
|
2151
2248
|
"pin"
|
|
2152
2249
|
]];
|
|
2153
2250
|
const SUGGEST = {
|
|
@@ -2165,6 +2262,8 @@ const SUGGEST = {
|
|
|
2165
2262
|
network: "egress",
|
|
2166
2263
|
networkPolicy: "egress",
|
|
2167
2264
|
tag: "tags",
|
|
2265
|
+
version: "revision",
|
|
2266
|
+
rev: "revision",
|
|
2168
2267
|
pins: "pin",
|
|
2169
2268
|
passRate: "gate",
|
|
2170
2269
|
threshold: "gate"
|
|
@@ -2235,6 +2334,7 @@ function refuseIllegalCompare(options, where) {
|
|
|
2235
2334
|
const baseline = options.compare !== null && typeof options.compare === "object" && "baseline" in options.compare ? options.compare.baseline : void 0;
|
|
2236
2335
|
if (options.environment !== void 0 && baseline !== void 0) throw new UsageError(`environment and compare.baseline can't both be set in ${where}. compare: { baseline, variants } names every arm, so drop environment, or drop baseline to make environment the control.`);
|
|
2237
2336
|
}
|
|
2337
|
+
const isRevision = (value) => typeof value === "number" && Number.isSafeInteger(value) || typeof value === "string" && value.trim() !== "";
|
|
2238
2338
|
/** Throws on keys that aren't options, and on values of the wrong shape, suggesting a fix. */
|
|
2239
2339
|
function checkOptions(options, where, allowed = KNOWN) {
|
|
2240
2340
|
for (const key of Object.keys(options)) {
|
|
@@ -2247,6 +2347,7 @@ function checkOptions(options, where, allowed = KNOWN) {
|
|
|
2247
2347
|
if (o.runs !== void 0 && !(Number.isInteger(o.runs) && o.runs >= 1)) throw new UsageError(`runs in ${where} must be a whole number of at least 1 (got ${JSON.stringify(o.runs)})`);
|
|
2248
2348
|
if (o.gate !== void 0) checkGate(o.gate, where);
|
|
2249
2349
|
if (o.tags !== void 0 && (!Array.isArray(o.tags) || o.tags.some((t) => typeof t !== "string"))) throw new UsageError(`tags in ${where} must be a list of strings, e.g. tags: ['smoke'] (got ${JSON.stringify(o.tags)})`);
|
|
2350
|
+
if (o.revision !== void 0 && !isRevision(o.revision)) throw new UsageError(`revision in ${where} must be a whole number or a non-empty string, e.g. revision: 2 (got ${JSON.stringify(o.revision)})`);
|
|
2250
2351
|
if (o.env !== void 0) checkEnv(o.env, where, UsageError);
|
|
2251
2352
|
if (o.egress !== void 0) checkEgress(o.egress, where, UsageError);
|
|
2252
2353
|
const judgeIssue = o.judge !== void 0 && judgeProblem(o.judge, {
|
|
@@ -2412,6 +2513,12 @@ function normalizeCompare(value) {
|
|
|
2412
2513
|
of: value.variants.length
|
|
2413
2514
|
};
|
|
2414
2515
|
}
|
|
2516
|
+
/**
|
|
2517
|
+
* Runs per cell when no level sets `runs`: a compare with variants runs `EFFECT_MIN_COUNTED` a
|
|
2518
|
+
* side, the fewest whose lift can reach a verdict; anything else, a pooled family's cases
|
|
2519
|
+
* included, runs once.
|
|
2520
|
+
*/
|
|
2521
|
+
const COMPARE_DEFAULT_RUNS = 4;
|
|
2415
2522
|
function applyLevel(out, options, where, egress = "own") {
|
|
2416
2523
|
for (const [key, value] of Object.entries(options)) if (value !== void 0) out.sources = {
|
|
2417
2524
|
...out.sources,
|
|
@@ -2425,6 +2532,7 @@ function applyLevel(out, options, where, egress = "own") {
|
|
|
2425
2532
|
if (options.runs !== void 0) out.runs = options.runs;
|
|
2426
2533
|
if (options.gate !== void 0) out.gate = options.gate;
|
|
2427
2534
|
if (options.timeout !== void 0) out.timeout = options.timeout;
|
|
2535
|
+
if (options.revision !== void 0) out.revision = options.revision;
|
|
2428
2536
|
Object.assign(out.env, options.env);
|
|
2429
2537
|
out.judge = mergeJudge(out.judge, options.judge);
|
|
2430
2538
|
if (options.capture !== void 0) {
|
|
@@ -2500,23 +2608,30 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2500
2608
|
const dupe = names.find((n, i) => names.indexOf(n) !== i);
|
|
2501
2609
|
if (dupe) throw new UsageError(`compare in ${where} lists two variants named ${dupe}. Give each variant its own name.`);
|
|
2502
2610
|
let skip;
|
|
2611
|
+
const harnesses = out.harness.map(harnessId);
|
|
2503
2612
|
const limit = profile?.limit;
|
|
2504
2613
|
if (profile && limit) {
|
|
2505
2614
|
const tag = `profile ${profile.name}`;
|
|
2506
2615
|
const pinned = (key) => out.pin.includes(key);
|
|
2507
|
-
|
|
2508
|
-
notes.push(`runs ${out.runs} → ${limit.runs} (${tag})`);
|
|
2509
|
-
out.runs = limit.runs;
|
|
2510
|
-
}
|
|
2616
|
+
let comparedLeft = true;
|
|
2511
2617
|
if (limit.harness && !pinned("harness")) {
|
|
2512
2618
|
const kept = out.harness.filter((h) => limit.harness?.includes(harnessId(h)));
|
|
2513
2619
|
if (kept.length === 0) skip = `no harness left: ${tag} allows ${limit.harness.join(", ") || "none"}`;
|
|
2514
2620
|
else if (kept.length < out.harness.length) {
|
|
2515
2621
|
notes.push(`harnesses ${out.harness.length} → ${kept.length} (${tag})`);
|
|
2516
2622
|
out.harness = kept;
|
|
2623
|
+
const scope = out.compare?.harness;
|
|
2624
|
+
if (out.compare?.variants.length && scope && !kept.some((h) => scope.includes(harnessId(h)))) {
|
|
2625
|
+
notes.push(`compare off: no compared harness left (${tag})`);
|
|
2626
|
+
out.compare = {
|
|
2627
|
+
...out.compare,
|
|
2628
|
+
variants: []
|
|
2629
|
+
};
|
|
2630
|
+
comparedLeft = false;
|
|
2631
|
+
}
|
|
2517
2632
|
}
|
|
2518
2633
|
}
|
|
2519
|
-
if (out.compare && limit.compare !== void 0 && !pinned("compare")) {
|
|
2634
|
+
if (out.compare && comparedLeft && limit.compare !== void 0 && !pinned("compare")) {
|
|
2520
2635
|
if (limit.compare === false) {
|
|
2521
2636
|
notes.push(`compare off (${tag})`);
|
|
2522
2637
|
out.compare = {
|
|
@@ -2531,7 +2646,16 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2531
2646
|
};
|
|
2532
2647
|
}
|
|
2533
2648
|
}
|
|
2534
|
-
|
|
2649
|
+
defaultRuns(out, notes, options);
|
|
2650
|
+
if (limit.runs !== void 0 && options.runs === void 0 && !pinned("runs") && (out.runs ?? 1) > limit.runs) {
|
|
2651
|
+
notes.push(`runs ${out.runs} → ${limit.runs} (${tag})`);
|
|
2652
|
+
out.runs = limit.runs;
|
|
2653
|
+
out.sources = {
|
|
2654
|
+
...out.sources,
|
|
2655
|
+
runs: `profiles.${profile.name}.limit`
|
|
2656
|
+
};
|
|
2657
|
+
}
|
|
2658
|
+
const pins = out.pin.filter((k) => k === "compare" ? limit.compare !== void 0 && out.compare && comparedLeft : limit[k] !== void 0);
|
|
2535
2659
|
if (pins.length) notes.push(`pinned: ${pins.join(", ")}`);
|
|
2536
2660
|
}
|
|
2537
2661
|
if (profile) {
|
|
@@ -2554,13 +2678,49 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2554
2678
|
if (narrowed) notes.push(`${plural(narrowed)} narrowed (profile ${profile.name})`);
|
|
2555
2679
|
out.questions = kept;
|
|
2556
2680
|
}
|
|
2557
|
-
|
|
2681
|
+
defaultRuns(out, notes, options);
|
|
2558
2682
|
return {
|
|
2559
|
-
resolved:
|
|
2683
|
+
resolved: {
|
|
2684
|
+
...out,
|
|
2685
|
+
runs: options.runs ?? out.runs ?? 1
|
|
2686
|
+
},
|
|
2560
2687
|
notes,
|
|
2688
|
+
harnesses,
|
|
2561
2689
|
...skip ? { skip } : {}
|
|
2562
2690
|
};
|
|
2563
2691
|
}
|
|
2692
|
+
/** Gives `out` its default runs when no level set them and `--runs` doesn't replace them. */
|
|
2693
|
+
function defaultRuns(out, notes, options) {
|
|
2694
|
+
if (out.runs !== void 0 || options.runs !== void 0) return;
|
|
2695
|
+
if (!out.compare?.variants.length || options.pooled) {
|
|
2696
|
+
out.runs = 1;
|
|
2697
|
+
return;
|
|
2698
|
+
}
|
|
2699
|
+
out.runs = COMPARE_DEFAULT_RUNS;
|
|
2700
|
+
out.sources = {
|
|
2701
|
+
...out.sources,
|
|
2702
|
+
runs: `compare default: ${COMPARE_DEFAULT_RUNS} a side`
|
|
2703
|
+
};
|
|
2704
|
+
notes.push(`runs ${COMPARE_DEFAULT_RUNS} (compare default)`);
|
|
2705
|
+
}
|
|
2706
|
+
/**
|
|
2707
|
+
* Refuses a profile's `limit.harness` id that no collected eval runs on: it keeps nothing, and a
|
|
2708
|
+
* limit that keeps nothing skips every eval. `harnesses` are the ids every eval in the loaded
|
|
2709
|
+
* files runs on before the limit, whatever the tag and name filters pick. When a path filter
|
|
2710
|
+
* loaded only some files (`partial`), an id may name a harness of the others, so only a limit
|
|
2711
|
+
* none of whose ids match is refused.
|
|
2712
|
+
*/
|
|
2713
|
+
function checkLimitHarness(profile, configFile, collected) {
|
|
2714
|
+
const ids = profile?.limit.harness;
|
|
2715
|
+
if (!profile || !ids?.length) return;
|
|
2716
|
+
const unknown = ids.filter((id) => !collected.harnesses.includes(id));
|
|
2717
|
+
if (!unknown.length || collected.partial && unknown.length < ids.length) return;
|
|
2718
|
+
const listed = unknown.map((id) => JSON.stringify(id)).join(", ");
|
|
2719
|
+
const have = [...collected.harnesses].sort();
|
|
2720
|
+
const evals = collected.partial ? "eval in the selected files" : "eval";
|
|
2721
|
+
const the = collected.partial ? "the evals in the selected files" : "the evals";
|
|
2722
|
+
throw new UsageError(`profiles.${profile.name}.limit.harness in ${configFile} lists ${listed}, which no ${evals} runs on. Harness ids ${the} run on: ${have.join(", ") || "none"}.`);
|
|
2723
|
+
}
|
|
2564
2724
|
/**
|
|
2565
2725
|
* The sandbox an attempt runs in: the resolved environment's (including an `.extend()`
|
|
2566
2726
|
* parent's), else the first variant's when this is an omitted-baseline empty control, else
|
|
@@ -2594,8 +2754,11 @@ function sandboxSetBy(planned, resolved, configFile) {
|
|
|
2594
2754
|
}
|
|
2595
2755
|
return resolved.sources?.environment ?? configFile;
|
|
2596
2756
|
}
|
|
2597
|
-
/**
|
|
2598
|
-
|
|
2757
|
+
/**
|
|
2758
|
+
* Per harness: the control, then each variant it compares on; then run numbers. `vertex` is the
|
|
2759
|
+
* run's `vertex()` settings, for each harness that resolved to Vertex AI.
|
|
2760
|
+
*/
|
|
2761
|
+
function expand(resolved, vertex) {
|
|
2599
2762
|
const { compare } = resolved;
|
|
2600
2763
|
const control = applied(compare ? compare.baseline : resolved.environment);
|
|
2601
2764
|
const policy = resolved.gate ?? "all";
|
|
@@ -2603,6 +2766,7 @@ function expand(resolved) {
|
|
|
2603
2766
|
const attempts = [];
|
|
2604
2767
|
for (const harness of resolved.harness) {
|
|
2605
2768
|
const hid = harnessId(harness);
|
|
2769
|
+
const onVertex = harness.gateway === "vertex" && harness.kind !== "fake" ? vertex : void 0;
|
|
2606
2770
|
const cells = [(() => {
|
|
2607
2771
|
const sandbox = attemptSandbox(resolved, control);
|
|
2608
2772
|
const sid = sandboxId(sandbox);
|
|
@@ -2646,11 +2810,12 @@ function expand(resolved) {
|
|
|
2646
2810
|
harness,
|
|
2647
2811
|
sandbox,
|
|
2648
2812
|
environment: env,
|
|
2649
|
-
egress: plannedEgress(resolved.egress, harness, resolved.env, env.layers),
|
|
2813
|
+
egress: plannedEgress(resolved.egress, harness, resolved.env, env.layers, onVertex),
|
|
2650
2814
|
gate,
|
|
2651
2815
|
...cell.role === "variant" && compare?.minLift !== void 0 ? { minLift: compare.minLift } : {},
|
|
2652
2816
|
n,
|
|
2653
|
-
label
|
|
2817
|
+
label,
|
|
2818
|
+
...onVertex ? { vertex: onVertex } : {}
|
|
2654
2819
|
});
|
|
2655
2820
|
}
|
|
2656
2821
|
}
|
|
@@ -2698,9 +2863,18 @@ function unbrokeredOnFake(env, attempts) {
|
|
|
2698
2863
|
}
|
|
2699
2864
|
/** The host variables an attempt's secrets read, brokered or not. */
|
|
2700
2865
|
const secretVarsIn = (env, attempt) => [...new Set(plannedSecrets(env, attempt.environment.layers).map((s) => s.from))];
|
|
2701
|
-
const brokerHint = (s) => /^mcp\.[^.]+\.headers\./.test(s.label) ? "give the MCP server an https:// URL (reeval brokers its header secrets to its own host
|
|
2702
|
-
const unbrokeredWarning = (s) => s.checksOnly ? `${s.label} reaches the checks (c.exec, setup steps) as its real value
|
|
2866
|
+
const brokerHint = (s) => /^mcp\.[^.]+\.headers\./.test(s.label) ? "give the MCP server an https:// URL (reeval brokers its header secrets over HTTPS only, to its own host unless they set hosts)" : `add the hosts it's sent to, e.g. secret.fromEnv('${s.from}', { hosts: ['api.example.com'] })`;
|
|
2867
|
+
const unbrokeredWarning = (s) => s.checksOnly ? `${s.label} reaches the checks (c.exec, setup steps) as its real value. It's never in the agent's own environment, but a process a setup step leaves running holds it while the agent works. To give the checks only a placeholder, ${brokerHint(s)}.` : `${s.label} is in the sandbox as its real value. To give the agent only a placeholder, ${brokerHint(s)}.`;
|
|
2703
2868
|
const unbrokeredError = (s, where, configFile) => `${s.label} in ${where} isn't brokered, and secrets.unbrokered is 'error' in ${configFile}. To broker it, ${brokerHint(s)}.`;
|
|
2869
|
+
/** The attempts' MCP headers written as plain strings that look like credentials, once each. */
|
|
2870
|
+
function plainHeaders(attempts) {
|
|
2871
|
+
const out = /* @__PURE__ */ new Map();
|
|
2872
|
+
for (const a of attempts) for (const h of plainCredentialHeaders(a.environment.layers)) out.set(h.label, h);
|
|
2873
|
+
return [...out.values()];
|
|
2874
|
+
}
|
|
2875
|
+
const templateHint = (h) => `read it from a host variable with secret.template, e.g. secret.template('Bearer \${VAR}')${h.https ? "" : " and give the MCP server an https:// URL"}`;
|
|
2876
|
+
const plainHeaderWarning = (h) => `${h.label} is a plain string, so it's in the sandbox as its real value. To give the agent only a placeholder, ${templateHint(h)}.`;
|
|
2877
|
+
const plainHeaderError = (h, where, configFile) => `${h.label} in ${where} is a plain string, and secrets.unbrokered is 'error' in ${configFile}. To broker it, ${templateHint(h)}.`;
|
|
2704
2878
|
const levelOf = (where) => where.startsWith("evaluate(") ? "eval" : where.startsWith("group(") ? "group" : where.startsWith("profiles.") ? "profile" : "config";
|
|
2705
2879
|
/**
|
|
2706
2880
|
* A warning per privilege an attempt's Docker sandbox adds, naming the level that set the
|
|
@@ -2774,7 +2948,8 @@ function evalPlan(r, attempts, origin) {
|
|
|
2774
2948
|
warnings: [
|
|
2775
2949
|
...environmentWarnings(attempts),
|
|
2776
2950
|
...skip ? [] : questionWarnings(resolved),
|
|
2777
|
-
...unbrokered(r.resolved.env, attempts).filter(reachesSandbox).map(unbrokeredWarning)
|
|
2951
|
+
...unbrokered(r.resolved.env, attempts).filter(reachesSandbox).map(unbrokeredWarning),
|
|
2952
|
+
...plainHeaders(attempts).map(plainHeaderWarning)
|
|
2778
2953
|
],
|
|
2779
2954
|
sandboxWarnings: skip ? [] : sandboxWarnings(attempts, resolved, origin),
|
|
2780
2955
|
brokered: [...new Set(attempts.flatMap((a) => brokeredIn(r.resolved.env, a)))],
|
|
@@ -2806,23 +2981,47 @@ function matchesTags(tags, filters) {
|
|
|
2806
2981
|
}
|
|
2807
2982
|
//#endregion
|
|
2808
2983
|
//#region src/gateway/resolve.ts
|
|
2809
|
-
/** The config's list when it sets none: each provider's own key
|
|
2984
|
+
/** The config's list when it sets none: each provider's own key. */
|
|
2810
2985
|
const DEFAULT_GATEWAYS = [{ kind: "direct" }];
|
|
2811
2986
|
/** Every gateway reeval can route through, by name. */
|
|
2812
2987
|
const GATEWAYS = {
|
|
2813
2988
|
direct: { cantServe: (h) => {
|
|
2814
|
-
|
|
2989
|
+
const fxEffort = fxEffortRefusal(h, "direct()");
|
|
2990
|
+
if (fxEffort) return fxEffort;
|
|
2991
|
+
if (h.kind === "fx" && h.model.includes("/")) return `fx runs a gateway id such as ${JSON.stringify(h.model)} only through Vercel AI Gateway or OpenRouter`;
|
|
2815
2992
|
if (h.kind === "opencode") {
|
|
2816
2993
|
const resolved = resolveOpenCodeProvider(h.model, process.env);
|
|
2817
2994
|
if ("error" in resolved) return resolved.error;
|
|
2818
2995
|
}
|
|
2819
2996
|
} },
|
|
2820
|
-
vercel:
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
if (
|
|
2997
|
+
vercel: routingEntry("vercel"),
|
|
2998
|
+
openrouter: routingEntry("openrouter"),
|
|
2999
|
+
vertex: { cantServe: (h) => {
|
|
3000
|
+
if (h.kind === "fake") return void 0;
|
|
3001
|
+
if (!VERTEX_ROUTES[h.kind]) return VERTEX_REFUSALS[h.kind] ?? `reeval doesn't run ${agentName(h)} on Vertex AI yet`;
|
|
3002
|
+
return vertexModelRefusalFor(h);
|
|
2824
3003
|
} }
|
|
2825
3004
|
};
|
|
3005
|
+
/**
|
|
3006
|
+
* Why fx can't take `effort` on a gateway: fx sends none itself, and reeval adds it as
|
|
3007
|
+
* `reasoning_effort` only to its calls on Vertex AI's Chat Completions endpoint.
|
|
3008
|
+
*/
|
|
3009
|
+
function fxEffortRefusal(h, gateway) {
|
|
3010
|
+
const effort = harnessEffort(h);
|
|
3011
|
+
if (h.kind !== "fx" || effort === void 0) return void 0;
|
|
3012
|
+
return `fx sends no effort of its own, and reeval adds fx's effort (${JSON.stringify(effort)}) only to its calls on Vertex AI, not on ${gateway}: drop effort or run fx on vertex()`;
|
|
3013
|
+
}
|
|
3014
|
+
/** A routing gateway's entry: its route table decides whom it serves, and with what effort. */
|
|
3015
|
+
function routingEntry(name) {
|
|
3016
|
+
const gateway = ROUTING_GATEWAYS[name];
|
|
3017
|
+
return { cantServe: (h) => {
|
|
3018
|
+
const fxEffort = fxEffortRefusal(h, gateway.name);
|
|
3019
|
+
if (fxEffort) return fxEffort;
|
|
3020
|
+
if (h.kind !== "fake" && !gateway.routes[h.kind]) return gateway.refusals[h.kind] ?? `reeval doesn't run ${agentName(h)} on ${gateway.name} yet`;
|
|
3021
|
+
const effort = harnessEffort(h);
|
|
3022
|
+
if (effort !== void 0 && h.kind !== "fake" && gateway.unprovenEffort[h.kind]?.(h.model)) return `reeval doesn't send ${agentName(h)}'s effort (${JSON.stringify(effort)}) to ${h.model} through ${gateway.name} yet: nothing checks the model gets it as its own setting, so drop effort or run ${agentName(h)} on another gateway`;
|
|
3023
|
+
} };
|
|
3024
|
+
}
|
|
2826
3025
|
/** The `reeval/gateway` helper that makes the gateway `name`, as messages name it. */
|
|
2827
3026
|
const gatewayHelper = (name) => `${name}()`;
|
|
2828
3027
|
/** The agent's name, for messages. */
|
|
@@ -2837,7 +3036,7 @@ function resolveGateway(gateways, harness, where, catalog = GATEWAYS) {
|
|
|
2837
3036
|
const why = (name) => catalog[name]?.cantServe(harness);
|
|
2838
3037
|
const pin = harness.gateway;
|
|
2839
3038
|
if (pin !== void 0) {
|
|
2840
|
-
if (!gateways.some((g) => g.kind === pin)) throw new UsageError(`${on}: it pins gateway '${pin}',
|
|
3039
|
+
if (!gateways.some((g) => g.kind === pin)) throw new UsageError(`${on}: it pins gateway '${pin}', ${notListed(pin, listed)}`);
|
|
2841
3040
|
const refused = why(pin);
|
|
2842
3041
|
if (refused) throw new UsageError(`${on}: it pins gateway '${pin}', which can't serve it (${refused}). A pin never falls back; pin another gateway, or remove the pin to take the first in the list that can.`);
|
|
2843
3042
|
return {
|
|
@@ -2859,10 +3058,13 @@ function resolveGateway(gateways, harness, where, catalog = GATEWAYS) {
|
|
|
2859
3058
|
}
|
|
2860
3059
|
throw new UsageError(`${on}: no gateway in the list can serve it (${skipped.map((s) => `${s.gateway}: ${s.why}`).join("; ")}). ${addHint(gateways, harness, catalog)}`);
|
|
2861
3060
|
}
|
|
2862
|
-
/**
|
|
3061
|
+
/**
|
|
3062
|
+
* What to change so the list serves `harness`: a list without `direct` has it turned off, so
|
|
3063
|
+
* turning it back on when it can serve, else adding the first gateway that can.
|
|
3064
|
+
*/
|
|
2863
3065
|
function addHint(gateways, harness, catalog) {
|
|
2864
3066
|
const serves = Object.keys(catalog).filter((name) => !gateways.some((g) => g.kind === name)).filter((name) => !catalog[name]?.cantServe(harness));
|
|
2865
|
-
if (serves.includes("direct")) return `
|
|
3067
|
+
if (serves.includes("direct")) return `Drop direct({ enabled: false }) from gateway to fall back to ${agentName(harness)}'s own key.`;
|
|
2866
3068
|
const [first] = serves;
|
|
2867
3069
|
return first ? `Add ${gatewayHelper(first)} to gateway.` : `No gateway reeval knows can serve it.`;
|
|
2868
3070
|
}
|
|
@@ -2881,30 +3083,40 @@ function routedHarness(harness, gateways) {
|
|
|
2881
3083
|
return harness;
|
|
2882
3084
|
}
|
|
2883
3085
|
}
|
|
2884
|
-
/**
|
|
2885
|
-
|
|
3086
|
+
/**
|
|
3087
|
+
* Why a pin names a gateway the list doesn't have: a list without `direct` has it turned off.
|
|
3088
|
+
*/
|
|
3089
|
+
const notListed = (pin, listed) => pin === "direct" ? `which the gateway list (${listed}) turns off with direct({ enabled: false }). Drop that, or pin one that's there.` : `which isn't in the gateway list (${listed}). Add ${gatewayHelper(pin)} to gateway, or pin one that's there.`;
|
|
3090
|
+
/**
|
|
3091
|
+
* `gateway: cursor:… falls back to direct (vercel can't serve it: …)`, or `→ vercel (…)` past
|
|
3092
|
+
* another gateway, for the plan's hints.
|
|
3093
|
+
*/
|
|
3094
|
+
const fallbackHint = (harness, resolved) => `gateway: ${harness} ${resolved.name === "direct" ? "falls back to" : "→"} ${resolved.name} (${resolved.skipped.map((s) => `${s.gateway} can't serve it: ${s.why}`).join("; ")})`;
|
|
2886
3095
|
//#endregion
|
|
2887
3096
|
//#region src/gateway/models.ts
|
|
2888
3097
|
const providerOf = (model) => model.slice(0, model.indexOf("/"));
|
|
2889
3098
|
/** Why each gateway can't serve a model by name, or undefined when it can. */
|
|
2890
3099
|
const MODEL_GATEWAYS = {
|
|
2891
3100
|
direct: (model) => DIRECT_PROVIDERS[providerOf(model)] ? void 0 : `reeval calls only ${Object.keys(DIRECT_PROVIDERS).map((p) => `${p}/*`).join(", ")} models on their own keys`,
|
|
2892
|
-
vercel: () => void 0
|
|
3101
|
+
vercel: () => void 0,
|
|
3102
|
+
openrouter: () => void 0,
|
|
3103
|
+
vertex: vertexModelRefusal
|
|
2893
3104
|
};
|
|
2894
3105
|
const gatewayScope = (config, profile) => ({ gateways: profile?.gateway ?? config.gateway });
|
|
2895
3106
|
/**
|
|
2896
|
-
* The gateway `model` goes through: its pin, or the first in the list that can serve it.
|
|
2897
|
-
*
|
|
2898
|
-
*
|
|
3107
|
+
* The gateway `model` goes through: its pin, or the first in the list that can serve it. A list
|
|
3108
|
+
* ends with `direct()` unless it's turned off, so a model no other gateway serves takes its
|
|
3109
|
+
* provider's own key. A model object or a fake calls through itself, and is undefined. Throws a UsageError
|
|
2899
3110
|
* naming each gateway tried.
|
|
2900
3111
|
*/
|
|
2901
3112
|
function resolveModelGateway(model, pin, scope, what) {
|
|
2902
3113
|
if (typeof model !== "string" || !model.includes("/")) return void 0;
|
|
2903
3114
|
const on = `${what} "${model}"`;
|
|
2904
3115
|
const listed = scope.gateways.map((g) => g.kind);
|
|
3116
|
+
if (typeof pin === "object" && pin.kind === "direct" && pin.enabled === false) throw new UsageError(directOffPin(on));
|
|
2905
3117
|
const name = typeof pin === "object" ? pin.kind : pin;
|
|
2906
3118
|
if (name !== void 0) {
|
|
2907
|
-
if (!listed.includes(name)) throw new UsageError(`${on} pins gateway '${name}',
|
|
3119
|
+
if (!listed.includes(name)) throw new UsageError(`${on} pins gateway '${name}', ${notListed(name, listed.join(", "))}`);
|
|
2908
3120
|
const refused = MODEL_GATEWAYS[name](model);
|
|
2909
3121
|
if (refused) throw new UsageError(`${on} pins gateway '${name}', which can't serve it (${refused}). A pin never falls back; pin another gateway, or remove the pin to take the first in the list that can.`);
|
|
2910
3122
|
return name;
|
|
@@ -2916,9 +3128,11 @@ function resolveModelGateway(model, pin, scope, what) {
|
|
|
2916
3128
|
skipped.push(`${kind}: ${refused}`);
|
|
2917
3129
|
}
|
|
2918
3130
|
const other = GATEWAY_NAMES.find((n) => !listed.includes(n) && !MODEL_GATEWAYS[n](model));
|
|
2919
|
-
const judge = other
|
|
2920
|
-
throw new UsageError(`${on}: no gateway in the list can serve it (${skipped.join("; ")}).${other ? ` Add ${gatewayHelper(other)} to gateway.${judge}` : ""}`);
|
|
3131
|
+
const judge = other && other !== "direct" && what.includes("judge") ? ` There a judge must be a decision model, such as '${other === "openrouter" ? OPENROUTER_JUDGE_MODEL : RECOMMENDED_JUDGE_MODEL}'.` : "";
|
|
3132
|
+
throw new UsageError(`${on}: no gateway in the list can serve it (${skipped.join("; ")}).${other === "direct" ? ` Drop direct({ enabled: false }) from gateway to fall back to its provider's own key.` : other ? ` Add ${gatewayHelper(other)} to gateway.${judge}` : ""}`);
|
|
2921
3133
|
}
|
|
3134
|
+
/** Whether an unpinned model resolved to `direct` past the list's other entries, for the plan and doctor. */
|
|
3135
|
+
const fellBackToDirect = (gateway, pin, scope) => gateway === "direct" && pin === void 0 && scope.gateways[0]?.kind !== "direct";
|
|
2922
3136
|
/** As `resolveModelGateway`, but undefined where it would throw, for checks the plan already makes. */
|
|
2923
3137
|
function modelGatewayOrNone(model, pin, scope) {
|
|
2924
3138
|
try {
|
|
@@ -3133,7 +3347,7 @@ function configFromData(data) {
|
|
|
3133
3347
|
* Resolves a config in its data form (a parsed `reeval.yaml`, or `ReevalConfigData` from another
|
|
3134
3348
|
* language) whose relative host paths resolve against `root`, its directory: `resolveConfig`'s
|
|
3135
3349
|
* checks name a mistake in their own words first, and the config contract refuses anything they
|
|
3136
|
-
* let through, such as a `null` or a
|
|
3350
|
+
* let through, such as a `null` or a misspelled nested key.
|
|
3137
3351
|
*/
|
|
3138
3352
|
function resolveConfigData(input, root, file) {
|
|
3139
3353
|
if (!isRecord(input)) throw new UsageError(`${file} must be a mapping of config keys, like harness: and judge: (got ${JSON.stringify(input)})`);
|
|
@@ -3237,12 +3451,13 @@ function checkProfile(name, profile, file) {
|
|
|
3237
3451
|
};
|
|
3238
3452
|
return p;
|
|
3239
3453
|
}
|
|
3240
|
-
const GATEWAY_EXAMPLE = "gateway: [
|
|
3454
|
+
const GATEWAY_EXAMPLE = "gateway: [vercel()]";
|
|
3241
3455
|
const isGatewayName = (v) => typeof v === "string" && GATEWAY_NAMES.includes(v);
|
|
3242
3456
|
/**
|
|
3243
3457
|
* A `gateway` list as descriptors: each entry is a helper's descriptor, or as data its name or a
|
|
3244
|
-
* one-key map from its name to its settings.
|
|
3245
|
-
*
|
|
3458
|
+
* one-key map from its name to its settings. `direct()` ends the list unless it's listed in
|
|
3459
|
+
* another place, and `direct({ enabled: false })` leaves it out, so the resolved list is the one
|
|
3460
|
+
* every route walks. Refuses an empty list, an unknown entry and a gateway listed twice.
|
|
3246
3461
|
*/
|
|
3247
3462
|
function checkGateways(value, where, file) {
|
|
3248
3463
|
if (!Array.isArray(value) || value.length === 0) throw new UsageError(`${where} in ${file} must be a list of gateways in the order you prefer them, e.g. ${GATEWAY_EXAMPLE} (got ${JSON.stringify(value)})`);
|
|
@@ -3273,10 +3488,20 @@ function checkGateways(value, where, file) {
|
|
|
3273
3488
|
const kind = isRecord(descriptor) ? descriptor.kind : void 0;
|
|
3274
3489
|
throw new UsageError(isGatewayName(kind) ? `${at} isn't a valid ${gatewayHelper(kind)}: ${parsed.error.issues[0]?.message ?? "unknown settings"} (got ${JSON.stringify(entry)})` : `${at} must be a gateway, e.g. direct() from 'reeval/gateway' (got ${describeValue(entry)})`);
|
|
3275
3490
|
}
|
|
3276
|
-
|
|
3491
|
+
const twice = out.find((g) => g.kind === parsed.data.kind);
|
|
3492
|
+
if (twice?.kind === "direct" && parsed.data.kind === "direct") {
|
|
3493
|
+
if (twice.enabled === false !== (parsed.data.enabled === false)) throw new UsageError(`${at} lists direct() and direct({ enabled: false }). direct() places the fallback to each provider's own key and direct({ enabled: false }) turns it off: keep one.`);
|
|
3494
|
+
}
|
|
3495
|
+
if (twice) throw new UsageError(`${at} lists ${parsed.data.kind} a second time. Each gateway is listed once, where you prefer it.`);
|
|
3277
3496
|
out.push(parsed.data);
|
|
3278
3497
|
});
|
|
3279
|
-
|
|
3498
|
+
const direct = out.find((g) => g.kind === "direct");
|
|
3499
|
+
if (direct?.enabled === false) {
|
|
3500
|
+
const rest = out.filter((g) => g.kind !== "direct");
|
|
3501
|
+
if (!rest.length) throw new UsageError(`${where} in ${file} turns direct() off and lists no other gateway, so no model call has one. Add a gateway such as vercel(), or drop direct({ enabled: false }).`);
|
|
3502
|
+
return rest;
|
|
3503
|
+
}
|
|
3504
|
+
return direct ? out.map((g) => g.kind === "direct" ? { kind: "direct" } : g) : [...out, { kind: "direct" }];
|
|
3280
3505
|
}
|
|
3281
3506
|
/** `--profile`, then `REEVAL_PROFILE`, then `defaultProfile.ci` in CI (or `.local`), then none. */
|
|
3282
3507
|
function selectProfile(config, flag, env = process.env) {
|
|
@@ -3352,7 +3577,7 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
|
3352
3577
|
harness,
|
|
3353
3578
|
sandbox: user.sandbox ?? docker(),
|
|
3354
3579
|
environment: user.environment,
|
|
3355
|
-
runs: user.runs
|
|
3580
|
+
runs: user.runs,
|
|
3356
3581
|
gate: user.gate,
|
|
3357
3582
|
env: user.env ?? {},
|
|
3358
3583
|
...user.egress !== void 0 ? { egress: user.egress } : {},
|
|
@@ -3378,35 +3603,27 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
|
3378
3603
|
}
|
|
3379
3604
|
const ANALYSIS_DEFAULTS = {
|
|
3380
3605
|
maxInputTokens: 2e4,
|
|
3381
|
-
maxOutputTokens: 8e3
|
|
3382
|
-
insights: true,
|
|
3383
|
-
maxTokens: 25e4
|
|
3606
|
+
maxOutputTokens: 8e3
|
|
3384
3607
|
};
|
|
3385
3608
|
function checkAnalysis(analysis, file) {
|
|
3386
3609
|
if (analysis === void 0) return { ...ANALYSIS_DEFAULTS };
|
|
3387
3610
|
if (typeof analysis !== "object" || analysis === null) throw new UsageError(`analysis in ${file} must be an object like { model: 'google/gemini-3.8-flash' } (got ${JSON.stringify(analysis)})`);
|
|
3388
3611
|
unknownKey(analysis, [
|
|
3389
3612
|
"model",
|
|
3390
|
-
"insights",
|
|
3391
|
-
"maxTokens",
|
|
3392
3613
|
"instructions",
|
|
3393
3614
|
"maxInputTokens",
|
|
3394
3615
|
"maxOutputTokens"
|
|
3395
3616
|
], "analysis", file);
|
|
3396
|
-
const { model,
|
|
3617
|
+
const { model, instructions, maxInputTokens = ANALYSIS_DEFAULTS.maxInputTokens, maxOutputTokens = ANALYSIS_DEFAULTS.maxOutputTokens } = analysis;
|
|
3397
3618
|
if (typeof model === "string") {
|
|
3398
3619
|
const problem = checkModelString(model, `analysis.model in ${file}`, [`google/${model}`]);
|
|
3399
3620
|
if (problem) throw new UsageError(problem);
|
|
3400
3621
|
} else if (model !== void 0 && !(isObjectLike(model) && model.kind === "fake-analysis")) throw new UsageError(`analysis.model in ${file} must be a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${JSON.stringify(model)})`);
|
|
3401
|
-
if (typeof insights !== "boolean") throw new UsageError(`analysis.insights in ${file} must be true or false (got ${JSON.stringify(insights)})`);
|
|
3402
|
-
if (!(Number.isInteger(maxTokens) && maxTokens >= 1)) throw new UsageError(`analysis.maxTokens in ${file} must be a whole number of tokens of at least 1, e.g. 250000 (got ${JSON.stringify(maxTokens)})`);
|
|
3403
3622
|
if (instructions !== void 0 && typeof instructions !== "string") throw new UsageError(`analysis.instructions in ${file} must be a string (got ${JSON.stringify(instructions)})`);
|
|
3404
3623
|
if (!(Number.isInteger(maxInputTokens) && maxInputTokens >= 1e3)) throw new UsageError(`analysis.maxInputTokens in ${file} must be a whole number of at least 1000 (got ${maxInputTokens})`);
|
|
3405
3624
|
if (!(Number.isInteger(maxOutputTokens) && maxOutputTokens >= 500)) throw new UsageError(`analysis.maxOutputTokens in ${file} must be a whole number of at least 500 (got ${maxOutputTokens})`);
|
|
3406
3625
|
return {
|
|
3407
3626
|
...analysis,
|
|
3408
|
-
insights,
|
|
3409
|
-
maxTokens,
|
|
3410
3627
|
maxInputTokens,
|
|
3411
3628
|
maxOutputTokens
|
|
3412
3629
|
};
|
|
@@ -3482,8 +3699,6 @@ function secretSources(env) {
|
|
|
3482
3699
|
}
|
|
3483
3700
|
/** The model that answers what each eval asks: `analysis.model` only, never the judge's. */
|
|
3484
3701
|
const analysisModelOf = (config) => config.analysis.model;
|
|
3485
|
-
/** Whether `analysis.model` is set, so a model can answer what each eval asks. */
|
|
3486
|
-
const hasAnalysisModel = (config) => analysisModelOf(config) !== void 0;
|
|
3487
3702
|
/** The run's judge model: the profile's when it sets one, else the config's. */
|
|
3488
3703
|
function runJudgeModel(config, profile) {
|
|
3489
3704
|
return profile ? profileJudge(profile).model ?? config.judge.model : config.judge.model;
|
|
@@ -3494,30 +3709,43 @@ function runModelGateway(config, profile, model, judge = false) {
|
|
|
3494
3709
|
}
|
|
3495
3710
|
/**
|
|
3496
3711
|
* Names the environment variables the run's providers need but don't have: the harnesses it
|
|
3497
|
-
* plans (default: the config's), the run's judge
|
|
3498
|
-
* and required config and profile secrets.
|
|
3499
|
-
* prints no insights, and never stops for it.
|
|
3712
|
+
* plans (default: the config's), the run's judge and the judges its attempts resolve, the models
|
|
3713
|
+
* that answer questions, and required config and profile secrets.
|
|
3500
3714
|
*/
|
|
3501
3715
|
function missingCredentials(config, env = process.env, options = {}) {
|
|
3502
3716
|
const missing = [];
|
|
3503
3717
|
const keys = new Set((options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []));
|
|
3504
3718
|
for (const key of keys) if (!env[key]) missing.push(key);
|
|
3505
3719
|
const judge = runJudgeModel(config, options.profile);
|
|
3506
|
-
const models = [
|
|
3507
|
-
|
|
3508
|
-
|
|
3509
|
-
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
|
|
3513
|
-
|
|
3514
|
-
|
|
3720
|
+
const models = [
|
|
3721
|
+
...typeof judge === "string" ? [{
|
|
3722
|
+
model: judge,
|
|
3723
|
+
gateway: runModelGateway(config, options.profile, judge, true),
|
|
3724
|
+
what: "the judge model"
|
|
3725
|
+
}] : [],
|
|
3726
|
+
...(options.judges ?? []).map((j) => ({
|
|
3727
|
+
...j,
|
|
3728
|
+
what: "the judge model"
|
|
3729
|
+
})),
|
|
3730
|
+
...(options.questionModels ?? []).map((model) => ({
|
|
3731
|
+
model,
|
|
3732
|
+
gateway: runModelGateway(config, options.profile, model),
|
|
3733
|
+
what: "questions"
|
|
3734
|
+
}))
|
|
3735
|
+
];
|
|
3515
3736
|
const named = /* @__PURE__ */ new Set();
|
|
3516
|
-
for (const { model,
|
|
3517
|
-
if (
|
|
3518
|
-
|
|
3519
|
-
|
|
3520
|
-
|
|
3737
|
+
for (const { model, gateway, what } of models) {
|
|
3738
|
+
if (gateway === "vertex") {
|
|
3739
|
+
const gateways = gatewayScope(config, options.profile).gateways;
|
|
3740
|
+
const line = `${VERTEX_PROJECT_ENV} (for vertex()) for ${what}, "${model}"`;
|
|
3741
|
+
if (!gateways.some((g) => g.kind === "vertex" && g.project) && !env["GOOGLE_CLOUD_PROJECT"] && !missing.includes(line)) missing.push(line);
|
|
3742
|
+
continue;
|
|
3743
|
+
}
|
|
3744
|
+
const keys = judgeCredentials(model, gateway);
|
|
3745
|
+
const once = `${keys.join(" ")}\0${model}`;
|
|
3746
|
+
if (named.has(once)) continue;
|
|
3747
|
+
named.add(once);
|
|
3748
|
+
if (!keys.some((k) => env[k])) missing.push(`${modelKeyOptions(model, gateway)} for ${what}, "${model}"`);
|
|
3521
3749
|
}
|
|
3522
3750
|
const vars = {
|
|
3523
3751
|
...config.env,
|
|
@@ -3527,16 +3755,16 @@ function missingCredentials(config, env = process.env, options = {}) {
|
|
|
3527
3755
|
return missing;
|
|
3528
3756
|
}
|
|
3529
3757
|
/**
|
|
3530
|
-
* Keys the run sends in request headers (the agents' credentials, the run's
|
|
3531
|
-
* questions' models') whose value has
|
|
3532
|
-
*
|
|
3533
|
-
*
|
|
3534
|
-
* rather than failing to authenticate.
|
|
3758
|
+
* Keys the run sends in request headers (the agents' credentials, the run's and attempts' judges'
|
|
3759
|
+
* and the questions' models') whose value has whitespace or a control character in it, with where
|
|
3760
|
+
* the first one is: a key pasted from a wrapped terminal line picks up a line break, and the agent
|
|
3761
|
+
* then hangs on a request it can't send rather than failing to authenticate.
|
|
3535
3762
|
*/
|
|
3536
3763
|
function malformedCredentials(config, env = process.env, options = {}) {
|
|
3537
3764
|
const names = (options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []);
|
|
3538
3765
|
const judge = runJudgeModel(config, options.profile);
|
|
3539
3766
|
if (judge !== void 0) names.push(...judgeCredentials(judge, runModelGateway(config, options.profile, judge, true)));
|
|
3767
|
+
for (const { model, gateway } of options.judges ?? []) names.push(...judgeCredentials(model, gateway));
|
|
3540
3768
|
for (const model of options.questionModels ?? []) names.push(...judgeCredentials(model, runModelGateway(config, options.profile, model)));
|
|
3541
3769
|
return [...new Set(names)].flatMap((name) => {
|
|
3542
3770
|
const at = badCharacterAt(env[name] ?? "");
|
|
@@ -3544,6 +3772,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
|
|
|
3544
3772
|
});
|
|
3545
3773
|
}
|
|
3546
3774
|
//#endregion
|
|
3547
|
-
export {
|
|
3775
|
+
export { isDirect as $, harnessId as A, badJudgeArtifacts as At, sandboxSetBy as B, routedHarness as C, captureWindowSchema as Ct, checkOptions as D, judgeProblem as Dt, checkLimitHarness as E, judgeModelNameSchema as Et, plannedSandboxes as F, resolveStateOptions as G, unbrokered as H, profileJudge as I, allowedHosts as J, DEFAULT_EGRESS as K, refuseIllegalCompare as L, matchesTags as M, filePartsOf as Mt, plainHeaderError as N, resolveJudgeArtifacts as Nt, evalPlan as O, judgeStateSchema as Ot, plainHeaders as P, gatewayPolicy as Q, sandboxId as R, resolveGateway as S, writePluginFiles as St, cascade as T, judgeFields as Tt, unbrokeredError as U, secretVarsIn as V, mergeJudge as W, directOnCloud as X, ciRunner as Y, egressLabel as Z, fellBackToDirect as _, expectedBlockedFor as _t, configPathFor as a, questionShows as at, resolveModelGateway as b, refusedMcp as bt, findConfig as c, resolvedEffortOf as ct, malformedCredentials as d, agentFor as dt, unenforceable as et, missingCredentials as f, agentMcp as ft, selectProfile as g, credentialOf as gt, runJudgeModel as h, claudeMcpServers as ht, configLabel as i, STOPPED_FIRST as it, levelOf as j, decideState as jt, expand as k, artifactRefsOf as kt, hostCapacity as l, servedEffortOf as lt, resolveConcurrency as m, brokeringFor as mt, analysisModelOf as n, createFakeSandbox as nt, defaultConcurrency as o, resolveQuestionState as ot, requireConfig as p, authHint as pt, DEFAULT_EGRESS_ALLOW as q, configFilesPhrase as r, DEFAULT_QUESTION_STATE as rt, defineConfig as s, harnessEffort as st, CONFIG_FILES as t, resolveOutput as tt, loadConfig as u, agentAuth as ut, gatewayScope as v, isAgentKind as vt, brokeredIn as w, fakeJudgeDataSchema as wt, fallbackHint as x, refusedPlugins as xt, modelGatewayOrNone as y, layoutOf as yt, sandboxProvider as z };
|
|
3548
3776
|
|
|
3549
|
-
//# sourceMappingURL=config-
|
|
3777
|
+
//# sourceMappingURL=config-k6m7XbAz.js.map
|