reeval 0.14.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/THIRD_PARTY_NOTICES.md +69 -69
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-BloGVZ_a.js → adapter-CRKYPkRt.js} +37 -8
- package/dist/adapter-CRKYPkRt.js.map +1 -0
- package/dist/{agent-m-_t-IuW.js → agent-BgH4-kFc.js} +58 -16
- package/dist/agent-BgH4-kFc.js.map +1 -0
- package/dist/{app-files-C-jwhRDt.js → app-files-BMVBr079.js} +4 -4
- package/dist/app-files-BMVBr079.js.map +1 -0
- package/dist/{assemble-BDEpelpi.js → assemble-BVbI6NW-.js} +214 -128
- package/dist/assemble-BVbI6NW-.js.map +1 -0
- package/dist/assemble-SiHa4__F.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-CpbZ9arV.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-CpbZ9arV.js.map} +1 -1
- package/dist/{match-Bz0nSdzN.js → attempt-host-CIxjs2Be.js} +104 -6
- package/dist/attempt-host-CIxjs2Be.js.map +1 -0
- package/dist/{capture-CVRwRTnb.js → capture-iQ6kYJV0.js} +2360 -1523
- package/dist/capture-iQ6kYJV0.js.map +1 -0
- package/dist/capture-px7Ey95_.js +1 -0
- package/dist/{cli-C_4gaJ50.js → cli-1YQU-66F.js} +153 -180
- package/dist/cli-1YQU-66F.js.map +1 -0
- package/dist/{cli-spec-CpnbATtI.js → cli-spec-CPWT1mAG.js} +10 -21
- package/dist/cli-spec-CPWT1mAG.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-D5hNjpEx.js → client-DMmOkbRA.js} +14 -7
- package/dist/client-DMmOkbRA.js.map +1 -0
- package/dist/{command-BXLUfw4h.js → command-8_Cg5v6b.js} +35 -22
- package/dist/command-8_Cg5v6b.js.map +1 -0
- package/dist/{command-I1euLLio.js → command-BlYCAqu-.js} +7 -7
- package/dist/{command-I1euLLio.js.map → command-BlYCAqu-.js.map} +1 -1
- package/dist/{validate-BhwqrVsm.js → command-C-HeGlap.js} +388 -60
- package/dist/command-C-HeGlap.js.map +1 -0
- package/dist/{command-Bxdy62dW.js → command-Cr1zjRvl.js} +32 -31
- package/dist/command-Cr1zjRvl.js.map +1 -0
- package/dist/{commands-p1S4nw_H.js → commands-Bj9NbmsO.js} +1 -1
- package/dist/{commands-jzigbNFb.js → commands-BvTNZ2-b.js} +181 -73
- package/dist/commands-BvTNZ2-b.js.map +1 -0
- package/dist/{config-BrdHA9bJ.js → config-CneBZB84.js} +883 -433
- package/dist/config-CneBZB84.js.map +1 -0
- package/dist/{contrasts-DRxMQQTC.js → contrasts-lhFsekco.js} +3 -3
- package/dist/{contrasts-DRxMQQTC.js.map → contrasts-lhFsekco.js.map} +1 -1
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-BnXyamuj.js +2 -0
- package/dist/db-Cnh3jIv_.js +1530 -0
- package/dist/db-Cnh3jIv_.js.map +1 -0
- package/dist/debug-CAx3qcaP.js +1 -0
- package/dist/{debug-D24NOQH1.js → debug-CgG3T071.js} +5 -2
- package/dist/debug-CgG3T071.js.map +1 -0
- package/dist/{docker-BnRlTpZA.js → docker-BihzJwbV.js} +11 -6
- package/dist/docker-BihzJwbV.js.map +1 -0
- package/dist/{docker-D01slDF3.js → docker-Cqq03H72.js} +26 -8
- package/dist/docker-Cqq03H72.js.map +1 -0
- package/dist/{docker-DivfZl2u.js → docker-DCAtFMo9.js} +1 -1
- package/dist/{doctor-DQ2p0euG.js → doctor-Dl4vMeqt.js} +275 -37
- package/dist/doctor-Dl4vMeqt.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-CO0P0soF.js → environment-Cf5HMjib.js} +116 -297
- package/dist/environment-Cf5HMjib.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-CHebohjc.js} +10 -4
- package/dist/{errors-r025B0Zb.js.map → errors-CHebohjc.js.map} +1 -1
- package/dist/eval-source-CAE4y2Sr.js +146 -0
- package/dist/eval-source-CAE4y2Sr.js.map +1 -0
- package/dist/{evaluate-BUmsN7yG.js → evaluate-CUtgyr43.js} +682 -150
- package/dist/evaluate-CUtgyr43.js.map +1 -0
- package/dist/evaluate-D35T51AC.d.ts +45 -0
- package/dist/files-BEnUJW1e.js +1 -0
- package/dist/{files-B-CJeeEh.js → files-BRViE7m-.js} +200 -87
- package/dist/files-BRViE7m-.js.map +1 -0
- package/dist/{files-kUCrswI6.js → files-BSqA2sZE.js} +20 -9
- package/dist/files-BSqA2sZE.js.map +1 -0
- package/dist/files-C3tQtD26.js +1 -0
- package/dist/files-CNiJ_fZf.js +37 -0
- package/dist/files-CNiJ_fZf.js.map +1 -0
- package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
- package/dist/files-D1y8Pg4Z.js.map +1 -0
- package/dist/files-Dts7EY3H.js +1 -0
- package/dist/{pairs-CHfZNRT6.js → gate-BC6SdQZs.js} +58 -4
- package/dist/gate-BC6SdQZs.js.map +1 -0
- package/dist/gateway.d.ts +22 -9
- package/dist/gateway.js +30 -8
- package/dist/gateway.js.map +1 -1
- package/dist/{git-Bk8DSwHy.js → git-UNxT_3z0.js} +3 -3
- package/dist/{git-Bk8DSwHy.js.map → git-UNxT_3z0.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +12 -8
- package/dist/harness.js +1 -1
- package/dist/{images-DzLeFjLx.js → images-8A_LzVfw.js} +8 -8
- package/dist/{images-DzLeFjLx.js.map → images-8A_LzVfw.js.map} +1 -1
- package/dist/index.d.ts +99 -11
- package/dist/index.js +5 -5
- package/dist/init-okoALIah.js +1 -0
- package/dist/{init-SRalM5xF.js → init-sFBgODhu.js} +10 -11
- package/dist/init-sFBgODhu.js.map +1 -0
- package/dist/{invocations-DzyKSJBa.js → invocations-C9ejlOYl.js} +10 -6
- package/dist/invocations-C9ejlOYl.js.map +1 -0
- package/dist/{judge-yLvjC5A3.js → judge-Bg4v4e_K.js} +236 -46
- package/dist/judge-Bg4v4e_K.js.map +1 -0
- package/dist/judge.d.ts +5 -7
- package/dist/judge.js +4 -8
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-DeMvnDD5.js} +3 -3
- package/dist/{live-DYd2QEyR.js.map → live-DeMvnDD5.js.map} +1 -1
- package/dist/{matchers-D4eUqx4j.js → matchers-BKPuaXlj.js} +14 -13
- package/dist/matchers-BKPuaXlj.js.map +1 -0
- package/dist/{model-1ZLr9Dcv.js → model-2rBZrdZA.js} +29 -36
- package/dist/model-2rBZrdZA.js.map +1 -0
- package/dist/{model-B45lDUV1.js → model-3MJtpho8.js} +22 -13
- package/dist/model-3MJtpho8.js.map +1 -0
- package/dist/{model-CgnOpss4.js → model-Bz7-j71_.js} +245 -68
- package/dist/model-Bz7-j71_.js.map +1 -0
- package/dist/model-DkPwypN4.js +1 -0
- package/dist/model-bREXVXwT.js +1 -0
- package/dist/{node-BSq5qHnQ.js → node-fP-H2uxn.js} +72 -50
- package/dist/node-fP-H2uxn.js.map +1 -0
- package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
- package/dist/object-BkXGPKWY.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js +34 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js.map +1 -0
- package/dist/{patch-DWf-I_Q_.js → patch-6T9XAipm.js} +7 -9
- package/dist/patch-6T9XAipm.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{plan-BIwUzlB8.js → plan-Dw2BjafJ.js} +79 -35
- package/dist/plan-Dw2BjafJ.js.map +1 -0
- package/dist/{plugins-Dkp_H3pB.js → plugins-ngQEJBjJ.js} +274 -17
- package/dist/plugins-ngQEJBjJ.js.map +1 -0
- package/dist/{preflight-D9y-HvZQ.js → preflight-BYXkweNN.js} +1 -1
- package/dist/{preflight-DJCD_BDn.js → preflight-CDRpog9o.js} +3 -3
- package/dist/{preflight-DJCD_BDn.js.map → preflight-CDRpog9o.js.map} +1 -1
- package/dist/{prepare-BJG_pD54.js → prepare-B3btUlT6.js} +2 -2
- package/dist/{prepare-CIxCGG0M.js → prepare-PkARUGoF.js} +8 -8
- package/dist/{prepare-CIxCGG0M.js.map → prepare-PkARUGoF.js.map} +1 -1
- package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-CfZpFpAh.js → provenance-7Hzg88f4.js} +4 -4
- package/dist/{provenance-CfZpFpAh.js.map → provenance-7Hzg88f4.js.map} +1 -1
- package/dist/{providers-D1Zxp2he.js → providers-DDNMbv1T.js} +1 -1
- package/dist/{providers-DFk_6W4K.js → providers-PKoJtGK0.js} +5 -5
- package/dist/{providers-DFk_6W4K.js.map → providers-PKoJtGK0.js.map} +1 -1
- package/dist/query-Bt6vLJ3m.js +1 -0
- package/dist/{query-CemclveB.js → query-nBXrgEwl.js} +3 -3
- package/dist/{query-CemclveB.js.map → query-nBXrgEwl.js.map} +1 -1
- package/dist/{questions-bAudIc04.js → questions-CkAwePPR.js} +110 -36
- package/dist/questions-CkAwePPR.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-Bp6AwPgb.js +1 -0
- package/dist/{rules-VhftPOOT.js → remarks-Bt0zwiQB.js} +330 -66
- package/dist/remarks-Bt0zwiQB.js.map +1 -0
- package/dist/{header-BLPP6aCU.js → remarks-IFFe_oty.js} +35 -9
- package/dist/remarks-IFFe_oty.js.map +1 -0
- package/dist/{remarks-mujJv1Ey.js → remarks-eBcrIZRu.js} +10 -5
- package/dist/remarks-eBcrIZRu.js.map +1 -0
- package/dist/{remote-CKcEKja4.js → remote-DfJYiP-R.js} +39 -99
- package/dist/remote-DfJYiP-R.js.map +1 -0
- package/dist/resolve-CT4XZIfV.js +1 -0
- package/dist/{resolve-CBoKzQMP.js → resolve-D_cLa4tW.js} +46 -30
- package/dist/resolve-D_cLa4tW.js.map +1 -0
- package/dist/results-DrKJuIQF.js +63 -0
- package/dist/results-DrKJuIQF.js.map +1 -0
- package/dist/{resume-B9SOSnK9.js → resume-BmIM52MJ.js} +47 -17
- package/dist/resume-BmIM52MJ.js.map +1 -0
- package/dist/{runtime-BgQRyCLl.js → runtime-CZVpkr04.js} +139 -48
- package/dist/runtime-CZVpkr04.js.map +1 -0
- package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmi6IQML.js} +2 -2
- package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmi6IQML.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CKJOnIB0.js +1 -0
- package/dist/{server-TGW4WyAz.js → server-DS8BGhGU.js} +75 -33
- package/dist/server-DS8BGhGU.js.map +1 -0
- package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/source-CgUqpumI.js +1 -0
- package/dist/source-DcXv4de-.js +59 -0
- package/dist/source-DcXv4de-.js.map +1 -0
- package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
- package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
- package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
- package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-DjRI-IeE.js → src-A8HLtO0a.js} +135 -35
- package/dist/{src-DjRI-IeE.js.map → src-A8HLtO0a.js.map} +1 -1
- package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/src-BTqJbvj6.js +2 -0
- package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
- package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BkiComGh.js +2 -0
- package/dist/src-BvgeZ4tS.js +2 -0
- package/dist/{src-8zoX_qrb.js → src-C0ySiej5.js} +572 -1067
- package/dist/src-C0ySiej5.js.map +1 -0
- package/dist/src-CgJrz5p0.js +3 -0
- package/dist/src-Cp-2Hdib.js +2 -0
- package/dist/{src-CyaDNG9r.js → src-DMo2eCOJ.js} +1 -1
- package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/src-Dk2Na0b-.js +2 -0
- package/dist/src-OESRtvOi.js +2 -0
- package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
- package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-h5uDlk3m.js +1 -0
- package/dist/state-C97udoh3.js +1 -0
- package/dist/{state-B1t143wG.js → state-C_kdhp5v.js} +15 -15
- package/dist/{state-B1t143wG.js.map → state-C_kdhp5v.js.map} +1 -1
- package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
- package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
- package/dist/suite-ByT1lsdD.js +2 -0
- package/dist/{suite-WgT7m6Vs.js → suite-DieLXvCT.js} +61 -65
- package/dist/suite-DieLXvCT.js.map +1 -0
- package/dist/{summary-DIGEVr08.js → summary-CSzyBVsF.js} +113 -277
- package/dist/summary-CSzyBVsF.js.map +1 -0
- package/dist/tarball-CHcahyNK.js +258 -0
- package/dist/tarball-CHcahyNK.js.map +1 -0
- package/dist/testing.d.ts +426 -143
- package/dist/testing.js +12 -10
- package/dist/testing.js.map +1 -1
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
- package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
- package/dist/{transcript-BTxJpW4R.js → transcript-BJrKfgRW.js} +13 -5
- package/dist/transcript-BJrKfgRW.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-DXBbP9Mk.d.ts → types-DVvW66ci.d.ts} +569 -895
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +548 -121
- package/dist/view/index.js +9 -9
- package/dist/{views-sXpGE6a2.js → views-B9STsLB4.js} +9 -10
- package/dist/{views-sXpGE6a2.js.map → views-B9STsLB4.js.map} +1 -1
- package/dist/views-Ca-rTixp.js +2 -0
- package/dist/{worker-entry-Duo9oAeT.js → worker-entry-C5bLCv6s.js} +48 -20
- package/dist/worker-entry-C5bLCv6s.js.map +1 -0
- package/dist/workspace-8uHtBSve.js +1 -0
- package/dist/{workspace-CDI68wtX.js → workspace-CBuI9Uds.js} +5 -5
- package/dist/{workspace-CDI68wtX.js.map → workspace-CBuI9Uds.js.map} +1 -1
- package/docker/seccomp-default.json +1 -0
- package/package.json +5 -1
- package/skills/eval-rubric/SKILL.md +46 -104
- package/dist/adapter-BloGVZ_a.js.map +0 -1
- package/dist/agent-m-_t-IuW.js.map +0 -1
- package/dist/app-files-C-jwhRDt.js.map +0 -1
- package/dist/assemble-BDEpelpi.js.map +0 -1
- package/dist/assemble-C5bfncHU.js +0 -1
- package/dist/capture-CVRwRTnb.js.map +0 -1
- package/dist/capture-CherbjzB.js +0 -1
- package/dist/cli-C_4gaJ50.js.map +0 -1
- package/dist/cli-spec-CpnbATtI.js.map +0 -1
- package/dist/client-D5hNjpEx.js.map +0 -1
- package/dist/command-BWAuwQNk.js +0 -91
- package/dist/command-BWAuwQNk.js.map +0 -1
- package/dist/command-BXLUfw4h.js.map +0 -1
- package/dist/command-Be6BSE2L.js +0 -335
- package/dist/command-Be6BSE2L.js.map +0 -1
- package/dist/command-Bxdy62dW.js.map +0 -1
- package/dist/commands-jzigbNFb.js.map +0 -1
- package/dist/config-BrdHA9bJ.js.map +0 -1
- package/dist/cost-D4elx933.js.map +0 -1
- package/dist/cost-DmnAod9x.js +0 -1
- package/dist/db-CAfksinX.js +0 -2
- package/dist/db-DO0p7xgP.js +0 -619
- package/dist/db-DO0p7xgP.js.map +0 -1
- package/dist/debug-D24NOQH1.js.map +0 -1
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-BnRlTpZA.js.map +0 -1
- package/dist/docker-D01slDF3.js.map +0 -1
- package/dist/doctor-DQ2p0euG.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-CO0P0soF.js.map +0 -1
- package/dist/evaluate-BUmsN7yG.js.map +0 -1
- package/dist/evaluate-C_sgRA-V.d.ts +0 -134
- package/dist/facts-DzOUTreB.js +0 -906
- package/dist/facts-DzOUTreB.js.map +0 -1
- package/dist/files-B-CJeeEh.js.map +0 -1
- package/dist/files-BAt0i00X.js +0 -1
- package/dist/files-Bp5j5BqI.js +0 -1
- package/dist/files-C2sjmXhk.js +0 -1
- package/dist/files-C_gQNVB-.js +0 -65
- package/dist/files-C_gQNVB-.js.map +0 -1
- package/dist/files-Ca0d6Uq3.js.map +0 -1
- package/dist/files-kUCrswI6.js.map +0 -1
- package/dist/generate-DZ7vp69F.js +0 -1647
- package/dist/generate-DZ7vp69F.js.map +0 -1
- package/dist/harness-COVXSwJ6.js +0 -354
- package/dist/harness-COVXSwJ6.js.map +0 -1
- package/dist/header-BLPP6aCU.js.map +0 -1
- package/dist/init-RgCQXZhb.js +0 -1
- package/dist/init-SRalM5xF.js.map +0 -1
- package/dist/insights-TyAvz9aV.js +0 -68
- package/dist/insights-TyAvz9aV.js.map +0 -1
- package/dist/invocations-DzyKSJBa.js.map +0 -1
- package/dist/judge-yLvjC5A3.js.map +0 -1
- package/dist/match-Bz0nSdzN.js.map +0 -1
- package/dist/matchers-D4eUqx4j.js.map +0 -1
- package/dist/model-1ZLr9Dcv.js.map +0 -1
- package/dist/model-B45lDUV1.js.map +0 -1
- package/dist/model-BTZq8HIZ.js +0 -1
- package/dist/model-CgnOpss4.js.map +0 -1
- package/dist/model-Ch8kX87u.js +0 -1
- package/dist/node-BSq5qHnQ.js.map +0 -1
- package/dist/object-CLf2-tNq.js.map +0 -1
- package/dist/opencode-providers-klkqgzvb.js +0 -3276
- package/dist/opencode-providers-klkqgzvb.js.map +0 -1
- package/dist/pairs-CHfZNRT6.js.map +0 -1
- package/dist/patch-DWf-I_Q_.js.map +0 -1
- package/dist/plan-BIwUzlB8.js.map +0 -1
- package/dist/plugins-Dkp_H3pB.js.map +0 -1
- package/dist/prices-BcLEXvdY.js.map +0 -1
- package/dist/query-Cvbwl804.js +0 -1
- package/dist/questions-bAudIc04.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-CXwSnSjR.js +0 -278
- package/dist/remarks-CXwSnSjR.js.map +0 -1
- package/dist/remarks-CyAu2DiB.js +0 -1
- package/dist/remarks-mujJv1Ey.js.map +0 -1
- package/dist/remote-CKcEKja4.js.map +0 -1
- package/dist/resolve-B5-uqmJ6.js +0 -1
- package/dist/resolve-CBoKzQMP.js.map +0 -1
- package/dist/results-CMgZEANy.js +0 -78
- package/dist/results-CMgZEANy.js.map +0 -1
- package/dist/resume-B9SOSnK9.js.map +0 -1
- package/dist/rules-VhftPOOT.js.map +0 -1
- package/dist/run-CwIEK16q.js +0 -88
- package/dist/run-CwIEK16q.js.map +0 -1
- package/dist/runtime-BgQRyCLl.js.map +0 -1
- package/dist/server-CHRSgKRp.js +0 -1
- package/dist/server-TGW4WyAz.js.map +0 -1
- package/dist/shared-BDUzStad.js.map +0 -1
- package/dist/source-B1WvBIQt.js +0 -40
- package/dist/source-B1WvBIQt.js.map +0 -1
- package/dist/source-DAzztZwU.js +0 -1
- package/dist/src-1jHbFaGW.js +0 -1
- package/dist/src-8zoX_qrb.js.map +0 -1
- package/dist/src-C_9wgowC.js +0 -2
- package/dist/src-ChhtI6BP.js.map +0 -1
- package/dist/src-D005Cbqz.js +0 -2
- package/dist/src-DIfRbp5a.js +0 -3
- package/dist/src-Dg8121Xy.js +0 -2
- package/dist/src-ElBqEN-S.js +0 -2
- package/dist/src-NSdQ9zI6.js.map +0 -1
- package/dist/src-YHGaUrGE.js +0 -2
- package/dist/src-mJVxw1Jm.js +0 -2
- package/dist/state-vznMh2wl.js +0 -1
- package/dist/suite-BsWGHzlG.js +0 -2
- package/dist/suite-WgT7m6Vs.js.map +0 -1
- package/dist/summary-DIGEVr08.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-DF3_p5pE.js +0 -1
- package/dist/validate-BhwqrVsm.js.map +0 -1
- package/dist/views-BW_C5eVH.js +0 -2
- package/dist/worker-entry-Duo9oAeT.js.map +0 -1
- package/dist/workspace-QZCacP8A.js +0 -1
|
@@ -1,26 +1,29 @@
|
|
|
1
|
-
import { n as isOneOf, t as isObjectLike } from "./object-
|
|
1
|
+
import { n as isOneOf, t as isObjectLike } from "./object-BkXGPKWY.js";
|
|
2
2
|
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
3
|
-
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, i as durationSchema, l as nonEmpty, o as genericProblem,
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
3
|
+
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, h as parseDuration, i as durationSchema, l as nonEmpty, o as genericProblem, s as got, t as closest, u as oneOf } from "./shared-B_lvGqWz.js";
|
|
4
|
+
import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-Cf5HMjib.js";
|
|
5
|
+
import { B as gatePolicySchema, D as pluginServerName, J as snapshotOptionSchema, L as egressPolicySchema, V as gateProblem, Y as snapshotProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-ngQEJBjJ.js";
|
|
6
|
+
import { _ as gatewayDescriptorSchema, f as FACTORY, g as directOffPin, h as GATEWAY_NAMES, m as harnessProblem, p as harnessDescriptorSchema, u as opencodeModelError, v as gatewayEntryDataSchema, y as gatewayNameSchema } from "./harness-3NVyTSGK.js";
|
|
6
7
|
import { a as fromCwd, l as sha256 } from "./ids-Bp6ut7ZD.js";
|
|
7
|
-
import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-
|
|
8
|
-
import {
|
|
9
|
-
import { r as
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
8
|
+
import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-Bz0l7OZI.js";
|
|
9
|
+
import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-Bg4v4e_K.js";
|
|
10
|
+
import { a as VERTEX_ROUTES, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV, t as SANDBOX_ACCESS_TOKEN } from "./vertex-DypYjx71.js";
|
|
11
|
+
import { r as badCharacterAt } from "./redact-DVIaahLv.js";
|
|
12
|
+
import { s as withoutBom } from "./text-DTigcA64.js";
|
|
13
|
+
import { t as inCI } from "./env-CARj5Axq.js";
|
|
12
14
|
import { u as ownFileURL } from "./root-Dlff-zp8.js";
|
|
13
|
-
import {
|
|
15
|
+
import { F as PACKAGE_HOSTS, N as DOCKER_EGRESS, P as HARNESS_HOSTS, ft as run, pt as shellQuote } from "./src-C0ySiej5.js";
|
|
14
16
|
import { t as InfraError } from "./types-C5vI3g1C.js";
|
|
15
|
-
import {
|
|
16
|
-
import
|
|
17
|
-
import {
|
|
17
|
+
import { n as gateLabel, u as pairFactors } from "./gate-BC6SdQZs.js";
|
|
18
|
+
import "./stats-hfpuYsAM.js";
|
|
19
|
+
import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-CRKYPkRt.js";
|
|
20
|
+
import { t as docker } from "./sandbox-Cmi6IQML.js";
|
|
18
21
|
import { existsSync, readFileSync } from "node:fs";
|
|
19
22
|
import { availableParallelism, tmpdir, totalmem } from "node:os";
|
|
20
23
|
import { dirname, join, resolve, win32 } from "node:path";
|
|
21
24
|
import { parseDocument } from "yaml";
|
|
22
25
|
import { z } from "zod";
|
|
23
|
-
import { isIP } from "node:net";
|
|
26
|
+
import { connect, isIP } from "node:net";
|
|
24
27
|
import { createHash, randomUUID } from "node:crypto";
|
|
25
28
|
import { spawn } from "node:child_process";
|
|
26
29
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
@@ -81,48 +84,65 @@ function decideState(json, files) {
|
|
|
81
84
|
}
|
|
82
85
|
//#endregion
|
|
83
86
|
//#region src/contract/judge.ts
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional()
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
87
|
+
/** The inputs every judge call and question shares; `c.judge()` adds `contents` and its own `artifacts`. */
|
|
88
|
+
const sharedInputFields = {
|
|
89
|
+
task: z.boolean().optional(),
|
|
90
|
+
output: z.boolean().optional(),
|
|
91
|
+
files: z.boolean().optional(),
|
|
92
|
+
diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
|
|
93
|
+
transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
|
|
94
|
+
checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
|
|
95
|
+
};
|
|
96
|
+
const turnsSchema = z.enum(["last", "all"]);
|
|
97
|
+
const { task, output, files, diff, transcript, checks } = sharedInputFields;
|
|
98
|
+
const judgeInputsSchema = z.strictObject({
|
|
99
|
+
task,
|
|
100
|
+
output,
|
|
101
|
+
files,
|
|
102
|
+
diff,
|
|
103
|
+
contents: z.array(nonEmpty()).optional(),
|
|
104
|
+
transcript,
|
|
105
|
+
checks,
|
|
106
|
+
artifacts: z.union([z.literal(true), z.array(z.custom())]).optional()
|
|
107
|
+
});
|
|
108
|
+
const judgeSubjectSchema = z.strictObject({
|
|
109
|
+
on: z.custom().optional(),
|
|
110
|
+
turns: turnsSchema.optional()
|
|
111
|
+
});
|
|
112
|
+
const INPUT_KEYS = Object.keys(judgeInputsSchema.shape);
|
|
113
|
+
/** True when an input's value turns it on: not `false`, unset or an empty list. */
|
|
114
|
+
const inputOn = (value) => value !== void 0 && value !== false && !(Array.isArray(value) && value.length === 0);
|
|
115
|
+
/** The input keys an options object turns on, in `keys`' order. */
|
|
116
|
+
const inputsOn = (value, keys) => keys.filter((k) => inputOn(value[k]));
|
|
117
|
+
/** `a`, `b` and `c` as prose: "a, b or c". */
|
|
118
|
+
const orList = (items) => items.length < 2 ? items[0] ?? "" : `${items.slice(0, -1).join(", ")} or ${items.at(-1)}`;
|
|
119
|
+
const NO_JUDGE_INPUT = `c.judge() shows the judge nothing: turn on at least one input, such as { diff: true } or { contents: ['src/**'] }. The inputs are ${orList(INPUT_KEYS)}.`;
|
|
94
120
|
const judgeFields = {
|
|
95
|
-
threshold: z.number().min(0, "must be between 0 and 1").max(1, "must be between 0 and 1").optional()
|
|
96
|
-
providerOptions: z.record(z.string(), z.record(z.string(), jsonValueSchema)).optional()
|
|
97
|
-
maxConcurrent: z.number().int().min(1).optional()
|
|
98
|
-
|
|
99
|
-
gateway: z.union([gatewayNameSchema, gatewayDescriptorSchema]).optional().describe("The gateway the judge's calls go through, by name or by its helper. It skips the config's `gateway` fallback: the plan refuses a gateway that isn't listed or can't serve the model.")
|
|
121
|
+
threshold: z.number().min(0, "must be between 0 and 1").max(1, "must be between 0 and 1").optional(),
|
|
122
|
+
providerOptions: z.record(z.string(), z.record(z.string(), jsonValueSchema)).optional(),
|
|
123
|
+
maxConcurrent: z.number().int().min(1).optional(),
|
|
124
|
+
gateway: z.union([gatewayNameSchema, gatewayDescriptorSchema]).optional()
|
|
100
125
|
};
|
|
101
126
|
const fakeJudgeDataSchema = z.strictObject({
|
|
102
127
|
kind: z.literal("fake-judge"),
|
|
103
|
-
score: z.number().min(0).max(1)
|
|
104
|
-
})
|
|
128
|
+
score: z.number().min(0).max(1)
|
|
129
|
+
});
|
|
105
130
|
const judgeModelNameSchema = z.string().regex(/\//, "must be a 'provider/model' string, e.g. 'google/gemini-3.8-flash'");
|
|
106
131
|
const configJudgeDataSchema = z.strictObject({
|
|
107
|
-
model: z.union([z.string(), fakeJudgeDataSchema]).optional()
|
|
132
|
+
model: z.union([z.string(), fakeJudgeDataSchema]).optional(),
|
|
108
133
|
...judgeFields
|
|
109
|
-
})
|
|
134
|
+
});
|
|
110
135
|
const judgeOptionsSchema = z.strictObject({
|
|
111
136
|
model: z.custom().optional(),
|
|
112
137
|
...judgeFields
|
|
113
138
|
});
|
|
114
|
-
const judgeCallSchema = judgeOptionsSchema.extend(
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
shows: "state",
|
|
122
|
-
show: "state",
|
|
123
|
-
input: "state"
|
|
124
|
-
};
|
|
125
|
-
const VALUES = {
|
|
139
|
+
const judgeCallSchema = judgeOptionsSchema.extend(judgeInputsSchema.shape).extend(judgeSubjectSchema.shape);
|
|
140
|
+
const CALL_ONLY_KEYS = [
|
|
141
|
+
...INPUT_KEYS,
|
|
142
|
+
"on",
|
|
143
|
+
"turns"
|
|
144
|
+
];
|
|
145
|
+
const VALUES$1 = {
|
|
126
146
|
task: "true or false",
|
|
127
147
|
output: "true or false",
|
|
128
148
|
files: "true or false",
|
|
@@ -159,7 +179,10 @@ function judgeProblem(judge, site) {
|
|
|
159
179
|
return `${key ? `${base}.${key}` : base} in ${site.at === "config" ? site.file : site.where}`;
|
|
160
180
|
};
|
|
161
181
|
const loc = site.at === "call" ? " in c.judge()" : site.at === "options" ? ` in ${site.where}` : ` in ${site.path === "judge" ? "" : `${site.path.replace(/\.judge$/, "")} in `}${site.file}`;
|
|
162
|
-
if (typeof judge !== "object" || judge === null || Array.isArray(judge))
|
|
182
|
+
if (typeof judge !== "object" || judge === null || Array.isArray(judge)) {
|
|
183
|
+
const example = site.at === "call" ? "{ diff: true }" : "{ threshold: 0.8 }";
|
|
184
|
+
return `${field()} must be an object such as ${example} (got ${shown(judge)})`;
|
|
185
|
+
}
|
|
163
186
|
const value = judge;
|
|
164
187
|
const model = value.model;
|
|
165
188
|
if (site.at === "config" && typeof model === "string") {
|
|
@@ -168,7 +191,7 @@ function judgeProblem(judge, site) {
|
|
|
168
191
|
}
|
|
169
192
|
const schema = site.at === "call" ? judgeCallSchema : site.at === "config" ? configJudgeDataSchema : judgeOptionsSchema;
|
|
170
193
|
const parsed = schema.safeParse(judge);
|
|
171
|
-
if (parsed.success) return void 0;
|
|
194
|
+
if (parsed.success) return site.at === "call" && !inputsOn(value, INPUT_KEYS).length ? NO_JUDGE_INPUT : void 0;
|
|
172
195
|
const numericFake = site.at === "config" && typeof model === "object" && model?.kind === "fake-judge" && typeof model.score === "number";
|
|
173
196
|
const issues = parsed.error.issues.filter((i) => !(numericFake && i.path[0] === "model"));
|
|
174
197
|
if (!issues.length) return void 0;
|
|
@@ -191,29 +214,22 @@ function judgeProblem(judge, site) {
|
|
|
191
214
|
const name = issues.find((i) => i.code === "unrecognized_keys" && i.path.length === 0)?.keys[0];
|
|
192
215
|
if (name === void 0) return void 0;
|
|
193
216
|
const allowed = Object.keys(schema.shape);
|
|
194
|
-
const hint =
|
|
195
|
-
const why = site.at !== "call" && (name
|
|
217
|
+
const hint = nearest(name, allowed);
|
|
218
|
+
const why = site.at !== "call" && CALL_ONLY_KEYS.includes(name) ? ` ${name} is per call: c.judge(question, { ${name} }).` : hint ? ` Did you mean "${hint}"?` : "";
|
|
196
219
|
return `Unknown judge option "${name}"${loc}.${why}`;
|
|
197
220
|
};
|
|
198
|
-
const
|
|
199
|
-
const issue = issues.find((i) => i.path[0] === "
|
|
221
|
+
const inputProblem = () => {
|
|
222
|
+
const issue = issues.find((i) => typeof i.path[0] === "string" && i.path[0] in VALUES$1);
|
|
200
223
|
if (!issue) return void 0;
|
|
201
|
-
const
|
|
202
|
-
|
|
203
|
-
if (issue.code === "unrecognized_keys" && issue.path.length === 1) {
|
|
204
|
-
const key = issue.keys[0] ?? "";
|
|
205
|
-
const hint = nearest(key, STATE_KEYS$1);
|
|
206
|
-
return `Unknown judge.state key "${key}"${loc}.${hint ? ` Did you mean "${hint}"?` : ""}`;
|
|
207
|
-
}
|
|
208
|
-
const key = issue.path[1];
|
|
209
|
-
return `${field(`state.${String(key)}`)} must be ${VALUES[key]} (got ${shown(state[key])})`;
|
|
224
|
+
const key = issue.path[0];
|
|
225
|
+
return `${field(key)} must be ${VALUES$1[key]} (got ${shown(value[key])})`;
|
|
210
226
|
};
|
|
211
227
|
const providerOptionsProblem = () => at("providerOptions") ? `${field("providerOptions")} must map each provider to an object of JSON values, e.g. { google: { thinkingConfig: { thinkingBudget: 0 } } } (got ${shown(value.providerOptions)})` : void 0;
|
|
212
228
|
const artifactsProblem = () => at("artifacts") ? `c.judge() artifacts ${badJudgeArtifacts(value.artifacts)}` : void 0;
|
|
213
229
|
const gatewayProblem = () => at("gateway") ? `${field("gateway")} names a gateway in the config's gateway list: ${GATEWAY_NAMES.map((g) => `'${g}'`).join(" or ")}, or is its helper (got ${shown(value.gateway)})` : void 0;
|
|
214
|
-
if (site.at === "config") order.push(modelProblem, keyProblem,
|
|
215
|
-
else if (site.at === "call") order.push(thresholdProblem, maxConcurrentProblem, keyProblem,
|
|
216
|
-
else order.push(maxConcurrentProblem, thresholdProblem, keyProblem,
|
|
230
|
+
if (site.at === "config") order.push(modelProblem, keyProblem, providerOptionsProblem, thresholdProblem, gatewayProblem);
|
|
231
|
+
else if (site.at === "call") order.push(thresholdProblem, maxConcurrentProblem, keyProblem, inputProblem, providerOptionsProblem, artifactsProblem, gatewayProblem);
|
|
232
|
+
else order.push(maxConcurrentProblem, thresholdProblem, keyProblem, providerOptionsProblem, gatewayProblem);
|
|
217
233
|
for (const check of order) {
|
|
218
234
|
const problem = check();
|
|
219
235
|
if (problem) return problem;
|
|
@@ -222,25 +238,18 @@ function judgeProblem(judge, site) {
|
|
|
222
238
|
}
|
|
223
239
|
//#endregion
|
|
224
240
|
//#region src/contract/questions.ts
|
|
225
|
-
const pairFactorSchema = z.enum(["environment", "harness"])
|
|
241
|
+
const pairFactorSchema = z.enum(["environment", "harness"]);
|
|
226
242
|
/** Evaluation models score yes/no questions only, so they can't answer a question. */
|
|
227
243
|
const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
|
|
228
|
-
const questionStateSchema = z.strictObject({
|
|
229
|
-
task: z.boolean().optional().describe("The prompts the attempt was sent, once for both sides when they match. Default true."),
|
|
230
|
-
output: z.boolean().optional().describe("The final message: the agent's own account of its work. Default false in a pair, true per attempt."),
|
|
231
|
-
files: z.boolean().optional().describe("`status path` lines for the changed files, under `diff`'s patterns. Default false."),
|
|
232
|
-
diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional().describe("The diff since the eval started. A list is gitignore-style patterns over workspace paths: the last matching pattern wins and `!` excludes. Default true."),
|
|
233
|
-
transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional().describe("The transcript as text lines: `true` is messages and tool calls with tool output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each harness's tools, so a pair of two harnesses isn't blind. Default false in a pair, true per attempt."),
|
|
234
|
-
turns: z.enum(["last", "all"]).optional().describe("`'all'`: every turn under `turns`, each with its task, output and (with `transcript`) transcript; `'last'`: the last turn only. Default `'all'`."),
|
|
235
|
-
checks: z.union([z.boolean(), z.array(nonEmpty())]).optional().describe("`c.exec` results; a list shows only those commands, as passed to `c.exec`. Default false.")
|
|
236
|
-
}).describe("What a question is shown of each attempt: the keys of `judge.state` but `contents` (the attempts' workspaces are gone when questions are asked). An unset key keeps its default: a pairwise question shows the task and the diff; one asked of each attempt also shows the final message and the transcript.");
|
|
237
244
|
const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty()).min(1)]);
|
|
245
|
+
const questionInputFields = {
|
|
246
|
+
...sharedInputFields,
|
|
247
|
+
artifacts: questionArtifactsSchema.optional()
|
|
248
|
+
};
|
|
249
|
+
z.strictObject(questionInputFields);
|
|
238
250
|
const questionModelSchema = z.string().refine((m) => m.includes("/"), { params: { provider: true } }).refine((m) => !isEvaluationModel(m), { params: { evaluation: true } });
|
|
239
|
-
const questionOptionsSchema = z.strictObject({
|
|
240
|
-
|
|
241
|
-
state: questionStateSchema.optional().describe("What it's shown of each attempt."),
|
|
242
|
-
artifacts: questionArtifactsSchema.optional().describe("Each attempt's own captures, by name; `true` sends all of them. Default none.")
|
|
243
|
-
}).describe("Defaults for every question: `model` and `artifacts` replaced, `state` merged key by key.");
|
|
251
|
+
const questionOptionsSchema = z.strictObject({ model: questionModelSchema.optional() });
|
|
252
|
+
const QUESTION_INPUT_KEYS = Object.keys(questionInputFields);
|
|
244
253
|
/** The first item a list holds twice. */
|
|
245
254
|
const twice = (items) => items.find((k, i) => items.indexOf(k) !== i);
|
|
246
255
|
const unique = (list, ctx) => {
|
|
@@ -251,35 +260,38 @@ const unique = (list, ctx) => {
|
|
|
251
260
|
};
|
|
252
261
|
const acrossSchema = z.union([pairFactorSchema, z.array(pairFactorSchema).min(1).superRefine(unique)]);
|
|
253
262
|
const choicesSchema = z.array(nonEmpty()).min(2).superRefine(unique);
|
|
254
|
-
const
|
|
263
|
+
const questionSchema = z.strictObject({
|
|
255
264
|
question: nonEmpty(),
|
|
256
|
-
model: questionModelSchema.optional()
|
|
257
|
-
across: acrossSchema.optional()
|
|
258
|
-
choices: choicesSchema.optional()
|
|
259
|
-
|
|
260
|
-
|
|
265
|
+
model: questionModelSchema.optional(),
|
|
266
|
+
across: acrossSchema.optional(),
|
|
267
|
+
choices: choicesSchema.optional(),
|
|
268
|
+
turns: turnsSchema.optional(),
|
|
269
|
+
...questionInputFields
|
|
261
270
|
}).superRefine((q, ctx) => {
|
|
262
271
|
if (q.across !== void 0 && q.choices !== void 0) ctx.addIssue({
|
|
263
272
|
code: "custom",
|
|
264
273
|
path: ["choices"],
|
|
265
274
|
message: "with across"
|
|
266
275
|
});
|
|
276
|
+
if (!inputsOn(q, QUESTION_INPUT_KEYS).length) ctx.addIssue({
|
|
277
|
+
code: "custom",
|
|
278
|
+
message: "no input"
|
|
279
|
+
});
|
|
267
280
|
});
|
|
268
|
-
const
|
|
269
|
-
const questionText = (q) => typeof q === "string" ? q : q.question;
|
|
281
|
+
const questionText = (q) => q.question;
|
|
270
282
|
const questionsSchema = z.array(questionSchema).superRefine((list, ctx) => {
|
|
271
283
|
if (twice(list.map(questionText)) !== void 0) ctx.addIssue({
|
|
272
284
|
code: "custom",
|
|
273
285
|
message: "twice"
|
|
274
286
|
});
|
|
275
287
|
});
|
|
276
|
-
const QUESTION_EXAMPLE = `questions: ['How did the agent find the failing test?']`;
|
|
288
|
+
const QUESTION_EXAMPLE = `questions: [{ question: 'How did the agent find the failing test?', transcript: true }]`;
|
|
277
289
|
const ACROSS_HINT = `'environment' (vary the environment, same harness) or 'harness' (vary the harness, same environment)`;
|
|
278
290
|
const ACROSS_MEAN = " Did you mean \"across\"? It is the factor a pairwise question varies: across: 'environment'.";
|
|
279
|
-
const QUESTION_KEYS = Object.keys(
|
|
280
|
-
const
|
|
281
|
-
const
|
|
282
|
-
const
|
|
291
|
+
const QUESTION_KEYS = Object.keys(questionSchema.shape);
|
|
292
|
+
const INPUTS = orList(QUESTION_INPUT_KEYS);
|
|
293
|
+
const QUESTION_SHAPE = `{ question, model?, across?, choices?, turns? } with at least one of ${INPUTS}`;
|
|
294
|
+
const VALUES = {
|
|
283
295
|
task: "true or false",
|
|
284
296
|
output: "true or false",
|
|
285
297
|
files: "true or false",
|
|
@@ -296,41 +308,31 @@ const describe$2 = (value) => {
|
|
|
296
308
|
}
|
|
297
309
|
return JSON.stringify(value) ?? String(value);
|
|
298
310
|
};
|
|
299
|
-
/**
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
const { state, artifacts } = value;
|
|
306
|
-
if (state !== void 0) {
|
|
307
|
-
if (!isObjectLike(state) || Array.isArray(state)) return `${name("state")} in ${where} must be an object such as { transcript: 'tools' } (got ${describe$2(state)})`;
|
|
308
|
-
for (const [key, v] of Object.entries(state)) {
|
|
309
|
-
if (key === "contents") return `${name("state.contents")} in ${where} can't be set: questions are asked after the attempts, when their workspaces are gone. Narrow state.diff instead.`;
|
|
310
|
-
const schema = questionStateSchema.shape[key];
|
|
311
|
-
if (!schema) {
|
|
312
|
-
const hint = closest(key, STATE_KEYS);
|
|
313
|
-
return `Unknown ${name("state")} key "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""}`;
|
|
314
|
-
}
|
|
315
|
-
if (!schema.safeParse(v).success) return `${name(`state.${key}`)} in ${where} must be ${STATE_VALUES[key]} (got ${describe$2(v)})`;
|
|
311
|
+
/** What's wrong with an entry's inputs and `turns`, as one sentence, or undefined. */
|
|
312
|
+
function questionInputsProblem(value, at, where) {
|
|
313
|
+
for (const [key, v] of Object.entries(value)) {
|
|
314
|
+
if (key === "artifacts") {
|
|
315
|
+
if (!questionArtifactsSchema.safeParse(v).success) return `${at} artifacts in ${where} must be true or a non-empty list of capture names (got ${describe$2(v)})`;
|
|
316
|
+
continue;
|
|
316
317
|
}
|
|
318
|
+
if (!(key in VALUES)) continue;
|
|
319
|
+
if (!(key === "turns" ? turnsSchema : sharedInputFields[key]).safeParse(v).success) return `${at} ${key} in ${where} must be ${VALUES[key]} (got ${describe$2(v)})`;
|
|
317
320
|
}
|
|
318
|
-
if (
|
|
321
|
+
if (!inputsOn(value, QUESTION_INPUT_KEYS).length) return `${at} in ${where} shows the model nothing: turn on at least one of ${INPUTS}, e.g. { question: '…', diff: true }.`;
|
|
319
322
|
}
|
|
320
323
|
/** What's wrong with the `question` option set in `where`, as one sentence, or undefined. */
|
|
321
324
|
function questionOptionsProblem(value, where) {
|
|
322
|
-
if (!isObjectLike(value) || Array.isArray(value)) return `question in ${where} is { model
|
|
325
|
+
if (!isObjectLike(value) || Array.isArray(value)) return `question in ${where} is { model? } (got ${describe$2(value)}), e.g. question: { model: 'google/gemini-3.8-flash' }`;
|
|
323
326
|
const keys = Object.keys(questionOptionsSchema.shape);
|
|
324
327
|
for (const key of Object.keys(value)) {
|
|
325
328
|
if (keys.includes(key)) continue;
|
|
326
|
-
if (key === "across" || key === "choices") return `Unknown question option "${key}" in ${where}. ${key} goes on each questions entry, { question, ${key} }:
|
|
329
|
+
if (key === "across" || key === "choices" || key === "turns" || key in questionInputFields) return `Unknown question option "${key}" in ${where}. ${key} goes on each questions entry, { question, ${key} }: what a question is and what it's shown are its own.`;
|
|
327
330
|
const hint = closest(key, keys);
|
|
328
|
-
return `Unknown question option "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""} question takes model
|
|
331
|
+
return `Unknown question option "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""} question takes model.`;
|
|
329
332
|
}
|
|
330
333
|
const { model } = value;
|
|
331
334
|
if (model !== void 0 && typeof model === "string" && isEvaluationModel(model)) return `question.model in ${where} names ${model}, an evaluation model: it scores yes/no questions and can't answer a question. Use a language model, e.g. 'google/gemini-3.8-flash'.`;
|
|
332
335
|
if (model !== void 0 && !questionModelSchema.safeParse(model).success) return `question.model in ${where} is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${describe$2(model)})`;
|
|
333
|
-
return questionPartsProblem(value, (key) => `question.${key}`, where);
|
|
334
336
|
}
|
|
335
337
|
/** What's wrong with an entry's `across`, or undefined. */
|
|
336
338
|
function acrossProblem(question, across, where) {
|
|
@@ -349,11 +351,11 @@ function choicesProblem(question, choices, where) {
|
|
|
349
351
|
if (blank !== void 0 && blank !== false) return `${at} has an answer that isn't a non-empty string (got ${describe$2(blank)}), e.g. choices: ['the test', 'the code']`;
|
|
350
352
|
return `${at} is a list of at least two answers, e.g. choices: ['the test', 'the code'] (got ${describe$2(choices)})`;
|
|
351
353
|
}
|
|
352
|
-
/** One question's sentence
|
|
354
|
+
/** One question's sentence. */
|
|
353
355
|
function questionProblem(item, where) {
|
|
354
|
-
if (typeof item
|
|
355
|
-
if (typeof item !== "object" || item === null || Array.isArray(item)) return `questions in ${where} takes strings or ${QUESTION_SHAPE} (got ${describe$2(item)}), e.g. ${QUESTION_EXAMPLE}.`;
|
|
356
|
+
if (typeof item !== "object" || item === null || Array.isArray(item)) return `questions in ${where} takes ${QUESTION_SHAPE} (got ${describe$2(item)}), e.g. ${QUESTION_EXAMPLE}.`;
|
|
356
357
|
const unknown = Object.keys(item).filter((k) => !QUESTION_KEYS.includes(k));
|
|
358
|
+
if (unknown.includes("contents")) return `questions in ${where} can't set contents: questions are asked after the attempts, when their workspaces are gone. Narrow diff instead.`;
|
|
357
359
|
if (unknown.length) {
|
|
358
360
|
const hint = unknown.some((k) => k === "cut" || k === "cuts") ? ACROSS_MEAN : "";
|
|
359
361
|
return `questions in ${where} takes ${QUESTION_SHAPE} (got ${unknown.map((k) => `"${k}"`).join(", ")}).${hint}`;
|
|
@@ -370,7 +372,7 @@ function questionProblem(item, where) {
|
|
|
370
372
|
if (problem) return problem;
|
|
371
373
|
}
|
|
372
374
|
if (across !== void 0 && choices !== void 0) return `questions "${text}" in ${where} sets both across and choices. choices are the answers a question asked of each attempt picks from, and across makes it pick between two attempts instead: drop choices to vote between pairs, or drop across to ask each attempt.`;
|
|
373
|
-
const parts =
|
|
375
|
+
const parts = questionInputsProblem(item, `questions "${text}"`, where);
|
|
374
376
|
if (parts) return parts;
|
|
375
377
|
if (typeof model === "string" && isEvaluationModel(model)) return `questions "${text}" in ${where} names ${model}, an evaluation model: it scores yes/no questions and can't answer a question. Use a language model, e.g. 'google/gemini-3.8-flash'.`;
|
|
376
378
|
return `questions "${text}" in ${where}: model is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${describe$2(model)})`;
|
|
@@ -386,28 +388,30 @@ function questionsProblem(value, where) {
|
|
|
386
388
|
//#endregion
|
|
387
389
|
//#region src/contract/config.ts
|
|
388
390
|
const count = () => z.number().int().min(1);
|
|
391
|
+
/** `runs`: a count, or `{ bestOf }` with an odd count of 3 or more. */
|
|
392
|
+
const runs = () => z.union([count(), z.strictObject({ bestOf: z.number().int().min(3).refine((n) => n % 2 === 1, "must be odd") })]);
|
|
389
393
|
const captureWindowSchema = z.strictObject({
|
|
390
|
-
width: count()
|
|
391
|
-
height: count()
|
|
392
|
-
})
|
|
394
|
+
width: count(),
|
|
395
|
+
height: count()
|
|
396
|
+
});
|
|
393
397
|
const captureOptionsSchema = z.strictObject({
|
|
394
|
-
viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional()
|
|
395
|
-
timeout: durationSchema.optional()
|
|
396
|
-
browser: z.boolean().optional()
|
|
397
|
-
model: z.string().includes("/").optional()
|
|
398
|
-
})
|
|
398
|
+
viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
|
|
399
|
+
timeout: durationSchema.optional(),
|
|
400
|
+
browser: z.boolean().optional(),
|
|
401
|
+
model: z.string().includes("/").optional()
|
|
402
|
+
});
|
|
399
403
|
const harnessList = z.union([harnessDescriptorSchema, z.array(harnessDescriptorSchema).min(1)]);
|
|
400
404
|
const budgetSchema = z.strictObject({
|
|
401
|
-
attempts: count().optional()
|
|
402
|
-
duration: durationSchema.optional()
|
|
403
|
-
})
|
|
405
|
+
attempts: count().optional(),
|
|
406
|
+
duration: durationSchema.optional()
|
|
407
|
+
});
|
|
404
408
|
const historyConfigSchema = z.strictObject({
|
|
405
|
-
dir: z.string().optional()
|
|
409
|
+
dir: z.string().optional(),
|
|
406
410
|
artifacts: z.enum([
|
|
407
411
|
"all",
|
|
408
412
|
"failures",
|
|
409
413
|
"none"
|
|
410
|
-
]).optional()
|
|
414
|
+
]).optional()
|
|
411
415
|
});
|
|
412
416
|
const recordMode = z.enum([
|
|
413
417
|
"manual",
|
|
@@ -415,32 +419,33 @@ const recordMode = z.enum([
|
|
|
415
419
|
"always"
|
|
416
420
|
]);
|
|
417
421
|
const profileDefaultsDataSchema = z.strictObject({
|
|
418
|
-
harness: harnessList.optional()
|
|
419
|
-
sandbox: sandboxDescriptorSchema.optional()
|
|
420
|
-
environment: environmentDataSchema.optional()
|
|
421
|
-
runs:
|
|
422
|
-
gate: gatePolicySchema.optional()
|
|
423
|
-
env: envVarsSchema.optional()
|
|
424
|
-
egress: egressPolicySchema.optional()
|
|
425
|
-
judge: configJudgeDataSchema.optional()
|
|
426
|
-
timeout: durationSchema.optional()
|
|
427
|
-
capture: captureOptionsSchema.optional()
|
|
428
|
-
question: questionOptionsSchema.optional()
|
|
429
|
-
|
|
422
|
+
harness: harnessList.optional(),
|
|
423
|
+
sandbox: sandboxDescriptorSchema.optional(),
|
|
424
|
+
environment: environmentDataSchema.optional(),
|
|
425
|
+
runs: runs().optional(),
|
|
426
|
+
gate: gatePolicySchema.optional(),
|
|
427
|
+
env: envVarsSchema.optional(),
|
|
428
|
+
egress: egressPolicySchema.optional(),
|
|
429
|
+
judge: configJudgeDataSchema.optional(),
|
|
430
|
+
timeout: durationSchema.optional(),
|
|
431
|
+
capture: captureOptionsSchema.optional(),
|
|
432
|
+
question: questionOptionsSchema.optional(),
|
|
433
|
+
snapshot: snapshotOptionSchema.optional()
|
|
434
|
+
});
|
|
430
435
|
const profileLimitSchema = z.strictObject({
|
|
431
|
-
runs: count().optional()
|
|
432
|
-
compare: z.union([z.literal(false), z.strictObject({ maxVariants: z.number().int().min(0) })]).optional()
|
|
433
|
-
harness: z.array(z.string()).optional()
|
|
434
|
-
})
|
|
436
|
+
runs: count().optional(),
|
|
437
|
+
compare: z.union([z.literal(false), z.strictObject({ maxVariants: z.number().int().min(0) })]).optional(),
|
|
438
|
+
harness: z.array(z.string()).optional()
|
|
439
|
+
});
|
|
435
440
|
const profileDataSchema = z.strictObject({
|
|
436
|
-
defaults: profileDefaultsDataSchema.optional()
|
|
437
|
-
limit: profileLimitSchema.optional()
|
|
438
|
-
judge: configJudgeDataSchema.optional()
|
|
439
|
-
budget: budgetSchema.optional()
|
|
440
|
-
record: recordMode.optional()
|
|
441
|
-
tags: z.array(z.string()).optional()
|
|
442
|
-
gateway: z.array(gatewayEntryDataSchema).min(1).optional()
|
|
443
|
-
})
|
|
441
|
+
defaults: profileDefaultsDataSchema.optional(),
|
|
442
|
+
limit: profileLimitSchema.optional(),
|
|
443
|
+
judge: configJudgeDataSchema.optional(),
|
|
444
|
+
budget: budgetSchema.optional(),
|
|
445
|
+
record: recordMode.optional(),
|
|
446
|
+
tags: z.array(z.string()).optional(),
|
|
447
|
+
gateway: z.array(gatewayEntryDataSchema).min(1).optional()
|
|
448
|
+
});
|
|
444
449
|
const fakeVote = z.enum([
|
|
445
450
|
"A",
|
|
446
451
|
"B",
|
|
@@ -449,7 +454,7 @@ const fakeVote = z.enum([
|
|
|
449
454
|
const failure = z.strictObject({ error: z.string() });
|
|
450
455
|
const fakeAnalysisDataSchema = z.strictObject({
|
|
451
456
|
kind: z.literal("fake-analysis"),
|
|
452
|
-
output: z.unknown().optional()
|
|
457
|
+
output: z.unknown().optional(),
|
|
453
458
|
vote: z.union([
|
|
454
459
|
fakeVote,
|
|
455
460
|
z.strictObject({
|
|
@@ -457,7 +462,7 @@ const fakeAnalysisDataSchema = z.strictObject({
|
|
|
457
462
|
reason: z.string().optional()
|
|
458
463
|
}),
|
|
459
464
|
failure
|
|
460
|
-
]).optional()
|
|
465
|
+
]).optional(),
|
|
461
466
|
answer: z.union([
|
|
462
467
|
z.string(),
|
|
463
468
|
z.strictObject({
|
|
@@ -465,46 +470,45 @@ const fakeAnalysisDataSchema = z.strictObject({
|
|
|
465
470
|
text: z.string().optional()
|
|
466
471
|
}),
|
|
467
472
|
failure
|
|
468
|
-
]).optional()
|
|
469
|
-
})
|
|
473
|
+
]).optional()
|
|
474
|
+
});
|
|
470
475
|
const analysisConfigDataSchema = z.strictObject({
|
|
471
|
-
model: z.union([z.string(), fakeAnalysisDataSchema]).optional()
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
maxOutputTokens: z.number().int().min(500).optional().describe("The most tokens the model may answer with (default 8000). reeval has no prices, so tokens are the cap.")
|
|
477
|
-
}).describe("`AnalysisConfig` as data: the model by name, or a fake analysis.");
|
|
476
|
+
model: z.union([z.string(), fakeAnalysisDataSchema]).optional(),
|
|
477
|
+
instructions: z.string().optional(),
|
|
478
|
+
maxInputTokens: z.number().int().min(1e3).optional(),
|
|
479
|
+
maxOutputTokens: z.number().int().min(500).optional()
|
|
480
|
+
});
|
|
478
481
|
const reevalConfigDataSchema = z.strictObject({
|
|
479
|
-
include: z.array(z.string()).optional()
|
|
480
|
-
harness: harnessList.optional()
|
|
481
|
-
sandbox: sandboxDescriptorSchema.optional()
|
|
482
|
-
environment: environmentDataSchema.optional()
|
|
483
|
-
runs:
|
|
484
|
-
gate: gatePolicySchema.optional()
|
|
485
|
-
env: envVarsSchema.optional()
|
|
486
|
-
egress: egressPolicySchema.optional()
|
|
487
|
-
judge: configJudgeDataSchema.optional()
|
|
488
|
-
timeout: durationSchema.optional()
|
|
489
|
-
capture: captureOptionsSchema.optional()
|
|
490
|
-
question: questionOptionsSchema.optional()
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
482
|
+
include: z.array(z.string()).optional(),
|
|
483
|
+
harness: harnessList.optional(),
|
|
484
|
+
sandbox: sandboxDescriptorSchema.optional(),
|
|
485
|
+
environment: environmentDataSchema.optional(),
|
|
486
|
+
runs: runs().optional(),
|
|
487
|
+
gate: gatePolicySchema.optional(),
|
|
488
|
+
env: envVarsSchema.optional(),
|
|
489
|
+
egress: egressPolicySchema.optional(),
|
|
490
|
+
judge: configJudgeDataSchema.optional(),
|
|
491
|
+
timeout: durationSchema.optional(),
|
|
492
|
+
capture: captureOptionsSchema.optional(),
|
|
493
|
+
question: questionOptionsSchema.optional(),
|
|
494
|
+
snapshot: snapshotOptionSchema.optional(),
|
|
495
|
+
concurrency: count().optional(),
|
|
496
|
+
record: recordMode.optional(),
|
|
497
|
+
history: historyConfigSchema.optional(),
|
|
498
|
+
budget: budgetSchema.optional(),
|
|
499
|
+
profiles: z.record(z.string(), profileDataSchema).optional(),
|
|
496
500
|
defaultProfile: z.strictObject({
|
|
497
501
|
ci: z.string().optional(),
|
|
498
502
|
local: z.string().optional()
|
|
499
|
-
}).optional()
|
|
500
|
-
secrets: z.strictObject({ unbrokered: z.enum(["warn", "error"]).optional() }).optional()
|
|
503
|
+
}).optional(),
|
|
504
|
+
secrets: z.strictObject({ unbrokered: z.enum(["warn", "error"]).optional() }).optional(),
|
|
501
505
|
debug: z.strictObject({
|
|
502
506
|
keep: count().optional(),
|
|
503
507
|
traceBufferMb: z.number().positive().optional()
|
|
504
|
-
}).optional()
|
|
505
|
-
analysis: analysisConfigDataSchema.optional()
|
|
506
|
-
gateway: z.array(gatewayEntryDataSchema).min(1).optional()
|
|
507
|
-
})
|
|
508
|
+
}).optional(),
|
|
509
|
+
analysis: analysisConfigDataSchema.optional(),
|
|
510
|
+
gateway: z.array(gatewayEntryDataSchema).min(1).optional()
|
|
511
|
+
});
|
|
508
512
|
/** The object schema an issue's path leads to through `schema`, for "Did you mean". */
|
|
509
513
|
function keysAt(schema, path, value) {
|
|
510
514
|
const def = schema.def;
|
|
@@ -573,6 +577,15 @@ function configDataProblem(value, file) {
|
|
|
573
577
|
}, v)}`;
|
|
574
578
|
}
|
|
575
579
|
//#endregion
|
|
580
|
+
//#region src/gateway/routes.ts
|
|
581
|
+
/** Every gateway that brokers a key, by name. */
|
|
582
|
+
const ROUTING_GATEWAYS = {
|
|
583
|
+
vercel: VERCEL_GATEWAY,
|
|
584
|
+
openrouter: OPENROUTER_GATEWAY
|
|
585
|
+
};
|
|
586
|
+
/** Whether a harness's `gateway` names one of {@link ROUTING_GATEWAYS}. */
|
|
587
|
+
const isRoutingGateway = (name) => name !== void 0 && Object.hasOwn(ROUTING_GATEWAYS, name);
|
|
588
|
+
//#endregion
|
|
576
589
|
//#region src/harness/agents.ts
|
|
577
590
|
const WEB_EVENTS = {
|
|
578
591
|
webSearch: "web-search",
|
|
@@ -710,12 +723,40 @@ function antigravityMcpServers(servers) {
|
|
|
710
723
|
}]));
|
|
711
724
|
}
|
|
712
725
|
/**
|
|
726
|
+
* A `{env:…}` or `{file:…}` OpenCode would substitute. It runs `\{env:[^}]+\}`, then
|
|
727
|
+
* `\{file:[^}]+\}`, over its whole config text with no escape, so a reference left open at the end
|
|
728
|
+
* of a value runs on into the JSON after it, and only `{env:}` and `{file:}` are left alone.
|
|
729
|
+
*/
|
|
730
|
+
const OPENCODE_REFERENCE = /\{(?:env|file):(?!\})/;
|
|
731
|
+
const fieldKey = (key) => /^[A-Za-z_$][\w$]*$/.test(key) ? `.${key}` : `[${JSON.stringify(key)}]`;
|
|
732
|
+
/**
|
|
733
|
+
* The first part of a server OpenCode would substitute, as what the server wouldn't get: its name,
|
|
734
|
+
* a value as `mcp.<name>.<field>`, or an env or header name. Names are in the same config text as
|
|
735
|
+
* values, so they're substituted too; a secret's value is never named.
|
|
736
|
+
*/
|
|
737
|
+
function openCodeReference(name, s) {
|
|
738
|
+
if (OPENCODE_REFERENCE.test(name)) return "its name";
|
|
739
|
+
const server = `mcp${fieldKey(name)}`;
|
|
740
|
+
const [map, kind, values] = s.transport === "stdio" ? [
|
|
741
|
+
"env",
|
|
742
|
+
"env",
|
|
743
|
+
s.env ?? {}
|
|
744
|
+
] : [
|
|
745
|
+
"headers",
|
|
746
|
+
"header",
|
|
747
|
+
s.headers ?? {}
|
|
748
|
+
];
|
|
749
|
+
return [...s.transport === "stdio" ? [[`${server}.command`, s.command], ...s.args.map((a, i) => [`${server}.args[${i}]`, a])] : [[`${server}.url`, s.url]], ...Object.entries(values).flatMap(([k, v]) => [[`the ${kind} name ${k} in ${server}.${map}`, k], [`${server}.${map}${fieldKey(k)}`, v]])].find(([, text]) => OPENCODE_REFERENCE.test(text))?.[0];
|
|
750
|
+
}
|
|
751
|
+
/**
|
|
713
752
|
* OpenCode's config shape (its `mcp.servers`), which the adapter sets `codemode: false` on.
|
|
714
|
-
* OpenCode has no SSE transport.
|
|
753
|
+
* OpenCode has no SSE transport, and refuses a value it would substitute.
|
|
715
754
|
*/
|
|
716
755
|
function opencodeMcpServers(servers) {
|
|
717
756
|
return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
|
|
718
757
|
if (s.transport === "sse") throw new InfraError(`OpenCode has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
|
|
758
|
+
const field = openCodeReference(name, s);
|
|
759
|
+
if (field) throw new InfraError(`OpenCode substitutes {env:…} and {file:…} anywhere in its config, with no escape, so the ${name} server wouldn't get ${field} as written: take the reference out, or leave OpenCode out of this cell`);
|
|
719
760
|
return s.transport === "stdio" ? [name, {
|
|
720
761
|
type: "local",
|
|
721
762
|
command: [s.command, ...s.args],
|
|
@@ -975,7 +1016,7 @@ async function writePluginFiles(sandbox, layout, plugins) {
|
|
|
975
1016
|
if (existing?.trim()) {
|
|
976
1017
|
let base;
|
|
977
1018
|
try {
|
|
978
|
-
base = JSON.parse(existing);
|
|
1019
|
+
base = JSON.parse(withoutBom(existing));
|
|
979
1020
|
} catch {}
|
|
980
1021
|
if (!isObject(base)) throw new InfraError(`${f.path} isn't a JSON object, so reeval can't enable the session's plugins in it: write it as one in files, or leave it out`);
|
|
981
1022
|
const ours = JSON.parse(f.content);
|
|
@@ -1087,7 +1128,7 @@ const AGENTS = {
|
|
|
1087
1128
|
},
|
|
1088
1129
|
unreported: cursorUnreported,
|
|
1089
1130
|
mcpServers: cursorMcpServers,
|
|
1090
|
-
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-
|
|
1131
|
+
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-DMo2eCOJ.js")).createCursorSdk({
|
|
1091
1132
|
auth,
|
|
1092
1133
|
...mcpServers ? { mcpServers } : {},
|
|
1093
1134
|
...descriptor.kind === "cursor" && descriptor.params ? { modelParams: descriptor.params } : {},
|
|
@@ -1124,13 +1165,14 @@ const AGENTS = {
|
|
|
1124
1165
|
}
|
|
1125
1166
|
],
|
|
1126
1167
|
mcpServers: claudeMcpServers,
|
|
1127
|
-
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./dist-CYrBwyHA.js")).createClaudeCode({
|
|
1168
|
+
adapter: async ({ auth, env, mcpServers, descriptor }) => (await import("./dist-CYrBwyHA.js")).createClaudeCode({
|
|
1128
1169
|
auth,
|
|
1129
1170
|
...mcpServers ? { mcpServers } : {},
|
|
1130
1171
|
...descriptor.kind === "claude-code" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1131
1172
|
env: {
|
|
1132
1173
|
IS_SANDBOX: "1",
|
|
1133
|
-
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1"
|
|
1174
|
+
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",
|
|
1175
|
+
...env
|
|
1134
1176
|
}
|
|
1135
1177
|
})
|
|
1136
1178
|
},
|
|
@@ -1153,8 +1195,9 @@ const AGENTS = {
|
|
|
1153
1195
|
},
|
|
1154
1196
|
servedModel: geminiServedModel,
|
|
1155
1197
|
mcpServers: antigravityMcpServers,
|
|
1156
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-
|
|
1198
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-OESRtvOi.js")).createAntigravity({
|
|
1157
1199
|
auth,
|
|
1200
|
+
...vertex ? { vertex } : {},
|
|
1158
1201
|
...mcpServers ? { mcpServers } : {},
|
|
1159
1202
|
...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
|
|
1160
1203
|
name,
|
|
@@ -1179,10 +1222,10 @@ const AGENTS = {
|
|
|
1179
1222
|
servedModel: geminiServedModel,
|
|
1180
1223
|
forModel: opencodeForModel,
|
|
1181
1224
|
mcpServers: opencodeMcpServers,
|
|
1182
|
-
adapter: async ({ auth, mcpServers, descriptor, model }) => (await import("./src-
|
|
1225
|
+
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-CgJrz5p0.js")).createOpenCode({
|
|
1183
1226
|
auth,
|
|
1184
1227
|
model,
|
|
1185
|
-
vars: opencodeVars(model),
|
|
1228
|
+
...vertex ? { vertex } : { vars: opencodeVars(model) },
|
|
1186
1229
|
...descriptor.kind === "opencode" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1187
1230
|
...mcpServers ? { mcpServers } : {},
|
|
1188
1231
|
requireCredentialBrokering: true
|
|
@@ -1197,14 +1240,18 @@ const AGENTS = {
|
|
|
1197
1240
|
endpoints: GOOSE_ENDPOINTS,
|
|
1198
1241
|
expectedBlocked: {
|
|
1199
1242
|
hosts: ["models.dev"],
|
|
1200
|
-
why: "Goose's model
|
|
1243
|
+
why: "Goose's model catalog refresh, which no setting turns off"
|
|
1201
1244
|
},
|
|
1202
1245
|
servedModel: geminiServedModel,
|
|
1203
1246
|
mcpServers: gooseMcpServers,
|
|
1204
|
-
adapter: async ({ auth, mcpServers, descriptor, model }) => (await import("./src-
|
|
1247
|
+
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-BTqJbvj6.js")).createGoose({
|
|
1248
|
+
...vertex ? { vertex } : {},
|
|
1205
1249
|
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1206
1250
|
auth,
|
|
1207
|
-
provider: (await import("./src-
|
|
1251
|
+
provider: (await import("./src-BTqJbvj6.js")).AI_GATEWAY_PROVIDER
|
|
1252
|
+
} : auth["OPENROUTER_API_KEY"] ? {
|
|
1253
|
+
auth,
|
|
1254
|
+
provider: (await import("./src-BTqJbvj6.js")).OPENROUTER_PROVIDER
|
|
1208
1255
|
} : { auth: { GOOGLE_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" } },
|
|
1209
1256
|
model,
|
|
1210
1257
|
...descriptor.kind === "goose" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
@@ -1225,8 +1272,17 @@ const AGENTS = {
|
|
|
1225
1272
|
endpoints: GEMINI_ENDPOINTS,
|
|
1226
1273
|
servedModel: geminiServedModel,
|
|
1227
1274
|
mcpServers: piMcpServers,
|
|
1228
|
-
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-
|
|
1229
|
-
auth
|
|
1275
|
+
adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-BkiComGh.js")).createPi({
|
|
1276
|
+
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1277
|
+
auth,
|
|
1278
|
+
gateway: (await import("./src-BkiComGh.js")).PI_AI_GATEWAY
|
|
1279
|
+
} : auth["OPENROUTER_API_KEY"] ? {
|
|
1280
|
+
auth,
|
|
1281
|
+
gateway: (await import("./src-BkiComGh.js")).PI_OPENROUTER
|
|
1282
|
+
} : {
|
|
1283
|
+
auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
|
|
1284
|
+
...vertex ? { vertex } : {}
|
|
1285
|
+
},
|
|
1230
1286
|
...descriptor.kind === "pi" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1231
1287
|
...mcpServers ? { mcpServers } : {},
|
|
1232
1288
|
requireCredentialBrokering: true
|
|
@@ -1255,9 +1311,9 @@ const AGENTS = {
|
|
|
1255
1311
|
},
|
|
1256
1312
|
servedModel: openaiServedModel,
|
|
1257
1313
|
mcpServers: codexMcpServers,
|
|
1258
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-
|
|
1314
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-BvgeZ4tS.js")).createCodex({
|
|
1259
1315
|
auth,
|
|
1260
|
-
...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-
|
|
1316
|
+
...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-BvgeZ4tS.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-BvgeZ4tS.js")).OPENROUTER_PROVIDER } : {},
|
|
1261
1317
|
...descriptor.kind === "codex" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1262
1318
|
...mcpServers ? { mcpServers } : {},
|
|
1263
1319
|
...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
|
|
@@ -1275,9 +1331,15 @@ const AGENTS = {
|
|
|
1275
1331
|
...fxForModel("provider/model"),
|
|
1276
1332
|
forModel: fxForModel,
|
|
1277
1333
|
mcpServers: acpMcpServers,
|
|
1278
|
-
adapter: async ({ auth, mcpServers, model }) => (await import("./src-
|
|
1334
|
+
adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-Dk2Na0b-.js")).createFx({
|
|
1279
1335
|
auth,
|
|
1280
1336
|
model,
|
|
1337
|
+
...auth["OPENROUTER_API_KEY"] ? { provider: "openrouter" } : {},
|
|
1338
|
+
...env?.FX_VERTEX_PROJECT ? { vertex: {
|
|
1339
|
+
project: env.FX_VERTEX_PROJECT,
|
|
1340
|
+
location: env.FX_VERTEX_LOCATION ?? ""
|
|
1341
|
+
} } : {},
|
|
1342
|
+
...descriptor.kind === "fx" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1281
1343
|
...mcpServers ? { mcpServers } : {},
|
|
1282
1344
|
requireCredentialBrokering: true
|
|
1283
1345
|
})
|
|
@@ -1293,18 +1355,26 @@ const AGENTS = {
|
|
|
1293
1355
|
modelResponses: XAI_MODEL_RESPONSES,
|
|
1294
1356
|
servedModel: xaiServedModel,
|
|
1295
1357
|
mcpServers: acpMcpServers,
|
|
1296
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins, model }) =>
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1358
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins, model, vertexChat }) => {
|
|
1359
|
+
const grok = await import("./src-Cp-2Hdib.js");
|
|
1360
|
+
return grok.createGrokBuild({
|
|
1361
|
+
...vertexChat ? {
|
|
1362
|
+
auth: { [grok.VERTEX_TOKEN_ENV]: SANDBOX_ACCESS_TOKEN },
|
|
1363
|
+
provider: grok.vertexProvider(vertexChat.project, vertexChat.region)
|
|
1364
|
+
} : {
|
|
1365
|
+
auth,
|
|
1366
|
+
...auth["AI_GATEWAY_API_KEY"] ? { provider: grok.AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: grok.OPENROUTER_PROVIDER } : {}
|
|
1367
|
+
},
|
|
1368
|
+
model,
|
|
1369
|
+
...plugins.length ? {
|
|
1370
|
+
pluginDirs: plugins.map((p) => p.dir),
|
|
1371
|
+
pluginMcpServers: plugins.flatMap((p) => p.servers.map((s) => s.slice(p.name.length + 1)))
|
|
1372
|
+
} : {},
|
|
1373
|
+
...descriptor.kind === "grok-build" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1374
|
+
...mcpServers ? { mcpServers } : {},
|
|
1375
|
+
requireCredentialBrokering: true
|
|
1376
|
+
});
|
|
1377
|
+
}
|
|
1308
1378
|
}
|
|
1309
1379
|
};
|
|
1310
1380
|
/** The harness of a config that names none: Cursor, on Cursor's own model. */
|
|
@@ -1316,20 +1386,31 @@ const DEFAULT_HARNESS = {
|
|
|
1316
1386
|
const HARNESS_EXAMPLE = "cursor({ model: 'composer-2.5' })";
|
|
1317
1387
|
const isAgentKind = (kind) => typeof kind === "string" && Object.hasOwn(AGENTS, kind);
|
|
1318
1388
|
/**
|
|
1389
|
+
* A server as the plan knows it: its names, literal values and each `secret.template`'s text. A
|
|
1390
|
+
* host secret's value is read when the attempt starts, so the plan has it as `''`.
|
|
1391
|
+
*/
|
|
1392
|
+
function plannedMcpServer(s) {
|
|
1393
|
+
const literal = (values) => Object.fromEntries(Object.entries(values).map(([k, v]) => [k, typeof v === "string" ? v : v.kind === "secret-template" ? v.template : ""]));
|
|
1394
|
+
return s.transport === "stdio" ? {
|
|
1395
|
+
transport: "stdio",
|
|
1396
|
+
command: s.command,
|
|
1397
|
+
args: s.args,
|
|
1398
|
+
env: literal(s.env)
|
|
1399
|
+
} : {
|
|
1400
|
+
transport: s.transport,
|
|
1401
|
+
url: s.url,
|
|
1402
|
+
headers: literal(s.headers)
|
|
1403
|
+
};
|
|
1404
|
+
}
|
|
1405
|
+
/**
|
|
1319
1406
|
* Why a harness's agent can't take the environment's MCP servers (Goose, OpenCode, pi and Codex
|
|
1320
|
-
* take no `mcp.sse
|
|
1321
|
-
* starts; undefined when it can.
|
|
1407
|
+
* take no `mcp.sse`; OpenCode would substitute a `{env:…}` or `{file:…}` in a name or value), in the
|
|
1408
|
+
* agent's own words, so the plan refuses them before any attempt starts; undefined when it can.
|
|
1322
1409
|
*/
|
|
1323
1410
|
function refusedMcp(harness, layers) {
|
|
1324
1411
|
if (!isAgentKind(harness.kind)) return void 0;
|
|
1325
1412
|
const servers = /* @__PURE__ */ new Map();
|
|
1326
|
-
for (const { spec } of layers) for (const [name, server] of Object.entries(spec.mcp ?? {})) servers.set(name, server
|
|
1327
|
-
...server,
|
|
1328
|
-
env: {}
|
|
1329
|
-
} : {
|
|
1330
|
-
...server,
|
|
1331
|
-
headers: {}
|
|
1332
|
-
});
|
|
1413
|
+
for (const { spec } of layers) for (const [name, server] of Object.entries(spec.mcp ?? {})) servers.set(name, plannedMcpServer(server));
|
|
1333
1414
|
if (servers.size === 0) return void 0;
|
|
1334
1415
|
try {
|
|
1335
1416
|
AGENTS[harness.kind].mcpServers(Object.fromEntries(servers));
|
|
@@ -1392,13 +1473,7 @@ function refusedPlugins(harness, plugins, own, unread = []) {
|
|
|
1392
1473
|
const fix = rest.length ? `use only: [${quoted(rest)}], or leave` : "leave";
|
|
1393
1474
|
return event ? `${name} can't run the ${p.name} plugin's ${event} hooks (hooks/hooks.json): it runs ${runs?.join(", ")} hooks; ${fix} the plugin out of this cell` : `${name} can't run the ${p.name} plugin's ${type} hooks (hooks/hooks.json): it runs only command hooks; ${fix} the plugin out of this cell`;
|
|
1394
1475
|
}
|
|
1395
|
-
const servers = Object.fromEntries(plugins.flatMap((p) => Object.entries(p.mcp).map(([server, s]) => [pluginServerName(p.name, server), s
|
|
1396
|
-
...s,
|
|
1397
|
-
env: {}
|
|
1398
|
-
} : {
|
|
1399
|
-
...s,
|
|
1400
|
-
headers: {}
|
|
1401
|
-
}])));
|
|
1476
|
+
const servers = Object.fromEntries(plugins.flatMap((p) => Object.entries(p.mcp).map(([server, s]) => [pluginServerName(p.name, server), plannedMcpServer(s)])));
|
|
1402
1477
|
if (agent && Object.keys(servers).length) {
|
|
1403
1478
|
const alongside = agent.pluginPlacement === "flat" ? Object.fromEntries(own.mcp.map((n) => [n, {
|
|
1404
1479
|
transport: "stdio",
|
|
@@ -1472,12 +1547,25 @@ function refusedPlugins(harness, plugins, own, unread = []) {
|
|
|
1472
1547
|
ruleOwner.set(file, what);
|
|
1473
1548
|
}
|
|
1474
1549
|
}
|
|
1475
|
-
/**
|
|
1476
|
-
|
|
1550
|
+
/**
|
|
1551
|
+
* A real agent's catalog entry, with what its model's provider decides, or its gateway's route:
|
|
1552
|
+
* on Vertex, the attempt's `vertex()` settings decide the project and region.
|
|
1553
|
+
*/
|
|
1554
|
+
function agentFor(h, vertex) {
|
|
1477
1555
|
const agent = AGENTS[h.kind];
|
|
1478
|
-
if (h.gateway === "
|
|
1479
|
-
const route =
|
|
1480
|
-
if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on
|
|
1556
|
+
if (h.gateway === "vertex") {
|
|
1557
|
+
const route = VERTEX_ROUTES[h.kind];
|
|
1558
|
+
if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on Vertex AI`);
|
|
1559
|
+
if (!vertex) throw new InfraError(`${agent.name} on Vertex AI has no vertex() settings`);
|
|
1560
|
+
return {
|
|
1561
|
+
...agent,
|
|
1562
|
+
...route(vertex)
|
|
1563
|
+
};
|
|
1564
|
+
}
|
|
1565
|
+
if (isRoutingGateway(h.gateway)) {
|
|
1566
|
+
const gateway = ROUTING_GATEWAYS[h.gateway];
|
|
1567
|
+
const route = gateway.routes[h.kind];
|
|
1568
|
+
if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on ${gateway.name}`);
|
|
1481
1569
|
return {
|
|
1482
1570
|
...agent,
|
|
1483
1571
|
...route(process.env)
|
|
@@ -1489,9 +1577,12 @@ function agentFor(h) {
|
|
|
1489
1577
|
} : agent;
|
|
1490
1578
|
}
|
|
1491
1579
|
/** The catalog entry for a harness's agent; undefined for the fake harness. */
|
|
1492
|
-
const agentOf = (h) => h.kind === "fake" ? void 0 : agentFor(h);
|
|
1493
|
-
/**
|
|
1494
|
-
|
|
1580
|
+
const agentOf = (h, vertex) => h.kind === "fake" ? void 0 : agentFor(h, vertex);
|
|
1581
|
+
/**
|
|
1582
|
+
* The host variable a harness authenticates with; undefined for the fake harness, and on Vertex,
|
|
1583
|
+
* whose token comes from Application Default Credentials.
|
|
1584
|
+
*/
|
|
1585
|
+
const credentialOf = (h) => h.kind === "fake" || h.gateway === "vertex" ? void 0 : agentFor(h).credential;
|
|
1495
1586
|
/** The fake agent's own layout. */
|
|
1496
1587
|
const FAKE_LAYOUT = {
|
|
1497
1588
|
rulesFile: "AGENTS.md",
|
|
@@ -1506,10 +1597,10 @@ function layoutOf(descriptor) {
|
|
|
1506
1597
|
}
|
|
1507
1598
|
/**
|
|
1508
1599
|
* The hosts a harness's agent is known to try in a session with or without plugins; a fake agent
|
|
1509
|
-
* with `layout` expects that agent's.
|
|
1600
|
+
* with `layout` expects that agent's. On Vertex, the attempt's `vertex()` settings pick the route.
|
|
1510
1601
|
*/
|
|
1511
|
-
function expectedBlockedFor(descriptor, plugins) {
|
|
1512
|
-
const expected = descriptor.kind !== "fake" ? agentFor(descriptor).expectedBlocked : descriptor.layout ? AGENTS[descriptor.layout].expectedBlocked : void 0;
|
|
1602
|
+
function expectedBlockedFor(descriptor, plugins, vertex) {
|
|
1603
|
+
const expected = descriptor.kind !== "fake" ? agentFor(descriptor, vertex).expectedBlocked : descriptor.layout ? AGENTS[descriptor.layout].expectedBlocked : void 0;
|
|
1513
1604
|
return expected?.withPlugins && !plugins ? void 0 : expected;
|
|
1514
1605
|
}
|
|
1515
1606
|
/**
|
|
@@ -1527,18 +1618,27 @@ const REFUSED_KEY = /\bHTTP 401\b|\b401\b.{0,40}\bunauthori[sz]ed\b|\bunauthori[
|
|
|
1527
1618
|
*/
|
|
1528
1619
|
function authHint(agent, message, refused = false) {
|
|
1529
1620
|
if (!refused && !REFUSED_KEY.test(message)) return void 0;
|
|
1621
|
+
if (agent.token) return `The API rejected the access token reeval minted from ${agent.credential}: check gcloud auth application-default login (or GOOGLE_APPLICATION_CREDENTIALS) and the project's Vertex AI access`;
|
|
1530
1622
|
return `The API rejected ${agent.credential}: set a valid key in the environment or .env.local`;
|
|
1531
1623
|
}
|
|
1532
|
-
/**
|
|
1624
|
+
/**
|
|
1625
|
+
* The agent's credential from the host, as the adapter's explicit `auth`. Fails closed. Empty for
|
|
1626
|
+
* a brokered token, which the adapter never sees.
|
|
1627
|
+
*/
|
|
1533
1628
|
function agentAuth(agent, env = process.env) {
|
|
1629
|
+
if (agent.token) return {};
|
|
1534
1630
|
const value = env[agent.credential];
|
|
1535
1631
|
if (!value) throw new InfraError(`${agent.credential} isn't set, so the ${agent.name} harness can't authenticate. Set it in the environment or .env.local.`);
|
|
1536
|
-
return {
|
|
1632
|
+
return {
|
|
1633
|
+
...agent.authEnv,
|
|
1634
|
+
[agent.authAs ?? agent.credential]: value
|
|
1635
|
+
};
|
|
1537
1636
|
}
|
|
1538
|
-
function brokeringFor(h) {
|
|
1539
|
-
const agent = agentOf(h);
|
|
1637
|
+
function brokeringFor(h, vertex) {
|
|
1638
|
+
const agent = agentOf(h, vertex);
|
|
1540
1639
|
if (!agent) return void 0;
|
|
1541
|
-
const { credential, endpoints, tokens, servedModel, askedModel, modelResponses } = agent;
|
|
1640
|
+
const { credential, endpoints, tokens, servedModel, askedModel, modelResponses, upstream } = agent;
|
|
1641
|
+
const { localResponses } = agent;
|
|
1542
1642
|
return {
|
|
1543
1643
|
credential,
|
|
1544
1644
|
endpoints,
|
|
@@ -1548,7 +1648,9 @@ function brokeringFor(h) {
|
|
|
1548
1648
|
} } : {},
|
|
1549
1649
|
...servedModel ? { servedModel } : {},
|
|
1550
1650
|
...askedModel ? { askedModel } : {},
|
|
1551
|
-
...modelResponses ? { modelResponses } : {}
|
|
1651
|
+
...modelResponses ? { modelResponses } : {},
|
|
1652
|
+
...upstream ? { upstream } : {},
|
|
1653
|
+
...localResponses?.length ? { localResponses } : {}
|
|
1552
1654
|
};
|
|
1553
1655
|
}
|
|
1554
1656
|
//#endregion
|
|
@@ -1605,11 +1707,10 @@ function resolvedEffortOf(models) {
|
|
|
1605
1707
|
/** The error a per-attempt answer carries when the run stopped before a model answered it. */
|
|
1606
1708
|
const STOPPED_FIRST = "the run stopped before it was answered";
|
|
1607
1709
|
/**
|
|
1608
|
-
* What
|
|
1609
|
-
* turn's prompt.
|
|
1610
|
-
* win a side-by-side pick on its wording; the changed files repeat what the diff shows.
|
|
1710
|
+
* What a recorded pairwise answer without `stateOptions` showed each side: the task and the diff,
|
|
1711
|
+
* every turn's prompt.
|
|
1611
1712
|
*/
|
|
1612
|
-
const
|
|
1713
|
+
const UNRECORDED_QUESTION_STATE = {
|
|
1613
1714
|
task: true,
|
|
1614
1715
|
output: false,
|
|
1615
1716
|
files: false,
|
|
@@ -1618,62 +1719,39 @@ const DEFAULT_QUESTION_STATE = {
|
|
|
1618
1719
|
turns: "all",
|
|
1619
1720
|
checks: false
|
|
1620
1721
|
};
|
|
1722
|
+
const KEYS = Object.keys(UNRECORDED_QUESTION_STATE);
|
|
1621
1723
|
/**
|
|
1622
|
-
*
|
|
1623
|
-
*
|
|
1624
|
-
* agent went about the work is the point.
|
|
1724
|
+
* Every key set, in the record's key order, so equal selections serialize the same: an input the
|
|
1725
|
+
* question didn't turn on is off, and `turns` is `'all'` unless it says `'last'`.
|
|
1625
1726
|
*/
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
const defaults = defaultQuestionState(pairwise);
|
|
1637
|
-
const out = {};
|
|
1638
|
-
for (const key of KEYS) out[key] = state?.[key] ?? defaults[key];
|
|
1639
|
-
return out;
|
|
1727
|
+
function resolveQuestionState(state) {
|
|
1728
|
+
return {
|
|
1729
|
+
task: state?.task ?? false,
|
|
1730
|
+
output: state?.output ?? false,
|
|
1731
|
+
files: state?.files ?? false,
|
|
1732
|
+
diff: state?.diff ?? false,
|
|
1733
|
+
transcript: state?.transcript ?? false,
|
|
1734
|
+
turns: state?.turns ?? "all",
|
|
1735
|
+
checks: state?.checks ?? false
|
|
1736
|
+
};
|
|
1640
1737
|
}
|
|
1641
|
-
/** `inner` over `outer
|
|
1738
|
+
/** `inner` over `outer`, field by field. */
|
|
1642
1739
|
function mergeQuestion(outer, inner) {
|
|
1643
|
-
|
|
1644
|
-
const out = {
|
|
1740
|
+
return inner ? {
|
|
1645
1741
|
...outer,
|
|
1646
1742
|
...inner
|
|
1647
|
-
};
|
|
1648
|
-
if (outer.state || inner.state) out.state = {
|
|
1649
|
-
...outer.state,
|
|
1650
|
-
...inner.state
|
|
1651
|
-
};
|
|
1652
|
-
return out;
|
|
1743
|
+
} : outer;
|
|
1653
1744
|
}
|
|
1654
|
-
/** A planned question
|
|
1745
|
+
/** A planned question under the cascaded `question` defaults: its own `model`, else theirs. */
|
|
1655
1746
|
function withQuestionDefaults(q, defaults) {
|
|
1656
|
-
const { state, artifacts } = mergeQuestion(defaults, {
|
|
1657
|
-
...q.state ? { state: q.state } : {},
|
|
1658
|
-
...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
|
|
1659
|
-
});
|
|
1660
|
-
const { state: _state, artifacts: _artifacts, ...rest } = q;
|
|
1661
1747
|
const model = q.model ?? defaults.model;
|
|
1662
|
-
return {
|
|
1663
|
-
...
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
...artifacts !== void 0 ? { artifacts } : {}
|
|
1667
|
-
};
|
|
1748
|
+
return model !== void 0 ? {
|
|
1749
|
+
...q,
|
|
1750
|
+
model
|
|
1751
|
+
} : q;
|
|
1668
1752
|
}
|
|
1669
1753
|
const isOff = (value) => value === false || Array.isArray(value) && !value.length;
|
|
1670
|
-
/**
|
|
1671
|
-
const showsNothing = (q) => {
|
|
1672
|
-
if (q.artifacts !== void 0) return false;
|
|
1673
|
-
const s = resolveQuestionState(q.state, q.across !== void 0);
|
|
1674
|
-
return KEYS.every((k) => k === "turns" || isOff(s[k]));
|
|
1675
|
-
};
|
|
1676
|
-
/** The state keys a question shows, in order, plus `artifacts` when it sends captures. */
|
|
1754
|
+
/** The inputs a question shows, in order, plus `artifacts` when it sends captures. */
|
|
1677
1755
|
function questionShows(state, artifacts) {
|
|
1678
1756
|
const on = KEYS.filter((k) => k !== "turns" && !isOff(state[k]));
|
|
1679
1757
|
return artifacts === void 0 ? on : [...on, "artifacts"];
|
|
@@ -1739,6 +1817,189 @@ function compareProblem(value, where, environment) {
|
|
|
1739
1817
|
if (!isObjectLike(value) || !Array.isArray(value.variants)) return `compare in ${where} takes { variants, baseline? } (got ${describe$1(value)}). Use compare: { variants: [a, b], baseline?, gate?, minLift?, harness? }.`;
|
|
1740
1818
|
return refusal(compareSchema(environment), value, explain(where), { order: ORDER });
|
|
1741
1819
|
}
|
|
1820
|
+
const POLL_MS = 200;
|
|
1821
|
+
/**
|
|
1822
|
+
* Prints `alive` while `pid` runs, then the sandbox's TCP tables, or `noproc` without them
|
|
1823
|
+
* (or without `tables`: the fake sandbox shares the host's, where other processes bind too).
|
|
1824
|
+
*/
|
|
1825
|
+
function probeScript(pid, tables = true) {
|
|
1826
|
+
return `${pid ? `kill -0 ${pid} 2>/dev/null && echo alive; ` : ""}${tables ? "if [ -r /proc/net/tcp ]; then cat /proc/net/tcp /proc/net/tcp6 2>/dev/null; else echo noproc; fi" : "echo noproc"}; exit 0`;
|
|
1827
|
+
}
|
|
1828
|
+
/** `0100007F` → `127.0.0.1`; 32 hex digits → an IPv6 address. Each 32-bit word is host-endian. */
|
|
1829
|
+
function procAddress(hex) {
|
|
1830
|
+
const bytes = (hex.match(/.{8}/g) ?? []).flatMap((w) => (w.match(/../g) ?? []).reverse().map((b) => Number.parseInt(b, 16)));
|
|
1831
|
+
if (bytes.length === 4) return bytes.join(".");
|
|
1832
|
+
if (bytes.slice(0, 10).every((b) => b === 0) && bytes[10] === 255 && bytes[11] === 255) return bytes.slice(12).join(".");
|
|
1833
|
+
const groups = [];
|
|
1834
|
+
for (let i = 0; i < 16; i += 2) groups.push(((bytes[i] ?? 0) << 8 | (bytes[i + 1] ?? 0)).toString(16));
|
|
1835
|
+
const full = groups.join(":");
|
|
1836
|
+
const longest = (full.match(/(^|:)0(:0)+(:|$)/g) ?? []).reduce((a, b) => b.length > a.length ? b : a, "");
|
|
1837
|
+
return longest ? full.replace(longest, "::") : full;
|
|
1838
|
+
}
|
|
1839
|
+
/** A listener the tunnel reaches: on loopback or every address, IPv4 or IPv6. */
|
|
1840
|
+
const reachable = (address) => address === "0.0.0.0" || address === "::" || address === "::1" || address.startsWith("127.");
|
|
1841
|
+
function parseProbe(stdout) {
|
|
1842
|
+
const lines = stdout.split("\n");
|
|
1843
|
+
const alive = lines.includes("alive");
|
|
1844
|
+
if (lines.includes("noproc")) return {
|
|
1845
|
+
proc: false,
|
|
1846
|
+
alive
|
|
1847
|
+
};
|
|
1848
|
+
const listeners = [];
|
|
1849
|
+
for (const l of lines) {
|
|
1850
|
+
const m = /^\s*\d+:\s+([0-9A-F]+):([0-9A-F]{4})\s+\S+\s+0A\s/i.exec(l);
|
|
1851
|
+
const address = m ? procAddress(m[1]) : "";
|
|
1852
|
+
if (m && reachable(address)) listeners.push({
|
|
1853
|
+
port: Number.parseInt(m[2], 16),
|
|
1854
|
+
address
|
|
1855
|
+
});
|
|
1856
|
+
}
|
|
1857
|
+
return {
|
|
1858
|
+
proc: true,
|
|
1859
|
+
alive,
|
|
1860
|
+
listeners
|
|
1861
|
+
};
|
|
1862
|
+
}
|
|
1863
|
+
/** The ports listening now that weren't before the command started, ascending. */
|
|
1864
|
+
const newPorts = (listeners, before) => [...new Set(listeners.map((l) => l.port).filter((p) => !before.has(p)))].sort((a, b) => a - b);
|
|
1865
|
+
/** The address to record for `port`: IPv4 first, since that's where the tunnel connects first. */
|
|
1866
|
+
const addressFor = (listeners, port) => {
|
|
1867
|
+
const all = listeners.filter((l) => l.port === port).map((l) => l.address);
|
|
1868
|
+
return all.find((a) => !a.includes(":")) ?? all.find((a) => a === "::") ?? all[0] ?? "";
|
|
1869
|
+
};
|
|
1870
|
+
/** `want` is taken whoever listens on it: reeval allocated it, or the eval named it. */
|
|
1871
|
+
function decide(s) {
|
|
1872
|
+
const { listeners, alive } = s.probe;
|
|
1873
|
+
const ports = newPorts(listeners, s.before);
|
|
1874
|
+
if (listeners.some((l) => l.port === s.want)) return {
|
|
1875
|
+
kind: "port",
|
|
1876
|
+
port: s.want,
|
|
1877
|
+
address: addressFor(listeners, s.want)
|
|
1878
|
+
};
|
|
1879
|
+
const late = s.elapsedMs >= s.timeoutMs;
|
|
1880
|
+
if (!s.given && ports.length && (s.stableMs >= 3e3 || late || !alive)) {
|
|
1881
|
+
if (ports.length === 1) {
|
|
1882
|
+
const port = ports[0];
|
|
1883
|
+
return {
|
|
1884
|
+
kind: "port",
|
|
1885
|
+
port,
|
|
1886
|
+
address: addressFor(listeners, port)
|
|
1887
|
+
};
|
|
1888
|
+
}
|
|
1889
|
+
return {
|
|
1890
|
+
kind: "several",
|
|
1891
|
+
ports
|
|
1892
|
+
};
|
|
1893
|
+
}
|
|
1894
|
+
if (!alive) return { kind: "exited" };
|
|
1895
|
+
return late ? {
|
|
1896
|
+
kind: "timeout",
|
|
1897
|
+
ports
|
|
1898
|
+
} : { kind: "wait" };
|
|
1899
|
+
}
|
|
1900
|
+
const plain = (output) => output.replace(/\x1b\[[0-9;?]*[A-Za-z]/g, "");
|
|
1901
|
+
/** The URLs `output` prints on one of `ports`, as Vite's `Local: http://localhost:5173/`. */
|
|
1902
|
+
function printedUrls(output, ports) {
|
|
1903
|
+
const found = /* @__PURE__ */ new Map();
|
|
1904
|
+
for (const m of plain(output).matchAll(/https?:\/\/(?:\[[0-9a-f:]+\]|[^\s/:[\]]+):(\d+)[^\s'"]*/gi)) {
|
|
1905
|
+
const port = Number(m[1]);
|
|
1906
|
+
if (ports.includes(port) && !found.has(m[0])) found.set(m[0], port);
|
|
1907
|
+
}
|
|
1908
|
+
return [...found].map(([url, port]) => ({
|
|
1909
|
+
url,
|
|
1910
|
+
port
|
|
1911
|
+
}));
|
|
1912
|
+
}
|
|
1913
|
+
/** Whether something on the host accepts a TCP connection on `port`, IPv4 or IPv6 loopback. */
|
|
1914
|
+
async function hostListening(port) {
|
|
1915
|
+
for (const host of ["127.0.0.1", "::1"]) if (await new Promise((resolve) => {
|
|
1916
|
+
const socket = connect({
|
|
1917
|
+
port,
|
|
1918
|
+
host
|
|
1919
|
+
});
|
|
1920
|
+
socket.once("connect", () => {
|
|
1921
|
+
socket.destroy();
|
|
1922
|
+
resolve(true);
|
|
1923
|
+
});
|
|
1924
|
+
socket.once("error", () => resolve(false));
|
|
1925
|
+
})) return host;
|
|
1926
|
+
}
|
|
1927
|
+
const sleep = (ms, signal) => new Promise((resolve) => {
|
|
1928
|
+
const done = () => {
|
|
1929
|
+
clearTimeout(timer);
|
|
1930
|
+
signal?.removeEventListener("abort", done);
|
|
1931
|
+
resolve();
|
|
1932
|
+
};
|
|
1933
|
+
const timer = setTimeout(done, ms);
|
|
1934
|
+
signal?.addEventListener("abort", done, { once: true });
|
|
1935
|
+
});
|
|
1936
|
+
const listOf = (items) => items.length > 1 ? `${items.slice(0, -1).join(", ")} and ${items.at(-1)}` : String(items[0]);
|
|
1937
|
+
/**
|
|
1938
|
+
* Polls until something listens on `want`, or, without a given port, until the ports that are
|
|
1939
|
+
* new since the command started have settled on one. Throws, with the command's last output,
|
|
1940
|
+
* when they settle on several, it exits, or nothing listens in time.
|
|
1941
|
+
*/
|
|
1942
|
+
async function awaitListening(command, w) {
|
|
1943
|
+
const timeoutMs = w.timeoutMs ?? 12e4;
|
|
1944
|
+
const started = Date.now();
|
|
1945
|
+
let last = "";
|
|
1946
|
+
let changed = started;
|
|
1947
|
+
const output = async () => plain(await w.output().catch(() => ""));
|
|
1948
|
+
const fail = async (why, out) => {
|
|
1949
|
+
const tail = (out ?? await output()).trimEnd().split("\n").slice(-20).join("\n").trim();
|
|
1950
|
+
return /* @__PURE__ */ new Error(`c.serve(${JSON.stringify(command)}) ${why}${tail ? `. Its last output:\n${tail}` : ""}`);
|
|
1951
|
+
};
|
|
1952
|
+
for (;;) {
|
|
1953
|
+
w.signal?.throwIfAborted();
|
|
1954
|
+
const probe = await w.probe();
|
|
1955
|
+
const now = Date.now();
|
|
1956
|
+
if (!probe.proc) {
|
|
1957
|
+
const address = await (w.hostListening ?? hostListening)(w.want);
|
|
1958
|
+
if (address !== void 0) return {
|
|
1959
|
+
port: w.want,
|
|
1960
|
+
address,
|
|
1961
|
+
detected: false,
|
|
1962
|
+
readyMs: now - started
|
|
1963
|
+
};
|
|
1964
|
+
if (!probe.alive) throw await fail(`exited before it listened on ${w.want}`);
|
|
1965
|
+
if (now - started >= timeoutMs) throw await fail(`didn't listen on ${w.want} within ${timeoutMs / 1e3}s. This sandbox can't list the ports it listens on, so it must listen on PORT or the port passed`);
|
|
1966
|
+
} else {
|
|
1967
|
+
const key = newPorts(probe.listeners, w.before).join(",");
|
|
1968
|
+
if (key !== last) {
|
|
1969
|
+
last = key;
|
|
1970
|
+
changed = now;
|
|
1971
|
+
}
|
|
1972
|
+
const d = decide({
|
|
1973
|
+
want: w.want,
|
|
1974
|
+
given: w.given,
|
|
1975
|
+
probe,
|
|
1976
|
+
before: w.before,
|
|
1977
|
+
stableMs: now - changed,
|
|
1978
|
+
elapsedMs: now - started,
|
|
1979
|
+
timeoutMs
|
|
1980
|
+
});
|
|
1981
|
+
if (d.kind === "port") return {
|
|
1982
|
+
port: d.port,
|
|
1983
|
+
address: d.address,
|
|
1984
|
+
detected: d.port !== w.want,
|
|
1985
|
+
readyMs: now - started
|
|
1986
|
+
};
|
|
1987
|
+
if (d.kind === "several") {
|
|
1988
|
+
const out = await output();
|
|
1989
|
+
const urls = printedUrls(out, d.ports);
|
|
1990
|
+
const printed = urls.length ? ` It printed ${listOf(urls.map((u) => u.url))}.` : "";
|
|
1991
|
+
const hint = urls[0]?.port ?? d.ports[0];
|
|
1992
|
+
throw await fail(`listens on ports ${listOf(d.ports)}, and not on PORT (${w.want}).${printed} Pass the one it serves on: c.serve(${JSON.stringify(command)}, ${hint})`, out);
|
|
1993
|
+
}
|
|
1994
|
+
if (d.kind === "exited") throw await fail("exited before it listened on a port");
|
|
1995
|
+
if (d.kind === "timeout") {
|
|
1996
|
+
const on = d.ports.length ? `; ${d.ports.length > 1 ? "ports" : "port"} ${listOf(d.ports)} started listening` : "";
|
|
1997
|
+
throw await fail(`didn't listen on ${w.given ? `the port passed (${w.want})` : "a port"} within ${timeoutMs / 1e3}s${on}`);
|
|
1998
|
+
}
|
|
1999
|
+
}
|
|
2000
|
+
await sleep(POLL_MS, w.signal);
|
|
2001
|
+
}
|
|
2002
|
+
}
|
|
1742
2003
|
//#endregion
|
|
1743
2004
|
//#region src/sandbox/fake.ts
|
|
1744
2005
|
/** Commands run on the host's own network, so the fake sandbox can neither restrict nor see it. */
|
|
@@ -1883,7 +2144,7 @@ async function createFakeSandbox(descriptor, env = {}) {
|
|
|
1883
2144
|
},
|
|
1884
2145
|
home: base,
|
|
1885
2146
|
readFile: (path) => readFile(resolve(path), "utf8").catch(() => null),
|
|
1886
|
-
getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}
|
|
2147
|
+
getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}://${await hostListening(port) === "::1" ? "[::1]" : "127.0.0.1"}:${port}` }),
|
|
1887
2148
|
async writeFile(path, content) {
|
|
1888
2149
|
const target = resolve(path);
|
|
1889
2150
|
await mkdir(dirname(target), { recursive: true });
|
|
@@ -1951,13 +2212,10 @@ function resolveOutput(input) {
|
|
|
1951
2212
|
const { flags, env } = input;
|
|
1952
2213
|
const ci = isCI(env);
|
|
1953
2214
|
const githubActions = set(env, "GITHUB_ACTIONS");
|
|
1954
|
-
const ui = pick("--ui", flags.ui ?? env.REEVAL_UI, [
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
]);
|
|
1959
|
-
const requested = ui && ui !== "auto" ? ui : input.agent ? "agent" : ci || !input.isTTY ? "plain" : "live";
|
|
1960
|
-
const mode = requested === "live" ? "plain" : requested;
|
|
2215
|
+
const ui = pick("--ui", flags.ui ?? env.REEVAL_UI, ["plain", "agent"]);
|
|
2216
|
+
const named = ui !== void 0 && ui !== "auto";
|
|
2217
|
+
const mode = named ? ui : input.agent ? "agent" : "plain";
|
|
2218
|
+
const statusLine = !named && mode === "plain" && !ci && input.isTTY;
|
|
1961
2219
|
const order = pick("--log-order", flags.logOrder ?? env.REEVAL_LOG_ORDER, ["stream", "grouped"]);
|
|
1962
2220
|
const logOrder = order && order !== "auto" ? order : ci ? "grouped" : "stream";
|
|
1963
2221
|
const prefix = pick("--log-prefix", flags.logPrefix ?? env.REEVAL_LOG_PREFIX, ["task", "none"]);
|
|
@@ -1968,8 +2226,8 @@ function resolveOutput(input) {
|
|
|
1968
2226
|
"none"
|
|
1969
2227
|
]);
|
|
1970
2228
|
return {
|
|
1971
|
-
requested,
|
|
1972
2229
|
mode,
|
|
2230
|
+
statusLine,
|
|
1973
2231
|
logOrder,
|
|
1974
2232
|
logPrefix,
|
|
1975
2233
|
outputLogs: logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only",
|
|
@@ -2003,8 +2261,8 @@ const isDirect = (policy) => policy === "allow-all";
|
|
|
2003
2261
|
* The hosts the harness needs: a real agent's own API (its model's provider's, for OpenCode), or
|
|
2004
2262
|
* a fake agent's with `layout` (it behaves like that agent). A plain fake agent talks to nothing.
|
|
2005
2263
|
*/
|
|
2006
|
-
function harnessHosts(harness) {
|
|
2007
|
-
if (harness.kind !== "fake" && isAgentKind(harness.kind)) return agentFor(harness).hosts ?? HARNESS_HOSTS[harness.kind];
|
|
2264
|
+
function harnessHosts(harness, vertex) {
|
|
2265
|
+
if (harness.kind !== "fake" && isAgentKind(harness.kind)) return agentFor(harness, vertex).hosts ?? HARNESS_HOSTS[harness.kind];
|
|
2008
2266
|
return harness.kind === "fake" && harness.layout ? HARNESS_HOSTS[harness.layout] : [];
|
|
2009
2267
|
}
|
|
2010
2268
|
/** MCP `http`/`sse` servers' hosts, local plugins' included: the agent connects to them from the sandbox. */
|
|
@@ -2020,7 +2278,7 @@ function mcpServerHosts(layers) {
|
|
|
2020
2278
|
* harness's, the brokered secrets' `hosts` and the MCP servers'. The cascade is config and
|
|
2021
2279
|
* profile defaults (`inherited`), then the environment's layers, then groups and the eval (`own`).
|
|
2022
2280
|
*/
|
|
2023
|
-
function plannedEgress(egress, harness, env, layers) {
|
|
2281
|
+
function plannedEgress(egress, harness, env, layers, vertex) {
|
|
2024
2282
|
const fromLayers = layers.findLast((l) => l.spec.egress !== void 0)?.spec.egress;
|
|
2025
2283
|
const policy = egress.own ?? fromLayers ?? egress.inherited ?? "proxy";
|
|
2026
2284
|
if (!isRestricted(policy)) return {
|
|
@@ -2029,7 +2287,7 @@ function plannedEgress(egress, harness, env, layers) {
|
|
|
2029
2287
|
};
|
|
2030
2288
|
const own = policy === "deny-all" ? [] : policy.allow;
|
|
2031
2289
|
const auto = [
|
|
2032
|
-
...harnessHosts(harness),
|
|
2290
|
+
...harnessHosts(harness, vertex),
|
|
2033
2291
|
...plannedSecrets(env, layers).flatMap((s) => s.hosts ?? []),
|
|
2034
2292
|
...mcpServerHosts(layers)
|
|
2035
2293
|
];
|
|
@@ -2104,34 +2362,28 @@ function egressLabel({ policy, auto }) {
|
|
|
2104
2362
|
}
|
|
2105
2363
|
//#endregion
|
|
2106
2364
|
//#region src/judge/state-options.ts
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2365
|
+
/**
|
|
2366
|
+
* What one call turned on, every key set (`false` when off), in the record's key order, so equal
|
|
2367
|
+
* selections serialize the same.
|
|
2368
|
+
*/
|
|
2369
|
+
function resolveStateOptions(call) {
|
|
2370
|
+
return {
|
|
2371
|
+
task: call.task ?? false,
|
|
2372
|
+
output: call.output ?? false,
|
|
2373
|
+
files: call.files ?? false,
|
|
2374
|
+
diff: call.diff ?? false,
|
|
2375
|
+
contents: call.contents ?? [],
|
|
2376
|
+
transcript: call.transcript ?? false,
|
|
2377
|
+
turns: call.turns ?? "last",
|
|
2378
|
+
checks: call.checks ?? false
|
|
2379
|
+
};
|
|
2122
2380
|
}
|
|
2123
|
-
/** `inner` over `outer`, field by field
|
|
2381
|
+
/** `inner` over `outer`, field by field. */
|
|
2124
2382
|
function mergeJudge(outer, inner) {
|
|
2125
|
-
|
|
2126
|
-
const out = {
|
|
2383
|
+
return inner ? {
|
|
2127
2384
|
...outer,
|
|
2128
2385
|
...inner
|
|
2129
|
-
};
|
|
2130
|
-
if (outer.state || inner.state) out.state = {
|
|
2131
|
-
...outer.state,
|
|
2132
|
-
...inner.state
|
|
2133
|
-
};
|
|
2134
|
-
return out;
|
|
2386
|
+
} : outer;
|
|
2135
2387
|
}
|
|
2136
2388
|
const KNOWN = [...[
|
|
2137
2389
|
"harness",
|
|
@@ -2147,7 +2399,9 @@ const KNOWN = [...[
|
|
|
2147
2399
|
"question",
|
|
2148
2400
|
"questions",
|
|
2149
2401
|
"timeout",
|
|
2402
|
+
"snapshot",
|
|
2150
2403
|
"tags",
|
|
2404
|
+
"revision",
|
|
2151
2405
|
"pin"
|
|
2152
2406
|
]];
|
|
2153
2407
|
const SUGGEST = {
|
|
@@ -2160,15 +2414,35 @@ const SUGGEST = {
|
|
|
2160
2414
|
repeats: "runs",
|
|
2161
2415
|
iterations: "runs",
|
|
2162
2416
|
trials: "runs",
|
|
2417
|
+
bestOf: "runs",
|
|
2163
2418
|
secrets: "env",
|
|
2164
2419
|
vars: "env",
|
|
2165
2420
|
network: "egress",
|
|
2166
2421
|
networkPolicy: "egress",
|
|
2167
2422
|
tag: "tags",
|
|
2423
|
+
version: "revision",
|
|
2424
|
+
rev: "revision",
|
|
2168
2425
|
pins: "pin",
|
|
2426
|
+
workspace: "snapshot",
|
|
2427
|
+
snapshots: "snapshot",
|
|
2169
2428
|
passRate: "gate",
|
|
2170
2429
|
threshold: "gate"
|
|
2171
2430
|
};
|
|
2431
|
+
/**
|
|
2432
|
+
* `snapshot` over the levels above: the shorthand sets `when`, the object sets the fields it
|
|
2433
|
+
* names, and `compress` carries through either.
|
|
2434
|
+
*/
|
|
2435
|
+
function mergeSnapshot(outer, inner) {
|
|
2436
|
+
if (inner === void 0) return outer;
|
|
2437
|
+
if (typeof inner !== "object") return {
|
|
2438
|
+
...outer,
|
|
2439
|
+
when: inner
|
|
2440
|
+
};
|
|
2441
|
+
return {
|
|
2442
|
+
when: inner.when ?? outer.when,
|
|
2443
|
+
compress: inner.compress ?? outer.compress
|
|
2444
|
+
};
|
|
2445
|
+
}
|
|
2172
2446
|
/** Throws unless `value` is `'all'` or `{ passRate }` with a rate in (0, 1]. */
|
|
2173
2447
|
function checkGate(value, where) {
|
|
2174
2448
|
const problem = gateProblem(value, where);
|
|
@@ -2235,18 +2509,45 @@ function refuseIllegalCompare(options, where) {
|
|
|
2235
2509
|
const baseline = options.compare !== null && typeof options.compare === "object" && "baseline" in options.compare ? options.compare.baseline : void 0;
|
|
2236
2510
|
if (options.environment !== void 0 && baseline !== void 0) throw new UsageError(`environment and compare.baseline can't both be set in ${where}. compare: { baseline, variants } names every arm, so drop environment, or drop baseline to make environment the control.`);
|
|
2237
2511
|
}
|
|
2512
|
+
const isRevision = (value) => typeof value === "number" && Number.isSafeInteger(value) || typeof value === "string" && value.trim() !== "";
|
|
2513
|
+
/** `runs: 3` or `runs: { bestOf: 5 }`, refused in the words of whichever it was meant to be. */
|
|
2514
|
+
function checkRuns(runs, where) {
|
|
2515
|
+
if (typeof runs === "number" && Number.isInteger(runs) && runs >= 1) return;
|
|
2516
|
+
if (typeof runs === "number") throw new UsageError(`runs in ${where} must be a whole number of at least 1 (got ${JSON.stringify(runs)})`);
|
|
2517
|
+
if (!isObjectLike(runs)) throw new UsageError(`runs in ${where} must be a whole number of at least 1, or { bestOf: n } (got ${JSON.stringify(runs)})`);
|
|
2518
|
+
const extra = Object.keys(runs).find((k) => k !== "bestOf");
|
|
2519
|
+
if (extra !== void 0) throw new UsageError(`runs in ${where} takes { bestOf: n } only (got the key "${extra}"), e.g. runs: { bestOf: 3 }`);
|
|
2520
|
+
const { bestOf } = runs;
|
|
2521
|
+
if (!(Number.isInteger(bestOf) && bestOf >= 3 && bestOf % 2 === 1)) throw new UsageError(`runs.bestOf in ${where} must be an odd whole number of 3 or more, so one side always has a majority, e.g. runs: { bestOf: 3 } (got ${JSON.stringify(bestOf)})`);
|
|
2522
|
+
}
|
|
2523
|
+
/** The most attempts `runs` plans a cell, and its `bestOf` when it has one. */
|
|
2524
|
+
const runsOf = (runs) => typeof runs === "number" ? { runs } : {
|
|
2525
|
+
runs: runs.bestOf,
|
|
2526
|
+
bestOf: runs.bestOf
|
|
2527
|
+
};
|
|
2528
|
+
/**
|
|
2529
|
+
* A best-of cell's gate: half or more of its counted attempts pass, which is its majority once it
|
|
2530
|
+
* has one (an infra error counts on neither side).
|
|
2531
|
+
*/
|
|
2532
|
+
const MAJORITY_GATE = { passRate: .5 };
|
|
2533
|
+
/** How many of a best-of cell's attempts always run: the majority. */
|
|
2534
|
+
const majorityOf = (bestOf) => (bestOf + 1) / 2;
|
|
2238
2535
|
/** Throws on keys that aren't options, and on values of the wrong shape, suggesting a fix. */
|
|
2239
2536
|
function checkOptions(options, where, allowed = KNOWN) {
|
|
2240
2537
|
for (const key of Object.keys(options)) {
|
|
2241
2538
|
if (allowed.includes(key)) continue;
|
|
2242
2539
|
const near = SUGGEST[key] ?? closest(key, allowed);
|
|
2243
|
-
const
|
|
2540
|
+
const value = options[key];
|
|
2541
|
+
const hint = key === "pool" ? " pool is an option of evaluate.each, beside the cases: evaluate.each(cases, { pool: false })." : key === "workspace" && typeof value !== "boolean" ? " workspace is an environment's starting tree: defineEnvironment(name, { workspace: git(…) })." : near ? ` Did you mean "${near}"?` : "";
|
|
2244
2542
|
throw new UsageError(`Unknown option "${key}" in ${where}.${hint}`);
|
|
2245
2543
|
}
|
|
2246
2544
|
const o = options;
|
|
2247
|
-
if (o.runs !== void 0
|
|
2545
|
+
if (o.runs !== void 0) checkRuns(o.runs, where);
|
|
2248
2546
|
if (o.gate !== void 0) checkGate(o.gate, where);
|
|
2249
2547
|
if (o.tags !== void 0 && (!Array.isArray(o.tags) || o.tags.some((t) => typeof t !== "string"))) throw new UsageError(`tags in ${where} must be a list of strings, e.g. tags: ['smoke'] (got ${JSON.stringify(o.tags)})`);
|
|
2548
|
+
if (o.revision !== void 0 && !isRevision(o.revision)) throw new UsageError(`revision in ${where} must be a whole number or a non-empty string, e.g. revision: 2 (got ${JSON.stringify(o.revision)})`);
|
|
2549
|
+
const snapshotIssue = snapshotProblem(o.snapshot, where);
|
|
2550
|
+
if (snapshotIssue) throw new UsageError(snapshotIssue);
|
|
2250
2551
|
if (o.env !== void 0) checkEnv(o.env, where, UsageError);
|
|
2251
2552
|
if (o.egress !== void 0) checkEgress(o.egress, where, UsageError);
|
|
2252
2553
|
const judgeIssue = o.judge !== void 0 && judgeProblem(o.judge, {
|
|
@@ -2377,17 +2678,18 @@ function checkUnique(list, where) {
|
|
|
2377
2678
|
seen.add(key);
|
|
2378
2679
|
}
|
|
2379
2680
|
}
|
|
2380
|
-
const plannedQuestions = (value) => value.map((q, i) =>
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2681
|
+
const plannedQuestions = (value) => value.map((q, i) => {
|
|
2682
|
+
const { question, model, across, choices, artifacts, ...rest } = q;
|
|
2683
|
+
const state = Object.fromEntries(Object.entries(rest).filter(([, v]) => v !== void 0));
|
|
2684
|
+
return {
|
|
2685
|
+
n: i + 1,
|
|
2686
|
+
question,
|
|
2687
|
+
...model !== void 0 ? { model } : {},
|
|
2688
|
+
...across !== void 0 ? { across } : {},
|
|
2689
|
+
...choices ? { choices } : {},
|
|
2690
|
+
state,
|
|
2691
|
+
...artifacts !== void 0 ? { artifacts } : {}
|
|
2692
|
+
};
|
|
2391
2693
|
});
|
|
2392
2694
|
/** Whether the eval's matrix varies each factor a pairwise question can vary. */
|
|
2393
2695
|
const variesBy = (r) => ({
|
|
@@ -2412,6 +2714,12 @@ function normalizeCompare(value) {
|
|
|
2412
2714
|
of: value.variants.length
|
|
2413
2715
|
};
|
|
2414
2716
|
}
|
|
2717
|
+
/**
|
|
2718
|
+
* Runs per cell when no level sets `runs`: a compare with variants runs `EFFECT_MIN_COUNTED` a
|
|
2719
|
+
* side, the fewest whose lift can reach a verdict; anything else, a pooled family's cases
|
|
2720
|
+
* included, runs once.
|
|
2721
|
+
*/
|
|
2722
|
+
const COMPARE_DEFAULT_RUNS = 4;
|
|
2415
2723
|
function applyLevel(out, options, where, egress = "own") {
|
|
2416
2724
|
for (const [key, value] of Object.entries(options)) if (value !== void 0) out.sources = {
|
|
2417
2725
|
...out.sources,
|
|
@@ -2422,9 +2730,15 @@ function applyLevel(out, options, where, egress = "own") {
|
|
|
2422
2730
|
if (options.sandbox !== void 0) out.sandbox = options.sandbox;
|
|
2423
2731
|
if (options.environment !== void 0) out.environment = options.environment;
|
|
2424
2732
|
if (options.compare !== void 0) out.compare = normalizeCompare(options.compare);
|
|
2425
|
-
if (options.runs !== void 0)
|
|
2733
|
+
if (options.runs !== void 0) {
|
|
2734
|
+
const { runs, bestOf } = runsOf(options.runs);
|
|
2735
|
+
out.runs = runs;
|
|
2736
|
+
out.bestOf = bestOf;
|
|
2737
|
+
}
|
|
2426
2738
|
if (options.gate !== void 0) out.gate = options.gate;
|
|
2427
2739
|
if (options.timeout !== void 0) out.timeout = options.timeout;
|
|
2740
|
+
out.snapshot = mergeSnapshot(out.snapshot, options.snapshot);
|
|
2741
|
+
if (options.revision !== void 0) out.revision = options.revision;
|
|
2428
2742
|
Object.assign(out.env, options.env);
|
|
2429
2743
|
out.judge = mergeJudge(out.judge, options.judge);
|
|
2430
2744
|
if (options.capture !== void 0) {
|
|
@@ -2470,11 +2784,7 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2470
2784
|
baseline: out.environment,
|
|
2471
2785
|
from: "environment"
|
|
2472
2786
|
};
|
|
2473
|
-
if (out.questions.length)
|
|
2474
|
-
out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
|
|
2475
|
-
const blank = out.questions.find(showsNothing);
|
|
2476
|
-
if (blank) throw new UsageError(`questions "${blank.question}" in ${where} shows the model nothing: every state key is off and it sends no artifacts. Turn on a state key, such as state: { diff: true }, or send artifacts.`);
|
|
2477
|
-
}
|
|
2787
|
+
if (out.questions.length) out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
|
|
2478
2788
|
checkUnique(out.harness, where);
|
|
2479
2789
|
const harnessFromProfile = profile?.defaults.harness !== void 0 && !levels.some((l) => l.options.harness);
|
|
2480
2790
|
const authored = variesBy(out);
|
|
@@ -2500,23 +2810,30 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2500
2810
|
const dupe = names.find((n, i) => names.indexOf(n) !== i);
|
|
2501
2811
|
if (dupe) throw new UsageError(`compare in ${where} lists two variants named ${dupe}. Give each variant its own name.`);
|
|
2502
2812
|
let skip;
|
|
2813
|
+
const harnesses = out.harness.map(harnessId);
|
|
2503
2814
|
const limit = profile?.limit;
|
|
2504
2815
|
if (profile && limit) {
|
|
2505
2816
|
const tag = `profile ${profile.name}`;
|
|
2506
2817
|
const pinned = (key) => out.pin.includes(key);
|
|
2507
|
-
|
|
2508
|
-
notes.push(`runs ${out.runs} → ${limit.runs} (${tag})`);
|
|
2509
|
-
out.runs = limit.runs;
|
|
2510
|
-
}
|
|
2818
|
+
let comparedLeft = true;
|
|
2511
2819
|
if (limit.harness && !pinned("harness")) {
|
|
2512
2820
|
const kept = out.harness.filter((h) => limit.harness?.includes(harnessId(h)));
|
|
2513
2821
|
if (kept.length === 0) skip = `no harness left: ${tag} allows ${limit.harness.join(", ") || "none"}`;
|
|
2514
2822
|
else if (kept.length < out.harness.length) {
|
|
2515
2823
|
notes.push(`harnesses ${out.harness.length} → ${kept.length} (${tag})`);
|
|
2516
2824
|
out.harness = kept;
|
|
2825
|
+
const scope = out.compare?.harness;
|
|
2826
|
+
if (out.compare?.variants.length && scope && !kept.some((h) => scope.includes(harnessId(h)))) {
|
|
2827
|
+
notes.push(`compare off: no compared harness left (${tag})`);
|
|
2828
|
+
out.compare = {
|
|
2829
|
+
...out.compare,
|
|
2830
|
+
variants: []
|
|
2831
|
+
};
|
|
2832
|
+
comparedLeft = false;
|
|
2833
|
+
}
|
|
2517
2834
|
}
|
|
2518
2835
|
}
|
|
2519
|
-
if (out.compare && limit.compare !== void 0 && !pinned("compare")) {
|
|
2836
|
+
if (out.compare && comparedLeft && limit.compare !== void 0 && !pinned("compare")) {
|
|
2520
2837
|
if (limit.compare === false) {
|
|
2521
2838
|
notes.push(`compare off (${tag})`);
|
|
2522
2839
|
out.compare = {
|
|
@@ -2531,7 +2848,18 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2531
2848
|
};
|
|
2532
2849
|
}
|
|
2533
2850
|
}
|
|
2534
|
-
|
|
2851
|
+
defaultRuns(out, notes, options);
|
|
2852
|
+
if (limit.runs !== void 0 && options.runs === void 0 && !pinned("runs") && (out.runs ?? 1) > limit.runs) {
|
|
2853
|
+
const was = out.bestOf !== void 0 ? `best of ${out.bestOf}` : String(out.runs);
|
|
2854
|
+
notes.push(`runs ${was} → ${limit.runs} (${tag})`);
|
|
2855
|
+
out.runs = limit.runs;
|
|
2856
|
+
out.bestOf = void 0;
|
|
2857
|
+
out.sources = {
|
|
2858
|
+
...out.sources,
|
|
2859
|
+
runs: `profiles.${profile.name}.limit`
|
|
2860
|
+
};
|
|
2861
|
+
}
|
|
2862
|
+
const pins = out.pin.filter((k) => k === "compare" ? limit.compare !== void 0 && out.compare && comparedLeft : limit[k] !== void 0);
|
|
2535
2863
|
if (pins.length) notes.push(`pinned: ${pins.join(", ")}`);
|
|
2536
2864
|
}
|
|
2537
2865
|
if (profile) {
|
|
@@ -2554,13 +2882,59 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2554
2882
|
if (narrowed) notes.push(`${plural(narrowed)} narrowed (profile ${profile.name})`);
|
|
2555
2883
|
out.questions = kept;
|
|
2556
2884
|
}
|
|
2557
|
-
if (options.runs !== void 0) out.
|
|
2885
|
+
if (options.runs !== void 0) out.bestOf = void 0;
|
|
2886
|
+
if (out.bestOf !== void 0 && out.compare?.variants.length) {
|
|
2887
|
+
const from = out.sources?.runs ?? "the config";
|
|
2888
|
+
const set = from === where ? "" : ` from ${from}`;
|
|
2889
|
+
throw new UsageError(`${where} compares environments, so it can't take runs: { bestOf: ${out.bestOf} }${set}: a lift needs the same number of attempts on each side, and best-of stops each cell on its own. Set a fixed count on it, e.g. runs: 4.`);
|
|
2890
|
+
}
|
|
2891
|
+
if (out.bestOf !== void 0) {
|
|
2892
|
+
if (out.gate !== void 0 && gateLabel(out.gate) !== gateLabel(MAJORITY_GATE)) notes.push(`gate ${gateLabel(out.gate)} → majority (runs.bestOf)`);
|
|
2893
|
+
out.gate = MAJORITY_GATE;
|
|
2894
|
+
}
|
|
2895
|
+
defaultRuns(out, notes, options);
|
|
2558
2896
|
return {
|
|
2559
|
-
resolved:
|
|
2897
|
+
resolved: {
|
|
2898
|
+
...out,
|
|
2899
|
+
runs: options.runs ?? out.runs ?? 1
|
|
2900
|
+
},
|
|
2560
2901
|
notes,
|
|
2902
|
+
harnesses,
|
|
2561
2903
|
...skip ? { skip } : {}
|
|
2562
2904
|
};
|
|
2563
2905
|
}
|
|
2906
|
+
/** Gives `out` its default runs when no level set them and `--runs` doesn't replace them. */
|
|
2907
|
+
function defaultRuns(out, notes, options) {
|
|
2908
|
+
if (out.runs !== void 0 || options.runs !== void 0) return;
|
|
2909
|
+
if (!out.compare?.variants.length || options.pooled) {
|
|
2910
|
+
out.runs = 1;
|
|
2911
|
+
return;
|
|
2912
|
+
}
|
|
2913
|
+
out.runs = COMPARE_DEFAULT_RUNS;
|
|
2914
|
+
out.sources = {
|
|
2915
|
+
...out.sources,
|
|
2916
|
+
runs: `compare default: ${COMPARE_DEFAULT_RUNS} a side`
|
|
2917
|
+
};
|
|
2918
|
+
notes.push(`runs ${COMPARE_DEFAULT_RUNS} (compare default)`);
|
|
2919
|
+
}
|
|
2920
|
+
/**
|
|
2921
|
+
* Refuses a profile's `limit.harness` id that no collected eval runs on: it keeps nothing, and a
|
|
2922
|
+
* limit that keeps nothing skips every eval. `harnesses` are the ids every eval in the loaded
|
|
2923
|
+
* files runs on before the limit, whatever the tag and name filters pick. When a path filter
|
|
2924
|
+
* loaded only some files (`partial`), an id may name a harness of the others, so only a limit
|
|
2925
|
+
* none of whose ids match is refused.
|
|
2926
|
+
*/
|
|
2927
|
+
function checkLimitHarness(profile, configFile, collected) {
|
|
2928
|
+
const ids = profile?.limit.harness;
|
|
2929
|
+
if (!profile || !ids?.length) return;
|
|
2930
|
+
const unknown = ids.filter((id) => !collected.harnesses.includes(id));
|
|
2931
|
+
if (!unknown.length || collected.partial && unknown.length < ids.length) return;
|
|
2932
|
+
const listed = unknown.map((id) => JSON.stringify(id)).join(", ");
|
|
2933
|
+
const have = [...collected.harnesses].sort();
|
|
2934
|
+
const evals = collected.partial ? "eval in the selected files" : "eval";
|
|
2935
|
+
const the = collected.partial ? "the evals in the selected files" : "the evals";
|
|
2936
|
+
throw new UsageError(`profiles.${profile.name}.limit.harness in ${configFile} lists ${listed}, which no ${evals} runs on. Harness ids ${the} run on: ${have.join(", ") || "none"}.`);
|
|
2937
|
+
}
|
|
2564
2938
|
/**
|
|
2565
2939
|
* The sandbox an attempt runs in: the resolved environment's (including an `.extend()`
|
|
2566
2940
|
* parent's), else the first variant's when this is an omitted-baseline empty control, else
|
|
@@ -2594,8 +2968,11 @@ function sandboxSetBy(planned, resolved, configFile) {
|
|
|
2594
2968
|
}
|
|
2595
2969
|
return resolved.sources?.environment ?? configFile;
|
|
2596
2970
|
}
|
|
2597
|
-
/**
|
|
2598
|
-
|
|
2971
|
+
/**
|
|
2972
|
+
* Per harness: the control, then each variant it compares on; then run numbers. `vertex` is the
|
|
2973
|
+
* run's `vertex()` settings, for each harness that resolved to Vertex AI.
|
|
2974
|
+
*/
|
|
2975
|
+
function expand(resolved, vertex) {
|
|
2599
2976
|
const { compare } = resolved;
|
|
2600
2977
|
const control = applied(compare ? compare.baseline : resolved.environment);
|
|
2601
2978
|
const policy = resolved.gate ?? "all";
|
|
@@ -2603,6 +2980,7 @@ function expand(resolved) {
|
|
|
2603
2980
|
const attempts = [];
|
|
2604
2981
|
for (const harness of resolved.harness) {
|
|
2605
2982
|
const hid = harnessId(harness);
|
|
2983
|
+
const onVertex = harness.gateway === "vertex" && harness.kind !== "fake" ? vertex : void 0;
|
|
2606
2984
|
const cells = [(() => {
|
|
2607
2985
|
const sandbox = attemptSandbox(resolved, control);
|
|
2608
2986
|
const sid = sandboxId(sandbox);
|
|
@@ -2646,11 +3024,13 @@ function expand(resolved) {
|
|
|
2646
3024
|
harness,
|
|
2647
3025
|
sandbox,
|
|
2648
3026
|
environment: env,
|
|
2649
|
-
egress: plannedEgress(resolved.egress, harness, resolved.env, env.layers),
|
|
3027
|
+
egress: plannedEgress(resolved.egress, harness, resolved.env, env.layers, onVertex),
|
|
2650
3028
|
gate,
|
|
2651
3029
|
...cell.role === "variant" && compare?.minLift !== void 0 ? { minLift: compare.minLift } : {},
|
|
3030
|
+
...resolved.bestOf !== void 0 ? { bestOf: resolved.bestOf } : {},
|
|
2652
3031
|
n,
|
|
2653
|
-
label
|
|
3032
|
+
label,
|
|
3033
|
+
...onVertex ? { vertex: onVertex } : {}
|
|
2654
3034
|
});
|
|
2655
3035
|
}
|
|
2656
3036
|
}
|
|
@@ -2698,9 +3078,18 @@ function unbrokeredOnFake(env, attempts) {
|
|
|
2698
3078
|
}
|
|
2699
3079
|
/** The host variables an attempt's secrets read, brokered or not. */
|
|
2700
3080
|
const secretVarsIn = (env, attempt) => [...new Set(plannedSecrets(env, attempt.environment.layers).map((s) => s.from))];
|
|
2701
|
-
const brokerHint = (s) => /^mcp\.[^.]+\.headers\./.test(s.label) ? "give the MCP server an https:// URL (reeval brokers its header secrets to its own host
|
|
2702
|
-
const unbrokeredWarning = (s) => s.checksOnly ? `${s.label} reaches the checks (c.exec, setup steps) as its real value
|
|
3081
|
+
const brokerHint = (s) => /^mcp\.[^.]+\.headers\./.test(s.label) ? "give the MCP server an https:// URL (reeval brokers its header secrets over HTTPS only, to its own host unless they set hosts)" : `add the hosts it's sent to, e.g. secret.fromEnv('${s.from}', { hosts: ['api.example.com'] })`;
|
|
3082
|
+
const unbrokeredWarning = (s) => s.checksOnly ? `${s.label} reaches the checks (c.exec, setup steps) as its real value. It's never in the agent's own environment, but a process a setup step leaves running holds it while the agent works. To give the checks only a placeholder, ${brokerHint(s)}.` : `${s.label} is in the sandbox as its real value. To give the agent only a placeholder, ${brokerHint(s)}.`;
|
|
2703
3083
|
const unbrokeredError = (s, where, configFile) => `${s.label} in ${where} isn't brokered, and secrets.unbrokered is 'error' in ${configFile}. To broker it, ${brokerHint(s)}.`;
|
|
3084
|
+
/** The attempts' MCP headers written as plain strings that look like credentials, once each. */
|
|
3085
|
+
function plainHeaders(attempts) {
|
|
3086
|
+
const out = /* @__PURE__ */ new Map();
|
|
3087
|
+
for (const a of attempts) for (const h of plainCredentialHeaders(a.environment.layers)) out.set(h.label, h);
|
|
3088
|
+
return [...out.values()];
|
|
3089
|
+
}
|
|
3090
|
+
const templateHint = (h) => `read it from a host variable with secret.template, e.g. secret.template('Bearer \${VAR}')${h.https ? "" : " and give the MCP server an https:// URL"}`;
|
|
3091
|
+
const plainHeaderWarning = (h) => `${h.label} is a plain string, so it's in the sandbox as its real value. To give the agent only a placeholder, ${templateHint(h)}.`;
|
|
3092
|
+
const plainHeaderError = (h, where, configFile) => `${h.label} in ${where} is a plain string, and secrets.unbrokered is 'error' in ${configFile}. To broker it, ${templateHint(h)}.`;
|
|
2704
3093
|
const levelOf = (where) => where.startsWith("evaluate(") ? "eval" : where.startsWith("group(") ? "group" : where.startsWith("profiles.") ? "profile" : "config";
|
|
2705
3094
|
/**
|
|
2706
3095
|
* A warning per privilege an attempt's Docker sandbox adds, naming the level that set the
|
|
@@ -2767,6 +3156,7 @@ function evalPlan(r, attempts, origin) {
|
|
|
2767
3156
|
} } : {},
|
|
2768
3157
|
...!skip && resolved.questions.length ? { questions: resolved.questions } : {},
|
|
2769
3158
|
runs: resolved.runs,
|
|
3159
|
+
...resolved.bestOf !== void 0 ? { bestOf: resolved.bestOf } : {},
|
|
2770
3160
|
...resolved.gate !== void 0 ? { gate: resolved.gate } : {},
|
|
2771
3161
|
attempts: skip ? 0 : attempts.length,
|
|
2772
3162
|
notes,
|
|
@@ -2774,7 +3164,8 @@ function evalPlan(r, attempts, origin) {
|
|
|
2774
3164
|
warnings: [
|
|
2775
3165
|
...environmentWarnings(attempts),
|
|
2776
3166
|
...skip ? [] : questionWarnings(resolved),
|
|
2777
|
-
...unbrokered(r.resolved.env, attempts).filter(reachesSandbox).map(unbrokeredWarning)
|
|
3167
|
+
...unbrokered(r.resolved.env, attempts).filter(reachesSandbox).map(unbrokeredWarning),
|
|
3168
|
+
...plainHeaders(attempts).map(plainHeaderWarning)
|
|
2778
3169
|
],
|
|
2779
3170
|
sandboxWarnings: skip ? [] : sandboxWarnings(attempts, resolved, origin),
|
|
2780
3171
|
brokered: [...new Set(attempts.flatMap((a) => brokeredIn(r.resolved.env, a)))],
|
|
@@ -2789,7 +3180,7 @@ function evalPlan(r, attempts, origin) {
|
|
|
2789
3180
|
/** A pairwise question that shows each side's transcript, asked of two harnesses: tool names unblind it. */
|
|
2790
3181
|
function questionWarnings(resolved) {
|
|
2791
3182
|
if (resolved.harness.length < 2) return [];
|
|
2792
|
-
return resolved.questions.filter((q) => pairFactors(q.across).includes("harness") && q.state?.transcript).map((q) => `question "${q.question}" shows
|
|
3183
|
+
return resolved.questions.filter((q) => pairFactors(q.across).includes("harness") && q.state?.transcript).map((q) => `question "${q.question}" shows the transcript, which names each harness's tools, so its harness pairs aren't blind. Ask it with across: 'environment', or leave the transcript out.`);
|
|
2793
3184
|
}
|
|
2794
3185
|
function captureNote(c) {
|
|
2795
3186
|
const parts = [];
|
|
@@ -2806,23 +3197,47 @@ function matchesTags(tags, filters) {
|
|
|
2806
3197
|
}
|
|
2807
3198
|
//#endregion
|
|
2808
3199
|
//#region src/gateway/resolve.ts
|
|
2809
|
-
/** The config's list when it sets none: each provider's own key
|
|
3200
|
+
/** The config's list when it sets none: each provider's own key. */
|
|
2810
3201
|
const DEFAULT_GATEWAYS = [{ kind: "direct" }];
|
|
2811
3202
|
/** Every gateway reeval can route through, by name. */
|
|
2812
3203
|
const GATEWAYS = {
|
|
2813
3204
|
direct: { cantServe: (h) => {
|
|
2814
|
-
|
|
3205
|
+
const fxEffort = fxEffortRefusal(h, "direct()");
|
|
3206
|
+
if (fxEffort) return fxEffort;
|
|
3207
|
+
if (h.kind === "fx" && h.model.includes("/")) return `fx runs a gateway id such as ${JSON.stringify(h.model)} only through Vercel AI Gateway or OpenRouter`;
|
|
2815
3208
|
if (h.kind === "opencode") {
|
|
2816
3209
|
const resolved = resolveOpenCodeProvider(h.model, process.env);
|
|
2817
3210
|
if ("error" in resolved) return resolved.error;
|
|
2818
3211
|
}
|
|
2819
3212
|
} },
|
|
2820
|
-
vercel:
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
if (
|
|
3213
|
+
vercel: routingEntry("vercel"),
|
|
3214
|
+
openrouter: routingEntry("openrouter"),
|
|
3215
|
+
vertex: { cantServe: (h) => {
|
|
3216
|
+
if (h.kind === "fake") return void 0;
|
|
3217
|
+
if (!VERTEX_ROUTES[h.kind]) return VERTEX_REFUSALS[h.kind] ?? `reeval doesn't run ${agentName(h)} on Vertex AI yet`;
|
|
3218
|
+
return vertexModelRefusalFor(h);
|
|
2824
3219
|
} }
|
|
2825
3220
|
};
|
|
3221
|
+
/**
|
|
3222
|
+
* Why fx can't take `effort` on a gateway: fx sends none itself, and reeval adds it as
|
|
3223
|
+
* `reasoning_effort` only to its calls on Vertex AI's Chat Completions endpoint.
|
|
3224
|
+
*/
|
|
3225
|
+
function fxEffortRefusal(h, gateway) {
|
|
3226
|
+
const effort = harnessEffort(h);
|
|
3227
|
+
if (h.kind !== "fx" || effort === void 0) return void 0;
|
|
3228
|
+
return `fx sends no effort of its own, and reeval adds fx's effort (${JSON.stringify(effort)}) only to its calls on Vertex AI, not on ${gateway}: drop effort or run fx on vertex()`;
|
|
3229
|
+
}
|
|
3230
|
+
/** A routing gateway's entry: its route table decides whom it serves, and with what effort. */
|
|
3231
|
+
function routingEntry(name) {
|
|
3232
|
+
const gateway = ROUTING_GATEWAYS[name];
|
|
3233
|
+
return { cantServe: (h) => {
|
|
3234
|
+
const fxEffort = fxEffortRefusal(h, gateway.name);
|
|
3235
|
+
if (fxEffort) return fxEffort;
|
|
3236
|
+
if (h.kind !== "fake" && !gateway.routes[h.kind]) return gateway.refusals[h.kind] ?? `reeval doesn't run ${agentName(h)} on ${gateway.name} yet`;
|
|
3237
|
+
const effort = harnessEffort(h);
|
|
3238
|
+
if (effort !== void 0 && h.kind !== "fake" && gateway.unprovenEffort[h.kind]?.(h.model)) return `reeval doesn't send ${agentName(h)}'s effort (${JSON.stringify(effort)}) to ${h.model} through ${gateway.name} yet: nothing checks the model gets it as its own setting, so drop effort or run ${agentName(h)} on another gateway`;
|
|
3239
|
+
} };
|
|
3240
|
+
}
|
|
2826
3241
|
/** The `reeval/gateway` helper that makes the gateway `name`, as messages name it. */
|
|
2827
3242
|
const gatewayHelper = (name) => `${name}()`;
|
|
2828
3243
|
/** The agent's name, for messages. */
|
|
@@ -2837,7 +3252,7 @@ function resolveGateway(gateways, harness, where, catalog = GATEWAYS) {
|
|
|
2837
3252
|
const why = (name) => catalog[name]?.cantServe(harness);
|
|
2838
3253
|
const pin = harness.gateway;
|
|
2839
3254
|
if (pin !== void 0) {
|
|
2840
|
-
if (!gateways.some((g) => g.kind === pin)) throw new UsageError(`${on}: it pins gateway '${pin}',
|
|
3255
|
+
if (!gateways.some((g) => g.kind === pin)) throw new UsageError(`${on}: it pins gateway '${pin}', ${notListed(pin, listed)}`);
|
|
2841
3256
|
const refused = why(pin);
|
|
2842
3257
|
if (refused) throw new UsageError(`${on}: it pins gateway '${pin}', which can't serve it (${refused}). A pin never falls back; pin another gateway, or remove the pin to take the first in the list that can.`);
|
|
2843
3258
|
return {
|
|
@@ -2859,10 +3274,13 @@ function resolveGateway(gateways, harness, where, catalog = GATEWAYS) {
|
|
|
2859
3274
|
}
|
|
2860
3275
|
throw new UsageError(`${on}: no gateway in the list can serve it (${skipped.map((s) => `${s.gateway}: ${s.why}`).join("; ")}). ${addHint(gateways, harness, catalog)}`);
|
|
2861
3276
|
}
|
|
2862
|
-
/**
|
|
3277
|
+
/**
|
|
3278
|
+
* What to change so the list serves `harness`: a list without `direct` has it turned off, so
|
|
3279
|
+
* turning it back on when it can serve, else adding the first gateway that can.
|
|
3280
|
+
*/
|
|
2863
3281
|
function addHint(gateways, harness, catalog) {
|
|
2864
3282
|
const serves = Object.keys(catalog).filter((name) => !gateways.some((g) => g.kind === name)).filter((name) => !catalog[name]?.cantServe(harness));
|
|
2865
|
-
if (serves.includes("direct")) return `
|
|
3283
|
+
if (serves.includes("direct")) return `Drop direct({ enabled: false }) from gateway to fall back to ${agentName(harness)}'s own key.`;
|
|
2866
3284
|
const [first] = serves;
|
|
2867
3285
|
return first ? `Add ${gatewayHelper(first)} to gateway.` : `No gateway reeval knows can serve it.`;
|
|
2868
3286
|
}
|
|
@@ -2881,30 +3299,40 @@ function routedHarness(harness, gateways) {
|
|
|
2881
3299
|
return harness;
|
|
2882
3300
|
}
|
|
2883
3301
|
}
|
|
2884
|
-
/**
|
|
2885
|
-
|
|
3302
|
+
/**
|
|
3303
|
+
* Why a pin names a gateway the list doesn't have: a list without `direct` has it turned off.
|
|
3304
|
+
*/
|
|
3305
|
+
const notListed = (pin, listed) => pin === "direct" ? `which the gateway list (${listed}) turns off with direct({ enabled: false }). Drop that, or pin one that's there.` : `which isn't in the gateway list (${listed}). Add ${gatewayHelper(pin)} to gateway, or pin one that's there.`;
|
|
3306
|
+
/**
|
|
3307
|
+
* `gateway: cursor:… falls back to direct (vercel can't serve it: …)`, or `→ vercel (…)` past
|
|
3308
|
+
* another gateway, for the plan's hints.
|
|
3309
|
+
*/
|
|
3310
|
+
const fallbackHint = (harness, resolved) => `gateway: ${harness} ${resolved.name === "direct" ? "falls back to" : "→"} ${resolved.name} (${resolved.skipped.map((s) => `${s.gateway} can't serve it: ${s.why}`).join("; ")})`;
|
|
2886
3311
|
//#endregion
|
|
2887
3312
|
//#region src/gateway/models.ts
|
|
2888
3313
|
const providerOf = (model) => model.slice(0, model.indexOf("/"));
|
|
2889
3314
|
/** Why each gateway can't serve a model by name, or undefined when it can. */
|
|
2890
3315
|
const MODEL_GATEWAYS = {
|
|
2891
3316
|
direct: (model) => DIRECT_PROVIDERS[providerOf(model)] ? void 0 : `reeval calls only ${Object.keys(DIRECT_PROVIDERS).map((p) => `${p}/*`).join(", ")} models on their own keys`,
|
|
2892
|
-
vercel: () => void 0
|
|
3317
|
+
vercel: () => void 0,
|
|
3318
|
+
openrouter: () => void 0,
|
|
3319
|
+
vertex: vertexModelRefusal
|
|
2893
3320
|
};
|
|
2894
3321
|
const gatewayScope = (config, profile) => ({ gateways: profile?.gateway ?? config.gateway });
|
|
2895
3322
|
/**
|
|
2896
|
-
* The gateway `model` goes through: its pin, or the first in the list that can serve it.
|
|
2897
|
-
*
|
|
2898
|
-
*
|
|
3323
|
+
* The gateway `model` goes through: its pin, or the first in the list that can serve it. A list
|
|
3324
|
+
* ends with `direct()` unless it's turned off, so a model no other gateway serves takes its
|
|
3325
|
+
* provider's own key. A model object or a fake calls through itself, and is undefined. Throws a UsageError
|
|
2899
3326
|
* naming each gateway tried.
|
|
2900
3327
|
*/
|
|
2901
3328
|
function resolveModelGateway(model, pin, scope, what) {
|
|
2902
3329
|
if (typeof model !== "string" || !model.includes("/")) return void 0;
|
|
2903
3330
|
const on = `${what} "${model}"`;
|
|
2904
3331
|
const listed = scope.gateways.map((g) => g.kind);
|
|
3332
|
+
if (typeof pin === "object" && pin.kind === "direct" && pin.enabled === false) throw new UsageError(directOffPin(on));
|
|
2905
3333
|
const name = typeof pin === "object" ? pin.kind : pin;
|
|
2906
3334
|
if (name !== void 0) {
|
|
2907
|
-
if (!listed.includes(name)) throw new UsageError(`${on} pins gateway '${name}',
|
|
3335
|
+
if (!listed.includes(name)) throw new UsageError(`${on} pins gateway '${name}', ${notListed(name, listed.join(", "))}`);
|
|
2908
3336
|
const refused = MODEL_GATEWAYS[name](model);
|
|
2909
3337
|
if (refused) throw new UsageError(`${on} pins gateway '${name}', which can't serve it (${refused}). A pin never falls back; pin another gateway, or remove the pin to take the first in the list that can.`);
|
|
2910
3338
|
return name;
|
|
@@ -2916,9 +3344,11 @@ function resolveModelGateway(model, pin, scope, what) {
|
|
|
2916
3344
|
skipped.push(`${kind}: ${refused}`);
|
|
2917
3345
|
}
|
|
2918
3346
|
const other = GATEWAY_NAMES.find((n) => !listed.includes(n) && !MODEL_GATEWAYS[n](model));
|
|
2919
|
-
const judge = other
|
|
2920
|
-
throw new UsageError(`${on}: no gateway in the list can serve it (${skipped.join("; ")}).${other ? ` Add ${gatewayHelper(other)} to gateway.${judge}` : ""}`);
|
|
3347
|
+
const judge = other && other !== "direct" && what.includes("judge") ? ` There a judge must be a decision model, such as '${other === "openrouter" ? OPENROUTER_JUDGE_MODEL : RECOMMENDED_JUDGE_MODEL}'.` : "";
|
|
3348
|
+
throw new UsageError(`${on}: no gateway in the list can serve it (${skipped.join("; ")}).${other === "direct" ? ` Drop direct({ enabled: false }) from gateway to fall back to its provider's own key.` : other ? ` Add ${gatewayHelper(other)} to gateway.${judge}` : ""}`);
|
|
2921
3349
|
}
|
|
3350
|
+
/** Whether an unpinned model resolved to `direct` past the list's other entries, for the plan and doctor. */
|
|
3351
|
+
const fellBackToDirect = (gateway, pin, scope) => gateway === "direct" && pin === void 0 && scope.gateways[0]?.kind !== "direct";
|
|
2922
3352
|
/** As `resolveModelGateway`, but undefined where it would throw, for checks the plan already makes. */
|
|
2923
3353
|
function modelGatewayOrNone(model, pin, scope) {
|
|
2924
3354
|
try {
|
|
@@ -2943,6 +3373,7 @@ const CONFIG_KEYS = [
|
|
|
2943
3373
|
"capture",
|
|
2944
3374
|
"question",
|
|
2945
3375
|
"timeout",
|
|
3376
|
+
"snapshot",
|
|
2946
3377
|
"concurrency",
|
|
2947
3378
|
"record",
|
|
2948
3379
|
"history",
|
|
@@ -2973,7 +3404,8 @@ const DEFAULTS_KEYS = [
|
|
|
2973
3404
|
"judge",
|
|
2974
3405
|
"capture",
|
|
2975
3406
|
"question",
|
|
2976
|
-
"timeout"
|
|
3407
|
+
"timeout",
|
|
3408
|
+
"snapshot"
|
|
2977
3409
|
];
|
|
2978
3410
|
const LIMIT_KEYS = [
|
|
2979
3411
|
"runs",
|
|
@@ -3133,7 +3565,7 @@ function configFromData(data) {
|
|
|
3133
3565
|
* Resolves a config in its data form (a parsed `reeval.yaml`, or `ReevalConfigData` from another
|
|
3134
3566
|
* language) whose relative host paths resolve against `root`, its directory: `resolveConfig`'s
|
|
3135
3567
|
* checks name a mistake in their own words first, and the config contract refuses anything they
|
|
3136
|
-
* let through, such as a `null` or a
|
|
3568
|
+
* let through, such as a `null` or a misspelled nested key.
|
|
3137
3569
|
*/
|
|
3138
3570
|
function resolveConfigData(input, root, file) {
|
|
3139
3571
|
if (!isRecord(input)) throw new UsageError(`${file} must be a mapping of config keys, like harness: and judge: (got ${JSON.stringify(input)})`);
|
|
@@ -3237,12 +3669,13 @@ function checkProfile(name, profile, file) {
|
|
|
3237
3669
|
};
|
|
3238
3670
|
return p;
|
|
3239
3671
|
}
|
|
3240
|
-
const GATEWAY_EXAMPLE = "gateway: [
|
|
3672
|
+
const GATEWAY_EXAMPLE = "gateway: [vercel()]";
|
|
3241
3673
|
const isGatewayName = (v) => typeof v === "string" && GATEWAY_NAMES.includes(v);
|
|
3242
3674
|
/**
|
|
3243
3675
|
* A `gateway` list as descriptors: each entry is a helper's descriptor, or as data its name or a
|
|
3244
|
-
* one-key map from its name to its settings.
|
|
3245
|
-
*
|
|
3676
|
+
* one-key map from its name to its settings. `direct()` ends the list unless it's listed in
|
|
3677
|
+
* another place, and `direct({ enabled: false })` leaves it out, so the resolved list is the one
|
|
3678
|
+
* every route walks. Refuses an empty list, an unknown entry and a gateway listed twice.
|
|
3246
3679
|
*/
|
|
3247
3680
|
function checkGateways(value, where, file) {
|
|
3248
3681
|
if (!Array.isArray(value) || value.length === 0) throw new UsageError(`${where} in ${file} must be a list of gateways in the order you prefer them, e.g. ${GATEWAY_EXAMPLE} (got ${JSON.stringify(value)})`);
|
|
@@ -3273,10 +3706,20 @@ function checkGateways(value, where, file) {
|
|
|
3273
3706
|
const kind = isRecord(descriptor) ? descriptor.kind : void 0;
|
|
3274
3707
|
throw new UsageError(isGatewayName(kind) ? `${at} isn't a valid ${gatewayHelper(kind)}: ${parsed.error.issues[0]?.message ?? "unknown settings"} (got ${JSON.stringify(entry)})` : `${at} must be a gateway, e.g. direct() from 'reeval/gateway' (got ${describeValue(entry)})`);
|
|
3275
3708
|
}
|
|
3276
|
-
|
|
3709
|
+
const twice = out.find((g) => g.kind === parsed.data.kind);
|
|
3710
|
+
if (twice?.kind === "direct" && parsed.data.kind === "direct") {
|
|
3711
|
+
if (twice.enabled === false !== (parsed.data.enabled === false)) throw new UsageError(`${at} lists direct() and direct({ enabled: false }). direct() places the fallback to each provider's own key and direct({ enabled: false }) turns it off: keep one.`);
|
|
3712
|
+
}
|
|
3713
|
+
if (twice) throw new UsageError(`${at} lists ${parsed.data.kind} a second time. Each gateway is listed once, where you prefer it.`);
|
|
3277
3714
|
out.push(parsed.data);
|
|
3278
3715
|
});
|
|
3279
|
-
|
|
3716
|
+
const direct = out.find((g) => g.kind === "direct");
|
|
3717
|
+
if (direct?.enabled === false) {
|
|
3718
|
+
const rest = out.filter((g) => g.kind !== "direct");
|
|
3719
|
+
if (!rest.length) throw new UsageError(`${where} in ${file} turns direct() off and lists no other gateway, so no model call has one. Add a gateway such as vercel(), or drop direct({ enabled: false }).`);
|
|
3720
|
+
return rest;
|
|
3721
|
+
}
|
|
3722
|
+
return direct ? out.map((g) => g.kind === "direct" ? { kind: "direct" } : g) : [...out, { kind: "direct" }];
|
|
3280
3723
|
}
|
|
3281
3724
|
/** `--profile`, then `REEVAL_PROFILE`, then `defaultProfile.ci` in CI (or `.local`), then none. */
|
|
3282
3725
|
function selectProfile(config, flag, env = process.env) {
|
|
@@ -3318,7 +3761,7 @@ function selectProfile(config, flag, env = process.env) {
|
|
|
3318
3761
|
function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
3319
3762
|
const user = withHostPaths(input, root, file);
|
|
3320
3763
|
if ("compare" in user) throw new UsageError(`compare can't be set in ${file}: it would multiply every eval. Put it on the group or eval that asks the question.`);
|
|
3321
|
-
if ("questions" in user) throw new UsageError(`questions can't be set in ${file}: each is about one eval's task. Put them on the group or eval they ask about; question: { model
|
|
3764
|
+
if ("questions" in user) throw new UsageError(`questions can't be set in ${file}: each is about one eval's task. Put them on the group or eval they ask about; question: { model } in ${file} sets every question's model.`);
|
|
3322
3765
|
checkOptions(user, file, CONFIG_KEYS);
|
|
3323
3766
|
const include = user.include ?? defaultIncludeGlobs();
|
|
3324
3767
|
if (!Array.isArray(include) || include.some((p) => typeof p !== "string")) throw new UsageError(`include in ${file} must be a list of glob patterns, e.g. include: ['evals/**/*.eval.ts'] (got ${JSON.stringify(include)})`);
|
|
@@ -3352,7 +3795,7 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
|
3352
3795
|
harness,
|
|
3353
3796
|
sandbox: user.sandbox ?? docker(),
|
|
3354
3797
|
environment: user.environment,
|
|
3355
|
-
runs: user.runs
|
|
3798
|
+
runs: user.runs,
|
|
3356
3799
|
gate: user.gate,
|
|
3357
3800
|
env: user.env ?? {},
|
|
3358
3801
|
...user.egress !== void 0 ? { egress: user.egress } : {},
|
|
@@ -3363,6 +3806,10 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
|
3363
3806
|
capture: user.capture ?? {},
|
|
3364
3807
|
question: user.question ?? {},
|
|
3365
3808
|
timeoutMs: parseDuration(user.timeout ?? "15m"),
|
|
3809
|
+
snapshot: mergeSnapshot({
|
|
3810
|
+
when: true,
|
|
3811
|
+
compress: false
|
|
3812
|
+
}, user.snapshot),
|
|
3366
3813
|
concurrency: user.concurrency === void 0 ? void 0 : checkConcurrency(user.concurrency, `concurrency in ${file}`),
|
|
3367
3814
|
record,
|
|
3368
3815
|
history: {
|
|
@@ -3378,35 +3825,27 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
|
3378
3825
|
}
|
|
3379
3826
|
const ANALYSIS_DEFAULTS = {
|
|
3380
3827
|
maxInputTokens: 2e4,
|
|
3381
|
-
maxOutputTokens: 8e3
|
|
3382
|
-
insights: true,
|
|
3383
|
-
maxTokens: 25e4
|
|
3828
|
+
maxOutputTokens: 8e3
|
|
3384
3829
|
};
|
|
3385
3830
|
function checkAnalysis(analysis, file) {
|
|
3386
3831
|
if (analysis === void 0) return { ...ANALYSIS_DEFAULTS };
|
|
3387
3832
|
if (typeof analysis !== "object" || analysis === null) throw new UsageError(`analysis in ${file} must be an object like { model: 'google/gemini-3.8-flash' } (got ${JSON.stringify(analysis)})`);
|
|
3388
3833
|
unknownKey(analysis, [
|
|
3389
3834
|
"model",
|
|
3390
|
-
"insights",
|
|
3391
|
-
"maxTokens",
|
|
3392
3835
|
"instructions",
|
|
3393
3836
|
"maxInputTokens",
|
|
3394
3837
|
"maxOutputTokens"
|
|
3395
3838
|
], "analysis", file);
|
|
3396
|
-
const { model,
|
|
3839
|
+
const { model, instructions, maxInputTokens = ANALYSIS_DEFAULTS.maxInputTokens, maxOutputTokens = ANALYSIS_DEFAULTS.maxOutputTokens } = analysis;
|
|
3397
3840
|
if (typeof model === "string") {
|
|
3398
3841
|
const problem = checkModelString(model, `analysis.model in ${file}`, [`google/${model}`]);
|
|
3399
3842
|
if (problem) throw new UsageError(problem);
|
|
3400
3843
|
} else if (model !== void 0 && !(isObjectLike(model) && model.kind === "fake-analysis")) throw new UsageError(`analysis.model in ${file} must be a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${JSON.stringify(model)})`);
|
|
3401
|
-
if (typeof insights !== "boolean") throw new UsageError(`analysis.insights in ${file} must be true or false (got ${JSON.stringify(insights)})`);
|
|
3402
|
-
if (!(Number.isInteger(maxTokens) && maxTokens >= 1)) throw new UsageError(`analysis.maxTokens in ${file} must be a whole number of tokens of at least 1, e.g. 250000 (got ${JSON.stringify(maxTokens)})`);
|
|
3403
3844
|
if (instructions !== void 0 && typeof instructions !== "string") throw new UsageError(`analysis.instructions in ${file} must be a string (got ${JSON.stringify(instructions)})`);
|
|
3404
3845
|
if (!(Number.isInteger(maxInputTokens) && maxInputTokens >= 1e3)) throw new UsageError(`analysis.maxInputTokens in ${file} must be a whole number of at least 1000 (got ${maxInputTokens})`);
|
|
3405
3846
|
if (!(Number.isInteger(maxOutputTokens) && maxOutputTokens >= 500)) throw new UsageError(`analysis.maxOutputTokens in ${file} must be a whole number of at least 500 (got ${maxOutputTokens})`);
|
|
3406
3847
|
return {
|
|
3407
3848
|
...analysis,
|
|
3408
|
-
insights,
|
|
3409
|
-
maxTokens,
|
|
3410
3849
|
maxInputTokens,
|
|
3411
3850
|
maxOutputTokens
|
|
3412
3851
|
};
|
|
@@ -3482,8 +3921,6 @@ function secretSources(env) {
|
|
|
3482
3921
|
}
|
|
3483
3922
|
/** The model that answers what each eval asks: `analysis.model` only, never the judge's. */
|
|
3484
3923
|
const analysisModelOf = (config) => config.analysis.model;
|
|
3485
|
-
/** Whether `analysis.model` is set, so a model can answer what each eval asks. */
|
|
3486
|
-
const hasAnalysisModel = (config) => analysisModelOf(config) !== void 0;
|
|
3487
3924
|
/** The run's judge model: the profile's when it sets one, else the config's. */
|
|
3488
3925
|
function runJudgeModel(config, profile) {
|
|
3489
3926
|
return profile ? profileJudge(profile).model ?? config.judge.model : config.judge.model;
|
|
@@ -3494,30 +3931,43 @@ function runModelGateway(config, profile, model, judge = false) {
|
|
|
3494
3931
|
}
|
|
3495
3932
|
/**
|
|
3496
3933
|
* Names the environment variables the run's providers need but don't have: the harnesses it
|
|
3497
|
-
* plans (default: the config's), the run's judge
|
|
3498
|
-
* and required config and profile secrets.
|
|
3499
|
-
* prints no insights, and never stops for it.
|
|
3934
|
+
* plans (default: the config's), the run's judge and the judges its attempts resolve, the models
|
|
3935
|
+
* that answer questions, and required config and profile secrets.
|
|
3500
3936
|
*/
|
|
3501
3937
|
function missingCredentials(config, env = process.env, options = {}) {
|
|
3502
3938
|
const missing = [];
|
|
3503
3939
|
const keys = new Set((options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []));
|
|
3504
3940
|
for (const key of keys) if (!env[key]) missing.push(key);
|
|
3505
3941
|
const judge = runJudgeModel(config, options.profile);
|
|
3506
|
-
const models = [
|
|
3507
|
-
|
|
3508
|
-
|
|
3509
|
-
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
|
|
3513
|
-
|
|
3514
|
-
|
|
3942
|
+
const models = [
|
|
3943
|
+
...typeof judge === "string" ? [{
|
|
3944
|
+
model: judge,
|
|
3945
|
+
gateway: runModelGateway(config, options.profile, judge, true),
|
|
3946
|
+
what: "the judge model"
|
|
3947
|
+
}] : [],
|
|
3948
|
+
...(options.judges ?? []).map((j) => ({
|
|
3949
|
+
...j,
|
|
3950
|
+
what: "the judge model"
|
|
3951
|
+
})),
|
|
3952
|
+
...(options.questionModels ?? []).map((model) => ({
|
|
3953
|
+
model,
|
|
3954
|
+
gateway: runModelGateway(config, options.profile, model),
|
|
3955
|
+
what: "questions"
|
|
3956
|
+
}))
|
|
3957
|
+
];
|
|
3515
3958
|
const named = /* @__PURE__ */ new Set();
|
|
3516
|
-
for (const { model,
|
|
3517
|
-
if (
|
|
3518
|
-
|
|
3519
|
-
|
|
3520
|
-
|
|
3959
|
+
for (const { model, gateway, what } of models) {
|
|
3960
|
+
if (gateway === "vertex") {
|
|
3961
|
+
const gateways = gatewayScope(config, options.profile).gateways;
|
|
3962
|
+
const line = `${VERTEX_PROJECT_ENV} (for vertex()) for ${what}, "${model}"`;
|
|
3963
|
+
if (!gateways.some((g) => g.kind === "vertex" && g.project) && !env["GOOGLE_CLOUD_PROJECT"] && !missing.includes(line)) missing.push(line);
|
|
3964
|
+
continue;
|
|
3965
|
+
}
|
|
3966
|
+
const keys = judgeCredentials(model, gateway);
|
|
3967
|
+
const once = `${keys.join(" ")}\0${model}`;
|
|
3968
|
+
if (named.has(once)) continue;
|
|
3969
|
+
named.add(once);
|
|
3970
|
+
if (!keys.some((k) => env[k])) missing.push(`${modelKeyOptions(model, gateway)} for ${what}, "${model}"`);
|
|
3521
3971
|
}
|
|
3522
3972
|
const vars = {
|
|
3523
3973
|
...config.env,
|
|
@@ -3527,16 +3977,16 @@ function missingCredentials(config, env = process.env, options = {}) {
|
|
|
3527
3977
|
return missing;
|
|
3528
3978
|
}
|
|
3529
3979
|
/**
|
|
3530
|
-
* Keys the run sends in request headers (the agents' credentials, the run's
|
|
3531
|
-
* questions' models') whose value has
|
|
3532
|
-
*
|
|
3533
|
-
*
|
|
3534
|
-
* rather than failing to authenticate.
|
|
3980
|
+
* Keys the run sends in request headers (the agents' credentials, the run's and attempts' judges'
|
|
3981
|
+
* and the questions' models') whose value has whitespace or a control character in it, with where
|
|
3982
|
+
* the first one is: a key pasted from a wrapped terminal line picks up a line break, and the agent
|
|
3983
|
+
* then hangs on a request it can't send rather than failing to authenticate.
|
|
3535
3984
|
*/
|
|
3536
3985
|
function malformedCredentials(config, env = process.env, options = {}) {
|
|
3537
3986
|
const names = (options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []);
|
|
3538
3987
|
const judge = runJudgeModel(config, options.profile);
|
|
3539
3988
|
if (judge !== void 0) names.push(...judgeCredentials(judge, runModelGateway(config, options.profile, judge, true)));
|
|
3989
|
+
for (const { model, gateway } of options.judges ?? []) names.push(...judgeCredentials(model, gateway));
|
|
3540
3990
|
for (const model of options.questionModels ?? []) names.push(...judgeCredentials(model, runModelGateway(config, options.profile, model)));
|
|
3541
3991
|
return [...new Set(names)].flatMap((name) => {
|
|
3542
3992
|
const at = badCharacterAt(env[name] ?? "");
|
|
@@ -3544,6 +3994,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
|
|
|
3544
3994
|
});
|
|
3545
3995
|
}
|
|
3546
3996
|
//#endregion
|
|
3547
|
-
export {
|
|
3997
|
+
export { egressLabel as $, harnessId as A, NO_JUDGE_INPUT as At, sandboxId as B, decideState as Bt, routedHarness as C, isAgentKind as Ct, checkOptions as D, writePluginFiles as Dt, checkLimitHarness as E, refusedPlugins as Et, plainHeaders as F, judgeProblem as Ft, unbrokeredError as G, sandboxSetBy as H, resolveJudgeArtifacts as Ht, plannedSandboxes as I, sharedInputFields as It, DEFAULT_EGRESS as J, mergeJudge as K, profileJudge as L, turnsSchema as Lt, majorityOf as M, inputsOn as Mt, matchesTags as N, judgeFields as Nt, evalPlan as O, captureWindowSchema as Ot, plainHeaderError as P, judgeModelNameSchema as Pt, directOnCloud as Q, refuseIllegalCompare as R, artifactRefsOf as Rt, resolveGateway as S, expectedBlockedFor as St, cascade as T, refusedMcp as Tt, secretVarsIn as U, sandboxProvider as V, filePartsOf as Vt, unbrokered as W, allowedHosts as X, DEFAULT_EGRESS_ALLOW as Y, ciRunner as Z, fellBackToDirect as _, agentMcp as _t, configPathFor as a, awaitListening as at, resolveModelGateway as b, claudeMcpServers as bt, findConfig as c, STOPPED_FIRST as ct, malformedCredentials as d, resolveQuestionState as dt, gatewayPolicy as et, missingCredentials as f, harnessEffort as ft, selectProfile as g, agentFor as gt, runJudgeModel as h, agentAuth as ht, configLabel as i, createFakeSandbox as it, levelOf as j, fakeJudgeDataSchema as jt, expand as k, INPUT_KEYS as kt, hostCapacity as l, UNRECORDED_QUESTION_STATE as lt, resolveConcurrency as m, servedEffortOf as mt, analysisModelOf as n, unenforceable as nt, defaultConcurrency as o, parseProbe as ot, requireConfig as p, resolvedEffortOf as pt, resolveStateOptions as q, configFilesPhrase as r, resolveOutput as rt, defineConfig as s, probeScript as st, CONFIG_FILES as t, isDirect as tt, loadConfig as u, questionShows as ut, gatewayScope as v, authHint as vt, brokeredIn as w, layoutOf as wt, fallbackHint as x, credentialOf as xt, modelGatewayOrNone as y, brokeringFor as yt, runsOf as z, badJudgeArtifacts as zt };
|
|
3548
3998
|
|
|
3549
|
-
//# sourceMappingURL=config-
|
|
3999
|
+
//# sourceMappingURL=config-CneBZB84.js.map
|