reeval 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -259
- package/THIRD_PARTY_NOTICES.md +121 -121
- package/assets/package.json +1 -1
- package/assets/pnpm-lock.yaml +41 -30
- package/dist/{adapter-DigeVFWC.js → adapter-BloGVZ_a.js} +20 -8
- package/dist/adapter-BloGVZ_a.js.map +1 -0
- package/dist/{agent-CatUEo5h.js → agent-m-_t-IuW.js} +54 -26
- package/dist/{agent-CatUEo5h.js.map → agent-m-_t-IuW.js.map} +1 -1
- package/dist/{analysis-Bxub9LMT.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-Bxub9LMT.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-C-jwhRDt.js +11 -0
- package/dist/app-files-C-jwhRDt.js.map +1 -0
- package/dist/{assemble-CF063ms6.js → assemble-BDEpelpi.js} +95 -38
- package/dist/assemble-BDEpelpi.js.map +1 -0
- package/dist/assemble-C5bfncHU.js +1 -0
- package/dist/{attempt-CGPV3BZY.js → attempt-jI0OWxlz.js} +2 -2
- package/dist/{attempt-CGPV3BZY.js.map → attempt-jI0OWxlz.js.map} +1 -1
- package/dist/{capture-BiD7lVaM.js → capture-CVRwRTnb.js} +574 -38
- package/dist/capture-CVRwRTnb.js.map +1 -0
- package/dist/capture-CherbjzB.js +1 -0
- package/dist/{cli-B5zQCR2r.js → cli-C_4gaJ50.js} +145 -142
- package/dist/cli-C_4gaJ50.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-CpnbATtI.js} +2 -2
- package/dist/{cli-spec-BXIBUlaK.js.map → cli-spec-CpnbATtI.js.map} +1 -1
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-D5hNjpEx.js} +105 -2
- package/dist/client-D5hNjpEx.js.map +1 -0
- package/dist/{command-C94CQNh8.js → command-BWAuwQNk.js} +12 -12
- package/dist/{command-C94CQNh8.js.map → command-BWAuwQNk.js.map} +1 -1
- package/dist/{command-CaZgCTIw.js → command-BXLUfw4h.js} +14 -14
- package/dist/command-BXLUfw4h.js.map +1 -0
- package/dist/{command-CVtgyqlM.js → command-Be6BSE2L.js} +12 -12
- package/dist/{command-CVtgyqlM.js.map → command-Be6BSE2L.js.map} +1 -1
- package/dist/{command-BMHXHqMT.js → command-Bxdy62dW.js} +15 -14
- package/dist/command-Bxdy62dW.js.map +1 -0
- package/dist/{command-DTgBZR0k.js → command-I1euLLio.js} +35 -15
- package/dist/command-I1euLLio.js.map +1 -0
- package/dist/{commands-CjpGBUWr.js → commands-jzigbNFb.js} +23 -14
- package/dist/commands-jzigbNFb.js.map +1 -0
- package/dist/{commands-BXzIDB3X.js → commands-p1S4nw_H.js} +1 -1
- package/dist/config-BrdHA9bJ.js +3549 -0
- package/dist/config-BrdHA9bJ.js.map +1 -0
- package/dist/{contrasts-CcoMLC_y.js → contrasts-DRxMQQTC.js} +3 -2
- package/dist/contrasts-DRxMQQTC.js.map +1 -0
- package/dist/cost-D4elx933.js +359 -0
- package/dist/cost-D4elx933.js.map +1 -0
- package/dist/cost-DmnAod9x.js +1 -0
- package/dist/db-CAfksinX.js +2 -0
- package/dist/{db-BeqRiWLl.js → db-DO0p7xgP.js} +25 -8
- package/dist/db-DO0p7xgP.js.map +1 -0
- package/dist/{dist-CMjp3qqF.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-CMjp3qqF.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-BbTlMmYi.js → docker-BnRlTpZA.js} +3 -3
- package/dist/{docker-BbTlMmYi.js.map → docker-BnRlTpZA.js.map} +1 -1
- package/dist/{docker-CCXdbREk.js → docker-D01slDF3.js} +29 -28
- package/dist/docker-D01slDF3.js.map +1 -0
- package/dist/{docker-DC69dKxB.js → docker-DivfZl2u.js} +1 -1
- package/dist/{doctor-CqttrZcs.js → doctor-DQ2p0euG.js} +136 -38
- package/dist/doctor-DQ2p0euG.js.map +1 -0
- package/dist/{environment-BBz1Yes9.js → environment-CO0P0soF.js} +31 -11
- package/dist/environment-CO0P0soF.js.map +1 -0
- package/dist/{evaluate-CXQ1o-pd.js → evaluate-BUmsN7yG.js} +68 -29
- package/dist/evaluate-BUmsN7yG.js.map +1 -0
- package/dist/{evaluate-yjZKUexW.d.ts → evaluate-C_sgRA-V.d.ts} +45 -3
- package/dist/{facts-DtENlf9s.js → facts-DzOUTreB.js} +13 -5
- package/dist/facts-DzOUTreB.js.map +1 -0
- package/dist/{files-B-eUpCZ1.js → files-B-CJeeEh.js} +14 -9
- package/dist/{files-B-eUpCZ1.js.map → files-B-CJeeEh.js.map} +1 -1
- package/dist/files-Bp5j5BqI.js +1 -0
- package/dist/files-C2sjmXhk.js +1 -0
- package/dist/{files-BsQbYz2i.js → files-C_gQNVB-.js} +4 -4
- package/dist/{files-BsQbYz2i.js.map → files-C_gQNVB-.js.map} +1 -1
- package/dist/{files-COHQLEHE.js → files-kUCrswI6.js} +3 -3
- package/dist/{files-COHQLEHE.js.map → files-kUCrswI6.js.map} +1 -1
- package/dist/gateway.d.ts +17 -0
- package/dist/gateway.js +17 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{generate-Cs2TMH_k.js → generate-DZ7vp69F.js} +18 -18
- package/dist/generate-DZ7vp69F.js.map +1 -0
- package/dist/{git-DhykwgKF.js → git-Bk8DSwHy.js} +3 -3
- package/dist/{git-DhykwgKF.js.map → git-Bk8DSwHy.js.map} +1 -1
- package/dist/{harness-BAjw1fcr.js → harness-COVXSwJ6.js} +46 -15
- package/dist/harness-COVXSwJ6.js.map +1 -0
- package/dist/harness.d.ts +26 -18
- package/dist/harness.js +1 -1
- package/dist/{header-kSmWatcO.js → header-BLPP6aCU.js} +2 -2
- package/dist/{header-kSmWatcO.js.map → header-BLPP6aCU.js.map} +1 -1
- package/dist/{images-BmJRuTpm.js → images-DzLeFjLx.js} +8 -8
- package/dist/{images-BmJRuTpm.js.map → images-DzLeFjLx.js.map} +1 -1
- package/dist/index.d.ts +3 -45
- package/dist/index.js +5 -6
- package/dist/init-RgCQXZhb.js +1 -0
- package/dist/{init-Bwc9byQr.js → init-SRalM5xF.js} +6 -8
- package/dist/init-SRalM5xF.js.map +1 -0
- package/dist/{insights-B2Xmhz9o.js → insights-TyAvz9aV.js} +5 -1
- package/dist/{insights-B2Xmhz9o.js.map → insights-TyAvz9aV.js.map} +1 -1
- package/dist/{invocations-CQTbP092.js → invocations-DzyKSJBa.js} +3 -3
- package/dist/{invocations-CQTbP092.js.map → invocations-DzyKSJBa.js.map} +1 -1
- package/dist/judge-yLvjC5A3.js +385 -0
- package/dist/judge-yLvjC5A3.js.map +1 -0
- package/dist/judge.d.ts +7 -6
- package/dist/judge.js +7 -5
- package/dist/judge.js.map +1 -1
- package/dist/match-Bz0nSdzN.js +204 -0
- package/dist/match-Bz0nSdzN.js.map +1 -0
- package/dist/{matchers-Ddmc2z7H.js → matchers-D4eUqx4j.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-D4eUqx4j.js.map} +1 -1
- package/dist/{model-B-mel-Gv.js → model-1ZLr9Dcv.js} +59 -35
- package/dist/model-1ZLr9Dcv.js.map +1 -0
- package/dist/model-B45lDUV1.js +68 -0
- package/dist/model-B45lDUV1.js.map +1 -0
- package/dist/model-BTZq8HIZ.js +1 -0
- package/dist/{model-CWlAqmbU.js → model-CgnOpss4.js} +58 -316
- package/dist/model-CgnOpss4.js.map +1 -0
- package/dist/model-Ch8kX87u.js +1 -0
- package/dist/{node-D0mtjRAO.js → node-BSq5qHnQ.js} +12 -5
- package/dist/node-BSq5qHnQ.js.map +1 -0
- package/dist/{opencode-providers-J9colP0-.js → opencode-providers-klkqgzvb.js} +72 -62
- package/dist/opencode-providers-klkqgzvb.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-CHfZNRT6.js} +8 -7
- package/dist/pairs-CHfZNRT6.js.map +1 -0
- package/dist/{patch-D4oluWzH.js → patch-DWf-I_Q_.js} +2 -2
- package/dist/{patch-D4oluWzH.js.map → patch-DWf-I_Q_.js.map} +1 -1
- package/dist/{plan-OkwUcJQ9.js → plan-BIwUzlB8.js} +63 -26
- package/dist/plan-BIwUzlB8.js.map +1 -0
- package/dist/{plugins-CUWMnSNb.js → plugins-Dkp_H3pB.js} +5 -5
- package/dist/{plugins-CUWMnSNb.js.map → plugins-Dkp_H3pB.js.map} +1 -1
- package/dist/{preflight-B6pY89bF.js → preflight-D9y-HvZQ.js} +1 -1
- package/dist/{preflight-Bjm7_ylE.js → preflight-DJCD_BDn.js} +3 -3
- package/dist/{preflight-Bjm7_ylE.js.map → preflight-DJCD_BDn.js.map} +1 -1
- package/dist/{prepare-DbdgtZKe.js → prepare-BJG_pD54.js} +2 -2
- package/dist/{prepare-CQmK3SBn.js → prepare-CIxCGG0M.js} +8 -8
- package/dist/{prepare-CQmK3SBn.js.map → prepare-CIxCGG0M.js.map} +1 -1
- package/dist/prices-BcLEXvdY.js +250 -0
- package/dist/prices-BcLEXvdY.js.map +1 -0
- package/dist/{provenance-B3Y17oCC.js → provenance-CfZpFpAh.js} +3 -3
- package/dist/{provenance-B3Y17oCC.js.map → provenance-CfZpFpAh.js.map} +1 -1
- package/dist/{providers-CXN2RFz-.js → providers-D1Zxp2he.js} +1 -1
- package/dist/{providers-eoniqK6w.js → providers-DFk_6W4K.js} +5 -5
- package/dist/{providers-eoniqK6w.js.map → providers-DFk_6W4K.js.map} +1 -1
- package/dist/{query-CLRZmWBT.js → query-CemclveB.js} +2 -2
- package/dist/{query-CLRZmWBT.js.map → query-CemclveB.js.map} +1 -1
- package/dist/query-Cvbwl804.js +1 -0
- package/dist/{questions-1XOulevV.js → questions-bAudIc04.js} +309 -47
- package/dist/questions-bAudIc04.js.map +1 -0
- package/dist/{redact-DJ9JCShu.js → redact-CRGdfV2s.js} +4 -2
- package/dist/redact-CRGdfV2s.js.map +1 -0
- package/dist/{remarks-DlJCqWRs.js → remarks-CXwSnSjR.js} +9 -9
- package/dist/{remarks-DlJCqWRs.js.map → remarks-CXwSnSjR.js.map} +1 -1
- package/dist/remarks-CyAu2DiB.js +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-mujJv1Ey.js} +5 -3
- package/dist/remarks-mujJv1Ey.js.map +1 -0
- package/dist/{remote-cE2VWfkJ.js → remote-CKcEKja4.js} +129 -174
- package/dist/remote-CKcEKja4.js.map +1 -0
- package/dist/resolve-B5-uqmJ6.js +1 -0
- package/dist/{resolve-C4zeccCH.js → resolve-CBoKzQMP.js} +9 -10
- package/dist/resolve-CBoKzQMP.js.map +1 -0
- package/dist/{results-Drtt0uFl.js → results-CMgZEANy.js} +8 -8
- package/dist/{results-Drtt0uFl.js.map → results-CMgZEANy.js.map} +1 -1
- package/dist/{resume-BEFazUfy.js → resume-B9SOSnK9.js} +3 -3
- package/dist/{resume-BEFazUfy.js.map → resume-B9SOSnK9.js.map} +1 -1
- package/dist/{root-DICDHCtp.js → root-Dlff-zp8.js} +5 -5
- package/dist/root-Dlff-zp8.js.map +1 -0
- package/dist/{rules-BFzQqPIm.js → rules-VhftPOOT.js} +53 -4
- package/dist/rules-VhftPOOT.js.map +1 -0
- package/dist/{run-BeMy92Us.js → run-CwIEK16q.js} +12 -12
- package/dist/run-CwIEK16q.js.map +1 -0
- package/dist/{runtime-PX_oXiCS.js → runtime-BgQRyCLl.js} +14 -2
- package/dist/runtime-BgQRyCLl.js.map +1 -0
- package/dist/{sandbox-CQaJj7n9.js → sandbox-BOYZ0hlo.js} +2 -2
- package/dist/{sandbox-CQaJj7n9.js.map → sandbox-BOYZ0hlo.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CHRSgKRp.js +1 -0
- package/dist/{server-xK0les4f.js → server-TGW4WyAz.js} +26 -27
- package/dist/{server-xK0les4f.js.map → server-TGW4WyAz.js.map} +1 -1
- package/dist/{shared-CA61BuPJ.js → shared-BDUzStad.js} +2 -2
- package/dist/shared-BDUzStad.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/{source-Q1hEKbCs.js → source-B1WvBIQt.js} +2 -2
- package/dist/{source-Q1hEKbCs.js.map → source-B1WvBIQt.js.map} +1 -1
- package/dist/source-DAzztZwU.js +1 -0
- package/dist/src-1jHbFaGW.js +1 -0
- package/dist/{src-LekQrr_S.js → src-8zoX_qrb.js} +614 -171
- package/dist/src-8zoX_qrb.js.map +1 -0
- package/dist/{src-C0CeKiLU.js → src-BYLM3hEz.js} +4 -3
- package/dist/src-BYLM3hEz.js.map +1 -0
- package/dist/{src-CFp86GtV.js → src-CHIwOy0o.js} +87 -55
- package/dist/{src-CFp86GtV.js.map → src-CHIwOy0o.js.map} +1 -1
- package/dist/src-C_9wgowC.js +2 -0
- package/dist/{src-C7_1SqDP.js → src-ChhtI6BP.js} +2 -2
- package/dist/{src-C7_1SqDP.js.map → src-ChhtI6BP.js.map} +1 -1
- package/dist/src-CyaDNG9r.js +2 -0
- package/dist/src-D005Cbqz.js +2 -0
- package/dist/src-DIfRbp5a.js +3 -0
- package/dist/{src-BhmaorOU.js → src-DKPVX2Us.js} +43 -17
- package/dist/{src-BhmaorOU.js.map → src-DKPVX2Us.js.map} +1 -1
- package/dist/src-Dg8121Xy.js +2 -0
- package/dist/{src-cbCb-MPy.js → src-DjF_M8_f.js} +44 -20
- package/dist/{src-cbCb-MPy.js.map → src-DjF_M8_f.js.map} +1 -1
- package/dist/{src-DIL0jST0.js → src-DjRI-IeE.js} +42 -14
- package/dist/src-DjRI-IeE.js.map +1 -0
- package/dist/{src-DiLMkIr9.js → src-DmUgeTlE.js} +47 -17
- package/dist/src-DmUgeTlE.js.map +1 -0
- package/dist/src-ElBqEN-S.js +2 -0
- package/dist/{src-qGoFkuJF.js → src-NSdQ9zI6.js} +88 -23
- package/dist/src-NSdQ9zI6.js.map +1 -0
- package/dist/src-YHGaUrGE.js +2 -0
- package/dist/src-mJVxw1Jm.js +2 -0
- package/dist/{state-BmZ8QVZ8.js → state-B1t143wG.js} +22 -20
- package/dist/{state-BmZ8QVZ8.js.map → state-B1t143wG.js.map} +1 -1
- package/dist/state-vznMh2wl.js +1 -0
- package/dist/stats-DSy5HfOv.js +315 -0
- package/dist/stats-DSy5HfOv.js.map +1 -0
- package/dist/suite-BsWGHzlG.js +2 -0
- package/dist/{suite-C8sw2pox.js → suite-WgT7m6Vs.js} +76 -13
- package/dist/suite-WgT7m6Vs.js.map +1 -0
- package/dist/{summary-0oQNGskv.js → summary-DIGEVr08.js} +58 -17
- package/dist/summary-DIGEVr08.js.map +1 -0
- package/dist/testing.d.ts +174 -2
- package/dist/testing.js +14 -3
- package/dist/testing.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-GraZtfHN.d.ts → types-DXBbP9Mk.d.ts} +302 -72
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/usage-DF3_p5pE.js +1 -0
- package/dist/{validate-BdoXUuL_.js → validate-BhwqrVsm.js} +48 -16
- package/dist/validate-BhwqrVsm.js.map +1 -0
- package/dist/view/index.d.ts +143 -91
- package/dist/view/index.js +7 -6
- package/dist/views-BW_C5eVH.js +2 -0
- package/dist/{views-BoUoX4_A.js → views-sXpGE6a2.js} +10 -8
- package/dist/views-sXpGE6a2.js.map +1 -0
- package/dist/{worker-entry-D0-GErmU.js → worker-entry-Duo9oAeT.js} +8 -8
- package/dist/{worker-entry-D0-GErmU.js.map → worker-entry-Duo9oAeT.js.map} +1 -1
- package/dist/{workspace-tjZFkjxC.js → workspace-CDI68wtX.js} +45 -11
- package/dist/workspace-CDI68wtX.js.map +1 -0
- package/dist/workspace-QZCacP8A.js +1 -0
- package/package.json +8 -2
- package/skills/eval-rubric/SKILL.md +9 -0
- package/dist/adapter-DigeVFWC.js.map +0 -1
- package/dist/assemble-CF063ms6.js.map +0 -1
- package/dist/assemble-CJoD_Rzr.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-BiD7lVaM.js.map +0 -1
- package/dist/capture-Dp3q7K7Y.js +0 -1
- package/dist/cli-B5zQCR2r.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-BMHXHqMT.js.map +0 -1
- package/dist/command-CaZgCTIw.js.map +0 -1
- package/dist/command-DTgBZR0k.js.map +0 -1
- package/dist/commands-CjpGBUWr.js.map +0 -1
- package/dist/config-Dv5MG7Rp.js +0 -753
- package/dist/config-Dv5MG7Rp.js.map +0 -1
- package/dist/contrasts-CcoMLC_y.js.map +0 -1
- package/dist/db-BeqRiWLl.js.map +0 -1
- package/dist/db-CJDRVQal.js +0 -2
- package/dist/docker-CCXdbREk.js.map +0 -1
- package/dist/doctor-CqttrZcs.js.map +0 -1
- package/dist/egress-BfK_O9ce.js +0 -1241
- package/dist/egress-BfK_O9ce.js.map +0 -1
- package/dist/environment-BBz1Yes9.js.map +0 -1
- package/dist/evaluate-CXQ1o-pd.js.map +0 -1
- package/dist/facts-DtENlf9s.js.map +0 -1
- package/dist/files-DqCBW7i8.js +0 -1
- package/dist/files-Ha-jsQXf.js +0 -1
- package/dist/generate-Cs2TMH_k.js.map +0 -1
- package/dist/harness-BAjw1fcr.js.map +0 -1
- package/dist/init-Bwc9byQr.js.map +0 -1
- package/dist/init-D7Y1ny1z.js +0 -1
- package/dist/judge-Dt6KW6_O.js +0 -153
- package/dist/judge-Dt6KW6_O.js.map +0 -1
- package/dist/model-B-mel-Gv.js.map +0 -1
- package/dist/model-BGKYVpoj.js +0 -1
- package/dist/model-C7tSwT7f.js +0 -44
- package/dist/model-C7tSwT7f.js.map +0 -1
- package/dist/model-CWlAqmbU.js.map +0 -1
- package/dist/model-DHY-dfoO.js +0 -1
- package/dist/node-D0mtjRAO.js.map +0 -1
- package/dist/opencode-providers-J9colP0-.js.map +0 -1
- package/dist/options-CPUW97_7.js +0 -1116
- package/dist/options-CPUW97_7.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/plan-OkwUcJQ9.js.map +0 -1
- package/dist/query-CmBDYt0_.js +0 -1
- package/dist/questions-1XOulevV.js.map +0 -1
- package/dist/redact-DJ9JCShu.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remote-cE2VWfkJ.js.map +0 -1
- package/dist/resolve-C4zeccCH.js.map +0 -1
- package/dist/resolve-D30yZYmB.js +0 -1
- package/dist/root-DICDHCtp.js.map +0 -1
- package/dist/rules-BFzQqPIm.js.map +0 -1
- package/dist/run-BeMy92Us.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-V4-2w0l8.js +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-BDow_oW8.js +0 -1
- package/dist/src-B9L9yGKI.js +0 -1
- package/dist/src-Bcah6ZkT.js +0 -3
- package/dist/src-C0CeKiLU.js.map +0 -1
- package/dist/src-C0xowZ3C.js +0 -2
- package/dist/src-CHIfIKUu.js +0 -2
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-DLtyz4w5.js +0 -2
- package/dist/src-DPIjmReZ.js +0 -2
- package/dist/src-DXNh5mTr.js +0 -2
- package/dist/src-DiLMkIr9.js.map +0 -1
- package/dist/src-LekQrr_S.js.map +0 -1
- package/dist/src-bCnsA6Vp.js +0 -2
- package/dist/src-qGoFkuJF.js.map +0 -1
- package/dist/src-ynBVosfX.js +0 -2
- package/dist/state-BQR_BCm4.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-CnYJijyI.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-BaHPmAGy.js +0 -1
- package/dist/suite-C8sw2pox.js.map +0 -1
- package/dist/summary-0oQNGskv.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-BdoXUuL_.js.map +0 -1
- package/dist/views-BoUoX4_A.js.map +0 -1
- package/dist/views-noVQNCUP.js +0 -2
- package/dist/workspace-B90YDU_a.js +0 -1
- package/dist/workspace-tjZFkjxC.js.map +0 -1
|
@@ -0,0 +1,3549 @@
|
|
|
1
|
+
import { n as isOneOf, t as isObjectLike } from "./object-CLf2-tNq.js";
|
|
2
|
+
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
3
|
+
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, i as durationSchema, l as nonEmpty, o as genericProblem, p as parseDuration, s as got, t as closest, u as oneOf } from "./shared-BDUzStad.js";
|
|
4
|
+
import { D as sandboxOf, E as sameWorkspaceOnly, F as sandboxDescriptorSchema, I as egressPolicySchema, L as envVarsSchema, M as workspaceLabel, N as workspaceOf, O as sandboxPaths, P as environmentDataSchema, R as gatePolicySchema, V as sandboxProblem, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, j as unbrokeredSecrets, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, w as plannedSecrets, y as isMovingRef, z as gateProblem } from "./environment-CO0P0soF.js";
|
|
5
|
+
import { _ as gatewayEntryDataSchema, f as FACTORY, g as gatewayDescriptorSchema, h as GATEWAY_NAMES, m as harnessProblem, p as harnessDescriptorSchema, u as opencodeModelError, v as gatewayNameSchema } from "./harness-COVXSwJ6.js";
|
|
6
|
+
import { a as fromCwd, l as sha256 } from "./ids-Bp6ut7ZD.js";
|
|
7
|
+
import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-klkqgzvb.js";
|
|
8
|
+
import { C as servedOnVercel, S as fxAskedModel, _ as VERCEL_HOST, a as checkModelString, b as VERCEL_ROUTES, g as FX_VERCEL_ENDPOINTS, h as FX_VERCEL_CALL, i as RECOMMENDED_JUDGE_MODEL, n as DIRECT_PROVIDERS, s as judgeCredentials, u as modelKeyOptions, v as VERCEL_KEY, x as VERCEL_UNPROVEN_EFFORT, y as VERCEL_REFUSALS } from "./judge-yLvjC5A3.js";
|
|
9
|
+
import { r as badCharacterAt } from "./redact-CRGdfV2s.js";
|
|
10
|
+
import { t as inCI } from "./env-BQqj93gu.js";
|
|
11
|
+
import { F as DOCKER_EGRESS, I as HARNESS_HOSTS, L as PACKAGE_HOSTS, dt as run, ft as shellQuote } from "./src-8zoX_qrb.js";
|
|
12
|
+
import { u as ownFileURL } from "./root-Dlff-zp8.js";
|
|
13
|
+
import { D as pluginServerName, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers } from "./plugins-Dkp_H3pB.js";
|
|
14
|
+
import { t as InfraError } from "./types-C5vI3g1C.js";
|
|
15
|
+
import { r as pairFactors } from "./pairs-CHfZNRT6.js";
|
|
16
|
+
import { a as reevalJiti, n as defaultIncludeGlobs } from "./adapter-BloGVZ_a.js";
|
|
17
|
+
import { t as docker } from "./sandbox-BOYZ0hlo.js";
|
|
18
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
19
|
+
import { availableParallelism, tmpdir, totalmem } from "node:os";
|
|
20
|
+
import { dirname, join, resolve, win32 } from "node:path";
|
|
21
|
+
import { parseDocument } from "yaml";
|
|
22
|
+
import { z } from "zod";
|
|
23
|
+
import { isIP } from "node:net";
|
|
24
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
25
|
+
import { spawn } from "node:child_process";
|
|
26
|
+
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
27
|
+
//#region src/judge/artifacts.ts
|
|
28
|
+
/** A capture handle: a name plus the file the attempt wrote. */
|
|
29
|
+
function isCaptureHandle(value) {
|
|
30
|
+
return isObjectLike(value) && typeof value.name === "string" && isObjectLike(value.file) && typeof value.file.path === "string";
|
|
31
|
+
}
|
|
32
|
+
/** The first `artifacts` value that isn't `true` or a list of capture handles or names. */
|
|
33
|
+
function badJudgeArtifacts(artifacts) {
|
|
34
|
+
if (artifacts === void 0 || artifacts === true) return void 0;
|
|
35
|
+
if (!Array.isArray(artifacts)) return `must be true or a list of capture handles or names (got ${JSON.stringify(artifacts)})`;
|
|
36
|
+
for (const item of artifacts) {
|
|
37
|
+
if (typeof item === "string" && item.trim()) continue;
|
|
38
|
+
if (isCaptureHandle(item)) continue;
|
|
39
|
+
return `must be true or a list of capture handles or names (got ${JSON.stringify(item)})`;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
const nameOf = (item) => typeof item === "string" ? item.trim() : item.name;
|
|
43
|
+
/**
|
|
44
|
+
* Resolves call `artifacts` (capture handles or their names, or `true` for every capture so far,
|
|
45
|
+
* in capture order) to PNG bytes this attempt captured. Unknown names throw: a mistake in the
|
|
46
|
+
* call, before the provider is asked.
|
|
47
|
+
*/
|
|
48
|
+
function resolveJudgeArtifacts(artifacts, blobs) {
|
|
49
|
+
const items = artifacts === true ? [...blobs.keys()] : artifacts;
|
|
50
|
+
if (!items?.length) return [];
|
|
51
|
+
return items.map((item) => {
|
|
52
|
+
const name = nameOf(item);
|
|
53
|
+
const data = blobs.get(name);
|
|
54
|
+
if (!data) throw new Error(`c.judge() artifacts names a capture this attempt doesn't have: ${JSON.stringify(name)}`);
|
|
55
|
+
return {
|
|
56
|
+
name,
|
|
57
|
+
path: `artifacts/${name}.png`,
|
|
58
|
+
sha256: sha256(data),
|
|
59
|
+
bytes: data.byteLength,
|
|
60
|
+
mediaType: "image/png",
|
|
61
|
+
data
|
|
62
|
+
};
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
const filePartsOf = (resolved) => resolved.map((a) => ({
|
|
66
|
+
type: "file",
|
|
67
|
+
mediaType: a.mediaType,
|
|
68
|
+
data: a.data
|
|
69
|
+
}));
|
|
70
|
+
const artifactRefsOf = (resolved) => resolved.map(({ data: _data, ...ref }) => ref);
|
|
71
|
+
/**
|
|
72
|
+
* JSON-only state is a plain object, as TypeSafe and other JSON judges read it.
|
|
73
|
+
* File parts are an ordered decide-state array: one JSON part, then each capture.
|
|
74
|
+
*/
|
|
75
|
+
function decideState(json, files) {
|
|
76
|
+
if (files.length === 0) return json;
|
|
77
|
+
return [{
|
|
78
|
+
type: "json",
|
|
79
|
+
value: json
|
|
80
|
+
}, ...files];
|
|
81
|
+
}
|
|
82
|
+
//#endregion
|
|
83
|
+
//#region src/contract/judge.ts
|
|
84
|
+
const judgeStateSchema = z.strictObject({
|
|
85
|
+
task: z.boolean().optional().describe("The graded turn's prompt. Default true."),
|
|
86
|
+
output: z.boolean().optional().describe("The graded turn's final message, or `on`'s value. Default true."),
|
|
87
|
+
files: z.boolean().optional().describe("`status path` lines for the changed files, under `diff`'s patterns. Default true."),
|
|
88
|
+
diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional().describe("The diff since the eval started. A list is gitignore-style patterns over workspace paths: the last matching pattern wins and `!` excludes. Default true."),
|
|
89
|
+
contents: z.array(nonEmpty()).optional().describe("Workspace files shown whole, as they are now, changed or not: exact paths or gitignore-style patterns (`'**'`, `'src/**'`; the last match wins, `!` excludes) over the files git sees in the workspace. A missing exact path is `null`; a pattern that matches nothing adds nothing. These files leave the diff. Default none."),
|
|
90
|
+
transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional().describe("The graded turn's transcript as text lines: `true` is messages and tool calls with tool output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. Default false."),
|
|
91
|
+
turns: z.enum(["last", "all"]).optional().describe("`'all'`: every turn up to the graded one under `turns`, each with its task, output and (with `transcript`) transcript. Default `'last'`."),
|
|
92
|
+
checks: z.union([z.boolean(), z.array(nonEmpty())]).optional().describe("`c.exec` results so far; a list shows only those commands, as passed to `c.exec`. Default false.")
|
|
93
|
+
}).describe("What the judge is given: each key is a key of the state in `judge-input.json`. An unset key keeps its default, which is what the judge gets with no `state` option at all.");
|
|
94
|
+
const judgeFields = {
|
|
95
|
+
threshold: z.number().min(0, "must be between 0 and 1").max(1, "must be between 0 and 1").optional().describe("Minimum score (0–1) for a `pass` verdict."),
|
|
96
|
+
providerOptions: z.record(z.string(), z.record(z.string(), jsonValueSchema)).optional().describe("AI SDK provider options for the judge's calls, keyed by provider."),
|
|
97
|
+
maxConcurrent: z.number().int().min(1).optional().describe("The most judge calls to this model's provider at once, across the run."),
|
|
98
|
+
state: judgeStateSchema.optional().describe("The judge's state: what the judge is given beside the question, not the workspace's or the attempt's state. Merged key by key down the cascade."),
|
|
99
|
+
gateway: z.union([gatewayNameSchema, gatewayDescriptorSchema]).optional().describe("The gateway the judge's calls go through, by name or by its helper. It skips the config's `gateway` fallback: the plan refuses a gateway that isn't listed or can't serve the model.")
|
|
100
|
+
};
|
|
101
|
+
const fakeJudgeDataSchema = z.strictObject({
|
|
102
|
+
kind: z.literal("fake-judge"),
|
|
103
|
+
score: z.number().min(0).max(1).describe("The score (0–1) every call gets.")
|
|
104
|
+
}).describe("`fakeJudge()` as data: the score every call gets.");
|
|
105
|
+
const judgeModelNameSchema = z.string().regex(/\//, "must be a 'provider/model' string, e.g. 'google/gemini-3.8-flash'");
|
|
106
|
+
const configJudgeDataSchema = z.strictObject({
|
|
107
|
+
model: z.union([z.string(), fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge."),
|
|
108
|
+
...judgeFields
|
|
109
|
+
}).describe("`ConfigJudgeOptions` as data: the model by name, or a fake judge with a fixed score.");
|
|
110
|
+
const judgeOptionsSchema = z.strictObject({
|
|
111
|
+
model: z.custom().optional(),
|
|
112
|
+
...judgeFields
|
|
113
|
+
});
|
|
114
|
+
const judgeCallSchema = judgeOptionsSchema.extend({
|
|
115
|
+
on: z.custom().optional(),
|
|
116
|
+
artifacts: z.union([z.literal(true), z.array(z.custom())]).optional()
|
|
117
|
+
});
|
|
118
|
+
const STATE_KEYS$1 = Object.keys(judgeStateSchema.shape);
|
|
119
|
+
const JUDGE_HINTS = {
|
|
120
|
+
see: "state",
|
|
121
|
+
shows: "state",
|
|
122
|
+
show: "state",
|
|
123
|
+
input: "state"
|
|
124
|
+
};
|
|
125
|
+
const VALUES = {
|
|
126
|
+
task: "true or false",
|
|
127
|
+
output: "true or false",
|
|
128
|
+
files: "true or false",
|
|
129
|
+
diff: "true, false or a non-empty list of patterns",
|
|
130
|
+
contents: "a list of paths or patterns",
|
|
131
|
+
transcript: "true, false, 'tools' or 'full'",
|
|
132
|
+
turns: "'last' or 'all'",
|
|
133
|
+
checks: "true, false or a list of commands"
|
|
134
|
+
};
|
|
135
|
+
function distance(a, b) {
|
|
136
|
+
const row = Array.from({ length: b.length + 1 }, (_, i) => i);
|
|
137
|
+
for (let i = 1; i <= a.length; i++) {
|
|
138
|
+
let prev = row[0] ?? 0;
|
|
139
|
+
row[0] = i;
|
|
140
|
+
for (let j = 1; j <= b.length; j++) {
|
|
141
|
+
const here = row[j] ?? 0;
|
|
142
|
+
row[j] = Math.min(here + 1, (row[j - 1] ?? 0) + 1, prev + (a[i - 1] === b[j - 1] ? 0 : 1));
|
|
143
|
+
prev = here;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return row[b.length] ?? 0;
|
|
147
|
+
}
|
|
148
|
+
const nearest = (key, keys) => keys.find((k) => distance(key.toLowerCase(), k.toLowerCase()) <= 2);
|
|
149
|
+
const shown = (value) => JSON.stringify(value) ?? String(value);
|
|
150
|
+
/**
|
|
151
|
+
* The first problem with a judge options object, as one sentence, or undefined. The checks run in
|
|
152
|
+
* the order each site has always run them, so a value with several problems names the same one.
|
|
153
|
+
*/
|
|
154
|
+
function judgeProblem(judge, site) {
|
|
155
|
+
/** A judge field and where it was set: `judge.threshold in evals/a.eval.ts › e`. */
|
|
156
|
+
const field = (key) => {
|
|
157
|
+
if (site.at === "call") return `c.judge() ${key ?? "options"}`;
|
|
158
|
+
const base = site.at === "config" ? site.path : "judge";
|
|
159
|
+
return `${key ? `${base}.${key}` : base} in ${site.at === "config" ? site.file : site.where}`;
|
|
160
|
+
};
|
|
161
|
+
const loc = site.at === "call" ? " in c.judge()" : site.at === "options" ? ` in ${site.where}` : ` in ${site.path === "judge" ? "" : `${site.path.replace(/\.judge$/, "")} in `}${site.file}`;
|
|
162
|
+
if (typeof judge !== "object" || judge === null || Array.isArray(judge)) return `${field()} must be an object such as { threshold: 0.8 } (got ${shown(judge)})`;
|
|
163
|
+
const value = judge;
|
|
164
|
+
const model = value.model;
|
|
165
|
+
if (site.at === "config" && typeof model === "string") {
|
|
166
|
+
const problem = checkModelString(model, field("model"));
|
|
167
|
+
if (problem) return problem;
|
|
168
|
+
}
|
|
169
|
+
const schema = site.at === "call" ? judgeCallSchema : site.at === "config" ? configJudgeDataSchema : judgeOptionsSchema;
|
|
170
|
+
const parsed = schema.safeParse(judge);
|
|
171
|
+
if (parsed.success) return void 0;
|
|
172
|
+
const numericFake = site.at === "config" && typeof model === "object" && model?.kind === "fake-judge" && typeof model.score === "number";
|
|
173
|
+
const issues = parsed.error.issues.filter((i) => !(numericFake && i.path[0] === "model"));
|
|
174
|
+
if (!issues.length) return void 0;
|
|
175
|
+
const at = (key) => issues.some((i) => i.path[0] === key);
|
|
176
|
+
const order = [];
|
|
177
|
+
const modelProblem = () => {
|
|
178
|
+
if (site.at !== "config" || !at("model")) return void 0;
|
|
179
|
+
if (typeof model === "object" && model?.kind === "fake-judge") return `${field("model")} must be fakeJudge({ score }) with a number. Pass a score function per group, eval or call instead.`;
|
|
180
|
+
return `${field("model")} must be a "provider/model" string or fakeJudge(). Pass a model object per group, eval or call instead.`;
|
|
181
|
+
};
|
|
182
|
+
const maxConcurrentProblem = () => {
|
|
183
|
+
if (site.at === "config" || !at("maxConcurrent")) return void 0;
|
|
184
|
+
return `${field("maxConcurrent")} must be a whole number of at least 1 (got ${shown(value.maxConcurrent)})`;
|
|
185
|
+
};
|
|
186
|
+
const thresholdProblem = () => {
|
|
187
|
+
if (!at("threshold")) return void 0;
|
|
188
|
+
return `${field("threshold")} must be between 0 and 1 (got ${shown(value.threshold)})`;
|
|
189
|
+
};
|
|
190
|
+
const keyProblem = () => {
|
|
191
|
+
const name = issues.find((i) => i.code === "unrecognized_keys" && i.path.length === 0)?.keys[0];
|
|
192
|
+
if (name === void 0) return void 0;
|
|
193
|
+
const allowed = Object.keys(schema.shape);
|
|
194
|
+
const hint = JUDGE_HINTS[name] ?? (STATE_KEYS$1.includes(name) ? `state.${name}` : nearest(name, allowed));
|
|
195
|
+
const why = site.at !== "call" && (name === "on" || name === "artifacts") ? ` ${name} is per call: c.judge(question, { ${name} }).` : hint ? ` Did you mean "${hint}"?` : "";
|
|
196
|
+
return `Unknown judge option "${name}"${loc}.${why}`;
|
|
197
|
+
};
|
|
198
|
+
const stateProblem = () => {
|
|
199
|
+
const issue = issues.find((i) => i.path[0] === "state");
|
|
200
|
+
if (!issue) return void 0;
|
|
201
|
+
const { state } = value;
|
|
202
|
+
if (typeof state !== "object" || state === null || Array.isArray(state)) return `${field("state")} must be an object such as { transcript: 'tools' } (got ${shown(state)})`;
|
|
203
|
+
if (issue.code === "unrecognized_keys" && issue.path.length === 1) {
|
|
204
|
+
const key = issue.keys[0] ?? "";
|
|
205
|
+
const hint = nearest(key, STATE_KEYS$1);
|
|
206
|
+
return `Unknown judge.state key "${key}"${loc}.${hint ? ` Did you mean "${hint}"?` : ""}`;
|
|
207
|
+
}
|
|
208
|
+
const key = issue.path[1];
|
|
209
|
+
return `${field(`state.${String(key)}`)} must be ${VALUES[key]} (got ${shown(state[key])})`;
|
|
210
|
+
};
|
|
211
|
+
const providerOptionsProblem = () => at("providerOptions") ? `${field("providerOptions")} must map each provider to an object of JSON values, e.g. { google: { thinkingConfig: { thinkingBudget: 0 } } } (got ${shown(value.providerOptions)})` : void 0;
|
|
212
|
+
const artifactsProblem = () => at("artifacts") ? `c.judge() artifacts ${badJudgeArtifacts(value.artifacts)}` : void 0;
|
|
213
|
+
const gatewayProblem = () => at("gateway") ? `${field("gateway")} names a gateway in the config's gateway list: ${GATEWAY_NAMES.map((g) => `'${g}'`).join(" or ")}, or is its helper (got ${shown(value.gateway)})` : void 0;
|
|
214
|
+
if (site.at === "config") order.push(modelProblem, keyProblem, stateProblem, providerOptionsProblem, thresholdProblem, gatewayProblem);
|
|
215
|
+
else if (site.at === "call") order.push(thresholdProblem, maxConcurrentProblem, keyProblem, stateProblem, providerOptionsProblem, artifactsProblem, gatewayProblem);
|
|
216
|
+
else order.push(maxConcurrentProblem, thresholdProblem, keyProblem, stateProblem, providerOptionsProblem, gatewayProblem);
|
|
217
|
+
for (const check of order) {
|
|
218
|
+
const problem = check();
|
|
219
|
+
if (problem) return problem;
|
|
220
|
+
}
|
|
221
|
+
return `judge options${loc} aren't valid: ${issues[0]?.message ?? "unknown problem"}`;
|
|
222
|
+
}
|
|
223
|
+
//#endregion
|
|
224
|
+
//#region src/contract/questions.ts
|
|
225
|
+
const pairFactorSchema = z.enum(["environment", "harness"]).describe("A matrix factor a pairwise question varies; the others are held.");
|
|
226
|
+
/** Evaluation models score yes/no questions only, so they can't answer a question. */
|
|
227
|
+
const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
|
|
228
|
+
const questionStateSchema = z.strictObject({
|
|
229
|
+
task: z.boolean().optional().describe("The prompts the attempt was sent, once for both sides when they match. Default true."),
|
|
230
|
+
output: z.boolean().optional().describe("The final message: the agent's own account of its work. Default false in a pair, true per attempt."),
|
|
231
|
+
files: z.boolean().optional().describe("`status path` lines for the changed files, under `diff`'s patterns. Default false."),
|
|
232
|
+
diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional().describe("The diff since the eval started. A list is gitignore-style patterns over workspace paths: the last matching pattern wins and `!` excludes. Default true."),
|
|
233
|
+
transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional().describe("The transcript as text lines: `true` is messages and tool calls with tool output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each harness's tools, so a pair of two harnesses isn't blind. Default false in a pair, true per attempt."),
|
|
234
|
+
turns: z.enum(["last", "all"]).optional().describe("`'all'`: every turn under `turns`, each with its task, output and (with `transcript`) transcript; `'last'`: the last turn only. Default `'all'`."),
|
|
235
|
+
checks: z.union([z.boolean(), z.array(nonEmpty())]).optional().describe("`c.exec` results; a list shows only those commands, as passed to `c.exec`. Default false.")
|
|
236
|
+
}).describe("What a question is shown of each attempt: the keys of `judge.state` but `contents` (the attempts' workspaces are gone when questions are asked). An unset key keeps its default: a pairwise question shows the task and the diff; one asked of each attempt also shows the final message and the transcript.");
|
|
237
|
+
const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty()).min(1)]);
|
|
238
|
+
const questionModelSchema = z.string().refine((m) => m.includes("/"), { params: { provider: true } }).refine((m) => !isEvaluationModel(m), { params: { evaluation: true } });
|
|
239
|
+
const questionOptionsSchema = z.strictObject({
|
|
240
|
+
model: questionModelSchema.optional().describe("`'provider/model'` that answers every question below it without its own `model`."),
|
|
241
|
+
state: questionStateSchema.optional().describe("What it's shown of each attempt."),
|
|
242
|
+
artifacts: questionArtifactsSchema.optional().describe("Each attempt's own captures, by name; `true` sends all of them. Default none.")
|
|
243
|
+
}).describe("Defaults for every question: `model` and `artifacts` replaced, `state` merged key by key.");
|
|
244
|
+
/** The first item a list holds twice. */
|
|
245
|
+
const twice = (items) => items.find((k, i) => items.indexOf(k) !== i);
|
|
246
|
+
const unique = (list, ctx) => {
|
|
247
|
+
if (twice(list) !== void 0) ctx.addIssue({
|
|
248
|
+
code: "custom",
|
|
249
|
+
message: "twice"
|
|
250
|
+
});
|
|
251
|
+
};
|
|
252
|
+
const acrossSchema = z.union([pairFactorSchema, z.array(pairFactorSchema).min(1).superRefine(unique)]);
|
|
253
|
+
const choicesSchema = z.array(nonEmpty()).min(2).superRefine(unique);
|
|
254
|
+
const questionObjectSchema = z.strictObject({
|
|
255
|
+
question: nonEmpty(),
|
|
256
|
+
model: questionModelSchema.optional().describe("`'provider/model'` that answers it, over the cascaded `question.model`."),
|
|
257
|
+
across: acrossSchema.optional().describe("The factor it varies when pairing, or both: `'environment'` holds the harness, `'harness'` holds the environment. Unset, it's asked of each attempt on its own."),
|
|
258
|
+
choices: choicesSchema.optional().describe("The answers a question asked of each attempt picks from, with a short reason."),
|
|
259
|
+
state: questionStateSchema.optional().describe("What it's shown of each attempt, over the cascaded `question.state`, key by key."),
|
|
260
|
+
artifacts: questionArtifactsSchema.optional().describe("Each attempt's own captures, by name; `true` sends all of them. Replaces `question.artifacts`.")
|
|
261
|
+
}).superRefine((q, ctx) => {
|
|
262
|
+
if (q.across !== void 0 && q.choices !== void 0) ctx.addIssue({
|
|
263
|
+
code: "custom",
|
|
264
|
+
path: ["choices"],
|
|
265
|
+
message: "with across"
|
|
266
|
+
});
|
|
267
|
+
});
|
|
268
|
+
const questionSchema = z.union([nonEmpty(), questionObjectSchema]).describe("A question a language model answers after the attempts: its text, or the text and its options. With `across`, it picks between two attempts that differ only in that factor, in both orders; without it, it's asked of each attempt on its own. Without a model, the cascaded `question.model` answers, else `analysis.model`, else the judge's when it's a language model. `state` and `artifacts` choose what it's shown of each attempt.");
|
|
269
|
+
const questionText = (q) => typeof q === "string" ? q : q.question;
|
|
270
|
+
const questionsSchema = z.array(questionSchema).superRefine((list, ctx) => {
|
|
271
|
+
if (twice(list.map(questionText)) !== void 0) ctx.addIssue({
|
|
272
|
+
code: "custom",
|
|
273
|
+
message: "twice"
|
|
274
|
+
});
|
|
275
|
+
});
|
|
276
|
+
const QUESTION_EXAMPLE = `questions: ['How did the agent find the failing test?']`;
|
|
277
|
+
const ACROSS_HINT = `'environment' (vary the environment, same harness) or 'harness' (vary the harness, same environment)`;
|
|
278
|
+
const ACROSS_MEAN = " Did you mean \"across\"? It is the factor a pairwise question varies: across: 'environment'.";
|
|
279
|
+
const QUESTION_KEYS = Object.keys(questionObjectSchema.shape);
|
|
280
|
+
const QUESTION_SHAPE = "{ question, model?, across?, choices?, state?, artifacts? }";
|
|
281
|
+
const STATE_KEYS = Object.keys(questionStateSchema.shape);
|
|
282
|
+
const STATE_VALUES = {
|
|
283
|
+
task: "true or false",
|
|
284
|
+
output: "true or false",
|
|
285
|
+
files: "true or false",
|
|
286
|
+
diff: "true, false or a non-empty list of patterns",
|
|
287
|
+
transcript: "true, false, 'tools' or 'full'",
|
|
288
|
+
turns: "'last' or 'all'",
|
|
289
|
+
checks: "true, false or a list of commands"
|
|
290
|
+
};
|
|
291
|
+
const describe$2 = (value) => {
|
|
292
|
+
if (Array.isArray(value)) return `a list of ${value.length}`;
|
|
293
|
+
if (typeof value === "object" && value !== null) {
|
|
294
|
+
const keys = Object.keys(value);
|
|
295
|
+
return keys.length ? `an object with ${keys.map((k) => `"${k}"`).join(", ")}` : "an object";
|
|
296
|
+
}
|
|
297
|
+
return JSON.stringify(value) ?? String(value);
|
|
298
|
+
};
|
|
299
|
+
/**
|
|
300
|
+
* What's wrong with a question's `state` or `artifacts`, as one sentence, or undefined. `name`
|
|
301
|
+
* turns a key into how the sentence names it: `question.state` on the option,
|
|
302
|
+
* `questions "…" state` on an entry.
|
|
303
|
+
*/
|
|
304
|
+
function questionPartsProblem(value, name, where) {
|
|
305
|
+
const { state, artifacts } = value;
|
|
306
|
+
if (state !== void 0) {
|
|
307
|
+
if (!isObjectLike(state) || Array.isArray(state)) return `${name("state")} in ${where} must be an object such as { transcript: 'tools' } (got ${describe$2(state)})`;
|
|
308
|
+
for (const [key, v] of Object.entries(state)) {
|
|
309
|
+
if (key === "contents") return `${name("state.contents")} in ${where} can't be set: questions are asked after the attempts, when their workspaces are gone. Narrow state.diff instead.`;
|
|
310
|
+
const schema = questionStateSchema.shape[key];
|
|
311
|
+
if (!schema) {
|
|
312
|
+
const hint = closest(key, STATE_KEYS);
|
|
313
|
+
return `Unknown ${name("state")} key "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""}`;
|
|
314
|
+
}
|
|
315
|
+
if (!schema.safeParse(v).success) return `${name(`state.${key}`)} in ${where} must be ${STATE_VALUES[key]} (got ${describe$2(v)})`;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
if (artifacts !== void 0 && !questionArtifactsSchema.safeParse(artifacts).success) return `${name("artifacts")} in ${where} must be true or a non-empty list of capture names (got ${describe$2(artifacts)})`;
|
|
319
|
+
}
|
|
320
|
+
/** What's wrong with the `question` option set in `where`, as one sentence, or undefined. */
|
|
321
|
+
function questionOptionsProblem(value, where) {
|
|
322
|
+
if (!isObjectLike(value) || Array.isArray(value)) return `question in ${where} is { model?, state?, artifacts? } (got ${describe$2(value)}), e.g. question: { model: 'google/gemini-3.8-flash' }`;
|
|
323
|
+
const keys = Object.keys(questionOptionsSchema.shape);
|
|
324
|
+
for (const key of Object.keys(value)) {
|
|
325
|
+
if (keys.includes(key)) continue;
|
|
326
|
+
if (key === "across" || key === "choices") return `Unknown question option "${key}" in ${where}. ${key} goes on each questions entry, { question, ${key} }: it says what kind of question that entry is.`;
|
|
327
|
+
const hint = closest(key, keys);
|
|
328
|
+
return `Unknown question option "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""} question takes model, state and artifacts.`;
|
|
329
|
+
}
|
|
330
|
+
const { model } = value;
|
|
331
|
+
if (model !== void 0 && typeof model === "string" && isEvaluationModel(model)) return `question.model in ${where} names ${model}, an evaluation model: it scores yes/no questions and can't answer a question. Use a language model, e.g. 'google/gemini-3.8-flash'.`;
|
|
332
|
+
if (model !== void 0 && !questionModelSchema.safeParse(model).success) return `question.model in ${where} is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${describe$2(model)})`;
|
|
333
|
+
return questionPartsProblem(value, (key) => `question.${key}`, where);
|
|
334
|
+
}
|
|
335
|
+
/** What's wrong with an entry's `across`, or undefined. */
|
|
336
|
+
function acrossProblem(question, across, where) {
|
|
337
|
+
if (acrossSchema.safeParse(across).success) return void 0;
|
|
338
|
+
const at = `questions "${question}" in ${where}`;
|
|
339
|
+
if (Array.isArray(across) && across.length > 0 && across.every((f) => pairFactorSchema.safeParse(f).success)) return `${at}: across lists "${twice(across)}" twice. List each factor once.`;
|
|
340
|
+
const bad = Array.isArray(across) ? across.find((f) => !pairFactorSchema.safeParse(f).success) : across;
|
|
341
|
+
return `${at}: across is ${ACROSS_HINT}, or a list of both (got ${describe$2(Array.isArray(across) && across.length === 0 ? across : bad)}). Leave it out to ask each attempt on its own.`;
|
|
342
|
+
}
|
|
343
|
+
/** What's wrong with an entry's `choices`, or undefined. */
|
|
344
|
+
function choicesProblem(question, choices, where) {
|
|
345
|
+
if (choicesSchema.safeParse(choices).success) return void 0;
|
|
346
|
+
const at = `questions "${question}" choices in ${where}`;
|
|
347
|
+
if (Array.isArray(choices) && choices.length >= 2 && choices.every((c) => nonEmpty().safeParse(c).success)) return `${at} lists "${twice(choices)}" twice. List each answer once.`;
|
|
348
|
+
const blank = Array.isArray(choices) && choices.find((c) => !nonEmpty().safeParse(c).success);
|
|
349
|
+
if (blank !== void 0 && blank !== false) return `${at} has an answer that isn't a non-empty string (got ${describe$2(blank)}), e.g. choices: ['the test', 'the code']`;
|
|
350
|
+
return `${at} is a list of at least two answers, e.g. choices: ['the test', 'the code'] (got ${describe$2(choices)})`;
|
|
351
|
+
}
|
|
352
|
+
/** One question's sentence: the union refuses it whole, so the branch it meant says why. */
|
|
353
|
+
function questionProblem(item, where) {
|
|
354
|
+
if (typeof item === "string") return `questions in ${where} has an empty string. Ask something, e.g. ${QUESTION_EXAMPLE}.`;
|
|
355
|
+
if (typeof item !== "object" || item === null || Array.isArray(item)) return `questions in ${where} takes strings or ${QUESTION_SHAPE} (got ${describe$2(item)}), e.g. ${QUESTION_EXAMPLE}.`;
|
|
356
|
+
const unknown = Object.keys(item).filter((k) => !QUESTION_KEYS.includes(k));
|
|
357
|
+
if (unknown.length) {
|
|
358
|
+
const hint = unknown.some((k) => k === "cut" || k === "cuts") ? ACROSS_MEAN : "";
|
|
359
|
+
return `questions in ${where} takes ${QUESTION_SHAPE} (got ${unknown.map((k) => `"${k}"`).join(", ")}).${hint}`;
|
|
360
|
+
}
|
|
361
|
+
const { question, model, across, choices } = item;
|
|
362
|
+
if (!nonEmpty().safeParse(question).success) return `questions in ${where} has an entry without its question${question === void 0 ? "" : ` (got ${describe$2(question)})`}, e.g. { question: 'Which is cleaner?', across: 'environment' }.`;
|
|
363
|
+
const text = question;
|
|
364
|
+
if (across !== void 0) {
|
|
365
|
+
const problem = acrossProblem(text, across, where);
|
|
366
|
+
if (problem) return problem;
|
|
367
|
+
}
|
|
368
|
+
if (choices !== void 0) {
|
|
369
|
+
const problem = choicesProblem(text, choices, where);
|
|
370
|
+
if (problem) return problem;
|
|
371
|
+
}
|
|
372
|
+
if (across !== void 0 && choices !== void 0) return `questions "${text}" in ${where} sets both across and choices. choices are the answers a question asked of each attempt picks from, and across makes it pick between two attempts instead: drop choices to vote between pairs, or drop across to ask each attempt.`;
|
|
373
|
+
const parts = questionPartsProblem(item, (key) => `questions "${text}" ${key}`, where);
|
|
374
|
+
if (parts) return parts;
|
|
375
|
+
if (typeof model === "string" && isEvaluationModel(model)) return `questions "${text}" in ${where} names ${model}, an evaluation model: it scores yes/no questions and can't answer a question. Use a language model, e.g. 'google/gemini-3.8-flash'.`;
|
|
376
|
+
return `questions "${text}" in ${where}: model is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${describe$2(model)})`;
|
|
377
|
+
}
|
|
378
|
+
/** What's wrong with the `questions` option set in `where`, as one sentence, or undefined. */
|
|
379
|
+
function questionsProblem(value, where) {
|
|
380
|
+
if (questionsSchema.safeParse(value).success) return void 0;
|
|
381
|
+
if (!Array.isArray(value)) return `questions in ${where} is a list (got ${describe$2(value)}), e.g. ${QUESTION_EXAMPLE}.`;
|
|
382
|
+
const bad = value.find((q) => !questionSchema.safeParse(q).success);
|
|
383
|
+
if (bad !== void 0) return questionProblem(bad, where);
|
|
384
|
+
return `questions in ${where} asks "${twice(value.map(questionText))}" twice. Ask each question once.`;
|
|
385
|
+
}
|
|
386
|
+
//#endregion
|
|
387
|
+
//#region src/contract/config.ts
|
|
388
|
+
const count = () => z.number().int().min(1);
|
|
389
|
+
const captureWindowSchema = z.strictObject({
|
|
390
|
+
width: count().describe("In CSS pixels."),
|
|
391
|
+
height: count().describe("In CSS pixels.")
|
|
392
|
+
}).describe("Live tab size for `c.capture()`. Default 1280×720 when the call and cascade omit `viewport`.");
|
|
393
|
+
const captureOptionsSchema = z.strictObject({
|
|
394
|
+
viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional().describe("The window and saved PNG size, or `'full-page'`. Default 1280×720."),
|
|
395
|
+
timeout: durationSchema.optional().describe("How long to poll before failing. Default `'60s'`."),
|
|
396
|
+
browser: z.boolean().optional().describe("`false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws."),
|
|
397
|
+
model: z.string().includes("/").optional().describe("Vision model for `instructions`. Cascades like `question`. Default is `analysis.model`, else the attempt's judge model, whichever is first a language model; never Jev.")
|
|
398
|
+
}).describe("Camera defaults for `c.capture()`. Cascades config → profile defaults → group → eval, shallow-merged. `viewport` on a level replaces the whole value. `port` is not an option: pass it as the first argument to `c.capture`.");
|
|
399
|
+
const harnessList = z.union([harnessDescriptorSchema, z.array(harnessDescriptorSchema).min(1)]);
|
|
400
|
+
const budgetSchema = z.strictObject({
|
|
401
|
+
attempts: count().optional().describe("The most attempts the plan may have."),
|
|
402
|
+
duration: durationSchema.optional().describe("How long the run may go on, from the first attempt's start.")
|
|
403
|
+
}).describe("Run-level caps. `attempts` is checked against the plan; `duration` while running.");
|
|
404
|
+
const historyConfigSchema = z.strictObject({
|
|
405
|
+
dir: z.string().optional().describe("Committed history folder (default: `.history` in the directory the evals live in)."),
|
|
406
|
+
artifacts: z.enum([
|
|
407
|
+
"all",
|
|
408
|
+
"failures",
|
|
409
|
+
"none"
|
|
410
|
+
]).optional().describe("Artifacts recorded next to each run: every eval's, failing evals' only, or none.")
|
|
411
|
+
});
|
|
412
|
+
const recordMode = z.enum([
|
|
413
|
+
"manual",
|
|
414
|
+
"ci",
|
|
415
|
+
"always"
|
|
416
|
+
]);
|
|
417
|
+
const profileDefaultsDataSchema = z.strictObject({
|
|
418
|
+
harness: harnessList.optional().describe("The agents every eval runs on, unless a group or eval replaces them."),
|
|
419
|
+
sandbox: sandboxDescriptorSchema.optional().describe("The default machine when the resolved environment omits `sandbox`."),
|
|
420
|
+
environment: environmentDataSchema.optional().describe("What every eval runs in when it names no environment."),
|
|
421
|
+
runs: count().optional().describe("Attempts per cell."),
|
|
422
|
+
gate: gatePolicySchema.optional().describe("How each cell's attempts decide its outcome."),
|
|
423
|
+
env: envVarsSchema.optional().describe("Variables and secrets for the sandbox, merged by name over the config's."),
|
|
424
|
+
egress: egressPolicySchema.optional().describe("Where the sandbox may connect; replaces the config's policy."),
|
|
425
|
+
judge: configJudgeDataSchema.optional().describe("Defaults for `c.judge()`, merged field by field over the config's."),
|
|
426
|
+
timeout: durationSchema.optional().describe("Per attempt."),
|
|
427
|
+
capture: captureOptionsSchema.optional().describe("Screenshot defaults for `c.capture()`, shallow-merged over the config's."),
|
|
428
|
+
question: questionOptionsSchema.optional().describe("Defaults for every question, merged over the config's.")
|
|
429
|
+
}).describe("`ProfileDefaults` as data: the environment without `extend`, and the judge as data.");
|
|
430
|
+
const profileLimitSchema = z.strictObject({
|
|
431
|
+
runs: count().optional().describe("The most runs per cell."),
|
|
432
|
+
compare: z.union([z.literal(false), z.strictObject({ maxVariants: z.number().int().min(0) })]).optional().describe("`false` runs the control only; `{ maxVariants }` keeps the first n variants."),
|
|
433
|
+
harness: z.array(z.string()).optional().describe("Harness ids to allow; evals left with none are skipped.")
|
|
434
|
+
}).describe("Over every eval, and it can only narrow; `pin` on a group or eval skips it.");
|
|
435
|
+
const profileDataSchema = z.strictObject({
|
|
436
|
+
defaults: profileDefaultsDataSchema.optional().describe("What evals get when they don't say otherwise: under groups and evals."),
|
|
437
|
+
limit: profileLimitSchema.optional().describe("Caps over every eval, which only narrow; `pin` on a group or eval skips them."),
|
|
438
|
+
judge: configJudgeDataSchema.optional().describe("Judge settings for this profile, the same as `defaults.judge`, which wins field by field."),
|
|
439
|
+
budget: budgetSchema.optional().describe("Merged over the config's `budget`."),
|
|
440
|
+
record: recordMode.optional().describe("When this profile's runs are copied into the history folder."),
|
|
441
|
+
tags: z.array(z.string()).optional().describe("Tag filters for this profile, like `--tag` (`'!slow'` excludes)."),
|
|
442
|
+
gateway: z.array(gatewayEntryDataSchema).min(1).optional().describe("Replaces the config's `gateway` list for this profile's runs.")
|
|
443
|
+
}).describe("`Profile` as data: its defaults and judge as data.");
|
|
444
|
+
const fakeVote = z.enum([
|
|
445
|
+
"A",
|
|
446
|
+
"B",
|
|
447
|
+
"tie"
|
|
448
|
+
]);
|
|
449
|
+
const failure = z.strictObject({ error: z.string() });
|
|
450
|
+
const fakeAnalysisDataSchema = z.strictObject({
|
|
451
|
+
kind: z.literal("fake-analysis"),
|
|
452
|
+
output: z.unknown().optional().describe("The model's structured answer."),
|
|
453
|
+
vote: z.union([
|
|
454
|
+
fakeVote,
|
|
455
|
+
z.strictObject({
|
|
456
|
+
winner: fakeVote,
|
|
457
|
+
reason: z.string().optional()
|
|
458
|
+
}),
|
|
459
|
+
failure
|
|
460
|
+
]).optional().describe("How it answers pairwise questions: a fixed pick, or `{ error }` to fail every call."),
|
|
461
|
+
answer: z.union([
|
|
462
|
+
z.string(),
|
|
463
|
+
z.strictObject({
|
|
464
|
+
choice: z.string().optional(),
|
|
465
|
+
text: z.string().optional()
|
|
466
|
+
}),
|
|
467
|
+
failure
|
|
468
|
+
]).optional().describe("How it answers questions asked of each attempt: a fixed answer, or `{ error }` to fail every call.")
|
|
469
|
+
}).describe("`fakeAnalysis()` as data: canned answers, without the functions of the digest or pair.");
|
|
470
|
+
const analysisConfigDataSchema = z.strictObject({
|
|
471
|
+
model: z.union([z.string(), fakeAnalysisDataSchema]).optional().describe("`'provider/model'`, e.g. `'google/gemini-3.8-flash'`. Insights use it only; `reeval analyze` and questions fall back to the judge's when it's a language model."),
|
|
472
|
+
insights: z.boolean().optional().describe("`reeval run` classifies each eval and explains its result with `model` once the attempts finish (default true when `model` is set; `--no-insights` turns it off for one run)."),
|
|
473
|
+
maxTokens: z.number().int().min(1).optional().describe("Tokens (in and out) that one run's insights may use, over every classification, insight and headline call (default 250000). Past it, the evals left print without insights; the run's outcome and exit code never change."),
|
|
474
|
+
instructions: z.string().optional().describe("Added to the prompt, e.g. \"Fewer tool calls count as a win. The skill under test is reeval-good.\""),
|
|
475
|
+
maxInputTokens: z.number().int().min(1e3).optional().describe("The digest's cap; evidence is dropped, lowest priority first, to fit (default 20000)."),
|
|
476
|
+
maxOutputTokens: z.number().int().min(500).optional().describe("The most tokens the model may answer with (default 8000). reeval has no prices, so tokens are the cap.")
|
|
477
|
+
}).describe("`AnalysisConfig` as data: the model by name, or a fake analysis.");
|
|
478
|
+
const reevalConfigDataSchema = z.strictObject({
|
|
479
|
+
include: z.array(z.string()).optional().describe("Globs of eval files, relative to the config's directory; default every `.eval.ts` under `evals`."),
|
|
480
|
+
harness: harnessList.optional().describe("The agents every eval runs on, unless a group or eval replaces them; default Cursor."),
|
|
481
|
+
sandbox: sandboxDescriptorSchema.optional().describe("The default machine when the resolved environment omits `sandbox`. Not an environment: it does not invent skills, files or a name. Implicit `docker()` when nobody sets one."),
|
|
482
|
+
environment: environmentDataSchema.optional().describe("What every eval runs in when it names no environment, and a baseline-less compare's control."),
|
|
483
|
+
runs: count().optional().describe("Attempts per cell; default 1."),
|
|
484
|
+
gate: gatePolicySchema.optional().describe("How each cell's attempts decide its outcome; default `'all'`."),
|
|
485
|
+
env: envVarsSchema.optional().describe("Variables and secrets for the sandbox, merged by name down the cascade."),
|
|
486
|
+
egress: egressPolicySchema.optional().describe("Where the sandbox may connect; default `'proxy'`."),
|
|
487
|
+
judge: configJudgeDataSchema.optional().describe("Defaults for `c.judge()`, merged field by field down the cascade."),
|
|
488
|
+
timeout: durationSchema.optional().describe("Per attempt; default `'15m'`."),
|
|
489
|
+
capture: captureOptionsSchema.optional().describe("Screenshot defaults for `c.capture()`. `REEVAL_CAPTURE_BROWSER=0` refuses the host binary when `c.capture()` runs; it does not change this object."),
|
|
490
|
+
question: questionOptionsSchema.optional().describe("Defaults for every question: the model that answers it and what it's shown of each attempt. Merged down the cascade."),
|
|
491
|
+
concurrency: count().optional().describe("How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it."),
|
|
492
|
+
record: recordMode.optional().describe("When runs are copied into the history folder: `reeval record` only, in CI, or on every run."),
|
|
493
|
+
history: historyConfigSchema.optional().describe("The history folder, and which attempts keep their artifacts there."),
|
|
494
|
+
budget: budgetSchema.optional().describe("Caps on a run; a profile's `budget` is merged over it."),
|
|
495
|
+
profiles: z.record(z.string(), profileDataSchema).optional().describe("Named presets of `defaults`, `limit`, `judge`, `budget`, `record` and `tags`."),
|
|
496
|
+
defaultProfile: z.strictObject({
|
|
497
|
+
ci: z.string().optional(),
|
|
498
|
+
local: z.string().optional()
|
|
499
|
+
}).optional().describe("The profile when neither `--profile` nor `REEVAL_PROFILE` names one: `ci` when `CI` is set."),
|
|
500
|
+
secrets: z.strictObject({ unbrokered: z.enum(["warn", "error"]).optional() }).optional().describe("`unbrokered: 'error'` refuses Docker secrets without `hosts` instead of warning (default `'warn'`)."),
|
|
501
|
+
debug: z.strictObject({
|
|
502
|
+
keep: count().optional(),
|
|
503
|
+
traceBufferMb: z.number().positive().optional()
|
|
504
|
+
}).optional().describe("Local debug data in `.reeval/debug`: how many runs to keep (default 20), and how many MB of raw trace each attempt buffers for trace.ndjson (default 8; the oldest entries go first)."),
|
|
505
|
+
analysis: analysisConfigDataSchema.optional().describe("Insights during `reeval run`, `reeval analyze`, and questions without a model."),
|
|
506
|
+
gateway: z.array(gatewayEntryDataSchema).min(1).optional().describe("The gateways model calls may go through, in the order you prefer them: each harness takes the first that can serve it. Default `[direct]`, each provider's own key.")
|
|
507
|
+
}).describe("`ReevalConfig` in its data forms, as `reeval.yaml` or another language carries it: the environment without `extend`, judge and analysis models by name or as fakes with fixed answers, and profiles the same way. AI SDK model objects and fake score functions are TypeScript only.");
|
|
508
|
+
/** The object schema an issue's path leads to through `schema`, for "Did you mean". */
|
|
509
|
+
function keysAt(schema, path, value) {
|
|
510
|
+
const def = schema.def;
|
|
511
|
+
if (def.innerType) return keysAt(def.innerType, path, value);
|
|
512
|
+
if (def.getter) return keysAt(def.getter(), path, value);
|
|
513
|
+
if (def.options) return [...new Set(def.options.flatMap((option) => keysAt(option, path, value)))];
|
|
514
|
+
const [key, ...rest] = path;
|
|
515
|
+
if (key === void 0) return def.shape ? Object.keys(def.shape) : [];
|
|
516
|
+
const next = valueAt(value, [key]);
|
|
517
|
+
if (def.shape) {
|
|
518
|
+
const field = def.shape[String(key)];
|
|
519
|
+
return field ? keysAt(field, rest, next) : [];
|
|
520
|
+
}
|
|
521
|
+
const inner = def.valueType ?? def.element;
|
|
522
|
+
return inner ? keysAt(inner, rest, next) : [];
|
|
523
|
+
}
|
|
524
|
+
/**
|
|
525
|
+
* The issue under a union that went furthest into the value: the branch the value was most
|
|
526
|
+
* likely meant to be. A union where every branch failed at its own path is left as it is.
|
|
527
|
+
*/
|
|
528
|
+
function deepest(issue) {
|
|
529
|
+
if (issue.code !== "invalid_union" || !("errors" in issue) || !issue.errors.length) return issue;
|
|
530
|
+
let best;
|
|
531
|
+
for (const branch of issue.errors) {
|
|
532
|
+
const inner = branch[0];
|
|
533
|
+
if (!inner) continue;
|
|
534
|
+
const found = deepest({
|
|
535
|
+
...inner,
|
|
536
|
+
path: [...issue.path, ...inner.path]
|
|
537
|
+
});
|
|
538
|
+
if (!best || found.path.length > best.path.length) best = found;
|
|
539
|
+
}
|
|
540
|
+
return best && best.path.length > issue.path.length ? best : issue;
|
|
541
|
+
}
|
|
542
|
+
/**
|
|
543
|
+
* reeval's sentence for what the config contract refuses in `value` (the parsed `reeval.yaml`,
|
|
544
|
+
* or a config another language sent), naming the field by its path, or undefined.
|
|
545
|
+
*/
|
|
546
|
+
function configDataProblem(value, file) {
|
|
547
|
+
const result = reevalConfigDataSchema.safeParse(value);
|
|
548
|
+
if (result.success) return void 0;
|
|
549
|
+
const { issues } = result.error;
|
|
550
|
+
const issue = deepest(issues.find((i) => i.code === "unrecognized_keys") ?? issues[0]);
|
|
551
|
+
const field = fieldPath(issue.path);
|
|
552
|
+
const at = field ? `${field} in ${file}` : file;
|
|
553
|
+
const v = valueAt(value, issue.path);
|
|
554
|
+
if (issue.code === "unrecognized_keys") {
|
|
555
|
+
const key = issue.keys[0];
|
|
556
|
+
const near = closest(key, keysAt(reevalConfigDataSchema, issue.path, value));
|
|
557
|
+
return `Unknown option "${key}" in ${at}.${near ? ` Did you mean "${near}"?` : ""}`;
|
|
558
|
+
}
|
|
559
|
+
if (issue.code === "invalid_union") {
|
|
560
|
+
const values = "options" in issue ? issue.options : void 0;
|
|
561
|
+
return values ? `${at} is ${oneOf(values)} (got ${got(v)})` : `${at} matches none of its forms (got ${got(v)})`;
|
|
562
|
+
}
|
|
563
|
+
if (issue.code === "invalid_type" && v === void 0) return `${at} is missing`;
|
|
564
|
+
if ((issue.code === "too_small" || issue.code === "too_big") && issue.origin === "number") {
|
|
565
|
+
const small = issue.code === "too_small";
|
|
566
|
+
const bound = small ? issue.minimum : issue.maximum;
|
|
567
|
+
const strict = small ? "above" : "below";
|
|
568
|
+
return `${at} must be ${issue.inclusive === false ? strict : small ? "at least" : "at most"} ${String(bound)} (got ${got(v)})`;
|
|
569
|
+
}
|
|
570
|
+
return `${at}${genericProblem("", {
|
|
571
|
+
...issue,
|
|
572
|
+
path: []
|
|
573
|
+
}, v)}`;
|
|
574
|
+
}
|
|
575
|
+
//#endregion
|
|
576
|
+
//#region src/harness/agents.ts
|
|
577
|
+
const WEB_EVENTS = {
|
|
578
|
+
webSearch: "web-search",
|
|
579
|
+
webFetch: "web-fetch"
|
|
580
|
+
};
|
|
581
|
+
/** `harnessMetadata.cursorSdk.unreportedTools` as transcript event types. */
|
|
582
|
+
function cursorUnreported(harnessMetadata) {
|
|
583
|
+
const tools = harnessMetadata?.cursorSdk?.unreportedTools;
|
|
584
|
+
return Array.isArray(tools) ? tools.flatMap((t) => WEB_EVENTS[t] ? [WEB_EVENTS[t]] : []) : [];
|
|
585
|
+
}
|
|
586
|
+
/** A Gemini API key's only request: `POST /v1beta/models/<model>:streamGenerateContent`. */
|
|
587
|
+
const GEMINI_ENDPOINTS = [{
|
|
588
|
+
host: "generativelanguage.googleapis.com",
|
|
589
|
+
path: { regex: "^/v1beta/models/[^/]+:streamGenerateContent$" },
|
|
590
|
+
method: ["POST"]
|
|
591
|
+
}];
|
|
592
|
+
/** Goose's: the model call, and the model list it fetches with the key at the start of a turn. */
|
|
593
|
+
const GOOSE_ENDPOINTS = [...GEMINI_ENDPOINTS, {
|
|
594
|
+
host: "generativelanguage.googleapis.com",
|
|
595
|
+
path: { regex: "^/v1beta/models$" },
|
|
596
|
+
method: ["GET"]
|
|
597
|
+
}];
|
|
598
|
+
/**
|
|
599
|
+
* What Grok Build sends the key on: the model list and the key's status as a session starts, and
|
|
600
|
+
* its model calls (Chat Completions, and the Responses API for a model or tool that takes it).
|
|
601
|
+
*/
|
|
602
|
+
const XAI_ENDPOINTS = [
|
|
603
|
+
{
|
|
604
|
+
host: "api.x.ai",
|
|
605
|
+
path: { exact: "/v1/models" },
|
|
606
|
+
method: ["GET"]
|
|
607
|
+
},
|
|
608
|
+
{
|
|
609
|
+
host: "api.x.ai",
|
|
610
|
+
path: { exact: "/v1/api-key" },
|
|
611
|
+
method: ["GET"]
|
|
612
|
+
},
|
|
613
|
+
{
|
|
614
|
+
host: "api.x.ai",
|
|
615
|
+
path: { exact: "/v1/chat/completions" },
|
|
616
|
+
method: ["POST"]
|
|
617
|
+
},
|
|
618
|
+
{
|
|
619
|
+
host: "api.x.ai",
|
|
620
|
+
path: { exact: "/v1/responses" },
|
|
621
|
+
method: ["POST"]
|
|
622
|
+
}
|
|
623
|
+
];
|
|
624
|
+
/** xAI's model calls, whose responses name the model that served them. */
|
|
625
|
+
const XAI_MODEL_RESPONSES = XAI_ENDPOINTS.filter((e) => e.path.exact === "/v1/chat/completions" || e.path.exact === "/v1/responses");
|
|
626
|
+
/** The model an xAI response says served it. */
|
|
627
|
+
const xaiServedModel = ({ host, responseModel }) => host === "api.x.ai" ? responseModel : void 0;
|
|
628
|
+
/** The model a Gemini API request names in its URL. */
|
|
629
|
+
const geminiServedModel = ({ host, path }) => host === "generativelanguage.googleapis.com" ? /^\/v1beta\/models\/([^/:]+):streamGenerateContent$/.exec(path)?.[1] : void 0;
|
|
630
|
+
/** Cursor's key exchange: the only request the Cursor SDK sends the API key on. */
|
|
631
|
+
const CURSOR_EXCHANGE = {
|
|
632
|
+
host: "api2.cursor.sh",
|
|
633
|
+
path: { exact: "/auth/exchange_user_api_key" },
|
|
634
|
+
method: ["POST"]
|
|
635
|
+
};
|
|
636
|
+
/** What Codex sends the key on: the Responses API. */
|
|
637
|
+
const OPENAI_ENDPOINTS = [{
|
|
638
|
+
host: "api.openai.com",
|
|
639
|
+
path: { exact: "/v1/responses" },
|
|
640
|
+
method: ["POST"]
|
|
641
|
+
}];
|
|
642
|
+
/**
|
|
643
|
+
* The model the Responses API says served a request, from its response (a dated snapshot for some
|
|
644
|
+
* models), and the reasoning effort Codex resolved for it, e.g. `gpt-5.6-luna[effort=medium]`.
|
|
645
|
+
*/
|
|
646
|
+
const openaiServedModel = ({ host, path, responseModel, responseEffort }) => host === "api.openai.com" && path === "/v1/responses" && responseModel ? `${responseModel}${responseEffort ? `[effort=${responseEffort}]` : ""}` : void 0;
|
|
647
|
+
/**
|
|
648
|
+
* OpenCode on the model's provider (`anthropic/claude-haiku-4-5` is Anthropic's): its key, the
|
|
649
|
+
* requests OpenCode sends it on, its hosts plus Exa's for web search, and the served model from
|
|
650
|
+
* the URL (Gemini's API, Bedrock's) or the response body. Bedrock's hosts are its `AWS_REGION`'s.
|
|
651
|
+
*/
|
|
652
|
+
function opencodeForModel(model, env = process.env) {
|
|
653
|
+
const resolved = resolveOpenCodeProvider(model, env);
|
|
654
|
+
if ("error" in resolved) throw new InfraError(`opencode(): ${resolved.error}`);
|
|
655
|
+
const { provider } = resolved;
|
|
656
|
+
const modelResponses = providerModelResponses(provider);
|
|
657
|
+
return {
|
|
658
|
+
credential: provider.env,
|
|
659
|
+
endpoints: providerEndpoints(provider),
|
|
660
|
+
...modelResponses.length ? { modelResponses } : {},
|
|
661
|
+
servedModel: (request) => providerServedModel(provider, request),
|
|
662
|
+
hosts: [...providerHosts(provider), "mcp.exa.ai"]
|
|
663
|
+
};
|
|
664
|
+
}
|
|
665
|
+
/**
|
|
666
|
+
* The variables besides its key that OpenCode's provider takes from the host, as they are:
|
|
667
|
+
* Bedrock's `AWS_REGION` (default `us-east-1`); none for most providers.
|
|
668
|
+
*/
|
|
669
|
+
function opencodeVars(model, env = process.env) {
|
|
670
|
+
const resolved = resolveOpenCodeProvider(model, env);
|
|
671
|
+
if ("error" in resolved) throw new InfraError(`opencode(): ${resolved.error}`);
|
|
672
|
+
return resolved.vars;
|
|
673
|
+
}
|
|
674
|
+
/** Codex's `config.toml` `[mcp_servers]` shape. */
|
|
675
|
+
function codexMcpServers(servers) {
|
|
676
|
+
return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
|
|
677
|
+
if (s.transport === "sse") throw new InfraError(`Codex has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
|
|
678
|
+
return s.transport === "stdio" ? [name, {
|
|
679
|
+
command: s.command,
|
|
680
|
+
args: s.args,
|
|
681
|
+
env: s.env ?? {}
|
|
682
|
+
}] : [name, {
|
|
683
|
+
url: s.url,
|
|
684
|
+
http_headers: s.headers ?? {}
|
|
685
|
+
}];
|
|
686
|
+
}));
|
|
687
|
+
}
|
|
688
|
+
/** The `.mcp.json` shape. */
|
|
689
|
+
function claudeMcpServers(servers) {
|
|
690
|
+
return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, {
|
|
691
|
+
type: "stdio",
|
|
692
|
+
command: s.command,
|
|
693
|
+
args: s.args,
|
|
694
|
+
env: s.env
|
|
695
|
+
}] : [name, {
|
|
696
|
+
type: s.transport,
|
|
697
|
+
url: s.url,
|
|
698
|
+
headers: s.headers
|
|
699
|
+
}]));
|
|
700
|
+
}
|
|
701
|
+
/** The Antigravity CLI's `mcp_config.json` shape: remote servers take `serverUrl`, not `url`. */
|
|
702
|
+
function antigravityMcpServers(servers) {
|
|
703
|
+
return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, {
|
|
704
|
+
command: s.command,
|
|
705
|
+
args: s.args,
|
|
706
|
+
env: s.env
|
|
707
|
+
}] : [name, {
|
|
708
|
+
serverUrl: s.url,
|
|
709
|
+
headers: s.headers
|
|
710
|
+
}]));
|
|
711
|
+
}
|
|
712
|
+
/**
|
|
713
|
+
* OpenCode's config shape (its `mcp.servers`), which the adapter sets `codemode: false` on.
|
|
714
|
+
* OpenCode has no SSE transport.
|
|
715
|
+
*/
|
|
716
|
+
function opencodeMcpServers(servers) {
|
|
717
|
+
return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
|
|
718
|
+
if (s.transport === "sse") throw new InfraError(`OpenCode has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
|
|
719
|
+
return s.transport === "stdio" ? [name, {
|
|
720
|
+
type: "local",
|
|
721
|
+
command: [s.command, ...s.args],
|
|
722
|
+
environment: s.env ?? {}
|
|
723
|
+
}] : [name, {
|
|
724
|
+
type: "remote",
|
|
725
|
+
url: s.url,
|
|
726
|
+
headers: s.headers ?? {},
|
|
727
|
+
oauth: false
|
|
728
|
+
}];
|
|
729
|
+
}));
|
|
730
|
+
}
|
|
731
|
+
/**
|
|
732
|
+
* The key Goose keeps an extension by (its `name_to_key`): whitespace dropped, anything but ASCII
|
|
733
|
+
* letters, digits, `_` and `-` made `_`, lower-cased. Of two extensions with one key it keeps one.
|
|
734
|
+
*/
|
|
735
|
+
const gooseKey = (name) => name.replace(/\s/g, "").replace(/[^A-Za-z0-9_-]/g, "_").toLowerCase();
|
|
736
|
+
/** Goose's extension shape (its `config.yaml`); it has no SSE transport. */
|
|
737
|
+
function gooseMcpServers(servers) {
|
|
738
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
739
|
+
for (const name of Object.keys(servers)) {
|
|
740
|
+
const key = gooseKey(name);
|
|
741
|
+
const other = byKey.get(key);
|
|
742
|
+
if (other !== void 0) throw new InfraError(`Goose keys the ${other} and ${name} MCP servers both as ${key}, and keeps only one: rename one`);
|
|
743
|
+
byKey.set(key, name);
|
|
744
|
+
}
|
|
745
|
+
return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
|
|
746
|
+
if (s.transport === "sse") throw new InfraError(`Goose has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
|
|
747
|
+
return s.transport === "stdio" ? [name, {
|
|
748
|
+
type: "stdio",
|
|
749
|
+
cmd: s.command,
|
|
750
|
+
args: s.args,
|
|
751
|
+
envs: s.env ?? {}
|
|
752
|
+
}] : [name, {
|
|
753
|
+
type: "streamable_http",
|
|
754
|
+
uri: s.url,
|
|
755
|
+
headers: s.headers ?? {}
|
|
756
|
+
}];
|
|
757
|
+
}));
|
|
758
|
+
}
|
|
759
|
+
/**
|
|
760
|
+
* pi's `mcp.json` shape, which the adapter escapes and sets `"exposure": "direct"` on. pi
|
|
761
|
+
* skips a server whose name isn't letters, digits, `_` and `-`, so the plan refuses it instead.
|
|
762
|
+
* A plugin's server, `<plugin>:<server>`, is named `<plugin>_<server>`, so its tools are
|
|
763
|
+
* `mcp__<plugin>_<server>__<tool>`.
|
|
764
|
+
*/
|
|
765
|
+
function piMcpServers(servers) {
|
|
766
|
+
const named = /* @__PURE__ */ new Map();
|
|
767
|
+
return Object.fromEntries(Object.entries(servers).map(([key, s]) => {
|
|
768
|
+
const name = /^[a-z0-9-]+:[^:]+$/.test(key) ? key.replace(":", "_") : key;
|
|
769
|
+
if (!/^[A-Za-z0-9_-]+$/.test(name)) throw new InfraError(`pi takes MCP server names of letters, digits, "_" and "-" only, so it can't use the ${key} server: rename it`);
|
|
770
|
+
const other = named.get(name);
|
|
771
|
+
if (other) throw new InfraError(`pi names the ${key} MCP server ${name}, as it does the ${other} server, and keeps only one: rename one`);
|
|
772
|
+
named.set(name, key);
|
|
773
|
+
if (s.transport === "sse") throw new InfraError(`pi has no SSE MCP transport, so it can't use the ${key} server (mcp.sse): use mcp.http or mcp.stdio`);
|
|
774
|
+
return s.transport === "stdio" ? [name, {
|
|
775
|
+
command: s.command,
|
|
776
|
+
args: s.args,
|
|
777
|
+
env: s.env ?? {}
|
|
778
|
+
}] : [name, {
|
|
779
|
+
url: s.url,
|
|
780
|
+
headers: s.headers ?? {}
|
|
781
|
+
}];
|
|
782
|
+
}));
|
|
783
|
+
}
|
|
784
|
+
/**
|
|
785
|
+
* ACP's `McpServer` shape, which fx and Grok Build take on `session/new`: `env` and `headers` as
|
|
786
|
+
* name-value lists.
|
|
787
|
+
*/
|
|
788
|
+
function acpMcpServers(servers) {
|
|
789
|
+
const pairs = (values = {}) => Object.entries(values).map(([name, value]) => ({
|
|
790
|
+
name,
|
|
791
|
+
value
|
|
792
|
+
}));
|
|
793
|
+
return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, s.command.startsWith("/") ? {
|
|
794
|
+
command: s.command,
|
|
795
|
+
args: s.args,
|
|
796
|
+
env: pairs(s.env)
|
|
797
|
+
} : {
|
|
798
|
+
command: "/usr/bin/env",
|
|
799
|
+
args: [s.command, ...s.args],
|
|
800
|
+
env: pairs(s.env)
|
|
801
|
+
}] : [name, {
|
|
802
|
+
type: s.transport,
|
|
803
|
+
url: s.url,
|
|
804
|
+
headers: pairs(s.headers)
|
|
805
|
+
}]));
|
|
806
|
+
}
|
|
807
|
+
/**
|
|
808
|
+
* fx on the model's provider: OpenAI's Chat Completions API for a bare OpenAI id, and Vercel AI
|
|
809
|
+
* Gateway for a gateway id, which only `vercel()` in the gateway list serves. Each response's
|
|
810
|
+
* first `model` names the model that served it.
|
|
811
|
+
*/
|
|
812
|
+
function fxForModel(model) {
|
|
813
|
+
if (!model.includes("/")) {
|
|
814
|
+
const endpoints = [{
|
|
815
|
+
host: "api.openai.com",
|
|
816
|
+
path: { exact: "/v1/chat/completions" },
|
|
817
|
+
method: ["POST"]
|
|
818
|
+
}];
|
|
819
|
+
return {
|
|
820
|
+
credential: "OPENAI_API_KEY",
|
|
821
|
+
endpoints,
|
|
822
|
+
modelResponses: endpoints,
|
|
823
|
+
servedModel: ({ host, responseModel }) => host === "api.openai.com" ? responseModel : void 0,
|
|
824
|
+
hosts: ["api.openai.com"]
|
|
825
|
+
};
|
|
826
|
+
}
|
|
827
|
+
return {
|
|
828
|
+
credential: VERCEL_KEY,
|
|
829
|
+
endpoints: FX_VERCEL_ENDPOINTS,
|
|
830
|
+
modelResponses: [FX_VERCEL_CALL],
|
|
831
|
+
servedModel: servedOnVercel,
|
|
832
|
+
askedModel: fxAskedModel,
|
|
833
|
+
hosts: [VERCEL_HOST]
|
|
834
|
+
};
|
|
835
|
+
}
|
|
836
|
+
/** The Cursor SDK's `McpServerConfig` shape. */
|
|
837
|
+
function cursorMcpServers(servers) {
|
|
838
|
+
return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, {
|
|
839
|
+
type: "stdio",
|
|
840
|
+
command: s.command,
|
|
841
|
+
args: s.args,
|
|
842
|
+
env: s.env
|
|
843
|
+
}] : [name, {
|
|
844
|
+
type: s.transport,
|
|
845
|
+
url: s.url,
|
|
846
|
+
headers: s.headers
|
|
847
|
+
}]));
|
|
848
|
+
}
|
|
849
|
+
/** Claude Code's hook events as Cursor names them, as the Cursor SDK converts Claude Code's own. */
|
|
850
|
+
const CURSOR_HOOK_EVENTS = {
|
|
851
|
+
SessionStart: "sessionStart",
|
|
852
|
+
SessionEnd: "sessionEnd",
|
|
853
|
+
UserPromptSubmit: "beforeSubmitPrompt",
|
|
854
|
+
PreToolUse: "preToolUse",
|
|
855
|
+
PostToolUse: "postToolUse",
|
|
856
|
+
SubagentStop: "subagentStop",
|
|
857
|
+
PreCompact: "preCompact",
|
|
858
|
+
Stop: "stop"
|
|
859
|
+
};
|
|
860
|
+
/** Claude Code's tool names as Cursor's hooks match them; null for one Cursor has no tool for. */
|
|
861
|
+
const CURSOR_HOOK_TOOLS = {
|
|
862
|
+
Bash: "Shell",
|
|
863
|
+
Read: "Read",
|
|
864
|
+
Write: "Write",
|
|
865
|
+
Edit: "Write",
|
|
866
|
+
Glob: null,
|
|
867
|
+
Grep: "Grep",
|
|
868
|
+
WebFetch: "WebFetch",
|
|
869
|
+
WebSearch: "WebSearch",
|
|
870
|
+
Task: "Task"
|
|
871
|
+
};
|
|
872
|
+
/**
|
|
873
|
+
* A Claude Code hook group's tool matcher as Cursor's, on the tool events only: each name mapped,
|
|
874
|
+
* `mcp__<server>__<tool>` as `MCP:<tool>`; null when it names only tools Cursor lacks.
|
|
875
|
+
*/
|
|
876
|
+
function cursorHookMatcher(event, matcher) {
|
|
877
|
+
if (event !== "PreToolUse" && event !== "PostToolUse") return void 0;
|
|
878
|
+
if (!matcher || matcher === "*") return void 0;
|
|
879
|
+
const out = [];
|
|
880
|
+
for (const raw of matcher.split("|")) {
|
|
881
|
+
const name = raw.trim();
|
|
882
|
+
const parts = name.split("__");
|
|
883
|
+
const tool = name.startsWith("mcp__") && parts.length >= 3 ? `MCP:${parts.slice(2).join("__")}` : CURSOR_HOOK_TOOLS[name] === void 0 ? name : CURSOR_HOOK_TOOLS[name];
|
|
884
|
+
if (tool && !out.includes(tool)) out.push(tool);
|
|
885
|
+
}
|
|
886
|
+
return out.length ? out.join("|") : null;
|
|
887
|
+
}
|
|
888
|
+
/** Where Cursor reads what reeval unpacks from a plugin, in the workspace (settingSources `project`). */
|
|
889
|
+
const CURSOR_PLUGIN_UNPACK = {
|
|
890
|
+
rules: ".cursor/rules",
|
|
891
|
+
agents: ".cursor/agents",
|
|
892
|
+
commands: ".cursor/commands",
|
|
893
|
+
hooks: {
|
|
894
|
+
file: ".cursor/hooks.json",
|
|
895
|
+
events: CURSOR_HOOK_EVENTS,
|
|
896
|
+
matcher: cursorHookMatcher
|
|
897
|
+
}
|
|
898
|
+
};
|
|
899
|
+
/**
|
|
900
|
+
* A plugin subagent's or command's frontmatter as OpenCode 2 takes it: the description, and a
|
|
901
|
+
* subagent's `mode`. OpenCode skips a file whose frontmatter its schema refuses, without saying,
|
|
902
|
+
* and Claude Code's `tools: Read, Grep`, `model: sonnet` and `allowed-tools` are such keys.
|
|
903
|
+
*/
|
|
904
|
+
function opencodeFrontmatter(part, front) {
|
|
905
|
+
const d = front.description;
|
|
906
|
+
const description = typeof d === "string" || typeof d === "number" || typeof d === "boolean" ? String(d) : void 0;
|
|
907
|
+
return {
|
|
908
|
+
...description ? { description } : {},
|
|
909
|
+
...part === "agents" ? { mode: "subagent" } : {}
|
|
910
|
+
};
|
|
911
|
+
}
|
|
912
|
+
/** Where OpenCode reads plugin subagents and commands, in the workspace's config directory. */
|
|
913
|
+
const OPENCODE_PLUGIN_UNPACK = {
|
|
914
|
+
agents: ".opencode/agents",
|
|
915
|
+
commands: ".opencode/commands",
|
|
916
|
+
frontmatter: opencodeFrontmatter
|
|
917
|
+
};
|
|
918
|
+
/**
|
|
919
|
+
* Where pi reads plugin commands, as prompt templates (`/<name>`), in the workspace, which its
|
|
920
|
+
* `--approve` trusts. Its frontmatter takes `description` and `argument-hint`, and ignores the
|
|
921
|
+
* rest of Claude Code's.
|
|
922
|
+
*/
|
|
923
|
+
const PI_PLUGIN_UNPACK = { commands: ".pi/prompts" };
|
|
924
|
+
/**
|
|
925
|
+
* What Claude Code's plugin format bundles, which Grok Build reads too: every part but rules,
|
|
926
|
+
* which neither loads from a plugin.
|
|
927
|
+
*/
|
|
928
|
+
const CLAUDE_FORMAT_PARTS = PLUGIN_PARTS.filter((p) => p !== "rules");
|
|
929
|
+
/** The marketplace Claude Code's settings name for the session's plugins. */
|
|
930
|
+
const CLAUDE_PLUGIN_MARKETPLACE = "reeval";
|
|
931
|
+
/**
|
|
932
|
+
* Claude Code's settings for the session's plugins: a `directory` marketplace at the directory
|
|
933
|
+
* they were copied to, listing each, and each enabled as `<plugin>@reeval`. Claude Code installs
|
|
934
|
+
* and loads them as it starts, since the bridge leaves every settings source on.
|
|
935
|
+
*/
|
|
936
|
+
function claudePluginFiles(plugins) {
|
|
937
|
+
const [first] = plugins;
|
|
938
|
+
if (!first) return [];
|
|
939
|
+
const json = (value) => `${JSON.stringify(value, null, 2)}\n`;
|
|
940
|
+
const marketplace = {
|
|
941
|
+
name: CLAUDE_PLUGIN_MARKETPLACE,
|
|
942
|
+
owner: { name: "reeval" },
|
|
943
|
+
plugins: plugins.map((p) => ({
|
|
944
|
+
name: p.name,
|
|
945
|
+
source: `./${p.name}`
|
|
946
|
+
}))
|
|
947
|
+
};
|
|
948
|
+
const settings = {
|
|
949
|
+
extraKnownMarketplaces: { [CLAUDE_PLUGIN_MARKETPLACE]: { source: {
|
|
950
|
+
source: "directory",
|
|
951
|
+
path: first.dir.slice(0, first.dir.lastIndexOf("/"))
|
|
952
|
+
} } },
|
|
953
|
+
enabledPlugins: Object.fromEntries(plugins.map((p) => [`${p.name}@${CLAUDE_PLUGIN_MARKETPLACE}`, true]))
|
|
954
|
+
};
|
|
955
|
+
return [{
|
|
956
|
+
path: `${PLUGIN_DIR}/.claude-plugin/marketplace.json`,
|
|
957
|
+
content: json(marketplace)
|
|
958
|
+
}, {
|
|
959
|
+
path: "~/.claude/settings.json",
|
|
960
|
+
content: json(settings),
|
|
961
|
+
merge: true
|
|
962
|
+
}];
|
|
963
|
+
}
|
|
964
|
+
const isObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
965
|
+
/**
|
|
966
|
+
* Writes the layout's plugin files into the sandbox and returns their paths. A `merge` file's keys
|
|
967
|
+
* go into the JSON object already at its path (an object-valued key merged one level down), so
|
|
968
|
+
* settings an environment's `files` wrote there stay.
|
|
969
|
+
*/
|
|
970
|
+
async function writePluginFiles(sandbox, layout, plugins) {
|
|
971
|
+
const written = [];
|
|
972
|
+
for (const f of layout.pluginFiles?.(plugins) ?? []) {
|
|
973
|
+
const existing = f.merge ? await sandbox.readFile(f.path) : null;
|
|
974
|
+
let content = f.content;
|
|
975
|
+
if (existing?.trim()) {
|
|
976
|
+
let base;
|
|
977
|
+
try {
|
|
978
|
+
base = JSON.parse(existing);
|
|
979
|
+
} catch {}
|
|
980
|
+
if (!isObject(base)) throw new InfraError(`${f.path} isn't a JSON object, so reeval can't enable the session's plugins in it: write it as one in files, or leave it out`);
|
|
981
|
+
const ours = JSON.parse(f.content);
|
|
982
|
+
const merged = { ...base };
|
|
983
|
+
for (const [k, v] of Object.entries(ours)) {
|
|
984
|
+
const was = base[k];
|
|
985
|
+
merged[k] = isObject(v) && isObject(was) ? {
|
|
986
|
+
...was,
|
|
987
|
+
...v
|
|
988
|
+
} : v;
|
|
989
|
+
}
|
|
990
|
+
content = `${JSON.stringify(merged, null, 2)}\n`;
|
|
991
|
+
}
|
|
992
|
+
await sandbox.writeFile(f.path, content);
|
|
993
|
+
written.push(f.path);
|
|
994
|
+
}
|
|
995
|
+
return written;
|
|
996
|
+
}
|
|
997
|
+
/**
|
|
998
|
+
* The MCP servers the agent is given: all of them, less the plugins' own where it takes plugin
|
|
999
|
+
* items namespaced, since it loads those from the plugin and would otherwise start each twice.
|
|
1000
|
+
*/
|
|
1001
|
+
function agentMcp(layout, mcp, plugins) {
|
|
1002
|
+
if (layout.pluginPlacement !== "namespaced") return mcp;
|
|
1003
|
+
const own = new Set(plugins.flatMap((p) => p.servers));
|
|
1004
|
+
return Object.fromEntries(Object.entries(mcp).filter(([name]) => !own.has(name)));
|
|
1005
|
+
}
|
|
1006
|
+
/** Where each agent with an image reads rules and skills. */
|
|
1007
|
+
const LAYOUTS = {
|
|
1008
|
+
cursor: {
|
|
1009
|
+
rulesFile: "AGENTS.md",
|
|
1010
|
+
workspaceSkillDir: ".cursor/skills",
|
|
1011
|
+
harnessSkillDir: ".agents/skills",
|
|
1012
|
+
pluginPlacement: "flat",
|
|
1013
|
+
pluginUnpack: CURSOR_PLUGIN_UNPACK,
|
|
1014
|
+
pluginHookEvents: Object.keys(CURSOR_HOOK_EVENTS)
|
|
1015
|
+
},
|
|
1016
|
+
"claude-code": {
|
|
1017
|
+
rulesFile: "CLAUDE.md",
|
|
1018
|
+
workspaceSkillDir: ".claude/skills",
|
|
1019
|
+
harnessSkillDir: ".claude/skills",
|
|
1020
|
+
pluginPlacement: "namespaced",
|
|
1021
|
+
pluginFiles: claudePluginFiles
|
|
1022
|
+
},
|
|
1023
|
+
antigravity: {
|
|
1024
|
+
rulesFile: "AGENTS.md",
|
|
1025
|
+
workspaceSkillDir: ".agents/skills",
|
|
1026
|
+
harnessSkillDir: ".gemini/config/skills",
|
|
1027
|
+
pluginPlacement: "namespaced",
|
|
1028
|
+
pluginHookEvents: [
|
|
1029
|
+
"SessionStart",
|
|
1030
|
+
"PreToolUse",
|
|
1031
|
+
"PostToolUse",
|
|
1032
|
+
"Stop"
|
|
1033
|
+
]
|
|
1034
|
+
},
|
|
1035
|
+
opencode: {
|
|
1036
|
+
rulesFile: "AGENTS.md",
|
|
1037
|
+
workspaceSkillDir: ".opencode/skills",
|
|
1038
|
+
harnessSkillDir: ".config/opencode/skills",
|
|
1039
|
+
pluginPlacement: "flat",
|
|
1040
|
+
pluginUnpack: OPENCODE_PLUGIN_UNPACK
|
|
1041
|
+
},
|
|
1042
|
+
goose: {
|
|
1043
|
+
rulesFile: "AGENTS.md",
|
|
1044
|
+
workspaceSkillDir: ".goose/skills",
|
|
1045
|
+
harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/goose/implementation/home/.config/goose/skills",
|
|
1046
|
+
pluginPlacement: "flat"
|
|
1047
|
+
},
|
|
1048
|
+
pi: {
|
|
1049
|
+
rulesFile: "AGENTS.md",
|
|
1050
|
+
workspaceSkillDir: ".pi/skills",
|
|
1051
|
+
harnessSkillDir: ".pi/agent/skills",
|
|
1052
|
+
pluginPlacement: "flat",
|
|
1053
|
+
pluginUnpack: PI_PLUGIN_UNPACK
|
|
1054
|
+
},
|
|
1055
|
+
codex: {
|
|
1056
|
+
rulesFile: "AGENTS.md",
|
|
1057
|
+
workspaceSkillDir: ".agents/skills",
|
|
1058
|
+
harnessSkillDir: ".agents/skills",
|
|
1059
|
+
pluginPlacement: "namespaced"
|
|
1060
|
+
},
|
|
1061
|
+
fx: {
|
|
1062
|
+
rulesFile: "AGENTS.md",
|
|
1063
|
+
workspaceSkillDir: ".fx/skills",
|
|
1064
|
+
harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/fx/implementation/home/.fx/skills",
|
|
1065
|
+
pluginPlacement: "flat"
|
|
1066
|
+
},
|
|
1067
|
+
"grok-build": {
|
|
1068
|
+
rulesFile: "AGENTS.md",
|
|
1069
|
+
workspaceSkillDir: ".grok/skills",
|
|
1070
|
+
harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/grok-build/implementation/home/.grok/skills",
|
|
1071
|
+
pluginPlacement: "namespaced"
|
|
1072
|
+
}
|
|
1073
|
+
};
|
|
1074
|
+
/** Every real agent reeval can drive, by harness kind. */
|
|
1075
|
+
const AGENTS = {
|
|
1076
|
+
cursor: {
|
|
1077
|
+
...LAYOUTS.cursor,
|
|
1078
|
+
pluginParts: PLUGIN_PARTS,
|
|
1079
|
+
name: "Cursor",
|
|
1080
|
+
factory: "cursor",
|
|
1081
|
+
credential: "CURSOR_API_KEY",
|
|
1082
|
+
endpoints: [CURSOR_EXCHANGE],
|
|
1083
|
+
tokens: {
|
|
1084
|
+
endpoint: CURSOR_EXCHANGE,
|
|
1085
|
+
fields: ["accessToken", "refreshToken"],
|
|
1086
|
+
hosts: ["cursor.sh", "*.cursor.sh"]
|
|
1087
|
+
},
|
|
1088
|
+
unreported: cursorUnreported,
|
|
1089
|
+
mcpServers: cursorMcpServers,
|
|
1090
|
+
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-CyaDNG9r.js")).createCursorSdk({
|
|
1091
|
+
auth,
|
|
1092
|
+
...mcpServers ? { mcpServers } : {},
|
|
1093
|
+
...descriptor.kind === "cursor" && descriptor.params ? { modelParams: descriptor.params } : {},
|
|
1094
|
+
...descriptor.kind === "cursor" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1095
|
+
requireCredentialBrokering: true
|
|
1096
|
+
})
|
|
1097
|
+
},
|
|
1098
|
+
"claude-code": {
|
|
1099
|
+
...LAYOUTS["claude-code"],
|
|
1100
|
+
pluginParts: CLAUDE_FORMAT_PARTS,
|
|
1101
|
+
name: "Claude Code",
|
|
1102
|
+
factory: "claudeCode",
|
|
1103
|
+
credential: "ANTHROPIC_API_KEY",
|
|
1104
|
+
endpoints: [
|
|
1105
|
+
{
|
|
1106
|
+
host: "api.anthropic.com",
|
|
1107
|
+
path: { exact: "/v1/messages" },
|
|
1108
|
+
method: ["POST"]
|
|
1109
|
+
},
|
|
1110
|
+
{
|
|
1111
|
+
host: "api.anthropic.com",
|
|
1112
|
+
path: { exact: "/v1/messages/count_tokens" },
|
|
1113
|
+
method: ["POST"]
|
|
1114
|
+
},
|
|
1115
|
+
{
|
|
1116
|
+
host: "api.anthropic.com",
|
|
1117
|
+
path: { exact: "/api/claude_code/settings" },
|
|
1118
|
+
method: ["GET"]
|
|
1119
|
+
},
|
|
1120
|
+
{
|
|
1121
|
+
host: "api.anthropic.com",
|
|
1122
|
+
path: { exact: "/api/claude_code/policy_limits" },
|
|
1123
|
+
method: ["GET"]
|
|
1124
|
+
}
|
|
1125
|
+
],
|
|
1126
|
+
mcpServers: claudeMcpServers,
|
|
1127
|
+
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./dist-CYrBwyHA.js")).createClaudeCode({
|
|
1128
|
+
auth,
|
|
1129
|
+
...mcpServers ? { mcpServers } : {},
|
|
1130
|
+
...descriptor.kind === "claude-code" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1131
|
+
env: {
|
|
1132
|
+
IS_SANDBOX: "1",
|
|
1133
|
+
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1"
|
|
1134
|
+
}
|
|
1135
|
+
})
|
|
1136
|
+
},
|
|
1137
|
+
antigravity: {
|
|
1138
|
+
...LAYOUTS.antigravity,
|
|
1139
|
+
pluginParts: PLUGIN_PARTS,
|
|
1140
|
+
pluginManifestsFirst: [
|
|
1141
|
+
"plugin.json",
|
|
1142
|
+
".antigravity-plugin/plugin.json",
|
|
1143
|
+
".gemini-plugin/plugin.json",
|
|
1144
|
+
"gemini-extension.json"
|
|
1145
|
+
],
|
|
1146
|
+
name: "Antigravity",
|
|
1147
|
+
factory: "antigravity",
|
|
1148
|
+
credential: "GEMINI_API_KEY",
|
|
1149
|
+
endpoints: GEMINI_ENDPOINTS,
|
|
1150
|
+
expectedBlocked: {
|
|
1151
|
+
hosts: ["play.googleapis.com", "antigravity-unleash.goog"],
|
|
1152
|
+
why: "Antigravity's telemetry and feature flags, which its CLI sends even with telemetry off"
|
|
1153
|
+
},
|
|
1154
|
+
servedModel: geminiServedModel,
|
|
1155
|
+
mcpServers: antigravityMcpServers,
|
|
1156
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-ElBqEN-S.js")).createAntigravity({
|
|
1157
|
+
auth,
|
|
1158
|
+
...mcpServers ? { mcpServers } : {},
|
|
1159
|
+
...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
|
|
1160
|
+
name,
|
|
1161
|
+
dir
|
|
1162
|
+
})) } : {},
|
|
1163
|
+
...descriptor.kind === "antigravity" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1164
|
+
requireCredentialBrokering: true
|
|
1165
|
+
})
|
|
1166
|
+
},
|
|
1167
|
+
opencode: {
|
|
1168
|
+
...LAYOUTS.opencode,
|
|
1169
|
+
pluginParts: [
|
|
1170
|
+
"skills",
|
|
1171
|
+
"agents",
|
|
1172
|
+
"commands",
|
|
1173
|
+
"mcp"
|
|
1174
|
+
],
|
|
1175
|
+
name: "OpenCode",
|
|
1176
|
+
factory: "opencode",
|
|
1177
|
+
credential: "GOOGLE_GENERATIVE_AI_API_KEY",
|
|
1178
|
+
endpoints: GEMINI_ENDPOINTS,
|
|
1179
|
+
servedModel: geminiServedModel,
|
|
1180
|
+
forModel: opencodeForModel,
|
|
1181
|
+
mcpServers: opencodeMcpServers,
|
|
1182
|
+
adapter: async ({ auth, mcpServers, descriptor, model }) => (await import("./src-DIfRbp5a.js")).createOpenCode({
|
|
1183
|
+
auth,
|
|
1184
|
+
model,
|
|
1185
|
+
vars: opencodeVars(model),
|
|
1186
|
+
...descriptor.kind === "opencode" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1187
|
+
...mcpServers ? { mcpServers } : {},
|
|
1188
|
+
requireCredentialBrokering: true
|
|
1189
|
+
})
|
|
1190
|
+
},
|
|
1191
|
+
goose: {
|
|
1192
|
+
...LAYOUTS.goose,
|
|
1193
|
+
pluginParts: ["skills", "mcp"],
|
|
1194
|
+
name: "Goose",
|
|
1195
|
+
factory: "goose",
|
|
1196
|
+
credential: "GOOGLE_GENERATIVE_AI_API_KEY",
|
|
1197
|
+
endpoints: GOOSE_ENDPOINTS,
|
|
1198
|
+
expectedBlocked: {
|
|
1199
|
+
hosts: ["models.dev"],
|
|
1200
|
+
why: "Goose's model catalogue refresh, which no setting turns off"
|
|
1201
|
+
},
|
|
1202
|
+
servedModel: geminiServedModel,
|
|
1203
|
+
mcpServers: gooseMcpServers,
|
|
1204
|
+
adapter: async ({ auth, mcpServers, descriptor, model }) => (await import("./src-Dg8121Xy.js")).createGoose({
|
|
1205
|
+
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1206
|
+
auth,
|
|
1207
|
+
provider: (await import("./src-Dg8121Xy.js")).AI_GATEWAY_PROVIDER
|
|
1208
|
+
} : { auth: { GOOGLE_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" } },
|
|
1209
|
+
model,
|
|
1210
|
+
...descriptor.kind === "goose" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1211
|
+
...mcpServers ? { mcpServers } : {},
|
|
1212
|
+
requireCredentialBrokering: true
|
|
1213
|
+
})
|
|
1214
|
+
},
|
|
1215
|
+
pi: {
|
|
1216
|
+
...LAYOUTS.pi,
|
|
1217
|
+
pluginParts: [
|
|
1218
|
+
"skills",
|
|
1219
|
+
"commands",
|
|
1220
|
+
"mcp"
|
|
1221
|
+
],
|
|
1222
|
+
name: "pi",
|
|
1223
|
+
factory: "pi",
|
|
1224
|
+
credential: "GOOGLE_GENERATIVE_AI_API_KEY",
|
|
1225
|
+
endpoints: GEMINI_ENDPOINTS,
|
|
1226
|
+
servedModel: geminiServedModel,
|
|
1227
|
+
mcpServers: piMcpServers,
|
|
1228
|
+
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-mJVxw1Jm.js")).createPi({
|
|
1229
|
+
auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
|
|
1230
|
+
...descriptor.kind === "pi" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1231
|
+
...mcpServers ? { mcpServers } : {},
|
|
1232
|
+
requireCredentialBrokering: true
|
|
1233
|
+
})
|
|
1234
|
+
},
|
|
1235
|
+
codex: {
|
|
1236
|
+
...LAYOUTS.codex,
|
|
1237
|
+
pluginParts: ["skills", "mcp"],
|
|
1238
|
+
pluginApps: "they need a ChatGPT login",
|
|
1239
|
+
pluginManifestsFirst: ["plugin.json", ".codex-plugin/plugin.json"],
|
|
1240
|
+
pluginRootManifestNeedsSchema: true,
|
|
1241
|
+
pluginServersByName: true,
|
|
1242
|
+
name: "Codex",
|
|
1243
|
+
factory: "codex",
|
|
1244
|
+
credential: "OPENAI_API_KEY",
|
|
1245
|
+
endpoints: OPENAI_ENDPOINTS,
|
|
1246
|
+
modelResponses: OPENAI_ENDPOINTS,
|
|
1247
|
+
expectedBlocked: {
|
|
1248
|
+
hosts: [
|
|
1249
|
+
"github.com",
|
|
1250
|
+
"api.github.com",
|
|
1251
|
+
"chatgpt.com"
|
|
1252
|
+
],
|
|
1253
|
+
why: "Codex's plugin sync, which it runs on every turn once plugins are on",
|
|
1254
|
+
withPlugins: true
|
|
1255
|
+
},
|
|
1256
|
+
servedModel: openaiServedModel,
|
|
1257
|
+
mcpServers: codexMcpServers,
|
|
1258
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-D005Cbqz.js")).createCodex({
|
|
1259
|
+
auth,
|
|
1260
|
+
...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-D005Cbqz.js")).AI_GATEWAY_PROVIDER } : {},
|
|
1261
|
+
...descriptor.kind === "codex" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1262
|
+
...mcpServers ? { mcpServers } : {},
|
|
1263
|
+
...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
|
|
1264
|
+
name,
|
|
1265
|
+
dir
|
|
1266
|
+
})) } : {},
|
|
1267
|
+
requireCredentialBrokering: true
|
|
1268
|
+
})
|
|
1269
|
+
},
|
|
1270
|
+
fx: {
|
|
1271
|
+
...LAYOUTS.fx,
|
|
1272
|
+
pluginParts: ["skills", "mcp"],
|
|
1273
|
+
name: "fx",
|
|
1274
|
+
factory: "fx",
|
|
1275
|
+
...fxForModel("provider/model"),
|
|
1276
|
+
forModel: fxForModel,
|
|
1277
|
+
mcpServers: acpMcpServers,
|
|
1278
|
+
adapter: async ({ auth, mcpServers, model }) => (await import("./src-YHGaUrGE.js")).createFx({
|
|
1279
|
+
auth,
|
|
1280
|
+
model,
|
|
1281
|
+
...mcpServers ? { mcpServers } : {},
|
|
1282
|
+
requireCredentialBrokering: true
|
|
1283
|
+
})
|
|
1284
|
+
},
|
|
1285
|
+
"grok-build": {
|
|
1286
|
+
...LAYOUTS["grok-build"],
|
|
1287
|
+
pluginParts: CLAUDE_FORMAT_PARTS,
|
|
1288
|
+
pluginServersByName: true,
|
|
1289
|
+
name: "Grok Build",
|
|
1290
|
+
factory: "grokBuild",
|
|
1291
|
+
credential: "XAI_API_KEY",
|
|
1292
|
+
endpoints: XAI_ENDPOINTS,
|
|
1293
|
+
modelResponses: XAI_MODEL_RESPONSES,
|
|
1294
|
+
servedModel: xaiServedModel,
|
|
1295
|
+
mcpServers: acpMcpServers,
|
|
1296
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins, model }) => (await import("./src-C_9wgowC.js")).createGrokBuild({
|
|
1297
|
+
auth,
|
|
1298
|
+
...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-C_9wgowC.js")).AI_GATEWAY_PROVIDER } : {},
|
|
1299
|
+
model,
|
|
1300
|
+
...plugins.length ? {
|
|
1301
|
+
pluginDirs: plugins.map((p) => p.dir),
|
|
1302
|
+
pluginMcpServers: plugins.flatMap((p) => p.servers.map((s) => s.slice(p.name.length + 1)))
|
|
1303
|
+
} : {},
|
|
1304
|
+
...descriptor.kind === "grok-build" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1305
|
+
...mcpServers ? { mcpServers } : {},
|
|
1306
|
+
requireCredentialBrokering: true
|
|
1307
|
+
})
|
|
1308
|
+
}
|
|
1309
|
+
};
|
|
1310
|
+
/** The harness of a config that names none: Cursor, on Cursor's own model. */
|
|
1311
|
+
const DEFAULT_HARNESS = {
|
|
1312
|
+
kind: "cursor",
|
|
1313
|
+
model: "composer-2.5"
|
|
1314
|
+
};
|
|
1315
|
+
/** A harness to name in messages that ask for one. */
|
|
1316
|
+
const HARNESS_EXAMPLE = "cursor({ model: 'composer-2.5' })";
|
|
1317
|
+
const isAgentKind = (kind) => typeof kind === "string" && Object.hasOwn(AGENTS, kind);
|
|
1318
|
+
/**
|
|
1319
|
+
* Why a harness's agent can't take the environment's MCP servers (Goose, OpenCode, pi and Codex
|
|
1320
|
+
* take no `mcp.sse`), in the agent's own words, so the plan refuses them before any attempt
|
|
1321
|
+
* starts; undefined when it can.
|
|
1322
|
+
*/
|
|
1323
|
+
function refusedMcp(harness, layers) {
|
|
1324
|
+
if (!isAgentKind(harness.kind)) return void 0;
|
|
1325
|
+
const servers = /* @__PURE__ */ new Map();
|
|
1326
|
+
for (const { spec } of layers) for (const [name, server] of Object.entries(spec.mcp ?? {})) servers.set(name, server.transport === "stdio" ? {
|
|
1327
|
+
...server,
|
|
1328
|
+
env: {}
|
|
1329
|
+
} : {
|
|
1330
|
+
...server,
|
|
1331
|
+
headers: {}
|
|
1332
|
+
});
|
|
1333
|
+
if (servers.size === 0) return void 0;
|
|
1334
|
+
try {
|
|
1335
|
+
AGENTS[harness.kind].mcpServers(Object.fromEntries(servers));
|
|
1336
|
+
return;
|
|
1337
|
+
} catch (error) {
|
|
1338
|
+
if (error instanceof InfraError) return error.message;
|
|
1339
|
+
throw error;
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
/** The first host variable a plugin MCP server reads (`${VAR}`), as `VAR`; undefined for none. */
|
|
1343
|
+
function hostVariable(s) {
|
|
1344
|
+
const values = Object.entries(s.transport === "stdio" ? s.env : s.headers);
|
|
1345
|
+
for (const [key, v] of values) {
|
|
1346
|
+
if (typeof v === "string") continue;
|
|
1347
|
+
if (v.kind === "secret") return v.from ?? key;
|
|
1348
|
+
return /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/.exec(v.template)?.[1] ?? key;
|
|
1349
|
+
}
|
|
1350
|
+
}
|
|
1351
|
+
const quoted = (parts) => parts.map((p) => `'${p}'`).join(", ");
|
|
1352
|
+
/**
|
|
1353
|
+
* Why a harness's agent can't take the plugins, in the agent's own words: a part it doesn't
|
|
1354
|
+
* load, apps it can't take (`pluginApps`), a hook event or type it doesn't run, a plugin MCP
|
|
1355
|
+
* server it refuses (named `<plugin>:<server>`) or whose `<plugin>:<server>` key `mcp` already has,
|
|
1356
|
+
* or, where it takes plugin items flat, a skill or MCP server whose name another already has, or a
|
|
1357
|
+
* subagent, command or rule another unpacks to the same file. `own` names the
|
|
1358
|
+
* environment's own skills and MCP servers, and `unread` the plugins not fetched yet (git and
|
|
1359
|
+
* npm ones, at plan time) by their call. Undefined when it can take them.
|
|
1360
|
+
*/
|
|
1361
|
+
function refusedPlugins(harness, plugins, own, unread = []) {
|
|
1362
|
+
if (plugins.length === 0 && unread.length === 0) return void 0;
|
|
1363
|
+
const agent = isAgentKind(harness.kind) ? AGENTS[harness.kind] : void 0;
|
|
1364
|
+
const mimics = harness.kind === "fake" && harness.layout ? AGENTS[harness.layout] : void 0;
|
|
1365
|
+
const loads = (agent ?? mimics)?.pluginParts ?? PLUGIN_PARTS;
|
|
1366
|
+
const name = agent?.name ?? (mimics ? `The fake agent with ${mimics.name}'s layout` : "The fake agent");
|
|
1367
|
+
if (loads.length === 0) return `${name} can't load plugins yet: leave ${plugins[0] ? `the ${plugins[0].name} plugin` : unread[0]} out of this cell`;
|
|
1368
|
+
for (const p of plugins) {
|
|
1369
|
+
const refused = p.kept.filter((part) => !loads.includes(part));
|
|
1370
|
+
if (refused.length === 0) continue;
|
|
1371
|
+
const can = p.kept.filter((part) => loads.includes(part));
|
|
1372
|
+
const what = refused.map((part) => `${part} (${PART_PATHS[part]})`).join(" and ");
|
|
1373
|
+
const fix = can.length ? `use only: [${quoted(can)}], or leave` : "leave";
|
|
1374
|
+
return `${name} can't load the ${p.name} plugin's ${what}: ${fix} the plugin out of this cell`;
|
|
1375
|
+
}
|
|
1376
|
+
const reads = agent ?? mimics;
|
|
1377
|
+
const first = reads?.pluginManifestsFirst ?? [];
|
|
1378
|
+
for (const p of plugins) {
|
|
1379
|
+
const other = p.manifests.find((m) => first.includes(m) && (m !== "plugin.json" || !reads?.pluginRootManifestNeedsSchema || p.rootSchema));
|
|
1380
|
+
if (other) return `${name} reads the ${p.name} plugin's ${other} in place of its ${PLUGIN_MANIFEST}, so reeval can't hold it to the parts it places: remove ${other} from the plugin, or leave the plugin out of this cell`;
|
|
1381
|
+
}
|
|
1382
|
+
const apps = (agent ?? mimics)?.pluginApps;
|
|
1383
|
+
const withApps = apps ? plugins.find((p) => p.apps) : void 0;
|
|
1384
|
+
if (withApps) return `${name} can't load the ${withApps.name} plugin's apps: ${apps}`;
|
|
1385
|
+
const layout = layoutOf(harness);
|
|
1386
|
+
const runs = layout.pluginHookEvents;
|
|
1387
|
+
for (const p of runs ? plugins : []) {
|
|
1388
|
+
const event = p.hooks.events.find((e) => !runs?.includes(e));
|
|
1389
|
+
const type = p.hooks.otherTypes[0];
|
|
1390
|
+
if (!event && !type) continue;
|
|
1391
|
+
const rest = p.kept.filter((part) => part !== "hooks");
|
|
1392
|
+
const fix = rest.length ? `use only: [${quoted(rest)}], or leave` : "leave";
|
|
1393
|
+
return event ? `${name} can't run the ${p.name} plugin's ${event} hooks (hooks/hooks.json): it runs ${runs?.join(", ")} hooks; ${fix} the plugin out of this cell` : `${name} can't run the ${p.name} plugin's ${type} hooks (hooks/hooks.json): it runs only command hooks; ${fix} the plugin out of this cell`;
|
|
1394
|
+
}
|
|
1395
|
+
const servers = Object.fromEntries(plugins.flatMap((p) => Object.entries(p.mcp).map(([server, s]) => [pluginServerName(p.name, server), s.transport === "stdio" ? {
|
|
1396
|
+
...s,
|
|
1397
|
+
env: {}
|
|
1398
|
+
} : {
|
|
1399
|
+
...s,
|
|
1400
|
+
headers: {}
|
|
1401
|
+
}])));
|
|
1402
|
+
if (agent && Object.keys(servers).length) {
|
|
1403
|
+
const alongside = agent.pluginPlacement === "flat" ? Object.fromEntries(own.mcp.map((n) => [n, {
|
|
1404
|
+
transport: "stdio",
|
|
1405
|
+
command: "true",
|
|
1406
|
+
args: [],
|
|
1407
|
+
env: {}
|
|
1408
|
+
}])) : {};
|
|
1409
|
+
try {
|
|
1410
|
+
agent.mcpServers({
|
|
1411
|
+
...alongside,
|
|
1412
|
+
...servers
|
|
1413
|
+
});
|
|
1414
|
+
} catch (error) {
|
|
1415
|
+
if (error instanceof InfraError) return error.message;
|
|
1416
|
+
throw error;
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1419
|
+
for (const p of plugins) for (const server of Object.keys(p.mcp)) {
|
|
1420
|
+
const key = pluginServerName(p.name, server);
|
|
1421
|
+
if (own.mcp.includes(key)) return `the ${p.name} plugin's MCP server "${server}" would join mcp as ${key}, which mcp already names: rename mcp["${key}"]`;
|
|
1422
|
+
}
|
|
1423
|
+
if (layout.pluginPlacement === "namespaced") {
|
|
1424
|
+
for (const p of plugins) {
|
|
1425
|
+
if (!p.kept.includes("mcp")) continue;
|
|
1426
|
+
for (const [server, s] of Object.entries(p.mcp)) {
|
|
1427
|
+
const secret = hostVariable(s);
|
|
1428
|
+
if (!secret) continue;
|
|
1429
|
+
const rest = p.kept.filter((part) => part !== "mcp");
|
|
1430
|
+
return `${name} loads the ${p.name} plugin's MCP server "${server}" itself, so reeval can't give it ${secret} from the host: use only: [${quoted(rest)}] and declare the server in mcp, where reeval fills it in`;
|
|
1431
|
+
}
|
|
1432
|
+
}
|
|
1433
|
+
if (!(agent ?? mimics)?.pluginServersByName) return void 0;
|
|
1434
|
+
const byName = new Map(own.mcp.map((n) => [n, `mcp.${n}`]));
|
|
1435
|
+
for (const p of plugins) for (const server of Object.keys(p.mcp)) {
|
|
1436
|
+
const other = byName.get(server);
|
|
1437
|
+
if (other) return `${name} names the ${p.name} plugin's MCP server "${server}" by its own name, as it does ${other}, and keeps only one: rename one, or leave mcp out of the plugin with only`;
|
|
1438
|
+
byName.set(server, `the ${p.name} plugin's MCP server "${server}"`);
|
|
1439
|
+
}
|
|
1440
|
+
return;
|
|
1441
|
+
}
|
|
1442
|
+
const serverOwner = new Map(own.mcp.map((n) => [n, `mcp.${n}`]));
|
|
1443
|
+
for (const p of plugins) for (const server of Object.keys(p.mcp)) {
|
|
1444
|
+
const other = serverOwner.get(server);
|
|
1445
|
+
if (other) return `the ${p.name} plugin's MCP server "${server}" clashes with ${other}`;
|
|
1446
|
+
serverOwner.set(server, `the ${p.name} plugin's MCP server "${server}"`);
|
|
1447
|
+
}
|
|
1448
|
+
const skillOwner = new Map(own.skills.map((s) => [s.name, `${s.call}'s "${s.name}"`]));
|
|
1449
|
+
for (const p of plugins) for (const { name: skill } of p.skills) {
|
|
1450
|
+
const other = skillOwner.get(skill);
|
|
1451
|
+
if (other) return `the ${p.name} plugin's skill "${skill}" and ${other} would both be placed at ~/${layout.harnessSkillDir}/${skill}: rename one, or narrow it with only or exclude`;
|
|
1452
|
+
skillOwner.set(skill, `the ${p.name} plugin's skill "${skill}"`);
|
|
1453
|
+
}
|
|
1454
|
+
for (const part of ["agents", "commands"]) {
|
|
1455
|
+
const dir = layout.pluginUnpack?.[part];
|
|
1456
|
+
if (!dir) continue;
|
|
1457
|
+
const owner = /* @__PURE__ */ new Map();
|
|
1458
|
+
for (const p of plugins) for (const item of p[part]) {
|
|
1459
|
+
const other = owner.get(item);
|
|
1460
|
+
const what = part === "agents" ? "subagent" : "command";
|
|
1461
|
+
if (other) return `the ${p.name} plugin's ${what} "${item}" and the ${other} plugin's would both be placed at ${dir}/${item}.md: rename one, or narrow one with only`;
|
|
1462
|
+
owner.set(item, p.name);
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
const rules = layout.pluginUnpack?.rules;
|
|
1466
|
+
const ruleOwner = /* @__PURE__ */ new Map();
|
|
1467
|
+
for (const p of rules ? plugins : []) for (const rule of p.rules) {
|
|
1468
|
+
const file = `${p.name}-${rule.replace(/\.mdc?$/, "")}.mdc`;
|
|
1469
|
+
const what = `the ${p.name} plugin's rule "${rule}"`;
|
|
1470
|
+
const other = ruleOwner.get(file);
|
|
1471
|
+
if (other) return `${what} and ${other} would both be placed at ${rules}/${file}: rename one, or narrow one with only`;
|
|
1472
|
+
ruleOwner.set(file, what);
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
/** A real agent's catalog entry, with what its model's provider decides. */
|
|
1476
|
+
function agentFor(h) {
|
|
1477
|
+
const agent = AGENTS[h.kind];
|
|
1478
|
+
if (h.gateway === "vercel") {
|
|
1479
|
+
const route = VERCEL_ROUTES[h.kind];
|
|
1480
|
+
if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on Vercel AI Gateway`);
|
|
1481
|
+
return {
|
|
1482
|
+
...agent,
|
|
1483
|
+
...route(process.env)
|
|
1484
|
+
};
|
|
1485
|
+
}
|
|
1486
|
+
return agent.forModel ? {
|
|
1487
|
+
...agent,
|
|
1488
|
+
...agent.forModel(h.model)
|
|
1489
|
+
} : agent;
|
|
1490
|
+
}
|
|
1491
|
+
/** The catalog entry for a harness's agent; undefined for the fake harness. */
|
|
1492
|
+
const agentOf = (h) => h.kind === "fake" ? void 0 : agentFor(h);
|
|
1493
|
+
/** The host variable a harness authenticates with; undefined for the fake harness. */
|
|
1494
|
+
const credentialOf = (h) => agentOf(h)?.credential;
|
|
1495
|
+
/** The fake agent's own layout. */
|
|
1496
|
+
const FAKE_LAYOUT = {
|
|
1497
|
+
rulesFile: "AGENTS.md",
|
|
1498
|
+
workspaceSkillDir: ".agents/skills",
|
|
1499
|
+
harnessSkillDir: ".agents/skills",
|
|
1500
|
+
pluginPlacement: "flat"
|
|
1501
|
+
};
|
|
1502
|
+
/** Where a harness's agent reads rules and skills; a fake one with `layout` mimics that agent. */
|
|
1503
|
+
function layoutOf(descriptor) {
|
|
1504
|
+
if (descriptor.kind !== "fake") return LAYOUTS[descriptor.kind];
|
|
1505
|
+
return descriptor.layout ? LAYOUTS[descriptor.layout] : FAKE_LAYOUT;
|
|
1506
|
+
}
|
|
1507
|
+
/**
|
|
1508
|
+
* The hosts a harness's agent is known to try in a session with or without plugins; a fake agent
|
|
1509
|
+
* with `layout` expects that agent's.
|
|
1510
|
+
*/
|
|
1511
|
+
function expectedBlockedFor(descriptor, plugins) {
|
|
1512
|
+
const expected = descriptor.kind !== "fake" ? agentFor(descriptor).expectedBlocked : descriptor.layout ? AGENTS[descriptor.layout].expectedBlocked : void 0;
|
|
1513
|
+
return expected?.withPlugins && !plugins ? void 0 : expected;
|
|
1514
|
+
}
|
|
1515
|
+
/**
|
|
1516
|
+
* How the APIs behind the harnesses say they refused a key: an HTTP 401, Anthropic's and OpenAI's
|
|
1517
|
+
* wording, Google's (which answers 400), and OpenCode's for any provider's 401 or 403. A bare 403 isn't here: the gateway and MCP servers send
|
|
1518
|
+
* those for reasons that have nothing to do with the harness's key. Grok Build checks its key as
|
|
1519
|
+
* it starts and, when xAI refuses it (a 400 or 401, not an outage), offers no `xai.api_key`
|
|
1520
|
+
* sign-in, which is all `@ai-sdk/harness-acp` reports.
|
|
1521
|
+
*/
|
|
1522
|
+
const REFUSED_KEY = /\bHTTP 401\b|\b401\b.{0,40}\bunauthori[sz]ed\b|\bunauthori[sz]ed\b.{0,40}\b401\b|\b(?:invalid|incorrect) (?:x-)?api[ _-]?key\b|\bapi key not valid\b|\bAPI_KEY_INVALID\b|\bauthentication_error\b|\bprovider authentication required\b|\bauthentication method "xai\.api_key" is not advertised\b/i;
|
|
1523
|
+
/**
|
|
1524
|
+
* What to do when the harness's API refused its key, naming the variable; else undefined. The
|
|
1525
|
+
* error's wording says so for some CLIs; `refused` is the egress record's word for every one: a
|
|
1526
|
+
* brokered request the API answered with 401, whatever the CLI made of it.
|
|
1527
|
+
*/
|
|
1528
|
+
function authHint(agent, message, refused = false) {
|
|
1529
|
+
if (!refused && !REFUSED_KEY.test(message)) return void 0;
|
|
1530
|
+
return `The API rejected ${agent.credential}: set a valid key in the environment or .env.local`;
|
|
1531
|
+
}
|
|
1532
|
+
/** The agent's credential from the host, as the adapter's explicit `auth`. Fails closed. */
|
|
1533
|
+
function agentAuth(agent, env = process.env) {
|
|
1534
|
+
const value = env[agent.credential];
|
|
1535
|
+
if (!value) throw new InfraError(`${agent.credential} isn't set, so the ${agent.name} harness can't authenticate. Set it in the environment or .env.local.`);
|
|
1536
|
+
return { [agent.authAs ?? agent.credential]: value };
|
|
1537
|
+
}
|
|
1538
|
+
function brokeringFor(h) {
|
|
1539
|
+
const agent = agentOf(h);
|
|
1540
|
+
if (!agent) return void 0;
|
|
1541
|
+
const { credential, endpoints, tokens, servedModel, askedModel, modelResponses } = agent;
|
|
1542
|
+
return {
|
|
1543
|
+
credential,
|
|
1544
|
+
endpoints,
|
|
1545
|
+
...tokens ? { tokens: {
|
|
1546
|
+
name: credential,
|
|
1547
|
+
...tokens
|
|
1548
|
+
} } : {},
|
|
1549
|
+
...servedModel ? { servedModel } : {},
|
|
1550
|
+
...askedModel ? { askedModel } : {},
|
|
1551
|
+
...modelResponses ? { modelResponses } : {}
|
|
1552
|
+
};
|
|
1553
|
+
}
|
|
1554
|
+
//#endregion
|
|
1555
|
+
//#region src/harness/effort.ts
|
|
1556
|
+
/**
|
|
1557
|
+
* The reasoning effort a harness asked for. Same user-facing name on every factory; the CLI's
|
|
1558
|
+
* native flag may say thinking. Undefined when the factory left it unset. Fast is not this.
|
|
1559
|
+
*/
|
|
1560
|
+
function harnessEffort(h) {
|
|
1561
|
+
return "effort" in h && typeof h.effort === "string" ? h.effort : void 0;
|
|
1562
|
+
}
|
|
1563
|
+
/**
|
|
1564
|
+
* The reasoning effort a served-model label recorded, e.g. `gpt-5.6-luna[effort=medium]` →
|
|
1565
|
+
* `medium`. Fast (`[fast=true]`) is not effort and is ignored.
|
|
1566
|
+
*/
|
|
1567
|
+
function effortFromModelLabel(model) {
|
|
1568
|
+
const inner = model?.match(/\[([^\]]+)]/)?.[1];
|
|
1569
|
+
if (!inner) return void 0;
|
|
1570
|
+
for (const part of inner.split(",")) {
|
|
1571
|
+
const eq = part.indexOf("=");
|
|
1572
|
+
if (eq < 0) continue;
|
|
1573
|
+
const key = part.slice(0, eq).trim();
|
|
1574
|
+
if (key === "fast") continue;
|
|
1575
|
+
if (/^(?:reasoning_)?effort$|^thinking$/.test(key)) {
|
|
1576
|
+
const value = part.slice(eq + 1).trim();
|
|
1577
|
+
if (value) return value;
|
|
1578
|
+
}
|
|
1579
|
+
}
|
|
1580
|
+
}
|
|
1581
|
+
/** The first reasoning-effort suffix among served-model labels; undefined when none name one. */
|
|
1582
|
+
function servedEffortOf(models) {
|
|
1583
|
+
if (!models) return void 0;
|
|
1584
|
+
for (const model of models) {
|
|
1585
|
+
const effort = effortFromModelLabel(model);
|
|
1586
|
+
if (effort) return effort;
|
|
1587
|
+
}
|
|
1588
|
+
}
|
|
1589
|
+
/** Unique reasoning-effort suffixes among served-model labels, in first-seen order. */
|
|
1590
|
+
function resolvedEffortOf(models) {
|
|
1591
|
+
if (!models) return [];
|
|
1592
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1593
|
+
const out = [];
|
|
1594
|
+
for (const model of models) {
|
|
1595
|
+
const effort = effortFromModelLabel(model);
|
|
1596
|
+
if (effort && !seen.has(effort)) {
|
|
1597
|
+
seen.add(effort);
|
|
1598
|
+
out.push(effort);
|
|
1599
|
+
}
|
|
1600
|
+
}
|
|
1601
|
+
return out;
|
|
1602
|
+
}
|
|
1603
|
+
//#endregion
|
|
1604
|
+
//#region src/compare/state.ts
|
|
1605
|
+
/** The error a per-attempt answer carries when the run stopped before a model answered it. */
|
|
1606
|
+
const STOPPED_FIRST = "the run stopped before it was answered";
|
|
1607
|
+
/**
|
|
1608
|
+
* What each side of a pairwise question shows with no `state`: the task and the diff, every
|
|
1609
|
+
* turn's prompt. The final message stays out, since a better-written account of the work would
|
|
1610
|
+
* win a side-by-side pick on its wording; the changed files repeat what the diff shows.
|
|
1611
|
+
*/
|
|
1612
|
+
const DEFAULT_QUESTION_STATE = {
|
|
1613
|
+
task: true,
|
|
1614
|
+
output: false,
|
|
1615
|
+
files: false,
|
|
1616
|
+
diff: true,
|
|
1617
|
+
transcript: false,
|
|
1618
|
+
turns: "all",
|
|
1619
|
+
checks: false
|
|
1620
|
+
};
|
|
1621
|
+
/**
|
|
1622
|
+
* What a question asked of each attempt on its own shows with no `state`: the task, the diff,
|
|
1623
|
+
* the final message and the transcript. Nothing is weighed against another side, so how the
|
|
1624
|
+
* agent went about the work is the point.
|
|
1625
|
+
*/
|
|
1626
|
+
const DEFAULT_ATTEMPT_QUESTION_STATE = {
|
|
1627
|
+
...DEFAULT_QUESTION_STATE,
|
|
1628
|
+
output: true,
|
|
1629
|
+
transcript: true
|
|
1630
|
+
};
|
|
1631
|
+
const KEYS = Object.keys(DEFAULT_QUESTION_STATE);
|
|
1632
|
+
/** A pairwise question's defaults when `pairwise`, a per-attempt question's otherwise. */
|
|
1633
|
+
const defaultQuestionState = (pairwise) => pairwise ? DEFAULT_QUESTION_STATE : DEFAULT_ATTEMPT_QUESTION_STATE;
|
|
1634
|
+
/** Every key set, in `DEFAULT_QUESTION_STATE`'s order, so equal selections serialise the same. */
|
|
1635
|
+
function resolveQuestionState(state, pairwise) {
|
|
1636
|
+
const defaults = defaultQuestionState(pairwise);
|
|
1637
|
+
const out = {};
|
|
1638
|
+
for (const key of KEYS) out[key] = state?.[key] ?? defaults[key];
|
|
1639
|
+
return out;
|
|
1640
|
+
}
|
|
1641
|
+
/** `inner` over `outer`: `state` key by key, `model` and `artifacts` replaced. */
|
|
1642
|
+
function mergeQuestion(outer, inner) {
|
|
1643
|
+
if (!inner) return outer;
|
|
1644
|
+
const out = {
|
|
1645
|
+
...outer,
|
|
1646
|
+
...inner
|
|
1647
|
+
};
|
|
1648
|
+
if (outer.state || inner.state) out.state = {
|
|
1649
|
+
...outer.state,
|
|
1650
|
+
...inner.state
|
|
1651
|
+
};
|
|
1652
|
+
return out;
|
|
1653
|
+
}
|
|
1654
|
+
/** A planned question with the cascaded `question` defaults under its own `model`, `state` and `artifacts`. */
|
|
1655
|
+
function withQuestionDefaults(q, defaults) {
|
|
1656
|
+
const { state, artifacts } = mergeQuestion(defaults, {
|
|
1657
|
+
...q.state ? { state: q.state } : {},
|
|
1658
|
+
...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
|
|
1659
|
+
});
|
|
1660
|
+
const { state: _state, artifacts: _artifacts, ...rest } = q;
|
|
1661
|
+
const model = q.model ?? defaults.model;
|
|
1662
|
+
return {
|
|
1663
|
+
...rest,
|
|
1664
|
+
...model !== void 0 ? { model } : {},
|
|
1665
|
+
...state && Object.keys(state).length ? { state } : {},
|
|
1666
|
+
...artifacts !== void 0 ? { artifacts } : {}
|
|
1667
|
+
};
|
|
1668
|
+
}
|
|
1669
|
+
const isOff = (value) => value === false || Array.isArray(value) && !value.length;
|
|
1670
|
+
/** True when the resolved state shows nothing and no captures are sent. */
|
|
1671
|
+
const showsNothing = (q) => {
|
|
1672
|
+
if (q.artifacts !== void 0) return false;
|
|
1673
|
+
const s = resolveQuestionState(q.state, q.across !== void 0);
|
|
1674
|
+
return KEYS.every((k) => k === "turns" || isOff(s[k]));
|
|
1675
|
+
};
|
|
1676
|
+
/** The state keys a question shows, in order, plus `artifacts` when it sends captures. */
|
|
1677
|
+
function questionShows(state, artifacts) {
|
|
1678
|
+
const on = KEYS.filter((k) => k !== "turns" && !isOff(state[k]));
|
|
1679
|
+
return artifacts === void 0 ? on : [...on, "artifacts"];
|
|
1680
|
+
}
|
|
1681
|
+
//#endregion
|
|
1682
|
+
//#region src/contract/compare.ts
|
|
1683
|
+
const compareGateSchema = z.enum([
|
|
1684
|
+
"variants",
|
|
1685
|
+
"all",
|
|
1686
|
+
"none"
|
|
1687
|
+
]);
|
|
1688
|
+
/** The compare object, with environments checked by `environment`. */
|
|
1689
|
+
const compareSchema = (environment) => z.strictObject({
|
|
1690
|
+
baseline: environment.optional(),
|
|
1691
|
+
variants: z.array(environment).min(1),
|
|
1692
|
+
gate: compareGateSchema.optional(),
|
|
1693
|
+
minLift: z.number().min(-1).max(1).optional(),
|
|
1694
|
+
harness: z.array(z.union([z.string(), harnessDescriptorSchema])).optional()
|
|
1695
|
+
});
|
|
1696
|
+
compareSchema(z.custom());
|
|
1697
|
+
const describe$1 = (value) => {
|
|
1698
|
+
if (Array.isArray(value)) return `a list of ${value.length}`;
|
|
1699
|
+
if (typeof value === "object" && value !== null) {
|
|
1700
|
+
const keys = Object.keys(value);
|
|
1701
|
+
return keys.length ? `an object with ${keys.map((k) => `"${k}"`).join(", ")}` : "an object";
|
|
1702
|
+
}
|
|
1703
|
+
return JSON.stringify(value) ?? String(value);
|
|
1704
|
+
};
|
|
1705
|
+
function explain(where) {
|
|
1706
|
+
return (issue, value) => {
|
|
1707
|
+
const spec = value;
|
|
1708
|
+
if (issue.code === "unrecognized_keys" && issue.path.length === 0) return `Unknown compare option "${issue.keys[0]}" in ${where}. compare takes ${Object.keys(compareSchema(z.any()).shape).join(", ")}.`;
|
|
1709
|
+
const [key, index] = issue.path;
|
|
1710
|
+
const v = spec[key];
|
|
1711
|
+
switch (key) {
|
|
1712
|
+
case "baseline": return `compare.baseline in ${where} must be an environment from defineEnvironment (got ${describe$1(v)}). It is the control; variants are not layered on it or on the eval's environment. Leave it out to use the eval's environment as the control.`;
|
|
1713
|
+
case "gate": return `compare.gate in ${where} is ${oneOf(compareGateSchema.options)} (got ${describe$1(v)})`;
|
|
1714
|
+
case "minLift": return `compare.minLift in ${where} is points from -1 to 1, e.g. minLift: 0.1 (got ${describe$1(v)})`;
|
|
1715
|
+
case "harness": {
|
|
1716
|
+
const item = index === void 0 ? void 0 : v[index];
|
|
1717
|
+
const kind = item?.kind;
|
|
1718
|
+
const problem = kind && kind in FACTORY ? harnessProblem(item) : void 0;
|
|
1719
|
+
return problem ? `compare.harness in ${where}: ${problem}` : `compare.harness in ${where} must be a list of harness ids, e.g. harness: ['cursor:composer-2.5'] (got ${describe$1(index === void 0 ? v : item)})`;
|
|
1720
|
+
}
|
|
1721
|
+
default: return index === void 0 ? `compare has no variants in ${where}. Give at least one environment.` : `compare.variants in ${where} must be environments from defineEnvironment (got ${describe$1(v[index])}). Variants share the eval's prompt and checks; if a variant needs different checks, write a separate eval.`;
|
|
1722
|
+
}
|
|
1723
|
+
};
|
|
1724
|
+
}
|
|
1725
|
+
/** The order the hand-written checks ran in, so the first refusal stays the same. */
|
|
1726
|
+
const ORDER = [
|
|
1727
|
+
"baseline",
|
|
1728
|
+
"gate",
|
|
1729
|
+
"minLift",
|
|
1730
|
+
"harness",
|
|
1731
|
+
"variants"
|
|
1732
|
+
];
|
|
1733
|
+
/**
|
|
1734
|
+
* What's wrong with a compare object whose environments `environment` checks, or undefined.
|
|
1735
|
+
* `where` names the group or eval it's set on.
|
|
1736
|
+
*/
|
|
1737
|
+
function compareProblem(value, where, environment) {
|
|
1738
|
+
if (Array.isArray(value)) return `compare in ${where} must be an object, not a list. Use compare: { baseline, variants: [...] }.`;
|
|
1739
|
+
if (!isObjectLike(value) || !Array.isArray(value.variants)) return `compare in ${where} takes { variants, baseline? } (got ${describe$1(value)}). Use compare: { variants: [a, b], baseline?, gate?, minLift?, harness? }.`;
|
|
1740
|
+
return refusal(compareSchema(environment), value, explain(where), { order: ORDER });
|
|
1741
|
+
}
|
|
1742
|
+
//#endregion
|
|
1743
|
+
//#region src/sandbox/fake.ts
|
|
1744
|
+
/** Commands run on the host's own network, so the fake sandbox can neither restrict nor see it. */
|
|
1745
|
+
const FAKE_EGRESS = {
|
|
1746
|
+
enforce: "none",
|
|
1747
|
+
record: false
|
|
1748
|
+
};
|
|
1749
|
+
/**
|
|
1750
|
+
* The POSIX shell commands run in: `sh`, which Windows lacks, so there the `sh.exe` on PATH or
|
|
1751
|
+
* Git for Windows'.
|
|
1752
|
+
*/
|
|
1753
|
+
function posixShell(platform = process.platform, env = process.env, exists = existsSync) {
|
|
1754
|
+
if (platform !== "win32") return "sh";
|
|
1755
|
+
const onPath = (env.PATH ?? env.Path ?? "").split(win32.delimiter).filter(Boolean);
|
|
1756
|
+
const git = [
|
|
1757
|
+
env.ProgramFiles,
|
|
1758
|
+
env["ProgramFiles(x86)"],
|
|
1759
|
+
env.LOCALAPPDATA && win32.join(env.LOCALAPPDATA, "Programs")
|
|
1760
|
+
].filter((dir) => !!dir).map((dir) => win32.join(dir, "Git", "bin"));
|
|
1761
|
+
const found = [...onPath, ...git].map((dir) => win32.join(dir, "sh.exe")).find(exists);
|
|
1762
|
+
if (found) return found;
|
|
1763
|
+
throw new UsageError("The fake sandbox runs commands with a POSIX sh, and there's none on this machine. Install Git for Windows (https://git-scm.com/download/win), which brings sh.exe, or run the evals in docker().");
|
|
1764
|
+
}
|
|
1765
|
+
const EXEC_TAG = "REEVAL_FAKE_EXEC";
|
|
1766
|
+
const KILL_TAGGED = `for pass in 1 2 3 4 5; do
|
|
1767
|
+
found=
|
|
1768
|
+
for f in $(grep -lzx "${EXEC_TAG}=$1" /proc/[0-9]*/environ 2>/dev/null); do
|
|
1769
|
+
p=\${f#/proc/}; kill -9 "\${p%/environ}" 2>/dev/null; found=1
|
|
1770
|
+
done
|
|
1771
|
+
[ -n "$found" ] || break
|
|
1772
|
+
done`;
|
|
1773
|
+
/**
|
|
1774
|
+
* Stops a command's MSYS processes on Windows. `taskkill /t` doesn't reach them: Git's `sh.exe`
|
|
1775
|
+
* starts bash, whose fork and exec leave each new process's Windows parent gone, so a timed-out
|
|
1776
|
+
* `sleep 30` would run on, holding the command's pipes and working directory.
|
|
1777
|
+
*/
|
|
1778
|
+
function killTagged(sh, tag) {
|
|
1779
|
+
runSh(sh, KILL_TAGGED, tag);
|
|
1780
|
+
}
|
|
1781
|
+
/**
|
|
1782
|
+
* Kills host processes whose cwd is the sandbox, including native `node.exe` `c.serve` starts.
|
|
1783
|
+
* Match only the unique `reeval-sandbox-*` folder name: Git Bash `pwd` and node.exe's
|
|
1784
|
+
* `/proc/cwd` disagree (`/tmp/...` vs `D:\\Users\\...\\Temp\\...`), and an empty
|
|
1785
|
+
* `"$prefix"/*` is `/*`, which SIGKILLs the Windows runner (exit 2304).
|
|
1786
|
+
*/
|
|
1787
|
+
const KILL_UNDER = `cd "$1" || exit 0
|
|
1788
|
+
base=$(pwd -P 2>/dev/null || pwd)
|
|
1789
|
+
name=\${base##*/}
|
|
1790
|
+
case "$name" in
|
|
1791
|
+
reeval-sandbox-?*) ;;
|
|
1792
|
+
*) exit 0 ;;
|
|
1793
|
+
esac
|
|
1794
|
+
for f in /proc/[0-9]*/cwd; do
|
|
1795
|
+
t=$(readlink "$f" 2>/dev/null) || continue
|
|
1796
|
+
case "$t" in
|
|
1797
|
+
*"$name"*) ;;
|
|
1798
|
+
*) continue ;;
|
|
1799
|
+
esac
|
|
1800
|
+
p=\${f#/proc/}; p=\${p%/cwd}
|
|
1801
|
+
[ -n "$p" ] || continue
|
|
1802
|
+
kill -9 "$p" 2>/dev/null
|
|
1803
|
+
wp=$(tr -d '\\r\\n' < /proc/$p/winpid 2>/dev/null)
|
|
1804
|
+
[ -n "$wp" ] && taskkill /F /PID "$wp" >/dev/null 2>&1
|
|
1805
|
+
done`;
|
|
1806
|
+
function runSh(sh, script, arg) {
|
|
1807
|
+
return new Promise((resolve) => {
|
|
1808
|
+
const child = spawn(sh, [
|
|
1809
|
+
"-c",
|
|
1810
|
+
script,
|
|
1811
|
+
"sh",
|
|
1812
|
+
arg
|
|
1813
|
+
], {
|
|
1814
|
+
stdio: "ignore",
|
|
1815
|
+
windowsHide: true
|
|
1816
|
+
});
|
|
1817
|
+
const done = () => resolve();
|
|
1818
|
+
child.on("error", done);
|
|
1819
|
+
child.on("close", done);
|
|
1820
|
+
});
|
|
1821
|
+
}
|
|
1822
|
+
/** Windows refuses to delete a directory a live process still has as cwd; retry after kills. */
|
|
1823
|
+
async function removeTree(path) {
|
|
1824
|
+
const deadline = Date.now() + 8e3;
|
|
1825
|
+
for (;;) {
|
|
1826
|
+
try {
|
|
1827
|
+
await rm(path, {
|
|
1828
|
+
recursive: true,
|
|
1829
|
+
force: true
|
|
1830
|
+
});
|
|
1831
|
+
if (!existsSync(path)) return;
|
|
1832
|
+
} catch {}
|
|
1833
|
+
if (Date.now() >= deadline) {
|
|
1834
|
+
await rm(path, {
|
|
1835
|
+
recursive: true,
|
|
1836
|
+
force: true
|
|
1837
|
+
}).catch(() => {});
|
|
1838
|
+
return;
|
|
1839
|
+
}
|
|
1840
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
/**
|
|
1844
|
+
* A temp directory on the host. Commands run unisolated on the host shell, so this is
|
|
1845
|
+
* only meant for tests and for the fake harness.
|
|
1846
|
+
*/
|
|
1847
|
+
async function createFakeSandbox(descriptor, env = {}) {
|
|
1848
|
+
const sh = posixShell();
|
|
1849
|
+
const parent = descriptor.dir ?? tmpdir();
|
|
1850
|
+
await mkdir(parent, { recursive: true });
|
|
1851
|
+
const base = await mkdtemp(join(parent, "reeval-sandbox-"));
|
|
1852
|
+
const workdir = join(base, "work");
|
|
1853
|
+
await mkdir(workdir);
|
|
1854
|
+
const stateDir = `${base}.state`;
|
|
1855
|
+
await mkdir(stateDir);
|
|
1856
|
+
let destroyed;
|
|
1857
|
+
const resolve = (path) => path.startsWith("~/") ? join(base, path.slice(2)) : join(workdir, path);
|
|
1858
|
+
return {
|
|
1859
|
+
workdir,
|
|
1860
|
+
stateDir,
|
|
1861
|
+
execOwn: (command, { signal } = {}) => run(sh, ["-c", command], {
|
|
1862
|
+
cwd: workdir,
|
|
1863
|
+
env: {
|
|
1864
|
+
HOME: base,
|
|
1865
|
+
...env
|
|
1866
|
+
},
|
|
1867
|
+
signal
|
|
1868
|
+
}),
|
|
1869
|
+
exec: (command, { cwd, env: extra, signal } = {}) => {
|
|
1870
|
+
const tag = process.platform === "win32" && signal ? randomUUID() : void 0;
|
|
1871
|
+
const stop = () => killTagged(sh, tag);
|
|
1872
|
+
if (tag) signal?.addEventListener("abort", stop, { once: true });
|
|
1873
|
+
return run(sh, ["-c", cwd ? `cd ${shellQuote(cwd)} && ${command}` : command], {
|
|
1874
|
+
cwd: workdir,
|
|
1875
|
+
env: {
|
|
1876
|
+
HOME: base,
|
|
1877
|
+
...env,
|
|
1878
|
+
...extra,
|
|
1879
|
+
...tag ? { [EXEC_TAG]: tag } : {}
|
|
1880
|
+
},
|
|
1881
|
+
signal
|
|
1882
|
+
}).finally(() => signal?.removeEventListener("abort", stop));
|
|
1883
|
+
},
|
|
1884
|
+
home: base,
|
|
1885
|
+
readFile: (path) => readFile(resolve(path), "utf8").catch(() => null),
|
|
1886
|
+
getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}://127.0.0.1:${port}` }),
|
|
1887
|
+
async writeFile(path, content) {
|
|
1888
|
+
const target = resolve(path);
|
|
1889
|
+
await mkdir(dirname(target), { recursive: true });
|
|
1890
|
+
await writeFile(target, content);
|
|
1891
|
+
},
|
|
1892
|
+
destroy: () => {
|
|
1893
|
+
destroyed ??= (async () => {
|
|
1894
|
+
await rm(stateDir, {
|
|
1895
|
+
recursive: true,
|
|
1896
|
+
force: true
|
|
1897
|
+
}).catch(() => {});
|
|
1898
|
+
if (process.platform === "win32") {
|
|
1899
|
+
if (existsSync(base)) await runSh(sh, KILL_UNDER, base);
|
|
1900
|
+
await removeTree(base);
|
|
1901
|
+
return;
|
|
1902
|
+
}
|
|
1903
|
+
await rm(base, {
|
|
1904
|
+
recursive: true,
|
|
1905
|
+
force: true
|
|
1906
|
+
}).catch(() => {});
|
|
1907
|
+
})().finally(() => {
|
|
1908
|
+
destroyed = void 0;
|
|
1909
|
+
});
|
|
1910
|
+
return destroyed;
|
|
1911
|
+
}
|
|
1912
|
+
};
|
|
1913
|
+
}
|
|
1914
|
+
//#endregion
|
|
1915
|
+
//#region src/ui/mode.ts
|
|
1916
|
+
const set = (env, name) => {
|
|
1917
|
+
const value = env[name];
|
|
1918
|
+
return !!value && value !== "false" && value !== "0";
|
|
1919
|
+
};
|
|
1920
|
+
const CI_VARS = [
|
|
1921
|
+
"BUILD_ID",
|
|
1922
|
+
"BUILD_NUMBER",
|
|
1923
|
+
"CI",
|
|
1924
|
+
"CI_APP_ID",
|
|
1925
|
+
"CI_BUILD_ID",
|
|
1926
|
+
"CI_BUILD_NUMBER",
|
|
1927
|
+
"CI_NAME",
|
|
1928
|
+
"CONTINUOUS_INTEGRATION",
|
|
1929
|
+
"RUN_ID",
|
|
1930
|
+
"TEAMCITY_VERSION",
|
|
1931
|
+
"GITHUB_ACTIONS"
|
|
1932
|
+
];
|
|
1933
|
+
const isCI = (env) => CI_VARS.some((name) => set(env, name));
|
|
1934
|
+
function pick(name, value, allowed) {
|
|
1935
|
+
if (value === void 0 || value === "") return void 0;
|
|
1936
|
+
if (value === "auto" || isOneOf(allowed, value)) return value;
|
|
1937
|
+
throw new UsageError(`${name} is one of ${[...allowed, "auto"].join(", ")} (got ${value})`);
|
|
1938
|
+
}
|
|
1939
|
+
function parseHeartbeat(value) {
|
|
1940
|
+
if (!value) return 6e4;
|
|
1941
|
+
const match = /^(\d+(?:\.\d+)?)(ms|s|m)?$/.exec(value);
|
|
1942
|
+
if (!match) throw new UsageError(`REEVAL_HEARTBEAT must be a duration like 30s, or 0 for none (got ${value})`);
|
|
1943
|
+
const unit = {
|
|
1944
|
+
ms: 1,
|
|
1945
|
+
s: 1e3,
|
|
1946
|
+
m: 6e4
|
|
1947
|
+
}[match[2] ?? "s"] ?? 1e3;
|
|
1948
|
+
return Number(match[1]) * unit;
|
|
1949
|
+
}
|
|
1950
|
+
function resolveOutput(input) {
|
|
1951
|
+
const { flags, env } = input;
|
|
1952
|
+
const ci = isCI(env);
|
|
1953
|
+
const githubActions = set(env, "GITHUB_ACTIONS");
|
|
1954
|
+
const ui = pick("--ui", flags.ui ?? env.REEVAL_UI, [
|
|
1955
|
+
"live",
|
|
1956
|
+
"plain",
|
|
1957
|
+
"agent"
|
|
1958
|
+
]);
|
|
1959
|
+
const requested = ui && ui !== "auto" ? ui : input.agent ? "agent" : ci || !input.isTTY ? "plain" : "live";
|
|
1960
|
+
const mode = requested === "live" ? "plain" : requested;
|
|
1961
|
+
const order = pick("--log-order", flags.logOrder ?? env.REEVAL_LOG_ORDER, ["stream", "grouped"]);
|
|
1962
|
+
const logOrder = order && order !== "auto" ? order : ci ? "grouped" : "stream";
|
|
1963
|
+
const prefix = pick("--log-prefix", flags.logPrefix ?? env.REEVAL_LOG_PREFIX, ["task", "none"]);
|
|
1964
|
+
const logPrefix = prefix && prefix !== "auto" ? prefix : githubActions && logOrder === "grouped" ? "none" : "task";
|
|
1965
|
+
const logs = pick("--output-logs", flags.outputLogs ?? env.REEVAL_OUTPUT_LOGS, [
|
|
1966
|
+
"full",
|
|
1967
|
+
"errors-only",
|
|
1968
|
+
"none"
|
|
1969
|
+
]);
|
|
1970
|
+
return {
|
|
1971
|
+
requested,
|
|
1972
|
+
mode,
|
|
1973
|
+
logOrder,
|
|
1974
|
+
logPrefix,
|
|
1975
|
+
outputLogs: logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only",
|
|
1976
|
+
color: env.NO_COLOR ? false : env.FORCE_COLOR !== void 0 && env.FORCE_COLOR !== "" ? set(env, "FORCE_COLOR") : mode === "agent" ? false : input.isTTY || githubActions,
|
|
1977
|
+
githubActions,
|
|
1978
|
+
heartbeatMs: mode === "agent" ? 0 : parseHeartbeat(env.REEVAL_HEARTBEAT)
|
|
1979
|
+
};
|
|
1980
|
+
}
|
|
1981
|
+
//#endregion
|
|
1982
|
+
//#region src/egress.ts
|
|
1983
|
+
/**
|
|
1984
|
+
* The package registries and GitHub, to spread into an allowlist that still installs:
|
|
1985
|
+
* `egress: { allow: [...DEFAULT_EGRESS_ALLOW, 'api.example.com'] }`.
|
|
1986
|
+
*/
|
|
1987
|
+
const DEFAULT_EGRESS_ALLOW = PACKAGE_HOSTS;
|
|
1988
|
+
/** What each sandbox kind can do about egress, as its provider declares it. */
|
|
1989
|
+
const SANDBOX_EGRESS = {
|
|
1990
|
+
docker: DOCKER_EGRESS,
|
|
1991
|
+
fake: FAKE_EGRESS
|
|
1992
|
+
};
|
|
1993
|
+
/** The policy an attempt runs under when nothing sets one. */
|
|
1994
|
+
const DEFAULT_EGRESS = "proxy";
|
|
1995
|
+
/** Whether the policy allows only some hosts: `deny-all` and allowlists. */
|
|
1996
|
+
const isRestricted = (policy) => policy !== "allow-all" && policy !== "proxy";
|
|
1997
|
+
/**
|
|
1998
|
+
* Whether traffic can leave around the gateway: under `allow-all` the container keeps its network,
|
|
1999
|
+
* so whatever ignores the proxy goes out directly, unfiltered and unrecorded.
|
|
2000
|
+
*/
|
|
2001
|
+
const isDirect = (policy) => policy === "allow-all";
|
|
2002
|
+
/**
|
|
2003
|
+
* The hosts the harness needs: a real agent's own API (its model's provider's, for OpenCode), or
|
|
2004
|
+
* a fake agent's with `layout` (it behaves like that agent). A plain fake agent talks to nothing.
|
|
2005
|
+
*/
|
|
2006
|
+
function harnessHosts(harness) {
|
|
2007
|
+
if (harness.kind !== "fake" && isAgentKind(harness.kind)) return agentFor(harness).hosts ?? HARNESS_HOSTS[harness.kind];
|
|
2008
|
+
return harness.kind === "fake" && harness.layout ? HARNESS_HOSTS[harness.layout] : [];
|
|
2009
|
+
}
|
|
2010
|
+
/** MCP `http`/`sse` servers' hosts, local plugins' included: the agent connects to them from the sandbox. */
|
|
2011
|
+
function mcpServerHosts(layers) {
|
|
2012
|
+
const urls = /* @__PURE__ */ new Map();
|
|
2013
|
+
for (const { spec } of layers) for (const [name, server] of Object.entries(spec.mcp ?? {})) if (server.transport === "stdio") urls.delete(name);
|
|
2014
|
+
else urls.set(name, server.url);
|
|
2015
|
+
for (const [name, server] of localPluginServers(layers)) if (server.transport !== "stdio") urls.set(name, server.url);
|
|
2016
|
+
return [...urls.values()].flatMap((url) => URL.parse(url)?.hostname ?? []);
|
|
2017
|
+
}
|
|
2018
|
+
/**
|
|
2019
|
+
* The policy an attempt runs under, and the hosts a restricted one allows on its own: the
|
|
2020
|
+
* harness's, the brokered secrets' `hosts` and the MCP servers'. The cascade is config and
|
|
2021
|
+
* profile defaults (`inherited`), then the environment's layers, then groups and the eval (`own`).
|
|
2022
|
+
*/
|
|
2023
|
+
function plannedEgress(egress, harness, env, layers) {
|
|
2024
|
+
const fromLayers = layers.findLast((l) => l.spec.egress !== void 0)?.spec.egress;
|
|
2025
|
+
const policy = egress.own ?? fromLayers ?? egress.inherited ?? "proxy";
|
|
2026
|
+
if (!isRestricted(policy)) return {
|
|
2027
|
+
policy,
|
|
2028
|
+
auto: []
|
|
2029
|
+
};
|
|
2030
|
+
const own = policy === "deny-all" ? [] : policy.allow;
|
|
2031
|
+
const auto = [
|
|
2032
|
+
...harnessHosts(harness),
|
|
2033
|
+
...plannedSecrets(env, layers).flatMap((s) => s.hosts ?? []),
|
|
2034
|
+
...mcpServerHosts(layers)
|
|
2035
|
+
];
|
|
2036
|
+
return {
|
|
2037
|
+
policy,
|
|
2038
|
+
auto: [...new Set(auto)].filter((h) => !own.includes(h))
|
|
2039
|
+
};
|
|
2040
|
+
}
|
|
2041
|
+
/** Every host a restricted policy lets through, or null under `proxy` and `allow-all`. */
|
|
2042
|
+
function allowedHosts({ policy, auto }) {
|
|
2043
|
+
if (!isRestricted(policy)) return null;
|
|
2044
|
+
return [.../* @__PURE__ */ new Set([...policy === "deny-all" ? [] : policy.allow, ...auto])];
|
|
2045
|
+
}
|
|
2046
|
+
/**
|
|
2047
|
+
* The policy the Docker sandbox's gateway enforces: `allow-all` under `proxy` and `allow-all`
|
|
2048
|
+
* (whether the container also keeps its network is {@link isDirect}), `deny-all` when a
|
|
2049
|
+
* restricted policy allows no host at all, else an allowlist. A listed IP address is an allowed
|
|
2050
|
+
* range too, since the gateway reaches a private address only through one.
|
|
2051
|
+
*/
|
|
2052
|
+
function gatewayPolicy(planned) {
|
|
2053
|
+
const hosts = allowedHosts(planned);
|
|
2054
|
+
if (hosts === null) return { mode: "allow-all" };
|
|
2055
|
+
if (!hosts.length) return { mode: "deny-all" };
|
|
2056
|
+
const cidrs = hosts.filter((h) => isIP(h)).map((h) => `${h}/${isIP(h) === 6 ? 128 : 32}`);
|
|
2057
|
+
return {
|
|
2058
|
+
mode: "custom",
|
|
2059
|
+
allowedHosts: hosts,
|
|
2060
|
+
...cidrs.length ? { allowedCIDRs: cidrs } : {}
|
|
2061
|
+
};
|
|
2062
|
+
}
|
|
2063
|
+
/** What enforcing the policy takes: nothing, turning egress off, or a host allowlist. */
|
|
2064
|
+
function egressNeeds(planned) {
|
|
2065
|
+
if (!isRestricted(planned.policy)) return "none";
|
|
2066
|
+
return allowedHosts(planned)?.length ? "allowlist" : "all-or-nothing";
|
|
2067
|
+
}
|
|
2068
|
+
const LEVELS = [
|
|
2069
|
+
"none",
|
|
2070
|
+
"all-or-nothing",
|
|
2071
|
+
"allowlist"
|
|
2072
|
+
];
|
|
2073
|
+
/** Why the sandbox can't enforce the attempt's policy, or undefined when it can. */
|
|
2074
|
+
function unenforceable(planned, sandbox, where) {
|
|
2075
|
+
const needs = egressNeeds(planned);
|
|
2076
|
+
const can = SANDBOX_EGRESS[sandbox.kind].enforce;
|
|
2077
|
+
if (LEVELS.indexOf(can) >= LEVELS.indexOf(needs)) return void 0;
|
|
2078
|
+
const hosts = allowedHosts(planned)?.join(", ");
|
|
2079
|
+
const what = needs !== "allowlist" ? "turning egress off" : planned.policy === "deny-all" ? `a host allowlist for its automatic hosts (${hosts})` : `a host allowlist (${hosts})`;
|
|
2080
|
+
const enforces = can === "none" ? "enforces no egress policy" : `only enforces ${can === "all-or-nothing" ? "'allow-all' or 'deny-all'" : can}`;
|
|
2081
|
+
const hint = sandbox.kind === "fake" ? " (its commands run on the host). Run it in docker(), or set egress: 'allow-all'" : "";
|
|
2082
|
+
return `${where} runs under egress ${describeEgress(planned.policy)}, which needs ${what}, but the ${sandbox.kind} sandbox ${enforces}${hint}.`;
|
|
2083
|
+
}
|
|
2084
|
+
const describeEgress = (policy) => typeof policy === "string" ? `'${policy}'` : `{ allow: [${policy.allow.map((h) => `'${h}'`).join(", ")}] }`;
|
|
2085
|
+
/** A CI runner as the environment names it, which usually runs on a cloud VM; null elsewhere. */
|
|
2086
|
+
function ciRunner(env) {
|
|
2087
|
+
if (env.GITHUB_ACTIONS === "true") return "a GitHub Actions runner (GITHUB_ACTIONS)";
|
|
2088
|
+
if (env.GITLAB_CI) return "a GitLab CI runner (GITLAB_CI)";
|
|
2089
|
+
if (env.BUILDKITE) return "a Buildkite agent (BUILDKITE)";
|
|
2090
|
+
if (env.CIRCLECI) return "a CircleCI runner (CIRCLECI)";
|
|
2091
|
+
return isCI(env) ? "a CI runner (CI)" : null;
|
|
2092
|
+
}
|
|
2093
|
+
/**
|
|
2094
|
+
* What `allow-all` exposes on a cloud VM or CI runner (`where`): the container keeps its network,
|
|
2095
|
+
* so anything in it reaches cloud metadata and the host's network around the gateway.
|
|
2096
|
+
*/
|
|
2097
|
+
const directOnCloud = (where) => `egress 'allow-all' on ${where}: anything in the sandbox can reach the cloud metadata service (169.254.169.254, which hands out the machine's credentials), the host's network and its services around the egress gateway, unfiltered and unrecorded`;
|
|
2098
|
+
/** `deny-all + auto: api2.cursor.sh, …` or `allow-all` for the plan; '' under `proxy`, the default. */
|
|
2099
|
+
function egressLabel({ policy, auto }) {
|
|
2100
|
+
if (policy === "proxy") return "";
|
|
2101
|
+
if (policy === "allow-all") return "allow-all";
|
|
2102
|
+
const own = policy === "deny-all" ? "deny-all" : policy.allow.length ? `allow ${policy.allow.join(", ")}` : "allow none";
|
|
2103
|
+
return auto.length ? `${own} + auto: ${auto.join(", ")}` : own;
|
|
2104
|
+
}
|
|
2105
|
+
//#endregion
|
|
2106
|
+
//#region src/judge/state-options.ts
|
|
2107
|
+
const DEFAULT_STATE = {
|
|
2108
|
+
task: true,
|
|
2109
|
+
output: true,
|
|
2110
|
+
files: true,
|
|
2111
|
+
diff: true,
|
|
2112
|
+
contents: [],
|
|
2113
|
+
transcript: false,
|
|
2114
|
+
turns: "last",
|
|
2115
|
+
checks: false
|
|
2116
|
+
};
|
|
2117
|
+
/** Every key set, in `STATE_KEYS` order, so equal selections serialise the same. */
|
|
2118
|
+
function resolveStateOptions(state) {
|
|
2119
|
+
const out = {};
|
|
2120
|
+
for (const key of STATE_KEYS$1) out[key] = state?.[key] ?? DEFAULT_STATE[key];
|
|
2121
|
+
return out;
|
|
2122
|
+
}
|
|
2123
|
+
/** `inner` over `outer`, field by field, and `state` key by key. */
|
|
2124
|
+
function mergeJudge(outer, inner) {
|
|
2125
|
+
if (!inner) return outer;
|
|
2126
|
+
const out = {
|
|
2127
|
+
...outer,
|
|
2128
|
+
...inner
|
|
2129
|
+
};
|
|
2130
|
+
if (outer.state || inner.state) out.state = {
|
|
2131
|
+
...outer.state,
|
|
2132
|
+
...inner.state
|
|
2133
|
+
};
|
|
2134
|
+
return out;
|
|
2135
|
+
}
|
|
2136
|
+
const KNOWN = [...[
|
|
2137
|
+
"harness",
|
|
2138
|
+
"sandbox",
|
|
2139
|
+
"environment",
|
|
2140
|
+
"compare",
|
|
2141
|
+
"runs",
|
|
2142
|
+
"gate",
|
|
2143
|
+
"env",
|
|
2144
|
+
"egress",
|
|
2145
|
+
"judge",
|
|
2146
|
+
"capture",
|
|
2147
|
+
"question",
|
|
2148
|
+
"questions",
|
|
2149
|
+
"timeout",
|
|
2150
|
+
"tags",
|
|
2151
|
+
"pin"
|
|
2152
|
+
]];
|
|
2153
|
+
const SUGGEST = {
|
|
2154
|
+
harnesses: "harness",
|
|
2155
|
+
sandboxes: "sandbox",
|
|
2156
|
+
environments: "environment",
|
|
2157
|
+
baseline: "environment",
|
|
2158
|
+
variants: "compare",
|
|
2159
|
+
repeat: "runs",
|
|
2160
|
+
repeats: "runs",
|
|
2161
|
+
iterations: "runs",
|
|
2162
|
+
trials: "runs",
|
|
2163
|
+
secrets: "env",
|
|
2164
|
+
vars: "env",
|
|
2165
|
+
network: "egress",
|
|
2166
|
+
networkPolicy: "egress",
|
|
2167
|
+
tag: "tags",
|
|
2168
|
+
pins: "pin",
|
|
2169
|
+
passRate: "gate",
|
|
2170
|
+
threshold: "gate"
|
|
2171
|
+
};
|
|
2172
|
+
/** Throws unless `value` is `'all'` or `{ passRate }` with a rate in (0, 1]. */
|
|
2173
|
+
function checkGate(value, where) {
|
|
2174
|
+
const problem = gateProblem(value, where);
|
|
2175
|
+
if (problem) throw new UsageError(problem);
|
|
2176
|
+
}
|
|
2177
|
+
const PINNABLE = [
|
|
2178
|
+
"harness",
|
|
2179
|
+
"runs",
|
|
2180
|
+
"compare"
|
|
2181
|
+
];
|
|
2182
|
+
const ID = /^[A-Za-z0-9._:@+\-=[\]]+$/;
|
|
2183
|
+
const sanitize = (text) => (text.split("/").at(-1) ?? text).replace(/[^A-Za-z0-9._:@-]+/g, "-");
|
|
2184
|
+
function harnessId(h) {
|
|
2185
|
+
if (h.id) return h.id;
|
|
2186
|
+
const effort = harnessEffort(h);
|
|
2187
|
+
const suffix = effort ? `[effort=${sanitize(effort)}]` : "";
|
|
2188
|
+
if (h.kind === "fake") return `fake${suffix}`;
|
|
2189
|
+
return `${h.model ? `${h.kind}:${sanitize(h.model)}` : h.kind}${suffix}`;
|
|
2190
|
+
}
|
|
2191
|
+
function installId(install) {
|
|
2192
|
+
if (!install?.length) return "";
|
|
2193
|
+
const key = JSON.stringify(install.map((step) => typeof step === "string" ? step : {
|
|
2194
|
+
run: step.run,
|
|
2195
|
+
timeout: step.timeout
|
|
2196
|
+
}));
|
|
2197
|
+
return `+${createHash("sha256").update(key).digest("hex").slice(0, 8)}`;
|
|
2198
|
+
}
|
|
2199
|
+
/** The provider a sandbox runs on: its kind, or `podman` for a Docker sandbox on Podman. */
|
|
2200
|
+
const sandboxProvider = (s) => s.kind === "docker" && s.engine === "podman" ? "podman" : s.kind;
|
|
2201
|
+
function sandboxId(s) {
|
|
2202
|
+
if (s.id) return s.id;
|
|
2203
|
+
if (s.kind === "fake") return "fake";
|
|
2204
|
+
const install = s.kind === "docker" ? installId(s.install) : "";
|
|
2205
|
+
const engine = s.engine ?? "docker";
|
|
2206
|
+
if (s.dockerfile) {
|
|
2207
|
+
const [file = "", dir = ""] = s.dockerfile.split(/[\\/]/).reverse();
|
|
2208
|
+
return `${engine}:${sanitize(file === "Dockerfile" ? dir || file : file)}${install}`;
|
|
2209
|
+
}
|
|
2210
|
+
const image = s.image ?? s.from;
|
|
2211
|
+
return image ? `${engine}:${sanitize(image)}${install}` : `${engine}${install}`;
|
|
2212
|
+
}
|
|
2213
|
+
/** The planned attempts' sandboxes by id, once each, or `fallback`'s when none is planned. */
|
|
2214
|
+
function plannedSandboxes(planned, fallback) {
|
|
2215
|
+
const ids = new Set(planned.map(sandboxId));
|
|
2216
|
+
return ids.size ? [...ids].join(", ") : sandboxId(fallback);
|
|
2217
|
+
}
|
|
2218
|
+
const cellKey = (harness, environment, sandbox) => `${harness}/${environment}/${sandbox}`;
|
|
2219
|
+
const isHarness = (v) => isObjectLike(v) && (v.kind === "fake" || isAgentKind(v.kind));
|
|
2220
|
+
function checkMaxConcurrent(value, where) {
|
|
2221
|
+
if (value === void 0 || typeof value === "number" && Number.isInteger(value) && value >= 1) return;
|
|
2222
|
+
throw new UsageError(`${where} must be a whole number of at least 1 (got ${JSON.stringify(value)})`);
|
|
2223
|
+
}
|
|
2224
|
+
const describe = (value) => {
|
|
2225
|
+
if (Array.isArray(value)) return `a list of ${value.length}`;
|
|
2226
|
+
if (typeof value === "object" && value !== null) {
|
|
2227
|
+
const keys = Object.keys(value);
|
|
2228
|
+
return keys.length ? `an object with ${keys.map((k) => `"${k}"`).join(", ")}` : "an object";
|
|
2229
|
+
}
|
|
2230
|
+
return JSON.stringify(value) ?? String(value);
|
|
2231
|
+
};
|
|
2232
|
+
/** `compare: false` and `environment` beside `compare.baseline` are never legal options. */
|
|
2233
|
+
function refuseIllegalCompare(options, where) {
|
|
2234
|
+
if (options.compare === false) throw new UsageError(`compare: false can't be set in ${where}. A profile turns compare off with limit.compare: false, which runs the baseline and strips the variants. On a group or eval, omit compare or pass compare: { baseline, variants }.`);
|
|
2235
|
+
const baseline = options.compare !== null && typeof options.compare === "object" && "baseline" in options.compare ? options.compare.baseline : void 0;
|
|
2236
|
+
if (options.environment !== void 0 && baseline !== void 0) throw new UsageError(`environment and compare.baseline can't both be set in ${where}. compare: { baseline, variants } names every arm, so drop environment, or drop baseline to make environment the control.`);
|
|
2237
|
+
}
|
|
2238
|
+
/** Throws on keys that aren't options, and on values of the wrong shape, suggesting a fix. */
|
|
2239
|
+
function checkOptions(options, where, allowed = KNOWN) {
|
|
2240
|
+
for (const key of Object.keys(options)) {
|
|
2241
|
+
if (allowed.includes(key)) continue;
|
|
2242
|
+
const near = SUGGEST[key] ?? closest(key, allowed);
|
|
2243
|
+
const hint = key === "pool" ? " pool is an option of evaluate.each, beside the cases: evaluate.each(cases, { pool: false })." : near ? ` Did you mean "${near}"?` : "";
|
|
2244
|
+
throw new UsageError(`Unknown option "${key}" in ${where}.${hint}`);
|
|
2245
|
+
}
|
|
2246
|
+
const o = options;
|
|
2247
|
+
if (o.runs !== void 0 && !(Number.isInteger(o.runs) && o.runs >= 1)) throw new UsageError(`runs in ${where} must be a whole number of at least 1 (got ${JSON.stringify(o.runs)})`);
|
|
2248
|
+
if (o.gate !== void 0) checkGate(o.gate, where);
|
|
2249
|
+
if (o.tags !== void 0 && (!Array.isArray(o.tags) || o.tags.some((t) => typeof t !== "string"))) throw new UsageError(`tags in ${where} must be a list of strings, e.g. tags: ['smoke'] (got ${JSON.stringify(o.tags)})`);
|
|
2250
|
+
if (o.env !== void 0) checkEnv(o.env, where, UsageError);
|
|
2251
|
+
if (o.egress !== void 0) checkEgress(o.egress, where, UsageError);
|
|
2252
|
+
const judgeIssue = o.judge !== void 0 && judgeProblem(o.judge, {
|
|
2253
|
+
at: "options",
|
|
2254
|
+
where
|
|
2255
|
+
});
|
|
2256
|
+
if (judgeIssue) throw new UsageError(judgeIssue);
|
|
2257
|
+
if (o.sandbox !== void 0) {
|
|
2258
|
+
const kind = o.sandbox.kind;
|
|
2259
|
+
if (kind !== "docker" && kind !== "fake") throw new UsageError(`sandbox in ${where} must come from docker(), podman() or fakeSandbox() (got ${describe(o.sandbox)})`);
|
|
2260
|
+
const max = o.sandbox.maxConcurrent;
|
|
2261
|
+
if (max !== void 0 && !(Number.isInteger(max) && max >= 1)) throw new UsageError(`sandbox.maxConcurrent in ${where} must be a whole number of at least 1 (got ${String(max)})`);
|
|
2262
|
+
const problem = sandboxProblem(o.sandbox);
|
|
2263
|
+
if (problem) throw new UsageError(`sandbox in ${where}: ${problem}`);
|
|
2264
|
+
}
|
|
2265
|
+
if (o.environment !== void 0) {
|
|
2266
|
+
if (Array.isArray(o.environment)) throw new UsageError(`environment takes one environment, not a list (${where}). To compare environments, use compare: { baseline, variants: [a, b] }. baseline is the control; variants are not layered on it.`);
|
|
2267
|
+
if (!isEnvironment(o.environment)) throw new UsageError(`environment in ${where} must come from defineEnvironment(…) (got ${describe(o.environment)})`);
|
|
2268
|
+
const problem = environmentDataProblem(o.environment, "environment", where);
|
|
2269
|
+
if (problem) throw new UsageError(problem);
|
|
2270
|
+
}
|
|
2271
|
+
refuseIllegalCompare(options, where);
|
|
2272
|
+
if (o.compare !== void 0) {
|
|
2273
|
+
const problem = compareProblem(o.compare, where, z.custom(isEnvironment));
|
|
2274
|
+
if (problem) throw new UsageError(problem);
|
|
2275
|
+
const { baseline, variants } = o.compare;
|
|
2276
|
+
for (const [label, env] of [["compare.baseline", baseline], ...variants.map((v, i) => [`compare.variants[${i}]`, v])]) {
|
|
2277
|
+
const bad = env && environmentDataProblem(env, label, where);
|
|
2278
|
+
if (bad) throw new UsageError(bad);
|
|
2279
|
+
}
|
|
2280
|
+
}
|
|
2281
|
+
if (o.pin !== void 0) {
|
|
2282
|
+
const bad = Array.isArray(o.pin) ? o.pin.filter((k) => !PINNABLE.includes(k)) : [o.pin];
|
|
2283
|
+
if (bad.length) throw new UsageError(`pin in ${where} lists harness, runs and/or compare (got ${bad.map((k) => JSON.stringify(k)).join(", ")})`);
|
|
2284
|
+
}
|
|
2285
|
+
if (o.capture !== void 0) checkCapture(o.capture, where);
|
|
2286
|
+
const questionIssue = o.question !== void 0 && questionOptionsProblem(o.question, where);
|
|
2287
|
+
if (questionIssue) throw new UsageError(questionIssue);
|
|
2288
|
+
const questionsIssue = o.questions !== void 0 && questionsProblem(o.questions, where);
|
|
2289
|
+
if (questionsIssue) throw new UsageError(questionsIssue);
|
|
2290
|
+
}
|
|
2291
|
+
const CAPTURE_KEYS = [
|
|
2292
|
+
"viewport",
|
|
2293
|
+
"timeout",
|
|
2294
|
+
"browser",
|
|
2295
|
+
"model"
|
|
2296
|
+
];
|
|
2297
|
+
const CAPTURE_SHAPE = "{ viewport?, timeout?, browser?, model? }";
|
|
2298
|
+
const CAPTURE_CALL_ONLY = /* @__PURE__ */ new Set([
|
|
2299
|
+
"instructions",
|
|
2300
|
+
"wait",
|
|
2301
|
+
"name",
|
|
2302
|
+
"path"
|
|
2303
|
+
]);
|
|
2304
|
+
/** Throws unless `capture` is a shallow-merge object with a viewport, timeout, browser and/or model. */
|
|
2305
|
+
function checkCapture(value, where) {
|
|
2306
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new UsageError(`capture in ${where} is ${CAPTURE_SHAPE} (got ${JSON.stringify(value)})`);
|
|
2307
|
+
const o = value;
|
|
2308
|
+
if (o.instructions !== void 0) throw new UsageError(`capture.instructions in ${where} can't be set: instructions go on the call, c.capture(port, { instructions }).`);
|
|
2309
|
+
const unknown = Object.keys(o).filter((k) => !CAPTURE_KEYS.includes(k));
|
|
2310
|
+
if (unknown.length) {
|
|
2311
|
+
const key = unknown[0] ?? "";
|
|
2312
|
+
if (key === "use" || key === "fake") throw new UsageError(`Unknown capture key "${key}" in ${where}. Set capture.model; tests pass fakeInstructions from reeval/testing as instructions.`);
|
|
2313
|
+
if (CAPTURE_CALL_ONLY.has(key)) throw new UsageError(`capture.${key} in ${where} can't be set: ${key} goes on the call, c.capture(port, { ${key} }). Use viewport, timeout, browser, model.`);
|
|
2314
|
+
throw new UsageError(`Unknown capture key "${key}" in ${where}. Use viewport, timeout, browser, model.`);
|
|
2315
|
+
}
|
|
2316
|
+
if (o.viewport !== void 0) checkViewport(o.viewport, where);
|
|
2317
|
+
if (o.timeout !== void 0) parseDuration(o.timeout);
|
|
2318
|
+
if (o.browser !== void 0 && typeof o.browser !== "boolean") throw new UsageError(`capture.browser in ${where} is true or false (got ${JSON.stringify(o.browser)})`);
|
|
2319
|
+
if (o.model !== void 0) checkCaptureModel(o.model, where);
|
|
2320
|
+
}
|
|
2321
|
+
function checkViewport(value, where) {
|
|
2322
|
+
if (value === "full-page") return;
|
|
2323
|
+
if (!isObjectLike(value) || Array.isArray(value)) throw new UsageError(`capture.viewport in ${where} is { width, height } or 'full-page' (got ${JSON.stringify(value)})`);
|
|
2324
|
+
const keys = Object.keys(value);
|
|
2325
|
+
if (keys.includes("fullPage")) throw new UsageError(`capture.viewport in ${where} is { width, height } or 'full-page', not { height, fullPage }. Use viewport: 'full-page' for a tall shot.`);
|
|
2326
|
+
const isSize = (n) => typeof n === "number" && Number.isInteger(n) && n > 0;
|
|
2327
|
+
if (keys.some((k) => k !== "width" && k !== "height") || !isSize(value.width) || !isSize(value.height)) throw new UsageError(`capture.viewport in ${where} is { width, height } or 'full-page' (got ${JSON.stringify(value)})`);
|
|
2328
|
+
}
|
|
2329
|
+
function checkCaptureModel(value, where) {
|
|
2330
|
+
if (typeof value !== "string" || !value.includes("/")) throw new UsageError(`capture.model in ${where} is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${JSON.stringify(value)})`);
|
|
2331
|
+
if (isEvaluationModel(value)) throw new UsageError(`capture.model "${value}" in ${where} is an evaluation model: it scores yes/no questions and can't drive a capture. Use a language model that accepts images, e.g. 'google/gemini-3.8-flash'.`);
|
|
2332
|
+
}
|
|
2333
|
+
function applyHarness(inherited, value, where) {
|
|
2334
|
+
if (value === void 0) return inherited;
|
|
2335
|
+
const next = typeof value === "function" ? value(inherited) : isHarnessEdit(value) ? editHarness(inherited, value, where) : value;
|
|
2336
|
+
const list = Array.isArray(next) ? next : [next];
|
|
2337
|
+
if (list.length === 0) throw new UsageError(`harness is an empty list in ${where}. Give at least one, e.g. harness: ${HARNESS_EXAMPLE}`);
|
|
2338
|
+
checkHarnesses(list, where);
|
|
2339
|
+
return list;
|
|
2340
|
+
}
|
|
2341
|
+
const isHarnessEdit = (v) => typeof v === "object" && v !== null && !Array.isArray(v) && !("kind" in v) && ("add" in v || "only" in v);
|
|
2342
|
+
/** `{ add }` appends to the inherited harnesses; `{ only }` keeps those with these ids. */
|
|
2343
|
+
function editHarness(inherited, edit, where) {
|
|
2344
|
+
if ("add" in edit && "only" in edit) throw new UsageError(`harness in ${where} takes { add } or { only }, not both. Use add to append to the inherited harnesses, only to keep some of them.`);
|
|
2345
|
+
if ("add" in edit) return [...inherited, ...Array.isArray(edit.add) ? edit.add : [edit.add]];
|
|
2346
|
+
const ids = Array.isArray(edit.only) ? edit.only : [edit.only];
|
|
2347
|
+
const have = inherited.map(harnessId);
|
|
2348
|
+
const missing = ids.filter((id) => typeof id !== "string" || !have.includes(id));
|
|
2349
|
+
if (missing.length) throw new UsageError(`harness { only: ${missing.map((id) => JSON.stringify(id)).join(", ")} } in ${where} names no inherited harness (inherited: ${have.join(", ")})`);
|
|
2350
|
+
return inherited.filter((h) => ids.includes(harnessId(h)));
|
|
2351
|
+
}
|
|
2352
|
+
/** Each harness is one, with a valid `maxConcurrent`. */
|
|
2353
|
+
function checkHarnesses(list, where) {
|
|
2354
|
+
for (const item of list) {
|
|
2355
|
+
if (!isHarness(item)) throw new UsageError(`harness in ${where} must be like ${HARNESS_EXAMPLE} (got ${describe(item)})`);
|
|
2356
|
+
if (item.kind === "opencode") {
|
|
2357
|
+
const why = opencodeModelError(item.model);
|
|
2358
|
+
if (why) throw new UsageError(`opencode() in ${where}: ${why}`);
|
|
2359
|
+
}
|
|
2360
|
+
checkMaxConcurrent(item.maxConcurrent, `harness.maxConcurrent in ${where}`);
|
|
2361
|
+
const { startTimeout } = item;
|
|
2362
|
+
if (startTimeout !== void 0) try {
|
|
2363
|
+
parseDuration(startTimeout);
|
|
2364
|
+
} catch (error) {
|
|
2365
|
+
throw new UsageError(`harness.startTimeout in ${where}: ${error.message}`);
|
|
2366
|
+
}
|
|
2367
|
+
const problem = harnessProblem(item);
|
|
2368
|
+
if (problem) throw new UsageError(`harness in ${where}: ${problem}`);
|
|
2369
|
+
}
|
|
2370
|
+
}
|
|
2371
|
+
function checkUnique(list, where) {
|
|
2372
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2373
|
+
for (const item of list) {
|
|
2374
|
+
const key = harnessId(item);
|
|
2375
|
+
if (!ID.test(key)) throw new UsageError(`The harness id "${key}" in ${where} may only use letters, digits and . _ : @ + - [ ] =`);
|
|
2376
|
+
if (seen.has(key)) throw new UsageError(`Two harnesses in ${where} resolve to the id "${key}". Give one an id, e.g. cursor({ model: '…', id: '…' })`);
|
|
2377
|
+
seen.add(key);
|
|
2378
|
+
}
|
|
2379
|
+
}
|
|
2380
|
+
const plannedQuestions = (value) => value.map((q, i) => typeof q === "string" ? {
|
|
2381
|
+
n: i + 1,
|
|
2382
|
+
question: q
|
|
2383
|
+
} : {
|
|
2384
|
+
n: i + 1,
|
|
2385
|
+
question: q.question,
|
|
2386
|
+
...q.model !== void 0 ? { model: q.model } : {},
|
|
2387
|
+
...q.across !== void 0 ? { across: q.across } : {},
|
|
2388
|
+
...q.choices ? { choices: q.choices } : {},
|
|
2389
|
+
...q.state ? { state: q.state } : {},
|
|
2390
|
+
...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
|
|
2391
|
+
});
|
|
2392
|
+
/** Whether the eval's matrix varies each factor a pairwise question can vary. */
|
|
2393
|
+
const variesBy = (r) => ({
|
|
2394
|
+
environment: (r.compare?.variants.length ?? 0) > 0,
|
|
2395
|
+
harness: r.harness.length > 1
|
|
2396
|
+
});
|
|
2397
|
+
const FACTOR_FIX = {
|
|
2398
|
+
environment: (_, where) => `varies the environment (across: 'environment'), but ${where} runs one environment, so it has no pairs to ask. Add compare: { variants: [...] }`,
|
|
2399
|
+
harness: (r, where) => `varies the harness (across: 'harness'), but ${where} runs one harness (${r.harness.map(harnessId).join(", ")}), so it has no pairs to ask. Run it on two or more, harness: [a, b]`
|
|
2400
|
+
};
|
|
2401
|
+
function normalizeCompare(value) {
|
|
2402
|
+
if (value === void 0) return void 0;
|
|
2403
|
+
return {
|
|
2404
|
+
...value.baseline ? { baseline: value.baseline } : {
|
|
2405
|
+
baseline: emptyEnvironment(EMPTY_BASELINE),
|
|
2406
|
+
from: "empty"
|
|
2407
|
+
},
|
|
2408
|
+
variants: value.variants,
|
|
2409
|
+
gate: value.gate ?? "variants",
|
|
2410
|
+
...value.minLift !== void 0 ? { minLift: value.minLift } : {},
|
|
2411
|
+
...value.harness ? { harness: value.harness.map((h) => typeof h === "string" ? h : harnessId(h)) } : {},
|
|
2412
|
+
of: value.variants.length
|
|
2413
|
+
};
|
|
2414
|
+
}
|
|
2415
|
+
function applyLevel(out, options, where, egress = "own") {
|
|
2416
|
+
for (const [key, value] of Object.entries(options)) if (value !== void 0) out.sources = {
|
|
2417
|
+
...out.sources,
|
|
2418
|
+
[key]: where
|
|
2419
|
+
};
|
|
2420
|
+
if (options.egress !== void 0) out.egress[egress] = options.egress;
|
|
2421
|
+
out.harness = applyHarness(out.harness, options.harness, where);
|
|
2422
|
+
if (options.sandbox !== void 0) out.sandbox = options.sandbox;
|
|
2423
|
+
if (options.environment !== void 0) out.environment = options.environment;
|
|
2424
|
+
if (options.compare !== void 0) out.compare = normalizeCompare(options.compare);
|
|
2425
|
+
if (options.runs !== void 0) out.runs = options.runs;
|
|
2426
|
+
if (options.gate !== void 0) out.gate = options.gate;
|
|
2427
|
+
if (options.timeout !== void 0) out.timeout = options.timeout;
|
|
2428
|
+
Object.assign(out.env, options.env);
|
|
2429
|
+
out.judge = mergeJudge(out.judge, options.judge);
|
|
2430
|
+
if (options.capture !== void 0) {
|
|
2431
|
+
const { viewport, ...rest } = options.capture;
|
|
2432
|
+
out.capture = {
|
|
2433
|
+
...out.capture,
|
|
2434
|
+
...rest
|
|
2435
|
+
};
|
|
2436
|
+
if (viewport !== void 0) out.capture.viewport = viewport;
|
|
2437
|
+
}
|
|
2438
|
+
out.question = mergeQuestion(out.question, options.question);
|
|
2439
|
+
if (options.questions !== void 0) out.questions = plannedQuestions(options.questions);
|
|
2440
|
+
for (const tag of options.tags ?? []) if (!out.tags.includes(tag)) out.tags.push(tag);
|
|
2441
|
+
for (const key of options.pin ?? []) if (!out.pin.includes(key)) out.pin.push(key);
|
|
2442
|
+
}
|
|
2443
|
+
/** What a profile sets over the config's judge: its `judge`, then its `defaults.judge`. */
|
|
2444
|
+
const profileJudge = (profile) => mergeJudge(profile.judge, profile.defaults.judge);
|
|
2445
|
+
/**
|
|
2446
|
+
* Config, then the profile's defaults, then each enclosing group, then the eval; then the
|
|
2447
|
+
* profile's limit (narrowing only, except pinned keys); then `--runs`.
|
|
2448
|
+
*/
|
|
2449
|
+
function cascade(base, levels, where, options = {}) {
|
|
2450
|
+
const out = {
|
|
2451
|
+
...base,
|
|
2452
|
+
env: { ...base.env },
|
|
2453
|
+
egress: { ...base.egress },
|
|
2454
|
+
judge: { ...base.judge },
|
|
2455
|
+
capture: { ...base.capture },
|
|
2456
|
+
question: { ...base.question },
|
|
2457
|
+
questions: [...base.questions],
|
|
2458
|
+
tags: [...base.tags],
|
|
2459
|
+
pin: [...base.pin]
|
|
2460
|
+
};
|
|
2461
|
+
const { profile } = options;
|
|
2462
|
+
const notes = [];
|
|
2463
|
+
if (profile) applyLevel(out, {
|
|
2464
|
+
...profile.defaults,
|
|
2465
|
+
judge: profileJudge(profile)
|
|
2466
|
+
}, `profiles.${profile.name}.defaults`, "inherited");
|
|
2467
|
+
for (const { options: own, where: at } of levels) applyLevel(out, own, at);
|
|
2468
|
+
if (out.compare?.from && out.environment) out.compare = {
|
|
2469
|
+
...out.compare,
|
|
2470
|
+
baseline: out.environment,
|
|
2471
|
+
from: "environment"
|
|
2472
|
+
};
|
|
2473
|
+
if (out.questions.length) {
|
|
2474
|
+
out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
|
|
2475
|
+
const blank = out.questions.find(showsNothing);
|
|
2476
|
+
if (blank) throw new UsageError(`questions "${blank.question}" in ${where} shows the model nothing: every state key is off and it sends no artifacts. Turn on a state key, such as state: { diff: true }, or send artifacts.`);
|
|
2477
|
+
}
|
|
2478
|
+
checkUnique(out.harness, where);
|
|
2479
|
+
const harnessFromProfile = profile?.defaults.harness !== void 0 && !levels.some((l) => l.options.harness);
|
|
2480
|
+
const authored = variesBy(out);
|
|
2481
|
+
if (harnessFromProfile) authored.harness ||= base.harness.length > 1;
|
|
2482
|
+
for (const q of options.skipped ? [] : out.questions) for (const factor of pairFactors(q.across)) {
|
|
2483
|
+
if (authored[factor]) continue;
|
|
2484
|
+
throw new UsageError(`questions "${q.question}" in ${where} ${FACTOR_FIX[factor](out, where)}, or leave out across to ask each attempt on its own.`);
|
|
2485
|
+
}
|
|
2486
|
+
if (profile?.defaults.harness !== void 0 && levels.some((l) => l.options.harness)) notes.push("harness set by the eval");
|
|
2487
|
+
const ids = out.harness.map(harnessId);
|
|
2488
|
+
const missing = (out.compare?.harness)?.filter((id) => !ids.includes(id)) ?? [];
|
|
2489
|
+
if (missing.length) throw new UsageError(`compare.harness lists ${missing.join(", ")}, which ${where} doesn't run (it runs ${ids.join(", ")}).`);
|
|
2490
|
+
for (const v of out.compare?.variants ?? []) {
|
|
2491
|
+
if (v.name === out.compare?.baseline.name) throw new UsageError(out.compare.from === "empty" ? `compare in ${where} lists a variant named ${v.name}, the name of the empty control a compare without a baseline or environment runs. Rename the variant, or set compare.baseline.` : out.compare.from === "environment" ? `compare in ${where} lists ${v.name}, which is the control: compare has no baseline, so the eval's environment is. List only the other environments.` : `compare in ${where} lists ${v.name}, which is the baseline. List only the other environments.`);
|
|
2492
|
+
const control = out.compare?.baseline;
|
|
2493
|
+
const shared = control && sameWorkspaceOnly(control, v);
|
|
2494
|
+
if (control && shared) {
|
|
2495
|
+
const at = shared.ref ? `at ref "${shared.ref}"` : "with no ref (the repository HEAD)";
|
|
2496
|
+
throw new UsageError(`compare in ${where} runs ${v.name} and ${control.name} on the same tree: past what they share, each sets only workspace, and both are git("${shared.repo}") ${at}${shared.path ? `, path "${shared.path}"` : ""}. A ref read from an unset variable is undefined, so check the variables the arms take their refs from, and give each arm its own ref.`);
|
|
2497
|
+
}
|
|
2498
|
+
}
|
|
2499
|
+
const names = out.compare?.variants.map((v) => v.name) ?? [];
|
|
2500
|
+
const dupe = names.find((n, i) => names.indexOf(n) !== i);
|
|
2501
|
+
if (dupe) throw new UsageError(`compare in ${where} lists two variants named ${dupe}. Give each variant its own name.`);
|
|
2502
|
+
let skip;
|
|
2503
|
+
const limit = profile?.limit;
|
|
2504
|
+
if (profile && limit) {
|
|
2505
|
+
const tag = `profile ${profile.name}`;
|
|
2506
|
+
const pinned = (key) => out.pin.includes(key);
|
|
2507
|
+
if (limit.runs !== void 0 && options.runs === void 0 && !pinned("runs") && out.runs > limit.runs) {
|
|
2508
|
+
notes.push(`runs ${out.runs} → ${limit.runs} (${tag})`);
|
|
2509
|
+
out.runs = limit.runs;
|
|
2510
|
+
}
|
|
2511
|
+
if (limit.harness && !pinned("harness")) {
|
|
2512
|
+
const kept = out.harness.filter((h) => limit.harness?.includes(harnessId(h)));
|
|
2513
|
+
if (kept.length === 0) skip = `no harness left: ${tag} allows ${limit.harness.join(", ") || "none"}`;
|
|
2514
|
+
else if (kept.length < out.harness.length) {
|
|
2515
|
+
notes.push(`harnesses ${out.harness.length} → ${kept.length} (${tag})`);
|
|
2516
|
+
out.harness = kept;
|
|
2517
|
+
}
|
|
2518
|
+
}
|
|
2519
|
+
if (out.compare && limit.compare !== void 0 && !pinned("compare")) {
|
|
2520
|
+
if (limit.compare === false) {
|
|
2521
|
+
notes.push(`compare off (${tag})`);
|
|
2522
|
+
out.compare = {
|
|
2523
|
+
...out.compare,
|
|
2524
|
+
variants: []
|
|
2525
|
+
};
|
|
2526
|
+
} else if (out.compare.variants.length > limit.compare.maxVariants) {
|
|
2527
|
+
notes.push(`variants ${out.compare.variants.length} → ${limit.compare.maxVariants} (${tag})`);
|
|
2528
|
+
out.compare = {
|
|
2529
|
+
...out.compare,
|
|
2530
|
+
variants: out.compare.variants.slice(0, limit.compare.maxVariants)
|
|
2531
|
+
};
|
|
2532
|
+
}
|
|
2533
|
+
}
|
|
2534
|
+
const pins = out.pin.filter((k) => k === "compare" ? limit.compare !== void 0 && out.compare : limit[k] !== void 0);
|
|
2535
|
+
if (pins.length) notes.push(`pinned: ${pins.join(", ")}`);
|
|
2536
|
+
}
|
|
2537
|
+
if (profile) {
|
|
2538
|
+
const varies = variesBy(out);
|
|
2539
|
+
let narrowed = 0;
|
|
2540
|
+
const kept = out.questions.flatMap((q) => {
|
|
2541
|
+
if (q.across === void 0) return [q];
|
|
2542
|
+
const factors = pairFactors(q.across).filter((f) => varies[f]);
|
|
2543
|
+
if (!factors.length) return [];
|
|
2544
|
+
if (factors.length === pairFactors(q.across).length) return [q];
|
|
2545
|
+
narrowed++;
|
|
2546
|
+
return [{
|
|
2547
|
+
...q,
|
|
2548
|
+
across: factors
|
|
2549
|
+
}];
|
|
2550
|
+
});
|
|
2551
|
+
const plural = (n) => `${n} pairwise ${n === 1 ? "question" : "questions"}`;
|
|
2552
|
+
const off = out.questions.length - kept.length;
|
|
2553
|
+
if (off) notes.push(`${plural(off)} off (profile ${profile.name})`);
|
|
2554
|
+
if (narrowed) notes.push(`${plural(narrowed)} narrowed (profile ${profile.name})`);
|
|
2555
|
+
out.questions = kept;
|
|
2556
|
+
}
|
|
2557
|
+
if (options.runs !== void 0) out.runs = options.runs;
|
|
2558
|
+
return {
|
|
2559
|
+
resolved: out,
|
|
2560
|
+
notes,
|
|
2561
|
+
...skip ? { skip } : {}
|
|
2562
|
+
};
|
|
2563
|
+
}
|
|
2564
|
+
/**
|
|
2565
|
+
* The sandbox an attempt runs in: the resolved environment's (including an `.extend()`
|
|
2566
|
+
* parent's), else the first variant's when this is an omitted-baseline empty control, else
|
|
2567
|
+
* the cascaded option (`defineConfig` / profile defaults / group / eval), else implicit Docker.
|
|
2568
|
+
* A named environment is a total override: the config environment's sandbox is not inherited.
|
|
2569
|
+
*/
|
|
2570
|
+
const attemptSandbox = (resolved, env) => {
|
|
2571
|
+
const own = env.layers.findLast((l) => l.spec.sandbox)?.spec.sandbox;
|
|
2572
|
+
if (own) return own;
|
|
2573
|
+
const variant = resolved.compare?.from === "empty" && env.name === "baseline" ? resolved.compare.variants[0] : void 0;
|
|
2574
|
+
return sandboxOf(variant?.layers ?? [], resolved.sandbox);
|
|
2575
|
+
};
|
|
2576
|
+
/**
|
|
2577
|
+
* Which level set this attempt's sandbox: the cascade source of the environment that named one
|
|
2578
|
+
* (an `.extend()` child still names that environment), else the first variant when this is an
|
|
2579
|
+
* omitted-baseline empty control, else the cascaded `sandbox` option, else the config (implicit
|
|
2580
|
+
* Docker).
|
|
2581
|
+
*/
|
|
2582
|
+
function sandboxSetBy(planned, resolved, configFile) {
|
|
2583
|
+
if (!planned.environment.layers.some((l) => l.spec.sandbox)) {
|
|
2584
|
+
if (resolved.compare?.from === "empty" && planned.cell.role === "control" && resolved.compare.variants[0]?.layers.some((l) => l.spec.sandbox)) return resolved.sources?.compare ?? configFile;
|
|
2585
|
+
return resolved.sources?.sandbox ?? configFile;
|
|
2586
|
+
}
|
|
2587
|
+
if (resolved.compare) {
|
|
2588
|
+
if (planned.cell.role === "control") {
|
|
2589
|
+
if (resolved.compare.from === "environment") return resolved.sources?.environment ?? configFile;
|
|
2590
|
+
if (resolved.compare.from === "empty") return configFile;
|
|
2591
|
+
return resolved.sources?.compare ?? configFile;
|
|
2592
|
+
}
|
|
2593
|
+
return resolved.sources?.compare ?? configFile;
|
|
2594
|
+
}
|
|
2595
|
+
return resolved.sources?.environment ?? configFile;
|
|
2596
|
+
}
|
|
2597
|
+
/** Per harness: the control, then each variant it compares on; then run numbers. */
|
|
2598
|
+
function expand(resolved) {
|
|
2599
|
+
const { compare } = resolved;
|
|
2600
|
+
const control = applied(compare ? compare.baseline : resolved.environment);
|
|
2601
|
+
const policy = resolved.gate ?? "all";
|
|
2602
|
+
const gateIf = (gated) => gated ? policy : "none";
|
|
2603
|
+
const attempts = [];
|
|
2604
|
+
for (const harness of resolved.harness) {
|
|
2605
|
+
const hid = harnessId(harness);
|
|
2606
|
+
const cells = [(() => {
|
|
2607
|
+
const sandbox = attemptSandbox(resolved, control);
|
|
2608
|
+
const sid = sandboxId(sandbox);
|
|
2609
|
+
return {
|
|
2610
|
+
env: control,
|
|
2611
|
+
sandbox,
|
|
2612
|
+
cell: {
|
|
2613
|
+
key: cellKey(hid, control.name, sid),
|
|
2614
|
+
harness: hid,
|
|
2615
|
+
environment: control.name,
|
|
2616
|
+
sandbox: sid,
|
|
2617
|
+
role: "control"
|
|
2618
|
+
},
|
|
2619
|
+
gate: gateIf(!compare || compare.variants.length === 0 || compare.gate === "all")
|
|
2620
|
+
};
|
|
2621
|
+
})()];
|
|
2622
|
+
if (compare && (!compare.harness || compare.harness.includes(hid))) for (const v of compare.variants) {
|
|
2623
|
+
const env = applied(v);
|
|
2624
|
+
const sandbox = attemptSandbox(resolved, env);
|
|
2625
|
+
const sid = sandboxId(sandbox);
|
|
2626
|
+
cells.push({
|
|
2627
|
+
env,
|
|
2628
|
+
sandbox,
|
|
2629
|
+
cell: {
|
|
2630
|
+
key: cellKey(hid, env.name, sid),
|
|
2631
|
+
harness: hid,
|
|
2632
|
+
environment: env.name,
|
|
2633
|
+
sandbox: sid,
|
|
2634
|
+
role: "variant",
|
|
2635
|
+
variant: v.name
|
|
2636
|
+
},
|
|
2637
|
+
gate: gateIf(compare.gate !== "none")
|
|
2638
|
+
});
|
|
2639
|
+
}
|
|
2640
|
+
for (const { env, sandbox, cell, gate } of cells) {
|
|
2641
|
+
const axes = [resolved.harness.length > 1 ? hid : "", compare ? cell.role === "control" ? "control" : env.name : ""].filter(Boolean);
|
|
2642
|
+
for (let n = 1; n <= resolved.runs; n++) {
|
|
2643
|
+
const label = [axes.join(" · "), resolved.runs > 1 ? `#${n}` : ""].filter(Boolean).join(" ");
|
|
2644
|
+
attempts.push({
|
|
2645
|
+
cell,
|
|
2646
|
+
harness,
|
|
2647
|
+
sandbox,
|
|
2648
|
+
environment: env,
|
|
2649
|
+
egress: plannedEgress(resolved.egress, harness, resolved.env, env.layers),
|
|
2650
|
+
gate,
|
|
2651
|
+
...cell.role === "variant" && compare?.minLift !== void 0 ? { minLift: compare.minLift } : {},
|
|
2652
|
+
n,
|
|
2653
|
+
label
|
|
2654
|
+
});
|
|
2655
|
+
}
|
|
2656
|
+
}
|
|
2657
|
+
}
|
|
2658
|
+
return attempts;
|
|
2659
|
+
}
|
|
2660
|
+
/** Warnings about what an eval's environments pull in, e.g. skills at a moving git ref. */
|
|
2661
|
+
function environmentWarnings(attempts) {
|
|
2662
|
+
const out = /* @__PURE__ */ new Set();
|
|
2663
|
+
for (const a of attempts) {
|
|
2664
|
+
const workspace = workspaceOf(a.environment.layers);
|
|
2665
|
+
if (workspace && isMovingRef(workspace.ref)) {
|
|
2666
|
+
const ref = workspace.ref ? `moving ref "${workspace.ref}"` : "no ref (the default branch)";
|
|
2667
|
+
out.add(`workspace git(${workspace.repo}) uses ${ref}: results aren't comparable over time. Pin a tag or SHA (the resolved commit is recorded either way).`);
|
|
2668
|
+
}
|
|
2669
|
+
for (const layer of a.environment.layers) for (const s of layer.spec.skills ?? []) {
|
|
2670
|
+
if (s.source === "git" && isMovingRef(s.ref)) {
|
|
2671
|
+
const ref = s.ref ? `moving ref "${s.ref}"` : "no ref (the default branch)";
|
|
2672
|
+
out.add(`skill.git(${s.repo}) uses ${ref}: results aren't comparable over time. Pin a tag or SHA (the resolved commit is recorded either way).`);
|
|
2673
|
+
}
|
|
2674
|
+
if (s.source === "registry" && !s.version) out.add(`skill.registry(${s.id}) has no version: results aren't comparable over time. Pin one (the resolved commit is recorded either way).`);
|
|
2675
|
+
}
|
|
2676
|
+
}
|
|
2677
|
+
return [...out];
|
|
2678
|
+
}
|
|
2679
|
+
/** Secrets a Docker attempt would get as real values, because they have no `hosts`. */
|
|
2680
|
+
function unbrokered(env, attempts) {
|
|
2681
|
+
const out = /* @__PURE__ */ new Map();
|
|
2682
|
+
for (const a of attempts) {
|
|
2683
|
+
if (a.sandbox.kind !== "docker") continue;
|
|
2684
|
+
for (const s of unbrokeredSecrets(env, a.environment.layers)) out.set(s.label, s);
|
|
2685
|
+
}
|
|
2686
|
+
return [...out.values()];
|
|
2687
|
+
}
|
|
2688
|
+
/** An optional secret whose host variable isn't set never reaches the sandbox, so it gets no warning. */
|
|
2689
|
+
const reachesSandbox = (s) => !s.optional || Boolean(process.env[s.from]);
|
|
2690
|
+
/** `GEMINI_API_KEY → generativelanguage.googleapis.com` for each secret with `hosts`. */
|
|
2691
|
+
const withHosts = (env, attempt) => [...new Set(plannedSecrets(env, attempt.environment.layers).flatMap((s) => s.hosts ? [`${s.label} → ${s.hosts.join(", ")}`] : []))];
|
|
2692
|
+
/** The secrets an attempt brokers: only Docker's egress gateway swaps a placeholder for its value. */
|
|
2693
|
+
const brokeredIn = (env, attempt) => attempt.sandbox.kind === "docker" ? withHosts(env, attempt) : [];
|
|
2694
|
+
/** The plan's note for secrets with `hosts` on the fake sandbox, which only ever has their placeholder. */
|
|
2695
|
+
function unbrokeredOnFake(env, attempts) {
|
|
2696
|
+
const names = [...new Set(attempts.flatMap((a) => a.sandbox.kind === "fake" ? withHosts(env, a) : []))];
|
|
2697
|
+
return names.length ? [`not brokered (fake sandbox, placeholder only): ${names.join("; ")}`] : [];
|
|
2698
|
+
}
|
|
2699
|
+
/** The host variables an attempt's secrets read, brokered or not. */
|
|
2700
|
+
const secretVarsIn = (env, attempt) => [...new Set(plannedSecrets(env, attempt.environment.layers).map((s) => s.from))];
|
|
2701
|
+
const brokerHint = (s) => /^mcp\.[^.]+\.headers\./.test(s.label) ? "give the MCP server an https:// URL (reeval brokers its header secrets to its own host, over HTTPS only)" : `add the hosts it's sent to, e.g. secret.fromEnv('${s.from}', { hosts: ['api.example.com'] })`;
|
|
2702
|
+
const unbrokeredWarning = (s) => s.checksOnly ? `${s.label} reaches the checks (c.exec, setup steps) as its real value; the agent never gets it. To give the checks only a placeholder, ${brokerHint(s)}.` : `${s.label} is in the sandbox as its real value. To give the agent only a placeholder, ${brokerHint(s)}.`;
|
|
2703
|
+
const unbrokeredError = (s, where, configFile) => `${s.label} in ${where} isn't brokered, and secrets.unbrokered is 'error' in ${configFile}. To broker it, ${brokerHint(s)}.`;
|
|
2704
|
+
const levelOf = (where) => where.startsWith("evaluate(") ? "eval" : where.startsWith("group(") ? "group" : where.startsWith("profiles.") ? "profile" : "config";
|
|
2705
|
+
/**
|
|
2706
|
+
* A warning per privilege an attempt's Docker sandbox adds, naming the level that set the
|
|
2707
|
+
* environment which owns that sandbox: the config when no profile, group or eval did.
|
|
2708
|
+
*/
|
|
2709
|
+
function sandboxWarnings(attempts, resolved, origin) {
|
|
2710
|
+
const out = [];
|
|
2711
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2712
|
+
for (const a of attempts) {
|
|
2713
|
+
if (a.sandbox.kind !== "docker") continue;
|
|
2714
|
+
const where = sandboxSetBy(a, resolved, origin.configFile);
|
|
2715
|
+
const level = levelOf(where);
|
|
2716
|
+
const file = level === "eval" || level === "group" ? origin.file : null;
|
|
2717
|
+
if (a.sandbox.root) {
|
|
2718
|
+
const message = `running as root in the sandbox: ${where} (docker({ root: true }))`;
|
|
2719
|
+
const key = `${file}\0${message}`;
|
|
2720
|
+
if (!seen.has(key)) {
|
|
2721
|
+
seen.add(key);
|
|
2722
|
+
out.push({
|
|
2723
|
+
kind: "root",
|
|
2724
|
+
level,
|
|
2725
|
+
where,
|
|
2726
|
+
file,
|
|
2727
|
+
message
|
|
2728
|
+
});
|
|
2729
|
+
}
|
|
2730
|
+
}
|
|
2731
|
+
if (a.sandbox.capabilities?.length) {
|
|
2732
|
+
const message = `extra capabilities in the sandbox: ${where} (docker({ capabilities: [${a.sandbox.capabilities.map((c) => `'${c}'`).join(", ")}] }))`;
|
|
2733
|
+
const key = `${file}\0${message}`;
|
|
2734
|
+
if (!seen.has(key)) {
|
|
2735
|
+
seen.add(key);
|
|
2736
|
+
out.push({
|
|
2737
|
+
kind: "capabilities",
|
|
2738
|
+
level,
|
|
2739
|
+
where,
|
|
2740
|
+
file,
|
|
2741
|
+
capabilities: [...a.sandbox.capabilities],
|
|
2742
|
+
message
|
|
2743
|
+
});
|
|
2744
|
+
}
|
|
2745
|
+
}
|
|
2746
|
+
}
|
|
2747
|
+
return out;
|
|
2748
|
+
}
|
|
2749
|
+
/** The plan row for an eval. */
|
|
2750
|
+
function evalPlan(r, attempts, origin) {
|
|
2751
|
+
const { resolved, skip } = r;
|
|
2752
|
+
const capture = captureNote(resolved.capture);
|
|
2753
|
+
const notes = [
|
|
2754
|
+
...r.notes,
|
|
2755
|
+
...capture ? [capture] : [],
|
|
2756
|
+
...unbrokeredOnFake(resolved.env, attempts)
|
|
2757
|
+
];
|
|
2758
|
+
return {
|
|
2759
|
+
harnesses: skip ? 0 : resolved.harness.length,
|
|
2760
|
+
variants: resolved.compare ? {
|
|
2761
|
+
run: resolved.compare.variants.length,
|
|
2762
|
+
of: resolved.compare.of
|
|
2763
|
+
} : null,
|
|
2764
|
+
...resolved.compare ? { compare: {
|
|
2765
|
+
baseline: resolved.compare.baseline.name,
|
|
2766
|
+
...resolved.compare.from ? { from: resolved.compare.from } : {}
|
|
2767
|
+
} } : {},
|
|
2768
|
+
...!skip && resolved.questions.length ? { questions: resolved.questions } : {},
|
|
2769
|
+
runs: resolved.runs,
|
|
2770
|
+
...resolved.gate !== void 0 ? { gate: resolved.gate } : {},
|
|
2771
|
+
attempts: skip ? 0 : attempts.length,
|
|
2772
|
+
notes,
|
|
2773
|
+
...skip ? { skip } : {},
|
|
2774
|
+
warnings: [
|
|
2775
|
+
...environmentWarnings(attempts),
|
|
2776
|
+
...skip ? [] : questionWarnings(resolved),
|
|
2777
|
+
...unbrokered(r.resolved.env, attempts).filter(reachesSandbox).map(unbrokeredWarning)
|
|
2778
|
+
],
|
|
2779
|
+
sandboxWarnings: skip ? [] : sandboxWarnings(attempts, resolved, origin),
|
|
2780
|
+
brokered: [...new Set(attempts.flatMap((a) => brokeredIn(r.resolved.env, a)))],
|
|
2781
|
+
egress: [...new Set(attempts.map((a) => egressLabel(a.egress)).filter(Boolean))],
|
|
2782
|
+
workspaces: [...new Set(attempts.flatMap((a) => {
|
|
2783
|
+
const w = workspaceOf(a.environment.layers);
|
|
2784
|
+
return w ? [workspaceLabel(w)] : [];
|
|
2785
|
+
}))],
|
|
2786
|
+
...capture ? { capture: resolved.capture } : {}
|
|
2787
|
+
};
|
|
2788
|
+
}
|
|
2789
|
+
/** A pairwise question that shows each side's transcript, asked of two harnesses: tool names unblind it. */
|
|
2790
|
+
function questionWarnings(resolved) {
|
|
2791
|
+
if (resolved.harness.length < 2) return [];
|
|
2792
|
+
return resolved.questions.filter((q) => pairFactors(q.across).includes("harness") && q.state?.transcript).map((q) => `question "${q.question}" shows state.transcript, which names each harness's tools, so its harness pairs aren't blind. Ask it with across: 'environment', or leave the transcript out.`);
|
|
2793
|
+
}
|
|
2794
|
+
function captureNote(c) {
|
|
2795
|
+
const parts = [];
|
|
2796
|
+
if (c.viewport === "full-page") parts.push("full-page");
|
|
2797
|
+
else if (c.viewport) parts.push(`${c.viewport.width}×${c.viewport.height}`);
|
|
2798
|
+
if (c.timeout !== void 0) parts.push(typeof c.timeout === "number" ? `${c.timeout}ms` : c.timeout);
|
|
2799
|
+
if (c.browser === false) parts.push("browser false");
|
|
2800
|
+
if (c.model) parts.push(`model ${c.model}`);
|
|
2801
|
+
return parts.length ? `capture: ${parts.join(" · ")}` : void 0;
|
|
2802
|
+
}
|
|
2803
|
+
/** `--tag a --tag !b`: every entry must hold. */
|
|
2804
|
+
function matchesTags(tags, filters) {
|
|
2805
|
+
return filters.every((f) => f.startsWith("!") ? !tags.includes(f.slice(1)) : tags.includes(f));
|
|
2806
|
+
}
|
|
2807
|
+
//#endregion
|
|
2808
|
+
//#region src/gateway/resolve.ts
|
|
2809
|
+
/** The config's list when it sets none: each provider's own key, as reeval has always run. */
|
|
2810
|
+
const DEFAULT_GATEWAYS = [{ kind: "direct" }];
|
|
2811
|
+
/** Every gateway reeval can route through, by name. */
|
|
2812
|
+
const GATEWAYS = {
|
|
2813
|
+
direct: { cantServe: (h) => {
|
|
2814
|
+
if (h.kind === "fx" && h.model.includes("/")) return `fx runs a gateway id such as ${JSON.stringify(h.model)} only through Vercel AI Gateway`;
|
|
2815
|
+
if (h.kind === "opencode") {
|
|
2816
|
+
const resolved = resolveOpenCodeProvider(h.model, process.env);
|
|
2817
|
+
if ("error" in resolved) return resolved.error;
|
|
2818
|
+
}
|
|
2819
|
+
} },
|
|
2820
|
+
vercel: { cantServe: (h) => {
|
|
2821
|
+
if (h.kind !== "fake" && !VERCEL_ROUTES[h.kind]) return VERCEL_REFUSALS[h.kind] ?? `reeval doesn't run ${agentName(h)} on Vercel AI Gateway yet`;
|
|
2822
|
+
const effort = harnessEffort(h);
|
|
2823
|
+
if (effort !== void 0 && h.kind !== "fake" && VERCEL_UNPROVEN_EFFORT.includes(h.kind)) return `reeval doesn't send ${agentName(h)}'s effort (${JSON.stringify(effort)}) through Vercel AI Gateway yet: nothing checks the model gets it as its own setting, so drop effort or run ${agentName(h)} on another gateway`;
|
|
2824
|
+
} }
|
|
2825
|
+
};
|
|
2826
|
+
/** The `reeval/gateway` helper that makes the gateway `name`, as messages name it. */
|
|
2827
|
+
const gatewayHelper = (name) => `${name}()`;
|
|
2828
|
+
/** The agent's name, for messages. */
|
|
2829
|
+
const agentName = (h) => isAgentKind(h.kind) ? AGENTS[h.kind].name : "the fake harness";
|
|
2830
|
+
/**
|
|
2831
|
+
* The gateway `harness` resolves to in `gateways`: its pin, or the first entry that can serve it,
|
|
2832
|
+
* with each entry before it and why it couldn't. Throws a UsageError naming every entry tried.
|
|
2833
|
+
*/
|
|
2834
|
+
function resolveGateway(gateways, harness, where, catalog = GATEWAYS) {
|
|
2835
|
+
const on = `${where} on ${harnessId(harness)}`;
|
|
2836
|
+
const listed = gateways.map((g) => g.kind).join(", ");
|
|
2837
|
+
const why = (name) => catalog[name]?.cantServe(harness);
|
|
2838
|
+
const pin = harness.gateway;
|
|
2839
|
+
if (pin !== void 0) {
|
|
2840
|
+
if (!gateways.some((g) => g.kind === pin)) throw new UsageError(`${on}: it pins gateway '${pin}', which isn't in the gateway list (${listed}). Add ${gatewayHelper(pin)} to gateway, or pin one that's there.`);
|
|
2841
|
+
const refused = why(pin);
|
|
2842
|
+
if (refused) throw new UsageError(`${on}: it pins gateway '${pin}', which can't serve it (${refused}). A pin never falls back; pin another gateway, or remove the pin to take the first in the list that can.`);
|
|
2843
|
+
return {
|
|
2844
|
+
name: pin,
|
|
2845
|
+
skipped: []
|
|
2846
|
+
};
|
|
2847
|
+
}
|
|
2848
|
+
const skipped = [];
|
|
2849
|
+
for (const { kind } of gateways) {
|
|
2850
|
+
const refused = why(kind);
|
|
2851
|
+
if (!refused) return {
|
|
2852
|
+
name: kind,
|
|
2853
|
+
skipped
|
|
2854
|
+
};
|
|
2855
|
+
skipped.push({
|
|
2856
|
+
gateway: kind,
|
|
2857
|
+
why: refused
|
|
2858
|
+
});
|
|
2859
|
+
}
|
|
2860
|
+
throw new UsageError(`${on}: no gateway in the list can serve it (${skipped.map((s) => `${s.gateway}: ${s.why}`).join("; ")}). ${addHint(gateways, harness, catalog)}`);
|
|
2861
|
+
}
|
|
2862
|
+
/** What to add to the list so it serves `harness`: `direct()` when it can, else the first that can. */
|
|
2863
|
+
function addHint(gateways, harness, catalog) {
|
|
2864
|
+
const serves = Object.keys(catalog).filter((name) => !gateways.some((g) => g.kind === name)).filter((name) => !catalog[name]?.cantServe(harness));
|
|
2865
|
+
if (serves.includes("direct")) return `Add direct() to the end of gateway to use ${agentName(harness)}'s own key.`;
|
|
2866
|
+
const [first] = serves;
|
|
2867
|
+
return first ? `Add ${gatewayHelper(first)} to gateway.` : `No gateway reeval knows can serve it.`;
|
|
2868
|
+
}
|
|
2869
|
+
/**
|
|
2870
|
+
* `harness` carrying the gateway it resolves to in `gateways`, as the plan gives it to the
|
|
2871
|
+
* worker; as written on `direct`, or when nothing serves it (the plan says why).
|
|
2872
|
+
*/
|
|
2873
|
+
function routedHarness(harness, gateways) {
|
|
2874
|
+
try {
|
|
2875
|
+
const { name } = resolveGateway(gateways, harness, "");
|
|
2876
|
+
return name === "direct" ? harness : {
|
|
2877
|
+
...harness,
|
|
2878
|
+
gateway: name
|
|
2879
|
+
};
|
|
2880
|
+
} catch {
|
|
2881
|
+
return harness;
|
|
2882
|
+
}
|
|
2883
|
+
}
|
|
2884
|
+
/** `gateway: claude-code:… → direct (vercel can't serve it: …)`, for the plan's hints. */
|
|
2885
|
+
const fallbackHint = (harness, resolved) => `gateway: ${harness} → ${resolved.name} (${resolved.skipped.map((s) => `${s.gateway} can't serve it: ${s.why}`).join("; ")})`;
|
|
2886
|
+
//#endregion
|
|
2887
|
+
//#region src/gateway/models.ts
|
|
2888
|
+
const providerOf = (model) => model.slice(0, model.indexOf("/"));
|
|
2889
|
+
/** Why each gateway can't serve a model by name, or undefined when it can. */
|
|
2890
|
+
const MODEL_GATEWAYS = {
|
|
2891
|
+
direct: (model) => DIRECT_PROVIDERS[providerOf(model)] ? void 0 : `reeval calls only ${Object.keys(DIRECT_PROVIDERS).map((p) => `${p}/*`).join(", ")} models on their own keys`,
|
|
2892
|
+
vercel: () => void 0
|
|
2893
|
+
};
|
|
2894
|
+
const gatewayScope = (config, profile) => ({ gateways: profile?.gateway ?? config.gateway });
|
|
2895
|
+
/**
|
|
2896
|
+
* The gateway `model` goes through: its pin, or the first in the list that can serve it. An unset
|
|
2897
|
+
* list is `[direct()]`, so a model takes its provider's own key unless the config lists another
|
|
2898
|
+
* gateway. A model object or a fake calls through itself, and is undefined. Throws a UsageError
|
|
2899
|
+
* naming each gateway tried.
|
|
2900
|
+
*/
|
|
2901
|
+
function resolveModelGateway(model, pin, scope, what) {
|
|
2902
|
+
if (typeof model !== "string" || !model.includes("/")) return void 0;
|
|
2903
|
+
const on = `${what} "${model}"`;
|
|
2904
|
+
const listed = scope.gateways.map((g) => g.kind);
|
|
2905
|
+
const name = typeof pin === "object" ? pin.kind : pin;
|
|
2906
|
+
if (name !== void 0) {
|
|
2907
|
+
if (!listed.includes(name)) throw new UsageError(`${on} pins gateway '${name}', which isn't in the gateway list (${listed.join(", ")}). Add ${gatewayHelper(name)} to gateway, or pin one that's there.`);
|
|
2908
|
+
const refused = MODEL_GATEWAYS[name](model);
|
|
2909
|
+
if (refused) throw new UsageError(`${on} pins gateway '${name}', which can't serve it (${refused}). A pin never falls back; pin another gateway, or remove the pin to take the first in the list that can.`);
|
|
2910
|
+
return name;
|
|
2911
|
+
}
|
|
2912
|
+
const skipped = [];
|
|
2913
|
+
for (const kind of listed) {
|
|
2914
|
+
const refused = MODEL_GATEWAYS[kind](model);
|
|
2915
|
+
if (!refused) return kind;
|
|
2916
|
+
skipped.push(`${kind}: ${refused}`);
|
|
2917
|
+
}
|
|
2918
|
+
const other = GATEWAY_NAMES.find((n) => !listed.includes(n) && !MODEL_GATEWAYS[n](model));
|
|
2919
|
+
const judge = other === "vercel" && what.includes("judge") ? ` There a judge must be a decision model, such as '${RECOMMENDED_JUDGE_MODEL}'.` : "";
|
|
2920
|
+
throw new UsageError(`${on}: no gateway in the list can serve it (${skipped.join("; ")}).${other ? ` Add ${gatewayHelper(other)} to gateway.${judge}` : ""}`);
|
|
2921
|
+
}
|
|
2922
|
+
/** As `resolveModelGateway`, but undefined where it would throw, for checks the plan already makes. */
|
|
2923
|
+
function modelGatewayOrNone(model, pin, scope) {
|
|
2924
|
+
try {
|
|
2925
|
+
return resolveModelGateway(model, pin, scope, "model");
|
|
2926
|
+
} catch {
|
|
2927
|
+
return;
|
|
2928
|
+
}
|
|
2929
|
+
}
|
|
2930
|
+
//#endregion
|
|
2931
|
+
//#region src/config.ts
|
|
2932
|
+
const CONFIG_KEYS = [
|
|
2933
|
+
"include",
|
|
2934
|
+
"gateway",
|
|
2935
|
+
"harness",
|
|
2936
|
+
"sandbox",
|
|
2937
|
+
"environment",
|
|
2938
|
+
"runs",
|
|
2939
|
+
"gate",
|
|
2940
|
+
"env",
|
|
2941
|
+
"egress",
|
|
2942
|
+
"judge",
|
|
2943
|
+
"capture",
|
|
2944
|
+
"question",
|
|
2945
|
+
"timeout",
|
|
2946
|
+
"concurrency",
|
|
2947
|
+
"record",
|
|
2948
|
+
"history",
|
|
2949
|
+
"budget",
|
|
2950
|
+
"profiles",
|
|
2951
|
+
"defaultProfile",
|
|
2952
|
+
"secrets",
|
|
2953
|
+
"debug",
|
|
2954
|
+
"analysis"
|
|
2955
|
+
];
|
|
2956
|
+
const PROFILE_KEYS = [
|
|
2957
|
+
"defaults",
|
|
2958
|
+
"limit",
|
|
2959
|
+
"judge",
|
|
2960
|
+
"budget",
|
|
2961
|
+
"record",
|
|
2962
|
+
"tags",
|
|
2963
|
+
"gateway"
|
|
2964
|
+
];
|
|
2965
|
+
const DEFAULTS_KEYS = [
|
|
2966
|
+
"harness",
|
|
2967
|
+
"sandbox",
|
|
2968
|
+
"environment",
|
|
2969
|
+
"runs",
|
|
2970
|
+
"gate",
|
|
2971
|
+
"env",
|
|
2972
|
+
"egress",
|
|
2973
|
+
"judge",
|
|
2974
|
+
"capture",
|
|
2975
|
+
"question",
|
|
2976
|
+
"timeout"
|
|
2977
|
+
];
|
|
2978
|
+
const LIMIT_KEYS = [
|
|
2979
|
+
"runs",
|
|
2980
|
+
"compare",
|
|
2981
|
+
"harness"
|
|
2982
|
+
];
|
|
2983
|
+
/**
|
|
2984
|
+
* The config: a plain object, or a callback that destructures the helpers it needs, e.g.
|
|
2985
|
+
* `defineConfig(({ secret }) => ({ env: { KEY: secret.fromEnv('KEY') } }))`.
|
|
2986
|
+
*/
|
|
2987
|
+
const defineConfig = (config) => config;
|
|
2988
|
+
/** The script configs `reeval` looks for in each directory, first match first, as Vitest orders its own. */
|
|
2989
|
+
const SCRIPT_FILES = [
|
|
2990
|
+
"ts",
|
|
2991
|
+
"mts",
|
|
2992
|
+
"cts",
|
|
2993
|
+
"js",
|
|
2994
|
+
"mjs",
|
|
2995
|
+
"cjs"
|
|
2996
|
+
].map((ext) => `reeval.config.${ext}`);
|
|
2997
|
+
/** The data configs, for projects without TypeScript: one of them, and no script config beside it. */
|
|
2998
|
+
const YAML_FILES = ["reeval.yaml", "reeval.yml"];
|
|
2999
|
+
/** Every file name `reeval` looks for in each directory. */
|
|
3000
|
+
const CONFIG_FILES = [...SCRIPT_FILES, ...YAML_FILES];
|
|
3001
|
+
/** True for a `reeval.yaml`-style config, which is data rather than a module. */
|
|
3002
|
+
const isYamlConfig = (path) => /\.ya?ml$/i.test(path);
|
|
3003
|
+
const DEFAULT_CONFIG_FILE = "reeval.config.ts";
|
|
3004
|
+
/** The names `findConfig` looks for, as doctor and the missing-config error list them. */
|
|
3005
|
+
const configFilesPhrase = () => `${CONFIG_FILES[0]} (or ${CONFIG_FILES.slice(1).join(", ")})`;
|
|
3006
|
+
/** Usage error when `findConfig()` finds nothing: same names as doctor, plus `reeval init`. */
|
|
3007
|
+
const missingConfigMessage = () => `No ${configFilesPhrase()} here or above. Run reeval init.`;
|
|
3008
|
+
/** A found config path, or the missing-config usage error. */
|
|
3009
|
+
function requireConfig(path) {
|
|
3010
|
+
if (path === void 0) throw new UsageError(missingConfigMessage());
|
|
3011
|
+
return path;
|
|
3012
|
+
}
|
|
3013
|
+
/**
|
|
3014
|
+
* The nearest config file in `from` or a directory above it, up to the filesystem root, the way
|
|
3015
|
+
* Vitest and ESLint find theirs. A git or workspace root doesn't stop the search: a nested
|
|
3016
|
+
* project's own config is nearer, so it wins.
|
|
3017
|
+
*/
|
|
3018
|
+
function findConfig(from = process.cwd()) {
|
|
3019
|
+
for (let dir = resolve(from);; dir = dirname(dir)) {
|
|
3020
|
+
const script = SCRIPT_FILES.find((name) => existsSync(join(dir, name)));
|
|
3021
|
+
const yaml = YAML_FILES.filter((name) => existsSync(join(dir, name)));
|
|
3022
|
+
const found = [...script ? [script] : [], ...yaml];
|
|
3023
|
+
if (found.length > 1) {
|
|
3024
|
+
const where = fromCwd(dir) || ".";
|
|
3025
|
+
throw new UsageError(`${found.join(" and ")} are both in ${where}: reeval reads one config per directory. Keep one, or pass --config.`);
|
|
3026
|
+
}
|
|
3027
|
+
if (found[0]) return join(dir, found[0]);
|
|
3028
|
+
if (dirname(dir) === dir) return void 0;
|
|
3029
|
+
}
|
|
3030
|
+
}
|
|
3031
|
+
/** The config a command reads: `--config` (from the current directory), else the nearest one. */
|
|
3032
|
+
const configPathFor = (flag) => flag === void 0 ? findConfig() : resolve(flag);
|
|
3033
|
+
/** The config file as the user would type it from the current directory. */
|
|
3034
|
+
const configLabel = (path) => fromCwd(path) || path;
|
|
3035
|
+
/**
|
|
3036
|
+
* Loads the config at `path` (see `configPathFor`); its directory is the project root. Without a
|
|
3037
|
+
* path, every key takes its default and the current directory is the root. `reeval run` and
|
|
3038
|
+
* `reeval image build` refuse a missing file before this.
|
|
3039
|
+
*/
|
|
3040
|
+
async function loadConfig(path, command = "run", profile = void 0) {
|
|
3041
|
+
if (path === void 0) return resolveConfig({}, process.cwd(), DEFAULT_CONFIG_FILE);
|
|
3042
|
+
const root = dirname(path);
|
|
3043
|
+
const configPath = configLabel(path);
|
|
3044
|
+
if (!existsSync(path)) throw new UsageError(`Config file not found: ${configPath}`);
|
|
3045
|
+
if (isYamlConfig(path)) return resolveConfigData(parseYamlConfig(readFileSync(path, "utf8"), configPath), root, configPath);
|
|
3046
|
+
const module = await (await reevalJiti(ownFileURL("dist/cli.js").href, { moduleCache: false })).import(path);
|
|
3047
|
+
if (!module || typeof module !== "object" || !("default" in module)) {
|
|
3048
|
+
const names = module && typeof module === "object" ? Object.keys(module) : [];
|
|
3049
|
+
const exportsNote = names.length ? ` (it exports ${names.join(", ")})` : "";
|
|
3050
|
+
throw new UsageError(`${configPath} has no default export${exportsNote}. Write export default defineConfig({…}).`);
|
|
3051
|
+
}
|
|
3052
|
+
let exported = module.default;
|
|
3053
|
+
if (typeof exported === "function") {
|
|
3054
|
+
const hostEnv = process.env;
|
|
3055
|
+
exported = exported({
|
|
3056
|
+
...helpersFor(root),
|
|
3057
|
+
ci: inCI(hostEnv),
|
|
3058
|
+
command,
|
|
3059
|
+
hostEnv,
|
|
3060
|
+
profile
|
|
3061
|
+
});
|
|
3062
|
+
if (exported instanceof Promise) throw new UsageError(`The defineConfig callback in ${configPath} returned a Promise. It runs once when the config loads and must return the config synchronously.`);
|
|
3063
|
+
}
|
|
3064
|
+
if (!exported || typeof exported !== "object" || exported instanceof Promise) throw new UsageError(`${configPath}'s default export must be defineConfig({…}) (got ${exported instanceof Promise ? "a Promise" : JSON.stringify(exported)}). Write export default defineConfig({…}).`);
|
|
3065
|
+
return resolveConfig(exported, root, configPath);
|
|
3066
|
+
}
|
|
3067
|
+
/** A `reeval.yaml`'s text as data: an empty file is an empty config. */
|
|
3068
|
+
function parseYamlConfig(text, file) {
|
|
3069
|
+
const doc = parseDocument(text, { prettyErrors: true });
|
|
3070
|
+
const [error] = doc.errors;
|
|
3071
|
+
if (error) throw new UsageError(`${file} isn't valid YAML: ${error.message.trimEnd()}`);
|
|
3072
|
+
return doc.toJS() ?? {};
|
|
3073
|
+
}
|
|
3074
|
+
const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3075
|
+
/** A config's Dockerfile that isn't there, named by its field. */
|
|
3076
|
+
const missingIn = (file) => (field, full) => {
|
|
3077
|
+
throw new UsageError(`${field} in ${file}: ${full} doesn't exist. Paths in ${file} are relative to its directory.`);
|
|
3078
|
+
};
|
|
3079
|
+
/**
|
|
3080
|
+
* The config with `sandbox` and each profile's `defaults.sandbox`, when written as data, given
|
|
3081
|
+
* absolute host paths against `root`. With `environments`, an environment written as data (a
|
|
3082
|
+
* `reeval.yaml`'s) gets its layers' paths resolved too; a script config's come from
|
|
3083
|
+
* `defineEnvironment`, which resolves them against its own file.
|
|
3084
|
+
*/
|
|
3085
|
+
function withHostPaths(config, root, file, { environments = false } = {}) {
|
|
3086
|
+
const missing = missingIn(file);
|
|
3087
|
+
const options = (value, prefix) => {
|
|
3088
|
+
if (!isRecord(value)) return value;
|
|
3089
|
+
const out = { ...value };
|
|
3090
|
+
if ("sandbox" in value) out.sandbox = sandboxPaths(value.sandbox, root, `${prefix}sandbox`, missing);
|
|
3091
|
+
if (environments && "environment" in value) out.environment = environmentPaths(value.environment, root, `${prefix}environment`, missing);
|
|
3092
|
+
return out;
|
|
3093
|
+
};
|
|
3094
|
+
const out = options(config, "");
|
|
3095
|
+
const { profiles } = out;
|
|
3096
|
+
if (!isRecord(profiles)) return out;
|
|
3097
|
+
const each = Object.entries(profiles).map(([name, profile]) => isRecord(profile) && "defaults" in profile ? [name, {
|
|
3098
|
+
...profile,
|
|
3099
|
+
defaults: options(profile.defaults, `profiles.${name}.defaults.`)
|
|
3100
|
+
}] : [name, profile]);
|
|
3101
|
+
return {
|
|
3102
|
+
...out,
|
|
3103
|
+
profiles: Object.fromEntries(each)
|
|
3104
|
+
};
|
|
3105
|
+
}
|
|
3106
|
+
/** The config from its data form: each environment's data becomes an `Environment` again. */
|
|
3107
|
+
function configFromData(data) {
|
|
3108
|
+
const withEnvironment = (options) => {
|
|
3109
|
+
if (!isObjectLike(options)) return options;
|
|
3110
|
+
const { environment } = options;
|
|
3111
|
+
return isEnvironment(environment) ? {
|
|
3112
|
+
...options,
|
|
3113
|
+
environment: environmentFrom(environment)
|
|
3114
|
+
} : options;
|
|
3115
|
+
};
|
|
3116
|
+
const config = withEnvironment(data);
|
|
3117
|
+
const { profiles } = config;
|
|
3118
|
+
if (typeof profiles !== "object" || profiles === null || Array.isArray(profiles)) return config;
|
|
3119
|
+
const each = Object.entries(profiles).map(([name, profile]) => {
|
|
3120
|
+
if (!isObjectLike(profile)) return [name, profile];
|
|
3121
|
+
const { defaults } = profile;
|
|
3122
|
+
return [name, defaults === void 0 ? profile : {
|
|
3123
|
+
...profile,
|
|
3124
|
+
defaults: withEnvironment(defaults)
|
|
3125
|
+
}];
|
|
3126
|
+
});
|
|
3127
|
+
return {
|
|
3128
|
+
...config,
|
|
3129
|
+
profiles: Object.fromEntries(each)
|
|
3130
|
+
};
|
|
3131
|
+
}
|
|
3132
|
+
/**
|
|
3133
|
+
* Resolves a config in its data form (a parsed `reeval.yaml`, or `ReevalConfigData` from another
|
|
3134
|
+
* language) whose relative host paths resolve against `root`, its directory: `resolveConfig`'s
|
|
3135
|
+
* checks name a mistake in their own words first, and the config contract refuses anything they
|
|
3136
|
+
* let through, such as a `null` or a misspelt nested key.
|
|
3137
|
+
*/
|
|
3138
|
+
function resolveConfigData(input, root, file) {
|
|
3139
|
+
if (!isRecord(input)) throw new UsageError(`${file} must be a mapping of config keys, like harness: and judge: (got ${JSON.stringify(input)})`);
|
|
3140
|
+
const data = withHostPaths(input, root, file, { environments: true });
|
|
3141
|
+
const { environment, profiles } = data;
|
|
3142
|
+
if ([environment, ...Object.values(typeof profiles === "object" && profiles ? profiles : {}).map((p) => p?.defaults?.environment)].some((e) => e !== void 0 && !isEnvironment(e))) {
|
|
3143
|
+
const problem = configDataProblem(data, file);
|
|
3144
|
+
if (problem) throw new UsageError(problem);
|
|
3145
|
+
}
|
|
3146
|
+
let resolved;
|
|
3147
|
+
try {
|
|
3148
|
+
resolved = resolveConfig(configFromData(data), root, file);
|
|
3149
|
+
} catch (error) {
|
|
3150
|
+
if (error instanceof UsageError) throw error;
|
|
3151
|
+
const problem = configDataProblem(data, file);
|
|
3152
|
+
if (problem) throw new UsageError(problem);
|
|
3153
|
+
throw error;
|
|
3154
|
+
}
|
|
3155
|
+
const problem = configDataProblem(data, file);
|
|
3156
|
+
if (problem) throw new UsageError(problem);
|
|
3157
|
+
return resolved;
|
|
3158
|
+
}
|
|
3159
|
+
const GLOB = /[*?[\]{}!]/;
|
|
3160
|
+
/** The directory every include pattern starts in: `evals/**\/*.eval.ts` → `evals`. */
|
|
3161
|
+
function evalRoot(include) {
|
|
3162
|
+
const dirs = include.map((pattern) => {
|
|
3163
|
+
const parts = pattern.split("/");
|
|
3164
|
+
const stop = parts.findIndex((part) => GLOB.test(part));
|
|
3165
|
+
return (stop < 0 ? parts.slice(0, -1) : parts.slice(0, stop)).filter((p) => p && p !== ".");
|
|
3166
|
+
});
|
|
3167
|
+
const common = [];
|
|
3168
|
+
for (let i = 0; dirs.every((d) => d[i] !== void 0 && d[i] === dirs[0]?.[i]); i++) common.push(dirs[0]?.[i]);
|
|
3169
|
+
return common.join("/");
|
|
3170
|
+
}
|
|
3171
|
+
const ARTIFACTS = [
|
|
3172
|
+
"all",
|
|
3173
|
+
"failures",
|
|
3174
|
+
"none"
|
|
3175
|
+
];
|
|
3176
|
+
const RECORD = [
|
|
3177
|
+
"manual",
|
|
3178
|
+
"ci",
|
|
3179
|
+
"always"
|
|
3180
|
+
];
|
|
3181
|
+
const unknownKey = (value, allowed, where, file) => {
|
|
3182
|
+
const key = Object.keys(value).find((k) => !allowed.includes(k));
|
|
3183
|
+
if (key === void 0) return;
|
|
3184
|
+
const hints = {
|
|
3185
|
+
reuse: "Reusing control results from .history isn’t built yet; every run runs its controls.",
|
|
3186
|
+
compare: "compare only goes on a group or eval; a profile can narrow it with limit.compare."
|
|
3187
|
+
};
|
|
3188
|
+
const near = closest(key, allowed);
|
|
3189
|
+
const hint = hints[key] ?? (near ? `Did you mean "${near}"?` : `Use ${allowed.join(", ")}.`);
|
|
3190
|
+
throw new UsageError(`Unknown option "${key}" in ${where} in ${file}. ${hint}`);
|
|
3191
|
+
};
|
|
3192
|
+
function checkBudget(budget, where, file) {
|
|
3193
|
+
if (budget === void 0) return {};
|
|
3194
|
+
if (typeof budget !== "object" || budget === null) throw new UsageError(`${where} in ${file} must be an object like { attempts: 20, duration: '1h' } (got ${JSON.stringify(budget)})`);
|
|
3195
|
+
if ("usd" in budget || "tokens" in budget) throw new UsageError(`${where}.${"usd" in budget ? "usd" : "tokens"} in ${file} isn't supported yet: reeval records the tokens harnesses report, but prices none and enforces no token budget. Cap the run with attempts or duration.`);
|
|
3196
|
+
unknownKey(budget, ["attempts", "duration"], where, file);
|
|
3197
|
+
const { attempts, duration } = budget;
|
|
3198
|
+
if (attempts !== void 0 && !(Number.isInteger(attempts) && attempts >= 1)) throw new UsageError(`${where}.attempts in ${file} must be a whole number of at least 1 (got ${JSON.stringify(attempts)})`);
|
|
3199
|
+
if (duration !== void 0) parseDuration(duration);
|
|
3200
|
+
return {
|
|
3201
|
+
...attempts ? { attempts } : {},
|
|
3202
|
+
...duration ? { duration } : {}
|
|
3203
|
+
};
|
|
3204
|
+
}
|
|
3205
|
+
const describeValue = (v) => {
|
|
3206
|
+
if (!isObjectLike(v)) return JSON.stringify(v);
|
|
3207
|
+
const { kind } = v;
|
|
3208
|
+
if (kind === "fake") return "fakeHarness({ … })";
|
|
3209
|
+
return isAgentKind(kind) ? `${AGENTS[kind].factory}({ … })` : "an object";
|
|
3210
|
+
};
|
|
3211
|
+
function checkProfile(name, profile, file) {
|
|
3212
|
+
const where = `profiles.${name}`;
|
|
3213
|
+
if (typeof profile !== "object" || profile === null) throw new UsageError(`${where} in ${file} must be an object of profile settings, e.g. { limit: { runs: 1 } } (got ${JSON.stringify(profile)})`);
|
|
3214
|
+
unknownKey(profile, PROFILE_KEYS, where, file);
|
|
3215
|
+
const p = profile;
|
|
3216
|
+
if (p.defaults !== void 0) {
|
|
3217
|
+
unknownKey(p.defaults, DEFAULTS_KEYS, `${where}.defaults`, file);
|
|
3218
|
+
checkOptions(p.defaults, `${where}.defaults in ${file}`);
|
|
3219
|
+
}
|
|
3220
|
+
if (p.limit !== void 0) {
|
|
3221
|
+
unknownKey(p.limit, LIMIT_KEYS, `${where}.limit`, file);
|
|
3222
|
+
const { runs, compare, harness } = p.limit;
|
|
3223
|
+
if (runs !== void 0 && !(Number.isInteger(runs) && runs >= 1)) throw new UsageError(`${where}.limit.runs in ${file} must be a whole number of at least 1 (got ${JSON.stringify(runs)})`);
|
|
3224
|
+
if (compare !== void 0 && compare !== false && !(Number.isInteger(compare?.maxVariants) && compare.maxVariants >= 0)) throw new UsageError(`${where}.limit.compare in ${file} must be false or { maxVariants: n }, e.g. { maxVariants: 1 } (got ${JSON.stringify(compare)})`);
|
|
3225
|
+
if (harness !== void 0) {
|
|
3226
|
+
const bad = Array.isArray(harness) ? harness.find((h) => typeof h !== "string") : harness;
|
|
3227
|
+
if (bad !== void 0) throw new UsageError(`${where}.limit.harness in ${file} can only list harness ids to allow (got ${describeValue(bad)}). To add harnesses, use ${where}.defaults.harness.`);
|
|
3228
|
+
}
|
|
3229
|
+
}
|
|
3230
|
+
if (p.tags !== void 0 && (!Array.isArray(p.tags) || p.tags.some((t) => typeof t !== "string"))) throw new UsageError(`${where}.tags in ${file} must be a list of tag filters, e.g. tags: ['smoke'] (got ${JSON.stringify(p.tags)})`);
|
|
3231
|
+
if (p.record !== void 0 && !RECORD.includes(p.record)) throw new UsageError(`${where}.record in ${file} is ${oneOf(RECORD)} (got ${JSON.stringify(p.record)})`);
|
|
3232
|
+
if (p.judge !== void 0) resolveJudge(p.judge, file, `${where}.judge`);
|
|
3233
|
+
checkBudget(p.budget, `${where}.budget`, file);
|
|
3234
|
+
if (p.gateway !== void 0) return {
|
|
3235
|
+
...p,
|
|
3236
|
+
gateway: checkGateways(p.gateway, `${where}.gateway`, file)
|
|
3237
|
+
};
|
|
3238
|
+
return p;
|
|
3239
|
+
}
|
|
3240
|
+
const GATEWAY_EXAMPLE = "gateway: [direct()]";
|
|
3241
|
+
const isGatewayName = (v) => typeof v === "string" && GATEWAY_NAMES.includes(v);
|
|
3242
|
+
/**
|
|
3243
|
+
* A `gateway` list as descriptors: each entry is a helper's descriptor, or as data its name or a
|
|
3244
|
+
* one-key map from its name to its settings. Refuses an empty list, an unknown entry and a
|
|
3245
|
+
* gateway listed twice.
|
|
3246
|
+
*/
|
|
3247
|
+
function checkGateways(value, where, file) {
|
|
3248
|
+
if (!Array.isArray(value) || value.length === 0) throw new UsageError(`${where} in ${file} must be a list of gateways in the order you prefer them, e.g. ${GATEWAY_EXAMPLE} (got ${JSON.stringify(value)})`);
|
|
3249
|
+
const names = GATEWAY_NAMES.map((n) => `'${n}'`).join(" or ");
|
|
3250
|
+
const out = [];
|
|
3251
|
+
value.forEach((entry, i) => {
|
|
3252
|
+
const at = `${where}[${i}] in ${file}`;
|
|
3253
|
+
const unknownName = (name) => {
|
|
3254
|
+
const near = closest(name, [...GATEWAY_NAMES]);
|
|
3255
|
+
return new UsageError(`${at} is ${JSON.stringify(name)}, which isn't a gateway reeval knows: ${names}.${near ? ` Did you mean '${near}'?` : ""}`);
|
|
3256
|
+
};
|
|
3257
|
+
let descriptor = entry;
|
|
3258
|
+
if (typeof entry === "string") {
|
|
3259
|
+
if (!isGatewayName(entry)) throw unknownName(entry);
|
|
3260
|
+
descriptor = { kind: entry };
|
|
3261
|
+
} else if (isRecord(entry) && typeof entry.kind === "string" && !isGatewayName(entry.kind)) throw unknownName(entry.kind);
|
|
3262
|
+
else if (isRecord(entry) && !("kind" in entry)) {
|
|
3263
|
+
const keys = Object.keys(entry);
|
|
3264
|
+
const [name] = keys;
|
|
3265
|
+
if (keys.length !== 1 || !isGatewayName(name)) throw new UsageError(`${at} must be a gateway's name or a map from one name to its settings, e.g. { direct: {} } (got ${JSON.stringify(entry)})`);
|
|
3266
|
+
descriptor = {
|
|
3267
|
+
kind: name,
|
|
3268
|
+
...isRecord(entry[name]) ? entry[name] : {}
|
|
3269
|
+
};
|
|
3270
|
+
}
|
|
3271
|
+
const parsed = gatewayDescriptorSchema.safeParse(descriptor);
|
|
3272
|
+
if (!parsed.success) {
|
|
3273
|
+
const kind = isRecord(descriptor) ? descriptor.kind : void 0;
|
|
3274
|
+
throw new UsageError(isGatewayName(kind) ? `${at} isn't a valid ${gatewayHelper(kind)}: ${parsed.error.issues[0]?.message ?? "unknown settings"} (got ${JSON.stringify(entry)})` : `${at} must be a gateway, e.g. direct() from 'reeval/gateway' (got ${describeValue(entry)})`);
|
|
3275
|
+
}
|
|
3276
|
+
if (out.some((g) => g.kind === parsed.data.kind)) throw new UsageError(`${at} lists ${parsed.data.kind} a second time. Each gateway is listed once, where you prefer it.`);
|
|
3277
|
+
out.push(parsed.data);
|
|
3278
|
+
});
|
|
3279
|
+
return out;
|
|
3280
|
+
}
|
|
3281
|
+
/** `--profile`, then `REEVAL_PROFILE`, then `defaultProfile.ci` in CI (or `.local`), then none. */
|
|
3282
|
+
function selectProfile(config, flag, env = process.env) {
|
|
3283
|
+
const pick = () => {
|
|
3284
|
+
if (flag) return [flag, "--profile"];
|
|
3285
|
+
if (env.REEVAL_PROFILE) return [env.REEVAL_PROFILE, "REEVAL_PROFILE"];
|
|
3286
|
+
const ci = inCI(env);
|
|
3287
|
+
if (ci && config.defaultProfile.ci) return [config.defaultProfile.ci, "auto: CI is set"];
|
|
3288
|
+
if (!ci && config.defaultProfile.local) return [config.defaultProfile.local, "default"];
|
|
3289
|
+
return null;
|
|
3290
|
+
};
|
|
3291
|
+
const picked = pick();
|
|
3292
|
+
if (!picked) return null;
|
|
3293
|
+
const [name, reason] = picked;
|
|
3294
|
+
const profile = config.profiles[name];
|
|
3295
|
+
if (!profile) {
|
|
3296
|
+
const names = Object.keys(config.profiles);
|
|
3297
|
+
throw new UsageError(`Unknown profile "${name}". ${names.length ? `Profiles in ${config.file}: ${names.join(", ")}.` : `${config.file} defines no profiles.`}`);
|
|
3298
|
+
}
|
|
3299
|
+
return {
|
|
3300
|
+
name,
|
|
3301
|
+
reason,
|
|
3302
|
+
defaults: profile.defaults ?? {},
|
|
3303
|
+
limit: profile.limit ?? {},
|
|
3304
|
+
judge: profile.judge ?? {},
|
|
3305
|
+
tags: profile.tags ?? [],
|
|
3306
|
+
budget: {
|
|
3307
|
+
...config.budget,
|
|
3308
|
+
...profile.budget
|
|
3309
|
+
},
|
|
3310
|
+
...profile.record ? { record: profile.record } : {},
|
|
3311
|
+
...profile.gateway ? { gateway: profile.gateway } : {}
|
|
3312
|
+
};
|
|
3313
|
+
}
|
|
3314
|
+
/**
|
|
3315
|
+
* The config with its defaults filled in and checked; `root` is its directory, which relative
|
|
3316
|
+
* host paths written as data resolve against.
|
|
3317
|
+
*/
|
|
3318
|
+
function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
3319
|
+
const user = withHostPaths(input, root, file);
|
|
3320
|
+
if ("compare" in user) throw new UsageError(`compare can't be set in ${file}: it would multiply every eval. Put it on the group or eval that asks the question.`);
|
|
3321
|
+
if ("questions" in user) throw new UsageError(`questions can't be set in ${file}: each is about one eval's task. Put them on the group or eval they ask about; question: { model, state, artifacts } in ${file} sets every question's defaults.`);
|
|
3322
|
+
checkOptions(user, file, CONFIG_KEYS);
|
|
3323
|
+
const include = user.include ?? defaultIncludeGlobs();
|
|
3324
|
+
if (!Array.isArray(include) || include.some((p) => typeof p !== "string")) throw new UsageError(`include in ${file} must be a list of glob patterns, e.g. include: ['evals/**/*.eval.ts'] (got ${JSON.stringify(include)})`);
|
|
3325
|
+
const record = user.record ?? "ci";
|
|
3326
|
+
if (!RECORD.includes(record)) throw new UsageError(`record in ${file} is ${oneOf(RECORD)} (got ${JSON.stringify(record)})`);
|
|
3327
|
+
const artifacts = user.history?.artifacts ?? "all";
|
|
3328
|
+
if (!ARTIFACTS.includes(artifacts)) throw new UsageError(`history.artifacts in ${file} is ${oneOf(ARTIFACTS)} (got ${JSON.stringify(artifacts)})`);
|
|
3329
|
+
const historyDir = resolve(root, user.history?.dir ?? join(evalRoot(include), ".history"));
|
|
3330
|
+
const harness = user.harness === void 0 ? [DEFAULT_HARNESS] : [user.harness].flat();
|
|
3331
|
+
if (harness.length === 0) throw new UsageError(`harness is an empty list in ${file}. Give at least one, e.g. harness: ${HARNESS_EXAMPLE}`);
|
|
3332
|
+
checkHarnesses(harness, file);
|
|
3333
|
+
const profiles = {};
|
|
3334
|
+
if (user.profiles !== void 0) {
|
|
3335
|
+
if (typeof user.profiles !== "object" || user.profiles === null) throw new UsageError(`profiles in ${file} must be an object of named profiles, e.g. profiles: { pr: { limit: { runs: 1 } } } (got ${JSON.stringify(user.profiles)})`);
|
|
3336
|
+
for (const [name, profile] of Object.entries(user.profiles)) profiles[name] = checkProfile(name, profile, file);
|
|
3337
|
+
}
|
|
3338
|
+
const defaultProfile = user.defaultProfile ?? {};
|
|
3339
|
+
unknownKey(defaultProfile, ["ci", "local"], "defaultProfile", file);
|
|
3340
|
+
for (const key of ["ci", "local"]) {
|
|
3341
|
+
const name = defaultProfile[key];
|
|
3342
|
+
if (name !== void 0 && !profiles[name]) {
|
|
3343
|
+
const names = Object.keys(profiles);
|
|
3344
|
+
throw new UsageError(`defaultProfile.${key} in ${file} names "${name}", which isn't a profile (${names.length ? `profiles: ${names.join(", ")}` : "it defines none"}). Add it under profiles, or name one that's there.`);
|
|
3345
|
+
}
|
|
3346
|
+
}
|
|
3347
|
+
return {
|
|
3348
|
+
root,
|
|
3349
|
+
file,
|
|
3350
|
+
optionKeys: Object.keys(user),
|
|
3351
|
+
include,
|
|
3352
|
+
harness,
|
|
3353
|
+
sandbox: user.sandbox ?? docker(),
|
|
3354
|
+
environment: user.environment,
|
|
3355
|
+
runs: user.runs ?? 1,
|
|
3356
|
+
gate: user.gate,
|
|
3357
|
+
env: user.env ?? {},
|
|
3358
|
+
...user.egress !== void 0 ? { egress: user.egress } : {},
|
|
3359
|
+
budget: checkBudget(user.budget, "budget", file),
|
|
3360
|
+
profiles,
|
|
3361
|
+
defaultProfile,
|
|
3362
|
+
judge: resolveJudge(user.judge, file),
|
|
3363
|
+
capture: user.capture ?? {},
|
|
3364
|
+
question: user.question ?? {},
|
|
3365
|
+
timeoutMs: parseDuration(user.timeout ?? "15m"),
|
|
3366
|
+
concurrency: user.concurrency === void 0 ? void 0 : checkConcurrency(user.concurrency, `concurrency in ${file}`),
|
|
3367
|
+
record,
|
|
3368
|
+
history: {
|
|
3369
|
+
dir: historyDir,
|
|
3370
|
+
artifacts
|
|
3371
|
+
},
|
|
3372
|
+
secrets: checkSecrets(user.secrets, file),
|
|
3373
|
+
debug: checkDebug(user.debug, file),
|
|
3374
|
+
analysis: checkAnalysis(user.analysis, file),
|
|
3375
|
+
gateway: user.gateway === void 0 ? DEFAULT_GATEWAYS : checkGateways(user.gateway, "gateway", file),
|
|
3376
|
+
gatewaySet: user.gateway !== void 0
|
|
3377
|
+
};
|
|
3378
|
+
}
|
|
3379
|
+
const ANALYSIS_DEFAULTS = {
|
|
3380
|
+
maxInputTokens: 2e4,
|
|
3381
|
+
maxOutputTokens: 8e3,
|
|
3382
|
+
insights: true,
|
|
3383
|
+
maxTokens: 25e4
|
|
3384
|
+
};
|
|
3385
|
+
function checkAnalysis(analysis, file) {
|
|
3386
|
+
if (analysis === void 0) return { ...ANALYSIS_DEFAULTS };
|
|
3387
|
+
if (typeof analysis !== "object" || analysis === null) throw new UsageError(`analysis in ${file} must be an object like { model: 'google/gemini-3.8-flash' } (got ${JSON.stringify(analysis)})`);
|
|
3388
|
+
unknownKey(analysis, [
|
|
3389
|
+
"model",
|
|
3390
|
+
"insights",
|
|
3391
|
+
"maxTokens",
|
|
3392
|
+
"instructions",
|
|
3393
|
+
"maxInputTokens",
|
|
3394
|
+
"maxOutputTokens"
|
|
3395
|
+
], "analysis", file);
|
|
3396
|
+
const { model, insights = ANALYSIS_DEFAULTS.insights, maxTokens = ANALYSIS_DEFAULTS.maxTokens, instructions, maxInputTokens = ANALYSIS_DEFAULTS.maxInputTokens, maxOutputTokens = ANALYSIS_DEFAULTS.maxOutputTokens } = analysis;
|
|
3397
|
+
if (typeof model === "string") {
|
|
3398
|
+
const problem = checkModelString(model, `analysis.model in ${file}`, [`google/${model}`]);
|
|
3399
|
+
if (problem) throw new UsageError(problem);
|
|
3400
|
+
} else if (model !== void 0 && !(isObjectLike(model) && model.kind === "fake-analysis")) throw new UsageError(`analysis.model in ${file} must be a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${JSON.stringify(model)})`);
|
|
3401
|
+
if (typeof insights !== "boolean") throw new UsageError(`analysis.insights in ${file} must be true or false (got ${JSON.stringify(insights)})`);
|
|
3402
|
+
if (!(Number.isInteger(maxTokens) && maxTokens >= 1)) throw new UsageError(`analysis.maxTokens in ${file} must be a whole number of tokens of at least 1, e.g. 250000 (got ${JSON.stringify(maxTokens)})`);
|
|
3403
|
+
if (instructions !== void 0 && typeof instructions !== "string") throw new UsageError(`analysis.instructions in ${file} must be a string (got ${JSON.stringify(instructions)})`);
|
|
3404
|
+
if (!(Number.isInteger(maxInputTokens) && maxInputTokens >= 1e3)) throw new UsageError(`analysis.maxInputTokens in ${file} must be a whole number of at least 1000 (got ${maxInputTokens})`);
|
|
3405
|
+
if (!(Number.isInteger(maxOutputTokens) && maxOutputTokens >= 500)) throw new UsageError(`analysis.maxOutputTokens in ${file} must be a whole number of at least 500 (got ${maxOutputTokens})`);
|
|
3406
|
+
return {
|
|
3407
|
+
...analysis,
|
|
3408
|
+
insights,
|
|
3409
|
+
maxTokens,
|
|
3410
|
+
maxInputTokens,
|
|
3411
|
+
maxOutputTokens
|
|
3412
|
+
};
|
|
3413
|
+
}
|
|
3414
|
+
/** Runs whose local debug data `.reeval/debug` keeps, and the trace buffer per attempt. */
|
|
3415
|
+
const DEBUG_DEFAULTS = {
|
|
3416
|
+
keep: 20,
|
|
3417
|
+
traceBufferMb: 8
|
|
3418
|
+
};
|
|
3419
|
+
function checkDebug(debug, file) {
|
|
3420
|
+
if (debug === void 0) return {
|
|
3421
|
+
keep: DEBUG_DEFAULTS.keep,
|
|
3422
|
+
traceBufferBytes: DEBUG_DEFAULTS.traceBufferMb * 2 ** 20
|
|
3423
|
+
};
|
|
3424
|
+
if (typeof debug !== "object" || debug === null) throw new UsageError(`debug in ${file} must be an object like { keep: 20, traceBufferMb: 8 } (got ${JSON.stringify(debug)})`);
|
|
3425
|
+
unknownKey(debug, ["keep", "traceBufferMb"], "debug", file);
|
|
3426
|
+
const { keep = DEBUG_DEFAULTS.keep, traceBufferMb = DEBUG_DEFAULTS.traceBufferMb } = debug;
|
|
3427
|
+
if (!(Number.isInteger(keep) && keep >= 1)) throw new UsageError(`debug.keep in ${file} must be a whole number of at least 1 (got ${JSON.stringify(keep)})`);
|
|
3428
|
+
if (!(typeof traceBufferMb === "number" && traceBufferMb > 0)) throw new UsageError(`debug.traceBufferMb in ${file} must be a number of megabytes above 0 (got ${JSON.stringify(traceBufferMb)})`);
|
|
3429
|
+
return {
|
|
3430
|
+
keep,
|
|
3431
|
+
traceBufferBytes: Math.round(traceBufferMb * 2 ** 20)
|
|
3432
|
+
};
|
|
3433
|
+
}
|
|
3434
|
+
function checkSecrets(secrets, file) {
|
|
3435
|
+
if (secrets === void 0) return { unbrokered: "warn" };
|
|
3436
|
+
if (typeof secrets !== "object" || secrets === null) throw new UsageError(`secrets in ${file} must be an object like { unbrokered: 'error' } (got ${JSON.stringify(secrets)})`);
|
|
3437
|
+
unknownKey(secrets, ["unbrokered"], "secrets", file);
|
|
3438
|
+
const unbrokered = secrets.unbrokered ?? "warn";
|
|
3439
|
+
if (unbrokered !== "warn" && unbrokered !== "error") throw new UsageError(`secrets.unbrokered in ${file} is 'warn' or 'error' (got ${JSON.stringify(unbrokered)})`);
|
|
3440
|
+
return { unbrokered };
|
|
3441
|
+
}
|
|
3442
|
+
const hostCapacity = () => ({
|
|
3443
|
+
cpus: availableParallelism(),
|
|
3444
|
+
memoryBytes: totalmem()
|
|
3445
|
+
});
|
|
3446
|
+
/**
|
|
3447
|
+
* Attempts mostly wait on a model, so the default is 8, but no more than one per CPU or per GB
|
|
3448
|
+
* of memory where the sandboxes run (the Docker VM, under Docker Desktop): an idle agent
|
|
3449
|
+
* container takes about 300 MB, and the rest is headroom for what the agent runs.
|
|
3450
|
+
*/
|
|
3451
|
+
function defaultConcurrency({ cpus, memoryBytes }) {
|
|
3452
|
+
return Math.max(1, Math.min(8, cpus, Math.floor(memoryBytes / 1e9)));
|
|
3453
|
+
}
|
|
3454
|
+
function checkConcurrency(value, where) {
|
|
3455
|
+
const n = typeof value === "string" && value.trim() !== "" ? Number(value) : value;
|
|
3456
|
+
if (typeof n !== "number" || !Number.isInteger(n) || n < 1) throw new UsageError(`${where} must be a whole number of at least 1 (got ${String(value)})`);
|
|
3457
|
+
return n;
|
|
3458
|
+
}
|
|
3459
|
+
/** `--concurrency`, then `REEVAL_CONCURRENCY`, then `concurrency` in the config, then the default. */
|
|
3460
|
+
async function resolveConcurrency(config, flag, capacity, env = process.env) {
|
|
3461
|
+
if (flag !== void 0) return checkConcurrency(flag, "--concurrency");
|
|
3462
|
+
if (env.REEVAL_CONCURRENCY) return checkConcurrency(env.REEVAL_CONCURRENCY, "REEVAL_CONCURRENCY");
|
|
3463
|
+
return config.concurrency ?? defaultConcurrency(await capacity());
|
|
3464
|
+
}
|
|
3465
|
+
function resolveJudge(judge, file, path = "judge") {
|
|
3466
|
+
judge ??= {};
|
|
3467
|
+
const problem = judgeProblem(judge, {
|
|
3468
|
+
at: "config",
|
|
3469
|
+
path,
|
|
3470
|
+
file
|
|
3471
|
+
});
|
|
3472
|
+
if (problem) throw new UsageError(problem);
|
|
3473
|
+
const threshold = judge.threshold ?? .8;
|
|
3474
|
+
return {
|
|
3475
|
+
...judge,
|
|
3476
|
+
threshold
|
|
3477
|
+
};
|
|
3478
|
+
}
|
|
3479
|
+
/** Host variables that required `env` secrets read, as `[sandbox name, host name]`. */
|
|
3480
|
+
function secretSources(env) {
|
|
3481
|
+
return Object.entries(env).flatMap(([name, value]) => isSecret(value) && !value.optional ? [[name, value.from ?? name]] : []);
|
|
3482
|
+
}
|
|
3483
|
+
/** The model that answers what each eval asks: `analysis.model` only, never the judge's. */
|
|
3484
|
+
const analysisModelOf = (config) => config.analysis.model;
|
|
3485
|
+
/** Whether `analysis.model` is set, so a model can answer what each eval asks. */
|
|
3486
|
+
const hasAnalysisModel = (config) => analysisModelOf(config) !== void 0;
|
|
3487
|
+
/** The run's judge model: the profile's when it sets one, else the config's. */
|
|
3488
|
+
function runJudgeModel(config, profile) {
|
|
3489
|
+
return profile ? profileJudge(profile).model ?? config.judge.model : config.judge.model;
|
|
3490
|
+
}
|
|
3491
|
+
/** The gateway a model the run calls itself goes through, as the plan resolves it. */
|
|
3492
|
+
function runModelGateway(config, profile, model, judge = false) {
|
|
3493
|
+
return modelGatewayOrNone(model, judge ? profile ? profileJudge(profile).gateway ?? config.judge.gateway : config.judge.gateway : void 0, gatewayScope(config, profile));
|
|
3494
|
+
}
|
|
3495
|
+
/**
|
|
3496
|
+
* Names the environment variables the run's providers need but don't have: the harnesses it
|
|
3497
|
+
* plans (default: the config's), the run's judge, the models that answer questions,
|
|
3498
|
+
* and required config and profile secrets. Insights' model isn't among them: without its key a run
|
|
3499
|
+
* prints no insights, and never stops for it.
|
|
3500
|
+
*/
|
|
3501
|
+
function missingCredentials(config, env = process.env, options = {}) {
|
|
3502
|
+
const missing = [];
|
|
3503
|
+
const keys = new Set((options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []));
|
|
3504
|
+
for (const key of keys) if (!env[key]) missing.push(key);
|
|
3505
|
+
const judge = runJudgeModel(config, options.profile);
|
|
3506
|
+
const models = [...judge !== void 0 ? [{
|
|
3507
|
+
model: judge,
|
|
3508
|
+
what: "the judge model",
|
|
3509
|
+
judge: true
|
|
3510
|
+
}] : [], ...(options.questionModels ?? []).map((model) => ({
|
|
3511
|
+
model,
|
|
3512
|
+
what: "questions",
|
|
3513
|
+
judge: false
|
|
3514
|
+
}))];
|
|
3515
|
+
const named = /* @__PURE__ */ new Set();
|
|
3516
|
+
for (const { model, what, judge: isJudge } of models) {
|
|
3517
|
+
if (typeof model !== "string" || named.has(model)) continue;
|
|
3518
|
+
named.add(model);
|
|
3519
|
+
const gateway = runModelGateway(config, options.profile, model, isJudge);
|
|
3520
|
+
if (!judgeCredentials(model, gateway).some((k) => env[k])) missing.push(`${modelKeyOptions(model, gateway)} for ${what}, "${model}"`);
|
|
3521
|
+
}
|
|
3522
|
+
const vars = {
|
|
3523
|
+
...config.env,
|
|
3524
|
+
...options.profile?.defaults.env
|
|
3525
|
+
};
|
|
3526
|
+
for (const [name, from] of secretSources(vars)) if (!env[from]) missing.push(from === name ? name : `${from} (for ${name})`);
|
|
3527
|
+
return missing;
|
|
3528
|
+
}
|
|
3529
|
+
/**
|
|
3530
|
+
* Keys the run sends in request headers (the agents' credentials, the run's judge's and the
|
|
3531
|
+
* questions' models') whose value has
|
|
3532
|
+
* whitespace or a control character in it, with where the first one is: a key pasted from a
|
|
3533
|
+
* wrapped terminal line picks up a line break, and the agent then hangs on a request it can't send
|
|
3534
|
+
* rather than failing to authenticate.
|
|
3535
|
+
*/
|
|
3536
|
+
function malformedCredentials(config, env = process.env, options = {}) {
|
|
3537
|
+
const names = (options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []);
|
|
3538
|
+
const judge = runJudgeModel(config, options.profile);
|
|
3539
|
+
if (judge !== void 0) names.push(...judgeCredentials(judge, runModelGateway(config, options.profile, judge, true)));
|
|
3540
|
+
for (const model of options.questionModels ?? []) names.push(...judgeCredentials(model, runModelGateway(config, options.profile, model)));
|
|
3541
|
+
return [...new Set(names)].flatMap((name) => {
|
|
3542
|
+
const at = badCharacterAt(env[name] ?? "");
|
|
3543
|
+
return at ? [`${name} (character ${at})`] : [];
|
|
3544
|
+
});
|
|
3545
|
+
}
|
|
3546
|
+
//#endregion
|
|
3547
|
+
export { DEFAULT_QUESTION_STATE as $, matchesTags as A, mergeJudge as B, brokeredIn as C, judgeStateSchema as Ct, expand as D, filePartsOf as Dt, evalPlan as E, decideState as Et, sandboxProvider as F, ciRunner as G, DEFAULT_EGRESS as H, sandboxSetBy as I, gatewayPolicy as J, directOnCloud as K, secretVarsIn as L, profileJudge as M, refuseIllegalCompare as N, harnessId as O, resolveJudgeArtifacts as Ot, sandboxId as P, createFakeSandbox as Q, unbrokered as R, routedHarness as S, judgeProblem as St, checkOptions as T, badJudgeArtifacts as Tt, DEFAULT_EGRESS_ALLOW as U, resolveStateOptions as V, allowedHosts as W, unenforceable as X, isDirect as Y, resolveOutput as Z, selectProfile as _, writePluginFiles as _t, configPathFor as a, servedEffortOf as at, fallbackHint as b, judgeFields as bt, findConfig as c, agentMcp as ct, loadConfig as d, claudeMcpServers as dt, STOPPED_FIRST as et, malformedCredentials as f, credentialOf as ft, runJudgeModel as g, refusedPlugins as gt, resolveConcurrency as h, refusedMcp as ht, configLabel as i, resolvedEffortOf as it, plannedSandboxes as j, levelOf as k, hasAnalysisModel as l, authHint as lt, requireConfig as m, layoutOf as mt, analysisModelOf as n, resolveQuestionState as nt, defaultConcurrency as o, agentAuth as ot, missingCredentials as p, expectedBlockedFor as pt, egressLabel as q, configFilesPhrase as r, harnessEffort as rt, defineConfig as s, agentFor as st, CONFIG_FILES as t, questionShows as tt, hostCapacity as u, brokeringFor as ut, gatewayScope as v, captureWindowSchema as vt, cascade as w, artifactRefsOf as wt, resolveGateway as x, judgeModelNameSchema as xt, resolveModelGateway as y, fakeJudgeDataSchema as yt, unbrokeredError as z };
|
|
3548
|
+
|
|
3549
|
+
//# sourceMappingURL=config-BrdHA9bJ.js.map
|