reeval 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/{adapter-CptK032x.js → adapter-C8xPuXYq.js} +5 -4
- package/dist/adapter-C8xPuXYq.js.map +1 -0
- package/dist/{agent-B63OzUX9.js → agent-CBoLndM7.js} +30 -10
- package/dist/{agent-B63OzUX9.js.map → agent-CBoLndM7.js.map} +1 -1
- package/dist/agents-C6zhkxp3.js +1151 -0
- package/dist/agents-C6zhkxp3.js.map +1 -0
- package/dist/{app-files-sgqwfjtn.js → app-files-DT5mJyV4.js} +3 -3
- package/dist/app-files-DT5mJyV4.js.map +1 -0
- package/dist/assemble-UPCRrtOz.js +1 -0
- package/dist/{assemble-Me7Whp-y.js → assemble-t6h018An.js} +152 -129
- package/dist/assemble-t6h018An.js.map +1 -0
- package/dist/{attempt-Dp7Wtm8M.js → attempt-CAdlwlNu.js} +2 -2
- package/dist/{attempt-Dp7Wtm8M.js.map → attempt-CAdlwlNu.js.map} +1 -1
- package/dist/{attempt-host-Cik0SoIn.js → attempt-host-CIm1-11m.js} +6 -6
- package/dist/{attempt-host-Cik0SoIn.js.map → attempt-host-CIm1-11m.js.map} +1 -1
- package/dist/boat-BjLqFJIZ.js +108 -0
- package/dist/boat-BjLqFJIZ.js.map +1 -0
- package/dist/boat-CJCtzfB1.js +322 -0
- package/dist/boat-CJCtzfB1.js.map +1 -0
- package/dist/bridge/index.mjs +300 -78
- package/dist/capture-DoxFUN3J.js +1 -0
- package/dist/{capture-C3RmjPph.js → capture-iBzFiC0j.js} +27 -25
- package/dist/capture-iBzFiC0j.js.map +1 -0
- package/dist/cases-Dy0mifal.js +1 -0
- package/dist/cases-z9E6DCd5.js +639 -0
- package/dist/cases-z9E6DCd5.js.map +1 -0
- package/dist/{cli-BBOlXKPG.js → cli-20Fpvs5H.js} +442 -707
- package/dist/cli-20Fpvs5H.js.map +1 -0
- package/dist/{cli-spec-DZ0bnP0_.js → cli-spec-Dsr79Qy1.js} +38 -11
- package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-tZ2WBDxf.js → client-KM4C5r5_.js} +93 -3
- package/dist/client-KM4C5r5_.js.map +1 -0
- package/dist/{command-qGS802uX.js → command-B5OyHpOu.js} +21 -80
- package/dist/command-B5OyHpOu.js.map +1 -0
- package/dist/{command-DhiwsZl1.js → command-BLNVDiIg.js} +9 -9
- package/dist/{command-DhiwsZl1.js.map → command-BLNVDiIg.js.map} +1 -1
- package/dist/{command-fO0BFsTq.js → command-CRh-cb6Z.js} +37 -37
- package/dist/command-CRh-cb6Z.js.map +1 -0
- package/dist/{command-CFQ9c1hk.js → command-aUJvENwy.js} +16 -16
- package/dist/command-aUJvENwy.js.map +1 -0
- package/dist/{commands-CC60TOT5.js → commands-CWy-WndG.js} +186 -85
- package/dist/commands-CWy-WndG.js.map +1 -0
- package/dist/{commands-xYTMNTPv.js → commands-CfdBX-mc.js} +1 -1
- package/dist/{config-C2UtS9vy.js → config-B9mB1kF5.js} +261 -1708
- package/dist/config-B9mB1kF5.js.map +1 -0
- package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
- package/dist/context-Bkv3ndjq.js.map +1 -0
- package/dist/{cost-BuVDamui.js → cost-BdRgySQl.js} +2 -2
- package/dist/{cost-BuVDamui.js.map → cost-BdRgySQl.js.map} +1 -1
- package/dist/cost-Cr26Alhq.js +1 -0
- package/dist/{cursor-catalog-BTK_apL-.js → cursor-catalog-BSn9xjmo.js} +3 -3
- package/dist/{cursor-catalog-BTK_apL-.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
- package/dist/db-P2fXw4nx.js +2 -0
- package/dist/{db-C5SZ0510.js → db-zgC1tvKB.js} +130 -84
- package/dist/db-zgC1tvKB.js.map +1 -0
- package/dist/debug-CF4p4Lp_.js +1 -0
- package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
- package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
- package/dist/{default-effort-FEidNAIi.js → default-effort-BAy8ixd1.js} +1 -1
- package/dist/{default-effort-BVmx7r7L.js → default-effort-DGhQmu95.js} +4 -4
- package/dist/{default-effort-BVmx7r7L.js.map → default-effort-DGhQmu95.js.map} +1 -1
- package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
- package/dist/dist-CAFD1yfv.js.map +1 -0
- package/dist/{docker-B2F8-PGC.js → docker-Cqw-CyWz.js} +3 -3
- package/dist/{docker-B2F8-PGC.js.map → docker-Cqw-CyWz.js.map} +1 -1
- package/dist/{docker-Dto8ROCq.js → docker-DAMnm3L9.js} +103 -73
- package/dist/docker-DAMnm3L9.js.map +1 -0
- package/dist/{docker-BlSopHpZ.js → docker-Dr4XeDc8.js} +1 -1
- package/dist/{doctor-BcvfKpBX.js → doctor-B2PohneB.js} +84 -27
- package/dist/doctor-B2PohneB.js.map +1 -0
- package/dist/{environment-CC1RV21n.js → environment-DB8o3hGi.js} +79 -39
- package/dist/environment-DB8o3hGi.js.map +1 -0
- package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
- package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
- package/dist/{eval-source-CzxdgFBW.js → eval-source-D3sbAzt-.js} +10 -10
- package/dist/{eval-source-CzxdgFBW.js.map → eval-source-D3sbAzt-.js.map} +1 -1
- package/dist/{evaluate-DToSB2bt.d.ts → evaluate-COdzf7Hl.d.ts} +17 -26
- package/dist/{evaluate-NTNJ8T9H.js → evaluate-CwKeN_fo.js} +377 -142
- package/dist/evaluate-CwKeN_fo.js.map +1 -0
- package/dist/files-B4KQUiM7.js +1 -0
- package/dist/files-BEwfjD8J.js +64 -0
- package/dist/files-BEwfjD8J.js.map +1 -0
- package/dist/{files-DsXULUi5.js → files-BRzV7aPh.js} +5 -5
- package/dist/{files-DsXULUi5.js.map → files-BRzV7aPh.js.map} +1 -1
- package/dist/{files-aAzoGNCB.js → files-BiFU16BW.js} +5 -5
- package/dist/{files-aAzoGNCB.js.map → files-BiFU16BW.js.map} +1 -1
- package/dist/files-C1Rk9FYA.js +1 -0
- package/dist/files-C3mCC8Mn.js +1 -0
- package/dist/{files-CZXrkPja.js → files-Cl7BQ5gE.js} +54 -13
- package/dist/{files-CZXrkPja.js.map → files-Cl7BQ5gE.js.map} +1 -1
- package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
- package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
- package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
- package/dist/files-DlQshsks.js.map +1 -0
- package/dist/files-DnlA1A7N.js +1 -0
- package/dist/gate-CgYdaprG.js +49 -0
- package/dist/gate-CgYdaprG.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-C1MB5bjr.js → git-fDiJ8UIV.js} +2 -2
- package/dist/{git-C1MB5bjr.js.map → git-fDiJ8UIV.js.map} +1 -1
- package/dist/{harness-DGsRZDSo.js → harness-C2berMiM.js} +3 -3
- package/dist/{harness-DGsRZDSo.js.map → harness-C2berMiM.js.map} +1 -1
- package/dist/{harness-DYSC4zMh.js → harness-DXv2gnaT.js} +4 -2
- package/dist/harness-DXv2gnaT.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
- package/dist/ids-C_advdJA.js.map +1 -0
- package/dist/{images-CNayXCp2.js → images-BNgcR77K.js} +98 -10
- package/dist/images-BNgcR77K.js.map +1 -0
- package/dist/index.d.ts +23 -3
- package/dist/index.js +5 -5
- package/dist/init-4RPeD-K-.js +1 -0
- package/dist/{init-D5HtmyMI.js → init-DbuZVRrW.js} +7 -6
- package/dist/init-DbuZVRrW.js.map +1 -0
- package/dist/{ink-list-Ca-5dunO.js → ink-list-BpfidQ1h.js} +121 -31
- package/dist/ink-list-BpfidQ1h.js.map +1 -0
- package/dist/{invocations-DrGXF9dT.js → invocations-CxGNOlh0.js} +3 -3
- package/dist/{invocations-DrGXF9dT.js.map → invocations-CxGNOlh0.js.map} +1 -1
- package/dist/judge-COMXT1Hx.js +230 -0
- package/dist/judge-COMXT1Hx.js.map +1 -0
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-m-mb0zv8.js → judgments-CzgpsTqv.js} +2 -2
- package/dist/{judgments-m-mb0zv8.js.map → judgments-CzgpsTqv.js.map} +1 -1
- package/dist/kinds-C1oY3qyC.js +697 -0
- package/dist/kinds-C1oY3qyC.js.map +1 -0
- package/dist/{summary-Uzc9osfV.js → layout-Cm85tQYk.js} +39 -150
- package/dist/layout-Cm85tQYk.js.map +1 -0
- package/dist/list-DuThsE28.js +1348 -0
- package/dist/list-DuThsE28.js.map +1 -0
- package/dist/live-CJCK11nh.js +1 -0
- package/dist/{live-B71o2TkX.js → live-CgYbO4Te.js} +6 -4
- package/dist/{live-B71o2TkX.js.map → live-CgYbO4Te.js.map} +1 -1
- package/dist/{live-uRURFDMZ.js → live-CnGrRM6j.js} +7 -7
- package/dist/{live-uRURFDMZ.js.map → live-CnGrRM6j.js.map} +1 -1
- package/dist/{matchers-Cs8aDqdU.js → matchers-CXLn-1jm.js} +3 -2
- package/dist/{matchers-Cs8aDqdU.js.map → matchers-CXLn-1jm.js.map} +1 -1
- package/dist/{model-CXVVB4RK.js → model-047Dcd-a.js} +1 -1
- package/dist/model-Cv4GuTsZ.js +1 -0
- package/dist/{model-1L19Zukh.js → model-CwwkrCb7.js} +5 -5
- package/dist/{model-1L19Zukh.js.map → model-CwwkrCb7.js.map} +1 -1
- package/dist/{model-BxBWFOZn.js → model-DJjLw-0r.js} +5 -5
- package/dist/{model-BxBWFOZn.js.map → model-DJjLw-0r.js.map} +1 -1
- package/dist/{model-Ce9l9_rF.js → model-V3mMXp6x.js} +203 -313
- package/dist/model-V3mMXp6x.js.map +1 -0
- package/dist/{node-DhIRSrrK.js → node-D7pAU--0.js} +14 -10
- package/dist/node-D7pAU--0.js.map +1 -0
- package/dist/{opencode-providers-Co7plCea.js → opencode-providers-BGiXt8_B.js} +104 -35
- package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
- package/dist/{openrouter-models-DNvLRYM7.js → openrouter-models-Cw1z8-sq.js} +2 -2
- package/dist/{openrouter-models-DNvLRYM7.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
- package/dist/options-CHubslQf.js +1 -0
- package/dist/{gate-Cp_pQoEe.js → pairs-BGb134Vu.js} +2 -48
- package/dist/pairs-BGb134Vu.js.map +1 -0
- package/dist/{patch-ug5HpMOj.js → patch-DH4e_Gkd.js} +2 -2
- package/dist/{patch-ug5HpMOj.js.map → patch-DH4e_Gkd.js.map} +1 -1
- package/dist/plain-BYUf_fUV.js +267 -0
- package/dist/plain-BYUf_fUV.js.map +1 -0
- package/dist/{plan-DqnzfDH1.js → plan-Cgq9NCEX.js} +110 -39
- package/dist/plan-Cgq9NCEX.js.map +1 -0
- package/dist/{plugins-B2KQ0y3P.js → plugins-BkmPxDiY.js} +10 -267
- package/dist/plugins-BkmPxDiY.js.map +1 -0
- package/dist/preflight-4gWSy0Pe.js +1 -0
- package/dist/{preflight-DyzmvK7i.js → preflight-BElE_q0u.js} +16 -9
- package/dist/preflight-BElE_q0u.js.map +1 -0
- package/dist/preflight-CtlloeOQ.js +68 -0
- package/dist/preflight-CtlloeOQ.js.map +1 -0
- package/dist/{preflight-DVXEvzFU.js → preflight-R-InZCQb.js} +1 -1
- package/dist/{prepare-21WMQZyt.js → prepare-C7f5mPl5.js} +9 -9
- package/dist/{prepare-21WMQZyt.js.map → prepare-C7f5mPl5.js.map} +1 -1
- package/dist/{prepare-BcSRa7-3.js → prepare-CzkAdbXm.js} +2 -2
- package/dist/{prices-DYJA0tJN.js → prices-5m3SzMZ3.js} +2 -2
- package/dist/{prices-DYJA0tJN.js.map → prices-5m3SzMZ3.js.map} +1 -1
- package/dist/{provenance-iRxjp4Pu.js → provenance-zsc4--dP.js} +4 -3
- package/dist/provenance-zsc4--dP.js.map +1 -0
- package/dist/provider-BnUoz4KB.js +1 -0
- package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
- package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
- package/dist/{providers-vXMXuwn-.js → providers-B8xYMeah.js} +152 -9
- package/dist/providers-B8xYMeah.js.map +1 -0
- package/dist/providers-CVu8e0xS.js +1 -0
- package/dist/{query-BhaKQ07P.js → query-Bb3SYICk.js} +2 -2
- package/dist/{query-BhaKQ07P.js.map → query-Bb3SYICk.js.map} +1 -1
- package/dist/query-DIfkUcPh.js +1 -0
- package/dist/{questions-BzdLdV3C.js → questions-CWlR9ffu.js} +9 -9
- package/dist/{questions-BzdLdV3C.js.map → questions-CWlR9ffu.js.map} +1 -1
- package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
- package/dist/record-6IUzwP9K.js.map +1 -0
- package/dist/{remarks-NMgPd30U.js → remarks-DPCONFxZ.js} +71 -50
- package/dist/remarks-DPCONFxZ.js.map +1 -0
- package/dist/{remarks-CauU408t.js → remarks-vtkPmQb9.js} +115 -35
- package/dist/remarks-vtkPmQb9.js.map +1 -0
- package/dist/{remote-Dwy3fyo3.js → remote-BgYWqogI.js} +6 -6
- package/dist/{remote-Dwy3fyo3.js.map → remote-BgYWqogI.js.map} +1 -1
- package/dist/{resolve-B1zMnPwK.js → resolve-Do0ZaYH6.js} +10 -9
- package/dist/resolve-Do0ZaYH6.js.map +1 -0
- package/dist/resolve-amw_2Odz.js +1 -0
- package/dist/{results-CfC0id_u.js → results-JyuM64gy.js} +6 -6
- package/dist/{results-CfC0id_u.js.map → results-JyuM64gy.js.map} +1 -1
- package/dist/{resume-Dkhrpjow.js → resume-AW5oXzUc.js} +169 -27
- package/dist/resume-AW5oXzUc.js.map +1 -0
- package/dist/retry-ChzISxmy.js +105 -0
- package/dist/retry-ChzISxmy.js.map +1 -0
- package/dist/{runtime-DcvQcr36.js → runtime-BpW1f1Mx.js} +26 -7
- package/dist/runtime-BpW1f1Mx.js.map +1 -0
- package/dist/sandbox/boat.d.ts +26 -0
- package/dist/sandbox/boat.js +37 -0
- package/dist/sandbox/boat.js.map +1 -0
- package/dist/{sandbox-8zCrtuCf.js → sandbox-kSgr7vo5.js} +31 -3
- package/dist/sandbox-kSgr7vo5.js.map +1 -0
- package/dist/sandbox.d.ts +20 -2
- package/dist/sandbox.js +1 -1
- package/dist/server-2_gUnVvf.js +1 -0
- package/dist/{server-Dplq8g0r.js → server-Cn8VqO9S.js} +9 -9
- package/dist/{server-Dplq8g0r.js.map → server-Cn8VqO9S.js.map} +1 -1
- package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
- package/dist/shared-Cfi45Bvy.js.map +1 -0
- package/dist/slug-IHh3f770.js +136 -0
- package/dist/slug-IHh3f770.js.map +1 -0
- package/dist/source-CY3etKzw.js +1 -0
- package/dist/{source-NWHqn5-o.js → source-CYOhR98B.js} +2 -2
- package/dist/{source-NWHqn5-o.js.map → source-CYOhR98B.js.map} +1 -1
- package/dist/src-6Zzyg4Qd.js +2 -0
- package/dist/{src-B9oE_mNS.js → src-BDhcPtnI.js} +2 -2
- package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
- package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
- package/dist/src-BRMsk9nh.js +2 -0
- package/dist/{src-BQXdf-Gw.js → src-CV_7DkPa.js} +47 -45
- package/dist/{src-BQXdf-Gw.js.map → src-CV_7DkPa.js.map} +1 -1
- package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
- package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
- package/dist/{src-BQ6BiXoc.js → src-Ck2r_EMT.js} +607 -72
- package/dist/src-Ck2r_EMT.js.map +1 -0
- package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
- package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
- package/dist/src-DB7N8TYJ.js +2 -0
- package/dist/src-DPDaAYwh.js +2 -0
- package/dist/{src-CUpoVVb8.js → src-Dbk19BiP.js} +2 -2
- package/dist/src-DtodMT59.js +1 -0
- package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
- package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
- package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
- package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
- package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
- package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
- package/dist/src-T-OeuJLr.js +2 -0
- package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
- package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
- package/dist/src-zntEDlnx.js +3 -0
- package/dist/{state-DDCGUhva.js → state-BOLt8pJg.js} +17 -17
- package/dist/{state-DDCGUhva.js.map → state-BOLt8pJg.js.map} +1 -1
- package/dist/state-kimtIhBP.js +1 -0
- package/dist/suite-CKBa6MKt.js +2 -0
- package/dist/{suite-M8_7xio7.js → suite-COglUhPd.js} +70 -92
- package/dist/suite-COglUhPd.js.map +1 -0
- package/dist/summary-DU5zaibx.js +148 -0
- package/dist/summary-DU5zaibx.js.map +1 -0
- package/dist/tart-BTVXp8rz.js +373 -0
- package/dist/tart-BTVXp8rz.js.map +1 -0
- package/dist/tart-cli-BIC9HqPg.js +37 -0
- package/dist/tart-cli-BIC9HqPg.js.map +1 -0
- package/dist/tart-cli-BMrkmBjI.js +1 -0
- package/dist/tart-images-DZ_Mbh-g.js +233 -0
- package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
- package/dist/tart-ref-7Qv3cm3B.js +1 -0
- package/dist/tart-ref-DYZijjrU.js +55 -0
- package/dist/tart-ref-DYZijjrU.js.map +1 -0
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +4 -4
- package/dist/trace-Dx2NBE1O.js +311 -0
- package/dist/trace-Dx2NBE1O.js.map +1 -0
- package/dist/traits-DKnYkqeK.js +66 -0
- package/dist/traits-DKnYkqeK.js.map +1 -0
- package/dist/{transcript-Cg4SeqRX.js → transcript-CCf07UmU.js} +2 -2
- package/dist/{transcript-Cg4SeqRX.js.map → transcript-CCf07UmU.js.map} +1 -1
- package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
- package/dist/types-C2mLpph3.js.map +1 -0
- package/dist/{types-DGvexkcp.d.ts → types-DBwFsGem.d.ts} +352 -76
- package/dist/{judge-DYOF69Oe.js → vercel-CGlSBv63.js} +3 -229
- package/dist/vercel-CGlSBv63.js.map +1 -0
- package/dist/{vertex-BLQMayc1.js → vertex-Eb2wqahk.js} +2 -2
- package/dist/{vertex-BLQMayc1.js.map → vertex-Eb2wqahk.js.map} +1 -1
- package/dist/vertex-O88TCwgY.js +1 -0
- package/dist/view/index.d.ts +101 -58
- package/dist/view/index.js +9 -8
- package/dist/{views-BISrsIjw.js → views-DAIiLiV3.js} +7 -6
- package/dist/views-DAIiLiV3.js.map +1 -0
- package/dist/views-DtRhj2Tn.js +2 -0
- package/dist/{worker-entry-DJD_nHkS.js → worker-entry-BIROTPX1.js} +11 -8
- package/dist/{worker-entry-DJD_nHkS.js.map → worker-entry-BIROTPX1.js.map} +1 -1
- package/dist/workspace-CcQRT31C.js +1 -0
- package/dist/{workspace-BO1VhUR9.js → workspace-uuE51x35.js} +6 -6
- package/dist/{workspace-BO1VhUR9.js.map → workspace-uuE51x35.js.map} +1 -1
- package/package.json +24 -11
- package/skills/eval-rubric/SKILL.md +15 -7
- package/dist/adapter-CptK032x.js.map +0 -1
- package/dist/app-files-sgqwfjtn.js.map +0 -1
- package/dist/assemble-CxBHkavM.js +0 -1
- package/dist/assemble-Me7Whp-y.js.map +0 -1
- package/dist/capture-C3RmjPph.js.map +0 -1
- package/dist/capture-YPwxp_iK.js +0 -1
- package/dist/cli-BBOlXKPG.js.map +0 -1
- package/dist/cli-spec-DZ0bnP0_.js.map +0 -1
- package/dist/client-tZ2WBDxf.js.map +0 -1
- package/dist/command-CFQ9c1hk.js.map +0 -1
- package/dist/command-fO0BFsTq.js.map +0 -1
- package/dist/command-qGS802uX.js.map +0 -1
- package/dist/commands-CC60TOT5.js.map +0 -1
- package/dist/config-C2UtS9vy.js.map +0 -1
- package/dist/context-BajfiH0S.js.map +0 -1
- package/dist/contrasts-BjOg7wyr.js +0 -101
- package/dist/contrasts-BjOg7wyr.js.map +0 -1
- package/dist/cost-BJU0MAji.js +0 -1
- package/dist/db-C5SZ0510.js.map +0 -1
- package/dist/db-puzLzX4k.js +0 -2
- package/dist/debug-BnXV7JbX.js +0 -1
- package/dist/dist-ByYk_wAs.js.map +0 -1
- package/dist/docker-Dto8ROCq.js.map +0 -1
- package/dist/doctor-BcvfKpBX.js.map +0 -1
- package/dist/environment-CC1RV21n.js.map +0 -1
- package/dist/evaluate-NTNJ8T9H.js.map +0 -1
- package/dist/files-COW5JBJM.js +0 -1
- package/dist/files-DFtuRQJx.js +0 -1
- package/dist/files-oOYNR7ZH.js +0 -1
- package/dist/files-p-uTdOjA.js.map +0 -1
- package/dist/gate-Cp_pQoEe.js.map +0 -1
- package/dist/harness-DYSC4zMh.js.map +0 -1
- package/dist/ids-DuK6xl4w.js.map +0 -1
- package/dist/images-CNayXCp2.js.map +0 -1
- package/dist/init-8o6W3_fn.js +0 -1
- package/dist/init-D5HtmyMI.js.map +0 -1
- package/dist/ink-list-Ca-5dunO.js.map +0 -1
- package/dist/judge-DYOF69Oe.js.map +0 -1
- package/dist/list-CM_YSJXq.js +0 -374
- package/dist/list-CM_YSJXq.js.map +0 -1
- package/dist/live-DQc8jr6o.js +0 -1
- package/dist/model-Ce9l9_rF.js.map +0 -1
- package/dist/model-Cxzo4cjw.js +0 -1
- package/dist/node-DhIRSrrK.js.map +0 -1
- package/dist/opencode-providers-Co7plCea.js.map +0 -1
- package/dist/plan-DqnzfDH1.js.map +0 -1
- package/dist/plugins-B2KQ0y3P.js.map +0 -1
- package/dist/preflight-DyzmvK7i.js.map +0 -1
- package/dist/provenance-iRxjp4Pu.js.map +0 -1
- package/dist/provider-DCdTHQpk.js +0 -1
- package/dist/providers-D0ob_tra.js +0 -1
- package/dist/providers-vXMXuwn-.js.map +0 -1
- package/dist/query-CvgUsvep.js +0 -1
- package/dist/record-DhzwkSXc.js.map +0 -1
- package/dist/remarks-CauU408t.js.map +0 -1
- package/dist/remarks-NMgPd30U.js.map +0 -1
- package/dist/resolve-B1zMnPwK.js.map +0 -1
- package/dist/resolve-BIIeVT12.js +0 -1
- package/dist/resume-Dkhrpjow.js.map +0 -1
- package/dist/runtime-DcvQcr36.js.map +0 -1
- package/dist/sandbox-8zCrtuCf.js.map +0 -1
- package/dist/server-0tuj8VSa.js +0 -1
- package/dist/shared-ogqPJ0uo.js.map +0 -1
- package/dist/source-llwLewOU.js +0 -1
- package/dist/src-BQ6BiXoc.js.map +0 -1
- package/dist/src-BXnv0ViU.js +0 -2
- package/dist/src-BiS8YNN_.js +0 -3
- package/dist/src-BkH4GOxu.js +0 -1
- package/dist/src-C-tH5ebd.js +0 -2
- package/dist/src-Cg1bFs9g.js +0 -2
- package/dist/src-DV_UorZv.js +0 -2
- package/dist/src-XxUpr2sa.js +0 -2
- package/dist/state-DZbAO95a.js +0 -1
- package/dist/suite-M8_7xio7.js.map +0 -1
- package/dist/suite-u2KJnGpY.js +0 -2
- package/dist/summary-Uzc9osfV.js.map +0 -1
- package/dist/types-C4OkYhZg.js.map +0 -1
- package/dist/vertex-h_peaZTN.js +0 -1
- package/dist/views-BISrsIjw.js.map +0 -1
- package/dist/views-Bfab8fM9.js +0 -2
- package/dist/workspace-B8EOCuxl.js +0 -1
|
@@ -1,33 +1,33 @@
|
|
|
1
|
-
import { n as isOneOf, t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
2
1
|
import { n as UsageError } from "./errors-ttMTnIYX.js";
|
|
3
|
-
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { a as
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
2
|
+
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as tokenUsageSchema, g as parseDuration, h as TIMER_LIMIT, i as durationSchema, l as nonEmpty$1, m as MAX_TIMER_MS, n as dataProblem, o as genericProblem, p as valueAt, s as got, t as closest, u as oneOf } from "./shared-Cfi45Bvy.js";
|
|
3
|
+
import { n as isOneOf, t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
4
|
+
import { _ as snapshotProblem, a as egressPolicySchema, c as gatePolicySchema, g as snapshotOptionSchema, l as gateProblem, s as envVarsSchema } from "./plain-BYUf_fUV.js";
|
|
5
|
+
import { a as isSandboxKind, d as sandboxKind, f as suggestedSandbox, i as idPart, n as capabilitiesOf, o as maxConcurrentProblem, t as SANDBOX_KINDS, u as sandboxHelpers } from "./kinds-C1oY3qyC.js";
|
|
6
|
+
import { C as plainCredentialHeaders, D as sandboxOf, E as sameWorkspaceOnly, F as environmentDataSchema, I as sandboxDescriptorSchema, L as traitsSchema, M as unbrokeredSecrets, N as workspaceLabel, O as sandboxPaths, P as workspaceOf, _ as isMovingRef, a as checkEgress, c as environmentDataProblem, h as isEnvironment, j as traitsOf, l as environmentFrom, m as helpersFor, o as checkEnv, r as applied, u as environmentPaths, v as isSecret, w as plannedSecrets, z as sandboxProblem } from "./environment-DB8o3hGi.js";
|
|
7
|
+
import { c as GATEWAY_NAMES, d as gatewayEntryDataSchema, f as gatewayNameSchema, i as claudeCodeTakesEffort, l as directOffPin, o as harnessDescriptorSchema, r as claudeCodeModelName, s as harnessProblem, t as FACTORY, u as gatewayDescriptorSchema } from "./harness-DXv2gnaT.js";
|
|
8
|
+
import { c as sha256, i as fromCwd } from "./ids-C_advdJA.js";
|
|
9
|
+
import { n as openCodeEffortBody, r as openCodeEffortRefusal, u as resolveOpenCodeProvider } from "./opencode-providers-BGiXt8_B.js";
|
|
10
|
+
import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV } from "./vertex-Eb2wqahk.js";
|
|
11
11
|
import { _ as redactValue, h as redactSecrets, r as badCharacterAt, t as REDACTED } from "./redact-BhjtnjrU.js";
|
|
12
|
-
import { o as
|
|
12
|
+
import { a as RECOMMENDED_JUDGE_MODEL, c as judgeCredentials, d as modelKeyOptions, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL } from "./judge-COMXT1Hx.js";
|
|
13
13
|
import { t as inCI } from "./env-8x1O8I1M.js";
|
|
14
14
|
import { u as ownFileURL } from "./root-Co0jhOT5.js";
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
import { n as
|
|
18
|
-
import {
|
|
15
|
+
import { B as PACKAGE_HOSTS, z as HARNESS_HOSTS } from "./src-Ck2r_EMT.js";
|
|
16
|
+
import { y as localPluginServers } from "./plugins-BkmPxDiY.js";
|
|
17
|
+
import { a as agentFor, b as isRoutingGateway, f as isAgentKind, n as DEFAULT_HARNESS, r as HARNESS_EXAMPLE, t as AGENTS, u as credentialOf, y as ROUTING_GATEWAYS } from "./agents-C6zhkxp3.js";
|
|
18
|
+
import { i as pairFactors } from "./pairs-BGb134Vu.js";
|
|
19
|
+
import { u as opencodeModelError } from "./harness-C2berMiM.js";
|
|
20
|
+
import { r as tartTagWarning } from "./tart-ref-DYZijjrU.js";
|
|
19
21
|
import "./stats-C18W4xPX.js";
|
|
20
|
-
import { n as
|
|
21
|
-
import {
|
|
22
|
+
import { n as gateLabel } from "./gate-CgYdaprG.js";
|
|
23
|
+
import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-C8xPuXYq.js";
|
|
24
|
+
import { t as docker } from "./sandbox-kSgr7vo5.js";
|
|
25
|
+
import { z } from "zod";
|
|
22
26
|
import { existsSync, readFileSync } from "node:fs";
|
|
23
|
-
import { availableParallelism,
|
|
24
|
-
import { dirname, join, resolve
|
|
27
|
+
import { availableParallelism, totalmem } from "node:os";
|
|
28
|
+
import { dirname, join, resolve } from "node:path";
|
|
25
29
|
import { parseDocument } from "yaml";
|
|
26
|
-
import {
|
|
27
|
-
import { connect, isIP } from "node:net";
|
|
28
|
-
import { createHash, randomUUID } from "node:crypto";
|
|
29
|
-
import { spawn } from "node:child_process";
|
|
30
|
-
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
30
|
+
import { isIP } from "node:net";
|
|
31
31
|
// src/judge/artifacts.ts
|
|
32
32
|
/** A capture handle: a name plus the file the attempt wrote. */
|
|
33
33
|
function isCaptureHandle(value) {
|
|
@@ -403,6 +403,7 @@ const captureOptionsSchema = z.strictObject({
|
|
|
403
403
|
model: z.string().includes("/").optional()
|
|
404
404
|
});
|
|
405
405
|
const harnessList = z.union([harnessDescriptorSchema, z.array(harnessDescriptorSchema).min(1)]);
|
|
406
|
+
const environmentList = z.union([environmentDataSchema, z.array(environmentDataSchema).min(1)]);
|
|
406
407
|
const budgetSchema = z.strictObject({
|
|
407
408
|
attempts: count().optional(),
|
|
408
409
|
duration: durationSchema.optional()
|
|
@@ -421,9 +422,9 @@ const recordMode = z.enum([
|
|
|
421
422
|
"always"
|
|
422
423
|
]);
|
|
423
424
|
const profileDefaultsDataSchema = z.strictObject({
|
|
424
|
-
|
|
425
|
+
harnesses: harnessList.optional(),
|
|
425
426
|
sandbox: sandboxDescriptorSchema.optional(),
|
|
426
|
-
|
|
427
|
+
environments: environmentList.optional(),
|
|
427
428
|
runs: runs().optional(),
|
|
428
429
|
gate: gatePolicySchema.optional(),
|
|
429
430
|
env: envVarsSchema.optional(),
|
|
@@ -437,7 +438,8 @@ const profileDefaultsDataSchema = z.strictObject({
|
|
|
437
438
|
const profileLimitSchema = z.strictObject({
|
|
438
439
|
runs: count().optional(),
|
|
439
440
|
compare: z.union([z.literal(false), z.strictObject({ maxVariants: z.number().int().min(0) })]).optional(),
|
|
440
|
-
|
|
441
|
+
harnesses: z.array(z.string()).optional(),
|
|
442
|
+
traits: z.record(z.string(), z.union([z.string(), z.array(z.string()).min(1)])).optional()
|
|
441
443
|
});
|
|
442
444
|
const profileDataSchema = z.strictObject({
|
|
443
445
|
defaults: profileDefaultsDataSchema.optional(),
|
|
@@ -482,9 +484,9 @@ const analysisConfigDataSchema = z.strictObject({
|
|
|
482
484
|
});
|
|
483
485
|
const reevalConfigDataSchema = z.strictObject({
|
|
484
486
|
include: z.array(z.string()).optional(),
|
|
485
|
-
|
|
487
|
+
harnesses: harnessList.optional(),
|
|
486
488
|
sandbox: sandboxDescriptorSchema.optional(),
|
|
487
|
-
|
|
489
|
+
environments: environmentList.optional(),
|
|
488
490
|
runs: runs().optional(),
|
|
489
491
|
gate: gatePolicySchema.optional(),
|
|
490
492
|
env: envVarsSchema.optional(),
|
|
@@ -564,6 +566,7 @@ function configDataProblem(value, file) {
|
|
|
564
566
|
}
|
|
565
567
|
if (issue.code === "invalid_union") {
|
|
566
568
|
const values = "options" in issue ? issue.options : void 0;
|
|
569
|
+
if (!values && issue.path.at(-1) === "environments") return `${at} must be an environment or a list of them (got ${got(v)})`;
|
|
567
570
|
return values ? `${at} is ${oneOf(values)} (got ${got(v)})` : `${at} matches none of its forms (got ${got(v)})`;
|
|
568
571
|
}
|
|
569
572
|
if (issue.code === "invalid_type" && v === void 0) return `${at} is missing`;
|
|
@@ -578,1126 +581,6 @@ function configDataProblem(value, file) {
|
|
|
578
581
|
path: []
|
|
579
582
|
}, v)}`;
|
|
580
583
|
}
|
|
581
|
-
// src/gateway/routes.ts
|
|
582
|
-
/** Every gateway that brokers a key, by name. */
|
|
583
|
-
const ROUTING_GATEWAYS = {
|
|
584
|
-
vercel: VERCEL_GATEWAY,
|
|
585
|
-
openrouter: OPENROUTER_GATEWAY
|
|
586
|
-
};
|
|
587
|
-
/** Whether a harness's `gateway` names one of {@link ROUTING_GATEWAYS}. */
|
|
588
|
-
const isRoutingGateway = (name) => name !== void 0 && Object.hasOwn(ROUTING_GATEWAYS, name);
|
|
589
|
-
// src/harness/agents.ts
|
|
590
|
-
const WEB_EVENTS = {
|
|
591
|
-
webSearch: "web-search",
|
|
592
|
-
webFetch: "web-fetch"
|
|
593
|
-
};
|
|
594
|
-
/** `harnessMetadata.cursorSdk.unreportedTools` as transcript event types. */
|
|
595
|
-
function cursorUnreported(harnessMetadata) {
|
|
596
|
-
const tools = harnessMetadata?.cursorSdk?.unreportedTools;
|
|
597
|
-
return Array.isArray(tools) ? tools.flatMap((t) => WEB_EVENTS[t] ? [WEB_EVENTS[t]] : []) : [];
|
|
598
|
-
}
|
|
599
|
-
/** A Gemini API key's only request: `POST /v1beta/models/<model>:streamGenerateContent`. */
|
|
600
|
-
const GEMINI_ENDPOINTS = [{
|
|
601
|
-
host: "generativelanguage.googleapis.com",
|
|
602
|
-
path: { regex: "^/v1beta/models/[^/]+:streamGenerateContent$" },
|
|
603
|
-
method: ["POST"]
|
|
604
|
-
}];
|
|
605
|
-
/** Antigravity's: its model calls, and the `:generateContent` call its `search_web` tool makes. */
|
|
606
|
-
const ANTIGRAVITY_ENDPOINTS = [{
|
|
607
|
-
host: "generativelanguage.googleapis.com",
|
|
608
|
-
path: { regex: "^/v1beta/models/[\\w.-]+:(?:streamGenerateContent|generateContent)$" },
|
|
609
|
-
method: ["POST"]
|
|
610
|
-
}];
|
|
611
|
-
/** Goose's: the model call, and the model list it fetches with the key at the start of a turn. */
|
|
612
|
-
const GOOSE_ENDPOINTS = [...GEMINI_ENDPOINTS, {
|
|
613
|
-
host: "generativelanguage.googleapis.com",
|
|
614
|
-
path: { regex: "^/v1beta/models$" },
|
|
615
|
-
method: ["GET"]
|
|
616
|
-
}];
|
|
617
|
-
/**
|
|
618
|
-
* What Grok Build sends the key on: the model list and the key's status as a session starts, and
|
|
619
|
-
* its model calls (Chat Completions, and the Responses API for a model or tool that takes it).
|
|
620
|
-
*/
|
|
621
|
-
const XAI_ENDPOINTS = [
|
|
622
|
-
{
|
|
623
|
-
host: "api.x.ai",
|
|
624
|
-
path: { exact: "/v1/models" },
|
|
625
|
-
method: ["GET"]
|
|
626
|
-
},
|
|
627
|
-
{
|
|
628
|
-
host: "api.x.ai",
|
|
629
|
-
path: { exact: "/v1/api-key" },
|
|
630
|
-
method: ["GET"]
|
|
631
|
-
},
|
|
632
|
-
{
|
|
633
|
-
host: "api.x.ai",
|
|
634
|
-
path: { exact: "/v1/chat/completions" },
|
|
635
|
-
method: ["POST"]
|
|
636
|
-
},
|
|
637
|
-
{
|
|
638
|
-
host: "api.x.ai",
|
|
639
|
-
path: { exact: "/v1/responses" },
|
|
640
|
-
method: ["POST"]
|
|
641
|
-
}
|
|
642
|
-
];
|
|
643
|
-
/** xAI's model calls, whose responses name the model that served them. */
|
|
644
|
-
const XAI_MODEL_RESPONSES = XAI_ENDPOINTS.filter((e) => e.path.exact === "/v1/chat/completions" || e.path.exact === "/v1/responses");
|
|
645
|
-
/** The model an xAI response says served it. */
|
|
646
|
-
const xaiServedModel = ({ host, responseModel }) => host === "api.x.ai" ? responseModel : void 0;
|
|
647
|
-
/** The model a Gemini API request names in its URL. */
|
|
648
|
-
const geminiServedModel = ({ host, path }) => host === "generativelanguage.googleapis.com" ? /^\/v1beta\/models\/([^/:]+):streamGenerateContent$/.exec(path)?.[1] : void 0;
|
|
649
|
-
/** Cursor's key exchange: the only request the Cursor SDK sends the API key on. */
|
|
650
|
-
const CURSOR_EXCHANGE = {
|
|
651
|
-
host: "api2.cursor.sh",
|
|
652
|
-
path: { exact: "/auth/exchange_user_api_key" },
|
|
653
|
-
method: ["POST"]
|
|
654
|
-
};
|
|
655
|
-
/** Codex's model calls: the Responses API, whose response names the served model. */
|
|
656
|
-
const OPENAI_MODEL_ENDPOINTS = [{
|
|
657
|
-
host: "api.openai.com",
|
|
658
|
-
path: { exact: "/v1/responses" },
|
|
659
|
-
method: ["POST"]
|
|
660
|
-
}];
|
|
661
|
-
/** What Codex sends the key on: its model calls, and its web search tool's own endpoint. */
|
|
662
|
-
const OPENAI_ENDPOINTS = [...OPENAI_MODEL_ENDPOINTS, {
|
|
663
|
-
host: "api.openai.com",
|
|
664
|
-
path: { exact: "/v1/alpha/search" },
|
|
665
|
-
method: ["POST"]
|
|
666
|
-
}];
|
|
667
|
-
/**
|
|
668
|
-
* The model the Responses API says served a request, from its response (a dated snapshot for some
|
|
669
|
-
* models), and the reasoning effort Codex resolved for it, e.g. `gpt-5.6-luna[effort=medium]`.
|
|
670
|
-
*/
|
|
671
|
-
const openaiServedModel = ({ host, path, responseModel, responseEffort }) => host === "api.openai.com" && path === "/v1/responses" && responseModel ? `${responseModel}${responseEffort ? `[effort=${responseEffort}]` : ""}` : void 0;
|
|
672
|
-
/**
|
|
673
|
-
* OpenCode on the model's provider (`anthropic/claude-haiku-4-5` is Anthropic's): its key, the
|
|
674
|
-
* requests OpenCode sends it on, its hosts plus Exa's for web search, and the served model from
|
|
675
|
-
* the URL (Gemini's API, Bedrock's) or the response body. Bedrock's hosts are its `AWS_REGION`'s.
|
|
676
|
-
*/
|
|
677
|
-
function opencodeForModel(model, env = process.env) {
|
|
678
|
-
const resolved = resolveOpenCodeProvider(model, env);
|
|
679
|
-
if ("error" in resolved) throw new InfraError(`opencode(): ${resolved.error}`);
|
|
680
|
-
const { provider } = resolved;
|
|
681
|
-
const modelResponses = providerModelResponses(provider);
|
|
682
|
-
return {
|
|
683
|
-
credential: provider.env,
|
|
684
|
-
endpoints: providerEndpoints(provider),
|
|
685
|
-
...modelResponses.length ? { modelResponses } : {},
|
|
686
|
-
servedModel: (request) => providerServedModel(provider, request),
|
|
687
|
-
hosts: [...providerHosts(provider), "mcp.exa.ai"]
|
|
688
|
-
};
|
|
689
|
-
}
|
|
690
|
-
/**
|
|
691
|
-
* The variables besides its key that OpenCode's provider takes from the host, as they are:
|
|
692
|
-
* Bedrock's `AWS_REGION` (default `us-east-1`); none for most providers.
|
|
693
|
-
*/
|
|
694
|
-
function opencodeVars(model, env = process.env) {
|
|
695
|
-
const resolved = resolveOpenCodeProvider(model, env);
|
|
696
|
-
if ("error" in resolved) throw new InfraError(`opencode(): ${resolved.error}`);
|
|
697
|
-
return resolved.vars;
|
|
698
|
-
}
|
|
699
|
-
/** Codex's `config.toml` `[mcp_servers]` shape. */
|
|
700
|
-
function codexMcpServers(servers) {
|
|
701
|
-
return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
|
|
702
|
-
if (s.transport === "sse") throw new InfraError(`Codex has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
|
|
703
|
-
return s.transport === "stdio" ? [name, {
|
|
704
|
-
command: s.command,
|
|
705
|
-
args: s.args,
|
|
706
|
-
env: s.env ?? {}
|
|
707
|
-
}] : [name, {
|
|
708
|
-
url: s.url,
|
|
709
|
-
http_headers: s.headers ?? {}
|
|
710
|
-
}];
|
|
711
|
-
}));
|
|
712
|
-
}
|
|
713
|
-
/** The `.mcp.json` shape. */
|
|
714
|
-
function claudeMcpServers(servers) {
|
|
715
|
-
return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, {
|
|
716
|
-
type: "stdio",
|
|
717
|
-
command: s.command,
|
|
718
|
-
args: s.args,
|
|
719
|
-
env: s.env
|
|
720
|
-
}] : [name, {
|
|
721
|
-
type: s.transport,
|
|
722
|
-
url: s.url,
|
|
723
|
-
headers: s.headers
|
|
724
|
-
}]));
|
|
725
|
-
}
|
|
726
|
-
/** The Antigravity CLI's `mcp_config.json` shape: remote servers take `serverUrl`, not `url`. */
|
|
727
|
-
function antigravityMcpServers(servers) {
|
|
728
|
-
return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, {
|
|
729
|
-
command: s.command,
|
|
730
|
-
args: s.args,
|
|
731
|
-
env: s.env
|
|
732
|
-
}] : [name, {
|
|
733
|
-
serverUrl: s.url,
|
|
734
|
-
headers: s.headers
|
|
735
|
-
}]));
|
|
736
|
-
}
|
|
737
|
-
/**
|
|
738
|
-
* A `{env:…}` or `{file:…}` OpenCode would substitute. It runs `\{env:[^}]+\}`, then
|
|
739
|
-
* `\{file:[^}]+\}`, over its whole config text with no escape, so a reference left open at the end
|
|
740
|
-
* of a value runs on into the JSON after it, and only `{env:}` and `{file:}` are left alone.
|
|
741
|
-
*/
|
|
742
|
-
const OPENCODE_REFERENCE = /\{(?:env|file):(?!\})/;
|
|
743
|
-
const fieldKey = (key) => /^[A-Za-z_$][\w$]*$/.test(key) ? `.${key}` : `[${JSON.stringify(key)}]`;
|
|
744
|
-
/**
|
|
745
|
-
* The first part of a server OpenCode would substitute, as what the server wouldn't get: its name,
|
|
746
|
-
* a value as `mcp.<name>.<field>`, or an env or header name. Names are in the same config text as
|
|
747
|
-
* values, so they're substituted too; a secret's value is never named.
|
|
748
|
-
*/
|
|
749
|
-
function openCodeReference(name, s) {
|
|
750
|
-
if (OPENCODE_REFERENCE.test(name)) return "its name";
|
|
751
|
-
const server = `mcp${fieldKey(name)}`;
|
|
752
|
-
const [map, kind, values] = s.transport === "stdio" ? [
|
|
753
|
-
"env",
|
|
754
|
-
"env",
|
|
755
|
-
s.env ?? {}
|
|
756
|
-
] : [
|
|
757
|
-
"headers",
|
|
758
|
-
"header",
|
|
759
|
-
s.headers ?? {}
|
|
760
|
-
];
|
|
761
|
-
return [...s.transport === "stdio" ? [[`${server}.command`, s.command], ...s.args.map((a, i) => [`${server}.args[${i}]`, a])] : [[`${server}.url`, s.url]], ...Object.entries(values).flatMap(([k, v]) => [[`the ${kind} name ${k} in ${server}.${map}`, k], [`${server}.${map}${fieldKey(k)}`, v]])].find(([, text]) => OPENCODE_REFERENCE.test(text))?.[0];
|
|
762
|
-
}
|
|
763
|
-
/**
|
|
764
|
-
* OpenCode's config shape (its `mcp.servers`), which the adapter sets `codemode: false` on.
|
|
765
|
-
* OpenCode has no SSE transport, and refuses a value it would substitute.
|
|
766
|
-
*/
|
|
767
|
-
function opencodeMcpServers(servers) {
|
|
768
|
-
return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
|
|
769
|
-
if (s.transport === "sse") throw new InfraError(`OpenCode has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
|
|
770
|
-
const field = openCodeReference(name, s);
|
|
771
|
-
if (field) throw new InfraError(`OpenCode substitutes {env:…} and {file:…} anywhere in its config, with no escape, so the ${name} server wouldn't get ${field} as written: take the reference out, or leave OpenCode out of this cell`);
|
|
772
|
-
return s.transport === "stdio" ? [name, {
|
|
773
|
-
type: "local",
|
|
774
|
-
command: [s.command, ...s.args],
|
|
775
|
-
environment: s.env ?? {}
|
|
776
|
-
}] : [name, {
|
|
777
|
-
type: "remote",
|
|
778
|
-
url: s.url,
|
|
779
|
-
headers: s.headers ?? {},
|
|
780
|
-
oauth: false
|
|
781
|
-
}];
|
|
782
|
-
}));
|
|
783
|
-
}
|
|
784
|
-
/**
|
|
785
|
-
* The key Goose keeps an extension by (its `name_to_key`): whitespace dropped, anything but ASCII
|
|
786
|
-
* letters, digits, `_` and `-` made `_`, lower-cased. Of two extensions with one key it keeps one.
|
|
787
|
-
*/
|
|
788
|
-
const gooseKey = (name) => name.replace(/\s/g, "").replace(/[^A-Za-z0-9_-]/g, "_").toLowerCase();
|
|
789
|
-
/** Goose's extension shape (its `config.yaml`); it has no SSE transport. */
|
|
790
|
-
function gooseMcpServers(servers) {
|
|
791
|
-
const byKey = /* @__PURE__ */ new Map();
|
|
792
|
-
for (const name of Object.keys(servers)) {
|
|
793
|
-
const key = gooseKey(name);
|
|
794
|
-
const other = byKey.get(key);
|
|
795
|
-
if (other !== void 0) throw new InfraError(`Goose keys the ${other} and ${name} MCP servers both as ${key}, and keeps only one: rename one`);
|
|
796
|
-
byKey.set(key, name);
|
|
797
|
-
}
|
|
798
|
-
return Object.fromEntries(Object.entries(servers).map(([name, s]) => {
|
|
799
|
-
if (s.transport === "sse") throw new InfraError(`Goose has no SSE MCP transport, so it can't use the ${name} server (mcp.sse): use mcp.http or mcp.stdio`);
|
|
800
|
-
return s.transport === "stdio" ? [name, {
|
|
801
|
-
type: "stdio",
|
|
802
|
-
cmd: s.command,
|
|
803
|
-
args: s.args,
|
|
804
|
-
envs: s.env ?? {}
|
|
805
|
-
}] : [name, {
|
|
806
|
-
type: "streamable_http",
|
|
807
|
-
uri: s.url,
|
|
808
|
-
headers: s.headers ?? {}
|
|
809
|
-
}];
|
|
810
|
-
}));
|
|
811
|
-
}
|
|
812
|
-
/**
|
|
813
|
-
* pi's `mcp.json` shape, which the adapter escapes and sets `"exposure": "direct"` on. pi
|
|
814
|
-
* skips a server whose name isn't letters, digits, `_` and `-`, so the plan refuses it instead.
|
|
815
|
-
* A plugin's server, `<plugin>:<server>`, is named `<plugin>_<server>`, so its tools are
|
|
816
|
-
* `mcp__<plugin>_<server>__<tool>`.
|
|
817
|
-
*/
|
|
818
|
-
function piMcpServers(servers) {
|
|
819
|
-
const named = /* @__PURE__ */ new Map();
|
|
820
|
-
return Object.fromEntries(Object.entries(servers).map(([key, s]) => {
|
|
821
|
-
const name = /^[a-z0-9-]+:[^:]+$/.test(key) ? key.replace(":", "_") : key;
|
|
822
|
-
if (!/^[A-Za-z0-9_-]+$/.test(name)) throw new InfraError(`pi takes MCP server names of letters, digits, "_" and "-" only, so it can't use the ${key} server: rename it`);
|
|
823
|
-
const other = named.get(name);
|
|
824
|
-
if (other) throw new InfraError(`pi names the ${key} MCP server ${name}, as it does the ${other} server, and keeps only one: rename one`);
|
|
825
|
-
named.set(name, key);
|
|
826
|
-
if (s.transport === "sse") throw new InfraError(`pi has no SSE MCP transport, so it can't use the ${key} server (mcp.sse): use mcp.http or mcp.stdio`);
|
|
827
|
-
return s.transport === "stdio" ? [name, {
|
|
828
|
-
command: s.command,
|
|
829
|
-
args: s.args,
|
|
830
|
-
env: s.env ?? {}
|
|
831
|
-
}] : [name, {
|
|
832
|
-
url: s.url,
|
|
833
|
-
headers: s.headers ?? {}
|
|
834
|
-
}];
|
|
835
|
-
}));
|
|
836
|
-
}
|
|
837
|
-
/**
|
|
838
|
-
* ACP's `McpServer` shape, which fx and Grok Build take on `session/new`: `env` and `headers` as
|
|
839
|
-
* name-value lists.
|
|
840
|
-
*/
|
|
841
|
-
function acpMcpServers(servers) {
|
|
842
|
-
const pairs = (values = {}) => Object.entries(values).map(([name, value]) => ({
|
|
843
|
-
name,
|
|
844
|
-
value
|
|
845
|
-
}));
|
|
846
|
-
return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, s.command.startsWith("/") ? {
|
|
847
|
-
command: s.command,
|
|
848
|
-
args: s.args,
|
|
849
|
-
env: pairs(s.env)
|
|
850
|
-
} : {
|
|
851
|
-
command: "/usr/bin/env",
|
|
852
|
-
args: [s.command, ...s.args],
|
|
853
|
-
env: pairs(s.env)
|
|
854
|
-
}] : [name, {
|
|
855
|
-
type: s.transport,
|
|
856
|
-
url: s.url,
|
|
857
|
-
headers: pairs(s.headers)
|
|
858
|
-
}]));
|
|
859
|
-
}
|
|
860
|
-
/**
|
|
861
|
-
* fx on the model's provider: OpenAI's Chat Completions API for a bare OpenAI id, and Vercel AI
|
|
862
|
-
* Gateway for a gateway id, which only `vercel()` in the gateway list serves. Each response's
|
|
863
|
-
* first `model` names the model that served it.
|
|
864
|
-
*/
|
|
865
|
-
function fxForModel(model) {
|
|
866
|
-
if (!model.includes("/")) {
|
|
867
|
-
const endpoints = [{
|
|
868
|
-
host: "api.openai.com",
|
|
869
|
-
path: { exact: "/v1/chat/completions" },
|
|
870
|
-
method: ["POST"]
|
|
871
|
-
}];
|
|
872
|
-
return {
|
|
873
|
-
credential: "OPENAI_API_KEY",
|
|
874
|
-
endpoints,
|
|
875
|
-
modelResponses: endpoints,
|
|
876
|
-
servedModel: ({ host, responseModel }) => host === "api.openai.com" ? responseModel : void 0,
|
|
877
|
-
hosts: ["api.openai.com"]
|
|
878
|
-
};
|
|
879
|
-
}
|
|
880
|
-
return {
|
|
881
|
-
credential: VERCEL_KEY,
|
|
882
|
-
endpoints: FX_VERCEL_ENDPOINTS,
|
|
883
|
-
modelResponses: [FX_VERCEL_CALL],
|
|
884
|
-
servedModel: servedOnVercel,
|
|
885
|
-
askedModel: fxAskedModel,
|
|
886
|
-
hosts: [VERCEL_HOST]
|
|
887
|
-
};
|
|
888
|
-
}
|
|
889
|
-
/** The Cursor SDK's `McpServerConfig` shape. */
|
|
890
|
-
function cursorMcpServers(servers) {
|
|
891
|
-
return Object.fromEntries(Object.entries(servers).map(([name, s]) => s.transport === "stdio" ? [name, {
|
|
892
|
-
type: "stdio",
|
|
893
|
-
command: s.command,
|
|
894
|
-
args: s.args,
|
|
895
|
-
env: s.env
|
|
896
|
-
}] : [name, {
|
|
897
|
-
type: s.transport,
|
|
898
|
-
url: s.url,
|
|
899
|
-
headers: s.headers
|
|
900
|
-
}]));
|
|
901
|
-
}
|
|
902
|
-
/** Claude Code's hook events as Cursor names them, as the Cursor SDK converts Claude Code's own. */
|
|
903
|
-
const CURSOR_HOOK_EVENTS = {
|
|
904
|
-
SessionStart: "sessionStart",
|
|
905
|
-
SessionEnd: "sessionEnd",
|
|
906
|
-
UserPromptSubmit: "beforeSubmitPrompt",
|
|
907
|
-
PreToolUse: "preToolUse",
|
|
908
|
-
PostToolUse: "postToolUse",
|
|
909
|
-
SubagentStop: "subagentStop",
|
|
910
|
-
PreCompact: "preCompact",
|
|
911
|
-
Stop: "stop"
|
|
912
|
-
};
|
|
913
|
-
/** Claude Code's tool names as Cursor's hooks match them; null for one Cursor has no tool for. */
|
|
914
|
-
const CURSOR_HOOK_TOOLS = {
|
|
915
|
-
Bash: "Shell",
|
|
916
|
-
Read: "Read",
|
|
917
|
-
Write: "Write",
|
|
918
|
-
Edit: "Write",
|
|
919
|
-
Glob: null,
|
|
920
|
-
Grep: "Grep",
|
|
921
|
-
WebFetch: "WebFetch",
|
|
922
|
-
WebSearch: "WebSearch",
|
|
923
|
-
Task: "Task"
|
|
924
|
-
};
|
|
925
|
-
/**
|
|
926
|
-
* A Claude Code hook group's tool matcher as Cursor's, on the tool events only: each name mapped,
|
|
927
|
-
* `mcp__<server>__<tool>` as `MCP:<tool>`; null when it names only tools Cursor lacks.
|
|
928
|
-
*/
|
|
929
|
-
function cursorHookMatcher(event, matcher) {
|
|
930
|
-
if (event !== "PreToolUse" && event !== "PostToolUse") return void 0;
|
|
931
|
-
if (!matcher || matcher === "*") return void 0;
|
|
932
|
-
const out = [];
|
|
933
|
-
for (const raw of matcher.split("|")) {
|
|
934
|
-
const name = raw.trim();
|
|
935
|
-
const parts = name.split("__");
|
|
936
|
-
const tool = name.startsWith("mcp__") && parts.length >= 3 ? `MCP:${parts.slice(2).join("__")}` : CURSOR_HOOK_TOOLS[name] === void 0 ? name : CURSOR_HOOK_TOOLS[name];
|
|
937
|
-
if (tool && !out.includes(tool)) out.push(tool);
|
|
938
|
-
}
|
|
939
|
-
return out.length ? out.join("|") : null;
|
|
940
|
-
}
|
|
941
|
-
/** Where Cursor reads what reeval unpacks from a plugin, in the workspace (settingSources `project`). */
|
|
942
|
-
const CURSOR_PLUGIN_UNPACK = {
|
|
943
|
-
rules: ".cursor/rules",
|
|
944
|
-
agents: ".cursor/agents",
|
|
945
|
-
commands: ".cursor/commands",
|
|
946
|
-
hooks: {
|
|
947
|
-
file: ".cursor/hooks.json",
|
|
948
|
-
events: CURSOR_HOOK_EVENTS,
|
|
949
|
-
matcher: cursorHookMatcher
|
|
950
|
-
}
|
|
951
|
-
};
|
|
952
|
-
/**
|
|
953
|
-
* A plugin subagent's or command's frontmatter as OpenCode 2 takes it: the description, and a
|
|
954
|
-
* subagent's `mode`. OpenCode skips a file whose frontmatter its schema refuses, without saying,
|
|
955
|
-
* and Claude Code's `tools: Read, Grep`, `model: sonnet` and `allowed-tools` are such keys.
|
|
956
|
-
*/
|
|
957
|
-
function opencodeFrontmatter(part, front) {
|
|
958
|
-
const d = front.description;
|
|
959
|
-
const description = typeof d === "string" || typeof d === "number" || typeof d === "boolean" ? String(d) : void 0;
|
|
960
|
-
return {
|
|
961
|
-
...description ? { description } : {},
|
|
962
|
-
...part === "agents" ? { mode: "subagent" } : {}
|
|
963
|
-
};
|
|
964
|
-
}
|
|
965
|
-
/** Where OpenCode reads plugin subagents and commands, in the workspace's config directory. */
|
|
966
|
-
const OPENCODE_PLUGIN_UNPACK = {
|
|
967
|
-
agents: ".opencode/agents",
|
|
968
|
-
commands: ".opencode/commands",
|
|
969
|
-
frontmatter: opencodeFrontmatter
|
|
970
|
-
};
|
|
971
|
-
/**
|
|
972
|
-
* Where pi reads plugin commands, as prompt templates (`/<name>`), in the workspace, which its
|
|
973
|
-
* `--approve` trusts. Its frontmatter takes `description` and `argument-hint`, and ignores the
|
|
974
|
-
* rest of Claude Code's.
|
|
975
|
-
*/
|
|
976
|
-
const PI_PLUGIN_UNPACK = { commands: ".pi/prompts" };
|
|
977
|
-
/**
|
|
978
|
-
* What Claude Code's plugin format bundles, which Grok Build reads too: every part but rules,
|
|
979
|
-
* which neither loads from a plugin.
|
|
980
|
-
*/
|
|
981
|
-
const CLAUDE_FORMAT_PARTS = PLUGIN_PARTS.filter((p) => p !== "rules");
|
|
982
|
-
/** The marketplace Claude Code's settings name for the session's plugins. */
|
|
983
|
-
const CLAUDE_PLUGIN_MARKETPLACE = "reeval";
|
|
984
|
-
/**
|
|
985
|
-
* Claude Code's settings for the session's plugins: a `directory` marketplace at the directory
|
|
986
|
-
* they were copied to, listing each, and each enabled as `<plugin>@reeval`. Claude Code installs
|
|
987
|
-
* and loads them as it starts, since the bridge leaves every settings source on.
|
|
988
|
-
*/
|
|
989
|
-
function claudePluginFiles(plugins) {
|
|
990
|
-
const [first] = plugins;
|
|
991
|
-
if (!first) return [];
|
|
992
|
-
const json = (value) => `${JSON.stringify(value, null, 2)}\n`;
|
|
993
|
-
const marketplace = {
|
|
994
|
-
name: CLAUDE_PLUGIN_MARKETPLACE,
|
|
995
|
-
owner: { name: "reeval" },
|
|
996
|
-
plugins: plugins.map((p) => ({
|
|
997
|
-
name: p.name,
|
|
998
|
-
source: `./${p.name}`
|
|
999
|
-
}))
|
|
1000
|
-
};
|
|
1001
|
-
const settings = {
|
|
1002
|
-
extraKnownMarketplaces: { [CLAUDE_PLUGIN_MARKETPLACE]: { source: {
|
|
1003
|
-
source: "directory",
|
|
1004
|
-
path: first.dir.slice(0, first.dir.lastIndexOf("/"))
|
|
1005
|
-
} } },
|
|
1006
|
-
enabledPlugins: Object.fromEntries(plugins.map((p) => [`${p.name}@${CLAUDE_PLUGIN_MARKETPLACE}`, true]))
|
|
1007
|
-
};
|
|
1008
|
-
return [{
|
|
1009
|
-
path: `${PLUGIN_DIR}/.claude-plugin/marketplace.json`,
|
|
1010
|
-
content: json(marketplace)
|
|
1011
|
-
}, {
|
|
1012
|
-
path: "~/.claude/settings.json",
|
|
1013
|
-
content: json(settings),
|
|
1014
|
-
merge: true
|
|
1015
|
-
}];
|
|
1016
|
-
}
|
|
1017
|
-
const isObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
1018
|
-
/**
|
|
1019
|
-
* Writes the layout's plugin files into the sandbox and returns their paths. A `merge` file's keys
|
|
1020
|
-
* go into the JSON object already at its path (an object-valued key merged one level down), so
|
|
1021
|
-
* settings an environment's `files` wrote there stay.
|
|
1022
|
-
*/
|
|
1023
|
-
async function writePluginFiles(sandbox, layout, plugins) {
|
|
1024
|
-
const written = [];
|
|
1025
|
-
for (const f of layout.pluginFiles?.(plugins) ?? []) {
|
|
1026
|
-
const existing = f.merge ? await sandbox.readFile(f.path) : null;
|
|
1027
|
-
let content = f.content;
|
|
1028
|
-
if (existing?.trim()) {
|
|
1029
|
-
let base;
|
|
1030
|
-
try {
|
|
1031
|
-
base = JSON.parse(withoutBom(existing));
|
|
1032
|
-
} catch {}
|
|
1033
|
-
if (!isObject(base)) throw new InfraError(`${f.path} isn't a JSON object, so reeval can't enable the session's plugins in it: write it as one in files, or leave it out`);
|
|
1034
|
-
const ours = JSON.parse(f.content);
|
|
1035
|
-
const merged = { ...base };
|
|
1036
|
-
for (const [k, v] of Object.entries(ours)) {
|
|
1037
|
-
const was = base[k];
|
|
1038
|
-
merged[k] = isObject(v) && isObject(was) ? {
|
|
1039
|
-
...was,
|
|
1040
|
-
...v
|
|
1041
|
-
} : v;
|
|
1042
|
-
}
|
|
1043
|
-
content = `${JSON.stringify(merged, null, 2)}\n`;
|
|
1044
|
-
}
|
|
1045
|
-
await sandbox.writeFile(f.path, content);
|
|
1046
|
-
written.push(f.path);
|
|
1047
|
-
}
|
|
1048
|
-
return written;
|
|
1049
|
-
}
|
|
1050
|
-
/**
|
|
1051
|
-
* The MCP servers the agent is given: all of them, less the plugins' own where it takes plugin
|
|
1052
|
-
* items namespaced, since it loads those from the plugin and would otherwise start each twice.
|
|
1053
|
-
*/
|
|
1054
|
-
function agentMcp(layout, mcp, plugins) {
|
|
1055
|
-
if (layout.pluginPlacement !== "namespaced") return mcp;
|
|
1056
|
-
const own = new Set(plugins.flatMap((p) => p.servers));
|
|
1057
|
-
return Object.fromEntries(Object.entries(mcp).filter(([name]) => !own.has(name)));
|
|
1058
|
-
}
|
|
1059
|
-
/** Where each agent with an image reads rules and skills. */
|
|
1060
|
-
const LAYOUTS = {
|
|
1061
|
-
cursor: {
|
|
1062
|
-
rulesFile: "AGENTS.md",
|
|
1063
|
-
workspaceSkillDir: ".cursor/skills",
|
|
1064
|
-
harnessSkillDir: ".agents/skills",
|
|
1065
|
-
pluginPlacement: "flat",
|
|
1066
|
-
pluginUnpack: CURSOR_PLUGIN_UNPACK,
|
|
1067
|
-
pluginHookEvents: Object.keys(CURSOR_HOOK_EVENTS)
|
|
1068
|
-
},
|
|
1069
|
-
"claude-code": {
|
|
1070
|
-
rulesFile: "CLAUDE.md",
|
|
1071
|
-
workspaceSkillDir: ".claude/skills",
|
|
1072
|
-
harnessSkillDir: ".claude/skills",
|
|
1073
|
-
pluginPlacement: "namespaced",
|
|
1074
|
-
pluginFiles: claudePluginFiles
|
|
1075
|
-
},
|
|
1076
|
-
antigravity: {
|
|
1077
|
-
rulesFile: "AGENTS.md",
|
|
1078
|
-
workspaceSkillDir: ".agents/skills",
|
|
1079
|
-
harnessSkillDir: ".gemini/config/skills",
|
|
1080
|
-
pluginPlacement: "namespaced",
|
|
1081
|
-
pluginHookEvents: [
|
|
1082
|
-
"SessionStart",
|
|
1083
|
-
"PreToolUse",
|
|
1084
|
-
"PostToolUse",
|
|
1085
|
-
"Stop"
|
|
1086
|
-
]
|
|
1087
|
-
},
|
|
1088
|
-
opencode: {
|
|
1089
|
-
rulesFile: "AGENTS.md",
|
|
1090
|
-
workspaceSkillDir: ".opencode/skills",
|
|
1091
|
-
harnessSkillDir: ".config/opencode/skills",
|
|
1092
|
-
pluginPlacement: "flat",
|
|
1093
|
-
pluginUnpack: OPENCODE_PLUGIN_UNPACK
|
|
1094
|
-
},
|
|
1095
|
-
goose: {
|
|
1096
|
-
rulesFile: "AGENTS.md",
|
|
1097
|
-
workspaceSkillDir: ".goose/skills",
|
|
1098
|
-
harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/goose/home/.config/goose/skills",
|
|
1099
|
-
pluginPlacement: "flat"
|
|
1100
|
-
},
|
|
1101
|
-
pi: {
|
|
1102
|
-
rulesFile: "AGENTS.md",
|
|
1103
|
-
workspaceSkillDir: ".pi/skills",
|
|
1104
|
-
harnessSkillDir: ".pi/agent/skills",
|
|
1105
|
-
pluginPlacement: "flat",
|
|
1106
|
-
pluginUnpack: PI_PLUGIN_UNPACK
|
|
1107
|
-
},
|
|
1108
|
-
codex: {
|
|
1109
|
-
rulesFile: "AGENTS.md",
|
|
1110
|
-
workspaceSkillDir: ".agents/skills",
|
|
1111
|
-
harnessSkillDir: ".agents/skills",
|
|
1112
|
-
pluginPlacement: "namespaced"
|
|
1113
|
-
},
|
|
1114
|
-
fx: {
|
|
1115
|
-
rulesFile: "AGENTS.md",
|
|
1116
|
-
workspaceSkillDir: ".fx/skills",
|
|
1117
|
-
harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/fx/home/.fx/skills",
|
|
1118
|
-
pluginPlacement: "flat"
|
|
1119
|
-
},
|
|
1120
|
-
"grok-build": {
|
|
1121
|
-
rulesFile: "AGENTS.md",
|
|
1122
|
-
workspaceSkillDir: ".grok/skills",
|
|
1123
|
-
harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/grok-build/home/.grok/skills",
|
|
1124
|
-
pluginPlacement: "namespaced"
|
|
1125
|
-
}
|
|
1126
|
-
};
|
|
1127
|
-
/** Every real agent reeval can drive, by harness kind. */
|
|
1128
|
-
const AGENTS = {
|
|
1129
|
-
cursor: {
|
|
1130
|
-
...LAYOUTS.cursor,
|
|
1131
|
-
pluginParts: PLUGIN_PARTS,
|
|
1132
|
-
name: "Cursor",
|
|
1133
|
-
factory: "cursor",
|
|
1134
|
-
reasoning: "streamed",
|
|
1135
|
-
reasoningTokens: false,
|
|
1136
|
-
credential: "CURSOR_API_KEY",
|
|
1137
|
-
endpoints: [CURSOR_EXCHANGE],
|
|
1138
|
-
tokens: {
|
|
1139
|
-
endpoint: CURSOR_EXCHANGE,
|
|
1140
|
-
fields: ["accessToken", "refreshToken"],
|
|
1141
|
-
hosts: ["cursor.sh", "*.cursor.sh"]
|
|
1142
|
-
},
|
|
1143
|
-
unreported: cursorUnreported,
|
|
1144
|
-
mcpServers: cursorMcpServers,
|
|
1145
|
-
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-BXnv0ViU.js")).createCursorSdk({
|
|
1146
|
-
auth,
|
|
1147
|
-
...mcpServers ? { mcpServers } : {},
|
|
1148
|
-
...descriptor.kind === "cursor" && descriptor.params ? { modelParams: descriptor.params } : {},
|
|
1149
|
-
...descriptor.kind === "cursor" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1150
|
-
requireCredentialBrokering: true
|
|
1151
|
-
})
|
|
1152
|
-
},
|
|
1153
|
-
"claude-code": {
|
|
1154
|
-
...LAYOUTS["claude-code"],
|
|
1155
|
-
pluginParts: CLAUDE_FORMAT_PARTS,
|
|
1156
|
-
name: "Claude Code",
|
|
1157
|
-
factory: "claudeCode",
|
|
1158
|
-
reasoning: "streamed",
|
|
1159
|
-
reasoningTokens: false,
|
|
1160
|
-
subagentCalls: true,
|
|
1161
|
-
credential: "ANTHROPIC_API_KEY",
|
|
1162
|
-
endpoints: [
|
|
1163
|
-
{
|
|
1164
|
-
host: "api.anthropic.com",
|
|
1165
|
-
path: { exact: "/v1/messages" },
|
|
1166
|
-
method: ["POST"]
|
|
1167
|
-
},
|
|
1168
|
-
{
|
|
1169
|
-
host: "api.anthropic.com",
|
|
1170
|
-
path: { exact: "/v1/messages/count_tokens" },
|
|
1171
|
-
method: ["POST"]
|
|
1172
|
-
},
|
|
1173
|
-
{
|
|
1174
|
-
host: "api.anthropic.com",
|
|
1175
|
-
path: { exact: "/api/claude_code/settings" },
|
|
1176
|
-
method: ["GET"]
|
|
1177
|
-
},
|
|
1178
|
-
{
|
|
1179
|
-
host: "api.anthropic.com",
|
|
1180
|
-
path: { exact: "/api/claude_code/policy_limits" },
|
|
1181
|
-
method: ["GET"]
|
|
1182
|
-
}
|
|
1183
|
-
],
|
|
1184
|
-
mcpServers: claudeMcpServers,
|
|
1185
|
-
adapter: async ({ auth, env, mcpServers, descriptor }) => (await import("./dist-ByYk_wAs.js")).createClaudeCode({
|
|
1186
|
-
auth,
|
|
1187
|
-
...mcpServers ? { mcpServers } : {},
|
|
1188
|
-
...descriptor.kind === "claude-code" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1189
|
-
env: {
|
|
1190
|
-
IS_SANDBOX: "1",
|
|
1191
|
-
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",
|
|
1192
|
-
...env
|
|
1193
|
-
}
|
|
1194
|
-
})
|
|
1195
|
-
},
|
|
1196
|
-
antigravity: {
|
|
1197
|
-
...LAYOUTS.antigravity,
|
|
1198
|
-
pluginParts: PLUGIN_PARTS,
|
|
1199
|
-
pluginManifestsFirst: [
|
|
1200
|
-
"plugin.json",
|
|
1201
|
-
".antigravity-plugin/plugin.json",
|
|
1202
|
-
".gemini-plugin/plugin.json",
|
|
1203
|
-
"gemini-extension.json"
|
|
1204
|
-
],
|
|
1205
|
-
name: "Antigravity",
|
|
1206
|
-
factory: "antigravity",
|
|
1207
|
-
reasoning: "none",
|
|
1208
|
-
reasoningTokens: true,
|
|
1209
|
-
credential: "GEMINI_API_KEY",
|
|
1210
|
-
endpoints: ANTIGRAVITY_ENDPOINTS,
|
|
1211
|
-
expectedBlocked: {
|
|
1212
|
-
hosts: ["play.googleapis.com", "antigravity-unleash.goog"],
|
|
1213
|
-
why: "Antigravity's telemetry and feature flags, which its CLI sends even with telemetry off"
|
|
1214
|
-
},
|
|
1215
|
-
servedModel: geminiServedModel,
|
|
1216
|
-
mcpServers: antigravityMcpServers,
|
|
1217
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-XxUpr2sa.js")).createAntigravity({
|
|
1218
|
-
auth,
|
|
1219
|
-
...vertex ? { vertex } : {},
|
|
1220
|
-
...mcpServers ? { mcpServers } : {},
|
|
1221
|
-
...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
|
|
1222
|
-
name,
|
|
1223
|
-
dir
|
|
1224
|
-
})) } : {},
|
|
1225
|
-
...descriptor.kind === "antigravity" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1226
|
-
requireCredentialBrokering: true
|
|
1227
|
-
})
|
|
1228
|
-
},
|
|
1229
|
-
opencode: {
|
|
1230
|
-
...LAYOUTS.opencode,
|
|
1231
|
-
pluginParts: [
|
|
1232
|
-
"skills",
|
|
1233
|
-
"agents",
|
|
1234
|
-
"commands",
|
|
1235
|
-
"mcp"
|
|
1236
|
-
],
|
|
1237
|
-
name: "OpenCode",
|
|
1238
|
-
factory: "opencode",
|
|
1239
|
-
configuredModelOnStart: true,
|
|
1240
|
-
reasoning: "streamed",
|
|
1241
|
-
reasoningTokens: "provider",
|
|
1242
|
-
credential: "GOOGLE_GENERATIVE_AI_API_KEY",
|
|
1243
|
-
endpoints: GEMINI_ENDPOINTS,
|
|
1244
|
-
servedModel: geminiServedModel,
|
|
1245
|
-
forModel: opencodeForModel,
|
|
1246
|
-
mcpServers: opencodeMcpServers,
|
|
1247
|
-
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-BiS8YNN_.js")).createOpenCode({
|
|
1248
|
-
auth,
|
|
1249
|
-
model,
|
|
1250
|
-
...vertex ? { vertex } : { vars: opencodeVars(model) },
|
|
1251
|
-
...descriptor.kind === "opencode" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1252
|
-
...mcpServers ? { mcpServers } : {},
|
|
1253
|
-
requireCredentialBrokering: true
|
|
1254
|
-
})
|
|
1255
|
-
},
|
|
1256
|
-
goose: {
|
|
1257
|
-
...LAYOUTS.goose,
|
|
1258
|
-
pluginParts: ["skills", "mcp"],
|
|
1259
|
-
name: "Goose",
|
|
1260
|
-
factory: "goose",
|
|
1261
|
-
configuredModelOnStart: true,
|
|
1262
|
-
reasoning: "streamed",
|
|
1263
|
-
reasoningTokens: false,
|
|
1264
|
-
credential: "GOOGLE_GENERATIVE_AI_API_KEY",
|
|
1265
|
-
endpoints: GOOSE_ENDPOINTS,
|
|
1266
|
-
expectedBlocked: {
|
|
1267
|
-
hosts: ["models.dev"],
|
|
1268
|
-
why: "Goose's model catalog refresh, which no setting turns off"
|
|
1269
|
-
},
|
|
1270
|
-
servedModel: geminiServedModel,
|
|
1271
|
-
mcpServers: gooseMcpServers,
|
|
1272
|
-
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-C-tH5ebd.js")).createGoose({
|
|
1273
|
-
...vertex ? { vertex } : {},
|
|
1274
|
-
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1275
|
-
auth,
|
|
1276
|
-
provider: (await import("./src-C-tH5ebd.js")).AI_GATEWAY_PROVIDER
|
|
1277
|
-
} : auth["OPENROUTER_API_KEY"] ? {
|
|
1278
|
-
auth,
|
|
1279
|
-
provider: (await import("./src-C-tH5ebd.js")).OPENROUTER_PROVIDER
|
|
1280
|
-
} : { auth: { GOOGLE_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" } },
|
|
1281
|
-
model,
|
|
1282
|
-
...descriptor.kind === "goose" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1283
|
-
...mcpServers ? { mcpServers } : {},
|
|
1284
|
-
requireCredentialBrokering: true
|
|
1285
|
-
})
|
|
1286
|
-
},
|
|
1287
|
-
pi: {
|
|
1288
|
-
...LAYOUTS.pi,
|
|
1289
|
-
pluginParts: [
|
|
1290
|
-
"skills",
|
|
1291
|
-
"commands",
|
|
1292
|
-
"mcp"
|
|
1293
|
-
],
|
|
1294
|
-
name: "pi",
|
|
1295
|
-
factory: "pi",
|
|
1296
|
-
reasoning: "streamed",
|
|
1297
|
-
reasoningTokens: true,
|
|
1298
|
-
credential: "GOOGLE_GENERATIVE_AI_API_KEY",
|
|
1299
|
-
endpoints: GEMINI_ENDPOINTS,
|
|
1300
|
-
servedModel: geminiServedModel,
|
|
1301
|
-
mcpServers: piMcpServers,
|
|
1302
|
-
adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-B9oE_mNS.js")).createPi({
|
|
1303
|
-
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1304
|
-
auth,
|
|
1305
|
-
gateway: (await import("./src-B9oE_mNS.js")).PI_AI_GATEWAY
|
|
1306
|
-
} : auth["OPENROUTER_API_KEY"] ? {
|
|
1307
|
-
auth,
|
|
1308
|
-
gateway: (await import("./src-B9oE_mNS.js")).PI_OPENROUTER
|
|
1309
|
-
} : {
|
|
1310
|
-
auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
|
|
1311
|
-
...vertex ? { vertex } : {}
|
|
1312
|
-
},
|
|
1313
|
-
...descriptor.kind === "pi" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1314
|
-
...mcpServers ? { mcpServers } : {},
|
|
1315
|
-
requireCredentialBrokering: true
|
|
1316
|
-
})
|
|
1317
|
-
},
|
|
1318
|
-
codex: {
|
|
1319
|
-
...LAYOUTS.codex,
|
|
1320
|
-
pluginParts: ["skills", "mcp"],
|
|
1321
|
-
pluginApps: "they need a ChatGPT login",
|
|
1322
|
-
pluginManifestsFirst: ["plugin.json", ".codex-plugin/plugin.json"],
|
|
1323
|
-
pluginRootManifestNeedsSchema: true,
|
|
1324
|
-
pluginServersByName: true,
|
|
1325
|
-
name: "Codex",
|
|
1326
|
-
factory: "codex",
|
|
1327
|
-
reasoning: "whole",
|
|
1328
|
-
reasoningTokens: true,
|
|
1329
|
-
credential: "OPENAI_API_KEY",
|
|
1330
|
-
endpoints: OPENAI_ENDPOINTS,
|
|
1331
|
-
modelResponses: OPENAI_MODEL_ENDPOINTS,
|
|
1332
|
-
expectedBlocked: {
|
|
1333
|
-
hosts: [
|
|
1334
|
-
"github.com",
|
|
1335
|
-
"api.github.com",
|
|
1336
|
-
"chatgpt.com"
|
|
1337
|
-
],
|
|
1338
|
-
why: "Codex's plugin sync, which it runs on every turn once plugins are on",
|
|
1339
|
-
withPlugins: true
|
|
1340
|
-
},
|
|
1341
|
-
servedModel: openaiServedModel,
|
|
1342
|
-
mcpServers: codexMcpServers,
|
|
1343
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-Cg1bFs9g.js")).createCodex({
|
|
1344
|
-
auth,
|
|
1345
|
-
...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-Cg1bFs9g.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-Cg1bFs9g.js")).OPENROUTER_PROVIDER } : {},
|
|
1346
|
-
...descriptor.kind === "codex" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1347
|
-
...mcpServers ? { mcpServers } : {},
|
|
1348
|
-
...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
|
|
1349
|
-
name,
|
|
1350
|
-
dir
|
|
1351
|
-
})) } : {},
|
|
1352
|
-
requireCredentialBrokering: true
|
|
1353
|
-
})
|
|
1354
|
-
},
|
|
1355
|
-
fx: {
|
|
1356
|
-
...LAYOUTS.fx,
|
|
1357
|
-
pluginParts: ["skills", "mcp"],
|
|
1358
|
-
name: "fx",
|
|
1359
|
-
factory: "fx",
|
|
1360
|
-
configuredModelOnStart: true,
|
|
1361
|
-
reasoning: "streamed",
|
|
1362
|
-
reasoningTokens: "provider",
|
|
1363
|
-
...fxForModel("provider/model"),
|
|
1364
|
-
forModel: fxForModel,
|
|
1365
|
-
mcpServers: acpMcpServers,
|
|
1366
|
-
adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-DV_UorZv.js")).createFx({
|
|
1367
|
-
auth,
|
|
1368
|
-
model,
|
|
1369
|
-
...auth["OPENROUTER_API_KEY"] ? { provider: "openrouter" } : {},
|
|
1370
|
-
...env?.FX_VERTEX_PROJECT ? { vertex: {
|
|
1371
|
-
project: env.FX_VERTEX_PROJECT,
|
|
1372
|
-
location: env.FX_VERTEX_LOCATION ?? ""
|
|
1373
|
-
} } : {},
|
|
1374
|
-
...descriptor.kind === "fx" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1375
|
-
...mcpServers ? { mcpServers } : {},
|
|
1376
|
-
requireCredentialBrokering: true
|
|
1377
|
-
})
|
|
1378
|
-
},
|
|
1379
|
-
"grok-build": {
|
|
1380
|
-
...LAYOUTS["grok-build"],
|
|
1381
|
-
pluginParts: CLAUDE_FORMAT_PARTS,
|
|
1382
|
-
pluginServersByName: true,
|
|
1383
|
-
name: "Grok Build",
|
|
1384
|
-
factory: "grokBuild",
|
|
1385
|
-
configuredModelOnStart: true,
|
|
1386
|
-
reasoning: "streamed",
|
|
1387
|
-
reasoningTokens: true,
|
|
1388
|
-
credential: "XAI_API_KEY",
|
|
1389
|
-
endpoints: XAI_ENDPOINTS,
|
|
1390
|
-
modelResponses: XAI_MODEL_RESPONSES,
|
|
1391
|
-
servedModel: xaiServedModel,
|
|
1392
|
-
mcpServers: acpMcpServers,
|
|
1393
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins, model, vertexChat }) => {
|
|
1394
|
-
const grok = await import("./src-CUpoVVb8.js");
|
|
1395
|
-
return grok.createGrokBuild({
|
|
1396
|
-
...vertexChat ? {
|
|
1397
|
-
auth: { [grok.VERTEX_TOKEN_ENV]: SANDBOX_ACCESS_TOKEN },
|
|
1398
|
-
provider: grok.vertexProvider(vertexChat.project, vertexChat.region)
|
|
1399
|
-
} : {
|
|
1400
|
-
auth,
|
|
1401
|
-
...auth["AI_GATEWAY_API_KEY"] ? { provider: grok.AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: grok.OPENROUTER_PROVIDER } : {}
|
|
1402
|
-
},
|
|
1403
|
-
model,
|
|
1404
|
-
...plugins.length ? {
|
|
1405
|
-
pluginDirs: plugins.map((p) => p.dir),
|
|
1406
|
-
pluginMcpServers: plugins.flatMap((p) => p.servers.map((s) => s.slice(p.name.length + 1)))
|
|
1407
|
-
} : {},
|
|
1408
|
-
...descriptor.kind === "grok-build" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1409
|
-
...mcpServers ? { mcpServers } : {},
|
|
1410
|
-
requireCredentialBrokering: true
|
|
1411
|
-
});
|
|
1412
|
-
}
|
|
1413
|
-
}
|
|
1414
|
-
};
|
|
1415
|
-
/** The harness of a config that names none: Cursor, on Cursor's own model. */
|
|
1416
|
-
const DEFAULT_HARNESS = {
|
|
1417
|
-
kind: "cursor",
|
|
1418
|
-
model: "composer-2.5"
|
|
1419
|
-
};
|
|
1420
|
-
/** A harness to name in messages that ask for one. */
|
|
1421
|
-
const HARNESS_EXAMPLE = "cursor({ model: 'composer-2.5' })";
|
|
1422
|
-
const isAgentKind = (kind) => typeof kind === "string" && Object.hasOwn(AGENTS, kind);
|
|
1423
|
-
/**
|
|
1424
|
-
* A server as the plan knows it: its names, literal values and each `secret.template`'s text. A
|
|
1425
|
-
* host secret's value is read when the attempt starts, so the plan has it as `''`.
|
|
1426
|
-
*/
|
|
1427
|
-
function plannedMcpServer(s) {
|
|
1428
|
-
const literal = (values) => Object.fromEntries(Object.entries(values).map(([k, v]) => [k, typeof v === "string" ? v : v.kind === "secret-template" ? v.template : ""]));
|
|
1429
|
-
return s.transport === "stdio" ? {
|
|
1430
|
-
transport: "stdio",
|
|
1431
|
-
command: s.command,
|
|
1432
|
-
args: s.args,
|
|
1433
|
-
env: literal(s.env)
|
|
1434
|
-
} : {
|
|
1435
|
-
transport: s.transport,
|
|
1436
|
-
url: s.url,
|
|
1437
|
-
headers: literal(s.headers)
|
|
1438
|
-
};
|
|
1439
|
-
}
|
|
1440
|
-
/**
|
|
1441
|
-
* Why a harness's agent can't take the environment's MCP servers (Goose, OpenCode, pi and Codex
|
|
1442
|
-
* take no `mcp.sse`; OpenCode would substitute a `{env:…}` or `{file:…}` in a name or value), in the
|
|
1443
|
-
* agent's own words, so the plan refuses them before any attempt starts; undefined when it can.
|
|
1444
|
-
*/
|
|
1445
|
-
function refusedMcp(harness, layers) {
|
|
1446
|
-
if (!isAgentKind(harness.kind)) return void 0;
|
|
1447
|
-
const servers = /* @__PURE__ */ new Map();
|
|
1448
|
-
for (const { spec } of layers) for (const [name, server] of Object.entries(spec.mcp ?? {})) servers.set(name, plannedMcpServer(server));
|
|
1449
|
-
if (servers.size === 0) return void 0;
|
|
1450
|
-
try {
|
|
1451
|
-
AGENTS[harness.kind].mcpServers(Object.fromEntries(servers));
|
|
1452
|
-
return;
|
|
1453
|
-
} catch (error) {
|
|
1454
|
-
if (error instanceof InfraError) return error.message;
|
|
1455
|
-
throw error;
|
|
1456
|
-
}
|
|
1457
|
-
}
|
|
1458
|
-
/** The first host variable a plugin MCP server reads (`${VAR}`), as `VAR`; undefined for none. */
|
|
1459
|
-
function hostVariable(s) {
|
|
1460
|
-
const values = Object.entries(s.transport === "stdio" ? s.env : s.headers);
|
|
1461
|
-
for (const [key, v] of values) {
|
|
1462
|
-
if (typeof v === "string") continue;
|
|
1463
|
-
if (v.kind === "secret") return v.from ?? key;
|
|
1464
|
-
return /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/.exec(v.template)?.[1] ?? key;
|
|
1465
|
-
}
|
|
1466
|
-
}
|
|
1467
|
-
const quoted = (parts) => parts.map((p) => `'${p}'`).join(", ");
|
|
1468
|
-
/**
|
|
1469
|
-
* Why a harness's agent can't take the plugins, in the agent's own words: a part it doesn't
|
|
1470
|
-
* load, apps it can't take (`pluginApps`), a hook event or type it doesn't run, a plugin MCP
|
|
1471
|
-
* server it refuses (named `<plugin>:<server>`) or whose `<plugin>:<server>` key `mcp` already has,
|
|
1472
|
-
* or, where it takes plugin items flat, a skill or MCP server whose name another already has, or a
|
|
1473
|
-
* subagent, command or rule another unpacks to the same file. `own` names the
|
|
1474
|
-
* environment's own skills and MCP servers, and `unread` the plugins not fetched yet (git and
|
|
1475
|
-
* npm ones, at plan time) by their call. Undefined when it can take them.
|
|
1476
|
-
*/
|
|
1477
|
-
function refusedPlugins(harness, plugins, own, unread = []) {
|
|
1478
|
-
if (plugins.length === 0 && unread.length === 0) return void 0;
|
|
1479
|
-
const agent = isAgentKind(harness.kind) ? AGENTS[harness.kind] : void 0;
|
|
1480
|
-
const mimics = harness.kind === "fake" && harness.layout ? AGENTS[harness.layout] : void 0;
|
|
1481
|
-
const loads = (agent ?? mimics)?.pluginParts ?? PLUGIN_PARTS;
|
|
1482
|
-
const name = agent?.name ?? (mimics ? `The fake agent with ${mimics.name}'s layout` : "The fake agent");
|
|
1483
|
-
if (loads.length === 0) return `${name} can't load plugins yet: leave ${plugins[0] ? `the ${plugins[0].name} plugin` : unread[0]} out of this cell`;
|
|
1484
|
-
for (const p of plugins) {
|
|
1485
|
-
const refused = p.kept.filter((part) => !loads.includes(part));
|
|
1486
|
-
if (refused.length === 0) continue;
|
|
1487
|
-
const can = p.kept.filter((part) => loads.includes(part));
|
|
1488
|
-
const what = refused.map((part) => `${part} (${PART_PATHS[part]})`).join(" and ");
|
|
1489
|
-
const fix = can.length ? `use only: [${quoted(can)}], or leave` : "leave";
|
|
1490
|
-
return `${name} can't load the ${p.name} plugin's ${what}: ${fix} the plugin out of this cell`;
|
|
1491
|
-
}
|
|
1492
|
-
const reads = agent ?? mimics;
|
|
1493
|
-
const first = reads?.pluginManifestsFirst ?? [];
|
|
1494
|
-
for (const p of plugins) {
|
|
1495
|
-
const other = p.manifests.find((m) => first.includes(m) && (m !== "plugin.json" || !reads?.pluginRootManifestNeedsSchema || p.rootSchema));
|
|
1496
|
-
if (other) return `${name} reads the ${p.name} plugin's ${other} in place of its ${PLUGIN_MANIFEST}, so reeval can't hold it to the parts it places: remove ${other} from the plugin, or leave the plugin out of this cell`;
|
|
1497
|
-
}
|
|
1498
|
-
const apps = (agent ?? mimics)?.pluginApps;
|
|
1499
|
-
const withApps = apps ? plugins.find((p) => p.apps) : void 0;
|
|
1500
|
-
if (withApps) return `${name} can't load the ${withApps.name} plugin's apps: ${apps}`;
|
|
1501
|
-
const layout = layoutOf(harness);
|
|
1502
|
-
const runs = layout.pluginHookEvents;
|
|
1503
|
-
for (const p of runs ? plugins : []) {
|
|
1504
|
-
const event = p.hooks.events.find((e) => !runs?.includes(e));
|
|
1505
|
-
const type = p.hooks.otherTypes[0];
|
|
1506
|
-
if (!event && !type) continue;
|
|
1507
|
-
const rest = p.kept.filter((part) => part !== "hooks");
|
|
1508
|
-
const fix = rest.length ? `use only: [${quoted(rest)}], or leave` : "leave";
|
|
1509
|
-
return event ? `${name} can't run the ${p.name} plugin's ${event} hooks (hooks/hooks.json): it runs ${runs?.join(", ")} hooks; ${fix} the plugin out of this cell` : `${name} can't run the ${p.name} plugin's ${type} hooks (hooks/hooks.json): it runs only command hooks; ${fix} the plugin out of this cell`;
|
|
1510
|
-
}
|
|
1511
|
-
const servers = Object.fromEntries(plugins.flatMap((p) => Object.entries(p.mcp).map(([server, s]) => [pluginServerName(p.name, server), plannedMcpServer(s)])));
|
|
1512
|
-
if (agent && Object.keys(servers).length) {
|
|
1513
|
-
const alongside = agent.pluginPlacement === "flat" ? Object.fromEntries(own.mcp.map((n) => [n, {
|
|
1514
|
-
transport: "stdio",
|
|
1515
|
-
command: "true",
|
|
1516
|
-
args: [],
|
|
1517
|
-
env: {}
|
|
1518
|
-
}])) : {};
|
|
1519
|
-
try {
|
|
1520
|
-
agent.mcpServers({
|
|
1521
|
-
...alongside,
|
|
1522
|
-
...servers
|
|
1523
|
-
});
|
|
1524
|
-
} catch (error) {
|
|
1525
|
-
if (error instanceof InfraError) return error.message;
|
|
1526
|
-
throw error;
|
|
1527
|
-
}
|
|
1528
|
-
}
|
|
1529
|
-
for (const p of plugins) for (const server of Object.keys(p.mcp)) {
|
|
1530
|
-
const key = pluginServerName(p.name, server);
|
|
1531
|
-
if (own.mcp.includes(key)) return `the ${p.name} plugin's MCP server "${server}" would join mcp as ${key}, which mcp already names: rename mcp["${key}"]`;
|
|
1532
|
-
}
|
|
1533
|
-
if (layout.pluginPlacement === "namespaced") {
|
|
1534
|
-
for (const p of plugins) {
|
|
1535
|
-
if (!p.kept.includes("mcp")) continue;
|
|
1536
|
-
for (const [server, s] of Object.entries(p.mcp)) {
|
|
1537
|
-
const secret = hostVariable(s);
|
|
1538
|
-
if (!secret) continue;
|
|
1539
|
-
const rest = p.kept.filter((part) => part !== "mcp");
|
|
1540
|
-
return `${name} loads the ${p.name} plugin's MCP server "${server}" itself, so reeval can't give it ${secret} from the host: use only: [${quoted(rest)}] and declare the server in mcp, where reeval fills it in`;
|
|
1541
|
-
}
|
|
1542
|
-
}
|
|
1543
|
-
if (!(agent ?? mimics)?.pluginServersByName) return void 0;
|
|
1544
|
-
const byName = new Map(own.mcp.map((n) => [n, `mcp.${n}`]));
|
|
1545
|
-
for (const p of plugins) for (const server of Object.keys(p.mcp)) {
|
|
1546
|
-
const other = byName.get(server);
|
|
1547
|
-
if (other) return `${name} names the ${p.name} plugin's MCP server "${server}" by its own name, as it does ${other}, and keeps only one: rename one, or leave mcp out of the plugin with only`;
|
|
1548
|
-
byName.set(server, `the ${p.name} plugin's MCP server "${server}"`);
|
|
1549
|
-
}
|
|
1550
|
-
return;
|
|
1551
|
-
}
|
|
1552
|
-
const serverOwner = new Map(own.mcp.map((n) => [n, `mcp.${n}`]));
|
|
1553
|
-
for (const p of plugins) for (const server of Object.keys(p.mcp)) {
|
|
1554
|
-
const other = serverOwner.get(server);
|
|
1555
|
-
if (other) return `the ${p.name} plugin's MCP server "${server}" clashes with ${other}`;
|
|
1556
|
-
serverOwner.set(server, `the ${p.name} plugin's MCP server "${server}"`);
|
|
1557
|
-
}
|
|
1558
|
-
const skillOwner = new Map(own.skills.map((s) => [s.name, `${s.call}'s "${s.name}"`]));
|
|
1559
|
-
for (const p of plugins) for (const { name: skill } of p.skills) {
|
|
1560
|
-
const other = skillOwner.get(skill);
|
|
1561
|
-
if (other) return `the ${p.name} plugin's skill "${skill}" and ${other} would both be placed at ~/${layout.harnessSkillDir}/${skill}: rename one, or narrow it with only or exclude`;
|
|
1562
|
-
skillOwner.set(skill, `the ${p.name} plugin's skill "${skill}"`);
|
|
1563
|
-
}
|
|
1564
|
-
for (const part of ["agents", "commands"]) {
|
|
1565
|
-
const dir = layout.pluginUnpack?.[part];
|
|
1566
|
-
if (!dir) continue;
|
|
1567
|
-
const owner = /* @__PURE__ */ new Map();
|
|
1568
|
-
for (const p of plugins) for (const item of p[part]) {
|
|
1569
|
-
const other = owner.get(item);
|
|
1570
|
-
const what = part === "agents" ? "subagent" : "command";
|
|
1571
|
-
if (other) return `the ${p.name} plugin's ${what} "${item}" and the ${other} plugin's would both be placed at ${dir}/${item}.md: rename one, or narrow one with only`;
|
|
1572
|
-
owner.set(item, p.name);
|
|
1573
|
-
}
|
|
1574
|
-
}
|
|
1575
|
-
const rules = layout.pluginUnpack?.rules;
|
|
1576
|
-
const ruleOwner = /* @__PURE__ */ new Map();
|
|
1577
|
-
for (const p of rules ? plugins : []) for (const rule of p.rules) {
|
|
1578
|
-
const file = `${p.name}-${rule.replace(/\.mdc?$/, "")}.mdc`;
|
|
1579
|
-
const what = `the ${p.name} plugin's rule "${rule}"`;
|
|
1580
|
-
const other = ruleOwner.get(file);
|
|
1581
|
-
if (other) return `${what} and ${other} would both be placed at ${rules}/${file}: rename one, or narrow one with only`;
|
|
1582
|
-
ruleOwner.set(file, what);
|
|
1583
|
-
}
|
|
1584
|
-
}
|
|
1585
|
-
/**
|
|
1586
|
-
* A real agent's catalog entry, with what its model's provider decides, or its gateway's route:
|
|
1587
|
-
* on Vertex, the attempt's `vertex()` settings decide the project and region.
|
|
1588
|
-
*/
|
|
1589
|
-
function agentFor(h, vertex) {
|
|
1590
|
-
const agent = AGENTS[h.kind];
|
|
1591
|
-
if (h.gateway === "vertex") {
|
|
1592
|
-
const route = VERTEX_ROUTES[h.kind];
|
|
1593
|
-
if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on Vertex AI`);
|
|
1594
|
-
if (!vertex) throw new InfraError(`${agent.name} on Vertex AI has no vertex() settings`);
|
|
1595
|
-
return {
|
|
1596
|
-
...agent,
|
|
1597
|
-
...route(vertex)
|
|
1598
|
-
};
|
|
1599
|
-
}
|
|
1600
|
-
if (isRoutingGateway(h.gateway)) {
|
|
1601
|
-
const gateway = ROUTING_GATEWAYS[h.gateway];
|
|
1602
|
-
const route = gateway.routes[h.kind];
|
|
1603
|
-
if (!route) throw new InfraError(`reeval doesn't run ${agent.name} on ${gateway.name}`);
|
|
1604
|
-
return {
|
|
1605
|
-
...agent,
|
|
1606
|
-
...route(process.env)
|
|
1607
|
-
};
|
|
1608
|
-
}
|
|
1609
|
-
return agent.forModel ? {
|
|
1610
|
-
...agent,
|
|
1611
|
-
...agent.forModel(h.model)
|
|
1612
|
-
} : agent;
|
|
1613
|
-
}
|
|
1614
|
-
/** The catalog entry for a harness's agent; undefined for the fake harness. */
|
|
1615
|
-
const agentOf = (h, vertex) => h.kind === "fake" ? void 0 : agentFor(h, vertex);
|
|
1616
|
-
/**
|
|
1617
|
-
* The host variable a harness authenticates with; undefined for the fake harness, and on Vertex,
|
|
1618
|
-
* whose token comes from Application Default Credentials.
|
|
1619
|
-
*/
|
|
1620
|
-
const credentialOf = (h) => h.kind === "fake" || h.gateway === "vertex" ? void 0 : agentFor(h).credential;
|
|
1621
|
-
/** The fake agent's own layout. */
|
|
1622
|
-
const FAKE_LAYOUT = {
|
|
1623
|
-
rulesFile: "AGENTS.md",
|
|
1624
|
-
workspaceSkillDir: ".agents/skills",
|
|
1625
|
-
harnessSkillDir: ".agents/skills",
|
|
1626
|
-
pluginPlacement: "flat"
|
|
1627
|
-
};
|
|
1628
|
-
/** Where a harness's agent reads rules and skills; a fake one with `layout` mimics that agent. */
|
|
1629
|
-
function layoutOf(descriptor) {
|
|
1630
|
-
if (descriptor.kind !== "fake") return LAYOUTS[descriptor.kind];
|
|
1631
|
-
return descriptor.layout ? LAYOUTS[descriptor.layout] : FAKE_LAYOUT;
|
|
1632
|
-
}
|
|
1633
|
-
/**
|
|
1634
|
-
* The hosts a harness's agent is known to try in a session with or without plugins; a fake agent
|
|
1635
|
-
* with `layout` expects that agent's. On Vertex, the attempt's `vertex()` settings pick the route.
|
|
1636
|
-
*/
|
|
1637
|
-
function expectedBlockedFor(descriptor, plugins, vertex) {
|
|
1638
|
-
const expected = descriptor.kind !== "fake" ? agentFor(descriptor, vertex).expectedBlocked : descriptor.layout ? AGENTS[descriptor.layout].expectedBlocked : void 0;
|
|
1639
|
-
return expected?.withPlugins && !plugins ? void 0 : expected;
|
|
1640
|
-
}
|
|
1641
|
-
/**
|
|
1642
|
-
* How the APIs behind the harnesses say they refused a key: an HTTP 401, Anthropic's and OpenAI's
|
|
1643
|
-
* wording, Google's (which answers 400), and OpenCode's for any provider's 401 or 403. A bare 403 isn't here: the gateway and MCP servers send
|
|
1644
|
-
* those for reasons that have nothing to do with the harness's key. Grok Build checks its key as
|
|
1645
|
-
* it starts and, when xAI refuses it (a 400 or 401, not an outage), offers no `xai.api_key`
|
|
1646
|
-
* sign-in, which is all `@ai-sdk/harness-acp` reports.
|
|
1647
|
-
*/
|
|
1648
|
-
const REFUSED_KEY = /\bHTTP 401\b|\b401\b.{0,40}\bunauthori[sz]ed\b|\bunauthori[sz]ed\b.{0,40}\b401\b|\b(?:invalid|incorrect) (?:x-)?api[ _-]?key\b|\bapi key not valid\b|\bAPI_KEY_INVALID\b|\bauthentication_error\b|\bprovider authentication required\b|\bauthentication method "xai\.api_key" is not advertised\b/i;
|
|
1649
|
-
/**
|
|
1650
|
-
* What to do when the harness's API refused its key, naming the variable; else undefined. The
|
|
1651
|
-
* error's wording says so for some CLIs; `refused` is the egress record's word for every one: a
|
|
1652
|
-
* brokered request the API answered with 401, whatever the CLI made of it.
|
|
1653
|
-
*/
|
|
1654
|
-
function authHint(agent, message, refused = false) {
|
|
1655
|
-
if (!refused && !REFUSED_KEY.test(message)) return void 0;
|
|
1656
|
-
if (agent.token) return `The API rejected the access token reeval minted from ${agent.credential}: check gcloud auth application-default login (or GOOGLE_APPLICATION_CREDENTIALS) and the project's Vertex AI access`;
|
|
1657
|
-
return keyRejected(agent.credential);
|
|
1658
|
-
}
|
|
1659
|
-
/** What to do when the API refused the key in `credential`, naming it. */
|
|
1660
|
-
const keyRejected = (credential) => `The API rejected ${credential}: set a valid key in the environment or .env.local`;
|
|
1661
|
-
/**
|
|
1662
|
-
* The agent's credential from the host, as the adapter's explicit `auth`. Fails closed. Empty for
|
|
1663
|
-
* a brokered token, which the adapter never sees.
|
|
1664
|
-
*/
|
|
1665
|
-
function agentAuth(agent, env = process.env) {
|
|
1666
|
-
if (agent.token) return {};
|
|
1667
|
-
const value = env[agent.credential];
|
|
1668
|
-
if (!value) throw new InfraError(`${agent.credential} isn't set, so the ${agent.name} harness can't authenticate. Set it in the environment or .env.local.`);
|
|
1669
|
-
return {
|
|
1670
|
-
...agent.authEnv,
|
|
1671
|
-
[agent.authAs ?? agent.credential]: value
|
|
1672
|
-
};
|
|
1673
|
-
}
|
|
1674
|
-
function brokeringFor(h, vertex) {
|
|
1675
|
-
if (h.kind === "fake") {
|
|
1676
|
-
if (!h.brokers) return void 0;
|
|
1677
|
-
const { credential, endpoints } = AGENTS[h.layout ?? "cursor"];
|
|
1678
|
-
return {
|
|
1679
|
-
credential,
|
|
1680
|
-
endpoints
|
|
1681
|
-
};
|
|
1682
|
-
}
|
|
1683
|
-
const agent = agentOf(h, vertex);
|
|
1684
|
-
if (!agent) return void 0;
|
|
1685
|
-
const { credential, endpoints, tokens, servedModel, askedModel, modelResponses, upstream } = agent;
|
|
1686
|
-
const { localResponses } = agent;
|
|
1687
|
-
return {
|
|
1688
|
-
credential,
|
|
1689
|
-
endpoints,
|
|
1690
|
-
...tokens ? { tokens: {
|
|
1691
|
-
name: credential,
|
|
1692
|
-
...tokens
|
|
1693
|
-
} } : {},
|
|
1694
|
-
...servedModel ? { servedModel } : {},
|
|
1695
|
-
...askedModel ? { askedModel } : {},
|
|
1696
|
-
...modelResponses ? { modelResponses } : {},
|
|
1697
|
-
...upstream ? { upstream } : {},
|
|
1698
|
-
...localResponses?.length ? { localResponses } : {}
|
|
1699
|
-
};
|
|
1700
|
-
}
|
|
1701
584
|
// src/harness/effort.ts
|
|
1702
585
|
/**
|
|
1703
586
|
* The reasoning effort a harness asked for. Same user-facing name on every factory; the CLI's
|
|
@@ -1749,13 +632,6 @@ function resolvedEffortOf(models) {
|
|
|
1749
632
|
}
|
|
1750
633
|
// src/contract/attempt.ts
|
|
1751
634
|
const nullableNumber = z.number().nullable();
|
|
1752
|
-
const tokenUsageSchema = z.strictObject({
|
|
1753
|
-
input: z.number(),
|
|
1754
|
-
output: z.number(),
|
|
1755
|
-
cacheRead: z.number(),
|
|
1756
|
-
cacheWrite: z.number(),
|
|
1757
|
-
reasoning: nullableNumber
|
|
1758
|
-
});
|
|
1759
635
|
const fileChangeSchema = z.strictObject({
|
|
1760
636
|
path: z.string(),
|
|
1761
637
|
status: z.enum([
|
|
@@ -2229,7 +1105,8 @@ const cellSchema = z.strictObject({
|
|
|
2229
1105
|
environment: z.string(),
|
|
2230
1106
|
sandbox: z.string(),
|
|
2231
1107
|
role: cellRoleSchema,
|
|
2232
|
-
variant: z.string().optional()
|
|
1108
|
+
variant: z.string().optional(),
|
|
1109
|
+
traits: traitsSchema.optional()
|
|
2233
1110
|
});
|
|
2234
1111
|
z.strictObject({
|
|
2235
1112
|
type: z.literal("start"),
|
|
@@ -2444,15 +1321,15 @@ function questionShows(state, artifacts) {
|
|
|
2444
1321
|
const compareGateSchema = z.enum([
|
|
2445
1322
|
"variants",
|
|
2446
1323
|
"all",
|
|
2447
|
-
"
|
|
1324
|
+
"observe"
|
|
2448
1325
|
]);
|
|
2449
1326
|
/** The compare object, with environments checked by `environment`. */
|
|
2450
1327
|
const compareSchema = (environment) => z.strictObject({
|
|
2451
|
-
|
|
1328
|
+
control: environment,
|
|
2452
1329
|
variants: z.array(environment).min(1),
|
|
2453
1330
|
gate: compareGateSchema.optional(),
|
|
2454
1331
|
minLift: z.number().min(-1).max(1).optional(),
|
|
2455
|
-
|
|
1332
|
+
harnesses: z.array(z.union([z.string(), harnessDescriptorSchema])).optional()
|
|
2456
1333
|
});
|
|
2457
1334
|
compareSchema(z.custom());
|
|
2458
1335
|
const describe$1 = (value) => {
|
|
@@ -2470,14 +1347,17 @@ function explain(where) {
|
|
|
2470
1347
|
const [key, index] = issue.path;
|
|
2471
1348
|
const v = spec[key];
|
|
2472
1349
|
switch (key) {
|
|
2473
|
-
case "
|
|
1350
|
+
case "control":
|
|
1351
|
+
if (v === void 0) return noControl(where);
|
|
1352
|
+
if (Array.isArray(v)) return `compare.control in ${where} takes one environment (got ${v.map((e) => e?.name ?? describe$1(e)).join(", ")}). For a grid without a compare, use environments.`;
|
|
1353
|
+
return `compare.control in ${where} must be an environment from defineEnvironment (got ${describe$1(v)}). It is what each variant is measured against; variants are not layered on it.`;
|
|
2474
1354
|
case "gate": return `compare.gate in ${where} is ${oneOf(compareGateSchema.options)} (got ${describe$1(v)})`;
|
|
2475
1355
|
case "minLift": return `compare.minLift in ${where} is points from -1 to 1, e.g. minLift: 0.1 (got ${describe$1(v)})`;
|
|
2476
|
-
case "
|
|
1356
|
+
case "harnesses": {
|
|
2477
1357
|
const item = index === void 0 ? void 0 : v[index];
|
|
2478
1358
|
const kind = item?.kind;
|
|
2479
1359
|
const problem = kind && kind in FACTORY ? harnessProblem(item) : void 0;
|
|
2480
|
-
return problem ? `compare.
|
|
1360
|
+
return problem ? `compare.harnesses in ${where}: ${problem}` : `compare.harnesses in ${where} must be a list of harness ids, e.g. harnesses: ['cursor:composer-2.5'] (got ${describe$1(index === void 0 ? v : item)})`;
|
|
2481
1361
|
}
|
|
2482
1362
|
default: return index === void 0 ? `compare has no variants in ${where}. Give at least one environment.` : `compare.variants in ${where} must be environments from defineEnvironment (got ${describe$1(v[index])}). Variants share the eval's prompt and checks; if a variant needs different checks, write a separate eval.`;
|
|
2483
1363
|
}
|
|
@@ -2485,393 +1365,23 @@ function explain(where) {
|
|
|
2485
1365
|
}
|
|
2486
1366
|
/** The order the hand-written checks ran in, so the first refusal stays the same. */
|
|
2487
1367
|
const ORDER = [
|
|
2488
|
-
"
|
|
1368
|
+
"control",
|
|
2489
1369
|
"gate",
|
|
2490
1370
|
"minLift",
|
|
2491
|
-
"
|
|
1371
|
+
"harnesses",
|
|
2492
1372
|
"variants"
|
|
2493
1373
|
];
|
|
1374
|
+
const noControl = (where) => `compare in ${where} has no control. Name it: compare: { control: app, variants: [...] }.`;
|
|
2494
1375
|
/**
|
|
2495
1376
|
* What's wrong with a compare object whose environments `environment` checks, or undefined.
|
|
2496
1377
|
* `where` names the group or eval it's set on.
|
|
2497
1378
|
*/
|
|
2498
1379
|
function compareProblem(value, where, environment) {
|
|
2499
|
-
if (Array.isArray(value)) return `compare in ${where} must be an object, not a list. Use compare: {
|
|
2500
|
-
if (
|
|
1380
|
+
if (Array.isArray(value)) return `compare in ${where} must be an object, not a list. Use compare: { control, variants: [...] }.`;
|
|
1381
|
+
if (isObjectLike(value) && "baseline" in value) return `Unknown compare option "baseline" in ${where}. Did you mean "control"?`;
|
|
1382
|
+
if (!isObjectLike(value) || !Array.isArray(value.variants)) return `compare in ${where} takes { control, variants } (got ${describe$1(value)}). Use compare: { control: a, variants: [b, c], gate?, minLift?, harnesses? }.`;
|
|
2501
1383
|
return refusal(compareSchema(environment), value, explain(where), { order: ORDER });
|
|
2502
1384
|
}
|
|
2503
|
-
const POLL_MS = 200;
|
|
2504
|
-
/**
|
|
2505
|
-
* Prints `alive` while `pid` runs, then the sandbox's TCP tables, or `noproc` without them
|
|
2506
|
-
* (or without `tables`: the fake sandbox shares the host's, where other processes bind too).
|
|
2507
|
-
*/
|
|
2508
|
-
function probeScript(pid, tables = true) {
|
|
2509
|
-
return `${pid ? `kill -0 ${pid} 2>/dev/null && echo alive; ` : ""}${tables ? "if [ -r /proc/net/tcp ]; then cat /proc/net/tcp /proc/net/tcp6 2>/dev/null; else echo noproc; fi" : "echo noproc"}; exit 0`;
|
|
2510
|
-
}
|
|
2511
|
-
/** `0100007F` → `127.0.0.1`; 32 hex digits → an IPv6 address. Each 32-bit word is host-endian. */
|
|
2512
|
-
function procAddress(hex) {
|
|
2513
|
-
const bytes = (hex.match(/.{8}/g) ?? []).flatMap((w) => (w.match(/../g) ?? []).reverse().map((b) => Number.parseInt(b, 16)));
|
|
2514
|
-
if (bytes.length === 4) return bytes.join(".");
|
|
2515
|
-
if (bytes.slice(0, 10).every((b) => b === 0) && bytes[10] === 255 && bytes[11] === 255) return bytes.slice(12).join(".");
|
|
2516
|
-
const groups = [];
|
|
2517
|
-
for (let i = 0; i < 16; i += 2) groups.push(((bytes[i] ?? 0) << 8 | (bytes[i + 1] ?? 0)).toString(16));
|
|
2518
|
-
const full = groups.join(":");
|
|
2519
|
-
const longest = (full.match(/(^|:)0(:0)+(:|$)/g) ?? []).reduce((a, b) => b.length > a.length ? b : a, "");
|
|
2520
|
-
return longest ? full.replace(longest, "::") : full;
|
|
2521
|
-
}
|
|
2522
|
-
/** A listener the tunnel reaches: on loopback or every address, IPv4 or IPv6. */
|
|
2523
|
-
const reachable = (address) => address === "0.0.0.0" || address === "::" || address === "::1" || address.startsWith("127.");
|
|
2524
|
-
function parseProbe(stdout) {
|
|
2525
|
-
const lines = stdout.split("\n");
|
|
2526
|
-
const alive = lines.includes("alive");
|
|
2527
|
-
if (lines.includes("noproc")) return {
|
|
2528
|
-
proc: false,
|
|
2529
|
-
alive
|
|
2530
|
-
};
|
|
2531
|
-
const listeners = [];
|
|
2532
|
-
for (const l of lines) {
|
|
2533
|
-
const m = /^\s*\d+:\s+([0-9A-F]+):([0-9A-F]{4})\s+\S+\s+0A\s/i.exec(l);
|
|
2534
|
-
const address = m ? procAddress(m[1]) : "";
|
|
2535
|
-
if (m && reachable(address)) listeners.push({
|
|
2536
|
-
port: Number.parseInt(m[2], 16),
|
|
2537
|
-
address
|
|
2538
|
-
});
|
|
2539
|
-
}
|
|
2540
|
-
return {
|
|
2541
|
-
proc: true,
|
|
2542
|
-
alive,
|
|
2543
|
-
listeners
|
|
2544
|
-
};
|
|
2545
|
-
}
|
|
2546
|
-
/** The ports listening now that weren't before the command started, ascending. */
|
|
2547
|
-
const newPorts = (listeners, before) => [...new Set(listeners.map((l) => l.port).filter((p) => !before.has(p)))].sort((a, b) => a - b);
|
|
2548
|
-
/** The address to record for `port`: IPv4 first, since that's where the tunnel connects first. */
|
|
2549
|
-
const addressFor = (listeners, port) => {
|
|
2550
|
-
const all = listeners.filter((l) => l.port === port).map((l) => l.address);
|
|
2551
|
-
return all.find((a) => !a.includes(":")) ?? all.find((a) => a === "::") ?? all[0] ?? "";
|
|
2552
|
-
};
|
|
2553
|
-
/** `want` is taken whoever listens on it: `servePort` found it free when the command started. */
|
|
2554
|
-
function decide(s) {
|
|
2555
|
-
const { listeners, alive } = s.probe;
|
|
2556
|
-
const ports = newPorts(listeners, s.before);
|
|
2557
|
-
if (listeners.some((l) => l.port === s.want)) return {
|
|
2558
|
-
kind: "port",
|
|
2559
|
-
port: s.want,
|
|
2560
|
-
address: addressFor(listeners, s.want)
|
|
2561
|
-
};
|
|
2562
|
-
const late = s.elapsedMs >= s.timeoutMs;
|
|
2563
|
-
if (!s.given && ports.length && (s.stableMs >= 3e3 || late || !alive)) {
|
|
2564
|
-
if (ports.length === 1) {
|
|
2565
|
-
const port = ports[0];
|
|
2566
|
-
return {
|
|
2567
|
-
kind: "port",
|
|
2568
|
-
port,
|
|
2569
|
-
address: addressFor(listeners, port)
|
|
2570
|
-
};
|
|
2571
|
-
}
|
|
2572
|
-
return {
|
|
2573
|
-
kind: "several",
|
|
2574
|
-
ports
|
|
2575
|
-
};
|
|
2576
|
-
}
|
|
2577
|
-
if (!alive) return { kind: "exited" };
|
|
2578
|
-
return late ? {
|
|
2579
|
-
kind: "timeout",
|
|
2580
|
-
ports
|
|
2581
|
-
} : { kind: "wait" };
|
|
2582
|
-
}
|
|
2583
|
-
const plain = (output) => output.replace(/\x1b\[[0-9;?]*[A-Za-z]/g, "");
|
|
2584
|
-
/** The URLs `output` prints on one of `ports`, as Vite's `Local: http://localhost:5173/`. */
|
|
2585
|
-
function printedUrls(output, ports) {
|
|
2586
|
-
const found = /* @__PURE__ */ new Map();
|
|
2587
|
-
for (const m of plain(output).matchAll(/https?:\/\/(?:\[[0-9a-f:]+\]|[^\s/:[\]]+):(\d+)[^\s'"]*/gi)) {
|
|
2588
|
-
const port = Number(m[1]);
|
|
2589
|
-
if (ports.includes(port) && !found.has(m[0])) found.set(m[0], port);
|
|
2590
|
-
}
|
|
2591
|
-
return [...found].map(([url, port]) => ({
|
|
2592
|
-
url,
|
|
2593
|
-
port
|
|
2594
|
-
}));
|
|
2595
|
-
}
|
|
2596
|
-
/** Whether something on the host accepts a TCP connection on `port`, IPv4 or IPv6 loopback. */
|
|
2597
|
-
async function hostListening(port) {
|
|
2598
|
-
for (const host of ["127.0.0.1", "::1"]) if (await new Promise((resolve) => {
|
|
2599
|
-
const socket = connect({
|
|
2600
|
-
port,
|
|
2601
|
-
host
|
|
2602
|
-
});
|
|
2603
|
-
socket.once("connect", () => {
|
|
2604
|
-
socket.destroy();
|
|
2605
|
-
resolve(true);
|
|
2606
|
-
});
|
|
2607
|
-
socket.once("error", () => resolve(false));
|
|
2608
|
-
})) return host;
|
|
2609
|
-
}
|
|
2610
|
-
/** How many times `c.serve` allocates `PORT` again when the sandbox already listens on it. */
|
|
2611
|
-
const ALLOCATE_TRIES = 5;
|
|
2612
|
-
/**
|
|
2613
|
-
* The port `c.serve` sets as `PORT` and waits on: `port`, or one from `allocate` that nothing
|
|
2614
|
-
* listens on yet. Throws when something already listens on `port`, since that server, not the
|
|
2615
|
-
* command's, would answer the checks: c.serve only stops what it started.
|
|
2616
|
-
*/
|
|
2617
|
-
async function servePort(command, port, allocate, taken) {
|
|
2618
|
-
if (port === void 0) {
|
|
2619
|
-
for (let i = 0; i < ALLOCATE_TRIES; i++) {
|
|
2620
|
-
const want = await allocate();
|
|
2621
|
-
if (!await taken(want)) return want;
|
|
2622
|
-
}
|
|
2623
|
-
throw new Error(`c.serve(${JSON.stringify(command)}) found every port it allocated for PORT already in use in the sandbox`);
|
|
2624
|
-
}
|
|
2625
|
-
if (await taken(port)) throw new Error(`c.serve(${JSON.stringify(command)}, ${port}): port ${port} was already in use before c.serve started (a server the agent left running?), so the checks would reach that server, not this command's. Stop it first, or drop the port to serve on PORT`);
|
|
2626
|
-
return port;
|
|
2627
|
-
}
|
|
2628
|
-
const sleep = (ms, signal) => new Promise((resolve) => {
|
|
2629
|
-
const done = () => {
|
|
2630
|
-
clearTimeout(timer);
|
|
2631
|
-
signal?.removeEventListener("abort", done);
|
|
2632
|
-
resolve();
|
|
2633
|
-
};
|
|
2634
|
-
const timer = setTimeout(done, ms);
|
|
2635
|
-
signal?.addEventListener("abort", done, { once: true });
|
|
2636
|
-
});
|
|
2637
|
-
const listOf = (items) => items.length > 1 ? `${items.slice(0, -1).join(", ")} and ${items.at(-1)}` : String(items[0]);
|
|
2638
|
-
/**
|
|
2639
|
-
* Polls until something listens on `want`, or, without a given port, until the ports that are
|
|
2640
|
-
* new since the command started have settled on one. Throws, with the command's last output,
|
|
2641
|
-
* when they settle on several, it exits, or nothing listens in time.
|
|
2642
|
-
*/
|
|
2643
|
-
async function awaitListening(command, w) {
|
|
2644
|
-
const timeoutMs = w.timeoutMs ?? 12e4;
|
|
2645
|
-
const started = Date.now();
|
|
2646
|
-
let last = "";
|
|
2647
|
-
let changed = started;
|
|
2648
|
-
const output = async () => plain(await w.output().catch(() => ""));
|
|
2649
|
-
const fail = async (why, out) => {
|
|
2650
|
-
const tail = (out ?? await output()).trimEnd().split("\n").slice(-20).join("\n").trim();
|
|
2651
|
-
return /* @__PURE__ */ new Error(`c.serve(${JSON.stringify(command)}) ${why}${tail ? `. Its last output:\n${tail}` : ""}`);
|
|
2652
|
-
};
|
|
2653
|
-
for (;;) {
|
|
2654
|
-
w.signal?.throwIfAborted();
|
|
2655
|
-
const probe = await w.probe();
|
|
2656
|
-
const now = Date.now();
|
|
2657
|
-
if (!probe.proc) {
|
|
2658
|
-
const address = await (w.hostListening ?? hostListening)(w.want);
|
|
2659
|
-
if (address !== void 0) return {
|
|
2660
|
-
port: w.want,
|
|
2661
|
-
address,
|
|
2662
|
-
detected: false,
|
|
2663
|
-
readyMs: now - started
|
|
2664
|
-
};
|
|
2665
|
-
if (!probe.alive) throw await fail(`exited before it listened on ${w.want}`);
|
|
2666
|
-
if (now - started >= timeoutMs) throw await fail(`didn't listen on ${w.want} within ${timeoutMs / 1e3}s. This sandbox can't list the ports it listens on, so it must listen on PORT or the port passed`);
|
|
2667
|
-
} else {
|
|
2668
|
-
const key = newPorts(probe.listeners, w.before).join(",");
|
|
2669
|
-
if (key !== last) {
|
|
2670
|
-
last = key;
|
|
2671
|
-
changed = now;
|
|
2672
|
-
}
|
|
2673
|
-
const d = decide({
|
|
2674
|
-
want: w.want,
|
|
2675
|
-
given: w.given,
|
|
2676
|
-
probe,
|
|
2677
|
-
before: w.before,
|
|
2678
|
-
stableMs: now - changed,
|
|
2679
|
-
elapsedMs: now - started,
|
|
2680
|
-
timeoutMs
|
|
2681
|
-
});
|
|
2682
|
-
if (d.kind === "port") return {
|
|
2683
|
-
port: d.port,
|
|
2684
|
-
address: d.address,
|
|
2685
|
-
detected: d.port !== w.want,
|
|
2686
|
-
readyMs: now - started
|
|
2687
|
-
};
|
|
2688
|
-
if (d.kind === "several") {
|
|
2689
|
-
const out = await output();
|
|
2690
|
-
const urls = printedUrls(out, d.ports);
|
|
2691
|
-
const printed = urls.length ? ` It printed ${listOf(urls.map((u) => u.url))}.` : "";
|
|
2692
|
-
const hint = urls[0]?.port ?? d.ports[0];
|
|
2693
|
-
throw await fail(`listens on ports ${listOf(d.ports)}, and not on PORT (${w.want}).${printed} Pass the one it serves on: c.serve(${JSON.stringify(command)}, ${hint})`, out);
|
|
2694
|
-
}
|
|
2695
|
-
if (d.kind === "exited") throw await fail("exited before it listened on a port");
|
|
2696
|
-
if (d.kind === "timeout") {
|
|
2697
|
-
const on = d.ports.length ? `; ${d.ports.length > 1 ? "ports" : "port"} ${listOf(d.ports)} started listening` : "";
|
|
2698
|
-
throw await fail(`didn't listen on ${w.given ? `the port passed (${w.want})` : "a port"} within ${timeoutMs / 1e3}s${on}`);
|
|
2699
|
-
}
|
|
2700
|
-
}
|
|
2701
|
-
await sleep(POLL_MS, w.signal);
|
|
2702
|
-
}
|
|
2703
|
-
}
|
|
2704
|
-
// src/sandbox/fake.ts
|
|
2705
|
-
/** Commands run on the host's own network, so the fake sandbox can neither restrict nor see it. */
|
|
2706
|
-
const FAKE_EGRESS = {
|
|
2707
|
-
enforce: "none",
|
|
2708
|
-
record: false
|
|
2709
|
-
};
|
|
2710
|
-
/**
|
|
2711
|
-
* The POSIX shell commands run in: `sh`, which Windows lacks, so there the `sh.exe` on PATH or
|
|
2712
|
-
* Git for Windows'.
|
|
2713
|
-
*/
|
|
2714
|
-
function posixShell(platform = process.platform, env = process.env, exists = existsSync) {
|
|
2715
|
-
if (platform !== "win32") return "sh";
|
|
2716
|
-
const onPath = (env.PATH ?? env.Path ?? "").split(win32.delimiter).filter(Boolean);
|
|
2717
|
-
const git = [
|
|
2718
|
-
env.ProgramFiles,
|
|
2719
|
-
env["ProgramFiles(x86)"],
|
|
2720
|
-
env.LOCALAPPDATA && win32.join(env.LOCALAPPDATA, "Programs")
|
|
2721
|
-
].filter((dir) => !!dir).map((dir) => win32.join(dir, "Git", "bin"));
|
|
2722
|
-
const found = [...onPath, ...git].map((dir) => win32.join(dir, "sh.exe")).find(exists);
|
|
2723
|
-
if (found) return found;
|
|
2724
|
-
throw new UsageError("The fake sandbox runs commands with a POSIX sh, and there's none on this machine. Install Git for Windows (https://git-scm.com/download/win), which brings sh.exe, or run the evals in docker().");
|
|
2725
|
-
}
|
|
2726
|
-
const EXEC_TAG = "REEVAL_FAKE_EXEC";
|
|
2727
|
-
const KILL_TAGGED = `for pass in 1 2 3 4 5; do
|
|
2728
|
-
found=
|
|
2729
|
-
for f in $(grep -lzx "${EXEC_TAG}=$1" /proc/[0-9]*/environ 2>/dev/null); do
|
|
2730
|
-
p=\${f#/proc/}; kill -9 "\${p%/environ}" 2>/dev/null; found=1
|
|
2731
|
-
done
|
|
2732
|
-
[ -n "$found" ] || break
|
|
2733
|
-
done`;
|
|
2734
|
-
/**
|
|
2735
|
-
* Stops a command's MSYS processes on Windows. `taskkill /t` doesn't reach them: Git's `sh.exe`
|
|
2736
|
-
* starts bash, whose fork and exec leave each new process's Windows parent gone, so a timed-out
|
|
2737
|
-
* `sleep 30` would run on, holding the command's pipes and working directory.
|
|
2738
|
-
*/
|
|
2739
|
-
function killTagged(sh, tag) {
|
|
2740
|
-
runSh(sh, KILL_TAGGED, tag);
|
|
2741
|
-
}
|
|
2742
|
-
/**
|
|
2743
|
-
* Kills host processes whose cwd is the sandbox, including native `node.exe` `c.serve` starts.
|
|
2744
|
-
* Match only the unique `reeval-sandbox-*` folder name: Git Bash `pwd` and node.exe's
|
|
2745
|
-
* `/proc/cwd` disagree (`/tmp/...` vs `D:\\Users\\...\\Temp\\...`), and an empty
|
|
2746
|
-
* `"$prefix"/*` is `/*`, which SIGKILLs the Windows runner (exit 2304).
|
|
2747
|
-
*/
|
|
2748
|
-
const KILL_UNDER = `cd "$1" || exit 0
|
|
2749
|
-
base=$(pwd -P 2>/dev/null || pwd)
|
|
2750
|
-
name=\${base##*/}
|
|
2751
|
-
case "$name" in
|
|
2752
|
-
reeval-sandbox-?*) ;;
|
|
2753
|
-
*) exit 0 ;;
|
|
2754
|
-
esac
|
|
2755
|
-
for f in /proc/[0-9]*/cwd; do
|
|
2756
|
-
t=$(readlink "$f" 2>/dev/null) || continue
|
|
2757
|
-
case "$t" in
|
|
2758
|
-
*"$name"*) ;;
|
|
2759
|
-
*) continue ;;
|
|
2760
|
-
esac
|
|
2761
|
-
p=\${f#/proc/}; p=\${p%/cwd}
|
|
2762
|
-
[ -n "$p" ] || continue
|
|
2763
|
-
kill -9 "$p" 2>/dev/null
|
|
2764
|
-
wp=$(tr -d '\\r\\n' < /proc/$p/winpid 2>/dev/null)
|
|
2765
|
-
[ -n "$wp" ] && taskkill /F /PID "$wp" >/dev/null 2>&1
|
|
2766
|
-
done`;
|
|
2767
|
-
function runSh(sh, script, arg) {
|
|
2768
|
-
return new Promise((resolve) => {
|
|
2769
|
-
const child = spawn(sh, [
|
|
2770
|
-
"-c",
|
|
2771
|
-
script,
|
|
2772
|
-
"sh",
|
|
2773
|
-
arg
|
|
2774
|
-
], {
|
|
2775
|
-
stdio: "ignore",
|
|
2776
|
-
windowsHide: true
|
|
2777
|
-
});
|
|
2778
|
-
const done = () => resolve();
|
|
2779
|
-
child.on("error", done);
|
|
2780
|
-
child.on("close", done);
|
|
2781
|
-
});
|
|
2782
|
-
}
|
|
2783
|
-
/** Windows refuses to delete a directory a live process still has as cwd; retry after kills. */
|
|
2784
|
-
async function removeTree(path) {
|
|
2785
|
-
const deadline = Date.now() + 8e3;
|
|
2786
|
-
for (;;) {
|
|
2787
|
-
try {
|
|
2788
|
-
await rm(path, {
|
|
2789
|
-
recursive: true,
|
|
2790
|
-
force: true
|
|
2791
|
-
});
|
|
2792
|
-
if (!existsSync(path)) return;
|
|
2793
|
-
} catch {}
|
|
2794
|
-
if (Date.now() >= deadline) {
|
|
2795
|
-
await rm(path, {
|
|
2796
|
-
recursive: true,
|
|
2797
|
-
force: true
|
|
2798
|
-
}).catch(() => {});
|
|
2799
|
-
return;
|
|
2800
|
-
}
|
|
2801
|
-
await new Promise((r) => setTimeout(r, 50));
|
|
2802
|
-
}
|
|
2803
|
-
}
|
|
2804
|
-
/**
|
|
2805
|
-
* A temp directory on the host. Commands run unisolated on the host shell, so this is
|
|
2806
|
-
* only meant for tests and for the fake harness.
|
|
2807
|
-
*/
|
|
2808
|
-
async function createFakeSandbox(descriptor, env = {}) {
|
|
2809
|
-
const sh = posixShell();
|
|
2810
|
-
const parent = descriptor.dir ?? tmpdir();
|
|
2811
|
-
await mkdir(parent, { recursive: true });
|
|
2812
|
-
const base = await mkdtemp(join(parent, "reeval-sandbox-"));
|
|
2813
|
-
const workdir = join(base, "work");
|
|
2814
|
-
await mkdir(workdir);
|
|
2815
|
-
const stateDir = `${base}.state`;
|
|
2816
|
-
await mkdir(stateDir);
|
|
2817
|
-
let destroyed;
|
|
2818
|
-
const resolve = (path) => path.startsWith("~/") ? join(base, path.slice(2)) : join(workdir, path);
|
|
2819
|
-
return {
|
|
2820
|
-
workdir,
|
|
2821
|
-
stateDir,
|
|
2822
|
-
execOwn: (command, { signal } = {}) => run(sh, ["-c", command], {
|
|
2823
|
-
cwd: workdir,
|
|
2824
|
-
env: {
|
|
2825
|
-
HOME: base,
|
|
2826
|
-
...env
|
|
2827
|
-
},
|
|
2828
|
-
signal
|
|
2829
|
-
}),
|
|
2830
|
-
exec: (command, { cwd, env: extra, signal } = {}) => {
|
|
2831
|
-
const tag = process.platform === "win32" && signal ? randomUUID() : void 0;
|
|
2832
|
-
const stop = () => killTagged(sh, tag);
|
|
2833
|
-
if (tag) signal?.addEventListener("abort", stop, { once: true });
|
|
2834
|
-
return run(sh, ["-c", cwd ? `cd ${shellQuote(cwd)} && ${command}` : command], {
|
|
2835
|
-
cwd: workdir,
|
|
2836
|
-
env: {
|
|
2837
|
-
HOME: base,
|
|
2838
|
-
...env,
|
|
2839
|
-
...extra,
|
|
2840
|
-
...tag ? { [EXEC_TAG]: tag } : {}
|
|
2841
|
-
},
|
|
2842
|
-
signal
|
|
2843
|
-
}).finally(() => signal?.removeEventListener("abort", stop));
|
|
2844
|
-
},
|
|
2845
|
-
home: base,
|
|
2846
|
-
readFile: (path) => readFile(resolve(path), "utf8").catch(() => null),
|
|
2847
|
-
getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}://${await hostListening(port) === "::1" ? "[::1]" : "127.0.0.1"}:${port}` }),
|
|
2848
|
-
async writeFile(path, content) {
|
|
2849
|
-
const target = resolve(path);
|
|
2850
|
-
await mkdir(dirname(target), { recursive: true });
|
|
2851
|
-
await writeFile(target, content);
|
|
2852
|
-
},
|
|
2853
|
-
destroy: () => {
|
|
2854
|
-
destroyed ??= (async () => {
|
|
2855
|
-
await rm(stateDir, {
|
|
2856
|
-
recursive: true,
|
|
2857
|
-
force: true
|
|
2858
|
-
}).catch(() => {});
|
|
2859
|
-
if (process.platform === "win32") {
|
|
2860
|
-
if (existsSync(base)) await runSh(sh, KILL_UNDER, base);
|
|
2861
|
-
await removeTree(base);
|
|
2862
|
-
return;
|
|
2863
|
-
}
|
|
2864
|
-
await rm(base, {
|
|
2865
|
-
recursive: true,
|
|
2866
|
-
force: true
|
|
2867
|
-
}).catch(() => {});
|
|
2868
|
-
})().finally(() => {
|
|
2869
|
-
destroyed = void 0;
|
|
2870
|
-
});
|
|
2871
|
-
return destroyed;
|
|
2872
|
-
}
|
|
2873
|
-
};
|
|
2874
|
-
}
|
|
2875
1385
|
// src/ui/mode.ts
|
|
2876
1386
|
const set = (env, name) => {
|
|
2877
1387
|
const value = env[name];
|
|
@@ -2944,11 +1454,7 @@ function resolveOutput(input) {
|
|
|
2944
1454
|
* `egress: { allow: [...DEFAULT_EGRESS_ALLOW, 'api.example.com'] }`.
|
|
2945
1455
|
*/
|
|
2946
1456
|
const DEFAULT_EGRESS_ALLOW = PACKAGE_HOSTS;
|
|
2947
|
-
|
|
2948
|
-
const SANDBOX_EGRESS = {
|
|
2949
|
-
docker: DOCKER_EGRESS,
|
|
2950
|
-
fake: FAKE_EGRESS
|
|
2951
|
-
};
|
|
1457
|
+
Object.fromEntries(Object.entries(SANDBOX_KINDS).map(([kind, k]) => [kind, k.capabilities.egress]));
|
|
2952
1458
|
/** The policy an attempt runs under when nothing sets one. */
|
|
2953
1459
|
const DEFAULT_EGRESS = "proxy";
|
|
2954
1460
|
/** Whether the policy allows only some hosts: `deny-all` and allowlists. */
|
|
@@ -3031,13 +1537,16 @@ const LEVELS = [
|
|
|
3031
1537
|
];
|
|
3032
1538
|
/** Why the sandbox can't enforce the attempt's policy, or undefined when it can. */
|
|
3033
1539
|
function unenforceable(planned, sandbox, where) {
|
|
1540
|
+
const capabilities = capabilitiesOf(sandbox);
|
|
1541
|
+
const can = capabilities.egress.enforce;
|
|
1542
|
+
if (planned.policy === "proxy" && capabilities.isolated && can === "none") return `${where} runs under egress 'proxy', where the egress gateway is the only way out, but the ${sandbox.kind} sandbox enforces no egress policy: its traffic that ignores the proxy goes out directly. Set egress: 'allow-all' to run it there, or run it in ${suggestedSandbox((c) => c.isolated && c.egress.enforce !== "none")}.`;
|
|
3034
1543
|
const needs = egressNeeds(planned);
|
|
3035
|
-
const can = SANDBOX_EGRESS[sandbox.kind].enforce;
|
|
3036
1544
|
if (LEVELS.indexOf(can) >= LEVELS.indexOf(needs)) return void 0;
|
|
3037
1545
|
const hosts = allowedHosts(planned)?.join(", ");
|
|
3038
1546
|
const what = needs !== "allowlist" ? "turning egress off" : planned.policy === "deny-all" ? `a host allowlist for its automatic hosts (${hosts})` : `a host allowlist (${hosts})`;
|
|
3039
1547
|
const enforces = can === "none" ? "enforces no egress policy" : `only enforces ${can === "all-or-nothing" ? "'allow-all' or 'deny-all'" : can}`;
|
|
3040
|
-
const
|
|
1548
|
+
const enforcing = suggestedSandbox((c) => LEVELS.indexOf(c.egress.enforce) >= LEVELS.indexOf(needs));
|
|
1549
|
+
const hint = capabilities.isolated ? "" : ` (its commands run on the host). ${enforcing ? `Run it in ${enforcing}, or set` : "Set"} egress: 'allow-all'`;
|
|
3041
1550
|
return `${where} runs under egress ${describeEgress(planned.policy)}, which needs ${what}, but the ${sandbox.kind} sandbox ${enforces}${hint}.`;
|
|
3042
1551
|
}
|
|
3043
1552
|
const describeEgress = (policy) => typeof policy === "string" ? `'${policy}'` : `{ allow: [${policy.allow.map((h) => `'${h}'`).join(", ")}] }`;
|
|
@@ -3061,10 +1570,12 @@ function egressLabel({ policy, auto }) {
|
|
|
3061
1570
|
const own = policy === "deny-all" ? "deny-all" : policy.allow.length ? `allow ${policy.allow.join(", ")}` : "allow none";
|
|
3062
1571
|
return auto.length ? `${own} + auto: ${auto.join(", ")}` : own;
|
|
3063
1572
|
}
|
|
3064
|
-
|
|
3065
|
-
|
|
1573
|
+
// src/options.ts
|
|
1574
|
+
/** Every eval/group option key `checkOptions` accepts. The feature matrix holds each to an inventory entry. */
|
|
1575
|
+
const EVAL_OPTION_KEYS = [
|
|
1576
|
+
"harnesses",
|
|
3066
1577
|
"sandbox",
|
|
3067
|
-
"
|
|
1578
|
+
"environments",
|
|
3068
1579
|
"compare",
|
|
3069
1580
|
"runs",
|
|
3070
1581
|
"gate",
|
|
@@ -3079,12 +1590,14 @@ const KNOWN = [...[
|
|
|
3079
1590
|
"tags",
|
|
3080
1591
|
"revision",
|
|
3081
1592
|
"pin"
|
|
3082
|
-
]
|
|
1593
|
+
];
|
|
1594
|
+
const KNOWN = [...EVAL_OPTION_KEYS];
|
|
3083
1595
|
const SUGGEST = {
|
|
3084
|
-
|
|
1596
|
+
harness: "harnesses",
|
|
3085
1597
|
sandboxes: "sandbox",
|
|
3086
|
-
|
|
3087
|
-
baseline: "
|
|
1598
|
+
environment: "environments",
|
|
1599
|
+
baseline: "compare",
|
|
1600
|
+
control: "compare",
|
|
3088
1601
|
variants: "compare",
|
|
3089
1602
|
repeat: "runs",
|
|
3090
1603
|
repeats: "runs",
|
|
@@ -3119,47 +1632,27 @@ function mergeSnapshot(outer, inner) {
|
|
|
3119
1632
|
compress: inner.compress ?? outer.compress
|
|
3120
1633
|
};
|
|
3121
1634
|
}
|
|
3122
|
-
/** Throws unless `value` is `'all'` or `{ passRate }` with a rate in (0, 1]. */
|
|
1635
|
+
/** Throws unless `value` is `'all'`, `'observe'` or `{ passRate }` with a rate in (0, 1]. */
|
|
3123
1636
|
function checkGate(value, where) {
|
|
3124
1637
|
const problem = gateProblem(value, where);
|
|
3125
1638
|
if (problem) throw new UsageError(problem);
|
|
3126
1639
|
}
|
|
3127
1640
|
const PINNABLE = [
|
|
3128
|
-
"
|
|
1641
|
+
"harnesses",
|
|
3129
1642
|
"runs",
|
|
3130
1643
|
"compare"
|
|
3131
1644
|
];
|
|
3132
1645
|
const ID = /^[A-Za-z0-9._:@+\-=[\]]+$/;
|
|
3133
|
-
const sanitize = (text) => (text.split("/").at(-1) ?? text).replace(/[^A-Za-z0-9._:@-]+/g, "-");
|
|
3134
1646
|
function harnessId(h) {
|
|
3135
1647
|
if (h.id) return h.id;
|
|
3136
1648
|
const effort = harnessEffort(h);
|
|
3137
|
-
const suffix = effort ? `[effort=${
|
|
1649
|
+
const suffix = effort ? `[effort=${idPart(effort)}]` : "";
|
|
3138
1650
|
if (h.kind === "fake") return `fake${suffix}`;
|
|
3139
|
-
return `${h.model ? `${h.kind}:${
|
|
3140
|
-
}
|
|
3141
|
-
function installId(install) {
|
|
3142
|
-
if (!install?.length) return "";
|
|
3143
|
-
const key = JSON.stringify(install.map((step) => typeof step === "string" ? step : {
|
|
3144
|
-
run: step.run,
|
|
3145
|
-
timeout: step.timeout
|
|
3146
|
-
}));
|
|
3147
|
-
return `+${createHash("sha256").update(key).digest("hex").slice(0, 8)}`;
|
|
1651
|
+
return `${h.model ? `${h.kind}:${idPart(h.model)}` : h.kind}${suffix}`;
|
|
3148
1652
|
}
|
|
3149
1653
|
/** The provider a sandbox runs on: its kind, or `podman` for a Docker sandbox on Podman. */
|
|
3150
|
-
const sandboxProvider = (s) => s.
|
|
3151
|
-
|
|
3152
|
-
if (s.id) return s.id;
|
|
3153
|
-
if (s.kind === "fake") return "fake";
|
|
3154
|
-
const install = s.kind === "docker" ? installId(s.install) : "";
|
|
3155
|
-
const engine = s.engine ?? "docker";
|
|
3156
|
-
if (s.dockerfile) {
|
|
3157
|
-
const [file = "", dir = ""] = s.dockerfile.split(/[\\/]/).reverse();
|
|
3158
|
-
return `${engine}:${sanitize(file === "Dockerfile" ? dir || file : file)}${install}`;
|
|
3159
|
-
}
|
|
3160
|
-
const image = s.image ?? s.from;
|
|
3161
|
-
return image ? `${engine}:${sanitize(image)}${install}` : `${engine}${install}`;
|
|
3162
|
-
}
|
|
1654
|
+
const sandboxProvider = (s) => sandboxKind(s).provider(s);
|
|
1655
|
+
const sandboxId = (s) => s.id ?? sandboxKind(s).id(s);
|
|
3163
1656
|
/** The planned attempts' sandboxes by id, once each, or `fallback`'s when none is planned. */
|
|
3164
1657
|
function plannedSandboxIds(planned, fallback) {
|
|
3165
1658
|
const ids = new Set(planned.map(sandboxId));
|
|
@@ -3181,11 +1674,14 @@ const describe = (value) => {
|
|
|
3181
1674
|
}
|
|
3182
1675
|
return JSON.stringify(value) ?? String(value);
|
|
3183
1676
|
};
|
|
3184
|
-
/** `compare: false` and `
|
|
1677
|
+
/** `compare: false` and `environments` beside `compare` are never legal options. */
|
|
3185
1678
|
function refuseIllegalCompare(options, where) {
|
|
3186
|
-
if (options.compare === false) throw new UsageError(`compare: false can't be set in ${where}. A profile turns compare off with limit.compare: false, which runs the
|
|
3187
|
-
|
|
3188
|
-
|
|
1679
|
+
if (options.compare === false) throw new UsageError(`compare: false can't be set in ${where}. A profile turns compare off with limit.compare: false, which runs the control and strips the variants. On a group or eval, omit compare or pass compare: { control, variants }.`);
|
|
1680
|
+
if (options.environments !== void 0 && options.compare !== void 0) {
|
|
1681
|
+
const first = Array.isArray(options.environments) ? options.environments[0] : options.environments;
|
|
1682
|
+
const name = isObjectLike(first) && typeof first.name === "string" ? first.name : "app";
|
|
1683
|
+
throw new UsageError(`${where} sets both environments and compare. With compare, the control is compare.control: move it there, as compare: { control: ${name}, variants: [...] }.`);
|
|
1684
|
+
}
|
|
3189
1685
|
}
|
|
3190
1686
|
const isRevision = (value) => typeof value === "number" && Number.isSafeInteger(value) || typeof value === "string" && value.trim() !== "";
|
|
3191
1687
|
/** `runs: 3` or `runs: { bestOf: 5 }`, refused in the words of whichever it was meant to be. */
|
|
@@ -3216,7 +1712,7 @@ function checkOptions(options, where, allowed = KNOWN) {
|
|
|
3216
1712
|
if (allowed.includes(key)) continue;
|
|
3217
1713
|
const near = SUGGEST[key] ?? closest(key, allowed);
|
|
3218
1714
|
const value = options[key];
|
|
3219
|
-
const hint = key === "
|
|
1715
|
+
const hint = key === "cases" || key === "caseKey" ? ` ${key} belongs to one eval: evaluate(name, { cases: [...] }, async (c, item) => { … }).` : key === "where" ? " where is an option of evaluate() only, beside cases: evaluate(name, { cases: [...], where: ({ case, environment }) => … }, …)." : key === "workspace" && typeof value !== "boolean" ? " workspace is an environment's starting tree: defineEnvironment(name, { workspace: git(…) })." : near ? ` Did you mean "${near}"?` : "";
|
|
3220
1716
|
throw new UsageError(`Unknown option "${key}" in ${where}.${hint}`);
|
|
3221
1717
|
}
|
|
3222
1718
|
const o = options;
|
|
@@ -3234,32 +1730,26 @@ function checkOptions(options, where, allowed = KNOWN) {
|
|
|
3234
1730
|
});
|
|
3235
1731
|
if (judgeIssue) throw new UsageError(judgeIssue);
|
|
3236
1732
|
if (o.sandbox !== void 0) {
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
if (max !== void 0 && !(Number.isInteger(max) && max >= 1)) throw new UsageError(`sandbox.maxConcurrent in ${where} must be a whole number of at least 1 (got ${String(max)})`);
|
|
1733
|
+
if (!isSandboxKind(o.sandbox.kind)) throw new UsageError(`sandbox in ${where} must come from ${sandboxHelpers()} (got ${describe(o.sandbox)})`);
|
|
1734
|
+
const limit = maxConcurrentProblem(o.sandbox, where);
|
|
1735
|
+
if (limit) throw new UsageError(limit);
|
|
3241
1736
|
const problem = sandboxProblem(o.sandbox);
|
|
3242
1737
|
if (problem) throw new UsageError(`sandbox in ${where}: ${problem}`);
|
|
3243
1738
|
}
|
|
3244
|
-
if (o.
|
|
3245
|
-
if (Array.isArray(o.environment)) throw new UsageError(`environment takes one environment, not a list (${where}). To compare environments, use compare: { baseline, variants: [a, b] }. baseline is the control; variants are not layered on it.`);
|
|
3246
|
-
if (!isEnvironment(o.environment)) throw new UsageError(`environment in ${where} must come from defineEnvironment(…) (got ${describe(o.environment)})`);
|
|
3247
|
-
const problem = environmentDataProblem(o.environment, "environment", where);
|
|
3248
|
-
if (problem) throw new UsageError(problem);
|
|
3249
|
-
}
|
|
1739
|
+
if (o.environments !== void 0) checkEnvironments(o.environments, where);
|
|
3250
1740
|
refuseIllegalCompare(options, where);
|
|
3251
1741
|
if (o.compare !== void 0) {
|
|
3252
1742
|
const problem = compareProblem(o.compare, where, z.custom(isEnvironment));
|
|
3253
1743
|
if (problem) throw new UsageError(problem);
|
|
3254
|
-
const {
|
|
3255
|
-
for (const [label, env] of [["compare.
|
|
1744
|
+
const { control, variants } = o.compare;
|
|
1745
|
+
for (const [label, env] of [["compare.control", control], ...variants.map((v, i) => [`compare.variants[${i}]`, v])]) {
|
|
3256
1746
|
const bad = env && environmentDataProblem(env, label, where);
|
|
3257
1747
|
if (bad) throw new UsageError(bad);
|
|
3258
1748
|
}
|
|
3259
1749
|
}
|
|
3260
1750
|
if (o.pin !== void 0) {
|
|
3261
1751
|
const bad = Array.isArray(o.pin) ? o.pin.filter((k) => !PINNABLE.includes(k)) : [o.pin];
|
|
3262
|
-
if (bad.length) throw new UsageError(`pin in ${where} lists
|
|
1752
|
+
if (bad.length) throw new UsageError(`pin in ${where} lists harnesses, runs and/or compare (got ${bad.map((k) => JSON.stringify(k)).join(", ")})`);
|
|
3263
1753
|
}
|
|
3264
1754
|
if (o.capture !== void 0) checkCapture(o.capture, where);
|
|
3265
1755
|
const questionIssue = o.question !== void 0 && questionOptionsProblem(o.question, where);
|
|
@@ -3309,42 +1799,82 @@ function checkCaptureModel(value, where) {
|
|
|
3309
1799
|
if (typeof value !== "string" || !value.includes("/")) throw new UsageError(`capture.model in ${where} is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${JSON.stringify(value)})`);
|
|
3310
1800
|
if (isEvaluationModel(value)) throw new UsageError(`capture.model "${value}" in ${where} is an evaluation model: it scores yes/no questions and can't drive a capture. Use a language model that accepts images, e.g. 'google/gemini-3.8-flash'.`);
|
|
3311
1801
|
}
|
|
1802
|
+
/** Each environment `environments` names, as one, a list or an edit, comes from defineEnvironment. */
|
|
1803
|
+
function checkEnvironments(value, where) {
|
|
1804
|
+
if (typeof value === "function") return;
|
|
1805
|
+
let list;
|
|
1806
|
+
let label = "environments";
|
|
1807
|
+
if (isEnvironmentEdit(value)) {
|
|
1808
|
+
if ("add" in value && "only" in value) throw new UsageError(`environments in ${where} takes { add } or { only }, not both. Use add to append to the inherited environments, only to keep some of them.`);
|
|
1809
|
+
if ("only" in value) {
|
|
1810
|
+
const names = [value.only].flat();
|
|
1811
|
+
if (!names.length || names.some((n) => typeof n !== "string")) throw new UsageError(`environments { only } in ${where} lists environment names, e.g. { only: ['app'] } (got ${describe(value.only)})`);
|
|
1812
|
+
return;
|
|
1813
|
+
}
|
|
1814
|
+
list = [value.add].flat();
|
|
1815
|
+
label = "environments.add";
|
|
1816
|
+
} else list = [value].flat();
|
|
1817
|
+
if (list.length === 0) throw new UsageError(`${label} is an empty list in ${where}. Give at least one, or leave environments out.`);
|
|
1818
|
+
list.forEach((env, i) => {
|
|
1819
|
+
const at = Array.isArray(value) || label !== "environments" ? `${label}[${i}]` : label;
|
|
1820
|
+
if (!isEnvironment(env)) throw new UsageError(`${at} in ${where} must come from defineEnvironment(…) (got ${describe(env)})`);
|
|
1821
|
+
const problem = environmentDataProblem(env, at, where);
|
|
1822
|
+
if (problem) throw new UsageError(problem);
|
|
1823
|
+
});
|
|
1824
|
+
}
|
|
1825
|
+
const isEnvironmentEdit = (v) => isObjectLike(v) && !Array.isArray(v) && !isEnvironment(v) && ("add" in v || "only" in v);
|
|
1826
|
+
function applyEnvironments(inherited, value, where) {
|
|
1827
|
+
if (value === void 0) return inherited;
|
|
1828
|
+
const list = [typeof value === "function" ? value(inherited) : isEnvironmentEdit(value) ? editEnvironments(inherited, value, where) : value].flat();
|
|
1829
|
+
if (list.length === 0) throw new UsageError(`environments is an empty list in ${where}. Give at least one, or leave environments out.`);
|
|
1830
|
+
if (typeof value === "function") checkEnvironments(list, where);
|
|
1831
|
+
return list;
|
|
1832
|
+
}
|
|
1833
|
+
/** `{ add }` appends to the inherited environments; `{ only }` keeps those with these names. */
|
|
1834
|
+
function editEnvironments(inherited, edit, where) {
|
|
1835
|
+
if ("add" in edit) return [...inherited, ...[edit.add].flat()];
|
|
1836
|
+
const names = [edit.only].flat();
|
|
1837
|
+
const have = inherited.map((e) => e.name);
|
|
1838
|
+
const missing = names.filter((n) => !have.includes(n));
|
|
1839
|
+
if (missing.length) throw new UsageError(`environments { only: ${missing.map((n) => JSON.stringify(n)).join(", ")} } in ${where} names no inherited environment (inherited: ${have.join(", ") || "none"})`);
|
|
1840
|
+
return inherited.filter((e) => names.includes(e.name));
|
|
1841
|
+
}
|
|
3312
1842
|
function applyHarness(inherited, value, where) {
|
|
3313
1843
|
if (value === void 0) return inherited;
|
|
3314
1844
|
const next = typeof value === "function" ? value(inherited) : isHarnessEdit(value) ? editHarness(inherited, value, where) : value;
|
|
3315
1845
|
const list = Array.isArray(next) ? next : [next];
|
|
3316
|
-
if (list.length === 0) throw new UsageError(`
|
|
1846
|
+
if (list.length === 0) throw new UsageError(`harnesses is an empty list in ${where}. Give at least one, e.g. harnesses: ${HARNESS_EXAMPLE}`);
|
|
3317
1847
|
checkHarnesses(list, where);
|
|
3318
1848
|
return list;
|
|
3319
1849
|
}
|
|
3320
1850
|
const isHarnessEdit = (v) => typeof v === "object" && v !== null && !Array.isArray(v) && !("kind" in v) && ("add" in v || "only" in v);
|
|
3321
1851
|
/** `{ add }` appends to the inherited harnesses; `{ only }` keeps those with these ids. */
|
|
3322
1852
|
function editHarness(inherited, edit, where) {
|
|
3323
|
-
if ("add" in edit && "only" in edit) throw new UsageError(`
|
|
1853
|
+
if ("add" in edit && "only" in edit) throw new UsageError(`harnesses in ${where} takes { add } or { only }, not both. Use add to append to the inherited harnesses, only to keep some of them.`);
|
|
3324
1854
|
if ("add" in edit) return [...inherited, ...Array.isArray(edit.add) ? edit.add : [edit.add]];
|
|
3325
1855
|
const ids = Array.isArray(edit.only) ? edit.only : [edit.only];
|
|
3326
1856
|
const have = inherited.map(harnessId);
|
|
3327
1857
|
const missing = ids.filter((id) => typeof id !== "string" || !have.includes(id));
|
|
3328
|
-
if (missing.length) throw new UsageError(`
|
|
1858
|
+
if (missing.length) throw new UsageError(`harnesses { only: ${missing.map((id) => JSON.stringify(id)).join(", ")} } in ${where} names no inherited harness (inherited: ${have.join(", ")})`);
|
|
3329
1859
|
return inherited.filter((h) => ids.includes(harnessId(h)));
|
|
3330
1860
|
}
|
|
3331
1861
|
/** Each harness is one, with a valid `maxConcurrent`. */
|
|
3332
1862
|
function checkHarnesses(list, where) {
|
|
3333
1863
|
for (const item of list) {
|
|
3334
|
-
if (!isHarness(item)) throw new UsageError(`
|
|
1864
|
+
if (!isHarness(item)) throw new UsageError(`harnesses in ${where} must be like ${HARNESS_EXAMPLE} (got ${describe(item)})`);
|
|
3335
1865
|
if (item.kind === "opencode") {
|
|
3336
1866
|
const why = opencodeModelError(item.model);
|
|
3337
1867
|
if (why) throw new UsageError(`opencode() in ${where}: ${why}`);
|
|
3338
1868
|
}
|
|
3339
|
-
checkMaxConcurrent(item.maxConcurrent, `
|
|
1869
|
+
checkMaxConcurrent(item.maxConcurrent, `harnesses.maxConcurrent in ${where}`);
|
|
3340
1870
|
const { startTimeout } = item;
|
|
3341
1871
|
if (startTimeout !== void 0) try {
|
|
3342
1872
|
parseDuration(startTimeout);
|
|
3343
1873
|
} catch (error) {
|
|
3344
|
-
throw new UsageError(`
|
|
1874
|
+
throw new UsageError(`harnesses.startTimeout in ${where}: ${error.message}`);
|
|
3345
1875
|
}
|
|
3346
1876
|
const problem = harnessProblem(item);
|
|
3347
|
-
if (problem) throw new UsageError(`
|
|
1877
|
+
if (problem) throw new UsageError(`harnesses in ${where}: ${problem}`);
|
|
3348
1878
|
}
|
|
3349
1879
|
}
|
|
3350
1880
|
function checkUnique(list, where) {
|
|
@@ -3371,24 +1901,21 @@ const plannedQuestions = (value) => value.map((q, i) => {
|
|
|
3371
1901
|
});
|
|
3372
1902
|
/** Whether the eval's matrix varies each factor a pairwise question can vary. */
|
|
3373
1903
|
const variesBy = (r) => ({
|
|
3374
|
-
environment:
|
|
1904
|
+
environment: r.compare ? r.compare.variants.length > 0 : r.environments.length > 1,
|
|
3375
1905
|
harness: r.harness.length > 1
|
|
3376
1906
|
});
|
|
3377
1907
|
const FACTOR_FIX = {
|
|
3378
|
-
environment: (_, where) => `varies the environment (across: 'environment'), but ${where} runs one environment, so it has no pairs to ask.
|
|
3379
|
-
harness: (r, where) => `varies the harness (across: 'harness'), but ${where} runs one harness (${r.harness.map(harnessId).join(", ")}), so it has no pairs to ask. Run it on two or more,
|
|
1908
|
+
environment: (_, where) => `varies the environment (across: 'environment'), but ${where} runs one environment, so it has no pairs to ask. Run it in two or more, environments: [a, b], or compare: { control, variants: [...] }`,
|
|
1909
|
+
harness: (r, where) => `varies the harness (across: 'harness'), but ${where} runs one harness (${r.harness.map(harnessId).join(", ")}), so it has no pairs to ask. Run it on two or more, harnesses: [a, b]`
|
|
3380
1910
|
};
|
|
3381
1911
|
function normalizeCompare(value) {
|
|
3382
1912
|
if (value === void 0) return void 0;
|
|
3383
1913
|
return {
|
|
3384
|
-
|
|
3385
|
-
baseline: emptyEnvironment(EMPTY_BASELINE),
|
|
3386
|
-
from: "empty"
|
|
3387
|
-
},
|
|
1914
|
+
control: value.control,
|
|
3388
1915
|
variants: value.variants,
|
|
3389
1916
|
gate: value.gate ?? "variants",
|
|
3390
1917
|
...value.minLift !== void 0 ? { minLift: value.minLift } : {},
|
|
3391
|
-
...value.
|
|
1918
|
+
...value.harnesses ? { harnesses: value.harnesses.map((h) => typeof h === "string" ? h : harnessId(h)) } : {},
|
|
3392
1919
|
of: value.variants.length
|
|
3393
1920
|
};
|
|
3394
1921
|
}
|
|
@@ -3403,9 +1930,9 @@ function applyLevel(out, options, where, egress = "own") {
|
|
|
3403
1930
|
[key]: where
|
|
3404
1931
|
};
|
|
3405
1932
|
if (options.egress !== void 0) out.egress[egress] = options.egress;
|
|
3406
|
-
out.harness = applyHarness(out.harness, options.
|
|
1933
|
+
out.harness = applyHarness(out.harness, options.harnesses, where);
|
|
3407
1934
|
if (options.sandbox !== void 0) out.sandbox = options.sandbox;
|
|
3408
|
-
|
|
1935
|
+
out.environments = applyEnvironments(out.environments, options.environments, where);
|
|
3409
1936
|
if (options.compare !== void 0) out.compare = normalizeCompare(options.compare);
|
|
3410
1937
|
if (options.runs !== void 0) {
|
|
3411
1938
|
const { runs, bestOf } = runsOf(options.runs);
|
|
@@ -3446,6 +1973,7 @@ function cascade(base, levels, where, options = {}) {
|
|
|
3446
1973
|
capture: { ...base.capture },
|
|
3447
1974
|
question: { ...base.question },
|
|
3448
1975
|
questions: [...base.questions],
|
|
1976
|
+
environments: [...base.environments],
|
|
3449
1977
|
tags: [...base.tags],
|
|
3450
1978
|
pin: [...base.pin]
|
|
3451
1979
|
};
|
|
@@ -3455,28 +1983,33 @@ function cascade(base, levels, where, options = {}) {
|
|
|
3455
1983
|
...profile.defaults,
|
|
3456
1984
|
judge: profileJudge(profile)
|
|
3457
1985
|
}, `profiles.${profile.name}.defaults`, "inherited");
|
|
3458
|
-
|
|
3459
|
-
|
|
3460
|
-
|
|
3461
|
-
|
|
3462
|
-
|
|
3463
|
-
}
|
|
1986
|
+
let compareAt;
|
|
1987
|
+
for (const { options: own, where: at } of levels) {
|
|
1988
|
+
if (compareAt && own.environments !== void 0) throw new UsageError(`${at} sets environments, but ${compareAt} sets compare, which runs its control in their place. Set compare: { control, variants: [...] } on ${at} instead, or move ${at} out of ${compareAt}.`);
|
|
1989
|
+
applyLevel(out, own, at);
|
|
1990
|
+
if (own.compare !== void 0) compareAt = at;
|
|
1991
|
+
}
|
|
3464
1992
|
if (out.questions.length) out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
|
|
3465
1993
|
checkUnique(out.harness, where);
|
|
3466
|
-
|
|
1994
|
+
if (!out.compare) {
|
|
1995
|
+
const names = out.environments.map((e) => e.name);
|
|
1996
|
+
const dupe = names.find((n, i) => names.indexOf(n) !== i);
|
|
1997
|
+
if (dupe) throw new UsageError(`environments in ${where} lists two environments named ${dupe}. Give each its own name.`);
|
|
1998
|
+
}
|
|
1999
|
+
const harnessFromProfile = profile?.defaults.harnesses !== void 0 && !levels.some((l) => l.options.harnesses);
|
|
3467
2000
|
const authored = variesBy(out);
|
|
3468
2001
|
if (harnessFromProfile) authored.harness ||= base.harness.length > 1;
|
|
3469
2002
|
for (const q of options.skipped ? [] : out.questions) for (const factor of pairFactors(q.across)) {
|
|
3470
2003
|
if (authored[factor]) continue;
|
|
3471
2004
|
throw new UsageError(`questions "${q.question}" in ${where} ${FACTOR_FIX[factor](out, where)}, or leave out across to ask each attempt on its own.`);
|
|
3472
2005
|
}
|
|
3473
|
-
if (profile?.defaults.
|
|
2006
|
+
if (profile?.defaults.harnesses !== void 0 && levels.some((l) => l.options.harnesses)) notes.push("harnesses set by the eval");
|
|
3474
2007
|
const ids = out.harness.map(harnessId);
|
|
3475
|
-
const missing = (out.compare?.
|
|
3476
|
-
if (missing.length) throw new UsageError(`compare.
|
|
2008
|
+
const missing = (out.compare?.harnesses)?.filter((id) => !ids.includes(id)) ?? [];
|
|
2009
|
+
if (missing.length) throw new UsageError(`compare.harnesses lists ${missing.join(", ")}, which ${where} doesn't run (it runs ${ids.join(", ")}).`);
|
|
3477
2010
|
for (const v of out.compare?.variants ?? []) {
|
|
3478
|
-
if (v.name === out.compare?.
|
|
3479
|
-
const control = out.compare?.
|
|
2011
|
+
if (v.name === out.compare?.control.name) throw new UsageError(`compare in ${where} lists ${v.name}, which is the control. List only the other environments.`);
|
|
2012
|
+
const control = out.compare?.control;
|
|
3480
2013
|
const shared = control && sameWorkspaceOnly(control, v);
|
|
3481
2014
|
if (control && shared) {
|
|
3482
2015
|
const at = shared.ref ? `at ref "${shared.ref}"` : "with no ref (the repository HEAD)";
|
|
@@ -3493,13 +2026,13 @@ function cascade(base, levels, where, options = {}) {
|
|
|
3493
2026
|
const tag = `profile ${profile.name}`;
|
|
3494
2027
|
const pinned = (key) => out.pin.includes(key);
|
|
3495
2028
|
let comparedLeft = true;
|
|
3496
|
-
if (limit.
|
|
3497
|
-
const kept = out.harness.filter((h) => limit.
|
|
3498
|
-
if (kept.length === 0) skip = `no harness left: ${tag} allows ${limit.
|
|
2029
|
+
if (limit.harnesses && !pinned("harnesses")) {
|
|
2030
|
+
const kept = out.harness.filter((h) => limit.harnesses?.includes(harnessId(h)));
|
|
2031
|
+
if (kept.length === 0) skip = `no harness left: ${tag} allows ${limit.harnesses.join(", ") || "none"}`;
|
|
3499
2032
|
else if (kept.length < out.harness.length) {
|
|
3500
2033
|
notes.push(`harnesses ${out.harness.length} → ${kept.length} (${tag})`);
|
|
3501
2034
|
out.harness = kept;
|
|
3502
|
-
const scope = out.compare?.
|
|
2035
|
+
const scope = out.compare?.harnesses;
|
|
3503
2036
|
if (out.compare?.variants.length && scope && !kept.some((h) => scope.includes(harnessId(h)))) {
|
|
3504
2037
|
notes.push(`compare off: no compared harness left (${tag})`);
|
|
3505
2038
|
out.compare = {
|
|
@@ -3565,6 +2098,8 @@ function cascade(base, levels, where, options = {}) {
|
|
|
3565
2098
|
const set = from === where ? "" : ` from ${from}`;
|
|
3566
2099
|
throw new UsageError(`${where} compares environments, so it can't take runs: { bestOf: ${out.bestOf} }${set}: a lift needs the same number of attempts on each side, and best-of stops each cell on its own. Set a fixed count on it, e.g. runs: 4.`);
|
|
3567
2100
|
}
|
|
2101
|
+
if (out.gate === "observe" && out.compare?.minLift !== void 0 && out.compare.variants.length) throw new UsageError(`${where} has gate: 'observe', so its compare can't take minLift: ${out.compare.minLift}: minLift gates the lift, and nothing in it gates. Drop minLift, or gate the eval.`);
|
|
2102
|
+
if (out.gate === "observe" && out.bestOf !== void 0) throw new UsageError(`${where} has gate: 'observe', so it can't take runs: { bestOf: ${out.bestOf} }: best-of decides each cell by its majority. Set a fixed count, e.g. runs: 3.`);
|
|
3568
2103
|
if (out.bestOf !== void 0) {
|
|
3569
2104
|
if (out.gate !== void 0 && gateLabel(out.gate) !== gateLabel(MAJORITY_GATE)) notes.push(`gate ${gateLabel(out.gate)} → majority (runs.bestOf)`);
|
|
3570
2105
|
out.gate = MAJORITY_GATE;
|
|
@@ -3595,14 +2130,14 @@ function defaultRuns(out, notes, options) {
|
|
|
3595
2130
|
notes.push(`runs ${COMPARE_DEFAULT_RUNS} (compare default)`);
|
|
3596
2131
|
}
|
|
3597
2132
|
/**
|
|
3598
|
-
* Refuses a profile's `limit.
|
|
2133
|
+
* Refuses a profile's `limit.harnesses` id that no collected eval runs on: it keeps nothing, and a
|
|
3599
2134
|
* limit that keeps nothing skips every eval. `harnesses` are the ids every eval in the loaded
|
|
3600
2135
|
* files runs on before the limit, whatever the tag and name filters pick. When a path filter
|
|
3601
2136
|
* loaded only some files (`partial`), an id may name a harness of the others, so only a limit
|
|
3602
2137
|
* none of whose ids match is refused.
|
|
3603
2138
|
*/
|
|
3604
2139
|
function checkLimitHarness(profile, configFile, collected) {
|
|
3605
|
-
const ids = profile?.limit.
|
|
2140
|
+
const ids = profile?.limit.harnesses;
|
|
3606
2141
|
if (!profile || !ids?.length) return;
|
|
3607
2142
|
const unknown = ids.filter((id) => !collected.harnesses.includes(id));
|
|
3608
2143
|
if (!unknown.length || collected.partial && unknown.length < ids.length) return;
|
|
@@ -3610,71 +2145,59 @@ function checkLimitHarness(profile, configFile, collected) {
|
|
|
3610
2145
|
const have = [...collected.harnesses].sort();
|
|
3611
2146
|
const evals = collected.partial ? "eval in the selected files" : "eval";
|
|
3612
2147
|
const the = collected.partial ? "the evals in the selected files" : "the evals";
|
|
3613
|
-
throw new UsageError(`profiles.${profile.name}.limit.
|
|
2148
|
+
throw new UsageError(`profiles.${profile.name}.limit.harnesses in ${configFile} lists ${listed}, which no ${evals} runs on. Harness ids ${the} run on: ${have.join(", ") || "none"}.`);
|
|
3614
2149
|
}
|
|
3615
2150
|
/**
|
|
3616
2151
|
* The sandbox an attempt runs in: the resolved environment's (including an `.extend()`
|
|
3617
|
-
* parent's), else the
|
|
3618
|
-
*
|
|
3619
|
-
*
|
|
2152
|
+
* parent's), else the cascaded option (`defineConfig` / profile defaults / group / eval), else
|
|
2153
|
+
* implicit Docker. A named environment is a total override: the config environment's sandbox is
|
|
2154
|
+
* not inherited.
|
|
3620
2155
|
*/
|
|
3621
|
-
const attemptSandbox = (resolved, env) =>
|
|
3622
|
-
const own = env.layers.findLast((l) => l.spec.sandbox)?.spec.sandbox;
|
|
3623
|
-
if (own) return own;
|
|
3624
|
-
const variant = resolved.compare?.from === "empty" && env.name === "baseline" ? resolved.compare.variants[0] : void 0;
|
|
3625
|
-
return sandboxOf(variant?.layers ?? [], resolved.sandbox);
|
|
3626
|
-
};
|
|
2156
|
+
const attemptSandbox = (resolved, env) => env.layers.findLast((l) => l.spec.sandbox)?.spec.sandbox ?? sandboxOf([], resolved.sandbox);
|
|
3627
2157
|
/**
|
|
3628
2158
|
* Which level set this attempt's sandbox: the cascade source of the environment that named one
|
|
3629
|
-
* (an `.extend()` child still names that environment), else the
|
|
3630
|
-
*
|
|
3631
|
-
* Docker).
|
|
2159
|
+
* (an `.extend()` child still names that environment), else the cascaded `sandbox` option, else
|
|
2160
|
+
* the config (implicit Docker).
|
|
3632
2161
|
*/
|
|
3633
2162
|
function sandboxSetBy(planned, resolved, configFile) {
|
|
3634
|
-
if (!planned.environment.layers.some((l) => l.spec.sandbox))
|
|
3635
|
-
|
|
3636
|
-
return resolved.sources?.sandbox ?? configFile;
|
|
3637
|
-
}
|
|
3638
|
-
if (resolved.compare) {
|
|
3639
|
-
if (planned.cell.role === "control") {
|
|
3640
|
-
if (resolved.compare.from === "environment") return resolved.sources?.environment ?? configFile;
|
|
3641
|
-
if (resolved.compare.from === "empty") return configFile;
|
|
3642
|
-
return resolved.sources?.compare ?? configFile;
|
|
3643
|
-
}
|
|
3644
|
-
return resolved.sources?.compare ?? configFile;
|
|
3645
|
-
}
|
|
3646
|
-
return resolved.sources?.environment ?? configFile;
|
|
2163
|
+
if (!planned.environment.layers.some((l) => l.spec.sandbox)) return resolved.sources?.sandbox ?? configFile;
|
|
2164
|
+
return (resolved.compare ? resolved.sources?.compare : resolved.sources?.environments) ?? configFile;
|
|
3647
2165
|
}
|
|
3648
2166
|
/**
|
|
3649
2167
|
* Per harness: the control, then each variant it compares on; then run numbers. `vertex` is the
|
|
3650
2168
|
* run's `vertex()` settings, for each harness that resolved to Vertex AI.
|
|
3651
2169
|
*/
|
|
2170
|
+
const cellTraits = (env) => {
|
|
2171
|
+
const traits = traitsOf(env.layers);
|
|
2172
|
+
return Object.keys(traits).length ? { traits } : {};
|
|
2173
|
+
};
|
|
3652
2174
|
function expand(resolved, vertex) {
|
|
3653
2175
|
const { compare } = resolved;
|
|
3654
|
-
const
|
|
3655
|
-
const policy = resolved.gate ?? "all";
|
|
2176
|
+
const controls = (compare ? [compare.control] : resolved.environments.length ? resolved.environments : [void 0]).map((e) => applied(e));
|
|
2177
|
+
const policy = resolved.gate === "observe" ? "none" : resolved.gate ?? "all";
|
|
3656
2178
|
const gateIf = (gated) => gated ? policy : "none";
|
|
3657
2179
|
const attempts = [];
|
|
3658
2180
|
for (const harness of resolved.harness) {
|
|
3659
2181
|
const hid = harnessId(harness);
|
|
3660
2182
|
const onVertex = harness.gateway === "vertex" && harness.kind !== "fake" ? vertex : void 0;
|
|
3661
|
-
const cells =
|
|
3662
|
-
const sandbox = attemptSandbox(resolved,
|
|
2183
|
+
const cells = controls.map((env) => {
|
|
2184
|
+
const sandbox = attemptSandbox(resolved, env);
|
|
3663
2185
|
const sid = sandboxId(sandbox);
|
|
3664
2186
|
return {
|
|
3665
|
-
env
|
|
2187
|
+
env,
|
|
3666
2188
|
sandbox,
|
|
3667
2189
|
cell: {
|
|
3668
|
-
key: cellKey(hid,
|
|
2190
|
+
key: cellKey(hid, env.name, sid),
|
|
3669
2191
|
harness: hid,
|
|
3670
|
-
environment:
|
|
2192
|
+
environment: env.name,
|
|
3671
2193
|
sandbox: sid,
|
|
3672
|
-
role: "control"
|
|
2194
|
+
role: "control",
|
|
2195
|
+
...cellTraits(env)
|
|
3673
2196
|
},
|
|
3674
2197
|
gate: gateIf(!compare || compare.variants.length === 0 || compare.gate === "all")
|
|
3675
2198
|
};
|
|
3676
|
-
})
|
|
3677
|
-
if (compare && (!compare.
|
|
2199
|
+
});
|
|
2200
|
+
if (compare && (!compare.harnesses || compare.harnesses.includes(hid))) for (const v of compare.variants) {
|
|
3678
2201
|
const env = applied(v);
|
|
3679
2202
|
const sandbox = attemptSandbox(resolved, env);
|
|
3680
2203
|
const sid = sandboxId(sandbox);
|
|
@@ -3687,13 +2210,14 @@ function expand(resolved, vertex) {
|
|
|
3687
2210
|
environment: env.name,
|
|
3688
2211
|
sandbox: sid,
|
|
3689
2212
|
role: "variant",
|
|
3690
|
-
variant: v.name
|
|
2213
|
+
variant: v.name,
|
|
2214
|
+
...cellTraits(env)
|
|
3691
2215
|
},
|
|
3692
|
-
gate: gateIf(compare.gate !== "
|
|
2216
|
+
gate: gateIf(compare.gate !== "observe")
|
|
3693
2217
|
});
|
|
3694
2218
|
}
|
|
3695
2219
|
for (const { env, sandbox, cell, gate } of cells) {
|
|
3696
|
-
const axes = [resolved.harness.length > 1 ? hid : "", compare ? cell.role === "control" ? "control" : env.name : ""].filter(Boolean);
|
|
2220
|
+
const axes = [resolved.harness.length > 1 ? hid : "", compare ? cell.role === "control" ? "control" : env.name : controls.length > 1 ? env.name : ""].filter(Boolean);
|
|
3697
2221
|
for (let n = 1; n <= resolved.runs; n++) {
|
|
3698
2222
|
const label = [axes.join(" · "), resolved.runs > 1 ? `#${n}` : ""].filter(Boolean).join(" ");
|
|
3699
2223
|
attempts.push({
|
|
@@ -3730,14 +2254,16 @@ function environmentWarnings(attempts) {
|
|
|
3730
2254
|
}
|
|
3731
2255
|
if (s.source === "registry" && !s.version) out.add(`skill.registry(${s.id}) has no version: results aren't comparable over time. Pin one (the resolved commit is recorded either way).`);
|
|
3732
2256
|
}
|
|
2257
|
+
const tag = a.sandbox.kind === "tart" ? tartTagWarning(a.sandbox.image) : null;
|
|
2258
|
+
if (tag) out.add(tag);
|
|
3733
2259
|
}
|
|
3734
2260
|
return [...out];
|
|
3735
2261
|
}
|
|
3736
|
-
/** Secrets
|
|
2262
|
+
/** Secrets an isolated sandbox's attempt would get as real values, because they have no `hosts`. */
|
|
3737
2263
|
function unbrokered(env, attempts) {
|
|
3738
2264
|
const out = /* @__PURE__ */ new Map();
|
|
3739
2265
|
for (const a of attempts) {
|
|
3740
|
-
if (a.sandbox.
|
|
2266
|
+
if (!capabilitiesOf(a.sandbox).isolated) continue;
|
|
3741
2267
|
for (const s of unbrokeredSecrets(env, a.environment.layers)) out.set(s.label, s);
|
|
3742
2268
|
}
|
|
3743
2269
|
return [...out.values()];
|
|
@@ -3746,12 +2272,21 @@ function unbrokered(env, attempts) {
|
|
|
3746
2272
|
const reachesSandbox = (s) => !s.optional || Boolean(process.env[s.from]);
|
|
3747
2273
|
/** `GEMINI_API_KEY → generativelanguage.googleapis.com` for each secret with `hosts`. */
|
|
3748
2274
|
const withHosts = (env, attempt) => [...new Set(plannedSecrets(env, attempt.environment.layers).flatMap((s) => s.hosts ? [`${s.label} → ${s.hosts.join(", ")}`] : []))];
|
|
3749
|
-
/** The secrets an attempt brokers: only
|
|
3750
|
-
const brokeredIn = (env, attempt) => attempt.sandbox.
|
|
3751
|
-
/** The plan's note for secrets with `hosts` on
|
|
3752
|
-
function
|
|
3753
|
-
const
|
|
3754
|
-
|
|
2275
|
+
/** The secrets an attempt brokers: only a sandbox that brokers swaps a placeholder for its value. */
|
|
2276
|
+
const brokeredIn = (env, attempt) => capabilitiesOf(attempt.sandbox).brokers ? withHosts(env, attempt) : [];
|
|
2277
|
+
/** The plan's note for secrets with `hosts` on a sandbox that can't broker them, which only ever has their placeholder. */
|
|
2278
|
+
function unbrokeredHere(env, attempts) {
|
|
2279
|
+
const byProvider = /* @__PURE__ */ new Map();
|
|
2280
|
+
for (const a of attempts) {
|
|
2281
|
+
if (capabilitiesOf(a.sandbox).brokers) continue;
|
|
2282
|
+
const names = withHosts(env, a);
|
|
2283
|
+
if (!names.length) continue;
|
|
2284
|
+
const provider = sandboxProvider(a.sandbox);
|
|
2285
|
+
const seen = byProvider.get(provider) ?? /* @__PURE__ */ new Set();
|
|
2286
|
+
for (const name of names) seen.add(name);
|
|
2287
|
+
byProvider.set(provider, seen);
|
|
2288
|
+
}
|
|
2289
|
+
return [...byProvider].map(([provider, names]) => `not brokered (${provider} sandbox, placeholder only): ${[...names].join("; ")}`);
|
|
3755
2290
|
}
|
|
3756
2291
|
/** The host variables an attempt's secrets read, brokered or not. */
|
|
3757
2292
|
const secretVarsIn = (env, attempt) => [...new Set(plannedSecrets(env, attempt.environment.layers).map((s) => s.from))];
|
|
@@ -3776,11 +2311,12 @@ function sandboxWarnings(attempts, resolved, origin) {
|
|
|
3776
2311
|
const out = [];
|
|
3777
2312
|
const seen = /* @__PURE__ */ new Set();
|
|
3778
2313
|
for (const a of attempts) {
|
|
3779
|
-
|
|
2314
|
+
const privileges = sandboxKind(a.sandbox).privileges(a.sandbox);
|
|
2315
|
+
if (!privileges.root && !privileges.capabilities.length) continue;
|
|
3780
2316
|
const where = sandboxSetBy(a, resolved, origin.configFile);
|
|
3781
2317
|
const level = levelOf(where);
|
|
3782
2318
|
const file = level === "eval" || level === "group" ? origin.file : null;
|
|
3783
|
-
if (
|
|
2319
|
+
if (privileges.root) {
|
|
3784
2320
|
const message = `running as root in the sandbox: ${where} (docker({ root: true }))`;
|
|
3785
2321
|
const key = `${file}\0${message}`;
|
|
3786
2322
|
if (!seen.has(key)) {
|
|
@@ -3794,8 +2330,8 @@ function sandboxWarnings(attempts, resolved, origin) {
|
|
|
3794
2330
|
});
|
|
3795
2331
|
}
|
|
3796
2332
|
}
|
|
3797
|
-
if (
|
|
3798
|
-
const message = `extra capabilities in the sandbox: ${where} (docker({ capabilities: [${
|
|
2333
|
+
if (privileges.capabilities.length) {
|
|
2334
|
+
const message = `extra capabilities in the sandbox: ${where} (docker({ capabilities: [${privileges.capabilities.map((c) => `'${c}'`).join(", ")}] }))`;
|
|
3799
2335
|
const key = `${file}\0${message}`;
|
|
3800
2336
|
if (!seen.has(key)) {
|
|
3801
2337
|
seen.add(key);
|
|
@@ -3804,7 +2340,7 @@ function sandboxWarnings(attempts, resolved, origin) {
|
|
|
3804
2340
|
level,
|
|
3805
2341
|
where,
|
|
3806
2342
|
file,
|
|
3807
|
-
capabilities: [...
|
|
2343
|
+
capabilities: [...privileges.capabilities],
|
|
3808
2344
|
message
|
|
3809
2345
|
});
|
|
3810
2346
|
}
|
|
@@ -3819,7 +2355,7 @@ function evalPlan(r, attempts, origin) {
|
|
|
3819
2355
|
const notes = [
|
|
3820
2356
|
...r.notes,
|
|
3821
2357
|
...capture ? [capture] : [],
|
|
3822
|
-
...
|
|
2358
|
+
...unbrokeredHere(resolved.env, attempts)
|
|
3823
2359
|
];
|
|
3824
2360
|
return {
|
|
3825
2361
|
harnesses: skip ? 0 : resolved.harness.length,
|
|
@@ -3827,10 +2363,7 @@ function evalPlan(r, attempts, origin) {
|
|
|
3827
2363
|
run: resolved.compare.variants.length,
|
|
3828
2364
|
of: resolved.compare.of
|
|
3829
2365
|
} : null,
|
|
3830
|
-
...resolved.compare ? { compare: {
|
|
3831
|
-
baseline: resolved.compare.baseline.name,
|
|
3832
|
-
...resolved.compare.from ? { from: resolved.compare.from } : {}
|
|
3833
|
-
} } : {},
|
|
2366
|
+
...resolved.compare ? { compare: { baseline: resolved.compare.control.name } } : {},
|
|
3834
2367
|
...!skip && resolved.questions.length ? { questions: resolved.questions } : {},
|
|
3835
2368
|
runs: resolved.runs,
|
|
3836
2369
|
...resolved.bestOf !== void 0 ? { bestOf: resolved.bestOf } : {},
|
|
@@ -4065,9 +2598,9 @@ function modelGatewayOrNone(model, pin, scope) {
|
|
|
4065
2598
|
const CONFIG_KEYS = [
|
|
4066
2599
|
"include",
|
|
4067
2600
|
"gateway",
|
|
4068
|
-
"
|
|
2601
|
+
"harnesses",
|
|
4069
2602
|
"sandbox",
|
|
4070
|
-
"
|
|
2603
|
+
"environments",
|
|
4071
2604
|
"runs",
|
|
4072
2605
|
"gate",
|
|
4073
2606
|
"env",
|
|
@@ -4097,9 +2630,9 @@ const PROFILE_KEYS = [
|
|
|
4097
2630
|
"gateway"
|
|
4098
2631
|
];
|
|
4099
2632
|
const DEFAULTS_KEYS = [
|
|
4100
|
-
"
|
|
2633
|
+
"harnesses",
|
|
4101
2634
|
"sandbox",
|
|
4102
|
-
"
|
|
2635
|
+
"environments",
|
|
4103
2636
|
"runs",
|
|
4104
2637
|
"gate",
|
|
4105
2638
|
"env",
|
|
@@ -4113,7 +2646,8 @@ const DEFAULTS_KEYS = [
|
|
|
4113
2646
|
const LIMIT_KEYS = [
|
|
4114
2647
|
"runs",
|
|
4115
2648
|
"compare",
|
|
4116
|
-
"
|
|
2649
|
+
"harnesses",
|
|
2650
|
+
"traits"
|
|
4117
2651
|
];
|
|
4118
2652
|
/**
|
|
4119
2653
|
* The config: a plain object, or a callback that destructures the helpers it needs, e.g.
|
|
@@ -4207,7 +2741,7 @@ function parseYamlConfig(text, file) {
|
|
|
4207
2741
|
return doc.toJS() ?? {};
|
|
4208
2742
|
}
|
|
4209
2743
|
const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4210
|
-
/** A config's Dockerfile that isn't there, named by its field. */
|
|
2744
|
+
/** A config's host file (a Dockerfile, an SSH key) that isn't there, named by its field. */
|
|
4211
2745
|
const missingIn = (file) => (field, full) => {
|
|
4212
2746
|
throw new UsageError(`${field} in ${file}: ${full} doesn't exist. Paths in ${file} are relative to its directory.`);
|
|
4213
2747
|
};
|
|
@@ -4223,7 +2757,10 @@ function withHostPaths(config, root, file, { environments = false } = {}) {
|
|
|
4223
2757
|
if (!isRecord(value)) return value;
|
|
4224
2758
|
const out = { ...value };
|
|
4225
2759
|
if ("sandbox" in value) out.sandbox = sandboxPaths(value.sandbox, root, `${prefix}sandbox`, missing);
|
|
4226
|
-
if (environments && "
|
|
2760
|
+
if (environments && "environments" in value) {
|
|
2761
|
+
const field = `${prefix}environments`;
|
|
2762
|
+
out.environments = Array.isArray(value.environments) ? value.environments.map((e, i) => environmentPaths(e, root, `${field}[${i}]`, missing)) : environmentPaths(value.environments, root, field, missing);
|
|
2763
|
+
}
|
|
4227
2764
|
return out;
|
|
4228
2765
|
};
|
|
4229
2766
|
const out = options(config, "");
|
|
@@ -4242,10 +2779,15 @@ function withHostPaths(config, root, file, { environments = false } = {}) {
|
|
|
4242
2779
|
function configFromData(data) {
|
|
4243
2780
|
const withEnvironment = (options) => {
|
|
4244
2781
|
if (!isObjectLike(options)) return options;
|
|
4245
|
-
const {
|
|
4246
|
-
|
|
2782
|
+
const { environments } = options;
|
|
2783
|
+
const from = (e) => isEnvironment(e) ? environmentFrom(e) : e;
|
|
2784
|
+
if (Array.isArray(environments)) return {
|
|
4247
2785
|
...options,
|
|
4248
|
-
|
|
2786
|
+
environments: environments.map(from)
|
|
2787
|
+
};
|
|
2788
|
+
return isEnvironment(environments) ? {
|
|
2789
|
+
...options,
|
|
2790
|
+
environments: from(environments)
|
|
4249
2791
|
} : options;
|
|
4250
2792
|
};
|
|
4251
2793
|
const config = withEnvironment(data);
|
|
@@ -4271,10 +2813,10 @@ function configFromData(data) {
|
|
|
4271
2813
|
* let through, such as a `null` or a misspelled nested key.
|
|
4272
2814
|
*/
|
|
4273
2815
|
function resolveConfigData(input, root, file) {
|
|
4274
|
-
if (!isRecord(input)) throw new UsageError(`${file} must be a mapping of config keys, like
|
|
2816
|
+
if (!isRecord(input)) throw new UsageError(`${file} must be a mapping of config keys, like harnesses: and judge: (got ${JSON.stringify(input)})`);
|
|
4275
2817
|
const data = withHostPaths(input, root, file, { environments: true });
|
|
4276
|
-
const { environment, profiles } = data;
|
|
4277
|
-
if ([environment, ...Object.values(typeof profiles === "object" && profiles ? profiles : {}).map((p) => p?.defaults?.
|
|
2818
|
+
const { environments: environment, profiles } = data;
|
|
2819
|
+
if ([environment, ...Object.values(typeof profiles === "object" && profiles ? profiles : {}).map((p) => p?.defaults?.environments)].flat().some((e) => e !== void 0 && !isEnvironment(e))) {
|
|
4278
2820
|
const problem = configDataProblem(data, file);
|
|
4279
2821
|
if (problem) throw new UsageError(problem);
|
|
4280
2822
|
}
|
|
@@ -4354,12 +2896,18 @@ function checkProfile(name, profile, file) {
|
|
|
4354
2896
|
}
|
|
4355
2897
|
if (p.limit !== void 0) {
|
|
4356
2898
|
unknownKey(p.limit, LIMIT_KEYS, `${where}.limit`, file);
|
|
4357
|
-
const { runs, compare, harness } = p.limit;
|
|
2899
|
+
const { runs, compare, harnesses: harness, traits } = p.limit;
|
|
4358
2900
|
if (runs !== void 0 && !(Number.isInteger(runs) && runs >= 1)) throw new UsageError(`${where}.limit.runs in ${file} must be a whole number of at least 1 (got ${JSON.stringify(runs)})`);
|
|
4359
2901
|
if (compare !== void 0 && compare !== false && !(Number.isInteger(compare?.maxVariants) && compare.maxVariants >= 0)) throw new UsageError(`${where}.limit.compare in ${file} must be false or { maxVariants: n }, e.g. { maxVariants: 1 } (got ${JSON.stringify(compare)})`);
|
|
4360
2902
|
if (harness !== void 0) {
|
|
4361
2903
|
const bad = Array.isArray(harness) ? harness.find((h) => typeof h !== "string") : harness;
|
|
4362
|
-
if (bad !== void 0) throw new UsageError(`${where}.limit.
|
|
2904
|
+
if (bad !== void 0) throw new UsageError(`${where}.limit.harnesses in ${file} can only list harness ids to allow (got ${describeValue(bad)}). To add harnesses, use ${where}.defaults.harnesses.`);
|
|
2905
|
+
}
|
|
2906
|
+
if (traits !== void 0) {
|
|
2907
|
+
const ok = (v) => typeof v === "string" || Array.isArray(v) && v.length > 0 && v.every((s) => typeof s === "string");
|
|
2908
|
+
const entries = typeof traits === "object" && traits !== null && !Array.isArray(traits) ? Object.entries(traits) : null;
|
|
2909
|
+
const bad = entries?.find(([, v]) => !ok(v));
|
|
2910
|
+
if (!entries || bad) throw new UsageError(`${where}.limit.traits in ${file} must map each trait key to a value or a list of values a cell's environment has, e.g. { lang: 'ts' } or { lang: ['ts', 'py'] } (got ${bad ? `${bad[0]}: ${JSON.stringify(bad[1])}` : JSON.stringify(traits)})`);
|
|
4363
2911
|
}
|
|
4364
2912
|
}
|
|
4365
2913
|
if (p.tags !== void 0 && (!Array.isArray(p.tags) || p.tags.some((t) => typeof t !== "string"))) throw new UsageError(`${where}.tags in ${file} must be a list of tag filters, e.g. tags: ['smoke'] (got ${JSON.stringify(p.tags)})`);
|
|
@@ -4473,8 +3021,8 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
|
4473
3021
|
const artifacts = user.history?.artifacts ?? "all";
|
|
4474
3022
|
if (!ARTIFACTS.includes(artifacts)) throw new UsageError(`history.artifacts in ${file} is ${oneOf(ARTIFACTS)} (got ${JSON.stringify(artifacts)})`);
|
|
4475
3023
|
const historyDir = resolve(root, user.history?.dir ?? join(evalRoot(include), ".history"));
|
|
4476
|
-
const harness = user.
|
|
4477
|
-
if (harness.length === 0) throw new UsageError(`
|
|
3024
|
+
const harness = user.harnesses === void 0 ? [DEFAULT_HARNESS] : [user.harnesses].flat();
|
|
3025
|
+
if (harness.length === 0) throw new UsageError(`harnesses is an empty list in ${file}. Give at least one, e.g. harnesses: ${HARNESS_EXAMPLE}`);
|
|
4478
3026
|
checkHarnesses(harness, file);
|
|
4479
3027
|
const profiles = {};
|
|
4480
3028
|
if (user.profiles !== void 0) {
|
|
@@ -4497,7 +3045,7 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
|
4497
3045
|
include,
|
|
4498
3046
|
harness,
|
|
4499
3047
|
sandbox: user.sandbox ?? docker(),
|
|
4500
|
-
|
|
3048
|
+
environments: user.environments === void 0 ? [] : [user.environments].flat(),
|
|
4501
3049
|
runs: user.runs,
|
|
4502
3050
|
gate: user.gate,
|
|
4503
3051
|
env: user.env ?? {},
|
|
@@ -4639,6 +3187,11 @@ function runModelGateway(config, profile, model, judge = false) {
|
|
|
4639
3187
|
*/
|
|
4640
3188
|
function missingCredentials(config, env = process.env, options = {}) {
|
|
4641
3189
|
const missing = [];
|
|
3190
|
+
for (const s of options.sandboxes ?? []) {
|
|
3191
|
+
if (s.kind !== "boat" || env[s.apiKey.from]) continue;
|
|
3192
|
+
const line = `${s.apiKey.from} (for boat())`;
|
|
3193
|
+
if (!missing.includes(line)) missing.push(line);
|
|
3194
|
+
}
|
|
4642
3195
|
const keys = new Set((options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []));
|
|
4643
3196
|
for (const key of keys) if (!env[key]) missing.push(key);
|
|
4644
3197
|
const judge = runJudgeModel(config, options.profile);
|
|
@@ -4696,6 +3249,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
|
|
|
4696
3249
|
return at ? [`${name} (character ${at})`] : [];
|
|
4697
3250
|
});
|
|
4698
3251
|
}
|
|
4699
|
-
export {
|
|
3252
|
+
export { runsOf as $, brokeredIn as A, execOptionsProblem as At, expand as B, decideState as Bt, openCodeModelOn as C, resolveStateOptions as Ct, EVAL_OPTION_KEYS as D, CAPTURE_NAME as Dt, COMPARE_DEFAULT_RUNS as E, observer as Et, checkHarnesses as F, NO_JUDGE_INPUT as Ft, mergeSnapshot as G, levelOf as H, resolveJudgeArtifacts as Ht, checkLimitHarness as I, inputsOn as It, plainHeaders as J, plainHeaderError as K, checkOptions as L, judgeProblem as Lt, cellKey as M, resolvedEffortOf as Mt, checkCapture as N, servedEffortOf as Nt, MAJORITY_GATE as O, CAPTURE_NAME_RULE as Ot, checkGate as P, INPUT_KEYS as Pt, refuseIllegalCompare as Q, environmentWarnings as R, artifactRefsOf as Rt, fallbackHint as S, mergeJudge as St, routedHarness as T, observedName as Tt, majorityOf as U, harnessId as V, filePartsOf as Vt, matchesTags as W, plannedSandboxes as X, plannedSandboxIds as Y, profileJudge as Z, fellBackToDirect as _, resolveOutput as _t, configPathFor as a, unbrokered as at, resolveModelGateway as b, questionShows as bt, findConfig as c, DEFAULT_EGRESS as ct, malformedCredentials as d, ciRunner as dt, sandboxId as et, missingCredentials as f, directOnCloud as ft, selectProfile as g, unenforceable as gt, runJudgeModel as h, isDirect as ht, configLabel as i, secretVarsIn as it, cascade as j, harnessEffort as jt, attemptSandbox as k, attemptRequestProblem as kt, hostCapacity as l, DEFAULT_EGRESS_ALLOW as lt, resolveConcurrency as m, gatewayPolicy as mt, analysisModelOf as n, sandboxSetBy as nt, defaultConcurrency as o, unbrokeredError as ot, requireConfig as p, egressLabel as pt, plainHeaderWarning as q, configFilesPhrase as r, sandboxWarnings as rt, defineConfig as s, unbrokeredWarning as st, CONFIG_FILES as t, sandboxProvider as tt, loadConfig as u, allowedHosts as ut, gatewayScope as v, STOPPED_FIRST as vt, resolveGateway as w, observedHere as wt, effortRefusedOn as x, resolveQuestionState as xt, modelGatewayOrNone as y, UNRECORDED_QUESTION_STATE as yt, evalPlan as z, badJudgeArtifacts as zt };
|
|
4700
3253
|
|
|
4701
|
-
//# sourceMappingURL=config-
|
|
3254
|
+
//# sourceMappingURL=config-B9mB1kF5.js.map
|